Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow multiple azure instances #47

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Allow multiple azure instances #47

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@ This project adheres to [Semantic Versioning](http://semver.org/).

### Added/Changed

- Added support for multiple Azure instances

### Fixed

- Don't crash on connection strings with trailing semicolon
Expand Down
15 changes: 15 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,21 @@ config :azurex, Azurex.Blob.Config,
storage_account_connection_string: "Storage=Account;Connection=String" # Required if storage account `name` and `key` not set
```

Each of these options is then overridable per-request, if you need to work with multiple instances:

```elixir
Azurex.Blob.list_blobs(container: "other", api_uri: "https://other.blob.net")

Azurex.Blob.get_blob("file.txt", [
storage_account_connection_string: "Account=Storage;String=Connection"
])

Azurex.Blob.put_blob("file.txt", "contents", "text/plain", [
storage_account_key: "key",
storage_account_name: "name"
])
```

## Documentation

Documentation can be found at [https://hexdocs.pm/azurex](https://hexdocs.pm/azurex). Or generated using [ExDoc](https://github.com/elixir-lang/ex_doc)
Expand Down
123 changes: 75 additions & 48 deletions lib/azurex/blob.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,17 @@ defmodule Azurex.Blob do

@typep optional_string :: String.t() | nil

def list_containers do
@spec list_containers(Config.config_overrides()) ::
{:ok, String.t()}
| {:error, HTTPoison.AsyncResponse.t() | HTTPoison.Error.t() | HTTPoison.Response.t()}
def list_containers(overrides \\ []) do
%HTTPoison.Request{
url: Config.api_url() <> "/",
url: Config.api_url(overrides) <> "/",
params: [comp: "list"]
}
|> SharedKey.sign(
storage_account_name: Config.storage_account_name(),
storage_account_key: Config.storage_account_key()
storage_account_name: Config.storage_account_name(overrides),
storage_account_key: Config.storage_account_key(overrides)
)
|> HTTPoison.request()
|> case do
Expand Down Expand Up @@ -55,6 +58,15 @@ defmodule Azurex.Blob do
iex> put_blob("filename.txt", "file contents", "text/plain", "container")
:ok

iex> put_blob("filename.txt", "file contents", "text/plain", [container: "container"])
:ok
jakobht marked this conversation as resolved.
Show resolved Hide resolved

iex> put_blob("filename.txt", "file contents", "text/plain", [storage_account_name: "name", storage_account_key: "key"])
:ok

iex> put_blob("filename.txt", "file contents", "text/plain", [storage_account_connection_string: "AccountName=name;AccountKey=key", container: "container"])
:ok

iex> put_blob("filename.txt", "file contents", "text/plain", nil, timeout: 10)
:ok

Expand All @@ -66,37 +78,38 @@ defmodule Azurex.Blob do
String.t(),
binary() | {:stream, Enumerable.t()},
optional_string,
optional_string,
Config.config_overrides(),
keyword
) ::
:ok
| {:error, HTTPoison.AsyncResponse.t() | HTTPoison.Error.t() | HTTPoison.Response.t()}
def put_blob(name, blob, content_type, container \\ nil, params \\ [])
def put_blob(name, blob, content_type, overrides \\ [], params \\ [])

def put_blob(name, {:stream, bitstream}, content_type, container, params) do
def put_blob(name, {:stream, bitstream}, content_type, overrides, params) do
content_type = content_type || "application/octet-stream"

bitstream
|> Stream.transform(
fn -> [] end,
fn chunk, acc ->
with {:ok, block_id} <- Block.put_block(container, chunk, name, params) do
with {:ok, block_id} <- Block.put_block(overrides, chunk, name, params) do
{[], [block_id | acc]}
end
end,
fn acc ->
Block.put_block_list(acc, container, name, content_type, params)
Block.put_block_list(acc, overrides, name, content_type, params)
end
)
|> Stream.run()
end

def put_blob(name, blob, content_type, container, params) do
def put_blob(name, blob, content_type, overrides, params) do
content_type = content_type || "application/octet-stream"
connection_params = Config.get_connection_params(overrides)

%HTTPoison.Request{
method: :put,
url: get_url(container, name),
url: get_url(name, connection_params),
params: params,
body: blob,
headers: [
Expand All @@ -107,8 +120,8 @@ defmodule Azurex.Blob do
options: [recv_timeout: :infinity]
}
|> SharedKey.sign(
storage_account_name: Config.storage_account_name(),
storage_account_key: Config.storage_account_key(),
storage_account_name: Config.storage_account_name(connection_params),
storage_account_key: Config.storage_account_key(connection_params),
content_type: content_type
)
|> HTTPoison.request()
Expand All @@ -130,18 +143,24 @@ defmodule Azurex.Blob do
iex> get_blob("filename.txt", "container")
{:ok, "file contents"}

iex> get_blob("filename.txt", [storage_account_name: "name", storage_account_key: "key", container: "container"])
{:ok, "file contents"}

iex> get_blob("filename.txt", [storage_account_connection_string: "AccountName=name;AccountKey=key"])
{:ok, "file contents"}

iex> get_blob("filename.txt", nil, timeout: 10)
{:ok, "file contents"}

iex> get_blob("filename.txt")
{:error, %HTTPoison.Response{}}

"""
@spec get_blob(String.t(), optional_string) ::
@spec get_blob(String.t(), Config.config_overrides(), keyword) ::
{:ok, binary()}
| {:error, HTTPoison.AsyncResponse.t() | HTTPoison.Error.t() | HTTPoison.Response.t()}
def get_blob(name, container \\ nil, params \\ []) do
blob_request(name, container, :get, params)
def get_blob(name, overrides \\ [], params \\ []) do
blob_request(name, overrides, :get, params)
|> HTTPoison.request()
|> case do
{:ok, %{body: blob, status_code: 200}} -> {:ok, blob}
Expand All @@ -153,11 +172,11 @@ defmodule Azurex.Blob do
@doc """
Checks if a blob exists, and returns metadata for the blob if it does
"""
@spec head_blob(String.t(), optional_string) ::
@spec head_blob(String.t(), Config.config_overrides(), keyword) ::
{:ok, list}
| {:error, :not_found | HTTPoison.Error.t() | HTTPoison.Response.t()}
def head_blob(name, container \\ nil, params \\ []) do
blob_request(name, container, :head, params)
def head_blob(name, overrides \\ [], params \\ []) do
blob_request(name, overrides, :head, params)
|> HTTPoison.request()
|> case do
{:ok, %HTTPoison.Response{status_code: 200, headers: details}} -> {:ok, details}
Expand All @@ -170,24 +189,27 @@ defmodule Azurex.Blob do
@doc """
Copies a blob to a destination.

The same configuration options (connection string, container, ...) are applied to both source and destination.

Note: Azure’s ‘[Copy Blob from URL](https://learn.microsoft.com/en-us/rest/api/storageservices/copy-blob-from-url)’
operation has a maximum size of 256 MiB.
"""
@spec copy_blob(String.t(), String.t(), optional_string) ::
@spec copy_blob(String.t(), String.t(), Config.config_overrides()) ::
{:ok, HTTPoison.Response.t()} | {:error, term()}
def copy_blob(source_name, destination_name, container \\ nil) do
def copy_blob(source_name, destination_name, overrides \\ []) do
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I suggest documenting that the overrides apply to both source and destination, so you can't use this method to copy between accounts.

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not super important IMO, but documentation is always good.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have added a line in the doc mentioning it

content_type = "application/octet-stream"
source_url = get_url(container, source_name)
connection_params = Config.get_connection_params(overrides)
source_url = get_url(source_name, connection_params)
headers = [{"x-ms-copy-source", source_url}, {"content-type", content_type}]

%HTTPoison.Request{
method: :put,
url: get_url(container, destination_name),
url: get_url(destination_name, connection_params),
headers: headers
}
|> SharedKey.sign(
storage_account_name: Config.storage_account_name(),
storage_account_key: Config.storage_account_key(),
storage_account_name: Config.storage_account_name(connection_params),
storage_account_key: Config.storage_account_key(connection_params),
content_type: content_type
)
|> HTTPoison.request()
Expand All @@ -198,10 +220,10 @@ defmodule Azurex.Blob do
end
end

@spec delete_blob(String.t(), optional_string()) ::
@spec delete_blob(String.t(), Config.config_overrides(), keyword) ::
:ok | {:error, :not_found | HTTPoison.Error.t() | HTTPoison.Response.t()}
def delete_blob(name, container \\ nil, params \\ []) do
blob_request(name, container, :delete, params)
def delete_blob(name, overrides \\ [], params \\ []) do
blob_request(name, overrides, :delete, params)
|> HTTPoison.request()
|> case do
{:ok, %HTTPoison.Response{status_code: 202}} -> :ok
Expand All @@ -211,17 +233,17 @@ defmodule Azurex.Blob do
end
end

defp blob_request(name, container, method, params, headers \\ [], options \\ []) do
defp blob_request(name, overrides, method, params) do
connection_params = Config.get_connection_params(overrides)

%HTTPoison.Request{
method: method,
url: get_url(container, name),
params: params,
headers: headers,
options: options
url: get_url(name, connection_params),
params: params
}
|> SharedKey.sign(
storage_account_name: Config.storage_account_name(),
storage_account_key: Config.storage_account_key()
storage_account_name: Config.storage_account_name(connection_params),
storage_account_key: Config.storage_account_key(connection_params)
)
end

Expand All @@ -233,24 +255,29 @@ defmodule Azurex.Blob do
iex> Azurex.Blob.list_blobs()
{:ok, "\uFEFF<?xml ...."}

iex> Azurex.Blob.list_blobs(storage_account_name: "name", storage_account_key: "key", container: "container")
{:ok, "\uFEFF<?xml ...."}

iex> Azurex.Blob.list_blobs()
{:error, %HTTPoison.Response{}}
"""
@spec list_blobs(optional_string) ::
@spec list_blobs(Config.config_overrides()) ::
{:ok, binary()}
| {:error, HTTPoison.AsyncResponse.t() | HTTPoison.Error.t() | HTTPoison.Response.t()}
def list_blobs(container \\ nil, params \\ []) do
def list_blobs(overrides \\ [], params \\ []) do
connection_params = Config.get_connection_params(overrides)

%HTTPoison.Request{
url: "#{Config.api_url()}/#{get_container(container)}",
url: get_url(connection_params),
params:
[
comp: "list",
restype: "container"
] ++ params
}
|> SharedKey.sign(
storage_account_name: Config.storage_account_name(),
storage_account_key: Config.storage_account_key()
storage_account_name: Config.storage_account_name(connection_params),
storage_account_key: Config.storage_account_key(connection_params)
)
|> HTTPoison.request()
|> case do
Expand All @@ -263,20 +290,20 @@ defmodule Azurex.Blob do
@doc """
Returns the url for a container (defaults to the one in `Azurex.Blob.Config`)
"""
@spec get_url(optional_string) :: String.t()
def get_url(container) do
"#{Config.api_url()}/#{get_container(container)}"
@spec get_url(keyword) :: String.t()
def get_url(connection_params) do
"#{Config.api_url(connection_params)}/#{get_container(connection_params)}"
end

@doc """
Returns the url for a file in a container (defaults to the one in `Azurex.Blob.Config`)
"""
@spec get_url(optional_string, String.t()) :: String.t()
def get_url(container, blob_name) do
"#{get_url(container)}/#{blob_name}"
@spec get_url(String.t(), keyword) :: String.t()
def get_url(blob_name, connection_params) do
"#{get_url(connection_params)}/#{blob_name}"
end

defp get_container(container) do
container || Config.default_container()
defp get_container(connection_params) do
Keyword.get(connection_params, :container) || Config.default_container()
end
end
22 changes: 12 additions & 10 deletions lib/azurex/blob/block.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,17 @@ defmodule Azurex.Blob.Block do

On success, returns an :ok tuple with the base64 encoded block_id.
"""
@spec put_block(String.t(), bitstring(), String.t(), list()) ::
@spec put_block(Config.config_overrides(), bitstring(), String.t(), list()) ::
{:ok, String.t()} | {:error, term()}
def put_block(container, chunk, name, params) do
def put_block(overrides \\ [], chunk, name, params) do
block_id = build_block_id()
content_type = "application/octet-stream"
params = [{:comp, "block"}, {:blockid, block_id} | params]
connection_params = Config.get_connection_params(overrides)

%HTTPoison.Request{
method: :put,
url: Blob.get_url(container, name),
url: Blob.get_url(name, connection_params),
params: params,
body: chunk,
headers: [
Expand All @@ -35,8 +36,8 @@ defmodule Azurex.Blob.Block do
]
}
|> SharedKey.sign(
storage_account_name: Config.storage_account_name(),
storage_account_key: Config.storage_account_key(),
storage_account_name: Config.storage_account_name(connection_params),
storage_account_key: Config.storage_account_key(connection_params),
content_type: content_type
)
|> HTTPoison.request()
Expand All @@ -52,12 +53,13 @@ defmodule Azurex.Blob.Block do

Block IDs should be base64 encoded, as returned by put_block/2.
"""
@spec put_block_list(list(), String.t(), String.t(), String.t() | nil, list()) ::
@spec put_block_list(list(), Config.config_overrides(), String.t(), String.t() | nil, list()) ::
:ok | {:error, term()}
def put_block_list(block_ids, container, name, blob_content_type, params) do
def put_block_list(block_ids, overrides \\ [], name, blob_content_type, params) do
params = [{:comp, "blocklist"} | params]
content_type = "text/plain; charset=UTF-8"
blob_content_type = blob_content_type || "application/octet-stream"
connection_params = Config.get_connection_params(overrides)

blocks =
block_ids
Expand All @@ -74,7 +76,7 @@ defmodule Azurex.Blob.Block do

%HTTPoison.Request{
method: :put,
url: Blob.get_url(container, name),
url: Blob.get_url(name, connection_params),
params: params,
body: body,
headers: [
Expand All @@ -83,8 +85,8 @@ defmodule Azurex.Blob.Block do
]
}
|> SharedKey.sign(
storage_account_name: Config.storage_account_name(),
storage_account_key: Config.storage_account_key(),
storage_account_name: Config.storage_account_name(connection_params),
storage_account_key: Config.storage_account_key(connection_params),
content_type: content_type
)
|> HTTPoison.request()
Expand Down
Loading
Loading