From 3305e0e96c98920689e8a70ce6ac4dfc6e163087 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Thu, 23 Nov 2023 17:19:00 +0700 Subject: [PATCH 1/6] Bootstrap can now choose least used node Signed-off-by: Zack Siri --- lib/uplink/clients/instellar/instance.ex | 3 +- lib/uplink/clients/lxd.ex | 4 + lib/uplink/clients/lxd/instance/manager.ex | 19 +++ lib/uplink/packages/install/execute.ex | 69 ++++++++-- lib/uplink/packages/instance/bootstrap.ex | 118 ++++++++++-------- lib/uplink/packages/instance/cleanup.ex | 43 +++++-- livebook/lxd.livemd | 32 +++++ .../clients/lxd/cluster/manager_test.exs | 2 +- 8 files changed, 209 insertions(+), 81 deletions(-) create mode 100644 livebook/lxd.livemd diff --git a/lib/uplink/clients/instellar/instance.ex b/lib/uplink/clients/instellar/instance.ex index 884c7da6..2df7569a 100644 --- a/lib/uplink/clients/instellar/instance.ex +++ b/lib/uplink/clients/instellar/instance.ex @@ -21,7 +21,8 @@ defmodule Uplink.Clients.Instellar.Instance do json: %{ "event" => %{ "name" => event_name, - "comment" => Keyword.get(options, :comment) + "comment" => Keyword.get(options, :comment), + "parameters" => Keyword.get(options, :parameters, %{}) } }, headers: Instellar.headers(install.deployment.hash) diff --git a/lib/uplink/clients/lxd.ex b/lib/uplink/clients/lxd.ex index 5b3914dd..04718101 100644 --- a/lib/uplink/clients/lxd.ex +++ b/lib/uplink/clients/lxd.ex @@ -20,6 +20,10 @@ defmodule Uplink.Clients.LXD do to: __MODULE__.Instance.Manager, as: :list + defdelegate list_instances(), + to: __MODULE__.Instance.Manager, + as: :list + defdelegate managed_network(), to: __MODULE__.Network.Manager, as: :managed diff --git a/lib/uplink/clients/lxd/instance/manager.ex b/lib/uplink/clients/lxd/instance/manager.ex index e4f16c95..77b713ad 100644 --- a/lib/uplink/clients/lxd/instance/manager.ex +++ b/lib/uplink/clients/lxd/instance/manager.ex @@ -6,6 +6,25 @@ defmodule Uplink.Clients.LXD.Instance.Manager do alias Clients.LXD alias LXD.Instance + + def list do + LXD.client() + |> Lexdee.list_instances(query: [{:recursion, 1}, {"all-projects", true}]) + |> case do + {:ok, %{body: instances}} -> + instances = + instances + |> Enum.map(fn instance -> + Instance.parse(instance) + end) + + instances + + error -> + error + end + end + def list(project) do LXD.client() |> Lexdee.list_instances(query: [recursion: 1, project: project]) diff --git a/lib/uplink/packages/install/execute.ex b/lib/uplink/packages/install/execute.ex index 21336949..d2351bf1 100644 --- a/lib/uplink/packages/install/execute.ex +++ b/lib/uplink/packages/install/execute.ex @@ -10,10 +10,13 @@ defmodule Uplink.Packages.Install.Execute do alias Members.Actor - alias Packages.{ - Install, - Metadata - } + alias Uplink.Packages + alias Uplink.Packages.Install + alias Uplink.Packages.Metadata + alias Uplink.Packages.Instance + + alias Uplink.Packages.Instance.Bootstrap + alias Uplink.Packages.Instance.Upgrade alias Clients.{ LXD, @@ -25,6 +28,14 @@ defmodule Uplink.Packages.Install.Execute do @state ~s(executing) + @task_supervisor Application.compile_env(:uplink, :task_supervisor) || + Task.Supervisor + + @transition_parameters %{ + "from" => "uplink", + "trigger" => false + } + def perform(%Oban.Job{ args: %{"install_id" => install_id, "actor_id" => actor_id} }) do @@ -54,16 +65,13 @@ defmodule Uplink.Packages.Install.Execute do project = Packages.get_project_name(client, metadata) - existing_instances_name = + existing_instances = LXD.list_instances(project) |> Enum.filter(&only_uplink_instance/1) - |> Enum.map(fn instance -> - instance.name - end) jobs = instances - |> Enum.map(&choose_execution_path(&1, existing_instances_name, state)) + |> Enum.map(&choose_execution_path(&1, existing_instances, state)) {:ok, jobs} end @@ -75,11 +83,46 @@ defmodule Uplink.Packages.Install.Execute do end defp choose_execution_path(instance, existing_instances, state) do + existing_instances_name = Enum.map(existing_instances, & &1.name) + event_name = - if instance.slug in existing_instances, do: "upgrade", else: "boot" + if instance.slug in existing_instances_name, do: "upgrade", else: "boot" - Instellar.transition_instance(instance.slug, state.install, event_name, - comment: "[Uplink.Packages.Install.Execute]" - ) + @task_supervisor.async_nolink(Uplink.TaskSupervisor, fn -> + Instellar.transition_instance(instance.slug, state.install, event_name, + comment: "[Uplink.Packages.Install.Execute]", + parameters: @transition_parameters + ) + end) + + case event_name do + "upgrade" -> + existing_instance = + Enum.find(existing_instances, &(&1.name == instance.slug)) + + %{ + "instance" => %{ + "slug" => instance.slug, + "node" => %{ + "slug" => existing_instance.location + } + }, + "install_id" => state.install.id, + "actor_id" => state.actor.id + } + |> Upgrade.new() + |> Oban.insert() + + "boot" -> + %{ + "instance" => %{ + "slug" => instance.slug, + }, + "install_id" => state.install.id, + "actor_id" => state.actor.id + } + Bootstrap.new(job_args) + |> Oban.insert() + end end end diff --git a/lib/uplink/packages/instance/bootstrap.ex b/lib/uplink/packages/instance/bootstrap.ex index a4859fa4..4ff8df31 100644 --- a/lib/uplink/packages/instance/bootstrap.ex +++ b/lib/uplink/packages/instance/bootstrap.ex @@ -3,26 +3,23 @@ defmodule Uplink.Packages.Instance.Bootstrap do queue: :instance, max_attempts: 1 - alias Uplink.{ - Members, - Clients, - Packages, - Repo - } - - alias Clients.{ - LXD, - Instellar - } - - alias LXD.Cluster - alias Cluster.Member - - alias Members.Actor - - alias Packages.{ - Install, - Instance + alias Uplink.Repo + alias Uplink.Members + alias Uplink.Members.Actor + + alias Uplink.Packages + alias Uplink.Packages.Install + alias Uplink.Packages.Instance + alias Uplink.Packages.Instance.Cleanup + + alias Uplink.Clients.LXD + alias Uplink.Clients.Instellar + alias Uplink.Clients.LXD.Cluster + alias Uplink.Clients.LXD.Cluster.Member + + @transition_parameters %{ + "from" => "uplink", + "trigger" => false } @default_params %{ @@ -30,17 +27,18 @@ defmodule Uplink.Packages.Instance.Bootstrap do "type" => "container" } + @task_supervisor Application.compile_env(:uplink, :task_supervisor) || + Task.Supervisor + import Ecto.Query, only: [preload: 2] def perform(%Oban.Job{ args: %{ - "instance" => %{ - "slug" => name, - "node" => %{ - "slug" => node_name - } - }, + "instance" => + %{ + "slug" => name, + } = instance_params, "install_id" => install_id, "actor_id" => actor_id } = job_args @@ -53,18 +51,35 @@ defmodule Uplink.Packages.Instance.Bootstrap do |> preload([:deployment]) |> Repo.get(install_id) + cluster_member_names = + LXD.list_cluster_members() + |> Enum.map(& &1.server_name) + + frequency = + LXD.list_instances() + |> Enum.frequencies_by(fn instance -> + instance.location + end) + + selected_member = + LXD.list_cluster_members() + |> Enum.min_by(fn m -> frequency[m.server_name] || 0 end) + with %{metadata: %{channel: channel} = metadata} <- Packages.build_install_state(install, actor), members when is_list(members) <- LXD.list_cluster_members(), %Member{architecture: architecture} <- members |> Enum.find(fn member -> - member.server_name == node_name + member.server_name == selected_member.server_name end), - {:ok, _transition} <- - Instellar.transition_instance(name, install, "boot", - comment: "[Uplink.Packages.Instance.Bootstrap]" - ) do + %Task{} <- + @task_supervisor.async_nolink(Uplink.TaskSupervisor, fn -> + Instellar.transition_instance(name, install, "boot", + comment: "[Uplink.Packages.Instance.Bootstrap]", + parameters: @transition_parameters + ) + end) do profile_name = Packages.profile_name(metadata) package = channel.package @@ -130,15 +145,28 @@ defmodule Uplink.Packages.Instance.Bootstrap do |> Oban.insert() {:error, error} -> - # will put instance in failing - Instellar.transition_instance(name, install, "fail", - comment: "[Uplink.Packages.Instance.Bootstrap] #{inspect(error)}" - ) - |> handle_event(job_args) + @task_supervisor.async_nolink(Uplink.TaskSupervisor, fn -> + Instellar.transition_instance(name, install, "fail", + comment: "[Uplink.Packages.Instance.Bootstrap] #{inspect(error)}", + parameters: @transition_parameters + ) + end) + + instance_params = Map.put(instance_params, "current_state", "failing") + + %{ + "instance" => instance_params, + "install_id" => install_id, + "actor_id" => actor_id + } + |> Cleanup.new() + |> Oban.insert() end else {:error, error} -> - Packages.transition_install_with(install, actor, "fail", comment: error) + Packages.transition_install_with(install, actor, "fail", + comment: "#{inspect(error)}" + ) nil -> Packages.transition_install_with(install, actor, "fail", @@ -146,20 +174,4 @@ defmodule Uplink.Packages.Instance.Bootstrap do ) end end - - defp handle_event({:ok, %{"name" => "fail"}}, %{ - "instance" => instance_params, - "install_id" => install_id, - "actor_id" => actor_id - }) do - job_args = %{ - "instance" => Map.merge(instance_params, %{"current_state" => "failing"}), - "install_id" => install_id, - "actor_id" => actor_id - } - - job_args - |> Packages.Instance.Cleanup.new() - |> Oban.insert() - end end diff --git a/lib/uplink/packages/instance/cleanup.ex b/lib/uplink/packages/instance/cleanup.ex index 00c54d9f..90c2fdb1 100644 --- a/lib/uplink/packages/instance/cleanup.ex +++ b/lib/uplink/packages/instance/cleanup.ex @@ -17,7 +17,15 @@ defmodule Uplink.Packages.Instance.Cleanup do } alias Packages.{ - Install + Install, + Instance + } + + alias Instance.Bootstrap + + @transition_parameters %{ + "from" => "uplink", + "trigger" => false } @cleanup_mappings %{ @@ -29,10 +37,7 @@ defmodule Uplink.Packages.Instance.Cleanup do args: %{ "instance" => %{ - "slug" => name, - "node" => %{ - "slug" => _node_name - } + "slug" => name }, "install_id" => install_id, "actor_id" => actor_id @@ -87,13 +92,25 @@ defmodule Uplink.Packages.Instance.Cleanup do end defp finalize(name, install, "deactivate_and_boot", args) do - comment = Map.get(args, "comment", "no comment") - - with {:ok, _transition} <- - Instellar.transition_instance(name, install, "deactivate", - comment: "[Uplink.Packages.Instance.Cleanup] #{inspect(comment)}" - ) do - Instellar.transition_instance(name, install, "boot", comment: comment) - end + Uplink.TaskSupervisor + |> @task_supervisor.async_nolink(fn -> + comment = Map.get(args, "comment", "no comment") + + with {:ok, _transition} <- + Instellar.transition_instance(name, install, "deactivate", + comment: + "[Uplink.Packages.Instance.Cleanup] #{inspect(comment)}", + parameters: @transition_parameters + ) do + Instellar.transition_instance(name, install, "boot", + comment: "[Uplink.Packages.Instance.Cleanup] #{inspect(comment)}", + parameters: @transition_parameters + ) + end + end) + + args + |> Bootstrap.new() + |> Oban.insert() end end diff --git a/livebook/lxd.livemd b/livebook/lxd.livemd new file mode 100644 index 00000000..3926195e --- /dev/null +++ b/livebook/lxd.livemd @@ -0,0 +1,32 @@ +# LXD Compute least used member + +## Update cache + +We need to update the cache with the key `:self` because that's where the credential for uplink's lxd is stored. + +```elixir +cert = File.read!(Path.expand("~/.config/lxc/client.crt")) +key = File.read!(Path.expand("~/.config/lxc/client.key")) + +credential = %{ + "endpoint" => "https://198.19.249.83:8443", + "certificate" => cert, + "private_key" => key +} + +Uplink.Cache.put(:self, %{"credential" => credential}) +``` + +## Query instances + +```elixir +alias Uplink.Clients.LXD + +frequency = + LXD.list_instances() + |> Enum.frequencies_by(fn i -> i.location end) + |> IO.inspect() + +LXD.list_cluster_members() +|> Enum.min_by(fn m -> frequency[m.server_name] || 0 end) +``` diff --git a/test/uplink/clients/lxd/cluster/manager_test.exs b/test/uplink/clients/lxd/cluster/manager_test.exs index 7f741e3a..12efebdd 100644 --- a/test/uplink/clients/lxd/cluster/manager_test.exs +++ b/test/uplink/clients/lxd/cluster/manager_test.exs @@ -38,7 +38,7 @@ defmodule Uplink.Clients.LXD.Cluster.ManagerTest do end) assert [member1] = Manager.list_members() - assert %Cluster.Member{} = member1 + assert %Cluster.Member{} = IO.inspect(member1) end end end From c1fd3e93ee657ee64170f279930a018ddd75eb9b Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Thu, 23 Nov 2023 17:35:45 +0700 Subject: [PATCH 2/6] Add node to parameter Signed-off-by: Zack Siri --- lib/uplink/packages/instance/bootstrap.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/uplink/packages/instance/bootstrap.ex b/lib/uplink/packages/instance/bootstrap.ex index 4ff8df31..0d51b087 100644 --- a/lib/uplink/packages/instance/bootstrap.ex +++ b/lib/uplink/packages/instance/bootstrap.ex @@ -65,6 +65,8 @@ defmodule Uplink.Packages.Instance.Bootstrap do LXD.list_cluster_members() |> Enum.min_by(fn m -> frequency[m.server_name] || 0 end) + transition_parameters = Map.put(@transition_parameters, "node", selected_member.server_name) + with %{metadata: %{channel: channel} = metadata} <- Packages.build_install_state(install, actor), members when is_list(members) <- LXD.list_cluster_members(), @@ -77,7 +79,7 @@ defmodule Uplink.Packages.Instance.Bootstrap do @task_supervisor.async_nolink(Uplink.TaskSupervisor, fn -> Instellar.transition_instance(name, install, "boot", comment: "[Uplink.Packages.Instance.Bootstrap]", - parameters: @transition_parameters + parameters: transition_parameters ) end) do profile_name = Packages.profile_name(metadata) From 008bd03184fe92f71534fa1eaddc4ab2017157aa Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Fri, 24 Nov 2023 14:41:29 +0700 Subject: [PATCH 3/6] Clean up all instance lifecycle workers Signed-off-by: Zack Siri --- lib/uplink/clients/caddy/config/builder.ex | 40 +++-- lib/uplink/clients/lxd.ex | 2 +- lib/uplink/clients/lxd/instance/manager.ex | 1 - lib/uplink/packages.ex | 4 + lib/uplink/packages/install/execute.ex | 50 ++++-- lib/uplink/packages/install/manager.ex | 11 ++ lib/uplink/packages/instance/bootstrap.ex | 66 ++++--- lib/uplink/packages/instance/cleanup.ex | 41 +++-- lib/uplink/packages/instance/finalize.ex | 40 +++++ lib/uplink/packages/instance/install.ex | 105 +++++++---- lib/uplink/packages/instance/upgrade.ex | 168 +++++++++++------- test/scenarios/mocks.ex | 2 +- .../clients/lxd/cluster/manager_test.exs | 2 +- .../uplink/packages/instance/upgrade_test.exs | 2 +- 14 files changed, 359 insertions(+), 175 deletions(-) create mode 100644 lib/uplink/packages/instance/finalize.ex diff --git a/lib/uplink/clients/caddy/config/builder.ex b/lib/uplink/clients/caddy/config/builder.ex index 75aa4344..45033f68 100644 --- a/lib/uplink/clients/caddy/config/builder.ex +++ b/lib/uplink/clients/caddy/config/builder.ex @@ -1,14 +1,12 @@ defmodule Uplink.Clients.Caddy.Config.Builder do - alias Uplink.{ - Clients, - Packages, - Repo - } + alias Uplink.Repo + alias Uplink.Cache - alias Clients.Caddy - alias Caddy.Admin - alias Caddy.Apps - alias Caddy.Storage + alias Uplink.Packages + + alias Uplink.Clients.Caddy.Admin + alias Uplink.Clients.Caddy.Apps + alias Uplink.Clients.Caddy.Storage def new do install_states = @@ -94,7 +92,10 @@ defmodule Uplink.Clients.Caddy.Config.Builder do end defp build_route( - %{install: %{deployment: %{app: _app}}, metadata: metadata} = _state + %{ + install: %{id: install_id, deployment: %{app: _app}}, + metadata: metadata + } = _state ) do main_routing = Map.get(metadata.main_port, :routing) @@ -112,6 +113,8 @@ defmodule Uplink.Clients.Caddy.Config.Builder do "installation_#{metadata.id}" end + valid_instances = find_valid_instances(metadata.instances, install_id) + main_route = %{ group: main_group, match: [ @@ -138,7 +141,7 @@ defmodule Uplink.Clients.Caddy.Config.Builder do } }, upstreams: - Enum.map(metadata.instances, fn instance -> + Enum.map(valid_instances, fn instance -> %{ dial: "#{instance.slug}:#{metadata.main_port.target}", max_requests: 80 @@ -196,7 +199,7 @@ defmodule Uplink.Clients.Caddy.Config.Builder do } }, upstreams: - Enum.map(metadata.instances, fn instance -> + Enum.map(valid_instances, fn instance -> %{ dial: "#{instance.slug}:#{port.target}", max_requests: 80 @@ -209,4 +212,17 @@ defmodule Uplink.Clients.Caddy.Config.Builder do [main_route | sub_routes] end + + defp find_valid_instances(instances, install_id) do + completed_instances = Cache.get({:install, install_id, "completed"}) + + if is_list(completed_instances) and Enum.count(completed_instances) > 0 do + instances + |> Enum.filter(fn instance -> + instance.slug in completed_instances + end) + else + instances + end + end end diff --git a/lib/uplink/clients/lxd.ex b/lib/uplink/clients/lxd.ex index 04718101..6e78fb96 100644 --- a/lib/uplink/clients/lxd.ex +++ b/lib/uplink/clients/lxd.ex @@ -20,7 +20,7 @@ defmodule Uplink.Clients.LXD do to: __MODULE__.Instance.Manager, as: :list - defdelegate list_instances(), + defdelegate list_instances(), to: __MODULE__.Instance.Manager, as: :list diff --git a/lib/uplink/clients/lxd/instance/manager.ex b/lib/uplink/clients/lxd/instance/manager.ex index 77b713ad..47178ee7 100644 --- a/lib/uplink/clients/lxd/instance/manager.ex +++ b/lib/uplink/clients/lxd/instance/manager.ex @@ -6,7 +6,6 @@ defmodule Uplink.Clients.LXD.Instance.Manager do alias Clients.LXD alias LXD.Instance - def list do LXD.client() |> Lexdee.list_instances(query: [{:recursion, 1}, {"all-projects", true}]) diff --git a/lib/uplink/packages.ex b/lib/uplink/packages.ex index 4152a807..0553882a 100644 --- a/lib/uplink/packages.ex +++ b/lib/uplink/packages.ex @@ -37,6 +37,10 @@ defmodule Uplink.Packages do to: Install.Manager, as: :transition_with + defdelegate maybe_mark_install_complete(install, actor), + to: Install.Manager, + as: :maybe_mark_complete + alias __MODULE__.Deployment defdelegate get_deployment(id), diff --git a/lib/uplink/packages/install/execute.ex b/lib/uplink/packages/install/execute.ex index d2351bf1..06d8ce8a 100644 --- a/lib/uplink/packages/install/execute.ex +++ b/lib/uplink/packages/install/execute.ex @@ -1,14 +1,13 @@ defmodule Uplink.Packages.Install.Execute do use Oban.Worker, queue: :install, max_attempts: 1 - alias Uplink.{ - Clients, - Members, - Packages, - Repo - } + alias Uplink.Repo + alias Uplink.Cache + + alias Uplink.Clients.LXD + alias Uplink.Clients.Instellar - alias Members.Actor + alias Uplink.Members.Actor alias Uplink.Packages alias Uplink.Packages.Install @@ -18,11 +17,6 @@ defmodule Uplink.Packages.Install.Execute do alias Uplink.Packages.Instance.Bootstrap alias Uplink.Packages.Instance.Upgrade - alias Clients.{ - LXD, - Instellar - } - import Ecto.Query, only: [where: 3, preload: 2] @@ -69,6 +63,14 @@ defmodule Uplink.Packages.Install.Execute do LXD.list_instances(project) |> Enum.filter(&only_uplink_instance/1) + Cache.put({:install, state.install.id, "completed"}, [], + ttl: :timer.hours(24) + ) + + Cache.put({:install, state.install.id, "executing"}, [], + ttl: :timer.hours(24) + ) + jobs = instances |> Enum.map(&choose_execution_path(&1, existing_instances, state)) @@ -95,6 +97,23 @@ defmodule Uplink.Packages.Install.Execute do ) end) + Cache.transaction( + [keys: [{:install, state.install.id, "executing"}]], + fn -> + Cache.get_and_update( + {:install, state.install.id, "executing"}, + fn current_value -> + executing_instances = + if current_value, + do: current_value ++ [instance.slug], + else: [instance.slug] + + {current_value, Enum.uniq(executing_instances)} + end + ) + end + ) + case event_name do "upgrade" -> existing_instance = @@ -114,14 +133,13 @@ defmodule Uplink.Packages.Install.Execute do |> Oban.insert() "boot" -> - %{ + Bootstrap.new(%{ "instance" => %{ - "slug" => instance.slug, + "slug" => instance.slug }, "install_id" => state.install.id, "actor_id" => state.actor.id - } - Bootstrap.new(job_args) + }) |> Oban.insert() end end diff --git a/lib/uplink/packages/install/manager.ex b/lib/uplink/packages/install/manager.ex index 3c80873d..66607d18 100644 --- a/lib/uplink/packages/install/manager.ex +++ b/lib/uplink/packages/install/manager.ex @@ -52,6 +52,17 @@ defmodule Uplink.Packages.Install.Manager do |> Repo.one() end + def maybe_mark_complete(%Install{} = install, actor) do + completed_instances = Cache.get({:install, install.id, "completed"}) + executing_instances = Cache.get({:install, install.id, "executing"}) + + if Enum.count(completed_instances) == Enum.count(executing_instances) do + Packages.transition_install_with(install, actor, "complete") + else + {:ok, :still_executing} + end + end + @spec build_state(%Install{}, %Actor{} | nil) :: %{ install: %Install{}, metadata: %Metadata{}, diff --git a/lib/uplink/packages/instance/bootstrap.ex b/lib/uplink/packages/instance/bootstrap.ex index 0d51b087..61f8d7bc 100644 --- a/lib/uplink/packages/instance/bootstrap.ex +++ b/lib/uplink/packages/instance/bootstrap.ex @@ -37,7 +37,7 @@ defmodule Uplink.Packages.Instance.Bootstrap do %{ "instance" => %{ - "slug" => name, + "slug" => name } = instance_params, "install_id" => install_id, "actor_id" => actor_id @@ -51,21 +51,33 @@ defmodule Uplink.Packages.Instance.Bootstrap do |> preload([:deployment]) |> Repo.get(install_id) - cluster_member_names = + cluster_member_names = LXD.list_cluster_members() |> Enum.map(& &1.server_name) - frequency = + frequency = LXD.list_instances() - |> Enum.frequencies_by(fn instance -> - instance.location + |> Enum.frequencies_by(fn instance -> + instance.location end) - selected_member = + selected_member = LXD.list_cluster_members() |> Enum.min_by(fn m -> frequency[m.server_name] || 0 end) - transition_parameters = Map.put(@transition_parameters, "node", selected_member.server_name) + transition_parameters = + Map.put(@transition_parameters, "node", selected_member.server_name) + + Uplink.TaskSupervisor + |> @task_supervisor.async_nolink( + fn -> + Instellar.transition_instance(name, install, "boot", + comment: "[Uplink.Packages.Instance.Bootstrap] Starting bootstrap...", + parameters: transition_parameters + ) + end, + shutdown: 30_000 + ) with %{metadata: %{channel: channel} = metadata} <- Packages.build_install_state(install, actor), @@ -74,13 +86,6 @@ defmodule Uplink.Packages.Instance.Bootstrap do members |> Enum.find(fn member -> member.server_name == selected_member.server_name - end), - %Task{} <- - @task_supervisor.async_nolink(Uplink.TaskSupervisor, fn -> - Instellar.transition_instance(name, install, "boot", - comment: "[Uplink.Packages.Instance.Bootstrap]", - parameters: transition_parameters - ) end) do profile_name = Packages.profile_name(metadata) package = channel.package @@ -128,31 +133,38 @@ defmodule Uplink.Packages.Instance.Bootstrap do formation_instance = Formation.new_lxd_instance(formation_instance_params) client - |> Formation.lxd_create(node_name, instance_params, project: project_name) + |> Formation.lxd_create(selected_member.server_name, instance_params, + project: project_name + ) |> Formation.lxd_start(name, project: project_name) |> Formation.setup_lxd_instance(formation_instance) |> case do {:ok, _message} -> %{ - instance: %{ - slug: name, - node: %{ - slug: node_name + "instance" => %{ + "slug" => name, + "node" => %{ + "slug" => selected_member.server_name } }, - install_id: install.id, - actor_id: actor_id + "install_id" => install.id, + "actor_id" => actor_id } |> Instance.Install.new() |> Oban.insert() {:error, error} -> - @task_supervisor.async_nolink(Uplink.TaskSupervisor, fn -> - Instellar.transition_instance(name, install, "fail", - comment: "[Uplink.Packages.Instance.Bootstrap] #{inspect(error)}", - parameters: @transition_parameters - ) - end) + Uplink.TaskSupervisor + |> @task_supervisor.async_nolink( + fn -> + Instellar.transition_instance(name, install, "fail", + comment: + "[Uplink.Packages.Instance.Bootstrap] #{inspect(error)}", + parameters: transition_parameters + ) + end, + shutdown: 30_000 + ) instance_params = Map.put(instance_params, "current_state", "failing") diff --git a/lib/uplink/packages/instance/cleanup.ex b/lib/uplink/packages/instance/cleanup.ex index 90c2fdb1..ed86492a 100644 --- a/lib/uplink/packages/instance/cleanup.ex +++ b/lib/uplink/packages/instance/cleanup.ex @@ -33,6 +33,9 @@ defmodule Uplink.Packages.Instance.Cleanup do "deactivating" => "off" } + @task_supervisor Application.compile_env(:uplink, :task_supervisor) || + Task.Supervisor + def perform(%Oban.Job{ args: %{ @@ -81,33 +84,37 @@ defmodule Uplink.Packages.Instance.Cleanup do "instance" => %{"current_state" => current_state} } = args ) do - event_name = Map.get(@cleanup_mappings, current_state, "off") - comment = Map.get(args, "comment", "no comment") - Caddy.schedule_config_reload(install) - Instellar.transition_instance(name, install, event_name, - comment: "[Uplink.Packages.Instance.Cleanup] #{inspect(comment)}" + Uplink.TaskSupervisor + |> @task_supervisor.async_nolink( + fn -> + event_name = Map.get(@cleanup_mappings, current_state, "off") + comment = Map.get(args, "comment", "no comment") + + Instellar.transition_instance(name, install, event_name, + comment: "[Uplink.Packages.Instance.Cleanup] #{inspect(comment)}" + ) + end, + shutdown: 30_000 ) + + {:ok, :cleaned} end defp finalize(name, install, "deactivate_and_boot", args) do Uplink.TaskSupervisor - |> @task_supervisor.async_nolink(fn -> - comment = Map.get(args, "comment", "no comment") - - with {:ok, _transition} <- - Instellar.transition_instance(name, install, "deactivate", - comment: - "[Uplink.Packages.Instance.Cleanup] #{inspect(comment)}", - parameters: @transition_parameters - ) do - Instellar.transition_instance(name, install, "boot", + |> @task_supervisor.async_nolink( + fn -> + comment = Map.get(args, "comment", "no comment") + + Instellar.transition_instance(name, install, "deactivate", comment: "[Uplink.Packages.Instance.Cleanup] #{inspect(comment)}", parameters: @transition_parameters ) - end - end) + end, + shutdown: 30_000 + ) args |> Bootstrap.new() diff --git a/lib/uplink/packages/instance/finalize.ex b/lib/uplink/packages/instance/finalize.ex new file mode 100644 index 00000000..5513db7c --- /dev/null +++ b/lib/uplink/packages/instance/finalize.ex @@ -0,0 +1,40 @@ +defmodule Uplink.Packages.Instance.Finalize do + use Oban.Worker, queue: :instance, max_attempts: 5 + + alias Uplink.Repo + alias Uplink.Clients.Instellar + alias Uplink.Packages.Install + + import Ecto.Query, only: [preload: 2] + + @transition_parameters %{ + "from" => "uplink", + "trigger" => false + } + + def perform(%Oban.Job{ + args: + %{ + "instance" => %{"slug" => name} = instance_params, + "comment" => comment, + "install_id" => install_id, + "actor_id" => actor_id + } = args + }) do + %Install{} = + install = + Install + |> preload([:deployment]) + |> Repo.get(install_id) + + node = Map.get(instance_params, "node", %{}) + + transition_parameters = + Map.put(@transition_parameters, "node", node["slug"]) + + Instellar.transition_instance(name, install, "complete", + comment: comment, + parameters: transition_parameters + ) + end +end diff --git a/lib/uplink/packages/instance/install.ex b/lib/uplink/packages/instance/install.ex index de64b820..35f98957 100644 --- a/lib/uplink/packages/instance/install.ex +++ b/lib/uplink/packages/instance/install.ex @@ -1,34 +1,37 @@ defmodule Uplink.Packages.Instance.Install do - use Oban.Worker, queue: :instance, max_attempts: 5 + use Oban.Worker, queue: :instance, max_attempts: 3 - alias Uplink.{ - Clients, - Packages, - Members, - Repo - } + alias Uplink.Repo + alias Uplink.Cache - alias Members.Actor + alias Uplink.Clients + alias Uplink.Clients.LXD + alias Uplink.Clients.Caddy + alias Uplink.Clients.Instellar - alias Clients.{ - LXD, - Caddy, - Instellar - } + alias Uplink.Packages + alias Uplink.Packages.Install + alias Uplink.Packages.Instance - alias Packages.Install + alias Uplink.Members.Actor import Ecto.Query, only: [preload: 2] + @transition_parameters %{ + "from" => "uplink", + "trigger" => false + } + + @task_supervisor Application.compile_env(:uplink, :task_supervisor) || + Task.Supervisor + def perform( %Oban.Job{ args: %{ - "instance" => %{ - "slug" => name, - "node" => %{ - "slug" => _node_name - } - }, + "instance" => + %{ + "slug" => name + } = instance_params, "install_id" => install_id, "actor_id" => actor_id } @@ -68,29 +71,69 @@ defmodule Uplink.Packages.Instance.Install do formation_instance = Formation.new_lxd_instance(formation_instance_params) + node = Map.get(instance_params, "node", %{}) + + transition_parameters = + Map.put(@transition_parameters, "node", node["slug"]) + + Uplink.TaskSupervisor + |> @task_supervisor.async_nolink( + fn -> + Instellar.transition_instance(name, install, "boot", + comment: + "[Uplink.Packages.Instance.Install] Installing #{package.slug}...", + parameters: transition_parameters + ) + end, + shutdown: 30_000 + ) + client |> Formation.add_package_and_restart_lxd_instance(formation_instance) |> case do {:ok, add_package_output} -> + Cache.transaction([keys: [{:install, install_id, "completed"}]], fn -> + Cache.get_and_update( + {:install, install_id, "completed"}, + fn current_value -> + completed_instances = + if current_value, do: current_value ++ [name], else: [name] + + {current_value, Enum.uniq(completed_instances)} + end + ) + end) + Caddy.schedule_config_reload(install) - Instellar.transition_instance( - formation_instance.slug, - install, - "complete", - comment: add_package_output - ) + Packages.maybe_mark_install_complete(install, actor) + + %{ + "instance" => instance_params, + "comment" => add_package_output, + "install_id" => install_id, + "actor_id" => actor_id + } + |> Instance.Finalize.new() + |> Oban.insert() {:error, %{"error" => "Instance is not running"}} -> {:snooze, 5} {:error, error} -> if job.attempt == job.max_attempts do - Instellar.transition_instance( - formation_instance.slug, - install, - "fail", - comment: "[Uplink.Packages.Instance.Install] #{inspect(error)}" + Uplink.TaskSupervisor + |> @task_supervisor.async_nolink( + fn -> + Instellar.transition_instance( + formation_instance.slug, + install, + "fail", + comment: "[Uplink.Packages.Instance.Install] #{inspect(error)}", + parameters: @transition_parameters + ) + end, + shutdown: 30_000 ) end diff --git a/lib/uplink/packages/instance/upgrade.ex b/lib/uplink/packages/instance/upgrade.ex index bb48bd4c..9a04a177 100644 --- a/lib/uplink/packages/instance/upgrade.ex +++ b/lib/uplink/packages/instance/upgrade.ex @@ -3,33 +3,36 @@ defmodule Uplink.Packages.Instance.Upgrade do queue: :instance, max_attempts: 1 - alias Uplink.{ - Members, - Clients, - Packages, - Repo - } + alias Uplink.Repo + alias Uplink.Cache - alias Members.Actor + alias Uplink.Packages + alias Uplink.Packages.Install + alias Uplink.Packages.Instance - alias Packages.{ - Install, - Instance - } + alias Uplink.Members.Actor - alias Clients.{ - LXD, - Instellar - } + alias Uplink.Clients.LXD + alias Uplink.Clients.Instellar + alias Uplink.Clients.Caddy import Ecto.Query, only: [limit: 2, where: 3, preload: 2, order_by: 2] + @transition_parameters %{ + "from" => "uplink", + "trigger" => false + } + + @task_supervisor Application.compile_env(:uplink, :task_supervisor) || + Task.Supervisor + def perform( %Oban.Job{ args: %{ - "instance" => %{ - "slug" => name - }, + "instance" => + %{ + "slug" => name + } = instance_params, "install_id" => install_id, "actor_id" => actor_id } @@ -43,29 +46,37 @@ defmodule Uplink.Packages.Instance.Upgrade do |> preload([:deployment]) |> Repo.get(install_id) - with %{metadata: %{channel: channel} = metadata} <- - Packages.build_install_state(install, actor), - {:ok, _transition} <- - Instellar.transition_instance(name, install, "upgrade", - comment: "[Uplink.Packages.Instance.Upgrade]" - ) do - client = LXD.client() - - project_name = Packages.get_project_name(client, metadata) - - Formation.new_lxd_instance(%{ - project: project_name, - slug: name, - repositories: [], - packages: [ - %{ - slug: channel.package.slug - } - ] - }) - |> validate_stack(install) - |> handle_upgrade(job, actor) - end + node = Map.get(instance_params, "node", %{}) + + transition_parameters = + Map.put(@transition_parameters, "node", node["slug"]) + + %{metadata: %{channel: channel} = metadata} = + Packages.build_install_state(install, actor) + + client = LXD.client() + project_name = Packages.get_project_name(client, metadata) + + @task_supervisor.async_nolink(Uplink.TaskSupervisor, fn -> + Instellar.transition_instance(name, install, "upgrade", + comment: + "[Uplink.Packages.Instance.Upgrade] Upgrading #{channel.package.slug} on #{name}...", + parameters: transition_parameters + ) + end) + + Formation.new_lxd_instance(%{ + project: project_name, + slug: name, + repositories: [], + packages: [ + %{ + slug: channel.package.slug + } + ] + }) + |> validate_stack(install) + |> handle_upgrade(job, actor) end defp validate_stack( @@ -101,31 +112,65 @@ defmodule Uplink.Packages.Instance.Upgrade do defp handle_upgrade( {:upgrade, formation_instance, install}, - %Job{} = job, + %Job{args: %{"instance" => instance_params}} = job, actor ) do + node = Map.get(instance_params, "node", %{}) + + transition_parameters = + Map.put(@transition_parameters, "node", node["slug"]) + LXD.client() |> Formation.lxd_upgrade_alpine_package(formation_instance) |> case do {:ok, upgrade_package_output} -> - Instellar.transition_instance( - formation_instance.slug, - install, - "complete", - comment: upgrade_package_output - ) - - maybe_mark_install_complete(install, actor) + Cache.transaction([keys: [{:install, install.id, "completed"}]], fn -> + Cache.get_and_update( + {:install, install.id, "completed"}, + fn current_value -> + completed_instances = + if current_value, + do: current_value ++ [formation_instance.slug], + else: [formation_instance.slug] + + {current_value, Enum.uniq(completed_instances)} + end + ) + end) + + Caddy.schedule_config_reload(install) + + Packages.maybe_mark_install_complete(install, actor) + + %{ + "instance" => instance_params, + "comment" => upgrade_package_output, + "install_id" => install.id, + "actor_id" => actor.id + } + |> Instance.Finalize.new() + |> Oban.insert() + + {:ok, upgrade_package_output} {:error, %{"err" => "Failed to retrieve PID of executing child process"}} -> - Instellar.transition_instance( - formation_instance.slug, - install, - "revert", - comment: - "Reverting please restart the underlying node and try upgrading again." + Uplink.TaskSupervisor + |> @task_supervisor.async_nolink( + fn -> + Instellar.transition_instance( + formation_instance.slug, + install, + "revert", + comment: + "Reverting please restart the underlying node and try upgrading again.", + parameters: transition_parameters + ) + end, + shutdown: 30_000 ) + {:ok, :reverted} + {:error, error} -> handle_error(error, job) end @@ -153,15 +198,4 @@ defmodule Uplink.Packages.Instance.Upgrade do |> Instance.Cleanup.new() |> Oban.insert() end - - defp maybe_mark_install_complete(install, actor) do - with {:ok, %{"current_state" => "synced"}} <- - Instellar.deployment_metadata(install), - {:ok, transition} <- - Packages.transition_install_with(install, actor, "complete") do - {:ok, transition} - else - _ -> :ok - end - end end diff --git a/test/scenarios/mocks.ex b/test/scenarios/mocks.ex index 0369cbb5..3f6454a5 100644 --- a/test/scenarios/mocks.ex +++ b/test/scenarios/mocks.ex @@ -1,6 +1,6 @@ Mox.defmock(Uplink.Drivers.Bucket.AwsMock, for: Uplink.Drivers.Behaviour) defmodule Uplink.TaskSupervisorMock do - def async_nolink(_supervisor, fun), + def async_nolink(_supervisor, fun, _options \\ []), do: fun.() end diff --git a/test/uplink/clients/lxd/cluster/manager_test.exs b/test/uplink/clients/lxd/cluster/manager_test.exs index 12efebdd..7f741e3a 100644 --- a/test/uplink/clients/lxd/cluster/manager_test.exs +++ b/test/uplink/clients/lxd/cluster/manager_test.exs @@ -38,7 +38,7 @@ defmodule Uplink.Clients.LXD.Cluster.ManagerTest do end) assert [member1] = Manager.list_members() - assert %Cluster.Member{} = IO.inspect(member1) + assert %Cluster.Member{} = member1 end end end diff --git a/test/uplink/packages/instance/upgrade_test.exs b/test/uplink/packages/instance/upgrade_test.exs index 094653a0..c2ea7aff 100644 --- a/test/uplink/packages/instance/upgrade_test.exs +++ b/test/uplink/packages/instance/upgrade_test.exs @@ -517,7 +517,7 @@ defmodule Uplink.Packages.Instance.UpgradeTest do end ) - assert {:ok, %{"id" => _, "name" => "revert"}} = + assert {:ok, :reverted} = perform_job(Upgrade, %{ instance: %{ slug: instance_slug, From 99058dd88d2181d3cec1bb2acc03de8a26638d4f Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Fri, 24 Nov 2023 16:00:01 +0700 Subject: [PATCH 4/6] Clean up tests Signed-off-by: Zack Siri --- lib/uplink/packages/install/manager.ex | 2 +- lib/uplink/packages/instance/upgrade.ex | 2 - .../packages/instance/bootstrap_test.exs | 8 +++ .../uplink/packages/instance/install_test.exs | 34 +++++++++- .../uplink/packages/instance/upgrade_test.exs | 68 ++++--------------- 5 files changed, 54 insertions(+), 60 deletions(-) diff --git a/lib/uplink/packages/install/manager.ex b/lib/uplink/packages/install/manager.ex index 66607d18..8f42ad53 100644 --- a/lib/uplink/packages/install/manager.ex +++ b/lib/uplink/packages/install/manager.ex @@ -59,7 +59,7 @@ defmodule Uplink.Packages.Install.Manager do if Enum.count(completed_instances) == Enum.count(executing_instances) do Packages.transition_install_with(install, actor, "complete") else - {:ok, :still_executing} + {:ok, :executing} end end diff --git a/lib/uplink/packages/instance/upgrade.ex b/lib/uplink/packages/instance/upgrade.ex index 9a04a177..86aaba3f 100644 --- a/lib/uplink/packages/instance/upgrade.ex +++ b/lib/uplink/packages/instance/upgrade.ex @@ -151,8 +151,6 @@ defmodule Uplink.Packages.Instance.Upgrade do |> Instance.Finalize.new() |> Oban.insert() - {:ok, upgrade_package_output} - {:error, %{"err" => "Failed to retrieve PID of executing child process"}} -> Uplink.TaskSupervisor |> @task_supervisor.async_nolink( diff --git a/test/uplink/packages/instance/bootstrap_test.exs b/test/uplink/packages/instance/bootstrap_test.exs index f2a85349..61b11a0f 100644 --- a/test/uplink/packages/instance/bootstrap_test.exs +++ b/test/uplink/packages/instance/bootstrap_test.exs @@ -38,6 +38,14 @@ defmodule Uplink.Packages.Instance.BootstrapTest do |> Plug.Conn.resp(200, cluster_members) end) + instances_list = File.read!("test/fixtures/lxd/instances/list/empty.json") + + Bypass.expect_once(bypass, "GET", "/1.0/instances", fn conn -> + conn + |> Plug.Conn.put_resp_header("content-type", "application/json") + |> Plug.Conn.resp(200, instances_list) + end) + Cache.delete(:cluster_members) create_instance = File.read!("test/fixtures/lxd/instances/create.json") diff --git a/test/uplink/packages/instance/install_test.exs b/test/uplink/packages/instance/install_test.exs index d9673a93..02a39689 100644 --- a/test/uplink/packages/instance/install_test.exs +++ b/test/uplink/packages/instance/install_test.exs @@ -105,6 +105,9 @@ defmodule Uplink.Packages.Instance.InstallTest do {:ok, %{resource: executing_install}} = Packages.transition_install_with(validating_install, actor, "execute") + Cache.put({:install, install.id, "completed"}, []) + Cache.put({:install, install.id, "executing"}, ["some-instance-01"]) + start_instance = File.read!("test/fixtures/lxd/instances/start.json") exec_instance = File.read!("test/fixtures/lxd/instances/exec.json") wait_for_operation = File.read!("test/fixtures/lxd/operations/wait.json") @@ -261,7 +264,7 @@ defmodule Uplink.Packages.Instance.InstallTest do end ) - Bypass.expect_once( + Bypass.expect( bypass, "POST", "/uplink/installations/#{install.instellar_installation_id}/instances/#{instance_slug}/events", @@ -269,7 +272,9 @@ defmodule Uplink.Packages.Instance.InstallTest do assert {:ok, body, conn} = Plug.Conn.read_body(conn) assert {:ok, body} = Jason.decode(body) - %{"event" => %{"name" => "complete" = event_name}} = body + %{"event" => %{"name" => event_name}} = body + + assert event_name in ["complete", "boot"] conn |> Plug.Conn.put_resp_header("content-type", "application/json") @@ -282,7 +287,7 @@ defmodule Uplink.Packages.Instance.InstallTest do end ) - assert {:ok, %{"id" => _id}} = + assert {:ok, %Oban.Job{worker: worker}} = perform_job(Install, %{ instance: %{ slug: instance_slug, @@ -294,6 +299,8 @@ defmodule Uplink.Packages.Instance.InstallTest do actor_id: actor.id }) + assert worker == "Uplink.Packages.Instance.Finalize" + assert_enqueued( worker: Uplink.Clients.Caddy.Config.Reload, args: %{install_id: install.id} @@ -348,6 +355,27 @@ defmodule Uplink.Packages.Instance.InstallTest do } do project_found = File.read!("test/fixtures/lxd/projects/show.json") + Bypass.expect( + bypass, + "POST", + "/uplink/installations/#{install.instellar_installation_id}/instances/#{instance_slug}/events", + fn conn -> + assert {:ok, body, conn} = Plug.Conn.read_body(conn) + assert {:ok, body} = Jason.decode(body) + + %{"event" => %{"name" => event_name}} = body + + conn + |> Plug.Conn.put_resp_header("content-type", "application/json") + |> Plug.Conn.resp( + 201, + Jason.encode!(%{ + "data" => %{"attributes" => %{"id" => 1, "name" => event_name}} + }) + ) + end + ) + Bypass.expect_once( bypass, "GET", diff --git a/test/uplink/packages/instance/upgrade_test.exs b/test/uplink/packages/instance/upgrade_test.exs index c2ea7aff..854ef6b3 100644 --- a/test/uplink/packages/instance/upgrade_test.exs +++ b/test/uplink/packages/instance/upgrade_test.exs @@ -4,10 +4,9 @@ defmodule Uplink.Packages.Instance.UpgradeTest do import Uplink.Scenarios.Deployment - alias Uplink.{ - Packages, - Repo - } + alias Uplink.Repo + alias Uplink.Cache + alias Uplink.Packages setup [:setup_endpoints, :setup_base] @@ -85,44 +84,12 @@ defmodule Uplink.Packages.Instance.UpgradeTest do } } - @uplink_installation_state_response %{ - "data" => %{ - "attributes" => %{ - "id" => 1, - "slug" => "uplink-web", - "main_port" => %{ - "slug" => "web", - "source" => 49142, - "target" => 4000 - }, - "current_state" => "synced", - "variables" => [ - %{"key" => "SOMETHING", "value" => "somevalue"} - ], - "channel" => %{ - "slug" => "develop", - "package" => %{ - "slug" => "something-1640927800", - "credential" => %{ - "public_key" => "public_key" - }, - "organization" => %{ - "slug" => "upmaru" - } - } - }, - "instances" => [ - %{ - "id" => 1, - "slug" => "something-1", - "node" => %{ - "slug" => "some-node" - } - } - ] - } - } - } + setup %{install: install} do + Cache.put({:install, install.id, "completed"}, []) + Cache.put({:install, install.id, "executing"}, ["some-instance-01"]) + + :ok + end describe "upgrade instance" do alias Uplink.Packages.Instance.Upgrade @@ -224,6 +191,8 @@ defmodule Uplink.Packages.Instance.UpgradeTest do end ) + complete_message = "upgrade complete" + Bypass.expect_once( bypass, "GET", @@ -236,7 +205,7 @@ defmodule Uplink.Packages.Instance.UpgradeTest do conn |> Plug.Conn.resp( 200, - "upgrade complete" + complete_message ) end ) @@ -277,16 +246,7 @@ defmodule Uplink.Packages.Instance.UpgradeTest do end ) - Bypass.expect(bypass, "GET", "/uplink/installations/1", fn conn -> - conn - |> Plug.Conn.put_resp_header("content-type", "application/json") - |> Plug.Conn.resp( - 200, - Jason.encode!(@uplink_installation_state_response) - ) - end) - - assert {:ok, %{event: event}} = + assert {:ok, %Oban.Job{worker: worker}} = perform_job(Upgrade, %{ instance: %{ slug: instance_slug, @@ -296,7 +256,7 @@ defmodule Uplink.Packages.Instance.UpgradeTest do actor_id: actor.id }) - assert event.name == "complete" + assert worker == "Uplink.Packages.Instance.Finalize" end test "on error it enqueue deactivate and bootstrap", %{ From 940e4a0588b30df08f09732e4de16186827ff73d Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Fri, 24 Nov 2023 16:16:39 +0700 Subject: [PATCH 5/6] Clean up tests Signed-off-by: Zack Siri --- lib/uplink/packages/instance/bootstrap.ex | 30 ++++++++----------- .../packages/instance/bootstrap_test.exs | 14 --------- 2 files changed, 12 insertions(+), 32 deletions(-) diff --git a/lib/uplink/packages/instance/bootstrap.ex b/lib/uplink/packages/instance/bootstrap.ex index 61f8d7bc..866241fa 100644 --- a/lib/uplink/packages/instance/bootstrap.ex +++ b/lib/uplink/packages/instance/bootstrap.ex @@ -4,7 +4,6 @@ defmodule Uplink.Packages.Instance.Bootstrap do max_attempts: 1 alias Uplink.Repo - alias Uplink.Members alias Uplink.Members.Actor alias Uplink.Packages @@ -12,9 +11,9 @@ defmodule Uplink.Packages.Instance.Bootstrap do alias Uplink.Packages.Instance alias Uplink.Packages.Instance.Cleanup - alias Uplink.Clients.LXD alias Uplink.Clients.Instellar - alias Uplink.Clients.LXD.Cluster + + alias Uplink.Clients.LXD alias Uplink.Clients.LXD.Cluster.Member @transition_parameters %{ @@ -33,15 +32,14 @@ defmodule Uplink.Packages.Instance.Bootstrap do import Ecto.Query, only: [preload: 2] def perform(%Oban.Job{ - args: - %{ - "instance" => - %{ - "slug" => name - } = instance_params, - "install_id" => install_id, - "actor_id" => actor_id - } = job_args + args: %{ + "instance" => + %{ + "slug" => name + } = instance_params, + "install_id" => install_id, + "actor_id" => actor_id + } }) do %Actor{} = actor = Repo.get(Actor, actor_id) @@ -51,10 +49,6 @@ defmodule Uplink.Packages.Instance.Bootstrap do |> preload([:deployment]) |> Repo.get(install_id) - cluster_member_names = - LXD.list_cluster_members() - |> Enum.map(& &1.server_name) - frequency = LXD.list_instances() |> Enum.frequencies_by(fn instance -> @@ -90,7 +84,7 @@ defmodule Uplink.Packages.Instance.Bootstrap do profile_name = Packages.profile_name(metadata) package = channel.package - instance_params = + lxd_instance_params = Map.merge(@default_params, %{ "name" => name, "architecture" => architecture, @@ -133,7 +127,7 @@ defmodule Uplink.Packages.Instance.Bootstrap do formation_instance = Formation.new_lxd_instance(formation_instance_params) client - |> Formation.lxd_create(selected_member.server_name, instance_params, + |> Formation.lxd_create(selected_member.server_name, lxd_instance_params, project: project_name ) |> Formation.lxd_start(name, project: project_name) diff --git a/test/uplink/packages/instance/bootstrap_test.exs b/test/uplink/packages/instance/bootstrap_test.exs index 61b11a0f..ba327101 100644 --- a/test/uplink/packages/instance/bootstrap_test.exs +++ b/test/uplink/packages/instance/bootstrap_test.exs @@ -70,20 +70,6 @@ defmodule Uplink.Packages.Instance.BootstrapTest do end describe "bootstrap instance" do - test "no matching cluster member", %{ - install: install, - actor: actor - } do - assert {:ok, %{resource: install}} = - perform_job(Bootstrap, %{ - instance: %{slug: "something-1", node: %{slug: "some-node-01"}}, - install_id: install.id, - actor_id: actor.id - }) - - assert install.current_state == "failed" - end - test "when project does not exist", %{ bypass: bypass, install: install, From bfca3808280f4727e6f32728095f1690db2ff730 Mon Sep 17 00:00:00 2001 From: Zack Siri Date: Fri, 24 Nov 2023 16:50:36 +0700 Subject: [PATCH 6/6] Add test for finalize worker Signed-off-by: Zack Siri --- lib/uplink/clients/caddy.ex | 2 +- lib/uplink/clients/caddy/config/reload.ex | 20 --- lib/uplink/packages/instance/finalize.ex | 5 +- .../clients/caddy/config/reload_test.exs | 33 ----- .../packages/instance/finalize_test.exs | 134 ++++++++++++++++++ .../uplink/packages/instance/install_test.exs | 2 +- 6 files changed, 140 insertions(+), 56 deletions(-) create mode 100644 test/uplink/packages/instance/finalize_test.exs diff --git a/lib/uplink/clients/caddy.ex b/lib/uplink/clients/caddy.ex index 28de440b..b8ded57a 100644 --- a/lib/uplink/clients/caddy.ex +++ b/lib/uplink/clients/caddy.ex @@ -29,7 +29,7 @@ defmodule Uplink.Clients.Caddy do end params - |> Config.Reload.new(schedule_in: 5) + |> Config.Reload.new(schedule_in: 1) |> Oban.insert() end diff --git a/lib/uplink/clients/caddy/config/reload.ex b/lib/uplink/clients/caddy/config/reload.ex index e4a9e3d9..ef8b895f 100644 --- a/lib/uplink/clients/caddy/config/reload.ex +++ b/lib/uplink/clients/caddy/config/reload.ex @@ -43,26 +43,6 @@ defmodule Uplink.Clients.Caddy.Config.Reload do end) end) - maybe_mark_install_complete(install, params) - :ok end - - defp maybe_mark_install_complete( - %Install{current_state: "refreshing"} = install, - params - ) do - actor_id = Map.get(params, "actor_id") - - actor = - if actor_id do - Repo.get(Members.Actor, actor_id) - else - Members.get_bot!() - end - - Packages.transition_install_with(install, actor, "complete") - end - - defp maybe_mark_install_complete(_install, _params), do: :ok end diff --git a/lib/uplink/packages/instance/finalize.ex b/lib/uplink/packages/instance/finalize.ex index 5513db7c..db2d6606 100644 --- a/lib/uplink/packages/instance/finalize.ex +++ b/lib/uplink/packages/instance/finalize.ex @@ -15,7 +15,10 @@ defmodule Uplink.Packages.Instance.Finalize do def perform(%Oban.Job{ args: %{ - "instance" => %{"slug" => name} = instance_params, + "instance" => + %{ + "slug" => name + } = instance_params, "comment" => comment, "install_id" => install_id, "actor_id" => actor_id diff --git a/test/uplink/clients/caddy/config/reload_test.exs b/test/uplink/clients/caddy/config/reload_test.exs index 44e07233..29f8ad31 100644 --- a/test/uplink/clients/caddy/config/reload_test.exs +++ b/test/uplink/clients/caddy/config/reload_test.exs @@ -3,7 +3,6 @@ defmodule Uplink.Clients.Caddy.Config.ReloadTest do use Oban.Testing, repo: Uplink.Repo alias Uplink.{ - Repo, Cache, Members, Packages @@ -192,37 +191,5 @@ defmodule Uplink.Clients.Caddy.Config.ReloadTest do assert :ok == perform_job(Config.Reload, %{install_id: install.id}) end - - test "mark install completed when refreshing", %{ - bypass: bypass, - install: install - } do - {:ok, install} = - install - |> Ecto.Changeset.cast(%{current_state: "refreshing"}, [:current_state]) - |> Repo.update() - - Bypass.expect(bypass, "POST", "/load", fn conn -> - conn - |> Plug.Conn.put_resp_header("content-type", "application/json") - |> Plug.Conn.resp(200, "") - end) - - Bypass.expect_once(bypass, "GET", "/uplink/installations/1", fn conn -> - conn - |> Plug.Conn.put_resp_header("content-type", "application/json") - |> Plug.Conn.resp( - 200, - Jason.encode!(@uplink_installation_state_response) - ) - end) - - assert :ok == - perform_job(Config.Reload, %{install_id: install.id}) - - install = Repo.reload(install) - - assert install.current_state == "completed" - end end end diff --git a/test/uplink/packages/instance/finalize_test.exs b/test/uplink/packages/instance/finalize_test.exs new file mode 100644 index 00000000..f2b866eb --- /dev/null +++ b/test/uplink/packages/instance/finalize_test.exs @@ -0,0 +1,134 @@ +defmodule Uplink.Packages.Instance.FinalizeTest do + use ExUnit.Case + use Oban.Testing, repo: Uplink.Repo + + alias Uplink.Cache + alias Uplink.Members + + alias Uplink.Packages + alias Uplink.Packages.Metadata + + @deployment_params %{ + "hash" => "some-hash", + "archive_url" => "http://localhost/archives/packages.zip", + "stack" => "alpine/3.14", + "channel" => "develop", + "metadata" => %{ + "id" => 1, + "slug" => "uplink-web", + "main_port" => %{ + "slug" => "web", + "source" => 49153, + "target" => 4000 + }, + "channel" => %{ + "slug" => "develop", + "package" => %{ + "slug" => "something-1640927800", + "credential" => %{ + "public_key" => "public_key" + }, + "organization" => %{ + "slug" => "upmaru" + } + } + }, + "instances" => [ + %{ + "id" => 1, + "slug" => "something-1", + "node" => %{ + "slug" => "some-node" + } + } + ] + } + } + + setup do + :ok = Ecto.Adapters.SQL.Sandbox.checkout(Uplink.Repo) + + bypass = Bypass.open() + + Cache.put(:self, %{ + "credential" => %{ + "endpoint" => "http://localhost:#{bypass.port}" + } + }) + + Application.put_env( + :uplink, + Uplink.Clients.Instellar, + endpoint: "http://localhost:#{bypass.port}/uplink" + ) + + {:ok, actor} = + Members.get_or_create_actor(%{ + "identifier" => "zacksiri", + "provider" => "instellar", + "id" => "1" + }) + + metadata = Map.get(@deployment_params, "metadata") + + {:ok, metadata} = Packages.parse_metadata(metadata) + + app = + metadata + |> Metadata.app_slug() + |> Packages.get_or_create_app() + + {:ok, deployment} = + Packages.get_or_create_deployment(app, @deployment_params) + + {:ok, install} = Packages.create_install(deployment, 1) + + {:ok, bypass: bypass, install: install, actor: actor} + end + + describe "perform" do + alias Uplink.Packages.Instance.Finalize + + test "calls instellar to mark instance complete", %{ + bypass: bypass, + install: install, + actor: actor + } do + instance_slug = "some-instance-01" + + Bypass.expect( + bypass, + "POST", + "/uplink/installations/#{install.instellar_installation_id}/instances/#{instance_slug}/events", + fn conn -> + assert {:ok, body, conn} = Plug.Conn.read_body(conn) + assert {:ok, body} = Jason.decode(body) + + %{"event" => %{"name" => event_name}} = body + + assert event_name in ["complete"] + + conn + |> Plug.Conn.put_resp_header("content-type", "application/json") + |> Plug.Conn.resp( + 201, + Jason.encode!(%{ + "data" => %{"attributes" => %{"id" => 1, "name" => event_name}} + }) + ) + end + ) + + assert {:ok, %{"id" => 1, "name" => "complete"}} = + perform_job( + Finalize, + %{ + "instance" => %{"slug" => instance_slug}, + "comment" => "all good!", + "install_id" => install.id, + "actor_id" => actor.id + } + ) + end + end +end diff --git a/test/uplink/packages/instance/install_test.exs b/test/uplink/packages/instance/install_test.exs index 02a39689..b286498b 100644 --- a/test/uplink/packages/instance/install_test.exs +++ b/test/uplink/packages/instance/install_test.exs @@ -274,7 +274,7 @@ defmodule Uplink.Packages.Instance.InstallTest do %{"event" => %{"name" => event_name}} = body - assert event_name in ["complete", "boot"] + assert event_name in ["boot"] conn |> Plug.Conn.put_resp_header("content-type", "application/json")