From ec1ea399b0e13effa9bf74ec3e323e57291596d4 Mon Sep 17 00:00:00 2001 From: Y86 Date: Sun, 1 Nov 2026 18:50:37 -0300 Subject: [PATCH 1/4] Add opt-in release pruning after switch Keep current and the highest lower SemVer, including pre-releases, to stop release directories accumulating on small deployment disks. Prune only after a verified switch so staged uploads survive deploy, and leave automatic cleanup opt-in for shared consumers. Add a standalone dry-run task and exercise guarded shell deletion through the fake executor against temporary release trees. Document rollback, serialization and rollout requirements. --- README.md | 49 ++++ lib/dicon.ex | 6 + lib/dicon/pruner.ex | 122 +++++++++ lib/mix/tasks/dicon.prune.ex | 50 ++++ lib/mix/tasks/dicon.switch.ex | 8 + test/mix/tasks/dicon.prune_test.exs | 371 ++++++++++++++++++++++++++++ test/test_helper.exs | 4 + 7 files changed, 610 insertions(+) create mode 100644 lib/dicon/pruner.ex create mode 100644 lib/mix/tasks/dicon.prune.ex create mode 100644 test/mix/tasks/dicon.prune_test.exs diff --git a/README.md b/README.md index 848e1ec..9394628 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,55 @@ end After you are done, run `mix deps.get` in your shell to fetch the dependencies. +## Release retention + +Inspect and prune existing releases on the configured hosts: + +```sh +mix dicon.prune --dry-run +mix dicon.prune --only app01 +``` + +Both commands print the retained versions and each deletion per host. `--only` +and `--skip` can be repeated, just as for deploy and switch. + +Pruning keeps the directory `current` resolves to and **one previous version**: +the highest version strictly below current according to `Version.compare/2`. +Previous may be an RC or another pre-release (e.g. current `1.7.6` keeps +`1.7.6-rc4` ahead of `1.7.5`). Current may itself be an RC. Build metadata does +not affect precedence; ties for previous are broken by directory name in +ascending lexical order. If no older version exists, only current is kept. + +Every other directory with a valid Elixir SemVer name is deleted, including +newer unswitched releases and abandoned experiments. Non-version entries such +as `log/`, files and symlinks are untouched. A missing, broken, non-version or +external `current` stops pruning without deleting releases on that host. + +For automatic cleanup, opt in per application: + +```elixir +config :dicon, prune_on_switch: true +``` + +Cleanup runs after each host's `dicon.switch` and verifies that `current` +resolves to the requested version before deleting anything. `dicon.deploy` +does not prune, so uploading a release before switching remains safe. Automatic +cleanup defaults to off because dicon is shared by services with different +staging and retention workflows; services must enable it to bound disk growth. + +Serialize deploy, switch and prune operations on each host. Never run standalone +pruning between deploy and switch: it removes the staged release. The commands +recheck current before each deletion but do not lock concurrent deployments. +Switch only changes the symlink, so stop processes using a release that pruning +would remove before switching/pruning (especially during rollback). + +Rollback to the retained previous version uses the usual `mix dicon.switch +VERSION`. With automatic pruning enabled, the release rolled back from is then +removed as newer than current; rolling forward again requires redeploying it. +There may be no previous version left after rollback. Pruning errors abort the +task; they do not undo an already completed switch or earlier deletions. Inspect +the reported host and rerun `mix dicon.prune --dry-run` after resolving the error. + ## License This software is licensed under [the ISC license](LICENSE). diff --git a/lib/dicon.ex b/lib/dicon.ex index 0fda48d..1f551be 100644 --- a/lib/dicon.ex +++ b/lib/dicon.ex @@ -29,6 +29,12 @@ defmodule Dicon do * `:executor` - a module that will be used to execute commands on servers. By default, it's `Dicon.SecureShell`. + * `:prune_on_switch` - a boolean, defaulting to `false`. When enabled, + `dicon.switch` prunes each host after switching, keeping current and the + highest SemVer below it (which may be a pre-release). All other version + directories, including newer versions, are removed. See `dicon.prune` and + the README for ordering and rollback requirements. + ### Configuration for hosts Each host listed in the `:hosts` configuration option mentioned above can be diff --git a/lib/dicon/pruner.ex b/lib/dicon/pruner.ex new file mode 100644 index 0000000..e27fc45 --- /dev/null +++ b/lib/dicon/pruner.ex @@ -0,0 +1,122 @@ +defmodule Dicon.Pruner do + @moduledoc false + + alias Dicon.Executor + + # SecureShell does not propagate remote exit statuses. Require this marker only + # after every shell operation succeeds, including a complete directory listing. + @success "dicon:prune:ok" + + def prune(%Executor{} = conn, target_dir, host, opts \\ []) when is_binary(target_dir) do + target_dir = if Path.type(target_dir) == :absolute, do: target_dir, else: "./" <> target_dir + + output = + checked_exec(conn, [ + "cd ", + quote_path(target_dir), + " && root=$(pwd -P) && test -L current", + " && current=$(cd current && pwd -P)", + " && printf '%s\\0' \"$root\" \"$current\"", + " && find . -mindepth 1 -maxdepth 1 -type d -print0" + ]) + + {root, current, releases} = parse_inventory(output, host) + + if expected = opts[:expected_current] do + if current != expected do + Mix.raise("#{host}: current is #{current}, expected #{expected}; refusing to prune") + end + end + + previous = + Enum.find(releases, fn {_, %Version{} = version} -> + Version.compare(version, current) == :lt + end) + + previous_name = if previous, do: elem(previous, 0) + Mix.shell().info("#{host}: keeping current #{current}, previous #{previous_name || "none"}") + + for {name, _} <- releases, name not in [current, previous_name] do + path = Path.join(root, name) + + if opts[:dry_run] do + Mix.shell().info("#{host}: would delete #{path}") + else + checked_exec(conn, [ + "cd ", + quote_path(root), + " && test -L current && test \"$(cd current && pwd -P)\" = ", + quote_path(Path.join(root, current)), + " && test ! -L ", + quote_path("./" <> name), + " && test -d ", + quote_path("./" <> name), + " && rm -rf -- ", + quote_path("./" <> name) + ]) + + Mix.shell().info("#{host}: deleted #{path}") + end + end + + :ok + end + + defp parse_inventory(output, host) when is_binary(output) do + case String.split(output, "\0", trim: false) do + ["/" <> _ = root, "/" <> _ = current_path | entries] -> + releases = + entries + |> Enum.flat_map(fn + "./" <> name -> + case Version.parse(name) do + {:ok, version} -> [{name, version}] + :error -> [] + end + + _ -> + [] + end) + |> Enum.sort(fn {left_name, left}, {right_name, right} -> + case Version.compare(left, right) do + :gt -> true + :lt -> false + :eq -> left_name <= right_name + end + end) + + current = Path.basename(current_path) + + if Path.dirname(current_path) != root or not List.keymember?(releases, current, 0) do + Mix.raise("#{host}: current must resolve to a version directory inside #{root}") + end + + {root, current, releases} + + _ -> + Mix.raise("#{host}: invalid release inventory; refusing to prune") + end + end + + defp checked_exec(%Executor{} = conn, command) do + {:ok, device} = StringIO.open("") + success = @success <> "\0" + + try do + Executor.exec(conn, [command, " && printf '%s\\0' ", quote_path(@success)], device) + {"", output} = StringIO.contents(device) + + if String.ends_with?(output, success) do + binary_part(output, 0, byte_size(output) - byte_size(success)) + else + Mix.raise("Pruning command failed or current changed; refusing to continue: #{output}") + end + after + StringIO.close(device) + end + end + + defp quote_path(path) when is_binary(path) do + "'" <> String.replace(path, "'", "'\\''") <> "'" + end +end diff --git a/lib/mix/tasks/dicon.prune.ex b/lib/mix/tasks/dicon.prune.ex new file mode 100644 index 0000000..11fdc1a --- /dev/null +++ b/lib/mix/tasks/dicon.prune.ex @@ -0,0 +1,50 @@ +defmodule Mix.Tasks.Dicon.Prune do + use Mix.Task + + @shortdoc "Keeps current and the previous release on the configured servers" + + @moduledoc """ + Deletes version directories except current and the highest SemVer below it. + The previous version may be a pre-release. Newer versions are also deleted. + Non-version entries and symlinks are left alone. + + The configured hosts and target directory come from the application's `:dicon` + configuration. Use `--only HOST` / `--skip HOST` to filter hosts and `--dry-run` + to print the deletions without changing anything. + + Run only after switching and stopping any process that uses a release to be + deleted. Serialize deploy, switch and prune operations for each host; never + prune between uploading a release and switching to it. An invalid or missing + `current` aborts pruning. See the README for rollback and automatic pruning. + + ## Usage + + mix dicon.prune --dry-run + mix dicon.prune --only app01 + """ + + import Dicon, only: [config: 1, config: 2, host_config: 1] + + alias Dicon.{Executor, Pruner} + + @options [strict: [only: :keep, skip: :keep, dry_run: :boolean]] + + def run(argv) do + case OptionParser.parse(argv, @options) do + {opts, [], []} -> + target_dir = config(:target_dir) + + for host <- config(:hosts, opts) do + {authority, host_config} = Keyword.pop!(host_config(host), :authority) + conn = Executor.connect(authority, host_config) + Pruner.prune(conn, target_dir, host, opts) + end + + {_opts, _commands, [switch | _]} -> + Mix.raise("Invalid option: " <> Mix.Dicon.switch_to_string(switch)) + + {_opts, _commands, _errors} -> + Mix.raise("Expected no arguments") + end + end +end diff --git a/lib/mix/tasks/dicon.switch.ex b/lib/mix/tasks/dicon.switch.ex index 7f4da9c..d6c7af4 100644 --- a/lib/mix/tasks/dicon.switch.ex +++ b/lib/mix/tasks/dicon.switch.ex @@ -14,6 +14,10 @@ defmodule Mix.Tasks.Dicon.Switch do the currently active release to the one specified by the given version. Switching versions won't affect the running application. + With `config :dicon, prune_on_switch: true`, each host is pruned after switching, + keeping current and the highest SemVer below it (including pre-releases). See + `mix help dicon.prune` and the README for the retention and rollback policy. + ## Usage mix dicon.switch VERSION @@ -43,6 +47,10 @@ defmodule Mix.Tasks.Dicon.Switch do {authority, host_config} = Keyword.pop!(host_config(host), :authority) conn = Executor.connect(authority, host_config) symlink(conn, [target_dir, ?/, version], [target_dir, "/current"]) + + if Application.get_env(:dicon, :prune_on_switch, false) do + Dicon.Pruner.prune(conn, config(:target_dir), host, expected_current: version) + end end {_opts, _commands, [switch | _]} -> diff --git a/test/mix/tasks/dicon.prune_test.exs b/test/mix/tasks/dicon.prune_test.exs new file mode 100644 index 0000000..cf202da --- /dev/null +++ b/test/mix/tasks/dicon.prune_test.exs @@ -0,0 +1,371 @@ +defmodule Mix.Tasks.Dicon.PruneTest do + use DiconTest.Case + + import Mix.Tasks.Dicon.Prune, only: [run: 1] + + setup do + root = Path.join(System.tmp_dir!(), "dicon-prune-#{System.unique_integer([:positive])}") + target = Path.join(root, "releases") + File.mkdir_p!(target) + + put_dicon_env(%{ + target_dir: "releases", + hosts: [:one], + one: [authority: "one"], + two: [authority: "two"], + prune_on_switch: false + }) + + # Keep the fake connection/notifications, but exercise the actual shell + # commands against disposable local releases. Like SecureShell, ignore exit + # status so the pruning success checks are exercised too. + on_exec(:all, fn command, device -> exec_locally(command, device, root) end) + + on_exit(fn -> + File.rm_rf!(root) + Application.delete_env(:dicon, :prune_on_switch) + end) + + {:ok, root: root, target: target} + end + + test "dry run reports each host's deletions without changing anything", %{target: target} do + versions = ~w(1.7.5 1.7.6-rc4 1.7.6 1.8.0-rc3-fcm-http2) + releases(target, versions, "1.7.6") + put_dicon_env(%{hosts: [:one, :two]}) + + run(["--dry-run"]) + + for host <- ["one", "two"] do + assert_receive {:dicon, _, :connect, [^host, []]} + assert_info("#{host}: keeping current 1.7.6, previous 1.7.6-rc4") + assert_info_suffix("#{host}: would delete ", "/1.8.0-rc3-fcm-http2") + assert_info_suffix("#{host}: would delete ", "/1.7.5") + end + + assert Enum.sort(File.ls!(target)) == Enum.sort(["current" | versions]) + refute_delete_command() + end + + test "deletes older and newer versions, keeping the highest previous RC", %{target: target} do + releases(target, ~w(1.7.5 1.7.6-rc4 1.7.6 1.8.0-rc3-fcm-http2), "1.7.6") + + run([]) + + assert Enum.sort(File.ls!(target)) == ~w(1.7.6 1.7.6-rc4 current) + assert File.read_link!(Path.join(target, "current")) == "1.7.6" + assert_info_suffix("one: deleted ", "/1.8.0-rc3-fcm-http2") + assert_info_suffix("one: deleted ", "/1.7.5") + end + + test "current can be an RC and alphanumeric identifiers sort lexically", %{target: target} do + releases(target, ~w(1.1.8 1.1.9-rc2 1.1.9-rc9 1.1.9-rc10 1.1.9 1.1.10), "1.1.9-rc9") + + run([]) + + # SemVer compares alphanumeric identifiers lexically: rc10 < rc2 < rc9. + assert Enum.sort(File.ls!(target)) == ~w(1.1.9-rc2 1.1.9-rc9 current) + end + + test "numeric pre-release identifiers and release components use SemVer", %{target: target} do + releases(target, ~w(1.9.0 1.10.0-rc.2 1.10.0-rc.10 1.10.0 2.0.0), "1.10.0") + + run([]) + + assert Enum.sort(File.ls!(target)) == ~w(1.10.0 1.10.0-rc.10 current) + end + + test "ignores metadata precedence and breaks previous ties deterministically", %{target: target} do + releases(target, ~w(1.0.0+a 1.0.0+b 1.1.0+a 1.1.0+b 1.1.0+c), "1.1.0+b") + + run([]) + + assert Enum.sort(File.ls!(target)) == ~w(1.0.0+a 1.1.0+b current) + end + + test "keeps only current when there is no lower version, and is idempotent", %{target: target} do + releases(target, ~w(1.0.0-rc1 1.0.0 1.1.0), "1.0.0-rc1") + + run([]) + run([]) + + assert Enum.sort(File.ls!(target)) == ~w(1.0.0-rc1 current) + assert_info("one: keeping current 1.0.0-rc1, previous none") + end + + test "never removes non-version entries, version files or symlinks", %{target: target} do + releases( + target, + ~w(1.3.1-rc1-bruce 1.4.0-incident.0 1.4.0-rc1-yuri-track-device-token-activity 1.5.0 1.6.0), + "1.6.0" + ) + + for name <- ["log", "data", "v1.0.0", "1.0", "notes\n1.0.0", "1.0.0; touch injected"] do + File.mkdir_p!(Path.join(target, name)) + end + + File.write!(Path.join(target, "0.1.0"), "not a directory") + File.ln_s!("log", Path.join(target, "1.2.0")) + File.ln_s!("missing", Path.join(target, "1.2.1")) + before = File.ls!(target) + + run([]) + + assert Enum.sort(before -- File.ls!(target)) == + ~w(1.3.1-rc1-bruce 1.4.0-incident.0 1.4.0-rc1-yuri-track-device-token-activity) + + assert File.read!(Path.join(target, "0.1.0")) == "not a directory" + assert File.read_link!(Path.join(target, "1.2.0")) == "log" + assert File.read_link!(Path.join(target, "1.2.1")) == "missing" + end + + test "supports absolute paths and shell metacharacters in the target", %{root: root} do + target = Path.join(root, "app's releases $(touch injected)") + releases(target, ~w(1.0.0 1.1.0 1.2.0), Path.join(target, "1.2.0")) + put_dicon_env(%{target_dir: target}) + + run([]) + + assert Enum.sort(File.ls!(target)) == ~w(1.1.0 1.2.0 current) + refute File.exists?(Path.join(root, "injected")) + end + + test "treats a relative target starting with a dash as a literal directory", %{root: root} do + target = Path.join(root, "-releases") + releases(target, ~w(1.0.0 1.1.0 1.2.0), "1.2.0") + put_dicon_env(%{target_dir: "-releases"}) + + run([]) + + assert Enum.sort(File.ls!(target)) == ~w(1.1.0 1.2.0 current) + end + + test "rejects a missing current", %{target: target} do + File.mkdir_p!(Path.join(target, "1.0.0")) + + assert_raise Mix.Error, ~r/Pruning command failed/, fn -> run([]) end + + assert File.dir?(Path.join(target, "1.0.0")) + refute_delete_command() + end + + test "rejects a dangling current", %{target: target} do + releases(target, ~w(1.0.0), "1.1.0") + + assert_raise Mix.Error, ~r/Pruning command failed/, fn -> run([]) end + + assert File.dir?(Path.join(target, "1.0.0")) + refute_delete_command() + end + + test "rejects a current directory instead of a symlink", %{target: target} do + File.mkdir_p!(Path.join(target, "current")) + + assert_raise Mix.Error, ~r/Pruning command failed/, fn -> run([]) end + + refute_delete_command() + end + + test "rejects a non-version current", %{target: target} do + releases(target, ~w(log 1.0.0), "log") + + assert_raise Mix.Error, ~r/current must resolve to a version directory/, fn -> run([]) end + + assert File.dir?(Path.join(target, "1.0.0")) + refute_delete_command() + end + + test "rejects an external current with the same basename as a release", %{ + root: root, + target: target + } do + outside = Path.join(root, "1.1.0") + File.mkdir_p!(outside) + releases(target, ~w(1.0.0 1.1.0), outside) + + assert_raise Mix.Error, ~r/current must resolve to a version directory/, fn -> run([]) end + + assert Enum.sort(File.ls!(target)) == ~w(1.0.0 1.1.0 current) + refute_delete_command() + end + + test "aborts if current changes between listing and deleting", %{root: root, target: target} do + releases(target, ~w(1.0.0 1.1.0 1.2.0), "1.2.0") + + on_exec(:all, fn command, device -> + if String.contains?(command, "rm -rf") do + File.rm!(Path.join(target, "current")) + File.ln_s!("1.0.0", Path.join(target, "current")) + end + + exec_locally(command, device, root) + end) + + assert_raise Mix.Error, ~r/current changed/, fn -> run([]) end + + assert Enum.sort(File.ls!(target)) == ~w(1.0.0 1.1.0 1.2.0 current) + end + + test "aborts if a candidate becomes a symlink", %{root: root, target: target} do + releases(target, ~w(1.0.0 1.1.0 1.2.0), "1.2.0") + + on_exec(:all, fn command, device -> + if String.contains?(command, "rm -rf") do + File.rm_rf!(Path.join(target, "1.0.0")) + File.ln_s!("1.2.0", Path.join(target, "1.0.0")) + end + + exec_locally(command, device, root) + end) + + assert_raise Mix.Error, ~r/Pruning command failed/, fn -> run([]) end + + assert File.read_link!(Path.join(target, "1.0.0")) == "1.2.0" + assert File.dir?(Path.join(target, "1.2.0")) + end + + test "rejects incomplete inventories even if the executor returns ok", %{target: target} do + releases(target, ~w(1.0.0 1.1.0 1.2.0), "1.2.0") + on_exec(:all, fn _command, device -> IO.write(device, "#{target}\0#{target}/1.2.0\0") end) + + assert_raise Mix.Error, ~r/Pruning command failed/, fn -> run([]) end + + refute_delete_command() + end + + test "reports a deletion failure even if the executor returns ok", %{root: root, target: target} do + releases(target, ~w(1.0.0 1.1.0 1.2.0), "1.2.0") + + on_exec(:all, fn command, device -> + if String.contains?(command, "rm -rf") do + IO.write(device, "rm: permission denied") + else + exec_locally(command, device, root) + end + end) + + assert_raise Mix.Error, ~r/rm: permission denied/, fn -> run([]) end + + assert File.dir?(Path.join(target, "1.0.0")) + end + + test "filters hosts and passes their connection options", %{target: target} do + releases(target, ~w(1.0.0), "1.0.0") + put_dicon_env(%{hosts: [:one, :two], one: [authority: "one", jump_authority: "jump"]}) + + run(["--dry-run", "--only", "one", "--only", "two", "--skip", "two"]) + + assert_receive {:dicon, ref, :connect, ["one", [jump_authority: "jump"]]} + flush_reply(ref) + refute_receive {:dicon, _, _, _} + + run(["--skip", "one", "--skip", "two"]) + refute_receive {:dicon, _, _, _} + + assert_raise Mix.Error, "Unknown host: \"unknown\"", fn -> + run(["--only", "unknown"]) + end + end + + test "validates arguments" do + assert_raise Mix.Error, "Expected no arguments", fn -> run(["1.0.0"]) end + assert_raise Mix.Error, "Invalid option: --invalid", fn -> run(["--invalid"]) end + end + + test "opted-in switch prunes after selecting the staged release and allows rollback", %{ + target: target + } do + releases(target, ~w(1.0.0 1.1.0 1.2.0), "1.1.0") + put_dicon_env(%{prune_on_switch: true}) + + Mix.Tasks.Dicon.Switch.run(["1.2.0"]) + + assert Enum.sort(File.ls!(target)) == ~w(1.1.0 1.2.0 current) + assert Path.basename(File.read_link!(Path.join(target, "current"))) == "1.2.0" + + Mix.Tasks.Dicon.Switch.run(["1.1.0"]) + + assert Enum.sort(File.ls!(target)) == ~w(1.1.0 current) + assert Path.basename(File.read_link!(Path.join(target, "current"))) == "1.1.0" + end + + test "does not prune if switch fails to select the requested version", %{target: target} do + releases(target, ~w(1.0.0 1.1.0 1.2.0), "1.1.0") + put_dicon_env(%{prune_on_switch: true}) + + on_exec("ln -snf $PWD/releases/1.2.0 $PWD/releases/current", fn device -> + IO.write(device, "ln: permission denied") + end) + + assert_raise Mix.Error, ~r/current is 1.1.0, expected 1.2.0/, fn -> + Mix.Tasks.Dicon.Switch.run(["1.2.0"]) + end + + assert Enum.sort(File.ls!(target)) == ~w(1.0.0 1.1.0 1.2.0 current) + refute_delete_command() + end + + test "switch and deploy retain all versions unless switch pruning is enabled", %{target: target} do + releases(target, ~w(1.0.0 1.1.0), "1.1.0") + put_dicon_env(%{prune_on_switch: true}) + source = PathHelpers.fixture_path("empty.tar.gz") + File.cp!(source, Path.join(target, "release.tar.gz")) + + Mix.Tasks.Dicon.Deploy.run([source, "1.2.0"]) + + assert Enum.sort(File.ls!(target)) == ~w(1.0.0 1.1.0 1.2.0 current) + refute_delete_command() + + Application.delete_env(:dicon, :prune_on_switch) + Mix.Tasks.Dicon.Switch.run(["1.2.0"]) + + assert Enum.sort(File.ls!(target)) == ~w(1.0.0 1.1.0 1.2.0 current) + refute_delete_command() + end + + defp releases(target, versions, current) do + for version <- versions do + File.mkdir_p!(Path.join(target, version)) + File.write!(Path.join([target, version, "release.txt"]), version) + end + + File.ln_s!(current, Path.join(target, "current")) + end + + defp exec_locally(command, device, root) do + {output, _status} = System.cmd("sh", ["-c", command], cd: root, stderr_to_stdout: true) + IO.binwrite(device, output) + end + + defp assert_info(expected) do + assert_receive {:mix_shell, :info, [^expected]} + end + + defp assert_info_suffix(prefix, suffix) do + messages = drain_messages() + + assert Enum.any?(messages, fn + {:mix_shell, :info, [message]} -> + String.starts_with?(message, prefix) and String.ends_with?(message, suffix) + + _ -> + false + end) + + Enum.each(messages, &send(self(), &1)) + end + + defp refute_delete_command() do + for {:dicon, _, :exec, [command]} <- drain_messages() do + refute String.contains?(command, "rm -rf") + end + end + + defp drain_messages() do + receive do + message -> [message | drain_messages()] + after + 0 -> [] + end + end +end diff --git a/test/test_helper.exs b/test/test_helper.exs index e37d0ad..9882b3c 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -108,6 +108,10 @@ defmodule DiconTest.Case do if callback do callback.(device) Application.put_env(:dicon, __MODULE__, env) + else + if handler = get_in(env, [:exec_callbacks, :all]) do + handler.(command, device) + end end :ok From 59203161943d233e7382f3fe57fcaea2247bc5f8 Mon Sep 17 00:00:00 2001 From: Y86 Date: Sun, 1 Nov 2026 18:51:43 -0300 Subject: [PATCH 2/4] Require a distinct success record before pruning Separate the command-success marker with NUL boundaries so a directory whose name ends with the marker cannot disguise an incomplete listing. Cover the false-success case through the fake executor. --- lib/dicon/pruner.ex | 4 ++-- test/mix/tasks/dicon.prune_test.exs | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/lib/dicon/pruner.ex b/lib/dicon/pruner.ex index e27fc45..3b62295 100644 --- a/lib/dicon/pruner.ex +++ b/lib/dicon/pruner.ex @@ -100,10 +100,10 @@ defmodule Dicon.Pruner do defp checked_exec(%Executor{} = conn, command) do {:ok, device} = StringIO.open("") - success = @success <> "\0" + success = "\0" <> @success <> "\0" try do - Executor.exec(conn, [command, " && printf '%s\\0' ", quote_path(@success)], device) + Executor.exec(conn, [command, " && printf '\\0%s\\0' ", quote_path(@success)], device) {"", output} = StringIO.contents(device) if String.ends_with?(output, success) do diff --git a/test/mix/tasks/dicon.prune_test.exs b/test/mix/tasks/dicon.prune_test.exs index cf202da..0bbbab3 100644 --- a/test/mix/tasks/dicon.prune_test.exs +++ b/test/mix/tasks/dicon.prune_test.exs @@ -249,6 +249,23 @@ defmodule Mix.Tasks.Dicon.PruneTest do assert File.dir?(Path.join(target, "1.0.0")) end + test "a directory name cannot masquerade as the success marker in a failed listing", %{ + target: target + } do + releases(target, ~w(1.0.0 1.1.0 1.2.0), "1.2.0") + + on_exec(:all, fn _command, device -> + IO.write( + device, + "#{target}\0#{target}/1.2.0\0./1.0.0\0./1.1.0\0./1.2.0\0./dicon:prune:ok\0" + ) + end) + + assert_raise Mix.Error, ~r/Pruning command failed/, fn -> run([]) end + + refute_delete_command() + end + test "filters hosts and passes their connection options", %{target: target} do releases(target, ~w(1.0.0), "1.0.0") put_dicon_env(%{hosts: [:one, :two], one: [authority: "one", jump_authority: "jump"]}) From 0a7f9166aa42b23a55f11e7d21a9ac16f1f1ac7e Mon Sep 17 00:00:00 2001 From: Y86 Date: Sun, 1 Nov 2026 18:59:32 -0300 Subject: [PATCH 3/4] Retain the pre-switch release for safe pruning Push Cartel switches before stopping the old BEAM. Capture current on each host before replacing it and retain that exact release so an unused RC cannot displace the running release or rollback target. Keep the rolled-back-from release available for rolling forward too. Without switch context, prefer the highest lower stable release and fall back to a pre-release only when no stable exists. Preserve rollback targets on same-version retries, validate captured releases, and cover the updated policies in tests and documentation. --- README.md | 46 ++++++----- lib/dicon.ex | 5 +- lib/dicon/pruner.ex | 85 ++++++++++++++------ lib/mix/tasks/dicon.prune.ex | 8 +- lib/mix/tasks/dicon.switch.ex | 15 +++- test/mix/tasks/dicon.prune_test.exs | 119 +++++++++++++++++++++++++--- 6 files changed, 217 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index 9394628..729b28e 100644 --- a/README.md +++ b/README.md @@ -29,12 +29,14 @@ mix dicon.prune --only app01 Both commands print the retained versions and each deletion per host. `--only` and `--skip` can be repeated, just as for deploy and switch. -Pruning keeps the directory `current` resolves to and **one previous version**: -the highest version strictly below current according to `Version.compare/2`. -Previous may be an RC or another pre-release (e.g. current `1.7.6` keeps -`1.7.6-rc4` ahead of `1.7.5`). Current may itself be an RC. Build metadata does -not affect precedence; ties for previous are broken by directory name in -ascending lexical order. If no older version exists, only current is kept. +Standalone pruning keeps the directory `current` resolves to and **one previous +version**: the highest stable (non-pre-release) version strictly below current +according to `Version.compare/2`. If no lower stable version exists, it falls +back to the highest pre-release below current. For example, current `1.7.6` +keeps `1.7.5` and deletes an unused `1.7.6-rc4`. Current may itself be an RC. +Build metadata does not affect precedence; ties for previous are broken by +directory name in ascending lexical order. If no older version exists, only +current is kept. Every other directory with a valid Elixir SemVer name is deleted, including newer unswitched releases and abandoned experiments. Non-version entries such @@ -47,24 +49,32 @@ For automatic cleanup, opt in per application: config :dicon, prune_on_switch: true ``` -Cleanup runs after each host's `dicon.switch` and verifies that `current` -resolves to the requested version before deleting anything. `dicon.deploy` -does not prune, so uploading a release before switching remains safe. Automatic -cleanup defaults to off because dicon is shared by services with different -staging and retention workflows; services must enable it to bound disk growth. +Automatic pruning retains **the exact release current pointed to before the +switch**, whether it is stable, an RC, or newer than the requested version during +rollback. Each host's old current is read and validated before switching; cleanup +then verifies that current resolves to the requested version. Thus switching +from `1.7.5` to `1.7.6` keeps `1.7.5` and deletes an unused `1.7.6-rc4`. This +preserves the old release throughout a `switch → stop → start` sequence. + +If current is absent on the first switch, cleanup keeps only the new release. +An invalid existing current aborts before switching. Switching to the +already-current version skips pruning so a retry preserves the rollback target. +`dicon.deploy` never prunes, so uploading before switching remains safe. Automatic +cleanup defaults to off because dicon is shared by services with different staging +and retention workflows; services must enable it to bound disk growth. Serialize deploy, switch and prune operations on each host. Never run standalone pruning between deploy and switch: it removes the staged release. The commands recheck current before each deletion but do not lock concurrent deployments. -Switch only changes the symlink, so stop processes using a release that pruning -would remove before switching/pruning (especially during rollback). +Standalone pruning has no switch history: inspect its dry run and stop any process +using a release it would delete before running it. It can remove an RC or newer +release that automatic pruning retained as the previous current. Rollback to the retained previous version uses the usual `mix dicon.switch -VERSION`. With automatic pruning enabled, the release rolled back from is then -removed as newer than current; rolling forward again requires redeploying it. -There may be no previous version left after rollback. Pruning errors abort the -task; they do not undo an already completed switch or earlier deletions. Inspect -the reported host and rerun `mix dicon.prune --dry-run` after resolving the error. +VERSION`. Automatic pruning keeps the release rolled back from, so rolling +forward to it requires no redeploy. Pruning errors abort the task; they do not +undo an already completed switch or earlier deletions. Inspect the reported host +before retrying; standalone pruning uses the stable-first policy described above. ## License diff --git a/lib/dicon.ex b/lib/dicon.ex index 1f551be..7ef910b 100644 --- a/lib/dicon.ex +++ b/lib/dicon.ex @@ -31,8 +31,9 @@ defmodule Dicon do * `:prune_on_switch` - a boolean, defaulting to `false`. When enabled, `dicon.switch` prunes each host after switching, keeping current and the - highest SemVer below it (which may be a pre-release). All other version - directories, including newer versions, are removed. See `dicon.prune` and + release current pointed to before the switch, even if it is an RC or newer + than current after rollback. All other version directories are removed. + Switching to the already-current version skips pruning. See `dicon.prune` and the README for ordering and rollback requirements. ### Configuration for hosts diff --git a/lib/dicon/pruner.ex b/lib/dicon/pruner.ex index 3b62295..6a9d44c 100644 --- a/lib/dicon/pruner.ex +++ b/lib/dicon/pruner.ex @@ -7,20 +7,17 @@ defmodule Dicon.Pruner do # after every shell operation succeeds, including a complete directory listing. @success "dicon:prune:ok" - def prune(%Executor{} = conn, target_dir, host, opts \\ []) when is_binary(target_dir) do - target_dir = if Path.type(target_dir) == :absolute, do: target_dir, else: "./" <> target_dir + def current_version(%Executor{} = conn, target_dir, host) when is_binary(target_dir) do + {_root, current, _releases} = inventory(conn, target_dir, host) + current + end - output = - checked_exec(conn, [ - "cd ", - quote_path(target_dir), - " && root=$(pwd -P) && test -L current", - " && current=$(cd current && pwd -P)", - " && printf '%s\\0' \"$root\" \"$current\"", - " && find . -mindepth 1 -maxdepth 1 -type d -print0" - ]) + def prune(%Executor{} = conn, target_dir, host, opts \\ []) when is_binary(target_dir) do + {root, current, releases} = inventory(conn, target_dir, host) - {root, current, releases} = parse_inventory(output, host) + if is_nil(current) do + Mix.raise("#{host}: current is missing; refusing to prune") + end if expected = opts[:expected_current] do if current != expected do @@ -28,12 +25,7 @@ defmodule Dicon.Pruner do end end - previous = - Enum.find(releases, fn {_, %Version{} = version} -> - Version.compare(version, current) == :lt - end) - - previous_name = if previous, do: elem(previous, 0) + previous_name = previous_version(releases, current, host, opts) Mix.shell().info("#{host}: keeping current #{current}, previous #{previous_name || "none"}") for {name, _} <- releases, name not in [current, previous_name] do @@ -62,9 +54,51 @@ defmodule Dicon.Pruner do :ok end + defp previous_version(releases, current, host, opts) do + case Keyword.fetch(opts, :keep_previous) do + {:ok, nil} -> + nil + + {:ok, previous} when is_binary(previous) -> + if not List.keymember?(releases, previous, 0) do + Mix.raise("#{host}: previous release #{previous} is missing; refusing to prune") + end + + previous + + :error -> + older = + Enum.filter(releases, fn {_, %Version{} = version} -> + Version.compare(version, current) == :lt + end) + + previous = + Enum.find(older, fn {_, %Version{pre: pre}} -> pre == [] end) || List.first(older) + + if previous, do: elem(previous, 0) + end + end + + defp inventory(%Executor{} = conn, target_dir, host) when is_binary(target_dir) do + target_dir = if Path.type(target_dir) == :absolute, do: target_dir, else: "./" <> target_dir + + output = + checked_exec(conn, [ + "cd ", + quote_path(target_dir), + " && root=$(pwd -P)", + " && if test ! -e current && test ! -L current; then current='';", + " else test -L current && current=$(cd current && pwd -P); fi", + " && printf '%s\\0' \"$root\" \"$current\"", + " && find . -mindepth 1 -maxdepth 1 -type d -print0" + ]) + + parse_inventory(output, host) + end + defp parse_inventory(output, host) when is_binary(output) do case String.split(output, "\0", trim: false) do - ["/" <> _ = root, "/" <> _ = current_path | entries] -> + ["/" <> _ = root, current_path | entries] -> releases = entries |> Enum.flat_map(fn @@ -85,11 +119,16 @@ defmodule Dicon.Pruner do end end) - current = Path.basename(current_path) + current = + if current_path != "" do + name = Path.basename(current_path) - if Path.dirname(current_path) != root or not List.keymember?(releases, current, 0) do - Mix.raise("#{host}: current must resolve to a version directory inside #{root}") - end + if Path.dirname(current_path) != root or not List.keymember?(releases, name, 0) do + Mix.raise("#{host}: current must resolve to a version directory inside #{root}") + end + + name + end {root, current, releases} diff --git a/lib/mix/tasks/dicon.prune.ex b/lib/mix/tasks/dicon.prune.ex index 11fdc1a..63a4853 100644 --- a/lib/mix/tasks/dicon.prune.ex +++ b/lib/mix/tasks/dicon.prune.ex @@ -4,9 +4,11 @@ defmodule Mix.Tasks.Dicon.Prune do @shortdoc "Keeps current and the previous release on the configured servers" @moduledoc """ - Deletes version directories except current and the highest SemVer below it. - The previous version may be a pre-release. Newer versions are also deleted. - Non-version entries and symlinks are left alone. + Deletes version directories except current and the highest stable SemVer below + it. If no lower stable version exists, keeps the highest pre-release below + current instead. All other versions, including newer versions, are deleted. + Non-version entries and symlinks are left alone. Unlike automatic pruning on + switch, this task has no record of the release previously selected by current. The configured hosts and target directory come from the application's `:dicon` configuration. Use `--only HOST` / `--skip HOST` to filter hosts and `--dry-run` diff --git a/lib/mix/tasks/dicon.switch.ex b/lib/mix/tasks/dicon.switch.ex index d6c7af4..884b17d 100644 --- a/lib/mix/tasks/dicon.switch.ex +++ b/lib/mix/tasks/dicon.switch.ex @@ -15,7 +15,8 @@ defmodule Mix.Tasks.Dicon.Switch do versions won't affect the running application. With `config :dicon, prune_on_switch: true`, each host is pruned after switching, - keeping current and the highest SemVer below it (including pre-releases). See + keeping current and the release current pointed to before the switch. Switching + to the already-current version skips pruning to preserve the rollback target. See `mix help dicon.prune` and the README for the retention and rollback policy. ## Usage @@ -46,10 +47,18 @@ defmodule Mix.Tasks.Dicon.Switch do for host <- config(:hosts, opts) do {authority, host_config} = Keyword.pop!(host_config(host), :authority) conn = Executor.connect(authority, host_config) + prune? = Application.get_env(:dicon, :prune_on_switch, false) + + previous = + if prune?, do: Dicon.Pruner.current_version(conn, config(:target_dir), host) + symlink(conn, [target_dir, ?/, version], [target_dir, "/current"]) - if Application.get_env(:dicon, :prune_on_switch, false) do - Dicon.Pruner.prune(conn, config(:target_dir), host, expected_current: version) + if prune? and previous != version do + Dicon.Pruner.prune(conn, config(:target_dir), host, + expected_current: version, + keep_previous: previous + ) end end diff --git a/test/mix/tasks/dicon.prune_test.exs b/test/mix/tasks/dicon.prune_test.exs index 0bbbab3..f9d0015 100644 --- a/test/mix/tasks/dicon.prune_test.exs +++ b/test/mix/tasks/dicon.prune_test.exs @@ -38,28 +38,37 @@ defmodule Mix.Tasks.Dicon.PruneTest do for host <- ["one", "two"] do assert_receive {:dicon, _, :connect, [^host, []]} - assert_info("#{host}: keeping current 1.7.6, previous 1.7.6-rc4") + assert_info("#{host}: keeping current 1.7.6, previous 1.7.5") assert_info_suffix("#{host}: would delete ", "/1.8.0-rc3-fcm-http2") - assert_info_suffix("#{host}: would delete ", "/1.7.5") + assert_info_suffix("#{host}: would delete ", "/1.7.6-rc4") end assert Enum.sort(File.ls!(target)) == Enum.sort(["current" | versions]) refute_delete_command() end - test "deletes older and newer versions, keeping the highest previous RC", %{target: target} do - releases(target, ~w(1.7.5 1.7.6-rc4 1.7.6 1.8.0-rc3-fcm-http2), "1.7.6") + test "standalone pruning prefers the highest stable below current over an RC", %{target: target} do + releases(target, ~w(1.7.4 1.7.5 1.7.6-rc4 1.7.6 1.8.0-rc3-fcm-http2), "1.7.6") run([]) - assert Enum.sort(File.ls!(target)) == ~w(1.7.6 1.7.6-rc4 current) + assert Enum.sort(File.ls!(target)) == ~w(1.7.5 1.7.6 current) assert File.read_link!(Path.join(target, "current")) == "1.7.6" assert_info_suffix("one: deleted ", "/1.8.0-rc3-fcm-http2") - assert_info_suffix("one: deleted ", "/1.7.5") + assert_info_suffix("one: deleted ", "/1.7.6-rc4") + assert_info_suffix("one: deleted ", "/1.7.4") + end + + test "RC current also prefers a lower stable release", %{target: target} do + releases(target, ~w(1.1.7 1.1.8 1.1.9-rc0 1.1.9-rc1 1.1.9), "1.1.9-rc1") + + run([]) + + assert Enum.sort(File.ls!(target)) == ~w(1.1.8 1.1.9-rc1 current) end - test "current can be an RC and alphanumeric identifiers sort lexically", %{target: target} do - releases(target, ~w(1.1.8 1.1.9-rc2 1.1.9-rc9 1.1.9-rc10 1.1.9 1.1.10), "1.1.9-rc9") + test "falls back to the highest RC below current when no lower stable exists", %{target: target} do + releases(target, ~w(1.1.8-rc1 1.1.9-rc2 1.1.9-rc9 1.1.9-rc10 1.1.9 1.1.10), "1.1.9-rc9") run([]) @@ -68,7 +77,7 @@ defmodule Mix.Tasks.Dicon.PruneTest do end test "numeric pre-release identifiers and release components use SemVer", %{target: target} do - releases(target, ~w(1.9.0 1.10.0-rc.2 1.10.0-rc.10 1.10.0 2.0.0), "1.10.0") + releases(target, ~w(1.9.0-rc.1 1.10.0-rc.2 1.10.0-rc.10 1.10.0 2.0.0), "1.10.0") run([]) @@ -143,7 +152,7 @@ defmodule Mix.Tasks.Dicon.PruneTest do test "rejects a missing current", %{target: target} do File.mkdir_p!(Path.join(target, "1.0.0")) - assert_raise Mix.Error, ~r/Pruning command failed/, fn -> run([]) end + assert_raise Mix.Error, ~r/current is missing/, fn -> run([]) end assert File.dir?(Path.join(target, "1.0.0")) refute_delete_command() @@ -289,7 +298,30 @@ defmodule Mix.Tasks.Dicon.PruneTest do assert_raise Mix.Error, "Invalid option: --invalid", fn -> run(["--invalid"]) end end - test "opted-in switch prunes after selecting the staged release and allows rollback", %{ + test "opted-in switch preserves the previous current when an unused RC is on disk", %{ + target: target + } do + releases(target, ~w(1.7.4 1.7.5 1.7.6-rc4 1.7.6), "1.7.5") + put_dicon_env(%{prune_on_switch: true}) + + Mix.Tasks.Dicon.Switch.run(["1.7.6"]) + + assert Enum.sort(File.ls!(target)) == ~w(1.7.5 1.7.6 current) + assert File.read!(Path.join([target, "1.7.5", "release.txt"])) == "1.7.5" + assert_info("one: keeping current 1.7.6, previous 1.7.5") + end + + test "opted-in switch retains an RC if it was the previous current", %{target: target} do + releases(target, ~w(1.7.5 1.7.6-rc4 1.7.6), "1.7.6-rc4") + put_dicon_env(%{prune_on_switch: true}) + + Mix.Tasks.Dicon.Switch.run(["1.7.6"]) + + assert Enum.sort(File.ls!(target)) == ~w(1.7.6 1.7.6-rc4 current) + assert_info("one: keeping current 1.7.6, previous 1.7.6-rc4") + end + + test "opted-in rollback preserves the rolled-back-from release for rolling forward", %{ target: target } do releases(target, ~w(1.0.0 1.1.0 1.2.0), "1.1.0") @@ -302,8 +334,71 @@ defmodule Mix.Tasks.Dicon.PruneTest do Mix.Tasks.Dicon.Switch.run(["1.1.0"]) - assert Enum.sort(File.ls!(target)) == ~w(1.1.0 current) + assert Enum.sort(File.ls!(target)) == ~w(1.1.0 1.2.0 current) assert Path.basename(File.read_link!(Path.join(target, "current"))) == "1.1.0" + assert_info("one: keeping current 1.1.0, previous 1.2.0") + + Mix.Tasks.Dicon.Switch.run(["1.2.0"]) + + assert Enum.sort(File.ls!(target)) == ~w(1.1.0 1.2.0 current) + assert Path.basename(File.read_link!(Path.join(target, "current"))) == "1.2.0" + end + + test "switching to current again preserves the rollback target", %{target: target} do + releases(target, ~w(1.7.6-rc4 1.7.6), "1.7.6") + put_dicon_env(%{prune_on_switch: true}) + + Mix.Tasks.Dicon.Switch.run(["1.7.6"]) + + assert Enum.sort(File.ls!(target)) == ~w(1.7.6 1.7.6-rc4 current) + refute_delete_command() + end + + test "first switch with no current keeps only the requested release", %{target: target} do + releases(target, ~w(1.7.5 1.7.6), "1.7.5") + File.rm!(Path.join(target, "current")) + put_dicon_env(%{prune_on_switch: true}) + + Mix.Tasks.Dicon.Switch.run(["1.7.6"]) + + assert Enum.sort(File.ls!(target)) == ~w(1.7.6 current) + assert_info("one: keeping current 1.7.6, previous none") + end + + test "invalid previous current aborts before switching", %{target: target} do + releases(target, ~w(log 1.7.5 1.7.6), "log") + put_dicon_env(%{prune_on_switch: true}) + + assert_raise Mix.Error, ~r/current must resolve to a version directory/, fn -> + Mix.Tasks.Dicon.Switch.run(["1.7.6"]) + end + + assert File.read_link!(Path.join(target, "current")) == "log" + assert Enum.sort(File.ls!(target)) == ~w(1.7.5 1.7.6 current log) + refute_delete_command() + end + + test "pruning aborts if the captured previous release disappears during switch", %{ + root: root, + target: target + } do + releases(target, ~w(1.7.4 1.7.5 1.7.6), "1.7.5") + put_dicon_env(%{prune_on_switch: true}) + + on_exec(:all, fn command, device -> + if String.starts_with?(command, "ln -snf") do + File.rm_rf!(Path.join(target, "1.7.5")) + end + + exec_locally(command, device, root) + end) + + assert_raise Mix.Error, ~r/previous release 1.7.5 is missing/, fn -> + Mix.Tasks.Dicon.Switch.run(["1.7.6"]) + end + + assert Enum.sort(File.ls!(target)) == ~w(1.7.4 1.7.6 current) + refute_delete_command() end test "does not prune if switch fails to select the requested version", %{target: target} do From e1371dda663f2cd4e20a4888a1a7b479ac28f2f8 Mon Sep 17 00:00:00 2001 From: Y86 Date: Tue, 3 Nov 2026 17:33:10 -0300 Subject: [PATCH 4/4] Format switch pruning with the pinned Elixir version --- lib/mix/tasks/dicon.switch.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/mix/tasks/dicon.switch.ex b/lib/mix/tasks/dicon.switch.ex index 884b17d..8e92f2a 100644 --- a/lib/mix/tasks/dicon.switch.ex +++ b/lib/mix/tasks/dicon.switch.ex @@ -49,8 +49,7 @@ defmodule Mix.Tasks.Dicon.Switch do conn = Executor.connect(authority, host_config) prune? = Application.get_env(:dicon, :prune_on_switch, false) - previous = - if prune?, do: Dicon.Pruner.current_version(conn, config(:target_dir), host) + previous = if prune?, do: Dicon.Pruner.current_version(conn, config(:target_dir), host) symlink(conn, [target_dir, ?/, version], [target_dir, "/current"])