Add AMI artifact management tool

README.md

@@ -60,3 +60,20 @@ The final AMI is tagged with:
 
 Do not deregister the AMI or delete either snapshot while runtime deployments
 use it.
+
+## Artifact Cleanup
+
+List published AMIs and cross-reference qx-deploy runtime registry records:
+
+```bash
+nix run ./tofu/ami-builder#ami-artifacts -- list
+```
+
+Delete an unused AMI and its root and `/nix` snapshots:
+
+```bash
+nix run ./tofu/ami-builder#ami-artifacts -- delete --ami-id ami-...
+```
+
+The delete command refuses to delete AMIs referenced by runtime deployment
+registry records unless you pass `--force`.

flake.nix

@@ -26,11 +26,65 @@
 
       packages = forAllSystems (system:
         let
+          pkgs = import nixpkgs { inherit system; };
           config = self.nixosConfigurations."baseline-${system}".config;
+          ami-artifacts-script = pkgs.stdenvNoCC.mkDerivation {
+            pname = "ami-artifacts-script";
+            version = "0.1.0";
+            src = ./manage-amis.ts;
+            nativeBuildInputs = [ pkgs.esbuild ];
+            dontUnpack = true;
+            buildPhase = ''
+              runHook preBuild
+              esbuild "$src" \
+                --bundle \
+                --platform=node \
+                --target=node24 \
+                --format=esm \
+                --outfile=manage-amis.mjs
+              runHook postBuild
+            '';
+            installPhase = ''
+              runHook preInstall
+              install -Dm755 manage-amis.mjs "$out/libexec/ami-artifacts/manage-amis.mjs"
+              runHook postInstall
+            '';
+          };
+          ami-artifacts = pkgs.writeShellApplication {
+            name = "ami-artifacts";
+            runtimeInputs = [
+              pkgs.awscli2
+              pkgs.git
+              pkgs.nodejs_24
+              pkgs.opentofu
+            ];
+            text = ''
+              git_root="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
+              if [ -d "$git_root/tofu/ami-builder" ]; then
+                repo_root="$git_root"
+                ami_builder_dir="$git_root/tofu/ami-builder"
+              else
+                repo_root="$(cd "$git_root/../.." && pwd)"
+                ami_builder_dir="$git_root"
+              fi
+              export QUIXOS_REPO_ROOT="$repo_root"
+              export QUIXOS_AMI_BUILDER_DIR="$ami_builder_dir"
+              exec ${pkgs.nodejs_24}/bin/node ${ami-artifacts-script}/libexec/ami-artifacts/manage-amis.mjs "$@"
+            '';
+          };
         in
         {
           rootImage = config.system.build.images.amazon;
           systemClosure = config.system.build.toplevel;
+          ami-artifacts = ami-artifacts;
+          default = ami-artifacts;
         });
+
+      apps = forAllSystems (system: {
+        ami-artifacts = {
+          type = "app";
+          program = "${self.packages.${system}.ami-artifacts}/bin/ami-artifacts";
+        };
+      });
     };
 }

manage-amis.ts

@@ -0,0 +1,304 @@
+#!/usr/bin/env node
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { spawnSync } from "node:child_process";
+import { fileURLToPath } from "node:url";
+
+const sourceScriptDir = path.dirname(fileURLToPath(import.meta.url));
+const repoRoot = process.env.QUIXOS_REPO_ROOT ?? path.resolve(sourceScriptDir, "../..");
+const amiBuilderDir = process.env.QUIXOS_AMI_BUILDER_DIR ?? path.resolve(repoRoot, "tofu/ami-builder");
+const centralDir = path.resolve(amiBuilderDir, "../central");
+
+const usage = `Usage: nix run ./tofu/ami-builder#ami-artifacts -- <list|delete> [options]
+
+Commands:
+  list                    List published AMI artifacts and deployment references
+  delete --ami-id <ami>   Deregister an AMI and delete its root and /nix snapshots
+
+Options:
+  --ami-id <ami>                  AMI to delete
+  --aws-region <region>          AWS region (default: AWS_REGION or us-west-2)
+  --name <name>                  AMI project tag (default: nixos-zfs-ec2)
+  --architecture <arm64|x86_64>  Filter by architecture
+  --runtime-state-bucket <name>  S3 bucket with qx-deploy registry objects
+  --force                        Delete even when registry deployments reference the AMI
+  -h, --help                     Show this help
+`;
+
+const run = (command, args, options = {}) => {
+  const result = spawnSync(command, args, {
+    cwd: options.cwd ?? repoRoot,
+    env: options.env ?? process.env,
+    encoding: "utf8",
+    stdio: options.capture ? ["ignore", "pipe", "pipe"] : "inherit",
+  });
+  if (result.error) {
+    throw result.error;
+  }
+  if (!options.allowFailure && result.status !== 0) {
+    const suffix = options.capture && result.stderr ? `\n${result.stderr.trimEnd()}` : "";
+    throw new Error(`Command failed: ${command} ${args.join(" ")}${suffix}`);
+  }
+  return result;
+};
+
+const capture = (command, args, options = {}) =>
+  run(command, args, { ...options, capture: true }).stdout.trim();
+
+const captureMaybe = (command, args, options = {}) =>
+  run(command, args, { ...options, capture: true, allowFailure: true });
+
+const requireCommand = (command) => {
+  const result = captureMaybe(command, ["--version"]);
+  if (result.status !== 0) {
+    throw new Error(`Missing required command: ${command}`);
+  }
+};
+
+const parseArgs = (argv) => {
+  const command = argv[0] ?? "";
+  if (!command || command === "-h" || command === "--help") {
+    console.error(usage.trimEnd());
+    process.exit(command ? 0 : 1);
+  }
+  if (command !== "list" && command !== "delete") {
+    throw new Error(`Unknown command: ${command}\n\n${usage}`);
+  }
+
+  const state = {
+    command,
+    amiId: "",
+    awsRegion: process.env.AWS_REGION ?? "us-west-2",
+    name: "nixos-zfs-ec2",
+    architecture: "",
+    runtimeStateBucket: process.env.QUIXOS_RUNTIME_STATE_BUCKET ?? "",
+    force: false,
+  };
+
+  const takeValue = (args, index, flag) => {
+    const value = args[index + 1];
+    if (!value) {
+      throw new Error(`Missing value for ${flag}`);
+    }
+    return value;
+  };
+
+  for (let i = 1; i < argv.length; i += 1) {
+    const arg = argv[i];
+    switch (arg) {
+      case "--ami-id":
+        state.amiId = takeValue(argv, i, arg);
+        i += 1;
+        break;
+      case "--aws-region":
+        state.awsRegion = takeValue(argv, i, arg);
+        i += 1;
+        break;
+      case "--name":
+        state.name = takeValue(argv, i, arg);
+        i += 1;
+        break;
+      case "--architecture":
+        state.architecture = takeValue(argv, i, arg);
+        i += 1;
+        break;
+      case "--runtime-state-bucket":
+        state.runtimeStateBucket = takeValue(argv, i, arg);
+        i += 1;
+        break;
+      case "--force":
+        state.force = true;
+        break;
+      case "-h":
+      case "--help":
+        console.error(usage.trimEnd());
+        process.exit(0);
+      default:
+        throw new Error(`Unknown argument: ${arg}\n\n${usage}`);
+    }
+  }
+
+  return state;
+};
+
+const resolveRuntimeStateBucket = (state) => {
+  if (state.runtimeStateBucket) {
+    return;
+  }
+  if (!fs.existsSync(centralDir)) {
+    return;
+  }
+  const result = captureMaybe("tofu", ["-chdir=" + centralDir, "output", "-raw", "runtime_state_bucket_name"]);
+  if (result.status === 0 && /^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(result.stdout.trim())) {
+    state.runtimeStateBucket = result.stdout.trim();
+  }
+};
+
+const awsJson = (args) => {
+  const text = capture("aws", [...args, "--output", "json"]);
+  return text ? JSON.parse(text) : null;
+};
+
+const amiTags = (image) => Object.fromEntries((image.Tags ?? []).map((tag) => [tag.Key, tag.Value]));
+
+const listImages = (state) => {
+  const filters = [
+    "Name=state,Values=available",
+    `Name=tag:Project,Values=${state.name}`,
+  ];
+  if (state.architecture) {
+    filters.push(`Name=tag:Architecture,Values=${state.architecture}`);
+  }
+
+  const result = awsJson([
+    "ec2",
+    "describe-images",
+    "--region",
+    state.awsRegion,
+    "--owners",
+    "self",
+    "--filters",
+    ...filters,
+  ]);
+
+  return (result?.Images ?? [])
+    .filter((image) => {
+      const tags = amiTags(image);
+      return tags.Role === "published-ami" || Boolean(tags.RootSnapshotId || tags.NixSnapshotId);
+    })
+    .sort((a, b) => String(b.CreationDate ?? "").localeCompare(String(a.CreationDate ?? "")));
+};
+
+const deploymentRegistry = (state) => {
+  if (!state.runtimeStateBucket) {
+    return [];
+  }
+
+  const listed = captureMaybe("aws", [
+    "s3api",
+    "list-objects-v2",
+    "--region",
+    state.awsRegion,
+    "--bucket",
+    state.runtimeStateBucket,
+    "--prefix",
+    "registry/deployments/",
+    "--query",
+    "Contents[].Key",
+    "--output",
+    "text",
+  ]);
+  if (listed.status !== 0 || !listed.stdout.trim() || listed.stdout.trim() === "None") {
+    return [];
+  }
+
+  const records = [];
+  const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "quixos-ami-registry."));
+  try {
+    for (const key of listed.stdout.trim().split(/\s+/)) {
+      if (!key.endsWith(".json")) {
+        continue;
+      }
+      const result = captureMaybe("aws", ["s3", "cp", `s3://${state.runtimeStateBucket}/${key}`, "-"]);
+      if (result.status === 0 && result.stdout.trim()) {
+        records.push(JSON.parse(result.stdout));
+      }
+    }
+  } finally {
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+  }
+  return records;
+};
+
+const listArtifacts = (state) => {
+  const registry = deploymentRegistry(state);
+  const references = new Map();
+  for (const record of registry) {
+    const amiId = record.config?.amiId;
+    if (!amiId) {
+      continue;
+    }
+    const entries = references.get(amiId) ?? [];
+    entries.push(record.id);
+    references.set(amiId, entries);
+  }
+
+  for (const image of listImages(state)) {
+    const tags = amiTags(image);
+    const refs = references.get(image.ImageId)?.join(",") ?? "-";
+    console.log([
+      image.ImageId,
+      image.Name ?? "",
+      image.CreationDate ?? "",
+      tags.Architecture ?? image.Architecture ?? "",
+      tags.BuilderGitRev ?? "",
+      tags.RootSnapshotId ?? "",
+      tags.NixSnapshotId ?? "",
+      refs,
+    ].join("\t"));
+  }
+};
+
+const deleteArtifact = (state) => {
+  if (!state.amiId) {
+    throw new Error("Missing --ami-id");
+  }
+
+  const registry = deploymentRegistry(state);
+  const refs = registry
+    .filter((record) => record.config?.amiId === state.amiId)
+    .map((record) => record.id);
+  if (refs.length > 0 && !state.force) {
+    throw new Error(`Refusing to delete ${state.amiId}; referenced by deployments: ${refs.join(", ")}\nPass --force after those deployments are destroyed or intentionally orphaned.`);
+  }
+
+  const result = awsJson([
+    "ec2",
+    "describe-images",
+    "--region",
+    state.awsRegion,
+    "--owners",
+    "self",
+    "--image-ids",
+    state.amiId,
+  ]);
+  const image = result?.Images?.[0];
+  if (!image) {
+    throw new Error(`AMI not found: ${state.amiId}`);
+  }
+  const tags = amiTags(image);
+  const rootSnapshot = tags.RootSnapshotId ?? "";
+  const nixSnapshot = tags.NixSnapshotId ?? "";
+
+  console.error(`Deregistering AMI: ${state.amiId}`);
+  run("aws", ["ec2", "deregister-image", "--region", state.awsRegion, "--image-id", state.amiId]);
+
+  if (rootSnapshot) {
+    console.error(`Deleting root snapshot: ${rootSnapshot}`);
+    run("aws", ["ec2", "delete-snapshot", "--region", state.awsRegion, "--snapshot-id", rootSnapshot]);
+  }
+  if (nixSnapshot) {
+    console.error(`Deleting /nix snapshot: ${nixSnapshot}`);
+    run("aws", ["ec2", "delete-snapshot", "--region", state.awsRegion, "--snapshot-id", nixSnapshot]);
+  }
+};
+
+const main = () => {
+  const state = parseArgs(process.argv.slice(2));
+  requireCommand("aws");
+  resolveRuntimeStateBucket(state);
+
+  if (state.command === "list") {
+    listArtifacts(state);
+    return;
+  }
+  deleteArtifact(state);
+};
+
+try {
+  main();
+} catch (error) {
+  console.error(error instanceof Error ? error.message : String(error));
+  process.exit(1);
+}

publish-ami.sh

@@ -261,6 +261,7 @@ aws ec2 create-tags \
   --tags \
     "Key=Name,Value=${ami_name}" \
     "Key=Project,Value=${name}" \
+    "Key=Role,Value=published-ami" \
     "Key=Architecture,Value=${architecture}" \
     "Key=System,Value=${system}" \
     "Key=InputHash,Value=${input_hash}" \