Document DNS-based early cache hints

README.md

@@ -15,13 +15,37 @@ also disables the default shell-oriented `amazon-init` service and enables a
 small generic service that treats EC2 user-data as `/etc/nixos/flake.nix`, then
 runs `nixos-rebuild switch --flake /etc/nixos`.
 
+## First-Boot Build Cache Hints
+
+The first user-data switch is built by the AMI's existing Nix configuration,
+before the target NixOS system has activated its own `nix.settings`. A caller
+flake can optionally expose `quixosEarlyBuildNixConfig` to make that first
+build use known binary caches without baking any project-specific cache into
+the AMI:
+
+```nix
+{
+  quixosEarlyBuildNixConfig = {
+    substituters = [ "http://nix-cache.internal.example.org/cache" ];
+    trustedPublicKeys = [ "nix-cache.internal.example.org-1:..." ];
+  };
+}
+```
+
+The AMI runner evaluates this output after writing `/etc/nixos/flake.nix` and
+passes the substituters and trusted public keys through `NIX_CONFIG` only for
+the first `nixos-rebuild switch`. It does not mutate `/etc/hosts`; first-boot
+substituter hostnames must already resolve through DNS. The target NixOS
+configuration should still declare the same cache in `nix.settings` for all
+later rebuilds.
+
 ## Publish
 
 The normal flow builds on a temporary EC2 instance, so your local machine does
 not need to be the same architecture as the target AMI:
 
 ```bash
-nix run ./tofu/ami-builder#publish-ami -- --builder-instance-type t4g.large
+nix run ./tofu/ami-builder#publish-ami
 ```
 
 `publish-ami.sh` applies this OpenTofu project to create a temporary builder
@@ -42,7 +66,7 @@ deregistering the AMI.
 Pass extra OpenTofu variables after `--`:
 
 ```bash
-nix run ./tofu/ami-builder#publish-ami -- --builder-instance-type t4g.large -- \
+nix run ./tofu/ami-builder#publish-ami -- -- \
   -var base_nixos_ami_id=ami-...
 ```
 
@@ -58,6 +82,14 @@ The final AMI is tagged with:
 - `NixSnapshotId`
 - `NixosSystemClosure`
 
+The publisher also writes SSM parameters that point to the AMI ID:
+
+- `/quixos/amis/<name>/<architecture>/recommended`
+- `/quixos/amis/<name>/<architecture>/by-builder-rev/<rev>`
+- `/quixos/amis/<name>/<architecture>/by-input-hash/<hash>`
+
+Central services read the `recommended` parameter by default.
+
 Do not deregister the AMI or delete either snapshot while runtime deployments
 use it.
 
@@ -69,6 +101,13 @@ List published AMIs and cross-reference qx-deploy runtime registry records:
 nix run ./tofu/ami-builder#ami-artifacts -- list
 ```
 
+Publish SSM parameters for an existing AMI, useful for AMIs created before SSM
+publication was added:
+
+```bash
+nix run ./tofu/ami-builder#ami-artifacts -- publish-ssm --ami-id ami-...
+```
+
 Delete an unused AMI and its root and `/nix` snapshots:
 
 ```bash

flake.nix

@@ -14,6 +14,7 @@
         nixpkgs.lib.nixosSystem {
           inherit system;
           modules = [
+            self.nixosModules.zfsNixStore
             ./nixos/baseline.nix
           ];
         };
@@ -111,5 +112,7 @@
           program = "${self.packages.${system}.publish-ami}/bin/publish-ami";
         };
       });
+
+      nixosModules.zfsNixStore = import ./nixos/modules/zfs-nix-store.nix;
     };
 }

main.tf

@@ -46,10 +46,11 @@ locals {
     ? "nixos/*-${var.system}"
     : var.base_nixos_ami_name_pattern
   )
-  base_nixos_ami_id = var.base_nixos_ami_id == null ? data.aws_ami.nixos[0].id : var.base_nixos_ami_id
-  resource_name     = "${var.name}-builder-${local.aws_architecture}-${substr(var.input_hash, 0, 12)}"
-  nix_device_id     = replace(aws_ebs_volume.nix.id, "-", "")
-  nix_device        = "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_${local.nix_device_id}"
+  builder_instance_type = var.builder_instance_type == null ? (var.system == "aarch64-linux" ? "c7g.2xlarge" : "c7i.2xlarge") : var.builder_instance_type
+  base_nixos_ami_id     = var.base_nixos_ami_id == null ? data.aws_ami.nixos[0].id : var.base_nixos_ami_id
+  resource_name         = "${var.name}-builder-${local.aws_architecture}-${substr(var.input_hash, 0, 12)}"
+  nix_device_id         = replace(aws_ebs_volume.nix.id, "-", "")
+  nix_device            = "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_${local.nix_device_id}"
   tags = {
     Name          = local.resource_name
     Project       = var.name
@@ -114,6 +115,8 @@ resource "aws_ebs_volume" "nix" {
   availability_zone = data.aws_subnet.selected.availability_zone
   size              = var.nix_volume_size_gb
   type              = "gp3"
+  iops              = var.nix_volume_iops
+  throughput        = var.nix_volume_throughput
 
   tags = merge(local.tags, {
     Name = "${local.resource_name}-nix"
@@ -122,16 +125,18 @@ resource "aws_ebs_volume" "nix" {
 
 resource "aws_instance" "builder" {
   ami                         = local.base_nixos_ami_id
-  instance_type               = var.builder_instance_type
+  instance_type               = local.builder_instance_type
   subnet_id                   = local.subnet_id
   vpc_security_group_ids      = [aws_security_group.builder.id]
   iam_instance_profile        = aws_iam_instance_profile.builder.name
   associate_public_ip_address = true
+  ebs_optimized               = true
   key_name                    = var.ssh_public_key == null ? null : aws_key_pair.admin[0].key_name
   user_data_replace_on_change = true
 
   user_data = templatefile("${path.module}/user-data.sh.tftpl", {
     builder_git_rev = var.builder_git_rev
+    builder_repo_url = var.builder_repo_url
     flake_lock_rev  = var.flake_lock_rev
     input_hash      = var.input_hash
     nix_device      = local.nix_device
@@ -143,6 +148,8 @@ resource "aws_instance" "builder" {
   root_block_device {
     volume_size = var.root_volume_size_gb
     volume_type = "gp3"
+    iops        = var.root_volume_iops
+    throughput  = var.root_volume_throughput
   }
 
   tags = local.tags

manage-amis.ts

@@ -10,14 +10,15 @@ const repoRoot = process.env.QUIXOS_REPO_ROOT ?? path.resolve(sourceScriptDir, "
 const amiBuilderDir = process.env.QUIXOS_AMI_BUILDER_DIR ?? path.resolve(repoRoot, "tofu/ami-builder");
 const centralDir = path.resolve(amiBuilderDir, "../central");
 
-const usage = `Usage: nix run ./tofu/ami-builder#ami-artifacts -- <list|delete> [options]
+const usage = `Usage: nix run ./tofu/ami-builder#ami-artifacts -- <list|delete|publish-ssm> [options]
 
 Commands:
-  list                    List published AMI artifacts and deployment references
-  delete --ami-id <ami>   Deregister an AMI and delete its root and /nix snapshots
+  list                         List published AMI artifacts and deployment references
+  delete --ami-id <ami>        Deregister an AMI and delete its root and /nix snapshots
+  publish-ssm --ami-id <ami>   Publish an existing AMI into the SSM artifact registry
 
 Options:
-  --ami-id <ami>                  AMI to delete
+  --ami-id <ami>                  AMI to operate on
   --aws-region <region>          AWS region (default: AWS_REGION or us-west-2)
   --name <name>                  AMI project tag (default: nixos-zfs-ec2)
   --architecture <arm64|x86_64>  Filter by architecture
@@ -62,7 +63,7 @@ const parseArgs = (argv) => {
     console.error(usage.trimEnd());
     process.exit(command ? 0 : 1);
   }
-  if (command !== "list" && command !== "delete") {
+  if (command !== "list" && command !== "delete" && command !== "publish-ssm") {
     throw new Error(`Unknown command: ${command}\n\n${usage}`);
   }
 
@@ -146,6 +147,7 @@ const listImages = (state) => {
   const filters = [
     "Name=state,Values=available",
     `Name=tag:Project,Values=${state.name}`,
+    "Name=tag:Role,Values=published-ami",
   ];
   if (state.architecture) {
     filters.push(`Name=tag:Architecture,Values=${state.architecture}`);
@@ -163,10 +165,6 @@ const listImages = (state) => {
   ]);
 
   return (result?.Images ?? [])
-    .filter((image) => {
-      const tags = amiTags(image);
-      return tags.Role === "published-ami" || Boolean(tags.RootSnapshotId || tags.NixSnapshotId);
-    })
     .sort((a, b) => String(b.CreationDate ?? "").localeCompare(String(a.CreationDate ?? "")));
 };
 
@@ -290,6 +288,72 @@ const deleteArtifact = (state) => {
   }
 };
 
+const getImage = (state) => {
+  const result = awsJson([
+    "ec2",
+    "describe-images",
+    "--region",
+    state.awsRegion,
+    "--owners",
+    "self",
+    "--image-ids",
+    state.amiId,
+  ]);
+  const image = result?.Images?.[0];
+  if (!image) {
+    throw new Error(`AMI not found: ${state.amiId}`);
+  }
+  return image;
+};
+
+const putSsmParameter = (state, name, value) => {
+  run("aws", [
+    "ssm",
+    "put-parameter",
+    "--region",
+    state.awsRegion,
+    "--name",
+    name,
+    "--type",
+    "String",
+    "--value",
+    value,
+    "--overwrite",
+  ]);
+};
+
+const publishSsmArtifact = (state) => {
+  if (!state.amiId) {
+    throw new Error("Missing --ami-id");
+  }
+
+  const image = getImage(state);
+  const tags = amiTags(image);
+  const architecture = tags.Architecture ?? image.Architecture ?? "";
+  const builderGitRev = tags.BuilderGitRev ?? "";
+  const inputHash = tags.InputHash ?? "";
+  if (!architecture) {
+    throw new Error(`AMI ${state.amiId} is missing Architecture metadata`);
+  }
+  if (!builderGitRev) {
+    throw new Error(`AMI ${state.amiId} is missing BuilderGitRev tag`);
+  }
+  if (!inputHash) {
+    throw new Error(`AMI ${state.amiId} is missing InputHash tag`);
+  }
+
+  const prefix = `/quixos/amis/${state.name}/${architecture}`;
+  const parameters = [
+    `${prefix}/recommended`,
+    `${prefix}/by-builder-rev/${builderGitRev}`,
+    `${prefix}/by-input-hash/${inputHash}`,
+  ];
+  for (const parameter of parameters) {
+    console.error(`Writing ${parameter} = ${state.amiId}`);
+    putSsmParameter(state, parameter, state.amiId);
+  }
+};
+
 const main = () => {
   const state = parseArgs(process.argv.slice(2));
   requireCommand("aws");
@@ -299,6 +363,10 @@ const main = () => {
     listArtifacts(state);
     return;
   }
+  if (state.command === "publish-ssm") {
+    publishSsmArtifact(state);
+    return;
+  }
   deleteArtifact(state);
 };
 

nixos/baseline.nix

@@ -1,8 +1,5 @@
 { config, lib, modulesPath, pkgs, ... }:
 
-let
-  nixPool = "nixos-zfs-nix";
-in
 {
   imports = [
     "${modulesPath}/virtualisation/amazon-image.nix"
@@ -22,28 +19,9 @@ in
   services.amazon-ssm-agent.enable = true;
   services.openssh.enable = lib.mkForce false;
 
-  boot.supportedFilesystems = [ "zfs" ];
-  boot.zfs.extraPools = [ nixPool ];
-  boot.zfs.forceImportRoot = false;
-
-  fileSystems."/nix" = {
-    device = "${nixPool}/nix";
-    fsType = "zfs";
-    neededForBoot = true;
-    options = [
-      "nofail"
-      "x-systemd.device-timeout=30s"
-    ];
-  };
-
-  systemd.sockets.nix-daemon = {
-    after = [ "nix.mount" ];
-    requires = [ "nix.mount" ];
-  };
-
-  systemd.services.nix-daemon = {
-    after = [ "nix.mount" ];
-    requires = [ "nix.mount" ];
+  quixos.modules.zfsNixStore = {
+    enable = true;
+    pool = "nixos-zfs-nix";
   };
 
   system.stateVersion = "25.05";

nixos/modules/zfs-nix-store.nix

@@ -0,0 +1,50 @@
+{ config, lib, ... }:
+
+let
+  cfg = config.quixos.modules.zfsNixStore;
+in
+{
+  # TODO: split this module into a small standalone repo if another AMI family
+  # needs to consume it outside the nixos-zfs-ec2-ami boundary.
+  options.quixos.modules.zfsNixStore = {
+    enable = lib.mkEnableOption "ZFS-backed /nix store mount";
+
+    pool = lib.mkOption {
+      type = lib.types.str;
+      default = "nixos-zfs-nix";
+      description = "ZFS pool containing the /nix dataset.";
+    };
+
+    dataset = lib.mkOption {
+      type = lib.types.str;
+      default = "nix";
+      description = "Dataset within the pool mounted at /nix.";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    boot.supportedFilesystems = [ "zfs" ];
+    boot.zfs.extraPools = [ cfg.pool ];
+    boot.zfs.forceImportRoot = false;
+
+    fileSystems."/nix" = {
+      device = "${cfg.pool}/${cfg.dataset}";
+      fsType = "zfs";
+      neededForBoot = true;
+      options = [
+        "nofail"
+        "x-systemd.device-timeout=30s"
+      ];
+    };
+
+    systemd.sockets.nix-daemon = {
+      after = [ "nix.mount" ];
+      requires = [ "nix.mount" ];
+    };
+
+    systemd.services.nix-daemon = {
+      after = [ "nix.mount" ];
+      requires = [ "nix.mount" ];
+    };
+  };
+}

publish-ami.sh

@@ -3,10 +3,11 @@ set -euo pipefail
 
 aws_region="${AWS_REGION:-us-west-2}"
 system="aarch64-linux"
-builder_instance_type="t4g.large"
+builder_instance_type=""
 name="nixos-zfs-ec2"
 keep_builder=false
 extra_tofu_args=()
+script_start_epoch="$(date +%s)"
 
 usage() {
   cat >&2 <<'EOF'
@@ -16,7 +17,7 @@ Options:
   --system <aarch64-linux|x86_64-linux>
                               Target NixOS system (default: aarch64-linux)
   --aws-region <region>       AWS region (default: AWS_REGION or us-west-2)
-  --builder-instance-type <t> Temporary builder EC2 type (default: t4g.large)
+  --builder-instance-type <t> Temporary builder EC2 type (default: Terraform chooses per architecture)
   --name <name>               AMI name prefix
   --keep-builder              Keep temporary builder resources after publish
   -h, --help                  Show this help
@@ -87,6 +88,31 @@ require() {
   fi
 }
 
+elapsed() {
+  local start="$1"
+  local end
+  end="$(date +%s)"
+  printf '%ss' "$((end - start))"
+}
+
+total_elapsed() {
+  elapsed "$script_start_epoch"
+}
+
+log() {
+  printf '[%s] %s\n' "$(date -Is)" "$*" >&2
+}
+
+step() {
+  local label="$1"
+  shift
+  local start
+  start="$(date +%s)"
+  log "starting: ${label}"
+  "$@"
+  log "finished: ${label} ($(elapsed "$start"), total $(total_elapsed))"
+}
+
 require aws
 require git
 require jq
@@ -104,15 +130,23 @@ flake_lock_rev="$(jq -r '.nodes.nixpkgs.locked.rev // .nodes.nixpkgs.locked.narH
 input_hash="$(printf '%s\n%s\n%s\n' "$builder_git_rev" "$flake_lock_rev" "$system" | sha256sum | awk '{ print $1 }')"
 ami_name="${name}-${architecture}-${input_hash:0:12}"
 
-tofu init
-tofu apply \
-  -var "aws_region=$aws_region" \
-  -var "name=$name" \
-  -var "system=$system" \
-  -var "builder_instance_type=$builder_instance_type" \
-  -var "input_hash=$input_hash" \
-  -var "builder_git_rev=$builder_git_rev" \
-  -var "flake_lock_rev=$flake_lock_rev" \
+common_tofu_vars=(
+  -var "aws_region=$aws_region"
+  -var "name=$name"
+  -var "system=$system"
+  -var "input_hash=$input_hash"
+  -var "builder_git_rev=$builder_git_rev"
+  -var "flake_lock_rev=$flake_lock_rev"
+)
+
+if [ -n "$builder_instance_type" ]; then
+  common_tofu_vars+=(-var "builder_instance_type=$builder_instance_type")
+fi
+
+step "tofu init" tofu init
+step "tofu apply builder resources" tofu apply \
+  -auto-approve \
+  "${common_tofu_vars[@]}" \
   "${extra_tofu_args[@]}"
 
 builder_instance_id="$(tofu output -raw builder_instance_id)"
@@ -120,9 +154,10 @@ nix_volume_id="$(tofu output -raw nix_volume_id)"
 nix_device_name="$(tofu output -raw nix_device_name)"
 nix_pool="$(tofu output -raw nix_pool)"
 
-echo "Waiting for builder instance to stop: $builder_instance_id" >&2
-aws ec2 wait instance-stopped --region "$aws_region" --instance-ids "$builder_instance_id"
+log "waiting for builder instance to stop: $builder_instance_id"
+step "wait for builder instance stop" aws ec2 wait instance-stopped --region "$aws_region" --instance-ids "$builder_instance_id"
 
+log "creating capture image from stopped builder"
 capture_image_id="$(
   aws ec2 create-image \
     --region "$aws_region" \
@@ -134,8 +169,9 @@ capture_image_id="$(
     --query ImageId \
     --output text
 )"
+log "capture image: $capture_image_id"
 
-aws ec2 wait image-available --region "$aws_region" --image-ids "$capture_image_id"
+step "wait for capture image" aws ec2 wait image-available --region "$aws_region" --image-ids "$capture_image_id"
 
 capture_image_json="$(aws ec2 describe-images --region "$aws_region" --image-ids "$capture_image_id")"
 root_snapshot_id="$(
@@ -214,6 +250,7 @@ block_device_mappings="$(
     ]'
 )"
 
+log "registering final AMI"
 image_id="$(
   aws ec2 register-image \
     --region "$aws_region" \
@@ -227,10 +264,11 @@ image_id="$(
     --query ImageId \
     --output text
 )"
+log "final AMI: $image_id"
 
-aws ec2 wait image-available --region "$aws_region" --image-ids "$image_id"
+step "wait for final AMI" aws ec2 wait image-available --region "$aws_region" --image-ids "$image_id"
 
-aws ec2 create-tags \
+step "tag root snapshot" aws ec2 create-tags \
   --region "$aws_region" \
   --resources "$root_snapshot_id" \
   --tags \
@@ -242,7 +280,7 @@ aws ec2 create-tags \
     "Key=BuilderGitRev,Value=${builder_git_rev}" \
     "Key=FlakeLockRev,Value=${flake_lock_rev}"
 
-aws ec2 create-tags \
+step "tag nix snapshot and volume" aws ec2 create-tags \
   --region "$aws_region" \
   --resources "$nix_snapshot_id" "$nix_volume_id" \
   --tags \
@@ -255,7 +293,7 @@ aws ec2 create-tags \
     "Key=FlakeLockRev,Value=${flake_lock_rev}" \
     "Key=NixPool,Value=${nix_pool}"
 
-aws ec2 create-tags \
+step "tag final AMI" aws ec2 create-tags \
   --region "$aws_region" \
   --resources "$image_id" \
   --tags \
@@ -271,7 +309,21 @@ aws ec2 create-tags \
     "Key=RootSnapshotId,Value=${root_snapshot_id}" \
     "Key=NixSnapshotId,Value=${nix_snapshot_id}"
 
-aws ec2 deregister-image --region "$aws_region" --image-id "$capture_image_id" >/dev/null
+ssm_ami_prefix="/quixos/amis/${name}/${architecture}"
+for ssm_ami_parameter_name in \
+  "${ssm_ami_prefix}/recommended" \
+  "${ssm_ami_prefix}/by-builder-rev/${builder_git_rev}" \
+  "${ssm_ami_prefix}/by-input-hash/${input_hash}"
+do
+  step "publish SSM ${ssm_ami_parameter_name}" aws ssm put-parameter \
+    --region "$aws_region" \
+    --name "$ssm_ami_parameter_name" \
+    --type String \
+    --value "$image_id" \
+    --overwrite >/dev/null
+done
+
+step "deregister capture image" aws ec2 deregister-image --region "$aws_region" --image-id "$capture_image_id" >/dev/null
 
 jq -n \
   --arg architecture "$architecture" \
@@ -285,6 +337,7 @@ jq -n \
   --arg rootDeviceName "$root_device_name" \
   --arg nixDeviceName "$nix_device_name" \
   --arg nixPool "$nix_pool" \
+  --arg ssmRecommendedParameter "${ssm_ami_prefix}/recommended" \
   '{
     architecture: $architecture,
     system: $system,
@@ -296,20 +349,17 @@ jq -n \
     nixSnapshotId: $nixSnapshotId,
     rootDeviceName: $rootDeviceName,
     nixDeviceName: $nixDeviceName,
-    nixPool: $nixPool
+    nixPool: $nixPool,
+    ssmRecommendedParameter: $ssmRecommendedParameter
   }' > artifact.json
 
 if [ "$keep_builder" = false ]; then
-  tofu destroy \
+  step "tofu destroy builder resources" tofu destroy \
     -auto-approve \
-    -var "aws_region=$aws_region" \
-    -var "name=$name" \
-    -var "system=$system" \
-    -var "builder_instance_type=$builder_instance_type" \
-    -var "input_hash=$input_hash" \
-    -var "builder_git_rev=$builder_git_rev" \
-    -var "flake_lock_rev=$flake_lock_rev" \
+    "${common_tofu_vars[@]}" \
     "${extra_tofu_args[@]}"
 fi
 
 echo "Published AMI: $image_id"
+echo "Published SSM parameter: ${ssm_ami_prefix}/recommended"
+log "total elapsed: $(total_elapsed)"

user-data.sh.tftpl

@@ -9,34 +9,45 @@ ready_marker="$state_dir/ami-ready"
 
 mkdir -p "$state_dir"
 
-write_bootstrap_config() {
+write_config() {
   mkdir -p /etc/nixos
   cat > /etc/nixos/flake.nix <<'EOF'
 {
   inputs = {
     nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+    amiBuilderModules.url = "git+${builder_repo_url}?rev=${builder_git_rev}";
   };
 
-  outputs = { nixpkgs, ... }: {
-    nixosConfigurations.ami-builder-bootstrap = nixpkgs.lib.nixosSystem {
-      system = "${system}";
-      modules = [
-        ({ config, lib, modulesPath, pkgs, ... }: {
+  outputs = { nixpkgs, amiBuilderModules, ... }:
+    let
+      baseModule = { lib, modulesPath, pkgs, ... }: {
           imports = [ "$${modulesPath}/virtualisation/amazon-image.nix" ];
 
-          networking.hostName = "nixos-zfs-ec2-builder";
           networking.hostId = "6e69787a";
 
           nix.settings.experimental-features = [ "nix-command" "flakes" ];
           boot.supportedFilesystems = [ "zfs" ];
           boot.zfs.forceImportRoot = false;
+          swapDevices = [
+            {
+              device = "/swapfile";
+              size = 4096;
+            }
+          ];
 
           environment.systemPackages = [
+            pkgs.coreutils
             pkgs.git
             pkgs.rsync
+            pkgs.systemd
             pkgs.zfs
           ];
 
+          programs.bash.shellAliases = {
+            nixrb = "sudo nix --refresh flake update --flake /etc/nixos && sudo nixos-rebuild switch --flake /etc/nixos#nixos-zfs-ec2";
+            nixfu = "sudo nix --refresh flake update --flake /etc/nixos";
+          };
+
           services.amazon-ssm-agent.enable = true;
 
           users.users.builder-admin = {
@@ -47,68 +58,24 @@ write_bootstrap_config() {
           security.sudo.wheelNeedsPassword = false;
 
           system.stateVersion = "25.05";
-        })
-      ];
-    };
-  };
-}
-EOF
-}
+        };
 
-write_final_config() {
-  mkdir -p /etc/nixos
-  cat > /etc/nixos/flake.nix <<'EOF'
-{
-  inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
-  };
-
-  outputs = { nixpkgs, ... }: {
-    nixosConfigurations.nixos-zfs-ec2 = nixpkgs.lib.nixosSystem {
-      system = "${system}";
-      modules = [
-        ({ config, lib, modulesPath, pkgs, ... }: {
-          imports = [ "$${modulesPath}/virtualisation/amazon-image.nix" ];
-
-          networking.hostName = "nixos-zfs-ec2";
-          networking.hostId = "6e69787a";
-
-          nix.settings.experimental-features = [ "nix-command" "flakes" ];
-          boot.supportedFilesystems = [ "zfs" ];
-          boot.zfs.extraPools = [ "${nix_pool}" ];
-          boot.zfs.forceImportRoot = false;
-
-          fileSystems."/nix" = {
-            device = "${nix_pool}/nix";
-            fsType = "zfs";
-            neededForBoot = true;
-          };
-
-          systemd.sockets.nix-daemon = {
-            after = [ "nix.mount" ];
-            requires = [ "nix.mount" ];
-          };
-
-          systemd.services.nix-daemon = {
-            after = [ "nix.mount" ];
-            requires = [ "nix.mount" ];
-          };
-
-          environment.systemPackages = [
-            pkgs.git
-            pkgs.rsync
-            pkgs.zfs
-          ];
-
-          services.amazon-ssm-agent.enable = true;
+      zfsNixModule = { lib, ... }: {
           systemd.services.amazon-init.enable = lib.mkForce false;
 
+          quixos.modules.zfsNixStore = {
+            enable = true;
+            pool = "${nix_pool}";
+          };
+        };
+
+      userDataApplyModule = { config, lib, pkgs, ... }: {
           systemd.services.nixos-user-data-flake = {
             description = "Apply NixOS flake from EC2 user data";
             wantedBy = [ "multi-user.target" ];
             after = [ "fetch-ec2-metadata.service" "network-online.target" "nix-daemon.socket" ];
             wants = [ "fetch-ec2-metadata.service" "network-online.target" "nix-daemon.socket" ];
-            path = [ pkgs.coreutils pkgs.git ];
+            path = [ pkgs.coreutils pkgs.git pkgs.jq pkgs.nix pkgs.systemd ];
             serviceConfig = {
               Type = "oneshot";
               RemainAfterExit = true;
@@ -134,29 +101,75 @@ write_final_config() {
               fi
 
               install -m 0644 "$user_data" /etc/nixos/flake.nix
+
+              early_build_config=/run/nixos-user-data-flake-early-build-config.json
+              early_build_error=/run/nixos-user-data-flake-early-build-config.err
+              early_nix_config=/run/nixos-user-data-flake-early-nix.conf
+              : > "$early_nix_config"
+
+              if nix --extra-experimental-features 'nix-command flakes' \
+                eval --json /etc/nixos#quixosEarlyBuildNixConfig \
+                > "$early_build_config" 2> "$early_build_error"; then
+                substituters="$(jq -r '(.substituters // []) | join(" ")' "$early_build_config")"
+                trusted_public_keys="$(jq -r '(.trustedPublicKeys // []) | join(" ")' "$early_build_config")"
+                if [ -n "$substituters" ]; then
+                  printf 'extra-substituters = %s\n' "$substituters" >> "$early_nix_config"
+                fi
+                if [ -n "$trusted_public_keys" ]; then
+                  printf 'extra-trusted-public-keys = %s\n' "$trusted_public_keys" >> "$early_nix_config"
+                fi
+              else
+                echo "No quixosEarlyBuildNixConfig flake output found; first switch will use the AMI Nix configuration."
+              fi
+
+              runner=/run/nixos-user-data-flake-apply
+              cat > "$runner" <<RUNNER_EOF
+              #!$${pkgs.runtimeShell}
+              set -eu
+              export PATH="$${lib.makeBinPath [ pkgs.coreutils pkgs.git pkgs.systemd ]}"
+              if [ -s "$early_nix_config" ]; then
+                export NIX_CONFIG="\$(cat "$early_nix_config")"
+              fi
+
               $${config.system.build.nixos-rebuild}/bin/nixos-rebuild switch --flake /etc/nixos
               printf '%s\n' "$new_hash" > "$applied_hash_file"
+RUNNER_EOF
+              chmod 0700 "$runner"
+
+              systemd-run \
+                --unit=nixos-user-data-flake-apply \
+                --collect \
+                --no-ask-password \
+                --service-type=exec \
+                "$${pkgs.runtimeShell}" "$runner"
             '';
           };
+        };
 
-          users.users.builder-admin = {
-            isNormalUser = true;
-            extraGroups = [ "wheel" ];
-            openssh.authorizedKeys.keys = ${ssh_keys};
-          };
-          security.sudo.wheelNeedsPassword = false;
-
-          system.stateVersion = "25.05";
-        })
+      mkHost = hostName: extraModules: nixpkgs.lib.nixosSystem {
+        system = "${system}";
+        modules = [
+          amiBuilderModules.nixosModules.zfsNixStore
+          baseModule
+          ({ ... }: {
+            networking.hostName = hostName;
+          })
+        ] ++ extraModules;
+      };
+    in
+    {
+      nixosConfigurations.ami-builder-bootstrap = mkHost "nixos-zfs-ec2-builder" [];
+      nixosConfigurations.nixos-zfs-ec2 = mkHost "nixos-zfs-ec2" [
+        zfsNixModule
+        userDataApplyModule
       ];
     };
-  };
 }
 EOF
 }
 
 if [ ! -e "$bootstrap_marker" ]; then
-  write_bootstrap_config
+  write_config
   nixos-rebuild boot --flake /etc/nixos#ami-builder-bootstrap
   touch "$bootstrap_marker"
   systemctl reboot
@@ -196,7 +209,7 @@ if ! mountpoint -q /mnt/nixos-zfs-nix; then
   mount -t zfs "$nix_pool/nix" /mnt/nixos-zfs-nix
 fi
 
-write_final_config
+write_config
 nixos-rebuild boot --flake /etc/nixos#nixos-zfs-ec2
 
 rsync -aHAX --numeric-ids \

variables.tf

@@ -31,6 +31,12 @@ variable "builder_git_rev" {
   description = "Git revision of this AMI builder."
 }
 
+variable "builder_repo_url" {
+  type        = string
+  default     = "https://gitea-external.egads.tutti.syntaxblitz.net/quixos/nixos-zfs-ec2-ami.git"
+  description = "Repository URL containing the AMI builder and shared NixOS modules."
+}
+
 variable "flake_lock_rev" {
   type        = string
   description = "Revision or hash representing flake.lock inputs."
@@ -38,8 +44,8 @@ variable "flake_lock_rev" {
 
 variable "builder_instance_type" {
   type        = string
-  default     = "t4g.large"
-  description = "Temporary EC2 instance type used to build and seed the AMI."
+  default     = null
+  description = "Temporary EC2 instance type used to build and seed the AMI. Defaults to c7g.2xlarge for aarch64-linux and c7i.2xlarge for x86_64-linux."
 }
 
 variable "vpc_id" {
@@ -78,12 +84,36 @@ variable "root_volume_size_gb" {
   description = "Temporary builder root EBS volume size. This becomes the final AMI root snapshot size."
 }
 
+variable "root_volume_iops" {
+  type        = number
+  default     = 6000
+  description = "Temporary builder root gp3 IOPS. This affects builder speed only; snapshots do not preserve provisioned IOPS."
+}
+
+variable "root_volume_throughput" {
+  type        = number
+  default     = 250
+  description = "Temporary builder root gp3 throughput in MiB/s. This affects builder speed only; snapshots do not preserve provisioned throughput."
+}
+
 variable "nix_volume_size_gb" {
   type        = number
   default     = 50
   description = "Temporary /nix EBS volume size. This becomes the final AMI /nix snapshot size."
 }
 
+variable "nix_volume_iops" {
+  type        = number
+  default     = 12000
+  description = "Temporary /nix gp3 IOPS used while building and seeding the AMI."
+}
+
+variable "nix_volume_throughput" {
+  type        = number
+  default     = 500
+  description = "Temporary /nix gp3 throughput in MiB/s used while building and seeding the AMI."
+}
+
 variable "nix_pool" {
   type        = string
   default     = "nixos-zfs-nix"