Build AMIs on temporary EC2 builders

2026-06-20 11:41:57 -07:00
parent e580f7e3b2
commit 149ffd374f
6 changed files with 632 additions and 353 deletions
@@ -0,0 +1,180 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+nix_device='${nix_device}'
+nix_pool='${nix_pool}'
+state_dir=/var/lib/nixos-zfs-ec2-ami
+bootstrap_marker="$state_dir/bootstrap-generation-installed"
+ready_marker="$state_dir/ami-ready"
+
+mkdir -p "$state_dir"
+
+write_bootstrap_config() {
+  mkdir -p /etc/nixos
+  cat > /etc/nixos/flake.nix <<'EOF'
+{
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+  };
+
+  outputs = { nixpkgs, ... }: {
+    nixosConfigurations.ami-builder-bootstrap = nixpkgs.lib.nixosSystem {
+      system = "${system}";
+      modules = [
+        ({ modulesPath, pkgs, ... }: {
+          imports = [ "$${modulesPath}/virtualisation/amazon-image.nix" ];
+
+          networking.hostName = "nixos-zfs-ec2-builder";
+          networking.hostId = "6e69787a";
+
+          nix.settings.experimental-features = [ "nix-command" "flakes" ];
+          boot.supportedFilesystems = [ "zfs" ];
+          boot.zfs.forceImportRoot = false;
+
+          environment.systemPackages = [
+            pkgs.git
+            pkgs.rsync
+            pkgs.zfs
+          ];
+
+          services.amazon-ssm-agent.enable = true;
+
+          users.users.builder-admin = {
+            isNormalUser = true;
+            extraGroups = [ "wheel" ];
+            openssh.authorizedKeys.keys = ${ssh_keys};
+          };
+          security.sudo.wheelNeedsPassword = false;
+
+          system.stateVersion = "25.05";
+        })
+      ];
+    };
+  };
+}
+EOF
+}
+
+write_final_config() {
+  mkdir -p /etc/nixos
+  cat > /etc/nixos/flake.nix <<'EOF'
+{
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
+  };
+
+  outputs = { nixpkgs, ... }: {
+    nixosConfigurations.nixos-zfs-ec2 = nixpkgs.lib.nixosSystem {
+      system = "${system}";
+      modules = [
+        ({ modulesPath, pkgs, ... }: {
+          imports = [ "$${modulesPath}/virtualisation/amazon-image.nix" ];
+
+          networking.hostName = "nixos-zfs-ec2";
+          networking.hostId = "6e69787a";
+
+          nix.settings.experimental-features = [ "nix-command" "flakes" ];
+          boot.supportedFilesystems = [ "zfs" ];
+          boot.zfs.extraPools = [ "${nix_pool}" ];
+          boot.zfs.forceImportRoot = false;
+
+          fileSystems."/nix" = {
+            device = "${nix_pool}/nix";
+            fsType = "zfs";
+            neededForBoot = true;
+          };
+
+          systemd.sockets.nix-daemon = {
+            after = [ "nix.mount" ];
+            requires = [ "nix.mount" ];
+          };
+
+          systemd.services.nix-daemon = {
+            after = [ "nix.mount" ];
+            requires = [ "nix.mount" ];
+          };
+
+          environment.systemPackages = [
+            pkgs.git
+            pkgs.rsync
+            pkgs.zfs
+          ];
+
+          services.amazon-ssm-agent.enable = true;
+
+          users.users.builder-admin = {
+            isNormalUser = true;
+            extraGroups = [ "wheel" ];
+            openssh.authorizedKeys.keys = ${ssh_keys};
+          };
+          security.sudo.wheelNeedsPassword = false;
+
+          system.stateVersion = "25.05";
+        })
+      ];
+    };
+  };
+}
+EOF
+}
+
+if [ ! -e "$bootstrap_marker" ]; then
+  write_bootstrap_config
+  nixos-rebuild boot --flake /etc/nixos#ami-builder-bootstrap
+  touch "$bootstrap_marker"
+  systemctl reboot
+  exit 0
+fi
+
+for _ in $(seq 1 600); do
+  if [ -e "$nix_device" ]; then
+    break
+  fi
+  sleep 1
+done
+
+if [ ! -e "$nix_device" ]; then
+  echo "Nix device did not appear: $nix_device" >&2
+  exit 1
+fi
+
+if ! zpool list -H "$nix_pool" >/dev/null 2>&1; then
+  if ! zpool import "$nix_pool" >/dev/null 2>&1; then
+    zpool create \
+      -f \
+      -o ashift=12 \
+      -O acltype=posixacl \
+      -O atime=off \
+      -O compression=zstd \
+      -O mountpoint=none \
+      -O xattr=sa \
+      "$nix_pool" "$nix_device"
+
+    zfs create -o mountpoint=legacy "$nix_pool/nix"
+  fi
+fi
+
+mkdir -p /mnt/nixos-zfs-nix
+if ! mountpoint -q /mnt/nixos-zfs-nix; then
+  mount -t zfs "$nix_pool/nix" /mnt/nixos-zfs-nix
+fi
+
+write_final_config
+nixos-rebuild boot --flake /etc/nixos#nixos-zfs-ec2
+
+rsync -aHAX --numeric-ids \
+  --exclude='/var/nix/daemon-socket/*' \
+  /nix/ /mnt/nixos-zfs-nix/
+rm -f /mnt/nixos-zfs-nix/var/nix/daemon-socket/socket
+
+umount /mnt/nixos-zfs-nix
+zpool export "$nix_pool"
+
+cat > "$ready_marker" <<EOF
+architecture=${system}
+input_hash=${input_hash}
+builder_git_rev=${builder_git_rev}
+flake_lock_rev=${flake_lock_rev}
+EOF
+
+systemctl poweroff