nixos-zfs-ec2-ami/user-data.sh.tftpl

#!/usr/bin/env bash
set -euo pipefail

nix_device='${nix_device}'
nix_pool='${nix_pool}'
state_dir=/var/lib/nixos-zfs-ec2-ami
bootstrap_marker="$state_dir/bootstrap-generation-installed"
ready_marker="$state_dir/ami-ready"

mkdir -p "$state_dir"

write_bootstrap_config() {
  mkdir -p /etc/nixos
  cat > /etc/nixos/flake.nix <<'EOF'
{
  inputs = {
    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
  };

  outputs = { nixpkgs, ... }: {
    nixosConfigurations.ami-builder-bootstrap = nixpkgs.lib.nixosSystem {
      system = "${system}";
      modules = [
        ({ modulesPath, pkgs, ... }: {
          imports = [ "$${modulesPath}/virtualisation/amazon-image.nix" ];

          networking.hostName = "nixos-zfs-ec2-builder";
          networking.hostId = "6e69787a";

          nix.settings.experimental-features = [ "nix-command" "flakes" ];
          boot.supportedFilesystems = [ "zfs" ];
          boot.zfs.forceImportRoot = false;

          environment.systemPackages = [
            pkgs.git
            pkgs.rsync
            pkgs.zfs
          ];

          services.amazon-ssm-agent.enable = true;

          users.users.builder-admin = {
            isNormalUser = true;
            extraGroups = [ "wheel" ];
            openssh.authorizedKeys.keys = ${ssh_keys};
          };
          security.sudo.wheelNeedsPassword = false;

          system.stateVersion = "25.05";
        })
      ];
    };
  };
}
EOF
}

write_final_config() {
  mkdir -p /etc/nixos
  cat > /etc/nixos/flake.nix <<'EOF'
{
  inputs = {
    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
  };

  outputs = { nixpkgs, ... }: {
    nixosConfigurations.nixos-zfs-ec2 = nixpkgs.lib.nixosSystem {
      system = "${system}";
      modules = [
        ({ modulesPath, pkgs, ... }: {
          imports = [ "$${modulesPath}/virtualisation/amazon-image.nix" ];

          networking.hostName = "nixos-zfs-ec2";
          networking.hostId = "6e69787a";

          nix.settings.experimental-features = [ "nix-command" "flakes" ];
          boot.supportedFilesystems = [ "zfs" ];
          boot.zfs.extraPools = [ "${nix_pool}" ];
          boot.zfs.forceImportRoot = false;

          fileSystems."/nix" = {
            device = "${nix_pool}/nix";
            fsType = "zfs";
            neededForBoot = true;
          };

          systemd.sockets.nix-daemon = {
            after = [ "nix.mount" ];
            requires = [ "nix.mount" ];
          };

          systemd.services.nix-daemon = {
            after = [ "nix.mount" ];
            requires = [ "nix.mount" ];
          };

          environment.systemPackages = [
            pkgs.git
            pkgs.rsync
            pkgs.zfs
          ];

          services.amazon-ssm-agent.enable = true;

          users.users.builder-admin = {
            isNormalUser = true;
            extraGroups = [ "wheel" ];
            openssh.authorizedKeys.keys = ${ssh_keys};
          };
          security.sudo.wheelNeedsPassword = false;

          system.stateVersion = "25.05";
        })
      ];
    };
  };
}
EOF
}

if [ ! -e "$bootstrap_marker" ]; then
  write_bootstrap_config
  nixos-rebuild boot --flake /etc/nixos#ami-builder-bootstrap
  touch "$bootstrap_marker"
  systemctl reboot
  exit 0
fi

for _ in $(seq 1 600); do
  if [ -e "$nix_device" ]; then
    break
  fi
  sleep 1
done

if [ ! -e "$nix_device" ]; then
  echo "Nix device did not appear: $nix_device" >&2
  exit 1
fi

if ! zpool list -H "$nix_pool" >/dev/null 2>&1; then
  if ! zpool import "$nix_pool" >/dev/null 2>&1; then
    zpool create \
      -f \
      -o ashift=12 \
      -O acltype=posixacl \
      -O atime=off \
      -O compression=zstd \
      -O mountpoint=none \
      -O xattr=sa \
      "$nix_pool" "$nix_device"

    zfs create -o mountpoint=legacy "$nix_pool/nix"
  fi
fi

mkdir -p /mnt/nixos-zfs-nix
if ! mountpoint -q /mnt/nixos-zfs-nix; then
  mount -t zfs "$nix_pool/nix" /mnt/nixos-zfs-nix
fi

write_final_config
nixos-rebuild boot --flake /etc/nixos#nixos-zfs-ec2

rsync -aHAX --numeric-ids \
  --exclude='/var/nix/daemon-socket/*' \
  /nix/ /mnt/nixos-zfs-nix/
rm -f /mnt/nixos-zfs-nix/var/nix/daemon-socket/socket

umount /mnt/nixos-zfs-nix
zpool export "$nix_pool"

cat > "$ready_marker" <<EOF
architecture=${system}
input_hash=${input_hash}
builder_git_rev=${builder_git_rev}
flake_lock_rev=${flake_lock_rev}
EOF

systemctl poweroff