Add NixOS ZFS EC2 AMI builder
This commit is contained in:
Timothy J. Aveni
Executable
+363
@@ -0,0 +1,363 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
aws_region="${AWS_REGION:-us-west-2}"
|
||||
architecture="arm64"
|
||||
bucket=""
|
||||
subnet_id=""
|
||||
security_group_id=""
|
||||
instance_profile_name=""
|
||||
instance_type="t4g.small"
|
||||
nix_volume_size_gb="50"
|
||||
name="nixos-zfs-ec2"
|
||||
|
||||
usage() {
|
||||
cat >&2 <<'EOF'
|
||||
Usage: publish-ami.sh --bucket s3://bucket[/prefix] [options]
|
||||
|
||||
Options:
|
||||
--architecture <arm64|x86_64> AMI architecture (default: arm64)
|
||||
--aws-region <region> AWS region (default: AWS_REGION or us-west-2)
|
||||
--bucket <s3-uri> S3 URI used by EC2 VM import
|
||||
--subnet-id <subnet> Subnet for the temporary seeder instance
|
||||
--security-group-id <sg> Security group for the temporary seeder instance
|
||||
--instance-profile-name <profile> Optional IAM instance profile for the seeder
|
||||
--instance-type <type> Seeder instance type (default: t4g.small)
|
||||
--nix-volume-size-gb <gb> /nix volume size (default: 50)
|
||||
--name <name> AMI name prefix
|
||||
EOF
|
||||
}
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--architecture)
|
||||
architecture="$2"
|
||||
shift 2
|
||||
;;
|
||||
--aws-region)
|
||||
aws_region="$2"
|
||||
shift 2
|
||||
;;
|
||||
--bucket)
|
||||
bucket="$2"
|
||||
shift 2
|
||||
;;
|
||||
--subnet-id)
|
||||
subnet_id="$2"
|
||||
shift 2
|
||||
;;
|
||||
--security-group-id)
|
||||
security_group_id="$2"
|
||||
shift 2
|
||||
;;
|
||||
--instance-profile-name)
|
||||
instance_profile_name="$2"
|
||||
shift 2
|
||||
;;
|
||||
--instance-type)
|
||||
instance_type="$2"
|
||||
shift 2
|
||||
;;
|
||||
--nix-volume-size-gb)
|
||||
nix_volume_size_gb="$2"
|
||||
shift 2
|
||||
;;
|
||||
--name)
|
||||
name="$2"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unknown argument: $1" >&2
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -z "$bucket" ]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$instance_profile_name" ]; then
|
||||
echo "Missing required --instance-profile-name for SSM seeding" >&2
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
case "$architecture" in
|
||||
arm64)
|
||||
nix_system="aarch64-linux"
|
||||
;;
|
||||
x86_64)
|
||||
nix_system="x86_64-linux"
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported architecture: $architecture" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
require() {
|
||||
if ! command -v "$1" >/dev/null 2>&1; then
|
||||
echo "Missing required command: $1" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
require aws
|
||||
require git
|
||||
require jq
|
||||
require nix
|
||||
require sha256sum
|
||||
require tofu
|
||||
|
||||
repo_root="$(git rev-parse --show-toplevel)"
|
||||
if ! git diff --quiet || ! git diff --cached --quiet; then
|
||||
echo "The AMI builder git tree must be clean before publishing." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
builder_git_rev="$(git rev-parse HEAD)"
|
||||
flake_lock_rev="$(jq -r '.nodes.nixpkgs.locked.rev // .nodes.nixpkgs.locked.narHash // "unknown"' "$repo_root/flake.lock")"
|
||||
input_hash="$(printf '%s\n%s\n%s\n' "$builder_git_rev" "$flake_lock_rev" "$architecture" | sha256sum | awk '{ print $1 }')"
|
||||
|
||||
root_image="$(nix build --print-out-paths ".#packages.${nix_system}.rootImage")"
|
||||
nixos_system_closure="$(nix build --print-out-paths ".#packages.${nix_system}.systemClosure")"
|
||||
|
||||
tmpdir="$(mktemp -d)"
|
||||
cleanup() {
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
bucket_no_scheme="${bucket#s3://}"
|
||||
bucket_name="${bucket_no_scheme%%/*}"
|
||||
bucket_prefix="${bucket_no_scheme#"$bucket_name"}"
|
||||
bucket_prefix="${bucket_prefix#/}"
|
||||
s3_key_prefix="${bucket_prefix:+$bucket_prefix/}${name}/${architecture}/${input_hash}"
|
||||
|
||||
image_file="$(find "$root_image" -type f \( -name '*.vhd' -o -name '*.raw' \) | head -n 1)"
|
||||
if [ -z "$image_file" ]; then
|
||||
echo "Could not locate built root image under $root_image" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
case "$image_file" in
|
||||
*.vhd)
|
||||
import_format="vhd"
|
||||
;;
|
||||
*.raw)
|
||||
import_format="raw"
|
||||
;;
|
||||
*)
|
||||
echo "Unsupported root image format: $image_file" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
aws s3 cp "$image_file" "s3://${bucket_name}/${s3_key_prefix}/root-image"
|
||||
|
||||
container_json="$tmpdir/container.json"
|
||||
jq -n \
|
||||
--arg bucket "$bucket_name" \
|
||||
--arg key "${s3_key_prefix}/root-image" \
|
||||
--arg format "$import_format" \
|
||||
'[{Description:"nixos-zfs-ec2-root",Format:$format,UserBucket:{S3Bucket:$bucket,S3Key:$key}}]' \
|
||||
> "$container_json"
|
||||
|
||||
import_task_id="$(
|
||||
aws ec2 import-image \
|
||||
--region "$aws_region" \
|
||||
--architecture "$architecture" \
|
||||
--platform Linux \
|
||||
--disk-containers "file://$container_json" \
|
||||
--query ImportTaskId \
|
||||
--output text
|
||||
)"
|
||||
|
||||
while true; do
|
||||
import_json="$(aws ec2 describe-import-image-tasks --region "$aws_region" --import-task-ids "$import_task_id")"
|
||||
status="$(jq -r '.ImportImageTasks[0].Status' <<<"$import_json")"
|
||||
case "$status" in
|
||||
completed)
|
||||
root_ami_id="$(jq -r '.ImportImageTasks[0].ImageId' <<<"$import_json")"
|
||||
break
|
||||
;;
|
||||
deleted|deleting)
|
||||
echo "$import_json" >&2
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
status_message="$(jq -r '.ImportImageTasks[0].StatusMessage // ""' <<<"$import_json")"
|
||||
echo "import-image: $status $status_message" >&2
|
||||
sleep 20
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
root_snapshot_id="$(
|
||||
aws ec2 describe-images \
|
||||
--region "$aws_region" \
|
||||
--image-ids "$root_ami_id" \
|
||||
--query 'Images[0].BlockDeviceMappings[?Ebs != `null`][0].Ebs.SnapshotId' \
|
||||
--output text
|
||||
)"
|
||||
|
||||
run_args=(
|
||||
--region "$aws_region"
|
||||
--image-id "$root_ami_id"
|
||||
--instance-type "$instance_type"
|
||||
--block-device-mappings "DeviceName=/dev/sdf,Ebs={VolumeSize=${nix_volume_size_gb},VolumeType=gp3,DeleteOnTermination=true}"
|
||||
--tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=${name}-seed-${input_hash:0:12}}]"
|
||||
--query 'Instances[0].InstanceId'
|
||||
--output text
|
||||
)
|
||||
|
||||
if [ -n "$subnet_id" ]; then
|
||||
run_args+=(--subnet-id "$subnet_id")
|
||||
fi
|
||||
if [ -n "$security_group_id" ]; then
|
||||
run_args+=(--security-group-ids "$security_group_id")
|
||||
fi
|
||||
if [ -n "$instance_profile_name" ]; then
|
||||
run_args+=(--iam-instance-profile "Name=$instance_profile_name")
|
||||
fi
|
||||
|
||||
instance_id="$(aws ec2 run-instances "${run_args[@]}")"
|
||||
aws ec2 wait instance-running --region "$aws_region" --instance-ids "$instance_id"
|
||||
aws ec2 wait instance-status-ok --region "$aws_region" --instance-ids "$instance_id"
|
||||
|
||||
cat > "$tmpdir/seed-nix.sh" <<'EOS'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
pool="nixos-zfs-nix"
|
||||
device=""
|
||||
for candidate in /dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_*; do
|
||||
if [ -e "$candidate" ] && ! findmnt -n -S "$candidate" >/dev/null 2>&1; then
|
||||
device="$candidate"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$device" ]; then
|
||||
echo "Unable to find unattached EBS device for /nix seed" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
zpool create \
|
||||
-f \
|
||||
-o ashift=12 \
|
||||
-O acltype=posixacl \
|
||||
-O atime=off \
|
||||
-O compression=zstd \
|
||||
-O mountpoint=none \
|
||||
-O xattr=sa \
|
||||
"$pool" "$device"
|
||||
|
||||
zfs create -o mountpoint=legacy "$pool/nix"
|
||||
mkdir -p /mnt/nixos-zfs-nix
|
||||
mount -t zfs "$pool/nix" /mnt/nixos-zfs-nix
|
||||
rsync -aHAX --numeric-ids \
|
||||
--exclude='/var/nix/daemon-socket/*' \
|
||||
/nix/ /mnt/nixos-zfs-nix/
|
||||
rm -f /mnt/nixos-zfs-nix/var/nix/daemon-socket/socket
|
||||
umount /mnt/nixos-zfs-nix
|
||||
zpool export "$pool"
|
||||
EOS
|
||||
|
||||
jq -n --rawfile commands "$tmpdir/seed-nix.sh" '{commands: [$commands]}' > "$tmpdir/ssm-parameters.json"
|
||||
|
||||
command_id="$(
|
||||
aws ssm send-command \
|
||||
--region "$aws_region" \
|
||||
--instance-ids "$instance_id" \
|
||||
--document-name AWS-RunShellScript \
|
||||
--parameters "file://$tmpdir/ssm-parameters.json" \
|
||||
--query Command.CommandId \
|
||||
--output text
|
||||
)"
|
||||
|
||||
while true; do
|
||||
invocation_json="$(aws ssm get-command-invocation --region "$aws_region" --command-id "$command_id" --instance-id "$instance_id" 2>/dev/null || true)"
|
||||
status="$(jq -r '.Status // "Pending"' <<<"$invocation_json")"
|
||||
case "$status" in
|
||||
Success)
|
||||
break
|
||||
;;
|
||||
Failed|Cancelled|TimedOut)
|
||||
echo "$invocation_json" >&2
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
echo "seed /nix: $status" >&2
|
||||
sleep 10
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
aws ec2 stop-instances --region "$aws_region" --instance-ids "$instance_id" >/dev/null
|
||||
aws ec2 wait instance-stopped --region "$aws_region" --instance-ids "$instance_id"
|
||||
|
||||
nix_volume_id="$(
|
||||
aws ec2 describe-instances \
|
||||
--region "$aws_region" \
|
||||
--instance-ids "$instance_id" \
|
||||
--query 'Reservations[0].Instances[0].BlockDeviceMappings[?DeviceName==`/dev/sdf`].Ebs.VolumeId | [0]' \
|
||||
--output text
|
||||
)"
|
||||
|
||||
nix_snapshot_id="$(
|
||||
aws ec2 create-snapshot \
|
||||
--region "$aws_region" \
|
||||
--volume-id "$nix_volume_id" \
|
||||
--description "${name} ${architecture} ZFS /nix ${input_hash}" \
|
||||
--tag-specifications "ResourceType=snapshot,Tags=[{Key=Name,Value=${name}-nix-${architecture}-${input_hash:0:12}},{Key=InputHash,Value=${input_hash}},{Key=BuilderGitRev,Value=${builder_git_rev}}]" \
|
||||
--query SnapshotId \
|
||||
--output text
|
||||
)"
|
||||
aws ec2 wait snapshot-completed --region "$aws_region" --snapshot-ids "$nix_snapshot_id"
|
||||
|
||||
tofu init
|
||||
tofu apply \
|
||||
-var "aws_region=$aws_region" \
|
||||
-var "name=$name" \
|
||||
-var "architecture=$architecture" \
|
||||
-var "root_snapshot_id=$root_snapshot_id" \
|
||||
-var "nix_snapshot_id=$nix_snapshot_id" \
|
||||
-var "input_hash=$input_hash" \
|
||||
-var "builder_git_rev=$builder_git_rev" \
|
||||
-var "flake_lock_rev=$flake_lock_rev" \
|
||||
-var "nixos_system_closure=$nixos_system_closure" \
|
||||
-var "nix_volume_size_gb=$nix_volume_size_gb"
|
||||
|
||||
ami_id="$(tofu output -raw ami_id)"
|
||||
|
||||
aws ec2 terminate-instances --region "$aws_region" --instance-ids "$instance_id" >/dev/null
|
||||
aws ec2 deregister-image --region "$aws_region" --image-id "$root_ami_id" >/dev/null || true
|
||||
|
||||
jq -n \
|
||||
--arg architecture "$architecture" \
|
||||
--arg inputHash "$input_hash" \
|
||||
--arg builderGitRev "$builder_git_rev" \
|
||||
--arg flakeLockRev "$flake_lock_rev" \
|
||||
--arg amiId "$ami_id" \
|
||||
--arg rootSnapshotId "$root_snapshot_id" \
|
||||
--arg nixSnapshotId "$nix_snapshot_id" \
|
||||
--arg nixosSystemClosure "$nixos_system_closure" \
|
||||
'{
|
||||
architecture: $architecture,
|
||||
inputHash: $inputHash,
|
||||
builderGitRev: $builderGitRev,
|
||||
flakeLockRev: $flakeLockRev,
|
||||
amiId: $amiId,
|
||||
rootSnapshotId: $rootSnapshotId,
|
||||
nixSnapshotId: $nixSnapshotId,
|
||||
nixosSystemClosure: $nixosSystemClosure
|
||||
}' > artifact.json
|
||||
|
||||
echo "Published AMI: $ami_id"
|
||||
Reference in New Issue
Block a user