Collapse runtime persistence to deployment-local persist
This commit is contained in:
Timothy J. Aveni
@@ -5,8 +5,7 @@ let
|
||||
cfg = config.services.quixosHost;
|
||||
secretsCfg = cfg.secrets;
|
||||
quixosPackages = self.packages.${pkgs.stdenv.hostPlatform.system};
|
||||
localPersistMount = "${utils.escapeSystemdPath "/persist-local"}.mount";
|
||||
sharedPersistMount = "${utils.escapeSystemdPath "/persist"}.mount";
|
||||
persistMount = "${utils.escapeSystemdPath "/persist"}.mount";
|
||||
devHomeMount = "${utils.escapeSystemdPath cfg.devHome.mountPoint}.mount";
|
||||
sourceCheckoutService = "quixos-source-checkout.service";
|
||||
devPrepService = "quixos-dev-prep.service";
|
||||
@@ -166,15 +165,6 @@ let
|
||||
quixosCodexRule = ''prefix_rule(pattern=[\x22qx-control\x22],decision=\x22allow\x22)'';
|
||||
in
|
||||
{
|
||||
imports = [
|
||||
(lib.mkRenamedOptionModule
|
||||
[ "services" "quixosHost" "caddyPersistZfsPool" ]
|
||||
[ "services" "quixosHost" "sharedPersistZfsPool" ])
|
||||
(lib.mkRenamedOptionModule
|
||||
[ "services" "quixosHost" "caddyPersistDevice" ]
|
||||
[ "services" "quixosHost" "sharedPersistDevice" ])
|
||||
];
|
||||
|
||||
options.services.quixosHost = {
|
||||
enable = lib.mkEnableOption "Quixos host";
|
||||
|
||||
@@ -185,25 +175,25 @@ in
|
||||
|
||||
packagesPath = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "/persist-local/srv/quixos/packages";
|
||||
default = "/persist/srv/quixos/packages";
|
||||
description = "Mutable package-tree checkout path.";
|
||||
};
|
||||
|
||||
dataRoot = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "/persist-local/var/lib/quixos";
|
||||
default = "/persist/var/lib/quixos";
|
||||
description = "Persistent Quixos runtime state root.";
|
||||
};
|
||||
|
||||
secretsEnvironmentFile = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "/persist-local/secrets/quixos.env";
|
||||
default = "/persist/secrets/quixos.env";
|
||||
description = "Environment file containing Quixos secrets.";
|
||||
};
|
||||
|
||||
packageTreeGitEnvironmentFile = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "/persist-local/secrets/quixos-git.env";
|
||||
default = "/persist/secrets/quixos-git.env";
|
||||
visible = false;
|
||||
description = "Optional environment file containing Git credentials for the package tree.";
|
||||
};
|
||||
@@ -229,25 +219,13 @@ in
|
||||
persistZfsPool = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "quixos-persist";
|
||||
description = "ZFS pool name on the deployment-local persistent block device mounted at /persist-local.";
|
||||
description = "ZFS pool name on the deployment persistent block device mounted at /persist.";
|
||||
};
|
||||
|
||||
persistDevice = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Stable device path for the deployment-local persistent block device. If set, Quixos initializes a blank device as persistZfsPool before mounting /persist-local.";
|
||||
};
|
||||
|
||||
sharedPersistZfsPool = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "quixos-shared";
|
||||
description = "ZFS pool name on the long-lived shared persistent block device mounted at /persist.";
|
||||
};
|
||||
|
||||
sharedPersistDevice = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Stable device path for the long-lived shared persistent block device. If set, Quixos initializes a blank device as sharedPersistZfsPool before mounting /persist.";
|
||||
description = "Stable device path for the deployment persistent block device. If set, Quixos initializes a blank device as persistZfsPool before mounting /persist.";
|
||||
};
|
||||
|
||||
nixZfsPool = lib.mkOption {
|
||||
@@ -283,7 +261,7 @@ in
|
||||
|
||||
mountPoint = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "/persist-local/home/quixos-dev";
|
||||
default = "/persist/home/quixos-dev";
|
||||
description = "Mount point for the development home dataset.";
|
||||
};
|
||||
|
||||
@@ -436,25 +414,25 @@ in
|
||||
files = {
|
||||
controlSecretPath = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "/persist-local/secrets/control-secret";
|
||||
default = "/persist/secrets/control-secret";
|
||||
description = "File backend path for the control secret.";
|
||||
};
|
||||
|
||||
stateContextSecretPath = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "/persist-local/secrets/state-context-secret";
|
||||
default = "/persist/secrets/state-context-secret";
|
||||
description = "File backend path for the state context secret.";
|
||||
};
|
||||
|
||||
packageTreeGitUsernamePath = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "/persist-local/secrets/package-tree-git-username";
|
||||
default = "/persist/secrets/package-tree-git-username";
|
||||
description = "File backend path for the package tree Git username.";
|
||||
};
|
||||
|
||||
packageTreeGitPasswordPath = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "/persist-local/secrets/package-tree-git-password";
|
||||
default = "/persist/secrets/package-tree-git-password";
|
||||
description = "File backend path for the package tree Git password.";
|
||||
};
|
||||
};
|
||||
@@ -543,7 +521,7 @@ in
|
||||
boot.zfs.forceImportRoot = false;
|
||||
services.zfs.autoScrub = {
|
||||
enable = true;
|
||||
pools = [ cfg.nixZfsPool cfg.persistZfsPool cfg.sharedPersistZfsPool ];
|
||||
pools = [ cfg.nixZfsPool cfg.persistZfsPool ];
|
||||
};
|
||||
|
||||
fileSystems = {
|
||||
@@ -553,7 +531,7 @@ in
|
||||
neededForBoot = true;
|
||||
};
|
||||
|
||||
"/persist-local" = {
|
||||
"/persist" = {
|
||||
device = "${cfg.persistZfsPool}/persist";
|
||||
fsType = "zfs";
|
||||
options =
|
||||
@@ -563,17 +541,6 @@ in
|
||||
"x-systemd.after=quixos-persist-init.service"
|
||||
];
|
||||
};
|
||||
|
||||
"/persist" = {
|
||||
device = "${cfg.sharedPersistZfsPool}/persist";
|
||||
fsType = "zfs";
|
||||
options =
|
||||
[ "defaults" ]
|
||||
++ lib.optionals (cfg.sharedPersistDevice != null) [
|
||||
"x-systemd.requires=quixos-shared-persist-init.service"
|
||||
"x-systemd.after=quixos-shared-persist-init.service"
|
||||
];
|
||||
};
|
||||
} // lib.optionalAttrs cfg.devHome.enable {
|
||||
${cfg.devHome.mountPoint} = {
|
||||
device = cfg.devHome.dataset;
|
||||
@@ -589,7 +556,7 @@ in
|
||||
systemd.services.quixos-persist-init = lib.mkIf (cfg.persistDevice != null) {
|
||||
description = "Initialize blank Quixos persistent ZFS volume";
|
||||
requiredBy = [ "zfs-import-${cfg.persistZfsPool}.service" ];
|
||||
before = [ "zfs-import-${cfg.persistZfsPool}.service" localPersistMount "local-fs.target" ];
|
||||
before = [ "zfs-import-${cfg.persistZfsPool}.service" persistMount "local-fs.target" ];
|
||||
after = [ "nix.mount" ];
|
||||
requires = [ "nix.mount" ];
|
||||
path = [ pkgs.coreutils pkgs.gnugrep pkgs.util-linux pkgs.zfs ];
|
||||
@@ -654,89 +621,12 @@ in
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.services.quixos-shared-persist-init = lib.mkIf (cfg.sharedPersistDevice != null) {
|
||||
description = "Initialize blank Quixos shared ZFS volume";
|
||||
requiredBy = [ "zfs-import-${cfg.sharedPersistZfsPool}.service" ];
|
||||
before = [ "zfs-import-${cfg.sharedPersistZfsPool}.service" sharedPersistMount "local-fs.target" ];
|
||||
after = [ "nix.mount" ];
|
||||
requires = [ "nix.mount" ];
|
||||
path = [ pkgs.coreutils pkgs.gnugrep pkgs.util-linux pkgs.zfs ];
|
||||
unitConfig = {
|
||||
DefaultDependencies = false;
|
||||
};
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
};
|
||||
script = ''
|
||||
set -eu
|
||||
|
||||
device=${lib.escapeShellArg cfg.sharedPersistDevice}
|
||||
pool=${lib.escapeShellArg cfg.sharedPersistZfsPool}
|
||||
legacy_pool=quixos-caddy
|
||||
dataset="$pool/persist"
|
||||
|
||||
for _ in $(seq 1 600); do
|
||||
if [ -e "$device" ]; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
if [ ! -e "$device" ]; then
|
||||
echo "Shared persist device did not appear: $device" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if zpool list -H "$pool" >/dev/null 2>&1; then
|
||||
if ! zfs list -H "$dataset" >/dev/null 2>&1; then
|
||||
echo "ZFS pool '$pool' is imported but dataset '$dataset' is missing." >&2
|
||||
exit 1
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if zpool import -d /dev/disk/by-id | grep -Eq "^[[:space:]]*pool:[[:space:]]+$pool$"; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$pool" != "$legacy_pool" ] && zpool import -d /dev/disk/by-id | grep -Eq "^[[:space:]]*pool:[[:space:]]+$legacy_pool$"; then
|
||||
zpool import -d /dev/disk/by-id "$legacy_pool" "$pool"
|
||||
if ! zfs list -H "$dataset" >/dev/null 2>&1; then
|
||||
echo "Imported legacy pool '$legacy_pool' as '$pool', but dataset '$dataset' is missing." >&2
|
||||
exit 1
|
||||
fi
|
||||
zpool export "$pool"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if wipefs --noheadings --output TYPE "$device" | grep -q .; then
|
||||
echo "Shared persist device has existing signatures but pool '$pool' could not be imported: $device" >&2
|
||||
wipefs "$device" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
zpool create \
|
||||
-f \
|
||||
-o ashift=12 \
|
||||
-O acltype=posixacl \
|
||||
-O atime=off \
|
||||
-O compression=zstd \
|
||||
-O mountpoint=none \
|
||||
-O xattr=sa \
|
||||
"$pool" "$device"
|
||||
|
||||
zfs create -o mountpoint=legacy "$dataset"
|
||||
zpool export "$pool"
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.services.quixos-dev-home-init = lib.mkIf cfg.devHome.enable {
|
||||
description = "Initialize Quixos development home ZFS dataset";
|
||||
requiredBy = [ devHomeMount ];
|
||||
before = [ devHomeMount "local-fs.target" ];
|
||||
after = [ localPersistMount ];
|
||||
requires = [ localPersistMount ];
|
||||
after = [ persistMount ];
|
||||
requires = [ persistMount ];
|
||||
path = [ pkgs.coreutils pkgs.zfs ];
|
||||
unitConfig = {
|
||||
DefaultDependencies = false;
|
||||
@@ -826,7 +716,7 @@ in
|
||||
environment.variables.QUIXOS_CONTROL_SOCKET = "/run/quixos/control.sock";
|
||||
environment.variables.QUIXOS_PACKAGES_PATH = cfg.packagesPath;
|
||||
environment.variables.QUIXOS_PACKAGE_TEMPLATES_DIR = "${self}/quixos-instance/package-templates";
|
||||
environment.variables.QUIXOS_WORKSPACE_ROOT = "/persist-local/srv/quixos";
|
||||
environment.variables.QUIXOS_WORKSPACE_ROOT = "/persist/srv/quixos";
|
||||
|
||||
systemd.sockets.nix-daemon = {
|
||||
after = [ "nix.mount" ];
|
||||
@@ -857,18 +747,15 @@ in
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
"z /persist 0755 root root -"
|
||||
"z /persist-local 0755 root root -"
|
||||
"d /persist/etc/ssh 0700 root root -"
|
||||
"d /persist-local/home 0755 root root -"
|
||||
"d /persist/home 0755 root root -"
|
||||
] ++ lib.optionals cfg.devHome.enable [
|
||||
"d ${cfg.devHome.mountPoint} 0710 ${cfg.devHome.user} quixos -"
|
||||
"z ${cfg.devHome.mountPoint} 0710 ${cfg.devHome.user} quixos -"
|
||||
] ++ [
|
||||
"d /persist-local/srv 0755 root root -"
|
||||
"d /persist-local/srv/quixos 0755 quixos quixos -"
|
||||
"d /persist-local/srv/quixos/packages 0755 quixos quixos -"
|
||||
"d /persist-local/var 0755 root root -"
|
||||
"d /persist-local/var/lib 0755 root root -"
|
||||
"d /persist/srv 0755 root root -"
|
||||
"d /persist/srv/quixos 0755 quixos quixos -"
|
||||
"d /persist/srv/quixos/packages 0755 quixos quixos -"
|
||||
"d /persist/var 0755 root root -"
|
||||
"d /persist/var/lib 0755 root root -"
|
||||
"d /persist/var/lib/quixos 0750 quixos quixos -"
|
||||
@@ -910,8 +797,8 @@ in
|
||||
"sshd-keygen.service"
|
||||
"sshd.service"
|
||||
];
|
||||
after = [ localPersistMount sharedPersistMount ] ++ lib.optionals cfg.devHome.enable [ devHomeMount ];
|
||||
requires = [ localPersistMount sharedPersistMount ] ++ lib.optionals cfg.devHome.enable [ devHomeMount ];
|
||||
after = [ persistMount ] ++ lib.optionals cfg.devHome.enable [ devHomeMount ];
|
||||
requires = [ persistMount ] ++ lib.optionals cfg.devHome.enable [ devHomeMount ];
|
||||
path = [ config.systemd.package ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
@@ -919,7 +806,7 @@ in
|
||||
};
|
||||
script = ''
|
||||
set -eu
|
||||
systemd-tmpfiles --create --prefix=/persist-local --prefix=/persist
|
||||
systemd-tmpfiles --create --prefix=/persist
|
||||
'';
|
||||
};
|
||||
|
||||
@@ -1141,7 +1028,7 @@ in
|
||||
WorkingDirectory =
|
||||
if cfg.orchestrator.source.enable
|
||||
then "${cfg.orchestrator.source.path}/orchestrator"
|
||||
else "/persist-local/srv/quixos";
|
||||
else "/persist/srv/quixos";
|
||||
Restart = "always";
|
||||
RestartSec = "5s";
|
||||
};
|
||||
@@ -1170,7 +1057,7 @@ in
|
||||
GIT_COMMITTER_NAME = "Quixos Runtime";
|
||||
GIT_COMMITTER_EMAIL = "quixos-runtime@localhost";
|
||||
XDG_DATA_HOME = cfg.dataRoot;
|
||||
QUIXOS_WORKSPACE_ROOT = "/persist-local/srv/quixos";
|
||||
QUIXOS_WORKSPACE_ROOT = "/persist/srv/quixos";
|
||||
QUIXOS_APP_ROOT =
|
||||
if cfg.orchestrator.source.enable
|
||||
then cfg.orchestrator.source.path
|
||||
@@ -1246,7 +1133,7 @@ in
|
||||
QUIXOS_APP_ROOT = "${self}";
|
||||
QUIXOS_PACKAGES_PATH = cfg.packagesPath;
|
||||
QUIXOS_PACKAGE_TEMPLATES_DIR = "${self}/quixos-instance/package-templates";
|
||||
QUIXOS_WORKSPACE_ROOT = "/persist-local/srv/quixos";
|
||||
QUIXOS_WORKSPACE_ROOT = "/persist/srv/quixos";
|
||||
QUIXOS_CONTROL_SOCKET = "/run/quixos/control.sock";
|
||||
};
|
||||
script = ''
|
||||
|
||||
+14
-15
@@ -8,8 +8,8 @@ This directory is split into two independent projects:
|
||||
with a matched root snapshot and preseeded ZFS `/nix` snapshot.
|
||||
- `central/`: singleton central infrastructure used by runtime deployments.
|
||||
Today this owns the S3 bucket for runtime deployment state/registry, the
|
||||
long-lived encrypted EBS volume for shared state such as Caddy ACME material
|
||||
and SSH host keys, the platform VPC/subnets/security groups, and the optional
|
||||
long-lived encrypted EBS volume for central `/persist` state such as Caddy
|
||||
ACME material and SSH host keys, the platform VPC/subnets/security groups, and the optional
|
||||
central app-server singleton. This is also the intended home for future
|
||||
central services such as auth, public site, VPCs, or other shared platform
|
||||
infrastructure.
|
||||
@@ -21,7 +21,7 @@ This directory is split into two independent projects:
|
||||
Destroying the runtime stack destroys the EC2/app/runtime resources only. It
|
||||
must not deregister the AMI or delete the AMI root or `/nix` snapshots. Those
|
||||
belong to the image-builder state. It also must not delete central
|
||||
infrastructure such as the runtime state bucket or shared persistent volume;
|
||||
infrastructure such as the runtime state bucket or central persist volume;
|
||||
those belong to `central`.
|
||||
|
||||
## Central Infrastructure
|
||||
@@ -61,7 +61,7 @@ overrides:
|
||||
|
||||
- `runtime_state_bucket_name`: S3 bucket for runtime deployment state and
|
||||
registry.
|
||||
- `shared_state_volume_id`: EBS volume mounted by runtime deployments at
|
||||
- `persist_volume_id`: EBS volume mounted by the central app server at
|
||||
`/persist`.
|
||||
- `platform_vpc_id` and `platform_runtime_subnet_id`: the managed VPC/subnet
|
||||
intended for runtime deployments.
|
||||
@@ -97,7 +97,7 @@ registry/deployments/<deployment-id>.json
|
||||
```
|
||||
|
||||
Use the same `aws_region`, `vpc_id`, and `subnet_id` values as the runtime
|
||||
stack if you override them. The central shared state volume must be in the same
|
||||
stack if you override them. The central persist volume must be in the same
|
||||
availability zone as the runtime EC2 instance.
|
||||
|
||||
## Runtime Deploy
|
||||
@@ -118,7 +118,7 @@ operator config: AWS region, Route53 zone, VPC/subnet overrides, package git
|
||||
credentials, instance sizing, and similar static environment inputs.
|
||||
|
||||
`qx-deploy` owns deployment choices: deployment id, runtime state key,
|
||||
package-tree ref, Quixos ref, AMI, shared state volume, generated domain, dev
|
||||
package-tree ref, Quixos ref, AMI, generated domain, dev
|
||||
mode, and SSH settings. For each run it resolves those values, rejects local
|
||||
tfvars that try to set deployment-owned keys such as `subdomain` or `dev_mode`,
|
||||
writes a temporary `zz-qx-deploy.auto.tfvars.json`, plans to a saved plan, and
|
||||
@@ -177,7 +177,7 @@ contains `quixos.rev`, that rev is deployed. Use `--quixos-ref master` to
|
||||
override it.
|
||||
|
||||
If the package tree or its submodules are private, provide HTTPS credentials
|
||||
through the wrapper. The token is written to `/persist-local/secrets/quixos-git.env`
|
||||
through the wrapper. The token is written to `/persist/secrets/quixos-git.env`
|
||||
and consumed by the package-tree systemd unit through `GIT_ASKPASS`.
|
||||
|
||||
```bash
|
||||
@@ -193,18 +193,17 @@ The runtime AMI is expected to provide `/nix` from the AMI-attached
|
||||
no filesystems; it is a NixOS flake consumed by the AMI's user-data-flake
|
||||
service.
|
||||
|
||||
The runtime `/persist-local` volume contains deployment-local Quixos workspace,
|
||||
secret, and package state. It uses a ZFS pool named
|
||||
The runtime `/persist` volume contains deployment-local Quixos workspace,
|
||||
secret, package state, service state, and dev home data. It uses a ZFS pool named
|
||||
`quixos-persist` with a `quixos-persist/persist` dataset. Runtime NixOS
|
||||
initializes that pool on the attached local persist EBS volume if the volume is
|
||||
initializes that pool on the attached runtime persist EBS volume if the volume is
|
||||
blank, imports it if it already exists, and fails on unknown existing filesystem
|
||||
signatures.
|
||||
|
||||
The `/persist` mount is reserved for long-lived shared state. It uses the
|
||||
central EBS volume and a ZFS pool named `quixos-shared` with a
|
||||
`quixos-shared/persist` dataset. Caddy's data directory is
|
||||
`/persist/var/lib/caddy`, and SSH host keys live under `/persist/etc/ssh`, so
|
||||
both survive runtime `tofu destroy`.
|
||||
Runtime `/persist` survives runtime reboots, but it is scoped to one runtime
|
||||
deployment and is destroyed with that deployment. It is never attached to
|
||||
central or to another runtime deployment. Central has its own `/persist` volume
|
||||
with central lifecycle.
|
||||
|
||||
The visualizer static derivation uses `project-visualizer/yarn-project.nix`,
|
||||
generated by `yarn-plugin-nixify`.
|
||||
|
||||
+2
-2
@@ -12,6 +12,6 @@
|
||||
- Define separate capabilities for service runtime, package-tree mutation, control socket access, local admin/debug access, and secret-file access.
|
||||
- Decide whether `quixos-control` should remain a group, and if so whether it is socket-only rather than a broad secret-reading group.
|
||||
- Decide which principals may read the control secret, state-context secret, and package-tree Git credentials.
|
||||
- Decide whether humans should directly write the canonical `/persist-local/srv/quixos/packages` workspace or work through control-plane operations.
|
||||
- Review file modes and ownership for `/run/quixos`, `/persist-local/srv/quixos`, `/persist-local/secrets`, and shared persistent state.
|
||||
- Decide whether humans should directly write the canonical `/persist/srv/quixos/packages` workspace or work through control-plane operations.
|
||||
- Review file modes and ownership for `/run/quixos`, `/persist/srv/quixos`, `/persist/secrets`, and persistent state.
|
||||
- Decide how and when to automatically commit the outer `packages` package-tree repo after submodule updates. This may belong in `qx-control cutover` or in another explicit package-tree publication command, but ordinary package staging should not imply a package-tree commit.
|
||||
|
||||
@@ -38,7 +38,7 @@ locals {
|
||||
app_server_ami_id = var.app_server_ami_id == null ? local.central_ami_id : var.app_server_ami_id
|
||||
app_server_enabled = local.app_server_ami_id != null
|
||||
app_server_flake_uri = "git+${var.app_server_repo_url}?dir=services/app-server&${var.app_server_ref_is_rev ? "rev" : "ref"}=${urlencode(var.app_server_ref)}"
|
||||
app_server_persist_device = "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_${replace(aws_ebs_volume.shared_state.id, "-", "")}"
|
||||
app_server_persist_device = "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_${replace(aws_ebs_volume.persist.id, "-", "")}"
|
||||
platform_vpc_id = var.platform_vpc_enable ? aws_vpc.platform[0].id : local.vpc_id
|
||||
platform_public_subnet_id = var.platform_vpc_enable ? aws_subnet.central_public[0].id : local.subnet_id
|
||||
platform_runtime_subnet_id = var.platform_vpc_enable ? aws_subnet.runtime_public[0].id : local.subnet_id
|
||||
@@ -173,7 +173,7 @@ resource "aws_s3_bucket_versioning" "runtime_state" {
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_ebs_volume" "shared_state" {
|
||||
resource "aws_ebs_volume" "persist" {
|
||||
availability_zone = data.aws_subnet.selected.availability_zone
|
||||
size = var.volume_size_gb
|
||||
snapshot_id = var.snapshot_id
|
||||
@@ -182,7 +182,7 @@ resource "aws_ebs_volume" "shared_state" {
|
||||
|
||||
tags = merge(local.tags, {
|
||||
Name = "${local.resource_name}-volume"
|
||||
Role = "shared-persistent-state"
|
||||
Role = "persistent-state"
|
||||
})
|
||||
}
|
||||
|
||||
@@ -367,7 +367,7 @@ resource "aws_instance" "app_server" {
|
||||
resource "aws_volume_attachment" "app_server_persist" {
|
||||
count = local.app_server_enabled ? 1 : 0
|
||||
device_name = "/dev/sdh"
|
||||
volume_id = aws_ebs_volume.shared_state.id
|
||||
volume_id = aws_ebs_volume.persist.id
|
||||
instance_id = aws_instance.app_server[0].id
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,6 @@ moved {
|
||||
}
|
||||
|
||||
moved {
|
||||
from = aws_ebs_volume.caddy_state
|
||||
to = aws_ebs_volume.shared_state
|
||||
from = aws_ebs_volume.shared_state
|
||||
to = aws_ebs_volume.persist
|
||||
}
|
||||
|
||||
@@ -10,17 +10,12 @@ output "runtime_state_prefix" {
|
||||
value = "runtime/deployments/"
|
||||
}
|
||||
|
||||
output "shared_state_volume_id" {
|
||||
value = aws_ebs_volume.shared_state.id
|
||||
}
|
||||
|
||||
output "volume_id" {
|
||||
value = aws_ebs_volume.shared_state.id
|
||||
description = "Deprecated alias for shared_state_volume_id."
|
||||
output "persist_volume_id" {
|
||||
value = aws_ebs_volume.persist.id
|
||||
}
|
||||
|
||||
output "availability_zone" {
|
||||
value = aws_ebs_volume.shared_state.availability_zone
|
||||
value = aws_ebs_volume.persist.availability_zone
|
||||
}
|
||||
|
||||
output "resource_name" {
|
||||
|
||||
@@ -31,7 +31,7 @@ variable "vpc_id" {
|
||||
variable "subnet_id" {
|
||||
type = string
|
||||
default = null
|
||||
description = "Subnet ID used to choose the shared state volume availability zone. Use the same subnet as runtime."
|
||||
description = "Subnet ID used to choose the central persist volume availability zone. Use the same subnet as the central app server."
|
||||
}
|
||||
|
||||
variable "platform_vpc_enable" {
|
||||
@@ -61,19 +61,19 @@ variable "platform_runtime_subnet_cidr" {
|
||||
variable "volume_size_gb" {
|
||||
type = number
|
||||
default = 2
|
||||
description = "Central shared persistent state EBS volume size."
|
||||
description = "Central persistent state EBS volume size."
|
||||
}
|
||||
|
||||
variable "volume_type" {
|
||||
type = string
|
||||
default = "gp3"
|
||||
description = "Central shared persistent state EBS volume type."
|
||||
description = "Central persistent state EBS volume type."
|
||||
}
|
||||
|
||||
variable "snapshot_id" {
|
||||
type = string
|
||||
default = null
|
||||
description = "Optional snapshot for restoring an existing central shared state volume."
|
||||
description = "Optional snapshot for restoring an existing central persist volume."
|
||||
}
|
||||
|
||||
variable "route53_zone_id" {
|
||||
|
||||
@@ -25,8 +25,6 @@ const deployExclusiveVars = new Set([
|
||||
"ami_id",
|
||||
"ami_system",
|
||||
"ami_project_name",
|
||||
"shared_persist_volume_id",
|
||||
"caddy_persist_volume_id",
|
||||
"central_proxy_security_group_id",
|
||||
"assign_ipv6_address",
|
||||
"proxy_registration_grant",
|
||||
@@ -64,8 +62,6 @@ Options:
|
||||
matching the current builder rev
|
||||
--ami-system <system> AMI target system for lookup (default: aarch64-linux)
|
||||
--ami-project-name <name> AMI builder project tag for lookup (default: nixos-zfs-ec2)
|
||||
--shared-persist-volume-id <vol>
|
||||
Long-lived shared-state EBS volume from tofu/central
|
||||
--central-proxy-security-group-id <sg>
|
||||
Limit runtime HTTP/HTTPS ingress to this central proxy SG
|
||||
--base-domain <name> Base DNS domain for generated deployment names
|
||||
@@ -459,7 +455,6 @@ const parseArgs = (argv) => {
|
||||
autoSelectAmi: false,
|
||||
amiSystem: process.env.QUIXOS_AMI_SYSTEM ?? "aarch64-linux",
|
||||
amiProjectName: process.env.QUIXOS_AMI_PROJECT_NAME ?? "nixos-zfs-ec2",
|
||||
sharedPersistVolumeId: process.env.QUIXOS_SHARED_PERSIST_VOLUME_ID ?? process.env.QUIXOS_CADDY_PERSIST_VOLUME_ID ?? "",
|
||||
centralProxySecurityGroupId: process.env.QUIXOS_CENTRAL_PROXY_SECURITY_GROUP_ID ?? "",
|
||||
proxyRegistrationUrl: process.env.QUIXOS_PROXY_REGISTRATION_URL ?? "",
|
||||
proxyRegistrationGrant: process.env.QUIXOS_PROXY_REGISTRATION_GRANT ?? "",
|
||||
@@ -524,11 +519,6 @@ const parseArgs = (argv) => {
|
||||
state.amiProjectName = takeValue(argv, i, arg);
|
||||
i += 1;
|
||||
break;
|
||||
case "--shared-persist-volume-id":
|
||||
case "--caddy-persist-volume-id":
|
||||
state.sharedPersistVolumeId = takeValue(argv, i, arg);
|
||||
i += 1;
|
||||
break;
|
||||
case "--central-proxy-security-group-id":
|
||||
state.centralProxySecurityGroupId = takeValue(argv, i, arg);
|
||||
i += 1;
|
||||
@@ -867,22 +857,6 @@ const resolveAmiId = async (state, prefetch) => {
|
||||
console.error(`Selected AMI: ${style.cyan(state.amiId)}`);
|
||||
};
|
||||
|
||||
const resolveSharedPersistVolume = (state, tofuEnv) => {
|
||||
if (state.sharedPersistVolumeId) {
|
||||
return;
|
||||
}
|
||||
|
||||
const volumeId = centralOutput("shared_state_volume_id", { env: tofuEnv, required: true });
|
||||
if (volumeId) {
|
||||
state.sharedPersistVolumeId = volumeId;
|
||||
console.error(`Using shared state volume from tofu/central: ${state.sharedPersistVolumeId}`);
|
||||
}
|
||||
|
||||
if (!state.sharedPersistVolumeId) {
|
||||
throw new Error("Missing long-lived shared state volume.\nCreate central infrastructure with: (cd tofu/central && tofu init && tofu apply)\nThen rerun deploy, or pass --shared-persist-volume-id vol-...");
|
||||
}
|
||||
};
|
||||
|
||||
const resolveRuntimeStateBucket = (state, tofuEnv) => {
|
||||
if (state.runtimeStateBucket) {
|
||||
return;
|
||||
@@ -1143,8 +1117,6 @@ const equivalentArgs = (state, resolvedPackagesRef) => {
|
||||
state.amiProjectName,
|
||||
"--aws-region",
|
||||
state.awsRegion,
|
||||
"--shared-persist-volume-id",
|
||||
state.sharedPersistVolumeId,
|
||||
"--packages-path",
|
||||
state.packagesPath,
|
||||
"--packages-ref",
|
||||
@@ -1297,7 +1269,6 @@ const writeGeneratedDeployVars = (state, resolvedPackagesRef) => {
|
||||
ami_id: state.amiId,
|
||||
ami_system: state.amiSystem,
|
||||
ami_project_name: state.amiProjectName,
|
||||
shared_persist_volume_id: state.sharedPersistVolumeId,
|
||||
central_proxy_security_group_id: state.centralProxySecurityGroupId || null,
|
||||
assign_ipv6_address: state.assignIpv6Address,
|
||||
proxy_registration_grant: state.proxyRegistrationGrant || null,
|
||||
@@ -1453,7 +1424,6 @@ const registryRecord = (state, resolvedPackagesRef, tofuEnv, status) => ({
|
||||
amiId: state.amiId,
|
||||
amiSystem: state.amiSystem,
|
||||
amiProjectName: state.amiProjectName,
|
||||
sharedPersistVolumeId: state.sharedPersistVolumeId,
|
||||
centralProxySecurityGroupId: state.centralProxySecurityGroupId,
|
||||
proxyRegistrationUrl: state.proxyRegistrationUrl,
|
||||
assignIpv6Address: state.assignIpv6Address,
|
||||
@@ -1522,7 +1492,6 @@ const tofuDestroyVarArgs = (state, record) => {
|
||||
addString("ami_project_name", config.amiProjectName);
|
||||
addString("vpc_id", config.vpcId);
|
||||
addString("subnet_id", config.subnetId);
|
||||
addString("shared_persist_volume_id", config.sharedPersistVolumeId);
|
||||
addString("central_proxy_security_group_id", config.centralProxySecurityGroupId);
|
||||
if (config.baseDomainSpecified) {
|
||||
addString("base_domain", config.baseDomain);
|
||||
|
||||
+5
-15
@@ -31,10 +31,8 @@ locals {
|
||||
ssh_keys = compact([var.ssh_public_key])
|
||||
ssm_parameter_prefix = "/${var.name}/${local.deployment_id}"
|
||||
effective_ami_id = var.ami_id == null ? "ami-00000000000000000" : var.ami_id
|
||||
shared_volume_id = var.shared_persist_volume_id == null ? var.caddy_persist_volume_id : var.shared_persist_volume_id
|
||||
central_proxy_enabled = var.central_proxy_security_group_id != null && var.proxy_registration_grant != null && var.proxy_registration_url != null
|
||||
persist_device = "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_${replace(aws_ebs_volume.persist_local.id, "-", "")}"
|
||||
shared_persist_device = local.shared_volume_id == null ? null : "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_${replace(local.shared_volume_id, "-", "")}"
|
||||
persist_device = "/dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_${replace(aws_ebs_volume.persist.id, "-", "")}"
|
||||
tags = {
|
||||
Name = local.resource_name
|
||||
Deployment = local.deployment_id
|
||||
@@ -234,7 +232,6 @@ resource "aws_instance" "host" {
|
||||
quixos_ref = var.quixos_ref
|
||||
quixos_repo_url = var.quixos_repo_url
|
||||
quixos_flake_uri = local.quixos_flake_uri
|
||||
shared_persist_device = local.shared_volume_id == null ? "null" : "\"${local.shared_persist_device}\""
|
||||
ssh_authorized_keys = jsonencode(local.ssh_keys)
|
||||
state_context_secret_parameter = aws_ssm_parameter.state_context_secret.name
|
||||
})
|
||||
@@ -247,27 +244,20 @@ resource "aws_instance" "host" {
|
||||
tags = local.tags
|
||||
}
|
||||
|
||||
resource "aws_ebs_volume" "persist_local" {
|
||||
resource "aws_ebs_volume" "persist" {
|
||||
availability_zone = data.aws_subnet.selected.availability_zone
|
||||
size = var.persist_volume_size_gb
|
||||
snapshot_id = var.persist_snapshot_id
|
||||
type = "gp3"
|
||||
|
||||
tags = merge(local.tags, {
|
||||
Name = "${local.resource_name}-persist-local"
|
||||
Name = "${local.resource_name}-persist"
|
||||
})
|
||||
}
|
||||
|
||||
resource "aws_volume_attachment" "persist_local" {
|
||||
resource "aws_volume_attachment" "persist" {
|
||||
device_name = "/dev/sdg"
|
||||
volume_id = aws_ebs_volume.persist_local.id
|
||||
instance_id = aws_instance.host.id
|
||||
}
|
||||
|
||||
resource "aws_volume_attachment" "shared_persist" {
|
||||
count = local.shared_volume_id == null ? 0 : 1
|
||||
device_name = "/dev/sdh"
|
||||
volume_id = local.shared_volume_id
|
||||
volume_id = aws_ebs_volume.persist.id
|
||||
instance_id = aws_instance.host.id
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
moved {
|
||||
from = aws_ebs_volume.persist_local
|
||||
to = aws_ebs_volume.persist
|
||||
}
|
||||
|
||||
moved {
|
||||
from = aws_volume_attachment.persist_local
|
||||
to = aws_volume_attachment.persist
|
||||
}
|
||||
@@ -10,8 +10,8 @@ output "ami_id" {
|
||||
value = local.effective_ami_id
|
||||
}
|
||||
|
||||
output "shared_persist_volume_id" {
|
||||
value = local.shared_volume_id
|
||||
output "persist_volume_id" {
|
||||
value = aws_ebs_volume.persist.id
|
||||
}
|
||||
|
||||
output "resource_name" {
|
||||
|
||||
@@ -27,8 +27,6 @@
|
||||
domain = "${domain}";
|
||||
persistDevice = "${persist_device}";
|
||||
persistZfsPool = "quixos-persist";
|
||||
sharedPersistDevice = ${shared_persist_device};
|
||||
sharedPersistZfsPool = "quixos-shared";
|
||||
nixZfsPool = "nixos-zfs-nix";
|
||||
packageTreeRepo = "${package_tree_repo_url}";
|
||||
packageTreeRef = "${package_tree_ref}";
|
||||
|
||||
@@ -90,19 +90,13 @@ variable "root_volume_size_gb" {
|
||||
variable "persist_volume_size_gb" {
|
||||
type = number
|
||||
default = 50
|
||||
description = "Deployment-local persistent EBS volume size mounted at /persist-local."
|
||||
description = "Deployment persistent EBS volume size mounted at /persist."
|
||||
}
|
||||
|
||||
variable "persist_snapshot_id" {
|
||||
type = string
|
||||
default = null
|
||||
description = "Optional snapshot for the runtime /persist-local ZFS volume. If unset, the volume is blank and NixOS initializes quixos-persist/persist."
|
||||
}
|
||||
|
||||
variable "shared_persist_volume_id" {
|
||||
type = string
|
||||
default = null
|
||||
description = "Long-lived EBS volume ID mounted at /persist for shared state such as Caddy ACME state and SSH host keys. Runtime only attaches it; manage the volume from tofu/central."
|
||||
description = "Optional snapshot for the runtime /persist ZFS volume. If unset, the volume is blank and NixOS initializes quixos-persist/persist."
|
||||
}
|
||||
|
||||
variable "central_proxy_security_group_id" {
|
||||
@@ -141,12 +135,6 @@ variable "proxy_registration_mode" {
|
||||
}
|
||||
}
|
||||
|
||||
variable "caddy_persist_volume_id" {
|
||||
type = string
|
||||
default = null
|
||||
description = "Deprecated alias for shared_persist_volume_id, kept so older local resolved tfvars files do not break destroy."
|
||||
}
|
||||
|
||||
variable "quixos_repo_url" {
|
||||
type = string
|
||||
default = "https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos.git"
|
||||
|
||||
Reference in New Issue
Block a user