Timothy J. Aveni
408 lines
14 KiB
Nix
408 lines
14 KiB
Nix
{ self }:
|
|
{ config, lib, pkgs, ... }:
|
|
|
|
let
|
|
cfg = config.services.quixosHost;
|
|
secretsCfg = cfg.secrets;
|
|
quixosPackages = self.packages.${pkgs.stdenv.hostPlatform.system};
|
|
secretFiles = {
|
|
controlSecret = "${secretsCfg.runtimeDirectory}/control-secret";
|
|
stateContextSecret = "${secretsCfg.runtimeDirectory}/state-context-secret";
|
|
packageTreeGitUsername = "${secretsCfg.runtimeDirectory}/package-tree-git-username";
|
|
packageTreeGitPassword = "${secretsCfg.runtimeDirectory}/package-tree-git-password";
|
|
};
|
|
fetchAwsSsmParameter = parameterName: destination: ''
|
|
${pkgs.awscli2}/bin/aws ssm get-parameter \
|
|
--with-decryption \
|
|
--name ${lib.escapeShellArg parameterName} \
|
|
--query Parameter.Value \
|
|
--output text > ${lib.escapeShellArg destination}.tmp
|
|
install -m 0640 -o root -g quixos-control ${lib.escapeShellArg destination}.tmp ${lib.escapeShellArg destination}
|
|
rm -f ${lib.escapeShellArg destination}.tmp
|
|
'';
|
|
installFileSecret = source: destination: ''
|
|
install -m 0640 -o root -g quixos-control ${lib.escapeShellArg source} ${lib.escapeShellArg destination}
|
|
'';
|
|
packageTreeGitAskPass = pkgs.writeShellScript "quixos-package-tree-git-askpass" ''
|
|
case "$1" in
|
|
*Username*)
|
|
username="$(cat ${lib.escapeShellArg secretFiles.packageTreeGitUsername} 2>/dev/null || true)"
|
|
if [ -z "$username" ]; then
|
|
username="oauth2"
|
|
fi
|
|
printf '%s\n' "$username"
|
|
;;
|
|
*Password*)
|
|
cat ${lib.escapeShellArg secretFiles.packageTreeGitPassword} 2>/dev/null || true
|
|
;;
|
|
*)
|
|
printf '\n'
|
|
;;
|
|
esac
|
|
'';
|
|
qxControl = pkgs.writeShellApplication {
|
|
name = "qx-control";
|
|
text = ''
|
|
if [ -z "''${QUIXOS_CONTROL_SECRET:-}" ] && [ -r ${lib.escapeShellArg secretFiles.controlSecret} ]; then
|
|
export QUIXOS_CONTROL_SECRET="$(cat ${lib.escapeShellArg secretFiles.controlSecret})"
|
|
fi
|
|
|
|
exec ${quixosPackages.qx-control}/bin/qx-control "$@"
|
|
'';
|
|
};
|
|
in
|
|
{
|
|
options.services.quixosHost = {
|
|
enable = lib.mkEnableOption "Quixos host";
|
|
|
|
domain = lib.mkOption {
|
|
type = lib.types.str;
|
|
description = "Public domain served by Caddy.";
|
|
};
|
|
|
|
packagesPath = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "/persist/srv/quixos/packages";
|
|
description = "Mutable package-tree checkout path.";
|
|
};
|
|
|
|
dataRoot = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "/persist/var/lib/quixos";
|
|
description = "Persistent Quixos runtime state root.";
|
|
};
|
|
|
|
secretsEnvironmentFile = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "/persist/secrets/quixos.env";
|
|
description = "Environment file containing Quixos secrets.";
|
|
};
|
|
|
|
packageTreeGitEnvironmentFile = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "/persist/secrets/quixos-git.env";
|
|
visible = false;
|
|
description = "Optional environment file containing Git credentials for the package tree.";
|
|
};
|
|
|
|
packageTreeRepo = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "https://gitea-external.egads.tutti.syntaxblitz.net/quixos-packages/packages.git";
|
|
description = "Git URL for the mutable package-tree super-repo.";
|
|
};
|
|
|
|
packageTreeRef = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "master";
|
|
description = "Package-tree ref to clone on first boot.";
|
|
};
|
|
|
|
forcePackageTreeRefOnBoot = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
description = "Whether each boot should force the package tree back to packageTreeRef.";
|
|
};
|
|
|
|
persistZfsPool = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "quixos-persist";
|
|
description = "ZFS pool name on the runtime persistent block device.";
|
|
};
|
|
|
|
nixZfsPool = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "nixos-zfs-nix";
|
|
description = "ZFS pool name supplied by the matched AMI /nix volume.";
|
|
};
|
|
|
|
hostId = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "71756978";
|
|
description = "Eight hex digit hostId required by ZFS.";
|
|
};
|
|
|
|
enableSsh = lib.mkOption {
|
|
type = lib.types.bool;
|
|
default = false;
|
|
description = "Enable inbound SSH in the NixOS host config. AWS ingress is managed by OpenTofu.";
|
|
};
|
|
|
|
secrets = {
|
|
provider = lib.mkOption {
|
|
type = lib.types.enum [ "files" "aws-ssm" ];
|
|
default = "files";
|
|
description = "Secret backend used to materialize Quixos runtime credentials.";
|
|
};
|
|
|
|
runtimeDirectory = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "/run/quixos/secrets";
|
|
description = "Runtime directory containing materialized secret files.";
|
|
};
|
|
|
|
files = {
|
|
controlSecretPath = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "/persist/secrets/control-secret";
|
|
description = "File backend path for the control secret.";
|
|
};
|
|
|
|
stateContextSecretPath = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "/persist/secrets/state-context-secret";
|
|
description = "File backend path for the state context secret.";
|
|
};
|
|
|
|
packageTreeGitUsernamePath = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "/persist/secrets/package-tree-git-username";
|
|
description = "File backend path for the package tree Git username.";
|
|
};
|
|
|
|
packageTreeGitPasswordPath = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "/persist/secrets/package-tree-git-password";
|
|
description = "File backend path for the package tree Git password.";
|
|
};
|
|
};
|
|
|
|
awsSsm = {
|
|
controlSecretParameter = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
description = "SSM parameter name for the control secret.";
|
|
};
|
|
|
|
stateContextSecretParameter = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
description = "SSM parameter name for the state context secret.";
|
|
};
|
|
|
|
packageTreeGitUsernameParameter = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
description = "SSM parameter name for the package tree Git username.";
|
|
};
|
|
|
|
packageTreeGitPasswordParameter = lib.mkOption {
|
|
type = lib.types.nullOr lib.types.str;
|
|
default = null;
|
|
description = "SSM parameter name for the package tree Git password.";
|
|
};
|
|
};
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = secretsCfg.provider != "aws-ssm" || (
|
|
secretsCfg.awsSsm.controlSecretParameter != null
|
|
&& secretsCfg.awsSsm.stateContextSecretParameter != null
|
|
&& secretsCfg.awsSsm.packageTreeGitUsernameParameter != null
|
|
&& secretsCfg.awsSsm.packageTreeGitPasswordParameter != null
|
|
);
|
|
message = "services.quixosHost.secrets.awsSsm.*Parameter options must all be set when provider is aws-ssm.";
|
|
}
|
|
];
|
|
|
|
nix.settings.experimental-features = [ "nix-command" "flakes" ];
|
|
|
|
networking.hostId = cfg.hostId;
|
|
networking.firewall.allowedTCPPorts = [ 80 443 ];
|
|
boot.supportedFilesystems = [ "zfs" ];
|
|
boot.zfs.extraPools = [ cfg.nixZfsPool cfg.persistZfsPool ];
|
|
boot.zfs.forceImportRoot = false;
|
|
services.zfs.autoScrub = {
|
|
enable = true;
|
|
pools = [ cfg.nixZfsPool cfg.persistZfsPool ];
|
|
};
|
|
|
|
fileSystems."/nix" = {
|
|
device = "${cfg.nixZfsPool}/nix";
|
|
fsType = "zfs";
|
|
neededForBoot = true;
|
|
};
|
|
|
|
fileSystems."/persist" = {
|
|
device = "${cfg.persistZfsPool}/persist";
|
|
fsType = "zfs";
|
|
neededForBoot = true;
|
|
};
|
|
|
|
users.groups.quixos = {};
|
|
users.groups.quixos-control = {};
|
|
users.users.quixos = {
|
|
isSystemUser = true;
|
|
group = "quixos";
|
|
extraGroups = [ "quixos-control" ];
|
|
home = cfg.dataRoot;
|
|
createHome = true;
|
|
};
|
|
users.users.caddy.extraGroups = [ "quixos" "quixos-control" ];
|
|
|
|
environment.systemPackages = [
|
|
pkgs.git
|
|
pkgs.nix
|
|
pkgs.amazon-ssm-agent
|
|
qxControl
|
|
];
|
|
environment.variables.QUIXOS_CONTROL_SOCKET = "/run/quixos/control.sock";
|
|
environment.variables.QUIXOS_WORKSPACE_ROOT = "/persist/srv/quixos";
|
|
|
|
systemd.sockets.nix-daemon = {
|
|
after = [ "nix.mount" ];
|
|
requires = [ "nix.mount" ];
|
|
};
|
|
|
|
systemd.services.nix-daemon = {
|
|
after = [ "nix.mount" ];
|
|
requires = [ "nix.mount" ];
|
|
};
|
|
|
|
services.amazon-ssm-agent.enable = true;
|
|
|
|
services.openssh = {
|
|
enable = lib.mkForce cfg.enableSsh;
|
|
hostKeys = [
|
|
{
|
|
path = "/persist/etc/ssh/ssh_host_ed25519_key";
|
|
type = "ed25519";
|
|
}
|
|
{
|
|
path = "/persist/etc/ssh/ssh_host_rsa_key";
|
|
type = "rsa";
|
|
bits = 4096;
|
|
}
|
|
];
|
|
};
|
|
|
|
systemd.tmpfiles.rules = [
|
|
"d /persist/etc/ssh 0700 root root -"
|
|
"d /persist/srv/quixos 0755 quixos quixos -"
|
|
"d ${cfg.dataRoot} 0750 quixos quixos -"
|
|
"d /persist/var/lib/acme 0750 caddy caddy -"
|
|
"L /var/lib/acme - - - - /persist/var/lib/acme"
|
|
];
|
|
|
|
systemd.services.quixos-secrets = {
|
|
description = "Materialize Quixos runtime secrets";
|
|
wantedBy = [ "multi-user.target" ];
|
|
before = [ "quixos-package-tree.service" "quixos-orchestrator.service" ];
|
|
after = [ "network-online.target" ];
|
|
wants = [ "network-online.target" ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
};
|
|
path = [ pkgs.awscli2 pkgs.coreutils ];
|
|
script =
|
|
''
|
|
set -eu
|
|
install -d -m 0750 -o root -g quixos-control ${lib.escapeShellArg secretsCfg.runtimeDirectory}
|
|
''
|
|
+ (
|
|
if secretsCfg.provider == "aws-ssm" then ''
|
|
${fetchAwsSsmParameter secretsCfg.awsSsm.controlSecretParameter secretFiles.controlSecret}
|
|
${fetchAwsSsmParameter secretsCfg.awsSsm.stateContextSecretParameter secretFiles.stateContextSecret}
|
|
${fetchAwsSsmParameter secretsCfg.awsSsm.packageTreeGitUsernameParameter secretFiles.packageTreeGitUsername}
|
|
${fetchAwsSsmParameter secretsCfg.awsSsm.packageTreeGitPasswordParameter secretFiles.packageTreeGitPassword}
|
|
'' else ''
|
|
${installFileSecret secretsCfg.files.controlSecretPath secretFiles.controlSecret}
|
|
${installFileSecret secretsCfg.files.stateContextSecretPath secretFiles.stateContextSecret}
|
|
${installFileSecret secretsCfg.files.packageTreeGitUsernamePath secretFiles.packageTreeGitUsername}
|
|
${installFileSecret secretsCfg.files.packageTreeGitPasswordPath secretFiles.packageTreeGitPassword}
|
|
''
|
|
);
|
|
};
|
|
|
|
systemd.services.quixos-package-tree = {
|
|
description = "Prepare Quixos package tree";
|
|
wantedBy = [ "multi-user.target" ];
|
|
before = [ "quixos-orchestrator.service" ];
|
|
after = [ "network-online.target" "persist.mount" "quixos-secrets.service" ];
|
|
wants = [ "network-online.target" "quixos-secrets.service" ];
|
|
requires = [ "quixos-secrets.service" ];
|
|
path = [ pkgs.git ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
User = "quixos";
|
|
Group = "quixos";
|
|
RemainAfterExit = true;
|
|
};
|
|
environment = {
|
|
GIT_ASKPASS = packageTreeGitAskPass;
|
|
GIT_TERMINAL_PROMPT = "0";
|
|
};
|
|
script = ''
|
|
set -eu
|
|
mkdir -p "$(dirname ${lib.escapeShellArg cfg.packagesPath})"
|
|
if [ ! -d ${lib.escapeShellArg cfg.packagesPath}/.git ]; then
|
|
git clone ${lib.escapeShellArg cfg.packageTreeRepo} ${lib.escapeShellArg cfg.packagesPath}
|
|
git -C ${lib.escapeShellArg cfg.packagesPath} checkout --detach ${lib.escapeShellArg cfg.packageTreeRef}
|
|
git -C ${lib.escapeShellArg cfg.packagesPath} submodule update --init --recursive
|
|
elif ${if cfg.forcePackageTreeRefOnBoot then "true" else "false"}; then
|
|
git -C ${lib.escapeShellArg cfg.packagesPath} fetch --tags --prune origin
|
|
git -C ${lib.escapeShellArg cfg.packagesPath} checkout --detach ${lib.escapeShellArg cfg.packageTreeRef}
|
|
git -C ${lib.escapeShellArg cfg.packagesPath} submodule sync --recursive
|
|
git -C ${lib.escapeShellArg cfg.packagesPath} submodule update --init --recursive
|
|
fi
|
|
'';
|
|
};
|
|
|
|
systemd.services.quixos-orchestrator = {
|
|
description = "Quixos orchestrator";
|
|
wantedBy = [ "multi-user.target" ];
|
|
after = [ "network-online.target" "nix-daemon.socket" "quixos-secrets.service" "quixos-package-tree.service" ];
|
|
wants = [ "network-online.target" "nix-daemon.socket" "quixos-secrets.service" "quixos-package-tree.service" ];
|
|
requires = [ "quixos-secrets.service" ];
|
|
serviceConfig = {
|
|
User = "quixos";
|
|
Group = "quixos-control";
|
|
SupplementaryGroups = [ "quixos" ];
|
|
RuntimeDirectory = "quixos";
|
|
RuntimeDirectoryMode = "0770";
|
|
WorkingDirectory = "/persist/srv/quixos";
|
|
Restart = "always";
|
|
RestartSec = "5s";
|
|
};
|
|
script = ''
|
|
set -eu
|
|
export QUIXOS_CONTROL_SECRET="$(cat ${lib.escapeShellArg secretFiles.controlSecret})"
|
|
export QUIXOS_STATE_CONTEXT_SECRET="$(cat ${lib.escapeShellArg secretFiles.stateContextSecret})"
|
|
exec ${quixosPackages.quixos-orchestrator}/bin/quixos-orchestrator --packages-path ${lib.escapeShellArg cfg.packagesPath}
|
|
'';
|
|
environment = {
|
|
XDG_DATA_HOME = cfg.dataRoot;
|
|
QUIXOS_WORKSPACE_ROOT = "/persist/srv/quixos";
|
|
QUIXOS_CONTROL_SOCKET = "/run/quixos/control.sock";
|
|
QUIXOS_DEVICE_ASSIGNMENTS_FILE = "${cfg.packagesPath}/device-assignments.json";
|
|
QUIXOS_REMOTE_DOM_RELAY_SOCKET = "/run/quixos/relay.sock";
|
|
QUIXOS_REMOTE_DOM_RELAY_PUBLIC_WS_URL = "wss://${cfg.domain}/relay/";
|
|
QUIXOS_REMOTE_DOM_RELAY_PUBLIC_HTTP_ORIGIN = "https://${cfg.domain}/relay";
|
|
};
|
|
};
|
|
|
|
services.caddy = {
|
|
enable = true;
|
|
virtualHosts.${cfg.domain}.extraConfig = ''
|
|
handle_path /control/* {
|
|
reverse_proxy unix//run/quixos/control.sock
|
|
}
|
|
|
|
handle_path /relay/* {
|
|
reverse_proxy unix//run/quixos/relay.sock {
|
|
header_up X-Forwarded-Prefix /relay
|
|
}
|
|
}
|
|
|
|
handle {
|
|
root * ${quixosPackages.project-visualizer-static}
|
|
file_server
|
|
}
|
|
'';
|
|
};
|
|
};
|
|
}
|