quixos/camino-package-runtime
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement workspace evolution, migrations, and runtime continuity

Browse Source 
Enable evolution by default for source-backed workspaces. Add stable
conformance ownership, semantic-major review, candidate typechecking,
and durable fenced cutover with explicit migrations and forward recovery.

Independently supervise package runtimes so unchanged resource owners keep
their processes and connections across cutover. Add scoped invocation
authority, resource sessions, and typed callback rebinding.

Wire opaque object references through generated bindings and RPCs. Add
canonical relationship sets, keyed maps, and ordered lists with scoped
transactional mutations, revision checks, and inverse consistency. Support
planned cascade deletion, protection, tombstones, and lifecycle foundations.

Add journaled structural edits, package/function/migration scaffolding,
managed repository creation, and resumable bottom-up dependency pin
publication. Document lifetime boundaries, revision pinning, prototype
compatibility policy, commands, and deferred work.

Validate with 210 tests, user-systemd process/connection continuity,
generated-package TypeScript checks, and Nix host/protocol checks.
TTL handoff, physical reclamation, general multi-step migrations, and
root-systemd migration isolation acceptance remain deferred.
This commit is contained in:
Timothy J. Aveni
2026-09-10 18:27:41 -07:00
parent 549c4539d5
commit ce6ae8f662
47 changed files with 1837 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/index.ts
+170 -32
View File
@@ -1,7 +1,12 @@
import http from "node:http";
import { readFileSync } from "node:fs";
import { createInvocationRegistry } from "./invocations.js";
import { isObjectReference, referenceFromWire, referenceToWire, assertReferenceFree, type QxObjectRef } from "./references.js";
export * from "./bindings.js";
export {relationshipMap, relationshipList, relationshipSet} from "./relationships.js";
import { AsyncLocalStorage } from "node:async_hooks";
import { randomUUID } from "node:crypto";
import { createHmac, randomBytes, randomUUID, timingSafeEqual } from "node:crypto";
export {createMigrationContext, migrationObjectId, serveMigration, type MigrationContext, type MigrationInput, type MigrationOutput, type MigrationEdge} from "./migration.js";
import { create, equals } from "@bufbuild/protobuf";
import { Code, ConnectError, createClient, type Client, type ConnectRouter } from "@connectrpc/connect";
import { connectNodeAdapter, createConnectTransport } from "@connectrpc/connect-node";
@@ -61,10 +66,11 @@ const isWrappedValue = (value: unknown): value is { $quixosValue: Value } =>
isRecord(value) && "$quixosValue" in value &&
isRecord(value.$quixosValue) && value.$quixosValue.$typeName === "camino.Value";
export const objectRef = (objectId: string) => ({ $quixosRef: objectId });
export const objectRef = (reference: QxObjectRef) => { referenceToWire(reference); return reference; };
export const liveValue = (value: Value) => ({ $quixosValue: value });
export const jsToProtoValue = (value: unknown): Value => {
if (isObjectReference(value)) return create(ValueSchema, {kind: {case: "refValue", value: create(RefValueSchema, {objectId: referenceToWire(value)})}});
if (isWrappedValue(value)) return value.$quixosValue;
if (value === null || value === undefined) {
return create(ValueSchema, { kind: { case: "nullValue", value: create(NullValueSchema, {}) } });
@@ -79,11 +85,7 @@ export const jsToProtoValue = (value: unknown): Value => {
kind: { case: "listValue", value: create(ListValueSchema, { values: value.map(jsToProtoValue) }) },
});
}
if (isRecord(value) && typeof value.$quixosRef === "string") {
return create(ValueSchema, {
kind: { case: "refValue", value: create(RefValueSchema, { objectId: value.$quixosRef }) },
});
}
if (isRecord(value) && "$quixosRef" in value) throw new Error("Raw ID wrappers are not object references");
if (isRecord(value) && typeof value.$quixosCrdtType === "string" &&
typeof value.$quixosCrdtPayload === "string") {
return create(ValueSchema, {
@@ -111,7 +113,7 @@ export const protoValueToJs = (value: Value | undefined): unknown => {
case "stringValue":
case "integerValue": return value.kind.value;
case "bytesValue": return bytesToBase64(value.kind.value);
case "refValue": return value.kind.value.objectId;
case "refValue": return referenceFromWire(value.kind.value.objectId);
case "listValue": return value.kind.value.values.map(protoValueToJs);
case "objectValue": return Object.fromEntries(
Object.entries(value.kind.value.fields).map(([key, entry]) => [key, protoValueToJs(entry)]),
@@ -136,24 +138,31 @@ export type StatePort<T = unknown> = {
export type EdgePort = {
edgeTypeId: string;
projectionId: string;
resolve(): Promise<string[]>;
connect(targetObjectId: string): Promise<void>;
disconnect(targetObjectId: string): Promise<void>;
resolve(): Promise<QxObjectRef[]>;
connect(target: QxObjectRef): Promise<void>;
disconnect(target: QxObjectRef): Promise<void>;
collection(): Promise<RelationshipCollection>;
replace(entries: RelationshipEntry[], expectedRevision: bigint): Promise<RelationshipCollection>;
};
export type RelationshipEntry<T extends QxObjectRef = QxObjectRef> = {edgeId?: string; target: T; key?: string | boolean | bigint};
export type RelationshipCollection<T extends QxObjectRef = QxObjectRef> = {revision: bigint; entries: RelationshipEntry<T>[]};
export type InterfacePort = {
objectId: string;
objectId: QxObjectRef;
interfaceRevisionId: string;
invoke(operationId: string, input?: Record<string, unknown>): Promise<unknown>;
live(operationId: string, input?: Record<string, unknown>): Promise<ReturnType<typeof liveValue>>;
};
export type ConstructorPort = {
atomId: string;
construct(input?: Record<string, unknown>): Promise<string>;
construct(input?: Record<string, unknown>): Promise<QxObjectRef>;
};
export type RuntimePort = StatePort | EdgePort | InterfacePort | ConstructorPort;
export type RuntimeContext = {
objectId: string;
/** Cooperative cancellation. Completion is acknowledged only after the handler returns. */
signal?: AbortSignal;
openSession?: () => Promise<RuntimeSession>;
objectId: QxObjectRef;
input: Record<string, unknown>;
inputProto: Record<string, Value>;
ports: ReadonlyMap<string, RuntimePort>;
@@ -163,6 +172,17 @@ export type RuntimeContext = {
constructor(portId: string): ConstructorPort;
};
export type RuntimeSession = {
id: string;
run<T>(work: (context: RuntimeContext) => Promise<T>): Promise<T>;
/** Close the external resource first, then close its host retention/session. */
close(): Promise<void>;
};
export class RuntimeAuthorityError extends Error {
readonly retryable: boolean;
constructor(message: string) { super(message); this.name = "RuntimeAuthorityError"; this.retryable = /WORKSPACE_FENCED|STALE_EPOCH/.test(message); }
}
const targetForEdge = (
edge: { firstObjectId: string; secondObjectId: string; firstProjectionId: string },
projectionId: string,
@@ -192,6 +212,7 @@ export const createRuntimeContext = (
return liveValue(value.value);
},
async set(value) {
assertReferenceFree(isWrappedValue(value) ? protoValueToJs(value.$quixosValue) : value);
await camino.writeState({ objectId: dependencyObjectId, slotId, value: jsToProtoValue(value) });
},
};
@@ -201,18 +222,34 @@ export const createRuntimeContext = (
case "edge": {
const { edgeTypeId, projectionId } = dependency.binding.value;
const dependencyObjectId = dependency.objectId || request.objectId;
const collectionResult = (response: {revision: bigint; entries: {edgeId: string; targetObjectId: string; key?: Value}[]}): RelationshipCollection => ({revision: response.revision,
entries: response.entries.map((entry) => ({edgeId: entry.edgeId, target: referenceFromWire(entry.targetObjectId),
...(entry.key ? {key: entry.key.kind.case === "integerValue" ? BigInt(entry.key.kind.value) : protoValueToJs(entry.key) as string | boolean} : {})}))});
const edge: EdgePort = {
edgeTypeId,
projectionId,
async collection() {
await recordDependency({kind: "edge", objectId: dependencyObjectId, attachmentId: edgeTypeId, projectionId});
return collectionResult(await camino.readCollection({objectId: dependencyObjectId, edgeTypeId, projectionId}));
},
async replace(entries, expectedRevision) {
return collectionResult(await camino.replaceCollection({objectId: dependencyObjectId, edgeTypeId, projectionId, expectedRevision,
entries: entries.map((entry) => {
assertReferenceFree(entry.key);
return {edgeId: entry.edgeId ?? "", targetObjectId: referenceToWire(entry.target), key: entry.key === undefined ? undefined : jsToProtoValue(entry.key)};
})}));
},
async resolve() {
await recordDependency({ kind: "edge", objectId: dependencyObjectId, attachmentId: edgeTypeId, projectionId });
const result = await camino.resolveEdge({ objectId: dependencyObjectId, edgeTypeId, projectionId });
return result.edges.map((entry) => targetForEdge(entry, projectionId));
return result.edges.map((entry) => referenceFromWire(targetForEdge(entry, projectionId)));
},
async connect(targetObjectId) {
async connect(target) {
const targetObjectId = referenceToWire(target);
await camino.connectEdge({ objectId: dependencyObjectId, edgeTypeId, projectionId, targetObjectId });
},
async disconnect(targetObjectId) {
async disconnect(target) {
const targetObjectId = referenceToWire(target);
const result = await camino.resolveEdge({ objectId: dependencyObjectId, edgeTypeId, projectionId });
for (const entry of result.edges) {
if (targetForEdge(entry, projectionId) === targetObjectId) await camino.disconnectEdge({ edgeId: entry.id });
@@ -251,7 +288,7 @@ export const createRuntimeContext = (
return response.result;
};
const capability: InterfacePort = {
objectId: dependencyObjectId,
objectId: referenceFromWire(dependencyObjectId),
interfaceRevisionId,
async invoke(operationId, input = {}) {
return protoValueToJs(await invoke(operationId, input));
@@ -275,7 +312,7 @@ export const createRuntimeContext = (
input: Object.fromEntries(Object.entries(input).map(([key, value]) => [key, jsToProtoValue(value)])),
});
if (!response.object) throw new Error(`Constructor ${atomId} returned no object`);
return response.object.id;
return referenceFromWire(response.object.id);
},
};
ports.set(dependency.portId, constructor);
@@ -288,7 +325,7 @@ export const createRuntimeContext = (
return port as T;
};
return {
objectId: request.objectId,
objectId: referenceFromWire(request.objectId),
input: protoFieldsToJs(request.input),
inputProto: request.input,
ports,
@@ -332,9 +369,12 @@ export const createPackageRuntimeRoutes = (config: {
caminoUrl?: string;
orchUrl?: string;
}) => {
const invocations = createInvocationRegistry();
const headers: Record<string, string> = {};
if (process.env.CAMINO_RUNTIME_AUTH_TOKEN) {
headers["x-camino-runtime-token"] = process.env.CAMINO_RUNTIME_AUTH_TOKEN;
const processToken = process.env.CAMINO_RUNTIME_AUTH_TOKEN ?? (process.env.CAMINO_RUNTIME_AUTH_TOKEN_FILE
? readFileSync(process.env.CAMINO_RUNTIME_AUTH_TOKEN_FILE, "utf8").trim() : "");
if (processToken) {
headers["x-camino-runtime-token"] = processToken;
} else if (process.env.CAMINO_RUNTIME_AUTH_REQUIRED === "1") {
throw new Error("CAMINO_RUNTIME_AUTH_TOKEN is required");
}
@@ -353,24 +393,114 @@ export const createPackageRuntimeRoutes = (config: {
httpVersion: "1.1",
}));
const authenticateInstance = (header: Headers) => {
if (!process.env.QUIXOS_RUNTIME_INSTANCE_ID) return; // standalone development ABI
const supplied = Buffer.from(header.get("x-quixos-instance-token") ?? "");
const expected = Buffer.from(processToken);
if (!expected.length || supplied.length !== expected.length || !timingSafeEqual(supplied, expected)) {
throw new ConnectError("Invalid runtime instance credential", Code.Unauthenticated);
}
};
const clientsFor = (request: { context?: { grant: string; instanceId: string; workspaceEpoch: string } }) => {
const context = request.context;
if (process.env.QUIXOS_RUNTIME_INSTANCE_ID && (!context?.grant || context.instanceId !== process.env.QUIXOS_RUNTIME_INSTANCE_ID || !context.workspaceEpoch)) {
throw new ConnectError("Managed invocation requires an exact instance and epoch grant", Code.Unauthenticated);
}
if (!context?.grant) return { camino, orch };
const transport = (url: string) => createConnectTransport({ baseUrl: url, httpVersion: "1.1", interceptors: [(next) => async (call) => {
call.header.set("x-quixos-invocation-grant", context.grant);
call.header.set("x-camino-runtime-token", processToken);
return next(call);
}] });
return {
camino: createClient(CaminoService, transport(config.caminoUrl ?? process.env.CAMINO_URL ?? "http://127.0.0.1:7310")),
orch: createClient(OrchestratorRuntime, transport(config.orchUrl ?? process.env.QUIXOS_ORCH_URL ?? "http://127.0.0.1:7311")),
};
};
const runtimeControl = async <T>(operation: string, input: unknown): Promise<T> => {
const response = await fetch(`${config.caminoUrl ?? process.env.CAMINO_URL ?? "http://127.0.0.1:7310"}/__runtime/${operation}`, {
method: "POST", headers: { "content-type": "application/json", "x-camino-runtime-token": processToken }, body: JSON.stringify(input), signal: AbortSignal.timeout(10_000),
});
const value = await response.json() as T & {error?: string};
if (!response.ok) throw new RuntimeAuthorityError(value.error ?? "Runtime authority request failed");
return value;
};
const attachSessions = (runtimeContext: RuntimeContext, request: RuntimeRequest & {context?: {grant: string; instanceId: string; workspaceEpoch: string; ownerConformanceId?: string}}) => {
if (!request.context?.grant || !request.context.ownerConformanceId) return;
runtimeContext.openSession = async () => {
const ownerId = request.context!.ownerConformanceId!;
const registration = { grant: request.context!.grant, objectId: request.objectId, ownerId,
sessionId: `session:${randomBytes(16).toString("hex")}`, token: randomBytes(32).toString("base64url") };
const register = () => runtimeControl<{sessionId: string; token: string}>("register-session", registration);
const registered = await register().catch((error) => {
// Retry a transport/lost-response failure with exactly the same identity.
// Admission/authority errors are definitive and must not be retried here.
if (error instanceof RuntimeAuthorityError) throw error;
return register();
});
let closed = false;
return {
id: registered.sessionId,
async run(work) {
if (closed) throw new RuntimeAuthorityError("SESSION_CLOSED");
// Acquisition happens before user code. A fence failure can be retried
// by the caller without replaying a side-effecting callback.
const grant = await runtimeControl<{grant: string; epoch: string; instanceId: string; invocationId: string; bindingDigest: string}>("acquire-session", registered);
const execution = invocations.begin(grant.invocationId);
const sessionRequest = { ...request, context: { grant: grant.grant, instanceId: grant.instanceId, workspaceEpoch: grant.epoch } };
const clients = clientsFor(sessionRequest);
const context = createRuntimeContext(clients.camino, clients.orch, sessionRequest);
context.signal = execution.signal;
try { return await work(context); }
finally {
execution.finish();
await runtimeControl("complete-invocation", { invocationId: grant.invocationId }).catch((error) => console.error("Session completion will be reconciled by the host", error));
}
},
async close() { await runtimeControl("close-session", registered); closed = true; },
};
};
};
return (router: ConnectRouter) => router.service(PackageRuntime, {
handshake: () => create(HandshakeResponseSchema, {
handshake: (request) => create(HandshakeResponseSchema, {
packageRevisionId: config.packageRevisionId,
runtimeProtocolVersion: "quixos-capabilities-v1",
exportIds: Object.keys(config.exports),
capabilities: ["invocation-completion-v1", "instance-authentication-v1", "epoch-grants-v1"],
instanceId: process.env.QUIXOS_RUNTIME_INSTANCE_ID ?? "",
authenticationProof: request.nonce && processToken ? createHmac("sha256", processToken)
.update(JSON.stringify([request.nonce, process.env.QUIXOS_RUNTIME_INSTANCE_ID ?? "", config.packageRevisionId]))
.digest("hex") : "",
}),
invoke: async (request) => {
getInvocationStatus: (request, context) => {
authenticateInstance(context.requestHeader);
return invocations.status(request.invocationId);
},
cancelInvocation: (request, context) => {
authenticateInstance(context.requestHeader);
return invocations.cancel(request.invocationId);
},
invoke: async (request, context) => {
authenticateInstance(context.requestHeader);
const { camino, orch } = clientsFor(request);
const exportId = request.export?.exportId;
const handler = exportId ? config.exports[exportId] : undefined;
if (!handler) throw new ConnectError(`Unknown export ${exportId ?? ""}`, Code.NotFound);
const execution = invocations.begin(request.invocationId || (process.env.QUIXOS_RUNTIME_INSTANCE_ID ? "" : randomUUID()));
try {
const result = await evaluate(handler, createRuntimeContext(camino, orch, request));
const runtimeContext = createRuntimeContext(camino, orch, request);
attachSessions(runtimeContext, request);
runtimeContext.signal = AbortSignal.any([context.signal, execution.signal]);
const result = await evaluate(handler, runtimeContext);
execution.finish();
return create(InvokeResponseSchema, {
ok: true,
result: result.value,
dependencies: protoDependencies(result.dependencies),
});
} catch (error) {
execution.finish(true);
return create(InvokeResponseSchema, {
ok: false,
error: error instanceof Error ? error.message : String(error),
@@ -378,13 +508,20 @@ export const createPackageRuntimeRoutes = (config: {
}
},
watch: async function* (request, context) {
authenticateInstance(context.requestHeader);
const { camino, orch } = clientsFor(request);
const exportId = request.export?.exportId;
const handler = exportId ? config.exports[exportId] : undefined;
if (!handler || !isDerived(handler)) {
throw new ConnectError(`Export ${exportId ?? ""} is not derived`, Code.FailedPrecondition);
}
const execution = invocations.begin(request.invocationId || (process.env.QUIXOS_RUNTIME_INSTANCE_ID ? "" : randomUUID()));
const signal = AbortSignal.any([context.signal, execution.signal]);
try {
const watchId = `watch:${randomUUID()}`;
const runtimeContext = createRuntimeContext(camino, orch, request);
attachSessions(runtimeContext, request);
runtimeContext.signal = signal;
type WatchOutcome = { key: string; done: boolean; error?: unknown };
type Subscription = {
dependency: RuntimeDependency;
@@ -404,7 +541,7 @@ export const createPackageRuntimeRoutes = (config: {
const establish = (async () => {
const controller = new AbortController();
const stream = camino.watchObject(
{ objectId: dependency.objectId, includeSnapshot: true },
{ objectId: dependency.objectId, includeSnapshot: true, attachmentIds: request.context?.grant ? [dependency.attachmentId] : [] },
{ signal: controller.signal },
)[Symbol.asyncIterator]();
try {
@@ -443,7 +580,7 @@ export const createPackageRuntimeRoutes = (config: {
subscription.controller.abort();
}
};
context.signal.addEventListener("abort", abortAll, { once: true });
signal.addEventListener("abort", abortAll, { once: true });
const evaluateWithStableSubscriptions = async () => {
// A direct state/edge port records its dependency before reading it,
@@ -459,6 +596,7 @@ export const createPackageRuntimeRoutes = (config: {
throw new Error("Derived dependency discovery did not stabilize after 32 passes");
};
try {
let current = await evaluateWithStableSubscriptions();
yield create(WatchEventSchema, {
watchId,
@@ -468,11 +606,10 @@ export const createPackageRuntimeRoutes = (config: {
});
const abort = new Promise<"abort">((resolve) => {
if (context.signal.aborted) resolve("abort");
else context.signal.addEventListener("abort", () => resolve("abort"), { once: true });
if (signal.aborted) resolve("abort");
else signal.addEventListener("abort", () => resolve("abort"), { once: true });
});
try {
while (!context.signal.aborted) {
while (!signal.aborted) {
if (subscriptions.size === 0) {
await abort;
break;
@@ -506,9 +643,10 @@ export const createPackageRuntimeRoutes = (config: {
current = updated;
}
} finally {
context.signal.removeEventListener("abort", abortAll);
signal.removeEventListener("abort", abortAll);
abortAll();
}
} finally { execution.finish(); }
},
});
};