import http from "node:http"; import { AsyncLocalStorage } from "node:async_hooks"; import { randomUUID } from "node:crypto"; import { create } from "@bufbuild/protobuf"; import { createClient, type Client } from "@connectrpc/connect"; import { connectNodeAdapter, createConnectTransport, } from "@connectrpc/connect-node"; import { CaminoService, CrdtValueSchema, ListValueSchema, NullValueSchema, ObjectValueSchema, RefValueSchema, ValueSchema, type Value, } from "./camino/api_pb.js"; import { HandshakeResponseSchema, InvokeResponseSchema, PackageRuntime, type InvokeRequest, } from "./quixos/runtime_pb.js"; import { FunctionRefSchema } from "./quixos/refs_pb.js"; declare const objectRefBrand: unique symbol; export type ObjectRef = string & { readonly [objectRefBrand]: ClassId; }; export type CaminoClient = Client; export type { InvokeRequest, Value }; export type DerivedDependency = | { kind: "object"; objectId: string; } | { kind: "field"; objectId: string; fieldName: string; } | { kind: "edges"; objectId: string; sourceField?: string; }; type DerivedDependencyTracker = { record(dependency: DerivedDependency): void; dependencies(): DerivedDependency[]; }; const dependencyKey = (dependency: DerivedDependency) => { switch (dependency.kind) { case "object": return `object:${dependency.objectId}`; case "field": return `field:${dependency.objectId}:${dependency.fieldName}`; case "edges": return `edges:${dependency.objectId}:${dependency.sourceField ?? ""}`; } }; const createDependencyTracker = (): DerivedDependencyTracker => { const dependencies = new Map(); return { record(dependency) { dependencies.set(dependencyKey(dependency), dependency); }, dependencies() { return [...dependencies.values()]; }, }; }; const dependencyStorage = new AsyncLocalStorage(); export const recordDerivedDependency = (dependency: DerivedDependency) => { dependencyStorage.getStore()?.record(dependency); }; export const getActiveDerivedDependencies = (): DerivedDependency[] => dependencyStorage.getStore()?.dependencies() ?? []; export const runWithDerivedDependencyTracking = async ( fn: () => Promise | T, ): Promise<{ value: T; dependencies: DerivedDependency[] }> => { const tracker = createDependencyTracker(); const value = await dependencyStorage.run(tracker, fn); return { value, dependencies: tracker.dependencies(), }; }; const logDerivedDependencies = (dependencies: DerivedDependency[]) => { if (process.env.QUIXOS_DERIVED_DEPS_LOG !== "1") { return; } process.stderr.write( `[derived-deps] ${JSON.stringify({ dependencies })}\n`, ); }; const logDerivedWatch = (event: string, detail: Record) => { if (process.env.QUIXOS_DERIVED_WATCH_LOG !== "1") { return; } process.stderr.write( `[derived-watch] ${JSON.stringify({ event, ...detail })}\n`, ); }; type WatchableContext = { camino: CaminoClient; request: InvokeRequest; }; type ActiveDerivedWatch = { watchId: string; context: Context; get: RuntimeHandler; controllers: Map; running: boolean; pending: boolean; stopped: boolean; lastValue: unknown; }; const dependencyObjectIds = (dependencies: DerivedDependency[]) => new Set(dependencies.map((dependency) => dependency.objectId)); const stopDerivedWatch = ( watch: ActiveDerivedWatch, ) => { if (watch.stopped) { return; } watch.stopped = true; for (const controller of watch.controllers.values()) { controller.abort(); } watch.controllers.clear(); logDerivedWatch("stopped", { watchId: watch.watchId }); }; export type Field = { get(): Promise; set(value: T): Promise; }; export type RuntimeHandler = ( context: Context, ) => Promise | unknown; export type FieldOperation = { get?: RuntimeHandler; set?: RuntimeHandler; watchStart?: RuntimeHandler; watchStop?: RuntimeHandler; }; export const fieldOps = ( operation: FieldOperation, ): FieldOperation => operation; export const derivedWithDeps = (operation: { get: RuntimeHandler; }): FieldOperation => { type DerivedContext = Context & WatchableContext; const watches = new Map>(); const rerun = async (watch: ActiveDerivedWatch) => { if (watch.stopped) { return; } if (watch.running) { watch.pending = true; return; } watch.running = true; try { do { watch.pending = false; const { value, dependencies } = await runWithDerivedDependencyTracking( () => watch.get(watch.context), ); watch.lastValue = value; logDerivedDependencies(dependencies); logDerivedWatch("evaluated", { watchId: watch.watchId, value, dependencies, }); reconcileSubscriptions(watch, dependencies); } while (watch.pending && !watch.stopped); } catch (error) { logDerivedWatch("error", { watchId: watch.watchId, error: error instanceof Error ? error.message : String(error), }); } finally { watch.running = false; } }; const scheduleRerun = ( watch: ActiveDerivedWatch, reason: string, ) => { if (watch.stopped) { return; } logDerivedWatch("scheduled", { watchId: watch.watchId, reason }); void rerun(watch); }; const startObjectWatch = ( watch: ActiveDerivedWatch, objectId: string, ) => { const controller = new AbortController(); watch.controllers.set(objectId, controller); logDerivedWatch("subscribed", { watchId: watch.watchId, objectId }); void (async () => { try { for await (const event of watch.context.camino.watchObject( { objectId, includeSnapshot: false, }, { signal: controller.signal }, )) { if (controller.signal.aborted || watch.stopped) { break; } scheduleRerun( watch, event.op?.id ? `op:${event.op.id}` : `object:${objectId}`, ); } } catch (error) { if (!controller.signal.aborted && !watch.stopped) { logDerivedWatch("watch-error", { watchId: watch.watchId, objectId, error: error instanceof Error ? error.message : String(error), }); } } finally { if (watch.controllers.get(objectId) === controller) { watch.controllers.delete(objectId); logDerivedWatch("unsubscribed", { watchId: watch.watchId, objectId }); } } })(); }; const reconcileSubscriptions = ( watch: ActiveDerivedWatch, dependencies: DerivedDependency[], ) => { const objectIds = dependencyObjectIds(dependencies); for (const [objectId, controller] of watch.controllers.entries()) { if (!objectIds.has(objectId)) { controller.abort(); watch.controllers.delete(objectId); logDerivedWatch("unsubscribed", { watchId: watch.watchId, objectId }); } } for (const objectId of objectIds) { if (!watch.controllers.has(objectId)) { startObjectWatch(watch, objectId); } } }; return fieldOps({ async get(context) { const { value, dependencies } = await runWithDerivedDependencyTracking( () => operation.get(context), ); logDerivedDependencies(dependencies); return value; }, async watchStart(context) { const watchId = `watch:${randomUUID()}`; const watch: ActiveDerivedWatch = { watchId, context: context as DerivedContext, get: operation.get as RuntimeHandler, controllers: new Map(), running: false, pending: false, stopped: false, lastValue: undefined, }; watches.set(watchId, watch); logDerivedWatch("started", { watchId }); await rerun(watch); return { watch_id: watchId, }; }, watchStop(context) { const input = protoFieldsToJs((context as DerivedContext).request.input); const watchId = typeof input.watch_id === "string" ? input.watch_id : typeof input.watchId === "string" ? input.watchId : ""; const watch = watches.get(watchId); if (!watch) { throw new Error(`Unknown derived watch: ${watchId}`); } stopDerivedWatch(watch); watches.delete(watchId); return {}; }, }); }; export type RuntimeFunctionSpec = { exportName: ExportName; symbol: string; operation?: string; }; const isRecord = (value: unknown): value is Record => Boolean(value) && typeof value === "object" && !Array.isArray(value); const bytesToBase64 = (value: Uint8Array) => Buffer.from(value).toString("base64"); const base64ToBytes = (value: string) => Buffer.from(value, "base64"); export const jsToProtoValue = (value: unknown): Value => { if (value === null || value === undefined) { return create(ValueSchema, { kind: { case: "nullValue", value: create(NullValueSchema, {}), }, }); } if (typeof value === "boolean") { return create(ValueSchema, { kind: { case: "boolValue", value }, }); } if (typeof value === "number") { return create(ValueSchema, { kind: { case: "numberValue", value }, }); } if (typeof value === "bigint") { return create(ValueSchema, { kind: { case: "integerValue", value: value.toString() }, }); } if (typeof value === "string") { return create(ValueSchema, { kind: { case: "stringValue", value }, }); } if (value instanceof Uint8Array) { return create(ValueSchema, { kind: { case: "bytesValue", value }, }); } if (Array.isArray(value)) { return create(ValueSchema, { kind: { case: "listValue", value: create(ListValueSchema, { values: value.map(jsToProtoValue), }), }, }); } if (isRecord(value)) { if (typeof value.$caminoRef === "string") { return create(ValueSchema, { kind: { case: "refValue", value: create(RefValueSchema, { objectId: value.$caminoRef, }), }, }); } if ( typeof value.$caminoBytes === "string" && Object.keys(value).every((key) => key === "$caminoBytes") ) { return create(ValueSchema, { kind: { case: "bytesValue", value: base64ToBytes(value.$caminoBytes), }, }); } if ( typeof value.$caminoCrdtType === "string" && typeof value.$caminoCrdtPayload === "string" ) { return create(ValueSchema, { kind: { case: "crdtValue", value: create(CrdtValueSchema, { type: value.$caminoCrdtType, encoding: typeof value.$caminoCrdtEncoding === "string" ? value.$caminoCrdtEncoding : "base64", payload: base64ToBytes(value.$caminoCrdtPayload), }), }, }); } return create(ValueSchema, { kind: { case: "objectValue", value: create(ObjectValueSchema, { fields: jsObjectToProtoFields(value), }), }, }); } throw new Error(`Unsupported Camino RPC value: ${typeof value}`); }; export const protoValueToJs = (value: Value | undefined): unknown => { switch (value?.kind.case) { case "nullValue": case undefined: return null; case "boolValue": case "numberValue": case "stringValue": return value.kind.value; case "integerValue": return value.kind.value; case "bytesValue": return bytesToBase64(value.kind.value); case "listValue": return value.kind.value.values.map(protoValueToJs); case "objectValue": return protoFieldsToJs(value.kind.value.fields); case "refValue": return value.kind.value.objectId; case "crdtValue": return { $caminoCrdtType: value.kind.value.type, $caminoCrdtEncoding: value.kind.value.encoding, $caminoCrdtPayload: bytesToBase64(value.kind.value.payload), }; } }; export const jsObjectToProtoFields = (value: Record) => Object.fromEntries( Object.entries(value) .filter(([, fieldValue]) => fieldValue !== undefined) .map(([key, fieldValue]) => [key, jsToProtoValue(fieldValue)]), ); export const protoFieldsToJs = ( fields: Record | undefined, ): Record => Object.fromEntries( Object.entries(fields ?? {}).map(([key, value]) => [ key, protoValueToJs(value), ]), ); export const objectRef = (objectId: string) => objectId as ObjectRef; export const createField = ( camino: CaminoClient, objectId: string, fieldName: string, ): Field => ({ async get() { recordDerivedDependency({ kind: "field", objectId, fieldName }); const response = await camino.getObject({ objectId }); return protoValueToJs(response.object?.fields[fieldName]) as T | undefined; }, async set(value) { await camino.setField({ objectId, fieldName, value: jsToProtoValue(value), }); }, }); const createDependencyTrackingCaminoClient = ( camino: CaminoClient, ): CaminoClient => new Proxy(camino, { get(target, property, receiver) { if (property === "getObject") { return async (request: { objectId: string }, ...args: unknown[]) => { if (request.objectId) { recordDerivedDependency({ kind: "object", objectId: request.objectId, }); } return await ( Reflect.get(target, property, receiver) as ( request: { objectId: string }, ...args: unknown[] ) => Promise )(request, ...args); }; } if (property === "listEdges") { return async ( request: { objectId: string; sourceField?: string }, ...args: unknown[] ) => { if (request.objectId) { recordDerivedDependency({ kind: "edges", objectId: request.objectId, ...(request.sourceField ? { sourceField: request.sourceField } : {}), }); } return await ( Reflect.get(target, property, receiver) as ( request: { objectId: string; sourceField?: string }, ...args: unknown[] ) => Promise )(request, ...args); }; } return Reflect.get(target, property, receiver); }, }); export const serveQuixosPackageRuntime = < Context, Spec extends readonly RuntimeFunctionSpec[], >(params: { packageNamespace: string; packageName: string; runtimeProtocolVersion: string; functions: Spec; implementation: { [Name in Spec[number]["exportName"]]: | RuntimeHandler | FieldOperation; }; createContext: (camino: CaminoClient, request: InvokeRequest) => Context; }) => { const host = process.env.QUIXOS_RUNTIME_HOST ?? "127.0.0.1"; const port = Number(process.env.QUIXOS_RUNTIME_PORT ?? "0"); const caminoUrl = process.env.CAMINO_URL ?? "http://127.0.0.1:7310"; const camino = createClient( CaminoService, createConnectTransport({ baseUrl: caminoUrl, httpVersion: "1.1" }), ); const trackingCamino = createDependencyTrackingCaminoClient(camino); if (!Number.isInteger(port) || port <= 0) { throw new Error("QUIXOS_RUNTIME_PORT must be a positive integer"); } const functionRefs = params.functions.map((entry) => create(FunctionRefSchema, { packageNamespace: params.packageNamespace, packageName: params.packageName, symbol: entry.symbol, operation: entry.operation ?? "", }), ); const handler = connectNodeAdapter({ routes: (router) => { router.service(PackageRuntime, { async handshake() { return create(HandshakeResponseSchema, { packageNamespace: params.packageNamespace, packageName: params.packageName, runtimeProtocolVersion: params.runtimeProtocolVersion, functions: functionRefs, }); }, async invoke(request) { try { const symbol = request.function?.symbol ?? ""; const spec = params.functions.find( (entry) => entry.symbol === symbol && (entry.operation ?? "") === (request.function?.operation ?? ""), ); if (!spec) { throw new Error( `Unknown ${params.packageName} function: ${symbol}${ request.function?.operation ? `.${request.function.operation}` : "" }`, ); } const implementationExport = ( params.implementation as Record< string, RuntimeHandler | FieldOperation > )[spec.exportName]; const runtimeFunction = spec.operation && typeof implementationExport === "object" ? implementationExport[spec.operation as keyof FieldOperation] : implementationExport; if (!runtimeFunction) { throw new Error( `Missing ${params.packageName} implementation export: ${spec.exportName}${ spec.operation ? `.${spec.operation}` : "" }`, ); } if (typeof runtimeFunction !== "function") { throw new Error( `${params.packageName} implementation export is not callable: ${spec.exportName}${ spec.operation ? `.${spec.operation}` : "" }`, ); } return create(InvokeResponseSchema, { ok: true, result: jsToProtoValue( await runtimeFunction(params.createContext(trackingCamino, request)), ), }); } catch (error) { return create(InvokeResponseSchema, { ok: false, result: jsToProtoValue(null), error: error instanceof Error ? error.message : String(error), }); } }, }); }, }); const server = http.createServer((request, response) => { void handler(request, response); }); const close = () => server.close(() => process.exit(0)); server.listen(port, host, () => { process.stderr.write( `${params.packageName} listening on ${host}:${port}\n`, ); }); process.on("SIGTERM", close); process.on("SIGINT", close); };