Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4a0eec33d8 | |||
| d6434efc28 | |||
| ccb48afc04 | |||
| e4e5c1f175 | |||
| 1b91e21d49 | |||
| fcfce6259e | |||
| dec3aa475a | |||
| 50b6442a41 | |||
| a9f42e14c3 |
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"version": 1,
|
"version": 1,
|
||||||
"sourceRepo": "https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos",
|
"sourceRepo": "https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos",
|
||||||
"sourceCommit": "932fe7714761fb8b5db4bd6401357cdff762a428",
|
"sourceCommit": "286ada5419c4fc2184b387cea4715fae1a74bde5",
|
||||||
"sourcePath": "quixos-instance/quixos-nix-helpers",
|
"sourcePath": "quixos-instance/quixos-nix-helpers",
|
||||||
"exportName": "quixos-nix-helpers",
|
"exportName": "quixos-nix-helpers",
|
||||||
"mirrorRemote": "https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos-nix-helpers.git"
|
"mirrorRemote": "https://gitea-external.egads.tutti.syntaxblitz.net/quixos/quixos-nix-helpers.git"
|
||||||
|
|||||||
+15
-1
@@ -1,5 +1,11 @@
|
|||||||
# Checked capability and component artifacts
|
# Checked capability and component artifacts
|
||||||
|
|
||||||
|
This is the **current helper/output format**, not the core package abstraction.
|
||||||
|
See [the architecture](../../docs/ARCHITECTURE.md) and
|
||||||
|
[current gaps](../../docs/ARCHITECTURE_STATUS.md). In particular, the universal
|
||||||
|
artifact inventory and component-only omission of a server are implementation
|
||||||
|
gaps to resolve, not requirements for arbitrary package servers.
|
||||||
|
|
||||||
`mkCaminoArtifacts` is the fresh package builder. It regenerates candidate
|
`mkCaminoArtifacts` is the fresh package builder. It regenerates candidate
|
||||||
bindings, typechecks the entry's dependency closure, bundles declared targets,
|
bindings, typechecks the entry's dependency closure, bundles declared targets,
|
||||||
generates a process service when requested, and emits content manifests and
|
generates a process service when requested, and emits content manifests and
|
||||||
@@ -105,7 +111,7 @@ and supplies an explicit SDK for value codecs, identities and declared graph por
|
|||||||
service = {
|
service = {
|
||||||
restricted = true;
|
restricted = true;
|
||||||
limits = {
|
limits = {
|
||||||
deadlineMs = 4000;
|
deadlineMs = 10000;
|
||||||
maxBytes = 1048576;
|
maxBytes = 1048576;
|
||||||
maxPortCalls = 1000;
|
maxPortCalls = 1000;
|
||||||
maxQueued = 8;
|
maxQueued = 8;
|
||||||
@@ -122,6 +128,14 @@ invocation identity; authored code cannot choose another frame or workspace.
|
|||||||
Native graph grants and transaction commit remain enforced by Camino outside
|
Native graph grants and transaction commit remain enforced by Camino outside
|
||||||
the guest. Initialization and synchronous loops are subject to the same deadline.
|
the guest. Initialization and synchronous loops are subject to the same deadline.
|
||||||
|
|
||||||
|
The default root/Package Invocation wall deadline is 10 seconds. The separate
|
||||||
|
root CPU allowance remains at most 4 CPU seconds; extra wall time accommodates
|
||||||
|
queueing and graph-port waits. Ordinary graph database transactions and idle
|
||||||
|
transactions allow 30 seconds, and ordinary client HTTP requests allow 45 seconds
|
||||||
|
so server failures can be returned. Nested calls inherit the remaining root
|
||||||
|
budget; explicit smaller limits still apply. Restore/activation jobs retain their
|
||||||
|
separate budgets. These defaults are starting limits, not latency targets.
|
||||||
|
|
||||||
Direct network access, native filesystem imports, ambient clock and entropy,
|
Direct network access, native filesystem imports, ambient clock and entropy,
|
||||||
parent environment credentials, and unlisted modules are denied. Explicit dates,
|
parent environment credentials, and unlisted modules are denied. Explicit dates,
|
||||||
`Intl.Segmenter`, deterministic hashes and normal JavaScript computation remain
|
`Intl.Segmenter`, deterministic hashes and normal JavaScript computation remain
|
||||||
|
|||||||
@@ -37,7 +37,7 @@ let
|
|||||||
mkdir -p "$out"
|
mkdir -p "$out"
|
||||||
cat > entry.mjs <<'JS'
|
cat > entry.mjs <<'JS'
|
||||||
export {definePortableRegistry,QxReject} from '${core}/dist/portable.js';
|
export {definePortableRegistry,QxReject} from '${core}/dist/portable.js';
|
||||||
export {encodeArgument,decodeArgument,canonicalJson,parseJson} from '${core}/dist/value.js';
|
export {encodeArgument,decodeArgument,canonicalJson,parseJson,hashJson} from '${core}/dist/value.js';
|
||||||
export {workspaceId,operationId,objectId,entryId,hashIdentity} from '${core}/dist/identity.js';
|
export {workspaceId,operationId,objectId,entryId,hashIdentity} from '${core}/dist/identity.js';
|
||||||
export {reference} from '${core}/dist/references.js';
|
export {reference} from '${core}/dist/references.js';
|
||||||
export {CodecError,ExecutionError} from '${core}/dist/limits.js';
|
export {CodecError,ExecutionError} from '${core}/dist/limits.js';
|
||||||
|
|||||||
@@ -108,7 +108,7 @@ export async function serveRestrictedRegistry(config, io = process) {
|
|||||||
const limits = {
|
const limits = {
|
||||||
maxBytes: config.maxBytes ?? 1024 * 1024,
|
maxBytes: config.maxBytes ?? 1024 * 1024,
|
||||||
maxPortCalls: config.maxPortCalls ?? 1000,
|
maxPortCalls: config.maxPortCalls ?? 1000,
|
||||||
deadlineMs: config.deadlineMs ?? 4000,
|
deadlineMs: config.deadlineMs ?? 10000,
|
||||||
maxQueued: config.maxQueued ?? 8,
|
maxQueued: config.maxQueued ?? 8,
|
||||||
maxActive: config.maxActive ?? 2,
|
maxActive: config.maxActive ?? 2,
|
||||||
maxNested: config.maxNested ?? 4,
|
maxNested: config.maxNested ?? 4,
|
||||||
@@ -148,6 +148,11 @@ export async function serveRestrictedRegistry(config, io = process) {
|
|||||||
engines = new Set(),
|
engines = new Set(),
|
||||||
idleEngines = [];
|
idleEngines = [];
|
||||||
const metrics = { starts: 0, reused: 0, completed: 0, peakRSSBytes: 0 };
|
const metrics = { starts: 0, reused: 0, completed: 0, peakRSSBytes: 0 };
|
||||||
|
const changed = () => {
|
||||||
|
try {
|
||||||
|
config.onChange?.();
|
||||||
|
} catch {}
|
||||||
|
};
|
||||||
let buffered = Buffer.alloc(0),
|
let buffered = Buffer.alloc(0),
|
||||||
stopped = false,
|
stopped = false,
|
||||||
failure;
|
failure;
|
||||||
@@ -234,16 +239,19 @@ export async function serveRestrictedRegistry(config, io = process) {
|
|||||||
}
|
}
|
||||||
const engine = call.engine;
|
const engine = call.engine;
|
||||||
if (engine) {
|
if (engine) {
|
||||||
// Revoke the frame token before returning an engine to the idle pool.
|
|
||||||
// A delayed child request can never acquire the next call's authority.
|
|
||||||
if (engine.current === call) engine.current = null;
|
|
||||||
if (!stopped && reason.code === "INVOCATION_ENDED" && !usageFailure && !engine.destroying) {
|
if (!stopped && reason.code === "INVOCATION_ENDED" && !usageFailure && !engine.destroying) {
|
||||||
const rss = (await engineUsage(engine)).memoryBytes;
|
const rss = (await engineUsage(engine)).memoryBytes;
|
||||||
metrics.peakRSSBytes = Math.max(metrics.peakRSSBytes, rss);
|
metrics.peakRSSBytes = Math.max(metrics.peakRSSBytes, rss);
|
||||||
// Bound retained engine heaps as well as lifetime. Hard aggregate
|
// Bound retained engine heaps as well as lifetime. Hard aggregate
|
||||||
// physical memory remains the supervising process/cgroup's contract.
|
// physical memory remains the supervising process/cgroup's contract.
|
||||||
if (rss <= 128 * 1024 * 1024) {
|
// Keep the lease busy through every await. Another artifact can evict
|
||||||
|
// an idle lease immediately; never publish an engine still retiring
|
||||||
|
// or one whose process died while its final usage read was pending.
|
||||||
|
if (rss <= 128 * 1024 * 1024 && !stopped && !engine.destroying && engine.current === call) {
|
||||||
|
// Revoke the frame token atomically with publishing idle capacity.
|
||||||
|
engine.current = null;
|
||||||
idleEngines.push(engine);
|
idleEngines.push(engine);
|
||||||
|
engine.idleUntil = Date.now() + limits.engineIdleMs;
|
||||||
engine.idleTimer = setTimeout(() => void destroyEngine(engine).catch(fail), limits.engineIdleMs);
|
engine.idleTimer = setTimeout(() => void destroyEngine(engine).catch(fail), limits.engineIdleMs);
|
||||||
} else await destroyEngine(engine);
|
} else await destroyEngine(engine);
|
||||||
} else await destroyEngine(engine);
|
} else await destroyEngine(engine);
|
||||||
@@ -300,6 +308,7 @@ export async function serveRestrictedRegistry(config, io = process) {
|
|||||||
} finally {
|
} finally {
|
||||||
engines.delete(engine);
|
engines.delete(engine);
|
||||||
await engine.release?.();
|
await engine.release?.();
|
||||||
|
changed();
|
||||||
}
|
}
|
||||||
})();
|
})();
|
||||||
return engine.destroying;
|
return engine.destroying;
|
||||||
@@ -418,6 +427,7 @@ export async function serveRestrictedRegistry(config, io = process) {
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
metrics.starts++;
|
metrics.starts++;
|
||||||
|
changed();
|
||||||
engine.exited = new Promise((resolve) => {
|
engine.exited = new Promise((resolve) => {
|
||||||
engine.child.once("exit", resolve);
|
engine.child.once("exit", resolve);
|
||||||
engine.child.once("error", resolve);
|
engine.child.once("error", resolve);
|
||||||
@@ -462,6 +472,9 @@ export async function serveRestrictedRegistry(config, io = process) {
|
|||||||
executionMs: call.executionMs ?? null,
|
executionMs: call.executionMs ?? null,
|
||||||
totalMs: performance.now() - call.receivedAt,
|
totalMs: performance.now() - call.receivedAt,
|
||||||
};
|
};
|
||||||
|
metrics.lastFailure =
|
||||||
|
reply.status === "failed" && /^[A-Z0-9_]{1,128}$/.test(reply.error?.code) ? reply.error.code : null;
|
||||||
|
changed();
|
||||||
send(call.cancellationReply ?? reply);
|
send(call.cancellationReply ?? reply);
|
||||||
pump();
|
pump();
|
||||||
})());
|
})());
|
||||||
@@ -470,6 +483,7 @@ export async function serveRestrictedRegistry(config, io = process) {
|
|||||||
const { message } = call;
|
const { message } = call;
|
||||||
call.startedAt = performance.now();
|
call.startedAt = performance.now();
|
||||||
active.set(message.invocation, call);
|
active.set(message.invocation, call);
|
||||||
|
changed();
|
||||||
const terminate = (code) => {
|
const terminate = (code) => {
|
||||||
if (!call.open) return;
|
if (!call.open) return;
|
||||||
void finish(call, fixedError(code), {
|
void finish(call, fixedError(code), {
|
||||||
@@ -687,6 +701,9 @@ export async function serveRestrictedRegistry(config, io = process) {
|
|||||||
queued: queued.length,
|
queued: queued.length,
|
||||||
engines: engines.size,
|
engines: engines.size,
|
||||||
idle: idleEngines.length,
|
idle: idleEngines.length,
|
||||||
|
idleDeadline: idleEngines.length
|
||||||
|
? new Date(Math.min(...idleEngines.map((engine) => engine.idleUntil))).toISOString()
|
||||||
|
: null,
|
||||||
}),
|
}),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -307,6 +307,47 @@ test("the checked portable SDK keeps codecs and nested graph ports inside the fr
|
|||||||
await f.close();
|
await f.close();
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
test("checked handlers hash canonical values without ambient capabilities", async () => {
|
||||||
|
const sdk = process.env.QUIXOS_TEST_SDK;
|
||||||
|
const C = await import(sdk);
|
||||||
|
const workspace = C.workspaceId("workspace:restricted");
|
||||||
|
const text = { kind: "scalar", name: "string" };
|
||||||
|
const value = {
|
||||||
|
kind: "record",
|
||||||
|
fields: {
|
||||||
|
text,
|
||||||
|
options: {
|
||||||
|
kind: "record",
|
||||||
|
fields: {
|
||||||
|
level: { kind: "scalar", name: "int32" },
|
||||||
|
enabled: { kind: "scalar", name: "bool" },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const declarations = [
|
||||||
|
{ id: "test", name: "test", effect: "snapshot", receiver: null, input: value, output: text, methods: [] },
|
||||||
|
];
|
||||||
|
const source = `import {definePortableRegistry,hashJson} from 'qx:core'; export const registry=definePortableRegistry(${JSON.stringify(declarations)},{test:ctx=>hashJson('test-content',ctx.input)});`;
|
||||||
|
const f = await fixture(source, { sdk: [{ name: "qx:core", path: sdk }] });
|
||||||
|
try {
|
||||||
|
const input = { text: "雨后的书店", options: { level: 3, enabled: true } };
|
||||||
|
let expected;
|
||||||
|
for (const candidate of [input, { options: input.options, text: input.text }, { ...input, text: "別の文" }]) {
|
||||||
|
f.start(C.encodeArgument(value, candidate, { workspace }));
|
||||||
|
const result = await f.receive();
|
||||||
|
assert.equal(result.status, "returned");
|
||||||
|
const actual = C.decodeArgument(text, result.output, { workspace });
|
||||||
|
assert.equal(actual, await C.hashJson("test-content", candidate));
|
||||||
|
if (candidate.text === input.text) {
|
||||||
|
expected ??= actual;
|
||||||
|
assert.equal(actual, expected, "Object key order must not invalidate content caches");
|
||||||
|
} else assert.notEqual(actual, expected, "Content changes must invalidate the key");
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
await f.close();
|
||||||
|
}
|
||||||
|
});
|
||||||
test("ambient clock, entropy, network and native imports are denied", async () => {
|
test("ambient clock, entropy, network and native imports are denied", async () => {
|
||||||
for (const code of [
|
for (const code of [
|
||||||
"Date.now()",
|
"Date.now()",
|
||||||
@@ -529,12 +570,25 @@ test("an expired frame token cannot borrow a later invocation's broker", async (
|
|||||||
});
|
});
|
||||||
|
|
||||||
test("idle engines evict without a graph poll and restart with fresh state", async () => {
|
test("idle engines evict without a graph poll and restart with fresh state", async () => {
|
||||||
const f = await fixture(registry("return request.input;"), { engineIdleMs: 30 });
|
const updates = [];
|
||||||
|
const f = await fixture(registry("return request.input;"), {
|
||||||
|
engineIdleMs: 30,
|
||||||
|
onChange: () => updates.push(f.stats()),
|
||||||
|
});
|
||||||
try {
|
try {
|
||||||
f.start("first");
|
f.start("first");
|
||||||
assert.equal((await f.receive()).output, "first");
|
assert.equal((await f.receive()).output, "first");
|
||||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||||
assert.equal(f.stats().engines, 0);
|
assert.equal(f.stats().engines, 0);
|
||||||
|
assert.ok(
|
||||||
|
updates.some((s) => s.active > 0),
|
||||||
|
"Inspectors observe active invocation admission",
|
||||||
|
);
|
||||||
|
assert.ok(
|
||||||
|
updates.some((s) => s.idle > 0 && s.idleDeadline),
|
||||||
|
"Idle eviction deadline is visible",
|
||||||
|
);
|
||||||
|
assert.equal(updates.at(-1).engines, 0, "Eviction pushes an update without a graph poll");
|
||||||
f.start("second");
|
f.start("second");
|
||||||
assert.equal((await f.receive()).output, "second");
|
assert.equal((await f.receive()).output, "second");
|
||||||
assert.equal(f.stats().starts, 2);
|
assert.equal(f.stats().starts, 2);
|
||||||
@@ -632,3 +686,39 @@ test("the shared engine budget evicts idle code families and rejects busy oversu
|
|||||||
}
|
}
|
||||||
assert.equal(engineBudget.stats().engines, 0);
|
assert.equal(engineBudget.stats().engines, 0);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test("engine retirement accounting keeps its admission lease busy", async () => {
|
||||||
|
const observations = [];
|
||||||
|
let idle;
|
||||||
|
const engineBudget = new RestrictedEngineBudget(1, async () => ({
|
||||||
|
usage: async () => {
|
||||||
|
observations.push(idle());
|
||||||
|
await new Promise((done) => setImmediate(done));
|
||||||
|
return { cpuMicroseconds: 0, memoryBytes: 1024 };
|
||||||
|
},
|
||||||
|
release: async () => {},
|
||||||
|
}));
|
||||||
|
const acquire = engineBudget.acquire.bind(engineBudget);
|
||||||
|
engineBudget.acquire = (isIdle, evict) => {
|
||||||
|
idle = isIdle;
|
||||||
|
return acquire(isIdle, evict);
|
||||||
|
};
|
||||||
|
const f = await fixture(registry("return request.input;"), { engineBudget });
|
||||||
|
try {
|
||||||
|
f.start("first");
|
||||||
|
assert.equal((await f.receive()).output, "first");
|
||||||
|
assert.ok(observations.length > 0);
|
||||||
|
assert.ok(
|
||||||
|
observations.every((value) => !value),
|
||||||
|
"An engine must not become evictable during asynchronous retirement accounting",
|
||||||
|
);
|
||||||
|
assert.equal(f.stats().idle, 1);
|
||||||
|
assert.equal(idle(), true);
|
||||||
|
f.start("second");
|
||||||
|
assert.equal((await f.receive()).output, "second");
|
||||||
|
assert.equal(f.stats().starts, 1);
|
||||||
|
} finally {
|
||||||
|
await f.close();
|
||||||
|
}
|
||||||
|
assert.equal(engineBudget.stats().engines, 0);
|
||||||
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user