Use a combined cache derivation by default

This commit is contained in:
Stéphan Kochen
2024-02-06 11:00:23 +01:00
parent 67fadc3f61
commit df1c72a4f6
13 changed files with 328 additions and 146 deletions
+7 -2
View File
@@ -79,8 +79,8 @@ jobs:
- name: Test nix-build
run: |
# This delete tests refetching a package with Nix.
nix-store --delete /nix/store/*--yarnpkg-core-npm-*
# This delete tests refetching the cache with Nix.
nix-store --delete /nix/store/*-yarn-cache
nix-build
- name: Test bin
@@ -91,6 +91,7 @@ jobs:
- name: Test isolated builds
run: |
# Matches example in ISOLATED_BUILDS.md
echo 'individualNixPackaging: true' >> .yarnrc.yml
echo 'isolatedNixBuilds: ["sqlite3"]' >> .yarnrc.yml
cat > default.nix << EOF
{ pkgs ? import <nixpkgs> { } }:
@@ -105,4 +106,8 @@ jobs:
EOF
yarn add sqlite3
# This delete tests refetching a package with Nix.
nix-store --delete /nix/store/*--yarnpkg-core-npm-*
nix-build
+3
View File
@@ -9,9 +9,12 @@ As an example, to create an isolated build of sqlite3, add the following to
your `.yarnrc.yml`:
```yml
individualNixPackaging: true
isolatedNixBuilds: ["sqlite3"]
```
(`individualNixPackaging` is required to use `isolatedNixBuilds`.)
In your Nix expression, separate options can be set to override attributes of
these derivations, which is often necessary to provide build inputs. For
sqlite3, you'd do the following in your `default.nix`:
+4 -3
View File
@@ -14,9 +14,6 @@ zero-install).
- A default install-phase that creates executables for you based on `"bin"` in
your `package.json`, making your package readily installable.
- Granular fetching of dependencies in Nix, speeding up rebuilds and
potentially allowing downloads to be shared between projects.
- Preloading of your Yarn cache into the Nix store, speeding up local
`nix-build`.
@@ -131,6 +128,10 @@ Some additional settings are available in `.yarnrc.yml`:
Preloading does mean another copy of dependencies on disk, even if you don't
do local Nix builds, but the size is usually not an issue on modern disks.
- `individualNixPackaging` can be set to `true` to create a derivation per
dependency. This is useful in some cases, but can significantly slow down
builds if cache is cold and preloading is not possible.
- `isolatedNixBuilds`, see [ISOLATED_BUILDS.md](./ISOLATED_BUILDS.md).
- `installNixBinariesForDependencies` can be set to also install executables
+2 -2
View File
@@ -91,10 +91,10 @@ const compiler = webpack({
`return plugin;`,
`}`,
`};`,
].join(`\n`)
].join(`\n`),
);
}
}
},
);
});
},
+1 -1
View File
@@ -14,7 +14,7 @@
"@babel/plugin-proposal-class-properties": "^7.10.4",
"@babel/plugin-proposal-decorators": "^7.10.5",
"@babel/preset-typescript": "^7.10.4",
"@types/node": "10",
"@types/node": "18",
"@yarnpkg/cli": "^4.0.0",
"@yarnpkg/core": "^4.0.0",
"@yarnpkg/fslib": "^3.0.0",
+58
View File
@@ -0,0 +1,58 @@
import { Command, Option } from "clipanion";
import {
Cache,
CommandContext,
Configuration,
Project,
StreamReport,
structUtils,
} from "@yarnpkg/core";
// Internal command that performs the fetch step.
// Used from within Nix to build the cache for the project.
export default class FetchCommand extends Command<CommandContext> {
static paths = [[`nixify`, `fetch`]];
locator = Option.String({ required: false });
async execute() {
const configuration = await Configuration.find(
this.context.cwd,
this.context.plugins,
);
const { project } = await Project.find(configuration, this.context.cwd);
const cache = await Cache.find(configuration);
const fetcher = configuration.makeFetcher();
const report = await StreamReport.start(
{ configuration, stdout: this.context.stdout },
async (report) => {
if (this.locator) {
const { locatorHash } = structUtils.parseLocator(this.locator, true);
const pkg = project.originalPackages.get(locatorHash);
if (!pkg) {
report.reportError(0, `Invalid locator: ${this.locator}`);
return;
}
await fetcher.fetch(pkg, {
checksums: project.storedChecksums,
project,
cache,
fetcher,
report,
});
} else {
await report.startTimerPromise(`Resolution step`, async () => {
await project.resolveEverything({ report, lockfileOnly: true });
});
await report.startTimerPromise(`Fetch step`, async () => {
await project.fetchEverything({ cache, report, fetcher });
});
}
},
);
return report.exitCode();
}
}
-50
View File
@@ -1,50 +0,0 @@
import { Command, Option } from "clipanion";
import {
Cache,
CommandContext,
Configuration,
Project,
StreamReport,
structUtils,
} from "@yarnpkg/core";
// Internal command that fetches a single locator.
// Used from within Nix to build the cache for the project.
export default class FetchOneCommand extends Command<CommandContext> {
static paths = [[`nixify`, `fetch-one`]];
locator = Option.String();
async execute() {
const configuration = await Configuration.find(
this.context.cwd,
this.context.plugins,
);
const { project } = await Project.find(configuration, this.context.cwd);
const cache = await Cache.find(configuration);
const fetcher = configuration.makeFetcher();
const report = await StreamReport.start(
{ configuration, stdout: this.context.stdout },
async (report) => {
const { locatorHash } = structUtils.parseLocator(this.locator, true);
const pkg = project.originalPackages.get(locatorHash);
if (!pkg) {
report.reportError(0, `Invalid locator: ${this.locator}`);
return;
}
await fetcher.fetch(pkg, {
checksums: project.storedChecksums,
project,
cache,
fetcher,
report,
});
},
);
return report.exitCode();
}
}
+140 -65
View File
@@ -20,8 +20,12 @@ import defaultExprTmpl from "./tmpl/default.nix.in";
import projectExprTmpl from "./tmpl/yarn-project.nix.in";
import {
computeFixedOutputStorePath,
hexToSri,
sanitizeDerivationName,
sriToHex,
} from "./nixUtils";
import { writeNarStream, writeNarStrings } from "./narUtils";
import { createHash } from "crypto";
const isYarn3 = YarnVersion?.startsWith("3.") || false;
@@ -48,6 +52,13 @@ export default async (
return;
}
// Config validation.
const isolatedBuilds = configuration.get(`isolatedNixBuilds`);
const individualDrvs = configuration.get(`individualNixPackaging`);
if (isolatedBuilds.length > 0 && !individualDrvs) {
throw Error(`isolatedNixBuilds requires individualNixPackaging to be set`);
}
// Determine relative paths for Nix path literals.
const nixExprPath = configuration.get(`nixExprPath`);
@@ -55,14 +66,13 @@ export default async (
let yarnBinExpr: string;
if (yarnPathAbs === null) {
// Assume the current running script is the correct Yarn.
const hexHash = await hashUtils.checksumFile(
const sha512 = await hashUtils.checksumFile(
process.argv[1] as PortablePath,
);
const sriHash = "sha512-" + Buffer.from(hexHash, "hex").toString("base64");
yarnBinExpr = [
"fetchurl {",
` url = "https://repo.yarnpkg.com/${YarnVersion!}/packages/yarnpkg-cli/bin/yarn.js";`,
` hash = "${sriHash}";`,
` hash = "${hexToSri(sha512)}";`,
"}",
].join("\n ");
} else if (yarnPathAbs.startsWith(cwd)) {
@@ -113,16 +123,14 @@ export default async (
ppath.resolve(cwd, "yarn.lock" as PortablePath),
);
// Build a list of cache entries so Nix can fetch them.
// TODO: See if we can use Nix fetchurl for npm: dependencies.
interface CacheEntry {
originalFilename: Filename;
filename: Filename;
sha512: string;
// Collect all the cache files used.
interface CacheFile {
pkg: Package;
checksum: string | undefined;
cachePath: PortablePath;
}
const cacheEntries: Map<string, CacheEntry> = new Map();
const cacheFiles = new Set(await xfs.readdirPromise(cache.cwd));
const cacheFiles = new Map<Filename, CacheFile>();
const allCacheFiles = new Set(await xfs.readdirPromise(cache.cwd));
const cacheOptions = { unstablePackages: project.conditionalLocators };
for (const pkg of project.storedPackages.values()) {
const { locatorHash } = pkg;
@@ -132,36 +140,83 @@ export default async (
: cache.getLocatorPath(pkg, checksum || null);
if (!cachePath) continue;
const filename = ppath.basename(cachePath);
if (!cacheFiles.has(filename)) continue;
if (!allCacheFiles.has(ppath.basename(cachePath))) continue;
const locatorStr = structUtils.stringifyLocator(pkg);
const sha512 = checksum
? checksum.split(`/`).pop()!
: await hashUtils.checksumFile(cachePath);
cacheEntries.set(locatorStr, {
originalFilename: filename,
// Rebuild the filename, because the cache file we're operating on may be
// from the mirror directory, which uses different naming.
filename: checksum
? cache.getChecksumFilename(pkg, checksum)
: cache.getVersionFilename(pkg),
sha512,
});
// Rebuild the filename, because the cache file we're operating on may be
// from the mirror directory, which uses different naming.
const filename = checksum
? cache.getChecksumFilename(pkg, checksum)
: cache.getVersionFilename(pkg);
cacheFiles.set(filename, { pkg, checksum, cachePath });
}
let cacheEntriesCode = `cacheEntries = {\n`;
for (const locatorStr of [...cacheEntries.keys()].sort()) {
const entry = cacheEntries.get(locatorStr)!;
cacheEntriesCode += `${json(locatorStr)} = { ${[
`filename = ${json(entry.filename)};`,
`sha512 = ${json(entry.sha512)};`,
].join(` `)} };\n`;
interface CacheEntry {
cachePath: PortablePath;
filename: Filename;
hash: string;
}
const cacheEntries = new Map<string, CacheEntry>();
let cacheEntriesCode = "";
let combinedHash = "";
if (individualDrvs) {
// Build a list of cache entries so Nix can fetch them.
// TODO: See if we can use Nix fetchurl for npm: dependencies.
for (const [
filename,
{ pkg, checksum, cachePath },
] of cacheFiles.entries()) {
const locatorStr = structUtils.stringifyLocator(pkg);
const sha512 = checksum
? checksum.split(`/`).pop()!
: await hashUtils.checksumFile(cachePath);
cacheEntries.set(locatorStr, {
cachePath,
filename,
hash: hexToSri(sha512),
});
}
cacheEntriesCode = `cacheEntries = {\n`;
for (const locatorStr of [...cacheEntries.keys()].sort()) {
const entry = cacheEntries.get(locatorStr)!;
cacheEntriesCode += `${json(locatorStr)} = { ${[
`filename = ${json(entry.filename)};`,
`hash = "${entry.hash}";`,
].join(` `)} };\n`;
}
cacheEntriesCode += `};`;
} else {
// Hash a NAR of just the cache files we use.
const hasher = createHash("sha512");
writeNarStrings(hasher, "nix-archive-1", "(", "type", "directory");
for (const filename of [...cacheFiles.keys()].sort()) {
const { cachePath } = cacheFiles.get(filename)!;
const { size } = await xfs.statPromise(cachePath);
writeNarStrings(
hasher,
"entry",
"(",
"name",
filename,
"node",
"(",
"type",
"regular",
"contents",
);
await writeNarStream(hasher, size, xfs.createReadStream(cachePath));
writeNarStrings(hasher, ")", ")");
}
writeNarStrings(hasher, ")");
hasher.end();
// Bit hacky, but hashers always produce a single read.
for await (const sha512 of hasher) {
combinedHash = hexToSri(sha512);
}
}
cacheEntriesCode += `};`;
// Generate Nix code for isolated builds.
const isolatedBuilds = configuration.get(`isolatedNixBuilds`);
let isolatedPackages = new Set<Package>();
let isolatedIntegration = [];
let isolatedCode = [];
@@ -320,6 +375,9 @@ export default async (
PROJECT_NAME: json(projectName),
YARN_BIN: yarnBinExpr,
LOCKFILE: lockfileExpr,
INDIVIDUAL_DRVS: individualDrvs,
COMBINED_DRV: !individualDrvs,
COMBINED_HASH: combinedHash,
CACHE_FOLDER: cacheFolderExpr,
CACHE_ENTRIES: cacheEntriesCode,
ISOLATED: isolatedCode.join("\n"),
@@ -350,30 +408,49 @@ export default async (
xfs.existsSync(npath.toPortablePath(`/nix/store`))
) {
await xfs.mktempPromise(async (tempDir) => {
const args = ["--add-fixed", "sha512"];
const toPreload: PortablePath[] = [];
for (const [
locator,
{ originalFilename, sha512 },
] of cacheEntries.entries()) {
const name = sanitizeDerivationName(locator);
if (individualDrvs) {
for (const [locator, { cachePath, hash }] of cacheEntries.entries()) {
const name = sanitizeDerivationName(locator);
// Check to see if the Nix store entry already exists.
const storePath = computeFixedOutputStorePath(name, hash);
if (!xfs.existsSync(storePath)) {
// The nix-store command requires a correct filename on disk, so we
// prepare a temporary directory containing all the files to preload.
//
// Because some names may conflict (e.g. 'typescript-npm-xyz' and
// 'typescript-patch-xyz' both have the same derivation name), we
// create subdirectories based on hash.
const subdir = ppath.join(
tempDir,
sriToHex(hash).slice(0, 7) as Filename,
);
await xfs.mkdirPromise(subdir);
const dst = ppath.join(subdir, name as Filename);
await xfs.copyFilePromise(cachePath, dst);
toPreload.push(dst);
}
}
} else {
args.unshift("--recursive");
// Check to see if the Nix store entry already exists.
const hash = Buffer.from(sha512, "hex");
const storePath = computeFixedOutputStorePath(name, `sha512`, hash);
const storePath = computeFixedOutputStorePath(
"yarn-cache",
combinedHash,
{ recursive: true },
);
if (!xfs.existsSync(storePath)) {
// The nix-store command requires a correct filename on disk, so we
// prepare a temporary directory containing all the files to preload.
//
// Because some names may conflict (e.g. 'typescript-npm-xyz' and
// 'typescript-patch-xyz' both have the same derivation name), we
// create subdirectories based on hash.
const subdir = ppath.join(tempDir, sha512.slice(0, 7) as Filename);
// Same as above, nix-store requires a correct filename.
const subdir = ppath.join(tempDir, "yarn-cache");
await xfs.mkdirPromise(subdir);
const src = ppath.join(cache.cwd, originalFilename);
const dst = ppath.join(subdir, name as Filename);
await xfs.copyFilePromise(src, dst);
toPreload.push(dst);
for (const [filename, { cachePath }] of cacheFiles.entries()) {
const dst = ppath.join(subdir, filename);
await xfs.copyFilePromise(cachePath, dst);
}
toPreload.push(subdir);
}
}
@@ -382,19 +459,17 @@ export default async (
const numToPreload = toPreload.length;
while (toPreload.length !== 0) {
const batch = toPreload.splice(0, 100);
await execUtils.execvp(
"nix-store",
["--add-fixed", "sha512", ...batch],
{
cwd: project.cwd,
strict: true,
},
);
await execUtils.execvp("nix-store", [...args, ...batch], {
cwd: project.cwd,
strict: true,
});
}
if (numToPreload !== 0) {
report.reportInfo(
0,
`Preloaded ${numToPreload} packages into the Nix store`,
individualDrvs
? `Preloaded ${numToPreload} packages into the Nix store`
: `Preloaded cache into the Nix store`,
);
}
} catch (err: any) {
+8 -2
View File
@@ -1,7 +1,7 @@
import { Hooks, Plugin, SettingsType } from "@yarnpkg/core";
import { PortablePath } from "@yarnpkg/fslib";
import FetchOneCommand from "./FetchOneCommand";
import FetchCommand from "./FetchCommand";
import InjectBuildCommand from "./InjectBuildCommand";
import InstallBinCommand from "./InstallBinCommand";
import generate from "./generate";
@@ -12,13 +12,14 @@ declare module "@yarnpkg/core" {
nixExprPath: PortablePath;
generateDefaultNix: boolean;
enableNixPreload: boolean;
individualNixPackaging: boolean;
isolatedNixBuilds: string[];
installNixBinariesForDependencies: boolean;
}
}
const plugin: Plugin<Hooks> = {
commands: [FetchOneCommand, InjectBuildCommand, InstallBinCommand],
commands: [FetchCommand, InjectBuildCommand, InstallBinCommand],
hooks: {
afterAllInstalled: async (project, opts) => {
if (
@@ -50,6 +51,11 @@ const plugin: Plugin<Hooks> = {
type: SettingsType.BOOLEAN,
default: true,
},
individualNixPackaging: {
description: `If true, generate one Nix derivation per package. If false, use a single derivation for the entire cache folder.`,
type: SettingsType.BOOLEAN,
default: false,
},
isolatedNixBuilds: {
description: `Dependencies with a build step that can be built in an isolated derivation`,
type: SettingsType.STRING,
+51
View File
@@ -0,0 +1,51 @@
import type { Readable, Writable } from "stream";
const MAX_UINT32 = 2 ** 32 - 1;
/** Write one or more NAR strings to the output. */
export const writeNarStrings = (out: Writable, ...input: string[]) => {
let size = 0;
const bufs = input.map((str) => {
const buf = Buffer.from(str);
// TODO: Support up to MAX_SAFE_INTEGER
if (buf.byteLength > MAX_UINT32) {
throw Error(`NAR string too long: ${buf.byteLength}`);
}
size += 8 + Math.ceil(buf.byteLength / 8) * 8;
return buf;
});
const res = Buffer.alloc(size);
let pos = 0;
for (const buf of bufs) {
res.writeUInt32LE(buf.byteLength, pos);
buf.copy(res, pos + 8);
pos += 8 + Math.ceil(buf.byteLength / 8) * 8;
}
out.write(res);
};
/** Write the contents of a stream as a NAR string to the output. */
export const writeNarStream = async (
out: Writable,
size: number,
input: Readable,
) => {
// TODO: Support up to MAX_SAFE_INTEGER
if (size > MAX_UINT32) {
throw Error(`NAR string too long: ${size}`);
}
const sizeBuf = Buffer.alloc(8);
sizeBuf.writeUInt32LE(size);
out.write(sizeBuf);
for await (const chunk of input) {
out.write(chunk);
}
const padding = 8 - (size % 8);
if (padding !== 8) {
out.write(Buffer.alloc(padding));
}
};
+17 -5
View File
@@ -44,13 +44,17 @@ export const encodeBase32 = (buf: Buffer) => {
*/
export const computeFixedOutputStorePath = (
name: string,
hashAlgorithm: string,
hash: Buffer,
storePath = `/nix/store` as PortablePath,
hash: string,
{
storePath = `/nix/store` as PortablePath,
recursive = false,
}: { storePath?: PortablePath; recursive?: boolean } = {},
) => {
const hashHex = hash.toString("hex");
const [hashAlgorithm, hash64] = hash.split("-");
const hashHex = Buffer.from(hash64, "base64").toString("hex");
const innerStr = `fixed:out:${hashAlgorithm}:${hashHex}:`;
const rec = recursive ? "r:" : "";
const innerStr = `fixed:out:${rec}${hashAlgorithm}:${hashHex}:`;
const innerHash = computeHash(`sha256`, innerStr);
const innerHashHex = innerHash.toString("hex");
@@ -71,3 +75,11 @@ export const sanitizeDerivationName = (name: string) =>
.replace(/^\.+/, "")
.replace(/[^a-zA-Z0-9+._?=-]+/g, "-")
.slice(0, 207) || "unknown";
/** Convert a hexadecimal hash to an SRI hash. */
export const hexToSri = (hash: string, algorithm = "sha512") =>
algorithm + "-" + Buffer.from(hash, "hex").toString("base64");
/** Convert an SRI hash to a hexadecimal hash. */
export const sriToHex = (hash: string) =>
Buffer.from(hash.split("-")[1], "base64").toString("hex");
+30 -11
View File
@@ -38,25 +38,38 @@ let
export yarn_enable_nixify=false
'';
#@@ IF COMBINED_DRV
cacheDrv = stdenv.mkDerivation {
name = "yarn-cache";
buildInputs = [ yarn git cacert ];
buildCommand = ''
cp --reflink=auto --recursive '${src}' ./src
cd ./src/
${buildVars}
HOME="$TMP" yarn_enable_global_cache=false yarn_cache_folder="$out" \
yarn nixify fetch
rm $out/.gitignore
'';
outputHashMode = "recursive";
outputHash = "@@COMBINED_HASH@@";
};
#@@ ENDIF COMBINED_DRV
#@@ IF INDIVIDUAL_DRVS
# Create derivations for fetching dependencies.
cacheDrvs = let
builder = writeShellScript "yarn-cache-builder" ''
source $stdenv/setup
cd "$src"
in lib.mapAttrs (locator: { filename, hash }: stdenv.mkDerivation {
name = lib.strings.sanitizeDerivationName locator;
buildInputs = [ yarn git cacert ];
buildCommand = ''
cd '${src}'
${buildVars}
HOME="$TMP" yarn_enable_global_cache=false yarn_cache_folder="$TMP" \
yarn nixify fetch-one $locator
yarn nixify fetch ${locator}
# Because we change the cache dir, Yarn may generate a different name.
mv "$TMP/$(sed 's/-[^-]*\.[^-]*$//' <<< "$outputFilename")"-* $out
'';
in lib.mapAttrs (locator: { filename, sha512 }: stdenv.mkDerivation {
inherit src builder locator;
name = lib.strings.sanitizeDerivationName locator;
buildInputs = [ yarn git cacert ];
outputFilename = filename;
outputHashMode = "flat";
outputHashAlgo = "sha512";
outputHash = sha512;
outputHash = hash;
}) cacheEntries;
# Create a shell snippet to copy dependencies from a list of derivations.
@@ -64,6 +77,7 @@ let
writeShellScript "collect-yarn-cache" (lib.concatMapStrings (drv: ''
cp --reflink=auto ${drv} '${drv.outputFilename}'
'') drvs);
#@@ ENDIF INDIVIDUAL_DRVS
#@@ IF NEED_ISOLATED_BUILD_SUPPRORT
# Create a shell snippet to copy dependencies from a list of locators.
@@ -119,9 +133,14 @@ let
# Copy over the Yarn cache.
rm -fr '${cacheFolder}'
mkdir -p '${cacheFolder}'
#@@ IF COMBINED_DRV
cp --reflink=auto --recursive ${cacheDrv}/* '${cacheFolder}/'
#@@ ENDIF COMBINED_DRV
#@@ IF INDIVIDUAL_DRVS
pushd '${cacheFolder}' > /dev/null
source ${mkCacheBuilderForDrvs (lib.attrValues cacheDrvs)}
popd > /dev/null
#@@ ENDIF INDIVIDUAL_DRVS
# Yarn may need a writable home directory.
export yarn_global_folder="$TMP"
+7 -5
View File
@@ -731,10 +731,12 @@ __metadata:
languageName: node
linkType: hard
"@types/node@npm:10":
version: 10.17.60
resolution: "@types/node@npm:10.17.60"
checksum: 0742294912a6e79786cdee9ed77cff6ee8ff007b55d8e21170fc3e5994ad3a8101fea741898091876f8dc32b0a5ae3d64537b7176799e92da56346028d2cbcd2
"@types/node@npm:18":
version: 18.19.14
resolution: "@types/node@npm:18.19.14"
dependencies:
undici-types: "npm:~5.26.4"
checksum: a614e85d3d75e9bb73c1155f77843804506806629d20774e0479e88678efd64ab57f7bb2ba290b226fe732b6aead037ac181db04851a2f37e1c2a14744ffdb96
languageName: node
linkType: hard
@@ -3665,7 +3667,7 @@ __metadata:
"@babel/plugin-proposal-class-properties": "npm:^7.10.4"
"@babel/plugin-proposal-decorators": "npm:^7.10.5"
"@babel/preset-typescript": "npm:^7.10.4"
"@types/node": "npm:10"
"@types/node": "npm:18"
"@yarnpkg/cli": "npm:^4.0.0"
"@yarnpkg/core": "npm:^4.0.0"
"@yarnpkg/fslib": "npm:^3.0.0"