Use a combined cache derivation by default
This commit is contained in:
@@ -79,8 +79,8 @@ jobs:
|
|||||||
|
|
||||||
- name: Test nix-build
|
- name: Test nix-build
|
||||||
run: |
|
run: |
|
||||||
# This delete tests refetching a package with Nix.
|
# This delete tests refetching the cache with Nix.
|
||||||
nix-store --delete /nix/store/*--yarnpkg-core-npm-*
|
nix-store --delete /nix/store/*-yarn-cache
|
||||||
nix-build
|
nix-build
|
||||||
|
|
||||||
- name: Test bin
|
- name: Test bin
|
||||||
@@ -91,6 +91,7 @@ jobs:
|
|||||||
- name: Test isolated builds
|
- name: Test isolated builds
|
||||||
run: |
|
run: |
|
||||||
# Matches example in ISOLATED_BUILDS.md
|
# Matches example in ISOLATED_BUILDS.md
|
||||||
|
echo 'individualNixPackaging: true' >> .yarnrc.yml
|
||||||
echo 'isolatedNixBuilds: ["sqlite3"]' >> .yarnrc.yml
|
echo 'isolatedNixBuilds: ["sqlite3"]' >> .yarnrc.yml
|
||||||
cat > default.nix << EOF
|
cat > default.nix << EOF
|
||||||
{ pkgs ? import <nixpkgs> { } }:
|
{ pkgs ? import <nixpkgs> { } }:
|
||||||
@@ -105,4 +106,8 @@ jobs:
|
|||||||
EOF
|
EOF
|
||||||
|
|
||||||
yarn add sqlite3
|
yarn add sqlite3
|
||||||
|
|
||||||
|
# This delete tests refetching a package with Nix.
|
||||||
|
nix-store --delete /nix/store/*--yarnpkg-core-npm-*
|
||||||
|
|
||||||
nix-build
|
nix-build
|
||||||
|
|||||||
@@ -9,9 +9,12 @@ As an example, to create an isolated build of sqlite3, add the following to
|
|||||||
your `.yarnrc.yml`:
|
your `.yarnrc.yml`:
|
||||||
|
|
||||||
```yml
|
```yml
|
||||||
|
individualNixPackaging: true
|
||||||
isolatedNixBuilds: ["sqlite3"]
|
isolatedNixBuilds: ["sqlite3"]
|
||||||
```
|
```
|
||||||
|
|
||||||
|
(`individualNixPackaging` is required to use `isolatedNixBuilds`.)
|
||||||
|
|
||||||
In your Nix expression, separate options can be set to override attributes of
|
In your Nix expression, separate options can be set to override attributes of
|
||||||
these derivations, which is often necessary to provide build inputs. For
|
these derivations, which is often necessary to provide build inputs. For
|
||||||
sqlite3, you'd do the following in your `default.nix`:
|
sqlite3, you'd do the following in your `default.nix`:
|
||||||
|
|||||||
@@ -14,9 +14,6 @@ zero-install).
|
|||||||
- A default install-phase that creates executables for you based on `"bin"` in
|
- A default install-phase that creates executables for you based on `"bin"` in
|
||||||
your `package.json`, making your package readily installable.
|
your `package.json`, making your package readily installable.
|
||||||
|
|
||||||
- Granular fetching of dependencies in Nix, speeding up rebuilds and
|
|
||||||
potentially allowing downloads to be shared between projects.
|
|
||||||
|
|
||||||
- Preloading of your Yarn cache into the Nix store, speeding up local
|
- Preloading of your Yarn cache into the Nix store, speeding up local
|
||||||
`nix-build`.
|
`nix-build`.
|
||||||
|
|
||||||
@@ -131,6 +128,10 @@ Some additional settings are available in `.yarnrc.yml`:
|
|||||||
Preloading does mean another copy of dependencies on disk, even if you don't
|
Preloading does mean another copy of dependencies on disk, even if you don't
|
||||||
do local Nix builds, but the size is usually not an issue on modern disks.
|
do local Nix builds, but the size is usually not an issue on modern disks.
|
||||||
|
|
||||||
|
- `individualNixPackaging` can be set to `true` to create a derivation per
|
||||||
|
dependency. This is useful in some cases, but can significantly slow down
|
||||||
|
builds if cache is cold and preloading is not possible.
|
||||||
|
|
||||||
- `isolatedNixBuilds`, see [ISOLATED_BUILDS.md](./ISOLATED_BUILDS.md).
|
- `isolatedNixBuilds`, see [ISOLATED_BUILDS.md](./ISOLATED_BUILDS.md).
|
||||||
|
|
||||||
- `installNixBinariesForDependencies` can be set to also install executables
|
- `installNixBinariesForDependencies` can be set to also install executables
|
||||||
|
|||||||
@@ -91,10 +91,10 @@ const compiler = webpack({
|
|||||||
`return plugin;`,
|
`return plugin;`,
|
||||||
`}`,
|
`}`,
|
||||||
`};`,
|
`};`,
|
||||||
].join(`\n`)
|
].join(`\n`),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
},
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
|
|||||||
+1
-1
@@ -14,7 +14,7 @@
|
|||||||
"@babel/plugin-proposal-class-properties": "^7.10.4",
|
"@babel/plugin-proposal-class-properties": "^7.10.4",
|
||||||
"@babel/plugin-proposal-decorators": "^7.10.5",
|
"@babel/plugin-proposal-decorators": "^7.10.5",
|
||||||
"@babel/preset-typescript": "^7.10.4",
|
"@babel/preset-typescript": "^7.10.4",
|
||||||
"@types/node": "10",
|
"@types/node": "18",
|
||||||
"@yarnpkg/cli": "^4.0.0",
|
"@yarnpkg/cli": "^4.0.0",
|
||||||
"@yarnpkg/core": "^4.0.0",
|
"@yarnpkg/core": "^4.0.0",
|
||||||
"@yarnpkg/fslib": "^3.0.0",
|
"@yarnpkg/fslib": "^3.0.0",
|
||||||
|
|||||||
@@ -0,0 +1,58 @@
|
|||||||
|
import { Command, Option } from "clipanion";
|
||||||
|
|
||||||
|
import {
|
||||||
|
Cache,
|
||||||
|
CommandContext,
|
||||||
|
Configuration,
|
||||||
|
Project,
|
||||||
|
StreamReport,
|
||||||
|
structUtils,
|
||||||
|
} from "@yarnpkg/core";
|
||||||
|
|
||||||
|
// Internal command that performs the fetch step.
|
||||||
|
// Used from within Nix to build the cache for the project.
|
||||||
|
export default class FetchCommand extends Command<CommandContext> {
|
||||||
|
static paths = [[`nixify`, `fetch`]];
|
||||||
|
locator = Option.String({ required: false });
|
||||||
|
|
||||||
|
async execute() {
|
||||||
|
const configuration = await Configuration.find(
|
||||||
|
this.context.cwd,
|
||||||
|
this.context.plugins,
|
||||||
|
);
|
||||||
|
const { project } = await Project.find(configuration, this.context.cwd);
|
||||||
|
const cache = await Cache.find(configuration);
|
||||||
|
|
||||||
|
const fetcher = configuration.makeFetcher();
|
||||||
|
const report = await StreamReport.start(
|
||||||
|
{ configuration, stdout: this.context.stdout },
|
||||||
|
async (report) => {
|
||||||
|
if (this.locator) {
|
||||||
|
const { locatorHash } = structUtils.parseLocator(this.locator, true);
|
||||||
|
const pkg = project.originalPackages.get(locatorHash);
|
||||||
|
if (!pkg) {
|
||||||
|
report.reportError(0, `Invalid locator: ${this.locator}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await fetcher.fetch(pkg, {
|
||||||
|
checksums: project.storedChecksums,
|
||||||
|
project,
|
||||||
|
cache,
|
||||||
|
fetcher,
|
||||||
|
report,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
await report.startTimerPromise(`Resolution step`, async () => {
|
||||||
|
await project.resolveEverything({ report, lockfileOnly: true });
|
||||||
|
});
|
||||||
|
await report.startTimerPromise(`Fetch step`, async () => {
|
||||||
|
await project.fetchEverything({ cache, report, fetcher });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
return report.exitCode();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,50 +0,0 @@
|
|||||||
import { Command, Option } from "clipanion";
|
|
||||||
|
|
||||||
import {
|
|
||||||
Cache,
|
|
||||||
CommandContext,
|
|
||||||
Configuration,
|
|
||||||
Project,
|
|
||||||
StreamReport,
|
|
||||||
structUtils,
|
|
||||||
} from "@yarnpkg/core";
|
|
||||||
|
|
||||||
// Internal command that fetches a single locator.
|
|
||||||
// Used from within Nix to build the cache for the project.
|
|
||||||
export default class FetchOneCommand extends Command<CommandContext> {
|
|
||||||
static paths = [[`nixify`, `fetch-one`]];
|
|
||||||
locator = Option.String();
|
|
||||||
|
|
||||||
async execute() {
|
|
||||||
const configuration = await Configuration.find(
|
|
||||||
this.context.cwd,
|
|
||||||
this.context.plugins,
|
|
||||||
);
|
|
||||||
const { project } = await Project.find(configuration, this.context.cwd);
|
|
||||||
const cache = await Cache.find(configuration);
|
|
||||||
|
|
||||||
const fetcher = configuration.makeFetcher();
|
|
||||||
|
|
||||||
const report = await StreamReport.start(
|
|
||||||
{ configuration, stdout: this.context.stdout },
|
|
||||||
async (report) => {
|
|
||||||
const { locatorHash } = structUtils.parseLocator(this.locator, true);
|
|
||||||
const pkg = project.originalPackages.get(locatorHash);
|
|
||||||
if (!pkg) {
|
|
||||||
report.reportError(0, `Invalid locator: ${this.locator}`);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
await fetcher.fetch(pkg, {
|
|
||||||
checksums: project.storedChecksums,
|
|
||||||
project,
|
|
||||||
cache,
|
|
||||||
fetcher,
|
|
||||||
report,
|
|
||||||
});
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
return report.exitCode();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+115
-40
@@ -20,8 +20,12 @@ import defaultExprTmpl from "./tmpl/default.nix.in";
|
|||||||
import projectExprTmpl from "./tmpl/yarn-project.nix.in";
|
import projectExprTmpl from "./tmpl/yarn-project.nix.in";
|
||||||
import {
|
import {
|
||||||
computeFixedOutputStorePath,
|
computeFixedOutputStorePath,
|
||||||
|
hexToSri,
|
||||||
sanitizeDerivationName,
|
sanitizeDerivationName,
|
||||||
|
sriToHex,
|
||||||
} from "./nixUtils";
|
} from "./nixUtils";
|
||||||
|
import { writeNarStream, writeNarStrings } from "./narUtils";
|
||||||
|
import { createHash } from "crypto";
|
||||||
|
|
||||||
const isYarn3 = YarnVersion?.startsWith("3.") || false;
|
const isYarn3 = YarnVersion?.startsWith("3.") || false;
|
||||||
|
|
||||||
@@ -48,6 +52,13 @@ export default async (
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Config validation.
|
||||||
|
const isolatedBuilds = configuration.get(`isolatedNixBuilds`);
|
||||||
|
const individualDrvs = configuration.get(`individualNixPackaging`);
|
||||||
|
if (isolatedBuilds.length > 0 && !individualDrvs) {
|
||||||
|
throw Error(`isolatedNixBuilds requires individualNixPackaging to be set`);
|
||||||
|
}
|
||||||
|
|
||||||
// Determine relative paths for Nix path literals.
|
// Determine relative paths for Nix path literals.
|
||||||
const nixExprPath = configuration.get(`nixExprPath`);
|
const nixExprPath = configuration.get(`nixExprPath`);
|
||||||
|
|
||||||
@@ -55,14 +66,13 @@ export default async (
|
|||||||
let yarnBinExpr: string;
|
let yarnBinExpr: string;
|
||||||
if (yarnPathAbs === null) {
|
if (yarnPathAbs === null) {
|
||||||
// Assume the current running script is the correct Yarn.
|
// Assume the current running script is the correct Yarn.
|
||||||
const hexHash = await hashUtils.checksumFile(
|
const sha512 = await hashUtils.checksumFile(
|
||||||
process.argv[1] as PortablePath,
|
process.argv[1] as PortablePath,
|
||||||
);
|
);
|
||||||
const sriHash = "sha512-" + Buffer.from(hexHash, "hex").toString("base64");
|
|
||||||
yarnBinExpr = [
|
yarnBinExpr = [
|
||||||
"fetchurl {",
|
"fetchurl {",
|
||||||
` url = "https://repo.yarnpkg.com/${YarnVersion!}/packages/yarnpkg-cli/bin/yarn.js";`,
|
` url = "https://repo.yarnpkg.com/${YarnVersion!}/packages/yarnpkg-cli/bin/yarn.js";`,
|
||||||
` hash = "${sriHash}";`,
|
` hash = "${hexToSri(sha512)}";`,
|
||||||
"}",
|
"}",
|
||||||
].join("\n ");
|
].join("\n ");
|
||||||
} else if (yarnPathAbs.startsWith(cwd)) {
|
} else if (yarnPathAbs.startsWith(cwd)) {
|
||||||
@@ -113,16 +123,14 @@ export default async (
|
|||||||
ppath.resolve(cwd, "yarn.lock" as PortablePath),
|
ppath.resolve(cwd, "yarn.lock" as PortablePath),
|
||||||
);
|
);
|
||||||
|
|
||||||
// Build a list of cache entries so Nix can fetch them.
|
// Collect all the cache files used.
|
||||||
// TODO: See if we can use Nix fetchurl for npm: dependencies.
|
interface CacheFile {
|
||||||
interface CacheEntry {
|
pkg: Package;
|
||||||
originalFilename: Filename;
|
checksum: string | undefined;
|
||||||
filename: Filename;
|
cachePath: PortablePath;
|
||||||
sha512: string;
|
|
||||||
}
|
}
|
||||||
const cacheEntries: Map<string, CacheEntry> = new Map();
|
const cacheFiles = new Map<Filename, CacheFile>();
|
||||||
|
const allCacheFiles = new Set(await xfs.readdirPromise(cache.cwd));
|
||||||
const cacheFiles = new Set(await xfs.readdirPromise(cache.cwd));
|
|
||||||
const cacheOptions = { unstablePackages: project.conditionalLocators };
|
const cacheOptions = { unstablePackages: project.conditionalLocators };
|
||||||
for (const pkg of project.storedPackages.values()) {
|
for (const pkg of project.storedPackages.values()) {
|
||||||
const { locatorHash } = pkg;
|
const { locatorHash } = pkg;
|
||||||
@@ -132,36 +140,83 @@ export default async (
|
|||||||
: cache.getLocatorPath(pkg, checksum || null);
|
: cache.getLocatorPath(pkg, checksum || null);
|
||||||
if (!cachePath) continue;
|
if (!cachePath) continue;
|
||||||
|
|
||||||
const filename = ppath.basename(cachePath);
|
if (!allCacheFiles.has(ppath.basename(cachePath))) continue;
|
||||||
if (!cacheFiles.has(filename)) continue;
|
|
||||||
|
|
||||||
|
// Rebuild the filename, because the cache file we're operating on may be
|
||||||
|
// from the mirror directory, which uses different naming.
|
||||||
|
const filename = checksum
|
||||||
|
? cache.getChecksumFilename(pkg, checksum)
|
||||||
|
: cache.getVersionFilename(pkg);
|
||||||
|
|
||||||
|
cacheFiles.set(filename, { pkg, checksum, cachePath });
|
||||||
|
}
|
||||||
|
|
||||||
|
interface CacheEntry {
|
||||||
|
cachePath: PortablePath;
|
||||||
|
filename: Filename;
|
||||||
|
hash: string;
|
||||||
|
}
|
||||||
|
const cacheEntries = new Map<string, CacheEntry>();
|
||||||
|
let cacheEntriesCode = "";
|
||||||
|
let combinedHash = "";
|
||||||
|
if (individualDrvs) {
|
||||||
|
// Build a list of cache entries so Nix can fetch them.
|
||||||
|
// TODO: See if we can use Nix fetchurl for npm: dependencies.
|
||||||
|
for (const [
|
||||||
|
filename,
|
||||||
|
{ pkg, checksum, cachePath },
|
||||||
|
] of cacheFiles.entries()) {
|
||||||
const locatorStr = structUtils.stringifyLocator(pkg);
|
const locatorStr = structUtils.stringifyLocator(pkg);
|
||||||
const sha512 = checksum
|
const sha512 = checksum
|
||||||
? checksum.split(`/`).pop()!
|
? checksum.split(`/`).pop()!
|
||||||
: await hashUtils.checksumFile(cachePath);
|
: await hashUtils.checksumFile(cachePath);
|
||||||
cacheEntries.set(locatorStr, {
|
cacheEntries.set(locatorStr, {
|
||||||
originalFilename: filename,
|
cachePath,
|
||||||
// Rebuild the filename, because the cache file we're operating on may be
|
filename,
|
||||||
// from the mirror directory, which uses different naming.
|
hash: hexToSri(sha512),
|
||||||
filename: checksum
|
|
||||||
? cache.getChecksumFilename(pkg, checksum)
|
|
||||||
: cache.getVersionFilename(pkg),
|
|
||||||
sha512,
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
let cacheEntriesCode = `cacheEntries = {\n`;
|
cacheEntriesCode = `cacheEntries = {\n`;
|
||||||
for (const locatorStr of [...cacheEntries.keys()].sort()) {
|
for (const locatorStr of [...cacheEntries.keys()].sort()) {
|
||||||
const entry = cacheEntries.get(locatorStr)!;
|
const entry = cacheEntries.get(locatorStr)!;
|
||||||
cacheEntriesCode += `${json(locatorStr)} = { ${[
|
cacheEntriesCode += `${json(locatorStr)} = { ${[
|
||||||
`filename = ${json(entry.filename)};`,
|
`filename = ${json(entry.filename)};`,
|
||||||
`sha512 = ${json(entry.sha512)};`,
|
`hash = "${entry.hash}";`,
|
||||||
].join(` `)} };\n`;
|
].join(` `)} };\n`;
|
||||||
}
|
}
|
||||||
cacheEntriesCode += `};`;
|
cacheEntriesCode += `};`;
|
||||||
|
} else {
|
||||||
|
// Hash a NAR of just the cache files we use.
|
||||||
|
const hasher = createHash("sha512");
|
||||||
|
writeNarStrings(hasher, "nix-archive-1", "(", "type", "directory");
|
||||||
|
for (const filename of [...cacheFiles.keys()].sort()) {
|
||||||
|
const { cachePath } = cacheFiles.get(filename)!;
|
||||||
|
const { size } = await xfs.statPromise(cachePath);
|
||||||
|
writeNarStrings(
|
||||||
|
hasher,
|
||||||
|
"entry",
|
||||||
|
"(",
|
||||||
|
"name",
|
||||||
|
filename,
|
||||||
|
"node",
|
||||||
|
"(",
|
||||||
|
"type",
|
||||||
|
"regular",
|
||||||
|
"contents",
|
||||||
|
);
|
||||||
|
await writeNarStream(hasher, size, xfs.createReadStream(cachePath));
|
||||||
|
writeNarStrings(hasher, ")", ")");
|
||||||
|
}
|
||||||
|
writeNarStrings(hasher, ")");
|
||||||
|
hasher.end();
|
||||||
|
// Bit hacky, but hashers always produce a single read.
|
||||||
|
for await (const sha512 of hasher) {
|
||||||
|
combinedHash = hexToSri(sha512);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Generate Nix code for isolated builds.
|
// Generate Nix code for isolated builds.
|
||||||
const isolatedBuilds = configuration.get(`isolatedNixBuilds`);
|
|
||||||
let isolatedPackages = new Set<Package>();
|
let isolatedPackages = new Set<Package>();
|
||||||
let isolatedIntegration = [];
|
let isolatedIntegration = [];
|
||||||
let isolatedCode = [];
|
let isolatedCode = [];
|
||||||
@@ -320,6 +375,9 @@ export default async (
|
|||||||
PROJECT_NAME: json(projectName),
|
PROJECT_NAME: json(projectName),
|
||||||
YARN_BIN: yarnBinExpr,
|
YARN_BIN: yarnBinExpr,
|
||||||
LOCKFILE: lockfileExpr,
|
LOCKFILE: lockfileExpr,
|
||||||
|
INDIVIDUAL_DRVS: individualDrvs,
|
||||||
|
COMBINED_DRV: !individualDrvs,
|
||||||
|
COMBINED_HASH: combinedHash,
|
||||||
CACHE_FOLDER: cacheFolderExpr,
|
CACHE_FOLDER: cacheFolderExpr,
|
||||||
CACHE_ENTRIES: cacheEntriesCode,
|
CACHE_ENTRIES: cacheEntriesCode,
|
||||||
ISOLATED: isolatedCode.join("\n"),
|
ISOLATED: isolatedCode.join("\n"),
|
||||||
@@ -350,15 +408,13 @@ export default async (
|
|||||||
xfs.existsSync(npath.toPortablePath(`/nix/store`))
|
xfs.existsSync(npath.toPortablePath(`/nix/store`))
|
||||||
) {
|
) {
|
||||||
await xfs.mktempPromise(async (tempDir) => {
|
await xfs.mktempPromise(async (tempDir) => {
|
||||||
|
const args = ["--add-fixed", "sha512"];
|
||||||
const toPreload: PortablePath[] = [];
|
const toPreload: PortablePath[] = [];
|
||||||
for (const [
|
if (individualDrvs) {
|
||||||
locator,
|
for (const [locator, { cachePath, hash }] of cacheEntries.entries()) {
|
||||||
{ originalFilename, sha512 },
|
|
||||||
] of cacheEntries.entries()) {
|
|
||||||
const name = sanitizeDerivationName(locator);
|
const name = sanitizeDerivationName(locator);
|
||||||
// Check to see if the Nix store entry already exists.
|
// Check to see if the Nix store entry already exists.
|
||||||
const hash = Buffer.from(sha512, "hex");
|
const storePath = computeFixedOutputStorePath(name, hash);
|
||||||
const storePath = computeFixedOutputStorePath(name, `sha512`, hash);
|
|
||||||
if (!xfs.existsSync(storePath)) {
|
if (!xfs.existsSync(storePath)) {
|
||||||
// The nix-store command requires a correct filename on disk, so we
|
// The nix-store command requires a correct filename on disk, so we
|
||||||
// prepare a temporary directory containing all the files to preload.
|
// prepare a temporary directory containing all the files to preload.
|
||||||
@@ -366,35 +422,54 @@ export default async (
|
|||||||
// Because some names may conflict (e.g. 'typescript-npm-xyz' and
|
// Because some names may conflict (e.g. 'typescript-npm-xyz' and
|
||||||
// 'typescript-patch-xyz' both have the same derivation name), we
|
// 'typescript-patch-xyz' both have the same derivation name), we
|
||||||
// create subdirectories based on hash.
|
// create subdirectories based on hash.
|
||||||
const subdir = ppath.join(tempDir, sha512.slice(0, 7) as Filename);
|
const subdir = ppath.join(
|
||||||
|
tempDir,
|
||||||
|
sriToHex(hash).slice(0, 7) as Filename,
|
||||||
|
);
|
||||||
await xfs.mkdirPromise(subdir);
|
await xfs.mkdirPromise(subdir);
|
||||||
|
|
||||||
const src = ppath.join(cache.cwd, originalFilename);
|
|
||||||
const dst = ppath.join(subdir, name as Filename);
|
const dst = ppath.join(subdir, name as Filename);
|
||||||
await xfs.copyFilePromise(src, dst);
|
await xfs.copyFilePromise(cachePath, dst);
|
||||||
|
|
||||||
toPreload.push(dst);
|
toPreload.push(dst);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
args.unshift("--recursive");
|
||||||
|
// Check to see if the Nix store entry already exists.
|
||||||
|
const storePath = computeFixedOutputStorePath(
|
||||||
|
"yarn-cache",
|
||||||
|
combinedHash,
|
||||||
|
{ recursive: true },
|
||||||
|
);
|
||||||
|
if (!xfs.existsSync(storePath)) {
|
||||||
|
// Same as above, nix-store requires a correct filename.
|
||||||
|
const subdir = ppath.join(tempDir, "yarn-cache");
|
||||||
|
await xfs.mkdirPromise(subdir);
|
||||||
|
for (const [filename, { cachePath }] of cacheFiles.entries()) {
|
||||||
|
const dst = ppath.join(subdir, filename);
|
||||||
|
await xfs.copyFilePromise(cachePath, dst);
|
||||||
|
}
|
||||||
|
toPreload.push(subdir);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Preload in batches, to keep the exec arguments reasonable.
|
// Preload in batches, to keep the exec arguments reasonable.
|
||||||
const numToPreload = toPreload.length;
|
const numToPreload = toPreload.length;
|
||||||
while (toPreload.length !== 0) {
|
while (toPreload.length !== 0) {
|
||||||
const batch = toPreload.splice(0, 100);
|
const batch = toPreload.splice(0, 100);
|
||||||
await execUtils.execvp(
|
await execUtils.execvp("nix-store", [...args, ...batch], {
|
||||||
"nix-store",
|
|
||||||
["--add-fixed", "sha512", ...batch],
|
|
||||||
{
|
|
||||||
cwd: project.cwd,
|
cwd: project.cwd,
|
||||||
strict: true,
|
strict: true,
|
||||||
},
|
});
|
||||||
);
|
|
||||||
}
|
}
|
||||||
if (numToPreload !== 0) {
|
if (numToPreload !== 0) {
|
||||||
report.reportInfo(
|
report.reportInfo(
|
||||||
0,
|
0,
|
||||||
`Preloaded ${numToPreload} packages into the Nix store`,
|
individualDrvs
|
||||||
|
? `Preloaded ${numToPreload} packages into the Nix store`
|
||||||
|
: `Preloaded cache into the Nix store`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
} catch (err: any) {
|
} catch (err: any) {
|
||||||
|
|||||||
+8
-2
@@ -1,7 +1,7 @@
|
|||||||
import { Hooks, Plugin, SettingsType } from "@yarnpkg/core";
|
import { Hooks, Plugin, SettingsType } from "@yarnpkg/core";
|
||||||
import { PortablePath } from "@yarnpkg/fslib";
|
import { PortablePath } from "@yarnpkg/fslib";
|
||||||
|
|
||||||
import FetchOneCommand from "./FetchOneCommand";
|
import FetchCommand from "./FetchCommand";
|
||||||
import InjectBuildCommand from "./InjectBuildCommand";
|
import InjectBuildCommand from "./InjectBuildCommand";
|
||||||
import InstallBinCommand from "./InstallBinCommand";
|
import InstallBinCommand from "./InstallBinCommand";
|
||||||
import generate from "./generate";
|
import generate from "./generate";
|
||||||
@@ -12,13 +12,14 @@ declare module "@yarnpkg/core" {
|
|||||||
nixExprPath: PortablePath;
|
nixExprPath: PortablePath;
|
||||||
generateDefaultNix: boolean;
|
generateDefaultNix: boolean;
|
||||||
enableNixPreload: boolean;
|
enableNixPreload: boolean;
|
||||||
|
individualNixPackaging: boolean;
|
||||||
isolatedNixBuilds: string[];
|
isolatedNixBuilds: string[];
|
||||||
installNixBinariesForDependencies: boolean;
|
installNixBinariesForDependencies: boolean;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const plugin: Plugin<Hooks> = {
|
const plugin: Plugin<Hooks> = {
|
||||||
commands: [FetchOneCommand, InjectBuildCommand, InstallBinCommand],
|
commands: [FetchCommand, InjectBuildCommand, InstallBinCommand],
|
||||||
hooks: {
|
hooks: {
|
||||||
afterAllInstalled: async (project, opts) => {
|
afterAllInstalled: async (project, opts) => {
|
||||||
if (
|
if (
|
||||||
@@ -50,6 +51,11 @@ const plugin: Plugin<Hooks> = {
|
|||||||
type: SettingsType.BOOLEAN,
|
type: SettingsType.BOOLEAN,
|
||||||
default: true,
|
default: true,
|
||||||
},
|
},
|
||||||
|
individualNixPackaging: {
|
||||||
|
description: `If true, generate one Nix derivation per package. If false, use a single derivation for the entire cache folder.`,
|
||||||
|
type: SettingsType.BOOLEAN,
|
||||||
|
default: false,
|
||||||
|
},
|
||||||
isolatedNixBuilds: {
|
isolatedNixBuilds: {
|
||||||
description: `Dependencies with a build step that can be built in an isolated derivation`,
|
description: `Dependencies with a build step that can be built in an isolated derivation`,
|
||||||
type: SettingsType.STRING,
|
type: SettingsType.STRING,
|
||||||
|
|||||||
@@ -0,0 +1,51 @@
|
|||||||
|
import type { Readable, Writable } from "stream";
|
||||||
|
|
||||||
|
const MAX_UINT32 = 2 ** 32 - 1;
|
||||||
|
|
||||||
|
/** Write one or more NAR strings to the output. */
|
||||||
|
export const writeNarStrings = (out: Writable, ...input: string[]) => {
|
||||||
|
let size = 0;
|
||||||
|
const bufs = input.map((str) => {
|
||||||
|
const buf = Buffer.from(str);
|
||||||
|
// TODO: Support up to MAX_SAFE_INTEGER
|
||||||
|
if (buf.byteLength > MAX_UINT32) {
|
||||||
|
throw Error(`NAR string too long: ${buf.byteLength}`);
|
||||||
|
}
|
||||||
|
size += 8 + Math.ceil(buf.byteLength / 8) * 8;
|
||||||
|
return buf;
|
||||||
|
});
|
||||||
|
|
||||||
|
const res = Buffer.alloc(size);
|
||||||
|
let pos = 0;
|
||||||
|
for (const buf of bufs) {
|
||||||
|
res.writeUInt32LE(buf.byteLength, pos);
|
||||||
|
buf.copy(res, pos + 8);
|
||||||
|
pos += 8 + Math.ceil(buf.byteLength / 8) * 8;
|
||||||
|
}
|
||||||
|
|
||||||
|
out.write(res);
|
||||||
|
};
|
||||||
|
|
||||||
|
/** Write the contents of a stream as a NAR string to the output. */
|
||||||
|
export const writeNarStream = async (
|
||||||
|
out: Writable,
|
||||||
|
size: number,
|
||||||
|
input: Readable,
|
||||||
|
) => {
|
||||||
|
// TODO: Support up to MAX_SAFE_INTEGER
|
||||||
|
if (size > MAX_UINT32) {
|
||||||
|
throw Error(`NAR string too long: ${size}`);
|
||||||
|
}
|
||||||
|
const sizeBuf = Buffer.alloc(8);
|
||||||
|
sizeBuf.writeUInt32LE(size);
|
||||||
|
out.write(sizeBuf);
|
||||||
|
|
||||||
|
for await (const chunk of input) {
|
||||||
|
out.write(chunk);
|
||||||
|
}
|
||||||
|
|
||||||
|
const padding = 8 - (size % 8);
|
||||||
|
if (padding !== 8) {
|
||||||
|
out.write(Buffer.alloc(padding));
|
||||||
|
}
|
||||||
|
};
|
||||||
+16
-4
@@ -44,13 +44,17 @@ export const encodeBase32 = (buf: Buffer) => {
|
|||||||
*/
|
*/
|
||||||
export const computeFixedOutputStorePath = (
|
export const computeFixedOutputStorePath = (
|
||||||
name: string,
|
name: string,
|
||||||
hashAlgorithm: string,
|
hash: string,
|
||||||
hash: Buffer,
|
{
|
||||||
storePath = `/nix/store` as PortablePath,
|
storePath = `/nix/store` as PortablePath,
|
||||||
|
recursive = false,
|
||||||
|
}: { storePath?: PortablePath; recursive?: boolean } = {},
|
||||||
) => {
|
) => {
|
||||||
const hashHex = hash.toString("hex");
|
const [hashAlgorithm, hash64] = hash.split("-");
|
||||||
|
const hashHex = Buffer.from(hash64, "base64").toString("hex");
|
||||||
|
|
||||||
const innerStr = `fixed:out:${hashAlgorithm}:${hashHex}:`;
|
const rec = recursive ? "r:" : "";
|
||||||
|
const innerStr = `fixed:out:${rec}${hashAlgorithm}:${hashHex}:`;
|
||||||
const innerHash = computeHash(`sha256`, innerStr);
|
const innerHash = computeHash(`sha256`, innerStr);
|
||||||
const innerHashHex = innerHash.toString("hex");
|
const innerHashHex = innerHash.toString("hex");
|
||||||
|
|
||||||
@@ -71,3 +75,11 @@ export const sanitizeDerivationName = (name: string) =>
|
|||||||
.replace(/^\.+/, "")
|
.replace(/^\.+/, "")
|
||||||
.replace(/[^a-zA-Z0-9+._?=-]+/g, "-")
|
.replace(/[^a-zA-Z0-9+._?=-]+/g, "-")
|
||||||
.slice(0, 207) || "unknown";
|
.slice(0, 207) || "unknown";
|
||||||
|
|
||||||
|
/** Convert a hexadecimal hash to an SRI hash. */
|
||||||
|
export const hexToSri = (hash: string, algorithm = "sha512") =>
|
||||||
|
algorithm + "-" + Buffer.from(hash, "hex").toString("base64");
|
||||||
|
|
||||||
|
/** Convert an SRI hash to a hexadecimal hash. */
|
||||||
|
export const sriToHex = (hash: string) =>
|
||||||
|
Buffer.from(hash.split("-")[1], "base64").toString("hex");
|
||||||
|
|||||||
@@ -38,25 +38,38 @@ let
|
|||||||
export yarn_enable_nixify=false
|
export yarn_enable_nixify=false
|
||||||
'';
|
'';
|
||||||
|
|
||||||
|
#@@ IF COMBINED_DRV
|
||||||
|
cacheDrv = stdenv.mkDerivation {
|
||||||
|
name = "yarn-cache";
|
||||||
|
buildInputs = [ yarn git cacert ];
|
||||||
|
buildCommand = ''
|
||||||
|
cp --reflink=auto --recursive '${src}' ./src
|
||||||
|
cd ./src/
|
||||||
|
${buildVars}
|
||||||
|
HOME="$TMP" yarn_enable_global_cache=false yarn_cache_folder="$out" \
|
||||||
|
yarn nixify fetch
|
||||||
|
rm $out/.gitignore
|
||||||
|
'';
|
||||||
|
outputHashMode = "recursive";
|
||||||
|
outputHash = "@@COMBINED_HASH@@";
|
||||||
|
};
|
||||||
|
#@@ ENDIF COMBINED_DRV
|
||||||
|
#@@ IF INDIVIDUAL_DRVS
|
||||||
# Create derivations for fetching dependencies.
|
# Create derivations for fetching dependencies.
|
||||||
cacheDrvs = let
|
cacheDrvs = let
|
||||||
builder = writeShellScript "yarn-cache-builder" ''
|
in lib.mapAttrs (locator: { filename, hash }: stdenv.mkDerivation {
|
||||||
source $stdenv/setup
|
name = lib.strings.sanitizeDerivationName locator;
|
||||||
cd "$src"
|
buildInputs = [ yarn git cacert ];
|
||||||
|
buildCommand = ''
|
||||||
|
cd '${src}'
|
||||||
${buildVars}
|
${buildVars}
|
||||||
HOME="$TMP" yarn_enable_global_cache=false yarn_cache_folder="$TMP" \
|
HOME="$TMP" yarn_enable_global_cache=false yarn_cache_folder="$TMP" \
|
||||||
yarn nixify fetch-one $locator
|
yarn nixify fetch ${locator}
|
||||||
# Because we change the cache dir, Yarn may generate a different name.
|
# Because we change the cache dir, Yarn may generate a different name.
|
||||||
mv "$TMP/$(sed 's/-[^-]*\.[^-]*$//' <<< "$outputFilename")"-* $out
|
mv "$TMP/$(sed 's/-[^-]*\.[^-]*$//' <<< "$outputFilename")"-* $out
|
||||||
'';
|
'';
|
||||||
in lib.mapAttrs (locator: { filename, sha512 }: stdenv.mkDerivation {
|
|
||||||
inherit src builder locator;
|
|
||||||
name = lib.strings.sanitizeDerivationName locator;
|
|
||||||
buildInputs = [ yarn git cacert ];
|
|
||||||
outputFilename = filename;
|
outputFilename = filename;
|
||||||
outputHashMode = "flat";
|
outputHash = hash;
|
||||||
outputHashAlgo = "sha512";
|
|
||||||
outputHash = sha512;
|
|
||||||
}) cacheEntries;
|
}) cacheEntries;
|
||||||
|
|
||||||
# Create a shell snippet to copy dependencies from a list of derivations.
|
# Create a shell snippet to copy dependencies from a list of derivations.
|
||||||
@@ -64,6 +77,7 @@ let
|
|||||||
writeShellScript "collect-yarn-cache" (lib.concatMapStrings (drv: ''
|
writeShellScript "collect-yarn-cache" (lib.concatMapStrings (drv: ''
|
||||||
cp --reflink=auto ${drv} '${drv.outputFilename}'
|
cp --reflink=auto ${drv} '${drv.outputFilename}'
|
||||||
'') drvs);
|
'') drvs);
|
||||||
|
#@@ ENDIF INDIVIDUAL_DRVS
|
||||||
|
|
||||||
#@@ IF NEED_ISOLATED_BUILD_SUPPRORT
|
#@@ IF NEED_ISOLATED_BUILD_SUPPRORT
|
||||||
# Create a shell snippet to copy dependencies from a list of locators.
|
# Create a shell snippet to copy dependencies from a list of locators.
|
||||||
@@ -119,9 +133,14 @@ let
|
|||||||
# Copy over the Yarn cache.
|
# Copy over the Yarn cache.
|
||||||
rm -fr '${cacheFolder}'
|
rm -fr '${cacheFolder}'
|
||||||
mkdir -p '${cacheFolder}'
|
mkdir -p '${cacheFolder}'
|
||||||
|
#@@ IF COMBINED_DRV
|
||||||
|
cp --reflink=auto --recursive ${cacheDrv}/* '${cacheFolder}/'
|
||||||
|
#@@ ENDIF COMBINED_DRV
|
||||||
|
#@@ IF INDIVIDUAL_DRVS
|
||||||
pushd '${cacheFolder}' > /dev/null
|
pushd '${cacheFolder}' > /dev/null
|
||||||
source ${mkCacheBuilderForDrvs (lib.attrValues cacheDrvs)}
|
source ${mkCacheBuilderForDrvs (lib.attrValues cacheDrvs)}
|
||||||
popd > /dev/null
|
popd > /dev/null
|
||||||
|
#@@ ENDIF INDIVIDUAL_DRVS
|
||||||
|
|
||||||
# Yarn may need a writable home directory.
|
# Yarn may need a writable home directory.
|
||||||
export yarn_global_folder="$TMP"
|
export yarn_global_folder="$TMP"
|
||||||
|
|||||||
@@ -731,10 +731,12 @@ __metadata:
|
|||||||
languageName: node
|
languageName: node
|
||||||
linkType: hard
|
linkType: hard
|
||||||
|
|
||||||
"@types/node@npm:10":
|
"@types/node@npm:18":
|
||||||
version: 10.17.60
|
version: 18.19.14
|
||||||
resolution: "@types/node@npm:10.17.60"
|
resolution: "@types/node@npm:18.19.14"
|
||||||
checksum: 0742294912a6e79786cdee9ed77cff6ee8ff007b55d8e21170fc3e5994ad3a8101fea741898091876f8dc32b0a5ae3d64537b7176799e92da56346028d2cbcd2
|
dependencies:
|
||||||
|
undici-types: "npm:~5.26.4"
|
||||||
|
checksum: a614e85d3d75e9bb73c1155f77843804506806629d20774e0479e88678efd64ab57f7bb2ba290b226fe732b6aead037ac181db04851a2f37e1c2a14744ffdb96
|
||||||
languageName: node
|
languageName: node
|
||||||
linkType: hard
|
linkType: hard
|
||||||
|
|
||||||
@@ -3665,7 +3667,7 @@ __metadata:
|
|||||||
"@babel/plugin-proposal-class-properties": "npm:^7.10.4"
|
"@babel/plugin-proposal-class-properties": "npm:^7.10.4"
|
||||||
"@babel/plugin-proposal-decorators": "npm:^7.10.5"
|
"@babel/plugin-proposal-decorators": "npm:^7.10.5"
|
||||||
"@babel/preset-typescript": "npm:^7.10.4"
|
"@babel/preset-typescript": "npm:^7.10.4"
|
||||||
"@types/node": "npm:10"
|
"@types/node": "npm:18"
|
||||||
"@yarnpkg/cli": "npm:^4.0.0"
|
"@yarnpkg/cli": "npm:^4.0.0"
|
||||||
"@yarnpkg/core": "npm:^4.0.0"
|
"@yarnpkg/core": "npm:^4.0.0"
|
||||||
"@yarnpkg/fslib": "npm:^3.0.0"
|
"@yarnpkg/fslib": "npm:^3.0.0"
|
||||||
|
|||||||
Reference in New Issue
Block a user