Isolated builds of dependencies
This commit is contained in:
Stéphan Kochen
@@ -82,15 +82,17 @@ Some examples of what's possible:
|
||||
|
||||
let
|
||||
|
||||
# Example of providing a different source tree.
|
||||
src = pkgs.lib.cleanSource ./.;
|
||||
|
||||
project = pkgs.callPackage ./yarn-project.nix {
|
||||
|
||||
# Example of selecting a specific version of Node.js.
|
||||
nodejs = pkgs.nodejs-14_x;
|
||||
|
||||
} src;
|
||||
} {
|
||||
|
||||
# Example of providing a different source tree.
|
||||
src = pkgs.lib.cleanSource ./.;
|
||||
|
||||
}
|
||||
|
||||
in project.overrideAttrs (oldAttrs: {
|
||||
|
||||
|
||||
@@ -10,6 +10,8 @@ const IS_PROD = process.argv[2] === `-p`;
|
||||
const EXTERNALS = [
|
||||
`@yarnpkg/core`,
|
||||
`@yarnpkg/fslib`,
|
||||
`@yarnpkg/parsers`,
|
||||
`@yarnpkg/plugin-patch`,
|
||||
`@yarnpkg/plugin-pnp`,
|
||||
`clipanion`,
|
||||
`crypto`,
|
||||
|
||||
Vendored
+1
-1
File diff suppressed because one or more lines are too long
@@ -17,6 +17,8 @@
|
||||
"@yarnpkg/cli": "^2.2.0",
|
||||
"@yarnpkg/core": "^2.2.0",
|
||||
"@yarnpkg/fslib": "^2.2.0",
|
||||
"@yarnpkg/parsers": "^2.2.0",
|
||||
"@yarnpkg/plugin-patch": "^2.1.1",
|
||||
"@yarnpkg/plugin-pnp": "^2.2.0",
|
||||
"babel-loader": "^8.1.0",
|
||||
"clipanion": "^2.4.4",
|
||||
|
||||
@@ -4,16 +4,16 @@ import {
|
||||
Cache,
|
||||
CommandContext,
|
||||
Configuration,
|
||||
LocatorHash,
|
||||
Project,
|
||||
StreamReport,
|
||||
structUtils,
|
||||
} from "@yarnpkg/core";
|
||||
|
||||
// Internal command that fetches a single locator.
|
||||
// Used from within Nix to build the cache for the project.
|
||||
export default class FetchOneCommand extends Command<CommandContext> {
|
||||
@Command.String()
|
||||
locatorHash: string = ``;
|
||||
locator: string = ``;
|
||||
|
||||
@Command.Path(`nixify`, `fetch-one`)
|
||||
async execute() {
|
||||
@@ -29,11 +29,10 @@ export default class FetchOneCommand extends Command<CommandContext> {
|
||||
const report = await StreamReport.start(
|
||||
{ configuration, stdout: this.context.stdout },
|
||||
async (report) => {
|
||||
const pkg = project.originalPackages.get(
|
||||
this.locatorHash as LocatorHash
|
||||
);
|
||||
const { locatorHash } = structUtils.parseLocator(this.locator, true);
|
||||
const pkg = project.originalPackages.get(locatorHash);
|
||||
if (!pkg) {
|
||||
report.reportError(0, `Invalid locator hash`);
|
||||
report.reportError(0, `Invalid locator`);
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
import { Command } from "clipanion";
|
||||
import { PortablePath, ppath, xfs } from "@yarnpkg/fslib";
|
||||
import { createHash } from "crypto";
|
||||
import { parseSyml, stringifySyml } from "@yarnpkg/parsers";
|
||||
|
||||
import {
|
||||
CommandContext,
|
||||
Configuration,
|
||||
execUtils,
|
||||
Locator,
|
||||
LocatorHash,
|
||||
Project,
|
||||
StreamReport,
|
||||
structUtils,
|
||||
} from "@yarnpkg/core";
|
||||
|
||||
// Internal command that injects an isolated build inside a Nix build.
|
||||
export default class InjectBuildCommand extends Command<CommandContext> {
|
||||
@Command.String()
|
||||
locator: string = ``;
|
||||
@Command.String()
|
||||
source: string = ``;
|
||||
@Command.String()
|
||||
installLocation: string = ``;
|
||||
|
||||
@Command.Path(`nixify`, `inject-build`)
|
||||
async execute() {
|
||||
const configuration = await Configuration.find(
|
||||
this.context.cwd,
|
||||
this.context.plugins
|
||||
);
|
||||
const { project } = await Project.find(configuration, this.context.cwd);
|
||||
|
||||
const report = await StreamReport.start(
|
||||
{ configuration, stdout: this.context.stdout },
|
||||
async (report) => {
|
||||
// To find virtualized packages, we need parse the lockfile.
|
||||
await project.resolveEverything({ report, lockfileOnly: true });
|
||||
|
||||
const locator = structUtils.parseLocator(this.locator, true);
|
||||
const pkg = project.storedPackages.get(locator.locatorHash);
|
||||
if (!pkg) {
|
||||
report.reportError(0, `Invalid locator`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Copy over the build directory.
|
||||
// Can't use xfs.copyPromise, because it doesn't know how to deal with
|
||||
// the read-only permissions of our source path in the Nix store.
|
||||
const installLocation = ppath.join(
|
||||
project.cwd,
|
||||
this.installLocation as PortablePath
|
||||
);
|
||||
await xfs.mkdirpPromise(ppath.dirname(installLocation));
|
||||
await execUtils.execvp("cp", ["-R", this.source, installLocation], {
|
||||
cwd: project.cwd,
|
||||
strict: true,
|
||||
});
|
||||
await execUtils.execvp("chmod", ["-R", "u+w", installLocation], {
|
||||
cwd: project.cwd,
|
||||
strict: true,
|
||||
});
|
||||
|
||||
// Imitate Project: generate the global hash
|
||||
const globalHashGenerator = createHash(`sha512`);
|
||||
globalHashGenerator.update(process.versions.node);
|
||||
|
||||
configuration.triggerHook(
|
||||
(hooks) => {
|
||||
return hooks.globalHashGeneration;
|
||||
},
|
||||
project,
|
||||
(data: Buffer | string) => {
|
||||
globalHashGenerator.update(`\0`);
|
||||
globalHashGenerator.update(data);
|
||||
}
|
||||
);
|
||||
|
||||
const globalHash = globalHashGenerator.digest(`hex`);
|
||||
|
||||
// Imitate Project: generate a package hash
|
||||
const packageHashMap = new Map<LocatorHash, string>();
|
||||
const getBaseHash = (locator: Locator) => {
|
||||
let hash = packageHashMap.get(locator.locatorHash);
|
||||
if (typeof hash !== `undefined`) return hash;
|
||||
|
||||
const pkg = project.storedPackages.get(locator.locatorHash);
|
||||
if (typeof pkg === `undefined`)
|
||||
throw new Error(
|
||||
`Assertion failed: The package should have been registered`
|
||||
);
|
||||
|
||||
const builder = createHash(`sha512`);
|
||||
builder.update(locator.locatorHash);
|
||||
|
||||
// To avoid the case where one dependency depends on itself somehow
|
||||
packageHashMap.set(locator.locatorHash, `<recursive>`);
|
||||
|
||||
for (const descriptor of pkg.dependencies.values()) {
|
||||
const resolution = project.storedResolutions.get(
|
||||
descriptor.descriptorHash
|
||||
);
|
||||
if (typeof resolution === `undefined`)
|
||||
throw new Error(
|
||||
`Assertion failed: The resolution (${structUtils.prettyDescriptor(
|
||||
project.configuration,
|
||||
descriptor
|
||||
)}) should have been registered`
|
||||
);
|
||||
|
||||
const dependency = project.storedPackages.get(resolution);
|
||||
if (typeof dependency === `undefined`)
|
||||
throw new Error(
|
||||
`Assertion failed: The package should have been registered`
|
||||
);
|
||||
|
||||
builder.update(getBaseHash(dependency));
|
||||
}
|
||||
|
||||
hash = builder.digest(`hex`);
|
||||
packageHashMap.set(locator.locatorHash, hash);
|
||||
|
||||
return hash;
|
||||
};
|
||||
|
||||
// Imitate Project: create a build hash that Yarn accepts
|
||||
const buildHash = createHash(`sha512`)
|
||||
.update(globalHash)
|
||||
.update(getBaseHash(pkg))
|
||||
.update(installLocation)
|
||||
.digest(`hex`);
|
||||
|
||||
// Update build state. The way we do this is crude, but we run
|
||||
// `yarn install` later, which should clean it up again.
|
||||
const bstatePath: PortablePath = configuration.get(`bstatePath`);
|
||||
const bstate: { [key: string]: string } = xfs.existsSync(bstatePath)
|
||||
? parseSyml(await xfs.readFilePromise(bstatePath, `utf8`))
|
||||
: {};
|
||||
bstate[pkg.locatorHash] = buildHash;
|
||||
await xfs.writeFilePromise(bstatePath, stringifySyml(bstate));
|
||||
}
|
||||
);
|
||||
|
||||
return report.exitCode();
|
||||
}
|
||||
}
|
||||
+13
-10
@@ -11,6 +11,7 @@ import {
|
||||
|
||||
import binWrapperNodeModulesTmpl from "./tmpl/bin-wrapper-node-modules.sh.in";
|
||||
import binWrapperPnpTmpl from "./tmpl/bin-wrapper-pnp.sh.in";
|
||||
import { renderTmpl } from "./textUtils";
|
||||
|
||||
const supportedLinkers = [`pnp`, `node-modules`];
|
||||
|
||||
@@ -59,22 +60,24 @@ export default class InstallBinCommand extends Command<CommandContext> {
|
||||
let script;
|
||||
switch (nodeLinker) {
|
||||
case `pnp`:
|
||||
script = binWrapperPnpTmpl
|
||||
.replace(`@@NODE_PATH@@`, process.execPath)
|
||||
.replace(`@@PNP_PATH@@`, pnpPath)
|
||||
.replace(`@@SCRIPT_PATH@@`, scriptPath);
|
||||
script = renderTmpl(binWrapperPnpTmpl, {
|
||||
NODE_PATH: process.execPath,
|
||||
PNP_PATH: pnpPath,
|
||||
SCRIPT_PATH: scriptPath,
|
||||
});
|
||||
break;
|
||||
case `node-modules`:
|
||||
script = binWrapperNodeModulesTmpl
|
||||
.replace(`@@NODE_PATH@@`, process.execPath)
|
||||
.replace(`@@SCRIPT_PATH@@`, scriptPath);
|
||||
script = renderTmpl(binWrapperNodeModulesTmpl, {
|
||||
NODE_PATH: process.execPath,
|
||||
SCRIPT_PATH: scriptPath,
|
||||
});
|
||||
break;
|
||||
default:
|
||||
throw Error(`Invalid nodeLinker ${nodeLinker}`);
|
||||
throw Error(`Assertion failed: Invalid nodeLinker ${nodeLinker}`);
|
||||
}
|
||||
|
||||
xfs.writeFileSync(binPath, script);
|
||||
xfs.chmodSync(binPath, 0o755);
|
||||
await xfs.writeFilePromise(binPath, script);
|
||||
await xfs.chmodPromise(binPath, 0o755);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
+202
-41
@@ -1,10 +1,17 @@
|
||||
import { Filename, npath, PortablePath, ppath, xfs } from "@yarnpkg/fslib";
|
||||
import { computeFixedOutputStorePath } from "./nixUtils";
|
||||
import { parseSyml } from "@yarnpkg/parsers";
|
||||
import { patchUtils } from "@yarnpkg/plugin-patch";
|
||||
import {
|
||||
computeFixedOutputStorePath,
|
||||
sanitizeDerivationName,
|
||||
} from "./nixUtils";
|
||||
import { json, indent, renderTmpl, upperCamelize } from "./textUtils";
|
||||
|
||||
import {
|
||||
Cache,
|
||||
execUtils,
|
||||
LocatorHash,
|
||||
Package,
|
||||
Project,
|
||||
Report,
|
||||
structUtils,
|
||||
@@ -14,21 +21,6 @@ import defaultExprTmpl from "./tmpl/default.nix.in";
|
||||
import projectExprTmpl from "./tmpl/yarn-project.nix.in";
|
||||
import { tmpdir } from "os";
|
||||
|
||||
interface CacheEntry {
|
||||
name: string;
|
||||
filename: Filename;
|
||||
sha512: string;
|
||||
locatorHash: LocatorHash;
|
||||
}
|
||||
|
||||
const cacheEntryToNix = (entry: CacheEntry) =>
|
||||
[
|
||||
`name = ${JSON.stringify(entry.name)};`,
|
||||
`filename = ${JSON.stringify(entry.filename)};`,
|
||||
`sha512 = ${JSON.stringify(entry.sha512)};`,
|
||||
`locatorHash = ${JSON.stringify(entry.locatorHash)};`,
|
||||
].join(` `);
|
||||
|
||||
// Generator function that runs after `yarn install`.
|
||||
export default async (project: Project, cache: Cache, report: Report) => {
|
||||
const { configuration, cwd } = project;
|
||||
@@ -39,7 +31,7 @@ export default async (project: Project, cache: Cache, report: Report) => {
|
||||
//
|
||||
// On macOS at least, we also need to get the real path of the OS temp dir,
|
||||
// because it goes through a symlink.
|
||||
const tempDir = xfs.realpathSync(npath.toPortablePath(tmpdir()));
|
||||
const tempDir = await xfs.realpathPromise(npath.toPortablePath(tmpdir()));
|
||||
if (project.cwd.startsWith(tempDir)) {
|
||||
report.reportInfo(
|
||||
0,
|
||||
@@ -81,11 +73,26 @@ export default async (project: Project, cache: Cache, report: Report) => {
|
||||
}
|
||||
}
|
||||
|
||||
// Determine relative paths for Nix path literals.
|
||||
const nixExprPath = configuration.get(`nixExprPath`);
|
||||
const lockfileFilename = configuration.get(`lockfileFilename`);
|
||||
const lockfileRel = ppath.relative(
|
||||
ppath.dirname(nixExprPath),
|
||||
lockfileFilename
|
||||
);
|
||||
const yarnPathRel = ppath.relative(ppath.dirname(nixExprPath), yarnPath);
|
||||
|
||||
// Build a list of cache entries so Nix can fetch them.
|
||||
let cacheEntries: CacheEntry[] = [];
|
||||
const cacheFiles = new Set(xfs.readdirSync(cache.cwd));
|
||||
// TODO: See if we can use Nix fetchurl for npm: dependencies.
|
||||
interface CacheEntry {
|
||||
filename: Filename;
|
||||
sha512: string;
|
||||
}
|
||||
const cacheEntries: Map<string, CacheEntry> = new Map();
|
||||
|
||||
const cacheFiles = new Set(await xfs.readdirPromise(cache.cwd));
|
||||
for (const pkg of project.storedPackages.values()) {
|
||||
const { version, locatorHash } = pkg;
|
||||
const { locatorHash } = pkg;
|
||||
const checksum = project.storedChecksums.get(locatorHash);
|
||||
if (!checksum) continue;
|
||||
|
||||
@@ -95,39 +102,192 @@ export default async (project: Project, cache: Cache, report: Report) => {
|
||||
const filename = ppath.basename(cachePath);
|
||||
if (!cacheFiles.has(filename)) continue;
|
||||
|
||||
let name = structUtils.slugifyIdent(pkg).replace(/^@/, "_at_");
|
||||
if (version) {
|
||||
name += `-${version}`;
|
||||
const locatorStr = structUtils.stringifyLocator(pkg);
|
||||
const sha512 = checksum.split(`/`).pop()!;
|
||||
cacheEntries.set(locatorStr, { filename, sha512 });
|
||||
}
|
||||
|
||||
let cacheEntriesCode = `cacheEntries = {\n`;
|
||||
for (const [locatorStr, entry] of cacheEntries) {
|
||||
cacheEntriesCode += `${json(locatorStr)} = { ${[
|
||||
`filename = ${json(entry.filename)};`,
|
||||
`sha512 = ${json(entry.sha512)};`,
|
||||
].join(` `)} };\n`;
|
||||
}
|
||||
cacheEntriesCode += `};`;
|
||||
|
||||
// Generate Nix code for isolated builds.
|
||||
const isolatedBuilds: string[] = configuration.get(`isolatedNixBuilds`);
|
||||
let isolatedPackages = new Set<Package>();
|
||||
let isolatedIntegration = [];
|
||||
let isolatedCode = [];
|
||||
|
||||
const nodeLinker = configuration.get(`nodeLinker`);
|
||||
const pnpUnpluggedFolder = configuration.get(`pnpUnpluggedFolder`);
|
||||
const bstatePath: PortablePath = configuration.get(`bstatePath`);
|
||||
const bstate: { [key: string]: string } = xfs.existsSync(bstatePath)
|
||||
? parseSyml(await xfs.readFilePromise(bstatePath, `utf8`))
|
||||
: {};
|
||||
|
||||
const collectTree = (pkg: Package, out: Set<string> = new Set()) => {
|
||||
const locatorStr = structUtils.stringifyLocator(pkg);
|
||||
if (cacheEntries.has(locatorStr)) {
|
||||
out.add(locatorStr);
|
||||
}
|
||||
|
||||
const sha512 = checksum.split(`/`).pop()!;
|
||||
cacheEntries.push({ name, filename, sha512, locatorHash });
|
||||
if (structUtils.isVirtualLocator(pkg)) {
|
||||
const devirtPkg = project.storedPackages.get(
|
||||
structUtils.devirtualizeLocator(pkg).locatorHash
|
||||
);
|
||||
if (!devirtPkg) {
|
||||
throw Error(
|
||||
`Assertion failed: The locator should have been registered`
|
||||
);
|
||||
}
|
||||
|
||||
collectTree(devirtPkg, out);
|
||||
}
|
||||
|
||||
if (pkg.reference.startsWith("patch:")) {
|
||||
const depatchPkg = project.storedPackages.get(
|
||||
patchUtils.parseLocator(pkg).sourceLocator.locatorHash
|
||||
);
|
||||
if (!depatchPkg) {
|
||||
throw Error(
|
||||
`Assertion failed: The locator should have been registered`
|
||||
);
|
||||
}
|
||||
|
||||
collectTree(depatchPkg, out);
|
||||
}
|
||||
|
||||
for (const dependency of pkg.dependencies.values()) {
|
||||
const resolution = project.storedResolutions.get(
|
||||
dependency.descriptorHash
|
||||
);
|
||||
if (!resolution) {
|
||||
throw Error(
|
||||
"Assertion failed: The descriptor should have been registered"
|
||||
);
|
||||
}
|
||||
|
||||
const depPkg = project.storedPackages.get(resolution);
|
||||
if (!depPkg) {
|
||||
throw Error(
|
||||
`Assertion failed: The locator should have been registered`
|
||||
);
|
||||
}
|
||||
|
||||
collectTree(depPkg, out);
|
||||
}
|
||||
|
||||
return out;
|
||||
};
|
||||
|
||||
for (const locatorHash of Object.keys(bstate)) {
|
||||
const pkg = project.storedPackages.get(locatorHash as LocatorHash);
|
||||
if (!pkg) {
|
||||
throw Error(`Assertion failed: The locator should have been registered`);
|
||||
}
|
||||
|
||||
// TODO: Better options for matching.
|
||||
if (!isolatedBuilds.includes(pkg.name)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// TODO: We can't currently support the node-modules linker, because it
|
||||
// always clears build state.
|
||||
let installLocation: PortablePath;
|
||||
switch (nodeLinker) {
|
||||
case `pnp`:
|
||||
installLocation = ppath.relative(
|
||||
project.cwd,
|
||||
ppath.join(
|
||||
pnpUnpluggedFolder,
|
||||
structUtils.slugifyLocator(pkg),
|
||||
structUtils.getIdentVendorPath(pkg)
|
||||
)
|
||||
);
|
||||
break;
|
||||
default:
|
||||
throw Error(
|
||||
`The nodeLinker ${nodeLinker} is not supported for isolated Nix builds`
|
||||
);
|
||||
}
|
||||
|
||||
// Virtualization typically happens when the package has peer dependencies,
|
||||
// and thus it depends on context how the package is built. But we
|
||||
// eliminate that context, so devirtualize.
|
||||
let devirtPkg = pkg;
|
||||
if (structUtils.isVirtualLocator(devirtPkg)) {
|
||||
const { locatorHash } = structUtils.devirtualizeLocator(devirtPkg);
|
||||
const pkg = project.storedPackages.get(locatorHash);
|
||||
if (!pkg) {
|
||||
throw Error(
|
||||
`Assertion failed: The locator should have been registered`
|
||||
);
|
||||
}
|
||||
devirtPkg = pkg;
|
||||
}
|
||||
|
||||
const buildLocatorStr = structUtils.stringifyLocator(devirtPkg);
|
||||
const injectLocatorStr = structUtils.stringifyLocator(pkg);
|
||||
const isolatedProp = `isolated.${json(buildLocatorStr)}`;
|
||||
|
||||
if (!isolatedPackages.has(devirtPkg)) {
|
||||
isolatedPackages.add(devirtPkg);
|
||||
|
||||
const locators = [...collectTree(pkg)]
|
||||
.sort()
|
||||
.map((v) => `${json(v)}\n`)
|
||||
.join(``);
|
||||
|
||||
const overrideArg = `override${upperCamelize(pkg.name)}Attrs`;
|
||||
isolatedCode.push(
|
||||
`${isolatedProp} = optionalOverride (args.${overrideArg} or null) (mkIsolatedBuild { ${[
|
||||
`pname = ${json(pkg.name)};`,
|
||||
`version = ${json(pkg.version)};`,
|
||||
`locators = [\n${locators}];`,
|
||||
].join(` `)} });`
|
||||
);
|
||||
}
|
||||
|
||||
if (isolatedIntegration.length === 0) {
|
||||
isolatedIntegration.push("# Copy in isolated builds.");
|
||||
}
|
||||
isolatedIntegration.push(
|
||||
`echo 'injecting build for ${pkg.name}'`,
|
||||
`yarn nixify inject-build \\`,
|
||||
` ${json(injectLocatorStr)} \\`,
|
||||
` $\{${isolatedProp}} \\`,
|
||||
` ${json(installLocation)}`
|
||||
);
|
||||
}
|
||||
if (isolatedIntegration.length > 0) {
|
||||
isolatedIntegration.push(`echo 'running yarn install'`);
|
||||
}
|
||||
|
||||
// Render the Nix expression.
|
||||
const ident = project.topLevelWorkspace.manifest.name;
|
||||
const projectName = ident ? structUtils.stringifyIdent(ident) : `workspace`;
|
||||
const projectExpr = projectExprTmpl
|
||||
.replace(`@@PROJECT_NAME@@`, JSON.stringify(projectName))
|
||||
.replace(`@@YARN_PATH@@`, JSON.stringify(yarnPath))
|
||||
.replace(`@@CACHE_FOLDER@@`, JSON.stringify(cacheFolder))
|
||||
.replace(
|
||||
`@@CACHE_ENTRIES@@`,
|
||||
`[\n` +
|
||||
cacheEntries
|
||||
.map((entry) => ` { ${cacheEntryToNix(entry)} }\n`)
|
||||
.sort()
|
||||
.join(``) +
|
||||
` ]`
|
||||
);
|
||||
xfs.writeFileSync(configuration.get(`nixExprPath`), projectExpr);
|
||||
const projectExpr = renderTmpl(projectExprTmpl, {
|
||||
PROJECT_NAME: json(projectName),
|
||||
YARN_PATH: yarnPathRel,
|
||||
LOCKFILE: lockfileRel,
|
||||
CACHE_FOLDER: json(cacheFolder),
|
||||
CACHE_ENTRIES: cacheEntriesCode,
|
||||
ISOLATED: isolatedCode.join("\n"),
|
||||
ISOLATED_INTEGRATION: indent(" ", isolatedIntegration.join("\n")),
|
||||
NEED_ISOLATED_BUILD_SUPPRORT: isolatedIntegration.length > 0,
|
||||
});
|
||||
await xfs.writeFilePromise(configuration.get(`nixExprPath`), projectExpr);
|
||||
|
||||
// Create a wrapper if it does not exist yet.
|
||||
if (configuration.get(`generateDefaultNix`)) {
|
||||
const defaultExprPath = ppath.join(cwd, `default.nix` as Filename);
|
||||
const flakeExprPath = ppath.join(cwd, `flake.nix` as Filename);
|
||||
if (!xfs.existsSync(defaultExprPath) && !xfs.existsSync(flakeExprPath)) {
|
||||
xfs.writeFileSync(defaultExprPath, defaultExprTmpl);
|
||||
await xfs.writeFilePromise(defaultExprPath, defaultExprTmpl);
|
||||
report.reportInfo(
|
||||
0,
|
||||
`A minimal default.nix was created. You may want to customize it.`
|
||||
@@ -142,7 +302,8 @@ export default async (project: Project, cache: Cache, report: Report) => {
|
||||
) {
|
||||
await xfs.mktempPromise(async (tempDir) => {
|
||||
const toPreload: PortablePath[] = [];
|
||||
for (const { name, filename, sha512 } of cacheEntries) {
|
||||
for (const [locator, { filename, sha512 }] of cacheEntries.entries()) {
|
||||
const name = sanitizeDerivationName(locator);
|
||||
// Check to see if the Nix store entry already exists.
|
||||
const hash = Buffer.from(sha512, "hex");
|
||||
const storePath = computeFixedOutputStorePath(name, `sha512`, hash);
|
||||
|
||||
+12
-4
@@ -1,10 +1,12 @@
|
||||
import FetchOneCommand from "./FetchOneCommand";
|
||||
import InstallBinCommand from "./InstallBinCommand";
|
||||
import generate from "./generate";
|
||||
import { Hooks, Plugin, SettingsType } from "@yarnpkg/core";
|
||||
|
||||
import FetchOneCommand from "./FetchOneCommand";
|
||||
import InjectBuildCommand from "./InjectBuildCommand";
|
||||
import InstallBinCommand from "./InstallBinCommand";
|
||||
import generate from "./generate";
|
||||
|
||||
const plugin: Plugin<Hooks> = {
|
||||
commands: [FetchOneCommand, InstallBinCommand],
|
||||
commands: [FetchOneCommand, InjectBuildCommand, InstallBinCommand],
|
||||
hooks: {
|
||||
afterAllInstalled: async (project, opts) => {
|
||||
if (
|
||||
@@ -36,6 +38,12 @@ const plugin: Plugin<Hooks> = {
|
||||
type: SettingsType.BOOLEAN,
|
||||
default: true,
|
||||
},
|
||||
isolatedNixBuilds: {
|
||||
description: `Dependencies with a build step that can be built in an isolated derivation`,
|
||||
type: SettingsType.STRING,
|
||||
default: [],
|
||||
isArray: true,
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
|
||||
@@ -60,3 +60,14 @@ export const computeFixedOutputStorePath = (
|
||||
|
||||
return ppath.join(storePath, `${outerHash32}-${name}` as Filename);
|
||||
};
|
||||
|
||||
/**
|
||||
* Creates a valid derivation name from a potentially invalid one.
|
||||
*
|
||||
* Matches lib.strings.sanitizeDerivationName in Nixpkgs.
|
||||
*/
|
||||
export const sanitizeDerivationName = (name: string) =>
|
||||
name
|
||||
.replace(/^\.+/, "")
|
||||
.replace(/[^a-zA-Z0-9+._?=-]+/g, "-")
|
||||
.slice(0, 207) || "unknown";
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
export const json = JSON.stringify;
|
||||
|
||||
/**
|
||||
* Capitalize the first character.
|
||||
*/
|
||||
export const ucfirst = (value: string) =>
|
||||
value.slice(0, 1).toUpperCase() + value.slice(1);
|
||||
|
||||
/**
|
||||
* Convert to a camel-case starting with uppercase.
|
||||
*
|
||||
* Example: `one two three` => `OneTwoThree`
|
||||
*/
|
||||
export const upperCamelize = (name: string) =>
|
||||
name
|
||||
.split(/[^a-zA-Z0-9]+/g)
|
||||
.filter((x) => x)
|
||||
.map((v) => ucfirst(v))
|
||||
.join("");
|
||||
|
||||
/**
|
||||
* Add a prefix to every line in some text.
|
||||
*/
|
||||
export const indent = (
|
||||
prefix: string,
|
||||
text: string,
|
||||
includeEmptyLines = false
|
||||
): string =>
|
||||
text
|
||||
.split("\n")
|
||||
.map((line: string) => (line || includeEmptyLines ? prefix + line : line))
|
||||
.join("\n");
|
||||
|
||||
/**
|
||||
* Basic templating rendering.
|
||||
*
|
||||
* String values in `vars` will be used for simple substitution
|
||||
* of `@@KEY@@`, while boolean values will be used for
|
||||
* conditional sections of code between `#@@ IF KEY` and `#@@
|
||||
* ENDIF KEY`.
|
||||
*/
|
||||
export const renderTmpl = (
|
||||
tmpl: string,
|
||||
vars: { [name: string]: string | boolean }
|
||||
): string => {
|
||||
let result = tmpl;
|
||||
for (const [name, value] of Object.entries(vars)) {
|
||||
if (typeof value === "string") {
|
||||
result = result.replace(`@@${name}@@`, value);
|
||||
}
|
||||
if (typeof value === "boolean") {
|
||||
const lines = result.split("\n");
|
||||
const startIdx = lines.indexOf(`#@@ IF ${name}`);
|
||||
const endIdx = lines.indexOf(`#@@ ENDIF ${name}`);
|
||||
if (startIdx !== -1 && endIdx > startIdx) {
|
||||
if (value) {
|
||||
lines.splice(endIdx, 1);
|
||||
lines.splice(startIdx, 1);
|
||||
} else {
|
||||
lines.splice(startIdx, endIdx - startIdx + 1);
|
||||
}
|
||||
result = lines.join("\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
return result;
|
||||
};
|
||||
@@ -3,4 +3,4 @@
|
||||
|
||||
{ pkgs ? import <nixpkgs> { } }:
|
||||
|
||||
pkgs.callPackage ./yarn-project.nix { } ./.
|
||||
pkgs.callPackage ./yarn-project.nix { } { src = ./.; }
|
||||
|
||||
+108
-66
@@ -1,103 +1,145 @@
|
||||
# This file is generated by running "yarn install" inside your project.
|
||||
# Manual changes might be lost - proceed with caution!
|
||||
|
||||
{ lib, coreutils, nodejs, stdenv, writeText }: src:
|
||||
{ lib, nodejs, stdenv, fetchurl, writeText }:
|
||||
{ src, overrideAttrs ? null, ... } @ args:
|
||||
|
||||
let
|
||||
|
||||
yarnPath = @@YARN_PATH@@;
|
||||
yarnPath = ./@@YARN_PATH@@;
|
||||
lockfile = ./@@LOCKFILE@@;
|
||||
cacheFolder = @@CACHE_FOLDER@@;
|
||||
cacheEntries = @@CACHE_ENTRIES@@;
|
||||
|
||||
# Fetch a single dependency.
|
||||
fetchOne = let
|
||||
# Call overrideAttrs on a derivation if a function is provided.
|
||||
optionalOverride = fn: drv:
|
||||
if fn == null then drv else drv.overrideAttrs fn;
|
||||
|
||||
# Common attributes between Yarn derivations.
|
||||
drvCommon = {
|
||||
# Make sure the build uses the right Node.js version everywhere.
|
||||
buildInputs = [ nodejs ];
|
||||
# Tell node-gyp to use the provided Node.js headers for native code builds.
|
||||
npm_config_nodedir = nodejs;
|
||||
# Tell node-pre-gyp to never fetch binaries / always build from source.
|
||||
npm_config_build_from_source = "true";
|
||||
# Defines the shell alias to run Yarn.
|
||||
postHook = ''
|
||||
yarn() {
|
||||
CI=1 node "${yarnPath}" "$@"
|
||||
}
|
||||
'';
|
||||
};
|
||||
|
||||
# Create derivations for fetching dependencies.
|
||||
cacheDrvs = let
|
||||
builder = builtins.toFile "builder.sh" ''
|
||||
source $stdenv/setup
|
||||
cd "$src"
|
||||
HOME="$TMP" yarn_cache_folder="$TMP" CI=1 \
|
||||
node '${yarnPath}' nixify fetch-one $locatorHash
|
||||
node '${yarnPath}' nixify fetch-one $locator
|
||||
mv "$TMP/$outputFilename" $out
|
||||
'';
|
||||
in { name, filename, sha512, locatorHash }: stdenv.mkDerivation {
|
||||
inherit name src builder locatorHash;
|
||||
in lib.mapAttrs (locator: { filename, sha512 }: stdenv.mkDerivation {
|
||||
inherit src builder locator;
|
||||
name = lib.strings.sanitizeDerivationName locator;
|
||||
buildInputs = [ nodejs ];
|
||||
outputFilename = filename;
|
||||
outputHashMode = "flat";
|
||||
outputHashAlgo = "sha512";
|
||||
outputHash = sha512;
|
||||
};
|
||||
}) cacheEntries;
|
||||
|
||||
# Shell snippet to collect all project dependencies.
|
||||
collectCacheScript = writeText "collect-cache.sh" (
|
||||
lib.concatMapStrings (args: ''
|
||||
cp ${fetchOne args} '${args.filename}'
|
||||
'') cacheEntries
|
||||
);
|
||||
# Create a shell snippet to copy dependencies from a list of derivations.
|
||||
mkCacheBuilderForDrvs = drvs:
|
||||
writeText "collect-cache.sh" (lib.concatMapStrings (drv: ''
|
||||
cp ${drv} '${drv.outputFilename}'
|
||||
'') drvs);
|
||||
|
||||
in stdenv.mkDerivation {
|
||||
name = @@PROJECT_NAME@@;
|
||||
inherit src;
|
||||
#@@ IF NEED_ISOLATED_BUILD_SUPPRORT
|
||||
# Create a shell snippet to copy dependencies from a list of locators.
|
||||
mkCacheBuilderForLocators = let
|
||||
pickCacheDrvs = map (locator: cacheDrvs.${locator});
|
||||
in locators:
|
||||
mkCacheBuilderForDrvs (pickCacheDrvs locators);
|
||||
|
||||
# Disable Nixify plugin to save on some unnecessary processing.
|
||||
yarn_enable_nixify = "false";
|
||||
# Tell node-gyp to use the provided Node.js headers for native code builds.
|
||||
npm_config_nodedir = nodejs;
|
||||
# Tell node-pre-gyp to never fetch binaries / always build from source.
|
||||
npm_config_build_from_source = "true";
|
||||
# Create a derivation that builds a node-pre-gyp module in isolation.
|
||||
mkIsolatedBuild = { pname, version, locators }: stdenv.mkDerivation (drvCommon // {
|
||||
inherit pname version;
|
||||
phases = [ "buildPhase" "installPhase" ];
|
||||
|
||||
# Make sure the build uses the right Node.js version everywhere.
|
||||
buildInputs = [ nodejs ];
|
||||
buildPhase = ''
|
||||
mkdir -p .yarn/cache
|
||||
pushd .yarn/cache > /dev/null
|
||||
source ${mkCacheBuilderForLocators locators}
|
||||
popd > /dev/null
|
||||
|
||||
# Defines the shell alias to run Yarn.
|
||||
postHook = ''
|
||||
yarn() {
|
||||
CI=1 node "$NIX_YARN_PATH" "$@"
|
||||
}
|
||||
'';
|
||||
echo '{ "dependencies": { "${pname}": "${version}" } }' > package.json
|
||||
install -m 0600 ${lockfile} ./yarn.lock
|
||||
yarn --immutable-cache
|
||||
'';
|
||||
|
||||
configurePhase = ''
|
||||
runHook preConfigure
|
||||
installPhase = ''
|
||||
unplugged=( .yarn/unplugged/${pname}-*/node_modules/* )
|
||||
if [[ ! -e "''${unplugged[@]}" ]]; then
|
||||
echo >&2 "Could not find the unplugged path for ${pname}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Copy over the Yarn cache.
|
||||
rm -fr '${cacheFolder}'
|
||||
mkdir -p '${cacheFolder}'
|
||||
pushd '${cacheFolder}' > /dev/null
|
||||
source ${collectCacheScript}
|
||||
popd > /dev/null
|
||||
mv "$unplugged" $out
|
||||
'';
|
||||
});
|
||||
#@@ ENDIF NEED_ISOLATED_BUILD_SUPPRORT
|
||||
|
||||
# Store the absolute path to Yarn for the 'yarn' alias.
|
||||
export NIX_YARN_PATH="$(readlink -f '${yarnPath}')"
|
||||
# Main project derivation.
|
||||
project = stdenv.mkDerivation (drvCommon // {
|
||||
inherit src;
|
||||
name = @@PROJECT_NAME@@;
|
||||
# Disable Nixify plugin to save on some unnecessary processing.
|
||||
yarn_enable_nixify = "false";
|
||||
|
||||
# Run normal Yarn install to complete dependency installation.
|
||||
yarn install --immutable --immutable-cache
|
||||
configurePhase = ''
|
||||
runHook preConfigure
|
||||
|
||||
runHook postConfigure
|
||||
'';
|
||||
# Copy over the Yarn cache.
|
||||
rm -fr '${cacheFolder}'
|
||||
mkdir -p '${cacheFolder}'
|
||||
pushd '${cacheFolder}' > /dev/null
|
||||
source ${mkCacheBuilderForDrvs (lib.attrValues cacheDrvs)}
|
||||
popd > /dev/null
|
||||
|
||||
buildPhase = ''
|
||||
runHook preBuild
|
||||
runHook postBuild
|
||||
'';
|
||||
@@ISOLATED_INTEGRATION@@
|
||||
|
||||
installPhase = ''
|
||||
runHook preInstall
|
||||
# Run normal Yarn install to complete dependency installation.
|
||||
yarn install --immutable --immutable-cache
|
||||
|
||||
mkdir -p $out/libexec $out/bin
|
||||
runHook postConfigure
|
||||
'';
|
||||
|
||||
# Move the entire project to the output directory.
|
||||
mv $PWD "$out/libexec/$sourceRoot"
|
||||
cd "$out/libexec/$sourceRoot"
|
||||
buildPhase = ''
|
||||
runHook preBuild
|
||||
runHook postBuild
|
||||
'';
|
||||
|
||||
# Update the path to Yarn.
|
||||
export NIX_YARN_PATH="$(readlink -f '${yarnPath}')"
|
||||
installPhase = ''
|
||||
runHook preInstall
|
||||
|
||||
# Invoke a plugin internal command to setup binaries.
|
||||
yarn nixify install-bin $out/bin
|
||||
mkdir -p $out/libexec $out/bin
|
||||
|
||||
runHook postInstall
|
||||
'';
|
||||
# Move the entire project to the output directory.
|
||||
mv $PWD "$out/libexec/$sourceRoot"
|
||||
cd "$out/libexec/$sourceRoot"
|
||||
|
||||
passthru = {
|
||||
inherit nodejs;
|
||||
};
|
||||
}
|
||||
# Invoke a plugin internal command to setup binaries.
|
||||
yarn nixify install-bin $out/bin
|
||||
|
||||
runHook postInstall
|
||||
'';
|
||||
|
||||
passthru = {
|
||||
inherit nodejs;
|
||||
};
|
||||
});
|
||||
|
||||
@@CACHE_ENTRIES@@
|
||||
@@ISOLATED@@
|
||||
in optionalOverride overrideAttrs project
|
||||
|
||||
Reference in New Issue
Block a user