Yarn v4 support

.github/workflows/build.yml

@@ -22,6 +22,9 @@ jobs:
         with:
           node-version: 18.x
 
+      - name: Enable corepack
+        run: corepack enable
+
       - name: Yarn cache
         uses: actions/cache@v3
         with:

.gitignore

@@ -1,6 +1,4 @@
-/.yarn/*
-!/.yarn/releases
-!/.yarn/plugins
+/.yarn
 /.pnp.*
 
 /dist/yarn-plugin-nixify.dev.js

.yarnrc.yml

@@ -1 +0,0 @@
-yarnPath: .yarn/releases/yarn-3.3.1.cjs

package.json

@@ -8,47 +8,24 @@
     "build-dev": "./build.js",
     "fmt": "prettier --write src"
   },
-  "packageManager": "yarn@3.3.1",
+  "packageManager": "yarn@4.0.1",
   "devDependencies": {
     "@babel/core": "^7.11.4",
     "@babel/plugin-proposal-class-properties": "^7.10.4",
     "@babel/plugin-proposal-decorators": "^7.10.5",
     "@babel/preset-typescript": "^7.10.4",
     "@types/node": "10",
-    "@yarnpkg/cli": "^3.0.0",
-    "@yarnpkg/core": "^3.0.0",
-    "@yarnpkg/fslib": "^2.5.1",
-    "@yarnpkg/plugin-patch": "^3.0.0",
-    "@yarnpkg/plugin-pnp": "^3.1.0",
-    "babel-loader": "^8.1.0",
-    "clipanion": "^3.0.0",
+    "@yarnpkg/cli": "^4.0.0",
+    "@yarnpkg/core": "^4.0.0",
+    "@yarnpkg/fslib": "^3.0.0",
+    "@yarnpkg/plugin-patch": "^4.0.0",
+    "@yarnpkg/plugin-pnp": "^4.0.0",
+    "babel-loader": "^9.1.3",
+    "clipanion": "^4.0.0-rc.2",
     "prettier": "3.0.3",
     "raw-loader": "^4.0.1",
-    "typescript": "4.5.4",
+    "typescript": "5.2.2",
     "webpack": "^5.35.0",
-    "webpack-sources": "^2.2.0"
-  },
-  "resolutions": {
-    "@yarnpkg/cli": "3.1.0",
-    "@yarnpkg/core": "3.1.0",
-    "@yarnpkg/fslib": "2.5.1",
-    "@yarnpkg/libzip": "2.2.2",
-    "@yarnpkg/plugin-compat": "3.0.0",
-    "@yarnpkg/plugin-dlx": "3.0.0",
-    "@yarnpkg/plugin-essentials": "3.0.0",
-    "@yarnpkg/plugin-file": "2.2.1",
-    "@yarnpkg/plugin-git": "2.4.0",
-    "@yarnpkg/plugin-github": "2.2.0",
-    "@yarnpkg/plugin-http": "2.1.3",
-    "@yarnpkg/plugin-init": "3.0.0",
-    "@yarnpkg/plugin-link": "2.1.2",
-    "@yarnpkg/plugin-node-modules": "2.3.0",
-    "@yarnpkg/plugin-npm": "2.5.0",
-    "@yarnpkg/plugin-npm-cli": "3.0.0",
-    "@yarnpkg/plugin-pack": "3.0.0",
-    "@yarnpkg/plugin-patch": "3.0.0",
-    "@yarnpkg/plugin-pnp": "3.1.0",
-    "@yarnpkg/plugin-stage": "3.0.0",
-    "@yarnpkg/shell": "3.0.0"
+    "webpack-sources": "^3.2.3"
   }
 }

src/generate.ts

@@ -1,13 +1,11 @@
+import { tmpdir } from "os";
+
 import { Filename, npath, PortablePath, ppath, xfs } from "@yarnpkg/fslib";
 import { patchUtils } from "@yarnpkg/plugin-patch";
-import {
-  computeFixedOutputStorePath,
-  sanitizeDerivationName,
-} from "./nixUtils";
 import { json, indent, renderTmpl, upperCamelize } from "./textUtils";
-
 import {
   Cache,
+  YarnVersion,
   execUtils,
   hashUtils,
   InstallMode,
@@ -20,7 +18,12 @@ import {
 
 import defaultExprTmpl from "./tmpl/default.nix.in";
 import projectExprTmpl from "./tmpl/yarn-project.nix.in";
-import { tmpdir } from "os";
+import {
+  computeFixedOutputStorePath,
+  sanitizeDerivationName,
+} from "./nixUtils";
+
+const isYarn3 = YarnVersion?.startsWith("3.") || false;
 
 // Generator function that runs after `yarn install`.
 export default async (
@@ -49,12 +52,24 @@ export default async (
   const nixExprPath = configuration.get(`nixExprPath`);
 
   const yarnPathAbs = configuration.get(`yarnPath`);
-  let yarnPathExpr: string;
-  if (yarnPathAbs.startsWith(cwd)) {
-    yarnPathExpr =
+  let yarnBinExpr: string;
+  if (yarnPathAbs === null) {
+    // Assume the current running script is the correct Yarn.
+    const hexHash = await hashUtils.checksumFile(
+      process.argv[1] as PortablePath,
+    );
+    const sriHash = "sha512-" + Buffer.from(hexHash, "hex").toString("base64");
+    yarnBinExpr = [
+      "fetchurl {",
+      `  url = "https://repo.yarnpkg.com/${YarnVersion!}/packages/yarnpkg-cli/bin/yarn.js";`,
+      `  hash = "${sriHash}";`,
+      "}",
+    ].join("\n  ");
+  } else if (yarnPathAbs.startsWith(cwd)) {
+    yarnBinExpr =
       "./" + ppath.relative(ppath.dirname(nixExprPath), yarnPathAbs);
   } else {
-    yarnPathExpr = json(yarnPathAbs);
+    yarnBinExpr = json(yarnPathAbs);
     report.reportWarning(
       0,
       `The Yarn path ${yarnPathAbs} is outside the project - it may not be reachable by the Nix build`,
@@ -65,6 +80,8 @@ export default async (
   let cacheFolderExpr: string;
   if (cacheFolderAbs.startsWith(cwd)) {
     cacheFolderExpr = json(ppath.relative(cwd, cacheFolderAbs));
+  } else if (!isYarn3 && configuration.get(`enableGlobalCache`)) {
+    cacheFolderExpr = '".yarn/cache"';
   } else {
     throw Error(
       `The cache folder ${cacheFolderAbs} is outside the project, this is currently not supported`,
@@ -99,6 +116,7 @@ export default async (
   // Build a list of cache entries so Nix can fetch them.
   // TODO: See if we can use Nix fetchurl for npm: dependencies.
   interface CacheEntry {
+    originalFilename: Filename;
     filename: Filename;
     sha512: string;
   }
@@ -109,7 +127,9 @@ export default async (
   for (const pkg of project.storedPackages.values()) {
     const { locatorHash } = pkg;
     const checksum = project.storedChecksums.get(locatorHash);
-    const cachePath = cache.getLocatorPath(pkg, checksum || null, cacheOptions);
+    const cachePath = isYarn3
+      ? (cache as any).getLocatorPath(pkg, checksum || null, cacheOptions)
+      : cache.getLocatorPath(pkg, checksum || null);
     if (!cachePath) continue;
 
     const filename = ppath.basename(cachePath);
@@ -119,7 +139,13 @@ export default async (
     const sha512 = checksum
       ? checksum.split(`/`).pop()!
       : await hashUtils.checksumFile(cachePath);
-    cacheEntries.set(locatorStr, { filename, sha512 });
+    cacheEntries.set(locatorStr, {
+      originalFilename: filename,
+      // Rebuild the filename, because the cache file we're operating on may be
+      // from the mirror directory, which uses different naming.
+      filename: cache.getChecksumFilename(pkg, sha512),
+      sha512,
+    });
   }
 
   let cacheEntriesCode = `cacheEntries = {\n`;
@@ -290,7 +316,7 @@ export default async (
     const projectName = ident ? structUtils.stringifyIdent(ident) : `workspace`;
     const projectExpr = renderTmpl(projectExprTmpl, {
       PROJECT_NAME: json(projectName),
-      YARN_PATH: yarnPathExpr,
+      YARN_BIN: yarnBinExpr,
       LOCKFILE: lockfileExpr,
       CACHE_FOLDER: cacheFolderExpr,
       CACHE_ENTRIES: cacheEntriesCode,
@@ -321,7 +347,10 @@ export default async (
   ) {
     await xfs.mktempPromise(async (tempDir) => {
       const toPreload: PortablePath[] = [];
-      for (const [locator, { filename, sha512 }] of cacheEntries.entries()) {
+      for (const [
+        locator,
+        { originalFilename, sha512 },
+      ] of cacheEntries.entries()) {
         const name = sanitizeDerivationName(locator);
         // Check to see if the Nix store entry already exists.
         const hash = Buffer.from(sha512, "hex");
@@ -336,7 +365,7 @@ export default async (
           const subdir = ppath.join(tempDir, sha512.slice(0, 7) as Filename);
           await xfs.mkdirPromise(subdir);
 
-          const src = ppath.join(cache.cwd, filename);
+          const src = ppath.join(cache.cwd, originalFilename);
           const dst = ppath.join(subdir, name as Filename);
           await xfs.copyFilePromise(src, dst);
 

src/tmpl/yarn-project.nix.in

@@ -6,6 +6,8 @@
 
 let
 
+  yarnBin = @@YARN_BIN@@;
+
   cacheFolder = @@CACHE_FOLDER@@;
   lockfile = @@LOCKFILE@@;
 
@@ -15,7 +17,7 @@ let
 
   # Simple stub that provides the global yarn command.
   yarn = writeShellScriptBin "yarn" ''
-    exec '${nodejs}/bin/node' '${@@YARN_PATH@@}' "$@"
+    exec '${nodejs}/bin/node' '${yarnBin}' "$@"
   '';
 
   # Common attributes between Yarn derivations.
@@ -42,7 +44,7 @@ let
       source $stdenv/setup
       cd "$src"
       ${buildVars}
-      HOME="$TMP" yarn_cache_folder="$TMP" \
+      HOME="$TMP" yarn_enable_global_cache=false yarn_cache_folder="$TMP" \
         yarn nixify fetch-one $locator
       # Because we change the cache dir, Yarn may generate a different name.
       mv "$TMP/$(sed 's/-[^-]*\.[^-]*$//' <<< "$outputFilename")"-* $out
@@ -89,7 +91,8 @@ let
       echo '{ "dependencies": { "${pname}": "${reference}" } }' > package.json
       install -m 0600 ${lockfile} ./yarn.lock
       export yarn_global_folder="$TMP"
-      export YARN_ENABLE_IMMUTABLE_INSTALLS=false
+      export yarn_enable_global_cache=false
+      export yarn_enable_immutable_installs=false
       yarn --immutable-cache
     '';
 
@@ -123,6 +126,11 @@ let
       # Yarn may need a writable home directory.
       export yarn_global_folder="$TMP"
 
+      # Ensure global cache is disabled. Cache must be part of our output.
+      touch .yarnrc.yml
+      sed -i -e '/^enableGlobalCache/d' .yarnrc.yml
+      echo 'enableGlobalCache: false' >> .yarnrc.yml
+
       # Some node-gyp calls may call out to npm, which could fail due to an
       # read-only home dir.
       export HOME="$TMP"