Initial commit

.gitignore

@@ -0,0 +1,6 @@
+/.yarn/*
+!/.yarn/releases
+!/.yarn/plugins
+/.pnp.*
+
+/dist/yarn-plugin-nixify.dev.js

.yarnrc.yml

@@ -0,0 +1 @@
+yarnPath: ".yarn/releases/yarn-berry.js"

README.md

@@ -0,0 +1,55 @@
+# yarn-plugin-nixify
+
+**WORK IN PROGRESS**
+
+Generates a Nix expression to build a Yarn v2 package.
+
+## Usage
+
+```sh
+yarn plugin import https://raw.githubusercontent.com/stephank/yarn-plugin-nixify/main/dist/yarn-plugin-nixify.js
+yarn
+yarn nixify
+nix-build
+```
+
+The `yarn nixify` command always updates `yarn-project.nix`, but only writes a
+(minimal) `default.nix` if it doesn't exist yet. The `default.nix` is intended
+to be customized, for example:
+
+```nix
+{ pkgs ? import <nixpkgs> {} }:
+
+let
+
+  project = pkgs.callPackage ./yarn-project.nix {
+    # Example of selecting a specific version of Node.js.
+    nodejs = pkgs.nodejs-14_x;
+  };
+
+in project.overrideAttrs (oldAttrs: {
+
+  # Example of adding dependencies to the environment.
+  # Native modules sometimes need these to build.
+  buildInputs = oldAttrs.buildInputs ++ [ python3 ];
+
+  # Example of invoking a build step in your project.
+  buildPhase = ''
+    yarn build
+  '';
+
+})
+```
+
+## Hacking
+
+```sh
+# In this directory:
+yarn
+yarn build-dev
+
+# In your test project:
+yarn plugin import /path/to/yarn-plugin-nixify/dist/yarn-plugin-nixify.dev.js
+```
+
+(Alternatively, add a direct reference in `.yarnrc.yml`.)

build.js

@@ -0,0 +1,106 @@
+#!/usr/bin/env node
+
+// TODO: This borrows from '@yarnpkg/builder', but the only thing we add here
+// is raw-loader for '.in` files.
+
+const webpack = require(`webpack`);
+const { RawSource } = require(`webpack-sources`);
+
+const IS_PROD = process.argv[2] === `-p`;
+const EXTERNALS = [
+  `@yarnpkg/cli`,
+  `@yarnpkg/core`,
+  `@yarnpkg/fslib`,
+  `@yarnpkg/plugin-pnp`,
+  `clipanion`,
+];
+
+const compiler = webpack({
+  context: __dirname,
+  entry: `.`,
+
+  mode: IS_PROD ? `production` : `development`,
+  devtool: false,
+
+  node: {
+    __dirname: false,
+    __filename: false,
+  },
+
+  output: {
+    filename: IS_PROD ? `yarn-plugin-nixify.js` : `yarn-plugin-nixify.dev.js`,
+    libraryTarget: `var`,
+    library: `plugin`,
+  },
+
+  resolve: {
+    extensions: [`.mjs`, `.js`, `.ts`, `.tsx`, `.json`],
+  },
+
+  externals: Object.fromEntries(
+    EXTERNALS.map((name) => [name, `commonjs ${name}`])
+  ),
+
+  module: {
+    rules: [
+      {
+        test: /\.ts$/,
+        loader: `babel-loader`,
+        options: {
+          plugins: [
+            [`@babel/plugin-proposal-decorators`, { legacy: true }],
+            [`@babel/plugin-proposal-class-properties`, { loose: true }],
+          ],
+          presets: ["@babel/preset-typescript"],
+        },
+      },
+      {
+        test: /\.in$/,
+        loader: `raw-loader`,
+      },
+    ],
+  },
+
+  plugins: [
+    // This plugin wraps the generated bundle so that it doesn't actually
+    // get evaluated right now - until after we give it a custom require
+    // function that will be able to fetch the dynamic modules.
+    {
+      apply: (compiler) => {
+        compiler.hooks.compilation.tap(`WrapYarn`, (compilation) => {
+          compilation.hooks.processAssets.tap(
+            {
+              name: `WrapYarn`,
+              stage: webpack.Compilation.PROCESS_ASSETS_STAGE_ADDITIONS,
+            },
+            (assets) => {
+              for (const file in assets) {
+                assets[file] = new RawSource(
+                  [
+                    `module.exports = {`,
+                    `name: "yarn-plugin-nixify",`,
+                    `factory: function (require) {`,
+                    assets[file].source(),
+                    `return plugin;`,
+                    `}`,
+                    `};`,
+                  ].join(`\n`)
+                );
+              }
+            }
+          );
+        });
+      },
+    },
+  ],
+});
+
+compiler.run((err, stats) => {
+  if (err) {
+    console.error(err);
+    process.exit(1);
+  } else if (stats && stats.compilation.errors.length > 0) {
+    console.error(stats.toString(`errors-only`));
+    process.exit(1);
+  }
+});

package.json

@@ -0,0 +1,27 @@
+{
+  "name": "yarn-plugin-nixify",
+  "version": "0.0.0",
+  "main": "./src/index.ts",
+  "devDependencies": {
+    "@babel/core": "^7.11.4",
+    "@babel/plugin-proposal-class-properties": "^7.10.4",
+    "@babel/plugin-proposal-decorators": "^7.10.5",
+    "@babel/preset-typescript": "^7.10.4",
+    "@types/node": "12",
+    "@yarnpkg/cli": "^2.1.1",
+    "@yarnpkg/core": "^2.1.1",
+    "@yarnpkg/fslib": "^2.1.0",
+    "@yarnpkg/plugin-pnp": "^2.1.0",
+    "babel-loader": "^8.1.0",
+    "clipanion": "^2.4.4",
+    "prettier": "^2.0.5",
+    "raw-loader": "^4.0.1",
+    "typescript": "^4.0.2",
+    "webpack": "next",
+    "webpack-sources": "^1.4.3"
+  },
+  "scripts": {
+    "build": "./build.js -p",
+    "build-dev": "./build.js"
+  }
+}

src/bin-wrapper.sh.in

@@ -0,0 +1,2 @@
+#!/bin/sh
+exec node -r '@@PNP_PATH@@' '@@SCRIPT_PATH@@' "$@"

src/default.nix.in

@@ -0,0 +1,3 @@
+{ pkgs ? import <nixpkgs> {} }:
+
+pkgs.callPackage ./yarn-project.nix {}

src/index.ts

@@ -0,0 +1,241 @@
+import { Command } from "clipanion";
+import { Filename, npath, ppath, xfs } from "@yarnpkg/fslib";
+import { getPnpPath } from "@yarnpkg/plugin-pnp";
+
+import {
+  Cache,
+  CommandContext,
+  Configuration,
+  Plugin,
+  Project,
+  Report,
+  StreamReport,
+  execUtils,
+  structUtils,
+} from "@yarnpkg/core";
+
+import binTmpl from "./bin-wrapper.sh.in";
+import defaultExprTmpl from "./default.nix.in";
+import projectExprTmpl from "./yarn-project.nix.in";
+
+// Actual body of NixifyCommand.
+const generate = async (project: Project, report: Report) => {
+  const { configuration, cwd } = project;
+  const yarnPathAbs = configuration.get(`yarnPath`);
+  const lockfileFilename = configuration.get(`lockfileFilename`);
+
+  // TODO: Should try to remove this. Our binary wrappers currently do
+  // `node -r .pnp.js <bin>`, but not even sure if that's supported.
+  if (configuration.get(`nodeLinker`) !== `pnp`) {
+    report.reportError(
+      0,
+      `Only 'pnp' is currently supported for the 'nodeLinker' setting.`
+    );
+    return;
+  }
+
+  const yarnPath = ppath.relative(cwd, yarnPathAbs);
+  if (yarnPath.startsWith(".")) {
+    report.reportError(
+      0,
+      `The Yarn path ${yarnPathAbs} is outside the project directory - it cannot be reached by the Nix build`
+    );
+    return;
+  }
+
+  // Simple check to see if the project has been setup, so we know the Yarn
+  // cache is populated. TODO: Can we also see if it is up-to-date?
+  if (
+    !xfs.existsSync(ppath.join(cwd, lockfileFilename)) ||
+    !xfs.existsSync(getPnpPath(project).main)
+  ) {
+    report.reportError(
+      0,
+      `The project in ${cwd}/package.json doesn't seem to have been installed - running an install there might help`
+    );
+    return;
+  }
+
+  // List files the derivation will depend on.
+  const yarnClosureInput = new Set([
+    `package.json`,
+    lockfileFilename,
+    yarnPath,
+  ]);
+  for (const source of configuration.sources.values()) {
+    if (!source.startsWith(`<`)) {
+      yarnClosureInput.add(source);
+    }
+  }
+  // TODO: Better way to find plugins? Re-parse rcfiles maybe?
+  const pluginsDir = ppath.join(cwd, `.yarn/plugins` as Filename);
+  if (xfs.existsSync(pluginsDir)) {
+    for (const filename of xfs.readdirSync(pluginsDir)) {
+      yarnClosureInput.add(ppath.join(pluginsDir, filename));
+    }
+  }
+
+  // Build Nix `filterSource` entries to match on.
+  const yarnClosureEntries = new Set();
+  for (const inputPath of yarnClosureInput) {
+    // Filter paths not reachable during the build and warn. (These are often
+    // just user global configuration files, but the warnings can help highlight
+    // dependencies on private registries.)
+    const relativePath = ppath.relative(cwd, inputPath);
+    if (relativePath.startsWith(`..`)) {
+      report.reportWarning(
+        0,
+        `The path ${inputPath} was ignored, because it cannot be reached by the Nix build`
+      );
+      continue;
+    }
+    // Add the file itself.
+    yarnClosureEntries.add(`regular:${relativePath}`);
+    // Add directories leading up to the file.
+    let dir = ppath.dirname(relativePath);
+    while (dir !== `.`) {
+      yarnClosureEntries.add(`directory:${dir}`);
+      dir = ppath.dirname(dir);
+    }
+  }
+
+  // Build the Nix output-hash by hashing the Yarn cache folder. The
+  // derivation should build the exact same.
+  const cacheFolder = configuration.get(`cacheFolder`);
+  const hasherResult = await execUtils.execvp(
+    `nix-hash`,
+    [`--type`, `sha256`, `--base32`, cacheFolder],
+    { cwd, encoding: `utf8`, strict: true }
+  );
+  const cacheHash = hasherResult.stdout.trim();
+
+  // Render the Nix expression.
+  const ident = project.topLevelWorkspace.manifest.name;
+  const projectName = ident ? structUtils.stringifyIdent(ident) : "workspace";
+  const projectExpr = projectExprTmpl
+    .replace(`@@PROJECT_NAME@@`, JSON.stringify(projectName))
+    .replace(`@@OFFLINE_CACHE_HASH@@`, JSON.stringify(cacheHash))
+    .replace(`@@YARN_PATH@@`, JSON.stringify(yarnPath))
+    .replace(
+      `@@YARN_CLOSURE_ENTRIES@@`,
+      `[ ` +
+        [...yarnClosureEntries]
+          .map((entry) => JSON.stringify(entry))
+          .join(` `) +
+        ` ]`
+    );
+  const projectExprPath = ppath.join(cwd, `yarn-project.nix` as Filename);
+  xfs.writeFileSync(projectExprPath, projectExpr);
+  report.reportInfo(0, `Wrote: ${projectExprPath}`);
+
+  // Create a wrapper if it does not exist yet.
+  const defaultExprPath = ppath.join(cwd, `default.nix` as Filename);
+  if (!xfs.existsSync(defaultExprPath)) {
+    xfs.writeFileSync(defaultExprPath, defaultExprTmpl);
+    report.reportInfo(0, `Wrote: ${defaultExprPath}`);
+  }
+};
+
+// Command that generates Nix expressions to setup the project.
+// TODO: Automate by running on every install instead? There's a hook in Yarn,
+// but it doesn't tell us if the persist flag is set.
+class NixifyCommand extends Command<CommandContext> {
+  @Command.Path(`nixify`)
+  async execute() {
+    const configuration = await Configuration.find(
+      this.context.cwd,
+      this.context.plugins
+    );
+    const { project } = await Project.find(configuration, this.context.cwd);
+
+    const report = await StreamReport.start(
+      { configuration, stdout: this.context.stdout },
+      async (report) => generate(project, report)
+    );
+
+    return report.exitCode();
+  }
+}
+
+// Internal command that does just the fetch part of `yarn install`.
+// Used inside the Nix offline-cache derivation to build the cache.
+class BuildCacheCommand extends Command<CommandContext> {
+  @Command.String()
+  out: string = ``;
+
+  @Command.Path(`nixify`, `build-cache`)
+  async execute() {
+    const configuration = await Configuration.find(
+      this.context.cwd,
+      this.context.plugins
+    );
+
+    // Overwrite the cache directory to our output directory.
+    configuration.use(
+      `<nixify>`,
+      { cacheFolder: this.out },
+      configuration.projectCwd!,
+      { overwrite: true }
+    );
+
+    const { project } = await Project.find(configuration, this.context.cwd);
+    const cache = await Cache.find(configuration);
+
+    // Run resolution and fetch steps.
+    const report = await StreamReport.start(
+      { configuration, stdout: this.context.stdout },
+      async (report) => {
+        await report.startTimerPromise(`Resolution step`, () =>
+          project.resolveEverything({ report, lockfileOnly: true })
+        );
+        await report.startTimerPromise(`Fetch step`, () =>
+          project.fetchEverything({ cache, report })
+        );
+      }
+    );
+
+    return report.exitCode();
+  }
+}
+
+// Internal command that creates wrappers for binaries.
+// Used inside the Nix install phase.
+class InstallBinCommand extends Command<CommandContext> {
+  @Command.String()
+  binDir: string = ``;
+
+  @Command.Path(`nixify`, `install-bin`)
+  async execute() {
+    const configuration = await Configuration.find(
+      this.context.cwd,
+      this.context.plugins
+    );
+    const { project, workspace } = await Project.find(
+      configuration,
+      this.context.cwd
+    );
+
+    if (workspace) {
+      const binDir = npath.toPortablePath(this.binDir);
+      const pnpPath = getPnpPath(project).main;
+      for (const [name, scriptInput] of workspace.manifest.bin) {
+        const binPath = ppath.join(binDir, name as Filename);
+        const scriptPath = ppath.join(
+          project.cwd,
+          npath.toPortablePath(scriptInput)
+        );
+        const script = binTmpl
+          .replace(`@@PNP_PATH@@`, pnpPath)
+          .replace(`@@SCRIPT_PATH@@`, scriptPath);
+        xfs.writeFileSync(binPath, script);
+        xfs.chmodSync(binPath, 0o755);
+      }
+    }
+  }
+}
+
+const plugin: Plugin = {
+  commands: [NixifyCommand, BuildCacheCommand, InstallBinCommand],
+};
+
+export default plugin;

src/types.d.ts

@@ -0,0 +1,4 @@
+declare module "*.in" {
+  const content: string;
+  export default content;
+}

src/yarn-project.nix.in

@@ -0,0 +1,108 @@
+{ lib, bash, nodejs, perl, stdenv, writeText }:
+
+with lib;
+
+let
+
+  # Variables provided by the generator.
+  project-name = @@PROJECT_NAME@@;
+  offline-cache-hash = @@OFFLINE_CACHE_HASH@@;
+  yarn-path = @@YARN_PATH@@;
+  yarn-closure-entries = @@YARN_CLOSURE_ENTRIES@@;
+
+  # Defines the shell alias to run Yarn.
+  yarn-alias = ''
+    yarn() {
+      CI=1 "$NIX_YARN_PATH" "$@"
+    }
+  '';
+
+  # Directory of just the files needed to run Yarn.
+  yarn-closure = cleanSourceWith {
+    src = ./.;
+    filter = let
+      srcStr = toString ./.;
+      srcRel = removePrefix "${srcStr}/";
+    in path: type:
+      elem "${type}:${srcRel path}" yarn-closure-entries;
+  };
+
+  # Build just the offline cache for the project.
+  offline-cache = stdenv.mkDerivation {
+    name = "${project-name}-offline-cache";
+    buildInputs = [ nodejs ];
+    builder = writeText "builder.sh" ''
+      source $stdenv/setup
+      cd ${yarn-closure}
+
+      # Yarn may need a writable home directory for the global cache mirror.
+      # TODO: Can't disable the mirror, because it changes cache filenames.
+      export HOME="$TEMP"
+
+      # Setup to environment so we can run Yarn.
+      export NIX_YARN_PATH="$PWD/${yarn-path}"
+      ${yarn-alias}
+
+      # Invoke a plugin internal command to build the cache.
+      yarn nixify build-cache $out
+    '';
+    outputHashMode = "recursive";
+    outputHashAlgo = "sha256";
+    outputHash = offline-cache-hash;
+  };
+
+in stdenv.mkDerivation {
+  name = project-name;
+  src = ./.;
+
+  # Tell node-gyp to use the provided Node.js headers for native code builds.
+  npm_config_nodedir = nodejs;
+  # Tell node-pre-gyp to never fetch binaries / always build from source.
+  npm_config_build_from_source = "true";
+
+  # Make sure the build uses the right Node.js version everywhere.
+  buildInputs = [ nodejs ];
+
+  # Define the Yarn alias in the build environment.
+  postHook = yarn-alias;
+
+  configurePhase = ''
+    runHook preConfigure
+
+    # Move the entire project to the output directory.
+    # TODO: Would rather do this in 'installPhase',
+    # but '.pnp.js' is generated with relative paths.
+    mkdir -p $out/libexec $out/bin
+    mv $PWD "$out/libexec/$sourceRoot"
+    cd "$out/libexec/$sourceRoot"
+
+    # Store the absolute path to Yarn for the 'yarn' alias.
+    export NIX_YARN_PATH="$PWD/${yarn-path}"
+
+    # Point Yarn to the offline cache built separately.
+    yarn config set cacheFolder '${offline-cache}'
+
+    # Run normal Yarn install to complete dependency installation.
+    yarn install --immutable --immutable-cache
+
+    runHook postConfigure
+  '';
+
+  buildPhase = ''
+    runHook preBuild
+    runHook postBuild
+  '';
+
+  installPhase = ''
+    runHook preInstall
+
+    # Invoke a plugin internal command to setup binaries.
+    yarn nixify install-bin $out/bin
+
+    runHook postInstall
+  '';
+
+  passthru = {
+    inherit yarn-closure offline-cache;
+  };
+}

tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "compilerOptions": {
+    "experimentalDecorators": true,
+    "module": "commonjs",
+    "target": "es2018",
+    "lib": ["es2018"],
+    "strict": true
+  },
+  "include": ["src/**/*.ts"]
+}