feat(prepare-sources): implement generate-manifests module (RHIDP-15834)

.sonarcloud.properties

@@ -1,2 +1,4 @@
 # Test fixtures are not production code; skip analysis.
 sonar.exclusions=prepare-sources/**/__fixtures__/**/*
+# Code duplication detection usually doesn't make sense in test files.
+sonar.cpd.exclusions=prepare-sources/lib/**/*.test.ts

prepare-sources/README.md

@@ -27,7 +27,7 @@ node lib/cli.ts \
 | --- | ------------------------- | ------------------------------------------------------------------------------ | --------------- |
 | 1   | `seed-frontend-lockfiles` | Seed `dist-dynamic/yarn.lock` for frontend plugins after the initial export    | Not implemented |
 | 2   | `make-self-contained`     | Merge repo-root `.yarn/` and `.yarnrc.yml` into the workspace (non-flat repos) | Implemented     |
-| 3   | `generate-manifests`      | Produce `manifest.json` and `backstage-manifest.json` for protocol resolution  | Not implemented |
+| 3   | `generate-manifests`      | Produce `manifest.json` and `backstage-manifest.json` for protocol resolution  | Implemented     |
 | 4   | `plugin-removal`          | Remove unsupported/community plugins and update `plugins-list.yaml`            | Not implemented |
 | 5   | `file-cleanup`            | Strip test files, mocks, stories, and dev-only artifacts                       | Not implemented |
 | 6   | `protocol-resolution`     | Resolve `workspace:^` and `backstage:^` protocols; generate `type-shims`       | Not implemented |
@@ -94,3 +94,7 @@ Run tests:
 ```bash
 vp test
 ```
+
+## TODO
+
+Remove `overrides.@yarnpkg/core.got` when [berry#7282](https://github.com/yarnpkg/berry/pull/7282) lands on npm.

prepare-sources/lib/manifest-types.ts

@@ -0,0 +1,39 @@
+/**
+ * Shape of `manifest.json` — inventory of all workspace packages.
+ * Written by `generate-manifests`, consumed by `protocol-resolution`.
+ */
+export type WorkspaceManifest = {
+  packages: WorkspacePackageEntry[];
+};
+
+export type WorkspacePackageEntry = {
+  name: string;
+  version: string;
+  path?: string;
+  dependencies?: Record<string, string>;
+  peerDependencies?: Record<string, string>;
+  peerDependenciesMeta?: Record<string, Record<string, unknown>>;
+  optionalDependencies?: Record<string, string>;
+  devDependencies?: Record<string, string>;
+  bin?: Record<string, string>;
+};
+
+/**
+ * Shape of `backstage-manifest.json` — Backstage release packages with
+ * dependency metadata. Written by `generate-manifests` (only when the
+ * workspace has `backstage:^` deps), consumed by `protocol-resolution`.
+ */
+export type BackstageManifest = {
+  backstageVersion: string;
+  packages: BackstagePackageEntry[];
+};
+
+export type BackstagePackageEntry = {
+  name: string;
+  version: string;
+  dependencies?: Record<string, string>;
+  peerDependencies?: Record<string, string>;
+  peerDependenciesMeta?: Record<string, Record<string, unknown>>;
+  optionalDependencies?: Record<string, string>;
+  bin?: Record<string, string>;
+};

prepare-sources/lib/modules.ts

@@ -1,6 +1,7 @@
 import type { ModuleContext, PipelineModule } from "./pipeline.ts";
 import { run as hermeticPrep } from "./modules/hermetic-prep/index.ts";
 import { run as makeSelfContained } from "./modules/make-self-contained/index.ts";
+import { run as generateManifests } from "./modules/generate-manifests/index.ts";
 
 async function notImplemented(ctx: ModuleContext): Promise<void> {
   ctx.log("not yet implemented");
@@ -10,7 +11,7 @@ async function notImplemented(ctx: ModuleContext): Promise<void> {
 export const MODULES: readonly PipelineModule[] = [
   { name: "seed-frontend-lockfiles", run: notImplemented },
   { name: "make-self-contained", run: makeSelfContained },
-  { name: "generate-manifests", run: notImplemented },
+  { name: "generate-manifests", run: generateManifests },
   { name: "plugin-removal", run: notImplemented },
   { name: "file-cleanup", run: notImplemented },
   { name: "protocol-resolution", run: notImplemented },

prepare-sources/lib/modules/generate-manifests/README.md

@@ -0,0 +1,110 @@
+# generate-manifests
+
+Builds `manifest.json` and (when needed) `backstage-manifest.json`: package
+inventories used to resolve `workspace:^` and `backstage:^` to concrete semver
+ranges.
+
+## Problem
+
+Downstream steps need a name → version (and dependency metadata) lookup for
+every workspace package and for Backstage release packages. That snapshot must
+be captured while all workspace packages are still on disk, because entries in
+`package.json` and `yarn.lock` can still reference packages that are removed
+later in the pipeline.
+
+## What the module does
+
+### `manifest.json`
+
+Inventory of all local workspace packages. Always generated.
+
+```json
+{
+  "packages": [
+    {
+      "name": "@scope/plugin-foo",
+      "version": "1.2.3",
+      "path": "plugins/plugin-foo/package.json",
+      "dependencies": { "@scope/plugin-bar": "workspace:^" },
+      "peerDependencies": { "react": "^18.0.0" },
+      "peerDependenciesMeta": { "react": { "optional": true } },
+      "optionalDependencies": { "@emotion/react": "^11.0.0" },
+      "devDependencies": { "@types/react": "^18.0.0" },
+      "bin": { "foo": "./dist/cli.js" }
+    }
+  ]
+}
+```
+
+All dependency fields are optional — only included when non-empty.
+
+**Package discovery:** Packages are discovered via the `workspaces` field in
+the root `package.json` (glob resolution), plus the root package itself. For
+flat repos (no `workspaces` field), only the root package is included.
+
+**Filtering:** Glob matches whose path segments include any of `node_modules`,
+`dist-dynamic`, `dist-scalprum`, `dist`, or `build` are excluded. This
+prevents build output directories from polluting the manifest.
+
+**`backstage:^` detection:** The module checks `dependencies`,
+`devDependencies`, `peerDependencies`, and `optionalDependencies` across all
+packages for `backstage:^` values. If none are found, `backstage-manifest.json`
+is skipped entirely.
+
+### `backstage-manifest.json`
+
+Backstage release packages with dependency metadata. Only generated when the
+workspace has `backstage:^` dependencies. Requires `backstage.json` at the
+workspace root (throws if missing).
+
+```json
+{
+  "backstageVersion": "1.42.5",
+  "packages": [
+    {
+      "name": "@backstage/core-plugin-api",
+      "version": "1.10.9",
+      "dependencies": { "@backstage/types": "npm:^1.2.3" },
+      "peerDependencies": { "react": "npm:^18.0.0" },
+      "peerDependenciesMeta": { "react": { "optional": true } },
+      "optionalDependencies": { "@emotion/react": "npm:^11.0.0" },
+      "bin": { "backstage-core": "./dist/cli.js" }
+    }
+  ]
+}
+```
+
+Note that dependency values in the backstage manifest use the Yarn-resolved
+format (`npm:^X.Y.Z`), since they come from `yarn.lock` rather than
+`package.json`.
+
+Neither file is included in the OCI artifact — they are build-time
+intermediates.
+
+## Why Backstage metadata from `yarn.lock`
+
+`yarn.lock` already holds resolved `@backstage/*` versions and dependency
+metadata after install. The [Backstage Yarn plugin](https://github.com/backstage/backstage/tree/master/packages/yarn-plugin) maps each `backstage:^`
+dependency to a concrete npm version (from `backstage.json` and the release
+manifest on `versions.backstage.io`); Yarn records the result in the lockfile.
+This module extracts that metadata from the lockfile for
+`backstage-manifest.json`.
+
+**Benefits:**
+
+- **No npm registry fan-out.** Dependency metadata for hundreds of
+  `@backstage/*` packages comes from the lockfile, not per-package registry
+  requests.
+- **Workspace-accurate.** The manifest reflects versions and dependency trees
+  Yarn actually resolved for this repo, not a generic registry view.
+
+**Release-line validation:** `backstage.json` names a single Backstage release
+(for example `1.42.5`). The manifest at
+`versions.backstage.io/v1/releases/<version>/manifest.json` lists every
+`@backstage/*` package version on that line. After extraction, the module
+fetches that manifest once and checks each lockfile package against it. A
+mismatch means the lockfile is not a coherent set for the claimed release —
+for example `backstage.json` was bumped without `yarn install`, a bad merge in
+`yarn.lock`, or a `resolutions` entry pinning a package off the release line.
+The build fails with per-package details instead of writing a
+`backstage-manifest.json` that downstream resolution would trust.

prepare-sources/lib/modules/generate-manifests/__fixtures__/backstage-json-no-version-error/error

@@ -0,0 +1 @@
+/no 'version' field/

prepare-sources/lib/modules/generate-manifests/__fixtures__/backstage-json-no-version-error/input/overlay/source.json

@@ -0,0 +1,6 @@
+{
+  "repo": "https://github.com/example/repo",
+  "repo-ref": "abc123",
+  "repo-flat": false,
+  "repo-backstage-version": "1.45.1"
+}

prepare-sources/lib/modules/generate-manifests/__fixtures__/backstage-json-no-version-error/input/workspace/backstage.json

@@ -0,0 +1,3 @@
+{
+  "description": "backstage.json without a version field"
+}

prepare-sources/lib/modules/generate-manifests/__fixtures__/backstage-json-no-version-error/input/workspace/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "@test/workspace-root",
+  "version": "1.0.0",
+  "private": true,
+  "workspaces": [
+    "plugins/*"
+  ]
+}

prepare-sources/lib/modules/generate-manifests/__fixtures__/backstage-json-no-version-error/input/workspace/plugins/my-plugin/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "@test/my-plugin",
+  "version": "1.0.0",
+  "dependencies": {
+    "@backstage/core-plugin-api": "backstage:^"
+  }
+}

prepare-sources/lib/modules/generate-manifests/__fixtures__/flat-repo-single-package/input/overlay/source.json

@@ -0,0 +1,6 @@
+{
+  "repo": "https://github.com/example/repo",
+  "repo-ref": "abc123",
+  "repo-flat": false,
+  "repo-backstage-version": "1.45.1"
+}

prepare-sources/lib/modules/generate-manifests/__fixtures__/flat-repo-single-package/input/workspace/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "@test/standalone-plugin",
+  "version": "3.0.1",
+  "bin": "./dist/cli.js",
+  "dependencies": {
+    "express": "^4.18.0"
+  }
+}

prepare-sources/lib/modules/generate-manifests/__fixtures__/flat-repo-single-package/output/workspace/manifest.json

@@ -0,0 +1,15 @@
+{
+  "packages": [
+    {
+      "name": "@test/standalone-plugin",
+      "version": "3.0.1",
+      "path": "package.json",
+      "dependencies": {
+        "express": "^4.18.0"
+      },
+      "bin": {
+        "standalone-plugin": "./dist/cli.js"
+      }
+    }
+  ]
+}

prepare-sources/lib/modules/generate-manifests/__fixtures__/flat-repo-single-package/output/workspace/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "@test/standalone-plugin",
+  "version": "3.0.1",
+  "bin": "./dist/cli.js",
+  "dependencies": {
+    "express": "^4.18.0"
+  }
+}

prepare-sources/lib/modules/generate-manifests/__fixtures__/missing-yarn-lock-error/error

@@ -0,0 +1 @@
+/yarn\.lock not found/

prepare-sources/lib/modules/generate-manifests/__fixtures__/missing-yarn-lock-error/input/overlay/source.json

@@ -0,0 +1,6 @@
+{
+  "repo": "https://github.com/example/repo",
+  "repo-ref": "abc123",
+  "repo-flat": false,
+  "repo-backstage-version": "1.45.1"
+}

prepare-sources/lib/modules/generate-manifests/__fixtures__/missing-yarn-lock-error/input/workspace/backstage.json

@@ -0,0 +1,3 @@
+{
+  "version": "1.42.5"
+}

prepare-sources/lib/modules/generate-manifests/__fixtures__/missing-yarn-lock-error/input/workspace/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "@test/workspace-root",
+  "version": "1.0.0",
+  "private": true,
+  "workspaces": [
+    "plugins/*"
+  ]
+}

prepare-sources/lib/modules/generate-manifests/__fixtures__/missing-yarn-lock-error/input/workspace/plugins/my-plugin/package.json

@@ -0,0 +1,7 @@
+{
+  "name": "@test/my-plugin",
+  "version": "1.0.0",
+  "dependencies": {
+    "@backstage/core-plugin-api": "backstage:^"
+  }
+}

prepare-sources/lib/modules/generate-manifests/__fixtures__/monorepo-no-backstage/input/overlay/source.json

@@ -0,0 +1,6 @@
+{
+  "repo": "https://github.com/example/repo",
+  "repo-ref": "abc123",
+  "repo-flat": false,
+  "repo-backstage-version": "1.45.1"
+}

prepare-sources/lib/modules/generate-manifests/__fixtures__/monorepo-no-backstage/input/workspace/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "@test/workspace-root",
+  "version": "1.0.0",
+  "private": true,
+  "workspaces": [
+    "plugins/*"
+  ]
+}

prepare-sources/lib/modules/generate-manifests/__fixtures__/monorepo-no-backstage/input/workspace/plugins/plugin-alpha/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "@test/plugin-alpha",
+  "version": "2.1.0",
+  "bin": {
+    "alpha-tool": "./dist/cli.js"
+  },
+  "dependencies": {
+    "@test/plugin-beta": "workspace:^",
+    "react": "^18.0.0"
+  },
+  "devDependencies": {
+    "@types/react": "^18.0.0",
+    "typescript": "~5.3.0"
+  },
+  "peerDependencies": {
+    "react-dom": "^18.0.0"
+  },
+  "peerDependenciesMeta": {
+    "react-dom": {
+      "optional": true
+    }
+  },
+  "optionalDependencies": {
+    "fsevents": "^2.3.0"
+  }
+}

prepare-sources/lib/modules/generate-manifests/__fixtures__/monorepo-no-backstage/input/workspace/plugins/plugin-beta/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "@test/plugin-beta",
+  "version": "1.3.0",
+  "bin": "./dist/index.js",
+  "dependencies": {
+    "lodash": "^4.17.21"
+  }
+}

prepare-sources/lib/modules/generate-manifests/__fixtures__/monorepo-no-backstage/output/workspace/manifest.json

@@ -0,0 +1,47 @@
+{
+  "packages": [
+    {
+      "name": "@test/plugin-alpha",
+      "version": "2.1.0",
+      "path": "plugins/plugin-alpha/package.json",
+      "dependencies": {
+        "@test/plugin-beta": "workspace:^",
+        "react": "^18.0.0"
+      },
+      "peerDependencies": {
+        "react-dom": "^18.0.0"
+      },
+      "peerDependenciesMeta": {
+        "react-dom": {
+          "optional": true
+        }
+      },
+      "optionalDependencies": {
+        "fsevents": "^2.3.0"
+      },
+      "devDependencies": {
+        "@types/react": "^18.0.0",
+        "typescript": "~5.3.0"
+      },
+      "bin": {
+        "alpha-tool": "./dist/cli.js"
+      }
+    },
+    {
+      "name": "@test/plugin-beta",
+      "version": "1.3.0",
+      "path": "plugins/plugin-beta/package.json",
+      "dependencies": {
+        "lodash": "^4.17.21"
+      },
+      "bin": {
+        "plugin-beta": "./dist/index.js"
+      }
+    },
+    {
+      "name": "@test/workspace-root",
+      "version": "1.0.0",
+      "path": "package.json"
+    }
+  ]
+}