feat(prepare-sources): implement file-cleanup module (RHIDP-15836)

.sonarcloud.properties

@@ -1,2 +1,4 @@
 # Test fixtures are not production code; skip analysis.
 sonar.exclusions=prepare-sources/**/__fixtures__/**/*
+# Code duplication detection usually doesn't make sense in test files.
+sonar.cpd.exclusions=prepare-sources/lib/**/*.test.ts

prepare-sources/lib/cli.test.ts

@@ -90,6 +90,7 @@ describe("main", () => {
     const inputs = {
       workspacePath: "/ws",
       overlayPath: "/ov",
+      overlayRepoRoot: "/",
       source: {
         repo: "https://github.com/example/repo",
         "repo-ref": "main",

prepare-sources/lib/manifest-types.ts

@@ -0,0 +1,39 @@
+/**
+ * Shape of `manifest.json` — inventory of all workspace packages.
+ * Written by `generate-manifests`, consumed by `protocol-resolution`.
+ */
+export type WorkspaceManifest = {
+  packages: WorkspacePackageEntry[];
+};
+
+export type WorkspacePackageEntry = {
+  name: string;
+  version: string;
+  path?: string;
+  dependencies?: Record<string, string>;
+  peerDependencies?: Record<string, string>;
+  peerDependenciesMeta?: Record<string, Record<string, unknown>>;
+  optionalDependencies?: Record<string, string>;
+  devDependencies?: Record<string, string>;
+  bin?: Record<string, string>;
+};
+
+/**
+ * Shape of `backstage-manifest.json` — Backstage release packages with
+ * dependency metadata. Written by `generate-manifests` (only when the
+ * workspace has `backstage:^` deps), consumed by `protocol-resolution`.
+ */
+export type BackstageManifest = {
+  backstageVersion: string;
+  packages: BackstagePackageEntry[];
+};
+
+export type BackstagePackageEntry = {
+  name: string;
+  version: string;
+  dependencies?: Record<string, string>;
+  peerDependencies?: Record<string, string>;
+  peerDependenciesMeta?: Record<string, Record<string, unknown>>;
+  optionalDependencies?: Record<string, string>;
+  bin?: Record<string, string>;
+};

prepare-sources/lib/modules.test.ts

@@ -29,6 +29,7 @@ describe("MODULES", () => {
     await stub.run({
       workspacePath: "/tmp/ws",
       overlayPath: "/tmp/overlay",
+      overlayRepoRoot: "/tmp",
       source: {
         repo: "https://github.com/example/repo",
         "repo-ref": "main",

prepare-sources/lib/modules.ts

@@ -1,6 +1,9 @@
 import type { ModuleContext, PipelineModule } from "./pipeline.ts";
 import { run as hermeticPrep } from "./modules/hermetic-prep/index.ts";
 import { run as makeSelfContained } from "./modules/make-self-contained/index.ts";
+import { run as generateManifests } from "./modules/generate-manifests/index.ts";
+import { run as pluginRemoval } from "./modules/plugin-removal/index.ts";
+import { run as fileCleanup } from "./modules/file-cleanup/index.ts";
 
 async function notImplemented(ctx: ModuleContext): Promise<void> {
   ctx.log("not yet implemented");
@@ -10,9 +13,9 @@ async function notImplemented(ctx: ModuleContext): Promise<void> {
 export const MODULES: readonly PipelineModule[] = [
   { name: "seed-frontend-lockfiles", run: notImplemented },
   { name: "make-self-contained", run: makeSelfContained },
-  { name: "generate-manifests", run: notImplemented },
-  { name: "plugin-removal", run: notImplemented },
-  { name: "file-cleanup", run: notImplemented },
+  { name: "generate-manifests", run: generateManifests },
+  { name: "plugin-removal", run: pluginRemoval },
+  { name: "file-cleanup", run: fileCleanup },
   { name: "protocol-resolution", run: notImplemented },
   { name: "package-cleanup", run: notImplemented },
   { name: "hermetic-prep", run: hermeticPrep },

prepare-sources/lib/modules/file-cleanup/README.md

@@ -0,0 +1,70 @@
+# file-cleanup
+
+Removes test, mock, and dev-only files from surviving workspace packages after
+`plugin-removal`. Reduces source size and avoids TypeScript errors from test
+files that reference devDependencies removed during scrubbing.
+
+**Pipeline position:** after `plugin-removal`, before `protocol-resolution` —
+see `modules.ts` for the current ordering.
+
+## Problem
+
+Surviving plugin directories still contain test specs, mock directories, and
+local dev utilities. Downstream Konflux builds do not run tests; these files
+only add noise and can fail `tsc` when they import types from scrubbed packages
+(for example `@types/jest`).
+
+## What the module does
+
+Recursively scans `ctx.workspacePath` and removes:
+
+| Kind                      | Patterns                                             |
+| ------------------------- | ---------------------------------------------------- |
+| Directories (entire tree) | `dev/`, `e2e-tests/`, `__tests__/`, `__mocks__/`     |
+| Files                     | `*.test.ts`, `*.test.tsx`, `*.spec.ts`, `*.spec.tsx` |
+
+Paths under `node_modules/` or `dist-dynamic/` are never touched.
+
+## Inputs and outputs
+
+| Input              | Location            |
+| ------------------ | ------------------- |
+| Scrubbed workspace | `ctx.workspacePath` |
+
+| Output            | Location                       |
+| ----------------- | ------------------------------ |
+| Cleaned workspace | `ctx.workspacePath` (in place) |
+
+No overlay files are read or written.
+
+## Non-obvious behavior
+
+**Full workspace scan.** The bash original searches the entire workspace tree
+(`$folder_name`), not only `plugins/`. This module does the same — a stray
+`packages/shared/src/foo.test.ts` is removed too.
+
+**Directory before file.** Removable directories are deleted deepest-first so
+nested content is not processed twice.
+
+**dist-dynamic is preserved.** Test files inside `dist-dynamic/` are kept
+because the downstream export step may reference lockfile metadata there; bash
+explicitly excludes `*/dist-dynamic/*`.
+
+## Differences from design-decisions summary
+
+The pipeline table mentions `*.stories.*` and `*.mock.*`; the legacy
+`sync-midstream.sh` `find` command does not remove those. This module follows
+the bash script (lines 917–924), not the broader table.
+
+## Error conditions
+
+This module does not throw for missing paths. An empty workspace is a no-op.
+
+## Tests
+
+Fixture-based I/O tests under `__fixtures__/`. Each subdirectory name is the
+Vitest case title; see the fixture catalog comment in `index.test.ts`.
+
+Typical layout: surviving plugin under `plugins/<name>/` with production
+source, test dirs, and spec files. Include `dist-dynamic/` and `node_modules/`
+subtrees when testing exclusion boundaries.

prepare-sources/lib/modules/file-cleanup/__fixtures__/preserves-files-under-dist-dynamic-and-node-modules/input/overlay/source.json

@@ -0,0 +1,6 @@
+{
+  "repo": "https://github.com/example/repo",
+  "repo-ref": "abc123",
+  "repo-flat": false,
+  "repo-backstage-version": "1.45.1"
+}

prepare-sources/lib/modules/file-cleanup/__fixtures__/preserves-files-under-dist-dynamic-and-node-modules/input/workspace/plugins/my-plugin/dist-dynamic/embedded.test.ts

@@ -0,0 +1 @@
+export const placeholder = true;

prepare-sources/lib/modules/file-cleanup/__fixtures__/preserves-files-under-dist-dynamic-and-node-modules/input/workspace/plugins/my-plugin/src/__tests__/keep-dir.ts

@@ -0,0 +1 @@
+export const placeholder = true;

prepare-sources/lib/modules/file-cleanup/__fixtures__/preserves-files-under-dist-dynamic-and-node-modules/input/workspace/plugins/my-plugin/src/index.test.ts

@@ -0,0 +1 @@
+export const placeholder = true;

prepare-sources/lib/modules/file-cleanup/__fixtures__/preserves-files-under-dist-dynamic-and-node-modules/input/workspace/plugins/my-plugin/src/index.ts

@@ -0,0 +1 @@
+export const plugin = "production";

prepare-sources/lib/modules/file-cleanup/__fixtures__/preserves-files-under-dist-dynamic-and-node-modules/output/workspace/plugins/my-plugin/dist-dynamic/embedded.test.ts

@@ -0,0 +1 @@
+export const placeholder = true;

prepare-sources/lib/modules/file-cleanup/__fixtures__/preserves-files-under-dist-dynamic-and-node-modules/output/workspace/plugins/my-plugin/src/index.ts

@@ -0,0 +1 @@
+export const plugin = "production";

prepare-sources/lib/modules/file-cleanup/__fixtures__/removes-test-and-dev-files-from-surviving-plugins/input/overlay/source.json

@@ -0,0 +1,6 @@
+{
+  "repo": "https://github.com/example/repo",
+  "repo-ref": "abc123",
+  "repo-flat": false,
+  "repo-backstage-version": "1.45.1"
+}

prepare-sources/lib/modules/file-cleanup/__fixtures__/removes-test-and-dev-files-from-surviving-plugins/input/workspace/plugins/my-plugin/__mocks__/mock.ts

@@ -0,0 +1 @@
+export const placeholder = true;

prepare-sources/lib/modules/file-cleanup/__fixtures__/removes-test-and-dev-files-from-surviving-plugins/input/workspace/plugins/my-plugin/dev/utils.ts

@@ -0,0 +1 @@
+export const placeholder = true;

prepare-sources/lib/modules/file-cleanup/__fixtures__/removes-test-and-dev-files-from-surviving-plugins/input/workspace/plugins/my-plugin/e2e-tests/spec.ts

@@ -0,0 +1 @@
+export const placeholder = true;

prepare-sources/lib/modules/file-cleanup/__fixtures__/removes-test-and-dev-files-from-surviving-plugins/input/workspace/plugins/my-plugin/src/__tests__/inside.test.ts

@@ -0,0 +1 @@
+export const placeholder = true;

prepare-sources/lib/modules/file-cleanup/__fixtures__/removes-test-and-dev-files-from-surviving-plugins/input/workspace/plugins/my-plugin/src/__tests__/nested.ts

@@ -0,0 +1 @@
+export const placeholder = true;

prepare-sources/lib/modules/file-cleanup/__fixtures__/removes-test-and-dev-files-from-surviving-plugins/input/workspace/plugins/my-plugin/src/bar.spec.ts

@@ -0,0 +1 @@
+export const placeholder = true;

prepare-sources/lib/modules/file-cleanup/__fixtures__/removes-test-and-dev-files-from-surviving-plugins/input/workspace/plugins/my-plugin/src/foo.spec.tsx

@@ -0,0 +1 @@
+export const placeholder = true;

prepare-sources/lib/modules/file-cleanup/__fixtures__/removes-test-and-dev-files-from-surviving-plugins/input/workspace/plugins/my-plugin/src/foo.test.tsx

@@ -0,0 +1 @@
+export const placeholder = true;

prepare-sources/lib/modules/file-cleanup/__fixtures__/removes-test-and-dev-files-from-surviving-plugins/input/workspace/plugins/my-plugin/src/index.mock.ts

@@ -0,0 +1 @@
+export const placeholder = true;

prepare-sources/lib/modules/file-cleanup/__fixtures__/removes-test-and-dev-files-from-surviving-plugins/input/workspace/plugins/my-plugin/src/index.test.ts

@@ -0,0 +1 @@
+export const placeholder = true;

prepare-sources/lib/modules/file-cleanup/__fixtures__/removes-test-and-dev-files-from-surviving-plugins/input/workspace/plugins/my-plugin/src/index.ts

@@ -0,0 +1 @@
+export const plugin = "production";

prepare-sources/lib/modules/file-cleanup/__fixtures__/removes-test-and-dev-files-from-surviving-plugins/input/workspace/plugins/my-plugin/src/stories/Button.stories.tsx

@@ -0,0 +1 @@
+export const placeholder = true;

prepare-sources/lib/modules/file-cleanup/__fixtures__/removes-test-and-dev-files-from-surviving-plugins/output/workspace/plugins/my-plugin/src/index.mock.ts

@@ -0,0 +1 @@
+export const placeholder = true;

prepare-sources/lib/modules/file-cleanup/__fixtures__/removes-test-and-dev-files-from-surviving-plugins/output/workspace/plugins/my-plugin/src/index.ts

@@ -0,0 +1 @@
+export const plugin = "production";

prepare-sources/lib/modules/file-cleanup/__fixtures__/removes-test-and-dev-files-from-surviving-plugins/output/workspace/plugins/my-plugin/src/stories/Button.stories.tsx

@@ -0,0 +1 @@
+export const placeholder = true;

prepare-sources/lib/modules/file-cleanup/index.test.ts

@@ -0,0 +1,18 @@
+import { describe } from "vite-plus/test";
+
+import { testInputOutputExpectations } from "../../test-utils.ts";
+import { run } from "./index.ts";
+
+/**
+ * Fixture catalog (`__fixtures__/<name>/` — name doubles as the Vitest case title):
+ *
+ * - `removes-test-and-dev-files-from-surviving-plugins` — all removable dir names
+ *   (`dev/`, `e2e-tests/`, `__tests__/`, `__mocks__/`), all test/spec file patterns
+ *   (`*.test.ts`, `*.test.tsx`, `*.spec.ts`, `*.spec.tsx`); production source,
+ *   stories, and `*.mock.ts` kept
+ * - `preserves-files-under-dist-dynamic-and-node-modules` — exclusion boundaries
+ *   for dist-dynamic/ and node_modules/
+ */
+describe("file-cleanup", () => {
+  testInputOutputExpectations(import.meta.dirname, run);
+});

prepare-sources/lib/modules/file-cleanup/index.ts

@@ -0,0 +1,78 @@
+import fs from "node:fs";
+import path from "node:path";
+import type { ModuleContext } from "../../pipeline.ts";
+
+/** Directory basenames removed wholesale (sync-midstream.sh find -type d). */
+export const REMOVABLE_DIR_NAMES = new Set(["dev", "e2e-tests", "__tests__", "__mocks__"]);
+
+const TEST_FILE_PATTERNS = [/\.test\.ts$/, /\.test\.tsx$/, /\.spec\.ts$/, /\.spec\.tsx$/];
+
+/** True when a relative workspace path is under node_modules or dist-dynamic. */
+export function isExcludedPath(relativePath: string): boolean {
+  const normalized = relativePath.split(path.sep).join("/");
+  return normalized.includes("/node_modules/") || normalized.includes("/dist-dynamic/");
+}
+
+export function shouldRemoveDir(dirName: string): boolean {
+  return REMOVABLE_DIR_NAMES.has(dirName);
+}
+
+export function shouldRemoveFile(fileName: string): boolean {
+  return TEST_FILE_PATTERNS.some((pattern) => pattern.test(fileName));
+}
+
+function walkEntries(dir: string): string[] {
+  const results: string[] = [];
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    const fullPath = path.join(dir, entry.name);
+    results.push(fullPath);
+    if (entry.isDirectory()) {
+      results.push(...walkEntries(fullPath));
+    }
+  }
+  return results;
+}
+
+export function findRemovablePaths(workspacePath: string): { dirs: string[]; files: string[] } {
+  const dirs: string[] = [];
+  const files: string[] = [];
+
+  for (const entryPath of walkEntries(workspacePath)) {
+    const rel = path.relative(workspacePath, entryPath);
+    if (isExcludedPath(rel)) continue;
+
+    const stat = fs.statSync(entryPath);
+    const baseName = path.basename(entryPath);
+
+    if (stat.isDirectory() && shouldRemoveDir(baseName)) {
+      dirs.push(entryPath);
+    } else if (stat.isFile() && shouldRemoveFile(baseName)) {
+      files.push(entryPath);
+    }
+  }
+
+  dirs.sort((a, b) => b.length - a.length);
+  return { dirs, files };
+}
+
+/**
+ * Removes test, mock, and dev-only files from surviving workspace packages.
+ * Matches sync-midstream.sh lines 917–924.
+ */
+export function run(ctx: ModuleContext): Promise<void> {
+  const { dirs, files } = findRemovablePaths(ctx.workspacePath);
+
+  for (const dir of dirs) {
+    fs.rmSync(dir, { recursive: true, force: true });
+    ctx.log(`removed ${path.relative(ctx.workspacePath, dir)}/`);
+  }
+
+  for (const file of files) {
+    // Files under removed dirs were already deleted with their parent.
+    if (!fs.existsSync(file)) continue;
+    fs.rmSync(file, { force: true });
+    ctx.log(`removed ${path.relative(ctx.workspacePath, file)}`);
+  }
+
+  return Promise.resolve();
+}