Add a tool to allow comparing Firebase App Hosting deployments

.gitignore

@@ -39,3 +39,6 @@ clean/
 # Dart/Flutter
 .dart_tool/
 **/pubspec.lock
+
+compare-report/
+compare-cache/

package.json

@@ -13,7 +13,7 @@
     "build:publish": "tsc --build tsconfig.publish.json && npm run copyfiles",
     "build:watch": "npm run build && tsc --watch",
     "clean": "node -e \"fs.rmSync('lib', { recursive: true, force: true }); fs.rmSync('dev', { recursive: true, force: true });\"",
-    "copyfiles": "node -e \"const fs = require('fs'); fs.copyFileSync('./src/dynamicImport.js', './lib/dynamicImport.js'); fs.cpSync('./src/firebase_studio', './lib/firebase_studio', {recursive: true, filter: (src) => fs.statSync(src).isDirectory() || src.endsWith('.md') || src.endsWith('.js')});\"",
+    "copyfiles": "node -e \"const fs = require('fs'); fs.copyFileSync('./src/dynamicImport.js', './lib/dynamicImport.js'); fs.cpSync('./src/firebase_studio', './lib/firebase_studio', {recursive: true, filter: (src) => fs.statSync(src).isDirectory() || src.endsWith('.md') || src.endsWith('.js')}); fs.cpSync('./src/apphosting/compare/public', './lib/apphosting/compare/public', {recursive: true});\"",
     "format": "npm run format:ts && npm run format:other",
     "format:other": "npm run lint:other -- --write",
     "format:ts": "npm run lint:ts -- --fix --quiet",

src/apphosting/compare/README.md

@@ -0,0 +1,136 @@
+# App Hosting N-Way Matrix Comparison Tool
+
+This is an experimental internal CLI tool built for the Firebase App Hosting team to dynamically verify the compatibility and performance of different infrastructure backends or application builds. 
+
+It takes an array of configurations (variants) and automatically deploys them, discovering their routes, and rendering an interactive $O(N^2)$ Pair-wise Cartesian Matrix heatmap dashboard.
+
+## Capabilities
+
+1. **Standard CLI Fidelity**: Deployments are executed by automatically constructing a temporary `firebase.json` and programmatically calling the `firebase deploy` CLI execution path. This guarantees that test deployments faithfully mirror the actual customer experience (including Secrets, AutoInit Env Vars, and custom headers).
+2. **Local vs Remote Build Verification**: Can deploy locally built bundles (e.g. `localBuild: true`) side-by-side with remote Cloud Build source zips.
+3. **Automated IAM & Secrets Management**: Intelligently creates a single mock secret in Secret Manager for each distinct codebase path, mapping the IAM `secretAccessor` roles simultaneously to all backends generated from that codebase.
+4. **Dynamic Spidering**: Automatically crawls Next.js / Angular apps recursively starting from `/` to discover hidden dynamic routes, alongside statically parsing `.next/prerender-manifest.json`.
+5. **Interactive Visualization Dashboard**: Hosts an interactive split-view dashboard showcasing global parity heatmaps, dynamic filtering, code diffs, and exact header mismatches.
+
+## Usage
+
+1. Create a `matrix-test.json` file to define your test cases:
+
+```json
+[
+  {
+    "name": "Node Matrix Test",
+    "variants": [
+      {
+        "id": "Local-Node24",
+        "path": "../next-sample-1",
+        "localBuild": true,
+        "runtime": "nodejs24"
+      },
+      {
+        "id": "Source-Node24",
+        "path": "../next-sample-1",
+        "localBuild": false,
+        "runtime": "nodejs24"
+      }
+    ]
+  }
+]
+```
+
+2. Run the deployment and recording phase:
+
+```bash
+FIREBASE_CLI_EXPERIMENTS=apphosting firebase apphosting:compare-suite --project <your-project> --suite-config src/apphosting/compare/matrices/matrix-test.json --record-only
+```
+
+3. Spin up the visual dashboard to inspect the matrices and diffs:
+
+```bash
+FIREBASE_CLI_EXPERIMENTS=apphosting firebase apphosting:compare-suite --serve --port 3000
+```
+
+
+## System Architecture & Subsystem Delineation
+
+The N-Way Matrix Comparison Tool is divided into two distinct subsystems to separate the CLI recording/deployment engine from the visual comparison dashboard (the Diff Viewer).
+
+
+```mermaid
+graph TD
+    %% Define styles
+    classDef diffViewer fill:#1e293b,stroke:#3b82f6,stroke-width:2px,color:#f8fafc;
+    classDef engine fill:#0f172a,stroke:#334155,stroke-width:1px,color:#94a3b8;
+    classDef ui fill:#1e1b4b,stroke:#818cf8,stroke-width:2px,color:#f8fafc;
+    classDef cache fill:#022c22,stroke:#10b981,stroke-width:2px,color:#f8fafc;
+
+    subgraph DiffViewerGroup["Diff Viewer Web Application (Visual Dashboard)"]
+        UI["User Interface (Web Dashboard UI)<br/>index.html / index.css / index.js"]:::ui
+        Server["Diff Viewer API Server<br/>server.ts"]:::diffViewer
+    end
+
+    subgraph EngineGroup["Deployment & Recording Engine (CLI Runner)"]
+        Cache["Cache Manager (JSON)<br/>cache.ts"]:::cache
+        Comp["Comparison Engine<br/>compare.ts"]:::engine
+        Crawler["Crawler & Route Discovery<br/>crawler.ts / discover.ts"]:::engine
+        Orch["Orchestration Engine<br/>suite.ts / lifecycle.ts"]:::engine
+    end
+
+    %% Relationships
+    UI -- "HTTP / API Requests" --> Server
+    Server -- "Read JSON Cache" --> Cache
+    Cache -- "Write Telemetry" --> Comp
+    Comp -- "Run Parity Analysis" --> Crawler
+    Crawler -- "Spider Dynamic Paths" --> Orch
+
+    class UI,Server diffViewer;
+    class Cache cache;
+    class Comp,Crawler,Orch engine;
+```
+
+
+### 1. The Diff Viewer Web Application (Visual Dashboard)
+This subsystem is responsible for hosting the interactive web interface, serving visualization assets, and exposing the matrix-calculation APIs.
+*   **`src/apphosting/compare/public/`**: Contains the frontend single-page application:
+    *   `index.html`: Layout structure, split panes, controls bar, and dialog anchors.
+    *   `index.css`: Stylesheet, custom dark-mode slate theme, responsive grid cards, and transition overlays.
+    *   `index.js`: Client-side logic, real-time matrix rendering using HSL interpolation, localStorage configuration persistence, inline table ignores, and dynamic filters.
+*   **`src/apphosting/compare/server.ts`**: The lightweight Express API backend. It reads recorded JSON files from the cache directory, serves the static assets from the `public/` directory, and provides endpoints:
+    *   `/api/recordings`: Lists all available cached test suite recordings.
+    *   `/api/matrix`: Recalculates pairwise parity similarity scores dynamically.
+    *   `/api/compare`: Performs exact body and header diffs between two selected variant recordings.
+    *   `/api/render`: Proxies the recorded response bodies as live visual rendering previews.
+
+### 2. The Deployment & Recording Engine (CLI Runner)
+This subsystem orchestrates the lifecycle of deploying temporary slots, discovering routes, spidering endpoints, and persisting telemetry.
+*   **`src/apphosting/compare/suite.ts`**: The main execution orchestrator for the CLI command. It processes test matrices, locks concurrency slots, and schedules deployment loops.
+*   **`src/apphosting/compare/lifecycle.ts`**: Handles programmatic interactions with the Firebase CLI (`firebase deploy`) and cleanups.
+*   **`src/apphosting/compare/discover.ts`**: Parses local build metadata (such as the Next.js pre-render manifest) to compile an initial target route list.
+*   **`src/apphosting/compare/crawler.ts`**: A recursive spider crawler that walks paths on live deployments to auto-discover dynamic routes.
+*   **`src/apphosting/compare/cache.ts`**: Manages reading and writing serialization records under the `.compare-cache/` workspace directory.
+*   **`src/apphosting/compare/compare.ts`**: Core parity engine containing the text similarity scoring (Levenshtein/Jaccard), status check rules, and header variation classifications.
+*   **`src/apphosting/compare/distance.ts`**: Algorithmic library for calculating string similarity distances.
+
+---
+
+
+## Instructions for AI Agents
+
+If you are an AI Agent tasked with running differential tests or adding new variants to a matrix, follow these strict guidelines:
+
+### 1. Matrix Configuration
+* Matrices must be an array of test cases. Each test case MUST have at least 2 variants.
+* Always point `path` to the correct target framework codebase directory relative to your working directory.
+* Valid configuration flags inside a variant: `id` (string), `path` (string), `localBuild` (boolean), `runtime` (string).
+
+### 2. Execution Phases
+The tool separates deploying/crawling from visual analysis.
+1. **Record Phase**: Run the suite using `--record-only` to deploy to App Hosting and generate the cache. This takes several minutes per variant. Wait for the command to finish.
+   * *Note: The tool globally locks GCP slots (e.g. `compare-slot-1-0`). Do NOT run multiple record suites concurrently.*
+2. **Analyze Phase**: Once recording is complete, start the dashboard using `--serve --port 8000` in the background (using `WaitMsBeforeAsync: 3000`).
+
+### 3. Visual Verification
+* You CANNOT read the raw test outputs or diff matrices from the terminal. All comparison analysis must be done visually.
+* After starting the `--serve` server, use your `browser_subagent` to open `http://localhost:8000`.
+* Instruct the browser subagent to click the heatmap cells, toggle the "Visual Split-View" tabs, apply "Runtime" or "Build Origin" dynamic dropdown filters, and capture screenshots of the diffs.
+* Read the screenshots returned by the subagent to interpret the parity results or detect UI rendering regressions.

src/apphosting/compare/cache.ts

@@ -0,0 +1,193 @@
+import * as fs from "fs-extra";
+import * as path from "path";
+import * as crypto from "crypto";
+import * as os from "os";
+import { logger } from "./logger";
+
+export interface RouteResponse {
+  status: number;
+  headers: Record<string, string>;
+  body: string;
+  isBinary: boolean;
+  latencyMs?: number;
+}
+
+export interface VariantRecording {
+  id: string;
+  testCaseName: string;
+  timestamp: string;
+  url: string;
+  routes: Record<string, RouteResponse>;
+  localBuild?: boolean;
+  runtime?: string;
+  deployTimeMs?: number;
+  path?: string;
+}
+
+export function isRouteResponse(obj: unknown): obj is RouteResponse {
+  if (typeof obj !== "object" || obj === null) return false;
+  const o = obj as Record<string, unknown>;
+
+  if (!(
+    typeof o.status === "number" &&
+    typeof o.headers === "object" && o.headers !== null &&
+    typeof o.body === "string" &&
+    typeof o.isBinary === "boolean"
+  )) {
+    return false;
+  }
+
+  if (o.latencyMs !== undefined && typeof o.latencyMs !== "number") {
+    return false;
+  }
+
+  return true;
+}
+
+export function isVariantRecording(obj: unknown): obj is VariantRecording {
+  if (typeof obj !== "object" || obj === null) return false;
+  const o = obj as Record<string, unknown>;
+
+  if (!(
+    typeof o.id === "string" &&
+    typeof o.testCaseName === "string" &&
+    typeof o.timestamp === "string" &&
+    typeof o.url === "string" &&
+    typeof o.routes === "object" && o.routes !== null
+  )) {
+    return false;
+  }
+
+  if (o.localBuild !== undefined && typeof o.localBuild !== "boolean") {
+    return false;
+  }
+
+  if (o.runtime !== undefined && typeof o.runtime !== "string") {
+    return false;
+  }
+
+  if (o.deployTimeMs !== undefined && typeof o.deployTimeMs !== "number") {
+    return false;
+  }
+
+  const routes = o.routes as Record<string, unknown>;
+  for (const key of Object.keys(routes)) {
+    if (!isRouteResponse(routes[key])) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+function isErrnoException(error: unknown): error is NodeJS.ErrnoException {
+  return error instanceof Error && "code" in error;
+}
+
+const CACHE_DIR = process.platform === "win32"
+  ? path.join(os.tmpdir(), "firebase-apphosting-compare-cache")
+  : "/tmp/firebase-apphosting-compare-cache";
+
+function getRecordingPath(testCaseName: string, variantId: string): string {
+  const tcHash = crypto.createHash("sha256").update(testCaseName).digest("hex").slice(0, 8);
+  const vHash = crypto.createHash("sha256").update(variantId).digest("hex").slice(0, 8);
+  const safeTestCase = `${testCaseName.replace(/[^a-zA-Z0-9_-]/g, "_")}_${tcHash}`;
+  const safeVariant = `${variantId.replace(/[^a-zA-Z0-9_-]/g, "_")}_${vHash}`;
+  const resolvedPath = path.resolve(path.join(CACHE_DIR, "recordings", safeTestCase, `${safeVariant}.json`));
+
+  if (!resolvedPath.startsWith(path.resolve(CACHE_DIR))) {
+    throw new Error("Path traversal restriction violation");
+  }
+  return resolvedPath;
+}
+
+/**
+ * Saves a variant recording to the cache atomically.
+ */
+export async function saveRecording(recording: VariantRecording): Promise<void> {
+  const filePath = getRecordingPath(recording.testCaseName, recording.id);
+  const tempPath = filePath + ".tmp";
+
+  await fs.ensureDir(path.dirname(tempPath));
+  await fs.writeJson(tempPath, recording, { spaces: 2 });
+  await fs.rename(tempPath, filePath);
+
+  logger.info(`Saved recording to cache: ${filePath}`);
+}
+
+/**
+ * Loads a variant recording from the cache.
+ */
+export async function loadRecording(testCaseName: string, variantId: string): Promise<VariantRecording> {
+  const filePath = getRecordingPath(testCaseName, variantId);
+  if (!(await fs.pathExists(filePath))) {
+    throw new Error(`No recording found in cache for variant "${variantId}" under test case "${testCaseName}"`);
+  }
+  const data = await fs.readJson(filePath);
+  if (!isVariantRecording(data)) {
+    throw new Error(`Invalid recording format in cache for variant "${variantId}" under test case "${testCaseName}"`);
+  }
+  return data;
+}
+
+/**
+ * Lists all recorded test cases and their variants.
+ */
+export async function listRecordings(): Promise<Record<string, string[]>> {
+  const recordingsDir = path.join(CACHE_DIR, "recordings");
+  if (!(await fs.pathExists(recordingsDir))) {
+    return {};
+  }
+
+  const result: Record<string, string[]> = {};
+  try {
+    const testCases = await fs.readdir(recordingsDir);
+    for (const tc of testCases) {
+      const tcDir = path.join(recordingsDir, tc);
+      try {
+        const stat = await fs.stat(tcDir);
+        if (!stat.isDirectory()) {
+          continue;
+        }
+        const files = await fs.readdir(tcDir);
+        const jsonFiles = files.filter((file) => file.endsWith(".json"));
+        if (jsonFiles.length === 0) {
+          continue;
+        }
+        const variantIds: string[] = [];
+        let originalTestCaseName = "";
+        for (const file of jsonFiles) {
+          try {
+            const data = await fs.readJson(path.join(tcDir, file));
+            if (isVariantRecording(data)) {
+              originalTestCaseName = data.testCaseName;
+              variantIds.push(data.id);
+            } else {
+              logger.debug(`Cache file ${file} does not match VariantRecording schema`);
+            }
+          } catch (readErr) {
+            // If a file is partially written or corrupted, skip it
+            logger.debug(`Failed to read metadata for cache file: ${file}`, readErr);
+          }
+        }
+        if (originalTestCaseName && variantIds.length > 0) {
+          result[originalTestCaseName] = variantIds;
+        }
+      } catch (err: unknown) {
+        if (isErrnoException(err) && err.code === "ENOENT") {
+          continue; // Directory was concurrently deleted/moved
+        }
+        throw err;
+      }
+    }
+  } catch (err: unknown) {
+    if (isErrnoException(err) && err.code === "ENOENT") {
+      return {};
+    }
+    throw err;
+  }
+
+  return result;
+}
+
+