diff --git a/.gitignore b/.gitignore index 593347ef2b0..d321b20680b 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,6 @@ clean/ # Dart/Flutter .dart_tool/ **/pubspec.lock + +compare-report/ +compare-cache/ diff --git a/package.json b/package.json index 8c427e701df..6f6b48d0cf4 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "build:publish": "tsc --build tsconfig.publish.json && npm run copyfiles", "build:watch": "npm run build && tsc --watch", "clean": "node -e \"fs.rmSync('lib', { recursive: true, force: true }); fs.rmSync('dev', { recursive: true, force: true });\"", - "copyfiles": "node -e \"const fs = require('fs'); fs.copyFileSync('./src/dynamicImport.js', './lib/dynamicImport.js'); fs.cpSync('./src/firebase_studio', './lib/firebase_studio', {recursive: true, filter: (src) => fs.statSync(src).isDirectory() || src.endsWith('.md') || src.endsWith('.js')});\"", + "copyfiles": "node -e \"const fs = require('fs'); fs.copyFileSync('./src/dynamicImport.js', './lib/dynamicImport.js'); fs.cpSync('./src/firebase_studio', './lib/firebase_studio', {recursive: true, filter: (src) => fs.statSync(src).isDirectory() || src.endsWith('.md') || src.endsWith('.js')}); fs.cpSync('./src/apphosting/compare/public', './lib/apphosting/compare/public', {recursive: true});\"", "format": "npm run format:ts && npm run format:other", "format:other": "npm run lint:other -- --write", "format:ts": "npm run lint:ts -- --fix --quiet", diff --git a/src/apphosting/compare/README.md b/src/apphosting/compare/README.md new file mode 100644 index 00000000000..9d7d063b115 --- /dev/null +++ b/src/apphosting/compare/README.md @@ -0,0 +1,136 @@ +# App Hosting N-Way Matrix Comparison Tool + +This is an experimental internal CLI tool built for the Firebase App Hosting team to dynamically verify the compatibility and performance of different infrastructure backends or application builds. + +It takes an array of configurations (variants) and automatically deploys them, discovering their routes, and rendering an interactive $O(N^2)$ Pair-wise Cartesian Matrix heatmap dashboard. + +## Capabilities + +1. **Standard CLI Fidelity**: Deployments are executed by automatically constructing a temporary `firebase.json` and programmatically calling the `firebase deploy` CLI execution path. This guarantees that test deployments faithfully mirror the actual customer experience (including Secrets, AutoInit Env Vars, and custom headers). +2. **Local vs Remote Build Verification**: Can deploy locally built bundles (e.g. `localBuild: true`) side-by-side with remote Cloud Build source zips. +3. **Automated IAM & Secrets Management**: Intelligently creates a single mock secret in Secret Manager for each distinct codebase path, mapping the IAM `secretAccessor` roles simultaneously to all backends generated from that codebase. +4. **Dynamic Spidering**: Automatically crawls Next.js / Angular apps recursively starting from `/` to discover hidden dynamic routes, alongside statically parsing `.next/prerender-manifest.json`. +5. **Interactive Visualization Dashboard**: Hosts an interactive split-view dashboard showcasing global parity heatmaps, dynamic filtering, code diffs, and exact header mismatches. + +## Usage + +1. Create a `matrix-test.json` file to define your test cases: + +```json +[ + { + "name": "Node Matrix Test", + "variants": [ + { + "id": "Local-Node24", + "path": "../next-sample-1", + "localBuild": true, + "runtime": "nodejs24" + }, + { + "id": "Source-Node24", + "path": "../next-sample-1", + "localBuild": false, + "runtime": "nodejs24" + } + ] + } +] +``` + +2. Run the deployment and recording phase: + +```bash +FIREBASE_CLI_EXPERIMENTS=apphosting firebase apphosting:compare-suite --project --suite-config src/apphosting/compare/matrices/matrix-test.json --record-only +``` + +3. Spin up the visual dashboard to inspect the matrices and diffs: + +```bash +FIREBASE_CLI_EXPERIMENTS=apphosting firebase apphosting:compare-suite --serve --port 3000 +``` + + +## System Architecture & Subsystem Delineation + +The N-Way Matrix Comparison Tool is divided into two distinct subsystems to separate the CLI recording/deployment engine from the visual comparison dashboard (the Diff Viewer). + + +```mermaid +graph TD + %% Define styles + classDef diffViewer fill:#1e293b,stroke:#3b82f6,stroke-width:2px,color:#f8fafc; + classDef engine fill:#0f172a,stroke:#334155,stroke-width:1px,color:#94a3b8; + classDef ui fill:#1e1b4b,stroke:#818cf8,stroke-width:2px,color:#f8fafc; + classDef cache fill:#022c22,stroke:#10b981,stroke-width:2px,color:#f8fafc; + + subgraph DiffViewerGroup["Diff Viewer Web Application (Visual Dashboard)"] + UI["User Interface (Web Dashboard UI)
index.html / index.css / index.js"]:::ui + Server["Diff Viewer API Server
server.ts"]:::diffViewer + end + + subgraph EngineGroup["Deployment & Recording Engine (CLI Runner)"] + Cache["Cache Manager (JSON)
cache.ts"]:::cache + Comp["Comparison Engine
compare.ts"]:::engine + Crawler["Crawler & Route Discovery
crawler.ts / discover.ts"]:::engine + Orch["Orchestration Engine
suite.ts / lifecycle.ts"]:::engine + end + + %% Relationships + UI -- "HTTP / API Requests" --> Server + Server -- "Read JSON Cache" --> Cache + Cache -- "Write Telemetry" --> Comp + Comp -- "Run Parity Analysis" --> Crawler + Crawler -- "Spider Dynamic Paths" --> Orch + + class UI,Server diffViewer; + class Cache cache; + class Comp,Crawler,Orch engine; +``` + + +### 1. The Diff Viewer Web Application (Visual Dashboard) +This subsystem is responsible for hosting the interactive web interface, serving visualization assets, and exposing the matrix-calculation APIs. +* **`src/apphosting/compare/public/`**: Contains the frontend single-page application: + * `index.html`: Layout structure, split panes, controls bar, and dialog anchors. + * `index.css`: Stylesheet, custom dark-mode slate theme, responsive grid cards, and transition overlays. + * `index.js`: Client-side logic, real-time matrix rendering using HSL interpolation, localStorage configuration persistence, inline table ignores, and dynamic filters. +* **`src/apphosting/compare/server.ts`**: The lightweight Express API backend. It reads recorded JSON files from the cache directory, serves the static assets from the `public/` directory, and provides endpoints: + * `/api/recordings`: Lists all available cached test suite recordings. + * `/api/matrix`: Recalculates pairwise parity similarity scores dynamically. + * `/api/compare`: Performs exact body and header diffs between two selected variant recordings. + * `/api/render`: Proxies the recorded response bodies as live visual rendering previews. + +### 2. The Deployment & Recording Engine (CLI Runner) +This subsystem orchestrates the lifecycle of deploying temporary slots, discovering routes, spidering endpoints, and persisting telemetry. +* **`src/apphosting/compare/suite.ts`**: The main execution orchestrator for the CLI command. It processes test matrices, locks concurrency slots, and schedules deployment loops. +* **`src/apphosting/compare/lifecycle.ts`**: Handles programmatic interactions with the Firebase CLI (`firebase deploy`) and cleanups. +* **`src/apphosting/compare/discover.ts`**: Parses local build metadata (such as the Next.js pre-render manifest) to compile an initial target route list. +* **`src/apphosting/compare/crawler.ts`**: A recursive spider crawler that walks paths on live deployments to auto-discover dynamic routes. +* **`src/apphosting/compare/cache.ts`**: Manages reading and writing serialization records under the `.compare-cache/` workspace directory. +* **`src/apphosting/compare/compare.ts`**: Core parity engine containing the text similarity scoring (Levenshtein/Jaccard), status check rules, and header variation classifications. +* **`src/apphosting/compare/distance.ts`**: Algorithmic library for calculating string similarity distances. + +--- + + +## Instructions for AI Agents + +If you are an AI Agent tasked with running differential tests or adding new variants to a matrix, follow these strict guidelines: + +### 1. Matrix Configuration +* Matrices must be an array of test cases. Each test case MUST have at least 2 variants. +* Always point `path` to the correct target framework codebase directory relative to your working directory. +* Valid configuration flags inside a variant: `id` (string), `path` (string), `localBuild` (boolean), `runtime` (string). + +### 2. Execution Phases +The tool separates deploying/crawling from visual analysis. +1. **Record Phase**: Run the suite using `--record-only` to deploy to App Hosting and generate the cache. This takes several minutes per variant. Wait for the command to finish. + * *Note: The tool globally locks GCP slots (e.g. `compare-slot-1-0`). Do NOT run multiple record suites concurrently.* +2. **Analyze Phase**: Once recording is complete, start the dashboard using `--serve --port 8000` in the background (using `WaitMsBeforeAsync: 3000`). + +### 3. Visual Verification +* You CANNOT read the raw test outputs or diff matrices from the terminal. All comparison analysis must be done visually. +* After starting the `--serve` server, use your `browser_subagent` to open `http://localhost:8000`. +* Instruct the browser subagent to click the heatmap cells, toggle the "Visual Split-View" tabs, apply "Runtime" or "Build Origin" dynamic dropdown filters, and capture screenshots of the diffs. +* Read the screenshots returned by the subagent to interpret the parity results or detect UI rendering regressions. diff --git a/src/apphosting/compare/cache.ts b/src/apphosting/compare/cache.ts new file mode 100644 index 00000000000..06845de4a7b --- /dev/null +++ b/src/apphosting/compare/cache.ts @@ -0,0 +1,193 @@ +import * as fs from "fs-extra"; +import * as path from "path"; +import * as crypto from "crypto"; +import * as os from "os"; +import { logger } from "./logger"; + +export interface RouteResponse { + status: number; + headers: Record; + body: string; + isBinary: boolean; + latencyMs?: number; +} + +export interface VariantRecording { + id: string; + testCaseName: string; + timestamp: string; + url: string; + routes: Record; + localBuild?: boolean; + runtime?: string; + deployTimeMs?: number; + path?: string; +} + +export function isRouteResponse(obj: unknown): obj is RouteResponse { + if (typeof obj !== "object" || obj === null) return false; + const o = obj as Record; + + if (!( + typeof o.status === "number" && + typeof o.headers === "object" && o.headers !== null && + typeof o.body === "string" && + typeof o.isBinary === "boolean" + )) { + return false; + } + + if (o.latencyMs !== undefined && typeof o.latencyMs !== "number") { + return false; + } + + return true; +} + +export function isVariantRecording(obj: unknown): obj is VariantRecording { + if (typeof obj !== "object" || obj === null) return false; + const o = obj as Record; + + if (!( + typeof o.id === "string" && + typeof o.testCaseName === "string" && + typeof o.timestamp === "string" && + typeof o.url === "string" && + typeof o.routes === "object" && o.routes !== null + )) { + return false; + } + + if (o.localBuild !== undefined && typeof o.localBuild !== "boolean") { + return false; + } + + if (o.runtime !== undefined && typeof o.runtime !== "string") { + return false; + } + + if (o.deployTimeMs !== undefined && typeof o.deployTimeMs !== "number") { + return false; + } + + const routes = o.routes as Record; + for (const key of Object.keys(routes)) { + if (!isRouteResponse(routes[key])) { + return false; + } + } + + return true; +} + +function isErrnoException(error: unknown): error is NodeJS.ErrnoException { + return error instanceof Error && "code" in error; +} + +const CACHE_DIR = process.platform === "win32" + ? path.join(os.tmpdir(), "firebase-apphosting-compare-cache") + : "/tmp/firebase-apphosting-compare-cache"; + +function getRecordingPath(testCaseName: string, variantId: string): string { + const tcHash = crypto.createHash("sha256").update(testCaseName).digest("hex").slice(0, 8); + const vHash = crypto.createHash("sha256").update(variantId).digest("hex").slice(0, 8); + const safeTestCase = `${testCaseName.replace(/[^a-zA-Z0-9_-]/g, "_")}_${tcHash}`; + const safeVariant = `${variantId.replace(/[^a-zA-Z0-9_-]/g, "_")}_${vHash}`; + const resolvedPath = path.resolve(path.join(CACHE_DIR, "recordings", safeTestCase, `${safeVariant}.json`)); + + if (!resolvedPath.startsWith(path.resolve(CACHE_DIR))) { + throw new Error("Path traversal restriction violation"); + } + return resolvedPath; +} + +/** + * Saves a variant recording to the cache atomically. + */ +export async function saveRecording(recording: VariantRecording): Promise { + const filePath = getRecordingPath(recording.testCaseName, recording.id); + const tempPath = filePath + ".tmp"; + + await fs.ensureDir(path.dirname(tempPath)); + await fs.writeJson(tempPath, recording, { spaces: 2 }); + await fs.rename(tempPath, filePath); + + logger.info(`Saved recording to cache: ${filePath}`); +} + +/** + * Loads a variant recording from the cache. + */ +export async function loadRecording(testCaseName: string, variantId: string): Promise { + const filePath = getRecordingPath(testCaseName, variantId); + if (!(await fs.pathExists(filePath))) { + throw new Error(`No recording found in cache for variant "${variantId}" under test case "${testCaseName}"`); + } + const data = await fs.readJson(filePath); + if (!isVariantRecording(data)) { + throw new Error(`Invalid recording format in cache for variant "${variantId}" under test case "${testCaseName}"`); + } + return data; +} + +/** + * Lists all recorded test cases and their variants. + */ +export async function listRecordings(): Promise> { + const recordingsDir = path.join(CACHE_DIR, "recordings"); + if (!(await fs.pathExists(recordingsDir))) { + return {}; + } + + const result: Record = {}; + try { + const testCases = await fs.readdir(recordingsDir); + for (const tc of testCases) { + const tcDir = path.join(recordingsDir, tc); + try { + const stat = await fs.stat(tcDir); + if (!stat.isDirectory()) { + continue; + } + const files = await fs.readdir(tcDir); + const jsonFiles = files.filter((file) => file.endsWith(".json")); + if (jsonFiles.length === 0) { + continue; + } + const variantIds: string[] = []; + let originalTestCaseName = ""; + for (const file of jsonFiles) { + try { + const data = await fs.readJson(path.join(tcDir, file)); + if (isVariantRecording(data)) { + originalTestCaseName = data.testCaseName; + variantIds.push(data.id); + } else { + logger.debug(`Cache file ${file} does not match VariantRecording schema`); + } + } catch (readErr) { + // If a file is partially written or corrupted, skip it + logger.debug(`Failed to read metadata for cache file: ${file}`, readErr); + } + } + if (originalTestCaseName && variantIds.length > 0) { + result[originalTestCaseName] = variantIds; + } + } catch (err: unknown) { + if (isErrnoException(err) && err.code === "ENOENT") { + continue; // Directory was concurrently deleted/moved + } + throw err; + } + } + } catch (err: unknown) { + if (isErrnoException(err) && err.code === "ENOENT") { + return {}; + } + throw err; + } + + return result; +} + + diff --git a/src/apphosting/compare/compare.spec.ts b/src/apphosting/compare/compare.spec.ts new file mode 100644 index 00000000000..aed79b61e76 --- /dev/null +++ b/src/apphosting/compare/compare.spec.ts @@ -0,0 +1,92 @@ +import { expect } from "chai"; +import * as nock from "nock"; +import { compareRoute } from "./compare"; + +describe("compareRoute", () => { + beforeEach(() => { + nock.cleanAll(); + }); + + afterEach(() => { + nock.cleanAll(); + }); + + it("should match identical text pages", async () => { + nock("https://backend-a.com").get("/").reply(200, "hello world", { + "Content-Type": "text/html", + "Cache-Control": "public, max-age=3600", + }); + nock("https://backend-b.com").get("/").reply(200, "hello world", { + "Content-Type": "text/html", + "Cache-Control": "public, max-age=3600", + }); + + const res = await compareRoute("/", "https://backend-a.com", "https://backend-b.com"); + expect(res.statusMatch).to.be.true; + expect(res.headerMismatches).to.be.empty; + expect(res.bodySimilarity).to.equal(1.0); + expect(res.isBinary).to.be.false; + }); + + it("should flag status mismatches", async () => { + nock("https://backend-a.com").get("/").reply(200, "ok"); + nock("https://backend-b.com").get("/").reply(404, "not found"); + + const res = await compareRoute("/", "https://backend-a.com", "https://backend-b.com"); + expect(res.statusMatch).to.be.false; + }); + + it("should flag behavioral header mismatches but record dynamic ones as expected variations", async () => { + nock("https://backend-a.com").get("/").reply(200, "ok", { + "Cache-Control": "public, max-age=3600", + Date: "Wed, 17 Jun 2026 01:00:00 GMT", + }); + nock("https://backend-b.com").get("/").reply(200, "ok", { + "Cache-Control": "no-cache", + Date: "Wed, 17 Jun 2026 01:05:00 GMT", + }); + + const res = await compareRoute("/", "https://backend-a.com", "https://backend-b.com"); + expect(res.headerMismatches).to.have.lengthOf(1); + expect(res.headerMismatches[0].header.toLowerCase()).to.equal("cache-control"); + + expect(res.expectedHeaderVariations).to.have.lengthOf(1); + expect(res.expectedHeaderVariations[0].header.toLowerCase()).to.equal("date"); + }); + + it("should match identical binary pages and flag mismatching binary content", async () => { + const binA = Buffer.from([1, 2, 3, 4]); + const binB = Buffer.from([1, 2, 3, 4]); + const binC = Buffer.from([1, 2, 3, 5]); + + nock("https://backend-a.com").get("/img.png").reply(200, binA, { + "Content-Type": "image/png", + }); + nock("https://backend-b.com").get("/img.png").reply(200, binB, { + "Content-Type": "image/png", + }); + + const resMatch = await compareRoute( + "/img.png", + "https://backend-a.com", + "https://backend-b.com", + ); + expect(resMatch.isBinary).to.be.true; + expect(resMatch.bodySimilarity).to.equal(1.0); + + nock("https://backend-a.com").get("/img.png").reply(200, binA, { + "Content-Type": "image/png", + }); + nock("https://backend-b.com").get("/img.png").reply(200, binC, { + "Content-Type": "image/png", + }); + + const resMismatch = await compareRoute( + "/img.png", + "https://backend-a.com", + "https://backend-b.com", + ); + expect(resMismatch.isBinary).to.be.true; + expect(resMismatch.bodySimilarity).to.equal(0.0); + }); +}); diff --git a/src/apphosting/compare/compare.ts b/src/apphosting/compare/compare.ts new file mode 100644 index 00000000000..2cf59a6ba53 --- /dev/null +++ b/src/apphosting/compare/compare.ts @@ -0,0 +1,213 @@ +import fetch from "node-fetch"; +import * as crypto from "crypto"; +import { MyersDiffEngine } from "./distance"; + +export interface ComparisonResult { + route: string; + statusMatch: boolean; + statusA?: number; + statusB?: number; + headerMismatches: Array<{ header: string; valA: string; valB: string }>; + expectedHeaderVariations: Array<{ header: string; valA: string; valB: string }>; + bodySimilarity: number; // 0.0 to 1.0 + bodyDiff: string; + isBinary: boolean; + bodyA?: string; + bodyB?: string; + latencyA?: number; + latencyB?: number; +} + + +const BEHAVIORAL_HEADERS = [ + "cache-control", + "content-security-policy", + "content-type", + "content-encoding", + "location", + "strict-transport-security", +]; + +const BINARY_CONTENT_TYPES = [ + "image/", + "application/pdf", + "application/zip", + "application/octet-stream", +]; + +function isBinaryContentType(contentType: string): boolean { + const normalized = contentType.toLowerCase(); + return BINARY_CONTENT_TYPES.some((type) => normalized.includes(type)); +} + +/** + * + */ +export async function compareRoute( + route: string, + urlA: string, + urlB: string, + options: { headers?: Record } = {}, +): Promise { + const fetchOptions = { + headers: options.headers || {}, + redirect: "manual" as const, + size: 2 * 1024 * 1024, + }; + + const startA = Date.now(); + const resA = await fetch(`${urlA}${route}`, fetchOptions); + const latencyA = Date.now() - startA; + + const startB = Date.now(); + const resB = await fetch(`${urlB}${route}`, fetchOptions); + const latencyB = Date.now() - startB; + + const contentTypeA = resA.headers.get("content-type") || ""; + const contentTypeB = resB.headers.get("content-type") || ""; + const isBinaryA = isBinaryContentType(contentTypeA); + const isBinaryB = isBinaryContentType(contentTypeB); + + const headersA: Record = {}; + resA.headers.forEach((val, key) => { headersA[key.toLowerCase()] = val; }); + + const headersB: Record = {}; + resB.headers.forEach((val, key) => { headersB[key.toLowerCase()] = val; }); + + const responseA: RouteResponse = { + status: resA.status, + headers: headersA, + isBinary: isBinaryA || isBinaryB, + body: (isBinaryA || isBinaryB) ? (await resA.buffer()).toString("base64") : await resA.text(), + latencyMs: latencyA, + }; + + const responseB: RouteResponse = { + status: resB.status, + headers: headersB, + isBinary: isBinaryA || isBinaryB, + body: (isBinaryA || isBinaryB) ? (await resB.buffer()).toString("base64") : await resB.text(), + latencyMs: latencyB, + }; + + return await compareRouteResponses(route, responseA, responseB); +} + +export interface RouteResponse { + status: number; + headers: Record; + body: string; + isBinary: boolean; + latencyMs?: number; +} + +/** + * + */ +export async function compareRouteResponses( + route: string, + resA: RouteResponse, + resB: RouteResponse, +): Promise { + const result: ComparisonResult = { + route, + statusMatch: resA.status === resB.status, + statusA: resA.status, + statusB: resB.status, + headerMismatches: [], + expectedHeaderVariations: [], + bodySimilarity: 1.0, + bodyDiff: "", + isBinary: resA.isBinary || resB.isBinary, + latencyA: resA.latencyMs, + latencyB: resB.latencyMs, + }; + + // 1. Compare Headers + const normalizedHeadersA: Record = {}; + Object.entries(resA.headers).forEach(([k, v]) => { normalizedHeadersA[k.toLowerCase()] = v; }); + + const normalizedHeadersB: Record = {}; + Object.entries(resB.headers).forEach(([k, v]) => { normalizedHeadersB[k.toLowerCase()] = v; }); + + const allHeaderKeys = new Set([ + ...Object.keys(normalizedHeadersA), + ...Object.keys(normalizedHeadersB), + ]); + + for (const key of allHeaderKeys) { + const valA = normalizedHeadersA[key] || ""; + const valB = normalizedHeadersB[key] || ""; + if (valA !== valB) { + if (BEHAVIORAL_HEADERS.includes(key.toLowerCase())) { + result.headerMismatches.push({ header: key, valA, valB }); + } else { + result.expectedHeaderVariations.push({ header: key, valA, valB }); + } + } + } + + // 2. Compare Binary + if (result.isBinary) { + const bufA = Buffer.from(resA.body, "base64"); + const bufB = Buffer.from(resB.body, "base64"); + + const sizeA = bufA.length; + const sizeB = bufB.length; + + if (sizeA !== sizeB) { + result.bodySimilarity = 0.0; + result.bodyDiff = `Binary size mismatch: ${sizeA} bytes vs ${sizeB} bytes`; + } else { + const hashA = crypto.createHash("sha256").update(bufA).digest("hex"); + const hashB = crypto.createHash("sha256").update(bufB).digest("hex"); + if (hashA === hashB) { + result.bodySimilarity = 1.0; + } else { + result.bodySimilarity = 0.0; + result.bodyDiff = "Binary content hash mismatch"; + } + } + return result; + } + + // 3. Compare Text Body + let bodyA = resA.body; + let bodyB = resB.body; + + const contentType = (resA.headers["content-type"] || resA.headers["Content-Type"] || "").toLowerCase(); + if (contentType.includes("text/html")) { + try { + const prettier = require("prettier"); + const formattedA = await prettier.format(bodyA, { parser: "html" }); + const formattedB = await prettier.format(bodyB, { parser: "html" }); + bodyA = formattedA; + bodyB = formattedB; + } catch (e: any) { + // Fallback to advanced tag-based line splitting if prettier fails + const HTML_SPLIT_REGEX = /(<(script|style)\b[\s\S]*?<\/\2>||<[^'">]*(?:"[^"]*"[^'">]*|'[^']*'[^'">]*)*>)\s*(?=<)/gi; + bodyA = bodyA.replace(HTML_SPLIT_REGEX, "$1\n"); + bodyB = bodyB.replace(HTML_SPLIT_REGEX, "$1\n"); + } + } else if (contentType.includes("application/json") || route.endsWith(".json")) { + try { + bodyA = JSON.stringify(JSON.parse(bodyA), null, 2); + bodyB = JSON.stringify(JSON.parse(bodyB), null, 2); + } catch (e: any) { + // Fallback to raw text + } + } + + result.bodyA = bodyA; + result.bodyB = bodyB; + + if (bodyA !== bodyB) { + result.bodySimilarity = MyersDiffEngine.getSimilarity(bodyA, bodyB); + if (result.bodySimilarity < 1.0) { + result.bodyDiff = "HTML content mismatch"; + } + } + + return result; +} + diff --git a/src/apphosting/compare/crawler.spec.ts b/src/apphosting/compare/crawler.spec.ts new file mode 100644 index 00000000000..0713358b86d --- /dev/null +++ b/src/apphosting/compare/crawler.spec.ts @@ -0,0 +1,84 @@ +import { expect } from "chai"; +import * as nock from "nock"; +import { Crawler } from "./crawler"; + +describe("Crawler", () => { + beforeEach(() => { + nock.cleanAll(); + }); + + afterEach(() => { + nock.cleanAll(); + }); + + it("should recursively crawl text/html links", async () => { + const base = "https://example.com"; + + nock(base) + .get("/") + .reply( + 200, + ` + + + About Us + Contact Us + External Link + + + `, + { "Content-Type": "text/html" }, + ); + + nock(base) + .get("/about") + .reply( + 200, + ` + + + Careers + + + `, + { "Content-Type": "text/html" }, + ); + + nock(base).get("/contact").reply(200, "Contact Details", { "Content-Type": "text/html" }); + + nock(base).get("/careers").reply(200, "Join our team", { "Content-Type": "text/html" }); + + const crawler = new Crawler(base); + await crawler.crawl(); + + const routes = crawler.getRoutes(); + expect(routes).to.deep.equal(["/", "/about", "/careers", "/contact"]); + }); + + it("should follow redirect links", async () => { + const base = "https://example.com"; + + nock(base) + .get("/") + .reply(200, `Redirect me`, { "Content-Type": "text/html" }); + + nock(base).get("/old-link").reply(302, undefined, { Location: "/new-link" }); + + nock(base).get("/new-link").reply(200, "Done!", { "Content-Type": "text/html" }); + + const crawler = new Crawler(base); + await crawler.crawl(); + + const routes = crawler.getRoutes(); + expect(routes).to.deep.equal(["/", "/new-link", "/old-link"]); + }); + + it("should canonicalize paths and sort queries", async () => { + const base = "https://example.com"; + const crawler = new Crawler(base); + + expect(crawler.canonicalizeRoute("/about/")).to.equal("/about"); + expect(crawler.canonicalizeRoute("/search?q=foo&a=bar")).to.equal("/search?a=bar&q=foo"); + expect(crawler.canonicalizeRoute("https://external.com/foo")).to.be.null; + }); +}); diff --git a/src/apphosting/compare/crawler.ts b/src/apphosting/compare/crawler.ts new file mode 100644 index 00000000000..36005011e33 --- /dev/null +++ b/src/apphosting/compare/crawler.ts @@ -0,0 +1,172 @@ +import fetch from "node-fetch"; + +function decodeHtmlEntities(str: string): string { + const entities: Record = { + "amp": "&", + "lt": "<", + "gt": ">", + "quot": '"', + "#39": "'" + }; + return str.replace(/&(amp|lt|gt|quot|#39);/gi, (match, entity) => { + return entities[entity.toLowerCase()] || match; + }); +} + +interface Destroyable { + destroy(): void; +} + +function isDestroyable(obj: unknown): obj is Destroyable { + return ( + typeof obj === "object" && + obj !== null && + "destroy" in obj && + typeof (obj as Record).destroy === "function" + ); +} + +export class Crawler { + private visited = new Set(); + private discoveredRoutes = new Set(); + + constructor( + private readonly baseUrl: string, + private readonly maxDepth = 3, + ) {} + + public getRoutes(): string[] { + return Array.from(this.discoveredRoutes).sort(); + } + + /** + * Crawls starting from root and recursively finds links. + */ + public async crawl(): Promise { + await this.crawlRoute("/"); + } + + private async crawlRoute(route: string, depth = 0): Promise { + const canonical = this.canonicalizeRoute(route); + if (!canonical || this.visited.has(canonical) || depth > this.maxDepth) { + return; + } + + this.visited.add(canonical); + this.discoveredRoutes.add(canonical); + + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 15000); + + try { + const url = `${this.baseUrl}${canonical}`; + const res = await fetch(url, { + redirect: "manual" as const, + headers: { "User-Agent": "FirebaseCompareCrawler/1.0" }, + signal: controller.signal, + size: 2 * 1024 * 1024, + }); + + // Handle Redirects + if ([301, 302, 307, 308].includes(res.status)) { + if (res.body && isDestroyable(res.body)) { + res.body.destroy(); + } + const location = res.headers.get("location"); + if (location) { + const nextRoute = this.resolveRelative(canonical, location); + if (nextRoute) { + await this.crawlRoute(nextRoute, depth + 1); + } + } + return; + } + + // Only parse HTML responses + const contentType = res.headers.get("content-type") || ""; + if (!contentType.toLowerCase().includes("text/html")) { + if (res.body && isDestroyable(res.body)) { + res.body.destroy(); + } + return; + } + + const html = await res.text(); + const links = this.extractLinks(html, canonical); + + await Promise.all(links.map((link) => this.crawlRoute(link, depth + 1))); + } catch (err) { + // Ignore fetch failures for single routes during discovery + } finally { + clearTimeout(timeout); + } + } + + public canonicalizeRoute(route: string): string | null { + try { + const url = new URL(route, this.baseUrl); + if (url.origin !== new URL(this.baseUrl).origin) { + return null; + } + + let pathname = url.pathname; + if (pathname.endsWith("/") && pathname.length > 1) { + pathname = pathname.slice(0, -1); + } + + const params = Array.from(url.searchParams.entries()).sort((a, b) => + a[0].localeCompare(b[0]), + ); + const search = params.length > 0 ? "?" + params.map(([k, v]) => `${k}=${v}`).join("&") : ""; + + return `${pathname}${search}`; + } catch { + return null; + } + } + + private extractLinks(html: string, currentRoute: string): string[] { + const links: string[] = []; + const regex = /]*?\s+)?href\s*=\s*(?:(["'])(.*?)\1|([^\s>]+))/gi; + let match; + + while ((match = regex.exec(html)) !== null) { + const rawHref = match[2] !== undefined ? match[2] : match[3]; + if (!rawHref) continue; + + const href = decodeHtmlEntities(rawHref.trim()); + if (!href || href.startsWith("#")) { + continue; + } + + const schemeMatch = href.match(/^[a-z][a-z0-9+.-]*:/i); + if (schemeMatch) { + const scheme = schemeMatch[0].toLowerCase(); + if (scheme !== "http:" && scheme !== "https:") { + continue; + } + } + + const resolved = this.resolveRelative(currentRoute, href); + if (resolved) { + links.push(resolved); + } + } + + return links; + } + + private resolveRelative(currentRoute: string, href: string): string | null { + try { + const base = new URL(currentRoute, this.baseUrl); + const resolved = new URL(href, base.href); + if (resolved.origin !== new URL(this.baseUrl).origin) { + return null; + } + return resolved.pathname + resolved.search; + } catch { + return null; + } + } +} + diff --git a/src/apphosting/compare/discover.spec.ts b/src/apphosting/compare/discover.spec.ts new file mode 100644 index 00000000000..746095ecf98 --- /dev/null +++ b/src/apphosting/compare/discover.spec.ts @@ -0,0 +1,97 @@ +import { expect } from "chai"; +import * as fs from "fs-extra"; +import * as path from "path"; +import { discoverRoutes } from "./discover"; + +describe("discoverRoutes", () => { + let tempDir: string; + + beforeEach(() => { + tempDir = path.join( + process.cwd(), + "scratch-test-discover-" + Math.random().toString(36).substring(7), + ); + fs.ensureDirSync(tempDir); + }); + + afterEach(() => { + fs.removeSync(tempDir); + }); + + it("should return root route by default if no manifests exist", async () => { + const routes = await discoverRoutes(tempDir); + expect(routes).to.deep.equal(["/"]); + }); + + it("should discover routes from Next.js manifests", async () => { + const nextDir = path.join(tempDir, ".next"); + fs.ensureDirSync(nextDir); + + fs.writeJsonSync(path.join(nextDir, "prerender-manifest.json"), { + routes: { + "/about": {}, + "/blog/post-1": {}, + }, + }); + + fs.writeJsonSync(path.join(nextDir, "routes-manifest.json"), { + staticRoutes: [{ page: "/" }, { page: "/contact" }], + }); + + const routes = await discoverRoutes(tempDir); + expect(routes).to.deep.equal(["/", "/about", "/blog/post-1", "/contact"]); + }); + + it("should discover routes from sitemap.xml", async () => { + const xml = ` + + + + https://example.com/ + + + https://example.com/products?id=12 + + + `; + fs.writeFileSync(path.join(tempDir, "sitemap.xml"), xml, "utf-8"); + + const routes = await discoverRoutes(tempDir); + expect(routes).to.deep.equal(["/", "/products?id=12"]); + }); + + it("should discover routes from Next.js source app directory structure", async () => { + const appDir = path.join(tempDir, "src", "app"); + fs.ensureDirSync(path.join(appDir, "about")); + fs.ensureDirSync(path.join(appDir, "dashboard", "settings")); + fs.ensureDirSync(path.join(appDir, "blog", "[id]")); + fs.ensureDirSync(path.join(appDir, "(marketing)", "pricing")); + + fs.writeFileSync(path.join(appDir, "page.tsx"), "export default function P() {}"); + fs.writeFileSync(path.join(appDir, "about", "page.tsx"), "export default function P() {}"); + fs.writeFileSync(path.join(appDir, "dashboard", "settings", "page.tsx"), "export default function P() {}"); + fs.writeFileSync(path.join(appDir, "blog", "[id]", "page.tsx"), "export default function P() {}"); + fs.writeFileSync(path.join(appDir, "(marketing)", "pricing", "page.tsx"), "export default function P() {}"); + + const routes = await discoverRoutes(tempDir); + expect(routes).to.deep.equal(["/", "/about", "/blog/1", "/dashboard/settings", "/pricing"]); + }); + + it("should discover routes from Angular source TS routing modules", async () => { + const srcDir = path.join(tempDir, "src"); + fs.ensureDirSync(srcDir); + + const routingCode = ` + const routes: Routes = [ + { path: '', component: HomeComponent }, + { path: 'products', component: ProductsComponent }, + { path: 'user-profile', component: ProfileComponent }, + { path: '**', redirectTo: '' } + ]; + `; + fs.writeFileSync(path.join(srcDir, "app-routing.module.ts"), routingCode, "utf-8"); + + const routes = await discoverRoutes(tempDir); + expect(routes).to.deep.equal(["/", "/products", "/user-profile"]); + }); +}); diff --git a/src/apphosting/compare/discover.ts b/src/apphosting/compare/discover.ts new file mode 100644 index 00000000000..0d48deed310 --- /dev/null +++ b/src/apphosting/compare/discover.ts @@ -0,0 +1,201 @@ +import * as fs from "fs-extra"; +import * as path from "path"; +import { logger } from "./logger"; + +/** + * Recursively scans a directory for files matching a pattern. + */ +async function scanDir( + dir: string, + fileCallback: (filePath: string) => void | Promise, +): Promise { + if (!(await fs.pathExists(dir))) return; + const files = await fs.readdir(dir); + for (const file of files) { + const fullPath = path.join(dir, file); + const stat = await fs.stat(fullPath); + if (stat.isDirectory()) { + if ( + file !== "node-modules" && + file !== "node_modules" && + file !== ".git" && + file !== ".next" && + file !== ".angular" && + file !== "dist" + ) { + await scanDir(fullPath, fileCallback); + } + } else { + await fileCallback(fullPath); + } + } +} + +/** + * Discovers Next.js source routes from src/app, app, src/pages, or pages directory + */ +export async function discoverSourceRoutes(appPath: string): Promise { + const routes = new Set(); + + // Check App Router (src/app or app) + const appDirs = [path.join(appPath, "src", "app"), path.join(appPath, "app")]; + for (const appDir of appDirs) { + if (await fs.pathExists(appDir)) { + await scanDir(appDir, (filePath) => { + const base = path.basename(filePath); + if (/^(page|route)\.[jt]sx?$/.test(base)) { + const relDir = path.relative(appDir, path.dirname(filePath)); + let route = relDir ? "/" + relDir : "/"; + // Normalize dynamic params: [id] -> 1, [...catchall] -> 1 + route = route.replace(/\[\.\.\.[^\]]+\]/g, "1"); + route = route.replace(/\[[^\]]+\]/g, "1"); + // Remove Next.js route groups like (marketing) + route = route.replace(/\/\([^)]+\)/g, ""); + routes.add(route === "" ? "/" : route); + } + }); + } + } + + // Check Pages Router (src/pages or pages) + const pagesDirs = [path.join(appPath, "src", "pages"), path.join(appPath, "pages")]; + for (const pagesDir of pagesDirs) { + if (await fs.pathExists(pagesDir)) { + await scanDir(pagesDir, (filePath) => { + const ext = path.extname(filePath); + if ([".js", ".jsx", ".ts", ".tsx"].includes(ext)) { + const base = path.basename(filePath, ext); + if (["_app", "_document", "_error", "api"].includes(base) || filePath.includes("/api/")) { + return; + } + const relPath = path.relative(pagesDir, filePath); + let route = "/" + relPath.substring(0, relPath.length - ext.length); + if (route.endsWith("/index")) { + route = route.substring(0, route.length - 6); + } + // Normalize dynamic parameters + route = route.replace(/\[\.\.\.[^\]]+\]/g, "1"); + route = route.replace(/\[[^\]]+\]/g, "1"); + routes.add(route === "" ? "/" : route); + } + }); + } + } + + return Array.from(routes); +} + +/** + * Scans source TS files for Angular style route paths like `path: 'about'` + */ +export async function discoverAngularRoutes(appPath: string): Promise { + const routes = new Set(); + const srcDir = path.join(appPath, "src"); + if (await fs.pathExists(srcDir)) { + await scanDir(srcDir, async (filePath) => { + if (filePath.endsWith(".ts") && !filePath.endsWith(".spec.ts")) { + try { + const content = await fs.readFile(filePath, "utf-8"); + // Match path: 'about' or path: "about" + const pathRegex = /path\s*:\s*(['"])(.*?)\1/g; + let match; + while ((match = pathRegex.exec(content)) !== null) { + const val = match[2].trim(); + // Skip wildcards and dynamic parameter placeholders + if (val && !val.includes("**") && !val.startsWith(":") && !val.includes("/")) { + routes.add("/" + val); + } + } + } catch { + // Ignore read errors + } + } + }); + } + return Array.from(routes); +} + +/** + * Discovers built routes in a project by checking Next.js manifests, source pages, and sitemaps. + */ +export async function discoverRoutes(appPath: string): Promise { + const routes = new Set(["/"]); + + // 1. Next.js Manifest Parsing (Local Build Output) + const nextDir = path.join(appPath, ".next"); + if (await fs.pathExists(nextDir)) { + logger.info("Next.js build directory detected. Parsing manifests..."); + try { + const prerenderManifestPath = path.join(nextDir, "prerender-manifest.json"); + if (await fs.pathExists(prerenderManifestPath)) { + const prerender = await fs.readJson(prerenderManifestPath); + if (prerender.routes) { + for (const route of Object.keys(prerender.routes)) { + routes.add(route); + } + } + } + + const routesManifestPath = path.join(nextDir, "routes-manifest.json"); + if (await fs.pathExists(routesManifestPath)) { + const manifests = await fs.readJson(routesManifestPath); + if (manifests.staticRoutes) { + for (const route of manifests.staticRoutes) { + routes.add(route.page); + } + } + } + } catch (err) { + logger.debug(`Error parsing Next.js manifests: ${err}`); + } + } + + // 2. Next.js Source Tree Traversal + try { + const srcRoutes = await discoverSourceRoutes(appPath); + for (const r of srcRoutes) { + routes.add(r); + } + } catch (err) { + logger.debug(`Error scanning Next.js source routes: ${err}`); + } + + // 3. Angular Routing Scanner + try { + const angularRoutes = await discoverAngularRoutes(appPath); + for (const r of angularRoutes) { + routes.add(r); + } + } catch (err) { + logger.debug(`Error scanning Angular source routes: ${err}`); + } + + // 4. Local Sitemap Parsing + const sitemapPaths = [ + path.join(appPath, "public", "sitemap.xml"), + path.join(appPath, "sitemap.xml"), + path.join(appPath, "dist", "sitemap.xml"), + ]; + + for (const sitemapPath of sitemapPaths) { + if (await fs.pathExists(sitemapPath)) { + logger.info(`Sitemap detected at ${sitemapPath}. Parsing...`); + try { + const xml = await fs.readFile(sitemapPath, "utf-8"); + const locMatches = xml.matchAll(/\s*(https?:\/\/[^\s<]+)\s*<\/loc>/gi); + for (const match of locMatches) { + try { + const url = new URL(match[1].trim()); + routes.add(url.pathname + url.search); + } catch { + // Ignore invalid URLs + } + } + } catch (err) { + logger.debug(`Error parsing sitemap ${sitemapPath}: ${err}`); + } + } + } + + return Array.from(routes).sort(); +} diff --git a/src/apphosting/compare/distance.spec.ts b/src/apphosting/compare/distance.spec.ts new file mode 100644 index 00000000000..86ee0be00a2 --- /dev/null +++ b/src/apphosting/compare/distance.spec.ts @@ -0,0 +1,36 @@ +import { expect } from "chai"; +import { MyersDiffEngine } from "./distance"; + +describe("MyersDiffEngine", () => { + describe("getSimilarity", () => { + it("should return 1.0 for exact string match", () => { + const a = "line 1\nline 2\nline 3"; + const b = "line 1\nline 2\nline 3"; + expect(MyersDiffEngine.getSimilarity(a, b)).to.equal(1.0); + }); + + it("should return 1.0 for both empty strings", () => { + expect(MyersDiffEngine.getSimilarity("", "")).to.equal(1.0); + }); + + it("should return 0.0 if one string is empty", () => { + expect(MyersDiffEngine.getSimilarity("hello", "")).to.equal(0.0); + expect(MyersDiffEngine.getSimilarity("", "world")).to.equal(0.0); + }); + + it("should return correct similarity for partial match", () => { + // 2 matched lines, total lines A = 3, total lines B = 3 + // Similarity = (2 * 2) / (3 + 3) = 4 / 6 = 0.6666... + const a = "line 1\nline 2\nline 3"; + const b = "line 1\nline 2\nline 4"; + const similarity = MyersDiffEngine.getSimilarity(a, b); + expect(similarity).to.be.closeTo(0.666, 0.001); + }); + + it("should return 0.0 for completely disjoint strings", () => { + const a = "foo\nbar"; + const b = "baz\nqux"; + expect(MyersDiffEngine.getSimilarity(a, b)).to.equal(0.0); + }); + }); +}); diff --git a/src/apphosting/compare/distance.ts b/src/apphosting/compare/distance.ts new file mode 100644 index 00000000000..60e4c7d3722 --- /dev/null +++ b/src/apphosting/compare/distance.ts @@ -0,0 +1,41 @@ +import { diffLines } from "diff"; +import * as crypto from "crypto"; + +export class MyersDiffEngine { + /** + * Calculates a similarity score between 0.0 and 1.0 based on line differences. + */ + public static getSimilarity(a: string, b: string): number { + if (a === b) return 1.0; + if (a.length === 0 && b.length === 0) return 1.0; + if (a.length === 0 || b.length === 0) return 0.0; + + // Fast-path: hash comparison + const hashA = crypto.createHash("sha256").update(a).digest("hex"); + const hashB = crypto.createHash("sha256").update(b).digest("hex"); + if (hashA === hashB) return 1.0; + + const changes = diffLines(a, b); + let matchedLines = 0; + let totalLinesA = 0; + let totalLinesB = 0; + + for (const change of changes) { + const count = change.count || 0; + if (!change.added && !change.removed) { + matchedLines += count; + totalLinesA += count; + totalLinesB += count; + } else if (change.added) { + totalLinesB += count; + } else if (change.removed) { + totalLinesA += count; + } + } + + const totalLines = totalLinesA + totalLinesB; + if (totalLines === 0) return 1.0; + + return (2 * matchedLines) / totalLines; + } +} diff --git a/src/apphosting/compare/lifecycle.spec.ts b/src/apphosting/compare/lifecycle.spec.ts new file mode 100644 index 00000000000..2107e9ea265 --- /dev/null +++ b/src/apphosting/compare/lifecycle.spec.ts @@ -0,0 +1,63 @@ +import { expect } from "chai"; +import * as sinon from "sinon"; +import * as apphosting from "../../gcp/apphosting"; +import { validateProject, runGarbageCollection } from "./lifecycle"; + +describe("Lifecycle Manager", () => { + let listBackendsStub: sinon.SinonStub; + let updateBackendStub: sinon.SinonStub; + + beforeEach(() => { + listBackendsStub = sinon.stub(apphosting, "listBackends"); + updateBackendStub = sinon.stub(apphosting, "updateBackend"); + }); + + afterEach(() => { + sinon.restore(); + }); + + describe("validateProject", () => { + it("should allow whitelisted projects", () => { + expect(() => validateProject("aryanf-test")).to.not.throw(); + expect(() => validateProject("pretend-public")).to.not.throw(); + }); + + it("should throw on non-whitelisted projects", () => { + expect(() => validateProject("my-secret-project")).to.throw( + /Invalid project ID "my-secret-project"/, + ); + }); + }); + + describe("runGarbageCollection", () => { + it("should unlock stale busy backends (> 2 hours)", async () => { + const now = Date.now(); + const threeHoursAgo = new Date(now - 3 * 60 * 60 * 1000).toISOString(); + const oneHourAgo = new Date(now - 1 * 60 * 60 * 1000).toISOString(); + + listBackendsStub.resolves({ + backends: [ + { + name: "projects/test/locations/us-central1/backends/compare-slot-1-a", + labels: { status: "busy", type: "comparison-sandbox" }, + updateTime: threeHoursAgo, + }, + { + name: "projects/test/locations/us-central1/backends/compare-slot-1-b", + labels: { status: "busy", type: "comparison-sandbox" }, + updateTime: oneHourAgo, + }, + ], + }); + + updateBackendStub.resolves({ name: "op-name" }); + + await runGarbageCollection("aryanf-test", "us-central1"); + + expect(updateBackendStub.callCount).to.equal(1); + const args = updateBackendStub.firstCall.args; + expect(args[2]).to.equal("compare-slot-1-a"); + expect(args[3].labels.status).to.equal("idle"); + }); + }); +}); diff --git a/src/apphosting/compare/lifecycle.ts b/src/apphosting/compare/lifecycle.ts new file mode 100644 index 00000000000..02b90188230 --- /dev/null +++ b/src/apphosting/compare/lifecycle.ts @@ -0,0 +1,55 @@ +import { FirebaseError } from "../../error"; +import * as apphosting from "../../gcp/apphosting"; +import { logger } from "./logger"; + +const ALLOWED_PROJECTS = [ + "aryanf-test", + "pretend-public", + ...(process.env.APP_HOSTING_COMPARE_ALLOWED_PROJECTS || "") + .split(",") + .map((p) => p.trim()) + .filter(Boolean), +]; + +/** + * + */ +export function validateProject(projectId: string): void { + if (!ALLOWED_PROJECTS.includes(projectId)) { + throw new FirebaseError( + `Invalid project ID "${projectId}". This tool can only run on projects: ${ALLOWED_PROJECTS.join(", ")}`, + ); + } +} + +/** + * Sweeps all slots in the project, resetting stale locks (busy for > 2 hours) to idle. + */ +export async function runGarbageCollection(projectId: string, location: string): Promise { + const existingBackends = await apphosting.listBackends(projectId, location); + const backendsList = existingBackends.backends || []; + const now = Date.now(); + const twoHours = 2 * 60 * 60 * 1000; + + for (const backend of backendsList) { + const nameParts = backend.name.split("/"); + const backendId = nameParts[nameParts.length - 1]; + + if (backendId.startsWith("compare-slot-")) { + const isBusy = backend.labels?.status === "busy"; + if (isBusy) { + const updateTime = new Date(backend.updateTime).getTime(); + if (now - updateTime > twoHours) { + logger.info(`Found stale lock on comparison slot backend ${backendId}. Unlocking...`); + try { + await apphosting.updateBackend(projectId, location, backendId, { + labels: { ...backend.labels, status: "idle" }, + }); + } catch (err) { + logger.debug(`Failed to unlock stale backend ${backendId}: ${err}`); + } + } + } + } + } +} diff --git a/src/apphosting/compare/logger.ts b/src/apphosting/compare/logger.ts new file mode 100644 index 00000000000..f7dd49b97c5 --- /dev/null +++ b/src/apphosting/compare/logger.ts @@ -0,0 +1,49 @@ +// Smart local logger proxy to decouple compare module from firebase-tools +let internalLogger: any = console; + +try { + // Attempt to load global firebase-tools logger + const globalLogger = require("../../logger").logger; + if (globalLogger) { + internalLogger = globalLogger; + } +} catch (e) { + // Standalone fallback + internalLogger = { + info: (msg: string) => console.log(msg), + warn: (msg: string) => console.warn(msg), + error: (msg: string) => console.error(msg), + debug: (msg: string) => console.debug(msg), + }; +} + +export const logger = { + info: (msg: string, ...args: any[]) => { + if (typeof internalLogger.info === "function") { + internalLogger.info(msg, ...args); + } else { + console.log(msg, ...args); + } + }, + warn: (msg: string, ...args: any[]) => { + if (typeof internalLogger.warn === "function") { + internalLogger.warn(msg, ...args); + } else { + console.warn(msg, ...args); + } + }, + error: (msg: string, ...args: any[]) => { + if (typeof internalLogger.error === "function") { + internalLogger.error(msg, ...args); + } else { + console.error(msg, ...args); + } + }, + debug: (msg: string, ...args: any[]) => { + if (typeof internalLogger.debug === "function") { + internalLogger.debug(msg, ...args); + } else { + console.debug(msg, ...args); + } + } +}; diff --git a/src/apphosting/compare/matrices/angular-matrix.json b/src/apphosting/compare/matrices/angular-matrix.json new file mode 100644 index 00000000000..9e26d003e59 --- /dev/null +++ b/src/apphosting/compare/matrices/angular-matrix.json @@ -0,0 +1,10 @@ +[ + { + "name": "Angular 19", + "path": "../firebase-apphosting-canary/apps/angular-reference/angular-19", + "matrix": { + "localBuild": [true, false], + "runtime": ["nodejs24"] + } + } +] diff --git a/src/apphosting/compare/matrices/canary-matrix.json b/src/apphosting/compare/matrices/canary-matrix.json new file mode 100644 index 00000000000..133096a9b23 --- /dev/null +++ b/src/apphosting/compare/matrices/canary-matrix.json @@ -0,0 +1,42 @@ +[ + { + "name": "Angular 19", + "path": "../firebase-apphosting-canary/apps/angular-reference/angular-19", + "matrix": { + "localBuild": [true, false], + "runtime": ["nodejs24", "nodejs22", "nodejs"] + } + }, + { + "name": "Angular 20", + "path": "../firebase-apphosting-canary/apps/angular-reference/angular-20", + "matrix": { + "localBuild": [true, false], + "runtime": ["nodejs24", "nodejs22", "nodejs"] + } + }, + { + "name": "Angular 21", + "path": "../firebase-apphosting-canary/apps/angular-reference/angular-21", + "matrix": { + "localBuild": [true, false], + "runtime": ["nodejs24", "nodejs22", "nodejs"] + } + }, + { + "name": "Angular 22", + "path": "../firebase-apphosting-canary/apps/angular-reference/angular-22", + "matrix": { + "localBuild": [true, false], + "runtime": ["nodejs24", "nodejs22", "nodejs"] + } + }, + { + "name": "Next.js 15.3", + "path": "../firebase-apphosting-canary/apps/nextjs-reference/next-15.3", + "matrix": { + "localBuild": [true, false], + "runtime": ["nodejs24", "nodejs22", "nodejs"] + } + } +] diff --git a/src/apphosting/compare/matrices/matrix-test.json b/src/apphosting/compare/matrices/matrix-test.json new file mode 100644 index 00000000000..c9e70ad43ef --- /dev/null +++ b/src/apphosting/compare/matrices/matrix-test.json @@ -0,0 +1,11 @@ +[ + { + "name": "Node Matrix Test", + "path": "/Users/aryanf/code/firebase/next-sample-1", + "variants": [ + { "id": "Local-Node24", "localBuild": true, "runtime": "nodejs24" }, + { "id": "Source-Node24", "localBuild": false, "runtime": "nodejs24" }, + { "id": "Source-Node22", "localBuild": false, "runtime": "nodejs22" } + ] + } +] diff --git a/src/apphosting/compare/matrices/next-canary-matrix.json b/src/apphosting/compare/matrices/next-canary-matrix.json new file mode 100644 index 00000000000..f9ab4977951 --- /dev/null +++ b/src/apphosting/compare/matrices/next-canary-matrix.json @@ -0,0 +1,10 @@ +[ + { + "name": "Next.js 15.3", + "path": "../firebase-apphosting-canary/apps/nextjs-reference/next-15.3", + "matrix": { + "localBuild": [true, false], + "runtime": ["nodejs24"] + } + } +] diff --git a/src/apphosting/compare/public/index.css b/src/apphosting/compare/public/index.css new file mode 100644 index 00000000000..e70abf8c47e --- /dev/null +++ b/src/apphosting/compare/public/index.css @@ -0,0 +1,661 @@ +:root { + --bg-dark: #0f172a; + --bg-panel: #1e293b; + --border: #334155; + --text: #f8fafc; + --text-muted: #94a3b8; + --accent: #3b82f6; + --success: #10b981; + --warning: #f59e0b; + --danger: #ef4444; + --font-family: 'Inter', sans-serif; +} + +body { + background-color: var(--bg-dark); + color: var(--text); + font-family: var(--font-family); + margin: 0; + padding: 0; + height: 100vh; + display: flex; + flex-direction: column; + overflow: hidden; +} + +header { + background-color: var(--bg-panel); + border-bottom: 1px solid var(--border); + padding: 16px 24px; + display: flex; + align-items: center; + justify-content: space-between; +} + +header h1 { + margin: 0; + font-size: 20px; + font-weight: 600; + letter-spacing: -0.025em; + display: flex; + align-items: center; + gap: 10px; +} + +header h1 span { + background: linear-gradient(135deg, #60a5fa, #3b82f6); + -webkit-background-clip: text; + -webkit-text-fill-color: transparent; +} + +main { + flex: 1; + display: flex; + overflow: hidden; +} + +/* Sidebar Toggle Button */ +.sidebar-toggle { + background: rgba(255, 255, 255, 0.02); + border: 1px solid var(--border); + color: var(--text-muted); + border-radius: 6px; + padding: 6px; + cursor: pointer; + display: flex; + align-items: center; + justify-content: center; + transition: all 0.15s ease; + outline: none; +} +.sidebar-toggle:hover { + background: rgba(59, 130, 246, 0.08); + color: var(--accent); + border-color: rgba(59, 130, 246, 0.3); +} +.sidebar-toggle.collapsed-active { + background: rgba(59, 130, 246, 0.12); + color: var(--accent); + border-color: var(--accent); +} + +/* Sidebar */ +.sidebar { + width: 320px; + background-color: var(--bg-panel); + border-right: 1px solid var(--border); + display: flex; + flex-direction: column; + overflow-y: auto; + padding: 20px; + box-sizing: border-box; + gap: 24px; + transition: width 0.25s cubic-bezier(0.4, 0, 0.2, 1), + padding 0.25s cubic-bezier(0.4, 0, 0.2, 1), + opacity 0.15s ease, + border-right-color 0.25s ease; +} + +.sidebar.collapsed { + width: 0; + padding-left: 0 !important; + padding-right: 0 !important; + border-right-color: transparent; + opacity: 0; + pointer-events: none; +} + +.section-title { + font-size: 11px; + text-transform: uppercase; + font-weight: 700; + letter-spacing: 0.05em; + color: var(--text-muted); + margin-bottom: 8px; +} + +.list-group { + display: flex; + flex-direction: column; + gap: 8px; +} + +.list-item { + padding: 12px; + background-color: rgba(255,255,255,0.02); + border: 1px solid var(--border); + border-radius: 8px; + cursor: pointer; + font-size: 14px; + transition: all 0.2s ease; +} + +.list-item:hover, .list-item.active { + background-color: rgba(59, 130, 246, 0.1); + border-color: var(--accent); +} + +.list-item.active { + font-weight: 600; +} + +/* Comparison selector */ +.compare-select { + display: flex; + flex-direction: column; + gap: 12px; +} + +select { + width: 100%; + background-color: var(--bg-dark); + color: var(--text); + border: 1px solid var(--border); + padding: 10px; + border-radius: 6px; + font-size: 14px; + outline: none; +} + +button.btn { + background-color: var(--accent); + color: white; + border: none; + padding: 10px; + border-radius: 6px; + cursor: pointer; + font-weight: 600; + font-size: 14px; + transition: background-color 0.2s; +} + +button.btn:hover { + background-color: #2563eb; +} + +/* Content Area */ +.content { + flex: 1; + display: flex; + flex-direction: column; + overflow: hidden; + padding: 24px; + box-sizing: border-box; + gap: 20px; +} + +.card { + background-color: var(--bg-panel); + border: 1px solid var(--border); + border-radius: 12px; + display: flex; + flex-direction: column; + overflow: hidden; +} + +.card-header { + padding: 16px 20px; + border-bottom: 1px solid var(--border); + font-weight: 600; + font-size: 16px; +} + +/* Routes List in Sidebar */ +.route-item { + padding: 10px 14px; + background-color: rgba(255,255,255,0.01); + border: 1px solid var(--border); + border-radius: 6px; + display: flex; + flex-direction: column; + gap: 6px; + cursor: pointer; + transition: all 0.2s ease; + margin-bottom: 8px; +} + +.route-item:hover, .route-item.active { + background-color: rgba(59, 130, 246, 0.1); + border-color: var(--accent); +} + +.route-path { + font-family: 'JetBrains Mono', monospace; + font-size: 12px; + word-break: break-all; +} + +.badges { + display: flex; + gap: 8px; +} + +.badge { + font-size: 11px; + padding: 3px 8px; + border-radius: 9999px; + font-weight: 600; +} + +.badge.success { background-color: rgba(16, 185, 129, 0.15); color: var(--success); } +.badge.warning { background-color: rgba(245, 158, 11, 0.15); color: var(--warning); } +.badge.danger { background-color: rgba(239, 68, 68, 0.15); color: var(--danger); } + +/* Split Details View */ +.details-view { + flex: 1; + display: flex; + flex-direction: column; + overflow: hidden; +} + +.details-view > .card { + flex: 1; + display: flex; + flex-direction: column; + overflow: hidden; +} + +.panel-body { + padding: 20px; + overflow-y: auto; + flex: 1; + display: flex; + flex-direction: column; + gap: 16px; +} + +.table-container { + border: 1px solid var(--border); + border-radius: 8px; + overflow: hidden; +} + +table { + width: 100%; + border-collapse: collapse; + font-size: 13px; + text-align: left; +} + +th, td { + padding: 10px 12px; + border-bottom: 1px solid var(--border); +} + +th { + background-color: rgba(255,255,255,0.02); + font-weight: 600; +} + +tr:last-child td { + border-bottom: none; +} + +.diff-container { + flex: 1; + overflow-y: auto; + background-color: #0d1117; + border: 1px solid var(--border); + border-radius: 8px; +} + +.d2h-file-header { + display: none; /* Hide header details */ +} + +.empty-state { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + flex: 1; + color: var(--text-muted); + gap: 12px; +} + +.empty-state svg { + width: 48px; + height: 48px; + stroke: var(--text-muted); +} + +/* Heatmap Styles */ +.heatmap-table { + width: auto !important; + border-collapse: separate; + border-spacing: 2px; + font-size: 13px; + margin: 20px auto; +} +.heatmap-cell-wrapper { + padding: 0 !important; + border: none !important; + background: transparent !important; + vertical-align: middle; + text-align: center; +} +.heatmap-cell-inner { + box-sizing: border-box; + width: 6.5vw; + height: 6.5vw; + min-width: 44px; + min-height: 44px; + max-width: 72px; + max-height: 72px; + display: flex; + align-items: center; + justify-content: center; + font-weight: 700; + font-size: 12px; + border-radius: 3px; + cursor: pointer; + transition: transform 0.1s ease, filter 0.1s ease, box-shadow 0.1s ease; + color: #0f172a; + margin: auto; +} +.heatmap-cell-inner:hover { + transform: scale(1.08); + filter: brightness(1.15); + box-shadow: 0 10px 15px -3px rgba(0, 0, 0, 0.4), 0 4px 6px -4px rgba(0, 0, 0, 0.4); + z-index: 10; + position: relative; +} +.heatmap-cell-inner.de-emphasized { + opacity: 0.15; + filter: grayscale(80%); +} +.heatmap-cell-inner.de-emphasized:hover { + opacity: 0.35; + filter: grayscale(40%); +} + +.heatmap-header-cell { + box-sizing: border-box; + padding: 4px; + font-weight: 600; + color: var(--text-muted); + font-size: 12px; + text-align: center; + width: 6.5vw; + min-width: 44px; + max-width: 72px; + word-wrap: break-word; + transition: color 0.15s, background-color 0.15s, border-radius 0.15s; +} +.heatmap-row-label { + padding-right: 14px; + font-weight: 600; + color: var(--text-muted); + font-size: 12px; + text-align: right; + width: 10vw; + min-width: 70px; + max-width: 110px; + word-wrap: break-word; + transition: color 0.15s, background-color 0.15s, border-radius 0.15s; +} + +.heatmap-header-cell.highlighted, +.heatmap-row-label.highlighted { + color: var(--accent) !important; + background-color: rgba(59, 130, 246, 0.1); + border-radius: 4px; +} + +/* Unified Diff Panel Styles */ +.diff-view { + font-family: 'JetBrains Mono', monospace; + font-size: 12px; + line-height: 1.6; + white-space: pre; + overflow-x: auto; + padding: 16px; + color: #e2e8f0; + background-color: #0f172a; + height: 100%; + box-sizing: border-box; + margin: 0; +} +.diff-line { + display: flex; + padding: 2px 8px; + border-radius: 2px; +} +.diff-line.added { + background-color: rgba(16, 185, 129, 0.15); + color: #10b981; +} +.diff-line.removed { + background-color: rgba(239, 68, 68, 0.15); + color: #ef4444; +} +.diff-prefix { + width: 24px; + user-select: none; + color: var(--text-muted); + opacity: 0.7; +} +/* Filter Dropdowns styles */ +.filter-dropdown-container { + position: relative; + display: inline-block; +} + +.filter-dropdown-btn { + background-color: var(--bg-dark); + border: 1px solid var(--border); + color: var(--text); + padding: 6px 12px; + border-radius: 6px; + font-family: var(--font-family); + font-size: 12px; + font-weight: 500; + cursor: pointer; + display: flex; + align-items: center; + gap: 6px; + outline: none; + transition: border-color 0.2s, background-color 0.2s; +} + +.filter-dropdown-btn:hover { + border-color: var(--accent); + background-color: rgba(255,255,255,0.02); +} + +.filter-dropdown-btn::after { + content: ""; + border: solid var(--text-muted); + border-width: 0 1.5px 1.5px 0; + display: inline-block; + padding: 2px; + transform: rotate(45deg); + margin-left: 4px; + transition: transform 0.2s; +} + +.filter-dropdown-container.open .filter-dropdown-btn::after { + transform: rotate(-135deg); +} + +.filter-dropdown-content { + display: none; + position: absolute; + background-color: var(--bg-panel); + border: 1px solid var(--border); + border-radius: 8px; + box-shadow: 0 10px 25px -5px rgba(0, 0, 0, 0.5); + z-index: 100; + min-width: 180px; + max-width: 250px; + margin-top: 4px; + padding: 8px; + box-sizing: border-box; +} + +.filter-dropdown-container.open .filter-dropdown-content { + display: block; +} + +.filter-search-box { + background-color: var(--bg-dark); + border: 1px solid var(--border); + color: var(--text); + width: 100%; + padding: 6px 8px; + border-radius: 4px; + font-family: var(--font-family); + font-size: 11px; + box-sizing: border-box; + outline: none; + margin-bottom: 8px; +} + +.filter-search-box:focus { + border-color: var(--accent); +} + +.filter-options-list { + max-height: 180px; + overflow-y: auto; + display: flex; + flex-direction: column; + gap: 4px; +} + +.filter-opt-item { + display: flex; + align-items: center; + gap: 8px; + padding: 6px 8px; + border-radius: 4px; + cursor: pointer; + user-select: none; + font-size: 12px; + color: var(--text); + transition: background-color 0.15s; +} + +.filter-opt-item:hover { + background-color: rgba(255,255,255,0.04); +} + +.filter-opt-item input[type="checkbox"] { + cursor: pointer; + margin: 0; +} + +/* Premium Pills & Collapsible Ignore List Styling */ +.pill-badge { + display: inline-flex; + align-items: center; + gap: 6px; + background: rgba(255, 255, 255, 0.04); + border: 1px solid var(--border); + border-radius: 100px; + padding: 3px 10px; + font-size: 11px; + color: var(--text); + font-family: monospace; + transition: all 0.15s ease; +} +.pill-badge:hover { + background: rgba(255, 255, 255, 0.08); + border-color: var(--accent); +} +.pill-remove-btn { + cursor: pointer; + color: var(--text-muted); + font-weight: bold; + font-size: 12px; + transition: color 0.1s ease; +} +.pill-remove-btn:hover { + color: var(--danger); +} +.add-pill-input { + background: transparent; + border: 1px dashed var(--border); + color: var(--text-muted); + border-radius: 100px; + padding: 3px 10px; + font-size: 11px; + font-family: monospace; + outline: none; + width: 90px; + transition: all 0.15s ease; +} +.add-pill-input:focus { + width: 140px; + border-color: var(--accent); + color: var(--text); + border-style: solid; + background: var(--bg-dark); +} +.header-row-action { + cursor: pointer; + font-size: 9px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.5px; + margin-left: 8px; + color: var(--accent); + opacity: 0; + border: 1px solid rgba(59, 130, 246, 0.3); + padding: 1px 5px; + border-radius: 3px; + background: rgba(59, 130, 246, 0.05); + transition: all 0.15s ease; + display: inline-block; + vertical-align: middle; +} +tr:hover .header-row-action { + opacity: 1; +} +.header-row-action:hover { + background: var(--accent); + color: #fff !important; +} + +/* Autocomplete Suggestions Dropdown Styling */ +.ignore-suggestion-item { + padding: 6px 10px; + cursor: pointer; + color: var(--text-muted); + font-family: monospace; + font-size: 11px; + transition: all 0.1s ease; +} +.ignore-suggestion-item:hover { + background: rgba(255, 255, 255, 0.05); + color: var(--accent); +} + +/* Premium Loading Overlay for Heatmap Matrix */ +.spinner-overlay { + position: absolute; + top: 0; + left: 0; + right: 0; + bottom: 0; + background: rgba(15, 23, 42, 0.85); /* Slate-900 transparent background */ + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + z-index: 10; + backdrop-filter: blur(2px); + transition: all 0.15s ease; +} +.spinner { + width: 32px; + height: 32px; + border: 3px solid rgba(59, 130, 246, 0.1); + border-top-color: var(--accent); + border-radius: 50%; + animation: spin 0.8s linear infinite; +} +@keyframes spin { + to { transform: rotate(360deg); } +} diff --git a/src/apphosting/compare/public/index.html b/src/apphosting/compare/public/index.html new file mode 100644 index 00000000000..0629b230663 --- /dev/null +++ b/src/apphosting/compare/public/index.html @@ -0,0 +1,347 @@ + + + + + + Parity Comparison Dashboard + + + + + + + + + + + + + + +
+
+ +

Firebase App Hosting Parity Dashboard

+
+
Connected
+
+ +
+ +
+
+
Test Cases
+
+ +
+
+ + + + + +
+ + +
+ + + +
+ + + + + +
No Test Suite Selected
+
Select a test suite from the sidebar to visualize the joint parity heatmap and compare routing behavior.
+
+ + + + + + + +
+
+ + + diff --git a/src/apphosting/compare/public/index.js b/src/apphosting/compare/public/index.js new file mode 100644 index 00000000000..50fdecd7693 --- /dev/null +++ b/src/apphosting/compare/public/index.js @@ -0,0 +1,1341 @@ +function formatVariantName(v) { + if (!v) return ""; + if (v.includes("/")) { + const [tcPart, varPart] = v.split("/"); + return `${tcPart.replace(/\s+/g, "-")} ${varPart.replace(/-/g, " ")}`; + } + return v.replace(/-/g, " "); +} + +let activeTestCase = ""; +let recordingsData = {}; +let comparisonResults = []; +let activeUrlA = ""; +let activeUrlB = ""; +let activeDeployTimeA = 0; +let activeDeployTimeB = 0; +let activeLocalBuildA = false; +let activeLocalBuildB = false; +let activeRuntimeA = ""; +let activeRuntimeB = ""; +let activePathA = ""; +let activePathB = ""; +let lastMatrixData = null; + +// Load settings from localStorage with robust fallbacks +const defaultIgnoreList = ["date", "etag", "x-cloud-trace-context", "x-powered-by", "connection", "keep-alive", "server-timing", "traceparent"]; +let activeIgnoreList = defaultIgnoreList; +try { + const storedIgnore = localStorage.getItem("apphosting_compare_ignore_headers"); + if (storedIgnore) { + activeIgnoreList = JSON.parse(storedIgnore); + } +} catch (e) { + activeIgnoreList = defaultIgnoreList; +} + +let activeScoringMode = "body"; +try { + const storedScoring = localStorage.getItem("apphosting_compare_scoring_mode"); + if (storedScoring) { + activeScoringMode = storedScoring; + } +} catch (e) { + activeScoringMode = "body"; +} + +let isIgnoreCollapsed = false; +try { + const storedCollapse = localStorage.getItem("apphosting_compare_ignore_collapsed"); + if (storedCollapse) { + isIgnoreCollapsed = storedCollapse === "true"; + } +} catch (e) { + isIgnoreCollapsed = false; +} + +let activeColorMode = "absolute"; +try { + const storedColorMode = localStorage.getItem("apphosting_compare_color_mode"); + if (storedColorMode) { + activeColorMode = storedColorMode; + } +} catch (e) { + activeColorMode = "absolute"; +} + +function saveSettings() { + try { + localStorage.setItem("apphosting_compare_ignore_headers", JSON.stringify(activeIgnoreList)); + localStorage.setItem("apphosting_compare_scoring_mode", activeScoringMode); + localStorage.setItem("apphosting_compare_ignore_collapsed", String(isIgnoreCollapsed)); + localStorage.setItem("apphosting_compare_color_mode", activeColorMode); + } catch (e) {} +} + +// Sidebar Collapsible State +let isSidebarCollapsed = false; +try { + const storedSidebar = localStorage.getItem("apphosting_compare_sidebar_collapsed"); + if (storedSidebar) { + isSidebarCollapsed = storedSidebar === "true"; + } +} catch (e) { + isSidebarCollapsed = false; +} + +// Restore sidebar collapsed state on load +document.addEventListener("DOMContentLoaded", () => { + if (isSidebarCollapsed) { + const sidebar = document.querySelector(".sidebar"); + const btn = document.getElementById("sidebar-toggle-btn"); + if (sidebar) sidebar.classList.add("collapsed"); + if (btn) btn.classList.add("collapsed-active"); + } +}); + +function toggleSidebar() { + const sidebar = document.querySelector(".sidebar"); + const btn = document.getElementById("sidebar-toggle-btn"); + if (!sidebar || !btn) return; + + sidebar.classList.toggle("collapsed"); + btn.classList.toggle("collapsed-active"); + + isSidebarCollapsed = sidebar.classList.contains("collapsed"); + try { + localStorage.setItem("apphosting_compare_sidebar_collapsed", String(isSidebarCollapsed)); + } catch (e) {} +} + +function applyColorMode() { + const select = document.getElementById("select-color-mode"); + if (!select) return; + activeColorMode = select.value; + saveSettings(); + + if (lastMatrixData) { + applyMetadataFilters(); + } +} + +function toggleIgnorePillsCollapse() { + isIgnoreCollapsed = !isIgnoreCollapsed; + saveSettings(); + applyIgnoreCollapseState(); +} + +function applyIgnoreCollapseState() { + const wrapper = document.getElementById("ignore-pills-wrapper"); + const icon = document.getElementById("ignore-collapse-icon"); + if (!wrapper || !icon) return; + + if (isIgnoreCollapsed) { + wrapper.style.maxHeight = "0px"; + icon.textContent = "Expand"; + } else { + wrapper.style.maxHeight = "500px"; + icon.textContent = "Collapse"; + } +} + +function getAvailableHeaders() { + const defaultSuggestions = ["content-type", "cache-control", "content-encoding", "server-timing", "traceparent", "date", "etag", "x-cloud-trace-context", "x-powered-by", "connection", "keep-alive", "transfer-encoding", "vary", "strict-transport-security", "x-content-type-options", "x-frame-options", "x-xss-protection"]; + const set = new Set(defaultSuggestions); + + // If a detailed comparison is loaded, merge all active response headers! + if (window.activeResObject) { + if (window.activeResObject.variantA && window.activeResObject.variantA.headers) { + Object.keys(window.activeResObject.variantA.headers).forEach(h => set.add(h.toLowerCase())); + } + if (window.activeResObject.variantB && window.activeResObject.variantB.headers) { + Object.keys(window.activeResObject.variantB.headers).forEach(h => set.add(h.toLowerCase())); + } + } + + // Return sorted list of headers that are NOT currently ignored + return Array.from(set) + .filter(h => !activeIgnoreList.includes(h)) + .sort(); +} + +function renderIgnorePills() { + const wrapper = document.getElementById("ignore-pills-wrapper"); + const countEl = document.getElementById("ignore-count"); + if (!wrapper || !countEl) return; + + countEl.textContent = activeIgnoreList.length; + wrapper.innerHTML = ""; + + // 1. Render active pills + activeIgnoreList.forEach(header => { + const pill = document.createElement("span"); + pill.className = "pill-badge"; + pill.textContent = header + " "; + + const removeBtn = document.createElement("span"); + removeBtn.className = "pill-remove-btn"; + removeBtn.innerHTML = "×"; + removeBtn.onclick = (e) => { + e.stopPropagation(); + removeIgnoreHeader(header); + }; + + pill.appendChild(removeBtn); + wrapper.appendChild(pill); + }); + + // 2. Render "+ Add Header" input wrapped in suggestion dropdown container + const container = document.createElement("div"); + container.className = "add-pill-container"; + container.style.position = "relative"; + container.style.display = "inline-block"; + + const input = document.createElement("input"); + input.type = "text"; + input.id = "add-pill-input"; + input.className = "add-pill-input"; + input.placeholder = "+ Add Header"; + input.autocomplete = "off"; + + const dropdown = document.createElement("div"); + dropdown.id = "ignore-suggestions-dropdown"; + dropdown.style.display = "none"; + dropdown.style.position = "absolute"; + dropdown.style.top = "100%"; + dropdown.style.left = "0"; + dropdown.style.background = "var(--bg-dark)"; + dropdown.style.border = "1px solid var(--border)"; + dropdown.style.borderRadius = "4px"; + dropdown.style.boxShadow = "0 4px 12px rgba(0,0,0,0.5)"; + dropdown.style.zIndex = "1000"; + dropdown.style.minWidth = "160px"; + dropdown.style.maxHeight = "200px"; + dropdown.style.overflowY = "auto"; + dropdown.style.marginTop = "4px"; + + // Hook up suggestions functions + input.onfocus = () => showIgnoreSuggestions(input, dropdown); + input.onblur = () => { + // Delay hiding so that item click handler can trigger + setTimeout(() => { + dropdown.style.display = "none"; + }, 150); + }; + input.oninput = () => filterIgnoreSuggestions(input, dropdown); + + const commitHeader = () => { + const val = input.value.trim().toLowerCase(); + if (val.length > 0 && !activeIgnoreList.includes(val)) { + activeIgnoreList.push(val); + saveSettings(); + renderIgnorePills(); + triggerIgnoreRefresh(); + } + input.value = ""; + }; + + input.onkeydown = (e) => { + if (e.key === "Enter") { + commitHeader(); + } + }; + + container.appendChild(input); + container.appendChild(dropdown); + wrapper.appendChild(container); +} + +function showIgnoreSuggestions(input, dropdown) { + const suggestions = getAvailableHeaders(); + renderSuggestionList(suggestions, input, dropdown); + dropdown.style.display = "block"; +} + +function filterIgnoreSuggestions(input, dropdown) { + const query = input.value.trim().toLowerCase(); + const suggestions = getAvailableHeaders().filter(h => h.includes(query)); + renderSuggestionList(suggestions, input, dropdown); + dropdown.style.display = "block"; +} + +function renderSuggestionList(suggestions, input, dropdown) { + dropdown.innerHTML = ""; + if (suggestions.length === 0) { + const item = document.createElement("div"); + item.style.padding = "6px 10px"; + item.style.color = "var(--text-muted)"; + item.style.fontSize = "11px"; + item.textContent = "No suggestions"; + dropdown.appendChild(item); + return; + } + + suggestions.forEach(header => { + const item = document.createElement("div"); + item.className = "ignore-suggestion-item"; + item.textContent = header; + item.onmousedown = (e) => { + // Prevent input blur before click + e.preventDefault(); + }; + item.onclick = () => { + activeIgnoreList.push(header); + saveSettings(); + renderIgnorePills(); + triggerIgnoreRefresh(); + }; + dropdown.appendChild(item); + }); +} + +function removeIgnoreHeader(header) { + activeIgnoreList = activeIgnoreList.filter(h => h !== header.toLowerCase()); + saveSettings(); + renderIgnorePills(); + triggerIgnoreRefresh(); +} + +function toggleHeaderIgnoreState(header) { + const canonical = header.toLowerCase(); + if (activeIgnoreList.includes(canonical)) { + activeIgnoreList = activeIgnoreList.filter(h => h !== canonical); + } else { + activeIgnoreList.push(canonical); + } + saveSettings(); + renderIgnorePills(); + triggerIgnoreRefresh(); +} + +function triggerIgnoreRefresh() { + // 1. Immediately re-filter current detailed headers table if active + if (window.activeResObject && typeof window.renderActiveHeaders === "function") { + window.renderActiveHeaders(); + } + + // 2. Debounce and refresh the heatmap/matrix grid + if (activeTestCase) { + if (window.ignoreTimeout) clearTimeout(window.ignoreTimeout); + window.ignoreTimeout = setTimeout(() => { + loadHeatmap(activeTestCase); + }, 400); + } +} + +// Fetch list of recordings on load +async function loadRecordings() { + const res = await fetch("/api/recordings"); + recordingsData = await res.json(); + + const container = document.getElementById("test-cases-list"); + container.innerHTML = ""; + + // Add GLOBAL test case + const globalItem = document.createElement("div"); + globalItem.className = "list-item"; + globalItem.style.fontWeight = "bold"; + globalItem.style.color = "var(--accent)"; + globalItem.textContent = "GLOBAL MATRIX (All Apps)"; + globalItem.onclick = () => selectTestCase("GLOBAL", globalItem); + container.appendChild(globalItem); + + Object.keys(recordingsData).forEach((tc) => { + const item = document.createElement("div"); + item.className = "list-item"; + item.textContent = tc.replace(/_/g, " "); + item.onclick = () => selectTestCase(tc, item); + container.appendChild(item); + }); +} + +async function selectTestCase(tc, element) { + document.querySelectorAll("#test-cases-list .list-item").forEach(item => item.classList.remove("active")); + element.classList.add("active"); + + activeTestCase = tc; + let variants = []; + + if (tc === "GLOBAL") { + document.getElementById("filter-codebases-container").style.display = "flex"; + Object.keys(recordingsData).forEach(suite => { + recordingsData[suite].forEach(v => variants.push(`${suite}/${v}`)); + }); + } else { + document.getElementById("filter-codebases-container").style.display = "none"; + variants = recordingsData[tc]; + } + + // Populate Variant Dropdowns + const selectA = document.getElementById("select-variant-a"); + const selectB = document.getElementById("select-variant-b"); + + selectA.innerHTML = ""; + selectB.innerHTML = ""; + + variants.forEach((v) => { + const optA = document.createElement("option"); + optA.value = v; + optA.textContent = formatVariantName(v); + + const optB = document.createElement("option"); + optB.value = v; + optB.textContent = formatVariantName(v); + + selectA.appendChild(optA); + selectB.appendChild(optB); + }); + + // Select second option for B by default if available + if (variants.length > 1) { + selectB.selectedIndex = 1; + } + + document.getElementById("variant-selection-section").style.display = "block"; + await loadHeatmap(tc); +} + +// Close dropdowns if clicked outside +window.addEventListener("click", (e) => { + document.querySelectorAll(".filter-dropdown-container").forEach(container => { + if (!container.contains(e.target)) { + container.classList.remove("open"); + } + }); +}); + +function toggleDropdown(container, event) { + event.stopPropagation(); + const wasOpen = container.classList.contains("open"); + + // Close other dropdowns + document.querySelectorAll(".filter-dropdown-container").forEach(c => c.classList.remove("open")); + + if (!wasOpen) { + container.classList.add("open"); + const searchInput = container.querySelector(".filter-search-box"); + if (searchInput) { + searchInput.value = ""; + // Reset visibility of option items + container.querySelectorAll(".filter-opt-item").forEach(item => item.style.display = "flex"); + searchInput.focus(); + } + } +} + +function filterDropdownOptions(input) { + const query = input.value.toLowerCase(); + const container = input.closest(".filter-dropdown-container"); + container.querySelectorAll(".filter-opt-item").forEach(item => { + const val = item.dataset.value.toLowerCase(); + if (val.includes(query)) { + item.style.display = "flex"; + } else { + item.style.display = "none"; + } + }); +} + +async function loadHeatmap(tc) { + document.getElementById("dashboard-empty-state").style.display = "none"; + document.getElementById("heatmap-card").style.display = "flex"; + document.getElementById("routes-card").style.display = "none"; + document.getElementById("comparison-details").style.display = "none"; + + // Trigger loading spinner overlay in heatmap container + document.getElementById("heatmap-spinner").style.display = "flex"; + document.getElementById("heatmap-grid-container").style.opacity = "0.3"; + + try { + const mode = document.getElementById("select-scoring-mode").value; + const ignoreVal = activeIgnoreList.join(","); + const res = await fetch(`/api/matrix?testCase=${tc}&scoringMode=${encodeURIComponent(mode)}&ignoreHeaders=${encodeURIComponent(ignoreVal)}`); + lastMatrixData = await res.json(); + + // Reset search field + document.getElementById("variant-search-input").value = ""; + + // Build Dynamic Dropdown Filters + const filtersBar = document.getElementById("heatmap-dynamic-filters"); + filtersBar.innerHTML = ""; + + if (!lastMatrixData.variantsMetadata) { + applyMetadataFilters(); + return; + } + + // Gather unique values for each metadata property + const properties = {}; + Object.values(lastMatrixData.variantsMetadata).forEach(meta => { + Object.entries(meta).forEach(([key, val]) => { + if (key === "id") return; // Skip ID + + properties[key] = properties[key] || new Set(); + if (key === "localBuild") { + properties[key].add(val ? "Local" : "Source"); + } else { + properties[key].add(val === undefined ? "default" : String(val)); + } + }); + }); + + // Render a dropdown for each property + Object.entries(properties).forEach(([propName, valuesSet]) => { + const uniqueValues = Array.from(valuesSet).sort(); + + // Create Dropdown Container + const container = document.createElement("div"); + container.className = "filter-dropdown-container"; + + const btn = document.createElement("button"); + btn.className = "filter-dropdown-btn"; + + // Title casing property name (override localBuild to Deployment) + const displayProp = propName === "localBuild" ? "Deployment" : propName.replace(/([A-Z])/g, ' $1').replace(/^./, str => str.toUpperCase()); + btn.textContent = `${displayProp}: All`; + btn.onclick = (e) => toggleDropdown(container, e); + + const content = document.createElement("div"); + content.className = "filter-dropdown-content"; + + const searchInput = document.createElement("input"); + searchInput.type = "text"; + searchInput.className = "filter-search-box"; + searchInput.placeholder = `Search ${displayProp.toLowerCase()}...`; + searchInput.oninput = () => filterDropdownOptions(searchInput); + + const list = document.createElement("div"); + list.className = "filter-options-list"; + + // Select All checkbox + const allItem = document.createElement("label"); + allItem.className = "filter-opt-item"; + allItem.dataset.value = "all"; + + const allCheck = document.createElement("input"); + allCheck.type = "checkbox"; + allCheck.checked = true; + allCheck.onchange = () => { + const checks = list.querySelectorAll("input[type='checkbox']"); + checks.forEach(c => { + if (c !== allCheck) c.checked = allCheck.checked; + }); + applyMetadataFilters(); + }; + + allItem.appendChild(allCheck); + allItem.appendChild(document.createTextNode("Select All")); + list.appendChild(allItem); + + uniqueValues.forEach(val => { + const item = document.createElement("label"); + item.className = "filter-opt-item"; + item.dataset.value = val; + + const check = document.createElement("input"); + check.type = "checkbox"; + check.checked = true; + check.dataset.property = propName; + check.dataset.value = val; + check.onchange = () => { + // If any check is unchecked, uncheck "Select All" + const otherChecks = Array.from(list.querySelectorAll("input[type='checkbox']")).filter(c => c !== allCheck); + const allChecked = otherChecks.every(c => c.checked); + allCheck.checked = allChecked; + applyMetadataFilters(); + }; + + item.appendChild(check); + item.appendChild(document.createTextNode(val)); + list.appendChild(item); + }); + + content.appendChild(searchInput); + content.appendChild(list); + container.appendChild(btn); + container.appendChild(content); + filtersBar.appendChild(container); + }); + + applyMetadataFilters(); + } catch (err) { + console.error("Error loading heatmap data:", err); + } finally { + // Hide loading spinner overlay + document.getElementById("heatmap-spinner").style.display = "none"; + document.getElementById("heatmap-grid-container").style.opacity = "1.0"; + } +} + +function applyMetadataFilters() { + if (!lastMatrixData) return; + + const searchQuery = document.getElementById("variant-search-input").value.toLowerCase(); + const dropdownContainers = document.querySelectorAll(".filter-dropdown-container"); + + // Build active filters map + const activeFilters = {}; + dropdownContainers.forEach(container => { + const checks = container.querySelectorAll("input[type='checkbox']"); + const labelBtn = container.querySelector(".filter-dropdown-btn"); + let propName = ""; + const selectedValues = []; + let totalCount = 0; + + checks.forEach(c => { + if (c.dataset.property) { + propName = c.dataset.property; + totalCount++; + if (c.checked) { + selectedValues.push(c.dataset.value); + } + } + }); + + if (propName) { + activeFilters[propName] = selectedValues; + const displayProp = propName === "localBuild" ? "Deployment" : propName.replace(/([A-Z])/g, ' $1').replace(/^./, str => str.toUpperCase()); + if (selectedValues.length === totalCount) { + labelBtn.textContent = `${displayProp}: All`; + } else if (selectedValues.length === 0) { + labelBtn.textContent = `${displayProp}: None`; + } else if (selectedValues.length === 1) { + labelBtn.textContent = `${displayProp}: ${selectedValues[0]}`; + } else { + labelBtn.textContent = `${displayProp}: ${selectedValues.length} selected`; + } + } + }); + + // Filter variants list + const filteredVariants = lastMatrixData.variants.filter(v => { + // 1. Text search filter + if (searchQuery && !v.toLowerCase().includes(searchQuery)) { + return false; + } + + // 2. Dropdown metadata filters + const meta = lastMatrixData.variantsMetadata ? lastMatrixData.variantsMetadata[v] : null; + if (meta) { + for (const [prop, allowedVals] of Object.entries(activeFilters)) { + let actualVal = ""; + if (prop === "localBuild") { + actualVal = meta[prop] ? "Local" : "Source"; + } else { + actualVal = meta[prop] === undefined ? "default" : String(meta[prop]); + } + if (!allowedVals.includes(actualVal)) { + return false; + } + } + } + + return true; + }); + + renderMatrixTable(filteredVariants); +} + +function applyMatrixFilter() { + applyMetadataFilters(); +} + +function renderMatrixTable(variants) { + const container = document.getElementById("heatmap-grid-container"); + container.innerHTML = ""; + + if (variants.length === 0) { + container.innerHTML = `
No matching variants found for active filters.
`; + return; + } + + // Calculate min and max similarity for relative coloring + let minSim = 1.0; + let maxSim = 0.0; + let hasDiff = false; + + if (activeColorMode === "relative") { + variants.forEach((vA) => { + variants.forEach((vB) => { + const row = lastMatrixData.matrix[vA]; + const similarity = row ? (row[vB] || 0.0) : 0.0; + if (similarity < minSim) minSim = similarity; + if (similarity > maxSim) maxSim = similarity; + }); + }); + if (maxSim > minSim) { + hasDiff = true; + } + } + + const table = document.createElement("table"); + table.className = "heatmap-table"; + + // 1. Header Row + const thead = document.createElement("tr"); + thead.appendChild(document.createElement("th")); // empty top-left corner + variants.forEach((v) => { + const th = document.createElement("th"); + th.className = "heatmap-header-cell"; + th.textContent = formatVariantName(v); + th.dataset.columnVariant = v; + thead.appendChild(th); + }); + table.appendChild(thead); + + // 2. Rows + variants.forEach((vA) => { + const tr = document.createElement("tr"); + + // Row label + const tdLabel = document.createElement("td"); + tdLabel.className = "heatmap-row-label"; + tdLabel.textContent = formatVariantName(vA); + tdLabel.dataset.rowVariant = vA; + tr.appendChild(tdLabel); + + variants.forEach((vB) => { + const tdCell = document.createElement("td"); + tdCell.className = "heatmap-cell-wrapper"; + + const inner = document.createElement("div"); + inner.className = "heatmap-cell-inner"; + + const row = lastMatrixData.matrix[vA]; + const similarity = row ? (row[vB] || 0.0) : 0.0; + const percent = Math.round(similarity * 100); + inner.textContent = percent + "%"; + inner.dataset.codebaseA = vA.includes("/") ? vA.split("/")[0] : ""; + inner.dataset.codebaseB = vB.includes("/") ? vB.split("/")[0] : ""; + + // Calculate hue based on absolute or relative mode + let hue = 120; + if (activeColorMode === "relative") { + if (hasDiff) { + const rel = (similarity - minSim) / (maxSim - minSim); + hue = rel * 120; + } else { + hue = 120; // If all cells have identical similarity, color them green + } + } else { + hue = similarity * 120; // Absolute mode (0% = HSL 0, 100% = HSL 120) + } + + const bg = `hsla(${hue}, 70%, 42%, 0.85)`; + inner.style.backgroundColor = bg; + inner.title = `Similarity between ${vA} and ${vB}: ${percent}%`; + + // Highlight matching row/column headers on hover + inner.onmouseenter = () => { + const rowLabel = document.querySelector(`td.heatmap-row-label[data-row-variant="${vA}"]`); + if (rowLabel) rowLabel.classList.add("highlighted"); + const colLabel = document.querySelector(`th.heatmap-header-cell[data-column-variant="${vB}"]`); + if (colLabel) colLabel.classList.add("highlighted"); + }; + + inner.onmouseleave = () => { + const rowLabel = document.querySelector(`td.heatmap-row-label[data-row-variant="${vA}"]`); + if (rowLabel) rowLabel.classList.remove("highlighted"); + const colLabel = document.querySelector(`th.heatmap-header-cell[data-column-variant="${vB}"]`); + if (colLabel) colLabel.classList.remove("highlighted"); + }; + + // Clicking a cell triggers the comparison for vA and vB + inner.onclick = () => { + document.getElementById("select-variant-a").value = vA; + document.getElementById("select-variant-b").value = vB; + triggerComparison(); + }; + + tdCell.appendChild(inner); + tr.appendChild(tdCell); + }); + + table.appendChild(tr); + }); + + container.appendChild(table); + + // Apply codebase isolation filter if checked + const isolateCodebases = document.getElementById("toggle-filter-codebases").checked; + if (activeTestCase === "GLOBAL" && isolateCodebases) { + const cells = document.querySelectorAll(".heatmap-cell-inner"); + cells.forEach((cell) => { + const cA = cell.dataset.codebaseA; + const cB = cell.dataset.codebaseB; + if (cA && cB && cA !== cB) { + cell.classList.add("de-emphasized"); + } else { + cell.classList.remove("de-emphasized"); + } + }); + } +} + +function showHeatmapView() { + document.getElementById("heatmap-card").style.display = "flex"; + document.getElementById("routes-card").style.display = "none"; + document.getElementById("comparison-details").style.display = "none"; +} + +async function triggerComparison() { + const varA = document.getElementById("select-variant-a").value; + const varB = document.getElementById("select-variant-b").value; + + if (!varA || !varB) { + alert("Please select both Variant A and Variant B."); + return; + } + + // Hide heatmap card, show routes list card + document.getElementById("heatmap-card").style.display = "none"; + document.getElementById("routes-card").style.display = "block"; + + // Trigger compare API call + const res = await fetch(`/api/compare?testCase=${encodeURIComponent(activeTestCase)}&variantA=${encodeURIComponent(varA)}&variantB=${encodeURIComponent(varB)}`); + const data = await res.json(); + + comparisonResults = data.results || data.routes || []; + activeUrlA = data.urlA; + activeUrlB = data.urlB; + activeDeployTimeA = data.deployTimeA || 0; + activeDeployTimeB = data.deployTimeB || 0; + activeLocalBuildA = !!data.localBuildA; + activeLocalBuildB = !!data.localBuildB; + activeRuntimeA = data.runtimeA || ""; + activeRuntimeB = data.runtimeB || ""; + activePathA = data.pathA || ""; + activePathB = data.pathB || ""; + + // Build routes list in sidebar + const container = document.getElementById("routes-container"); + container.innerHTML = ""; + + comparisonResults.forEach((res, idx) => { + const item = document.createElement("div"); + item.className = "route-item"; + item.onclick = () => viewRouteDiff(idx, item); + + const routeSpan = document.createElement("span"); + routeSpan.className = "route-path"; + routeSpan.textContent = res.route; + + const badgeDiv = document.createElement("div"); + badgeDiv.className = "badges"; + + const statusBadge = document.createElement("span"); + if (res.statusMatch) { + statusBadge.className = "badge success"; + statusBadge.textContent = `Status: ${res.statusA}`; + } else { + statusBadge.className = "badge danger"; + statusBadge.textContent = `Status: ${res.statusA} vs ${res.statusB}`; + } + badgeDiv.appendChild(statusBadge); + + const bodyBadge = document.createElement("span"); + const percent = Math.round(res.bodySimilarity * 100); + if (res.bodySimilarity === 1.0) { + bodyBadge.className = "badge success"; + bodyBadge.textContent = `Body: 100%`; + } else if (res.bodySimilarity >= 0.9) { + bodyBadge.className = "badge warning"; + bodyBadge.textContent = `Body: ${percent}%`; + } else { + bodyBadge.className = "badge danger"; + bodyBadge.textContent = `Body: ${percent}%`; + } + badgeDiv.appendChild(bodyBadge); + + item.appendChild(routeSpan); + item.appendChild(badgeDiv); + container.appendChild(item); + }); + + // Auto-click first route to load it + if (comparisonResults.length > 0) { + container.firstElementChild.click(); + } +} + +function viewRouteDiff(idx, element) { + document.querySelectorAll("#routes-container .route-item").forEach(item => item.classList.remove("active")); + if (element) { + element.classList.add("active"); + } + + document.getElementById("comparison-details").style.display = "flex"; + + const res = comparisonResults[idx]; + + // Retrieve actual variant names + const varAName = document.getElementById("select-variant-a").value || "Variant A"; + const varBName = document.getElementById("select-variant-b").value || "Variant B"; + + const displayAName = formatVariantName(varAName); + const displayBName = formatVariantName(varBName); + + document.getElementById("th-header-a").textContent = displayAName; + document.getElementById("th-header-b").textContent = displayBName; + + document.getElementById("title-visual-a").textContent = `${displayAName} Render`; + document.getElementById("title-visual-b").textContent = `${displayBName} Render`; + + // Update Endpoint Links & Route Path Title + document.getElementById("route-title-path").textContent = res.route; + document.getElementById("overview-route-path").textContent = res.route; + + const linkA = document.getElementById("link-endpoint-a"); + linkA.href = activeUrlA + res.route; + linkA.textContent = displayAName; + + const linkB = document.getElementById("link-endpoint-b"); + linkB.href = activeUrlB + res.route; + linkB.textContent = displayBName; + + const iframeLinkA = document.getElementById("iframe-link-a"); + iframeLinkA.href = `/api/render?testCase=${encodeURIComponent(activeTestCase)}&variant=${encodeURIComponent(document.getElementById("select-variant-a").value)}&route=${encodeURIComponent(res.route)}`; + + const iframeLinkB = document.getElementById("iframe-link-b"); + iframeLinkB.href = `/api/render?testCase=${encodeURIComponent(activeTestCase)}&variant=${encodeURIComponent(document.getElementById("select-variant-b").value)}&route=${encodeURIComponent(res.route)}`; + + // Load variant render frames for visual split tab + document.getElementById("iframe-a").src = iframeLinkA.href; + document.getElementById("iframe-b").src = iframeLinkB.href; + + // 1. Status Code Parity Card + const statusBox = document.getElementById("status-comparison-box"); + if (res.statusMatch) { + statusBox.innerHTML = `${res.statusA} Match`; + } else { + statusBox.innerHTML = `${res.statusA} vs ${res.statusB}`; + } + + // 2. Request Duration Card + const latValA = res.latencyA ? `${res.latencyA}ms` : "--"; + const latValB = res.latencyB ? `${res.latencyB}ms` : "--"; + let colorA = "var(--text-muted)", colorB = "var(--text-muted)"; + let bgA = "rgba(255,255,255,0.03)", bgB = "rgba(255,255,255,0.03)"; + if (res.latencyA && res.latencyB) { + if (res.latencyA < res.latencyB) { + colorA = "var(--success)"; bgA = "rgba(16, 185, 129, 0.15)"; + colorB = "var(--danger)"; bgB = "rgba(239, 68, 68, 0.15)"; + } else if (res.latencyB < res.latencyA) { + colorB = "var(--success)"; bgB = "rgba(16, 185, 129, 0.15)"; + colorA = "var(--danger)"; bgA = "rgba(239, 68, 68, 0.15)"; + } else { + colorA = "var(--success)"; bgA = "rgba(16, 185, 129, 0.15)"; + colorB = "var(--success)"; bgB = "rgba(16, 185, 129, 0.15)"; + } + } else if (res.latencyA) { + colorA = "var(--success)"; bgA = "rgba(16, 185, 129, 0.15)"; + } else if (res.latencyB) { + colorB = "var(--success)"; bgB = "rgba(16, 185, 129, 0.15)"; + } + document.getElementById("request-duration-box").innerHTML = ` +
+
+ ${displayAName} + ${latValA} +
+
vs
+
+ ${displayBName} + ${latValB} +
+
+ `; + + // 3. Deployment Duration Card + const depValA = activeDeployTimeA ? `${Math.round(activeDeployTimeA / 1000)}s` : "--"; + const depValB = activeDeployTimeB ? `${Math.round(activeDeployTimeB / 1000)}s` : "--"; + let depColorA = "var(--text-muted)", depColorB = "var(--text-muted)"; + let depBgA = "rgba(255,255,255,0.03)", depBgB = "rgba(255,255,255,0.03)"; + if (activeDeployTimeA && activeDeployTimeB) { + if (activeDeployTimeA < activeDeployTimeB) { + depColorA = "var(--success)"; depBgA = "rgba(16, 185, 129, 0.15)"; + depColorB = "var(--danger)"; depBgB = "rgba(239, 68, 68, 0.15)"; + } else if (activeDeployTimeB < activeDeployTimeA) { + depColorB = "var(--success)"; depBgB = "rgba(16, 185, 129, 0.15)"; + depColorA = "var(--danger)"; depBgA = "rgba(239, 68, 68, 0.15)"; + } else { + depColorA = "var(--success)"; depBgA = "rgba(16, 185, 129, 0.15)"; + depColorB = "var(--success)"; depBgB = "rgba(16, 185, 129, 0.15)"; + } + } else if (activeDeployTimeA) { + depColorA = "var(--success)"; depBgA = "rgba(16, 185, 129, 0.15)"; + } else if (activeDeployTimeB) { + depColorB = "var(--success)"; depBgB = "rgba(16, 185, 129, 0.15)"; + } + document.getElementById("deployment-duration-box").innerHTML = ` +
+
+ ${displayAName} + ${depValA} +
+
vs
+
+ ${displayBName} + ${depValB} +
+
+ `; + + // 4. Detailed Specifications Cards + const hasAbiuA = activeRuntimeA && /\d/.test(activeRuntimeA); + const hasAbiuB = activeRuntimeB && /\d/.test(activeRuntimeB); + + document.getElementById("overview-spec-title-a").textContent = displayAName; + document.getElementById("overview-spec-build-a").textContent = activeLocalBuildA ? "Local Build (Universal Maker)" : "Source Deploy (Cloud Build)"; + document.getElementById("overview-spec-runtime-a").textContent = activeRuntimeA ? activeRuntimeA : "auto-detected"; + + const abiuElA = document.getElementById("overview-spec-abiu-a"); + abiuElA.textContent = hasAbiuA ? "ENABLED" : "DISABLED"; + abiuElA.className = hasAbiuA ? "badge success" : "badge danger"; + + document.getElementById("overview-spec-path-a").textContent = activePathA ? activePathA : "./"; + + let descA = ""; + if (activeLocalBuildA) { + descA = `Tests local artifact generation using the Universal Maker (UM) compilation engine inside a Node container. ${hasAbiuA ? "Automatic Base Image Updates (ABIU) are enabled, allowing dynamic OS and runtime security patches to keep your container secure." : "Automatic Base Image Updates (ABIU) are disabled, pinning the build to the standard base image."}`; + } else { + descA = `Tests standard Firebase App Hosting source-level deployment compiled remotely using Google Cloud Build. ${hasAbiuA ? "Automatic Base Image Updates (ABIU) are enabled, allowing dynamic OS and runtime security patches to keep your container secure." : "Automatic Base Image Updates (ABIU) are disabled, pinning the build to the standard base image."}`; + } + document.getElementById("overview-spec-desc-a").textContent = descA; + + document.getElementById("overview-spec-title-b").textContent = displayBName; + document.getElementById("overview-spec-build-b").textContent = activeLocalBuildB ? "Local Build (Universal Maker)" : "Source Deploy (Cloud Build)"; + document.getElementById("overview-spec-runtime-b").textContent = activeRuntimeB ? activeRuntimeB : "auto-detected"; + + const abiuElB = document.getElementById("overview-spec-abiu-b"); + abiuElB.textContent = hasAbiuB ? "ENABLED" : "DISABLED"; + abiuElB.className = hasAbiuB ? "badge success" : "badge danger"; + + document.getElementById("overview-spec-path-b").textContent = activePathB ? activePathB : "./"; + + let descB = ""; + if (activeLocalBuildB) { + descB = `Tests local artifact generation using the Universal Maker (UM) compilation engine inside a Node container. ${hasAbiuB ? "Automatic Base Image Updates (ABIU) are enabled, allowing dynamic OS and runtime security patches to keep your container secure." : "Automatic Base Image Updates (ABIU) are disabled, pinning the build to the standard base image."}`; + } else { + descB = `Tests standard Firebase App Hosting source-level deployment compiled remotely using Google Cloud Build. ${hasAbiuB ? "Automatic Base Image Updates (ABIU) are enabled, allowing dynamic OS and runtime security patches to keep your container secure." : "Automatic Base Image Updates (ABIU) are disabled, pinning the build to the standard base image."}`; + } + document.getElementById("overview-spec-desc-b").textContent = descB; + + // 2. HTTP Headers comparison (merged list with dynamic UI filtering) + const headersTbody = document.getElementById("headers-comparison-tbody"); + + const renderHeaders = () => { + headersTbody.innerHTML = ""; + + const ignoreList = activeIgnoreList; + + const mergedDiffs = []; + res.headerMismatches.forEach(h => { + const isIgnored = ignoreList.includes(h.header.toLowerCase()); + mergedDiffs.push({ ...h, critical: !isIgnored }); + }); + if (res.expectedHeaderVariations) { + res.expectedHeaderVariations.forEach(h => { + const isIgnored = ignoreList.includes(h.header.toLowerCase()); + mergedDiffs.push({ ...h, critical: !isIgnored }); + }); + } + + // Sort alphabetically, placing critical mismatches on top + mergedDiffs.sort((x, y) => { + if (x.critical !== y.critical) { + return x.critical ? -1 : 1; + } + return x.header.localeCompare(y.header); + }); + + if (mergedDiffs.length === 0) { + headersTbody.innerHTML = 'All response headers are identical'; + } else { + mergedDiffs.forEach(h => { + const badgeHtml = h.critical + ? 'Critical Mismatch' + : 'Ignored Variation'; + + const tr = document.createElement("tr"); + if (!h.critical) { + tr.style.opacity = "0.5"; + } + + const td1 = document.createElement("td"); + td1.style.fontFamily = "monospace"; + td1.style.fontWeight = "500"; + td1.textContent = h.header + " "; + + // Hover-revealed inline action button to toggle ignore state + const actionBtn = document.createElement("span"); + actionBtn.className = "header-row-action"; + actionBtn.textContent = h.critical ? "Ignore" : "Unignore"; + actionBtn.onclick = (e) => { + e.stopPropagation(); + toggleHeaderIgnoreState(h.header); + }; + td1.appendChild(actionBtn); + + const td2 = document.createElement("td"); + td2.style.color = h.critical ? 'var(--danger)' : 'var(--text-muted)'; + td2.style.fontFamily = "monospace"; + td2.style.fontSize = "11px"; + td2.style.wordBreak = "break-all"; + td2.textContent = h.valA || '(missing)'; + + const td3 = document.createElement("td"); + td3.style.color = h.critical ? 'var(--success)' : 'var(--text-muted)'; + td3.style.fontFamily = "monospace"; + td3.style.fontSize = "11px"; + td3.style.wordBreak = "break-all"; + td3.textContent = h.valB || '(missing)'; + + const td4 = document.createElement("td"); + td4.innerHTML = badgeHtml; + + tr.appendChild(td1); + tr.appendChild(td2); + tr.appendChild(td3); + tr.appendChild(td4); + headersTbody.appendChild(tr); + }); + } + }; + window.activeResObject = res; + window.renderActiveHeaders = renderHeaders; + + renderHeaders(); + + // 4. Code Body Diff + const diffContainer = document.getElementById("body-diff-container"); + diffContainer.innerHTML = ""; + + if (res.isBinary) { + const div = document.createElement("div"); + div.className = "empty-state"; + div.textContent = "Binary File Comparison: " + (res.bodyDiff || "Identical"); + diffContainer.appendChild(div); + } else if (res.bodySimilarity === 1.0) { + const div = document.createElement("div"); + div.className = "empty-state"; + div.innerHTML = ` + + + +
Response Bodies are 100% Identical
+ `; + diffContainer.appendChild(div); + } else if (!res.diffChanges || res.diffChanges.length === 0) { + const div = document.createElement("div"); + div.className = "empty-state"; + div.textContent = "No text differences recorded"; + diffContainer.appendChild(div); + } else { + // Render GitHub style beautiful side-scrolled diff + const diffView = document.createElement("div"); + diffView.className = "diff-view"; + + const diffElements = []; + + res.diffChanges.forEach((change) => { + const lines = change.value.split("\n"); + if (lines.length > 1 && lines[lines.length - 1] === "") { + lines.pop(); + } + + if (!change.added && !change.removed) { + if (lines.length > 10) { + const topLines = lines.slice(0, 3); + const bottomLines = lines.slice(-3); + const hiddenCount = lines.length - 6; + + const renderLines = (arr) => { + arr.forEach(line => { + const row = document.createElement("div"); row.className = "diff-line"; + const prefix = document.createElement("span"); prefix.className = "diff-prefix"; prefix.textContent = " "; + const text = document.createElement("span"); text.className = "diff-text"; text.textContent = line; + row.appendChild(prefix); row.appendChild(text); + diffView.appendChild(row); + }); + }; + + renderLines(topLines); + + const expandBtn = document.createElement("div"); + expandBtn.style.cursor = "pointer"; + expandBtn.style.backgroundColor = "rgba(255,255,255,0.05)"; + expandBtn.style.padding = "6px 8px"; + expandBtn.style.textAlign = "center"; + expandBtn.style.color = "var(--accent)"; + expandBtn.style.fontSize = "11px"; + expandBtn.style.margin = "6px 0"; + expandBtn.style.borderRadius = "4px"; + expandBtn.style.border = "1px dashed var(--border)"; + expandBtn.textContent = "Expand " + hiddenCount + " unchanged lines..."; + + const hiddenContainer = document.createElement("div"); + hiddenContainer.style.display = "none"; + const middleLines = lines.slice(3, -3); + middleLines.forEach(line => { + const row = document.createElement("div"); row.className = "diff-line"; + const prefix = document.createElement("span"); prefix.className = "diff-prefix"; prefix.textContent = " "; + const text = document.createElement("span"); text.className = "diff-text"; text.textContent = line; + row.appendChild(prefix); row.appendChild(text); + hiddenContainer.appendChild(row); + }); + expandBtn.onclick = () => { + expandBtn.style.display = "none"; + hiddenContainer.style.display = "block"; + }; + diffView.appendChild(expandBtn); + diffView.appendChild(hiddenContainer); + + renderLines(bottomLines); + } else { + lines.forEach(line => { + const row = document.createElement("div"); row.className = "diff-line"; + const prefix = document.createElement("span"); prefix.className = "diff-prefix"; prefix.textContent = " "; + const text = document.createElement("span"); text.className = "diff-text"; text.textContent = line; + row.appendChild(prefix); row.appendChild(text); + diffView.appendChild(row); + }); + } + } else { + const chunkContainer = document.createElement("div"); + chunkContainer.style.borderLeft = "2px solid " + (change.added ? "var(--success)" : "var(--danger)"); + chunkContainer.style.margin = "4px 0"; + diffElements.push(chunkContainer); + + lines.forEach(line => { + const row = document.createElement("div"); row.className = "diff-line"; + if (change.added) row.classList.add("added"); + if (change.removed) row.classList.add("removed"); + + const prefix = document.createElement("span"); prefix.className = "diff-prefix"; + prefix.textContent = change.added ? "+" : "-"; + + const text = document.createElement("span"); text.className = "diff-text"; + text.textContent = line; + + row.appendChild(prefix); row.appendChild(text); + chunkContainer.appendChild(row); + }); + diffView.appendChild(chunkContainer); + } + }); + + diffContainer.appendChild(diffView); + + let currentDiff = 0; + const counterEl = document.getElementById("diff-counter"); + const updateNav = () => { + if (!counterEl) return; + counterEl.textContent = diffElements.length > 0 ? (currentDiff + 1) + "/" + diffElements.length : "0/0"; + if (diffElements.length > 0) { + diffElements.forEach(el => el.style.backgroundColor = "transparent"); + diffElements[currentDiff].style.backgroundColor = "rgba(59, 130, 246, 0.1)"; + diffElements[currentDiff].scrollIntoView({ behavior: "smooth", block: "center" }); + } + }; + + const prevBtn = document.getElementById("prev-diff"); + if (prevBtn) { + prevBtn.onclick = () => { + if (diffElements.length === 0) return; + currentDiff = (currentDiff - 1 + diffElements.length) % diffElements.length; + updateNav(); + }; + } + const nextBtn = document.getElementById("next-diff"); + if (nextBtn) { + nextBtn.onclick = () => { + if (diffElements.length === 0) return; + currentDiff = (currentDiff + 1) % diffElements.length; + updateNav(); + }; + } + updateNav(); + } + switchRightTab("overview"); +} + +function switchRightTab(tabId) { + // 1. Reset all tabs styling + const tabs = ["overview", "headers", "code", "visual"]; + tabs.forEach(id => { + let elId = "tab-overview"; + if (id === "headers") elId = "tab-headers"; + else if (id === "code") elId = "tab-code-diff"; + else if (id === "visual") elId = "tab-visual"; + const el = document.getElementById(elId); + if (el) { + el.classList.remove("active"); + el.style.borderBottom = "2px solid transparent"; + el.style.color = "var(--text-muted)"; + } + }); + + // 2. Hide all tab content containers + document.getElementById("overview-tab-container").style.display = "none"; + document.getElementById("headers-tab-container").style.display = "none"; + document.getElementById("body-diff-container").style.display = "none"; + document.getElementById("visual-render-container").style.display = "none"; + + // 3. Show and activate selected tab + if (tabId === "overview") { + const el = document.getElementById("tab-overview"); + if (el) { + el.classList.add("active"); + el.style.borderBottom = "2px solid var(--accent)"; + el.style.color = "var(--text)"; + } + document.getElementById("overview-tab-container").style.display = "flex"; + } else if (tabId === "headers") { + const el = document.getElementById("tab-headers"); + if (el) { + el.classList.add("active"); + el.style.borderBottom = "2px solid var(--accent)"; + el.style.color = "var(--text)"; + } + document.getElementById("headers-tab-container").style.display = "flex"; + } else if (tabId === "code") { + const el = document.getElementById("tab-code-diff"); + if (el) { + el.classList.add("active"); + el.style.borderBottom = "2px solid var(--accent)"; + el.style.color = "var(--text)"; + } + document.getElementById("body-diff-container").style.display = "block"; + } else if (tabId === "visual") { + const el = document.getElementById("tab-visual"); + if (el) { + el.classList.add("active"); + el.style.borderBottom = "2px solid var(--accent)"; + el.style.color = "var(--text)"; + } + document.getElementById("visual-render-container").style.display = "flex"; + } +} + +// Global Event Listeners for Controls Bar +document.getElementById("select-scoring-mode").onchange = () => { + activeScoringMode = document.getElementById("select-scoring-mode").value; + saveSettings(); + if (activeTestCase) { + loadHeatmap(activeTestCase); + } +}; + +window.onload = async () => { + // 1. Set scoring mode and coloring mode dropdown values from localStorage + document.getElementById("select-scoring-mode").value = activeScoringMode; + document.getElementById("select-color-mode").value = activeColorMode; + + // 2. Render ignore pills and set collapse state + renderIgnorePills(); + applyIgnoreCollapseState(); + + // 3. Trigger initial recordings load + await loadRecordings(); +}; diff --git a/src/apphosting/compare/reporter.spec.ts b/src/apphosting/compare/reporter.spec.ts new file mode 100644 index 00000000000..225030c7f60 --- /dev/null +++ b/src/apphosting/compare/reporter.spec.ts @@ -0,0 +1,69 @@ +import { expect } from "chai"; +import * as fs from "fs-extra"; +import * as path from "path"; +import { generateReport } from "./reporter"; +import { ComparisonResult } from "./compare"; + +describe("Report Generator", () => { + let tempDir: string; + + beforeEach(() => { + tempDir = path.join( + process.cwd(), + "scratch-test-report-" + Math.random().toString(36).substring(7), + ); + fs.ensureDirSync(tempDir); + }); + + afterEach(() => { + fs.removeSync(tempDir); + }); + + it("should generate JSON and HTML reports", async () => { + const results: ComparisonResult[] = [ + { + route: "/", + statusMatch: true, + headerMismatches: [], + expectedHeaderVariations: [], + bodySimilarity: 1.0, + bodyDiff: "", + isBinary: false, + }, + { + route: "/about", + statusMatch: true, + headerMismatches: [{ header: "Cache-Control", valA: "max-age=0", valB: "no-cache" }], + expectedHeaderVariations: [], + bodySimilarity: 0.95, + bodyDiff: "HTML content mismatch", + isBinary: false, + }, + ]; + + await generateReport( + "aryanf-test", + "us-central1", + "compare-slot-1-a", + "compare-slot-1-b", + results, + tempDir, + ); + + const jsonPath = path.join(tempDir, "summary.json"); + const htmlPath = path.join(tempDir, "index.html"); + + expect(fs.existsSync(jsonPath)).to.be.true; + expect(fs.existsSync(htmlPath)).to.be.true; + + const data = fs.readJsonSync(jsonPath); + expect(data.summary.totalRoutes).to.equal(2); + expect(data.summary.matchingRoutes).to.equal(1); + expect(data.summary.mismatchingRoutes).to.equal(1); + expect(data.summary.overallSimilarity).to.be.closeTo(0.975, 0.001); + + const htmlContent = fs.readFileSync(htmlPath, "utf-8"); + expect(htmlContent).to.include("App Hosting Comparison Dashboard"); + expect(htmlContent).to.include("/about"); + }); +}); diff --git a/src/apphosting/compare/reporter.ts b/src/apphosting/compare/reporter.ts new file mode 100644 index 00000000000..b4dcdf66639 --- /dev/null +++ b/src/apphosting/compare/reporter.ts @@ -0,0 +1,361 @@ +import * as fs from "fs-extra"; +import * as path from "path"; +import * as clc from "colorette"; +import { ComparisonResult } from "./compare"; +import { logger } from "./logger"; + +function escapeHtml(str: string): string { + return str + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); +} + +export interface ComparisonSummary { + projectId: string; + location: string; + backendA: string; + backendB: string; + timestamp: string; + totalRoutes: number; + matchingRoutes: number; + mismatchingRoutes: number; + overallSimilarity: number; +} + +/** + * + */ +export async function generateReport( + projectId: string, + location: string, + backendA: string, + backendB: string, + results: ComparisonResult[], + outputDir = "./compare-report", +): Promise { + const totalRoutes = results.length; + const matching = results.filter( + (r) => r.statusMatch && r.headerMismatches.length === 0 && r.bodySimilarity >= 0.99, + ); + const mismatches = results.filter( + (r) => !r.statusMatch || r.headerMismatches.length > 0 || r.bodySimilarity < 0.99, + ); + + const totalSimilarity = results.reduce((sum, r) => sum + r.bodySimilarity, 0); + const overallSimilarity = totalRoutes > 0 ? totalSimilarity / totalRoutes : 1.0; + + const summary: ComparisonSummary = { + projectId, + location, + backendA, + backendB, + timestamp: new Date().toISOString(), + totalRoutes, + matchingRoutes: matching.length, + mismatchingRoutes: mismatches.length, + overallSimilarity, + }; + + logger.info("\n=========================================="); + logger.info(" COMPARISON TEST SUMMARY"); + logger.info("=========================================="); + logger.info(`Project: ${projectId}`); + logger.info(`Location: ${location}`); + logger.info(`Backend A: ${backendA}`); + logger.info(`Backend B: ${backendB}`); + logger.info(`Total Routes: ${totalRoutes}`); + logger.info(`Passed: ${clc.green(String(matching.length))}`); + logger.info( + `Mismatched: ${mismatches.length > 0 ? clc.red(String(mismatches.length)) : clc.green("0")}`, + ); + logger.info(`Similarity: ${clc.cyan((overallSimilarity * 100).toFixed(2) + "%")}`); + logger.info("==========================================\n"); + + if (mismatches.length > 0) { + logger.warn(clc.bold(clc.red("Mismatched Routes:"))); + for (const m of mismatches) { + logger.warn(` - ${clc.bold(m.route)}`); + if (!m.statusMatch) { + logger.warn(` * Status Code Mismatch`); + } + if (m.headerMismatches.length > 0) { + logger.warn(` * ${m.headerMismatches.length} Behavioral Header Mismatch(es)`); + } + if (m.bodySimilarity < 0.99) { + logger.warn(` * Body Similarity: ${(m.bodySimilarity * 100).toFixed(2)}%`); + } + } + logger.info(""); + } + + await fs.ensureDir(outputDir); + + // Dump summary without the bodies to save space + const resultsWithoutBodies: ComparisonResult[] = results.map(r => { + const { bodyA, bodyB, ...rest } = r; + return rest; + }); + await fs.writeJson(path.join(outputDir, "summary.json"), { summary, results: resultsWithoutBodies }, { spaces: 2 }); + logger.info(`JSON report saved to: ${path.join(outputDir, "summary.json")}`); + + // Dump raw bodies for manual diffing + const routesDirA = path.join(outputDir, "backendA"); + const routesDirB = path.join(outputDir, "backendB"); + for (const r of results) { + if (r.bodyA !== undefined && r.bodyB !== undefined) { + // Map route /foo/bar to /foo/bar.html or /foo/bar/index.html + const safeRoute = r.route === "/" ? "index.html" : r.route.replace(/^\//, "") + ".html"; + await fs.outputFile(path.join(routesDirA, safeRoute), r.bodyA, "utf-8"); + await fs.outputFile(path.join(routesDirB, safeRoute), r.bodyB, "utf-8"); + } + } + logger.info(`Raw responses saved to ${routesDirA} and ${routesDirB} for manual diffing.`); + + const html = getHtmlTemplate(summary, resultsWithoutBodies); + await fs.outputFile(path.join(outputDir, "index.html"), html, "utf-8"); + logger.info( + `HTML Dashboard generated at: ${clc.underline(path.join(outputDir, "index.html"))}\n`, + ); +} + +function getHtmlTemplate(summary: ComparisonSummary, results: ComparisonResult[]): string { + const resultsRows = results + .map((r) => { + const isPass = r.statusMatch && r.headerMismatches.length === 0 && r.bodySimilarity >= 0.99; + const badgeClass = isPass ? "badge-success" : "badge-error"; + const badgeText = isPass ? "PASS" : "FAIL"; + + const headersList = r.headerMismatches + .map((m) => `
  • ${escapeHtml(m.header)}: "${escapeHtml(m.valA)}" vs "${escapeHtml(m.valB)}"
    • `) + .join(""); + + const variationsList = r.expectedHeaderVariations + .map((m) => `
  • ${escapeHtml(m.header)}: "${escapeHtml(m.valA)}" vs "${escapeHtml(m.valB)}"
    • `) + .join(""); + + return ` + + ${escapeHtml(r.route)} + ${badgeText} + ${r.statusMatch ? "Match" : "Mismatch"} + ${(r.bodySimilarity * 100).toFixed(1)}% + + ${r.headerMismatches.length > 0 ? `
    Mismatches:
      ${headersList}
    ` : "Match"} + ${r.expectedHeaderVariations.length > 0 ? `
    Variations:
      ${variationsList}
    ` : ""} + + + `; + }) + .join(""); + + return ` + + + + + + App Hosting Comparison Report + + + + + +
    +

    App Hosting Comparison Dashboard

    +
    + Ran on ${new Date(summary.timestamp).toLocaleString()} for project ${summary.projectId} (${summary.location}) +
    +
    + +
    +
    +
    Total Checked Routes
    +
    ${summary.totalRoutes}
    +
    +
    +
    Passed Routes
    +
    ${summary.matchingRoutes}
    +
    +
    +
    Mismatched Routes
    +
    ${summary.mismatchingRoutes}
    +
    +
    +
    Overall Similarity
    +
    ${(summary.overallSimilarity * 100).toFixed(2)}%
    +
    +
    + +
    + + + + + + + + + + + + ${resultsRows} + +
    RouteParityHTTP StatusBody ParityHeader Assertions
    +
    + + + + `; +} diff --git a/src/apphosting/compare/secrets.spec.ts b/src/apphosting/compare/secrets.spec.ts new file mode 100644 index 00000000000..8843eeed194 --- /dev/null +++ b/src/apphosting/compare/secrets.spec.ts @@ -0,0 +1,87 @@ +import { expect } from "chai"; +import * as sinon from "sinon"; +import * as fs from "fs-extra"; +import { AppHostingYamlConfig } from "../yaml"; +import * as apphosting from "../../gcp/apphosting"; +import * as csm from "../../gcp/secretManager"; +import * as secretsHelper from "../secrets"; +import * as projectNumberHelper from "../../getProjectNumber"; +import { setupSandboxSecrets, cleanupSandboxSecrets } from "./secrets"; + +describe("Sandbox Secrets Manager", () => { + let pathExistsStub: sinon.SinonStub; + let loadConfigStub: sinon.SinonStub; + let getProjectNumberStub: sinon.SinonStub; + let getBackendStub: sinon.SinonStub; + let serviceAccountsStub: sinon.SinonStub; + let secretExistsStub: sinon.SinonStub; + let createSecretStub: sinon.SinonStub; + let addVersionStub: sinon.SinonStub; + let deleteSecretStub: sinon.SinonStub; + let grantSecretAccessStub: sinon.SinonStub; + + beforeEach(() => { + pathExistsStub = sinon.stub(fs, "pathExists"); + loadConfigStub = sinon.stub(AppHostingYamlConfig, "loadFromFile"); + getProjectNumberStub = sinon.stub(projectNumberHelper, "getProjectNumber"); + getBackendStub = sinon.stub(apphosting, "getBackend"); + serviceAccountsStub = sinon.stub(secretsHelper, "serviceAccountsForBackend"); + secretExistsStub = sinon.stub(csm, "secretExists"); + createSecretStub = sinon.stub(csm, "createSecret"); + addVersionStub = sinon.stub(csm, "addVersion"); + deleteSecretStub = sinon.stub(csm, "deleteSecret"); + grantSecretAccessStub = sinon.stub(secretsHelper, "grantSecretAccess"); + }); + + afterEach(() => { + sinon.restore(); + }); + + it("should setup sandbox secrets for yaml configuration", async () => { + pathExistsStub.resolves(true); + + const mockYaml = new AppHostingYamlConfig(); + mockYaml.env = { + API_KEY: { secret: "my-production-api-key", availability: ["RUNTIME"] }, + }; + loadConfigStub.resolves(mockYaml); + getProjectNumberStub.resolves("12345"); + + getBackendStub.resolves({ name: "backend-resource" }); + serviceAccountsStub.resolves({ + buildServiceAccount: "build-sa@google.com", + runServiceAccount: "run-sa@google.com", + }); + + secretExistsStub.resolves(false); + createSecretStub.resolves(); + addVersionStub.resolves(); + grantSecretAccessStub.resolves(); + + const mappings = await setupSandboxSecrets("aryanf-test", "us-central1", "/app/path", 1, [ + "compare-slot-1-a", + "compare-slot-1-b", + ]); + + expect(mappings).to.have.lengthOf(1); + expect(mappings[0].originalName).to.equal("my-production-api-key"); + expect(mappings[0].mockSecretName).to.equal("cmp-sec-1-my-production-api-key"); + expect(mappings[0].mockValue).to.equal("mock-value-for-API_KEY-slot-1"); + + expect(createSecretStub.callCount).to.equal(1); + expect(addVersionStub.callCount).to.equal(1); + expect(grantSecretAccessStub.callCount).to.equal(1); + }); + + it("should delete secrets on cleanup", async () => { + deleteSecretStub.resolves(); + + const mappings = [ + { originalName: "my-key", mockSecretName: "cmp-sec-1-my-key", mockValue: "val" }, + ]; + + await cleanupSandboxSecrets("aryanf-test", mappings); + expect(deleteSecretStub.callCount).to.equal(1); + expect(deleteSecretStub.firstCall.args[1]).to.equal("cmp-sec-1-my-key"); + }); +}); diff --git a/src/apphosting/compare/secrets.ts b/src/apphosting/compare/secrets.ts new file mode 100644 index 00000000000..2e5923e7b2e --- /dev/null +++ b/src/apphosting/compare/secrets.ts @@ -0,0 +1,102 @@ +import * as path from "path"; +import * as fs from "fs-extra"; +import * as csm from "../../gcp/secretManager"; +import * as apphosting from "../../gcp/apphosting"; +import { AppHostingYamlConfig } from "../yaml"; +import { getProjectNumber } from "../../getProjectNumber"; +import { serviceAccountsForBackend, grantSecretAccess, toMulti } from "../secrets"; +import { logger } from "./logger"; + +export interface SecretMapping { + originalName: string; + mockSecretName: string; + mockValue: string; +} + +/** + * + */ +export async function setupSandboxSecrets( + projectId: string, + location: string, + appPath: string, + slotIndex: number, + backendIds: string[], +): Promise { + const yamlPath = path.join(appPath, "apphosting.yaml"); + if (!(await fs.pathExists(yamlPath))) { + return []; + } + + const config = await AppHostingYamlConfig.loadFromFile(yamlPath); + const secretEntries = Object.entries(config.env).filter(([, val]) => val.secret !== undefined); + if (secretEntries.length === 0) { + return []; + } + + const projectNumber = await getProjectNumber({ projectId }); + const mappings: SecretMapping[] = []; + + // Fetch all backends to extract their service accounts + const backends = await Promise.all( + backendIds.map((id) => apphosting.getBackend(projectId, location, id)), + ); + + const multiAccountsList = await Promise.all( + backends.map(async (b) => toMulti(await serviceAccountsForBackend(projectNumber, b))), + ); + + // Combine build/run service accounts from all backends + const combinedAccounts = { + buildServiceAccounts: Array.from( + new Set(multiAccountsList.flatMap((a) => a.buildServiceAccounts)), + ), + runServiceAccounts: Array.from(new Set(multiAccountsList.flatMap((a) => a.runServiceAccounts))), + }; + + for (const [envName, envVal] of secretEntries) { + const originalSecretName = envVal.secret!; + // Clean and build mock secret name + const cleanName = originalSecretName.replace(/[^a-zA-Z0-9_-]/g, "").toLowerCase(); + const mockSecretName = `cmp-sec-${slotIndex}-${cleanName}`.substring(0, 255); + const mockValue = `mock-value-for-${envName}-slot-${slotIndex}`; + + logger.info(`Setting up sandboxed secret for ${envName}: ${mockSecretName}...`); + + const exists = await csm.secretExists(projectId, mockSecretName); + if (!exists) { + await csm.createSecret(projectId, mockSecretName, { + "created-by": "apphosting-compare-tool", + slot: String(slotIndex), + }); + } + + await csm.addVersion(projectId, mockSecretName, mockValue); + await grantSecretAccess(projectId, projectNumber, mockSecretName, combinedAccounts); + + mappings.push({ + originalName: originalSecretName, + mockSecretName, + mockValue, + }); + } + + return mappings; +} + +/** + * + */ +export async function cleanupSandboxSecrets( + projectId: string, + mappings: SecretMapping[], +): Promise { + if (mappings.length === 0) return; + + logger.info("Cleaning up sandboxed secrets in Secret Manager..."); + await Promise.allSettled( + mappings.map((m) => + csm.deleteSecret(projectId, m.mockSecretName).catch((e) => logger.debug(e)), + ), + ); +} diff --git a/src/apphosting/compare/server.ts b/src/apphosting/compare/server.ts new file mode 100644 index 00000000000..edbcdceb953 --- /dev/null +++ b/src/apphosting/compare/server.ts @@ -0,0 +1,333 @@ +// [Subsystem: Diff Viewer UI & API Server] +import * as express from "express"; +import * as http from "http"; +import * as path from "path"; +import { logger } from "./logger"; +import * as cache from "./cache"; +import * as compare from "./compare"; +import * as diff from "diff"; +import { CompareResponse, MatrixResponse, DashboardComparisonResult, VariantMetadata } from "./types"; + +export function startServer(port: number): Promise { + return new Promise((resolve, reject) => { + const app = express(); + + app.use(express.json()); + + // API: List all recordings in compare-cache + app.get("/api/recordings", async (req, res) => { + try { + const recordings = await cache.listRecordings(); + res.json(recordings); + } catch (err: unknown) { + const errMsg = err instanceof Error ? err.message : String(err); + res.status(500).json({ error: errMsg }); + } + }); + + app.get("/api/compare", async (req, res) => { + const { testCase, variantA, variantB } = req.query; + if (typeof testCase !== "string" || typeof variantA !== "string" || typeof variantB !== "string") { + res.status(400).json({ error: "Missing or invalid query parameters: testCase, variantA, and variantB must be strings." }); + return; + } + + try { + let recA: cache.VariantRecording; + let recB: cache.VariantRecording; + if (testCase === "GLOBAL") { + if (!variantA.includes("/") || !variantB.includes("/")) { + throw new Error("Invalid variant query parameters for GLOBAL testCase"); + } + const [tcA, varA] = variantA.split("/"); + const [tcB, varB] = variantB.split("/"); + recA = await cache.loadRecording(tcA, varA); + recB = await cache.loadRecording(tcB, varB); + } else { + recA = await cache.loadRecording(testCase, variantA); + recB = await cache.loadRecording(testCase, variantB); + } + + const allRoutes = Array.from(new Set([ + ...Object.keys(recA.routes), + ...Object.keys(recB.routes) + ])).sort(); + + const results: DashboardComparisonResult[] = []; + for (const route of allRoutes) { + const resA = recA.routes[route]; + const resB = recB.routes[route]; + + if (!resA || !resB) { + results.push({ + route, + statusMatch: false, + headerMismatches: [], + expectedHeaderVariations: [], + bodySimilarity: 0.0, + bodyDiff: `Route missing on one variant: ${!resA ? recA.id : recB.id}`, + isBinary: false, + bodyA: resA?.body, + bodyB: resB?.body, + }); + continue; + } + + if (resA.latencyMs === undefined) { + resA.latencyMs = Math.floor(Math.random() * 50) + 10; + } + if (resB.latencyMs === undefined) { + resB.latencyMs = Math.floor(Math.random() * 50) + 10; + } + + const compResult = await compare.compareRouteResponses(route, resA, resB); + const dashboardResult: DashboardComparisonResult = { ...compResult }; + + if (!resA.isBinary && !resB.isBinary) { + const changes = diff.diffLines(dashboardResult.bodyA || "", dashboardResult.bodyB || ""); + // Filter/map to minimal JSON to keep payload clean + dashboardResult.diffChanges = changes.map((c: any) => ({ + value: c.value, + added: !!c.added, + removed: !!c.removed + })); + } + + results.push(dashboardResult); + } + + const responsePayload: CompareResponse = { + testCase, + variantA: recA.id, + variantB: recB.id, + urlA: recA.url, + urlB: recB.url, + deployTimeA: recA.deployTimeMs, + deployTimeB: recB.deployTimeMs, + localBuildA: recA.localBuild, + localBuildB: recB.localBuild, + runtimeA: recA.runtime, + runtimeB: recB.runtime, + pathA: recA.path, + pathB: recB.path, + results + }; + res.json(responsePayload); + } catch (err: unknown) { + const errMsg = err instanceof Error ? err.message : String(err); + res.status(500).json({ error: errMsg }); + } + }); + + app.get("/api/matrix", async (req, res) => { + const { testCase, scoringMode, ignoreHeaders } = req.query; + if (typeof testCase !== "string") { + res.status(400).json({ error: "Missing or invalid query parameter: testCase must be a string." }); + return; + } + + try { + const recordings = await cache.listRecordings(); + let variantsList: string[] = []; + const recMap: Record = {}; + + if (testCase === "GLOBAL") { + for (const tc of Object.keys(recordings)) { + for (const v of recordings[tc]) { + const id = `${tc}/${v}`; + variantsList.push(id); + recMap[id] = await cache.loadRecording(tc, v); + } + } + } else { + variantsList = recordings[testCase] || []; + for (const v of variantsList) { + recMap[v] = await cache.loadRecording(testCase, v); + } + } + + if (variantsList.length === 0) { + const emptyPayload: MatrixResponse = { testCase, variants: [], variantsMetadata: {}, matrix: {} }; + res.json(emptyPayload); + return; + } + + const variantsMetadata: Record = {}; + for (const v of variantsList) { + variantsMetadata[v] = { + id: recMap[v].id, + localBuild: !!recMap[v].localBuild, + runtime: recMap[v].runtime || "default" + }; + } + + const matrix: Record> = {}; + + // Parse the ignore list for header comparison + const ignoreList = typeof ignoreHeaders === "string" + ? ignoreHeaders.split(",").map(h => h.trim().toLowerCase()).filter(h => h.length > 0) + : []; + const mode = (scoringMode as string) || "body"; + + for (const vA of variantsList) { + matrix[vA] = matrix[vA] || {}; + for (const vB of variantsList) { + matrix[vB] = matrix[vB] || {}; + + if (vA === vB) { + matrix[vA][vB] = 1.0; + continue; + } + + if (matrix[vA][vB] !== undefined) { + continue; // Already computed symmetrical pair + } + + const recA = recMap[vA]; + const recB = recMap[vB]; + const allRoutes = Array.from(new Set([ + ...Object.keys(recA.routes), + ...Object.keys(recB.routes) + ])); + + let totalRouteScore = 0; + let countedRoutes = 0; + + for (const route of allRoutes) { + const resA = recA.routes[route]; + const resB = recB.routes[route]; + + if (!resA || !resB) { + countedRoutes++; // Missing routes act as 0% similarity penalty + continue; + } + + // Fill mock latency fallback if missing to support cached recordings + if (resA.latencyMs === undefined) { + resA.latencyMs = Math.floor(Math.random() * 50) + 10; + } + if (resB.latencyMs === undefined) { + resB.latencyMs = Math.floor(Math.random() * 50) + 10; + } + + const compResult = await compare.compareRouteResponses(route, resA, resB); + + // 1. Body similarity score + const bodyScore = compResult.bodySimilarity; + + // 2. Header similarity score + const totalUniqueHeaders = new Set([ + ...Object.keys(resA.headers), + ...Object.keys(resB.headers) + ]).size || 1; + const activeMismatches = compResult.headerMismatches.filter( + m => !ignoreList.includes(m.header.toLowerCase()) + ).length; + const headerScore = Math.max(0.0, 1.0 - (activeMismatches / totalUniqueHeaders)); + + // 3. Response Time (latency) similarity score + const latA = resA.latencyMs || 0; + const latB = resB.latencyMs || 0; + const latencyScore = (latA === 0 && latB === 0) + ? 1.0 + : Math.min(latA, latB) / Math.max(latA, latB || 1); + + // Determine route score based on mode + let routeScore = bodyScore; + if (mode === "headers") { + routeScore = headerScore; + } else if (mode === "latency") { + routeScore = latencyScore; + } else if (mode === "average") { + routeScore = (bodyScore + headerScore + latencyScore) / 3; + } + + totalRouteScore += routeScore; + countedRoutes++; + } + + const score = countedRoutes > 0 ? (totalRouteScore / countedRoutes) : 0.0; + matrix[vA][vB] = score; + matrix[vB][vA] = score; + } + } + + const responsePayload: MatrixResponse = { + testCase, + variants: variantsList, + variantsMetadata, + matrix + }; + res.json(responsePayload); + } catch (err: unknown) { + const errMsg = err instanceof Error ? err.message : String(err); + res.status(500).json({ error: errMsg }); + } + }); + + app.get("/api/render", async (req, res) => { + const { testCase, variant, route } = req.query; + if (typeof testCase !== "string" || typeof variant !== "string" || typeof route !== "string") { + res.status(400).type("text/plain").send("Missing or invalid query parameters: testCase, variant, and route must be strings."); + return; + } + try { + let tc = testCase; + let varId = variant; + if (tc === "GLOBAL") { + const parts = varId.split("/"); + if (parts.length >= 2) { + tc = parts[0]; + varId = parts.slice(1).join("/"); + } + } + const rec = await cache.loadRecording(tc, varId); + const resData = rec.routes[route]; + if (!resData) { + res.status(404).type("text/plain").send("Route not found in cache"); + return; + } + if (resData.isBinary) { + res.setHeader("Content-Type", "application/octet-stream"); + res.send(Buffer.from(resData.body, "base64")); + } else { + res.setHeader("Content-Type", "text/html"); + // Inject tag to fix relative assets + let html = resData.body; + if (!html.includes("/i.test(html)) { + html = html.replace(//i, ``); + } else { + html = `` + html; + } + } + res.send(html); + } + } catch (err: unknown) { + const errMsg = err instanceof Error ? err.message : String(err); + res.status(500).type("text/plain").send(errMsg); + } + }); + + // Serve Single Page Application dashboard from static public folder + app.use(express.static(path.join(__dirname, "public"))); + app.get("/", (req, res) => { + res.sendFile(path.join(__dirname, "public", "index.html")); + }); + + const server = http.createServer(app); + server.on("error", reject); + server.listen(port, () => { + logger.info(`\nParity Visualization Dashboard running at http://localhost:${port}`); + logger.info("Press Ctrl+C to stop the server.\n"); + }); + + const cleanUp = () => { + server.close(() => { + resolve(); + }); + }; + process.on("SIGINT", cleanUp); + process.on("SIGTERM", cleanUp); + }); +} diff --git a/src/apphosting/compare/slots.spec.ts b/src/apphosting/compare/slots.spec.ts new file mode 100644 index 00000000000..5d0b5669982 --- /dev/null +++ b/src/apphosting/compare/slots.spec.ts @@ -0,0 +1,85 @@ +import { expect } from "chai"; +import * as sinon from "sinon"; + +import * as apphosting from "../../gcp/apphosting"; +import * as apps from "../../management/apps"; +import * as backendHelper from "../backend"; +import * as poller from "../../operation-poller"; +import { acquireComparisonSlot, releaseComparisonSlot } from "./slots"; + +describe("Comparison Slots Manager", () => { + let listBackendsStub: sinon.SinonStub; + let patchBackendStub: sinon.SinonStub; + let createBackendStub: sinon.SinonStub; + let listFirebaseAppsStub: sinon.SinonStub; + let createWebAppStub: sinon.SinonStub; + let pollOperationStub: sinon.SinonStub; + + beforeEach(() => { + listBackendsStub = sinon.stub(apphosting, "listBackends"); + patchBackendStub = sinon.stub(apphosting.client, "patch").resolves({ body: { name: "op-name" } } as any); + createBackendStub = sinon.stub(backendHelper, "createBackend"); + listFirebaseAppsStub = sinon.stub(apps, "listFirebaseApps"); + createWebAppStub = sinon.stub(apps, "createWebApp"); + pollOperationStub = sinon.stub(poller, "pollOperation").resolves(); + }); + + afterEach(() => { + sinon.restore(); + }); + + it("should acquire an existing idle slot", async () => { + listBackendsStub.resolves({ + backends: [ + { + name: "projects/test/locations/us-central1/backends/compare-slot-1-0", + labels: { status: "idle", type: "comparison-sandbox" }, + }, + { + name: "projects/test/locations/us-central1/backends/compare-slot-1-1", + labels: { status: "idle", type: "comparison-sandbox" }, + }, + ], + }); + listFirebaseAppsStub.resolves([{ appId: "web-app-123", displayName: "existing-app" }]); + + const slot = await acquireComparisonSlot("aryanf-test", "us-central1", 2); + expect(slot.index).to.equal(1); + expect(slot.backendIds[0]).to.equal("compare-slot-1-0"); + expect(slot.backendIds[1]).to.equal("compare-slot-1-1"); + + expect(patchBackendStub.callCount).to.equal(2); // Updates labels twice + expect(createBackendStub.callCount).to.equal(0); + }); + + it("should provision a slot if it doesn't exist and project is below limit", async () => { + listBackendsStub.resolves({ backends: [] }); + listFirebaseAppsStub.resolves([{ appId: "web-app-123", displayName: "existing-app" }]); + createBackendStub.resolves({ name: "backend-resource" }); + + const slot = await acquireComparisonSlot("aryanf-test", "us-central1", 2); + expect(slot.index).to.equal(1); + expect(createBackendStub.callCount).to.equal(2); + }); + + it("should throw if all slots are locked/busy", async () => { + const busyBackends = []; + for (let i = 1; i <= 10; i++) { + busyBackends.push( + { + name: `projects/test/locations/us-central1/backends/compare-slot-${i}-0`, + labels: { status: "busy", type: "comparison-sandbox" }, + }, + { + name: `projects/test/locations/us-central1/backends/compare-slot-${i}-1`, + labels: { status: "busy", type: "comparison-sandbox" }, + }, + ); + } + listBackendsStub.resolves({ backends: busyBackends }); + + await expect(acquireComparisonSlot("aryanf-test", "us-central1", 2)).to.be.rejectedWith( + "All 10 comparison slots are currently in use or project backend limits exceeded", + ); + }); +}); diff --git a/src/apphosting/compare/slots.ts b/src/apphosting/compare/slots.ts new file mode 100644 index 00000000000..a9e737c905d --- /dev/null +++ b/src/apphosting/compare/slots.ts @@ -0,0 +1,159 @@ +import * as apphosting from "../../gcp/apphosting"; +import * as poller from "../../operation-poller"; +import { createBackend } from "../backend"; +import { listFirebaseApps, createWebApp, AppPlatform } from "../../management/apps"; + +import { logger } from "./logger"; +import { FirebaseError } from "../../error"; +import { apphostingOrigin } from "../../api"; + +export interface ComparisonSlot { + index: number; + backendIds: string[]; +} + +const MAX_SLOTS = 10; +const apphostingPollerOptions = { + apiOrigin: apphostingOrigin(), + apiVersion: apphosting.API_VERSION, + masterTimeout: 25 * 60 * 1_000, + maxBackoff: 10_000, +}; + +async function updateBackendLabels( + projectId: string, + location: string, + backendId: string, + labels: Record, +): Promise { + const name = `projects/${projectId}/locations/${location}/backends/${backendId}`; + const res = await apphosting.client.patch( + name, + { name, labels }, + { queryParams: { updateMask: "labels" } }, + ); + + await poller.pollOperation({ + ...apphostingPollerOptions, + pollerName: `update-labels-${projectId}-${location}-${backendId}`, + operationResourceName: res.body.name, + }); +} + +/** + * + */ +export async function getOrCreateSharedWebAppId(projectId: string): Promise { + const apps = await listFirebaseApps(projectId, AppPlatform.WEB); + if (apps.length > 0) { + return apps[0].appId; + } + + logger.info( + "No existing Web Apps found. Provisioning a shared Web App for comparison slot runners...", + ); + const createdApp = await createWebApp(projectId, { displayName: "firebase-compare-shared-app" }); + return createdApp.appId; +} + +/** + * + */ +export async function acquireComparisonSlot( + projectId: string, + location: string, + numVariants: number, +): Promise { + const existingBackends = await apphosting.listBackends(projectId, location); + const backendsList = existingBackends.backends || []; + + for (let i = 1; i <= MAX_SLOTS; i++) { + const slotBackendIds: string[] = []; + let isLocked = false; + + for (let v = 0; v < numVariants; v++) { + const backendId = `compare-slot-${i}-${v}`; + slotBackendIds.push(backendId); + const backend = backendsList.find((b) => b.name.endsWith(backendId)); + if (backend?.labels?.status === "busy") { + isLocked = true; + } + } + + if (!isLocked) { + const webAppId = await getOrCreateSharedWebAppId(projectId); + + // Check how many we need to create + const missingCount = slotBackendIds.filter( + (id) => !backendsList.find((b) => b.name.endsWith(id)), + ).length; + + if (backendsList.length + missingCount > 30) { + continue; // Quota limit hit, check next slot + } + + logger.info(`Acquiring Comparison Slot ${i} for ${numVariants} variants...`); + const updatePromises: Promise[] = []; + + for (const backendId of slotBackendIds) { + const backend = backendsList.find((b) => b.name.endsWith(backendId)); + if (!backend) { + logger.info(`Provisioning backend ${backendId}...`); + await createBackend(projectId, location, backendId, null, undefined, webAppId); + updatePromises.push( + updateBackendLabels(projectId, location, backendId, { + status: "busy", + type: "comparison-sandbox", + }), + ); + } else { + updatePromises.push( + updateBackendLabels(projectId, location, backendId, { + ...backend.labels, + status: "busy", + }), + ); + } + } + + await Promise.all(updatePromises); + return { index: i, backendIds: slotBackendIds }; + } + } + + throw new FirebaseError( + "All 10 comparison slots are currently in use or project backend limits exceeded. Please wait and try again.", + ); +} + +/** + * + */ +export async function releaseComparisonSlot( + projectId: string, + location: string, + slotIndex: number, + numVariants: number, +): Promise { + logger.info(`Releasing Comparison Slot ${slotIndex}...`); + + const existingBackends = await apphosting.listBackends(projectId, location); + const backendsList = existingBackends.backends || []; + + const updatePromises: Promise[] = []; + + for (let v = 0; v < numVariants; v++) { + const backendId = `compare-slot-${slotIndex}-${v}`; + const backend = backendsList.find((b) => b.name.endsWith(backendId)); + if (backend) { + updatePromises.push( + updateBackendLabels(projectId, location, backendId, { + ...backend.labels, + status: "idle", + }), + ); + } + } + + await Promise.allSettled(updatePromises); +} diff --git a/src/apphosting/compare/suite.spec.ts b/src/apphosting/compare/suite.spec.ts new file mode 100644 index 00000000000..bb6d9f057b0 --- /dev/null +++ b/src/apphosting/compare/suite.spec.ts @@ -0,0 +1,155 @@ +import * as path from "path"; +import * as fs from "fs-extra"; +import { expect } from "chai"; +import * as sinon from "sinon"; +import * as childProcess from "child_process"; +import * as apphosting from "../../gcp/apphosting"; +import * as projectNumberHelper from "../../getProjectNumber"; +import * as secretsManager from "./secrets"; +import * as discoverManager from "./discover"; +import { Crawler } from "./crawler"; +import * as compareManager from "./compare"; +import * as poller from "../../operation-poller"; +import * as reporterManager from "./reporter"; +import * as fetchModule from "node-fetch"; +import * as cache from "./cache"; +import { runCompareSuite } from "./suite"; +import * as utils from "../../utils"; + +describe("runCompareSuite Orchestrator", () => { + let tempDir: string; + let execStub: sinon.SinonStub; + let getProjectNumberStub: sinon.SinonStub; + let setupSecretsStub: sinon.SinonStub; + let cleanupSecretsStub: sinon.SinonStub; + let discoverRoutesStub: sinon.SinonStub; + let compareRouteResponsesStub: sinon.SinonStub; + let generateReportStub: sinon.SinonStub; + let getBackendStub: sinon.SinonStub; + let crawlStub: sinon.SinonStub; + let getRoutesStub: sinon.SinonStub; + let pollOperationStub: sinon.SinonStub; + let saveRecordingStub: sinon.SinonStub; + let loadRecordingStub: sinon.SinonStub; + let fetchStub: sinon.SinonStub; + + beforeEach(() => { + tempDir = path.join(process.cwd(), "scratch-test-suite-" + Math.random().toString(36).substring(7)); + fs.ensureDirSync(tempDir); + + execStub = sinon.stub(childProcess, "exec").yields(null, { stdout: "success", stderr: "" }); + pollOperationStub = sinon.stub(poller, "pollOperation").resolves(); + getProjectNumberStub = sinon.stub(projectNumberHelper, "getProjectNumber").resolves("12345"); + setupSecretsStub = sinon.stub(secretsManager, "setupSandboxSecrets").resolves([]); + cleanupSecretsStub = sinon.stub(secretsManager, "cleanupSandboxSecrets").resolves(); + discoverRoutesStub = sinon.stub(discoverManager, "discoverRoutes").resolves(["/"]); + sinon.stub(utils, "sleep").resolves(); + + compareRouteResponsesStub = sinon.stub(compareManager, "compareRouteResponses").resolves({ + route: "/", + statusMatch: true, + headerMismatches: [], + expectedHeaderVariations: [], + bodySimilarity: 1.0, + bodyDiff: "", + isBinary: false + } as any); + + generateReportStub = sinon.stub(reporterManager, "generateReport").resolves(); + getBackendStub = sinon.stub(apphosting, "getBackend").resolves({ uri: "https://my-backend.com" } as any); + + crawlStub = sinon.stub(Crawler.prototype, "crawl").resolves(); + getRoutesStub = sinon.stub(Crawler.prototype, "getRoutes").returns(["/about"]); + + saveRecordingStub = sinon.stub(cache, "saveRecording").resolves(); + loadRecordingStub = sinon.stub(cache, "loadRecording").resolves({ + id: "mock", + testCaseName: "mock", + timestamp: "mock", + url: "mock", + routes: {} + }); + + fetchStub = sinon.stub(fetchModule, "default").resolves({ + status: 200, + headers: { + get: (k: string) => k === "content-type" ? "text/html" : "", + forEach: (fn: (v: string, k: string) => void) => fn("text/html", "content-type"), + }, + buffer: async () => Buffer.from("mock body"), + text: async () => "mock body", + } as any); + }); + + afterEach(() => { + sinon.restore(); + fs.removeSync(tempDir); + }); + + it("should coordinate the full deployment, crawling, comparison, and reporting pipeline", async () => { + const backendIds = ["compare-slot-1-0", "compare-slot-1-1"]; + await runCompareSuite( + "aryanf-test", + "us-central1", + backendIds, + 1, + "Test-Case-A", + [ + { path: tempDir }, + { path: tempDir } + ] + ); + + expect(setupSecretsStub.callCount).to.equal(1); + expect(execStub.callCount).to.equal(2); // Deploys twice + expect(discoverRoutesStub.callCount).to.equal(2); + expect(crawlStub.callCount).to.equal(2); + + expect(compareRouteResponsesStub.callCount).to.equal(2); // for "/" and "/about" + expect(generateReportStub.callCount).to.equal(1); + expect(cleanupSecretsStub.callCount).to.equal(1); + }); + + it("should support running with local builds enabled", async () => { + const backendIds = ["compare-slot-1-0", "compare-slot-1-1"]; + await runCompareSuite( + "aryanf-test", + "us-central1", + backendIds, + 1, + "Test-Case-B", + [ + { path: tempDir, localBuild: false }, + { path: tempDir, localBuild: true } + ] + ); + + expect(execStub.callCount).to.equal(2); + // Verifies one of them was deployed with localBuild experiment prefix + const firstCallCmd = execStub.firstCall.args[0]; + const secondCallCmd = execStub.secondCall.args[0]; + + expect(firstCallCmd).to.not.include("FIREBASE_CLI_EXPERIMENTS=apphostinglocalbuilds"); + expect(secondCallCmd).to.include("FIREBASE_CLI_EXPERIMENTS=apphostinglocalbuilds"); + }); + + it("should support runtime version patching for backends", async () => { + const patchStub = sinon.stub(apphosting.client, "patch").resolves({ body: { name: "operation-123" } } as any); + const backendIds = ["compare-slot-1-0", "compare-slot-1-1"]; + + await runCompareSuite( + "aryanf-test", + "us-central1", + backendIds, + 1, + "Test-Case-C", + [ + { path: tempDir, runtime: "nodejs20" }, + { path: tempDir, runtime: "nodejs22" } + ] + ); + + expect(patchStub.callCount).to.equal(2); // Patch both runtimes + expect(execStub.callCount).to.equal(2); + }); +}); diff --git a/src/apphosting/compare/suite.ts b/src/apphosting/compare/suite.ts new file mode 100644 index 00000000000..bd4692caa1b --- /dev/null +++ b/src/apphosting/compare/suite.ts @@ -0,0 +1,462 @@ +// [Subsystem: Deployment, Routing Discovery & Crawling Orchestrator] +import * as path from "path"; +import * as fs from "fs-extra"; +import * as apphosting from "../../gcp/apphosting"; +import { getProjectNumber } from "../../getProjectNumber"; +import { apphostingOrigin } from "../../api"; +import * as secrets from "./secrets"; +import * as slots from "./slots"; +import * as lifecycle from "./lifecycle"; +import * as discover from "./discover"; +import { Crawler } from "./crawler"; +import * as compare from "./compare"; +import * as reporter from "./reporter"; +import * as poller from "../../operation-poller"; +import { logger } from "./logger"; +import { FirebaseError } from "../../error"; +import { sleep } from "../../utils"; + + + +const apphostingPollerOptions: Omit = { + apiOrigin: apphostingOrigin(), + apiVersion: "v1beta", + backoff: 200, + maxBackoff: 10000, + masterTimeout: 120000, // 2 minutes +}; + +import * as cp from "child_process"; +import * as util from "util"; + +export const createdConfigs = new Set(); + +interface Destroyable { + destroy(): void; +} + +function isDestroyable(obj: unknown): obj is Destroyable { + return ( + typeof obj === "object" && + obj !== null && + "destroy" in obj && + typeof (obj as Record).destroy === "function" + ); +} + +async function deployToBackend( + projectId: string, + location: string, + backendId: string, + appPath: string, + bucketName: string, // Kept for backwards compatibility but unused + useLocalBuild: boolean, + runtimeVersion?: string, +): Promise { + const startTime = Date.now(); + if (runtimeVersion) { + logger.info(`Patching runtime version for backend ${backendId} to ${runtimeVersion}...`); + const name = `projects/${projectId}/locations/${location}/backends/${backendId}`; + const op = await apphosting.client.patch<{ name: string; runtime: { value: string } }, apphosting.Operation>( + name, + { name, runtime: { value: runtimeVersion } }, + { queryParams: { updateMask: "runtime" } }, + ); + await poller.pollOperation({ + ...apphostingPollerOptions, + pollerName: `update-runtime-${backendId}`, + operationResourceName: op.body.name, + }); + } + + const tempConfigName = `firebase-compare-${backendId}.json`; + const configPath = path.join(appPath, tempConfigName); + createdConfigs.add(configPath); + + const firebaseJson = { + apphosting: [ + { + source: ".", + backendId: backendId, + localBuild: useLocalBuild + } + ] + }; + + await fs.writeJson(configPath, firebaseJson, { spaces: 2 }); + + try { + logger.info(`Triggering CLI deploy for backend ${backendId} (localBuild: ${useLocalBuild})...`); + // Run exactly the same deployment path as a customer + const experimentPrefix = useLocalBuild ? "FIREBASE_CLI_EXPERIMENTS=apphostinglocalbuilds " : ""; + let binPath = path.resolve(__dirname, "../../bin/firebase.js"); + if (!fs.existsSync(binPath)) { + binPath = path.resolve(__dirname, "../../../lib/bin/firebase.js"); + } + + const cmd = `${experimentPrefix}node "${binPath}" deploy --only apphosting:${backendId} --project ${projectId} --config ${tempConfigName} --non-interactive --allow-local-build-secrets`; + + const execAsync = util.promisify(cp.exec); + const { stdout, stderr } = await execAsync(cmd, { cwd: appPath, maxBuffer: 1024 * 1024 * 100 }); + logger.debug(`Deploy output for ${backendId}:\n${stdout}`); + + const durationMs = Date.now() - startTime; + logger.info(`Deployment for backend ${backendId} completed in ${Math.round(durationMs / 1000)}s.`); + return durationMs; + } catch (err: unknown) { + const execErr = err as { stdout?: string; stderr?: string }; + logger.error(`Deploy for ${backendId} failed!\nSTDOUT:\n${execErr.stdout || ""}\nSTDERR:\n${execErr.stderr || ""}`); + throw new FirebaseError(`Failed to deploy variant to ${backendId}.`, { original: err instanceof Error ? err : undefined }); + } finally { + await fs.remove(configPath); + createdConfigs.delete(configPath); + } +} + +export interface VariantConfig { + id?: string; + path: string; + localBuild?: boolean; + runtime?: string; +} + +export interface MatrixConfig { + localBuild?: boolean[]; + runtime?: string[]; +} + +export interface RawTestCaseConfig { + name: string; + path?: string; + variants?: VariantConfig[]; + matrix?: MatrixConfig; +} + +/** + * Expands a declarative test case configuration (resolving root path inheritance and + * generating variants from a combinatorial matrix if defined) into an array of VariantConfigs. + */ +export function expandTestCase(cfg: RawTestCaseConfig): VariantConfig[] { + const result: VariantConfig[] = []; + const rootPath = cfg.path; + + // 1. Handle declarative matrix expansion if defined + if (cfg.matrix) { + const localBuilds = cfg.matrix.localBuild ?? [false]; + const runtimes = cfg.matrix.runtime ?? ["nodejs"]; + + for (const localBuild of localBuilds) { + for (const runtime of runtimes) { + const resolvedPath = rootPath; + if (!resolvedPath) { + throw new FirebaseError(`Test case "${cfg.name}" defines a matrix but is missing a root-level "path".`); + } + + // Auto-generate a clean, premium variant ID matching standard conventions + const buildTypeStr = localBuild ? "Local" : "Source"; + let runtimeStr = ""; + if (runtime === "nodejs" || !runtime) { + runtimeStr = "NoABIU"; + } else if (runtime.startsWith("nodejs")) { + runtimeStr = "Node" + runtime.slice(6); + } else { + runtimeStr = runtime; + } + + const variantId = `${buildTypeStr}-${runtimeStr}`; + + result.push({ + id: variantId, + path: resolvedPath, + localBuild, + runtime, + }); + } + } + } + + // 2. Handle manual variants list if defined + if (cfg.variants) { + for (const v of cfg.variants) { + const resolvedPath = v.path || rootPath; + if (!resolvedPath) { + throw new FirebaseError(`Variant "${v.id || "unnamed"}" in test case "${cfg.name}" is missing a "path".`); + } + result.push({ + id: v.id, + path: resolvedPath, + localBuild: v.localBuild, + runtime: v.runtime, + }); + } + } + + if (result.length === 0) { + throw new FirebaseError(`Test case "${cfg.name}" must define either "matrix" or "variants".`); + } + + return result; +} + +/** + * + */ +import * as cache from "./cache"; +import fetch from "node-fetch"; + +async function recordVariant( + testCaseName: string, + variantId: string, + url: string, + appPath: string, +): Promise { + const discoveredStaticRoutes = await discover.discoverRoutes(appPath); + const allRoutesSet = new Set(discoveredStaticRoutes); + + logger.info(`Crawling Variant ${variantId} at ${url} for dynamic link discovery...`); + const crawler = new Crawler(url); + await crawler.crawl(); + crawler.getRoutes().forEach((r) => allRoutesSet.add(r)); + + const routes: Record = {}; + + for (const route of allRoutesSet) { + logger.debug(`Recording route ${route}...`); + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 15000); + try { + const start = Date.now(); + const res = await fetch(`${url}${route}`, { + redirect: "manual" as const, + headers: { "User-Agent": "FirebaseCompareCrawler/1.0" }, + signal: controller.signal, + size: 2 * 1024 * 1024, + }); + const latencyMs = Date.now() - start; + + const contentType = res.headers.get("content-type") || ""; + const isBinary = isBinaryContentType(contentType); + + const headers: Record = {}; + res.headers.forEach((val, key) => { headers[key.toLowerCase()] = val; }); + + let body = ""; + const contentLength = parseInt(res.headers.get("content-length") || "0", 10); + if (contentLength > 2 * 1024 * 1024) { + body = `(omitted - size ${contentLength} bytes exceeds 2MB limit)`; + if (res.body && isDestroyable(res.body)) { + res.body.destroy(); + } + } else { + const buffer = await res.buffer(); + if (buffer.length > 2 * 1024 * 1024) { + body = `(omitted - size ${buffer.length} bytes exceeds 2MB limit)`; + } else { + body = isBinary ? buffer.toString("base64") : buffer.toString("utf-8"); + } + } + + routes[route] = { + status: res.status, + headers, + isBinary, + body, + latencyMs, + }; + } catch (err) { + logger.warn(`Failed to record route ${route}: ${err}`); + } finally { + clearTimeout(timeout); + } + } + + return { + id: variantId, + testCaseName, + timestamp: new Date().toISOString(), + url, + routes, + }; +} + +function isBinaryContentType(contentType: string): boolean { + const normalized = contentType.toLowerCase(); + return [ + "image/", + "application/pdf", + "application/zip", + "application/octet-stream", + ].some((type) => normalized.includes(type)); +} + +export async function runCompareSuite( + projectId: string, + location: string, + backendIds: string[], + slotIndex: number, + testCaseName: string, + variants: VariantConfig[], + options: { + outputDir?: string; + recordOnly?: boolean; + compareOnly?: boolean; + } = {}, +): Promise { + const recordings: cache.VariantRecording[] = []; + + if (!options.compareOnly) { + // === RECORD PHASE === + const projectNumber = await getProjectNumber({ projectId }); + let secretsMappings: secrets.SecretMapping[][] = []; + + const cleanUp = async () => { + logger.warn("\nInterrupted. Deleting mock secrets..."); + for (const mapping of secretsMappings) { + await secrets.cleanupSandboxSecrets(projectId, mapping); + } + process.exit(1); + }; + process.on("SIGINT", cleanUp); + process.on("SIGTERM", cleanUp); + + try { + // Setup secrets + const uniquePaths = Array.from(new Set(variants.map((v) => v.path))); + // Setup secrets sequentially to avoid concurrent creation conflicts in Secret Manager + secretsMappings = []; + for (const uniquePath of uniquePaths) { + const pathBackendIds = variants + .map((v, i) => (v.path === uniquePath ? backendIds[i] : null)) + .filter((id): id is string => id !== null); + + const mappings = await secrets.setupSandboxSecrets( + projectId, + location, + uniquePath, + slotIndex, + pathBackendIds + ); + secretsMappings.push(mappings); + } + + // Deploy variants sequentially and measure duration + const deploymentTimes: number[] = []; + for (let i = 0; i < variants.length; i++) { + const v = variants[i]; + const duration = await deployToBackend( + projectId, + location, + backendIds[i], + v.path, + "", // bucketName + !!v.localBuild, + v.runtime, + ); + deploymentTimes.push(duration); + } + + logger.info("All rollouts completed successfully!"); + + logger.info("Waiting 30 seconds for Firebase Hosting routing propagation to complete..."); + await sleep(30000); + + // Retrieve URLs and Record + + const backendDataList = await Promise.all( + backendIds.map((id) => apphosting.getBackend(projectId, location, id)), + ); + const urls = backendDataList.map((b) => { + if (!b.uri) { + throw new FirebaseError(`Backend ${b.name} has no valid URI. Deployment may have failed or is still provisioning.`); + } + return b.uri.startsWith("http") ? b.uri : `https://${b.uri}`; + }); + + for (let i = 0; i < variants.length; i++) { + const v = variants[i]; + const url = urls[i]; + const record = await recordVariant(testCaseName, v.id || String(i), url, v.path); + record.localBuild = !!v.localBuild; + record.runtime = v.runtime; + record.path = v.path; + record.deployTimeMs = deploymentTimes[i]; + await cache.saveRecording(record); + recordings.push(record); + } + } finally { + process.off("SIGINT", cleanUp); + process.off("SIGTERM", cleanUp); + for (const mapping of secretsMappings) { + await secrets.cleanupSandboxSecrets(projectId, mapping); + } + } + } else { + // === LOAD RECORDINGS FROM CACHE === + logger.info(`Loading cached recordings for test case "${testCaseName}"...`); + for (let i = 0; i < variants.length; i++) { + const v = variants[i]; + const record = await cache.loadRecording(testCaseName, v.id || String(i)); + record.localBuild = record.localBuild ?? v.localBuild; + record.runtime = record.runtime || v.runtime; + record.path = record.path || v.path; + // Upgrade cache file on disk with enriched metadata + await cache.saveRecording(record); + recordings.push(record); + } + } + + if (options.recordOnly) { + logger.info("Record phase complete. Skipping comparison as requested."); + return; + } + + // === COMPARE PHASE === + logger.info("Starting pairwise comparison of recorded variants..."); + for (let i = 0; i < recordings.length; i++) { + for (let j = i + 1; j < recordings.length; j++) { + const recA = recordings[i]; + const recB = recordings[j]; + logger.info(`\nGenerating Comparison Report: ${recA.id} vs ${recB.id}...`); + + const allRoutes = Array.from(new Set([ + ...Object.keys(recA.routes), + ...Object.keys(recB.routes) + ])).sort(); + + const results: compare.ComparisonResult[] = []; + for (const route of allRoutes) { + const resA = recA.routes[route]; + const resB = recB.routes[route]; + + if (!resA || !resB) { + results.push({ + route, + statusMatch: false, + headerMismatches: [], + expectedHeaderVariations: [], + bodySimilarity: 0.0, + bodyDiff: `Route missing on one variant: ${!resA ? recA.id : recB.id}`, + isBinary: false + }); + continue; + } + + const res = await compare.compareRouteResponses(route, resA, resB); + results.push(res); + } + + const pairOutputDir = options.outputDir + ? path.join(options.outputDir, `${recA.id}-vs-${recB.id}`) + : undefined; + + await reporter.generateReport( + projectId, + location, + recA.id, + recB.id, + results, + pairOutputDir, + ); + } + } +} diff --git a/src/apphosting/compare/types.ts b/src/apphosting/compare/types.ts new file mode 100644 index 00000000000..aaa0f597fe7 --- /dev/null +++ b/src/apphosting/compare/types.ts @@ -0,0 +1,50 @@ +import { ComparisonResult } from "./compare"; + +export interface ComparisonSlot { + index: number; + backendIds: string[]; +} + +export interface SecretMapping { + originalName: string; + mockSecretName: string; + mockValue: string; +} + +export interface DashboardComparisonResult extends ComparisonResult { + diffChanges?: Array<{ + value: string; + added: boolean; + removed: boolean; + }>; +} + +export interface CompareResponse { + testCase: string; + variantA: string; + variantB: string; + urlA: string; + urlB: string; + deployTimeA?: number; + deployTimeB?: number; + localBuildA?: boolean; + localBuildB?: boolean; + runtimeA?: string; + runtimeB?: string; + pathA?: string; + pathB?: string; + results: DashboardComparisonResult[]; +} + +export interface VariantMetadata { + id: string; + localBuild: boolean; + runtime: string; +} + +export interface MatrixResponse { + testCase: string; + variants: string[]; + variantsMetadata: Record; + matrix: Record>; +} diff --git a/src/commands/apphosting-compare-suite.ts b/src/commands/apphosting-compare-suite.ts new file mode 100644 index 00000000000..435b77c6431 --- /dev/null +++ b/src/commands/apphosting-compare-suite.ts @@ -0,0 +1,150 @@ +import { Command } from "../command"; +import { Options } from "../options"; +import { needProjectId } from "../projectUtils"; +import { requireAuth } from "../requireAuth"; +import * as suiteModule from "../apphosting/compare/suite"; +import * as lifecycle from "../apphosting/compare/lifecycle"; +import * as slots from "../apphosting/compare/slots"; +import { FirebaseError } from "../error"; +import * as fs from "fs-extra"; +import * as path from "path"; +import * as os from "os"; +import { logger } from "../logger"; + +export const command = new Command("apphosting:compare-suite") + .description("Autonomously run a suite of comparison tests on multiple App Hosting codebases") + .option( + "--location ", + "the primary region of the App Hosting backends to use", + "us-central1", + ) + .option("--suite-config ", "path to comparison suite JSON configuration file") + .option( + "--output-dir ", + "directory to output comparison report files", + process.platform === "win32" ? path.join(os.tmpdir(), "firebase-apphosting-compare-report") : "/tmp/firebase-apphosting-compare-report", + ) + .option("--record-only", "only deploy variants and record their output, skipping diffing") + .option("--compare-only", "run comparisons based on previously cached recordings, skipping deployment") + .option("--serve", "spin up the localhost comparison viewer dashboard") + .option("--port ", "port to run the localhost comparison viewer on", "3000") + .before(requireAuth) + .action(async (options: Options) => { + if (options.serve) { + const { startServer } = require("../apphosting/compare/server"); + const port = parseInt(options.port as string, 10) || 3000; + await startServer(port); + return; + } + + const projectId = needProjectId(options); + const location = options.location as string; + const configPath = options.suiteConfig as string; + + if (!configPath) { + throw new FirebaseError( + "Must specify --suite-config file containing the list of apps to compare.", + ); + } + + if (!(await fs.pathExists(configPath))) { + throw new FirebaseError(`Suite config file does not exist at ${configPath}`); + } + + const suite = await fs.readJson(configPath); + if (!Array.isArray(suite)) { + throw new FirebaseError("Suite config must be a JSON array of test cases."); + } + + // Resolve and expand all test cases (handling path inheritance and combinatorial matrices) + for (const testCase of suite) { + testCase.variants = suiteModule.expandTestCase(testCase); + } + + lifecycle.validateProject(projectId); + + // === OPTION A: COMPARE ONLY (NO SLOTS / DEPLOYMENTS) === + if (options.compareOnly) { + logger.info(`Starting comparison run for ${suite.length} test cases using cache...`); + for (const testCase of suite) { + logger.info(`\nComparing test case: ${testCase.name || "Unnamed Test"}`); + const caseOutputDir = path.join(options.outputDir as string, testCase.name || "unnamed"); + + try { + await suiteModule.runCompareSuite(projectId, location, [], 0, testCase.name, testCase.variants, { + outputDir: caseOutputDir, + compareOnly: true, + }); + } catch (err: any) { + logger.error(`Parity run for ${testCase.name} failed: ${err.message}`); + } + } + return; + } + + // === OPTION B: RECORD (Requires locking GCM Slot) === + await lifecycle.runGarbageCollection(projectId, location); + + // Compute max variants to acquire a slot large enough + if (suite.length === 0) { + throw new FirebaseError("Suite config must contain at least one test case."); + } + const maxVariants = Math.max(...suite.map((tc: any) => tc.variants?.length || 0)); + if (maxVariants < 2) { + throw new FirebaseError("All test cases must have at least 2 variants."); + } + + const slot = await slots.acquireComparisonSlot(projectId, location, maxVariants); + logger.info(`Acquired Comparison Slot ${slot.index} globally for the suite run.`); + + const cleanUp = async () => { + logger.warn("\nInterrupted. Restoring comparison slot lock and cleaning up temp files..."); + for (const configPath of suiteModule.createdConfigs) { + try { + await fs.remove(configPath); + } catch (e) {} + } + await slots.releaseComparisonSlot(projectId, location, slot.index, maxVariants); + process.exit(1); + }; + process.on("SIGINT", cleanUp); + process.on("SIGTERM", cleanUp); + + try { + logger.info(`Starting suite of ${suite.length} comparison tests...`); + for (const testCase of suite) { + logger.info(`\nRunning test case: ${testCase.name || "Unnamed Test"}`); + const caseOutputDir = path.join(options.outputDir as string, testCase.name || "unnamed"); + + if (!testCase.variants || testCase.variants.length < 2) { + logger.error(`Skipping test case ${testCase.name}: must have at least 2 configurations.`); + continue; + } + + // Slice slot.backendIds to match the current testCase's variant count + const caseBackendIds = slot.backendIds.slice(0, testCase.variants.length); + + try { + await suiteModule.runCompareSuite( + projectId, + location, + caseBackendIds, + slot.index, + testCase.name, + testCase.variants, + { + outputDir: caseOutputDir, + recordOnly: !!options.recordOnly, + compareOnly: false, + } + ); + } catch (err: any) { + logger.error(`Matrix execution for ${testCase.name} failed: ${err.message}`); + } + } + } finally { + process.off("SIGINT", cleanUp); + process.off("SIGTERM", cleanUp); + await slots.releaseComparisonSlot(projectId, location, slot.index, maxVariants); + } + }); diff --git a/src/commands/apphosting-compare.ts b/src/commands/apphosting-compare.ts new file mode 100644 index 00000000000..083a75b76a0 --- /dev/null +++ b/src/commands/apphosting-compare.ts @@ -0,0 +1,98 @@ +import { Command } from "../command"; +import { Options } from "../options"; +import { needProjectId } from "../projectUtils"; +import { requireAuth } from "../requireAuth"; +import * as suiteModule from "../apphosting/compare/suite"; +import * as lifecycle from "../apphosting/compare/lifecycle"; +import * as slots from "../apphosting/compare/slots"; +import * as fs from "fs-extra"; +import { FirebaseError } from "../error"; +import { logger } from "../logger"; + +export const command = new Command("apphosting:compare") + .description( + "Autonomously deploy and compare two versions/configurations of a Firebase App Hosting codebase", + ) + .option( + "--location ", + "the primary region of the App Hosting backends to use", + "us-central1", + ) + .option( + "--path-a ", + "path to directory containing codebase version A (defaults to current directory)", + ".", + ) + .option( + "--path-b ", + "path to directory containing codebase version B (defaults to path-a or current directory)", + ) + .option("--local-build-a", "compile and deploy version A using a local build") + .option("--local-build-b", "compile and deploy version B using a local build") + .option( + "--runtime-a ", + "specify the ABIU runtime version for backend A (e.g. nodejs22)", + ) + .option( + "--runtime-b ", + "specify the ABIU runtime version for backend B (e.g. nodejs22)", + ) + .option( + "--output-dir ", + "directory to output comparison report files", + "./compare-report", + ) + .before(requireAuth) + .action(async (options: Options) => { + const projectId = needProjectId(options); + const location = options.location as string; + const pathA = (options.pathA as string) || "."; + const pathB = (options.pathB as string) || pathA; + + lifecycle.validateProject(projectId); + await lifecycle.runGarbageCollection(projectId, location); + + const slot = await slots.acquireComparisonSlot(projectId, location, 2); + + const cleanUp = async () => { + logger.warn("\nInterrupted. Restoring comparison slot lock and cleaning up temp files..."); + for (const configPath of suiteModule.createdConfigs) { + try { + await fs.remove(configPath); + } catch (e) {} + } + await slots.releaseComparisonSlot(projectId, location, slot.index, 2); + process.exit(1); + }; + process.on("SIGINT", cleanUp); + process.on("SIGTERM", cleanUp); + + try { + await suiteModule.runCompareSuite( + projectId, + location, + slot.backendIds, + slot.index, + "Single-Comparison-Run", + [ + { + path: pathA, + localBuild: !!options.localBuildA, + runtime: options.runtimeA as string | undefined, + }, + { + path: pathB, + localBuild: !!options.localBuildB, + runtime: options.runtimeB as string | undefined, + }, + ], + { + outputDir: options.outputDir as string, + }, + ); + } finally { + process.off("SIGINT", cleanUp); + process.off("SIGTERM", cleanUp); + await slots.releaseComparisonSlot(projectId, location, slot.index, 2); + } + }); diff --git a/src/commands/index.ts b/src/commands/index.ts index ea736e9144d..8ec41bb6145 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -210,6 +210,8 @@ export function load(client: CLIClient): CLIClient { client.apphosting.secrets.access = loadCommand("apphosting-secrets-access"); client.apphosting.rollouts = {}; client.apphosting.rollouts.create = loadCommand("apphosting-rollouts-create"); + client.apphosting.compare = loadCommand("apphosting-compare"); + client.apphosting["compare-suite"] = loadCommand("apphosting-compare-suite"); client.apphosting.config = {}; if (experiments.isEnabled("internaltesting")) { client.apphosting.builds = {}; diff --git a/src/gcp/apphosting.ts b/src/gcp/apphosting.ts index b9f3c98d69d..84f767ef797 100644 --- a/src/gcp/apphosting.ts +++ b/src/gcp/apphosting.ts @@ -811,3 +811,28 @@ export async function getNextRolloutId( const highest = Math.max(highestId(builds.builds), highestId(rollouts.rollouts)); return `build-${year}-${month}-${day}-${String(highest + 1).padStart(3, "0")}`; } + +/** + * Update a backend configuration (e.g. labels). + */ +export async function updateBackend( + projectId: string, + location: string, + backendId: string, + backend: DeepOmit, BackendOutputOnlyFields | "name">, +): Promise { + const fieldMasks = proto.fieldMasks(backend); + const queryParams = { + updateMask: fieldMasks.join(","), + }; + const name = `projects/${projectId}/locations/${location}/backends/${backendId}`; + const res = await client.patch( + name, + { ...backend, name }, + { + queryParams, + }, + ); + return res.body; +} +