diff --git a/plugins/codex-security/.codex-plugin/plugin.json b/plugins/codex-security/.codex-plugin/plugin.json index 43fa0c2a9..ae87118e5 100644 --- a/plugins/codex-security/.codex-plugin/plugin.json +++ b/plugins/codex-security/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "codex-security", - "version": "0.1.94", + "version": "0.1.96", "description": "Codex Security workflows for security scans, analysis, and investigation.", "author": { "name": "OpenAI" diff --git a/plugins/codex-security/scripts/finalize_scan_contract.py b/plugins/codex-security/scripts/finalize_scan_contract.py index 58cac8f1f..7006ed2a5 100644 --- a/plugins/codex-security/scripts/finalize_scan_contract.py +++ b/plugins/codex-security/scripts/finalize_scan_contract.py @@ -44,6 +44,28 @@ "directory_snapshot": {"snapshotDigest"}, } DISPOSITIONS = {"reported", "no_issue_found", "rejected", "not_applicable", "needs_follow_up"} +NON_COVERAGE_TARGET_WARNINGS = { + ( + "Directory contents changed while the scan was running; " + "results were saved for the original snapshot." + ), + ( + "The scanned Git repository became unavailable while the scan was running; " + "results were saved for the original revision." + ), + ( + "Repository HEAD changed while the scan was running; " + "results were saved for the original revision." + ), + ( + "Working-tree contents changed while the scan was running; " + "results were saved for the original snapshot." + ), + ( + "The scan target became unavailable while the scan was running; " + "results were saved for the original revision or snapshot." + ), +} SARIF_LEVELS = { "critical": "error", "high": "error", @@ -1093,6 +1115,26 @@ def _recover_unsealed_coverage( partial = True if partial: coverage["completeness"] = "partial" + elif ( + completeness == "partial" + and coverage.get("mode") == "deep_repository" + and coverage["surfaces"] + and not coverage["deferred"] + and all( + warning in NON_COVERAGE_TARGET_WARNINGS + or re.fullmatch( + r"Recovered finding [0-9]+: " + r"(?:normalized [a-z, ]+|retained stronger duplicate logical finding)\.", + warning, + ) + or re.fullmatch( + r"Skipped malformed finding [0-9]+: duplicate logical finding\.", warning + ) + for warning in warnings + ) + ): + coverage["completeness"] = "complete" + warnings.append("Recovered Deep Scan coverage marked partial without deferred review work.") def _recover_unsealed_hardening( diff --git a/plugins/codex-security/scripts/workbench_saved_results.py b/plugins/codex-security/scripts/workbench_saved_results.py index 8c015a65d..b2a0a1e32 100644 --- a/plugins/codex-security/scripts/workbench_saved_results.py +++ b/plugins/codex-security/scripts/workbench_saved_results.py @@ -47,6 +47,9 @@ _PUBLICATION_FOLLOW_UP_WARNING = ( "Saved scan evidence remains on disk; result publication needs follow-up:" ) +_UNVERIFIED_COVERAGE_WARNING = ( + "Saved scan source is incomplete or has unverified coverage; coverage remains partial." +) @dataclass(frozen=True) @@ -472,6 +475,11 @@ def merge_saved_results( if frozen_source_digests is None or allow_frozen_legacy_parent: try: parent_manifest, parent = _read_saved_parent_result(scan_dir, scan_id) + if ( + parent.get("complete", True) is not True + and _UNVERIFIED_COVERAGE_WARNING not in warnings + ): + warnings.append(_UNVERIFIED_COVERAGE_WARNING) except (ContractError, OSError, ValueError) as exc: if not stopped: raise @@ -503,6 +511,7 @@ def merge_saved_results( paths: dict[str, str | None] = {} reducer_paths: set[str] = set() current_results: set[str] = set() + required_results: set[str] = set() reducer_outputs: list[tuple[Any, str, list[str], int]] = [] reducer = _latest_successful_reducer(workers) latest_reducer: str | None = None @@ -510,6 +519,7 @@ def merge_saved_results( try: latest_reducer = Path(reducer["result_manifest_path"]).relative_to(scan_dir).as_posix() paths[latest_reducer] = None + required_results.add(latest_reducer) reducer_paths.add(latest_reducer) except ValueError: warnings.append("Skipped a reducer result outside the scan directory.") @@ -570,6 +580,8 @@ def reducer_output(directory: str, attempt: int, reducer_worker: Any) -> None: current_path = Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix() paths[current_path] = worker["id"] current_results.add(current_path) + if worker["status"] == "succeeded": + required_results.add(current_path) except ValueError: warnings.append("Skipped a worker result outside the scan directory.") @@ -597,6 +609,8 @@ def reducer_output(directory: str, attempt: int, reducer_worker: Any) -> None: except (ContractError, OSError, ValueError) as exc: if (scan_dir / relative).exists(): warnings.append(f"Preserved unreadable checkpoint {relative}: {exc}") + elif relative in required_results and _UNVERIFIED_COVERAGE_WARNING not in warnings: + warnings.append(_UNVERIFIED_COVERAGE_WARNING) if frozen_source_digests is not None: if frozen_source_digests.keys() - source_digests.keys(): raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") @@ -781,7 +795,7 @@ def valid_finding(value: Any) -> bool: superseded = ( worker_id is None and parent is not None - and parent.get("complete") is not False + and parent.get("complete", True) is True and relative != "parent" and (not stopped_parent_seal or relative in parent_preserved_sources) ) or ( @@ -789,20 +803,37 @@ def valid_finding(value: Any) -> bool: and any( saved_worker == worker_id and saved_path in current_results - and current.get("complete") is not False + and current.get("complete", True) is True for saved_path, current, saved_worker in sources ) ) - if ( - (relative != "parent" or not parent_manifest) - and not superseded - and ( - draft.get("complete") is False - or draft["coverage"].get("completeness") != "complete" - ) - and coverage.get("completeness") in {"complete", "unknown"} + if relative in required_results or ( + (relative != "parent" or not parent_manifest) and not superseded ): - coverage["completeness"] = "partial" + source_coverage = draft["coverage"] + source_completeness = source_coverage.get("completeness") + source_complete = draft.get("complete", True) is True + if ( + not source_complete + or ( + source_completeness != "complete" + and ( + worker_id is not None + or relative in required_results + or source_completeness != "partial" + or coverage.get("completeness") == "unknown" + ) + ) + or any( + not isinstance(source_coverage.get(field), list) + for field in ("surfaces", "explicitExclusions", "deferred") + ) + ) and _UNVERIFIED_COVERAGE_WARNING not in warnings: + warnings.append(_UNVERIFIED_COVERAGE_WARNING) + if (not source_complete or source_completeness != "complete") and coverage.get( + "completeness" + ) in {"complete", "unknown"}: + coverage["completeness"] = "partial" if ( superseded and not stopped @@ -1031,7 +1062,11 @@ def valid_finding(value: Any) -> bool: used.add(item["id"]) if field == "surfaces": item.setdefault("receiptRefs", []) - if stopped or any(warning not in initial_warnings for warning in warnings): + if ( + stopped + or _UNVERIFIED_COVERAGE_WARNING in warnings + or any(warning not in initial_warnings for warning in warnings) + ): coverage["completeness"] = "partial" if stopped: if not isinstance(coverage.get("deferred"), list): diff --git a/plugins/codex-security/tests/test_deep_scan_coverage_recovery.py b/plugins/codex-security/tests/test_deep_scan_coverage_recovery.py new file mode 100644 index 000000000..eebf77615 --- /dev/null +++ b/plugins/codex-security/tests/test_deep_scan_coverage_recovery.py @@ -0,0 +1,279 @@ +from __future__ import annotations + +import copy +import json +from pathlib import Path +from types import ModuleType +from typing import Any + +import pytest +from workbench_test_support import write_checkpoint, write_completed_contract + + +@pytest.fixture +def scan_dir(tmp_path: Path) -> Path: + target = tmp_path / "target" + target.mkdir() + scan_dir = tmp_path / "scan" + scan_dir.mkdir() + write_completed_contract(scan_dir, "synthetic-scan", target, coverage_mode="deep_repository") + manifest = json.loads((scan_dir / "scan-manifest.json").read_text()) + manifest["scan"]["complete"] = True + (scan_dir / "scan-manifest.json").write_text(json.dumps(manifest)) + coverage = json.loads((scan_dir / "coverage.json").read_text()) + coverage.update( + completeness="partial", openQuestions=[{"question": "Which deployment controls apply?"}] + ) + (scan_dir / "coverage.json").write_text(json.dumps(coverage)) + return scan_dir + + +def recover_saved_scan( + saved: ModuleType, + scan_dir: Path, + workers: list[dict[str, Any]], + warnings: list[str], +) -> tuple[dict[str, Any], dict[str, Any]]: + scan = json.loads((scan_dir / "scan-manifest.json").read_text())["scan"] + binding = { + "scanId": scan["id"], + "startedAt": scan["startedAt"], + "completedAt": scan["completedAt"], + "producer": scan["producer"], + "allowedTargetKinds": [scan["target"]["kind"]], + "target": scan["target"], + "scope": scan["scope"], + "coverageMode": "deep_repository", + "status": "completed", + } + drafts = saved.merge_saved_results( + scan_dir, scan["id"], binding, workers, warnings, stopped=False, reason="" + ) + prepared = saved._prepare_scan_finalization( + scan_dir, + draft_documents=drafts, + completion_binding=binding, + completion_warnings=warnings, + ) + _, findings, coverage = saved._write_prepared_scan_finalization(prepared) + return findings, coverage + + +@pytest.mark.parametrize( + "case", + [ + pytest.param({"expected": "complete"}, id="reviewed"), + pytest.param( + { + "warnings": ["Recovered finding 1: normalized semantic anchor."], + "expected": "complete", + }, + id="normalized", + ), + pytest.param( + { + "warnings": [ + ( + "Repository HEAD changed while the scan was running; " + "results were saved for the original revision." + ) + ], + "expected": "complete", + }, + id="target-changed", + ), + pytest.param( + {"warnings": ["Recovered finding 1: discarded unverified evidence."]}, + id="unverified-evidence", + ), + pytest.param({"warnings": ["Saved checkpoint could not be read."]}, id="unknown-warning"), + pytest.param({"coverage": {"surfaces": []}}, id="no-surfaces"), + pytest.param( + {"coverage": {"deferred": [{"id": "pending", "reason": "Review is incomplete."}]}}, + id="deferred", + ), + pytest.param({"disposition": "needs_follow_up"}, id="follow-up"), + pytest.param({"receipt": "artifacts/missing-receipt.json"}, id="missing-receipt"), + pytest.param({"coverage": {"explicitExclusions": [None]}}, id="invalid-exclusion"), + pytest.param({"discarded": True}, id="discarded-finding"), + pytest.param({"coverage": {"mode": "repository"}}, id="standard-scan"), + pytest.param( + {"coverage": {"completeness": "unknown"}, "expected": "unknown"}, + id="unknown-completeness", + ), + ], +) +def test_recovers_only_fully_reviewed_deep_coverage(scan_dir: Path, workbench_api, case) -> None: + coverage = json.loads((scan_dir / "coverage.json").read_text()) + coverage.update(case.get("coverage", {})) + if "disposition" in case: + coverage["surfaces"][0]["disposition"] = case["disposition"] + if "receipt" in case: + coverage["surfaces"][0]["receiptRefs"] = [case["receipt"]] + (scan_dir / "coverage.json").write_text(json.dumps(coverage)) + if case.get("discarded"): + findings = json.loads((scan_dir / "findings.json").read_text()) + findings["findings"].append(None) + (scan_dir / "findings.json").write_text(json.dumps(findings)) + warnings = list(case.get("warnings", [])) + prepared = workbench_api["_prepare_scan_finalization"](scan_dir, completion_warnings=warnings) + recovered = prepared[4] + assert recovered["completeness"] == case.get("expected", "partial") + assert recovered["openQuestions"] == [{"question": "Which deployment controls apply?"}] + + +@pytest.mark.parametrize( + "case", + [ + pytest.param({"expected": "complete"}, id="complete-worker"), + pytest.param({"complete": True, "expected": "complete"}, id="explicit-complete"), + *[ + pytest.param({"complete": marker}, id=f"worker-marker-{marker}") + for marker in (False, None, 0, "false") + ], + *[ + pytest.param({"parent_complete": marker}, id=f"parent-marker-{marker}") + for marker in (False, None, 0, "false") + ], + *[ + pytest.param({"worker": status}, id=f"worker-coverage-{status}") + for status in ("unknown", "partial", None, "invalid", []) + ], + *[ + pytest.param({"malformed": field}, id=f"worker-{field}") + for field in ("surfaces", "explicitExclusions", "deferred") + ], + pytest.param({"parent": "complete", "worker": "partial"}, id="complete-parent"), + pytest.param({"parent": "unknown", "worker": "partial"}, id="unknown-parent-partial"), + pytest.param({"parent": "unknown", "expected": "unknown"}, id="unknown-parent"), + pytest.param({"missing": True}, id="missing-successful-worker"), + pytest.param({"kind": "dedup", "missing": True}, id="missing-reducer"), + pytest.param({"kind": "dedup", "expected": "complete"}, id="complete-reducer"), + *[ + pytest.param({"kind": "dedup", "complete": marker}, id=f"reducer-marker-{marker}") + for marker in (False, None, 0, "false") + ], + pytest.param({"kind": "dedup", "worker": "partial"}, id="partial-reducer"), + pytest.param({"kind": "dedup", "malformed": "surfaces"}, id="malformed-reducer"), + pytest.param( + {"status": "failed", "missing": True, "expected": "complete"}, + id="missing-failed-worker", + ), + pytest.param( + {"status": "canceled", "missing": True, "expected": "complete"}, + id="missing-canceled-worker", + ), + ], +) +def test_preserves_source_coverage_and_retry_warnings(scan_dir: Path, workbench_api, case) -> None: + saved = workbench_api["saved_results"] + manifest = json.loads((scan_dir / "scan-manifest.json").read_text()) + if "parent_complete" in case: + manifest["scan"]["complete"] = case["parent_complete"] + (scan_dir / "scan-manifest.json").write_text(json.dumps(manifest)) + coverage = json.loads((scan_dir / "coverage.json").read_text()) + coverage["completeness"] = case.get("parent", "partial") + (scan_dir / "coverage.json").write_text(json.dumps(coverage)) + + output = scan_dir / "worker" + output.mkdir() + result = output / "result.json" + worker = { + "id": "synthetic-worker", + "kind": case.get("kind", "discovery"), + "status": case.get("status", "succeeded"), + "artifact_dir": str(output), + "result_manifest_path": str(result), + "completed_at": "2026-08-29T00:00:00Z", + "attempt": 1, + } + worker_coverage = { + "completeness": case.get("worker", "complete"), + "surfaces": [], + "explicitExclusions": [], + "deferred": [], + } + if "malformed" in case: + worker_coverage[case["malformed"]] = "unverified review" + draft = {"scanId": "synthetic-scan", "findings": [], "coverage": worker_coverage} + if "complete" in case: + draft["complete"] = case["complete"] + if not case.get("missing"): + result.write_text(json.dumps(draft)) + + expected = case.get("expected", "partial") + warnings: list[str] = [] + for _ in range(2): + _, recovered = recover_saved_scan(saved, scan_dir, [worker], warnings) + assert recovered["completeness"] == expected + assert warnings.count(saved._UNVERIFIED_COVERAGE_WARNING) == (expected == "partial") + + +@pytest.mark.parametrize("owner", ["discovery", "parent"]) +@pytest.mark.parametrize("marker", [None, 0, "false", False, True]) +def test_unverified_replacement_preserves_checkpoint_findings( + scan_dir: Path, workbench_api, owner: str, marker +) -> None: + findings = json.loads((scan_dir / "findings.json").read_text()) + checkpoint = { + "scanId": "synthetic-scan", + "complete": False, + "findings": findings["findings"], + "coverage": { + "completeness": "partial", + "surfaces": [], + "explicitExclusions": [], + "deferred": [], + }, + } + findings["findings"] = [] + (scan_dir / "findings.json").write_text(json.dumps(findings)) + workers = [] + output = scan_dir + if owner == "discovery": + output = scan_dir / "worker" + output.mkdir() + result = output / "result.json" + current = copy.deepcopy(checkpoint) + current.update(complete=marker, findings=[]) + current["coverage"]["completeness"] = "complete" + result.write_text(json.dumps(current)) + workers.append( + { + "id": "synthetic-worker", + "kind": "discovery", + "status": "succeeded", + "artifact_dir": str(output), + "result_manifest_path": str(result), + "completed_at": "2026-08-29T00:00:00Z", + "attempt": 1, + } + ) + else: + manifest = json.loads((scan_dir / "scan-manifest.json").read_text()) + manifest["scan"]["complete"] = marker + (scan_dir / "scan-manifest.json").write_text(json.dumps(manifest)) + write_checkpoint(output / "checkpoints", checkpoint) + + recovered, coverage = recover_saved_scan(workbench_api["saved_results"], scan_dir, workers, []) + assert len(recovered["findings"]) == (0 if marker is True else 1) + assert coverage["completeness"] == ("complete" if marker is True else "partial") + + +@pytest.mark.parametrize("severities", [("low", "high"), ("high", "low"), ("high", "high")]) +def test_lossless_duplicates_do_not_prevent_complete_coverage( + scan_dir: Path, workbench_api, severities +) -> None: + findings = json.loads((scan_dir / "findings.json").read_text()) + baseline = findings["findings"][0] + findings["findings"] = [] + for severity in severities: + finding = copy.deepcopy(baseline) + finding["severity"]["level"] = severity + findings["findings"].append(finding) + (scan_dir / "findings.json").write_text(json.dumps(findings)) + prepared = workbench_api["_prepare_scan_finalization"](scan_dir, completion_warnings=[]) + + assert prepared[4]["completeness"] == "complete" + assert len(prepared[3]["findings"]) == 1 + assert prepared[3]["findings"][0]["severity"]["level"] == "high" diff --git a/sdk/typescript/src/version.ts b/sdk/typescript/src/version.ts index 51643cf9b..d85b482b7 100644 --- a/sdk/typescript/src/version.ts +++ b/sdk/typescript/src/version.ts @@ -9,7 +9,7 @@ const PACKAGE_VERSIONS = packageVersions( export const VERSION = PACKAGE_VERSIONS.package; export const CODEX_SDK_VERSION = PACKAGE_VERSIONS.sdk; export const CODEX_EXECUTABLE_VERSION = PACKAGE_VERSIONS.executable; -export const BUNDLED_PLUGIN_VERSION = "0.1.94" as const; +export const BUNDLED_PLUGIN_VERSION = "0.1.96" as const; const PACKAGE_NAME = "@openai/codex-security"; diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.ts index d090bc1d2..f55cbf333 100644 --- a/sdk/typescript/tests-ts/runtime.test.ts +++ b/sdk/typescript/tests-ts/runtime.test.ts @@ -1967,8 +1967,10 @@ describe("plugin runtime preparation", () => { "0.1.59", "0.1.60", "0.1.79", + "0.1.83", "0.1.92", "0.1.93", + "0.1.94", ])( "upgrades a cached %s plugin and restores with the SDK-owned helper", async (previousVersion) => { @@ -1978,6 +1980,10 @@ describe("plugin runtime preparation", () => { join(previous, "scripts", "workbench_scan_history.py"), "print('previous bundled scan history')\n", ); + await writeFile( + join(previous, "scripts", "finalize_scan_contract.py"), + "# stale synthetic validator\n", + ); await writeFile( join(previous, ".mcp.json"), JSON.stringify({ mcpServers: { "codex-security": { env_vars: [] } } }), @@ -2016,6 +2022,12 @@ describe("plugin runtime preparation", () => { const options = { codexCommand: command, environment }; const stale = await bootstrapPlugin(home, previous, options); + expect( + await readFile( + join(stale.installedRoot, "scripts", "finalize_scan_contract.py"), + "utf8", + ), + ).toBe("# stale synthetic validator\n"); const upgraded = await bootstrapPlugin(home, PLUGIN_ROOT, options); expect(stale.version).toBe(previousVersion); @@ -2030,6 +2042,7 @@ describe("plugin runtime preparation", () => { "workbench_target.py", "finalize_scan_contract.py", "workbench_scan_history.py", + "workbench_saved_results.py", ]) { expect(await readFile(join(pluginRoot, "scripts", script))).toEqual( await readFile(join(PLUGIN_ROOT, "scripts", script)),