Repository navigation
Expand file tree
/
Copy pathmain.py
More file actions
6591 lines (6182 loc) · 310 KB
/
Copy pathmain.py
File metadata and controls
6591 lines (6182 loc) · 310 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import json
import sys
import os
import uuid
import hashlib
import dataclasses
import fnmatch
import subprocess
import warnings
from pathlib import Path
from typing import Any, Optional
# Suppress noisy ADK preview/experimental feature notices
warnings.filterwarnings("ignore", message=r".*\[EXPERIMENTAL\].*")
# Keep LiteLLM 100% hermetic before `google.adk` imports `litellm`
os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "True")
os.environ.setdefault("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "True")
os.environ.setdefault("LITELLM_MODE", "PRODUCTION")
from google.genai import types
from google.adk.runners import Runner, RunConfig
from google.adk.agents.run_config import StreamingMode
from google.adk.agents.invocation_context import LlmCallsLimitExceededError
from google.adk.sessions.sqlite_session_service import SqliteSessionService
from google.adk.sessions.base_session_service import BaseSessionService
from google.adk.apps.app import App, ResumabilityConfig
from google.adk.apps.compaction import EventsCompactionConfig
from google.adk.agents.context_cache_config import ContextCacheConfig
# Ensure the `reference/` package root is on sys.path even when Python runs
# under isolated import policy (`PYTHONSAFEPATH=1` / `-P`).
_REF_ROOT = os.path.dirname(os.path.abspath(__file__))
if _REF_ROOT not in sys.path:
sys.path.insert(0, _REF_ROOT)
from core.budget import (
BudgetConfig,
BudgetController,
BudgetExceededError,
CampaignBudgetScope,
should_credit_coverage,
)
from core.database import init_db, read_findings, read_risk_scores, update_status
from core.sandbox import build_sandbox
from core.graph_loader import load_workflow_from_json, DEFAULT_SEED_PROMPT, prefix_reuse_applies
from core.context import RunContext, current_run_context
from core.transcript import TranscriptWriter, event_transcript_rows
from core.paths import resolve_db_path
from core.config import (
ConfigValidationError,
EXIT_AUTH,
MantisAuthError,
ResilientLiteLlm,
format_auth_error_message,
get_run_llm_kwargs,
is_auth_error,
)
from core.config import configure_llm_streaming, llm_streaming_enabled
from core.compactor import MantisEventsSummarizer
from core.llm_gateway import strip_terminal_control
APP_NAME = "mantis_graph"
USER_ID = "user1"
# Event error codes that mean the provider refused the credential. ADK sets
# error_code to the exception's class name (litellm's AuthenticationError,
# google-auth's RefreshError and friends, our own MantisAuthError) or to a
# provider status. Matched exactly; the free-text error_message is never
# consulted for this, because a model can write it.
_AUTH_ERROR_CODES = frozenset({
"MantisAuthError",
"AuthenticationError",
"PermissionDeniedError",
"RefreshError",
"DefaultCredentialsError",
"ReauthError",
"ReauthFailError",
"ReauthSamlChallengeFailError",
"OAuthError",
"UserAccessTokenError",
"UNAUTHENTICATED",
"PERMISSION_DENIED",
})
def cprint(*args, **kwargs) -> None:
"""Console emitter for untrusted content (model text, tool responses, DB rows).
SECURITY: strips terminal escape sequences and control characters so scanned
repository content cannot repaint, reset or relocate the operator's terminal,
or forge trusted-looking pipeline banners in the scroll-back.
"""
cleaned = [strip_terminal_control(a) if isinstance(a, str) else a for a in args]
print(*cleaned, **kwargs)
async def execute_sub_task(
runner: Runner,
session_service: BaseSessionService,
filepath: str,
run_id: str,
db_path: str = "",
status_map: dict[str, str] | None = None,
seed_prompt_template: str = DEFAULT_SEED_PROMPT,
budget_controller: Optional[BudgetController] = None,
slice_briefing: str = "",
focus_directive: str = "",
prior_memory: str = "",
coverage_note: str = "",
hypotheses: str = "",
transcript: Optional["TranscriptWriter"] = None,
session_suffix: str = "",
) -> bool:
"""Executes the workflow graph for a single target file. Returns True if an error was encountered."""
sanitized_filepath = str(filepath).replace("\n", "").replace("\r", "").strip()
target_hash = hashlib.sha256(sanitized_filepath.encode("utf-8")).hexdigest()[:8]
session_id = f"session_run_{run_id}_{target_hash}{session_suffix}"
existing_session = await session_service.get_session(app_name=APP_NAME, user_id=USER_ID, session_id=session_id)
if existing_session is None:
initial_state = {"db_path": db_path, "run_id": run_id, "filepath": sanitized_filepath}
await session_service.create_session(app_name=APP_NAME, user_id=USER_ID, session_id=session_id, state=initial_state)
elif hasattr(existing_session, "state") and isinstance(existing_session.state, dict):
existing_session.state.setdefault("db_path", db_path)
existing_session.state.setdefault("run_id", run_id)
existing_session.state.setdefault("filepath", sanitized_filepath)
# Detect and record VCS metadata into SQLite knowledge base for reporting provenance
if db_path and os.path.exists(os.path.dirname(os.path.abspath(db_path)) or "."):
try:
from tools.research_tools import detect_vcs_info
from core.database import record_artifact
vcs_meta = detect_vcs_info(sanitized_filepath)
vcs_json = json.dumps(vcs_meta, indent=2)
record_artifact(db_path, run_id, "vcs_info", "workspace/.structured/vcs_info.json", vcs_json)
except Exception as e:
print(f"[PROVENANCE WARNING] Failed to record VCS provenance: {e}", file=sys.stderr)
resumed_invocation_id: Optional[str] = None
if existing_session and existing_session.events:
root_agent_name = getattr(getattr(runner, "agent", None), "name", None) or "mantis_vulnerability_pipeline"
for ev in reversed(existing_session.events):
inv_id = getattr(ev, "invocation_id", None)
if inv_id:
has_ended = any(
getattr(e, "invocation_id", None) == inv_id
and getattr(getattr(e, "actions", None), "end_of_agent", False)
and getattr(e, "author", None) in (root_agent_name, "mantis_vulnerability_pipeline")
for e in existing_session.events
)
if not has_ended:
resumed_invocation_id = inv_id
break
# The briefing, memory, leads, coverage note and focus, for the stages
# that decide where to look. The seed message below is read by the first
# node only (each node runs with include_contents="none", and ADK starts
# a node's turn at the previous node's reply), so these are also handed
# to the architect, threat modeler, planner, researchers, reviewer and
# critic through their instructions (core.graph_loader.CONTEXT_STAGES).
# Set on both paths: a resumed invocation sends no seed at all. Same
# order as the seed: data first, operator-authored notes last and
# outside any fence.
_ctx_for_context = current_run_context.get()
if _ctx_for_context is not None:
_ctx_for_context.campaign_context = "".join(
part for part in (slice_briefing, prior_memory, hypotheses, coverage_note, focus_directive) if part
)
if resumed_invocation_id:
# Bookkeeping only (INV-6): a malformed event must cost the resumed
# campaign its coverage/spend reconstruction, never the campaign.
try:
_rehydrate_session_metrics(
existing_session,
runner,
budget_controller=budget_controller,
run_ctx=_ctx_for_context,
)
if (
_ctx_for_context is not None
and int(getattr(_ctx_for_context, "pre_pause_llm_calls", 0) or 0) > 0
and getattr(_SPEND_REPORT.get("controller"), "run_id", None) == run_id
):
_SPEND_REPORT["prior_llm_calls"] = int(_SPEND_REPORT.get("prior_llm_calls") or 0) + int(
_ctx_for_context.pre_pause_llm_calls
)
except Exception as exc:
print(f"[RESUME METRICS WARNING] {filepath}: {exc}", file=sys.stderr)
new_message = None
else:
new_message = _build_seed_message(
seed_prompt_template, sanitized_filepath, run_id,
slice_briefing, prior_memory, hypotheses, coverage_note, focus_directive,
)
print(f"\n[GRAPH EXECUTION] Triggered via: {filepath}")
print("-" * 60)
try:
return await _drive_campaign(
runner, session_id, new_message, resumed_invocation_id,
filepath=filepath, run_id=run_id, db_path=db_path, status_map=status_map,
budget_controller=budget_controller, transcript=transcript,
)
except Exception as exc:
# A campaign interrupted mid-stage resumes inside its old invocation,
# and ADK rebuilds that stage's context from everything the stage did
# before the interruption. When that history no longer fits the
# model's window the resume fails the same way every time, so the
# campaign would never finish under any number of --resume passes.
# Start it over instead: fresh session, the seed message, the full
# graph. Once only -- the second attempt is not a resume.
if not (resumed_invocation_id and _is_context_overflow(exc)):
raise
print(
f"\n[RESUME] {filepath}: the interrupted campaign's history no longer fits "
f"the model window ({type(exc).__name__}); starting the campaign over.",
file=sys.stderr,
)
if budget_controller is not None:
budget_controller.observe("overflow_restarts")
if _ctx_for_context is not None:
# pre_pause_tokens and resumed_mid_invocation stay as rehydrated:
# the pre-pause slices were real spend for this campaign, so the
# spend row records the full cost, and a campaign that was never
# flagged resumed is not flagged by the restart.
if hasattr(_ctx_for_context, "node_context"):
_ctx_for_context.node_context.clear()
restart_handoff = _prepare_overflow_restart_handoff(db_path, run_id, filepath)
if _ctx_for_context is not None and restart_handoff:
_ctx_for_context.campaign_context = "".join(
part
for part in (
slice_briefing,
restart_handoff,
prior_memory,
hypotheses,
coverage_note,
focus_directive,
)
if part
)
await session_service.delete_session(app_name=APP_NAME, user_id=USER_ID, session_id=session_id)
await session_service.create_session(
app_name=APP_NAME, user_id=USER_ID, session_id=session_id,
state={"db_path": db_path, "run_id": run_id, "filepath": sanitized_filepath},
)
if restart_handoff:
slice_briefing = slice_briefing + restart_handoff
fresh = _build_seed_message(
seed_prompt_template, sanitized_filepath, run_id,
slice_briefing, prior_memory, hypotheses, coverage_note, focus_directive,
)
return await _drive_campaign(
runner, session_id, fresh, None,
filepath=filepath, run_id=run_id, db_path=db_path, status_map=status_map,
budget_controller=budget_controller, transcript=transcript,
)
def _rehydrate_session_metrics(
existing_session: Any,
runner: Any,
budget_controller: Optional[BudgetController] = None,
run_ctx: Optional[RunContext] = None,
) -> None:
"""Reconstructs pre-pause `audit_code_reads` and `pre_pause_tokens` across
all prior slices of an unfinished session so multi-slice `--resume` runs
earn coverage credit when an earlier slice read target code and record the
full campaign token total on completion without billing prior slices against
the resumed process's fresh token allowance."""
if run_ctx is not None:
run_ctx.resumed_mid_invocation = True
run_ctx.pre_pause_tokens = 0
run_ctx.pre_pause_llm_calls = 0
events = list(getattr(existing_session, "events", None) or [])
if not events:
return
from core.budget import AUDIT_NODES, CODE_READ_TOOLS
root_agent_name = getattr(getattr(runner, "agent", None), "name", None) or "mantis_vulnerability_pipeline"
start_idx = 0
for idx, ev in enumerate(events):
if (
getattr(getattr(ev, "actions", None), "end_of_agent", False)
and getattr(ev, "author", None) in (root_agent_name, "mantis_vulnerability_pipeline")
):
start_idx = idx + 1
active_events = events[start_idx:]
if not active_events:
return
aborted_fc_ids: set[str] = set()
for ev in active_events:
if getattr(ev, "partial", False):
continue
for part in getattr(getattr(ev, "content", None), "parts", None) or []:
fn_resp = getattr(part, "function_response", None)
if fn_resp is None:
continue
fr_id = str(getattr(fn_resp, "id", "") or "")
raw_resp = getattr(fn_resp, "response", None)
if fr_id and isinstance(raw_resp, dict):
err_txt = str(raw_resp.get("error") or "")
if "Invocation was aborted" in err_txt or "Interrupted before tool completed" in err_txt:
aborted_fc_ids.add(fr_id)
discount = budget_controller.config.cache_discount if budget_controller is not None else 0.1
prior_reads = 0
prior_tokens = 0
prior_llm_calls = 0
prior_reviewer_fallbacks = 0
seen_nodes_order: list[str] = []
seen_nodes_set: set[str] = set()
for ev in active_events:
if getattr(ev, "partial", False):
continue
usage = getattr(ev, "usage_metadata", None)
if usage and getattr(usage, "total_token_count", None):
total_t = max(0, int(usage.total_token_count or 0))
if total_t > 0:
prior_llm_calls += 1
cached_t = max(0, min(int(getattr(usage, "cached_content_token_count", 0) or 0), total_t))
prior_tokens += max(0, int((total_t - cached_t) + cached_t * discount))
node_info = getattr(ev, "node_info", None)
npath = str(getattr(node_info, "path", "") or "") if node_info is not None else ""
clean_node = (
npath.rsplit("/", 1)[-1].split("@", 1)[0]
if npath
else str(getattr(ev, "author", "") or "")
).split(":", 1)[0].strip()
if clean_node and not getattr(ev, "error_code", None) and clean_node not in seen_nodes_set:
seen_nodes_set.add(clean_node)
seen_nodes_order.append(clean_node)
if clean_node == "reviewer" or (clean_node.startswith("reviewer_") and "classifier" not in clean_node):
ev_parts = getattr(getattr(ev, "content", None), "parts", None) or []
ev_text = "".join(
getattr(p, "text", "") or ""
for p in ev_parts
if not getattr(p, "thought", False)
).strip()
if ev_text.startswith("{"):
try:
ev_obj = json.loads(ev_text)
except (TypeError, ValueError):
ev_obj = None
if (
isinstance(ev_obj, dict)
and str(ev_obj.get("route") or "").strip().lower() == "false_positive"
and str(ev_obj.get("reason") or "").startswith("Fallback:")
):
prior_reviewer_fallbacks += 1
for part in getattr(getattr(ev, "content", None), "parts", None) or []:
fc = getattr(part, "function_call", None)
if fc is None:
continue
fc_id = str(getattr(fc, "id", "") or "")
if fc_id and fc_id in aborted_fc_ids:
continue
call_name = str(getattr(fc, "name", "") or "")
call_args = getattr(fc, "args", None) or {}
if (
clean_node in AUDIT_NODES
and call_name in CODE_READ_TOOLS
and _is_in_scope_code_read(run_ctx, call_name, call_args)
):
prior_reads += 1
# On shared-prefix (`prefix_group`) workflows, `researcher` relies on files
# already read into the inherited stem by upstream stages (e.g. `planner`,
# `architect`, `threat_modeler`) and does not re-read them. An unbroken run
# credits those inherited in-scope reads on `researcher` Turn 1 inside
# `graph_loader.fork_before_model`; when a campaign pauses after `researcher`
# starts (e.g. during `critic`) and resumes, reconstruct that same credit so
# `[COVERAGE GUARD]` does not falsely withhold coverage.
raw_state = getattr(existing_session, "state", None)
if hasattr(raw_state, "to_dict"):
try:
state_dict = raw_state.to_dict()
except Exception:
state_dict = {}
elif isinstance(raw_state, dict):
state_dict = raw_state
else:
state_dict = {}
if state_dict and seen_nodes_order:
from core.graph_loader import (
PREFIX_U_STATE_KEY_PREFIX,
_count_inherited_in_scope_code_reads,
forked_prefix_contents,
)
def _has_prefix_opener(node_name: str) -> bool:
val = state_dict.get(f"{PREFIX_U_STATE_KEY_PREFIX}{node_name}")
return isinstance(val, str) and bool(val)
resumed_prefix_researchers = [
n
for n in seen_nodes_order
if (n == "researcher" or n.startswith("researcher_")) and _has_prefix_opener(n)
]
if resumed_prefix_researchers:
last_researcher_idx = max(
seen_nodes_order.index(rn) for rn in resumed_prefix_researchers
)
stem_ancestors = [
n
for n in seen_nodes_order[:last_researcher_idx]
if n not in AUDIT_NODES
and not (n == "researcher" or n.startswith("researcher_"))
and _has_prefix_opener(n)
]
if stem_ancestors:
stem_contents = forked_prefix_contents(
active_events,
stem_ancestors,
state=state_dict,
preserve_signatures=False,
)
prior_reads += _count_inherited_in_scope_code_reads(run_ctx, stem_contents)
if run_ctx is not None and isinstance(getattr(run_ctx, "node_context", None), dict):
for rn in resumed_prefix_researchers:
slot = run_ctx.node_context.setdefault(
rn, {"tokens": 0, "limit": 0, "continuations": 0, "handoff": ""}
)
slot["credited_inherited_reads"] = True
if budget_controller is not None and prior_reads > 0:
budget_controller.audit_code_reads += prior_reads
if run_ctx is not None:
run_ctx.pre_pause_tokens = prior_tokens
run_ctx.pre_pause_llm_calls = prior_llm_calls
if prior_reviewer_fallbacks > 0:
run_ctx.reviewer_verdict_fallbacks = max(
int(getattr(run_ctx, "reviewer_verdict_fallbacks", 0) or 0),
prior_reviewer_fallbacks,
)
def _rehydrate_campaign_resume_state(
branch_ctx: RunContext,
db_path: str,
run_id: str,
scan_item: str,
pass_idx: int = 1,
pass_resource: str = "",
) -> None:
"""Restores persisted per-finding dynamic sink evidence (`reproduced` flags)
and reviewer/critic `promoted_finding_ids` for an interrupted campaign
resuming under `run_id`."""
if not db_path or not run_id:
return
try:
from core.database import CONFIRMED_LIVE_STATUSES, _artifact_resource, read_artifacts_like, read_findings
want_res = str(pass_resource or scan_item or "")
ev_rows = read_artifacts_like(
db_path, run_id, r"workspace/.structured/evidence\_%.json", artifact_type="finding_evidence"
)
for row in ev_rows:
owner = _artifact_resource(row.get("metadata_json"))
if want_res and owner not in (want_res, ""):
continue
try:
payload = json.loads(row.get("content") or "{}")
except (TypeError, ValueError):
continue
if not isinstance(payload, dict) or not payload.get("reproduced"):
continue
fid = payload.get("finding_id")
if fid is None:
continue
key = str(int(fid))
branch_ctx.finding_evidence[key] = {
"reproduced": True,
"reattacked": False,
"reattack_present": False,
"sink_symbol": str(payload.get("sink_symbol") or ""),
"reason": str(payload.get("reason") or ""),
}
branch_ctx.sandbox_executed = True
# The cohort graph_loader._persist_finding_promotions rebuilds after a
# campaign-level reviewer/critic verdict, so a resumed critic narrows
# to it.
confirmed_live = set(CONFIRMED_LIVE_STATUSES)
existing_rows = read_findings(
db_path,
run_id=run_id,
scope_path=scan_item or None,
campaign=scan_item or None,
pass_index=max(1, int(pass_idx or 1)),
)
promoted: set[int] = set()
for r in existing_rows:
rid_val = r.get("id")
if rid_val is None:
continue
fid_int = int(rid_val)
st = str(r.get("status") or "").strip().lower()
if st in confirmed_live:
promoted.add(fid_int)
if st in ("dynamic_confirmed", "patch_verified"):
slot = branch_ctx.finding_evidence.setdefault(
str(fid_int),
{
"reproduced": True,
"reattacked": False,
"reattack_present": False,
"sink_symbol": "",
"reason": "rehydrated from finding status",
},
)
slot["reproduced"] = True
branch_ctx.sandbox_executed = True
if promoted:
branch_ctx.promoted_finding_ids = promoted
# Restore durable reviewer-fallback count recorded for THIS attempt of
# the campaign so a pause/Ctrl-C/SIGTERM between reviewer fallback and
# campaign end cannot zero out the counter on `--resume`. Keyed by
# `(resource, pass_idx, fallback_attempt)` so Attempt 1's failed record
# never bleeds into a clean Attempt 2 retry.
try:
from core.config import reviewer_fallback_artifact_path
from core.database import read_artifact_record
cur_attempt = max(1, int(getattr(branch_ctx, "fallback_attempt", 1) or 1))
fb_path = reviewer_fallback_artifact_path(
want_res, pass_index=max(1, int(pass_idx or 1)), attempt=cur_attempt
)
fb_rec = read_artifact_record(
db_path,
filepath=fb_path,
artifact_type="reviewer_fallback",
run_id=run_id,
resource=None,
)
if fb_rec is not None:
fb_payload = json.loads(fb_rec.get("content") or "{}")
if isinstance(fb_payload, dict):
fb_count = int(fb_payload.get("count", 0) or 0)
if fb_count > 0:
branch_ctx.reviewer_verdict_fallbacks = max(
int(getattr(branch_ctx, "reviewer_verdict_fallbacks", 0) or 0),
fb_count,
)
except Exception:
pass
except Exception:
pass
DEFAULT_MAX_REVIEWER_FALLBACK_ATTEMPTS = 2
def _max_reviewer_fallback_attempts() -> int:
raw = str(os.environ.get("MANTIS_MAX_REVIEWER_FALLBACK_ATTEMPTS", "") or "").strip()
if raw:
try:
return max(1, int(raw))
except (TypeError, ValueError):
pass
return DEFAULT_MAX_REVIEWER_FALLBACK_ATTEMPTS
def _evaluate_reviewer_fallback_outcome(
*,
task_failed: bool,
rev_fallbacks: int,
unreviewed_reported: int,
prior_fallback_attempts: int,
max_attempts: int,
) -> dict[str, Any]:
"""Determines whether a campaign with synthesized reviewer fallback verdicts
should fail for `--resume` retry or settle into a terminal `review_incomplete`
end state (preventing infinite `--resume` loops while keeping findings
`'reported'` and withholding coverage credit)."""
if task_failed or rev_fallbacks <= 0 or unreviewed_reported <= 0:
return {
"triggered": False,
"task_failed": bool(task_failed),
"terminal": False,
"attempt_num": 0,
"max_attempts": max(1, int(max_attempts)),
}
cap = max(1, int(max_attempts))
attempt_num = max(0, int(prior_fallback_attempts)) + 1
terminal = attempt_num >= cap
return {
"triggered": True,
"task_failed": not terminal,
"terminal": terminal,
"attempt_num": attempt_num,
"max_attempts": cap,
}
def _build_operator_steering_blocks(focus: str, seed_report: str) -> tuple[str, str]:
"""Builds `(operator_focus_block, seed_report_block)` for per-campaign
`HUNTING_STAGES` injection across all scan modes (`whole`, `file-by-file`,
`cross-functional`, `changed`).
Preserves the planner's strict trust split (`INV-2`, CP-4):
- `operator_focus_block`: sanitized unfenced operator directive.
- `seed_report_block`: sanitized CP-4 fenced untrusted bug-report evidence.
"""
from core.planner import (
_render_focus_section,
_render_seed_section,
_sanitize_focus,
_sanitize_seed,
)
clean_focus = _sanitize_focus(focus)
clean_seed = _sanitize_seed(seed_report)[:3000].strip()
focus_block = _render_focus_section(clean_focus) if clean_focus else ""
seed_block = _render_seed_section(clean_seed) if clean_seed else ""
return focus_block, seed_block
def _prepare_overflow_restart_handoff(db_path: str, run_id: str, filepath: str) -> str:
"""Clears exhausted continuation handoffs and renders a bounded, CP-4-fenced
summary of artifacts already saved for `(run_id, filepath)` before the
resumed invocation overflowed."""
if not db_path or not run_id or not os.path.exists(db_path):
return ""
try:
from core.database import read_artifact, read_findings, reset_campaign_handoffs
from core.llm_gateway import wrap_untrusted_content
reset_campaign_handoffs(db_path, run_id, resource=filepath or "")
sections: list[str] = []
for label, kind in (
("Saved Architecture Summary", "summary"),
("Saved Threat Model", "threat_model"),
("Saved Review Plan", "plan"),
):
raw = read_artifact(db_path, artifact_type=kind, run_id=run_id, resource=filepath or None)
if raw and str(raw).strip():
snippet = str(raw).strip()[:1200]
sections.append(f"### {label}\n{snippet}")
saved_findings = read_findings(
db_path, run_id=run_id, scope_path=filepath or None, campaign=filepath or None
)
if saved_findings:
lines = [
f"- #{f.get('id')}: [{f.get('severity')}] {f.get('filepath')} — "
f"{str(f.get('title') or '')[:120]} (status={f.get('status')})"
for f in saved_findings[:15]
]
sections.append("### Already-Recorded Campaign Findings\n" + "\n".join(lines))
if not sections:
return ""
body = (
"Prior interrupted attempt on this campaign persisted the following "
"artifacts in the database before context restart (avoid duplicating "
"findings already recorded below):\n\n" + "\n\n".join(sections)
)
return "\n\n" + wrap_untrusted_content(body, filename="overflow_restart_handoff")
except Exception:
return ""
# Provider wording for "this request does not fit the model's window". Used
# only to decide whether a RESUMED campaign starts over, so a false match
# costs one fresh run of that campaign and nothing else.
_CONTEXT_OVERFLOW_MARKERS = (
"prompt is too long",
"context length",
"context_length_exceeded",
"contextwindowexceeded",
"maximum context",
"exceeds the maximum number of tokens",
"input token count",
"max_model_len",
"too many tokens",
"token limit",
"request too large",
)
# Exception class-name tokens emitted by provider SDKs and LiteLLM when an
# HTTP 400/413/422 context-window rejection occurs. Generic Python classes
# (RuntimeError, ValueError, Exception) are deliberately excluded so target
# code or tool strings cannot trigger a restart by text alone.
_PROVIDER_OVERFLOW_CLASS_TOKENS = (
"badrequest",
"invalidrequest",
"invalidargument",
"apierror",
"openaierror",
"clienterror",
"unprocessableentity",
"httperror",
"litellm",
)
def _single_exc_is_overflow(exc: BaseException) -> bool:
name = type(exc).__name__.lower()
text = f"{name} {exc}".lower()
# Mantis's own pre-dispatch refusal, node context ceiling exhaustion, and
# litellm's typed overflow: deterministic in the request, by class.
if (
"contextbudgetexceeded" in name
or "contextwindowexceeded" in name
or "nodecontextexhausted" in name
):
return True
return any(tok in name for tok in _PROVIDER_OVERFLOW_CLASS_TOKENS) and any(
m in text for m in _CONTEXT_OVERFLOW_MARKERS
)
def _is_context_overflow(exc: BaseException) -> bool:
seen: set[int] = set()
cur: Optional[BaseException] = exc
while cur is not None and id(cur) not in seen:
seen.add(id(cur))
if _single_exc_is_overflow(cur):
return True
cur = getattr(cur, "__cause__", None) or getattr(cur, "__context__", None)
return False
def _build_seed_message(
seed_prompt_template: str,
sanitized_filepath: str,
run_id: str,
slice_briefing: str,
prior_memory: str,
hypotheses: str,
coverage_note: str,
focus_directive: str,
) -> types.Content:
"""The first node's seed: the operator template with the target substituted,
then the data-side context, then the operator-authored notes."""
# SECURITY: literal substitution, not str.format(). A template containing a
# format spec such as "{filepath:>9999999999}" would otherwise be evaluated
# here, and conversion/attribute syntax would traverse object internals.
query_text = (
str(seed_prompt_template)
.replace("{filepath}", str(sanitized_filepath))
.replace("{run_id}", str(run_id))
)
# Appended AFTER substitution, deliberately. The briefing is built from
# repository bytes, and a directory named "{filepath}" would otherwise be
# substituted into rather than merely quoted. Concatenating afterwards means
# repo content is never on the template side of an expansion.
if slice_briefing:
query_text += slice_briefing
# Prior-run evidence. Like the briefing it is LLM-written text carrying its own
# CP-4 fencing, so it sits with the briefing on the data side of the prompt,
# ahead of the operator instruction below.
if prior_memory:
query_text += prior_memory
# Cross-area leads. Derived mechanically from earlier findings' symbols and
# weakness classes, but the subject matter still originates in LLM output, so
# this carries its own CP-4 fencing and belongs on the data side too.
if hypotheses:
query_text += hypotheses
# The focus directive and the coverage note are operator-authored text chosen by
# a repository-derived key, so unlike the briefing they are not fenced as
# untrusted. They go LAST so they are not enclosed by the briefing's
# untrusted-data delimiters: inside them they would read as content the agent has
# been told to distrust, which is the opposite of an instruction. Same
# literal-concatenation rule as above.
if coverage_note:
query_text += coverage_note
if focus_directive:
query_text += focus_directive
return types.Content(
parts=[types.Part.from_text(text=query_text)],
role="user"
)
def _is_in_scope_code_read(
ctx: Optional[RunContext], tool_name: str, call_args: Any
) -> bool:
"""True when `tool_name(call_args)` reads source code inside the campaign's
assigned target scope (never `workspace/*` artifacts or unrelated sibling
subtrees on a directory-slice campaign)."""
from core.budget import CODE_READ_TOOLS
if tool_name not in CODE_READ_TOOLS:
return True
if not isinstance(call_args, dict):
return False
raw_fp = str(call_args.get("filepath") or "").strip()
if not raw_fp:
return False
clean_fp = raw_fp.replace("\\", "/")
while clean_fp.startswith("./") or clean_fp.startswith("/"):
clean_fp = clean_fp[2:] if clean_fp.startswith("./") else clean_fp[1:]
if (
clean_fp in ("workspace", "findings", "mantis-summary.md", "learnings.jsonl")
or clean_fp.startswith("workspace/")
or clean_fp.startswith("findings/")
):
return False
if ctx is None or not ctx.jail_dir or not ctx.target_file:
return True
try:
jail_real = os.path.realpath(ctx.jail_dir).rstrip("/")
if (
os.path.isfile(ctx.target_file)
and len(getattr(ctx, "group_targets", ()) or ()) <= 1
and getattr(ctx, "scan_mode", "") != SCAN_MODE_CROSS_FUNCTIONAL
):
return True
scope_roots = [
os.path.realpath(str(p)).rstrip("/")
for p in (ctx.target_file, *getattr(ctx, "group_targets", ()))
if p
]
if not scope_roots or any(sr == jail_real for sr in scope_roots):
return True
sb = getattr(ctx, "sandbox", None)
workdir = str(getattr(sb, "working_dir", "") or "/workspace").replace("\\", "/").rstrip("/") or "/workspace"
rel_cand = clean_fp
if rel_cand == workdir or rel_cand.startswith(workdir + "/"):
rel_cand = rel_cand[len(workdir):].lstrip("/")
candidates = [os.path.realpath(os.path.join(jail_real, rel_cand)).rstrip("/")]
path_root = str(getattr(ctx, "path_root", "") or "").strip()
if path_root and os.path.isdir(path_root):
candidates.append(os.path.realpath(os.path.join(path_root, rel_cand)).rstrip("/"))
return any(
cand == sr or cand.startswith(sr + os.sep)
for cand in candidates
for sr in scope_roots
)
except OSError:
return True
def _render_slice_scope_directive(
scan_item: str, group: dict, jail_dir: str
) -> str:
"""Operator-authored scope directive anchoring a directory-slice or
multi-file depth-batch campaign to its assigned targets."""
if not scan_item or not jail_dir:
return ""
try:
jail_real = os.path.realpath(str(jail_dir)).rstrip("/")
targets = [str(t) for t in (group.get("targets") or [scan_item]) if t]
dir_rels = []
file_rels = []
for t in targets:
t_real = os.path.realpath(t).rstrip("/")
if t_real == jail_real:
continue
if not t_real.startswith(jail_real + os.sep):
continue
rel = os.path.relpath(t_real, jail_real).replace(os.sep, "/")
if os.path.isdir(t_real):
dir_rels.append(rel)
elif os.path.isfile(t_real):
file_rels.append(rel)
if dir_rels:
shown = ", ".join(f"'{r}/'" for r in dir_rels[:6])
primary = dir_rels[0]
return (
f"\nAssigned Campaign Target Scope: {shown}. "
f"Focus your enumeration, planning, and source reads on {shown} first "
f"(list_files('') defaults to '{primary}/'). You may inspect callers or "
"callees outside this slice to verify cross-boundary dataflow, but "
f"coverage credit requires auditing source files inside {shown}."
)
if len(file_rels) > 1:
shown_files = file_rels[:40]
bullet_list = "\n".join(f" - {r}" for r in shown_files)
extra = (
f"\n - ... ({len(file_rels) - len(shown_files)} additional assigned file(s); call list_files('') for all)"
if len(file_rels) > len(shown_files)
else ""
)
return (
f"\nAssigned Campaign File Batch ({len(file_rels)} file(s)):\n"
f"{bullet_list}{extra}\n"
"Focus your planning, exploration, and vulnerability analysis on these "
"assigned batch files first (list_files('') lists this batch). You may "
"read helpers, middleware, sanitizers, or RTL submodules outside the batch "
"to trace cross-file dataflow, and report findings across all assigned files."
)
return ""
except OSError:
return ""
async def _drive_campaign(
runner: Runner,
session_id: str,
new_message: Optional[types.Content],
resumed_invocation_id: Optional[str],
*,
filepath: str,
run_id: str,
db_path: str,
status_map: dict[str, str] | None,
budget_controller: Optional[BudgetController],
transcript: Optional["TranscriptWriter"],
) -> bool:
"""Streams one campaign's graph run and applies the per-event bookkeeping.
Returns True if an error was encountered."""
pending_node_errors: dict[str, str] = {}
fatal_tool_errors: set[str] = set()
stamped_nodes: set[str] = set()
pending_in_scope_reads: dict[str, str] = {}
pending_in_scope_by_name: dict[str, list[str]] = {}
last_banner: tuple[str | None, str | None] = (None, None)
current_active_node: str | None = None
streamed_partial_text: bool = False
ctx_init = current_run_context.get()
is_parallel = bool(ctx_init and int(getattr(ctx_init, "parallel_workers", 1) or 1) > 1)
camp_tag = ""
if filepath:
jail_for_tag = str(getattr(ctx_init, "jail_dir", "") or "") if ctx_init else ""
try:
if jail_for_tag and os.path.realpath(filepath).startswith(os.path.realpath(jail_for_tag) + os.sep):
camp_tag = os.path.relpath(os.path.realpath(filepath), os.path.realpath(jail_for_tag)).replace(os.sep, "/")
else:
camp_tag = os.path.basename(filepath.rstrip("/")) or filepath
except Exception:
camp_tag = os.path.basename(filepath.rstrip("/")) or filepath
line_buf: list[str] = []
def _flush_parallel_lines(*, force: bool = False) -> None:
if not is_parallel or not line_buf:
return
joined = "".join(line_buf)
line_buf.clear()
if force:
for ln in joined.splitlines():
if ln.strip():
cprint(f"[{camp_tag}] {ln}", flush=True)
return
if "\n" not in joined:
line_buf.append(joined)
return
parts = joined.split("\n")
for ln in parts[:-1]:
if ln.strip():
cprint(f"[{camp_tag}] {ln}", flush=True)
if parts[-1]:
line_buf.append(parts[-1])
max_calls = 0
if budget_controller and budget_controller.config.max_llm_calls > 0:
max_calls = budget_controller.config.max_llm_calls
run_cfg = RunConfig(
max_llm_calls=max_calls,
streaming_mode=StreamingMode.SSE if llm_streaming_enabled() else StreamingMode.NONE,
)
try:
async for event in runner.run_async(
user_id=USER_ID,
session_id=session_id,
invocation_id=resumed_invocation_id,
new_message=new_message,
run_config=run_cfg,
):
node_path = getattr(getattr(event, "node_info", None), "path", None)
route = getattr(getattr(event, "actions", None), "route", None)
if transcript is not None:
transcript.write(event_transcript_rows(
event,
node=(node_path.split("/")[-1].split("@")[0] if node_path else (current_active_node or "")),
campaign=camp_tag if is_parallel else "",
))
if node_path:
node_name = node_path.split("/")[-1].split("@")[0]
if node_name != current_active_node:
_flush_parallel_lines(force=True)
current_active_node = node_name
streamed_partial_text = False
ctx = current_run_context.get()
if ctx:
ctx.active_node = node_name
if budget_controller:
budget_controller.record_step(node_name)