From ea7d782946806b0a908b84977202fb8e26c839f8 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Fri, 6 Nov 2026 15:38:45 +0800 Subject: [PATCH 01/25] ci(oceanbase): bound startup and balance migration workers --- .../setup-upgrade-oceanbase/action.yml | 105 +++---- .github/ci-paths.toml | 6 + .github/oceanbase-migration-durations.json | 67 +++++ .github/workflows/e2e-harness.yml | 284 ++++++------------ .../check_atomic_memory_migration_shards.py | 13 +- scripts/oceanbase_ci.py | 233 ++++++++++++++ scripts/run_atomic_memory_clean_migration.py | 77 ++++- tests/test_ci_changes.py | 5 + tests/test_oceanbase_ci.py | 183 +++++++++++ 9 files changed, 694 insertions(+), 279 deletions(-) create mode 100644 .github/oceanbase-migration-durations.json create mode 100644 scripts/oceanbase_ci.py create mode 100644 tests/test_oceanbase_ci.py diff --git a/.github/actions/setup-upgrade-oceanbase/action.yml b/.github/actions/setup-upgrade-oceanbase/action.yml index 61a110c2ea..160e9f9d1a 100644 --- a/.github/actions/setup-upgrade-oceanbase/action.yml +++ b/.github/actions/setup-upgrade-oceanbase/action.yml @@ -8,75 +8,48 @@ runs: env: OCEANBASE_IMAGE: ghcr.io/oceanbase/oceanbase-ce:4.3.5.6-106000012026040916 run: | - docker run --detach --name upgrade-oceanbase \ + mkdir -p "$RUNNER_TEMP/oceanbase-startup" + timeout --kill-after=10s 180s docker run --detach --name upgrade-oceanbase \ --publish 127.0.0.1:2881:2881 \ --ulimit nofile=65536:65536 \ --env MODE=slim \ --env OB_DATABASE=pc_upgrade_test \ --env OB_TENANT_PASSWORD=powercontext-e2e \ "$OCEANBASE_IMAGE" - for attempt in $(seq 1 120); do - if docker exec upgrade-oceanbase \ - obclient -h127.0.0.1 -P2881 -uroot@test -ppowercontext-e2e \ - -Dpc_upgrade_test -e 'SELECT 1' >/dev/null 2>&1; then - # The slim image's default 1 GB maximum is too small for the - # indexes created across the clean migration acceptance cases. - docker exec upgrade-oceanbase \ - obclient -h127.0.0.1 -P2881 -uroot@sys \ - -e "ALTER SYSTEM SET datafile_maxsize='8G'; ALTER SYSTEM SET datafile_next='512M'; ALTER SYSTEM SET datafile_size='4G';" - datafile_ready=false - for config_attempt in $(seq 1 30); do - docker exec upgrade-oceanbase \ - obclient -h127.0.0.1 -P2881 -uroot@sys --batch --skip-column-names \ - -e "SHOW PARAMETERS WHERE name IN ('datafile_size', 'datafile_maxsize', 'datafile_next');" \ - > "$RUNNER_TEMP/upgrade-datafile.tsv" - if awk ' - $5 == "datafile_size" && $7 == "4G" { size = 1 } - $5 == "datafile_maxsize" && $7 == "8G" { maximum = 1 } - $5 == "datafile_next" && $7 == "512M" { increment = 1 } - END { exit !(size && maximum && increment) } - ' "$RUNNER_TEMP/upgrade-datafile.tsv"; then - datafile_ready=true - break - fi - sleep 2 - done - cat "$RUNNER_TEMP/upgrade-datafile.tsv" - if [ "$datafile_ready" != true ]; then - echo 'OceanBase datafile configuration did not become effective within one minute.' >&2 - exit 1 - fi - docker exec upgrade-oceanbase \ - obclient -h127.0.0.1 -P2881 -uroot@test -ppowercontext-e2e \ - -Dpc_upgrade_test \ - -e " - CREATE DATABASE pc_upgrade_fresh; - CREATE DATABASE pc_upgrade_code_test; - CREATE DATABASE pc_upgrade_code_fresh; - CREATE DATABASE pc_upgrade_vector_test; - CREATE DATABASE pc_upgrade_vector_fresh; - CREATE DATABASE pc_upgrade_vector_code_test; - CREATE DATABASE pc_upgrade_vector_code_fresh; - CREATE DATABASE pc_upgrade_140_test; - CREATE DATABASE pc_upgrade_140_fresh; - CREATE DATABASE pc_upgrade_140_code_test; - CREATE DATABASE pc_upgrade_140_code_fresh; - CREATE DATABASE pc_upgrade_140_vector_test; - CREATE DATABASE pc_upgrade_140_vector_fresh; - CREATE DATABASE pc_upgrade_140_vector_code_test; - CREATE DATABASE pc_upgrade_140_vector_code_fresh; - SET GLOBAL recyclebin=OFF; - SELECT VERSION(); - SHOW VARIABLES LIKE 'ob_compatibility_mode'; - SHOW VARIABLES LIKE 'recyclebin'; - " - exit 0 - fi - if [ "$(docker inspect --format '{{.State.Running}}' upgrade-oceanbase)" != true ]; then - echo 'OceanBase exited before becoming ready.' >&2 - exit 1 - fi - sleep 5 - done - echo 'OceanBase did not become ready within 10 minutes.' >&2 - exit 1 + python scripts/oceanbase_ci.py wait --container upgrade-oceanbase --database pc_upgrade_test \ + --output "$RUNNER_TEMP/oceanbase-startup" + timeout --kill-after=5s 60s docker exec upgrade-oceanbase \ + timeout --kill-after=1s 55s obclient -h127.0.0.1 -P2881 -uroot@test -ppowercontext-e2e \ + --connect-timeout=5 -Dpc_upgrade_test -e " + CREATE DATABASE pc_upgrade_fresh; + CREATE DATABASE pc_upgrade_code_test; + CREATE DATABASE pc_upgrade_code_fresh; + CREATE DATABASE pc_upgrade_vector_test; + CREATE DATABASE pc_upgrade_vector_fresh; + CREATE DATABASE pc_upgrade_vector_code_test; + CREATE DATABASE pc_upgrade_vector_code_fresh; + CREATE DATABASE pc_upgrade_140_test; + CREATE DATABASE pc_upgrade_140_fresh; + CREATE DATABASE pc_upgrade_140_code_test; + CREATE DATABASE pc_upgrade_140_code_fresh; + CREATE DATABASE pc_upgrade_140_vector_test; + CREATE DATABASE pc_upgrade_140_vector_fresh; + CREATE DATABASE pc_upgrade_140_vector_code_test; + CREATE DATABASE pc_upgrade_140_vector_code_fresh; + " + - name: Collect failed startup diagnostics + if: failure() + shell: bash + run: | + python scripts/oceanbase_ci.py collect --container upgrade-oceanbase \ + --output "$RUNNER_TEMP/oceanbase-startup/failure" + timeout 15s docker logs --tail 2000 upgrade-oceanbase \ + > "$RUNNER_TEMP/oceanbase-startup/console.log" 2>&1 || true + - name: Save failed startup diagnostics + if: failure() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: oceanbase-startup-${{ github.job }}-${{ strategy.job-index || 0 }} + path: ${{ runner.temp }}/oceanbase-startup/ + if-no-files-found: warn + retention-days: 7 diff --git a/.github/ci-paths.toml b/.github/ci-paths.toml index 65ed21856d..c291b059e5 100644 --- a/.github/ci-paths.toml +++ b/.github/ci-paths.toml @@ -28,6 +28,7 @@ tests = [ "scripts/sync_dsh_plugin_tree.py", "scripts/generate_integration_manifest_docs.py", "scripts/run_atomic_memory_clean_migration.py", "scripts/scan_e2e_evidence.sh", "scripts/check_atomic_memory_migration_shards.py", + "scripts/oceanbase_ci.py", ".github/oceanbase-migration-durations.json", "scripts/ci_role_smoke.py", "scripts/ci_installation_roles.py", # Release tooling is covered by unit tests; the packaging and publication workflows are manual. ".github/workflows/desktop-candidate.yml", "scripts/desktop_candidate.py", @@ -49,6 +50,7 @@ database = [ ".github/workflows/e2e-harness.yml", ".github/actions/setup-upgrade-oceanbase/**", "tests/native/test_oceanbase_engine_recovery.py", "tests/*migration*.py", "scripts/check_atomic_memory_migration_shards.py", + "scripts/oceanbase_ci.py", ".github/oceanbase-migration-durations.json", ] dify = ["integrations/dify/**", "tests/*dify*"] hermes = ["integrations/hermes/**", "tests/*hermes*"] @@ -72,6 +74,7 @@ website = [ docs_tests = ["docs/**", "README*.md"] native = ["tests/native/**", "tests/test_service*.py", "tests/test_cursor_secret.py", "tests/test_server.py"] harness = [ + "scripts/oceanbase_ci.py", "e2e/**", "integrations/bub/**", "scripts/scan_e2e_evidence.sh", "tests/test_e2e_workflow_contract.py", ".github/workflows/e2e-harness.yml", "tests/e2e/test_atomic_memory_search_authorization_snapshot.py", "tests/e2e/test_atomic_memory_read_snapshot.py", @@ -84,6 +87,7 @@ zcode = [ ] kimi = ["integrations/kimi-code/**", "tests/*kimi*", ".github/workflows/kimi-code-acceptance.yml"] upgrade = [ + "scripts/oceanbase_ci.py", "scripts/run_atomic_memory_clean_migration.py", "scripts/build_ops_bundle.py", "release/**", "scripts/upgrade_*.py", "scripts/ci_legacy_upgrade_rejection.py", "scripts/ci_release_smoke.py", "scripts/ci_ops_bundle_smoke.py", "scripts/capture_release_schema.py", @@ -114,11 +118,13 @@ managed_resources = [ ".github/workflows/managed-resource-candidates.yml", ] candidate_replay = [ + "scripts/oceanbase_ci.py", "src/powercontext/builtin/persistence/**", "src/powercontext/builtin/artifacts/**", "pyproject.toml", "uv.lock", "tests/builtin/persistence/**", "scripts/replay_candidate_migration.py", ".github/workflows/candidate-schema-replay.yml", ".github/actions/setup-upgrade-oceanbase/**", ] release_schema = [ + "scripts/oceanbase_ci.py", "scripts/capture_release_schema.py", "scripts/upgrade_database_probe.py", "scripts/upgrade_process.py", "tests/release_upgrades/test_full_server.py", ".github/workflows/capture-release-schema.yml", ".github/actions/setup-upgrade-oceanbase/**", diff --git a/.github/oceanbase-migration-durations.json b/.github/oceanbase-migration-durations.json new file mode 100644 index 0000000000..b0bad10593 --- /dev/null +++ b/.github/oceanbase-migration-durations.json @@ -0,0 +1,67 @@ +{ + "source_run": "https://github.com/oceanbase/powercontext/actions/runs/42143824175", + "source_commit": "4b6cf42a4a9fc438fb8bc56890de22962357d7b7", + "durations_seconds": { + "tests/e2e/test_atomic_artifact_search_http.py::test_long_query_recovery_probe_preserves_lexical_coverage[oceanbase]": 27.992, + "tests/e2e/test_atomic_artifact_search_http.py::test_supported_long_query_preserves_all_terms_in_both_http_searches[oceanbase-hybrid-ascii-1000]": 136.261, + "tests/e2e/test_atomic_artifact_search_http.py::test_supported_long_query_preserves_all_terms_in_both_http_searches[oceanbase-hybrid-cjk-600]": 135.768, + "tests/e2e/test_atomic_artifact_search_http.py::test_supported_long_query_preserves_all_terms_in_both_http_searches[oceanbase-text-ascii-1000]": 163.24, + "tests/e2e/test_atomic_artifact_search_http.py::test_supported_long_query_preserves_all_terms_in_both_http_searches[oceanbase-text-cjk-600]": 131.607, + "tests/e2e/test_atomic_memory_grant_migration.py::test_apply_repairs_receipts_left_by_completed_migration[oceanbase]": 115.042, + "tests/e2e/test_atomic_memory_grant_migration.py::test_legacy_grant_replay_and_conflicts_survive_migration[oceanbase-False]": 138.787, + "tests/e2e/test_atomic_memory_grant_migration.py::test_legacy_grant_replay_and_conflicts_survive_migration[oceanbase-True]": 118.128, + "tests/e2e/test_atomic_memory_grant_migration.py::test_migration_rejects_unverifiable_grant_receipts[oceanbase-False-actor]": 118.128, + "tests/e2e/test_atomic_memory_grant_migration.py::test_migration_rejects_unverifiable_grant_receipts[oceanbase-False-hash]": 144.134, + "tests/e2e/test_atomic_memory_grant_migration.py::test_migration_rejects_unverifiable_grant_receipts[oceanbase-False-missing]": 142.398, + "tests/e2e/test_atomic_memory_grant_migration.py::test_migration_rejects_unverifiable_grant_receipts[oceanbase-False-result]": 106.418, + "tests/e2e/test_atomic_memory_grant_migration.py::test_migration_rejects_unverifiable_grant_receipts[oceanbase-True-actor]": 108.233, + "tests/e2e/test_atomic_memory_grant_migration.py::test_migration_rejects_unverifiable_grant_receipts[oceanbase-True-hash]": 117.798, + "tests/e2e/test_atomic_memory_grant_migration.py::test_migration_rejects_unverifiable_grant_receipts[oceanbase-True-missing]": 118.907, + "tests/e2e/test_atomic_memory_grant_migration.py::test_migration_rejects_unverifiable_grant_receipts[oceanbase-True-result]": 127.015, + "tests/e2e/test_atomic_memory_grant_migration.py::test_server_startup_keeps_grant_projection_current[oceanbase-adapter]": 141.768, + "tests/e2e/test_atomic_memory_grant_migration.py::test_server_startup_keeps_grant_projection_current[oceanbase-factory]": 116.791, + "tests/e2e/test_atomic_memory_long_fts_query.py::test_long_fts_query_related_recall_preserves_threshold[oceanbase]": 27.319, + "tests/e2e/test_atomic_memory_long_fts_query.py::test_long_fts_query_search_preserves_recovery_probe[oceanbase-hybrid]": 35.902, + "tests/e2e/test_atomic_memory_long_fts_query.py::test_long_fts_query_search_preserves_recovery_probe[oceanbase-text]": 30.618, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_apply_replaces_the_collection_anchor_left_by_an_earlier_import[oceanbase]": 26.173, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_candidate_left_without_evidence_blocks_until_an_explicit_replacement[oceanbase]": 27.876, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_clean_migration_retries_after_interrupted_public_column_removal[oceanbase]": 31.498, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_collections_citing_each_other_are_removed_regardless_of_identifier_order[oceanbase]": 29.306, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_development_projection_layout_is_recreated_and_rebuilt_after_apply[oceanbase]": 31.406, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_enforced_migration_repairs_ownerless_entry_from_explicit_decision[oceanbase]": 24.61, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_handoff_collection_citation_is_archived_and_removed[oceanbase]": 27.58, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_handoff_supported_only_by_a_collection_blocks_before_any_change[oceanbase]": 26.378, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_legacy_entry_write_source_retains_target_and_stays_out_of_projection[oceanbase]": 24.899, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_merge_keeps_migrated_input_history_without_unrelated_collection_sources[oceanbase]": 28.306, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_migrated_collection_leaves_public_artifact_reads[oceanbase]": 28.539, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_migrated_entry_projects_only_its_exact_sources[oceanbase-alpha-1-expected0]": 26.38, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_migrated_entry_projects_only_its_exact_sources[oceanbase-alpha-2-expected2]": 26.69, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_migrated_entry_projects_only_its_exact_sources[oceanbase-beta-1-expected1]": 28.391, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_migration_converts_exact_citations_and_archives_collection_relationships[oceanbase]": 26.477, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_other_origin_does_not_broaden_migrated_entry_root_groups_or_selection[oceanbase-atomic]": 27.882, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_other_origin_does_not_broaden_migrated_entry_root_groups_or_selection[oceanbase-experience]": 24.88, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_owner_repair_failure_rolls_back_entry_import[oceanbase]": 25.595, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_pending_owner_repair_blocks_plan_and_verify[oceanbase]": 26.244, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_rerun_completes_a_database_left_by_an_earlier_reference_conversion[oceanbase]": 32.053, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_restoration_of_migrated_history_does_not_add_unrelated_sources[oceanbase]": 26.356, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_runtime_start_does_not_read_the_archive_or_legacy_entry_tables[oceanbase]": 26.302, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_snapshot_read_failure_preserves_authority_and_allows_restart[oceanbase-cancel]": 24.691, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_snapshot_read_failure_preserves_authority_and_allows_restart[oceanbase-memory]": 23.861, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_source_flush_supplies_the_migrated_related_entry_current_content[oceanbase]": 27.338, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_undo_merge_preserves_each_migrated_entry_evidence[oceanbase]": 27.206, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_unenveloped_source_mentioning_memory_migrates_unchanged[oceanbase]": 29.014, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_unenveloped_work_source_citations_convert_and_keep_their_layout[oceanbase]": 29.171, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_unfinished_dream_pinning_a_rewritten_source_blocks_migration[oceanbase]": 22.01, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_unfinished_dream_receipt_address_stays_readable_without_rewriting_its_source[oceanbase]": 27.359, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_unfinished_dream_requires_current_artifact_snapshot_format[oceanbase-False]": 30.471, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_unfinished_dream_requires_current_artifact_snapshot_format[oceanbase-True]": 24.548, + "tests/e2e/test_atomic_memory_migration_evidence.py::test_work_text_mentioning_legacy_field_names_migrates_unchanged[oceanbase]": 26.647, + "tests/e2e/test_atomic_memory_migration_vector.py::test_clean_migration_and_rebuild_publish_searchable_vectors[oceanbase]": 33.975, + "tests/e2e/test_background_metrics.py::test_background_cli_serves_only_authorized_metrics_and_stops_cleanly[oceanbase]": 121.608, + "tests/e2e/test_managed_startup_read_consistency.py::test_concurrent_publication_does_not_reject_managed_startup[oceanbase-background-False]": 122.048, + "tests/e2e/test_managed_startup_read_consistency.py::test_concurrent_publication_does_not_reject_managed_startup[oceanbase-sdk-True]": 128.565, + "tests/e2e/test_managed_startup_read_consistency.py::test_concurrent_publication_does_not_reject_managed_startup[oceanbase-server-False]": 127.705, + "tests/e2e/test_scope_search_migration.py::test_interrupted_scope_search_upgrade_recovers_constraints[oceanbase-False]": 3.485, + "tests/e2e/test_scope_search_migration.py::test_interrupted_scope_search_upgrade_recovers_constraints[oceanbase-True]": 2.602 + } +} diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index f4b2fce18b..189abb1918 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -94,74 +94,17 @@ jobs: python-version: "3.12" - name: Start the isolated OceanBase test instance + timeout-minutes: 14 run: | - docker run --detach --name recall-effort-oceanbase \ + timeout --kill-after=10s 180s docker run --detach --name recall-effort-oceanbase \ --publish 127.0.0.1:2881:2881 \ --ulimit nofile=65536:65536 \ --env MODE=slim \ --env OB_DATABASE=powercontext_test \ --env OB_TENANT_PASSWORD=powercontext-e2e \ "$OCEANBASE_IMAGE" - for attempt in $(seq 1 120); do - if docker exec recall-effort-oceanbase \ - obclient -h127.0.0.1 -P2881 -uroot@test -ppowercontext-e2e \ - -Dpowercontext_test -e 'SELECT 1' >/dev/null 2>&1; then - # The slim image's default 1 GB maximum is too small for the - # indexes created across the full Server recovery scenarios. - docker exec recall-effort-oceanbase \ - obclient -h127.0.0.1 -P2881 -uroot@sys \ - -e "ALTER SYSTEM SET datafile_maxsize='8G'; ALTER SYSTEM SET datafile_next='512M'; ALTER SYSTEM SET datafile_size='4G';" - datafile_ready=false - for config_attempt in $(seq 1 30); do - docker exec recall-effort-oceanbase \ - obclient -h127.0.0.1 -P2881 -uroot@sys --batch --skip-column-names \ - -e "SHOW PARAMETERS WHERE name IN ('datafile_size', 'datafile_maxsize', 'datafile_next');" \ - > "$RUNNER_TEMP/recall-effort-datafile.tsv" - if awk ' - $5 == "datafile_size" && $7 == "4G" { size = 1 } - $5 == "datafile_maxsize" && $7 == "8G" { maximum = 1 } - $5 == "datafile_next" && $7 == "512M" { increment = 1 } - END { exit !(size && maximum && increment) } - ' "$RUNNER_TEMP/recall-effort-datafile.tsv"; then - datafile_ready=true - break - fi - sleep 2 - done - cat "$RUNNER_TEMP/recall-effort-datafile.tsv" - if [ "$datafile_ready" != true ]; then - echo 'OceanBase datafile configuration did not become effective within one minute.' >&2 - exit 1 - fi - docker exec recall-effort-oceanbase \ - obclient -h127.0.0.1 -P2881 -uroot@test -ppowercontext-e2e \ - -Dpowercontext_test \ - -e "SET GLOBAL recyclebin=OFF; SELECT VERSION(); SHOW VARIABLES LIKE 'ob_compatibility_mode'; SHOW GLOBAL VARIABLES LIKE 'recyclebin';" - exit 0 - fi - if [ "$(docker inspect --format '{{.State.Running}}' recall-effort-oceanbase)" != true ]; then - echo 'OceanBase exited before becoming ready.' >&2 - exit 1 - fi - sleep 5 - done - echo 'OceanBase did not become ready within 10 minutes.' >&2 - exit 1 - - - name: Resolve Python version for hook cache - id: python - run: echo "version=$(python -c 'import platform; print(platform.python_version())')" >> "$GITHUB_OUTPUT" - - - name: Cache prek environments - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 - with: - path: ~/.cache/prek - key: prek-v1-${{ runner.os }}-${{ runner.arch }}-${{ steps.python.outputs.version }}-${{ hashFiles('.pre-commit-config.yaml') }} - - - name: Check code and types on Linux - run: | - uv lock --check - uv run --locked prek run -a + python scripts/oceanbase_ci.py wait --container recall-effort-oceanbase \ + --output ".powercontext/recall-effort-ci/startup" - name: Run real OceanBase database bootstrap without skips timeout-minutes: 5 @@ -171,8 +114,10 @@ jobs: POWERCONTEXT_TEST_BOOTSTRAP_PRIVILEGES: "1" run: | mkdir -p .powercontext/recall-effort-ci - uv run --locked python -m pytest tests/e2e/test_database_bootstrap.py -k oceanbase -q -rs \ - --junitxml=.powercontext/recall-effort-ci/bootstrap.xml + python scripts/oceanbase_ci.py run --container recall-effort-oceanbase \ + --timeout 240 --output .powercontext/recall-effort-ci/bootstrap-process -- \ + uv run --locked python -m pytest tests/e2e/test_database_bootstrap.py -k oceanbase -q -rs \ + --durations=0 --junitxml=.powercontext/recall-effort-ci/bootstrap.xml uv run --locked python - <<'PYCODE' from xml.etree import ElementTree @@ -185,12 +130,13 @@ jobs: - name: Run OceanBase contracts and Jev integration without skips run: | mkdir -p .powercontext/recall-effort-ci - uv run --locked pytest \ + python scripts/oceanbase_ci.py run --container recall-effort-oceanbase \ + --timeout 600 --output .powercontext/recall-effort-ci/contracts-process -- uv run --locked pytest \ tests/builtin/persistence/test_oceanbase_profile.py::test_live_oceanbase_profile_smoke \ 'tests/builtin/persistence/test_recall_effort_statistics.py::test_recall_effort_schema_upgrade_and_atomic_daily_scope_accounting[oceanbase]' \ tests/builtin/persistence/test_oceanbase_profile.py::test_live_oceanbase_profile_rejects_old_schema_collation \ tests/e2e/test_jev_memory_reranking.py \ - --junitxml=.powercontext/recall-effort-ci/oceanbase.xml -q -rs + --durations=0 --junitxml=.powercontext/recall-effort-ci/oceanbase.xml -q -rs uv run --locked python - <<'PY' from pathlib import Path from xml.etree import ElementTree @@ -203,8 +149,10 @@ jobs: - name: Run the real OceanBase recovery upgrade without skips run: | - uv run --locked pytest tests/e2e/test_topic_memory_recovery_migration.py -k oceanbase \ - --junitxml=.powercontext/recall-effort-ci/topic-recovery.xml -q -rs + python scripts/oceanbase_ci.py run --container recall-effort-oceanbase \ + --timeout 600 --output .powercontext/recall-effort-ci/topic-recovery-process -- \ + uv run --locked pytest tests/e2e/test_topic_memory_recovery_migration.py -k oceanbase \ + --durations=0 --junitxml=.powercontext/recall-effort-ci/topic-recovery.xml -q -rs uv run --locked python - <<'PY' from pathlib import Path from xml.etree import ElementTree @@ -215,30 +163,17 @@ jobs: raise SystemExit("Both live OceanBase recovery upgrades must execute; skipped tests are not acceptance.") PY - - name: Run recall, statistics and prepared-context regressions - run: | - uv run --locked pytest \ - tests/builtin/runtime/test_recall_sufficiency.py \ - tests/e2e/test_recall_sufficiency_gate.py \ - tests/builtin/runtime/test_statistics.py \ - tests/builtin/runtime/test_model_usage_recorder.py \ - tests/builtin/runtime/test_prepared_context.py \ - tests/e2e/test_context_text_assembly.py \ - tests/builtin/persistence/test_recall_effort_statistics.py \ - tests/builtin/persistence/test_mysql_schema.py \ - tests/builtin/persistence/test_sqlite_profile.py \ - tests/builtin/persistence/test_database_snapshots.py \ - --junitxml=.powercontext/recall-effort-ci/regressions.xml -q -rs - + # General regressions run in Main unit/e2e jobs before this native job. - name: Recover interrupted Server initialization on real OceanBase without skips env: POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL: >- mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_migration_probe_server_recovery?charset=utf8mb4 run: | - docker exec recall-effort-oceanbase \ + timeout 15s docker exec recall-effort-oceanbase \ obclient -h127.0.0.1 -P2881 -uroot@test -ppowercontext-e2e \ -Dpowercontext_test -e 'CREATE DATABASE pc_migration_probe_server_recovery' - timeout --signal=TERM --kill-after=10s 600s uv run --locked pytest -vv -s -rs \ + python scripts/oceanbase_ci.py run --container recall-effort-oceanbase \ + --timeout 600 --output .powercontext/recall-effort-ci/full-server-initialization-process -- uv run --locked pytest -vv -s -rs \ tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_full_server_resumes_interrupted_initialization_before_manifest_seed \ -o faulthandler_timeout=180 --durations=0 \ -k oceanbase --junitxml=.powercontext/recall-effort-ci/full-server-initialization.xml @@ -248,7 +183,8 @@ jobs: POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL: >- mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_migration_probe_server_recovery?charset=utf8mb4 run: | - timeout --signal=TERM --kill-after=10s 900s uv run --locked pytest -vv -s -rs \ + python scripts/oceanbase_ci.py run --container recall-effort-oceanbase \ + --timeout 900 --output .powercontext/recall-effort-ci/historical-server-upgrade-process -- uv run --locked pytest -vv -s -rs \ tests/builtin/persistence/test_mysql_migration_runner.py::test_historical_memory_upgrades_and_resumes_native_ddl \ -o faulthandler_timeout=180 --durations=0 \ -k oceanbase --junitxml=.powercontext/recall-effort-ci/historical-server-upgrade.xml @@ -262,7 +198,8 @@ jobs: POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL: >- mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_migration_probe_server_recovery?charset=utf8mb4 run: | - timeout --signal=TERM --kill-after=10s 900s uv run --locked pytest -vv -s -rs \ + python scripts/oceanbase_ci.py run --container recall-effort-oceanbase \ + --timeout 900 --output .powercontext/recall-effort-ci/full-server-checkpoints-process -- uv run --locked pytest -vv -s -rs \ tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_full_server_interruption_preserves_history_and_original_maintenance \ -o faulthandler_timeout=180 --durations=0 \ -k oceanbase --junitxml=.powercontext/recall-effort-ci/full-server-checkpoints.xml @@ -272,7 +209,8 @@ jobs: POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL: >- mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_migration_probe_server_recovery?charset=utf8mb4 run: | - timeout --signal=TERM --kill-after=10s 600s uv run --locked pytest -vv -s -rs \ + python scripts/oceanbase_ci.py run --container recall-effort-oceanbase \ + --timeout 600 --output .powercontext/recall-effort-ci/full-server-client-exit-process -- uv run --locked pytest -vv -s -rs \ tests/builtin/persistence/test_mysql_migration_runner.py::test_oceanbase_full_server_client_exit_requires_acknowledgement_and_preserves_recovery \ -o faulthandler_timeout=180 --durations=0 \ -k oceanbase --junitxml=.powercontext/recall-effort-ci/full-server-client-exit.xml @@ -282,7 +220,8 @@ jobs: POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL: >- mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_migration_probe_server_recovery?charset=utf8mb4 run: | - timeout --signal=TERM --kill-after=10s 600s uv run --locked pytest -vv -s -rs \ + python scripts/oceanbase_ci.py run --container recall-effort-oceanbase \ + --timeout 600 --output .powercontext/recall-effort-ci/full-server-maintenance-process -- uv run --locked pytest -vv -s -rs \ tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_registered_processing_mode_switch_preserves_revision_and_history \ tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_atomic_compatibility_command_uses_shared_server_upgrade \ tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_registered_projection_repair_preserves_authority_and_revision \ @@ -307,17 +246,20 @@ jobs: - name: Collect the database log if: always() + timeout-minutes: 3 run: | mkdir -p .powercontext/recall-effort-ci - docker logs recall-effort-oceanbase > .powercontext/recall-effort-ci/oceanbase.log 2>&1 || true + python scripts/oceanbase_ci.py collect --container recall-effort-oceanbase \ + --output .powercontext/recall-effort-ci/diagnostics + timeout 15s docker logs --tail 20000 recall-effort-oceanbase > .powercontext/recall-effort-ci/oceanbase.log 2>&1 || true for logfile in observer.log observer.log.wf; do - docker exec recall-effort-oceanbase tail -n 20000 "/root/demo/log/$logfile" \ + timeout 15s docker exec recall-effort-oceanbase tail -n 20000 "/root/demo/log/$logfile" \ > ".powercontext/recall-effort-ci/$logfile" 2>&1 || true done df -h / > .powercontext/recall-effort-ci/runner-disk.txt 2>&1 || true - docker exec recall-effort-oceanbase df -h /root/demo \ + timeout 15s docker exec recall-effort-oceanbase df -h /root/demo \ > .powercontext/recall-effort-ci/container-disk.txt 2>&1 || true - docker exec recall-effort-oceanbase \ + timeout 15s docker exec recall-effort-oceanbase \ obclient -h127.0.0.1 -P2881 -uroot@sys \ -e "SHOW PARAMETERS WHERE name IN ('datafile_size', 'datafile_maxsize', 'datafile_next'); SELECT data_disk_allocated, data_disk_capacity, data_disk_in_use FROM oceanbase.GV\$OB_SERVERS;" \ > .powercontext/recall-effort-ci/database-disk.txt 2>&1 || true @@ -334,10 +276,10 @@ jobs: - name: Stop and remove the test instance if: always() - run: docker rm --force recall-effort-oceanbase || true + run: timeout 30s docker rm --force recall-effort-oceanbase || true atomic-memory-clean-oceanbase-shards: - name: Atomic Memory clean migration (OceanBase shard ${{ matrix.shard }}/3) + name: Atomic Memory clean migration (OceanBase shard ${{ matrix.shard }}/2) needs: changes if: >- ${{ !cancelled() && (github.event_name != 'workflow_dispatch' || inputs.suite != 'recall-effort') && @@ -349,8 +291,9 @@ jobs: max-parallel: 2 fail-fast: false matrix: - shard: [0, 1, 2] + shard: [0, 1] env: + OCEANBASE_CI_CONTAINER: atomic-memory-clean-oceanbase OCEANBASE_IMAGE: ghcr.io/oceanbase/oceanbase-ce:4.3.5.6-106000012026040916 POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL: >- mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/powercontext_test?charset=utf8mb4 @@ -368,65 +311,23 @@ jobs: python-version: "3.12" - name: Start the isolated OceanBase test instance + timeout-minutes: 14 run: | - docker run --detach --name atomic-memory-clean-oceanbase \ + timeout --kill-after=10s 180s docker run --detach --name atomic-memory-clean-oceanbase \ --publish 127.0.0.1:2881:2881 \ --ulimit nofile=65536:65536 \ --env MODE=slim \ --env OB_DATABASE=powercontext_test \ --env OB_TENANT_PASSWORD=powercontext-e2e \ "$OCEANBASE_IMAGE" - for attempt in $(seq 1 120); do - if docker exec atomic-memory-clean-oceanbase \ - obclient -h127.0.0.1 -P2881 -uroot@test -ppowercontext-e2e \ - -Dpowercontext_test -e 'SELECT 1' >/dev/null 2>&1; then - # The slim image's default 1 GB maximum is too small for the - # indexes created across the clean migration acceptance cases. - docker exec atomic-memory-clean-oceanbase \ - obclient -h127.0.0.1 -P2881 -uroot@sys \ - -e "ALTER SYSTEM SET datafile_maxsize='8G'; ALTER SYSTEM SET datafile_next='512M'; ALTER SYSTEM SET datafile_size='4G';" - datafile_ready=false - for config_attempt in $(seq 1 30); do - docker exec atomic-memory-clean-oceanbase \ - obclient -h127.0.0.1 -P2881 -uroot@sys --batch --skip-column-names \ - -e "SHOW PARAMETERS WHERE name IN ('datafile_size', 'datafile_maxsize', 'datafile_next');" \ - > "$RUNNER_TEMP/atomic-memory-datafile.tsv" - if awk ' - $5 == "datafile_size" && $7 == "4G" { size = 1 } - $5 == "datafile_maxsize" && $7 == "8G" { maximum = 1 } - $5 == "datafile_next" && $7 == "512M" { increment = 1 } - END { exit !(size && maximum && increment) } - ' "$RUNNER_TEMP/atomic-memory-datafile.tsv"; then - datafile_ready=true - break - fi - sleep 2 - done - cat "$RUNNER_TEMP/atomic-memory-datafile.tsv" - if [ "$datafile_ready" != true ]; then - echo 'OceanBase datafile configuration did not become effective within one minute.' >&2 - exit 1 - fi - docker exec atomic-memory-clean-oceanbase \ - obclient -h127.0.0.1 -P2881 -uroot@test -ppowercontext-e2e \ - -Dpowercontext_test \ - -e "SET GLOBAL recyclebin=OFF; SELECT VERSION(); SHOW VARIABLES LIKE 'ob_compatibility_mode'; SHOW VARIABLES LIKE 'recyclebin';" - exit 0 - fi - if [ "$(docker inspect --format '{{.State.Running}}' atomic-memory-clean-oceanbase)" != true ]; then - echo 'OceanBase exited before becoming ready.' >&2 - exit 1 - fi - sleep 5 - done - echo 'OceanBase did not become ready within 10 minutes.' >&2 - exit 1 + python scripts/oceanbase_ci.py wait --container atomic-memory-clean-oceanbase \ + --output "$MIGRATION_REPORTS/startup" - name: Test interrupted Scope search migrations without skips timeout-minutes: 10 run: | uv run --locked python scripts/run_atomic_memory_clean_migration.py \ - oceanbase --shard "${{ matrix.shard }}/3" --output "$MIGRATION_REPORTS/scope-search" --tests \ + oceanbase --shard "${{ matrix.shard }}/2" --durations .github/oceanbase-migration-durations.json --output "$MIGRATION_REPORTS/scope-search" --tests \ tests/e2e/test_scope_search_migration.py - name: Test background listener lifecycle without skips @@ -434,57 +335,61 @@ jobs: timeout-minutes: 10 run: | uv run --locked --extra server python scripts/run_atomic_memory_clean_migration.py \ - oceanbase --test-timeout 300 --shard "${{ matrix.shard }}/3" --output "$MIGRATION_REPORTS/background-metrics" --tests \ + oceanbase --test-timeout 300 --shard "${{ matrix.shard }}/2" --durations .github/oceanbase-migration-durations.json --output "$MIGRATION_REPORTS/background-metrics" --tests \ tests/e2e/test_background_metrics.py - name: Run clean migration evidence, grants and vectors without skips - # Up to 16 cases per shard at 300s each, plus collection and termination grace. + # Measured clean migration work is about 21m per worker; retain the 90m group ceiling. timeout-minutes: 90 run: | uv run --locked python scripts/run_atomic_memory_clean_migration.py \ - oceanbase --test-timeout 300 --shard "${{ matrix.shard }}/3" --output "$MIGRATION_REPORTS/clean-migration" + oceanbase --test-timeout 300 --shard "${{ matrix.shard }}/2" --durations .github/oceanbase-migration-durations.json --output "$MIGRATION_REPORTS/clean-migration" - name: Test concurrent managed startup without skips - # Each shard runs one full Server case (300s), plus collection (180s). + # Up to two full Server cases (300s each), plus collection (180s). timeout-minutes: 15 run: | uv run --locked --extra server python scripts/run_atomic_memory_clean_migration.py \ - oceanbase --test-timeout 300 --shard "${{ matrix.shard }}/3" --output "$MIGRATION_REPORTS/startup-snapshot" --tests \ + oceanbase --test-timeout 300 --shard "${{ matrix.shard }}/2" --durations .github/oceanbase-migration-durations.json --output "$MIGRATION_REPORTS/startup-snapshot" --tests \ tests/e2e/test_managed_startup_read_consistency.py test "$(wc -l < "$MIGRATION_REPORTS/startup-snapshot/nodes.txt")" -eq 3 - name: Test long-query retrieval and admission without skips - # Up to three full Server cases per shard at 300s each, plus collection (180s). + # Four full Server cases per shard at 300s each, plus collection (180s). timeout-minutes: 30 run: | uv run --locked python scripts/run_atomic_memory_clean_migration.py \ - oceanbase --test-timeout 300 --shard "${{ matrix.shard }}/3" --output "$MIGRATION_REPORTS/long-query" --tests \ + oceanbase --test-timeout 300 --shard "${{ matrix.shard }}/2" --durations .github/oceanbase-migration-durations.json --output "$MIGRATION_REPORTS/long-query" --tests \ tests/e2e/test_atomic_artifact_search_http.py \ tests/e2e/test_atomic_memory_long_fts_query.py - name: Collect the database log if: always() + timeout-minutes: 3 run: | mkdir -p "$MIGRATION_REPORTS" - docker logs atomic-memory-clean-oceanbase > "$MIGRATION_REPORTS/oceanbase.log" 2>&1 || true + python scripts/oceanbase_ci.py collect --container atomic-memory-clean-oceanbase \ + --output "$MIGRATION_REPORTS/diagnostics" + timeout 15s docker logs --tail 20000 atomic-memory-clean-oceanbase > "$MIGRATION_REPORTS/oceanbase.log" 2>&1 || true for logfile in observer.log observer.log.wf; do - docker exec atomic-memory-clean-oceanbase tail -n 20000 "/root/demo/log/$logfile" \ + timeout 15s docker exec atomic-memory-clean-oceanbase tail -n 20000 "/root/demo/log/$logfile" \ > "$MIGRATION_REPORTS/$logfile" 2>&1 || true done - docker inspect --format 'MemoryBytes={{.HostConfig.Memory}} NanoCpus={{.HostConfig.NanoCpus}} CpuQuota={{.HostConfig.CpuQuota}}' \ + timeout 15s docker inspect --format 'MemoryBytes={{.HostConfig.Memory}} NanoCpus={{.HostConfig.NanoCpus}} CpuQuota={{.HostConfig.CpuQuota}}' \ atomic-memory-clean-oceanbase > "$MIGRATION_REPORTS/container-resources.txt" 2>&1 || true - docker stats --no-stream atomic-memory-clean-oceanbase \ + timeout 15s docker stats --no-stream atomic-memory-clean-oceanbase \ > "$MIGRATION_REPORTS/container-stats.txt" 2>&1 || true - docker exec atomic-memory-clean-oceanbase \ + timeout 15s docker exec atomic-memory-clean-oceanbase \ obclient -h127.0.0.1 -P2881 -uroot@sys \ -e "SHOW PARAMETERS WHERE name IN ('datafile_size', 'datafile_maxsize', 'datafile_next', 'memory_limit', 'system_memory', 'cpu_count');" \ > "$MIGRATION_REPORTS/database-parameters.txt" 2>&1 || true - name: Save complete observer logs on failure if: failure() + timeout-minutes: 3 run: | mkdir -p "$MIGRATION_REPORTS/observer" - docker cp atomic-memory-clean-oceanbase:/root/demo/log/. "$MIGRATION_REPORTS/observer/" || true + timeout 120s docker cp atomic-memory-clean-oceanbase:/root/demo/log/. "$MIGRATION_REPORTS/observer/" || true - name: Save clean migration reports if: always() @@ -498,7 +403,7 @@ jobs: - name: Stop and remove the test instance if: always() - run: docker rm --force atomic-memory-clean-oceanbase || true + run: timeout 30s docker rm --force atomic-memory-clean-oceanbase || true atomic-memory-clean-oceanbase: name: Atomic Memory clean migration (OceanBase) @@ -538,7 +443,7 @@ jobs: - name: Verify complete execution coverage run: >- python scripts/check_atomic_memory_migration_shards.py - --reports "${{ runner.temp }}/atomic-memory-clean-oceanbase-shards" --shards 3 + --reports "${{ runner.temp }}/atomic-memory-clean-oceanbase-shards" --shards 2 --durations .github/oceanbase-migration-durations.json acceptance: needs: validate @@ -572,41 +477,13 @@ jobs: COMPOSE_PROJECT_NAME: powercontext-e2e-oceanbase POWERCONTEXT_E2E_OUTPUT: ${{ github.workspace }}/.powercontext-e2e/oceanbase/acceptance run: | - mkdir -p "$POWERCONTEXT_E2E_OUTPUT" + native_deadline=$((SECONDS + 2850)) + mkdir -p "$POWERCONTEXT_E2E_OUTPUT" "$RUNNER_TEMP/oceanbase-acceptance" compose=(docker compose -f e2e/bub/compose.yaml -f e2e/bub/compose.oceanbase.yaml) - "${compose[@]}" up --detach --wait --wait-timeout 660 oceanbase + timeout --kill-after=10s 700s "${compose[@]}" up --detach --wait --wait-timeout 660 oceanbase container_id=$("${compose[@]}" ps -q oceanbase) - # The slim image's default 1 GB maximum is too small for the - # indexes created across isolated acceptance databases. - docker exec "$container_id" \ - obclient -h127.0.0.1 -P2881 -uroot@sys \ - -e "ALTER SYSTEM SET datafile_maxsize='8G'; ALTER SYSTEM SET datafile_next='512M'; ALTER SYSTEM SET datafile_size='4G';" - datafile_ready=false - for config_attempt in $(seq 1 30); do - docker exec "$container_id" \ - obclient -h127.0.0.1 -P2881 -uroot@sys --batch --skip-column-names \ - -e "SHOW PARAMETERS WHERE name IN ('datafile_size', 'datafile_maxsize', 'datafile_next');" \ - > "$RUNNER_TEMP/acceptance-datafile.tsv" - if awk ' - $5 == "datafile_size" && $7 == "4G" { size = 1 } - $5 == "datafile_maxsize" && $7 == "8G" { maximum = 1 } - $5 == "datafile_next" && $7 == "512M" { increment = 1 } - END { exit !(size && maximum && increment) } - ' "$RUNNER_TEMP/acceptance-datafile.tsv"; then - datafile_ready=true - break - fi - sleep 2 - done - cat "$RUNNER_TEMP/acceptance-datafile.tsv" - if [ "$datafile_ready" != true ]; then - echo 'OceanBase datafile configuration did not become effective within one minute.' >&2 - exit 1 - fi - docker exec "$container_id" \ - obclient -h127.0.0.1 -P2881 -uroot@test -ppowercontext-e2e \ - -Dpowercontext \ - -e "SET GLOBAL recyclebin=OFF; SELECT VERSION(); SHOW VARIABLES LIKE 'ob_compatibility_mode'; SHOW VARIABLES LIKE 'recyclebin';" + python scripts/oceanbase_ci.py wait --container "$container_id" --database powercontext \ + --timeout 90 --output "$RUNNER_TEMP/oceanbase-acceptance/startup" oceanbase_ip=$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$container_id") test -n "$oceanbase_ip" export POWERCONTEXT_TEST_OCEANBASE_URL="mysql+aoceanbase://root%40test:powercontext-e2e@${oceanbase_ip}:2881/powercontext?charset=utf8mb4" @@ -628,21 +505,23 @@ jobs: asyncio.run(version()) PY - uv run --locked --extra server python -m pytest -vv -ra \ + python scripts/oceanbase_ci.py run --container "$container_id" \ + --timeout "$((native_deadline - SECONDS))" --output "$RUNNER_TEMP/oceanbase-acceptance/process" -- \ + uv run --locked --extra server python -m pytest -vv -ra --durations=0 \ tests/e2e/test_atomic_memory_search_authorization_snapshot.py \ tests/e2e/test_atomic_memory_read_snapshot.py \ tests/e2e/test_server_metrics_oceanbase.py \ tests/e2e/test_runtime_server.py \ 'tests/e2e/test_portable_archive.py::test_archive_batches_validation_and_rechecks_after_dry_run[server]' \ -k 'oceanbase or archive_batches' \ - --junitxml="$RUNNER_TEMP/atomic-memory-oceanbase-snapshots.xml" + --junitxml="$RUNNER_TEMP/oceanbase-acceptance/junit.xml" uv run --locked --extra server python - <<'PY' import json import os import xml.etree.ElementTree as ET from pathlib import Path - suites = ET.parse(Path(os.environ["RUNNER_TEMP"]) / "atomic-memory-oceanbase-snapshots.xml").iter("testsuite") + suites = ET.parse(Path(os.environ["RUNNER_TEMP"]) / "oceanbase-acceptance" / "junit.xml").iter("testsuite") counts = {key: 0 for key in ("tests", "failures", "errors", "skipped")} for suite in suites: for key in counts: @@ -660,6 +539,25 @@ jobs: POWERCONTEXT_E2E_OUTPUT: ${{ github.workspace }}/.powercontext-e2e/${{ matrix.database }}/acceptance run: make harness-compose-acceptance + - name: Collect OceanBase native diagnostics + if: always() && matrix.database == 'oceanbase' + timeout-minutes: 2 + env: + COMPOSE_PROJECT_NAME: powercontext-e2e-oceanbase + run: | + container_id=$(timeout 15s docker compose -f e2e/bub/compose.yaml -f e2e/bub/compose.oceanbase.yaml ps --all -q oceanbase) || true + python scripts/oceanbase_ci.py collect --container "${container_id:-missing-oceanbase}" \ + --output "$RUNNER_TEMP/oceanbase-acceptance/diagnostics" + + - name: Upload OceanBase native reports + if: always() && matrix.database == 'oceanbase' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: oceanbase-native-acceptance-${{ github.sha }} + path: ${{ runner.temp }}/oceanbase-acceptance/ + if-no-files-found: error + retention-days: 14 + - name: Scan replay evidence id: evidence_scan if: always() diff --git a/scripts/check_atomic_memory_migration_shards.py b/scripts/check_atomic_memory_migration_shards.py index 4cc4db1f65..ff25ff8752 100644 --- a/scripts/check_atomic_memory_migration_shards.py +++ b/scripts/check_atomic_memory_migration_shards.py @@ -22,10 +22,12 @@ from collections import Counter from pathlib import Path +from run_atomic_memory_clean_migration import assigned_nodes, read_durations + GROUPS = ("scope-search", "background-metrics", "clean-migration", "startup-snapshot", "long-query") -def verify_group(paths: list[Path], shards: int) -> int: +def verify_group(paths: list[Path], shards: int, durations: dict[str, float] | None = None) -> int: """Require matching collections and successful, disjoint execution of every node.""" collection: list[str] | None = None passed: list[str] = [] @@ -64,7 +66,7 @@ def verify_group(paths: list[Path], shards: int) -> int: if nodes != collection: message = f"{path}: full collections differ between shards" raise ValueError(message) - if actual != nodes[index::shards]: + if actual != assigned_nodes(nodes, index, shards, durations): message = f"{path}: passed nodes differ from the assigned shard" raise ValueError(message) indices.add(index) @@ -75,14 +77,14 @@ def verify_group(paths: list[Path], shards: int) -> int: return len(passed) -def verify_reports(directory: Path, shards: int) -> None: +def verify_reports(directory: Path, shards: int, durations: dict[str, float] | None = None) -> None: """Verify every acceptance group in exactly one artifact per expected shard.""" artifacts = sorted(path for path in directory.iterdir() if path.is_dir()) if len(artifacts) != shards: message = f"expected {shards} shard artifacts, found {len(artifacts)}" raise ValueError(message) for group in GROUPS: - count = verify_group([artifact / group / "summary.json" for artifact in artifacts], shards) + count = verify_group([artifact / group / "summary.json" for artifact in artifacts], shards, durations) print(f"{group}: all {count} OceanBase cases passed exactly once without skips.", flush=True) @@ -92,11 +94,12 @@ def main() -> int: "--reports", type=Path, required=True, help="Directory containing one artifact directory per shard" ) parser.add_argument("--shards", type=int, required=True) + parser.add_argument("--durations", type=Path, help="The same duration baseline used by the migration jobs") args = parser.parse_args() if args.shards < 1: parser.error("--shards must be positive") try: - verify_reports(args.reports, args.shards) + verify_reports(args.reports, args.shards, read_durations(args.durations)) except (OSError, TypeError, ValueError) as error: print(f"Shard verification failed: {error}", file=sys.stderr) return 1 diff --git a/scripts/oceanbase_ci.py b/scripts/oceanbase_ci.py new file mode 100644 index 0000000000..69570af491 --- /dev/null +++ b/scripts/oceanbase_ci.py @@ -0,0 +1,233 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Bound startup and preserve diagnostics for disposable OceanBase CI containers.""" + +from __future__ import annotations + +import argparse +import json +import math +import signal +import subprocess +import time +from pathlib import Path +from typing import IO + + +def sql(container: str, statement: str, *, database: str = "oceanbase", system: bool = False) -> list[str]: + # Bound the client inside the container too: killing docker exec alone does + # not guarantee that its remote process exits. These are disposable CI credentials. + return [ + "docker", + "exec", + container, + "timeout", + "--kill-after=1s", + "8s", + "obclient", + "-h127.0.0.1", + "-P2881", + "-uroot@sys" if system else "-uroot@test", + *([] if system else ["-ppowercontext-e2e"]), + "--connect-timeout=5", + "--batch", + "--skip-column-names", + f"-D{database}", + "--init-command=SET ob_query_timeout=5000000", + "-e", + statement, + ] + + +def capture(command: list[str], path: Path, deadline: float, *, probe_timeout: float = 10) -> int: + """Each probe has a local deadline within the caller's overall budget.""" + remaining = deadline - time.monotonic() + if remaining <= 0: + return 124 + with path.open("w") as output: + try: + result = subprocess.run( # noqa: S603 -- fixed CI commands, no shell + command, stdout=output, stderr=subprocess.STDOUT, timeout=min(probe_timeout, remaining), check=False + ) + except subprocess.TimeoutExpired: + output.write("\nCI probe deadline exceeded.\n") + return 124 + except OSError as error: + output.write(f"\nCI probe unavailable: {type(error).__name__}\n") + return 127 + else: + return result.returncode + + +def collect(container: str, output: Path, *, budget: float = 30) -> None: + """Best-effort bounded snapshots; omit statement text, row keys and container env.""" + output.mkdir(parents=True, exist_ok=True) + started = time.monotonic() + queries = { + "sessions": "SELECT ID, USER, DB, COMMAND, TIME, STATE FROM information_schema.PROCESSLIST LIMIT 100", + "locks": "SELECT TENANT_ID, TRANS_ID, TYPE, ID1, LMODE, REQUEST, CTIME, BLOCK FROM oceanbase.GV$OB_LOCKS LIMIT 100", + "transactions": "SELECT TENANT_ID, SESSION_ID, TX_ID, STATE, ACTIVE_TIME, EXPIRE_TIME FROM oceanbase.GV$OB_TRANSACTION_SCHEDULERS LIMIT 100", + } + commands = {name: sql(container, query) for name, query in queries.items()} + commands.update({ + "processes": ["docker", "top", container, "-eo", "pid,ppid,stat,etime,comm"], + "resources": ["docker", "stats", "--no-stream", container], + "disk": ["docker", "exec", container, "df", "-h", "/root/demo"], + "state": ["docker", "inspect", "--format", "{{json .State}}", container], + }) + results = { + name: capture(command, output / f"{name}.txt", started + budget, probe_timeout=min(3, budget / len(commands))) + for name, command in commands.items() + } + (output / "snapshot.json").write_text( + json.dumps( + { + "captured_at_unix_seconds": time.time(), + "elapsed_seconds": time.monotonic() - started, + "exit_codes": results, + }, + indent=2, + ) + + "\n" + ) + + +def wait_ready(container: str, database: str, output: Path, timeout: float) -> int: + """Wait for the tenant, configure capacity, and verify it within one deadline.""" + output.mkdir(parents=True, exist_ok=True) + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if capture(sql(container, "SELECT 1", database=database), output / "readiness.txt", deadline) == 0: + break + state = output / "state.txt" + code = capture(["docker", "inspect", "--format", "{{.State.Running}}", container], state, deadline) + if code == 0 and state.read_text().strip() == "false": + print("OceanBase exited before becoming ready.", flush=True) + return 1 + time.sleep(min(2, max(0, deadline - time.monotonic()))) + else: + print(f"OceanBase did not become ready within {timeout:g} seconds.", flush=True) + return 124 + + statement = "ALTER SYSTEM SET datafile_maxsize='8G'; ALTER SYSTEM SET datafile_next='512M'; ALTER SYSTEM SET datafile_size='4G';" + if capture(sql(container, statement, system=True), output / "configure.txt", deadline): + return 1 + expected = {"datafile_size": "4G", "datafile_maxsize": "8G", "datafile_next": "512M"} + parameters = output / "datafile.tsv" + config_deadline = min(deadline, time.monotonic() + 60) + while time.monotonic() < config_deadline: + code = capture( + sql( + container, + "SHOW PARAMETERS WHERE name IN ('datafile_size', 'datafile_maxsize', 'datafile_next');", + system=True, + ), + parameters, + config_deadline, + ) + rows = [line.split("\t") for line in parameters.read_text().splitlines()] if parameters.exists() else [] + values = {row[4]: row[6] for row in rows if len(row) > 6} + if code == 0 and all(values.get(key) == value for key, value in expected.items()): + return capture( + sql( + container, + "SET GLOBAL recyclebin=OFF; SELECT VERSION(); SHOW VARIABLES LIKE 'ob_compatibility_mode'; SHOW VARIABLES LIKE 'recyclebin';", + database=database, + ), + output / "engine.txt", + deadline, + ) + time.sleep(min(2, max(0, config_deadline - time.monotonic()))) + print("OceanBase datafile configuration did not become effective before its deadline.", flush=True) + return 124 + + +def run_command( + command: list[str], *, container: str, output: Path, timeout: float, stdout: IO[str] | None = None +) -> int: + """Keep GNU timeout's process deadline while collecting evidence before TERM.""" + output.mkdir(parents=True, exist_ok=True) + started = time.monotonic() + bounded = ["timeout", "--signal=TERM", "--kill-after=10s", f"{timeout:g}s", *command] + cancelled_signal: int | None = None + with subprocess.Popen(bounded, stdout=stdout, stderr=subprocess.STDOUT) as process: # noqa: S603 + # Forward Actions cancellation to GNU timeout so its normal process-group + # termination and kill-after grace remain effective through this wrapper. + def forward_signal(number: int, _frame) -> None: + nonlocal cancelled_signal + cancelled_signal = number + process.send_signal(number) + + previous = {number: signal.getsignal(number) for number in (signal.SIGINT, signal.SIGTERM)} + for number in previous: + signal.signal(number, forward_signal) + try: + try: + code = process.wait(timeout=max(0, timeout - min(30, timeout / 2))) + except subprocess.TimeoutExpired: + # GNU timeout still enforces the original deadline even if a + # diagnostic SQL probe is unresponsive. Collection never retries tests. + collect(container, output / "before-timeout", budget=min(20, timeout / 3)) + code = process.wait() + finally: + for number, handler in previous.items(): + signal.signal(number, handler) + if cancelled_signal is not None: + code = 128 + cancelled_signal + (output / "execution.json").write_text( + json.dumps( + { + "exit_code": code, + "cancelled_signal": cancelled_signal, + "deadline_seconds": timeout, + "elapsed_seconds": time.monotonic() - started, + }, + indent=2, + ) + + "\n" + ) + return code + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("operation", choices=("wait", "collect", "run")) + parser.add_argument("--container", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--database", default="powercontext_test") + parser.add_argument("--timeout", type=float, default=600) + # parse_known_args keeps the command after -- intact. + args, command = parser.parse_known_args() + if ( + not math.isfinite(args.timeout) + or args.timeout <= 0 + or (command and (args.operation != "run" or command[0] != "--")) + ): + parser.error("A positive timeout and a command after -- (run only) are required") + if args.operation == "run": + if len(command) < 2: + parser.error("run requires a command after --") + return run_command(command[1:], container=args.container, output=args.output, timeout=args.timeout) + if args.operation == "wait": + code = wait_ready(args.container, args.database, args.output, args.timeout) + if code: + collect(args.container, args.output / "failure", budget=20) + return code + collect(args.container, args.output) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_atomic_memory_clean_migration.py b/scripts/run_atomic_memory_clean_migration.py index 6c16aac14d..742318f33f 100644 --- a/scripts/run_atomic_memory_clean_migration.py +++ b/scripts/run_atomic_memory_clean_migration.py @@ -18,11 +18,17 @@ import argparse import json +import math +import os +import statistics import subprocess import sys +import time from pathlib import Path from xml.etree import ElementTree +from oceanbase_ci import run_command + SUITES = ( "tests/e2e/test_atomic_memory_migration_evidence.py", "tests/e2e/test_atomic_memory_grant_migration.py", @@ -42,24 +48,54 @@ def parse_shard(value: str) -> tuple[int, int]: raise argparse.ArgumentTypeError(message) -def run_pytest(arguments: list[str], log: Path, *, timeout_seconds: int = 180) -> int: +def read_durations(path: Path | None) -> dict[str, float] | None: + if path is None: + return None + values = json.loads(path.read_text())["durations_seconds"] + if ( + not isinstance(values, dict) + or not values + or any( + not isinstance(node, str) or type(seconds) not in (int, float) or not math.isfinite(seconds) or seconds <= 0 + for node, seconds in values.items() + ) + ): + raise ValueError("Expected positive, finite per-node durations") # noqa: TRY003 + return values + + +def assigned_nodes(nodes: list[str], index: int, count: int, durations: dict[str, float] | None) -> list[str]: + """Pack expensive cases first; unknown cases remain covered using the median estimate.""" + if durations is None: + return nodes[index::count] + fallback = statistics.median(durations.values()) + loads = [0.0] * count + assignments: dict[str, int] = {} + for node in sorted(nodes, key=lambda node: (-durations.get(node, fallback), node)): + shard = min(range(count), key=lambda shard: (loads[shard], shard)) + assignments[node] = shard + loads[shard] += durations.get(node, fallback) + # Preserve collection order within each independent database instance. + return [node for node in nodes if assignments[node] == index] + + +def run_pytest(arguments: list[str], log: Path, *, timeout_seconds: int = 180, container: str | None = None) -> int: # GNU timeout also terminates native child processes; never retry a failed node. - command = [ - "timeout", - "--signal=TERM", - "--kill-after=10s", - f"{timeout_seconds}s", - sys.executable, - "-m", - "pytest", - "--color=no", - *arguments, - ] + command = [sys.executable, "-m", "pytest", "--color=no", *arguments] with log.open("w") as output: + if container: + return run_command( + command, + container=container, + output=log.with_suffix(".diagnostics"), + timeout=timeout_seconds, + stdout=output, + ) + command = ["timeout", "--signal=TERM", "--kill-after=10s", f"{timeout_seconds}s", *command] return subprocess.run(command, stdout=output, stderr=subprocess.STDOUT, check=False).returncode # noqa: S603 -def run_node(node: str, directory: Path, *, timeout_seconds: int) -> bool: +def run_node(node: str, directory: Path, *, timeout_seconds: int, container: str | None = None) -> bool: directory.mkdir() (directory / "node.txt").write_text(node + "\n") report = directory / "junit.xml" @@ -82,6 +118,7 @@ def run_node(node: str, directory: Path, *, timeout_seconds: int) -> bool: ], log, timeout_seconds=timeout_seconds, + container=container, ) passed = False try: @@ -106,6 +143,7 @@ def main() -> int: parser.add_argument("backend", choices=("seekdb", "oceanbase")) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--collect-only", action="store_true") + parser.add_argument("--durations", type=Path, help="Measured OceanBase per-node durations for balanced assignment") parser.add_argument( "--test-timeout", type=int, @@ -119,16 +157,22 @@ def main() -> int: args = parser.parse_args() if args.test_timeout <= 0: parser.error("--test-timeout must be a positive number of seconds") + if args.durations and args.backend != "oceanbase": + parser.error("Duration-based scheduling is only enabled for OceanBase") + durations = read_durations(args.durations) + container = os.environ.get("OCEANBASE_CI_CONTAINER") if args.backend == "oceanbase" else None output = args.output.resolve() output.mkdir(parents=True, exist_ok=False) shard_index, shard_count = args.shard passed_nodes: list[str] = [] + elapsed_seconds: dict[str, float] = {} summary = { "backend": args.backend, "shard_index": shard_index, "shard_count": shard_count, "collected_nodes": [], "passed_nodes": passed_nodes, + "elapsed_seconds": elapsed_seconds, "complete": False, } summary_path = output / "summary.json" @@ -153,13 +197,16 @@ def main() -> int: print(f"Collected {len(nodes)} {args.backend} cases.", flush=True) if args.collect_only: return 0 - selected = nodes[shard_index::shard_count] + selected = assigned_nodes(nodes, shard_index, shard_count, durations) print(f"Shard {shard_index}/{shard_count}: running {len(selected)} cases.", flush=True) # SeekDB must exit between cases. OceanBase cases share one tenant, so keep # their schema creation and cleanup separate from other test cases. try: for index, node in enumerate(selected, 1): - if not run_node(node, output / f"{index:03d}", timeout_seconds=args.test_timeout): + started = time.monotonic() + passed = run_node(node, output / f"{index:03d}", timeout_seconds=args.test_timeout, container=container) + elapsed_seconds[node] = round(time.monotonic() - started, 3) + if not passed: return 1 passed_nodes.append(node) summary["complete"] = True diff --git a/tests/test_ci_changes.py b/tests/test_ci_changes.py index f08d11b24c..d408c76a20 100644 --- a/tests/test_ci_changes.py +++ b/tests/test_ci_changes.py @@ -60,6 +60,11 @@ (["scripts/scan_e2e_evidence.sh"], {"tests", "harness"}), (["scripts/run_atomic_memory_clean_migration.py"], {"tests", "database", "upgrade"}), (["scripts/check_atomic_memory_migration_shards.py"], {"tests", "database"}), + ([".github/oceanbase-migration-durations.json"], {"tests", "database"}), + ( + ["scripts/oceanbase_ci.py"], + {"tests", "database", "harness", "upgrade", "candidate_replay", "release_schema"}, + ), (["scripts/sync_dsh_plugin_tree.py"], {"tests", "dsh"}), (["desktop/ui/src/app/App.tsx"], {"desktop"}), (["release/upgrade-support.json"], {"upgrade", "packaging"}), diff --git a/tests/test_oceanbase_ci.py b/tests/test_oceanbase_ci.py new file mode 100644 index 0000000000..cef66666bd --- /dev/null +++ b/tests/test_oceanbase_ci.py @@ -0,0 +1,183 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CI process budgets and migration coverage, independent of a live database.""" + +from __future__ import annotations + +import json +import os +import shutil +import signal +import subprocess +import sys +import time +from pathlib import Path + +import pytest +from check_atomic_memory_migration_shards import verify_group +from oceanbase_ci import collect, run_command, wait_ready +from run_atomic_memory_clean_migration import assigned_nodes, read_durations + + +@pytest.fixture +def docker_probe(tmp_path, monkeypatch): + """Exercise real subprocess deadlines with a controlled Docker command boundary.""" + if sys.platform == "win32": + pytest.skip("OceanBase CI runs on POSIX runners") + executable = tmp_path / "docker" + executable.write_text( + f"#!{sys.executable}\n" + "import os, sys, time\n" + "mode = os.environ.get('CI_PROBE_MODE', 'ready')\n" + "args = sys.argv[1:]\n" + "if mode == 'hang': time.sleep(60)\n" + "if args[0] == 'inspect':\n" + " print('false' if mode == 'exited' else 'true')\n" + "elif args[0] == 'exec':\n" + " if mode == 'exited': sys.exit(1)\n" + " if 'SHOW PARAMETERS' in args[-1]:\n" + " for name, value in [('datafile_size', '4G'), ('datafile_maxsize', '8G'), ('datafile_next', '512M')]:\n" + " print('server\\t2882\\tzone\\tscope\\t'+name+'\\ttype\\t'+value)\n" + " else: print('1')\n" + ) + executable.chmod(0o755) + monkeypatch.setenv("PATH", str(tmp_path) + os.pathsep + os.environ["PATH"]) + return tmp_path / "diagnostics" + + +def test_startup_checks_configuration_and_exits_when_the_container_stops(docker_probe, monkeypatch): + assert wait_ready("test", "test", docker_probe, 5) == 0 + assert (docker_probe / "engine.txt").read_text().strip() == "1" + monkeypatch.setenv("CI_PROBE_MODE", "exited") + assert wait_ready("test", "test", docker_probe, 5) == 1 + + +def test_unresponsive_startup_and_diagnostics_respect_overall_deadlines(docker_probe, monkeypatch): + monkeypatch.setenv("CI_PROBE_MODE", "hang") + started = time.monotonic() + assert wait_ready("test", "test", docker_probe, 0.2) == 124 + collect("test", docker_probe / "failure", budget=0.2) + assert time.monotonic() - started < 3 + snapshot = json.loads((docker_probe / "failure/snapshot.json").read_text()) + assert set(snapshot["exit_codes"].values()) == {124} + + +@pytest.mark.parametrize(("exit_code", "delay", "expected"), [(0, 0, 0), (7, 0, 7), (0, 60, 124)]) +def test_command_preserves_failures_and_collects_before_timeout(docker_probe, monkeypatch, exit_code, delay, expected): + timeout = shutil.which("timeout") or shutil.which("gtimeout") + if timeout is None: + pytest.skip("GNU timeout is required by the Linux CI command supervisor") + (docker_probe.parent / "timeout").symlink_to(timeout) + # Diagnostic failure must not change the test exit status or prevent termination. + monkeypatch.setenv("CI_PROBE_MODE", "hang") + report = docker_probe / "command" + started = time.monotonic() + code = run_command( + [sys.executable, "-c", f"import sys,time; time.sleep({delay}); sys.exit({exit_code})"], + container="test", + output=report, + timeout=2, + ) + assert code == expected + assert time.monotonic() - started < 5 + assert json.loads((report / "execution.json").read_text())["exit_code"] == expected + assert (report / "before-timeout/snapshot.json").exists() is (expected == 124) + + +def test_cancellation_reaches_the_command_without_waiting_for_its_deadline(docker_probe): + timeout = shutil.which("timeout") or shutil.which("gtimeout") + if timeout is None: + pytest.skip("GNU timeout is required by the Linux CI command supervisor") + (docker_probe.parent / "timeout").symlink_to(timeout) + marker = docker_probe.parent / "started" + process = subprocess.Popen([ + sys.executable, + str(Path(__file__).parents[1] / "scripts/oceanbase_ci.py"), + "run", + "--container", + "test", + "--output", + str(docker_probe), + "--timeout", + "60", + "--", + sys.executable, + "-c", + f"from pathlib import Path; import time; Path({str(marker)!r}).touch(); time.sleep(60)", + ]) + try: + deadline = time.monotonic() + 10 + while not marker.exists() and time.monotonic() < deadline: + time.sleep(0.02) + assert marker.exists() + process.send_signal(signal.SIGTERM) + assert process.wait(timeout=5) == 143 + finally: + if process.poll() is None: + process.kill() + process.wait() + + +def test_duration_assignment_balances_work_and_includes_unknown_nodes(): + nodes = ["slow-a", "slow-b", "fast-a", "fast-b", "new-case"] + durations = {"slow-a": 100.0, "slow-b": 90.0, "fast-a": 5.0, "fast-b": 5.0} + assignments = [assigned_nodes(nodes, index, 2, durations) for index in range(2)] + assert sorted(node for group in assignments for node in group) == sorted(nodes) + assert not set(assignments[0]) & set(assignments[1]) + assert ("slow-a" in assignments[0]) != ("slow-b" in assignments[0]) + assert assignments == [assigned_nodes(nodes, index, 2, durations) for index in range(2)] + # No duration file preserves the seekDB/default round-robin contract. + assert assigned_nodes(nodes, 1, 3, None) == nodes[1::3] + + +@pytest.mark.parametrize("corruption", [None, "missing", "duplicate", "incomplete", "collection"]) +def test_balanced_shards_must_cover_every_collected_node_exactly_once(tmp_path, corruption): + nodes = ["slow", "fast", "new"] + durations = {"slow": 100.0, "fast": 1.0} + paths = [] + for index in range(2): + passed = assigned_nodes(nodes, index, 2, durations) + report = { + "backend": "oceanbase", + "shard_index": index, + "shard_count": 2, + "complete": True, + "collected_nodes": nodes.copy(), + "passed_nodes": passed, + } + if index == 0: + if corruption == "missing": + passed.pop() + elif corruption == "duplicate": + passed.append(passed[0]) + elif corruption == "incomplete": + report["complete"] = False + elif corruption == "collection": + report["collected_nodes"] = [*nodes, "unexpected"] + path = tmp_path / f"{index}.json" + path.write_text(json.dumps(report)) + paths.append(path) + if corruption: + with pytest.raises(ValueError): + verify_group(paths, 2, durations) + else: + assert verify_group(paths, 2, durations) == len(nodes) + + +def test_invalid_duration_baseline_cannot_silently_change_scheduling(tmp_path): + path = tmp_path / "durations.json" + path.write_text(json.dumps({"durations_seconds": {"test": -1}})) + with pytest.raises(ValueError, match="positive"): + read_durations(path) From a9fc9f1fdfdfcce541f451cd02ee17271359c355 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Fri, 6 Nov 2026 15:57:59 +0800 Subject: [PATCH 02/25] fix(ci): preserve test results when diagnostics cannot be written --- scripts/oceanbase_ci.py | 66 ++++++++++++++++++++++---------------- tests/test_oceanbase_ci.py | 38 ++++++++++++++++++++++ 2 files changed, 77 insertions(+), 27 deletions(-) diff --git a/scripts/oceanbase_ci.py b/scripts/oceanbase_ci.py index 69570af491..75dddb5be3 100644 --- a/scripts/oceanbase_ci.py +++ b/scripts/oceanbase_ci.py @@ -21,6 +21,7 @@ import math import signal import subprocess +import sys import time from pathlib import Path from typing import IO @@ -71,9 +72,22 @@ def capture(command: list[str], path: Path, deadline: float, *, probe_timeout: f return result.returncode +def write_report(path: Path, report: dict[str, object]) -> None: + """Diagnostic storage failures must not replace the observed command result.""" + try: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(report, indent=2) + "\n") + except OSError as error: + print(f"OceanBase CI diagnostic report unavailable: {type(error).__name__}", file=sys.stderr) + + def collect(container: str, output: Path, *, budget: float = 30) -> None: """Best-effort bounded snapshots; omit statement text, row keys and container env.""" - output.mkdir(parents=True, exist_ok=True) + try: + output.mkdir(parents=True, exist_ok=True) + except OSError as error: + print(f"OceanBase CI diagnostic directory unavailable: {type(error).__name__}", file=sys.stderr) + return started = time.monotonic() queries = { "sessions": "SELECT ID, USER, DB, COMMAND, TIME, STATE FROM information_schema.PROCESSLIST LIMIT 100", @@ -87,20 +101,22 @@ def collect(container: str, output: Path, *, budget: float = 30) -> None: "disk": ["docker", "exec", container, "df", "-h", "/root/demo"], "state": ["docker", "inspect", "--format", "{{json .State}}", container], }) - results = { - name: capture(command, output / f"{name}.txt", started + budget, probe_timeout=min(3, budget / len(commands))) - for name, command in commands.items() - } - (output / "snapshot.json").write_text( - json.dumps( - { - "captured_at_unix_seconds": time.time(), - "elapsed_seconds": time.monotonic() - started, - "exit_codes": results, - }, - indent=2, - ) - + "\n" + results = {} + for name, command in commands.items(): + try: + results[name] = capture( + command, output / f"{name}.txt", started + budget, probe_timeout=min(3, budget / len(commands)) + ) + except OSError as error: + results[name] = 74 # EX_IOERR: the diagnostic output could not be written. + print(f"OceanBase CI diagnostic probe unavailable: {type(error).__name__}", file=sys.stderr) + write_report( + output / "snapshot.json", + { + "captured_at_unix_seconds": time.time(), + "elapsed_seconds": time.monotonic() - started, + "exit_codes": results, + }, ) @@ -158,7 +174,6 @@ def run_command( command: list[str], *, container: str, output: Path, timeout: float, stdout: IO[str] | None = None ) -> int: """Keep GNU timeout's process deadline while collecting evidence before TERM.""" - output.mkdir(parents=True, exist_ok=True) started = time.monotonic() bounded = ["timeout", "--signal=TERM", "--kill-after=10s", f"{timeout:g}s", *command] cancelled_signal: int | None = None @@ -186,17 +201,14 @@ def forward_signal(number: int, _frame) -> None: signal.signal(number, handler) if cancelled_signal is not None: code = 128 + cancelled_signal - (output / "execution.json").write_text( - json.dumps( - { - "exit_code": code, - "cancelled_signal": cancelled_signal, - "deadline_seconds": timeout, - "elapsed_seconds": time.monotonic() - started, - }, - indent=2, - ) - + "\n" + write_report( + output / "execution.json", + { + "exit_code": code, + "cancelled_signal": cancelled_signal, + "deadline_seconds": timeout, + "elapsed_seconds": time.monotonic() - started, + }, ) return code diff --git a/tests/test_oceanbase_ci.py b/tests/test_oceanbase_ci.py index cef66666bd..d421e60393 100644 --- a/tests/test_oceanbase_ci.py +++ b/tests/test_oceanbase_ci.py @@ -74,6 +74,44 @@ def test_unresponsive_startup_and_diagnostics_respect_overall_deadlines(docker_p assert set(snapshot["exit_codes"].values()) == {124} +def test_one_unwritable_probe_does_not_discard_other_diagnostics(docker_probe): + (docker_probe / "sessions.txt").mkdir(parents=True) + collect("test", docker_probe) + snapshot = json.loads((docker_probe / "snapshot.json").read_text()) + assert snapshot["exit_codes"]["sessions"] != 0 + assert snapshot["exit_codes"]["transactions"] == 0 + assert (docker_probe / "transactions.txt").read_text().strip() == "1" + + +@pytest.mark.parametrize( + ("exit_code", "delay", "blocked_path", "expected"), + [(0, 0, "directory", 0), (7, 0, "execution.json", 7), (0, 60, "before-timeout", 124)], +) +def test_diagnostic_storage_failure_preserves_the_command_result( + docker_probe, exit_code, delay, blocked_path, expected +): + timeout = shutil.which("timeout") or shutil.which("gtimeout") + if timeout is None: + pytest.skip("GNU timeout is required by the Linux CI command supervisor") + (docker_probe.parent / "timeout").symlink_to(timeout) + if blocked_path == "directory": + docker_probe.write_text("A file prevents creating the diagnostic directory") + elif blocked_path == "execution.json": + (docker_probe / blocked_path).mkdir(parents=True) + else: + docker_probe.mkdir() + (docker_probe / blocked_path).write_text("A file prevents creating the snapshot directory") + assert ( + run_command( + [sys.executable, "-c", f"import sys,time; time.sleep({delay}); sys.exit({exit_code})"], + container="test", + output=docker_probe, + timeout=4, + ) + == expected + ) + + @pytest.mark.parametrize(("exit_code", "delay", "expected"), [(0, 0, 0), (7, 0, 7), (0, 60, 124)]) def test_command_preserves_failures_and_collects_before_timeout(docker_probe, monkeypatch, exit_code, delay, expected): timeout = shutil.which("timeout") or shutil.which("gtimeout") From d8492f56aae3ffd092c8ac81946ad314dc83d9b0 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Fri, 6 Nov 2026 22:59:47 +0800 Subject: [PATCH 03/25] test(observability): await matching worker failure statistics --- tests/e2e/test_worker_otlp.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/tests/e2e/test_worker_otlp.py b/tests/e2e/test_worker_otlp.py index 5b9d14c0f3..8f143de711 100644 --- a/tests/e2e/test_worker_otlp.py +++ b/tests/e2e/test_worker_otlp.py @@ -306,14 +306,24 @@ def test_builtin_spawned_worker_exports_model_and_execution_spans_to_otlp( ) if timeout: assert outcomes["powercontext.operation.outcome"] == "failure" - stats = client.post( - "/v1/stats", - json={"selection": {"mode": "exact", "scope_ids": [scope]}, "include_processing": True}, - ) - assert stats.status_code == 200 - failed = next(row for row in stats.json()["by_scope"][0]["processing"] if row["family"] == family) - assert failed["last_error_worker_id"] - assert failed["last_error_trace_id"] == execution.trace_id.hex() + # Spans finish before the supervisor reaps the worker and publishes + # its failure state. Wait for this execution within the original deadline. + while True: + stats = client.post( + "/v1/stats", + json={"selection": {"mode": "exact", "scope_ids": [scope]}, "include_processing": True}, + ) + assert stats.status_code == 200 + failed = next( + row for row in stats.json()["by_scope"][0]["processing"] if row["family"] == family + ) + if ( + failed["last_error_worker_id"] and failed["last_error_trace_id"] == execution.trace_id.hex() + ) or time.monotonic() >= deadline: + break + time.sleep(min(0.05, max(0, deadline - time.monotonic()))) + assert failed["last_error_worker_id"], failed + assert failed["last_error_trace_id"] == execution.trace_id.hex(), failed assert outcomes["powercontext.error.category"] == "timeout" else: assert outcomes["powercontext.operation.outcome"] == "success", [ From 101733202483d9a31768ec52c2e2d5651149bba2 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Sat, 7 Nov 2026 00:41:49 +0800 Subject: [PATCH 04/25] ci: scope OceanBase acceptance independently from harness replay --- .github/ci-paths.toml | 24 ++++++- .github/workflows/e2e-harness.yml | 104 ++++++++++++++++++++++++--- .github/workflows/master.yml | 5 +- Makefile | 8 ++- docs/en/development/ci-scope.md | 19 +++++ docs/zh/development/ci-scope.md | 19 +++++ scripts/ci_changes.py | 1 + tests/test_ci_changes.py | 58 ++++++++++++++- tests/test_e2e_workflow_contract.py | 106 +++++++++++++++++++++++++++- 9 files changed, 326 insertions(+), 18 deletions(-) diff --git a/.github/ci-paths.toml b/.github/ci-paths.toml index 7b6e359b23..f100edd647 100644 --- a/.github/ci-paths.toml +++ b/.github/ci-paths.toml @@ -74,8 +74,15 @@ native = ["tests/native/**", "tests/test_service*.py", "tests/test_cursor_secret harness = [ "e2e/**", "integrations/bub/**", "scripts/scan_e2e_evidence.sh", "tests/test_e2e_workflow_contract.py", ".github/workflows/e2e-harness.yml", +] +oceanbase_acceptance = [ + "e2e/bub/compose*.yaml", "e2e/bub/run.sh", + ".github/workflows/e2e-harness.yml", "tests/test_e2e_workflow_contract.py", "tests/e2e/test_atomic_memory_search_authorization_snapshot.py", "tests/e2e/test_atomic_memory_read_snapshot.py", - "tests/e2e/test_runtime_server.py", + "tests/e2e/atomic_memory_migration_backend.py", "tests/e2e/test_runtime_server.py", + "tests/e2e/test_server_metrics_oceanbase.py", "tests/e2e/test_portable_archive.py", + "tests/e2e/server_storage.py", "tests/e2e/dream_support.py", "tests/builtin/persistence/contract.py", + "tests/e2e/atomic_memory_migration_fixture.py", "tests/legacy_memory.py", ] # ZCode acceptance also exercises the Pi plugin and cross-host handoff fixtures. zcode = [ @@ -138,6 +145,18 @@ skill_guidance = [ ".github/workflows/skill-guidance.yml", ] +# Audited SQLite/in-memory tests retain ordinary unit/e2e coverage. Exact paths +# avoid exempting future native tests matched by the broad database patterns. +# Exclusions affect direct matches only; shared and full-scope rules take precedence. +# Remove a file from this list if it gains native database cases. +[exclusions] +database = [ + "tests/builtin/runtime/test_recall_sufficiency.py", + "tests/builtin/runtime/test_statistics.py", "tests/builtin/runtime/test_model_usage_recorder.py", + "tests/builtin/runtime/test_prepared_context.py", "tests/e2e/test_recall_sufficiency_gate.py", + "tests/e2e/test_context_text_assembly.py", +] + # Packaged guidance and its generators are inputs to the native host tests. [shared.guidance] patterns = [ @@ -152,7 +171,7 @@ suites = ["tests", "dify", "hermes", "dsh", "pi", "opencode", "openclaw", "zcode patterns = ["src/**", "openapi/**", "pyproject.toml", "uv.lock"] suites = [ "tests", "evaluation", "database", "dify", "hermes", "dsh", "cindy", "pi", "opencode", "openclaw", "opendal", - "website", "native", "harness", "zcode", "kimi", "upgrade", "observability", "packaging", "desktop", + "website", "native", "harness", "oceanbase_acceptance", "zcode", "kimi", "upgrade", "observability", "packaging", "desktop", "managed_resources", ] @@ -199,6 +218,7 @@ website = ["check-website", "website-link-validation-windows"] docs_tests = ["docs-tests"] native = ["native-service"] harness = ["harness-validation"] +oceanbase_acceptance = ["harness-validation"] zcode = ["zcode-acceptance"] kimi = ["kimi-acceptance"] upgrade = ["seekdb-managed-startup", "upgrade-acceptance"] diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index 815c481a2e..37aa4daafa 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -7,6 +7,10 @@ on: description: Scope already selected by the Main orchestrator required: true type: string + preflight-checked: + description: Main has already passed repository-wide lint and formatting checks + type: boolean + default: false suite: type: string default: all @@ -43,9 +47,13 @@ jobs: needs: changes if: >- ${{ !cancelled() && (github.event_name != 'workflow_dispatch' || inputs.suite != 'recall-effort') && - ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').harness) }} + ((inputs.suites == '' && needs.changes.result != 'success') || + fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').harness || + fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').oceanbase_acceptance) }} runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'powercontext-ci-light' || 'ubuntu-24.04' }} timeout-minutes: 15 + outputs: + matrix: ${{ steps.plan.outputs.matrix }} steps: - name: Check out uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -53,7 +61,32 @@ jobs: fetch-depth: 0 persist-credentials: false + - name: Plan selected acceptance work + id: plan + env: + CI_SUITES: ${{ inputs.suites || needs.changes.outputs.suites || '{"harness":true,"oceanbase_acceptance":true}' }} + run: | + python3 - <<'PY' + import json + import os + + suites = json.loads(os.environ['CI_SUITES']) + replay = suites.get('harness', False) + native = suites.get('oceanbase_acceptance', False) + include = [] + if replay: + include.append({'database': 'sqlite', 'replay': True, 'native': False}) + if replay or native: + include.append({'database': 'oceanbase', 'replay': replay, 'native': native}) + if not include: + raise SystemExit('Acceptance was selected without any work') + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + output.write(f'matrix={json.dumps({"include": include})}\n') + output.write(f'replay={json.dumps(replay)}\n') + PY + - name: Install uv + if: steps.plan.outputs.replay == 'true' uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 with: version: ${{ env.UV_VERSION }} @@ -63,12 +96,18 @@ jobs: # checks out with tags because the root version is derived from them, which is what makes that # class of failure visible. - name: Verify dependencies resolve from scratch + if: steps.plan.outputs.replay == 'true' run: uv lock --upgrade --dry-run + - name: Check harness formatting for standalone runs + if: steps.plan.outputs.replay == 'true' && !inputs.preflight-checked + run: make harness-lint + - name: Validate harness and Compose definitions + if: steps.plan.outputs.replay == 'true' run: | make harness-sync - make harness-check + make harness-test make harness-compose-check recall-effort: @@ -545,12 +584,11 @@ jobs: # Main supplies scope, so the detector ancestor is intentionally skipped. if: ${{ !cancelled() && needs.validate.result == 'success' }} runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'powercontext-ci-standard' || 'ubuntu-24.04' }} - # Reserve 50m for OceanBase regressions, 15m for replay, and 10m for setup/evidence/cleanup. + # Keep the existing overall budget; selected suites share setup and cleanup. timeout-minutes: ${{ matrix.database == 'oceanbase' && 75 || 40 }} strategy: fail-fast: false - matrix: - database: [sqlite, oceanbase] + matrix: ${{ fromJSON(needs.validate.outputs.matrix) }} name: Acceptance (${{ matrix.database }}) steps: - name: Check out @@ -564,9 +602,9 @@ jobs: with: version: ${{ env.UV_VERSION }} - - name: Verify OceanBase Atomic Memory, HTTP and archive regressions - # The 22 tests alone took 39m; this step also starts the database and installs dependencies. - timeout-minutes: 50 + - name: Start and configure the shared OceanBase instance + id: oceanbase_setup + timeout-minutes: 15 if: matrix.database == 'oceanbase' env: COMPOSE_PROJECT_NAME: powercontext-e2e-oceanbase @@ -611,6 +649,14 @@ jobs: test -n "$oceanbase_ip" export POWERCONTEXT_TEST_OCEANBASE_URL="mysql+aoceanbase://root%40test:powercontext-e2e@${oceanbase_ip}:2881/powercontext?charset=utf8mb4" export POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL="$POWERCONTEXT_TEST_OCEANBASE_URL" + echo "POWERCONTEXT_TEST_OCEANBASE_URL=$POWERCONTEXT_TEST_OCEANBASE_URL" >> "$GITHUB_ENV" + echo "POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL=$POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL" >> "$GITHUB_ENV" + + - name: Verify OceanBase Atomic Memory, HTTP and archive regressions + id: native_acceptance + timeout-minutes: 50 + if: matrix.native + run: | uv run --locked --extra server python - <<'PY' import asyncio import os @@ -653,6 +699,7 @@ jobs: PY - name: Run deterministic scenarios + if: matrix.replay timeout-minutes: ${{ matrix.database == 'oceanbase' && 15 || 40 }} id: acceptance_scenarios env: @@ -662,7 +709,7 @@ jobs: - name: Scan replay evidence id: evidence_scan - if: always() + if: always() && matrix.replay env: DATABASE: ${{ matrix.database }} DIAGNOSTICS_PATH: ${{ runner.temp }}/powercontext-e2e-diagnostics-${{ matrix.database }} @@ -672,7 +719,7 @@ jobs: - name: Upload sanitized replay diagnostics if: >- - always() && + always() && matrix.replay && (steps.evidence_scan.outcome != 'success' || steps.acceptance_scenarios.outcome != 'success') uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 @@ -683,7 +730,8 @@ jobs: retention-days: 14 - name: Upload replay evidence - if: always() && steps.evidence_scan.outcome == 'success' + id: replay_evidence + if: always() && matrix.replay && steps.evidence_scan.outcome == 'success' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: e2e-acceptance-${{ matrix.database }}-${{ github.sha }} @@ -692,6 +740,40 @@ jobs: if-no-files-found: error retention-days: 14 + - name: Upload native acceptance results + id: native_results + if: always() && matrix.native + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: oceanbase-acceptance-results-${{ github.sha }} + path: ${{ runner.temp }}/atomic-memory-oceanbase-snapshots.xml + if-no-files-found: error + retention-days: 14 + + - name: Require all selected acceptance work and evidence + if: always() + env: + ACCEPTANCE_PLAN: ${{ toJSON(matrix) }} + STEP_RESULTS: ${{ toJSON(steps) }} + run: | + python3 - <<'PY' + import json + import os + + plan = json.loads(os.environ['ACCEPTANCE_PLAN']) + results = json.loads(os.environ['STEP_RESULTS']) + required = ['oceanbase_setup'] if plan['database'] == 'oceanbase' else [] + if plan['native']: + required.extend(['native_acceptance', 'native_results']) + if plan['replay']: + required.extend(['acceptance_scenarios', 'evidence_scan', 'replay_evidence']) + failed = {step: results.get(step, {}).get('outcome', 'missing') for step in required + if results.get(step, {}).get('outcome') != 'success'} + if failed: + raise SystemExit(f'Selected acceptance work did not succeed: {failed}') + print('All selected acceptance work and evidence passed.') + PY + - name: Stop harness environment if: always() env: diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 5fdf5e0d97..4043314ddf 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -1144,10 +1144,13 @@ jobs: if: >- ${{ !cancelled() && needs.quality.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.tests.result) && - (fromJSON(needs.quality.outputs.suites || '{}').database || fromJSON(needs.quality.outputs.suites || '{}').harness) }} + (fromJSON(needs.quality.outputs.suites || '{}').database || + fromJSON(needs.quality.outputs.suites || '{}').harness || + fromJSON(needs.quality.outputs.suites || '{}').oceanbase_acceptance) }} uses: ./.github/workflows/e2e-harness.yml with: suites: ${{ needs.quality.outputs.suites }} + preflight-checked: true gateway-validation: needs: [quality, tests] diff --git a/Makefile b/Makefile index 6cdb994e07..e0be08b822 100644 --- a/Makefile +++ b/Makefile @@ -114,9 +114,15 @@ opendal-test: ## Validate the standalone OpenDAL Connector against this checkout @$(OPENDAL_TEST_RUN) powercontext-connector-opendal --help >/dev/null .PHONY: harness-check -harness-check: ## Validate the Bub replay harness and committed scenarios. +harness-check: harness-lint harness-test ## Validate the Bub replay harness and committed scenarios. + +.PHONY: harness-lint +harness-lint: ## Check harness formatting (also covered by Main's preflight). @uv run ruff check e2e/bub @uv run ruff format --check e2e/bub + +.PHONY: harness-test +harness-test: ## Check harness types, scenarios and CLI without repeating formatting. @uv run ty check --project e2e/bub --python e2e/bub/.venv --python-version 3.12 e2e/bub/src integrations/bub/src @uv run --project e2e/bub python -m pytest e2e/bub/tests @uv run --project e2e/bub powercontext-e2e --help >/dev/null diff --git a/docs/en/development/ci-scope.md b/docs/en/development/ci-scope.md index 99c6514695..0e52d14c21 100644 --- a/docs/en/development/ci-scope.md +++ b/docs/en/development/ci-scope.md @@ -63,6 +63,25 @@ Helm charts select deployment-contract checks. Managed resource producers select Shared helpers select their actual consumers: the clean-migration runner also selects managed-startup and upgrade acceptance; the pnpm cache action also selects the Python test setup and Cindy package. A path filter must follow dependencies, not only directory names. +## Harness replay and OceanBase acceptance + +The reusable E2E workflow selects replay (`harness`) separately from native snapshot, HTTP, metrics and archive regressions (`oceanbase_acceptance`). + +| Changed input | Replay workers | Native OceanBase acceptance | +| --- | --- | --- | +| Bub adapter or replay scenarios | SQLite and OceanBase | Not selected | +| Native HTTP, metrics or archive test files | None | Selected | +| Shared Compose definitions or harness startup script | SQLite and OceanBase | Selected | +| Shared runtime, API or root dependencies | SQLite and OceanBase | Selected, alongside the other affected suites | + +When both scopes are selected, their OceanBase work shares one worker and container. Database startup and capacity configuration run for either scope. Native-only changes do not allocate a SQLite replay worker or install the Bub validation environment. Selected tests and evidence must succeed; missing reports and unexpected skips fail the acceptance gate. Native JUnit results are retained independently of replay artifacts. + +Main's preflight covers repository-wide lint and formatting. Its harness validation runs `make harness-test`, retaining harness type checks, scenario tests and CLI validation, plus Compose validation. Standalone runs retain formatting through `make harness-lint`; local `make harness-check` runs both targets. + +The exact test paths in `[exclusions].database` use SQLite or in-memory behavior and retain ordinary Python coverage, plus any other matching scope. Their test-only edits do not start the broad database suite. Remove an exclusion if that file gains native database cases. Exclusions cannot suppress another changed file, shared fixtures, shared runtime selection or the full fallback. Selected upgrade acceptance still runs all 12 configurations automatically. + +These rules reduce unnecessary work for local changes. Shared-runtime and CI-policy changes retain broad coverage; runner minutes and queue latency require measurement after rollout. + ## Parallelism and cancellation budgets | Matrix | Maximum parallel jobs per run | Failure behavior | diff --git a/docs/zh/development/ci-scope.md b/docs/zh/development/ci-scope.md index d4f18804b4..77bf3f5e41 100644 --- a/docs/zh/development/ci-scope.md +++ b/docs/zh/development/ci-scope.md @@ -63,6 +63,25 @@ Helm chart 改动选择部署合约检查。托管资源生产器通过 Main 选 共享辅助文件按实际消费者选择：干净迁移 runner 脚本同时触发 managed-startup 和升级验收；pnpm 缓存 action 同时覆盖 Python 测试环境及 Cindy 包。路径规则需要沿依赖关系判定，不能只看目录名称。 +## Harness 回放与 OceanBase 验收 + +可复用 E2E 工作流分别选择回放范围 `harness`，以及原生快照、HTTP、指标和归档回归范围 `oceanbase_acceptance`。 + +| 变更输入 | 回放任务 | 原生 OceanBase 验收 | +| --- | --- | --- | +| Bub 适配器或回放场景 | SQLite、OceanBase | 不选择 | +| 原生 HTTP、指标或归档测试文件 | 无 | 选择 | +| 共用 Compose 定义或 harness 启动脚本 | SQLite、OceanBase | 选择 | +| 公共运行时、API 或根依赖 | SQLite、OceanBase | 选择，同时保留其他相关范围 | + +两种范围同时选中时，OceanBase 部分共用一个任务和容器。任一种范围选中都会完成数据库启动与容量配置。仅选择原生验收时，不分配 SQLite 回放任务，也不安装 Bub 验证环境。选中的测试和证据必须成功，缺少报告或意外跳过都会使验收门禁失败。原生 JUnit 结果独立于回放产物保留。 + +Main 的 preflight 已检查全仓库的 lint 和格式；其 harness 验证运行 `make harness-test`，保留 harness 类型检查、场景测试、CLI 验证及 Compose 验证。独立运行仍通过 `make harness-lint` 检查格式；本地 `make harness-check` 同时执行两个目标。 + +`[exclusions].database` 中的精确测试路径使用 SQLite 或内存行为，仍由普通 Python 测试及其他匹配范围覆盖。仅修改这些测试时，不启动整套数据库验收。文件新增原生数据库用例时，需要移除对应例外。例外不能抵消其他变更文件、共享夹具、公共运行时选择或全量兜底。升级验收被选中时，仍自动运行全部 12 个配置。 + +这些规则减少局部改动带来的无关任务。公共运行时和 CI 策略改动仍保留广泛覆盖；runner 分钟和排队时间需要上线后实测。 + ## 并发预算与取消 | 矩阵 | 每次运行的最大并发 | 失败策略 | diff --git a/scripts/ci_changes.py b/scripts/ci_changes.py index 061c64d755..8a859fe091 100644 --- a/scripts/ci_changes.py +++ b/scripts/ci_changes.py @@ -40,6 +40,7 @@ def select_suites(paths: list[str], rules: dict[str, Any]) -> dict[str, bool]: suite for suite, patterns in rules["suites"].items() if any(fnmatchcase(path, pattern) for pattern in patterns) + and path not in rules.get("exclusions", {}).get(suite, []) } for group in rules["shared"].values(): if any(fnmatchcase(path, pattern) for pattern in group["patterns"]): diff --git a/tests/test_ci_changes.py b/tests/test_ci_changes.py index 5290f8d6bd..109e98f499 100644 --- a/tests/test_ci_changes.py +++ b/tests/test_ci_changes.py @@ -62,7 +62,25 @@ (["scripts/generate_js_operations.py"], {"tests", "dsh", "cindy"}), (["tests/builtin/persistence/test_mysql_schema.py"], {"tests", "database", "upgrade", "candidate_replay"}), (["tests/native/test_personal_service_lifecycle.py"], {"tests", "native"}), - (["e2e/bub/compose.yaml"], {"harness"}), + (["e2e/bub/compose.yaml"], {"harness", "oceanbase_acceptance"}), + (["e2e/bub/compose.oceanbase.yaml"], {"harness", "oceanbase_acceptance"}), + (["e2e/bub/run.sh"], {"harness", "oceanbase_acceptance"}), + (["e2e/bub/scenarios/new.json"], {"harness"}), + (["tests/e2e/test_runtime_server.py"], {"tests", "oceanbase_acceptance"}), + (["tests/e2e/test_server_metrics_oceanbase.py"], {"tests", "oceanbase_acceptance"}), + (["tests/e2e/test_portable_archive.py"], {"tests", "oceanbase_acceptance"}), + (["tests/e2e/test_atomic_memory_read_snapshot.py"], {"tests", "database", "oceanbase_acceptance"}), + (["tests/builtin/runtime/test_recall_sufficiency.py"], {"tests"}), + (["tests/builtin/runtime/test_statistics.py"], {"tests"}), + (["tests/builtin/runtime/test_prepared_context.py"], {"tests"}), + (["tests/builtin/runtime/test_model_usage_recorder.py"], {"tests", "observability"}), + (["tests/e2e/test_recall_sufficiency_gate.py"], {"tests"}), + (["tests/e2e/test_context_text_assembly.py"], {"tests"}), + (["tests/builtin/runtime/test_recall_new_backend.py"], {"tests", "database"}), + ( + ["tests/builtin/runtime/test_recall_sufficiency.py", "tests/builtin/runtime/test_recall_new_backend.py"], + {"tests", "database"}, + ), (["scripts/scan_e2e_evidence.sh"], {"tests", "harness"}), (["scripts/run_atomic_memory_clean_migration.py"], {"tests", "database", "upgrade"}), (["scripts/check_atomic_memory_migration_shards.py"], {"tests", "database"}), @@ -164,6 +182,8 @@ def test_shared_runtime_inputs_keep_backend_upgrade_and_consumer_coverage(path: "zcode", "cindy", "managed_resources", + "harness", + "oceanbase_acceptance", } <= {suite for suite, enabled in selected.items() if enabled} assert not selected["release_schema"] assert selected["seekdb_lock"] is (path != "src/powercontext/server/authz/owner_recovery.py") @@ -268,7 +288,41 @@ def test_harness_workflow_change_selects_acceptance_and_database_suites(reposito selected = run_selector(repository, "pull_request", base, head) - assert {suite for suite, enabled in selected.items() if enabled} == {"harness", "database"} + assert {suite for suite, enabled in selected.items() if enabled} == {"harness", "database", "oceanbase_acceptance"} + + +@pytest.mark.parametrize( + "path", + [ + "tests/e2e/atomic_memory_migration_backend.py", + "tests/e2e/atomic_memory_migration_fixture.py", + "tests/e2e/server_storage.py", + "tests/e2e/dream_support.py", + "tests/builtin/persistence/contract.py", + "tests/legacy_memory.py", + ], +) +def test_native_acceptance_helper_changes_keep_real_backend_coverage(path: str) -> None: + selected = select_suites([path], RULES) + assert selected["tests"] + assert selected["oceanbase_acceptance"] + + +def test_test_only_exclusion_cannot_suppress_shared_or_full_coverage() -> None: + for path in ("src/powercontext/server/settings.py", "tests/conftest.py", "uv.lock", "new-component/code.py"): + assert select_suites(["tests/builtin/runtime/test_recall_sufficiency.py", path], RULES) == select_suites( + [path], RULES + ) + + +def test_native_acceptance_moves_and_deletions_preserve_coverage(repository: Path) -> None: + base = commit(repository, "tests/e2e/test_runtime_server.py", "tests/e2e/test_server_metrics_oceanbase.py") + (repository / "e2e/bub").mkdir(parents=True) + git(repository, "mv", "tests/e2e/test_runtime_server.py", "e2e/bub/moved.py") + git(repository, "rm", "tests/e2e/test_server_metrics_oceanbase.py") + git(repository, "commit", "-m", "Move and remove native test inputs") + selected = run_selector(repository, "pull_request", base, git(repository, "rev-parse", "HEAD")) + assert {suite for suite, enabled in selected.items() if enabled} == {"tests", "harness", "oceanbase_acceptance"} def test_diff_covers_all_pr_commits_and_excludes_base_branch_changes(repository: Path) -> None: diff --git a/tests/test_e2e_workflow_contract.py b/tests/test_e2e_workflow_contract.py index 179f295576..3e86561b4f 100644 --- a/tests/test_e2e_workflow_contract.py +++ b/tests/test_e2e_workflow_contract.py @@ -12,8 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json +import os +import subprocess +import sys from pathlib import Path +import pytest import yaml WORKFLOW = (Path(__file__).parents[1] / ".github" / "workflows" / "e2e-harness.yml").read_text(encoding="utf-8") @@ -27,7 +32,106 @@ def test_supplied_scope_keeps_acceptance_runnable_after_the_detector_is_skipped( # A successful validate job does not clear the skipped ancestor for Actions' # implicit success(). Acceptance needs its own status function and success gate. assert jobs["acceptance"]["if"] == "${{ !cancelled() && needs.validate.result == 'success' }}" - assert set(jobs["acceptance"]["strategy"]["matrix"]["database"]) == {"sqlite", "oceanbase"} + assert jobs["acceptance"]["strategy"]["matrix"] == "${{ fromJSON(needs.validate.outputs.matrix) }}" + + +def run_inline_python(step, env): + script = step["run"].removeprefix("python3 - <<'PY'\n").removesuffix("PY\n") + return subprocess.run( + [sys.executable, "-c", script], env={**os.environ, **env}, capture_output=True, text=True, check=False + ) + + +@pytest.mark.parametrize( + ("suites", "expected"), + [ + ( + {"harness": True}, + [ + {"database": "sqlite", "replay": True, "native": False}, + {"database": "oceanbase", "replay": True, "native": False}, + ], + ), + ({"oceanbase_acceptance": True}, [{"database": "oceanbase", "replay": False, "native": True}]), + ( + {"harness": True, "oceanbase_acceptance": True}, + [ + {"database": "sqlite", "replay": True, "native": False}, + {"database": "oceanbase", "replay": True, "native": True}, + ], + ), + ], +) +def test_acceptance_plan_only_allocates_selected_workers(tmp_path, suites, expected): + step = next(step for step in yaml.safe_load(WORKFLOW)["jobs"]["validate"]["steps"] if step.get("id") == "plan") + output = tmp_path / "output" + result = run_inline_python(step, {"CI_SUITES": json.dumps(suites), "GITHUB_OUTPUT": str(output)}) + assert result.returncode == 0, result.stderr + values = dict(line.split("=", 1) for line in output.read_text().splitlines()) + assert json.loads(values["matrix"]) == {"include": expected} + assert json.loads(values["replay"]) is suites.get("harness", False) + + +@pytest.mark.parametrize("mode", ["replay", "native", "both"]) +@pytest.mark.parametrize("outcome", ["success", "skipped", "failure", "missing"]) +def test_acceptance_gate_rejects_unexpected_skips_failures_and_missing_evidence(mode, outcome): + jobs = yaml.safe_load(WORKFLOW)["jobs"] + step = next(step for step in jobs["acceptance"]["steps"] if "STEP_RESULTS" in step.get("env", {})) + plan = {"database": "oceanbase", "replay": mode != "native", "native": mode != "replay"} + selected = ["oceanbase_setup"] + if plan["replay"]: + selected.extend(["acceptance_scenarios", "evidence_scan", "replay_evidence"]) + if plan["native"]: + selected.extend(["native_acceptance", "native_results"]) + # Unselected work is intentionally skipped. Every selected step is mandatory. + for target in selected: + results = { + name: {"outcome": "success" if name in selected else "skipped"} + for name in ( + "oceanbase_setup", + "acceptance_scenarios", + "evidence_scan", + "replay_evidence", + "native_acceptance", + "native_results", + ) + } + if outcome == "missing": + results.pop(target) + else: + results[target]["outcome"] = outcome + result = run_inline_python(step, {"ACCEPTANCE_PLAN": json.dumps(plan), "STEP_RESULTS": json.dumps(results)}) + assert (result.returncode == 0) is (outcome == "success"), result.stderr + if outcome != "success": + assert target in result.stderr + + +def test_shared_oceanbase_setup_and_evidence_are_independent_of_replay_selection(): + jobs = yaml.safe_load(WORKFLOW)["jobs"] + steps = {step.get("id"): step for step in jobs["acceptance"]["steps"] if "id" in step} + assert steps["oceanbase_setup"]["if"] == "matrix.database == 'oceanbase'" + for setting in ("datafile_size='4G'", "datafile_maxsize='8G'", "datafile_next='512M'"): + assert setting in steps["oceanbase_setup"]["run"] + assert steps["native_acceptance"]["if"] == "matrix.native" + assert steps["acceptance_scenarios"]["if"] == "matrix.replay" + for name in ("evidence_scan", "replay_evidence"): + assert "matrix.replay" in steps[name]["if"] + assert steps["native_results"]["if"] == "always() && matrix.native" + for name in ("native_results", "replay_evidence"): + assert steps[name]["with"]["if-no-files-found"] == "error" + + +def test_standalone_harness_retains_formatting_and_full_fallback(): + workflow = yaml.safe_load(WORKFLOW) + assert workflow[True]["workflow_call"]["inputs"]["preflight-checked"]["default"] is False + steps = workflow["jobs"]["validate"]["steps"] + lint = next(step for step in steps if step.get("run") == "make harness-lint") + assert "!inputs.preflight-checked" in lint["if"] + plan = next(step for step in steps if step.get("id") == "plan") + assert '\'{"harness":true,"oceanbase_acceptance":true}\'' in plan["env"]["CI_SUITES"] + main = yaml.safe_load((Path(__file__).parents[1] / ".github/workflows/master.yml").read_text()) + assert main["jobs"]["harness-validation"]["with"]["preflight-checked"] is True + assert ".oceanbase_acceptance" in main["jobs"]["harness-validation"]["if"] def test_e2e_scan_keeps_raw_scanner_output_out_of_artifacts() -> None: From 996c730fcc5c03c278c8dde48455553bf4b0f70a Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Sun, 8 Nov 2026 00:17:35 +0800 Subject: [PATCH 05/25] fix(ci): cache license tooling and reduce checkout transfers --- .github/actions/setup-license-eye/action.yml | 71 ++++++++++++++++++++ .github/ci-paths.toml | 1 + .github/workflows/master.yml | 30 ++++----- docs/en/development/ci-scope.md | 4 ++ docs/zh/development/ci-scope.md | 4 ++ tests/test_ci_changes.py | 27 ++++++++ tests/test_ci_license_eye.py | 63 +++++++++++++++++ tests/test_ci_pipeline.py | 10 ++- 8 files changed, 191 insertions(+), 19 deletions(-) create mode 100644 .github/actions/setup-license-eye/action.yml create mode 100644 tests/test_ci_license_eye.py diff --git a/.github/actions/setup-license-eye/action.yml b/.github/actions/setup-license-eye/action.yml new file mode 100644 index 0000000000..2430bde280 --- /dev/null +++ b/.github/actions/setup-license-eye/action.yml @@ -0,0 +1,71 @@ +name: Setup license-eye +description: Cache the pinned native license checker and compile only on a cache miss + +outputs: + executable: + description: Absolute path to the license checker + value: ${{ runner.temp }}/license-eye/bin/license-eye + +runs: + using: composite + steps: + - name: Restore the compiled checker + id: cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ${{ runner.temp }}/license-eye/bin + key: license-eye-v1-${{ runner.os }}-${{ runner.arch }}-a196742f472feaffafea537ce5a2a4c3c53a8de4-go1.25.14 + + - name: Fetch the pinned checker source + id: source + if: steps.cache.outputs.cache-hit != 'true' + shell: bash + env: + LICENSE_EYE_REVISION: a196742f472feaffafea537ce5a2a4c3c53a8de4 + run: | + source_dir="$RUNNER_TEMP/license-eye/source" + mkdir -p "$source_dir" + git -C "$source_dir" init + timeout --signal=TERM --kill-after=10s 180s \ + git -C "$source_dir" fetch --depth=1 --no-tags \ + https://github.com/apache/skywalking-eyes.git "$LICENSE_EYE_REVISION" + git -C "$source_dir" checkout --detach FETCH_HEAD + test "$(git -C "$source_dir" rev-parse HEAD)" = "$LICENSE_EYE_REVISION" + test -f "$source_dir/go.sum" + + - name: Prepare Go only for a cold tool cache + id: go + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 + with: + go-version: '1.25.14' + cache-dependency-path: ${{ runner.temp }}/license-eye/source/go.sum + + - name: Compile the native checker + id: build + if: steps.cache.outputs.cache-hit != 'true' + shell: bash + working-directory: ${{ runner.temp }}/license-eye/source + env: + GOTOOLCHAIN: local + GO111MODULE: 'on' + run: | + mkdir -p "$RUNNER_TEMP/license-eye/bin" + go build -o "$RUNNER_TEMP/license-eye/bin/license-eye" \ + -ldflags '-X github.com/apache/skywalking-eyes/commands.version=0.9.0' \ + cmd/license-eye/main.go + + - name: Verify the installed checker + id: verify + shell: bash + run: | + test -x "$RUNNER_TEMP/license-eye/bin/license-eye" + "$RUNNER_TEMP/license-eye/bin/license-eye" --help > /dev/null + + # Save before source validation so a license violation does not discard the tool. + - name: Save the compiled checker + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ${{ runner.temp }}/license-eye/bin + key: ${{ steps.cache.outputs.cache-primary-key }} diff --git a/.github/ci-paths.toml b/.github/ci-paths.toml index 01c1fd739c..e6117fdfe0 100644 --- a/.github/ci-paths.toml +++ b/.github/ci-paths.toml @@ -5,6 +5,7 @@ full = [ ".mise.toml", ".pre-commit-config.yaml", ".github/actions/setup-python-env/**", ".github/workflows/ci-changes.yml", ".github/workflows/master.yml", ".github/ci-paths.toml", "scripts/ci_changes.py", "tests/test_ci_changes.py", "tests/test_ci_pipeline.py", + ".github/actions/setup-license-eye/**", "tests/test_ci_license_eye.py", "tests/conftest.py", "tests/**/conftest.py", "tests/fixtures/**", "integrations/capabilities.toml", "integrations/langchain/pyproject.toml", "integrations/langgraph/pyproject.toml", "integrations/opendal/pyproject.toml", diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 58f1b051ed..be4e3465c0 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -176,16 +176,20 @@ jobs: quality: name: CI preflight if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} - timeout-minutes: 30 + # Leave room for the first tool build; warm runs skip Go and compilation. + timeout-minutes: 40 runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'powercontext-ci-light' || 'ubuntu-24.04' }} outputs: suites: ${{ steps.select.outputs.suites }} test-matrix: ${{ steps.select.outputs.test-matrix }} required-jobs: ${{ steps.select.outputs.required-jobs }} steps: - - name: Check out the complete branch history + - name: Check out history without historical file contents uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + timeout-minutes: 10 with: + # Keep merge bases and version tags; fetch old file contents only on demand. + filter: blob:none fetch-depth: 0 persist-credentials: false @@ -200,24 +204,18 @@ jobs: GH_TOKEN: ${{ github.token }} run: python3 scripts/ci_changes.py - - name: Prepare a writable license-eye prefix - # Install license-eye via the action's supported INSTALL_DIR override, - # using a user-writable path that the action resolves from PATH. - env: - LICENSE_EYE_BIN: ${{ runner.temp }}/license-eye-bin - run: | - mkdir -p "$LICENSE_EYE_BIN" - echo "$LICENSE_EYE_BIN" >> "$GITHUB_PATH" + - name: Prepare the cached license checker + id: license_tool + timeout-minutes: 15 + uses: ./.github/actions/setup-license-eye - name: Check License Header - timeout-minutes: 10 - uses: apache/skywalking-eyes/header@a196742f472feaffafea537ce5a2a4c3c53a8de4 # v0.9.0 + id: license_check + timeout-minutes: 3 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - INSTALL_DIR: ${{ runner.temp }}/license-eye-bin - with: - config: .licenserc.yaml - mode: check + LICENSE_EYE: ${{ steps.license_tool.outputs.executable }} + run: '"$LICENSE_EYE" -v info -c .licenserc.yaml header check' - name: Set up the environment uses: ./.github/actions/setup-python-env diff --git a/docs/en/development/ci-scope.md b/docs/en/development/ci-scope.md index 1270f0b98c..727fd74204 100644 --- a/docs/en/development/ci-scope.md +++ b/docs/en/development/ci-scope.md @@ -31,6 +31,10 @@ Draft PRs allocate no automatic CI runners. Marking a PR ready validates its com ## Validation profiles +Preflight keeps the complete commit history and tags using a `blob:none` partial clone; historical file contents are fetched only when needed. This preserves merge-base scope selection and version calculation while reducing historical file transfers. The checkout has a ten-minute limit. + +License validation caches a native license-eye executable built from the pinned v0.9.0 commit. The cache key includes the source commit, Go version, runner OS and architecture; cache hits skip source fetching, Go setup and compilation. Cold starts use pinned Go 1.25.14 and cache dependencies against the fetched repository's root `go.sum`. The checker is installed under the runner's temporary directory, verified as executable, and saved before source-header validation so a license violation does not discard the tool. Tool preparation and header validation have separate fifteen-minute and three-minute limits; preflight allows forty minutes for a cold start plus the remaining checks. These are ceilings, not added work. Cold builds still depend on network availability, and measured savings require CI observations. + | Entry point or input | Python validation | Other acceptance | | --- | --- | --- | | Ordinary ready PR | 3.11/3.14 unit tests and 3.12 E2E: 3 combinations | Complete selected backend, upgrade, installation-role, platform and host acceptance | diff --git a/docs/zh/development/ci-scope.md b/docs/zh/development/ci-scope.md index ed570febdc..6e5a476d27 100644 --- a/docs/zh/development/ci-scope.md +++ b/docs/zh/development/ci-scope.md @@ -31,6 +31,10 @@ flowchart TD ## 验证层级 +前置检查通过 `blob:none` 部分克隆保留完整提交历史和标签，历史文件内容按需获取。这会减少历史文件传输，同时保留 merge base 范围检测和版本计算。checkout 上限为十分钟。 + +许可证检查缓存由固定 v0.9.0 提交编译的原生 license-eye 程序。缓存键包含源码提交、Go 版本、runner 系统和架构；命中后跳过源码获取、Go 环境准备和编译。冷启动使用固定的 Go 1.25.14，以源码仓库根目录实际存在的 `go.sum` 缓存依赖。工具安装在 runner 临时目录，验证可执行后、检查源码许可证之前保存缓存，因此源码许可证错误不会丢掉工具缓存。工具准备和许可证检查分别限制为十五分钟和三分钟，前置任务共预留四十分钟容纳冷启动和后续检查。这些是时间上限，不会增加实际工作量；首次编译仍受网络影响，实际节省时间需要通过 CI 测量。 + | 入口或输入 | Python 验证 | 其他验收 | | --- | --- | --- | | 普通 ready PR | 3.11/3.14 单元测试，3.12 E2E，共 3 个组合 | 运行变更选中的完整后端、升级、安装角色、平台和宿主验收 | diff --git a/tests/test_ci_changes.py b/tests/test_ci_changes.py index 47d1051174..ddc6477793 100644 --- a/tests/test_ci_changes.py +++ b/tests/test_ci_changes.py @@ -369,6 +369,33 @@ def test_moves_and_deletions_keep_the_original_suite(repository: Path) -> None: } +def test_blobless_checkout_preserves_pr_scope_and_version_tags(repository: Path) -> None: + historical = repository / "old.bin" + historical.write_bytes(b"historical content that is no longer checked out") + git(repository, "add", "old.bin") + git(repository, "commit", "-m", "Historical content") + old_blob = git(repository, "rev-parse", "HEAD:old.bin") + git(repository, "rm", "old.bin") + git(repository, "commit", "-m", "Remove historical content") + git(repository, "tag", "v1.0.0") + git(repository, "checkout", "-b", "feature") + head = commit(repository, "docs/zh/说明.md") + git(repository, "checkout", "master") + base = commit(repository, "src/powercontext/unrelated.py") + git(repository, "checkout", "feature") + git(repository, "config", "uploadpack.allowFilter", "true") + clone = repository.parent / f"{repository.name}-partial" + git(repository, "clone", "--filter=blob:none", repository.as_uri(), str(clone)) + + assert git(clone, "rev-parse", "--is-shallow-repository") == "false" + assert git(clone, "describe", "--tags") == git(repository, "describe", "--tags") + selected = run_selector(clone, "pull_request", base, head) + assert {suite for suite, enabled in selected.items() if enabled} == {"website", "docs_tests"} + # Inspect missing objects without triggering a lazy fetch of the old content. + missing = git(clone, "rev-list", "--objects", "--all", "--missing=print") + assert f"?{old_blob}" in missing.splitlines() + + def test_large_diff_includes_code_after_hundreds_of_docs(repository: Path) -> None: base = git(repository, "rev-parse", "HEAD") head = commit(repository, *(f"docs/page-{index}.md" for index in range(350)), "src/powercontext/client.py") diff --git a/tests/test_ci_license_eye.py b/tests/test_ci_license_eye.py new file mode 100644 index 0000000000..c46ea9db3c --- /dev/null +++ b/tests/test_ci_license_eye.py @@ -0,0 +1,63 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +from pathlib import Path + +import pytest +import yaml + +ROOT = Path(__file__).parents[1] +ACTION = yaml.safe_load((ROOT / ".github/actions/setup-license-eye/action.yml").read_text()) +STEPS = ACTION["runs"]["steps"] +BY_ID = {step["id"]: step for step in STEPS if "id" in step} + + +def test_warm_cache_skips_source_download_toolchain_setup_and_compilation(): + for name in ("source", "go", "build"): + assert BY_ID[name]["if"] == "steps.cache.outputs.cache-hit != 'true'" + key = BY_ID["cache"]["with"]["key"] + assert BY_ID["source"]["env"]["LICENSE_EYE_REVISION"] in key + assert BY_ID["go"]["with"]["go-version"] in key + assert "${{ runner.os }}" in key and "${{ runner.arch }}" in key + assert "restore-keys" not in BY_ID["cache"]["with"] + + +def test_cold_cache_uses_the_fetched_repository_root_for_go_dependencies(): + assert STEPS.index(BY_ID["source"]) < STEPS.index(BY_ID["go"]) + assert BY_ID["go"]["with"]["cache-dependency-path"] == "${{ runner.temp }}/license-eye/source/go.sum" + assert 'test -f "$source_dir/go.sum"' in BY_ID["source"]["run"] + assert "180s" in BY_ID["source"]["run"] + + +@pytest.mark.parametrize("exit_code", [0, 7]) +def test_cached_checker_must_be_executable_before_it_can_be_saved(tmp_path, exit_code): + binary = tmp_path / "license-eye/bin/license-eye" + binary.parent.mkdir(parents=True) + binary.write_text(f"#!/bin/sh\nexit {exit_code}\n") + binary.chmod(0o755) + result = subprocess.run( + ["bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", BY_ID["verify"]["run"]], # noqa: S607 + env={**os.environ, "RUNNER_TEMP": str(tmp_path)}, + check=False, + ) + assert result.returncode == exit_code + save = next(step for step in STEPS if step.get("uses", "").startswith("actions/cache/save@")) + assert STEPS.index(BY_ID["verify"]) < STEPS.index(save) + assert "always()" not in save["if"] + assert save["with"]["path"] == BY_ID["cache"]["with"]["path"] + assert save["with"]["key"] == "${{ steps.cache.outputs.cache-primary-key }}" + for step in STEPS: + assert not step.get("continue-on-error") diff --git a/tests/test_ci_pipeline.py b/tests/test_ci_pipeline.py index 2d9988011c..a82fbda488 100644 --- a/tests/test_ci_pipeline.py +++ b/tests/test_ci_pipeline.py @@ -197,13 +197,17 @@ def test_preflight_uses_one_runner_and_keeps_all_admission_checks_mandatory(): checkouts = [step for step in steps if step.get("uses", "").startswith("actions/checkout@")] assert len(checkouts) == 1 assert checkouts[0]["with"]["fetch-depth"] == 0 + assert checkouts[0]["with"]["filter"] == "blob:none" selection = next(step for step in steps if step.get("id") == "select") - license_check = next(step for step in steps if step.get("uses", "").startswith("apache/skywalking-eyes/header@")) + license_tool = next(step for step in steps if step.get("id") == "license_tool") + license_check = next(step for step in steps if step.get("id") == "license_check") quality = next(step for step in steps if step.get("run") == "make check") contract = next(step for step in steps if step.get("run") == "make contract-test") assert selection["env"]["CI_BASELINE_WORKFLOW"] == "master.yml" - assert license_check["with"] == {"config": ".licenserc.yaml", "mode": "check"} - for step in (selection, license_check, quality, contract): + assert license_tool["uses"] == "./.github/actions/setup-license-eye" + assert license_check["run"] == '"$LICENSE_EYE" -v info -c .licenserc.yaml header check' + assert license_check["env"]["LICENSE_EYE"] == "${{ steps.license_tool.outputs.executable }}" + for step in (selection, license_tool, license_check, quality, contract): assert not step.get("continue-on-error") assert "if" not in step # A preceding failure prevents further admission work. for output in ("suites", "test-matrix", "required-jobs"): From 105e7a94cee3bced7b9a60e08b78dd046899bc72 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Sun, 8 Nov 2026 00:20:15 +0800 Subject: [PATCH 06/25] fix(ci): make cached license checker independent of libc --- .github/actions/setup-license-eye/action.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup-license-eye/action.yml b/.github/actions/setup-license-eye/action.yml index 2430bde280..40fc0895a4 100644 --- a/.github/actions/setup-license-eye/action.yml +++ b/.github/actions/setup-license-eye/action.yml @@ -14,7 +14,7 @@ runs: uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: ${{ runner.temp }}/license-eye/bin - key: license-eye-v1-${{ runner.os }}-${{ runner.arch }}-a196742f472feaffafea537ce5a2a4c3c53a8de4-go1.25.14 + key: license-eye-static-v1-${{ runner.os }}-${{ runner.arch }}-a196742f472feaffafea537ce5a2a4c3c53a8de4-go1.25.14 - name: Fetch the pinned checker source id: source @@ -49,6 +49,8 @@ runs: env: GOTOOLCHAIN: local GO111MODULE: 'on' + # Share the tool across Linux runner images without a host libc dependency. + CGO_ENABLED: '0' run: | mkdir -p "$RUNNER_TEMP/license-eye/bin" go build -o "$RUNNER_TEMP/license-eye/bin/license-eye" \ From 4e2762f7770e5f046a3a168d222be9147d343587 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Sun, 8 Nov 2026 16:20:24 +0800 Subject: [PATCH 07/25] fix(ci): reserve E2E cleanup time and run serial cases once --- .github/workflows/master.yml | 15 +++------ tests/test_ci_pipeline.py | 61 ++++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 10 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index be4e3465c0..87610dd48d 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -240,8 +240,8 @@ jobs: name: tests (${{ matrix.python-version }}, ${{ matrix.suite }}) needs: [quality] if: ${{ !cancelled() && needs.quality.result == 'success' && (fromJSON(needs.quality.outputs.suites || '{}').tests) }} - # Allow the Python 3.14 end-to-end suite's subprocess tail to finish. - timeout-minutes: ${{ matrix.suite == 'e2e' && matrix.python-version == '3.14' && 55 || 40 }} + # E2E: 50m for tests plus 10m for setup and post-job cache/cleanup on every Python version. + timeout-minutes: ${{ matrix.suite == 'e2e' && 60 || 40 }} runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'powercontext-ci-standard' || 'ubuntu-24.04' }} strategy: fail-fast: true @@ -289,15 +289,10 @@ jobs: PYTEST_ADDOPTS: --require-dsh-config-runtime --require-claude-code-python-floor run: make unit-test - - name: Run the ZCode service chain with native Hook deadlines + - name: Run end-to-end tests if: matrix.suite == 'e2e' - # Preserve the real Hook deadlines without competing with xdist workers. - run: uv run python -m pytest -n 0 tests/e2e/test_zcode_service_chain.py - - - name: Run remaining end-to-end tests - if: matrix.suite == 'e2e' - env: - PYTEST_ADDOPTS: --ignore=tests/e2e/test_zcode_service_chain.py + timeout-minutes: 50 + # The Makefile runs ordinary cases in parallel, then Hook cases once in isolation. run: make e2e-test isolated-installations: diff --git a/tests/test_ci_pipeline.py b/tests/test_ci_pipeline.py index a82fbda488..1f0bdd4378 100644 --- a/tests/test_ci_pipeline.py +++ b/tests/test_ci_pipeline.py @@ -306,3 +306,64 @@ def test_actual_ci_result_script_rejects_missing_selected_acceptance( check=False, ) assert (run.returncode == 0) == passes, run.stdout + run.stderr + + +def test_e2e_deadline_reserves_setup_and_cache_time_for_every_python(): + job = workflow("master.yml")["jobs"]["tests"] + assert job["timeout-minutes"] == "${{ matrix.suite == 'e2e' && 60 || 40 }}" + steps = [step for step in job["steps"] if step.get("if") == "matrix.suite == 'e2e'"] + assert steps + assert all(isinstance(step.get("timeout-minutes"), int) for step in steps) + assert sum(step["timeout-minutes"] for step in steps) <= 50 + + +@pytest.mark.skipif(sys.platform == "win32", reason="CI commands require POSIX make and shell") +def test_ci_e2e_runs_each_case_once_and_keeps_hook_tests_serial(tmp_path, monkeypatch): + make = shutil.which("make") + if make is None: + pytest.skip("make is required to exercise the CI entry point") + shutil.copyfile(ROOT / "Makefile", tmp_path / "Makefile") + cases = tmp_path / "tests/e2e" + cases.mkdir(parents=True) + for name in ("test_other", "test_zcode_service_chain"): + (cases / f"{name}.py").write_text( + "import os\nfrom pathlib import Path\n" + "def test_observable_execution():\n" + f" with Path(os.environ['CI_TEST_EVENTS'], '{name}').open('a') as output:\n" + " output.write(os.environ.get('PYTEST_XDIST_WORKER', 'serial') + '\\n')\n" + ) + commands = tmp_path / "bin" + commands.mkdir() + uv = commands / "uv" + uv.write_text( + f"#!{sys.executable}\n" + "import os, sys\n" + "assert sys.argv[1:5] == ['run', 'python', '-m', 'pytest'], sys.argv\n" + "os.execv(sys.executable, [sys.executable, '-m', 'pytest', '-p', 'xdist.plugin', *sys.argv[5:]])\n" + ) + uv.chmod(0o755) + events = tmp_path / "events" + events.mkdir() + monkeypatch.setenv("PATH", str(commands) + os.pathsep + os.environ["PATH"]) + monkeypatch.setenv("CI_TEST_EVENTS", str(events)) + monkeypatch.setenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1") + monkeypatch.setenv("PYTEST_WORKERS", "2") + monkeypatch.delenv("PYTEST_ADDOPTS", raising=False) + # Execute the actual workflow commands and Makefile against small fixture + # cases; pytest itself handles ignore flags, xdist and serial selection. + for step in workflow("master.yml")["jobs"]["tests"]["steps"]: + if step.get("if") == "matrix.suite == 'e2e'": + result = subprocess.run( + ["/bin/sh", "-e", "-c", step["run"]], + cwd=tmp_path, + env={**os.environ, **step.get("env", {})}, + capture_output=True, + text=True, + check=False, + timeout=30, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert (events / "test_zcode_service_chain").read_text().splitlines() == ["serial"] + parallel_runs = (events / "test_other").read_text().splitlines() + assert len(parallel_runs) == 1 + assert parallel_runs[0].startswith("gw") From ac059e8921b51204418366330774d6aca7750cc8 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Sun, 8 Nov 2026 17:03:07 +0800 Subject: [PATCH 08/25] test(ci): inspect child worker state under parallel pytest --- tests/test_ci_pipeline.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_ci_pipeline.py b/tests/test_ci_pipeline.py index 1f0bdd4378..16b71596e3 100644 --- a/tests/test_ci_pipeline.py +++ b/tests/test_ci_pipeline.py @@ -325,12 +325,14 @@ def test_ci_e2e_runs_each_case_once_and_keeps_hook_tests_serial(tmp_path, monkey shutil.copyfile(ROOT / "Makefile", tmp_path / "Makefile") cases = tmp_path / "tests/e2e" cases.mkdir(parents=True) + # Read the child pytest configuration: its environment can retain the outer + # xdist worker ID even when the child runs serially. for name in ("test_other", "test_zcode_service_chain"): (cases / f"{name}.py").write_text( "import os\nfrom pathlib import Path\n" - "def test_observable_execution():\n" + "def test_observable_execution(request):\n" f" with Path(os.environ['CI_TEST_EVENTS'], '{name}').open('a') as output:\n" - " output.write(os.environ.get('PYTEST_XDIST_WORKER', 'serial') + '\\n')\n" + " output.write(getattr(request.config, 'workerinput', {}).get('workerid', 'serial') + '\\n')\n" ) commands = tmp_path / "bin" commands.mkdir() From ed6f0268927dd3178d19ac6258c7d68fca6b114b Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Sun, 8 Nov 2026 18:07:25 +0800 Subject: [PATCH 09/25] fix(ci): bound migration shards and pin mise setup --- .github/workflows/deploy-docs.yml | 3 +++ .github/workflows/desktop-candidate.yml | 2 ++ .github/workflows/desktop.yml | 6 ++++++ .github/workflows/e2e-harness.yml | 8 ++++---- .github/workflows/master.yml | 27 +++++++++++++++++++++++++ .github/workflows/release.yml | 6 ++++++ .github/workflows/upgrade-release.yml | 3 +++ tests/test_oceanbase_ci.py | 23 ++++++++++++++++++++- 8 files changed, 73 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index ceedebdd01..ef3a9b19f3 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -36,6 +36,9 @@ jobs: - name: Set up website tools uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + with: + # Avoid the mutable release index during CI setup. + version: "2026.11.3" - name: Cache website dependencies uses: ./.github/actions/cache-pnpm diff --git a/.github/workflows/desktop-candidate.yml b/.github/workflows/desktop-candidate.yml index cac0912561..0ae487b9b2 100644 --- a/.github/workflows/desktop-candidate.yml +++ b/.github/workflows/desktop-candidate.yml @@ -114,6 +114,8 @@ jobs: --version "$DESKTOP_VERSION" --server-version "$SERVER_VERSION" --source-sha "$SOURCE_SHA" --server-source-sha "$SERVER_SOURCE_SHA" - uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 with: + # Avoid the mutable release index during CI setup. + version: "2026.11.3" working_directory: desktop - name: Install pinned Rust target run: rustup toolchain install "1.95.0-$RUST_TARGET" --profile minimal diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 06c2240dbe..a762f5b968 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -60,6 +60,8 @@ jobs: } - uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 with: + # Avoid the mutable release index during CI setup. + version: "2026.11.3" working_directory: desktop - name: Install pinned Windows Rust toolchain run: rustup toolchain install 1.95.0-x86_64-pc-windows-msvc --profile minimal --component rustfmt --component clippy @@ -186,6 +188,8 @@ jobs: persist-credentials: false - uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 with: + # Avoid the mutable release index during CI setup. + version: "2026.11.3" working_directory: desktop - run: rustup toolchain install 1.95.0 --profile minimal --component rustfmt --component clippy - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 @@ -316,6 +320,8 @@ jobs: pkg-config - uses: jdx/mise-action@9b200bcecbfd6303e69a4a261b2e09e3757440a7 # v5.2.0 with: + # Avoid the mutable release index during CI setup. + version: "2026.11.3" working_directory: desktop - name: Install pinned Linux Rust toolchain run: rustup toolchain install 1.95.0 --profile minimal --component rustfmt --component clippy diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index 849706c7ed..435e6f6e0d 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -325,8 +325,8 @@ jobs: ${{ !cancelled() && (github.event_name != 'workflow_dispatch' || inputs.suite != 'recall-effort') && ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').database) }} runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'powercontext-ci-standard' || 'ubuntu-24.04' }} - # Test steps total 155m; reserve 25m for setup, diagnostics and artifact upload. - timeout-minutes: 180 + # Test steps total 205m; reserve 25m for setup, diagnostics and artifact upload. + timeout-minutes: 230 strategy: max-parallel: 2 fail-fast: false @@ -379,8 +379,8 @@ jobs: tests/e2e/test_background_metrics.py - name: Run clean migration evidence, grants and vectors without skips - # Measured clean migration work is about 21m per worker; retain the 90m group ceiling. - timeout-minutes: 90 + # The larger shard has 25 cases: 25 * (300s + 10s grace) + 190s collection < 140m. + timeout-minutes: 140 run: | uv run --locked python scripts/run_atomic_memory_clean_migration.py \ oceanbase --test-timeout 300 --shard "${{ matrix.shard }}/2" --durations .github/oceanbase-migration-durations.json --output "$MIGRATION_REPORTS/clean-migration" diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 87610dd48d..d7bcd1cf49 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -263,6 +263,9 @@ jobs: - name: Set up project tools uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + with: + # Avoid the mutable release index during CI setup. + version: "2026.11.3" - name: Cache DSH configuration dependencies if: matrix.suite == 'unit' @@ -859,6 +862,9 @@ jobs: - name: Set up project tools uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + with: + # Avoid the mutable release index during CI setup. + version: "2026.11.3" - name: Check daemon serialization, host casting, Workflow selectors and SDK HTTP readback env: @@ -909,6 +915,9 @@ jobs: - name: Set up project tools uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + with: + # Avoid the mutable release index during CI setup. + version: "2026.11.3" - name: Cache DSH dependencies uses: ./.github/actions/cache-pnpm @@ -959,6 +968,9 @@ jobs: - name: Set up project tools uses: jdx/mise-action@2d8d4cafcbd33be2ea37d2b6f5ad595363d1f1ca # v5.1.1 + with: + # Avoid the mutable release index during CI setup. + version: "2026.11.3" - name: Cache shared JavaScript build tooling uses: ./.github/actions/cache-pnpm @@ -1002,6 +1014,9 @@ jobs: - name: Set up project tools uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + with: + # Avoid the mutable release index during CI setup. + version: "2026.11.3" - name: Cache Pi dependencies uses: ./.github/actions/cache-pnpm @@ -1028,6 +1043,9 @@ jobs: uses: ./.github/actions/setup-python-env - uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + with: + # Avoid the mutable release index during CI setup. + version: "2026.11.3" - name: Cache OpenCode dependencies uses: ./.github/actions/cache-pnpm with: @@ -1052,6 +1070,9 @@ jobs: uses: ./.github/actions/setup-python-env - uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + with: + # Avoid the mutable release index during CI setup. + version: "2026.11.3" # The pinned OpenClaw SDK requires the SQLite runtime shipped with Node 24.15+. - run: mise install node@24.15.0 - name: Cache OpenClaw dependencies @@ -1098,6 +1119,9 @@ jobs: - name: Set up website tools uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + with: + # Avoid the mutable release index during CI setup. + version: "2026.11.3" - name: Cache website dependencies uses: ./.github/actions/cache-pnpm @@ -1127,6 +1151,9 @@ jobs: - name: Set up website tools uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + with: + # Avoid the mutable release index during CI setup. + version: "2026.11.3" - name: Cache website dependencies uses: ./.github/actions/cache-pnpm diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 668140dc01..d7365ad4cb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -123,6 +123,9 @@ jobs: - name: Set up project tools uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + with: + # Avoid the mutable release index during CI setup. + version: "2026.11.3" - name: Cache DSH dependencies uses: ./.github/actions/cache-pnpm @@ -302,6 +305,9 @@ jobs: - name: Set up website tools uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + with: + # Avoid the mutable release index during CI setup. + version: "2026.11.3" - name: Cache website dependencies uses: ./.github/actions/cache-pnpm diff --git a/.github/workflows/upgrade-release.yml b/.github/workflows/upgrade-release.yml index 97ec9f782e..3d5245ce38 100644 --- a/.github/workflows/upgrade-release.yml +++ b/.github/workflows/upgrade-release.yml @@ -103,6 +103,9 @@ jobs: - name: Set up website tools if: inputs.release uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + with: + # Avoid the mutable release index during CI setup. + version: "2026.11.3" - name: Cache website dependencies if: inputs.release diff --git a/tests/test_oceanbase_ci.py b/tests/test_oceanbase_ci.py index d421e60393..940b9519a6 100644 --- a/tests/test_oceanbase_ci.py +++ b/tests/test_oceanbase_ci.py @@ -18,6 +18,7 @@ import json import os +import shlex import shutil import signal import subprocess @@ -26,9 +27,10 @@ from pathlib import Path import pytest +import yaml from check_atomic_memory_migration_shards import verify_group from oceanbase_ci import collect, run_command, wait_ready -from run_atomic_memory_clean_migration import assigned_nodes, read_durations +from run_atomic_memory_clean_migration import SUITES, assigned_nodes, read_durations @pytest.fixture @@ -219,3 +221,22 @@ def test_invalid_duration_baseline_cannot_silently_change_scheduling(tmp_path): path.write_text(json.dumps({"durations_seconds": {"test": -1}})) with pytest.raises(ValueError, match="positive"): read_durations(path) + + +def test_clean_migration_shard_budget_covers_slow_cases_and_keeps_cleanup_time(): + root = Path(__file__).parents[1] + workflow = yaml.safe_load((root / ".github/workflows/e2e-harness.yml").read_text()) + job = workflow["jobs"]["atomic-memory-clean-oceanbase-shards"] + steps = [step for step in job["steps"] if "run_atomic_memory_clean_migration.py" in step.get("run", "")] + clean = next(step for step in steps if '--output "$MIGRATION_REPORTS/clean-migration"' in step["run"]) + args = shlex.split(clean["run"].replace("\\\n", "")) + case_timeout = int(args[args.index("--test-timeout") + 1]) + durations = read_durations(root / ".github/oceanbase-migration-durations.json") + assert durations is not None + nodes = [node for node in durations if node.split("::")[0] in SUITES] + shard_count = len(job["strategy"]["matrix"]["shard"]) + largest_shard = max(len(assigned_nodes(nodes, index, shard_count, durations)) for index in range(shard_count)) + # Slow, passing cases can approach their deadline before the final case times out. + # Include the runner's 180s collection timeout and 10s process termination grace. + assert clean["timeout-minutes"] * 60 >= largest_shard * (case_timeout + 10) + 180 + 10 + assert job["timeout-minutes"] >= sum(step["timeout-minutes"] for step in steps) + 25 From 5caea8fbdfd14d14cab0aa9ec70912751ff7338c Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Sun, 8 Nov 2026 18:22:23 +0800 Subject: [PATCH 10/25] fix(ci): pin verified mise binary checksums --- .github/workflows/deploy-docs.yml | 3 ++- .github/workflows/desktop-candidate.yml | 4 +++- .github/workflows/desktop.yml | 10 ++++++---- .github/workflows/master.yml | 21 +++++++++++---------- .github/workflows/release.yml | 5 +++-- .github/workflows/upgrade-release.yml | 3 ++- 6 files changed, 27 insertions(+), 19 deletions(-) diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index ef3a9b19f3..77fed683ad 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -37,8 +37,9 @@ jobs: - name: Set up website tools uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 with: - # Avoid the mutable release index during CI setup. + # Version and binary hashes are pinned from the verified, minisign-signed release checksums. version: "2026.11.3" + sha256: c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b - name: Cache website dependencies uses: ./.github/actions/cache-pnpm diff --git a/.github/workflows/desktop-candidate.yml b/.github/workflows/desktop-candidate.yml index 0ae487b9b2..77158cdb6c 100644 --- a/.github/workflows/desktop-candidate.yml +++ b/.github/workflows/desktop-candidate.yml @@ -114,8 +114,10 @@ jobs: --version "$DESKTOP_VERSION" --server-version "$SERVER_VERSION" --source-sha "$SOURCE_SHA" --server-source-sha "$SERVER_SOURCE_SHA" - uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 with: - # Avoid the mutable release index during CI setup. + # Version and binary hashes are pinned from the verified, minisign-signed release checksums. version: "2026.11.3" + sha256: >- + ${{ runner.os == 'Windows' && '4f796228c826ad6225d608e2029278b31677c8919508574498877543ed3431d8' || (runner.arch == 'ARM64' && '6d39119ab914c54ab865204dee9337c71b83a7ed1e730f663c2fc3453e38fe9c' || '49f32abd93f54147af0599a1c9583d2092df8e7250c9fe635783a7932c4087ef') }} working_directory: desktop - name: Install pinned Rust target run: rustup toolchain install "1.95.0-$RUST_TARGET" --profile minimal diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index a762f5b968..66f17424f4 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -60,8 +60,9 @@ jobs: } - uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 with: - # Avoid the mutable release index during CI setup. + # Version and binary hashes are pinned from the verified, minisign-signed release checksums. version: "2026.11.3" + sha256: 4f796228c826ad6225d608e2029278b31677c8919508574498877543ed3431d8 working_directory: desktop - name: Install pinned Windows Rust toolchain run: rustup toolchain install 1.95.0-x86_64-pc-windows-msvc --profile minimal --component rustfmt --component clippy @@ -188,8 +189,9 @@ jobs: persist-credentials: false - uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 with: - # Avoid the mutable release index during CI setup. version: "2026.11.3" + sha256: >- + ${{ runner.arch == 'ARM64' && '6d39119ab914c54ab865204dee9337c71b83a7ed1e730f663c2fc3453e38fe9c' || '49f32abd93f54147af0599a1c9583d2092df8e7250c9fe635783a7932c4087ef' }} working_directory: desktop - run: rustup toolchain install 1.95.0 --profile minimal --component rustfmt --component clippy - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 @@ -318,10 +320,10 @@ jobs: libxdo-dev \ openjdk-17-jre-headless \ pkg-config - - uses: jdx/mise-action@9b200bcecbfd6303e69a4a261b2e09e3757440a7 # v5.2.0 + - uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 with: - # Avoid the mutable release index during CI setup. version: "2026.11.3" + sha256: c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b working_directory: desktop - name: Install pinned Linux Rust toolchain run: rustup toolchain install 1.95.0 --profile minimal --component rustfmt --component clippy diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index d7bcd1cf49..3ef905b55c 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -264,8 +264,9 @@ jobs: - name: Set up project tools uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 with: - # Avoid the mutable release index during CI setup. + # Version and binary hashes are pinned from the verified, minisign-signed release checksums. version: "2026.11.3" + sha256: c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b - name: Cache DSH configuration dependencies if: matrix.suite == 'unit' @@ -863,8 +864,8 @@ jobs: - name: Set up project tools uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 with: - # Avoid the mutable release index during CI setup. version: "2026.11.3" + sha256: c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b - name: Check daemon serialization, host casting, Workflow selectors and SDK HTTP readback env: @@ -916,8 +917,8 @@ jobs: - name: Set up project tools uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 with: - # Avoid the mutable release index during CI setup. version: "2026.11.3" + sha256: c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b - name: Cache DSH dependencies uses: ./.github/actions/cache-pnpm @@ -967,10 +968,10 @@ jobs: uses: ./.github/actions/setup-python-env - name: Set up project tools - uses: jdx/mise-action@2d8d4cafcbd33be2ea37d2b6f5ad595363d1f1ca # v5.1.1 + uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 with: - # Avoid the mutable release index during CI setup. version: "2026.11.3" + sha256: c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b - name: Cache shared JavaScript build tooling uses: ./.github/actions/cache-pnpm @@ -1015,8 +1016,8 @@ jobs: - name: Set up project tools uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 with: - # Avoid the mutable release index during CI setup. version: "2026.11.3" + sha256: c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b - name: Cache Pi dependencies uses: ./.github/actions/cache-pnpm @@ -1044,8 +1045,8 @@ jobs: - uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 with: - # Avoid the mutable release index during CI setup. version: "2026.11.3" + sha256: c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b - name: Cache OpenCode dependencies uses: ./.github/actions/cache-pnpm with: @@ -1071,8 +1072,8 @@ jobs: - uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 with: - # Avoid the mutable release index during CI setup. version: "2026.11.3" + sha256: c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b # The pinned OpenClaw SDK requires the SQLite runtime shipped with Node 24.15+. - run: mise install node@24.15.0 - name: Cache OpenClaw dependencies @@ -1120,8 +1121,8 @@ jobs: - name: Set up website tools uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 with: - # Avoid the mutable release index during CI setup. version: "2026.11.3" + sha256: 4f796228c826ad6225d608e2029278b31677c8919508574498877543ed3431d8 - name: Cache website dependencies uses: ./.github/actions/cache-pnpm @@ -1152,8 +1153,8 @@ jobs: - name: Set up website tools uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 with: - # Avoid the mutable release index during CI setup. version: "2026.11.3" + sha256: c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b - name: Cache website dependencies uses: ./.github/actions/cache-pnpm diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d7365ad4cb..796e4aaa49 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -124,8 +124,9 @@ jobs: - name: Set up project tools uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 with: - # Avoid the mutable release index during CI setup. + # Version and binary hashes are pinned from the verified, minisign-signed release checksums. version: "2026.11.3" + sha256: c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b - name: Cache DSH dependencies uses: ./.github/actions/cache-pnpm @@ -306,8 +307,8 @@ jobs: - name: Set up website tools uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 with: - # Avoid the mutable release index during CI setup. version: "2026.11.3" + sha256: c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b - name: Cache website dependencies uses: ./.github/actions/cache-pnpm diff --git a/.github/workflows/upgrade-release.yml b/.github/workflows/upgrade-release.yml index 3d5245ce38..7d5a36c31a 100644 --- a/.github/workflows/upgrade-release.yml +++ b/.github/workflows/upgrade-release.yml @@ -104,8 +104,9 @@ jobs: if: inputs.release uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 with: - # Avoid the mutable release index during CI setup. + # Version and binary hashes are pinned from the verified, minisign-signed release checksums. version: "2026.11.3" + sha256: c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b - name: Cache website dependencies if: inputs.release From b512749f2a8b183f70bd561d40fd34504025bfdc Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Sun, 8 Nov 2026 18:40:32 +0800 Subject: [PATCH 11/25] fix(ci): fall back to verified mise release API downloads --- .../actions/setup-project-tools/action.yml | 44 ++++++++++++++++++ .github/workflows/master.yml | 46 ++++--------------- 2 files changed, 53 insertions(+), 37 deletions(-) create mode 100644 .github/actions/setup-project-tools/action.yml diff --git a/.github/actions/setup-project-tools/action.yml b/.github/actions/setup-project-tools/action.yml new file mode 100644 index 0000000000..dc17024a53 --- /dev/null +++ b/.github/actions/setup-project-tools/action.yml @@ -0,0 +1,44 @@ +name: Setup project tools +description: Install pinned mise tools with a verified Linux Release API fallback + +runs: + using: composite + steps: + - name: Install project tools + id: primary + continue-on-error: ${{ runner.os == 'Linux' && runner.arch == 'X64' }} + uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + with: + version: "2026.11.3" + sha256: >- + ${{ runner.os == 'Windows' && '4f796228c826ad6225d608e2029278b31677c8919508574498877543ed3431d8' || 'c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b' }} + + - name: Bootstrap verified mise through the Release API + id: fallback + if: steps.primary.outcome == 'failure' && runner.os == 'Linux' && runner.arch == 'X64' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + archive="$(mktemp)" + binary="$(mktemp)" + trap 'rm -f "$archive" "$binary"' EXIT + # These archive and binary hashes were verified against the release's signed SHASUMS256.txt. + # The API asset is the same v2026.11.3 Linux x64 tarball as the primary download. + timeout --kill-after=10s 180s gh api -H 'Accept: application/octet-stream' \ + repos/jdx/mise/releases/assets/716895278 > "$archive" + printf '837acae6defb8c9ad74823f6e38856633bacb04c65c1ac67d952749847543bd9 %s\n' "$archive" | sha256sum --check + tar -xOzf "$archive" mise/bin/mise > "$binary" + printf 'c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b %s\n' "$binary" | sha256sum --check + mise_directory="${MISE_DATA_DIR:-${XDG_DATA_HOME:-$HOME/.local/share}/mise}" + mkdir -p "$mise_directory/bin" + install -m 755 "$binary" "$mise_directory/bin/mise" + printf '2026.11.3' > "$mise_directory/bin/mise-version" + + - name: Install project tools with the verified bootstrap + if: steps.primary.outcome == 'failure' && steps.fallback.outcome == 'success' + uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + with: + version: "2026.11.3" + sha256: c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 3ef905b55c..6d419831a8 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -262,11 +262,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Set up project tools - uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 - with: - # Version and binary hashes are pinned from the verified, minisign-signed release checksums. - version: "2026.11.3" - sha256: c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b + uses: ./.github/actions/setup-project-tools - name: Cache DSH configuration dependencies if: matrix.suite == 'unit' @@ -862,10 +858,7 @@ jobs: cache-dependency-path: .artifacts/dify-daemon/go.sum - name: Set up project tools - uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 - with: - version: "2026.11.3" - sha256: c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b + uses: ./.github/actions/setup-project-tools - name: Check daemon serialization, host casting, Workflow selectors and SDK HTTP readback env: @@ -915,10 +908,7 @@ jobs: uses: ./.github/actions/setup-python-env - name: Set up project tools - uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 - with: - version: "2026.11.3" - sha256: c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b + uses: ./.github/actions/setup-project-tools - name: Cache DSH dependencies uses: ./.github/actions/cache-pnpm @@ -968,10 +958,7 @@ jobs: uses: ./.github/actions/setup-python-env - name: Set up project tools - uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 - with: - version: "2026.11.3" - sha256: c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b + uses: ./.github/actions/setup-project-tools - name: Cache shared JavaScript build tooling uses: ./.github/actions/cache-pnpm @@ -1014,10 +1001,7 @@ jobs: uses: ./.github/actions/setup-python-env - name: Set up project tools - uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 - with: - version: "2026.11.3" - sha256: c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b + uses: ./.github/actions/setup-project-tools - name: Cache Pi dependencies uses: ./.github/actions/cache-pnpm @@ -1043,10 +1027,7 @@ jobs: - name: Set up the Python environment uses: ./.github/actions/setup-python-env - - uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 - with: - version: "2026.11.3" - sha256: c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b + - uses: ./.github/actions/setup-project-tools - name: Cache OpenCode dependencies uses: ./.github/actions/cache-pnpm with: @@ -1070,10 +1051,7 @@ jobs: - name: Set up the Python environment uses: ./.github/actions/setup-python-env - - uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 - with: - version: "2026.11.3" - sha256: c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b + - uses: ./.github/actions/setup-project-tools # The pinned OpenClaw SDK requires the SQLite runtime shipped with Node 24.15+. - run: mise install node@24.15.0 - name: Cache OpenClaw dependencies @@ -1119,10 +1097,7 @@ jobs: persist-credentials: false - name: Set up website tools - uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 - with: - version: "2026.11.3" - sha256: 4f796228c826ad6225d608e2029278b31677c8919508574498877543ed3431d8 + uses: ./.github/actions/setup-project-tools - name: Cache website dependencies uses: ./.github/actions/cache-pnpm @@ -1151,10 +1126,7 @@ jobs: uses: ./.github/actions/setup-python-env - name: Set up website tools - uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 - with: - version: "2026.11.3" - sha256: c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b + uses: ./.github/actions/setup-project-tools - name: Cache website dependencies uses: ./.github/actions/cache-pnpm From ee12e4648e9622aeaa7a5f66dc64e3c6676a65a8 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Sun, 8 Nov 2026 18:53:14 +0800 Subject: [PATCH 12/25] fix(ci): share verified bootstrap with desktop Linux --- .github/actions/setup-project-tools/action.yml | 7 +++++++ .github/workflows/desktop.yml | 6 ++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/actions/setup-project-tools/action.yml b/.github/actions/setup-project-tools/action.yml index dc17024a53..cf014408dd 100644 --- a/.github/actions/setup-project-tools/action.yml +++ b/.github/actions/setup-project-tools/action.yml @@ -1,6 +1,11 @@ name: Setup project tools description: Install pinned mise tools with a verified Linux Release API fallback +inputs: + working-directory: + description: Directory containing the mise tool configuration + default: . + runs: using: composite steps: @@ -10,6 +15,7 @@ runs: uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 with: version: "2026.11.3" + working_directory: ${{ inputs.working-directory }} sha256: >- ${{ runner.os == 'Windows' && '4f796228c826ad6225d608e2029278b31677c8919508574498877543ed3431d8' || 'c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b' }} @@ -41,4 +47,5 @@ runs: uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 with: version: "2026.11.3" + working_directory: ${{ inputs.working-directory }} sha256: c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 66f17424f4..26dc563ec3 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -320,11 +320,9 @@ jobs: libxdo-dev \ openjdk-17-jre-headless \ pkg-config - - uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + - uses: ./.github/actions/setup-project-tools with: - version: "2026.11.3" - sha256: c2f7e814cb39f60b7f67a0ff36cb805e94128bf2dff09d175309d0f095b8cc3b - working_directory: desktop + working-directory: desktop - name: Install pinned Linux Rust toolchain run: rustup toolchain install 1.95.0 --profile minimal --component rustfmt --component clippy - name: Cache desktop dependencies From 4a670a2e2f5c446f4d7c46db69255ed045e1b149 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Sun, 8 Nov 2026 21:09:51 +0800 Subject: [PATCH 13/25] fix(ci): separate cold image pulls and stabilize setup checks --- .../setup-upgrade-oceanbase/action.yml | 10 +- .github/workflows/candidate-schema-replay.yml | 2 +- .github/workflows/capture-release-schema.yml | 2 +- .github/workflows/e2e-harness.yml | 25 ++++- .github/workflows/master.yml | 2 +- .github/workflows/skill-guidance.yml | 15 ++- .github/workflows/upgrade-release.yml | 2 +- .../powercontext/tests/status-http.spec.ts | 14 ++- tests/test_oceanbase_ci.py | 99 ++++++++++++++++++- tests/test_upgrade_release_gate.py | 2 +- 10 files changed, 154 insertions(+), 19 deletions(-) diff --git a/.github/actions/setup-upgrade-oceanbase/action.yml b/.github/actions/setup-upgrade-oceanbase/action.yml index 160e9f9d1a..519f585769 100644 --- a/.github/actions/setup-upgrade-oceanbase/action.yml +++ b/.github/actions/setup-upgrade-oceanbase/action.yml @@ -3,13 +3,21 @@ description: Start the disposable MySQL tenant used by installed-release upgrade runs: using: composite steps: + - name: Pull the OceanBase image + shell: bash + env: + OCEANBASE_IMAGE: ghcr.io/oceanbase/oceanbase-ce:4.3.5.6-106000012026040916 + run: | + if ! timeout --kill-after=5s 15s docker image inspect "$OCEANBASE_IMAGE" >/dev/null 2>&1; then + timeout --kill-after=10s 600s docker pull "$OCEANBASE_IMAGE" + fi - name: Start and verify OceanBase shell: bash env: OCEANBASE_IMAGE: ghcr.io/oceanbase/oceanbase-ce:4.3.5.6-106000012026040916 run: | mkdir -p "$RUNNER_TEMP/oceanbase-startup" - timeout --kill-after=10s 180s docker run --detach --name upgrade-oceanbase \ + timeout --kill-after=10s 180s docker run --pull never --detach --name upgrade-oceanbase \ --publish 127.0.0.1:2881:2881 \ --ulimit nofile=65536:65536 \ --env MODE=slim \ diff --git a/.github/workflows/candidate-schema-replay.yml b/.github/workflows/candidate-schema-replay.yml index 8b063819e4..30d4b043bc 100644 --- a/.github/workflows/candidate-schema-replay.yml +++ b/.github/workflows/candidate-schema-replay.yml @@ -29,7 +29,7 @@ jobs: if: ${{ !cancelled() && ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').candidate_replay) }} name: Candidate replay (${{ matrix.backend }}) runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'powercontext-ci-standard' || 'ubuntu-24.04' }} - timeout-minutes: 30 + timeout-minutes: ${{ matrix.backend == 'oceanbase' && 45 || 30 }} strategy: max-parallel: 2 fail-fast: false diff --git a/.github/workflows/capture-release-schema.yml b/.github/workflows/capture-release-schema.yml index 331b9f4ea3..120543c9f7 100644 --- a/.github/workflows/capture-release-schema.yml +++ b/.github/workflows/capture-release-schema.yml @@ -29,7 +29,7 @@ jobs: if: ${{ !cancelled() && ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').release_schema) }} name: Capture 1.4.0 (${{ matrix.backend }}, ${{ matrix.configuration }}) runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'powercontext-ci-standard' || 'ubuntu-24.04' }} - timeout-minutes: 45 + timeout-minutes: ${{ matrix.backend == 'oceanbase' && 60 || 45 }} strategy: max-parallel: 2 fail-fast: false diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index 435e6f6e0d..e2322691b7 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -115,7 +115,8 @@ jobs: if: ${{ !cancelled() && ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').database) }} name: Recall effort (OceanBase) runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'powercontext-ci-standard' || 'ubuntu-24.04' }} - timeout-minutes: 60 + # Bounded tests total about 91m; allow separate image pull/startup and final evidence. + timeout-minutes: 130 env: OCEANBASE_IMAGE: ghcr.io/oceanbase/oceanbase-ce:4.3.5.6-106000012026040916 POWERCONTEXT_TEST_OCEANBASE_URL: >- @@ -132,10 +133,17 @@ jobs: with: python-version: "3.12" + - name: Pull the OceanBase image + timeout-minutes: 11 + run: | + if ! timeout --kill-after=5s 15s docker image inspect "$OCEANBASE_IMAGE" >/dev/null 2>&1; then + timeout --kill-after=10s 600s docker pull "$OCEANBASE_IMAGE" + fi + - name: Start the isolated OceanBase test instance timeout-minutes: 14 run: | - timeout --kill-after=10s 180s docker run --detach --name recall-effort-oceanbase \ + timeout --kill-after=10s 180s docker run --pull never --detach --name recall-effort-oceanbase \ --publish 127.0.0.1:2881:2881 \ --ulimit nofile=65536:65536 \ --env MODE=slim \ @@ -325,8 +333,8 @@ jobs: ${{ !cancelled() && (github.event_name != 'workflow_dispatch' || inputs.suite != 'recall-effort') && ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').database) }} runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'powercontext-ci-standard' || 'ubuntu-24.04' }} - # Test steps total 205m; reserve 25m for setup, diagnostics and artifact upload. - timeout-minutes: 230 + # Test steps total 205m; reserve 40m for image pull, setup, diagnostics and artifact upload. + timeout-minutes: 245 strategy: max-parallel: 2 fail-fast: false @@ -350,10 +358,17 @@ jobs: with: python-version: "3.12" + - name: Pull the OceanBase image + timeout-minutes: 11 + run: | + if ! timeout --kill-after=5s 15s docker image inspect "$OCEANBASE_IMAGE" >/dev/null 2>&1; then + timeout --kill-after=10s 600s docker pull "$OCEANBASE_IMAGE" + fi + - name: Start the isolated OceanBase test instance timeout-minutes: 14 run: | - timeout --kill-after=10s 180s docker run --detach --name atomic-memory-clean-oceanbase \ + timeout --kill-after=10s 180s docker run --pull never --detach --name atomic-memory-clean-oceanbase \ --publish 127.0.0.1:2881:2881 \ --ulimit nofile=65536:65536 \ --env MODE=slim \ diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 6d419831a8..5f1153dfa9 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -118,7 +118,7 @@ jobs: needs: [quality, tests] if: ${{ !cancelled() && needs.quality.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.tests.result) && (fromJSON(needs.quality.outputs.suites || '{}').database) }} runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'powercontext-ci-standard' || 'ubuntu-24.04' }} - timeout-minutes: 20 + timeout-minutes: 35 env: POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL: mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_migration_probe_engine_crash?charset=utf8mb4 steps: diff --git a/.github/workflows/skill-guidance.yml b/.github/workflows/skill-guidance.yml index 9e47c1e16b..c1b89996e3 100644 --- a/.github/workflows/skill-guidance.yml +++ b/.github/workflows/skill-guidance.yml @@ -36,10 +36,19 @@ jobs: - uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7 with: python-version: '3.11' - cache: pip - cache-dependency-path: evaluation/skills/skill-up/requirements-dev.txt + - name: Set up uv + uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 + with: + version: "0.10.12" + enable-cache: true + cache-suffix: skill-guidance-py311 + cache-dependency-glob: evaluation/skills/skill-up/requirements-dev.txt - name: Install validation dependencies - run: python -m pip install -r evaluation/skills/skill-up/requirements-dev.txt + run: | + uv venv --python "$(command -v python)" "$RUNNER_TEMP/skill-guidance-env" + uv pip install --python "$RUNNER_TEMP/skill-guidance-env/bin/python" \ + -r evaluation/skills/skill-up/requirements-dev.txt + echo "$RUNNER_TEMP/skill-guidance-env/bin" >> "$GITHUB_PATH" - name: Install checksum-pinned skill-up shell: bash run: | diff --git a/.github/workflows/upgrade-release.yml b/.github/workflows/upgrade-release.yml index 7d5a36c31a..e87a3a4579 100644 --- a/.github/workflows/upgrade-release.yml +++ b/.github/workflows/upgrade-release.yml @@ -253,7 +253,7 @@ jobs: needs: prepare if: ${{ !inputs.skip_upgrade_acceptance }} runs-on: ${{ inputs.internal_runner && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'powercontext-ci-standard' || 'ubuntu-24.04' }} - timeout-minutes: 60 + timeout-minutes: ${{ matrix.backend == 'oceanbase' && 75 || 60 }} strategy: max-parallel: 3 fail-fast: false diff --git a/integrations/dsh/plugins/powercontext/tests/status-http.spec.ts b/integrations/dsh/plugins/powercontext/tests/status-http.spec.ts index 5c10ef7671..ae738abf34 100644 --- a/integrations/dsh/plugins/powercontext/tests/status-http.spec.ts +++ b/integrations/dsh/plugins/powercontext/tests/status-http.spec.ts @@ -34,6 +34,8 @@ const FLUSH = '/v1/memory/flush' const PREPARE = '/v1/context/prepare' const REQUEST_ID = 'req-review-http' const PRIVATE = 'private-http-body-marker' +// Healthy setup requests must tolerate runner scheduling delays before the fault under test. +const REQUEST_TIMEOUT_MS = 1000 const cleanup: Array<() => Promise> = [] function json(res: ServerResponse, value: unknown, status = 200) { @@ -52,7 +54,7 @@ function fault(status: number, mode: 'complete' | 'stall' | 'disconnect' | 'over } } -async function fixture(target: string, respond: (res: ServerResponse, occurrence: number) => void) { +async function fixture(target: string, respond: (res: ServerResponse, occurrence: number) => void, scopeDelayMs = 0) { const requests: string[] = [] const logs: Record[] = [] let occurrence = 0 @@ -61,6 +63,10 @@ async function fixture(target: string, respond: (res: ServerResponse, occurrence const path = new URL(req.url!, 'http://localhost').pathname requests.push(path) if (path === target) { respond(res, ++occurrence); return } + if (path === '/v1/scope-bindings/resolve' && scopeDelayMs > 0) { + setTimeout(() => json(res, { scope_id: 'review-scope' }), scopeDelayMs) + return + } json(res, path === '/v1/scope-bindings/resolve' ? { scope_id: 'review-scope' } : path === PREPARE ? { schema: 'powercontext.prepared-context.v1', status: 'ready', content: 'fixture', content_bytes: 7 } : path === CAPTURE ? { position: 2 } : path === FLUSH ? { current_cursor: 2 } : { status: 'ok' }, path === CAPTURE ? 202 : 200) @@ -85,7 +91,7 @@ async function fixture(target: string, respond: (res: ServerResponse, occurrence on: (name: string, listener: Hook) => { if (name === 'agent/pre-step') hook = listener }, logger: { debug: () => {}, warn: (line: string) => logs.push(JSON.parse(line)) }, } as unknown as Context, { baseUrl, authorization: 'Bearer http-fixture', timeoutMs: 10000, - requestTimeoutMs: 250, flushOnCapture: true, flushMaxCalls: 3 }) + requestTimeoutMs: REQUEST_TIMEOUT_MS, flushOnCapture: true, flushMaxCalls: 3 }) const agent = { session: { header: { id: 'http-review', cwd: '/review-workspace' } } } const messages = [{ content: [{ type: 'text', text: 'fixture prompt' }], source: { kind: 'user' } }] const run = (signal = new AbortController().signal) => hook({ agent, signal, turn: 1, messages }, @@ -93,7 +99,7 @@ async function fixture(target: string, respond: (res: ServerResponse, occurrence const command = (rawInput = '') => pc({ agent, rawInput, signal: new AbortController().signal }) const status = async () => JSON.parse((await command()).text.split('\nautomatic=')[1]) return { run, status, command, requests, logs, baseUrl, - client: new PowerContextClient({ baseUrl, requestTimeoutMs: 250 }) } + client: new PowerContextClient({ baseUrl, requestTimeoutMs: REQUEST_TIMEOUT_MS }) } } afterEach(async () => { @@ -103,7 +109,7 @@ afterEach(async () => { describe('registered /pc with real HTTP response failures', () => { it.each([CAPTURE, FLUSH])('distinguishes a complete 401 rejection at %s from an unknown write', async path => { - const h = await fixture(path, fault(401)) + const h = await fixture(path, fault(401), 350) await h.run() const stages = (await h.status()).stages expect(stages[path === CAPTURE ? 'capture' : 'flush']).toMatchObject({ state: 'unavailable', diff --git a/tests/test_oceanbase_ci.py b/tests/test_oceanbase_ci.py index 940b9519a6..d01f8a2474 100644 --- a/tests/test_oceanbase_ci.py +++ b/tests/test_oceanbase_ci.py @@ -18,6 +18,7 @@ import json import os +import re import shlex import shutil import signal @@ -239,4 +240,100 @@ def test_clean_migration_shard_budget_covers_slow_cases_and_keeps_cleanup_time() # Slow, passing cases can approach their deadline before the final case times out. # Include the runner's 180s collection timeout and 10s process termination grace. assert clean["timeout-minutes"] * 60 >= largest_shard * (case_timeout + 10) + 180 + 10 - assert job["timeout-minutes"] >= sum(step["timeout-minutes"] for step in steps) + 25 + pull = next(step for step in job["steps"] if step["name"] == "Pull the OceanBase image") + assert job["timeout-minutes"] >= sum(step["timeout-minutes"] for step in steps) + pull["timeout-minutes"] + 25 + + +@pytest.mark.parametrize("caller", ["recall-effort", "atomic-memory-clean-oceanbase-shards", "upgrade"]) +@pytest.mark.parametrize("cache_state", ["cold", "warm", "pull_failure"]) +def test_cold_image_download_does_not_consume_the_container_start_deadline(tmp_path, caller, cache_state): + if sys.platform == "win32": + pytest.skip("OceanBase CI runs on POSIX runners") + root = Path(__file__).parents[1] + if caller == "upgrade": + action = yaml.safe_load((root / ".github/actions/setup-upgrade-oceanbase/action.yml").read_text()) + steps = action["runs"]["steps"] + else: + workflow = yaml.safe_load((root / ".github/workflows/e2e-harness.yml").read_text()) + steps = workflow["jobs"][caller]["steps"] + startup = "\n".join( + step["run"] + for step in steps + if step.get("name") + in {"Pull the OceanBase image", "Start the isolated OceanBase test instance", "Start and verify OceanBase"} + ) + binaries = tmp_path / "bin" + binaries.mkdir() + # Model a cold image requiring five minutes, without spending that time in the test. + (binaries / "timeout").write_text( + f"#!{sys.executable}\n" + "import os, sys\n" + "args = sys.argv[1:]\n" + "while args[0].startswith('--'): args.pop(0)\n" + "os.environ['PC_DOCKER_DEADLINE'] = args.pop(0).removesuffix('s')\n" + "os.execvp(args[0], args)\n" + ) + (binaries / "docker").write_text( + f"#!{sys.executable}\n" + "import os, sys\n" + "from pathlib import Path\n" + "args = sys.argv[1:]\n" + "root = Path(os.environ['PC_DOCKER_STATE'])\n" + "image = root / 'image'\n" + "if args[:2] == ['image', 'inspect']: sys.exit(0 if image.exists() else 1)\n" + "if args[0] == 'pull':\n" + " (root / 'pull').touch()\n" + " if os.environ['PC_CACHE_STATE'] == 'pull_failure': sys.exit(124)\n" + " if int(os.environ['PC_DOCKER_DEADLINE']) < 300: sys.exit(124)\n" + " image.touch()\n" + "if args[0] == 'run':\n" + " if not image.exists():\n" + " if '--pull' in args and args[args.index('--pull') + 1] == 'never': sys.exit(125)\n" + " if int(os.environ['PC_DOCKER_DEADLINE']) < 300: sys.exit(124)\n" + " image.touch()\n" + " (root / 'container').touch()\n" + ) + (binaries / "python").write_text("#!/bin/sh\nexit 0\n") + for executable in binaries.iterdir(): + executable.chmod(0o755) + if cache_state == "warm": + (tmp_path / "image").touch() + env = { + **os.environ, + "PATH": str(binaries) + os.pathsep + os.environ["PATH"], + "PC_DOCKER_STATE": str(tmp_path), + "PC_CACHE_STATE": cache_state, + "OCEANBASE_IMAGE": "fixture-image", + "RUNNER_TEMP": str(tmp_path), + "MIGRATION_REPORTS": str(tmp_path / "reports"), + } + bash = shutil.which("bash") + assert bash is not None + result = subprocess.run( + [bash, "-e", "-o", "pipefail", "-c", startup], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=10, + check=False, + ) + assert result.returncode == (124 if cache_state == "pull_failure" else 0), result.stderr + assert (tmp_path / "container").exists() is (cache_state != "pull_failure") + assert (tmp_path / "pull").exists() is (cache_state != "warm") + + +def test_recall_job_preserves_startup_and_cleanup_beyond_command_deadlines(): + root = Path(__file__).parents[1] + workflow = yaml.safe_load((root / ".github/workflows/e2e-harness.yml").read_text()) + job = workflow["jobs"]["recall-effort"] + commands = [step["run"] for step in job["steps"] if "scripts/oceanbase_ci.py run" in step.get("run", "")] + deadlines = [int(value) for command in commands for value in re.findall(r"--timeout (\d+)", command)] + assert len(deadlines) == len(commands) + test_seconds = sum(deadline + 10 for deadline in deadlines) + setup_minutes = sum( + step["timeout-minutes"] + for step in job["steps"] + if step["name"] in {"Pull the OceanBase image", "Start the isolated OceanBase test instance"} + ) + assert job["timeout-minutes"] >= test_seconds / 60 + setup_minutes + 10 diff --git a/tests/test_upgrade_release_gate.py b/tests/test_upgrade_release_gate.py index 7e25bbc80a..3fc7a0ed6c 100644 --- a/tests/test_upgrade_release_gate.py +++ b/tests/test_upgrade_release_gate.py @@ -564,7 +564,7 @@ def test_oceanbase_sources_and_profiles_use_separate_provisioned_databases(tmp_p action = yaml.safe_load( (Path(__file__).parents[1] / ".github/actions/setup-upgrade-oceanbase/action.yml").read_text() ) - setup = action["runs"]["steps"][0]["run"] + setup = next(step["run"] for step in action["runs"]["steps"] if step["name"] == "Start and verify OceanBase") provisioned = set(re.findall(r"CREATE DATABASE (pc_upgrade[a-z0-9_]*)", setup)) provisioned.update(re.findall(r"--env OB_DATABASE=(pc_upgrade[a-z0-9_]*)", setup)) selected = set() From bac1a28a72986085c262185538416981bb9651d8 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Sun, 8 Nov 2026 21:12:34 +0800 Subject: [PATCH 14/25] test(ci): preserve background lifecycle progress on timeout --- tests/e2e/test_background_metrics.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/tests/e2e/test_background_metrics.py b/tests/e2e/test_background_metrics.py index d4f8201d22..2bc9dead6f 100644 --- a/tests/e2e/test_background_metrics.py +++ b/tests/e2e/test_background_metrics.py @@ -34,6 +34,7 @@ async def _database_objects(config: OceanBaseConfig) -> tuple[tuple[str, str], . @pytest.mark.parametrize("migration_backend", ["oceanbase"], indirect=True) def test_background_cli_serves_only_authorized_metrics_and_stops_cleanly(tmp_path, migration_backend): + import json import os import signal import socket @@ -44,6 +45,17 @@ def test_background_cli_serves_only_authorized_metrics_and_stops_cleanly(tmp_pat import httpx assert isinstance(migration_backend.config, OceanBaseConfig) + reports = os.environ.get("MIGRATION_REPORTS") + evidence = Path(reports) / "background-metrics" if reports else tmp_path + evidence.mkdir(parents=True, exist_ok=True) + started = time.monotonic() + + def checkpoint(stage: str) -> None: + # Persist progress even if the external deadline terminates pytest before it prints capture. + with (evidence / "lifecycle.jsonl").open("a") as output: + output.write(json.dumps({"stage": stage, "elapsed_seconds": time.monotonic() - started}) + "\n") + + checkpoint("inspect_empty_database") with socket.socket() as candidate: candidate.bind(("127.0.0.1", 0)) port = candidate.getsockname()[1] @@ -67,6 +79,7 @@ def test_background_cli_serves_only_authorized_metrics_and_stops_cleanly(tmp_pat ] before = asyncio.run(_database_objects(migration_backend.config)) assert before == () + checkpoint("reject_uninitialized_background") rejected = subprocess.run(command, env=environment, capture_output=True, text=True, timeout=60, check=False) assert rejected.returncode != 0, rejected.stdout + rejected.stderr assert "migration_required" in rejected.stdout + rejected.stderr @@ -81,7 +94,9 @@ def test_background_cli_serves_only_authorized_metrics_and_stops_cleanly(tmp_pat evidence_dir=tmp_path / "migration-evidence", lock_coordination="single-host", ) + checkpoint("plan_initialization") plan = runner.plan(backup_policy="skip") + checkpoint("apply_initialization") initialized = runner.apply( plan_id=plan.plan_id, accepted=True, @@ -91,9 +106,8 @@ def test_background_cli_serves_only_authorized_metrics_and_stops_cleanly(tmp_pat ) assert initialized.state == "ready" assert initialized.readiness_scope == "server" - reports = os.environ.get("MIGRATION_REPORTS") - log_path = Path(reports) / "background-metrics" / "background.log" if reports else tmp_path / "background.log" - log_path.parent.mkdir(parents=True, exist_ok=True) + checkpoint("start_background") + log_path = evidence / "background.log" with log_path.open("w") as output: process = subprocess.Popen( command, @@ -117,15 +131,18 @@ def test_background_cli_serves_only_authorized_metrics_and_stops_cleanly(tmp_pat f"Background CLI did not become live (returncode={process.poll()}); log tail:\n" + log_path.read_text(errors="replace")[-16000:] ) + checkpoint("verify_metrics") assert client.get(base + "/metrics").status_code == 401 headers = {"Authorization": "Bearer background-fixture-token"} response = client.get(base + "/metrics", headers=headers) assert response.status_code == 200 assert 'powercontext_server_readiness_state{state="ready"} 1.0' in response.text assert client.get(base + "/v1/scopes/default", headers=headers).status_code == 404 + checkpoint("stop_background") process.send_signal(signal.SIGTERM) assert process.wait(timeout=10) == 0 finally: if process.poll() is None: process.kill() process.wait(timeout=5) + checkpoint("complete") From 28d5afa16435c506b840e0d5d5787239223a2ab2 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Sun, 8 Nov 2026 21:15:55 +0800 Subject: [PATCH 15/25] fix(ci): restore executable harness entrypoint Reuse the entrypoint fix from oceanbase/powercontext#2537 to unblock backend acceptance. Co-authored-by: knqiufan --- e2e/bub/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/e2e/bub/Dockerfile b/e2e/bub/Dockerfile index fdafa03abc..32be4f0d97 100644 --- a/e2e/bub/Dockerfile +++ b/e2e/bub/Dockerfile @@ -51,4 +51,6 @@ RUN --mount=type=cache,id=powercontext-bub-e2e-uv,target=/root/.cache/uv \ COPY --chmod=0755 e2e/bub/container-entrypoint.sh /usr/local/bin/powercontext-e2e-container -ENTRYPOINT ["powercontext-e2e-container"] +RUN chmod 0755 /usr/local/bin/powercontext-e2e-container && test -x /usr/local/bin/powercontext-e2e-container + +ENTRYPOINT ["/usr/local/bin/powercontext-e2e-container"] From 38d4296c6efc2f3a347a13ae2de61d0d277a942c Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Sun, 8 Nov 2026 21:40:59 +0800 Subject: [PATCH 16/25] fix(ci): budget full server initialization before lifecycle checks --- .github/workflows/e2e-harness.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index e2322691b7..0452923975 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -333,8 +333,8 @@ jobs: ${{ !cancelled() && (github.event_name != 'workflow_dispatch' || inputs.suite != 'recall-effort') && ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').database) }} runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'powercontext-ci-standard' || 'ubuntu-24.04' }} - # Test steps total 205m; reserve 40m for image pull, setup, diagnostics and artifact upload. - timeout-minutes: 245 + # Test steps total 210m; reserve 40m for image pull, setup, diagnostics and artifact upload. + timeout-minutes: 250 strategy: max-parallel: 2 fail-fast: false @@ -386,11 +386,12 @@ jobs: tests/e2e/test_scope_search_migration.py - name: Test background listener lifecycle without skips - # One full Server case (300s), collection (180s), and termination grace. - timeout-minutes: 10 + # Full Server DDL was still progressing at 277s in CI, before listener verification. + # One 600s lifecycle, 180s collection and process termination grace fit within 15m. + timeout-minutes: 15 run: | uv run --locked --extra server python scripts/run_atomic_memory_clean_migration.py \ - oceanbase --test-timeout 300 --shard "${{ matrix.shard }}/2" --durations .github/oceanbase-migration-durations.json --output "$MIGRATION_REPORTS/background-metrics" --tests \ + oceanbase --test-timeout 600 --shard "${{ matrix.shard }}/2" --durations .github/oceanbase-migration-durations.json --output "$MIGRATION_REPORTS/background-metrics" --tests \ tests/e2e/test_background_metrics.py - name: Run clean migration evidence, grants and vectors without skips From be601d51b4cb4c1e58fa5736c9def60f9aa56433 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Sun, 8 Nov 2026 22:35:24 +0800 Subject: [PATCH 17/25] test(memory): preserve sequence locators in retry expectations --- .../e2e/test_atomic_memory_reconciliation_retry.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/e2e/test_atomic_memory_reconciliation_retry.py b/tests/e2e/test_atomic_memory_reconciliation_retry.py index 6d1717a08c..0e8fab3c05 100644 --- a/tests/e2e/test_atomic_memory_reconciliation_retry.py +++ b/tests/e2e/test_atomic_memory_reconciliation_retry.py @@ -30,6 +30,7 @@ from powercontext.builtin.runtime import BuiltinConfig, CaptureSource, RuntimeConfig, open_builtin_runtime from powercontext.builtin.runtime.config import InferenceConfig from powercontext.builtin.scope import ScopeDraft +from powercontext.sources import SourceSequenceLocator REVISE_RULE = "revise must have exactly one published identity" @@ -343,7 +344,9 @@ async def scenario(): assert len(entries) == len({_PUBLISHED_WORKFLOW, proposal_text}) if proposal_text != _PUBLISHED_WORKFLOW: retained = next(entry for entry in entries if entry.ref != original.ref) - assert retained.artifact.lineage.sources == (receipt.source_ref,) + assert retained.artifact.lineage.sources == ( + receipt.source_ref.model_copy(update={"locator": SourceSequenceLocator(sequence=receipt.sequence)}), + ) assert len(provider.retry_feedback) == max_requests - 1 await memory.flush() assert (await memory.list()).items == entries @@ -366,7 +369,9 @@ async def scenario(): provider.proposal_text, } retained = next(entry for entry in current if entry.artifact.content.text == provider.proposal_text) - assert retained.artifact.lineage.sources == (later.source_ref,) + assert retained.artifact.lineage.sources == ( + later.source_ref.model_copy(update={"locator": SourceSequenceLocator(sequence=later.sequence)}), + ) assert await memory.get(original.ref.artifact_id) == original history = await runtime.records.for_scope(scope.scope_id).list_artifact_revisions( "atomic-memory", original.ref.artifact_id, limit=10, cursor=None @@ -390,7 +395,9 @@ def test_changed_workflow_revises_published_fact_and_preserves_history(monkeypat assert current.ref.artifact_id == original.ref.artifact_id assert current.ref.revision == original.ref.revision + 1 assert current.artifact.content.text == _UPDATED_WORKFLOW - assert current.artifact.lineage.sources == (receipt.source_ref,) + assert current.artifact.lineage.sources == ( + receipt.source_ref.model_copy(update={"locator": SourceSequenceLocator(sequence=receipt.sequence)}), + ) assert current.artifact.lineage.artifacts == (original.ref,) assert len(history.items) == 2 assert not provider.retry_feedback From acce1dde1a3d1031831990c95232d50ac98740f5 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Sun, 8 Nov 2026 23:28:35 +0800 Subject: [PATCH 18/25] fix(ci): reserve unit-test setup and cache cleanup time --- .github/workflows/master.yml | 6 ++++-- tests/test_ci_pipeline.py | 19 ++++++++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 68e2297fb9..11681840d8 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -240,8 +240,8 @@ jobs: name: tests (${{ matrix.python-version }}, ${{ matrix.suite }}) needs: [quality] if: ${{ !cancelled() && needs.quality.result == 'success' && (fromJSON(needs.quality.outputs.suites || '{}').tests) }} - # E2E: 50m for tests plus 10m for setup and post-job cache/cleanup on every Python version. - timeout-minutes: ${{ matrix.suite == 'e2e' && 60 || 40 }} + # E2E: 50m + 10m setup/cleanup. Unit: 40m + 5m floor check + 15m setup/cleanup. + timeout-minutes: 60 runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'powercontext-ci-standard' || 'ubuntu-24.04' }} strategy: fail-fast: true @@ -285,12 +285,14 @@ jobs: - name: Run unit tests if: matrix.suite == 'unit' + timeout-minutes: 40 env: PYTEST_ADDOPTS: --require-dsh-config-runtime --require-claude-code-python-floor run: make unit-test - name: Check Source serialization with minimum Pydantic if: matrix.suite == 'unit' && matrix.python-version == '3.11' + timeout-minutes: 5 run: >- uv run --with 'pydantic==2.12.0' python -m pytest tests/test_source_locators.py tests/test_source_observations.py tests/test_api_contract.py -q diff --git a/tests/test_ci_pipeline.py b/tests/test_ci_pipeline.py index 16b71596e3..89b240fdcc 100644 --- a/tests/test_ci_pipeline.py +++ b/tests/test_ci_pipeline.py @@ -308,13 +308,22 @@ def test_actual_ci_result_script_rejects_missing_selected_acceptance( assert (run.returncode == 0) == passes, run.stdout + run.stderr -def test_e2e_deadline_reserves_setup_and_cache_time_for_every_python(): +@pytest.mark.parametrize( + ("test_names", "reserve_minutes"), + [ + (("Run unit tests", "Check Source serialization with minimum Pydantic"), 15), + (("Run end-to-end tests",), 10), + ], + ids=("unit", "e2e"), +) +def test_suite_deadline_reserves_setup_and_cache_time_for_every_python(test_names, reserve_minutes): job = workflow("master.yml")["jobs"]["tests"] - assert job["timeout-minutes"] == "${{ matrix.suite == 'e2e' && 60 || 40 }}" - steps = [step for step in job["steps"] if step.get("if") == "matrix.suite == 'e2e'"] - assert steps + # One common job budget covers every supported Python version, including the floor check. + assert isinstance(job["timeout-minutes"], int) + steps = [step for step in job["steps"] if step.get("name") in test_names] + assert len(steps) == len(test_names) assert all(isinstance(step.get("timeout-minutes"), int) for step in steps) - assert sum(step["timeout-minutes"] for step in steps) <= 50 + assert job["timeout-minutes"] >= sum(step["timeout-minutes"] for step in steps) + reserve_minutes @pytest.mark.skipif(sys.platform == "win32", reason="CI commands require POSIX make and shell") From 62967a3c2bf29422ef4469324d3490c19d4f8f57 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Mon, 9 Nov 2026 10:55:47 +0800 Subject: [PATCH 19/25] fix(ci): reuse search schemas and preserve native cleanup budgets --- .github/workflows/e2e-harness.yml | 16 +++---- .github/workflows/master.yml | 14 +++--- ...ic_memory_search_authorization_snapshot.py | 46 ++++++++++++------- tests/e2e/test_background_metrics.py | 3 +- 4 files changed, 47 insertions(+), 32 deletions(-) diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index 0452923975..8f603ea35e 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -115,8 +115,8 @@ jobs: if: ${{ !cancelled() && ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').database) }} name: Recall effort (OceanBase) runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'powercontext-ci-standard' || 'ubuntu-24.04' }} - # Bounded tests total about 91m; allow separate image pull/startup and final evidence. - timeout-minutes: 130 + # Bounded tests total about 106m; allow separate image pull/startup and final evidence. + timeout-minutes: 145 env: OCEANBASE_IMAGE: ghcr.io/oceanbase/oceanbase-ce:4.3.5.6-106000012026040916 POWERCONTEXT_TEST_OCEANBASE_URL: >- @@ -231,7 +231,7 @@ jobs: mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_migration_probe_server_recovery?charset=utf8mb4 run: | python scripts/oceanbase_ci.py run --container recall-effort-oceanbase \ - --timeout 900 --output .powercontext/recall-effort-ci/historical-server-upgrade-process -- uv run --locked pytest -vv -s -rs \ + --timeout 1800 --output .powercontext/recall-effort-ci/historical-server-upgrade-process -- uv run --locked pytest -vv -s -rs \ tests/builtin/persistence/test_mysql_migration_runner.py::test_historical_memory_upgrades_and_resumes_native_ddl \ -o faulthandler_timeout=180 --durations=0 \ -k oceanbase --junitxml=.powercontext/recall-effort-ci/historical-server-upgrade.xml @@ -333,8 +333,8 @@ jobs: ${{ !cancelled() && (github.event_name != 'workflow_dispatch' || inputs.suite != 'recall-effort') && ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').database) }} runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'powercontext-ci-standard' || 'ubuntu-24.04' }} - # Test steps total 210m; reserve 40m for image pull, setup, diagnostics and artifact upload. - timeout-minutes: 250 + # Test steps total 230m; reserve 40m for image pull, setup, diagnostics and artifact upload. + timeout-minutes: 270 strategy: max-parallel: 2 fail-fast: false @@ -395,11 +395,11 @@ jobs: tests/e2e/test_background_metrics.py - name: Run clean migration evidence, grants and vectors without skips - # The larger shard has 25 cases: 25 * (300s + 10s grace) + 190s collection < 140m. - timeout-minutes: 140 + # The larger shard has 25 cases: 25 * (360s + 10s grace) + 190s collection < 160m. + timeout-minutes: 160 run: | uv run --locked python scripts/run_atomic_memory_clean_migration.py \ - oceanbase --test-timeout 300 --shard "${{ matrix.shard }}/2" --durations .github/oceanbase-migration-durations.json --output "$MIGRATION_REPORTS/clean-migration" + oceanbase --test-timeout 360 --shard "${{ matrix.shard }}/2" --durations .github/oceanbase-migration-durations.json --output "$MIGRATION_REPORTS/clean-migration" - name: Test concurrent managed startup without skips # Up to two full Server cases (300s each), plus collection (180s). diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 11681840d8..5a6e054eaf 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -493,9 +493,9 @@ jobs: needs: [quality, tests] if: ${{ !cancelled() && needs.quality.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.tests.result) && (fromJSON(needs.quality.outputs.suites || '{}').database) }} # Shard 0 also runs indexing, bootstrap (20m), and archive validation (5m) - # before migrations (40m). Preserve each step's deadline and leave time + # before migrations (50m). Preserve each step's deadline and leave time # for environment setup and diagnostic uploads. - timeout-minutes: 80 + timeout-minutes: 90 runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'powercontext-ci-standard' || 'ubuntu-24.04' }} strategy: fail-fast: false @@ -555,8 +555,8 @@ jobs: - name: Test unified migrations with embedded seekdb if: ${{ !cancelled() && steps.seekdb_setup.outcome == 'success' }} # Last-client shutdown adds native startup work. Partition runner nodes; - # Source evidence expands the batch; every node keeps its 180-second deadline. - timeout-minutes: 40 + # Full Server cases can pass at 175s; allow process startup and native shutdown as well. + timeout-minutes: 50 env: SEEKDB_MIGRATION_SHARD: ${{ matrix.migration-shard }} POWERCONTEXT_TEST_MIGRATION_SEEKDB: "1" @@ -606,7 +606,7 @@ jobs: fi seekdb_runner_node="${seekdb_runner_nodes[$index]}" uv run --locked --extra seekdb --extra code python .github/scripts/run_seekdb_migration_test.py \ - "$seekdb_runner_node" \ + --timeout 210 "$seekdb_runner_node" \ 2>&1 | tee -a seekdb-migrations.log done # These are required independent runs, not retries: any failure stops @@ -644,7 +644,7 @@ jobs: needs: [quality, seekdb-managed-startup, tests] if: ${{ !cancelled() && needs.quality.result == 'success' && contains(fromJSON('["success","skipped"]'), needs.tests.result) && (fromJSON(needs.quality.outputs.suites || '{}').database) }} name: Atomic Memory clean migration (SeekDB) - timeout-minutes: 55 + timeout-minutes: 65 runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'powercontext-ci-standard' || 'ubuntu-24.04' }} env: POWERCONTEXT_TEST_MIGRATION_SEEKDB: "1" @@ -662,7 +662,7 @@ jobs: python-version: "3.12" - name: Run clean migration evidence, grants and vectors without skips - timeout-minutes: 35 + timeout-minutes: 45 env: TMPDIR: ${{ runner.temp }}/am-seekdb run: | diff --git a/tests/e2e/test_atomic_memory_search_authorization_snapshot.py b/tests/e2e/test_atomic_memory_search_authorization_snapshot.py index 1aa7eafa5b..8f5b00f19d 100644 --- a/tests/e2e/test_atomic_memory_search_authorization_snapshot.py +++ b/tests/e2e/test_atomic_memory_search_authorization_snapshot.py @@ -19,7 +19,7 @@ import asyncio import importlib.util import os -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Iterator from contextlib import asynccontextmanager from pathlib import Path from tempfile import TemporaryDirectory @@ -130,6 +130,22 @@ async def _isolated_database(backend: str, tmp_path: Path) -> AsyncIterator[Data await connection.exec_driver_sql(f"DROP DATABASE `{name}`") +@pytest.fixture(scope="module") +def search_database( + request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[DatabaseConfig]: + # These cases exercise Scope authorization and snapshots, not schema migration. + # Reuse the schema while each case creates its own Scope and idempotency keys. + # Keep the management connection's creation and cleanup on the same event loop. + with asyncio.Runner() as runner: + context = _isolated_database(request.param, tmp_path_factory.mktemp("search-snapshots")) + database = runner.run(context.__aenter__()) + try: + yield database + finally: + runner.run(context.__aexit__(None, None, None)) + + @asynccontextmanager async def _server(database: DatabaseConfig, provider: str, tmp_path: Path, embedding: _Embedding): settings = ServerSettings( @@ -170,7 +186,9 @@ async def _server(database: DatabaseConfig, provider: str, tmp_path: Path, embed async def _seed(admin: PowerContextClient): scope_id = ( await admin.create_scope( - CreateScopeRequest(title="Search snapshot", summary="Concurrent access", idempotency_key="snapshot") + CreateScopeRequest( + title="Search snapshot", summary="Concurrent access", idempotency_key=f"snapshot-{uuid4().hex}" + ) ) ).scope_id remembered = await admin.remember_memory( @@ -181,7 +199,7 @@ async def _seed(admin: PowerContextClient): "subject": {"type": VIEWER.type, "id": VIEWER.id}, "resource": {"type": "scope", "scope_id": scope_id}, "role": "scope.viewer", - "idempotency_key": "viewer-scope", + "idempotency_key": f"viewer-scope-{scope_id}", }) ) return scope_id, remembered.records[0], binding @@ -190,7 +208,7 @@ async def _seed(admin: PowerContextClient): async def _revoke_and_create(admin: PowerContextClient, scope_id: str, binding): revoked = await admin.revoke_access_binding( RevokeAccessBindingRequest( - binding_id=binding.binding_id, expected_version=binding.version, idempotency_key="revoke-viewer" + binding_id=binding.binding_id, expected_version=binding.version, idempotency_key=f"revoke-viewer-{scope_id}" ) ) assert revoked.state == "revoked" @@ -207,16 +225,15 @@ async def _assert_current_access_revoked(viewer: PowerContextClient, scope_id: s await viewer.get_artifact(scope_id, "atomic-memory", created.artifact.artifact_id) -@pytest.mark.parametrize("backend", ["sqlite", "oceanbase", "seekdb"]) +@pytest.mark.parametrize("search_database", ["sqlite", "oceanbase", "seekdb"], indirect=True) @pytest.mark.parametrize("provider", ["builtin", "casbin"]) @pytest.mark.parametrize("mode", ["vector", "hybrid"]) -def test_search_rechecks_access_after_query_embedding(backend: str, provider: str, mode: str, tmp_path: Path) -> None: +def test_search_rechecks_access_after_query_embedding( + search_database: DatabaseConfig, provider: str, mode: str, tmp_path: Path +) -> None: async def scenario() -> None: embedding = _Embedding() - async with ( - _isolated_database(backend, tmp_path) as database, - _server(database, provider, tmp_path, embedding) as (_, admin, viewer), - ): + async with _server(search_database, provider, tmp_path, embedding) as (_, admin, viewer): scope_id, _, binding = await _seed(admin) embedding.pause_query = True pending = asyncio.create_task( @@ -391,17 +408,14 @@ async def scenario(): asyncio.run(scenario()) -@pytest.mark.parametrize("backend", ["sqlite", "oceanbase", "seekdb"]) +@pytest.mark.parametrize("search_database", ["sqlite", "oceanbase", "seekdb"], indirect=True) @pytest.mark.parametrize("provider", ["builtin", "casbin"]) @pytest.mark.parametrize("mode", ["text", "vector", "hybrid"]) def test_search_pins_access_and_content_without_blocking_writers( - backend: str, provider: str, mode: str, tmp_path: Path, monkeypatch + search_database: DatabaseConfig, provider: str, mode: str, tmp_path: Path, monkeypatch ) -> None: async def scenario() -> None: - async with ( - _isolated_database(backend, tmp_path) as database, - _server(database, provider, tmp_path, _Embedding()) as (_, admin, viewer), - ): + async with _server(search_database, provider, tmp_path, _Embedding()) as (_, admin, viewer): scope_id, original, binding = await _seed(admin) filters = AtomicMemoryAccess.filters authorized = asyncio.Event() diff --git a/tests/e2e/test_background_metrics.py b/tests/e2e/test_background_metrics.py index 2bc9dead6f..1a7e6be1d9 100644 --- a/tests/e2e/test_background_metrics.py +++ b/tests/e2e/test_background_metrics.py @@ -117,7 +117,8 @@ def checkpoint(stage: str) -> None: ) try: base = f"http://127.0.0.1:{port}" - deadline = time.monotonic() + 20 + # The fresh process must import the Server and inspect the initialized native schema. + deadline = time.monotonic() + 60 ready = False with httpx.Client(timeout=0.5) as client: while time.monotonic() < deadline and process.poll() is None: From 78c118408f35dbc6ab0384a8f8a1bf2d2d3e4956 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Mon, 9 Nov 2026 10:55:47 +0800 Subject: [PATCH 20/25] test(ci): align integration fixtures with current contracts --- .../src-tauri/tests/deployment_identity.rs | 57 ++++++++++++++++--- .../powercontext/scripts/e2e-server.mjs | 3 +- .../powercontext_connector_opendal/source.py | 12 ++-- 3 files changed, 57 insertions(+), 15 deletions(-) diff --git a/desktop/src-tauri/tests/deployment_identity.rs b/desktop/src-tauri/tests/deployment_identity.rs index b818b16743..729f51d2be 100644 --- a/desktop/src-tauri/tests/deployment_identity.rs +++ b/desktop/src-tauri/tests/deployment_identity.rs @@ -19,7 +19,7 @@ mod common; use powercontext_desktop::{ connections::{ profiles::ProfileRepository, - session::{ConnectionManager, WriteStatus}, + session::{CompatibilityProfile, ConnectionManager, WriteStatus}, }, credentials::NativeVault, error::SafeError, @@ -85,6 +85,16 @@ impl Fixture { Self::with_profile(authenticated, None).await } async fn with_profile(authenticated: bool, compatibility: Option<&str>) -> Self { + Self::with_evidence(authenticated, compatibility, false).await + } + async fn with_legacy_evidence(authenticated: bool, compatibility: &str) -> Self { + Self::with_evidence(authenticated, Some(compatibility), true).await + } + async fn with_evidence( + authenticated: bool, + compatibility: Option<&str>, + synthetic_legacy_evidence: bool, + ) -> Self { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let endpoint = format!("http://{}", listener.local_addr().unwrap()); let state = Arc::new(Mutex::new(State { @@ -217,14 +227,34 @@ impl Fixture { Arc::new(NativeVault), ) .unwrap(); - let manager = Arc::new(if compatibility.is_some() { + let mut profile_id = compatibility.unwrap_or(common::PROFILE_ID).to_owned(); + let manager = Arc::new(if synthetic_legacy_evidence { + let profiles: Vec = + serde_json::from_str(include_str!("../src/connections/compatibility.json")) + .unwrap(); + let mut evidence = profiles.into_iter().find(|p| p.id == profile_id).unwrap(); + let contract: Value = + serde_json::from_str(include_str!("../src/transport/operations.json")).unwrap(); + // These HTTP fixtures prove fencing behavior, not release compatibility. + // Keep the bundled qualification records unchanged when the client contract advances. + profile_id = format!("synthetic-{}", evidence.id); + evidence.id = profile_id.clone(); + evidence.authenticated_clients = Some( + [( + contract["contractSha256"].as_str().unwrap().to_owned(), + "Synthetic HTTP fixture; not release qualification".to_owned(), + )] + .into(), + ); + ConnectionManager::with_compatibility(repository, vec![evidence]) + } else if compatibility.is_some() { ConnectionManager::new(repository) } else { common::connection_manager(repository) }); let saved = manager.save_profile(serde_json::from_value(json!({"id":null,"revision":null,"name":"Synthetic", "endpoint":endpoint,"authentication":if authenticated {"bearer"} else {"unauthenticated_loopback"}, - "caPem":null,"compatibility":compatibility.unwrap_or(common::PROFILE_ID),"keepCredential":false, + "caPem":null,"compatibility":profile_id,"keepCredential":false, "credential":if authenticated {json!({"secret":"synthetic-token","storage":"session_only"})} else {Value::Null}})).unwrap()).unwrap(); Self { manager, @@ -270,9 +300,9 @@ impl Fixture { } #[tokio::test] -async fn bundled_authenticated_legacy_profile_keeps_its_operations_and_principal_fence() { +async fn authenticated_legacy_evidence_keeps_its_operations_and_principal_fence() { for compatibility in ["sqlite-atomic-112db11e-v1", "sqlite-atomic-3cefa45a-v1"] { - let fixture = Fixture::with_profile(true, Some(compatibility)).await; + let fixture = Fixture::with_legacy_evidence(true, compatibility).await; fixture.state.lock().unwrap().discovery.1["feature_contracts"] .as_object_mut() .unwrap() @@ -322,10 +352,21 @@ async fn bundled_authenticated_legacy_profile_keeps_its_operations_and_principal } } +#[tokio::test] +async fn unqualified_bundled_legacy_profiles_do_not_enable_current_client_operations() { + for compatibility in ["sqlite-atomic-112db11e-v1", "sqlite-atomic-3cefa45a-v1"] { + let fixture = Fixture::with_profile(true, Some(compatibility)).await; + let state = fixture.manager.check(&fixture.id, true).await.unwrap(); + let report = state.active.unwrap().report; + assert!(!report.compatibility_verified); + assert!(report.supported_operations.is_empty()); + } +} + #[tokio::test] async fn authenticated_client_evidence_requires_bearer_mode_and_a_valid_principal() { for bearer in [false, true] { - let fixture = Fixture::with_profile(bearer, Some("sqlite-atomic-112db11e-v1")).await; + let fixture = Fixture::with_legacy_evidence(bearer, "sqlite-atomic-112db11e-v1").await; fixture.state.lock().unwrap().principal = None; let state = fixture.manager.check(&fixture.id, true).await.unwrap(); let report = state.active.unwrap().report; @@ -742,7 +783,7 @@ async fn a_late_readiness_probe_cannot_repopulate_an_invalidated_report() { #[tokio::test] async fn failed_discovery_cannot_downgrade_an_active_deployment_to_legacy() { for status in [404, 401, 403, 503] { - let fixture = Fixture::with_profile(true, Some("sqlite-atomic-112db11e-v1")).await; + let fixture = Fixture::with_legacy_evidence(true, "sqlite-atomic-112db11e-v1").await; let generation = fixture.activate().await; fixture.state.lock().unwrap().discovery = (status, json!({})); assert!(fixture.manager.scopes(generation, "", None).await.is_err()); @@ -752,7 +793,7 @@ async fn failed_discovery_cannot_downgrade_an_active_deployment_to_legacy() { #[tokio::test] async fn a_legacy_session_requires_explicit_recheck_when_discovery_appears() { - let fixture = Fixture::with_profile(true, Some("sqlite-atomic-112db11e-v1")).await; + let fixture = Fixture::with_legacy_evidence(true, "sqlite-atomic-112db11e-v1").await; fixture.state.lock().unwrap().discovery = (404, json!({})); let generation = fixture.activate().await; assert!(fixture.manager.scopes(generation, "", None).await.is_ok()); diff --git a/integrations/dsh/plugins/powercontext/scripts/e2e-server.mjs b/integrations/dsh/plugins/powercontext/scripts/e2e-server.mjs index a931ab9839..2a0b59cbc9 100644 --- a/integrations/dsh/plugins/powercontext/scripts/e2e-server.mjs +++ b/integrations/dsh/plugins/powercontext/scripts/e2e-server.mjs @@ -126,7 +126,8 @@ export async function startPowerContextServer(options = {}) { child.stderr?.on('data', (chunk) => logs.push(String(chunk))) const baseUrl = `http://127.0.0.1:${port}` try { - await waitForUrl(`${baseUrl}/health/live`, 30000, () => { + // Cold Python imports can consume 30s on CI; keep cleanup inside the 60s hook budget. + await waitForUrl(`${baseUrl}/health/live`, 45000, () => { if (child.exitCode !== null) throw new Error(`Server exited with code ${child.exitCode}`) }) } catch (error) { diff --git a/integrations/opendal/src/powercontext_connector_opendal/source.py b/integrations/opendal/src/powercontext_connector_opendal/source.py index 920719eccc..a71885c5be 100644 --- a/integrations/opendal/src/powercontext_connector_opendal/source.py +++ b/integrations/opendal/src/powercontext_connector_opendal/source.py @@ -157,12 +157,12 @@ def project(self, source: TextFileSnapshotSource, /) -> BaseModel: metadata["provider_version"] = source.provider_version if source.modified_at is not None: metadata["modified_at"] = source.modified_at.isoformat() - return _TEXT_EVIDENCE_MODEL( - source_type=TEXT_FILE_SNAPSHOT_SOURCE_NAME, - source_id=source.name, - content=source.content, - metadata=metadata, - ) + return _TEXT_EVIDENCE_MODEL.model_validate({ + "source_type": TEXT_FILE_SNAPSHOT_SOURCE_NAME, + "source_id": source.name, + "content": source.content, + "metadata": metadata, + }) def _snapshot_id(namespace: str, path: str, content_digest: str) -> str: From af4ada37560d88b4d607599bf643c5f090fc107a Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Mon, 9 Nov 2026 10:55:48 +0800 Subject: [PATCH 21/25] fix(harness): reuse Python during agent bootstrap --- .../src/powercontext_e2e/agent_bootstrap.py | 49 +++++++++++ e2e/bub/src/powercontext_e2e/harbor_agent.py | 8 +- e2e/bub/tests/test_agent_bootstrap.py | 83 +++++++++++++++++++ 3 files changed, 135 insertions(+), 5 deletions(-) create mode 100644 e2e/bub/src/powercontext_e2e/agent_bootstrap.py create mode 100644 e2e/bub/tests/test_agent_bootstrap.py diff --git a/e2e/bub/src/powercontext_e2e/agent_bootstrap.py b/e2e/bub/src/powercontext_e2e/agent_bootstrap.py new file mode 100644 index 0000000000..3cbe7925cb --- /dev/null +++ b/e2e/bub/src/powercontext_e2e/agent_bootstrap.py @@ -0,0 +1,49 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Prepare the ACP runner without reinstalling an existing Python runtime.""" + +from __future__ import annotations + +import shlex + +# Match bub-acp-server's SDK pin and the harness container's uv release. +ACP_SDK_REQUIREMENT = "agent-client-protocol==1.0.0rc2" +UV_REQUIREMENT = "uv==0.10.12" + + +def runner_setup_command(runner_path: str, fallback: str, *, log_path: str = "/logs/agent/bootstrap.log") -> str: + """Reuse Python images, retain the upstream bootstrap for other task images, and preserve failure logs.""" + runner = shlex.quote(runner_path) + python = shlex.quote(f"{runner_path}/bin/python") + log = shlex.quote(log_path) + return f""" +set -euo pipefail +mkdir -p "$(dirname {log})" +{{ + if python3 -c 'import ensurepip, ssl, venv' >/dev/null 2>&1; then + echo 'ACP bootstrap: reuse the task image Python runtime' + if [ ! -x {python} ]; then + python3 -m venv {runner} + fi + else + echo 'ACP bootstrap: install the missing system runtime' + bash -c {shlex.quote(fallback)} + fi + echo 'ACP bootstrap: install pinned SDK and uv' + {python} -m pip install --disable-pip-version-check --no-input --retries 1 --timeout 30 \ + {shlex.quote(ACP_SDK_REQUIREMENT)} {shlex.quote(UV_REQUIREMENT)} + echo 'ACP bootstrap: ready' +}} 2>&1 | tee {log} +""".strip() diff --git a/e2e/bub/src/powercontext_e2e/harbor_agent.py b/e2e/bub/src/powercontext_e2e/harbor_agent.py index b347f13721..c4d5ddcde6 100644 --- a/e2e/bub/src/powercontext_e2e/harbor_agent.py +++ b/e2e/bub/src/powercontext_e2e/harbor_agent.py @@ -26,6 +26,8 @@ from harbor.environments.base import BaseEnvironment from harbor.models.agent.context import AgentContext +from .agent_bootstrap import ACP_SDK_REQUIREMENT, runner_setup_command + AGENT_ID = "powercontext-bub-acp" REMOTE_BIN_DIR = "/installed-agent/bin" REMOTE_BUB_HOME = "/installed-agent/bub-home" @@ -41,10 +43,6 @@ POWERCONTEXT_VERSION = version("powercontext") # Change ACP_SDK_REQUIREMENT together with this version: it repeats the server's own pin of the ACP SDK. BUB_ACP_SERVER_VERSION = "0.1.3" -# bub-acp-server 0.1.3 pins this pre-release. uv accepts a pre-release only when a direct requirement names it, so the -# install must repeat the pin or it cannot resolve the server at all. A server release that pins another SDK version -# conflicts with this requirement and fails to install. -ACP_SDK_REQUIREMENT = "agent-client-protocol==1.0.0rc2" STEP_FAILURE_MARKER = "/logs/agent/powercontext-step-failed" # Harbor uploads each step's tests to this directory before running the step's verifier and leaves them there. STEP_TESTS_DIR = "/tests" @@ -103,7 +101,7 @@ async def run(self, instruction: str, environment: BaseEnvironment, context: Age async def install(self, environment: BaseEnvironment) -> None: await self.exec_as_root( environment, - command=self._build_dependencies_command("uvx"), + command=runner_setup_command(self._RUNNER_VENV_PATH, self._build_dependencies_command("uvx")), env={"DEBIAN_FRONTEND": "noninteractive", "PIP_INDEX_URL": pip_index_url()}, ) await self.exec_as_root(environment, command=_install_bub_command(powercontext=self._powercontext)) diff --git a/e2e/bub/tests/test_agent_bootstrap.py b/e2e/bub/tests/test_agent_bootstrap.py new file mode 100644 index 0000000000..009bbb72eb --- /dev/null +++ b/e2e/bub/tests/test_agent_bootstrap.py @@ -0,0 +1,83 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Exercise ACP bootstrap with offline packages and a real Python virtual environment.""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +import zipfile + +import pytest + +from powercontext_e2e.agent_bootstrap import runner_setup_command + + +@pytest.mark.parametrize("complete", [True, False]) +def test_python_image_bootstrap_avoids_system_install_and_preserves_package_failures(tmp_path, complete): + if sys.platform == "win32": + pytest.skip("Harbor task images use Linux bash") + wheels = tmp_path / "wheels" + wheels.mkdir() + packages = [("agent_client_protocol", "1.0.0rc2")] + if complete: + packages.append(("uv", "0.10.12")) + for name, version in packages: + dist = f"{name}-{version}.dist-info" + with zipfile.ZipFile(wheels / f"{name}-{version}-py3-none-any.whl", "w") as wheel: + wheel.writestr(f"{dist}/METADATA", f"Metadata-Version: 2.1\nName: {name}\nVersion: {version}\n") + wheel.writestr(f"{dist}/WHEEL", "Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n") + wheel.writestr(f"{dist}/RECORD", "") + binaries = tmp_path / "bin" + binaries.mkdir() + (binaries / "python3").symlink_to(sys.executable) + runner, log = tmp_path / "runner", tmp_path / "logs" / "bootstrap.log" + # A prepared Python image must work even when system package installation is unavailable. + command = runner_setup_command(str(runner), "echo unexpected-system-install; exit 67", log_path=str(log)) + shell = shutil.which("bash") + assert shell is not None + result = subprocess.run( # noqa: S603 -- checked-in bootstrap and an isolated offline fixture + [shell, "-c", command], + env={ + **os.environ, + "PATH": f"{binaries}{os.pathsep}{os.environ['PATH']}", + "PIP_NO_INDEX": "1", + "PIP_FIND_LINKS": str(wheels), + }, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + evidence = log.read_text() + assert "unexpected-system-install" not in evidence + if complete: + assert result.returncode == 0, result.stdout + result.stderr + subprocess.run( # noqa: S603 -- interpreter just created in the isolated fixture + [ + str(runner / "bin/python"), + "-c", + "from importlib.metadata import version; " + "assert version('agent-client-protocol') == '1.0.0rc2'; assert version('uv') == '0.10.12'", + ], + check=True, + ) + assert "ACP bootstrap: ready" in evidence + else: + assert result.returncode != 0 + assert "uv==0.10.12" in evidence + assert "ACP bootstrap: ready" not in evidence From 74906306cc47f9b0b178946c0ddc748b4837a7c2 Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Mon, 9 Nov 2026 11:53:16 +0800 Subject: [PATCH 22/25] test(oceanbase): support synchronous capability probes in scoring fixture --- tests/builtin/persistence/test_atomic_memory_search_scoring.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/builtin/persistence/test_atomic_memory_search_scoring.py b/tests/builtin/persistence/test_atomic_memory_search_scoring.py index 5e5daab0ce..5e35c40e9f 100644 --- a/tests/builtin/persistence/test_atomic_memory_search_scoring.py +++ b/tests/builtin/persistence/test_atomic_memory_search_scoring.py @@ -203,6 +203,9 @@ def __init__(self): self.sql = "" self.parameters = {} + async def run_sync(self, function): + return function(SimpleNamespace(engine=self.sync_engine)) + async def exec_driver_sql(self, _statement): return SimpleNamespace(close=lambda: None) From 3cb815f0fc0a9a6d4355f7c36e405008968eafcc Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Mon, 9 Nov 2026 14:12:07 +0800 Subject: [PATCH 23/25] fix(ci): recover pinned release downloads through GitHub API --- .../workflows/managed-resource-candidates.yml | 2 + scripts/prepare_managed_resources.py | 50 ++++++++++++++++++- tests/test_managed_resource_producer.py | 47 +++++++++++++++++ 3 files changed, 97 insertions(+), 2 deletions(-) diff --git a/.github/workflows/managed-resource-candidates.yml b/.github/workflows/managed-resource-candidates.yml index 59bbb90619..632aa860c1 100644 --- a/.github/workflows/managed-resource-candidates.yml +++ b/.github/workflows/managed-resource-candidates.yml @@ -76,6 +76,8 @@ jobs: uv run --no-sync ruff format --check scripts/prepare_managed_resources.py scripts/produce_managed_resources.py scripts/ci_managed_resource_smoke.py scripts/build_intel_managed_wheel.py tests/test_managed_resource_producer.py uv run --no-sync ty check scripts/prepare_managed_resources.py scripts/produce_managed_resources.py scripts/ci_managed_resource_smoke.py scripts/build_intel_managed_wheel.py tests/test_managed_resource_producer.py - name: Acquire pinned interpreter and freeze native wheel closure + env: + GH_TOKEN: ${{ github.token }} run: | uv run --no-sync python scripts/prepare_managed_resources.py --target "$RESOURCE_TARGET" --source-ref "$PACKAGE_SOURCE_SHA" --server-version "$PACKAGE_VERSION" --output build/resource-inputs --download-cache build/resource-downloads uv run --no-sync python - <<'PY' diff --git a/scripts/prepare_managed_resources.py b/scripts/prepare_managed_resources.py index c5f51390ee..6fd59ef7f1 100644 --- a/scripts/prepare_managed_resources.py +++ b/scripts/prepare_managed_resources.py @@ -34,11 +34,12 @@ import subprocess import tarfile import tempfile +import time import tomllib from concurrent.futures import ThreadPoolExecutor from pathlib import Path, PurePosixPath from typing import Any -from urllib.parse import unquote, urlsplit +from urllib.parse import quote, unquote, urlsplit import build_intel_managed_wheel from packaging.requirements import Requirement @@ -105,7 +106,48 @@ def target_spec(target: str, *, native: bool = True) -> dict[str, Any]: return spec +def _download_release_asset(url: str, partial: Path, env: dict[str, str], deadline: float) -> bool: + """Use the official API for the same pinned asset when the release route fails.""" + parsed = urlsplit(url) + parts = parsed.path.strip("/").split("/") + if ( + parsed.netloc != "github.com" + or parsed.query + or len(parts) != 6 + or parts[2:4] != ["releases", "download"] + or shutil.which("gh") is None + or time.monotonic() >= deadline + ): + return False + owner, repository, _, _, tag, filename = parts + endpoint = f"repos/{owner}/{repository}/releases" + command = ["gh", "api", "--hostname", "github.com"] + result = subprocess.run( + [*command, f"{endpoint}/tags/{quote(unquote(tag), safe='')}"], + check=True, + capture_output=True, + text=True, + env=env, + timeout=min(30, max(0.001, deadline - time.monotonic())), + ) + assets = json.loads(result.stdout)["assets"] + asset = next((asset for asset in assets if asset["name"] == unquote(filename)), None) + if asset is None or type(asset.get("id")) is not int or asset["id"] <= 0: + raise ValueError("The pinned GitHub release asset is unavailable") + partial.unlink(missing_ok=True) + with partial.open("xb") as output: + subprocess.run( + [*command, f"{endpoint}/assets/{asset['id']}", "--header", "Accept: application/octet-stream"], + check=True, + stdout=output, + env=env, + timeout=max(0.001, deadline - time.monotonic()), + ) + return True + + def _download(url: str, partial: Path) -> None: + deadline = time.monotonic() + 1900 command = [ "curl", "--fail", @@ -127,8 +169,12 @@ def _download(url: str, partial: Path) -> None: env = environment() for attempt in range(2): try: - subprocess.run(command, check=True, env=env, timeout=1900) + subprocess.run(command, check=True, env=env, timeout=max(0.001, deadline - time.monotonic())) except subprocess.CalledProcessError as error: + if error.returncode in {5, 6, 7, 18, 22, 28, 35, 52, 55, 56} and _download_release_asset( + url, partial, env, deadline + ): + return # curl's own retry cannot recover from a process crash on the Windows runner. if attempt or error.returncode != _WINDOWS_ACCESS_VIOLATION: raise diff --git a/tests/test_managed_resource_producer.py b/tests/test_managed_resource_producer.py index 782e973f87..01abbd41cd 100644 --- a/tests/test_managed_resource_producer.py +++ b/tests/test_managed_resource_producer.py @@ -30,6 +30,7 @@ from typing import Any import httpx +import prepare_managed_resources import pytest from build_intel_managed_wheel import _check_linkage, native_build_environment, verify_backend from ci_managed_resource_smoke import MAX_DIAGNOSTIC_BYTES, preserve_failure @@ -246,6 +247,52 @@ def download(command: list[str], **_options: Any) -> None: assert next(results, None) is None +@pytest.mark.parametrize("curl_error", [22, 56]) +@pytest.mark.parametrize("corrupt_asset", [False, True]) +def test_acquire_release_asset_after_gateway_failure_preserves_digest_and_budget( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, curl_error: int, corrupt_asset: bool +) -> None: + content = b"locked candidate bytes" + item = { + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20261026/python%2Bbuild.tar.gz", + "sha256": hashlib.sha256(content).hexdigest(), + "size": len(content), + } + elapsed = 0.0 + monkeypatch.setattr(prepare_managed_resources.time, "monotonic", lambda: elapsed) + monkeypatch.setattr(shutil, "which", lambda _name: "/tools/gh") + monkeypatch.setenv("GH_TOKEN", "test-token-not-for-command-line") + + def download(command: list[str], **options: Any) -> subprocess.CompletedProcess[str]: + nonlocal elapsed + assert "test-token-not-for-command-line" not in " ".join(command) + assert 0 < options["timeout"] <= 1900 - elapsed + if command[0] == "curl": + Path(command[command.index("--output") + 1]).write_bytes(b"partial gateway response") + elapsed += 1800 + raise subprocess.CalledProcessError(curl_error, command) + assert command[:4] == ["gh", "api", "--hostname", "github.com"] + assert options["env"]["GH_TOKEN"] == os.environ["GH_TOKEN"] + if command[4].endswith("/tags/20261026"): + elapsed += 20 + return subprocess.CompletedProcess( + command, 0, json.dumps({"assets": [{"id": 42, "name": "python+build.tar.gz"}]}) + ) + assert command[4].endswith("/assets/42") + options["stdout"].write(b"corrupt" if corrupt_asset else content) + return subprocess.CompletedProcess(command, 0, "") + + monkeypatch.setattr(subprocess, "run", download) + artifact = tmp_path / "python.tar.gz" + if corrupt_asset: + with pytest.raises(ValueError, match="locked digest and size"): + acquire(item, artifact) + assert not artifact.exists() + else: + acquire(item, artifact) + assert artifact.read_bytes() == content + + def test_windows_layout_requires_console_and_windowless_interpreters(tmp_path: Path) -> None: source = tmp_path / "python.tar.gz" _runtime_archive(source) From 370473d7b139b050493be9ad069898934b9b94be Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Mon, 9 Nov 2026 14:30:04 +0800 Subject: [PATCH 24/25] fix(ci): share verified release downloads across tool bootstraps --- .github/ci-paths.toml | 3 + .github/workflows/desktop.yml | 6 + .github/workflows/skill-guidance.yml | 9 +- scripts/download_ci_asset.py | 141 ++++++++++++++++++++++++ scripts/prepare_managed_resources.py | 85 +------------- scripts/prepare_windows_ci_tools.py | 55 +++++++++ tests/test_ci_changes.py | 2 + tests/test_managed_resource_producer.py | 22 ++-- 8 files changed, 233 insertions(+), 90 deletions(-) create mode 100644 scripts/download_ci_asset.py create mode 100644 scripts/prepare_windows_ci_tools.py diff --git a/.github/ci-paths.toml b/.github/ci-paths.toml index 02cea2c39b..b68c190a58 100644 --- a/.github/ci-paths.toml +++ b/.github/ci-paths.toml @@ -123,10 +123,12 @@ packaging = [ ".github/workflows/build-docker.yml", "docker/**", ] desktop = [ + "scripts/download_ci_asset.py", "scripts/prepare_windows_ci_tools.py", "desktop/**", "tests/test_desktop_fixture_qualification.py", "website/assets/**", ".github/workflows/desktop.yml", ] managed_resources = [ + "scripts/download_ci_asset.py", "scripts/*managed_resource*", "scripts/build_ops_bundle.py", "scripts/ci_ops_bundle_smoke.py", "scripts/build_intel_managed_wheel.py", "tests/test_managed_resource_producer.py", "tests/test_ops_resources.py", ".github/workflows/managed-resource-candidates.yml", @@ -151,6 +153,7 @@ seekdb_lock = [ "src/powercontext/builtin/persistence/**", "pyproject.toml", "uv.lock", ] skill_guidance = [ + "scripts/download_ci_asset.py", "evaluation/skills/skill-up/**", "integrations/agent-plugin/powercontext/skills/powercontext-project-context/**", ".github/workflows/skill-guidance.yml", diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml index 26dc563ec3..a16edc4e9e 100644 --- a/.github/workflows/desktop.yml +++ b/.github/workflows/desktop.yml @@ -138,6 +138,12 @@ jobs: - name: Test native diagnostics with isolated real CLI working-directory: . run: uv run --no-sync python desktop/tests/real_cli.py + - name: Prepare checksum-pinned Windows bundler tools + working-directory: . + timeout-minutes: 11 + env: + GH_TOKEN: ${{ github.token }} + run: uv run --no-sync python scripts/prepare_windows_ci_tools.py - name: Build fixture-qualified unsigned internal Windows installer run: pnpm desktop:build:windows:ci - name: Prepare matching WebView2 driver diff --git a/.github/workflows/skill-guidance.yml b/.github/workflows/skill-guidance.yml index c1b89996e3..912d455a72 100644 --- a/.github/workflows/skill-guidance.yml +++ b/.github/workflows/skill-guidance.yml @@ -51,12 +51,17 @@ jobs: echo "$RUNNER_TEMP/skill-guidance-env/bin" >> "$GITHUB_PATH" - name: Install checksum-pinned skill-up shell: bash + timeout-minutes: 6 + env: + GH_TOKEN: ${{ github.token }} run: | set -euo pipefail mkdir -p "$RUNNER_TEMP/skill-up" + python scripts/download_ci_asset.py \ + --url https://github.com/alibaba/skill-up/releases/download/v0.12.0/skill-up_0.12.0_linux_amd64.tar.gz \ + --sha256 ec2631e45702d460b69cd700e1847f5562ef2e971f09c23be2efcec4e0f583c6 \ + --output "$RUNNER_TEMP/skill-up/skill-up.tar.gz" --timeout 300 cd "$RUNNER_TEMP/skill-up" - curl -fsSL https://github.com/alibaba/skill-up/releases/download/v0.12.0/skill-up_0.12.0_linux_amd64.tar.gz -o skill-up.tar.gz - echo 'ec2631e45702d460b69cd700e1847f5562ef2e971f09c23be2efcec4e0f583c6 skill-up.tar.gz' | sha256sum --check - tar -xzf skill-up.tar.gz echo "$RUNNER_TEMP/skill-up" >> "$GITHUB_PATH" - name: Verify pin, fixtures, reports and native schema diff --git a/scripts/download_ci_asset.py b/scripts/download_ci_asset.py new file mode 100644 index 0000000000..52ada359bc --- /dev/null +++ b/scripts/download_ci_asset.py @@ -0,0 +1,141 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Acquire checksum-pinned CI assets with bounded official Release API recovery.""" + +# ruff: noqa: TRY003, S603 - fixed CI input URLs and host tools + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shutil +import subprocess +import time +from pathlib import Path +from urllib.parse import quote, unquote, urlsplit + +_WINDOWS_ACCESS_VIOLATION = 0xC0000005 + + +def _download_release_asset(url: str, partial: Path, env: dict[str, str], deadline: float) -> bool: + """Use the official API for the same pinned asset when the release route fails.""" + parsed = urlsplit(url) + parts = parsed.path.strip("/").split("/") + if ( + parsed.netloc != "github.com" + or parsed.query + or len(parts) != 6 + or parts[2:4] != ["releases", "download"] + or shutil.which("gh") is None + or time.monotonic() >= deadline + ): + return False + owner, repository, _, _, tag, filename = parts + endpoint = f"repos/{owner}/{repository}/releases" + command = ["gh", "api", "--hostname", "github.com"] + result = subprocess.run( + [*command, f"{endpoint}/tags/{quote(unquote(tag), safe='')}"], + check=True, + capture_output=True, + text=True, + env=env, + timeout=min(30, max(0.001, deadline - time.monotonic())), + ) + assets = json.loads(result.stdout)["assets"] + asset = next((asset for asset in assets if asset["name"] == unquote(filename)), None) + if asset is None or type(asset.get("id")) is not int or asset["id"] <= 0: + raise ValueError("The pinned GitHub release asset is unavailable") + partial.unlink(missing_ok=True) + with partial.open("xb") as output: + subprocess.run( + [*command, f"{endpoint}/assets/{asset['id']}", "--header", "Accept: application/octet-stream"], + check=True, + stdout=output, + env=env, + timeout=max(0.001, deadline - time.monotonic()), + ) + return True + + +def download(url: str, partial: Path, *, env: dict[str, str], timeout: float = 1900) -> None: + deadline = time.monotonic() + timeout + command = [ + "curl", + "--fail", + "--silent", + "--show-error", + "--location", + "--proto", + "=https", + "--proto-redir", + "=https", + "--retry", + "2", + "--max-time", + str(min(1800, timeout)), + "--output", + str(partial), + url, + ] + for attempt in range(2): + try: + subprocess.run(command, check=True, env=env, timeout=max(0.001, deadline - time.monotonic())) + except subprocess.CalledProcessError as error: + if error.returncode in {5, 6, 7, 18, 22, 28, 35, 52, 55, 56} and _download_release_asset( + url, partial, env, deadline + ): + return + # curl's own retry cannot recover from a process crash on the Windows runner. + if attempt or error.returncode != _WINDOWS_ACCESS_VIOLATION: + raise + partial.unlink(missing_ok=True) + else: + return + + +def acquire_asset(url: str, destination: Path, digest: str, *, timeout: float = 300) -> None: + """Publish a complete file only after verifying its checked-in SHA256.""" + parsed = urlsplit(url) + if parsed.scheme != "https" or parsed.username or parsed.password or parsed.fragment: + raise ValueError("CI assets require fixed HTTPS source URLs") + if not re.fullmatch(r"[0-9a-f]{64}", digest) or timeout <= 0: + raise ValueError("CI assets require a SHA256 digest and positive deadline") + destination.parent.mkdir(parents=True, exist_ok=True) + partial = destination.with_name(destination.name + ".partial") + if destination.exists() or partial.exists(): + raise FileExistsError("CI download output must be fresh") + download(url, partial, env=os.environ.copy(), timeout=timeout) + with partial.open("rb") as stream: + actual = hashlib.file_digest(stream, "sha256").hexdigest() + if actual != digest: + raise ValueError("Downloaded CI asset does not match its pinned SHA256") + partial.rename(destination) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--url", required=True) + parser.add_argument("--sha256", required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--timeout", type=float, default=300) + args = parser.parse_args() + acquire_asset(args.url, args.output, args.sha256, timeout=args.timeout) + + +if __name__ == "__main__": + main() diff --git a/scripts/prepare_managed_resources.py b/scripts/prepare_managed_resources.py index 6fd59ef7f1..fe5a388e18 100644 --- a/scripts/prepare_managed_resources.py +++ b/scripts/prepare_managed_resources.py @@ -34,14 +34,14 @@ import subprocess import tarfile import tempfile -import time import tomllib from concurrent.futures import ThreadPoolExecutor from pathlib import Path, PurePosixPath from typing import Any -from urllib.parse import quote, unquote, urlsplit +from urllib.parse import unquote, urlsplit import build_intel_managed_wheel +from download_ci_asset import download from packaging.requirements import Requirement from packaging.utils import canonicalize_name, parse_wheel_filename from packaging.version import Version @@ -52,7 +52,6 @@ _INTERPRETERS = _SCRIPTS / "managed_resource_interpreters.json" _BUILD_TOOLS = _SCRIPTS / "managed_resource_build_tools.json" _SOURCE_REPOSITORY = "https://github.com/oceanbase/powercontext" -_WINDOWS_ACCESS_VIOLATION = 0xC0000005 def json_bytes(value: object) -> bytes: @@ -68,6 +67,7 @@ def producer_identity() -> dict[str, Any]: """Bind executable producer sources separately from the packaged Server source.""" names = ( "prepare_managed_resources.py", + "download_ci_asset.py", "produce_managed_resources.py", "build_ops_bundle.py", "managed_resource_interpreters.json", @@ -106,83 +106,6 @@ def target_spec(target: str, *, native: bool = True) -> dict[str, Any]: return spec -def _download_release_asset(url: str, partial: Path, env: dict[str, str], deadline: float) -> bool: - """Use the official API for the same pinned asset when the release route fails.""" - parsed = urlsplit(url) - parts = parsed.path.strip("/").split("/") - if ( - parsed.netloc != "github.com" - or parsed.query - or len(parts) != 6 - or parts[2:4] != ["releases", "download"] - or shutil.which("gh") is None - or time.monotonic() >= deadline - ): - return False - owner, repository, _, _, tag, filename = parts - endpoint = f"repos/{owner}/{repository}/releases" - command = ["gh", "api", "--hostname", "github.com"] - result = subprocess.run( - [*command, f"{endpoint}/tags/{quote(unquote(tag), safe='')}"], - check=True, - capture_output=True, - text=True, - env=env, - timeout=min(30, max(0.001, deadline - time.monotonic())), - ) - assets = json.loads(result.stdout)["assets"] - asset = next((asset for asset in assets if asset["name"] == unquote(filename)), None) - if asset is None or type(asset.get("id")) is not int or asset["id"] <= 0: - raise ValueError("The pinned GitHub release asset is unavailable") - partial.unlink(missing_ok=True) - with partial.open("xb") as output: - subprocess.run( - [*command, f"{endpoint}/assets/{asset['id']}", "--header", "Accept: application/octet-stream"], - check=True, - stdout=output, - env=env, - timeout=max(0.001, deadline - time.monotonic()), - ) - return True - - -def _download(url: str, partial: Path) -> None: - deadline = time.monotonic() + 1900 - command = [ - "curl", - "--fail", - "--silent", - "--show-error", - "--location", - "--proto", - "=https", - "--proto-redir", - "=https", - "--retry", - "2", - "--max-time", - "1800", - "--output", - str(partial), - url, - ] - env = environment() - for attempt in range(2): - try: - subprocess.run(command, check=True, env=env, timeout=max(0.001, deadline - time.monotonic())) - except subprocess.CalledProcessError as error: - if error.returncode in {5, 6, 7, 18, 22, 28, 35, 52, 55, 56} and _download_release_asset( - url, partial, env, deadline - ): - return - # curl's own retry cannot recover from a process crash on the Windows runner. - if attempt or error.returncode != _WINDOWS_ACCESS_VIOLATION: - raise - partial.unlink(missing_ok=True) - else: - return - - def acquire(item: dict[str, Any], destination: Path, cache: Path | None = None) -> None: """Download fixed HTTPS bytes; incomplete and corrupt files cannot be reused.""" if destination.exists(): @@ -202,7 +125,7 @@ def acquire(item: dict[str, Any], destination: Path, cache: Path | None = None) partial = destination.with_name(destination.name + ".partial") if partial.exists(): raise ValueError("An incomplete input exists; inspect it before retrying") - _download(item["url"], partial) + download(item["url"], partial, env=environment()) if sha256(partial) != item["sha256"] or ("size" in item and partial.stat().st_size != item["size"]): raise ValueError("Downloaded input does not match its locked digest and size") partial.rename(destination) diff --git a/scripts/prepare_windows_ci_tools.py b/scripts/prepare_windows_ci_tools.py new file mode 100644 index 0000000000..9cef34a24c --- /dev/null +++ b/scripts/prepare_windows_ci_tools.py @@ -0,0 +1,55 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Stage the checksum-pinned NSIS tools required by the locked Tauri CLI.""" + +from __future__ import annotations + +import os +import shutil +import tempfile +import zipfile +from pathlib import Path + +from download_ci_asset import acquire_asset + + +def main() -> None: + # Tauri CLI 2.11.4 uses this cache layout and still verifies the plugin itself. + # Source: tauri-apps/tauri, tauri-cli-v2.11.4, crates/tauri-bundler/src/bundle/windows/nsis/mod.rs. + target = Path(os.environ["LOCALAPPDATA"]) / "tauri" / "NSIS" + with tempfile.TemporaryDirectory(prefix="powercontext-nsis-") as directory: + root = Path(directory) + archive = root / "nsis.zip" + acquire_asset( + "https://github.com/tauri-apps/binary-releases/releases/download/nsis-3.11/nsis-3.11.zip", + archive, + "c7d27f780ddb6cffb4730138cd1591e841f4b7edb155856901cdf5f214394fa1", + ) + plugin = root / "nsis_tauri_utils.dll" + acquire_asset( + "https://github.com/tauri-apps/nsis-tauri-utils/releases/download/nsis_tauri_utils-v0.5.3/nsis_tauri_utils.dll", + plugin, + "5ba143b5db4a87d32d6e7802e033330aae56cbceabe0d1e3ba41948385ad4709", + ) + with zipfile.ZipFile(archive) as source: + source.extractall(root) + shutil.copytree(root / "nsis-3.11", target, dirs_exist_ok=True) + plugins = target / "Plugins" / "x86-unicode" / "additional" + plugins.mkdir(parents=True, exist_ok=True) + shutil.copyfile(plugin, plugins / plugin.name) + + +if __name__ == "__main__": + main() diff --git a/tests/test_ci_changes.py b/tests/test_ci_changes.py index bb048e29cf..06b2bf2294 100644 --- a/tests/test_ci_changes.py +++ b/tests/test_ci_changes.py @@ -94,6 +94,8 @@ (["scripts/sync_dsh_plugin_tree.py"], {"tests", "dsh"}), (["scripts/generate_integration_lifecycle.py"], {"tests", "hermes"}), (["desktop/ui/src/app/App.tsx"], {"desktop"}), + (["scripts/prepare_windows_ci_tools.py"], {"desktop"}), + (["scripts/download_ci_asset.py"], {"desktop", "managed_resources", "skill_guidance"}), (["release/upgrade-support.json"], {"upgrade", "packaging"}), ([".github/workflows/desktop.yml"], {"desktop"}), ([".github/workflows/desktop-candidate.yml"], {"tests"}), diff --git a/tests/test_managed_resource_producer.py b/tests/test_managed_resource_producer.py index 01abbd41cd..7617125e6d 100644 --- a/tests/test_managed_resource_producer.py +++ b/tests/test_managed_resource_producer.py @@ -29,8 +29,8 @@ from pathlib import Path, PureWindowsPath from typing import Any +import download_ci_asset import httpx -import prepare_managed_resources import pytest from build_intel_managed_wheel import _check_linkage, native_build_environment, verify_backend from ci_managed_resource_smoke import MAX_DIAGNOSTIC_BYTES, preserve_failure @@ -249,17 +249,18 @@ def download(command: list[str], **_options: Any) -> None: @pytest.mark.parametrize("curl_error", [22, 56]) @pytest.mark.parametrize("corrupt_asset", [False, True]) +@pytest.mark.parametrize("consumer", ["producer", "ci-tool"]) def test_acquire_release_asset_after_gateway_failure_preserves_digest_and_budget( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, curl_error: int, corrupt_asset: bool + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, curl_error: int, corrupt_asset: bool, consumer: str ) -> None: content = b"locked candidate bytes" - item = { + item: dict[str, Any] = { "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20261026/python%2Bbuild.tar.gz", "sha256": hashlib.sha256(content).hexdigest(), "size": len(content), } elapsed = 0.0 - monkeypatch.setattr(prepare_managed_resources.time, "monotonic", lambda: elapsed) + monkeypatch.setattr(download_ci_asset.time, "monotonic", lambda: elapsed) monkeypatch.setattr(shutil, "which", lambda _name: "/tools/gh") monkeypatch.setenv("GH_TOKEN", "test-token-not-for-command-line") @@ -284,12 +285,19 @@ def download(command: list[str], **options: Any) -> subprocess.CompletedProcess[ monkeypatch.setattr(subprocess, "run", download) artifact = tmp_path / "python.tar.gz" - if corrupt_asset: - with pytest.raises(ValueError, match="locked digest and size"): + + def acquire_candidate() -> None: + if consumer == "producer": acquire(item, artifact) + else: + download_ci_asset.acquire_asset(item["url"], artifact, item["sha256"], timeout=1900) + + if corrupt_asset: + with pytest.raises(ValueError, match=r"locked digest and size|pinned SHA256"): + acquire_candidate() assert not artifact.exists() else: - acquire(item, artifact) + acquire_candidate() assert artifact.read_bytes() == content From b95e6a55e9c9498b7cf35f9187d036a1289ee28f Mon Sep 17 00:00:00 2001 From: Frey <370805886@qq.com> Date: Mon, 9 Nov 2026 14:31:32 +0800 Subject: [PATCH 25/25] fix(ci): authorize pinned asset fallback for desktop candidates --- .github/workflows/desktop-candidate.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/desktop-candidate.yml b/.github/workflows/desktop-candidate.yml index 77158cdb6c..a163c28f04 100644 --- a/.github/workflows/desktop-candidate.yml +++ b/.github/workflows/desktop-candidate.yml @@ -96,6 +96,8 @@ jobs: if: matrix.target == 'macos-x86_64' run: echo 'DEVELOPER_DIR=/Applications/Xcode_16.4.app/Contents/Developer' >> "$GITHUB_ENV" - name: Produce and smoke-test the pinned 09-C resources + env: + GH_TOKEN: ${{ github.token }} run: | uv run --no-sync python scripts/prepare_managed_resources.py \ --target "$RESOURCE_TARGET" --source-ref "$SERVER_SOURCE_SHA" --server-version "$SERVER_VERSION" \