Repository navigation
Animate Orleans grain networks and routed request streams #137
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarks | |
| on: | |
| push: | |
| branches: [main] | |
| workflow_dispatch: | |
| permissions: {contents: read, actions: read} | |
| concurrency: | |
| group: keyload-benchmarks-${{ github.ref }} | |
| cancel-in-progress: false | |
| jobs: | |
| comparison-build: | |
| name: Build and check KeyLoad | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| steps: | |
| - name: Download source code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| - name: Set up .NET | |
| uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 | |
| with: | |
| global-json-file: global.json | |
| - name: Restore .NET packages | |
| run: dotnet restore KeyLoad.slnx | |
| - name: Build KeyLoad | |
| run: dotnet build KeyLoad.slnx --no-restore --configuration Release | |
| - name: Check code formatting | |
| run: dotnet format KeyLoad.slnx --verify-no-changes --no-restore | |
| - name: Check repository rules | |
| run: node scripts/Features/RepositoryGovernance/verify.mjs | |
| - name: Test benchmark contracts | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/KeyLoad.UnitTests*/*/*' --KeyLoadTests:ResultsDirectory=TestResults/benchmark-contracts --KeyLoadTests:ReportTrx=true | |
| - name: Test benchmark contracts without CPU intrinsics | |
| env: | |
| DOTNET_EnableHWIntrinsic: '0' | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/KeyLoad.UnitTests*/*/*' --KeyLoadTests:ResultsDirectory=TestResults/benchmark-contracts-scalar --KeyLoadTests:ReportTrx=true | |
| - name: Test prompt termination on Aspire resource failures | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/AspireFailure*/*' --KeyLoadTests:ResultsDirectory=TestResults/aspire-failure --KeyLoadTests:ReportTrx=true | |
| - name: Save benchmark contract test results | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: benchmark-contract-tests | |
| path: TestResults/** | |
| if-no-files-found: error | |
| - name: Save build diagnostics | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: code-quality-comparisons | |
| path: artifacts/code-quality/** | |
| if-no-files-found: error | |
| comparison-plan: | |
| name: Plan benchmark runs | |
| if: github.repository == 'managedcode/KeyLoad' && github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| outputs: | |
| databases: ${{ steps.plan.outputs.database_matrices }} | |
| steps: | |
| - name: Download source code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 | |
| - name: Plan all benchmark scenarios | |
| id: plan | |
| run: node scripts/Features/BenchmarkComparisons/isolated-plan.mjs --output="$RUNNER_TEMP/isolated-plan.json" --scale-output="$RUNNER_TEMP/scaled-plan.json" --vector-output="$RUNNER_TEMP/vector-plan.json" --composite-output="$RUNNER_TEMP/composite-plan.json" --open-loop-output="$RUNNER_TEMP/open-loop-plan.json" --github-output="$GITHUB_OUTPUT" | |
| - name: Save benchmark plan | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-plan | |
| path: | | |
| ${{ runner.temp }}/isolated-plan.json | |
| ${{ runner.temp }}/scaled-plan.json | |
| ${{ runner.temp }}/vector-plan.json | |
| ${{ runner.temp }}/composite-plan.json | |
| ${{ runner.temp }}/open-loop-plan.json | |
| if-no-files-found: error | |
| retention-days: 90 | |
| comparison-images: | |
| name: Build Docker images | |
| if: github.repository == 'managedcode/KeyLoad' && github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| permissions: {contents: read, actions: read} | |
| outputs: | |
| bundle-artifact-id: ${{ steps.bundle.outputs.artifact-id }} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| KEYLOAD_COMPARISON_JOB_NAME: Build Docker images | |
| steps: | |
| - name: Download source code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 | |
| - name: Set up .NET | |
| uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 | |
| with: | |
| global-json-file: global.json | |
| - name: Check Docker | |
| run: docker version | |
| - name: Restore .NET packages | |
| run: dotnet restore KeyLoad.slnx | |
| - name: Build benchmark tests | |
| id: benchmark-build | |
| run: dotnet build tests/KeyLoad.ComparisonTests --no-restore --configuration Release | |
| - name: Check bounded Docker registry readiness | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/ImageRegistryReadinessTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/registry-readiness | |
| - name: Build KeyLoad server and benchmark Docker images | |
| id: images | |
| run: node scripts/Features/BenchmarkComparisons/prepare-images.mjs | |
| - name: Configure Docker image references | |
| env: | |
| KEYLOAD_SERVER_IMAGE: ${{ steps.images.outputs.server-image }} | |
| KEYLOAD_RUNNER_IMAGE: ${{ steps.images.outputs.load-generator-image }} | |
| run: | | |
| test -n "$KEYLOAD_SERVER_IMAGE" && test -n "$KEYLOAD_RUNNER_IMAGE" | |
| printf 'KeyLoad__ContainerImages__Server=%s\nBenchmarks__ContainerImages__LoadGenerator=%s\nKEYLOAD_IMAGE_RECEIPT=%s\n' "$KEYLOAD_SERVER_IMAGE" "$KEYLOAD_RUNNER_IMAGE" "$RUNNER_TEMP/keyload-images/image-receipt.json" >> "$GITHUB_ENV" | |
| - name: Check native KurrentDB discovery settings | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/IsolatedKurrentDiscoverySettingsTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/kurrent-discovery | |
| - name: Check exact OpenSearch vector query contract | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/OpenSearchVectorQueryTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/opensearch-query | |
| - name: Check exact OpenSearch vector response contract | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/OpenSearchVectorResponseTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/opensearch-response | |
| - name: Check TimescaleDB Docker image and startup tools | |
| env: | |
| KEYLOAD_TIMESERIES_IMAGE_FACTS_DIRECTORY: ${{ runner.temp }}/keyload-timeseries-image-facts | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/TimeSeriesIntensivePinnedImageTests/*' --KeyLoadTests:ResultsDirectory=TestResults/timeseries-image | |
| - name: Save TimescaleDB image checks | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: timeseries-native-pinned-image-facts | |
| path: | | |
| ${{ runner.temp }}/keyload-timeseries-image-facts/** | |
| **/TestResults/** | |
| artifacts/code-quality/** | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Check Docker image export and import | |
| id: roundtrip | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/ImageBundleRealTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/image-roundtrip | |
| - name: Verify GitHub job identity | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/IsolatedGitHubCurrentJobTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/job-identity | |
| - name: Test database cluster resource models | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/Isolated*Resource*/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/resource-models | |
| - name: Test MongoDB readiness configuration | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/MongoReadinessResourceTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/mongo-resource | |
| - name: Test MongoDB node identity | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/MongoReadinessIdentityTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/mongo-identity | |
| - name: Test MongoDB startup failure handling | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/MongoReadinessTaskFailureTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/mongo-task-failure | |
| - name: Test KurrentDB storage failure handling | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/IsolatedKurrentVolumeRegressionFailureTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/kurrent-volume-failure | |
| - name: Test KeyLoad request admission | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/IsolatedKeyLoadAdmissionTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/keyload-admission | |
| - name: Test bounded database log buffers | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/ComparisonResourceLogBufferTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/log-buffer | |
| - name: Test live benchmark progress capture | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/ComparisonProgress*/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/live-progress | |
| - name: Test live benchmark process entry | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/NativeBenchmarkProgressEntryTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/live-entry | |
| - name: Test benchmark phase and attempt counters | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/ComparisonLiveProgress*/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/live-counters | |
| - name: Test KeyLoad replay admission | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/IsolatedKeyLoadReplayAdmissionTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/replay-admission | |
| - name: Test Redis replica error reporting | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/RedisReplicaDiagnosticTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/redis-diagnostics | |
| - name: Test MongoDB startup error reporting | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/MongoBootstrapDiagnosticTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/mongo-diagnostics | |
| - name: Test KurrentDB cleanup error reporting | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/KurrentCleanupDiagnosticTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/kurrent-diagnostics | |
| - name: Test KurrentDB stream ownership | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/KurrentStreamOwnershipTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/kurrent-stream-ownership | |
| - name: Test database replay logs | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/ComparisonReplayDiagnosticLogTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/replay-log | |
| - name: Test database replay log retention | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/ComparisonReplayDiagnosticRetentionTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/replay-retention | |
| - name: Test time-series workload correctness | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/TimeSeriesWorkloadTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/timeseries-workload | |
| - name: Check published time-series package version | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/TimeSeriesPackageVersionTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/timeseries-package | |
| - name: Test KeyLoad time-series cluster configuration | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/IsolatedTimeSeriesKeyLoadResourceTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/timeseries-keyload-model | |
| - name: Test TimescaleDB cluster configuration | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/IsolatedTimeSeriesTimescaleResourceTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/timeseries-timescale-model | |
| - name: Test time-series workload configuration | |
| run: dotnet run --project src/KeyLoad.AppHost --no-build --no-restore --configuration Release -- --KeyLoadTests:Suite=comparison '--KeyLoadTests:Filter=/*/*/IsolatedTimeSeriesBenchmarkResourceTests/*' --KeyLoadTests:ResultsDirectory=TestResults/comparison-images/timeseries-family-model | |
| - name: Save Docker images | |
| id: bundle | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-image-bundle | |
| path: ${{ runner.temp }}/keyload-image-bundle/* | |
| if-no-files-found: error | |
| compression-level: 0 | |
| retention-days: 90 | |
| - name: Clean up Docker registry after image failure | |
| if: ${{ always() && steps.roundtrip.outcome != 'success' && (steps.images.outcome == 'success' || steps.images.outcome == 'failure' || steps.images.outcome == 'cancelled') }} | |
| run: node scripts/Features/BenchmarkComparisons/cleanup-images.mjs | |
| - name: Save Docker image checks and test results | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-images-qualification | |
| path: | | |
| ${{ runner.temp }}/keyload-images/** | |
| ${{ runner.temp }}/keyload-images-build/** | |
| ${{ runner.temp }}/keyload-cell-github/** | |
| **/TestResults/** | |
| artifacts/code-quality/** | |
| if-no-files-found: error | |
| retention-days: 90 | |
| comparison-keyload: | |
| name: ${{ matrix.jobName }} | |
| needs: [comparison-plan, comparison-images] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 150 | |
| permissions: {contents: read, actions: read} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-plan.outputs.databases).keyload }} | |
| env: &database-environment | |
| GH_TOKEN: ${{ github.token }} | |
| KEYLOAD_COMPARISON_CELL_ID: ${{ matrix.id }} | |
| KEYLOAD_COMPARISON_JOB_NAME: ${{ matrix.jobName }} | |
| Benchmarks__Target: ${{ matrix.target }} | |
| KEYLOAD_MATRIX_KIND: ${{ matrix.kind }} | |
| steps: &database-steps | |
| - name: Download source code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 | |
| - name: Download canonical comparison plan | |
| uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 | |
| with: | |
| name: comparison-plan | |
| path: ${{ runner.temp }}/comparison-plan | |
| - name: Resolve full matrix row from the canonical plan | |
| env: | |
| KEYLOAD_MATRIX_PLAN_DIRECTORY: ${{ runner.temp }}/comparison-plan | |
| run: node scripts/Features/BenchmarkComparisons/isolated-matrix-entry.mjs | |
| - name: Prepare database containers | |
| id: images | |
| uses: ./.github/workflows/Features/BenchmarkComparisons/IsolatedCellSetup | |
| with: | |
| bundle-artifact-id: ${{ needs.comparison-images.outputs.bundle-artifact-id }} | |
| - name: Restore .NET packages | |
| run: dotnet restore KeyLoad.slnx | |
| - name: Build benchmark tests | |
| id: benchmark-build | |
| run: dotnet build tests/KeyLoad.ComparisonTests --no-restore --configuration Release | |
| - name: Run database workload | |
| id: workload | |
| if: ${{ !cancelled() }} | |
| env: | |
| PREPARATION_OUTCOME: ${{ steps.images.outcome }} | |
| BUILD_OUTCOME: ${{ steps.benchmark-build.outcome }} | |
| run: | | |
| if [ "$PREPARATION_OUTCOME" != success ] || [ "$BUILD_OUTCOME" != success ]; then | |
| echo '::error::Benchmark preparation failed; no measurement data is available.' | |
| exit 1 | |
| fi | |
| node scripts/Features/BenchmarkComparisons/run-workload.mjs | |
| - name: Record benchmark availability | |
| if: ${{ !cancelled() }} | |
| env: | |
| KEYLOAD_WORKLOAD_OUTCOME: ${{ steps.workload.outcome }} | |
| run: node scripts/Features/BenchmarkComparisons/finalize-worker.mjs | |
| - name: Save benchmark results | |
| if: ${{ !cancelled() }} | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: ${{ matrix.kind == 'preflight' && 'comparison-preflight-' || matrix.kind == 'proof' && 'comparison-open-loop-proof-' || matrix.kind == 'open-loop' && 'comparison-open-loop-worker-' || 'comparison-worker-' }}${{ matrix.id }} | |
| path: artifacts/comparisons/isolated/workers/${{ matrix.id }}/* | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Clean up containers and save logs | |
| if: always() | |
| uses: ./.github/workflows/Features/BenchmarkComparisons/IsolatedCellTeardown | |
| with: | |
| setup-outcome: ${{ steps.images.outcome }} | |
| artifact-name: ${{ matrix.kind == 'preflight' && 'comparison-preflight-qualification-' || matrix.kind == 'proof' && 'comparison-open-loop-proof-qualification-' || matrix.kind == 'open-loop' && 'comparison-open-loop-case-qualification-' || 'comparison-case-qualification-' }}${{ matrix.id }} | |
| comparison-postgresql: | |
| name: ${{ matrix.jobName }} | |
| needs: [comparison-plan, comparison-images] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 150 | |
| permissions: {contents: read, actions: read} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-plan.outputs.databases).postgresql }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-qdrant: | |
| name: ${{ matrix.jobName }} | |
| needs: [comparison-plan, comparison-images] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 150 | |
| permissions: {contents: read, actions: read} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-plan.outputs.databases).qdrant }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-rabbitmq: | |
| name: ${{ matrix.jobName }} | |
| needs: [comparison-plan, comparison-images] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 150 | |
| permissions: {contents: read, actions: read} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-plan.outputs.databases).rabbitmq }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-redis: | |
| name: ${{ matrix.jobName }} | |
| needs: [comparison-plan, comparison-images] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 150 | |
| permissions: {contents: read, actions: read} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-plan.outputs.databases).redis }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-neo4j: | |
| name: ${{ matrix.jobName }} | |
| needs: [comparison-plan, comparison-images] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 150 | |
| permissions: {contents: read, actions: read} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-plan.outputs.databases).neo4j }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-mongodb: | |
| name: ${{ matrix.jobName }} | |
| needs: [comparison-plan, comparison-images] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 150 | |
| permissions: {contents: read, actions: read} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-plan.outputs.databases).mongodb }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-opensearch: | |
| name: ${{ matrix.jobName }} | |
| needs: [comparison-plan, comparison-images] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 150 | |
| permissions: {contents: read, actions: read} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-plan.outputs.databases).opensearch }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-kurrentdb: | |
| name: ${{ matrix.jobName }} | |
| needs: [comparison-plan, comparison-images] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 150 | |
| permissions: {contents: read, actions: read} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-plan.outputs.databases).kurrentdb }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-surrealdb: | |
| name: ${{ matrix.jobName }} | |
| needs: [comparison-plan, comparison-images] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 150 | |
| permissions: {contents: read, actions: read} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-plan.outputs.databases).surrealdb }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-helixdb: | |
| name: ${{ matrix.jobName }} | |
| needs: [comparison-plan, comparison-images] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 150 | |
| permissions: {contents: read, actions: read} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.comparison-plan.outputs.databases).helixdb }} | |
| env: *database-environment | |
| steps: *database-steps | |
| comparison-aggregate: | |
| name: Combine benchmark results | |
| needs: [comparison-build, comparison-plan, comparison-images, comparison-keyload, comparison-postgresql, comparison-qdrant, comparison-rabbitmq, comparison-redis, comparison-neo4j, comparison-mongodb, comparison-opensearch, comparison-kurrentdb, comparison-surrealdb, comparison-helixdb] | |
| if: ${{ always() && !cancelled() && needs.comparison-plan.result == 'success' && needs.comparison-images.result == 'success' }} | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 150 | |
| permissions: {contents: read, actions: read} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| steps: | |
| - name: Download source code | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 | |
| - name: Verify benchmark plan | |
| id: plan | |
| run: node scripts/Features/BenchmarkComparisons/isolated-plan.mjs --output="$RUNNER_TEMP/isolated-plan.json" --scale-output="$RUNNER_TEMP/scaled-plan.json" --vector-output="$RUNNER_TEMP/vector-plan.json" --composite-output="$RUNNER_TEMP/composite-plan.json" --open-loop-output="$RUNNER_TEMP/open-loop-plan.json" | |
| - name: Download benchmark results | |
| run: node scripts/Features/BenchmarkComparisons/isolated-github-collect.mjs --plan="$RUNNER_TEMP/isolated-plan.json" --scale-plan="$RUNNER_TEMP/scaled-plan.json" --vector-plan="$RUNNER_TEMP/vector-plan.json" --input="$RUNNER_TEMP/isolated-capture" | |
| - name: Check control and complete scale accounting | |
| id: composite-aggregate | |
| run: node scripts/Features/BenchmarkComparisons/scaled-cohort-aggregate-cli.mjs --plan="$RUNNER_TEMP/isolated-plan.json" --scale-plan="$RUNNER_TEMP/scaled-plan.json" --vector-plan="$RUNNER_TEMP/vector-plan.json" --composite-plan="$RUNNER_TEMP/composite-plan.json" --input="$RUNNER_TEMP/isolated-capture" --proof-root="$RUNNER_TEMP/isolated-capture/github" --output="$RUNNER_TEMP/isolated-aggregate" --scale-output="$RUNNER_TEMP/scale-cohort" | |
| - name: Save combined benchmark results | |
| if: ${{ always() && !cancelled() }} | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-isolated-suite | |
| path: ${{ runner.temp }}/isolated-aggregate/** | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Save internal scaled cohort receipt | |
| if: ${{ always() && !cancelled() }} | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-scaled-internal-cohort | |
| path: ${{ runner.temp }}/scale-cohort/** | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Save GitHub result verification | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-isolated-provider-evidence | |
| path: ${{ runner.temp }}/isolated-capture/github/** | |
| if-no-files-found: error | |
| retention-days: 90 | |
| - name: Download authenticated open-loop worker results | |
| id: open-loop-intake | |
| if: ${{ always() && !cancelled() && steps.plan.outcome == 'success' }} | |
| run: node scripts/Features/BenchmarkComparisons/open-loop-github-collect.mjs --capture="$RUNNER_TEMP/isolated-capture/github" --plan="$RUNNER_TEMP/open-loop-plan.json" --output="$RUNNER_TEMP/open-loop-intake" | |
| - name: Aggregate open-loop cohort | |
| id: open-loop-aggregate | |
| if: ${{ always() && !cancelled() && steps.open-loop-intake.outcome == 'success' }} | |
| run: node scripts/Features/BenchmarkComparisons/open-loop-cohort-aggregate-cli.mjs --input="$RUNNER_TEMP/open-loop-intake" --output="$RUNNER_TEMP/open-loop-aggregate" | |
| - name: Save original open-loop worker inputs | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-open-loop-intake | |
| path: ${{ runner.temp }}/open-loop-intake/** | |
| if-no-files-found: warn | |
| retention-days: 90 | |
| - name: Save open-loop cohort disposition | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a | |
| with: | |
| name: comparison-open-loop-cohort | |
| path: ${{ runner.temp }}/open-loop-aggregate/** | |
| if-no-files-found: warn | |
| retention-days: 90 | |
| website-trigger: | |
| name: Trigger Website | |
| needs: [comparison-aggregate] | |
| if: >- | |
| always() && !cancelled() && | |
| github.repository == 'managedcode/KeyLoad' && github.event.repository.id == 477801965 && | |
| github.ref == 'refs/heads/main' && | |
| (github.event_name == 'push' || github.event_name == 'workflow_dispatch') | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| permissions: {actions: write} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| GH_REPO: ${{ github.repository }} | |
| BENCHMARK_RUN_ID: ${{ github.run_id }} | |
| steps: | |
| - name: Start the separate Website workflow | |
| run: timeout 120s gh api --method POST "repos/$GH_REPO/actions/workflows/website.yml/dispatches" -f ref=main -f "inputs[benchmark_run_id]=$BENCHMARK_RUN_ID" |