Repository navigation
Replace metrics table with model comparisons and preserve benchmark p… #27
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: verify | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| workflow_dispatch: | |
| inputs: | |
| gguf_model_id: {description: 'GGUF catalog package ID (input overrides repository variable)', type: string, required: false} | |
| gguf_model_file: {description: 'GGUF filename within the selected package', type: string, required: false} | |
| model_set: {description: 'Functional catalog model set', type: string, required: false} | |
| embedding_model_set: {description: 'Embedding catalog model set', type: string, required: false} | |
| foundry_model_set: {description: 'Tracked Foundry model-set JSON path', type: string, required: false} | |
| foundry_anchor: {description: 'Foundry functional model alias from the selected set', type: string, required: false} | |
| env: | |
| SYNAPSE_GGUF_MODEL_ID: ${{ inputs.gguf_model_id || vars.SYNAPSE_GGUF_MODEL_ID || 'qwen2.5-0.5b-instruct-q8_0' }} | |
| SYNAPSE_GGUF_MODEL_FILE: ${{ inputs.gguf_model_file || vars.SYNAPSE_GGUF_MODEL_FILE || 'qwen2.5-0.5b-instruct-q8_0.gguf' }} | |
| SYNAPSE_MODEL_SET: ${{ inputs.model_set || vars.SYNAPSE_MODEL_SET || 'family-small' }} | |
| SYNAPSE_EMBEDDING_MODEL_SET: ${{ inputs.embedding_model_set || vars.SYNAPSE_EMBEDDING_MODEL_SET || 'embedding-small' }} | |
| SYNAPSE_FOUNDRY_MODEL_SET: ${{ inputs.foundry_model_set || vars.SYNAPSE_FOUNDRY_MODEL_SET || 'experiments/Synapse.ReferenceBenchmarks/Features/Benchmarking/ModelSets/foundry-local-families.json' }} | |
| SYNAPSE_FOUNDRY_ANCHOR: ${{ inputs.foundry_anchor || vars.SYNAPSE_FOUNDRY_ANCHOR || 'qwen2.5-0.5b' }} | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: verify-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| test: | |
| name: ${{ matrix.name }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: macOS 15 ARM64 primary | |
| os: macos-15 | |
| runtime_identifier: osx-arm64 | |
| executable_suffix: '' | |
| llama_binary_directory: build/bin | |
| - name: Ubuntu 24.04 x64 portable | |
| os: ubuntu-24.04 | |
| runtime_identifier: linux-x64 | |
| executable_suffix: '' | |
| llama_binary_directory: build/bin | |
| - name: Windows Server 2025 x64 portable | |
| os: windows-2025 | |
| runtime_identifier: win-x64 | |
| executable_suffix: .exe | |
| llama_binary_directory: build/bin/Release | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 35 | |
| env: | |
| DOTNET_NOLOGO: true | |
| DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true | |
| SYNAPSE_REQUIRED_RUNTIME_IDENTIFIER: ${{ matrix.runtime_identifier }} | |
| SYNAPSE_DOTLLM_VERSION: d88040451d7db56e5dfef9d5754ad0955b0f7fe5 | |
| SYNAPSE_LLAMACPP_VERSION: b29c606e28a01b1bc8c1351026a0fa6e616bf6c4 | |
| SYNAPSE_MODEL_ROOT: ${{ github.workspace }}/artifacts/models | |
| SYNAPSE_FOUNDRY_CACHE: ${{ github.workspace }}/artifacts/foundry-local | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - name: Checkout pinned dotLLM benchmark subject | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| repository: kkokosa/dotLLM | |
| ref: d88040451d7db56e5dfef9d5754ad0955b0f7fe5 | |
| path: _external/dotLLM | |
| persist-credentials: false | |
| - name: Checkout pinned native llama.cpp benchmark subject | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| repository: ggml-org/llama.cpp | |
| ref: b29c606e28a01b1bc8c1351026a0fa6e616bf6c4 | |
| path: _external/llama.cpp | |
| persist-credentials: false | |
| - name: Install pinned .NET SDK | |
| uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6.0.0 | |
| with: | |
| global-json-file: global.json | |
| - name: Install pinned Rust toolchain | |
| run: rustup toolchain install 1.98.1 --profile minimal --component clippy,rustfmt | |
| - name: Restore locked dependencies | |
| run: dotnet restore Synapse.slnx --locked-mode | |
| - name: Build .NET | |
| run: dotnet build Synapse.slnx --configuration Release --no-restore | |
| - name: Restore verified model cache | |
| uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 | |
| with: | |
| path: ${{ env.SYNAPSE_MODEL_ROOT }} | |
| key: synapse-models-${{ runner.os }}-${{ matrix.runtime_identifier }}-${{ env.SYNAPSE_GGUF_MODEL_ID }}-${{ env.SYNAPSE_GGUF_MODEL_FILE }}-${{ env.SYNAPSE_MODEL_SET }}-${{ env.SYNAPSE_EMBEDDING_MODEL_SET }}-${{ hashFiles('models/catalog.json') }} | |
| - name: Fetch verified small model packages | |
| shell: bash | |
| run: | | |
| dotnet run --project src/Synapse.Cli --configuration Release --no-build -- \ | |
| model fetch --id "${SYNAPSE_GGUF_MODEL_ID}" --output "${SYNAPSE_MODEL_ROOT}" | |
| dotnet run --project src/Synapse.Cli --configuration Release --no-build -- \ | |
| model fetch --set "${SYNAPSE_MODEL_SET}" --output "${SYNAPSE_MODEL_ROOT}" | |
| dotnet run --project src/Synapse.Cli --configuration Release --no-build -- \ | |
| model fetch --set "${SYNAPSE_EMBEDDING_MODEL_SET}" --output "${SYNAPSE_MODEL_ROOT}" | |
| - name: Prepare Synapse model before runtime checks | |
| shell: bash | |
| run: | | |
| source_model="${SYNAPSE_MODEL_ROOT}/${SYNAPSE_GGUF_MODEL_ID}/${SYNAPSE_GGUF_MODEL_FILE}" | |
| prepared_model="${source_model%.gguf}.synapse" | |
| if [[ -f "${prepared_model}" ]]; then | |
| dotnet run --project src/Synapse.Cli --configuration Release --no-build -- model inspect --model "${prepared_model}" | |
| else | |
| dotnet run --project src/Synapse.Cli --configuration Release --no-build -- model compile --source "${source_model}" --output "${prepared_model}" | |
| fi | |
| - name: Restore Foundry Local anchor model cache | |
| uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 | |
| with: | |
| path: ${{ env.SYNAPSE_FOUNDRY_CACHE }} | |
| key: foundry-local-anchor-${{ runner.os }}-${{ env.SYNAPSE_FOUNDRY_ANCHOR }}-${{ env.SYNAPSE_FOUNDRY_MODEL_SET }}-${{ hashFiles(env.SYNAPSE_FOUNDRY_MODEL_SET) }} | |
| - name: Fetch only the Foundry Local anchor model for functional checks | |
| shell: bash | |
| run: | | |
| dotnet experiments/Synapse.FoundryLocalBenchmarks/bin/Release/net10.0/Synapse.FoundryLocalBenchmarks.dll \ | |
| fetch --set "${SYNAPSE_FOUNDRY_MODEL_SET}" --alias "${SYNAPSE_FOUNDRY_ANCHOR}" --cache "${SYNAPSE_FOUNDRY_CACHE}" | |
| - name: Build pinned dotLLM benchmark subject | |
| run: dotnet build _external/dotLLM/src/DotLLM.Cli/DotLLM.Cli.csproj --configuration Release | |
| - name: Build pinned CPU llama.cpp benchmark subjects | |
| shell: bash | |
| run: | | |
| cmake -S _external/llama.cpp -B _external/llama.cpp/build \ | |
| -DCMAKE_BUILD_TYPE=Release -DGGML_METAL=OFF \ | |
| -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_SERVER=OFF | |
| cmake --build _external/llama.cpp/build --config Release --parallel 4 \ | |
| --target llama-completion llama-bench | |
| - name: Register dotLLM smoke-test executable | |
| shell: bash | |
| run: echo "SYNAPSE_DOTLLM_EXECUTABLE=${GITHUB_WORKSPACE}/_external/dotLLM/src/DotLLM.Cli/bin/Release/net10.0/DotLLM.Cli${{ matrix.executable_suffix }}" >> "${GITHUB_ENV}" | |
| - name: Register native llama.cpp smoke-test executable | |
| shell: bash | |
| run: echo "SYNAPSE_LLAMACPP_EXECUTABLE=${GITHUB_WORKSPACE}/_external/llama.cpp/${{ matrix.llama_binary_directory }}/llama-completion${{ matrix.executable_suffix }}" >> "${GITHUB_ENV}" | |
| - name: Test .NET | |
| run: >- | |
| dotnet test Synapse.slnx --configuration Release --no-build -- | |
| --report-trx --results-directory "${{ runner.temp }}/test-results" | |
| --minimum-expected-tests 1 --zero-tests-policy strict | |
| - name: Export real .NET test results as JSON | |
| if: always() | |
| shell: bash | |
| run: | | |
| dotnet experiments/Synapse.ReferenceBenchmarks/bin/Release/net10.0/Synapse.ReferenceBenchmarks.dll test-report \ | |
| --results "${RUNNER_TEMP}/test-results" --runner "${{ matrix.runtime_identifier }}" \ | |
| --output "${RUNNER_TEMP}/test-results.json" | |
| - name: Publish .NET test JSON | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: test-results-${{ matrix.runtime_identifier }} | |
| path: ${{ runner.temp }}/test-results.json | |
| if-no-files-found: warn | |
| retention-days: 90 | |
| - name: Preserve raw .NET TRX evidence | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: test-trx-${{ matrix.runtime_identifier }} | |
| path: ${{ runner.temp }}/test-results/*.trx | |
| if-no-files-found: warn | |
| retention-days: 90 | |
| - name: Check Rust formatting | |
| run: cargo fmt --manifest-path native/Cargo.toml --all --check | |
| - name: Lint Rust | |
| run: cargo clippy --manifest-path native/Cargo.toml --workspace --all-targets -- -D warnings | |
| - name: Test Rust | |
| run: cargo test --manifest-path native/Cargo.toml --locked | |
| - name: Exercise both doctor entry points | |
| shell: bash | |
| run: | | |
| state_dir="${RUNNER_TEMP}/synapse-doctor-state" | |
| dotnet run --project src/Synapse.Cli --configuration Release --no-build -- \ | |
| doctor --memory-budget-bytes 1073741824 --state-directory "${state_dir}" | |
| cargo run --manifest-path native/Cargo.toml --locked --quiet \ | |
| -p synapse-runtime -- doctor --memory-budget-bytes 1073741824 |