Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
41 changes: 41 additions & 0 deletions .github/actions/setup-disk-benchmark/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT license.

# Shared setup for disk-index benchmark workflows.
# Installs system dependencies and downloads a dataset from big-ann-benchmarks.

name: Setup Disk Benchmark
description: Install dependencies and download a benchmark dataset

inputs:
dataset:
description: 'Dataset name (e.g. wikipedia-100K, openai-100K)'
required: true
archive:
description: 'Archive filename to download (e.g. wikipedia-100K.tar.gz)'
required: true
extract-to:
description: 'Directory to extract the dataset into'
required: true

runs:
using: composite
steps:
- name: Install Rust
shell: bash
run: rustup show

- name: Install system dependencies
shell: bash
run: |
sudo apt-get update
sudo apt-get install -y openssl libssl-dev pkg-config

- name: Download ${{ inputs.dataset }} dataset
shell: bash
env:
BAB_RELEASE_URL: https://github.com/harsha-simhadri/big-ann-benchmarks/releases/download/v0.4.0
run: |
mkdir -p ${{ inputs.extract-to }}
curl -L -o ${{ inputs.archive }} ${{ env.BAB_RELEASE_URL }}/${{ inputs.archive }}
tar xzf ${{ inputs.archive }} -C ${{ inputs.extract-to }}
123 changes: 67 additions & 56 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,11 @@ env:
# This ensures that we do not overwrite the parameters in `.cargo/config.toml` unnecessarily.
RUST_CONFIG: 'build.rustflags=["-Dwarnings"]'
RUST_BACKTRACE: 1
CARGO_TERM_COLOR: always
# The features we want to explicitly test. For example, the `flatbuffers-build` feature
# of `diskann-quantization` requires additional setup and so must not be included by default.
DISKANN_FEATURES: "virtual_storage,bf_tree,spherical-quantization,product-quantization,tracing,experimental_diversity_search,disk-index,flatbuffers,linalg,codegen"

# Use the Rust version specified in rust-toolchain.toml
rust_stable: "1.92"

# Intel SDE version used for baseline and AVX-512 emulation jobs.
SDE_VERSION: "sde-external-10.7.0-2026-02-18-lin"

Expand All @@ -37,6 +35,49 @@ permissions:
contents: read

jobs:
# Branch-protection gate
#
# Add "CI gate" as the sole required status check in branch protection.
#
# How it works:
#
# * If no upstream job fails or is cancelled, the `if` condition is `false`
# and this job is *skipped*. GitHub treats a skipped required check as
# passing.
#
# Note: jobs that are conditionally skipped (e.g. `coverage` excludes
# Dependabot) are neither failed nor cancelled, so the gate will still
# pass. This is intentional -- conditional skips represent expected
# behavior, not failures.
#
# See: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/about-status-checks
#
# * If any upstream job fails or is cancelled, the `if` condition is `true`.
# This job runs and exits 1, blocking the merge.
#
# All merge-blocking jobs should be listed in `needs` below.
ci-gate:
name: CI gate
runs-on: ubuntu-latest
needs:
- fmt
- clippy-default-features
- clippy-features
- clippy-no-default-features
- basics
- codeql
- baseline
- sde
- test-workspace
- test-workspace-features
- coverage
if: ${{ always() && (cancelled() || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'failure')) }}
steps:
- name: Gate failure
run: |
echo "::error::One or more required CI jobs failed or were cancelled."
exit 1

# Basic checks that must pass before we kick off more expensive tests.
basics:
name: basic checks
Expand All @@ -56,11 +97,8 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install Rust ${{ env.rust_stable }}
uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{ env.rust_stable }}
components: rustfmt
- name: Install Rust
run: rustup show && rustup component add rustfmt
- uses: Swatinem/rust-cache@v2
# Check fmt
- name: "cargo fmt --check"
Expand All @@ -83,11 +121,8 @@ jobs:
runs-on: ${{ matrix.runner }}
steps:
- uses: actions/checkout@v4
- name: Install Rust ${{ env.rust_stable }}
uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{ env.rust_stable }}
components: clippy
- name: Install Rust
run: rustup show && rustup component add clippy
- uses: Swatinem/rust-cache@v2
- name: "clippy --workspace --all-targets"
run: cargo clippy --locked --workspace --all-targets --no-deps --config "$RUST_CONFIG" -- -Dwarnings
Expand All @@ -104,11 +139,8 @@ jobs:
runs-on: ${{ matrix.runner }}
steps:
- uses: actions/checkout@v4
- name: Install Rust ${{ env.rust_stable }}
uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{ env.rust_stable }}
components: clippy
- name: Install Rust
run: rustup show && rustup component add clippy
- uses: Swatinem/rust-cache@v2
- name: "clippy --workspace --all-targets"
run: |
Expand All @@ -130,11 +162,8 @@ jobs:
with:
lfs: true

- name: Install Rust ${{ env.rust_stable }}
uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{ env.rust_stable }}
components: clippy
- name: Install Rust
run: rustup show && rustup component add clippy

- uses: Swatinem/rust-cache@v2

Expand Down Expand Up @@ -180,10 +209,8 @@ jobs:
with:
languages: rust

- name: Install Rust ${{ env.rust_stable }}
uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{ env.rust_stable }}
- name: Install Rust
run: rustup show

- uses: Swatinem/rust-cache@v2

Expand All @@ -201,10 +228,8 @@ jobs:
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v4
# - name: Install Rust ${{ env.rust_stable }}
# uses: dtolnay/rust-toolchain@master
# with:
# toolchain: ${{ env.rust_stable }}
# - name: Install Rust
# run: rustup show
# - uses: Swatinem/rust-cache@v2
# - name: "doc --workspace --no-deps"
# run: cargo doc --workspace --no-deps --document-private-items
Expand Down Expand Up @@ -241,10 +266,8 @@ jobs:
steps:
- uses: actions/checkout@v4

- name: Install Rust ${{ env.rust_stable }}
uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{ env.rust_stable }}
- name: Install Rust
run: rustup show

- uses: Swatinem/rust-cache@v2

Expand Down Expand Up @@ -304,10 +327,8 @@ jobs:
steps:
- uses: actions/checkout@v4

- name: Install Rust ${{ env.rust_stable }}
uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{ env.rust_stable }}
- name: Install Rust
run: rustup show

- uses: Swatinem/rust-cache@v2

Expand Down Expand Up @@ -357,10 +378,8 @@ jobs:
with:
lfs: true

- name: Install Rust ${{ env.rust_stable }}
uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{ env.rust_stable }}
- name: Install Rust
run: rustup show

- name: Install cargo-nextest
uses: taiki-e/install-action@v2
Expand Down Expand Up @@ -391,10 +410,8 @@ jobs:
with:
lfs: true

- name: Install Rust ${{ env.rust_stable }}
uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{ env.rust_stable }}
- name: Install Rust
run: rustup show

- name: Install cargo-nextest
uses: taiki-e/install-action@v2
Expand Down Expand Up @@ -424,11 +441,8 @@ jobs:
lfs: true
fetch-depth: 0

- name: Install Rust ${{ env.rust_stable }}
uses: dtolnay/rust-toolchain@master
with:
toolchain: ${{ env.rust_stable }}
components: llvm-tools-preview
- name: Install Rust
run: rustup show && rustup component add llvm-tools-preview

- name: Install cargo-llvm-cov
uses: taiki-e/install-action@v2
Expand Down Expand Up @@ -488,10 +502,7 @@ jobs:
lfs: true

- name: Install Rust nightly with miri
uses: dtolnay/rust-toolchain@master
with:
toolchain: nightly
components: miri
run: rustup toolchain install nightly --component miri

- name: Install cargo-nextest
uses: taiki-e/install-action@v2
Expand Down
Loading