Dev Build & Publish #2444
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Dev Build & Publish | |
| on: | |
| schedule: | |
| - cron: '0 * * * *' # Every hour | |
| workflow_dispatch: | |
| concurrency: | |
| group: dev-build-publish | |
| cancel-in-progress: false | |
| permissions: | |
| contents: read | |
| actions: read | |
| env: | |
| GCP_REGION: ${{ vars.GCP_REGION }} | |
| GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }} | |
| GCP_REPO_NAME: ${{ vars.GCP_REPO_NAME }} | |
| PROTOC_VERSION: "28.3" | |
| # arduino/setup-protoc@v3.0.0 has no Node.js 24 release yet; remove once a node24 version ships | |
| FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true | |
| jobs: | |
| # --------------------------------------------------------------------------- | |
| # Gate: skip scheduled runs when nothing changed since last success | |
| # --------------------------------------------------------------------------- | |
| check-changes: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| outputs: | |
| has_changes: ${{ steps.check.outputs.has_changes }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| with: | |
| fetch-depth: 0 | |
| # Anchored on the commit we last *published*, not on a wall-clock | |
| # timestamp. The old form asked "any commit newer than the last | |
| # successful run's updatedAt?", which had two compounding faults: | |
| # | |
| # 1. A commit that lands while a run is in flight is older than that | |
| # run's completion time, so the next run's window excludes it. | |
| # 2. A gate-skipped run still concludes `success`, so it became the | |
| # new anchor — making the loss permanent rather than transient. | |
| # | |
| # Together those wedged the schedule: 7817ed50 landed at 14:51:48Z | |
| # during a run that finished at 14:52:56Z, and every hourly run after | |
| # it compared against the previous skip and reported "no new commits". | |
| # The tip of main went unpublished indefinitely. | |
| # | |
| # A commit SHA has neither failure mode: it advances only when a | |
| # publish actually succeeds, so an in-flight commit is simply picked up | |
| # by the next run, and a skipped run moves nothing. | |
| - name: Check for changes since last published commit | |
| id: check | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| set -euo pipefail | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| echo "has_changes=true" >> "$GITHUB_OUTPUT" | |
| echo "Manual trigger — always run" | |
| exit 0 | |
| fi | |
| # The publish job uploads `last-published-sha` only on success, so | |
| # the newest unexpired copy identifies the last published commit via | |
| # its run's head_sha. Sorted explicitly rather than trusting the | |
| # listing order, which the API does not document as stable. | |
| # | |
| # Guarded rather than assigned bare: under `set -e` a failing command | |
| # substitution aborts the step, which would leave `has_changes` unset | |
| # and skip the publish on any transient API error -- the one anchor | |
| # state that resolves itself. An unreachable API is an unknown anchor, | |
| # so it takes the same fall-open path as the two cases below. | |
| if ! last_sha=$(gh api \ | |
| "repos/${GITHUB_REPOSITORY}/actions/artifacts?name=last-published-sha&per_page=100" \ | |
| --jq '[.artifacts[] | select(.expired == false)] | |
| | sort_by(.created_at) | last | |
| | .workflow_run.head_sha // empty'); then | |
| echo "::warning::artifacts API lookup failed — treating the anchor as unknown and building" | |
| last_sha="" | |
| fi | |
| # No marker means either the first run after this change shipped or | |
| # the artifact aged out of retention. Publishing is idempotent | |
| # (`twine upload --skip-existing`), so building is the safe default. | |
| if [ -z "$last_sha" ]; then | |
| echo "has_changes=true" >> "$GITHUB_OUTPUT" | |
| echo "No published-commit marker found (first run or expired retention) — running" | |
| exit 0 | |
| fi | |
| # A force-push or history rewrite can leave a marker naming a commit | |
| # this checkout has never heard of; `rev-list` would abort the step. | |
| if ! git cat-file -e "${last_sha}^{commit}" 2>/dev/null; then | |
| echo "has_changes=true" >> "$GITHUB_OUTPUT" | |
| echo "Last published commit ${last_sha} is not in this history — running" | |
| exit 0 | |
| fi | |
| commit_count=$(git rev-list --count "${last_sha}..HEAD") | |
| if [ "$commit_count" -gt 0 ]; then | |
| echo "has_changes=true" >> "$GITHUB_OUTPUT" | |
| echo "${commit_count} new commit(s) since published ${last_sha}" | |
| else | |
| echo "has_changes=false" >> "$GITHUB_OUTPUT" | |
| echo "HEAD is already published as ${last_sha} — skipping" | |
| fi | |
| # --------------------------------------------------------------------------- | |
| # Stage 1: Compute dev version | |
| # --------------------------------------------------------------------------- | |
| version: | |
| needs: check-changes | |
| if: needs.check-changes.outputs.has_changes == 'true' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| outputs: | |
| dev_version: ${{ steps.compute.outputs.dev_version }} | |
| cargo_version: ${{ steps.compute.outputs.cargo_version }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Compute dev version | |
| id: compute | |
| run: | | |
| base_version=$(python3 -c " | |
| import tomllib | |
| with open('pyproject.toml', 'rb') as f: | |
| print(tomllib.load(f)['project']['version']) | |
| ") | |
| timestamp=$(date +%s) | |
| short_sha=$(git rev-parse --short HEAD) | |
| dev_version="${base_version}.dev${timestamp}+g${short_sha}" | |
| cargo_version="${base_version}-dev.${timestamp}+g${short_sha}" | |
| echo "dev_version=${dev_version}" >> "$GITHUB_OUTPUT" | |
| echo "cargo_version=${cargo_version}" >> "$GITHUB_OUTPUT" | |
| echo "PEP 440 version (pyproject.toml): ${dev_version}" | |
| echo "SemVer version (Cargo.toml): ${cargo_version}" | |
| echo "${dev_version}" > dev_version.txt | |
| echo "${cargo_version}" > cargo_version.txt | |
| - name: Upload version artifact | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: dev-version | |
| path: | | |
| dev_version.txt | |
| cargo_version.txt | |
| # --------------------------------------------------------------------------- | |
| # Stage 2: Build wheels (matrix) + sdist | |
| # --------------------------------------------------------------------------- | |
| build-wheels: | |
| needs: version | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| include: | |
| # Linux glibc x86_64 | |
| - os: ubuntu-latest | |
| target: x86_64-unknown-linux-gnu | |
| manylinux: auto | |
| # Linux glibc aarch64 (native ARM64 runner — avoids ring crate cross-compilation issues) | |
| - os: ubuntu-24.04-arm | |
| target: aarch64-unknown-linux-gnu | |
| manylinux: auto | |
| # Linux musl x86_64 (Alpine Docker images) | |
| - os: ubuntu-latest | |
| target: x86_64-unknown-linux-musl | |
| manylinux: musllinux_1_2 | |
| # Linux musl aarch64 (Alpine on ARM) | |
| - os: ubuntu-24.04-arm | |
| target: aarch64-unknown-linux-musl | |
| manylinux: musllinux_1_2 | |
| # macOS x86_64 (cross-compiled on arm64 runner) | |
| - os: macos-14 | |
| target: x86_64-apple-darwin | |
| # macOS arm64 | |
| - os: macos-14 | |
| target: aarch64-apple-darwin | |
| # Windows x86_64 | |
| - os: windows-latest | |
| target: x86_64-pc-windows-msvc | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 30 | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Download version artifact | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: dev-version | |
| path: version-artifact | |
| continue-on-error: true | |
| - name: Resolve dev version | |
| shell: bash | |
| run: | | |
| if [ -f version-artifact/dev_version.txt ]; then | |
| DEV_VERSION=$(cat version-artifact/dev_version.txt) | |
| CARGO_VERSION=$(cat version-artifact/cargo_version.txt) | |
| else | |
| DEV_VERSION="${{ needs.version.outputs.dev_version }}" | |
| CARGO_VERSION="${{ needs.version.outputs.cargo_version }}" | |
| fi | |
| echo "DEV_VERSION=${DEV_VERSION}" >> "$GITHUB_ENV" | |
| echo "CARGO_VERSION=${CARGO_VERSION}" >> "$GITHUB_ENV" | |
| - name: Install protoc | |
| if: runner.os != 'Linux' | |
| uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 | |
| with: | |
| version: ${{ env.PROTOC_VERSION }} | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Inject dev version | |
| shell: bash | |
| env: | |
| DEV_VERSION: ${{ needs.version.outputs.dev_version }} | |
| CARGO_VERSION: ${{ needs.version.outputs.cargo_version }} | |
| run: | | |
| python3 -c " | |
| import re, os, pathlib | |
| p = pathlib.Path('pyproject.toml') | |
| text = p.read_text() | |
| v = os.environ['DEV_VERSION'] | |
| text = re.sub(r'version\s*=\s*\"[^\"]+\"', f'version = \"{v}\"', text, count=1) | |
| p.write_text(text) | |
| " | |
| python3 -c " | |
| import re, os, pathlib | |
| p = pathlib.Path('rust/Cargo.toml') | |
| text = p.read_text() | |
| v = os.environ['CARGO_VERSION'] | |
| text = re.sub(r'^version\s*=\s*\"[^\"]+\"', f'version = \"{v}\"', text, count=1, flags=re.MULTILINE) | |
| p.write_text(text) | |
| " | |
| cargo generate-lockfile --manifest-path rust/Cargo.toml | |
| grep '^version' pyproject.toml rust/Cargo.toml | |
| - name: Verify version injection | |
| shell: bash | |
| run: | | |
| python3 -c " | |
| import tomllib, os | |
| with open('pyproject.toml', 'rb') as f: | |
| v = tomllib.load(f)['project']['version'] | |
| expected = os.environ['DEV_VERSION'] | |
| assert v == expected, f'Expected {expected}, got {v}' | |
| print(f'Version injection verified: {v}') | |
| " | |
| - name: Build wheels | |
| uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1.50.1 | |
| with: | |
| target: ${{ matrix.target }} | |
| args: --release --out dist | |
| manylinux: ${{ matrix.manylinux || 'auto' }} | |
| before-script-linux: | | |
| TARGET="${{ matrix.target }}" | |
| PROTOC_VERSION="${{ env.PROTOC_VERSION }}" | |
| case "$TARGET" in | |
| x86_64-*) | |
| PROTOC_ARCH="linux-x86_64" | |
| PROTOC_SHA256="0ad949f04a6a174da83cdcbdb36dee0a4925272a5b6d83f79a6bf9852076d53f" | |
| ;; | |
| aarch64-*) | |
| PROTOC_ARCH="linux-aarch_64" | |
| PROTOC_SHA256="1de522032a8b194002fe35cab86d747848238b5e4de4f99648372079f5b46f9a" | |
| ;; | |
| *) | |
| echo "Unsupported target: $TARGET" >&2 | |
| exit 1 | |
| ;; | |
| esac | |
| PROTOC_ZIP="protoc-${PROTOC_VERSION}-${PROTOC_ARCH}.zip" | |
| curl -fsSLO "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/${PROTOC_ZIP}" | |
| echo "${PROTOC_SHA256} ${PROTOC_ZIP}" | sha256sum -c - | |
| python3 -m zipfile -e "${PROTOC_ZIP}" /usr/local | |
| chmod +x /usr/local/bin/protoc | |
| rm "${PROTOC_ZIP}" | |
| - name: Validate wheel platform tags (x86_64 cross-compile) | |
| if: matrix.target == 'x86_64-apple-darwin' | |
| shell: bash | |
| run: | | |
| echo "Validating wheel platform tags for x86_64 cross-compile..." | |
| bad=0 | |
| for wheel in dist/*.whl; do | |
| echo " $wheel" | |
| if [[ "$wheel" != *"x86_64"* ]]; then | |
| echo "ERROR: expected x86_64 platform tag but got: $wheel" | |
| bad=1 | |
| fi | |
| done | |
| if [ "$bad" -ne 0 ]; then | |
| echo "One or more wheels have incorrect platform tags." | |
| exit 1 | |
| fi | |
| echo "All wheels have correct macosx_*_x86_64 platform tags." | |
| - name: Upload wheels | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: wheels-${{ matrix.target }} | |
| path: dist/*.whl | |
| build-sdist: | |
| needs: version | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Download version artifact | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: dev-version | |
| path: version-artifact | |
| continue-on-error: true | |
| - name: Resolve dev version | |
| run: | | |
| if [ -f version-artifact/dev_version.txt ]; then | |
| DEV_VERSION=$(cat version-artifact/dev_version.txt) | |
| CARGO_VERSION=$(cat version-artifact/cargo_version.txt) | |
| else | |
| DEV_VERSION="${{ needs.version.outputs.dev_version }}" | |
| CARGO_VERSION="${{ needs.version.outputs.cargo_version }}" | |
| fi | |
| echo "DEV_VERSION=${DEV_VERSION}" >> "$GITHUB_ENV" | |
| echo "CARGO_VERSION=${CARGO_VERSION}" >> "$GITHUB_ENV" | |
| - name: Inject dev version | |
| env: | |
| DEV_VERSION: ${{ needs.version.outputs.dev_version }} | |
| CARGO_VERSION: ${{ needs.version.outputs.cargo_version }} | |
| run: | | |
| python3 -c " | |
| import re, os, pathlib | |
| p = pathlib.Path('pyproject.toml') | |
| text = p.read_text() | |
| v = os.environ['DEV_VERSION'] | |
| text = re.sub(r'version\s*=\s*\"[^\"]+\"', f'version = \"{v}\"', text, count=1) | |
| p.write_text(text) | |
| " | |
| python3 -c " | |
| import re, os, pathlib | |
| p = pathlib.Path('rust/Cargo.toml') | |
| text = p.read_text() | |
| v = os.environ['CARGO_VERSION'] | |
| text = re.sub(r'^version\s*=\s*\"[^\"]+\"', f'version = \"{v}\"', text, count=1, flags=re.MULTILINE) | |
| p.write_text(text) | |
| " | |
| cargo generate-lockfile --manifest-path rust/Cargo.toml | |
| grep '^version' pyproject.toml rust/Cargo.toml | |
| - name: Verify version injection | |
| run: | | |
| python3 -c " | |
| import tomllib, os | |
| with open('pyproject.toml', 'rb') as f: | |
| v = tomllib.load(f)['project']['version'] | |
| expected = os.environ['DEV_VERSION'] | |
| assert v == expected, f'Expected {expected}, got {v}' | |
| print(f'Version injection verified: {v}') | |
| " | |
| - name: Build sdist | |
| uses: PyO3/maturin-action@04ac600d27cdf7a9a280dadf7147097c42b757ad # v1.50.1 | |
| with: | |
| command: sdist | |
| args: --out dist | |
| - name: Upload sdist | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: sdist | |
| path: dist/*.tar.gz | |
| # --------------------------------------------------------------------------- | |
| # Stage 3: Publish to Artifact Registry | |
| # --------------------------------------------------------------------------- | |
| publish: | |
| needs: [build-wheels, build-sdist] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - name: Download all artifacts | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| path: dist | |
| pattern: '{wheels-*,sdist}' | |
| merge-multiple: true | |
| - name: Validate artifact counts | |
| run: | | |
| echo "Artifacts to publish:" | |
| ls -la dist/ | |
| whl_count=$(ls dist/*.whl 2>/dev/null | wc -l) | |
| sdist_count=$(ls dist/*.tar.gz 2>/dev/null | wc -l) | |
| echo "Wheels: $whl_count, Sdists: $sdist_count" | |
| # 7 platforms × 1 abi3 wheel (cp310-abi3, covers Python 3.10+) = 7 wheels | |
| if [ "$whl_count" -lt 7 ]; then | |
| echo "::error::expected at least 7 wheels but got $whl_count — one or more matrix legs may be missing" | |
| exit 1 | |
| fi | |
| if [ "$sdist_count" -lt 1 ]; then | |
| echo "::error::sdist missing" | |
| exit 1 | |
| fi | |
| - name: Authenticate to Google Cloud | |
| uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 | |
| with: | |
| workload_identity_provider: ${{ vars.GCP_WORKLOAD_IDENTITY_PROVIDER }} | |
| service_account: ${{ vars.GCP_SERVICE_ACCOUNT }} | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version: "3.12" | |
| - name: Install twine | |
| run: pip install twine==6.1.0 | |
| - name: Configure Artifact Registry credentials | |
| run: | | |
| pip install keyring==25.6.0 keyrings.google-artifactregistry-auth==1.1.2 | |
| - name: Validate wheel metadata | |
| run: twine check dist/* | |
| - name: Upload to Artifact Registry | |
| run: | | |
| twine upload \ | |
| --skip-existing \ | |
| --repository-url "https://${GCP_REGION}-python.pkg.dev/${GCP_PROJECT_ID}/${GCP_REPO_NAME}/" \ | |
| dist/* | |
| # The gate's anchor. Written only after twine returns 0, so a failed | |
| # publish leaves the previous anchor standing and the next run retries | |
| # the same commit instead of stepping over it. The file content is for | |
| # humans reading the artifact; the gate reads the SHA from the artifact's | |
| # associated run, which cannot drift from what this job checked out. | |
| - name: Record published commit | |
| run: | | |
| echo "${{ github.sha }}" > last-published-sha.txt | |
| - name: Upload published-commit marker | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: last-published-sha | |
| path: last-published-sha.txt | |
| # --------------------------------------------------------------------------- | |
| # Stage 4: Smoke tests against the just-published wheel (informational only) | |
| # --------------------------------------------------------------------------- | |
| smoke-tests: | |
| name: Smoke tests (informational) | |
| needs: publish | |
| if: success() | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| continue-on-error: true | |
| permissions: | |
| contents: read | |
| env: | |
| PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }} | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version: "3.12" | |
| - name: Download linux x86_64 wheel | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: wheels-x86_64-unknown-linux-gnu | |
| path: dist | |
| - name: Install wheel and test deps | |
| run: | | |
| set -euo pipefail | |
| wheel=$(ls dist/*.whl | head -n1) | |
| echo "Installing ${wheel}" | |
| pip install --upgrade pip | |
| pip install "${wheel}" pytest pytest-asyncio python-dotenv | |
| - name: Run smoke suite | |
| id: smoke | |
| continue-on-error: true | |
| run: | | |
| # The source `pinecone/` directory from actions/checkout shadows the | |
| # wheel-installed package on sys.path (cwd is on the path). The source | |
| # tree has `pinecone/grpc/__init__.py` but not the compiled | |
| # `_grpc.abi3.so` extension, which only ships inside the wheel — so | |
| # any test that constructs a GrpcIndex would crash with | |
| # `ModuleNotFoundError: No module named 'pinecone._grpc'`. Smoke runs | |
| # the published wheel only, so we can safely delete the source. | |
| rm -rf pinecone rust target | |
| pytest tests/smoke/ \ | |
| -v -s -rs \ | |
| --ignore=tests/smoke/test_pod_collections_sync.py \ | |
| --ignore=tests/smoke/test_pod_collections_async.py \ | |
| --junitxml=smoke-results.xml | |
| - name: Append summary | |
| if: always() | |
| run: | | |
| if [ "${{ steps.smoke.outcome }}" = "success" ]; then | |
| echo "## Smoke tests: PASSED" >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| echo "## Smoke tests: FAILED (informational — does not block publish)" >> "$GITHUB_STEP_SUMMARY" | |
| echo "See the 'Run smoke suite' step output and the smoke-results artifact." >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| - name: Orphan cleanup | |
| if: always() | |
| continue-on-error: true | |
| run: | | |
| python tests/smoke/scripts/cleanup_orphans.py || true | |
| - name: Upload smoke results | |
| if: always() | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: smoke-results | |
| path: smoke-results.xml |