diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml index 440353af8..055bcea55 100644 --- a/.github/workflows/checks.yaml +++ b/.github/workflows/checks.yaml @@ -74,7 +74,7 @@ jobs: - name: Ensure SHA pinned actions uses: zgosalvez/github-actions-ensure-sha-pinned-actions@70c4af2ed5282c51ba40566d026d6647852ffa3e # v5.0.1 - js-build: + js-test: runs-on: ${{ matrix.os }} timeout-minutes: 30 strategy: @@ -82,19 +82,39 @@ jobs: matrix: os: [ubuntu-latest, windows-latest] node-version: [20, 22] + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: ${{ matrix.node-version }} + - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 + - name: Install dependencies + run: pnpm install --frozen-lockfile + - name: Build + run: pnpm run build + - name: Generate docs + working-directory: js + run: pnpm run docs + - name: Run hermetic tests + working-directory: js + run: pnpm run test:checks + + js-build: + runs-on: ubuntu-latest + timeout-minutes: 20 outputs: artifact-name: ${{ steps.artifact.outputs.name }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: - node-version: ${{ matrix.node-version }} + node-version: 20 registry-url: "https://registry.npmjs.org" - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 - - name: Verify checks - shell: bash - run: | - make js-verify-checks + - name: Install dependencies + run: pnpm install --frozen-lockfile + - name: Build + run: pnpm run build - name: Determine artifact name id: artifact env: @@ -162,7 +182,7 @@ jobs: - name: Upload build artifacts uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: ${{ steps.artifact.outputs.name }}-${{ matrix.node-version }}-dist + name: ${{ steps.artifact.outputs.name }}-dist path: | js/artifacts/${{ steps.prepare_artifact.outputs.packed_tarball }} js/artifacts/${{ steps.prepare_browser_artifact.outputs.packed_browser_tarball }} @@ -273,10 +293,10 @@ jobs: with: node-version-file: .tool-versions - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 - - name: Download build artifact (node 20) + - name: Download build artifacts uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: - name: ${{ needs.js-build.outputs.artifact-name }}-20-dist + name: ${{ needs.js-build.outputs.artifact-name }}-dist path: js/artifacts run-id: ${{ github.run_id }} github-token: ${{ github.token }} @@ -439,11 +459,9 @@ jobs: working-directory: ./js run: | pnpm run build - - name: Run hermetic JS tests + - name: Run hermetic tests working-directory: ./js - shell: bash - run: | - make test-checks + run: pnpm run test:checks - name: Run zod v3 tests if: matrix.zod-version == '3.25.34' working-directory: ./js @@ -463,6 +481,7 @@ jobs: - check-typings - dead-code - ensure-pinned-actions + - js-test - js-build - js-api-compatibility - js-smoke-discover @@ -494,6 +513,7 @@ jobs: check_result "check-typings" "${{ needs.check-typings.result }}" check_result "dead-code" "${{ needs.dead-code.result }}" check_result "ensure-pinned-actions" "${{ needs.ensure-pinned-actions.result }}" + check_result "js-test" "${{ needs.js-test.result }}" check_result "js-build" "${{ needs.js-build.result }}" check_result "js-api-compatibility" "${{ needs.js-api-compatibility.result }}" check_result "js-smoke-discover" "${{ needs.js-smoke-discover.result }}" diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 0c9ca87de..32a0807b4 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -10,7 +10,6 @@ on: - "pnpm-lock.yaml" - "pnpm-workspace.yaml" - "turbo.json" - - "Makefile" push: branches: [main] @@ -37,10 +36,14 @@ jobs: node-version: ${{ matrix.node-version }} registry-url: "https://registry.npmjs.org" - uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 - - name: Run provider-backed JS tests + - name: Install dependencies + run: pnpm install --frozen-lockfile + - name: Build + run: pnpm run build + - name: Run provider-backed tests + working-directory: js shell: bash - run: | - make js-test-external + run: pnpm run test:external e2e: runs-on: ubuntu-latest diff --git a/.github/workflows/publish-js-sdk.yaml b/.github/workflows/publish-js-sdk.yaml index c9c11b187..acfeb244a 100644 --- a/.github/workflows/publish-js-sdk.yaml +++ b/.github/workflows/publish-js-sdk.yaml @@ -139,7 +139,7 @@ jobs: working-directory: js env: RELEASE_BRANCH: ${{ env.TARGET_BRANCH }} - run: make publish-sdk-js + run: pnpm run publish:validate - name: Create and push release tag run: | diff --git a/CLAUDE.md b/CLAUDE.md index 603573999..8659ace61 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,5 +24,4 @@ pnpm run formatting # Check formatting pnpm run lint # Run lint checks pnpm run fix:formatting # Auto-fix formatting pnpm run fix:lint # Auto-fix lint issues -make test # Full JS-oriented test flow used in this repo ``` diff --git a/Makefile b/Makefile deleted file mode 100644 index 581f89bd5..000000000 --- a/Makefile +++ /dev/null @@ -1,75 +0,0 @@ -SHELL := /bin/bash -ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) - -.PHONY: install-dev -install-dev: - mise install - -.PHONY: install-deps -install-deps: - pnpm install - -.PHONY: test -test: js-test - -.PHONY: lint -lint: - pnpm run lint - -.PHONY: fixup -fixup: - pnpm run fix:formatting && pnpm run fix:lint - -# -# js stuff -# -# - -.PHONY: js-build js-test js-test-checks js-test-external js-docs js-verify-checks js-verify-ci - -js-build: - $(MISE) pnpm install --frozen-lockfile - $(MISE) pnpm run build - -js-test-checks: js-build - cd js && make test-checks - -js-test-external: js-build - cd js && make test-external - -js-test: js-test-checks js-test-external - -js-docs: js-build - cd js && make docs - -js-verify-checks: js-docs js-test-checks - -js-verify-ci: js-verify-checks js-test-external - -js-test-otel-docker: - @echo "Building Docker images for otel-js tests..." - @if [ -z "$$NODE_VERSION" ]; then \ - NODE_VER=22; \ - else \ - NODE_VER=$$NODE_VERSION; \ - fi; \ - echo "Building otel-v1 test container..."; \ - docker build -f integrations/otel-js/Dockerfile.test --build-arg NODE_VERSION=$$NODE_VER --build-arg TEST_DIR=otel-v1 -t otel-js-test-v1 . && \ - echo "Building otel-v2 test container..."; \ - docker build -f integrations/otel-js/Dockerfile.test --build-arg NODE_VERSION=$$NODE_VER --build-arg TEST_DIR=otel-v2 -t otel-js-test-v2 . - @echo "Running otel-v1 tests in Docker container..." - @docker run --rm otel-js-test-v1 - @echo "Running otel-v2 tests in Docker container..." - @docker run --rm otel-js-test-v2 - @echo "✅ All otel-js Docker tests passed" - - -# ------------------------------------------------------------------------------------------------- -# Stable release publishing -# Preferred: trigger publish-js-sdk.yaml from GitHub Actions UI -# Fallback: make release-js-sdk [RELEASE_TYPE=stable|prerelease|canary] [BRANCH=] dispatches the same workflow via gh -# ------------------------------------------------------------------------------------------------- -.PHONY: release-js-sdk - -release-js-sdk: - ./js/scripts/dispatch-release-workflow.sh diff --git a/PUBLISHING_JS.md b/PUBLISHING_JS.md index afb4f44a5..3c6358dec 100644 --- a/PUBLISHING_JS.md +++ b/PUBLISHING_JS.md @@ -78,20 +78,20 @@ The workflow writes a short run summary with the published version and recent co If you do not want to open GitHub Actions manually, you can dispatch the same workflow from the terminal: ```bash -make release-js-sdk +./js/scripts/dispatch-release-workflow.sh ``` To target a different remote branch: ```bash -make release-js-sdk BRANCH= +BRANCH= ./js/scripts/dispatch-release-workflow.sh ``` To dispatch a prerelease or canary instead of a stable release: ```bash -make release-js-sdk RELEASE_TYPE=prerelease -make release-js-sdk RELEASE_TYPE=canary +RELEASE_TYPE=prerelease ./js/scripts/dispatch-release-workflow.sh +RELEASE_TYPE=canary ./js/scripts/dispatch-release-workflow.sh ``` Notes: diff --git a/js/CLAUDE.md b/js/CLAUDE.md index b7bfa228a..a0a02ef16 100644 --- a/js/CLAUDE.md +++ b/js/CLAUDE.md @@ -5,7 +5,27 @@ Run commands from the `js/` directory unless noted otherwise. ## Running Tests ```bash -make test # Full JS test suite (core + wrappers) +pnpm run test:checks # Hermetic tests (core + vitest wrapper) +pnpm run test:external # Provider-backed tests (OpenAI, Anthropic, etc.) +pnpm run test:all # Everything (checks + external) +pnpm test # Core vitest suite only +``` + +**Individual provider tests:** + +```bash +pnpm run test:external:openai # OpenAI wrapper +pnpm run test:external:anthropic # Anthropic wrapper +pnpm run test:external:google-genai # Google GenAI wrapper +pnpm run test:external:ai-sdk # AI SDK (v5 + v6) +pnpm run test:external:claude-agent-sdk # Claude Agent SDK +``` + +**Test a specific version of a provider:** + +```bash +./scripts/test-provider.sh test:openai openai@4.92.1 +./scripts/test-provider.sh test:anthropic @anthropic-ai/sdk@0.39.0 ``` **Run a single test:** diff --git a/js/Makefile b/js/Makefile deleted file mode 100644 index cc9a84264..000000000 --- a/js/Makefile +++ /dev/null @@ -1,191 +0,0 @@ -# Default target -help: - @echo "Braintrust JS SDK Makefile" - @echo "" - @echo "Available commands:" - @echo " make help - Show this help message" - @echo " make build - Build the SDK" - @echo " make clean - Clean build artifacts" - @echo " make install-optional-deps - Install optional dependencies" - @echo " make test - Run all tests (core + wrappers + ai-sdk)" - @echo " make test-checks - Run hermetic tests only" - @echo " make test-external - Run provider-backed tests only" - @echo " make test-core - Run core tests only" - @echo " make test-openai - Run OpenAI wrapper tests" - @echo " make test-anthropic - Run Anthropic wrapper tests" - @echo " make test-google-genai - Run Google GenAI wrapper tests" - @echo " make test-ai-sdk - Run AI SDK wrapper tests (v5 + v6)" - @echo " make test-ai-sdk-v5 - Run AI SDK v5 wrapper tests" - @echo " make test-ai-sdk-v6 - Run AI SDK v6 wrapper tests" - @echo " make test-claude-agent-sdk - Run Claude Agent SDK wrapper tests" - @echo " make test-vitest - Run Vitest wrapper tests" - @echo " make test-api-compat - Run API compatibility tests" - @echo " make bench - Run queue performance benchmarks" - @echo " make test-latest - Run core + latest versions of wrappers" - @echo " make verify-checks - Build, docs, and hermetic tests" - @echo "" - @echo "Smoke tests (mimics CI workflow):" - @echo " make test-smoke - Run all smoke tests (auto-prepares)" - @echo " make test-smoke deno - Run deno smoke test only" - @echo " make test-smoke span - Run span smoke test only" - @echo "" - @echo "See smoke/README.md for details on smoke test infrastructure" - -.PHONY: help bench build clean test test-checks test-external test-core test-openai test-anthropic test-google-genai test-ai-sdk test-ai-sdk-v5 test-ai-sdk-v6 test-claude-agent-sdk test-vitest test-latest install-optional-deps publish-beta-local test-smoke - -# ------------------------------------------------------------------------------------------------- # -# Anthropic testing -# ------------------------------------------------------------------------------------------------- - -test-anthropic: test-anthropic-latest test-anthropic-0.39.0 test-anthropic-0.38.0 - -test-anthropic-latest: - pnpm prune - $(call pnpm_install_no_save,@anthropic-ai/sdk) - pnpm test:anthropic - -test-anthropic-%: - pnpm prune - $(call pnpm_install_no_save,@anthropic-ai/sdk@$*) - pnpm test:anthropic - -# ---- -# Google GenAI -# ---- -test-google-genai: test-google-genai-latest - -test-google-genai-latest: - pnpm prune - $(call pnpm_install_no_save,@google/genai) - pnpm test:google-genai - -# ------------------------------------------------------------------------------------------------- -# AI SDK testing -# ------------------------------------------------------------------------------------------------- - -test-ai-sdk: test-ai-sdk-v5 test-ai-sdk-v6 - -test-ai-sdk-v5: - cd src/wrappers/ai-sdk/tests/v5 && pnpm install --ignore-workspace && pnpm test - -test-ai-sdk-v6: - cd src/wrappers/ai-sdk/tests/v6 && pnpm install --ignore-workspace && pnpm test - -# ------------------------------------------------------------------------------------------------- -# Claude Agent SDK testing -# ------------------------------------------------------------------------------------------------- - -test-claude-agent-sdk: - cd src/wrappers/claude-agent-sdk && pnpm install && pnpm test - -# ------------------------------------------------------------------------------------------------- -# Vitest testing -# ------------------------------------------------------------------------------------------------- - -test-vitest: - cd src/wrappers/vitest && pnpm install && pnpm test - -# ------------------------------------------------------------------------------------------------- -# OpenAI testing -# ------------------------------------------------------------------------------------------------- - -.PHONY: test-openai test-openai-latest test-openai-% - -test-openai: test-openai-latest test-openai-4.92.1 test-openai-4.91.0 test-openai-4.86.0 -test-openai-latest: - pnpm prune - $(call pnpm_install_no_save,openai) - pnpm test:openai - -test-openai-%: - pnpm prune - $(call pnpm_install_no_save,openai@$*) - pnpm test:openai - - -# ------------------------------------------------------------------------------------------------- -# common things -# ------------------------------------------------------------------------------------------------- - -.PHONY: test-pnpm test test-checks test-external test-latest prune installing-optional-deps docs build verify-checks verify-ci bench test-api-compat - -install-optional-deps: - npm_config_save=false npm_config_lockfile=false pnpm add \ - "openai" \ - "@anthropic-ai/sdk" - - -# Test everything but the wrappers. -test-core: - pnpm prune - pnpm test - -# Test API compatibility -test-api-compat: - pnpm test:api-compat - -# Test only hermetic suites. -test-checks: test-core test-vitest - -# Test only provider-backed suites. -test-external: test-openai test-anthropic test-google-genai test-ai-sdk test-claude-agent-sdk - -# Test everything -test: test-checks test-external - -# Test the core and the latest versions of wrappers. -test-latest: test-core test-anthropic-latest test-openai-latest test-google-genai test-ai-sdk test-vitest test-claude-agent-sdk - - -prune: - pnpm prune - -docs: - pnpm run docs - -build: - npm run build - -publish-beta-local: build - pnpm publish --tag beta - -bench: - npx tsx src/queue.bench.ts - - -# note: we don't use the docs in ci, but this makes sure they keep building - -verify-checks: build docs test-checks - -verify-ci: verify-checks test-external - -publish-sdk-js: - ./scripts/validate-release.sh - npm install - npm run build - npm publish - -# This is the only method I could find to install a package without explicitly -# adding a dependency or modifying lock files. -define pnpm_install_no_save - @echo "No save installing "$(1)"" - npm_config_save=false npm_config_lockfile=false pnpm add "$(1)" -endef - -clean: - npm run clean - -# ------------------------------------------------------------------------------------------------- -# Smoke tests -# ------------------------------------------------------------------------------------------------- - -SMOKE_DIR := smoke - -# Run smoke tests - delegates to smoke/Makefile which handles auto-discovery and tarball creation -# Usage: make test-smoke [TEST_NAME...] -test-smoke: - @cd $(SMOKE_DIR) && $(MAKE) test $(filter-out test-smoke,$(MAKECMDGOALS)) - -# Allow passing test names as make targets (they become no-ops) -cloudflare-worker deno nextjs-instrumentation otel-v1 span span-jest: - @: diff --git a/js/package.json b/js/package.json index c7e264293..781d0d05b 100644 --- a/js/package.json +++ b/js/package.json @@ -112,6 +112,17 @@ "clean": "rm -r dist/* && rm -r dev/dist/*", "docs": "npx typedoc --options typedoc.json src/node/index.ts", "test": "vitest run --exclude \"src/wrappers/**/*.test.ts\" --exclude \"src/otel/**/*.test.ts\" --exclude \"smoke/**/*.test.ts\" --exclude \"src/zod/**/*.test.ts\" --exclude \"tests/api-compatibility/**\"", + "test:core": "pnpm prune && pnpm test", + "test:checks": "pnpm run test:core && pnpm run test:vitest", + "test:external": "pnpm run test:external:openai && pnpm run test:external:anthropic && pnpm run test:external:google-genai && pnpm run test:external:ai-sdk && pnpm run test:external:claude-agent-sdk", + "test:external:openai": "bash scripts/test-provider.sh test:openai openai", + "test:external:anthropic": "bash scripts/test-provider.sh test:anthropic @anthropic-ai/sdk", + "test:external:google-genai": "bash scripts/test-provider.sh test:google-genai @google/genai", + "test:external:ai-sdk": "pnpm run test:external:ai-sdk-v5 && pnpm run test:external:ai-sdk-v6", + "test:external:ai-sdk-v5": "cd src/wrappers/ai-sdk/tests/v5 && pnpm install --ignore-workspace && pnpm test", + "test:external:ai-sdk-v6": "cd src/wrappers/ai-sdk/tests/v6 && pnpm install --ignore-workspace && pnpm test", + "test:external:claude-agent-sdk": "cd src/wrappers/claude-agent-sdk && pnpm install && pnpm test", + "test:all": "pnpm run test:checks && pnpm run test:external", "test:api-compat": "vitest run tests/api-compatibility/api-compatibility.test.ts", "test:anthropic": "vitest run src/wrappers/anthropic.test.ts", "test:openai": "vitest run src/wrappers/oai.test.ts", @@ -127,6 +138,8 @@ "test:claude-agent-sdk": "pnpm --filter @braintrust/claude-agent-sdk-tests test", "test:vitest": "pnpm --filter @braintrust/vitest-wrapper-tests test", "test:output": "tsx scripts/test-output.ts --with-comparison --with-metrics --with-progress", + "bench": "npx tsx src/queue.bench.ts", + "publish:validate": "./scripts/validate-release.sh && npm install && npm run build && npm publish", "lint": "eslint .", "fix:lint": "eslint --fix .", "playground": "tsx playground.ts", diff --git a/js/scripts/test-provider.sh b/js/scripts/test-provider.sh new file mode 100755 index 000000000..ba2876d38 --- /dev/null +++ b/js/scripts/test-provider.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# +# Run a provider wrapper test, optionally installing a specific package version first. +# +# Usage: +# ./scripts/test-provider.sh [package@version] +# +# Examples: +# ./scripts/test-provider.sh test:openai # uses whatever is installed +# ./scripts/test-provider.sh test:openai openai # installs latest openai +# ./scripts/test-provider.sh test:openai openai@4.92.1 # installs openai@4.92.1 +# ./scripts/test-provider.sh test:anthropic @anthropic-ai/sdk@0.39.0 +# +set -euo pipefail + +SCRIPT="${1:?Usage: test-provider.sh [package@version]}" +PACKAGE="${2:-}" + +pnpm prune + +if [ -n "$PACKAGE" ]; then + echo "Installing $PACKAGE (no-save)..." + npm_config_save=false npm_config_lockfile=false pnpm add "$PACKAGE" +fi + +pnpm run "$SCRIPT"