From beaf56ade32595d93cf91f4837190857f5993b04 Mon Sep 17 00:00:00 2001 From: F-OBrien Date: Mon, 31 Aug 2026 21:06:11 +0100 Subject: [PATCH 1/4] fix: disable Blockscout's internal transaction fetcher pallet-revive's eth-rpc surfaces native Substrate extrinsics as pseudo-Ethereum transactions: SCALE-encoded input, a `modl*` pallet account as `to`, and zero gas. They carry no EVM call frames, so debug_traceTransaction answers "No Ethereum extrinsic found" and the internal transaction fetcher retries those blocks forever, flooding the logs. Nothing is lost by turning it off, since debug_traceBlockByNumber returns [] regardless. Exposed as POLYMESH_BLOCKSCOUT_DISABLE_INTERNAL_TX_FETCHER (default true) so it can be re-enabled if a future eth-rpc image supports tracing. --- README.md | 2 ++ compose.yaml | 4 ++++ envs/8.0 | 3 +++ envs/latest | 3 +++ envs/local | 3 +++ envs/template | 3 +++ 6 files changed, 18 insertions(+) diff --git a/README.md b/README.md index 3e21dc3..1748e68 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,8 @@ Notes: - RPC/subscription coverage can vary by image tag. Keep `POLYMESH_ETH_RPC_IMAGE` configurable in env files and pin known-good tags. - Some paritypr image tags can be architecture-specific. Override `POLYMESH_ETH_RPC_PLATFORM` if needed. +- `eth-rpc` surfaces some native Substrate extrinsics as pseudo-Ethereum transactions: SCALE-encoded `input`, a `modl*` pallet account as `to`, and zero gas. They are visible in Blockscout but cannot be traced — `debug_traceTransaction` returns `No Ethereum extrinsic found`. Blockscout's internal transaction fetcher is therefore disabled by default (`POLYMESH_BLOCKSCOUT_DISABLE_INTERNAL_TX_FETCHER`), since it would otherwise retry those blocks indefinitely and flood the logs. +- Receipts from `eth-rpc` report the Substrate extrinsic index as `transactionIndex`, so it can disagree with the position of the hash in the block's `transactions` array. ## Additional Notes diff --git a/compose.yaml b/compose.yaml index 1b011ec..4ac92f8 100644 --- a/compose.yaml +++ b/compose.yaml @@ -323,6 +323,10 @@ services: NETWORK: ${POLYMESH_BLOCKSCOUT_NETWORK:-Polymesh} SUBNETWORK: ${POLYMESH_BLOCKSCOUT_SUBNETWORK:-Polymesh Dev} INDEXER_DISABLE_PENDING_TRANSACTIONS_FETCHER: ${POLYMESH_BLOCKSCOUT_DISABLE_PENDING_TX_FETCHER:-true} + # Native Substrate extrinsics appear as pseudo-Ethereum transactions with + # no EVM call frames, so tracing them fails and the fetcher retries those + # blocks forever. Nothing is lost: their traces are empty either way. + INDEXER_DISABLE_INTERNAL_TRANSACTIONS_FETCHER: ${POLYMESH_BLOCKSCOUT_DISABLE_INTERNAL_TX_FETCHER:-true} DISABLE_MARKET: 'true' ACCOUNT_ENABLED: 'false' DISABLE_WEBAPP: 'false' diff --git a/envs/8.0 b/envs/8.0 index 1175003..5b5707b 100644 --- a/envs/8.0 +++ b/envs/8.0 @@ -34,6 +34,9 @@ POLYMESH_SUBQUERY_GRAPHQL_PORT=3000 # POLYMESH_BLOCKSCOUT_REDIS_IMAGE=redis:alpine # POLYMESH_BLOCKSCOUT_RPC_VARIANT=geth # POLYMESH_BLOCKSCOUT_DISABLE_PENDING_TX_FETCHER=true +# Native Substrate extrinsics appear as pseudo-Ethereum transactions that +# cannot be traced, so the fetcher would retry those blocks forever. +# POLYMESH_BLOCKSCOUT_DISABLE_INTERNAL_TX_FETCHER=true # POLYMESH_BLOCKSCOUT_NETWORK=Polymesh # POLYMESH_BLOCKSCOUT_SUBNETWORK=Polymesh Dev # POLYMESH_BLOCKSCOUT_COIN=POLYX diff --git a/envs/latest b/envs/latest index 51f3155..1c24233 100644 --- a/envs/latest +++ b/envs/latest @@ -33,6 +33,9 @@ POLYMESH_SUBQUERY_GRAPHQL_PORT=3000 # POLYMESH_BLOCKSCOUT_REDIS_IMAGE=redis:alpine # POLYMESH_BLOCKSCOUT_RPC_VARIANT=geth # POLYMESH_BLOCKSCOUT_DISABLE_PENDING_TX_FETCHER=true +# Native Substrate extrinsics appear as pseudo-Ethereum transactions that +# cannot be traced, so the fetcher would retry those blocks forever. +# POLYMESH_BLOCKSCOUT_DISABLE_INTERNAL_TX_FETCHER=true # POLYMESH_BLOCKSCOUT_NETWORK=Polymesh # POLYMESH_BLOCKSCOUT_SUBNETWORK=Polymesh Dev # POLYMESH_BLOCKSCOUT_COIN=POLYX diff --git a/envs/local b/envs/local index 5b3f567..a04cb01 100644 --- a/envs/local +++ b/envs/local @@ -33,6 +33,9 @@ POLYMESH_SUBQUERY_GRAPHQL_PORT=3000 # POLYMESH_BLOCKSCOUT_REDIS_IMAGE=redis:alpine # POLYMESH_BLOCKSCOUT_RPC_VARIANT=geth # POLYMESH_BLOCKSCOUT_DISABLE_PENDING_TX_FETCHER=true +# Native Substrate extrinsics appear as pseudo-Ethereum transactions that +# cannot be traced, so the fetcher would retry those blocks forever. +# POLYMESH_BLOCKSCOUT_DISABLE_INTERNAL_TX_FETCHER=true # POLYMESH_BLOCKSCOUT_NETWORK=Polymesh # POLYMESH_BLOCKSCOUT_SUBNETWORK=Polymesh Dev # POLYMESH_BLOCKSCOUT_COIN=POLYX diff --git a/envs/template b/envs/template index 8cf7df7..e1cf42e 100644 --- a/envs/template +++ b/envs/template @@ -35,6 +35,9 @@ POLYMESH_SUBQUERY_GRAPHQL_PORT=3000 # POLYMESH_BLOCKSCOUT_REDIS_IMAGE=redis:alpine # POLYMESH_BLOCKSCOUT_RPC_VARIANT=geth # POLYMESH_BLOCKSCOUT_DISABLE_PENDING_TX_FETCHER=true +# Native Substrate extrinsics appear as pseudo-Ethereum transactions that +# cannot be traced, so the fetcher would retry those blocks forever. +# POLYMESH_BLOCKSCOUT_DISABLE_INTERNAL_TX_FETCHER=true # POLYMESH_BLOCKSCOUT_NETWORK=Polymesh # POLYMESH_BLOCKSCOUT_SUBNETWORK=Polymesh Dev # POLYMESH_BLOCKSCOUT_COIN=POLYX From 67c1289bdeca6af468bf40f6aa202874cdb296bc Mon Sep 17 00:00:00 2001 From: F-OBrien Date: Mon, 31 Aug 2026 21:08:53 +0100 Subject: [PATCH 2/4] fix: point the Blockscout frontend at a host-reachable RPC URL The frontend's Contract Read/Write tabs and its "Add to wallet" network config are driven by wagmi/viem in the browser, so the RPC endpoint has to resolve from the host rather than from inside the Docker network. Without NEXT_PUBLIC_NETWORK_RPC_URL those calls have no endpoint to use. Defaults to http://localhost: and is overridable via POLYMESH_BLOCKSCOUT_NETWORK_RPC_URL for setups that publish the eth-rpc port elsewhere or run the browser on another host. --- compose.yaml | 4 ++++ envs/8.0 | 3 +++ envs/latest | 3 +++ envs/local | 3 +++ envs/template | 4 ++++ 5 files changed, 17 insertions(+) diff --git a/compose.yaml b/compose.yaml index 4ac92f8..df9a3ac 100644 --- a/compose.yaml +++ b/compose.yaml @@ -369,6 +369,10 @@ services: # Required to enable in-browser wallet interaction (contract Write tab, # MetaMask, etc). Get a free project ID from https://cloud.reown.com NEXT_PUBLIC_WALLET_CONNECT_PROJECT_ID: ${POLYMESH_BLOCKSCOUT_WALLET_CONNECT_PROJECT_ID:-} + # RPC endpoint the browser (wagmi/viem) uses for Contract Read/Write + # calls and the "Add to wallet" network config. Must be reachable from + # the host, not the Docker network, so use the published eth-rpc port. + NEXT_PUBLIC_NETWORK_RPC_URL: ${POLYMESH_BLOCKSCOUT_NETWORK_RPC_URL:-http://localhost:${POLYMESH_ETH_RPC_PORT:-8545}} # The upstream frontend image enables third-party ad slots by default # (a banner on the home page and a text ad on token pages). They are # disabled here; set a provider name to re-enable. diff --git a/envs/8.0 b/envs/8.0 index 5b5707b..343c179 100644 --- a/envs/8.0 +++ b/envs/8.0 @@ -44,6 +44,9 @@ POLYMESH_SUBQUERY_GRAPHQL_PORT=3000 # Enables in-browser wallet interaction (contract Write tab, MetaMask, etc). # Get a free project ID from https://cloud.reown.com # POLYMESH_BLOCKSCOUT_WALLET_CONNECT_PROJECT_ID= +# RPC URL the browser uses for Contract Read/Write and "Add to wallet". +# Defaults to http://localhost: in compose.yaml. +# POLYMESH_BLOCKSCOUT_NETWORK_RPC_URL= # Third-party ad slots are disabled by default. Set a provider name # (e.g. slise, coinzilla) to re-enable them. # POLYMESH_BLOCKSCOUT_AD_BANNER_PROVIDER=none diff --git a/envs/latest b/envs/latest index 1c24233..0fbda20 100644 --- a/envs/latest +++ b/envs/latest @@ -43,6 +43,9 @@ POLYMESH_SUBQUERY_GRAPHQL_PORT=3000 # Enables in-browser wallet interaction (contract Write tab, MetaMask, etc). # Get a free project ID from https://cloud.reown.com # POLYMESH_BLOCKSCOUT_WALLET_CONNECT_PROJECT_ID= +# RPC URL the browser uses for Contract Read/Write and "Add to wallet". +# Defaults to http://localhost: in compose.yaml. +# POLYMESH_BLOCKSCOUT_NETWORK_RPC_URL= # Third-party ad slots are disabled by default. Set a provider name # (e.g. slise, coinzilla) to re-enable them. # POLYMESH_BLOCKSCOUT_AD_BANNER_PROVIDER=none diff --git a/envs/local b/envs/local index a04cb01..f28fda5 100644 --- a/envs/local +++ b/envs/local @@ -43,6 +43,9 @@ POLYMESH_SUBQUERY_GRAPHQL_PORT=3000 # Enables in-browser wallet interaction (contract Write tab, MetaMask, etc). # Get a free project ID from https://cloud.reown.com # POLYMESH_BLOCKSCOUT_WALLET_CONNECT_PROJECT_ID= +# RPC URL the browser uses for Contract Read/Write and "Add to wallet". +# Defaults to http://localhost: in compose.yaml. +# POLYMESH_BLOCKSCOUT_NETWORK_RPC_URL= # Third-party ad slots are disabled by default. Set a provider name # (e.g. slise, coinzilla) to re-enable them. # POLYMESH_BLOCKSCOUT_AD_BANNER_PROVIDER=none diff --git a/envs/template b/envs/template index e1cf42e..6526350 100644 --- a/envs/template +++ b/envs/template @@ -45,6 +45,10 @@ POLYMESH_SUBQUERY_GRAPHQL_PORT=3000 # Enables in-browser wallet interaction (contract Write tab, MetaMask, etc). # Get a free project ID from https://cloud.reown.com # POLYMESH_BLOCKSCOUT_WALLET_CONNECT_PROJECT_ID= +# RPC URL the browser uses for Contract Read/Write and "Add to wallet". +# Defaults to http://localhost:; override if the +# eth-rpc port is published elsewhere or the browser runs on another host. +# POLYMESH_BLOCKSCOUT_NETWORK_RPC_URL= # Optional frontend overrides # POLYMESH_BLOCKSCOUT_FRONTEND_API_HOST=localhost From af9489d22c6d7a6513546ab8595f906f4bb61654 Mon Sep 17 00:00:00 2001 From: F-OBrien Date: Tue, 1 Sep 2026 10:40:16 +0100 Subject: [PATCH 3/4] feat: put the REST API and Vault behind a `rest-api` compose profile Starting everything meant five containers for the REST APIs and Vault even when a workflow never signs through them, which is a real cost on a constrained machine. They are now opt-in via `--profile rest-api`, leaving the node, Postgres and the Subquery indexer as the default set. `environment-ready` waited on a marker written by the REST API account init, which no longer runs in the default set, so it would have hung forever. It now gates that wait on POLYMESH_WAIT_FOR_REST_API and otherwise reports ready once the core services are healthy, which its new depends_on guarantees. `start-env.sh` sets the variable to match the profiles it was given; running `docker compose` directly needs it set by hand. `--profile evm` no longer implies Vault and the REST APIs, so working on contracts no longer pays for them. `--profile rest-api,evm` restores the previous behaviour. `stop-env.sh` force-enables both optional profiles on teardown so nothing is left running, and the test scripts enable `rest-api` themselves since the suite signs with Vault. --- .github/workflows/run-tests.yml | 4 +-- README.md | 64 +++++++++++++++++++++++++++------ compose.yaml | 30 ++++++++++++++++ envs/template | 7 ++++ scripts/start-env.sh | 7 ++++ scripts/stop-env.sh | 16 +++++---- scripts/test.sh | 20 +++++++++-- tests/README.md | 13 +++---- tests/package.json | 5 ++- 9 files changed, 134 insertions(+), 32 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 262e629..30f31b0 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -36,8 +36,8 @@ jobs: - name: dump environment logs if: failure() run: | - docker compose --env-file envs/latest ps --all || true - docker compose --env-file envs/latest logs --no-color --timestamps || true + docker compose --env-file envs/latest --profile rest-api ps --all || true + docker compose --env-file envs/latest --profile rest-api logs --no-color --timestamps || true - name: stop environment if: always() run: yarn test:stop diff --git a/README.md b/README.md index 1748e68..4ca3a1d 100644 --- a/README.md +++ b/README.md @@ -10,24 +10,44 @@ A Docker Compose file and auxiliary scripts for running a [Polymesh](https://pol ## Running 1. **Configuration**: Copy an environment file (e.g., `envs/8.0`) to `.env` in the project root: `cp envs/8.0 .env`. This file specifies the Docker images to use. Alternatively, provide the path directly: `docker compose --env-file=envs/8.0 up`. -2. **Start**: Run `docker compose up -d` to start the services in detached mode. -3. **Stop**: Run `docker compose down` to stop the services. `docker compose down --volumes` will also remove associated volumes (chain data, database data, vault data), this will reset the environment. +2. **Start**: Run `docker compose up -d` to start the services in detached mode. This brings up the core services only; see [Runtime Modes](#runtime-modes) for the optional `rest-api` and `evm` profiles. +3. **Stop**: Run `docker compose down` to stop the services, repeating any `--profile` flags you started with — without them the profiled containers (Blockscout, eth-rpc, Vault, the REST APIs) are left running. `docker compose down --volumes` will also remove associated volumes (chain data, database data, vault data), this will reset the environment. 4. **Restart**: To apply changes (e.g., updated `.env` file), run `docker compose down && docker compose up -d`. +The `scripts/start-env.sh` and `scripts/stop-env.sh` helpers wrap these commands and manage the profile flags for you. + Full variable reference is available in `envs/template` (also linked as `.env.example`). -This setup will launch the following services: +This setup will launch the following services by default: - `polymesh-node`: A single Polymesh node running in development mode (`--dev`). - `postgres`: PostgreSQL database, primarily for the Subquery indexer. - `subquery-node`: Polymesh Subquery indexer service. - `subquery-graphql`: GraphQL API server for querying indexed data. +- `environment-ready`: Indicates when the core services and initial setup scripts have completed successfully. + +The REST API and Vault services are opt-in via the `rest-api` profile, since not +every workflow signs through them and they add five containers: + - `polymesh-rest-api-local-sm`: The Polymesh REST API service that uses local signers (Alice, Bob, Charlie, etc) - `polymesh-rest-api-vault-sm`: The Polymesh REST API service with HashiCorp Vault as the signing manager. - `polymesh-rest-api-vault-sm-init`: Creates test accounts and identities via the REST API. - `vault`: HashiCorp Vault for key management. - `vault-init`: Initializes and unseals Vault on first run, unseals on subsequent runs. -- `environment-ready`: Indicates when the core services and initial setup scripts have completed successfully. + +```bash +./scripts/start-env.sh --env-file envs/8.0 --profile rest-api +# or, driving compose directly +POLYMESH_WAIT_FOR_REST_API=true docker compose --env-file=envs/8.0 --profile rest-api up -d +``` + +`POLYMESH_WAIT_FOR_REST_API` tells `environment-ready` to wait for the REST API +account setup rather than reporting ready as soon as the core services are +healthy. `start-env.sh` sets it to match the profiles it was given, so it is +only needed when invoking `docker compose` yourself. + +The integration test suite needs the REST API and Vault, so `yarn test` and +`yarn test:start` enable the profile themselves. ## Checking Environment Status @@ -37,7 +57,7 @@ The environment involves several services starting up and performing initial set docker compose logs environment-ready ``` -Wait for the message indicating completion: +With the `rest-api` profile enabled, wait for the message indicating completion: ```text ************************************************************************************ @@ -47,6 +67,15 @@ Wait for the message indicating completion: If the environment was already initialized in a previous run, it will indicate readiness much faster. +Without that profile there is no account setup to wait for, so readiness is +reported as soon as the core services are healthy: + +```text +************************************************************************************ +*** Polymesh Environment Ready! (core services healthy) **************************** +************************************************************************************ +``` + ## Vault VS Local Signing Manager This environment supports usage of either the Local or Vault Signing Manager @@ -58,22 +87,33 @@ To facilitate this choice, the environment runs two independent instances of the ## Runtime Modes -This repository now has two runtime options: +Optional services are grouped into profiles, so a run only pays for what it +uses: -1. Default mode (no EVM tooling) -2. EVM tooling mode (`--profile evm`), which starts `polymesh-eth-rpc`, Blockscout backend/frontend, and dedicated Blockscout Postgres/Redis. +| Profile | Adds | Containers | +| ---------- | -------------------------------------------------------------------------- | ---------- | +| _(none)_ | node, Postgres, Subquery indexer + GraphQL, readiness check | 5 | +| `rest-api` | both REST API instances, their init job, Vault, Vault init | +5 | +| `evm` | `polymesh-eth-rpc`, Blockscout backend/frontend, Blockscout Postgres/Redis | +5 | + +Profiles combine: `--profile rest-api,evm` starts everything, which is what +earlier versions of this repository did on `--profile evm` alone. If you are +only exercising smart contracts, `--profile evm` on its own leaves out Vault and +the REST APIs entirely. Polymesh v8 includes Revive-based smart contract support, so optional EVM tooling follows Polkadot smart-contracts guidance: - [Get started with smart contracts](https://docs.polkadot.com/smart-contracts/get-started/) - [JSON-RPC APIs for Ethereum developers](https://docs.polkadot.com/smart-contracts/for-eth-devs/json-rpc-apis/) -### Option 1: Default mode (without EVM tooling) +### Option 1: Core mode (node and indexer only) ```bash ./scripts/start-env.sh --env-file envs/8.0 ``` +Add `--profile rest-api` if you need the REST API or Vault. + `--env-file` is optional; if omitted, both `start-env.sh` and `stop-env.sh` default to `envs/latest`. You can also override the default via the `COMPOSE_ENV` environment variable. @@ -103,8 +143,8 @@ for the next start, pass `--keep-volumes`: ./scripts/stop-env.sh --env-file envs/8.0 --keep-volumes ``` -`stop-env.sh` always tears down the EVM tooling as well, so you do not need to -repeat `--profile evm` when stopping. +`stop-env.sh` always tears down the profiled services as well, so you do not +need to repeat `--profile evm` or `--profile rest-api` when stopping. ### Option 2: With EVM tooling @@ -134,6 +174,7 @@ Explorer URLs: Notes: - EVM tooling remains opt-in and is not started by default. +- `--profile evm` no longer pulls in Vault and the REST APIs. Blockscout and `eth-rpc` talk to the node directly, so this is the leanest way to work on contracts. Add `--profile rest-api,evm` if you want both. - `--allow-unprotected-txs` is enabled for local experimentation with legacy transaction flows. - Blockscout uses dedicated Postgres and Redis services, separate from Subquery services. - If you customize ports, keep frontend `NEXT_PUBLIC_API_PORT` aligned with `POLYMESH_BLOCKSCOUT_BACKEND_PORT`. @@ -171,6 +212,7 @@ Notes: **Vault Automation:** +- Vault, the REST APIs and the account setup below only run with `--profile rest-api`. `scripts/get-vault-token.sh` and anything reading ports 3004/3005/8200 need that profile enabled. - The `vault-init` service automatically initializes Vault (on the very first run with the profile) and unseals it every time the services start. - It creates one unseal key and a root token, storing them in the `vault-root-token` named volume (accessible within the `vault-init` container at `/vault-token/` as `.unseal_key` and `.token`). - It also automatically enables the `transit` secrets engine and creates ED25519 keys named `admin`, `signer1`, `signer2`, `signer3`, and `signer4`. diff --git a/compose.yaml b/compose.yaml index df9a3ac..efe7a8b 100644 --- a/compose.yaml +++ b/compose.yaml @@ -131,6 +131,8 @@ services: # https://github.com/PolymeshAssociation/polymesh-rest-api#signing-managers polymesh-rest-api-vault-sm: image: ${POLYMESH_REST_API_IMAGE} + profiles: + - rest-api depends_on: polymesh-node: condition: service_healthy @@ -169,6 +171,8 @@ services: polymesh-rest-api-vault-sm-init: image: alpine:3.19 + profiles: + - rest-api depends_on: polymesh-rest-api-vault-sm: condition: service_healthy @@ -191,6 +195,8 @@ services: # https://github.com/PolymeshAssociation/polymesh-rest-api#signing-managers polymesh-rest-api-local-sm: image: ${POLYMESH_REST_API_IMAGE} + profiles: + - rest-api depends_on: polymesh-node: condition: service_healthy @@ -387,6 +393,8 @@ services: # non-root and its binary carries no cap_ipc_lock file capability, so mlock # fails wherever the host's memlock rlimit is finite. image: hashicorp/vault:${VAULT_VERSION:-latest} + profiles: + - rest-api networks: - polymesh restart: unless-stopped @@ -403,6 +411,8 @@ services: # This will make sure vault is initialized and unsealed on the first run and unsealed on subsequent runs vault-init: image: hashicorp/vault:${VAULT_VERSION:-latest} + profiles: + - rest-api user: '0:0' networks: - polymesh @@ -435,10 +445,30 @@ services: image: alpine:latest volumes: - rest-api-accounts-init:/status:ro + depends_on: + polymesh-node: + condition: service_healthy + subquery-graphql: + condition: service_healthy + environment: + # The REST API and Vault services sit behind the `rest-api` profile, so + # the account-init marker they write is only produced when that profile + # is enabled. Without it, readiness is just the core services being + # healthy, which `depends_on` above already guarantees. `start-env.sh` + # sets this whenever it enables the profile. + WAIT_FOR_REST_API: ${POLYMESH_WAIT_FOR_REST_API:-false} command: > sh -c ' start_time=$$(date +%s) + if [ "$$WAIT_FOR_REST_API" != "true" ]; then + echo "************************************************************************************" + echo "*** Polymesh Environment Ready! (core services healthy) ****************************" + echo "*** REST API and Vault are not running: start with --profile rest-api to add them **" + echo "************************************************************************************" + exit 0 + fi + if [ -f /status/.setup-complete ]; then echo "************************************************************************************" echo "*** Environment previously initialized and is ready! *******************************" diff --git a/envs/template b/envs/template index 6526350..ddd69b3 100644 --- a/envs/template +++ b/envs/template @@ -15,6 +15,13 @@ POLYMESH_SUBQUERY_IMAGE=polymeshassociation/polymesh-subquery:v19.6.0 POLYMESH_SUBQUERY_GRAPHQL_IMAGE=onfinality/subql-query:v2.25.0 POLYMESH_SUBQUERY_GRAPHQL_PORT=3000 +# The REST API and Vault services are opt-in (`--profile rest-api`). This makes +# `environment-ready` wait for their account setup instead of reporting ready +# once the core services are healthy; `start-env.sh` sets it to match the +# profiles it was given, so it is only needed when running docker compose +# directly. +# POLYMESH_WAIT_FOR_REST_API=false + # Optional EVM tooling (v8+ only, used with --profile evm) # POLYMESH_EVM_CHAIN_ID=1641818 # POLYMESH_ETH_RPC_IMAGE=paritypr/eth-rpc:stable2606-73b734d9 diff --git a/scripts/start-env.sh b/scripts/start-env.sh index d037055..0a88af4 100755 --- a/scripts/start-env.sh +++ b/scripts/start-env.sh @@ -59,6 +59,13 @@ if [[ -n "$COMPOSE_PROFILES" ]]; then done fi +# `environment-ready` waits on a marker written by the REST API account init, +# which only runs with the `rest-api` profile. Tell it which of the two it is. +case ",$COMPOSE_PROFILES," in + *,rest-api,*) export POLYMESH_WAIT_FOR_REST_API=true ;; + *) export POLYMESH_WAIT_FOR_REST_API=false ;; +esac + UP_ARGS=(--detach) if [[ -n "$COMPOSE_PULL_POLICY" ]]; then UP_ARGS+=(--pull "$COMPOSE_PULL_POLICY") diff --git a/scripts/stop-env.sh b/scripts/stop-env.sh index cc135ab..aa5d5f6 100755 --- a/scripts/stop-env.sh +++ b/scripts/stop-env.sh @@ -35,13 +35,15 @@ while [[ $# -gt 0 ]]; do esac done -# Always tear down with the `evm` profile enabled so profile-gated services -# (eth-rpc, Blockscout, ...) are removed even when the caller omits --profile. -# Without this, `down` leaves those containers running. -case ",$COMPOSE_PROFILES," in - *,evm,*) ;; - *) COMPOSE_PROFILES="${COMPOSE_PROFILES:+$COMPOSE_PROFILES,}evm" ;; -esac +# Always tear down with the optional profiles enabled so profile-gated services +# (eth-rpc, Blockscout, Vault, the REST APIs, ...) are removed even when the +# caller omits --profile. Without this, `down` leaves those containers running. +for required_profile in evm rest-api; do + case ",$COMPOSE_PROFILES," in + *",$required_profile,"*) ;; + *) COMPOSE_PROFILES="${COMPOSE_PROFILES:+$COMPOSE_PROFILES,}$required_profile" ;; + esac +done if [[ "${COMPOSE_ENV}" != /* ]]; then COMPOSE_ENV="${SCRIPT_DIR}/../${COMPOSE_ENV}" diff --git a/scripts/test.sh b/scripts/test.sh index ca7fdaf..2309177 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -6,14 +6,19 @@ set -e # Get the directory where this script is located, regardless of where it's called from SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -START_STOP_ARGS=() +ENV_FILE_ARGS=() +PROFILES="" # Args that only apply to start-env.sh (stop-env.sh would reject them). START_ONLY_ARGS=() while [[ $# -gt 0 ]]; do case "$1" in - --env-file|--profile) - START_STOP_ARGS+=("$1" "$2") + --env-file) + ENV_FILE_ARGS=("$1" "$2") + shift 2 + ;; + --profile) + PROFILES="$2" shift 2 ;; --pull) @@ -36,6 +41,15 @@ while [[ $# -gt 0 ]]; do esac done +# The suite signs with Vault and drives the REST API, both of which live behind +# the `rest-api` profile. +case ",$PROFILES," in + *,rest-api,*) ;; + *) PROFILES="${PROFILES:+$PROFILES,}rest-api" ;; +esac + +START_STOP_ARGS=("${ENV_FILE_ARGS[@]}" --profile "$PROFILES") + echo "[ENV TEST] Starting environment..." "${SCRIPT_DIR}/start-env.sh" "${START_STOP_ARGS[@]}" "${START_ONLY_ARGS[@]}" diff --git a/tests/README.md b/tests/README.md index 8102bbb..3b02c58 100644 --- a/tests/README.md +++ b/tests/README.md @@ -12,15 +12,16 @@ yarn ## Runtime options -Two environment modes are supported: +Optional services are grouped into compose profiles: -1. Default mode (without EVM tooling) -2. EVM tooling mode (`--profile evm`), which includes eth-rpc and Blockscout api and explorer +- `rest-api` adds both REST API instances and Vault. The suite signs with Vault + and drives the REST API, so the test scripts enable this profile themselves. +- `evm` adds eth-rpc and the Blockscout API and explorer. ## Default mode workflow ```sh -yarn test:start # starts environment with default profile set +yarn test:start # starts environment with --profile rest-api yarn test:run # runs integration tests yarn test:stop # stops and removes environment ``` @@ -34,10 +35,10 @@ yarn test ## EVM tooling workflow ```sh -yarn test:start:evm # starts environment with --profile evm +yarn test:start:evm # starts environment with --profile rest-api,evm yarn test:evm:smoke # EVM RPC + Blockscout API smoke checks yarn test:run # optional: run integration tests while env is up -yarn test:stop:evm # stops and removes env started with --profile evm +yarn test:stop # stops the environment, whichever profiles it used ``` ## Notes diff --git a/tests/package.json b/tests/package.json index 4ce8bed..2c7fafb 100644 --- a/tests/package.json +++ b/tests/package.json @@ -5,11 +5,10 @@ "main": "index.js", "scripts": { "test": "../scripts/test.sh", - "test:start": "../scripts/start-env.sh", - "test:start:evm": "../scripts/start-env.sh --profile evm", + "test:start": "../scripts/start-env.sh --profile rest-api", + "test:start:evm": "../scripts/start-env.sh --profile rest-api,evm", "test:run": "../scripts/run-tests.sh", "test:stop": "../scripts/stop-env.sh", - "test:stop:evm": "../scripts/stop-env.sh --profile evm", "test:vault-token": "../scripts/get-vault-token.sh", "test:evm:smoke": "../scripts/evm-smoke-test.sh && ../scripts/blockscout-smoke-test.sh", "lint": "eslint src --ext .js,.jsx,.ts,.tsx", From 366d0102e4f789dda97ff899e69e12bac26d047f Mon Sep 17 00:00:00 2001 From: F-OBrien Date: Tue, 1 Sep 2026 11:54:52 +0100 Subject: [PATCH 4/4] feat: keep named volumes on teardown by default `stop-env.sh` removed chain data, Vault keys and the Blockscout database on every stop, so resuming from existing state meant remembering `--keep-volumes` and forgetting it meant re-syncing from genesis. Losing that data by accident is a lot more expensive than typing a flag when you actually want a clean slate, so the default is inverted: `--volumes` now opts into removal. `--keep-volumes` is still accepted as a no-op so existing invocations keep working. The full `yarn test` cycle owns the environment it creates and leaves identities and assets on chain that a rerun would trip over, so it now passes `--volumes` explicitly, as does CI via a new `test:stop:clean`. --- .github/workflows/run-tests.yml | 2 +- README.md | 11 +++++++---- scripts/stop-env.sh | 20 ++++++++++++-------- scripts/test.sh | 5 ++++- tests/README.md | 6 +++++- tests/package.json | 1 + 6 files changed, 30 insertions(+), 15 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 30f31b0..57f15ec 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -40,5 +40,5 @@ jobs: docker compose --env-file envs/latest --profile rest-api logs --no-color --timestamps || true - name: stop environment if: always() - run: yarn test:stop + run: yarn test:stop:clean working-directory: tests diff --git a/README.md b/README.md index 4ca3a1d..64d187b 100644 --- a/README.md +++ b/README.md @@ -135,14 +135,17 @@ file uses floating tags such as `latest`), add `--pull always`: `build`); a bare `--pull` is shorthand for `always`. The same behaviour can be set via the `COMPOSE_PULL_POLICY` environment variable. -By default `stop-env.sh` removes the named volumes (chain data, Vault keys, -Blockscout DB, ...) for a clean slate. To stop the containers but keep the data -for the next start, pass `--keep-volumes`: +`stop-env.sh` keeps the named volumes (chain data, Vault keys, Blockscout DB, +...) so the next start resumes from the existing state. For a clean slate, pass +`--volumes` (or `-v`): ```bash -./scripts/stop-env.sh --env-file envs/8.0 --keep-volumes +./scripts/stop-env.sh --env-file envs/8.0 --volumes ``` +`--keep-volumes` is still accepted, as a no-op, so older invocations keep +working. + `stop-env.sh` always tears down the profiled services as well, so you do not need to repeat `--profile evm` or `--profile rest-api` when stopping. diff --git a/scripts/stop-env.sh b/scripts/stop-env.sh index aa5d5f6..154600b 100755 --- a/scripts/stop-env.sh +++ b/scripts/stop-env.sh @@ -9,10 +9,10 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" COMPOSE_ENV_DEFAULT="${SCRIPT_DIR}/../envs/latest" COMPOSE_ENV="${COMPOSE_ENV:-$COMPOSE_ENV_DEFAULT}" COMPOSE_PROFILES="${COMPOSE_PROFILES:-}" -# By default the teardown removes named volumes (chain data, Vault keys, -# Blockscout DB, ...). Pass --keep-volumes to stop the containers but retain -# the data so the next start resumes from the existing state. -KEEP_VOLUMES=false +# The teardown keeps named volumes (chain data, Vault keys, Blockscout DB, ...) +# so the next start resumes from the existing state. Pass --volumes for a clean +# slate. +REMOVE_VOLUMES=false while [[ $# -gt 0 ]]; do case "$1" in @@ -24,8 +24,12 @@ while [[ $# -gt 0 ]]; do COMPOSE_PROFILES="$2" shift 2 ;; + --volumes|-v) + REMOVE_VOLUMES=true + shift + ;; --keep-volumes) - KEEP_VOLUMES=true + # Now the default; still accepted so existing invocations work. shift ;; *) @@ -64,11 +68,11 @@ if [[ -n "$COMPOSE_PROFILES" ]]; then fi DOWN_ARGS=(down) -if [[ "$KEEP_VOLUMES" == true ]]; then - echo "[STOP ENV] Stopping the docker environment (named volumes preserved)..." -else +if [[ "$REMOVE_VOLUMES" == true ]]; then DOWN_ARGS+=(--volumes) echo "[STOP ENV] Cleaning up the docker environment (removing named volumes)..." +else + echo "[STOP ENV] Stopping the docker environment (named volumes preserved)..." fi docker compose "${COMPOSE_ARGS[@]}" "${DOWN_ARGS[@]}" diff --git a/scripts/test.sh b/scripts/test.sh index 2309177..7ae8c48 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -56,5 +56,8 @@ echo "[ENV TEST] Starting environment..." echo "[ENV TEST] Running tests..." "${SCRIPT_DIR}/run-tests.sh" +# A full test cycle owns the environment it created, and the suite leaves +# identities and assets on chain that a rerun would trip over, so this teardown +# removes the volumes even though `stop-env.sh` keeps them by default. echo "[ENV TEST] Cleaning up environment..." -"${SCRIPT_DIR}/stop-env.sh" "${START_STOP_ARGS[@]}" +"${SCRIPT_DIR}/stop-env.sh" "${START_STOP_ARGS[@]}" --volumes diff --git a/tests/README.md b/tests/README.md index 3b02c58..234acb7 100644 --- a/tests/README.md +++ b/tests/README.md @@ -23,7 +23,8 @@ Optional services are grouped into compose profiles: ```sh yarn test:start # starts environment with --profile rest-api yarn test:run # runs integration tests -yarn test:stop # stops and removes environment +yarn test:stop # stops environment, keeping volumes +yarn test:stop:clean # stops and also removes named volumes (clean slate) ``` Or run the full flow with one command: @@ -43,6 +44,9 @@ yarn test:stop # stops the environment, whichever profiles it used ## Notes +- `test:stop` leaves the named volumes in place so the next start resumes from + the existing state; `test:stop:clean` removes them. Calling `stop-env.sh` + directly, the flag is `--volumes` (or `-v`). - EVM tooling is only expected on chain v8+ presets. - The suite requires a **chain v8 preset** (`envs/8.0`, `envs/latest`). The Polymesh SDK dropped v7 support in v31 and throws on connecting to a v7 node, so `envs/7.2` can no longer be used here. diff --git a/tests/package.json b/tests/package.json index 2c7fafb..30c065f 100644 --- a/tests/package.json +++ b/tests/package.json @@ -9,6 +9,7 @@ "test:start:evm": "../scripts/start-env.sh --profile rest-api,evm", "test:run": "../scripts/run-tests.sh", "test:stop": "../scripts/stop-env.sh", + "test:stop:clean": "../scripts/stop-env.sh --volumes", "test:vault-token": "../scripts/get-vault-token.sh", "test:evm:smoke": "../scripts/evm-smoke-test.sh && ../scripts/blockscout-smoke-test.sh", "lint": "eslint src --ext .js,.jsx,.ts,.tsx",