feat: migrate test infrastructure to xnet

.env.docker

@@ -1,4 +1,4 @@
 DB_CONNECTION_STRING="postgres://postgres:xmtp@db:5432/postgres?sslmode=disable"
-XMTP_GRPC_ADDRESS="node:5556"
+XMTP_GRPC_ADDRESS="xnet-100:5050"
 LOG_ENCODING=console
-API_PORT="8080"
+API_PORT="8080"

.env.local

@@ -1,4 +1,4 @@
 DB_CONNECTION_STRING="postgres://postgres:xmtp@localhost:25432/postgres?sslmode=disable"
-XMTP_GRPC_ADDRESS="localhost:25556"
+XMTP_GRPC_ADDRESS="localhost:5556"
 LOG_ENCODING=console
 API_PORT="8080"

.github/actions/setup-xnet/action.yml

@@ -0,0 +1,38 @@
+name: "Setup xnet"
+description: "Install Nix and xnet-cli for XMTP network testing"
+
+inputs:
+  github-token:
+    description: "GitHub token for Nix installation"
+    required: true
+  cachix-auth-token:
+    description: "Cachix auth token for cache pushes"
+    required: false
+
+runs:
+  using: "composite"
+  steps:
+    - uses: cachix/install-nix-action@v31
+      with:
+        github_access_token: ${{ inputs.github-token }}
+        extra_nix_config: |
+          accept-flake-config = true
+          extra-trusted-public-keys = xmtp.cachix.org-1:nFPFrqLQ9kjYQKiWL7gKq6llcNEeaV4iI+Ka1F+Tmq0=
+          extra-substituters = https://xmtp.cachix.org
+    - uses: cachix/cachix-action@v16
+      with:
+        name: xmtp
+        authToken: ${{ inputs.cachix-auth-token }}
+    - name: Install xnet-cli
+      shell: bash
+      run: ./dev/install-xnet
+    - name: Setup DNS
+      shell: bash
+      run: |
+        sudo mkdir -p /etc/systemd/resolved.conf.d
+        sudo tee /etc/systemd/resolved.conf.d/xmtp.conf <<EOF
+        [Resolve]
+        DNS=127.0.0.1:5354
+        Domains=~xmtpd.local
+        EOF
+        sudo systemctl restart systemd-resolved

.github/workflows/test.yml

@@ -10,12 +10,26 @@ jobs:
       - uses: actions/setup-go@v3
         with:
           go-version-file: go.mod
+      - uses: ./.github/actions/setup-xnet
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
       - run: ./dev/up
       - name: Run Tests
         run: go test -p 1 ./...
   integration:
-    name: Integration
+    name: Integration (${{ matrix.message-source }})
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        message-source:
+          - v3-direct
+          - v4-with-migrator
+          # - v4-direct
     steps:
       - uses: actions/checkout@v3
-      - run: ./dev/integration
+      - uses: ./.github/actions/setup-xnet
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
+      - run: ./dev/integration ${{ matrix.message-source }}

Dockerfile

@@ -27,7 +27,7 @@ LABEL description="XMTP Example Notification Server"
 ENV GOLOG_LOG_FMT=nocolor
 
 # go-waku default port
-EXPOSE 5556
+EXPOSE 8080
 
 COPY --from=builder /app/bin/notifications-server /usr/bin/
 

README.md

@@ -14,6 +14,7 @@ Many applications will have different needs for push notifications (different de
 
 1. Go 1.26
 2. Docker and Docker Compose
+3. Nix
 
 ## Local Setup
 
@@ -23,6 +24,8 @@ To start the XMTP service and database, run:
 ./dev/up
 ```
 
+If `xnet-cli` is not already installed, `./dev/up` will try to install it with Nix.
+
 You should then be able to build the server using:
 
 ```sh

cmd/server/main.go

@@ -104,6 +104,9 @@ func main() {
 
 	if opts.Api.Enabled {
 		apiServer = api.NewApiServer(logger, opts.Api, installationsService, subscriptionsService, interfaces.ListenerType(opts.Xmtp.ListenerType))
+		if notifListener != nil {
+			apiServer.SetReadyCheck(notifListener.Ready)
+		}
 		apiServer.Start()
 	}
 

dev/down

@@ -1,4 +1,11 @@
 #!/bin/bash
 set -eou pipefail
 
-docker compose down
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+xnet() { "$SCRIPT_DIR/xnet" "$@"; }
+
+docker compose down -v
+
+if docker network inspect xnet >/dev/null 2>&1; then
+  xnet delete
+fi

dev/install-xnet

@@ -0,0 +1,52 @@
+#!/bin/bash
+set -euo pipefail
+
+XNET_COMMIT="001ce0a76a8727319e3209b1bd509792388d67d2"
+XNET_FLAKE="github:xmtp/libxmtp/${XNET_COMMIT}"
+XNET_REF="${XNET_FLAKE}#xnet-cli"
+XNET_LOCKED_PREFIX="${XNET_FLAKE}"
+
+have_command() { command -v "$1" >/dev/null 2>&1; }
+
+ensure_nix_profile_path() {
+  local nix_path
+
+  for nix_path in \
+    "$HOME/.nix-profile/bin" \
+    "$HOME/.local/state/nix/profile/bin" \
+    "/nix/var/nix/profiles/default/bin"
+  do
+    if [[ -d "$nix_path" ]] && [[ ":$PATH:" != *":$nix_path:"* ]]; then
+      PATH="$nix_path:$PATH"
+    fi
+  done
+}
+
+ensure_nix_profile_path
+
+if ! have_command nix; then
+  echo "Nix is required to install xnet-cli." >&2
+  exit 1
+fi
+
+if nix profile list --json --no-pretty | jq -e --arg locked_prefix "$XNET_LOCKED_PREFIX" '
+  .elements
+  | [to_entries[]
+      | select(
+          .value.active
+          and ((.value.attrPath // "") | endswith(".xnet-cli"))
+        )
+    ] as $xnet_entries
+  | ($xnet_entries | length) > 0
+    and all(
+      $xnet_entries[];
+      ((.value.url // "") | startswith($locked_prefix))
+    )
+' >/dev/null; then
+  echo "xnet-cli already matches pinned commit $XNET_COMMIT"
+  exit 0
+fi
+
+echo "Installing xnet-cli from $XNET_REF"
+nix profile remove xnet-cli >/dev/null 2>&1 || true
+nix --accept-flake-config profile add "$XNET_REF"

dev/integration

@@ -1,6 +1,47 @@
 #!/bin/bash
 set -eou pipefail
 
-docker compose up -d node
-docker compose up integration
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+
+# message-source controls both how the XMTP client writes messages and what
+# listener topology the notification server is expected to exercise:
+# - v3-direct:
+#   client writes directly to node-go and the notification server listens to the
+#   same v3 node with the v3 listener.
+# - v4-with-migrator:
+#   client writes to node-go, the migrator forwards those messages into xmtpd,
+#   and the notification server listens to xmtpd with the v4 listener.
+# - v4-direct:
+#   client writes to xmtpd after cutover + d14n activation, and the
+#   notification server also listens directly to xmtpd with the v4 listener.
+MESSAGE_SOURCE="${1:-v3-direct}"
+source "$SCRIPT_DIR/xnet-env" "$MESSAGE_SOURCE"
+
+"$SCRIPT_DIR/down"
+"$SCRIPT_DIR/up" "$MESSAGE_SOURCE"
+
+echo "message-source=$MESSAGE_SOURCE listener_type=$LISTENER_TYPE listener_address=$XMTP_LISTENER_ADDRESS migrator_running=$XMTP_MIGRATOR_RUNNING xmtp_client_url=$XMTP_CLIENT_URL xmtp_gateway_url=${XMTP_GATEWAY_URL:-<unset>}"
+
+docker compose build -q notification_server integration
+
+# Get CoreDNS container IP for DNS resolution inside containers
+COREDNS_IP=$(docker inspect xnet-coredns -f '{{.NetworkSettings.Networks.xnet.IPAddress}}')
+export COREDNS_IP
+
+docker compose up -d notification_server
+notification_server_container_id="$(docker compose ps -q notification_server)"
+for _ in $(seq 1 30); do
+  if [ "$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{end}}' "$notification_server_container_id")" = "healthy" ]; then
+    break
+  fi
+  sleep 1
+done
+
+if [ "$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{end}}' "$notification_server_container_id")" != "healthy" ]; then
+  echo "notification_server did not become healthy for message-source $MESSAGE_SOURCE" >&2
+  docker compose logs notification_server >&2 || true
+  exit 1
+fi
+
+docker compose run --rm --use-aliases integration npx vitest run
 exit $?

dev/up

@@ -1,4 +1,89 @@
 #!/bin/bash
 set -eou pipefail
 
-docker compose up -d node
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+ensure_nix_profile_path() {
+  local nix_path
+
+  for nix_path in \
+    "$HOME/.nix-profile/bin" \
+    "$HOME/.local/state/nix/profile/bin" \
+    "/nix/var/nix/profiles/default/bin"
+  do
+    if [[ -d "$nix_path" ]] && [[ ":$PATH:" != *":$nix_path:"* ]]; then
+      PATH="$nix_path:$PATH"
+    fi
+  done
+}
+
+ensure_xnet_cli() {
+  "$SCRIPT_DIR/install-xnet"
+  ensure_nix_profile_path
+
+  if ! command -v xnet-cli >/dev/null 2>&1; then
+    echo "xnet-cli is required. Install Nix and rerun ./dev/up, or install xnet-cli manually." >&2
+    exit 1
+  fi
+}
+
+xnet() { "$SCRIPT_DIR/xnet" "$@"; }
+is_node_running() { docker inspect xnet-100 --format '{{.State.Running}}' 2>/dev/null | grep -q '^true$'; }
+is_node_registered() { xnet addresses 2>&1 | grep -q "xnet-100"; }
+is_v3_node_running() { docker inspect xnet-node --format '{{.State.Running}}' 2>/dev/null | grep -q '^true$'; }
+
+MESSAGE_SOURCE="${1:-v3-direct}"
+source "$SCRIPT_DIR/xnet-env" "$MESSAGE_SOURCE"
+
+ensure_xnet_cli
+xnet up --paused
+
+if [[ "$XMTP_CLIENT_URL" == "http://xnet-node:5556" ]]; then
+  for _ in $(seq 1 30); do
+    if is_v3_node_running; then
+      break
+    fi
+    sleep 1
+  done
+
+  if ! is_v3_node_running; then
+    echo "xnet-node did not become ready for message-source $MESSAGE_SOURCE" >&2
+    exit 1
+  fi
+fi
+
+if [[ "$XMTP_NEEDS_XMTPD_NODE" == "true" ]]; then
+  xnet node add --migrator
+
+  if [[ "$XMTP_SHOULD_ENABLE_D14N" == "true" ]]; then
+    xnet migrate
+    xnet activate-d14n
+  fi
+
+  for _ in $(seq 1 30); do
+    if is_node_running; then
+      break
+    fi
+    sleep 1
+  done
+
+  if ! is_node_running; then
+    echo "xnet-100 did not become ready for message-source $MESSAGE_SOURCE" >&2
+    exit 1
+  fi
+
+  if [[ "$XMTP_SHOULD_ENABLE_D14N" == "true" ]]; then
+    for _ in $(seq 1 90); do
+      if is_node_registered; then
+        break
+      fi
+      sleep 1
+    done
+
+    if ! is_node_registered; then
+      echo "xnet-100 did not become registered for message-source $MESSAGE_SOURCE" >&2
+      exit 1
+    fi
+  fi
+fi
+
+docker compose up -d db

dev/xnet

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -eou pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+xnet-cli -c "$SCRIPT_DIR/../xnet.toml" "$@"

dev/xnet-env

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+MESSAGE_SOURCE="${1:-v3-direct}"
+
+case "$MESSAGE_SOURCE" in
+  v3-direct)
+    export MESSAGE_SOURCE
+    export LISTENER_TYPE="v3"
+    export XMTP_LISTENER_ADDRESS="xnet-node:5556"
+    export XMTP_CLIENT_URL="http://xnet-node:5556"
+    export XMTP_GATEWAY_URL=""
+    export XMTP_NEEDS_XMTPD_NODE="false"
+    export XMTP_SHOULD_ENABLE_D14N="false"
+    export XMTP_MIGRATOR_RUNNING="false"
+    ;;
+  v4-with-migrator)
+    export MESSAGE_SOURCE
+    export LISTENER_TYPE="v4"
+    export XMTP_LISTENER_ADDRESS="xnet-100:5050"
+    export XMTP_CLIENT_URL="http://xnet-node:5556"
+    export XMTP_GATEWAY_URL=""
+    export XMTP_NEEDS_XMTPD_NODE="true"
+    export XMTP_SHOULD_ENABLE_D14N="false"
+    export XMTP_MIGRATOR_RUNNING="true"
+    ;;
+  v4-direct)
+    export MESSAGE_SOURCE
+    export LISTENER_TYPE="v4"
+    export XMTP_LISTENER_ADDRESS="xnet-100:5050"
+    export XMTP_CLIENT_URL="http://xnet-100:5050"
+    export XMTP_GATEWAY_URL="http://xnet-gateway:5050"
+    export XMTP_NEEDS_XMTPD_NODE="true"
+    export XMTP_SHOULD_ENABLE_D14N="true"
+    export XMTP_MIGRATOR_RUNNING="false"
+    ;;
+  *)
+    echo "Unknown message-source: $MESSAGE_SOURCE (use v3-direct, v4-with-migrator, or v4-direct)" >&2
+    return 1 2>/dev/null || exit 1
+    ;;
+esac