From ea72ddd424a6ef0282fe740afa140611d49283ad Mon Sep 17 00:00:00 2001 From: Jim Schaff Date: Wed, 8 Jul 2026 16:42:12 -0400 Subject: [PATCH] Remove repo-local /loki-query command; superseded by global skill The /loki-query slash command and tools/loki/ scripts are replaced by a global 'loki-query' Claude skill (hosted in the Mantis repo, symlinked into ~/.claude/skills/), now generalized to query Loki for any app on the cluster (VCell, sms-api, compose-api, ...). All VCell-specific guidance (container tables, broker/supervisord archaeology, JMS-wedge workflow) is preserved in the skill's VCell section. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01QtHfpXHt1sXcDUUsXWYEpM --- .claude/commands/loki-query.md | 175 --------------------------------- tools/loki/loki-query.sh | 20 ---- tools/loki/setup.sh | 101 ------------------- tools/loki/teardown.sh | 17 ---- 4 files changed, 313 deletions(-) delete mode 100644 .claude/commands/loki-query.md delete mode 100755 tools/loki/loki-query.sh delete mode 100755 tools/loki/setup.sh delete mode 100755 tools/loki/teardown.sh diff --git a/.claude/commands/loki-query.md b/.claude/commands/loki-query.md deleted file mode 100644 index 1feaa5f1c3..0000000000 --- a/.claude/commands/loki-query.md +++ /dev/null @@ -1,175 +0,0 @@ ---- -description: Query VCell logs from Loki via logcli, across prod/stage/dev (port-forwarded automatically) ---- - -Query the VCell Loki instance to investigate incidents, monitor failures, or trace user-reported errors. Same Loki, same tenant — choose the namespace per query. - -The wrapper script handles installation, port-forwarding, and tenant headers. - -## When invoked - -The argument `$ARGUMENTS` is a free-form description of what to look for. Translate it into one or more `logcli query` invocations using the wrapper script: - -``` -bash tools/loki/loki-query.sh [logcli-args...] '' -``` - -The wrapper auto-runs `tools/loki/setup.sh` (idempotent — installs logcli if missing, starts a port-forward to `loki-read` if not already running, exports `LOKI_ADDR` and `LOKI_ORG_ID`). - -If the user hasn't given specific time bounds, default to `--since=15m`. For incidents, narrow the window to ±5–10 min around the reported timestamp using `--from=` and `--to=` (Better Stack timestamps are HDT = UTC-10). - -## Picking the namespace - -Always include a `namespace=` label in every query. The same Loki instance covers all three VCell environments: - -| `namespace=` | Site | URL | -| ------------ | ----------- | ---------------------------- | -| `prod` | Production | `vcell.cam.uchc.edu` | -| `stage` | Staging | `vcell-stage.cam.uchc.edu` | -| `dev` | Development | `vcell-dev.cam.uchc.edu` | - -If the user mentions a specific URL or "Better Stack incident", pick `prod` unless they say otherwise. If unsure, ask — don't guess across environments. - -## VCell containers (per namespace) - -Common to all three: - -| `container=` | Service | -| ------------ | ------------------------------------------------ | -| `api` | Legacy Restlet `/api/v0/` (HealthService, BMDB) | -| `data` | SimDataServer - Data RPC consumer | -| `db` | Database RPC consumer | -| `sched` | Simulation dispatcher | -| `submit` | Sim submit / ServerJobDispatcher | -| `webapp` | Angular webapp | -| `export` | Export server | - -Environment-specific: -- `prod` only: `rest` (Quarkus `/api/v1/`), `mongodb` (sidecar) -- `stage` only: `sif-prepull` (Apptainer SIF pre-build) - -When in doubt, discover with `logcli series` (after `eval "$(bash tools/loki/setup.sh --quiet)"`): -```bash -logcli series --quiet --since=1h '{namespace="dev"}' | grep -oE 'container="[^"]+"' | sort -u -``` - -## Useful queries - -```bash -# Recent ERRORs across all VCell pods in prod -bash tools/loki/loki-query.sh --since=1h \ - '{namespace="prod", container=~"api|data|db|sched|submit|rest"} |~ "ERROR"' - -# Same sweep, but in dev (note: no `rest` container in dev) -bash tools/loki/loki-query.sh --since=1h \ - '{namespace="dev", container=~"api|data|db|sched|submit"} |~ "ERROR"' - -# Around a specific incident, raw output for jq parsing -bash tools/loki/loki-query.sh --output=raw --limit=500 \ - --from="2026-05-05T14:15:00Z" --to="2026-05-05T14:25:00Z" \ - '{namespace="prod", container="data"} |~ "FailoverTransport|Exception"' - -# Compare same container across two environments side-by-side -bash tools/loki/loki-query.sh --since=15m --output=raw \ - '{namespace=~"prod|stage", container="data"} |~ "ERROR"' - -# Activity volume by minute (sanity that a container is even running) -bash tools/loki/loki-query.sh --output=raw --limit=20000 --since=1h \ - '{namespace="prod", container="data"}' \ - | jq -r '.["@timestamp"][:16] // empty' | sort | uniq -c -``` - -For raw-output queries, log lines are JSON; useful fields include `["@timestamp"]`, `log_level`, `["log.logger"]`, `["process.thread.name"]`, `message`. Pipe through `jq -r '...'` to extract a clean digest. - -## Direct kubectl logs (when Loki isn't enough) - -Use `kubectl logs` directly when: -- **A pod just crashed/restarted** — `--previous` shows the prior container's logs. Critical when a `JmsFailoverWatchdog` `System.exit` recycles a pod and you need to see why. -- **Broker pods** — `activemqint` (ActiveMQ Classic, legacy daemon-to-daemon traffic — counterpart to client-side JMS errors in `data`/`db`/`sched`/`submit`) and `activemqsim` (sim/solver traffic). These may not be in Loki's collection set; check Loki first and fall back here. -- **Real-time tail** of a single pod, or windows that have aged out of Loki retention. - -Setup (same kubeconfig as Loki): -```bash -KCFG="${LOKI_KUBECONFIG:-$HOME/.kube/kubeconfig_vxrails.yaml}" -NS=prod # or stage / dev -``` - -Discover the actual workload names — deployment vs statefulset, exact selectors: -```bash -kubectl --kubeconfig "$KCFG" -n "$NS" get deployments,statefulsets,pods \ - | grep -iE "data|activemq|api|db|sched|submit" -``` - -Common log patterns: -```bash -# Last 200 lines from the data pod (the one that wedged on 2026-05-06) -kubectl --kubeconfig "$KCFG" -n "$NS" logs deployment/data --tail=200 - -# Real-time tail -kubectl --kubeconfig "$KCFG" -n "$NS" logs -f deployment/data - -# Previous container instance — after a crash, OOM, or watchdog-driven exit -kubectl --kubeconfig "$KCFG" -n "$NS" logs deployment/data --previous --tail=500 - -# Time-bounded -kubectl --kubeconfig "$KCFG" -n "$NS" logs deployment/data --since=10m -kubectl --kubeconfig "$KCFG" -n "$NS" logs deployment/data --since-time="2026-05-06T06:30:00Z" - -``` - -### Broker pods are special — supervisord hides the broker log - -`activemqint`, `activemqsim`, and `artemismq` all run as `Deployments` in `prod` (and the same naming applies in `stage`/`dev` if present), but their container PID 1 is **supervisord**. The actual broker logs to a file inside the pod, not stdout. Consequences: - -- `kubectl logs deployment/activemqint` returns ~10 lines of supervisord lifecycle, frozen at pod startup (44d old in prod). It does **not** show broker activity. -- Loki's promtail isn't scraping these pods either (verified by `logcli series '{namespace="prod"}'` — no `activemq`/`artemis` containers appear). -- The only path to broker events is `kubectl exec` against the in-pod log file. - -```bash -# ActiveMQ Classic (activemqint, activemqsim) — log at /var/log/activemq/activemq.log -kubectl --kubeconfig "$KCFG" -n "$NS" exec deployment/activemqint -- \ - tail -200 /var/log/activemq/activemq.log - -# Around an incident window (in-pod awk filter — efficient on a multi-MB log) -kubectl --kubeconfig "$KCFG" -n "$NS" exec deployment/activemqint -- \ - bash -c 'awk "/2026-05-06 06:[34][0-9]/" /var/log/activemq/activemq.log' - -# Errors only -kubectl --kubeconfig "$KCFG" -n "$NS" exec deployment/activemqint -- \ - grep -E "WARN|ERROR" /var/log/activemq/activemq.log - -# Artemis (artemismq) — different layout; discover the log path -kubectl --kubeconfig "$KCFG" -n "$NS" exec deployment/artemismq -- \ - bash -c 'find / -name "*.log" -size +1k 2>/dev/null | head -10' -``` - -The in-pod ActiveMQ log file rotates after roughly 14h of activity. For older incidents, the broker side is not recoverable — a real operational gap when investigating wedges that took longer than that to manifest. - -If `deployment/` doesn't resolve (e.g. a broker is later deployed as a StatefulSet), discover the pod by label or by listing pods. - -**Worth fixing**: reconfigure these brokers to log to stdout (or have supervisord forward child stdout/stderr). Once stdout flows, both `kubectl logs` and Loki's promtail pick them up automatically — no more `exec` archaeology and no more 14h horizon. - -### Putting it together for a JMS-wedge incident - -Pull both sides around the same window: -- **Client side** (`data`/`db`/`sched`/`submit`) via Loki — the `IllegalStateException: Timer already cancelled.` WARNs surface here, but the *triggering* network event happened earlier and silently. -- **Broker side** (`activemqint`/`activemqsim`) via `kubectl exec` — `Transport Connection to: tcp://: failed: java.io.EOFException` lines tell you which client connection actually dropped, and at what timestamp. Compare to client-side reconnect attempts to confirm the wedge was already latent before the trigger. - -When to switch back to Loki: multi-pod sweeps, time-range queries spanning multiple containers, structured filters (`|~ "ERROR"`, regex), and historical incidents older than the kubelet's local log retention. - -## Workflow - -1. Read the user's request and identify: namespace (prod/stage/dev), time window, suspected container(s), keywords. -2. If the request is broad (e.g., "what's wrong with prod"), start with an ERROR-only sweep across all containers in that namespace, then drill in. -3. If the volume is large, save to `/tmp/.txt` with `--output=raw` and parse with `jq`. -4. Report the finding crisply: timestamp, namespace, thread/logger, key log lines. Cite the file:line in vcell source code if the stack frame is visible. -5. When done, leave the port-forward running (subsequent calls reuse it). If you want to stop it, run `bash tools/loki/teardown.sh`. - -## First-time setup hints (for the user) - -- Requires `kubectl` and a kubeconfig with cluster access. Default path is `~/.kube/kubeconfig_vxrails.yaml`; override via `LOKI_KUBECONFIG`. -- macOS-only auto-install of `logcli` via `brew install logcli`. On Linux, install manually before first use. -- Tenant is `uchc`; the script sets `X-Scope-OrgID` automatically. -- Bypasses the `loki-gateway` (whose ED25519 TLS cert LibreSSL/macOS curl can't handshake) by port-forwarding `loki-read` on plain HTTP. - -User request: $ARGUMENTS diff --git a/tools/loki/loki-query.sh b/tools/loki/loki-query.sh deleted file mode 100755 index ea9fd1c81c..0000000000 --- a/tools/loki/loki-query.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -# Query VCell prod Loki via logcli, after auto-starting the port-forward. -# -# Usage: -# bash tools/loki/loki-query.sh [logcli-args...] -# bash tools/loki/loki-query.sh --since=15m '{namespace="prod", container="data"} |~ "ERROR"' -# bash tools/loki/loki-query.sh --from="2026-05-05T14:15:00Z" --to="2026-05-05T14:25:00Z" \ -# --output=raw --limit=500 '{namespace="prod", container="api"} |~ "HealthService"' -# -# All arguments are passed verbatim to `logcli query`. - -set -euo pipefail - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -# Ensure the port-forward is up; capture its env exports. -EXPORTS="$(bash "$DIR/setup.sh" --quiet)" -eval "$EXPORTS" - -exec logcli query --quiet "$@" diff --git a/tools/loki/setup.sh b/tools/loki/setup.sh deleted file mode 100755 index 54ce342d8b..0000000000 --- a/tools/loki/setup.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env bash -# Configure logcli access to the VCell prod Loki instance. -# -# Idempotent: safe to run multiple times. Installs logcli if missing, -# starts a kubectl port-forward to loki-read in the background if one -# isn't already running on the chosen port, and prints the env vars -# required for logcli. -# -# Usage: -# bash tools/loki/setup.sh # start + print env (default port 3100) -# bash tools/loki/setup.sh --port 3101 -# eval "$(bash tools/loki/setup.sh --quiet)" # set env in current shell -# -# Required: -# - kubectl in PATH -# - kubeconfig with access to the prod cluster (default: $LOKI_KUBECONFIG -# or ~/.kube/kubeconfig_vxrails.yaml) -# - brew (for first-time logcli install on macOS) - -set -euo pipefail - -PORT=3100 -QUIET=0 -KUBECONFIG_PATH="${LOKI_KUBECONFIG:-$HOME/.kube/kubeconfig_vxrails.yaml}" -LOKI_NS="logging" -LOKI_SVC="loki-read" -LOKI_TENANT="uchc" -PF_LABEL="vcell-loki-pf" -PIDFILE="/tmp/${PF_LABEL}.pid" -LOGFILE="/tmp/${PF_LABEL}.log" - -while [[ $# -gt 0 ]]; do - case "$1" in - --port) PORT="$2"; shift 2 ;; - --quiet) QUIET=1; shift ;; - --kubeconfig) KUBECONFIG_PATH="$2"; shift 2 ;; - -h|--help) - grep '^#' "$0" | sed 's/^# \{0,1\}//' - exit 0 ;; - *) echo "unknown arg: $1" >&2; exit 2 ;; - esac -done - -log() { [[ $QUIET -eq 1 ]] || echo "[loki-setup] $*" >&2; } -die() { echo "[loki-setup] ERROR: $*" >&2; exit 1; } - -command -v kubectl >/dev/null || die "kubectl not found in PATH" -[[ -f "$KUBECONFIG_PATH" ]] || die "kubeconfig not found at $KUBECONFIG_PATH (set LOKI_KUBECONFIG)" - -# Install logcli on macOS via brew (no-op if already installed). -if ! command -v logcli >/dev/null; then - log "logcli not found" - if [[ "$(uname)" == "Darwin" ]] && command -v brew >/dev/null; then - log "installing logcli via brew..." - brew install logcli >/dev/null - else - die "install logcli manually: https://grafana.com/docs/loki/latest/query/logcli/" - fi -fi - -# Reuse existing port-forward if it's still alive on the same port. -if [[ -f "$PIDFILE" ]]; then - OLD_PID="$(cat "$PIDFILE")" - if kill -0 "$OLD_PID" 2>/dev/null \ - && lsof -iTCP:"$PORT" -sTCP:LISTEN -P 2>/dev/null | grep -q "[ ]$OLD_PID[ ]"; then - log "reusing existing port-forward (pid $OLD_PID, port $PORT)" - else - log "stale pidfile, cleaning up" - rm -f "$PIDFILE" - fi -fi - -# Start a new port-forward if needed. -if [[ ! -f "$PIDFILE" ]]; then - if lsof -iTCP:"$PORT" -sTCP:LISTEN -P >/dev/null 2>&1; then - die "port $PORT already in use by another process — pass --port " - fi - log "starting port-forward: svc/$LOKI_SVC -n $LOKI_NS :$PORT" - nohup kubectl --kubeconfig "$KUBECONFIG_PATH" \ - -n "$LOKI_NS" port-forward "svc/$LOKI_SVC" "$PORT:3100" \ - > "$LOGFILE" 2>&1 & - echo $! > "$PIDFILE" - # Wait for the local port to accept connections. - for _ in $(seq 1 20); do - if curl -fsS "http://localhost:$PORT/ready" >/dev/null 2>&1; then break; fi - sleep 0.5 - done - if ! curl -fsS "http://localhost:$PORT/ready" >/dev/null 2>&1; then - die "port-forward did not become ready — see $LOGFILE" - fi -fi - -# Verify Loki is reachable with the tenant header. -if ! curl -fsS -H "X-Scope-OrgID: $LOKI_TENANT" \ - "http://localhost:$PORT/loki/api/v1/labels" >/dev/null; then - die "Loki not responding for tenant '$LOKI_TENANT' — port-forward may be stale" -fi - -log "ready: tenant=$LOKI_TENANT, port=$PORT" -echo "export LOKI_ADDR=http://localhost:$PORT" -echo "export LOKI_ORG_ID=$LOKI_TENANT" diff --git a/tools/loki/teardown.sh b/tools/loki/teardown.sh deleted file mode 100755 index e3f1ceb61c..0000000000 --- a/tools/loki/teardown.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -# Stop the loki-read port-forward started by setup.sh. - -set -euo pipefail -PF_LABEL="vcell-loki-pf" -PIDFILE="/tmp/${PF_LABEL}.pid" - -if [[ -f "$PIDFILE" ]]; then - PID="$(cat "$PIDFILE")" - if kill -0 "$PID" 2>/dev/null; then - kill "$PID" - echo "[loki-teardown] killed port-forward pid $PID" >&2 - fi - rm -f "$PIDFILE" -else - echo "[loki-teardown] no port-forward pidfile — nothing to do" >&2 -fi