Skip to content

fix(scripts): make the Postgres smoke report the code under test, not the environment #149

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
eldonm merged 2 commits into from
Aug 8, 2026
Merged

fix(scripts): make the Postgres smoke report the code under test, not the environment #149

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
63e63f0
fix(scripts): make postgres smoke run env-correct and concurrency-safe
Aug 8, 2026
77fb5d7
fix(scripts): anchor the smoke app root to the repo, and reject scaff…
Aug 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 90 additions & 29 deletions scripts/smoke_postgres.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,27 +21,57 @@
# (`pip install asyncpg`). Agent-turn checks need a model key in the app's
# .env; without one they are skipped and the persistence checks still run.
#
# Exits non-zero with the number of failed checks. See docs/postgres.md.
# The jvagent entrypoint is derived from $PYTHON (default python3) so the run
# exercises the same environment as PYBIN, not whatever `jvagent` PATH happens
# to resolve to. Container name and ports are unique per run, and the server is
# tracked by PID, so concurrent runs do not kill each other.
#
# Exits non-zero with the number of failed checks. The temp workdir (bootstrap
# and server logs) is kept when any check fails. See docs/postgres.md.

set -u

APP_ROOT="examples/jvagent_app"
# Resolve the default app relative to the REPO, not the caller's cwd. The app's
# .env is gitignored, so a cwd-relative default silently yields a keyless app
# when the script is run from a worktree or anywhere else — and a keyless run
# still answers, with prompt scaffolding, so it fails three steps later at the
# recall check instead of here.
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
APP_ROOT="$REPO_ROOT/examples/jvagent_app"
USE_DOCKER=1
for arg in "$@"; do
case "$arg" in
--no-docker) USE_DOCKER=0 ;;
-h|--help) sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
-h|--help) sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
*) APP_ROOT="$arg" ;;
esac
done

CONTAINER="${JVAGENT_SMOKE_CONTAINER:-jvagent-smoke-pg}"
PGPORT="${JVAGENT_SMOKE_PGPORT:-55432}"
PORT="${JVAGENT_SMOKE_PORT:-8123}"
BASE="http://127.0.0.1:$PORT"
WORKDIR="$(mktemp -d)"
PYBIN="${PYTHON:-python3}"

# Run the *same* install PYBIN belongs to. A bare `jvagent` off PATH can be a
# different environment entirely (different jvspatial, missing asyncpg), which
# turns an environment problem into a wall of bogus check failures.
PYBIN_PATH="$(command -v "$PYBIN" 2>/dev/null || printf '%s' "$PYBIN")"
if [ -x "$(dirname "$PYBIN_PATH")/jvagent" ]; then
JVAGENT_CMD=("$(dirname "$PYBIN_PATH")/jvagent")
else
JVAGENT_CMD=("$PYBIN" -m jvagent)
fi

# Everything externally visible is per-run unique so two overlapping runs do not
# fight over a container name, a port, or each other's server process.
RUN_ID=$$
port_in_use() { (exec 3<>"/dev/tcp/127.0.0.1/$1") >/dev/null 2>&1; }
pick_port() { local p=$1; while port_in_use "$p"; do p=$((p + 1)); done; printf '%s' "$p"; }

CONTAINER="${JVAGENT_SMOKE_CONTAINER:-jvagent-smoke-pg-$RUN_ID}"
PGPORT="${JVAGENT_SMOKE_PGPORT:-$(pick_port $((55432 + RUN_ID % 1000)))}"
PORT="${JVAGENT_SMOKE_PORT:-$(pick_port $((8123 + RUN_ID % 1000)))}"
BASE="http://127.0.0.1:$PORT"
SERVER_PID=""

pass=0
fail=0
skip=0
Expand All @@ -50,10 +80,40 @@ ok() { printf 'PASS %s\n' "$1"; pass=$((pass + 1)); }
bad() { printf 'FAIL %s\n' "$1"; fail=$((fail + 1)); }
note() { printf 'SKIP %s\n' "$1"; skip=$((skip + 1)); }

start_server() {
nohup "${JVAGENT_CMD[@]}" "$APP_ROOT" > "$WORKDIR/server_$1.log" 2>&1 &
SERVER_PID=$!
for _ in $(seq 1 30); do
curl -s -m 2 "$BASE/health" >/dev/null 2>&1 && return 0
kill -0 "$SERVER_PID" 2>/dev/null || return 1 # died before serving
sleep 3
done
return 1
}
# Kill by PID, never `pkill -f jvagent <app root>` — that pattern matches every
# concurrent run's server too. The trailing redirect swallows bash's
# "Terminated: 15" job report, which is expected here, not a failure.
stop_server() {
[ -n "$SERVER_PID" ] || return 0
pkill -P "$SERVER_PID"
kill "$SERVER_PID"
for _ in $(seq 1 15); do
kill -0 "$SERVER_PID" 2>/dev/null || break
sleep 1
done
kill -9 "$SERVER_PID"
wait "$SERVER_PID"
SERVER_PID=""
} 2>/dev/null

cleanup() {
pkill -f "jvagent $APP_ROOT" 2>/dev/null
stop_server
[ "$USE_DOCKER" = "1" ] && docker rm -f "$CONTAINER" >/dev/null 2>&1
rm -rf "$WORKDIR"
if [ "$fail" -gt 0 ]; then
printf '\nlogs kept: %s\n' "$WORKDIR"
else
rm -rf "$WORKDIR"
fi
}
trap cleanup EXIT

Expand Down Expand Up @@ -92,22 +152,6 @@ export JVSPATIAL_ENVIRONMENT=development

psql_q() { docker exec "$CONTAINER" psql -U jvagent -d jvagent_smoke -tA -c "$1" 2>/dev/null; }

start_server() {
nohup jvagent "$APP_ROOT" > "$WORKDIR/server_$1.log" 2>&1 &
for _ in $(seq 1 30); do
curl -s -m 2 "$BASE/health" >/dev/null 2>&1 && return 0
sleep 3
done
return 1
}
stop_server() {
pkill -f "jvagent $APP_ROOT" 2>/dev/null
for _ in $(seq 1 15); do
pgrep -f "jvagent $APP_ROOT" >/dev/null || return 0
sleep 1
done
}

login() {
curl -s -m 15 -X POST "$BASE/api/auth/login" -H 'Content-Type: application/json' \
-d "{\"email\":\"$JVAGENT_ADMIN_EMAIL\",\"password\":\"$JVAGENT_ADMIN_PASSWORD\"}" |
Expand All @@ -122,11 +166,11 @@ say() { # token agent utterance
}

step "1. bootstrap graph onto Postgres"
if jvagent "$APP_ROOT" bootstrap > "$WORKDIR/bootstrap.log" 2>&1; then
if "${JVAGENT_CMD[@]}" "$APP_ROOT" bootstrap > "$WORKDIR/bootstrap.log" 2>&1; then
ok "jvagent bootstrap"
else
bad "jvagent bootstrap — see $WORKDIR/bootstrap.log"
grep -E "ValueError|Unsupported database type" "$WORKDIR/bootstrap.log" | head -2
grep -E "ValueError|Unsupported database type|ImportError" "$WORKDIR/bootstrap.log" | head -2
fi

if [ "$USE_DOCKER" = "1" ]; then
Expand All @@ -151,15 +195,32 @@ AGENT=$(curl -s -m 15 "$BASE/api/agents" -H "Authorization: Bearer $TOK" |
"$PYBIN" -c 'import sys,json; a=json.load(sys.stdin).get("agents",[]); print(a[0]["id"] if a else "")' 2>/dev/null)
[ -n "$AGENT" ] && ok "agent listed ($AGENT)" || bad "no agents returned"

# A reply is not automatically an answer. Without a model key the agent still
# responds — by echoing the prompt scaffolding it was handed — so a bare
# non-empty test passes on output that proves nothing, and the run only comes
# apart at the recall check two steps later. Reject the scaffolding markers and
# require the agent to have actually taken the utterance.
reply_is_substantive() {
case "$1" in
*"MANDATORY directive"*|*"[Deliver every"*|*"User message:"*) return 1 ;;
esac
[ "${#1}" -ge 12 ]
}

step "4. agent turn"
TURNS=1
r1=$(say "$TOK" "$AGENT" "Remember the number 8675309.")
if [ -n "$r1" ]; then
if [ -z "$r1" ]; then
TURNS=0
note "agent turn — no reply (model key missing?); persistence checks continue"
elif reply_is_substantive "$r1"; then
printf 'agent: %s\n' "$r1"
ok "turn produced a reply"
else
TURNS=0
note "agent turn — no reply (model key missing?); persistence checks continue"
printf 'agent: %s\n' "$r1"
note "agent turn — reply was prompt scaffolding, not an answer (model key \
missing or misconfigured?); persistence checks continue"
fi

if [ "$USE_DOCKER" = "1" ] && [ "$TURNS" = "1" ]; then
Expand Down