From 9483ddd00667ae4315ecb72dd14872249909335d Mon Sep 17 00:00:00 2001 From: Eric Date: Sun, 13 Sep 2026 03:08:34 +0000 Subject: [PATCH 1/6] use compose stack for integration tests --- .github/workflows/test.yml | 35 ++++++++--------------------------- docker-compose.yml | 2 ++ 2 files changed, 10 insertions(+), 27 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3dc225f..59f2a70 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,31 +38,6 @@ jobs: name: Integration Tests runs-on: ubuntu-latest - services: - redis: - image: redis:7-alpine - ports: - - 6379:6379 - options: >- - --health-cmd "redis-cli ping" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - postgres: - image: postgres:16-alpine - env: - POSTGRES_DB: quotient_test - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - ports: - - 5432:5432 - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - steps: - name: Checkout code uses: actions/checkout@v7 @@ -78,6 +53,12 @@ jobs: - name: Install dependencies run: go mod download + - name: Set up Docker Compose + uses: docker/setup-compose-action@v2 + + - name: Setup Compose stack + run: docker compose up db redis -d + - name: Run integration tests # -p 1 runs one package binary at a time. These packages share a single # Postgres database and a single Redis instance, and they use fixed @@ -91,8 +72,8 @@ jobs: POSTGRES_HOST: localhost POSTGRES_PORT: 5432 POSTGRES_DB: quotient_test - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres + POSTGRES_USER: engineuser + POSTGRES_PASSWORD: postgres_password security-scanner: name: Security Scanner diff --git a/docker-compose.yml b/docker-compose.yml index dff04f5..780bb04 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,8 @@ services: restart: always env_file: - .env + ports: + - "127.0.0.1:5432:5432" networks: - quotient_network volumes: From 7690e51b6b5f6767f34a1bb13ed1f3b8cc4375bc Mon Sep 17 00:00:00 2001 From: Eric Date: Sun, 13 Sep 2026 03:08:42 +0000 Subject: [PATCH 2/6] simplify parameter env lookup with cmp.Or --- tests/testutil/testutil.go | 36 ++++++++---------------------------- 1 file changed, 8 insertions(+), 28 deletions(-) diff --git a/tests/testutil/testutil.go b/tests/testutil/testutil.go index c7ea421..6e49fcb 100644 --- a/tests/testutil/testutil.go +++ b/tests/testutil/testutil.go @@ -1,6 +1,7 @@ package testutil import ( + "cmp" "fmt" "os" "sync/atomic" @@ -30,14 +31,8 @@ func (r *RedisContainer) Close() error { func startRedis(t *testing.T) *RedisContainer { t.Helper() - host := os.Getenv("REDIS_HOST") - if host == "" { - host = "localhost" - } - port := os.Getenv("REDIS_PORT") - if port == "" { - port = "6379" - } + host := cmp.Or(os.Getenv("REDIS_HOST"), "localhost") + port := cmp.Or(os.Getenv("REDIS_PORT"), "6379") client := redis.NewClient(&redis.Options{ Addr: fmt.Sprintf("%s:%s", host, port), @@ -60,26 +55,11 @@ func (p *PostgresContainer) Close() error { func startPostgres(t *testing.T) *PostgresContainer { t.Helper() - host := os.Getenv("POSTGRES_HOST") - if host == "" { - host = "localhost" - } - port := os.Getenv("POSTGRES_PORT") - if port == "" { - port = "5432" - } - dbname := os.Getenv("POSTGRES_DB") - if dbname == "" { - dbname = "quotient_test" - } - user := os.Getenv("POSTGRES_USER") - if user == "" { - user = "postgres" - } - password := os.Getenv("POSTGRES_PASSWORD") - if password == "" { - password = "postgres" - } + host := cmp.Or(os.Getenv("POSTGRES_HOST"), "localhost") + port := cmp.Or(os.Getenv("POSTGRES_PORT"), "5432") + dbname := cmp.Or(os.Getenv("POSTGRES_DB"), "quotient_test") + user := cmp.Or(os.Getenv("POSTGRES_USER"), "postgres") + password := cmp.Or(os.Getenv("POSTGRES_PASSWORD"), "postgres") connString := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", host, port, user, password, dbname) From 27def20fa39895dff4db8589d0c6b06e441c981b Mon Sep 17 00:00:00 2001 From: Eric Date: Sun, 13 Sep 2026 06:18:01 +0000 Subject: [PATCH 3/6] wrong database --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 59f2a70..3db1d81 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -71,7 +71,7 @@ jobs: REDIS_PORT: 6379 POSTGRES_HOST: localhost POSTGRES_PORT: 5432 - POSTGRES_DB: quotient_test + POSTGRES_DB: engine POSTGRES_USER: engineuser POSTGRES_PASSWORD: postgres_password From 6d43a8240966704b5066a81ea53f2b54cde13f34 Mon Sep 17 00:00:00 2001 From: Eric Date: Sun, 13 Sep 2026 06:20:58 +0000 Subject: [PATCH 4/6] redis auth --- .github/workflows/test.yml | 1 + tests/testutil/testutil.go | 10 ++++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3db1d81..520f403 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -69,6 +69,7 @@ jobs: env: REDIS_HOST: localhost REDIS_PORT: 6379 + REDIS_PASSWORD: redis_password POSTGRES_HOST: localhost POSTGRES_PORT: 5432 POSTGRES_DB: engine diff --git a/tests/testutil/testutil.go b/tests/testutil/testutil.go index 6e49fcb..734a4b7 100644 --- a/tests/testutil/testutil.go +++ b/tests/testutil/testutil.go @@ -33,9 +33,11 @@ func startRedis(t *testing.T) *RedisContainer { host := cmp.Or(os.Getenv("REDIS_HOST"), "localhost") port := cmp.Or(os.Getenv("REDIS_PORT"), "6379") + password := cmp.Or(os.Getenv("REDIS_PASSWORD"), "redis_password") client := redis.NewClient(&redis.Options{ - Addr: fmt.Sprintf("%s:%s", host, port), + Addr: fmt.Sprintf("%s:%s", host, port), + Password: password, }) return &RedisContainer{Client: client} @@ -57,9 +59,9 @@ func startPostgres(t *testing.T) *PostgresContainer { host := cmp.Or(os.Getenv("POSTGRES_HOST"), "localhost") port := cmp.Or(os.Getenv("POSTGRES_PORT"), "5432") - dbname := cmp.Or(os.Getenv("POSTGRES_DB"), "quotient_test") - user := cmp.Or(os.Getenv("POSTGRES_USER"), "postgres") - password := cmp.Or(os.Getenv("POSTGRES_PASSWORD"), "postgres") + dbname := cmp.Or(os.Getenv("POSTGRES_DB"), "engine") + user := cmp.Or(os.Getenv("POSTGRES_USER"), "engineuser") + password := cmp.Or(os.Getenv("POSTGRES_PASSWORD"), "postgres_password") connString := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", host, port, user, password, dbname) From 2564829d2327358419fcbbe83065c32e18f921ff Mon Sep 17 00:00:00 2001 From: Eric Date: Sun, 13 Sep 2026 01:40:22 -0700 Subject: [PATCH 5/6] use makefile within github ci instead --- .github/workflows/test.yml | 10 +--------- Makefile | 31 +++++++------------------------ 2 files changed, 8 insertions(+), 33 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 520f403..c653087 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,16 +56,8 @@ jobs: - name: Set up Docker Compose uses: docker/setup-compose-action@v2 - - name: Setup Compose stack - run: docker compose up db redis -d - - name: Run integration tests - # -p 1 runs one package binary at a time. These packages share a single - # Postgres database and a single Redis instance, and they use fixed - # names for both (the `tasks` and `results` lists, one schema). Running - # them concurrently lets one package consume the other's queued tasks - # and reset its rows. - run: go test -v -race -p 1 ./tests/integration/... ./engine/... -timeout 10m + run: make test-integration env: REDIS_HOST: localhost REDIS_PORT: 6379 diff --git a/Makefile b/Makefile index bfb2c86..f279f92 100644 --- a/Makefile +++ b/Makefile @@ -1,37 +1,20 @@ -.PHONY: test test-unit test-integration test-coverage clean test-deps-start test-deps-stop test-deps-wait +.PHONY: test test-unit test-integration test-coverage clean test-deps-start test-deps-stop test: - go test -race -p 1 ./... + go test -v -race -p 1 ./... test-unit: - go test -race -short ./engine/... ./www/... ./runner/... + go test -v -race -short ./engine/... ./www/... ./runner/... -test-integration: test-deps-start test-deps-wait - POSTGRES_PASSWORD=postgres go test -race ./tests/integration/... -timeout 10m +test-integration: test-deps-start + go test -v -race -p 1 ./tests/integration/... ./engine/... -timeout 10m @$(MAKE) test-deps-stop test-deps-start: - @docker ps -q -f name=quotient-test-postgres | grep -q . || \ - docker run -d --name quotient-test-postgres \ - -e POSTGRES_PASSWORD=postgres \ - -e POSTGRES_DB=quotient_test \ - -p 5432:5432 \ - postgres:16-alpine - @docker ps -q -f name=quotient-test-redis | grep -q . || \ - docker run -d --name quotient-test-redis \ - -p 6379:6379 \ - redis:7-alpine - -test-deps-wait: - @echo "Waiting for Postgres..." - @until docker exec quotient-test-postgres pg_isready -U postgres > /dev/null 2>&1; do sleep 1; done - @echo "Waiting for Redis..." - @until docker exec quotient-test-redis redis-cli ping > /dev/null 2>&1; do sleep 1; done - @echo "Services ready." + docker compose up redis db --wait -d test-deps-stop: - @docker stop quotient-test-postgres quotient-test-redis 2>/dev/null || true - @docker rm quotient-test-postgres quotient-test-redis 2>/dev/null || true + docker compose down test-coverage: go test -race -p 1 -coverprofile=coverage.out ./... From cad6e120cfbf1461d64d26409193caabb07895ee Mon Sep 17 00:00:00 2001 From: Eric Date: Sun, 13 Sep 2026 02:02:00 -0700 Subject: [PATCH 6/6] split out test db / redis but with shared components from actual db / redis --- .github/workflows/test.yml | 2 +- Makefile | 4 +-- docker-compose.base.yml | 27 ++++++++++++++++++++ docker-compose.yml | 52 +++++++++++++++++++------------------- tests/testutil/testutil.go | 2 +- 5 files changed, 57 insertions(+), 30 deletions(-) create mode 100644 docker-compose.base.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c653087..b978ee6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,7 +60,7 @@ jobs: run: make test-integration env: REDIS_HOST: localhost - REDIS_PORT: 6379 + REDIS_PORT: 6380 REDIS_PASSWORD: redis_password POSTGRES_HOST: localhost POSTGRES_PORT: 5432 diff --git a/Makefile b/Makefile index f279f92..eb933a6 100644 --- a/Makefile +++ b/Makefile @@ -11,10 +11,10 @@ test-integration: test-deps-start @$(MAKE) test-deps-stop test-deps-start: - docker compose up redis db --wait -d + docker compose up test-redis test-db --wait -d test-deps-stop: - docker compose down + docker compose down test-redis test-db test-coverage: go test -race -p 1 -coverprofile=coverage.out ./... diff --git a/docker-compose.base.yml b/docker-compose.base.yml new file mode 100644 index 0000000..1a84a9f --- /dev/null +++ b/docker-compose.base.yml @@ -0,0 +1,27 @@ +services: + db-base: + image: postgres:17 + env_file: + - .env + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + interval: 5s + timeout: 5s + retries: 5 + start_period: 3s + networks: + - quotient_network + + redis-base: + image: redis:alpine + env_file: + - .env + command: ["redis-server", "--requirepass", "${REDIS_PASSWORD:?REDIS_PASSWORD variable is not set}"] + healthcheck: + test: ["CMD-SHELL", "redis-cli -a $$REDIS_PASSWORD ping | grep -q PONG"] + interval: 5s + timeout: 5s + retries: 5 + start_period: 3s + networks: + - quotient_network \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 780bb04..a9975f9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,22 +1,21 @@ services: + test-db: + extends: + file: docker-compose.base.yml + service: db-base + profiles: + - test + ports: + - "127.0.0.1:5432:5432" + db: + extends: + file: docker-compose.base.yml + service: db-base container_name: quotient_database - image: postgres:17 restart: always - env_file: - - .env - ports: - - "127.0.0.1:5432:5432" - networks: - - quotient_network volumes: - db_data:/var/lib/postgresql/data - healthcheck: - test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] - interval: 5s - timeout: 5s - retries: 5 - start_period: 3s server: container_name: quotient_server @@ -40,23 +39,24 @@ services: - ./engine:/app/engine - ./static:/app/static + test-redis: + extends: + file: docker-compose.base.yml + service: redis-base + profiles: + - test + ports: + # Don't conflict with actual redis instance + - "127.0.0.1:6380:6379" + redis: - image: redis:alpine - container_name: quotient_redis + extends: + file: docker-compose.base.yml + service: redis-base restart: always - env_file: - - .env + container_name: quotient_redis ports: - "127.0.0.1:6379:6379" - networks: - - quotient_network - command: ["redis-server", "--requirepass", "${REDIS_PASSWORD:?REDIS_PASSWORD variable is not set}"] - healthcheck: - test: ["CMD-SHELL", "redis-cli -a $$REDIS_PASSWORD ping | grep -q PONG"] - interval: 5s - timeout: 5s - retries: 5 - start_period: 3s runner: build: diff --git a/tests/testutil/testutil.go b/tests/testutil/testutil.go index 734a4b7..40f8b75 100644 --- a/tests/testutil/testutil.go +++ b/tests/testutil/testutil.go @@ -32,7 +32,7 @@ func startRedis(t *testing.T) *RedisContainer { t.Helper() host := cmp.Or(os.Getenv("REDIS_HOST"), "localhost") - port := cmp.Or(os.Getenv("REDIS_PORT"), "6379") + port := cmp.Or(os.Getenv("REDIS_PORT"), "6380") password := cmp.Or(os.Getenv("REDIS_PASSWORD"), "redis_password") client := redis.NewClient(&redis.Options{