From 9c06a07bc69a2b3c08719756b119b00438591256 Mon Sep 17 00:00:00 2001 From: ems-sato Date: Tue, 1 Sep 2026 17:04:15 +0100 Subject: [PATCH 1/4] Update README for corporate mac setup --- README.md | 101 +++++++++++++++++++++++++++++++++++++----------- environment.yml | 8 ++++ 2 files changed, 86 insertions(+), 23 deletions(-) create mode 100644 environment.yml diff --git a/README.md b/README.md index 1cb7e7ad..2e4bd1cd 100644 --- a/README.md +++ b/README.md @@ -4,42 +4,87 @@ An API for validating survey schemas. ## Setup -In order to run locally you'll need Node.js, Poetry and Python installed. -It's recommended that Python is installed via pyenv but pyenv is optional. +In order to run locally you'll need Node.js, Poetry and Python installed, managed via a conda +environment. -### Install NVM and pyenv +### Pre-Requisites -NVM and pyenv will manage your versions of Node and Python and these commands will install the -required versions of them which will be read from `.nvmrc` and `.python-version`. +The following must be installed and working before you start: +- Miniconda: Python, Node and system package management (install from Self Service) +- Podman: Container runtime for supporting services (machine created and running) +- gcloud: Pulling images from Google Artifact Registry + +Verify each is available: + +```shell +conda --version +podman --version +gcloud --version +``` + +If `conda` reports `command not found` after installing from Self Service, the installer did not +write the conda block into `~/.zshrc`. Confirm the install is present and wire it in: + +```shell +ls -d /opt/miniconda3 +/opt/miniconda3/bin/conda init zsh +``` + +Open a new terminal tab and re-check `conda --version`. + +### Conda environment + +Python and Node.js versions are pinned in the committed `environment.yml`, matching +`.python-version` and `.nvmrc` as closely as conda-forge availability allows: + +> Note: conda-forge does not publish every Node patch release (it jumps from `22.13.0` to +> `22.17.0`). Where the exact `.nvmrc` version is unavailable, pin the closest available patch +> below it and note the substitution in `environment.yml`. + +If `.python-version` or `.nvmrc` change, update `environment.yml` to match. + +Create and activate the environment: + +```shell +conda env create -f environment.yml +conda activate eq-validator +``` + +Version can be changed by editing `environment.yml` and running: ```shell -brew install nvm pyenv -nvm use -pyenv install +conda env update -f environment.yml --prune ``` -If you get a message in the command line after running `nvm use` that the version of Node specified -in the `.nvmrc` file isn't installed, just follow the commands to install it. +### Poetry -e.g. +Poetry must install into the conda environment rather than creating its own virtualenv. Set this +on the environment so no configuration file is left in the repository: ```shell -nvm install v22.15.0 +conda env config vars set POETRY_VIRTUALENVS_CREATE=false +conda deactivate && conda activate eq-validator ``` -### Install JS dependencies +Confirm it took effect: this must print `false`: ```shell -npm install +echo $POETRY_VIRTUALENVS_CREATE ``` -### Install Poetry and Python dependencies +### Install dependencies + +With the conda environment active, install the Python dependencies: ```shell -curl -sSL https://install.python-poetry.org | python3 - --version 2.1.2 poetry install ``` +Install the JavaScript dependencies: +```shell +npm ci +``` + ## Running locally To run the app: @@ -217,7 +262,8 @@ best practices and maintaining consistency across the repository without the nee MegaLinter examines various file types and tools, including GitHub Actions, Shell scripts, Dockerfile, etc. It is configured using the `.mega-linter.yml` file. -To run MegaLinter, ensure you have **Docker** installed on your system. +To run MegaLinter, ensure you have **Podman** installed and running on your system +(see [Running with Docker](#running-with-docker) below for setup and the `docker` command shim). > Note: The initial run may take some time to download the Docker image. However, subsequent executions will be > considerably faster due to Docker caching. @@ -230,24 +276,33 @@ make megalint ## Running with Docker -To install Docker run: +Install Podman for your system as the container runtime. + +Make sure the Podman machine is started every time you want to use container images: ```shell -brew install docker +podman machine start ``` -On MacOS install container runtimes, e.g. Colima: +This repo's Makefile and commands below only use `docker run` (no compose), so provide a +`docker` command that points at Podman: ```shell -brew install colima +mkdir -p ~/.local/bin +ln -s "$(which podman)" ~/.local/bin/docker +hash -r ``` -Make sure Colima is started every time you want to use Docker images: +`~/.local/bin` must be on your `PATH`. Verify: ```shell -colima start +docker --version ``` +> Note: `eq-questionnaire-runner` pulls and runs these images via `docker-compose-schema-validator.yml`. +> That repo's Makefile uses `docker-compose` (not `docker compose`), so if you also work in that +> repo you additionally need `podman-compose`, installed with `conda install -c conda-forge podman-compose`. + When PRs are merged in this repo there is a GitHub workflow that builds 2 Docker images one for Validator and one for the Ajv validator and then pushes them to our GAR in GCP. These images can then be pulled down and run locally with Docker. diff --git a/environment.yml b/environment.yml new file mode 100644 index 00000000..dbe3cadc --- /dev/null +++ b/environment.yml @@ -0,0 +1,8 @@ +name: eq-validator +channels: + - conda-forge +dependencies: + - python=3.14.6 + - nodejs=22.13.0 # closest available patch to .nvmrc's v22.15.0 + - poetry=2.1.2 + - pip From d146e56e39cc2b8dd240ae31020c92e48b10f80f Mon Sep 17 00:00:00 2001 From: ems-sato Date: Thu, 3 Sep 2026 09:51:49 +0100 Subject: [PATCH 2/4] Update READMEs and port change for corporate Macs --- .development.env | 2 +- .github/workflows/pull_request.yml | 4 ++-- Dockerfile | 2 +- Dockerfile-ajv | 2 +- README.md | 14 +++++++------- ajv/app.js | 2 +- api.py | 3 +++ environment.yml | 2 +- poetry.lock | 19 +++++++++++++++++-- pyproject.toml | 1 + 10 files changed, 35 insertions(+), 16 deletions(-) diff --git a/.development.env b/.development.env index 5c6c97ed..ca0f1d2d 100644 --- a/.development.env +++ b/.development.env @@ -1,4 +1,4 @@ LOG_LEVEL="INFO" AJV_VALIDATOR_SCHEME=http AJV_VALIDATOR_HOST=localhost -AJV_VALIDATOR_PORT=5002 +AJV_VALIDATOR_PORT=5005 diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 7638a6de..e539e50d 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -141,7 +141,7 @@ jobs: - name: Spin Up AJV Validator run: | - docker run --network validator-network --name ajv-validator -d -p 5002:5002 ${{ secrets.GAR_LOCATION }}/${{ secrets.GAR_PROJECT_ID }}/docker-images/eq-questionnaire-validator-ajv:$TAG + docker run --network validator-network --name ajv-validator -d -p 5005:5005 ${{ secrets.GAR_LOCATION }}/${{ secrets.GAR_PROJECT_ID }}/docker-images/eq-questionnaire-validator-ajv:$TAG - name: Spin Up Python Validator run: > docker run @@ -162,7 +162,7 @@ jobs: - name: Check AJV Validator is Running run: | for i in {1..10}; do - if curl -f http://localhost:5002/status; then + if curl -f http://localhost:5005/status; then echo "AJV Validator is up!" exit 0 fi diff --git a/Dockerfile b/Dockerfile index 89ca168d..639da2a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ COPY api.py poetry.lock pyproject.toml /usr/src/ ENV AJV_VALIDATOR_SCHEME=http ENV AJV_VALIDATOR_HOST=localhost -ENV AJV_VALIDATOR_PORT=5002 +ENV AJV_VALIDATOR_PORT=5005 RUN poetry install --only main diff --git a/Dockerfile-ajv b/Dockerfile-ajv index 76fb9e92..5c79beef 100644 --- a/Dockerfile-ajv +++ b/Dockerfile-ajv @@ -11,7 +11,7 @@ COPY schemas ../schemas RUN npm install --production=true -ENV AJV_VALIDATOR_PORT=5002 +ENV AJV_VALIDATOR_PORT=5005 ENV DEBUG=ajv-schema-validator WORKDIR /usr/src diff --git a/README.md b/README.md index 2e4bd1cd..20df1808 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ To run the app: make run ``` -Validator runs on two ports, `5001` is the main validator app and `5002` is Ajv validator. +Validator runs on two ports, `5001` is the main validator app and `5005` is Ajv validator. ### Validator @@ -110,7 +110,7 @@ if you want to run the app locally using multiple server workers you need to set ### Ajv validator -Ajv validator defaults to running on `http://localhost:5002`. +Ajv validator defaults to running on `http://localhost:5005`. You can override this by setting the `AJV_VALIDATOR_SCHEME`, `AJV_VALIDATOR_HOST`, and `AJV_VALIDATOR_PORT` environment variables. @@ -121,7 +121,7 @@ The defaults for these are: - `AJV_VALIDATOR_SCHEME` = http - `AJV_VALIDATOR_HOST` = localhost -- `AJV_VALIDATOR_PORT` = 5002 +- `AJV_VALIDATOR_PORT` = 5005 Alternatively, you can override the entire URL by setting the `AJV_VALIDATOR_URL` environment variable directly. (**Note**: These values are also defined in the Dockerfiles, so if you choose to run @@ -169,7 +169,7 @@ and the Validator app itself). However, if you want to start Ajv individually, r make start-ajv ``` -This defaults to running on port `5002`, set `AJV_VALIDATOR_PORT` in your .env file if you need to change this. +This defaults to running on port `5005`, set `AJV_VALIDATOR_PORT` in your .env file if you need to change this. Running the Ajv server returns either an empty json response when the questionnaire is valid, or a response containing an "errors" key. @@ -193,7 +193,7 @@ To run the app's Python tests: make test-python ``` -Make sure you don't already have Ajv running on localhost:5002 by running `lsof -i tcp:5002` if you +Make sure you don't already have Ajv running on localhost:5005 by running `lsof -i tcp:5005` if you do make a note of the PID (process identifier) and then run `kill -9 `, replacing `` with the process id from the previous command. @@ -322,7 +322,7 @@ docker run -it -p 5001:5001 europe-west2-docker.pkg.dev/ons-eq-ci/docker-images/ - Ajv validator: ```shell -docker run -it -p 5002:5002 europe-west2-docker.pkg.dev/ons-eq-ci/docker-images/eq-questionnaire-validator-ajv +docker run -it -p 5005:5005 europe-west2-docker.pkg.dev/ons-eq-ci/docker-images/eq-questionnaire-validator-ajv ``` To stop these containers you may need to use the `docker kill` command: @@ -347,6 +347,6 @@ docker kill | `LOG_LEVEL` | Sets the minimum log level, can be set to `DEBUG` to increase this level | `INFO` | | `AJV_VALIDATOR_SCHEME` | Sets the scheme for the URL that Ajv validator will run on | `http` | | `AJV_VALIDATOR_HOST` | Sets the host for the URL that Ajv validator will run on | `localhost` | -| `AJV_VALIDATOR_PORT` | Sets the port for the URL that Ajv validator will run on | `5002` | +| `AJV_VALIDATOR_PORT` | Sets the port for the URL that Ajv validator will run on | `5005` | | `AJV_VALIDATOR_URL` | Sets complete URL that Ajv validator will run on | `://:/validate` | | `VALIDATOR_VERSION` | Sets the version of the validator, this is used in the response from the `/validate` endpoint | `0.0.0` | diff --git a/ajv/app.js b/ajv/app.js index f1d56fbb..07c4b21f 100644 --- a/ajv/app.js +++ b/ajv/app.js @@ -6,7 +6,7 @@ import Debug from "debug"; const debug = Debug("ajv-schema-validator"); const app = express(); -const AJV_VALIDATOR_PORT = process.env.AJV_VALIDATOR_PORT || 5002; +const AJV_VALIDATOR_PORT = process.env.AJV_VALIDATOR_PORT || 5005; app.use( express.json({ diff --git a/api.py b/api.py index 107c4b9b..4a194723 100644 --- a/api.py +++ b/api.py @@ -30,6 +30,7 @@ import requests import structlog import uvicorn +from dotenv import load_dotenv from fastapi import Body, FastAPI from fastapi.responses import JSONResponse, Response from requests import RequestException @@ -45,6 +46,8 @@ ALLOWED_REPO_OWNERS = {"ONSdigital"} +load_dotenv() + AJV_VALIDATOR_SCHEME = os.getenv("AJV_VALIDATOR_SCHEME") AJV_VALIDATOR_HOST = os.getenv("AJV_VALIDATOR_HOST") diff --git a/environment.yml b/environment.yml index dbe3cadc..9f6a0620 100644 --- a/environment.yml +++ b/environment.yml @@ -4,5 +4,5 @@ channels: dependencies: - python=3.14.6 - nodejs=22.13.0 # closest available patch to .nvmrc's v22.15.0 - - poetry=2.1.2 + - poetry=2.4.2 - pip diff --git a/poetry.lock b/poetry.lock index a0d94095..48ed6861 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. +c# This file is automatically @generated by Poetry 2.4.2 and should not be changed by hand. [[package]] name = "annotated-doc" @@ -1084,6 +1084,21 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "python-dotenv" +version = "1.2.3" +description = "Read key-value pairs from a .env file and set them as environment variables" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "python_dotenv-1.2.3-py3-none-any.whl", hash = "sha256:904552145e8bfed22162c09dab1c2b9b54fefa7b23ba780f4f26ca0316b0f0d9"}, + {file = "python_dotenv-1.2.3.tar.gz", hash = "sha256:a20a594dabeaa385725aa239d5244871c143ecb356add8a20fcf23773a6c3a35"}, +] + +[package.extras] +cli = ["click (>=5.0)"] + [[package]] name = "pytokens" version = "0.4.1" @@ -1487,4 +1502,4 @@ standard = ["colorama (>=0.4) ; sys_platform == \"win32\"", "httptools (>=0.8.0) [metadata] lock-version = "2.1" python-versions = "^3.14" -content-hash = "136e240ff1186316e2fb9693c777481b7f6c73579215b29b294289b14d0915e1" +content-hash = "4c2509bd3f6cf585c8f8deb17d7020bc19144d1791ad0b3e61a454cb8e15c789" diff --git a/pyproject.toml b/pyproject.toml index 13467e48..acd3eb71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ referencing = "^0.37.0" jsonpath-ng = "1.8.0" starlette = "^1.0.0" requests = "^2.33.1" +python-dotenv = "^1.2.3" [tool.poetry.group.dev.dependencies] pytest = "^9.0.3" From f39dee4079b1ef017b3192fe09c8e948ac454c76 Mon Sep 17 00:00:00 2001 From: ems-sato Date: Thu, 3 Sep 2026 11:03:47 +0100 Subject: [PATCH 3/4] README npm upgrade command --- README.md | 11 +++++++++++ environment.yml | 2 +- poetry.lock | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 20df1808..f09f0b97 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ environment. ### Pre-Requisites The following must be installed and working before you start: + - Miniconda: Python, Node and system package management (install from Self Service) - Podman: Container runtime for supporting services (machine created and running) - gcloud: Pulling images from Google Artifact Registry @@ -81,10 +82,20 @@ poetry install ``` Install the JavaScript dependencies: + ```shell npm ci ``` +Temporary fix, this section can be removed once conda-forge has a nodejs build bundling npm 11: +conda-forge's `nodejs=22.13.0` bundles npm 10.9.2. `make run` will not start the Python app on npm 10: +`npm run start` holds the terminal despite the trailing `&`, so make never reaches `poetry run python api.py`. +Upgrade npm after creating the environment: + +```shell +npm install -g npm@11.6.1 +``` + ## Running locally To run the app: diff --git a/environment.yml b/environment.yml index 9f6a0620..6c971590 100644 --- a/environment.yml +++ b/environment.yml @@ -3,6 +3,6 @@ channels: - conda-forge dependencies: - python=3.14.6 - - nodejs=22.13.0 # closest available patch to .nvmrc's v22.15.0 + - nodejs=22.13.0 # closest available patch to .nvmrc's v22.15.0 - poetry=2.4.2 - pip diff --git a/poetry.lock b/poetry.lock index 48ed6861..1fc0b8cb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -c# This file is automatically @generated by Poetry 2.4.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.2 and should not be changed by hand. [[package]] name = "annotated-doc" From ffda65b0a9d84018bc13ddb8dc548fec63aa009e Mon Sep 17 00:00:00 2001 From: ems-sato Date: Wed, 9 Sep 2026 13:52:35 +0100 Subject: [PATCH 4/4] Move Poetry config from README to environment.yml --- README.md | 16 ---------------- environment.yml | 2 ++ 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index f09f0b97..0b4a4922 100644 --- a/README.md +++ b/README.md @@ -57,22 +57,6 @@ Version can be changed by editing `environment.yml` and running: conda env update -f environment.yml --prune ``` -### Poetry - -Poetry must install into the conda environment rather than creating its own virtualenv. Set this -on the environment so no configuration file is left in the repository: - -```shell -conda env config vars set POETRY_VIRTUALENVS_CREATE=false -conda deactivate && conda activate eq-validator -``` - -Confirm it took effect: this must print `false`: - -```shell -echo $POETRY_VIRTUALENVS_CREATE -``` - ### Install dependencies With the conda environment active, install the Python dependencies: diff --git a/environment.yml b/environment.yml index 6c971590..f32f940e 100644 --- a/environment.yml +++ b/environment.yml @@ -6,3 +6,5 @@ dependencies: - nodejs=22.13.0 # closest available patch to .nvmrc's v22.15.0 - poetry=2.4.2 - pip +variables: + POETRY_VIRTUALENVS_CREATE: "false"