From 3f2bd29aaa336b69d52c0c3d11c76788143d4ca8 Mon Sep 17 00:00:00 2001 From: Joe Niland Date: Thu, 18 Jun 2026 17:35:59 +1000 Subject: [PATCH 1/8] docs: add overview section and document download command --- README.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/README.md b/README.md index 151b074..d892ce3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,29 @@ Populate Gitlab Project Variables from .env file ================================================= +## Overview + +A command-line tool for managing a Gitlab project's CI/CD variables, scoped to a +Gitlab [environment](https://docs.gitlab.com/ee/ci/environments/) (e.g. `uat`, +`production`). It talks to the Gitlab API using a personal access token and lets +you move variables between a local `.env` file and Gitlab in both directions. + +It provides four commands: + +- `write` — read a local `.env` file and create or update the matching + project variables in the given environment scope. Supports `--include` / + `--exclude` filtering and `--mask` to mask values whose key contains `KEY`, + `SECRET`, or `TOKEN`. +- `list` — print the variables for an environment in a table. Masked values are + hidden unless you pass `--sensitive`. +- `get` — print the variables for an environment, optionally appending them to a + `.env` file with `--export`. +- `download` — write an environment's variables to a `.env` file, + prompting before overwriting an existing file. + +All commands target both the requested environment and globally-scoped (`*`) +variables. Requires a `GITLAB_TOKEN` environment variable. + ## Install Install as a global user tool (isolated environment, command on your PATH): @@ -56,3 +79,9 @@ populate-secrets-gitlab write \ ```shell populate-secrets-gitlab get --environment uat --gitlab-host gitlab.example.com --project my-group/my-project --export ``` + +### Download variables to an .env file + +```shell +populate-secrets-gitlab download --environment uat --gitlab-host gitlab.example.com --project my-group/my-project --output-dir . +``` From 5a06d9a649d127066a24a29d5a2029de9b7bac29 Mon Sep 17 00:00:00 2001 From: Joe Niland Date: Thu, 18 Jun 2026 18:10:53 +1000 Subject: [PATCH 2/8] fix: write to fresh environment scope, narrow exception handler, consistent token error, and idempotent get --export - Use .get(environment, []) instead of [environment] on the scope dict so write does not KeyError on a brand-new environment with zero existing variables. - Narrow except Exception to except gitlab.exceptions.GitlabError so programming errors (like the above KeyError) surface instead of being swallowed as "unexpected error". - Use click.ClickException in the get command for missing GITLAB_TOKEN, matching write/list/download (clean message, no traceback). - Track which scope files have been opened during get --export so each file is truncated on first write and appended thereafter, preventing duplicate lines on re-run. --- src/populate_secrets_gitlab/app.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/populate_secrets_gitlab/app.py b/src/populate_secrets_gitlab/app.py index 25342eb..fff4dcd 100644 --- a/src/populate_secrets_gitlab/app.py +++ b/src/populate_secrets_gitlab/app.py @@ -138,7 +138,7 @@ def write(env_file, environment, gitlab_host, project, include, exclude, mask, d # Write to Gitlab API try: - if key in gitlab_project_variable_keys_by_scope[environment]: + if key in gitlab_project_variable_keys_by_scope.get(environment, []): is_update = True # Update project_var = [v for v in gl_project_vars if v.key == key][0] @@ -164,8 +164,8 @@ def write(env_file, environment, gitlab_host, project, include, exclude, mask, d logger.info("Failed to write {} due to error from Gitlab API".format(key)) print_exc() continue - except Exception: - logger.info("Failed to write {} due to unexpected error".format(key)) + except gitlab.exceptions.GitlabError: + logger.info("Failed to write {} due to unexpected Gitlab error".format(key)) print_exc() continue @@ -209,7 +209,7 @@ def get(environment, gitlab_host, project, export, debug): try: gitlab_token = os.environ["GITLAB_TOKEN"] except KeyError: - raise Exception( + raise click.ClickException( f"GITLAB_TOKEN must be set. Get token from https://{gitlab_host}/-/profile/personal_access_tokens" ) @@ -232,6 +232,7 @@ def get(environment, gitlab_host, project, export, debug): gitlabProjectVariables = gitlabProject.variables.list(get_all=True) + export_opened = set() for variable in sorted(gitlabProjectVariables, key=lambda v: v.key): scope = 'global' if variable.environment_scope == '*' else variable.environment_scope if scope == environment or scope == 'global': @@ -239,8 +240,10 @@ def get(environment, gitlab_host, project, export, debug): if export: logger.debug(f"Writing {variable.key} to {scope}.env") - with open(f"{scope}.env", 'a+') as f: + mode = "a" if scope in export_opened else "w" + with open(f"{scope}.env", mode) as f: f.write(f"{variable.key}={variable.value}\n") + export_opened.add(scope) logger.info("Done") From f87c049e3d3823709212e132416da12e2b8f2f00 Mon Sep 17 00:00:00 2001 From: Joe Niland Date: Thu, 18 Jun 2026 18:11:03 +1000 Subject: [PATCH 3/8] test: add pytest suite covering write, masking, token errors, export, and scope filtering Covers the critical logic paths: - Regression test for write to fresh environment scope (Task 1 fix) - Masking heuristic (KEY/SECRET/TOKEN substring match) - Consistent click.ClickException on missing GITLAB_TOKEN for all commands - get --export idempotence (no duplicate lines on re-run) - Scope filtering (global * vars included, other-env vars excluded) - prepare_gitlab_host URL normalisation --- pyproject.toml | 3 + tests/__init__.py | 0 tests/test_app.py | 287 +++++++++++++++++++++++++++++++++++++++++++++ tests/test_util.py | 18 +++ 4 files changed, 308 insertions(+) create mode 100644 tests/__init__.py create mode 100644 tests/test_app.py create mode 100644 tests/test_util.py diff --git a/pyproject.toml b/pyproject.toml index 63a52e2..2c59f09 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,5 +25,8 @@ Issues = "https://github.com/deploymode/populate-secrets-gitlab/issues" [project.scripts] populate-secrets-gitlab = "populate_secrets_gitlab.__main__:main" +[dependency-groups] +dev = ["pytest>=8"] + [tool.setuptools.packages.find] where = ["src"] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_app.py b/tests/test_app.py new file mode 100644 index 0000000..613ab16 --- /dev/null +++ b/tests/test_app.py @@ -0,0 +1,287 @@ +"""Tests for app.py commands. + +Mocks only at the gitlab API boundary (gitlab.Gitlab client / project.variables). +""" + +import os +from unittest.mock import MagicMock, patch + +import click.testing +import pytest + +from populate_secrets_gitlab.app import cli + + +def _make_variable(key, value, environment_scope="*", masked=False): + v = MagicMock() + v.key = key + v.value = value + v.environment_scope = environment_scope + v.masked = masked + return v + + +def _make_project(name="test-project", project_id=42, variables=None): + proj = MagicMock() + proj.name = name + proj.id = project_id + proj.variables.list.return_value = variables or [] + return proj + + +def _make_gitlab_client(project): + client = MagicMock() + client.projects.get.return_value = project + return client + + +def _invoke_write(tmp_path, env_content, environment, extra_args=None, variables=None): + env_file = tmp_path / ".env" + env_file.write_text(env_content) + + project = _make_project(variables=variables or []) + client = _make_gitlab_client(project) + + args = [ + "write", + "--env-file", str(env_file), + "--environment", environment, + "--gitlab-host", "gitlab.example.com", + "--project", "test/project", + ] + if extra_args: + args.extend(extra_args) + + with patch.dict(os.environ, {"GITLAB_TOKEN": "fake-token"}): + with patch("populate_secrets_gitlab.app.gitlab_client", return_value=client): + runner = click.testing.CliRunner() + result = runner.invoke(cli, args) + + return result, project + + +# --- Task 1 regression: write to a fresh (empty) environment scope --- + +class TestWriteFreshEnvironment: + """The write command must create variables when the target environment + has zero existing variables (no entry in the scope dict).""" + + def test_create_vars_when_scope_has_no_existing_variables(self, tmp_path): + result, project = _invoke_write(tmp_path, "APP_NAME=hello\n", "staging") + + assert result.exit_code == 0, result.output + project.variables.create.assert_called_once_with({ + "key": "APP_NAME", + "value": "hello", + "environment_scope": "staging", + }) + + def test_updates_existing_var_in_known_scope(self, tmp_path): + existing_var = _make_variable("DB_HOST", "old-value", environment_scope="uat") + result, project = _invoke_write( + tmp_path, "DB_HOST=localhost\n", "uat", variables=[existing_var], + ) + + assert result.exit_code == 0, result.output + assert existing_var.value == "localhost" + existing_var.save.assert_called_once() + project.variables.create.assert_not_called() + + +# --- Masking heuristic --- + +class TestMaskingHeuristic: + def test_key_containing_KEY_is_masked(self, tmp_path): + result, project = _invoke_write( + tmp_path, "API_KEY=secret123\n", "prod", extra_args=["--mask"], + ) + + assert result.exit_code == 0, result.output + project.variables.create.assert_called_once() + payload = project.variables.create.call_args[0][0] + assert payload["masked"] is True + + def test_key_containing_SECRET_is_masked(self, tmp_path): + _, project = _invoke_write( + tmp_path, "MY_SECRET=shhh\n", "prod", extra_args=["--mask"], + ) + payload = project.variables.create.call_args[0][0] + assert payload["masked"] is True + + def test_key_containing_TOKEN_is_masked(self, tmp_path): + _, project = _invoke_write( + tmp_path, "AUTH_TOKEN=abc\n", "prod", extra_args=["--mask"], + ) + payload = project.variables.create.call_args[0][0] + assert payload["masked"] is True + + def test_non_matching_key_is_not_masked(self, tmp_path): + _, project = _invoke_write( + tmp_path, "APP_NAME=myapp\n", "prod", extra_args=["--mask"], + ) + payload = project.variables.create.call_args[0][0] + assert "masked" not in payload + + def test_masking_not_applied_without_flag(self, tmp_path): + _, project = _invoke_write(tmp_path, "API_KEY=secret123\n", "prod") + payload = project.variables.create.call_args[0][0] + assert "masked" not in payload + + +# --- Token-missing error consistency (Task 3) --- + +class TestMissingTokenError: + """All four commands should raise click.ClickException (clean exit, no traceback).""" + + @pytest.fixture(autouse=True) + def _clear_token(self): + with patch.dict(os.environ, {}, clear=True): + yield + + def _invoke(self, args): + runner = click.testing.CliRunner() + return runner.invoke(cli, args) + + def test_write_missing_token(self, tmp_path): + env_file = tmp_path / ".env" + env_file.write_text("X=1\n") + result = self._invoke([ + "write", "--env-file", str(env_file), + "--environment", "uat", "--gitlab-host", "h", "--project", "p", + ]) + assert result.exit_code == 1 + assert "GITLAB_TOKEN" in result.output + assert "Traceback" not in result.output + + def test_get_missing_token(self): + result = self._invoke([ + "get", "--environment", "uat", "--gitlab-host", "h", "--project", "p", + ]) + assert result.exit_code == 1 + assert "GITLAB_TOKEN" in result.output + assert "Traceback" not in result.output + + def test_list_missing_token(self): + result = self._invoke([ + "list", "--environment", "uat", "--gitlab-host", "h", "--project", "p", + ]) + assert result.exit_code == 1 + assert "GITLAB_TOKEN" in result.output + assert "Traceback" not in result.output + + def test_download_missing_token(self): + result = self._invoke([ + "download", "--environment", "uat", "--gitlab-host", "h", "--project", "p", + ]) + assert result.exit_code == 1 + assert "GITLAB_TOKEN" in result.output + assert "Traceback" not in result.output + + +# --- get --export idempotence (Task 4) --- + +class TestGetExportOverwrite: + """Running get --export twice should not duplicate lines.""" + + def test_export_does_not_duplicate_on_rerun(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + + variables = [ + _make_variable("A_VAR", "1", environment_scope="uat"), + _make_variable("B_VAR", "2", environment_scope="uat"), + ] + project = _make_project(variables=variables) + client = _make_gitlab_client(project) + + runner = click.testing.CliRunner() + with patch.dict(os.environ, {"GITLAB_TOKEN": "fake-token"}): + with patch("populate_secrets_gitlab.app.gitlab_client", return_value=client): + for _ in range(2): + result = runner.invoke(cli, [ + "get", + "--environment", "uat", + "--gitlab-host", "gitlab.example.com", + "--project", "test/project", + "--export", + ]) + assert result.exit_code == 0, result.output + + env_file = tmp_path / "uat.env" + lines = env_file.read_text().strip().splitlines() + assert len(lines) == 2 + assert "A_VAR=1" in lines + assert "B_VAR=2" in lines + + def test_export_writes_global_scope_to_global_env(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + + variables = [ + _make_variable("GLOBAL_VAR", "g", environment_scope="*"), + ] + project = _make_project(variables=variables) + client = _make_gitlab_client(project) + + runner = click.testing.CliRunner() + with patch.dict(os.environ, {"GITLAB_TOKEN": "fake-token"}): + with patch("populate_secrets_gitlab.app.gitlab_client", return_value=client): + result = runner.invoke(cli, [ + "get", + "--environment", "uat", + "--gitlab-host", "gitlab.example.com", + "--project", "test/project", + "--export", + ]) + assert result.exit_code == 0, result.output + + global_file = tmp_path / "global.env" + assert global_file.exists() + assert "GLOBAL_VAR=g" in global_file.read_text() + + +# --- Scope filtering (used by list/get/download) --- + +class TestScopeFiltering: + """Variables scoped to '*' are treated as global and included for any + environment. Variables scoped to a specific environment are only included + for that environment.""" + + def test_global_scope_included_for_any_environment(self): + variables = [ + _make_variable("GLOBAL_VAR", "g", environment_scope="*"), + _make_variable("UAT_VAR", "u", environment_scope="uat"), + ] + project = _make_project(variables=variables) + client = _make_gitlab_client(project) + + runner = click.testing.CliRunner() + with patch.dict(os.environ, {"GITLAB_TOKEN": "fake-token"}): + with patch("populate_secrets_gitlab.app.gitlab_client", return_value=client): + result = runner.invoke(cli, [ + "list", + "--environment", "uat", + "--gitlab-host", "gitlab.example.com", + "--project", "test/project", + ]) + + assert result.exit_code == 0 + assert "GLOBAL_VAR" in result.output + assert "UAT_VAR" in result.output + + def test_other_environment_vars_excluded(self): + variables = [ + _make_variable("PROD_VAR", "p", environment_scope="prod"), + ] + project = _make_project(variables=variables) + client = _make_gitlab_client(project) + + runner = click.testing.CliRunner() + with patch.dict(os.environ, {"GITLAB_TOKEN": "fake-token"}): + with patch("populate_secrets_gitlab.app.gitlab_client", return_value=client): + result = runner.invoke(cli, [ + "list", + "--environment", "uat", + "--gitlab-host", "gitlab.example.com", + "--project", "test/project", + ]) + + assert "PROD_VAR" not in result.output diff --git a/tests/test_util.py b/tests/test_util.py new file mode 100644 index 0000000..0045fc5 --- /dev/null +++ b/tests/test_util.py @@ -0,0 +1,18 @@ +from populate_secrets_gitlab.util import prepare_gitlab_host + + +class TestPrepareGitlabHost: + def test_bare_host_gets_https_prefix(self): + assert prepare_gitlab_host("gitlab.example.com") == "https://gitlab.example.com" + + def test_https_host_unchanged(self): + assert prepare_gitlab_host("https://gitlab.example.com") == "https://gitlab.example.com" + + def test_http_host_unchanged(self): + assert prepare_gitlab_host("http://gitlab.example.com") == "http://gitlab.example.com" + + def test_trailing_slash_preserved(self): + assert prepare_gitlab_host("https://gitlab.example.com/") == "https://gitlab.example.com/" + + def test_bare_host_with_trailing_slash(self): + assert prepare_gitlab_host("gitlab.example.com/") == "https://gitlab.example.com/" From 3825af3007311cbbba8a37b03c54c2442ac80fd5 Mon Sep 17 00:00:00 2001 From: Joe Niland Date: Thu, 18 Jun 2026 18:11:48 +1000 Subject: [PATCH 4/8] refactor: drop invalid %s strftime code and trailing whitespace - Remove .%s from datefmt in logging.basicConfig (%s is not a valid strftime directive and renders literally). - Remove trailing blank line in gitlab_server.py. --- src/populate_secrets_gitlab/app.py | 2 +- src/populate_secrets_gitlab/gitlab_server.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/populate_secrets_gitlab/app.py b/src/populate_secrets_gitlab/app.py index fff4dcd..5b4d9a0 100644 --- a/src/populate_secrets_gitlab/app.py +++ b/src/populate_secrets_gitlab/app.py @@ -16,7 +16,7 @@ logging.basicConfig( level=logging.INFO, format='%(asctime)s %(levelname)s\t%(message)s', - datefmt='%Y-%m-%d_%H:%M:%S.%s', + datefmt='%Y-%m-%d_%H:%M:%S', handlers=[ logging.StreamHandler() ], diff --git a/src/populate_secrets_gitlab/gitlab_server.py b/src/populate_secrets_gitlab/gitlab_server.py index 705dad0..813533b 100644 --- a/src/populate_secrets_gitlab/gitlab_server.py +++ b/src/populate_secrets_gitlab/gitlab_server.py @@ -3,5 +3,3 @@ def gitlab_client(gitlab_host, gitlab_token): return gitlab.Gitlab(util.prepare_gitlab_host(gitlab_host), private_token=gitlab_token) - - \ No newline at end of file From 853c50fe3bbe05e096c69dca31c6793e971609c5 Mon Sep 17 00:00:00 2001 From: Joe Niland Date: Thu, 18 Jun 2026 18:12:03 +1000 Subject: [PATCH 5/8] docs: clarify masking heuristic is a substring match and one-way --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d892ce3..2884cd8 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,10 @@ It provides four commands: - `write` — read a local `.env` file and create or update the matching project variables in the given environment scope. Supports `--include` / - `--exclude` filtering and `--mask` to mask values whose key contains `KEY`, - `SECRET`, or `TOKEN`. + `--exclude` filtering and `--mask` to mask values whose key contains the + substring `KEY`, `SECRET`, or `TOKEN` (e.g. `APP_KEY`, `PUBLIC_KEY`, + `AUTH_TOKEN` will all be masked). Masking is one-way: an already-masked + variable is never un-masked by this tool. - `list` — print the variables for an environment in a table. Masked values are hidden unless you pass `--sensitive`. - `get` — print the variables for an environment, optionally appending them to a From af57bb93da4a4e52ee656bf72e732a2b6613199b Mon Sep 17 00:00:00 2001 From: Joe Niland Date: Thu, 18 Jun 2026 18:12:22 +1000 Subject: [PATCH 6/8] ci: add lint and test check workflow on pull requests --- .github/workflows/check.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/check.yml diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml new file mode 100644 index 0000000..9ca50d2 --- /dev/null +++ b/.github/workflows/check.yml @@ -0,0 +1,25 @@ +name: Check + +on: + pull_request: + push: + branches-ignore: + - master + +jobs: + lint-and-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v6 + + - uses: actions/setup-python@v5 + with: + python-version: "3.9" + + - run: uv sync --group dev + + - run: uv run ruff check . + + - run: uv run pytest From b60e2c3f97d836c6677eb425115263ea39e267f3 Mon Sep 17 00:00:00 2001 From: Joe Niland Date: Thu, 18 Jun 2026 19:14:07 +1000 Subject: [PATCH 7/8] ci: add release-please config so only fix/feat commits trigger releases Without explicit config, the Python release type treats every conventional commit as release-worthy. The config marks docs, test, ci, refactor, and chore sections as hidden so they ride along in the changelog but do not trigger a version bump on their own. --- .github/workflows/release-please.yml | 2 -- .release-please-manifest.json | 3 +++ release-please-config.json | 17 +++++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 .release-please-manifest.json create mode 100644 release-please-config.json diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 2675dc0..a2f7401 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -14,5 +14,3 @@ jobs: runs-on: ubuntu-latest steps: - uses: googleapis/release-please-action@v4 - with: - release-type: python diff --git a/.release-please-manifest.json b/.release-please-manifest.json new file mode 100644 index 0000000..2537c1f --- /dev/null +++ b/.release-please-manifest.json @@ -0,0 +1,3 @@ +{ + ".": "0.4.0" +} diff --git a/release-please-config.json b/release-please-config.json new file mode 100644 index 0000000..8e801bf --- /dev/null +++ b/release-please-config.json @@ -0,0 +1,17 @@ +{ + "packages": { + ".": { + "release-type": "python", + "changelog-sections": [ + {"type": "feat", "section": "Features"}, + {"type": "fix", "section": "Bug Fixes"}, + {"type": "perf", "section": "Performance"}, + {"type": "docs", "section": "Documentation", "hidden": true}, + {"type": "test", "section": "Tests", "hidden": true}, + {"type": "ci", "section": "CI", "hidden": true}, + {"type": "refactor", "section": "Refactoring", "hidden": true}, + {"type": "chore", "section": "Miscellaneous", "hidden": true} + ] + } + } +} From 0e92fb9a815b2c6e9f7cf36990e4ccaad0b6c7ed Mon Sep 17 00:00:00 2001 From: Joe Niland Date: Thu, 18 Jun 2026 19:21:19 +1000 Subject: [PATCH 8/8] ci: fix check workflow to use uv run --group dev uv sync --group dev failed in CI. Use uv run --group dev instead, which auto-syncs before running. Add ruff to dev dependency group so it resolves through the project. --- .github/workflows/check.yml | 6 ++---- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 9ca50d2..559e0b7 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -18,8 +18,6 @@ jobs: with: python-version: "3.9" - - run: uv sync --group dev + - run: uv run --group dev ruff check . - - run: uv run ruff check . - - - run: uv run pytest + - run: uv run --group dev pytest diff --git a/pyproject.toml b/pyproject.toml index 2c59f09..3af4764 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ Issues = "https://github.com/deploymode/populate-secrets-gitlab/issues" populate-secrets-gitlab = "populate_secrets_gitlab.__main__:main" [dependency-groups] -dev = ["pytest>=8"] +dev = ["pytest>=8", "ruff>=0.8"] [tool.setuptools.packages.find] where = ["src"]