From ddcd84e8e266fe7edd7d7ea023d432a7c431e5b8 Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Thu, 16 Jul 2026 12:06:55 +0000 Subject: [PATCH 1/7] fix(auth): updates templates, adds example 2 --- .../templates/example.yaml.jinja | 12 ++--- .../templates/divisionyaml.yaml | 1 + uv.lock | 46 +++++++++---------- 3 files changed, 30 insertions(+), 29 deletions(-) diff --git a/src/copier_template/src/{{ project_name }}/templates/example.yaml.jinja b/src/copier_template/src/{{ project_name }}/templates/example.yaml.jinja index 53c3aa7..3d15c68 100644 --- a/src/copier_template/src/{{ project_name }}/templates/example.yaml.jinja +++ b/src/copier_template/src/{{ project_name }}/templates/example.yaml.jinja @@ -2,11 +2,11 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: generateName: hera-example- - namespace: None + namespace: ks10000-3 annotations: workflows.argoproj.io/description: |- Replicates the functionality of - example.yaml + example.yaml workflows.argoproj.io/title: example remade via hera workflows.diamond.ac.uk/repository: https://github.com/{{github_org}}/{{repo_name}} labels: @@ -56,7 +56,7 @@ spec: value: '{{tasks.create-image.outputs.parameters.out-paths}}' - name: install-dependencies script: - image: None + image: python:3.10 source: |- import os import sys @@ -84,7 +84,7 @@ spec: valueFrom: path: /tmp/parameters.json script: - image: None + image: python:3.10 source: |- import os import sys @@ -129,7 +129,7 @@ spec: valueFrom: path: /tmp/{{inputs.parameters.extension}}-path.json script: - image: None + image: python:3.10 source: |- import os import sys @@ -178,7 +178,7 @@ spec: archive: none: {} script: - image: None + image: python:3.10 source: |- import os import sys diff --git a/src/python_interface_to_workflows/templates/divisionyaml.yaml b/src/python_interface_to_workflows/templates/divisionyaml.yaml index ecb2814..e825918 100644 --- a/src/python_interface_to_workflows/templates/divisionyaml.yaml +++ b/src/python_interface_to_workflows/templates/divisionyaml.yaml @@ -2,6 +2,7 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: generateName: hera-division- + namespace: ks10000-3 annotations: workflows.argoproj.io/description: |- Takes a numerical input and returns diff --git a/uv.lock b/uv.lock index 271b45a..40041c5 100644 --- a/uv.lock +++ b/uv.lock @@ -618,11 +618,11 @@ wheels = [ [[package]] name = "filelock" -version = "3.31.2" +version = "3.32.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/99/2c/c603f8aedff281295f7afce455d5c05f33459c7bf684abb46228f844a1f0/filelock-3.31.2.tar.gz", hash = "sha256:e6d35965c709527915a184837a8421826d18bc3f9d7e9a5a0c8114a782475d66", size = 200476, upload-time = "2026-07-21T04:04:46.471Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c0/80/8232b582c4b318b817cf1274ba74976b07b34d35ef439b3eb948f98645a1/filelock-3.32.0.tar.gz", hash = "sha256:7be2ad23a14607ccc71808e68fe30848aeace7058ace17852f68e2a68e310402", size = 213757, upload-time = "2026-07-21T13:17:42.898Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/29/4c/c0ec6b645dcfefdd158313edb68c676d36c3f93ce8d2d9f0725d6341e506/filelock-3.31.2-py3-none-any.whl", hash = "sha256:18a4179901809ad1905c6631d907f1c3a41806c0d3506f7f9862565576a81a16", size = 97486, upload-time = "2026-07-21T04:04:44.903Z" }, + { url = "https://files.pythonhosted.org/packages/06/79/b4c714bef36bc4ec2beeae1e0c124f0223888cd8c6feb1cdc56038116920/filelock-3.32.0-py3-none-any.whl", hash = "sha256:d396bea984af47333ef05e50eae7eff88c84256de6112aea0ec48a233c064fe3", size = 97732, upload-time = "2026-07-21T13:17:41.55Z" }, ] [[package]] @@ -1372,11 +1372,11 @@ wheels = [ [[package]] name = "platformdirs" -version = "4.10.1" +version = "4.11.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/52/cd/4f25b2f95b23f5d2c9c1fe43e49841bff5800562149b2666afc09309aa8f/platformdirs-4.10.1.tar.gz", hash = "sha256:ceab4084426fe6319ce18e86deada8ab1b7487c7aee7040c55e277c9ae793695", size = 31678, upload-time = "2026-07-18T03:53:43.808Z" } +sdist = { url = "https://files.pythonhosted.org/packages/78/9b/560e4be8e26f6fd133a03630a8df0c663b9e8d61b4ade152b72005aec83b/platformdirs-4.11.0.tar.gz", hash = "sha256:0555d18370482847566ffabcaa53ad7c6c1c29f195989ae1ed634a05f76ea1e0", size = 31953, upload-time = "2026-07-21T13:09:36.565Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/73/6fd0bb9ce84138c3857f12e9de63bc901852975a092d545f18087a204aa2/platformdirs-4.10.1-py3-none-any.whl", hash = "sha256:0e4eff26be2d75293977f7cddc153fd9b8eaa7fb0c7b64ffe4076cb443117443", size = 22906, upload-time = "2026-07-18T03:53:42.576Z" }, + { url = "https://files.pythonhosted.org/packages/7d/68/d8d58938dfb1370b266a1a729e6d77a985be23689a0496498ee17b2cbf90/platformdirs-4.11.0-py3-none-any.whl", hash = "sha256:360ccded2b7fce0af0ff80cc8f5942a1c5d99b0e856033acb030bfc634709e74", size = 23247, upload-time = "2026-07-21T13:09:35.422Z" }, ] [[package]] @@ -1676,14 +1676,14 @@ wheels = [ [[package]] name = "pyproject-api" -version = "1.10.1" +version = "1.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "packaging" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/62/62/0fe346fe380b1aafaf819c8cb195d3241bb4f355f908e6339814131a830b/pyproject_api-1.10.1.tar.gz", hash = "sha256:c2b2726bd7aa9217b6c50b621fef5b2ae5def4d55b779c9e0694c15e0a8517ba", size = 23477, upload-time = "2026-05-28T14:22:14.049Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ab/bd/2f985c12bf33fdd8637e3a8f9418d6806f177601dee7c4924d0b2bb28650/pyproject_api-1.11.0.tar.gz", hash = "sha256:b8807d85a293e6c9f133e6575946fed45f1d42b22d58c780b33aa2421a799549", size = 23787, upload-time = "2026-07-21T13:09:33.744Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/79/d7/29e1e5e882f79133631f7bcace42d23db493f616463c157a1ab614bf69dd/pyproject_api-1.10.1-py3-none-any.whl", hash = "sha256:fa9e6f66c35b5017e909825d8f2b5d5482ea699d7be809d21c03bd1f7317f36a", size = 12992, upload-time = "2026-05-28T14:22:12.711Z" }, + { url = "https://files.pythonhosted.org/packages/a3/6a/2822ee12bafe87af8f6cc620f7d1f126e77f82ce56ba888b94a9af5a979c/pyproject_api-1.11.0-py3-none-any.whl", hash = "sha256:860060c8832dce983b5eec6f41c4c43eb3ec06ff7332387a63acdf5ca27b68d8", size = 13275, upload-time = "2026-07-21T13:09:32.559Z" }, ] [[package]] @@ -1731,15 +1731,15 @@ wheels = [ [[package]] name = "python-discovery" -version = "1.4.4" +version = "1.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "filelock" }, { name = "platformdirs" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4c/81/58c70036dffeccb7fe7d79d6260c69f7a28272bbd3909c29a01ea9422744/python_discovery-1.4.4.tar.gz", hash = "sha256:5cad33982d412c1f3ffb8f9ca4ea292c9680bca3942451d30b69c37fce53a4a3", size = 72212, upload-time = "2026-07-08T23:06:50.691Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/51/276f964496a5714ab9f320896195639086881c2b39c03b5ad13de84acbb8/python_discovery-1.5.0.tar.gz", hash = "sha256:3e014c6327154d3dda27939a9a0dc9c5c000439f1906d3f303b48f984bd2ecef", size = 72483, upload-time = "2026-07-21T13:14:14.641Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/ae/84bc0d2440c95772272bb6f4b3d09ccf08b2898fce89b3d4f969a9fc74e9/python_discovery-1.4.4-py3-none-any.whl", hash = "sha256:abebe9120b43453b68c908acfb1e72a19d1a959ed2cb620ad38fc57d08056dbe", size = 34181, upload-time = "2026-07-08T23:06:49.402Z" }, + { url = "https://files.pythonhosted.org/packages/c7/7b/14882602ddee241d7984a742fcb423cb4a30fb0d6efc546ac3129fba475a/python_discovery-1.5.0-py3-none-any.whl", hash = "sha256:70c4fc61b4e7404e44f01d6fc44a715c4d685ca6cea83d295922f05891877c98", size = 34205, upload-time = "2026-07-21T13:14:13.398Z" }, ] [[package]] @@ -2020,7 +2020,7 @@ wheels = [ [[package]] name = "tox" -version = "4.57.2" +version = "4.58.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cachetools" }, @@ -2034,34 +2034,34 @@ dependencies = [ { name = "tomli-w" }, { name = "virtualenv" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/09/c5/511b56f901c9a2c47f081a53ff7929a20e69d010930710acb70ff1ac96d4/tox-4.57.2.tar.gz", hash = "sha256:d23465ed07fd743f3b20b21a4991f4932ed4edb392bf32c0b9f7ce4415d2e89d", size = 296928, upload-time = "2026-07-21T01:15:25.841Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6e/8e/4d2b1b2a81f4de1cd4e54fa40df1ab5f9bb88fe2e37461fc44aba8f9d302/tox-4.58.0.tar.gz", hash = "sha256:ab0b126a04dd56bc18e6d216386db09335247f2289b54cf534deb5c4ae3a8d2e", size = 296926, upload-time = "2026-07-21T13:10:36.622Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/d7/201d7adfd7372ff3354a39e85cb70f95e36b91aff6bcc5547155d6ea6a5c/tox-4.57.2-py3-none-any.whl", hash = "sha256:2c187558a0d336b5e658025649f3c0e61aad080fdc674740e213d93bd48506ee", size = 223298, upload-time = "2026-07-21T01:15:24.115Z" }, + { url = "https://files.pythonhosted.org/packages/6b/3d/7ba55871e9d794d40b6c8424f2e5d1b267ea5d9a4bd2175e08b57960ba13/tox-4.58.0-py3-none-any.whl", hash = "sha256:dcae21f5f015f3a67658e35644cce0d1aa0dedcd06f3927f95d84e1717f6cea5", size = 223298, upload-time = "2026-07-21T13:10:34.731Z" }, ] [[package]] name = "tox-uv" -version = "1.35.2" +version = "1.36.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "tox-uv-bare" }, { name = "uv" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/ca/dc/6e9994c799bdbb309f829dd6b8d98764dd0757302f3433c380438a3a127b/tox_uv-1.35.2-py3-none-any.whl", hash = "sha256:2d99b0e3c782ba49e7cbe521c8d344758595961b17a3633738d67096641c1bde", size = 6565, upload-time = "2026-05-05T01:34:16.07Z" }, + { url = "https://files.pythonhosted.org/packages/f9/d7/3fce976b9e295218a2d7e541e0f1e9b1259cb893de7a8d277952a66a798d/tox_uv-1.36.0-py3-none-any.whl", hash = "sha256:5f81b39be3fe4e14c6b9bb7ba637ed97d34efa6214efd5f525d95ee9559a99ac", size = 6564, upload-time = "2026-07-21T13:09:54.316Z" }, ] [[package]] name = "tox-uv-bare" -version = "1.35.2" +version = "1.36.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "packaging" }, { name = "tox" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/0a/cb/168dc1ccf24e4065a9a0a33df55709ed2b5eb73bd2b13ddd53187e5dffb8/tox_uv_bare-1.35.2.tar.gz", hash = "sha256:49e28a804c97f23ea17e25859960c0fa78f35bccb7e14344cfd840e89a9aade9", size = 32333, upload-time = "2026-05-05T01:34:18.916Z" } +sdist = { url = "https://files.pythonhosted.org/packages/56/df/9f90a59de8c87cece6e691eee53dc1cb0a824d73aea8f380ae2f6ecb71de/tox_uv_bare-1.36.0.tar.gz", hash = "sha256:d9b0a2fd0f74fa65d9597108f8a0ef7abb08651c9196a998a6073b781b45cfd0", size = 32548, upload-time = "2026-07-21T13:09:56.435Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5f/53/4a33dc81da39db7b31e5622333df361e8fe055b7ec636bd5fea762c9182d/tox_uv_bare-1.35.2-py3-none-any.whl", hash = "sha256:c0d590a41d1054a1ad0874e9e5943ff52402786e3d4599d8f8d37a65b566ef53", size = 22307, upload-time = "2026-05-05T01:34:17.681Z" }, + { url = "https://files.pythonhosted.org/packages/f7/0a/6dc462e4fb543305283a6157c80f43e3d12ca4702da6ae6521d541c6b55c/tox_uv_bare-1.36.0-py3-none-any.whl", hash = "sha256:ba397dd0396df95a75744d4e42a50ee27207c0ffcf277b62ffba9c3de455a939", size = 22489, upload-time = "2026-07-21T13:09:55.389Z" }, ] [[package]] @@ -2131,7 +2131,7 @@ wheels = [ [[package]] name = "virtualenv" -version = "21.6.1" +version = "21.7.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "distlib" }, @@ -2139,9 +2139,9 @@ dependencies = [ { name = "platformdirs" }, { name = "python-discovery" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/34/d9/b477fddb68840b570af8b22afe9b035cbc277b5fb7b33dea390617a8b10f/virtualenv-21.6.1.tar.gz", hash = "sha256:15f978b7cd329f24855ff4a0c4b4899cc7678589f49adbdcbbb4d3232e641128", size = 5526620, upload-time = "2026-07-10T19:33:53.312Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/25/e367a7229b0914772ca8d81b41fde012d9feda68523b52644a571bb21ce8/virtualenv-21.7.0.tar.gz", hash = "sha256:7f9519b9432ff11b6e1a3e94061664efc2ff99ea21780e3cf4f6bd0a5da8b37c", size = 5527510, upload-time = "2026-07-21T13:12:14.109Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/7c/4e7225d46d634a0d8d534dd8a6ce0c319d09b4d0cf0337eb314ca4789d8c/virtualenv-21.6.1-py3-none-any.whl", hash = "sha256:afe991df855715a2b2f60edfcc0107ef95a79fdfd8cb4cdaa71603d1c12e463b", size = 5506392, upload-time = "2026-07-10T19:33:51.629Z" }, + { url = "https://files.pythonhosted.org/packages/a5/7a/ae29312b1e88a22e81f5d21fc11526d2a114089776c2550d2b205b6c2a47/virtualenv-21.7.0-py3-none-any.whl", hash = "sha256:a8370c1c5530fbabf955e40b8fbbc68a431648b10f9433faa587db30a06e51dd", size = 5507078, upload-time = "2026-07-21T13:12:12.136Z" }, ] [[package]] From 828731a20dddd6d63988d4c6e5603c341bb89a1f Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Fri, 17 Jul 2026 10:42:22 +0000 Subject: [PATCH 2/7] feat(auth): updates templates to align with keycloak, .env variables --- .devcontainer/devcontainer.json | 3 ++ src/copier_template/src/.env-sample | 5 ++ .../auth/keycloak_checker.py | 47 +++++++++++++++++++ .../{{ project_name }}/auth/open_auth_url.py | 7 ++- .../templates/example.yaml.jinja | 2 +- .../tests/test_keycloak_checker.py.jinja | 2 +- .../tests/test_open_auth_url.py.jinja | 4 +- .../auth/open_auth_url.py | 3 ++ 8 files changed, 67 insertions(+), 6 deletions(-) create mode 100644 src/copier_template/src/.env-sample create mode 100644 src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 9c6f9c8..5173e8f 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -53,6 +53,9 @@ "--env-file", "src/.env" ], + "forwardPorts": [ + 5173 + ], "portsAttributes": { "5173": { "onAutoForward": "ignore" diff --git a/src/copier_template/src/.env-sample b/src/copier_template/src/.env-sample new file mode 100644 index 0000000..15fddf5 --- /dev/null +++ b/src/copier_template/src/.env-sample @@ -0,0 +1,5 @@ +HOST= +IMAGE= +TOKEN= +NAMESPACE= +AUTH= diff --git a/src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py b/src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py new file mode 100644 index 0000000..5cd279f --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py @@ -0,0 +1,47 @@ +import os + +import dotenv +from keycloak import KeycloakOpenID +from keycloak.pkce_utils import generate_code_challenge, generate_code_verifier + +from python_interface_to_workflows.auth.open_auth_url import open_auth_url + + +def set_token_env_variable(staging: bool) -> str: + match staging: + case True: + keycloak_openid = KeycloakOpenID( + server_url="https://identity-test.diamond.ac.uk/", + client_id="workflows-ui-dev", + realm_name="dls", + client_secret_key="", + pool_maxsize=1, + ) + case False: + keycloak_openid = KeycloakOpenID( + client_id="workflows-dashboard", + server_url="https://identity.diamond.ac.uk/", + realm_name="dls", + client_secret_key="", + pool_maxsize=1, + ) + code_verifier = generate_code_verifier() + code_challenge, code_challenge_method = generate_code_challenge(code_verifier) + auth_url = keycloak_openid.auth_url( + redirect_uri="http://localhost:5173/", + scope="openid posix-uid profile email fedid", + state="", + code_challenge=code_challenge, + code_challenge_method=code_challenge_method, + ) + open_auth_url(auth_url) + token: dict[str, str] = ( # pyright: ignore[reportUnknownVariableType] + keycloak_openid.token( # pyright: ignore[reportUnknownMemberType] + grant_type="authorization_code", + code=os.environ["AUTH"], + redirect_uri="http://localhost:5173/", + code_verifier=code_verifier, + ) + ) + dotenv.set_key("src/.env", "TOKEN", token["access_token"].strip("'")) + return token["access_token"] # pyright: ignore[reportUnknownArgumentType] diff --git a/src/copier_template/src/{{ project_name }}/auth/open_auth_url.py b/src/copier_template/src/{{ project_name }}/auth/open_auth_url.py index 5a18266..72430a9 100644 --- a/src/copier_template/src/{{ project_name }}/auth/open_auth_url.py +++ b/src/copier_template/src/{{ project_name }}/auth/open_auth_url.py @@ -5,6 +5,8 @@ from http.server import BaseHTTPRequestHandler, HTTPServer from typing import cast +import dotenv + class _ReusingHTTPServer(HTTPServer): allow_reuse_address = True @@ -26,12 +28,13 @@ def do_GET(self): self.wfile.write(b"Missing authorization code.") -def open_auth_url(auth_url: str) -> None: - httpd = _ReusingHTTPServer(("localhost", 8000), CallbackHandler) +def open_auth_url(auth_url: str): + httpd = _ReusingHTTPServer(("localhost", 5173), CallbackHandler) webbrowser.open(auth_url) try: httpd.handle_request() os.environ["AUTH"] = httpd.auth_code + dotenv.set_key("src/.env", "AUTH", httpd.auth_code) except OSError: os.environ["AUTH"] = "" print("ERROR: Port in use. Please restart your terminal.") diff --git a/src/copier_template/src/{{ project_name }}/templates/example.yaml.jinja b/src/copier_template/src/{{ project_name }}/templates/example.yaml.jinja index 3d15c68..9fc76d2 100644 --- a/src/copier_template/src/{{ project_name }}/templates/example.yaml.jinja +++ b/src/copier_template/src/{{ project_name }}/templates/example.yaml.jinja @@ -6,7 +6,7 @@ metadata: annotations: workflows.argoproj.io/description: |- Replicates the functionality of - example.yaml + example.yaml workflows.argoproj.io/title: example remade via hera workflows.diamond.ac.uk/repository: https://github.com/{{github_org}}/{{repo_name}} labels: diff --git a/src/copier_template/tests/test_keycloak_checker.py.jinja b/src/copier_template/tests/test_keycloak_checker.py.jinja index eb6459b..a56ea98 100644 --- a/src/copier_template/tests/test_keycloak_checker.py.jinja +++ b/src/copier_template/tests/test_keycloak_checker.py.jinja @@ -28,7 +28,7 @@ def test_return_key( "refresh_token": "fake_refresh", } assert set_token_env_variable() == "fake_token" - mock_open_auth_url.assert_called_once_with("https://mock.site", port) + mock_open_auth_url.assert_called_once_with("https://mock.site", 8000) keycloak.token.assert_called_once_with( grant_type="authorization_code", code="auth_url_code", diff --git a/src/copier_template/tests/test_open_auth_url.py.jinja b/src/copier_template/tests/test_open_auth_url.py.jinja index 4873694..f6d3393 100644 --- a/src/copier_template/tests/test_open_auth_url.py.jinja +++ b/src/copier_template/tests/test_open_auth_url.py.jinja @@ -15,7 +15,7 @@ def test_open_auth_url_normal_function( ): server = mock_http_server.return_value server.auth_code = "this_is_your_code" - open_auth_url("url", 5173) + open_auth_url("url") server.handle_request.assert_called_once() mock_open_browser.assert_called_once_with("url") server.socket.shutdown.assert_called_once() @@ -34,7 +34,7 @@ def test_open_auth_url_raises_error( server = mock_http_server.return_value server.auth_code = "this_is_your_code" server.handle_request.side_effect = OSError - open_auth_url("url", 5173) + open_auth_url("url") mock_open_browser.assert_called_once_with("url") assert os.environ["AUTH"] == "" mock_exit.assert_called_once_with(1) diff --git a/src/python_interface_to_workflows/auth/open_auth_url.py b/src/python_interface_to_workflows/auth/open_auth_url.py index 8f938c9..2f0dcf9 100644 --- a/src/python_interface_to_workflows/auth/open_auth_url.py +++ b/src/python_interface_to_workflows/auth/open_auth_url.py @@ -5,6 +5,8 @@ from http.server import BaseHTTPRequestHandler, HTTPServer from typing import cast +import dotenv + class _ReusingHTTPServer(HTTPServer): allow_reuse_address = True @@ -32,6 +34,7 @@ def open_auth_url(auth_url: str, port: int) -> None: try: httpd.handle_request() os.environ["AUTH"] = httpd.auth_code + dotenv.set_key("src/.env", "AUTH", httpd.auth_code) except OSError: os.environ["AUTH"] = "" print("ERROR: Port in use. Please restart your terminal.") From de8af43143154cded740a19b23c379d4259751c0 Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Tue, 21 Jul 2026 09:52:28 +0000 Subject: [PATCH 3/7] feat(copier): updates copier template to new standards --- src/copier_template/src/.env-sample | 5 -- .../auth/keycloak_checker.py | 47 ------------------- .../{{ project_name }}/auth/open_auth_url.py | 7 +-- .../templates/example.yaml.jinja | 10 ++-- .../tests/test_keycloak_checker.py.jinja | 2 - .../templates/divisionyaml.yaml | 1 - 6 files changed, 7 insertions(+), 65 deletions(-) delete mode 100644 src/copier_template/src/.env-sample delete mode 100644 src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py diff --git a/src/copier_template/src/.env-sample b/src/copier_template/src/.env-sample deleted file mode 100644 index 15fddf5..0000000 --- a/src/copier_template/src/.env-sample +++ /dev/null @@ -1,5 +0,0 @@ -HOST= -IMAGE= -TOKEN= -NAMESPACE= -AUTH= diff --git a/src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py b/src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py deleted file mode 100644 index 5cd279f..0000000 --- a/src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py +++ /dev/null @@ -1,47 +0,0 @@ -import os - -import dotenv -from keycloak import KeycloakOpenID -from keycloak.pkce_utils import generate_code_challenge, generate_code_verifier - -from python_interface_to_workflows.auth.open_auth_url import open_auth_url - - -def set_token_env_variable(staging: bool) -> str: - match staging: - case True: - keycloak_openid = KeycloakOpenID( - server_url="https://identity-test.diamond.ac.uk/", - client_id="workflows-ui-dev", - realm_name="dls", - client_secret_key="", - pool_maxsize=1, - ) - case False: - keycloak_openid = KeycloakOpenID( - client_id="workflows-dashboard", - server_url="https://identity.diamond.ac.uk/", - realm_name="dls", - client_secret_key="", - pool_maxsize=1, - ) - code_verifier = generate_code_verifier() - code_challenge, code_challenge_method = generate_code_challenge(code_verifier) - auth_url = keycloak_openid.auth_url( - redirect_uri="http://localhost:5173/", - scope="openid posix-uid profile email fedid", - state="", - code_challenge=code_challenge, - code_challenge_method=code_challenge_method, - ) - open_auth_url(auth_url) - token: dict[str, str] = ( # pyright: ignore[reportUnknownVariableType] - keycloak_openid.token( # pyright: ignore[reportUnknownMemberType] - grant_type="authorization_code", - code=os.environ["AUTH"], - redirect_uri="http://localhost:5173/", - code_verifier=code_verifier, - ) - ) - dotenv.set_key("src/.env", "TOKEN", token["access_token"].strip("'")) - return token["access_token"] # pyright: ignore[reportUnknownArgumentType] diff --git a/src/copier_template/src/{{ project_name }}/auth/open_auth_url.py b/src/copier_template/src/{{ project_name }}/auth/open_auth_url.py index 72430a9..5a18266 100644 --- a/src/copier_template/src/{{ project_name }}/auth/open_auth_url.py +++ b/src/copier_template/src/{{ project_name }}/auth/open_auth_url.py @@ -5,8 +5,6 @@ from http.server import BaseHTTPRequestHandler, HTTPServer from typing import cast -import dotenv - class _ReusingHTTPServer(HTTPServer): allow_reuse_address = True @@ -28,13 +26,12 @@ def do_GET(self): self.wfile.write(b"Missing authorization code.") -def open_auth_url(auth_url: str): - httpd = _ReusingHTTPServer(("localhost", 5173), CallbackHandler) +def open_auth_url(auth_url: str) -> None: + httpd = _ReusingHTTPServer(("localhost", 8000), CallbackHandler) webbrowser.open(auth_url) try: httpd.handle_request() os.environ["AUTH"] = httpd.auth_code - dotenv.set_key("src/.env", "AUTH", httpd.auth_code) except OSError: os.environ["AUTH"] = "" print("ERROR: Port in use. Please restart your terminal.") diff --git a/src/copier_template/src/{{ project_name }}/templates/example.yaml.jinja b/src/copier_template/src/{{ project_name }}/templates/example.yaml.jinja index 9fc76d2..53c3aa7 100644 --- a/src/copier_template/src/{{ project_name }}/templates/example.yaml.jinja +++ b/src/copier_template/src/{{ project_name }}/templates/example.yaml.jinja @@ -2,7 +2,7 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: generateName: hera-example- - namespace: ks10000-3 + namespace: None annotations: workflows.argoproj.io/description: |- Replicates the functionality of @@ -56,7 +56,7 @@ spec: value: '{{tasks.create-image.outputs.parameters.out-paths}}' - name: install-dependencies script: - image: python:3.10 + image: None source: |- import os import sys @@ -84,7 +84,7 @@ spec: valueFrom: path: /tmp/parameters.json script: - image: python:3.10 + image: None source: |- import os import sys @@ -129,7 +129,7 @@ spec: valueFrom: path: /tmp/{{inputs.parameters.extension}}-path.json script: - image: python:3.10 + image: None source: |- import os import sys @@ -178,7 +178,7 @@ spec: archive: none: {} script: - image: python:3.10 + image: None source: |- import os import sys diff --git a/src/copier_template/tests/test_keycloak_checker.py.jinja b/src/copier_template/tests/test_keycloak_checker.py.jinja index a56ea98..2eda176 100644 --- a/src/copier_template/tests/test_keycloak_checker.py.jinja +++ b/src/copier_template/tests/test_keycloak_checker.py.jinja @@ -1,8 +1,6 @@ import os from unittest.mock import MagicMock, patch -from pytest import mark - from {{project_name}}.auth.keycloak_checker import set_token_env_variable diff --git a/src/python_interface_to_workflows/templates/divisionyaml.yaml b/src/python_interface_to_workflows/templates/divisionyaml.yaml index e825918..ecb2814 100644 --- a/src/python_interface_to_workflows/templates/divisionyaml.yaml +++ b/src/python_interface_to_workflows/templates/divisionyaml.yaml @@ -2,7 +2,6 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: generateName: hera-division- - namespace: ks10000-3 annotations: workflows.argoproj.io/description: |- Takes a numerical input and returns From e615563d9b3b7cb14b64fe6670dc33c85e96234e Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Wed, 22 Jul 2026 09:41:42 +0000 Subject: [PATCH 4/7] feat(api): reworks submission to argo --- .../submit_to_argo.py.jinja | 23 ++++++++++++++++++ .../templates/example.yaml.jinja | 3 +-- .../create_example_template.py.jinja | 3 +-- .../tests/test_prepare_to_submit.py.jinja | 18 -------------- .../tests/test_submit_workflow_to_argo.jinja | 20 ++++++++++++++++ .../prepare_to_submit.py | 14 ----------- .../submit_to_argo.py | 24 +++++++++++++++++++ .../templates/divisionyaml.yaml | 2 +- .../templates/example.yaml | 3 +-- .../create_division_yaml.py | 4 +--- .../create_example_template.py | 3 +-- tests/test_prepare_to_submit.py | 18 -------------- tests/test_submit_to_argo.py | 20 ++++++++++++++++ 13 files changed, 93 insertions(+), 62 deletions(-) create mode 100644 src/copier_template/src/{{ project_name }}/submit_to_argo.py.jinja delete mode 100644 src/copier_template/tests/test_prepare_to_submit.py.jinja create mode 100644 src/copier_template/tests/test_submit_workflow_to_argo.jinja delete mode 100644 src/python_interface_to_workflows/prepare_to_submit.py create mode 100644 src/python_interface_to_workflows/submit_to_argo.py delete mode 100644 tests/test_prepare_to_submit.py create mode 100644 tests/test_submit_to_argo.py diff --git a/src/copier_template/src/{{ project_name }}/submit_to_argo.py.jinja b/src/copier_template/src/{{ project_name }}/submit_to_argo.py.jinja new file mode 100644 index 0000000..ff9bcfa --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/submit_to_argo.py.jinja @@ -0,0 +1,23 @@ +import os +from typing import cast + +import dotenv +from hera.shared import global_config +from hera.workflows import Workflow, WorkflowsService +from hera.workflows import models as m + +from {{project_name}}.auth.keycloak_checker import set_token_env_variable + + +def submit_workflow_to_argo(w: Workflow): + set_token_env_variable(staging=True) + dotenv.load_dotenv(dotenv_path="src/.env", override=True) + w.namespace = os.environ.get("NAMESPACE") + w.workflows_service = WorkflowsService( + host=str(os.environ.get("HOST")).strip("'"), + token=str(os.environ.get("TOKEN")).strip("'"), + ) + submitted_w = cast(m.Workflow, w.create()) + name = submitted_w.metadata.name + namespace = submitted_w.metadata.namespace + print(f"submitted {name} to {namespace}") diff --git a/src/copier_template/src/{{ project_name }}/templates/example.yaml.jinja b/src/copier_template/src/{{ project_name }}/templates/example.yaml.jinja index 53c3aa7..017bfb2 100644 --- a/src/copier_template/src/{{ project_name }}/templates/example.yaml.jinja +++ b/src/copier_template/src/{{ project_name }}/templates/example.yaml.jinja @@ -2,11 +2,10 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: generateName: hera-example- - namespace: None annotations: workflows.argoproj.io/description: |- Replicates the functionality of - example.yaml + example.yaml workflows.argoproj.io/title: example remade via hera workflows.diamond.ac.uk/repository: https://github.com/{{github_org}}/{{repo_name}} labels: diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja index bca5375..669cb23 100644 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja @@ -152,14 +152,13 @@ def to_hdf5(paths: str): with Workflow( generate_name="hera-example-", # when running on graphql this should be name entrypoint="workflowentry", - namespace=str(os.environ.get("NAMESPACE")), api_version="argoproj.io/v1alpha1", kind="Workflow", # ClusterWorkflowTemplate", when on graphql labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, annotations={ "workflows.argoproj.io/title": "example remade via hera", "workflows.argoproj.io/description": """Replicates the functionality of - example.yaml""", +example.yaml""", "workflows.diamond.ac.uk/repository": "https://github.com/{{github_org}}/{{repo_name}}", }, volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), diff --git a/src/copier_template/tests/test_prepare_to_submit.py.jinja b/src/copier_template/tests/test_prepare_to_submit.py.jinja deleted file mode 100644 index b9fba6a..0000000 --- a/src/copier_template/tests/test_prepare_to_submit.py.jinja +++ /dev/null @@ -1,18 +0,0 @@ -from unittest.mock import MagicMock, call, patch - -from {{ project_name }}.prepare_to_submit import prepare_to_submit - - -@patch("{{ project_name }}.prepare_to_submit.os.environ.get") -@patch("{{ project_name }}.prepare_to_submit.dotenv.load_dotenv") -@patch("{{ project_name }}.prepare_to_submit.set_token_env_variable") -def test_prep_submit_a_workflow( - mock_key: MagicMock, mock_load_env: MagicMock, mock_get_env_var: MagicMock -): - - prepare_to_submit(True) - mock_key.assert_called_once_with(staging=True) - mock_load_env.assert_called_once_with(dotenv_path="src/.env", override=True) - mock_get_env_var.assert_has_calls( - [call("HOST"), call("IMAGE"), call("TOKEN")], any_order=True - ) diff --git a/src/copier_template/tests/test_submit_workflow_to_argo.jinja b/src/copier_template/tests/test_submit_workflow_to_argo.jinja new file mode 100644 index 0000000..c6eef98 --- /dev/null +++ b/src/copier_template/tests/test_submit_workflow_to_argo.jinja @@ -0,0 +1,20 @@ +from unittest.mock import MagicMock, call, patch + +from {{project_name}}.submit_to_argo import submit_workflow_to_argo + + +@patch("{{project_name}}.submit_to_argo.Workflow") +@patch("{{project_name}}.submit_to_argo.os.environ.get") +@patch("{{project_name}}.submit_to_argo.dotenv.load_dotenv") +@patch("{{project_name}}.submit_to_argo.set_token_env_variable") +def test_submit_workflow_to_argo( + mock_key: MagicMock, + mock_load_env: MagicMock, + mock_get_env_var: MagicMock, + mock_workflow: MagicMock, +): + mock_workflow = MagicMock() + submit_workflow_to_argo(mock_workflow) + mock_key.assert_called_once_with(staging=True) + mock_load_env.assert_called_once_with(dotenv_path="src/.env", override=True) + mock_get_env_var.assert_has_calls([call("HOST"), call("TOKEN")], any_order=True) diff --git a/src/python_interface_to_workflows/prepare_to_submit.py b/src/python_interface_to_workflows/prepare_to_submit.py deleted file mode 100644 index d595c5e..0000000 --- a/src/python_interface_to_workflows/prepare_to_submit.py +++ /dev/null @@ -1,14 +0,0 @@ -import os - -import dotenv -from hera.shared import global_config - -from python_interface_to_workflows.auth.keycloak_checker import set_token_env_variable - - -def prepare_to_submit(staging: bool): - set_token_env_variable(staging=staging) - dotenv.load_dotenv(dotenv_path="src/.env", override=True) - global_config.host = str(os.environ.get("HOST")) - global_config.image = str(os.environ.get("IMAGE")) - global_config.token = str(os.environ.get("TOKEN")) diff --git a/src/python_interface_to_workflows/submit_to_argo.py b/src/python_interface_to_workflows/submit_to_argo.py new file mode 100644 index 0000000..343d423 --- /dev/null +++ b/src/python_interface_to_workflows/submit_to_argo.py @@ -0,0 +1,24 @@ +import os +from typing import cast + +import dotenv +from hera.shared import global_config +from hera.workflows import Workflow, WorkflowsService +from hera.workflows import models as m + +from python_interface_to_workflows.auth.keycloak_checker import set_token_env_variable + + +def submit_workflow_to_argo(w: Workflow): + set_token_env_variable(staging=True) + dotenv.load_dotenv(dotenv_path="src/.env", override=True) + w.namespace = os.environ.get("NAMESPACE") + w.workflows_service = WorkflowsService( + host=str(os.environ.get("HOST")).strip("'"), + token=str(os.environ.get("TOKEN")).strip("'"), + ) + global_config.image = str(os.environ.get("IMAGE")) + submitted_w = cast(m.Workflow, w.create()) + name = submitted_w.metadata.name + namespace = submitted_w.metadata.namespace + print(f"submitted {name} to {namespace}") diff --git a/src/python_interface_to_workflows/templates/divisionyaml.yaml b/src/python_interface_to_workflows/templates/divisionyaml.yaml index ecb2814..34bed16 100644 --- a/src/python_interface_to_workflows/templates/divisionyaml.yaml +++ b/src/python_interface_to_workflows/templates/divisionyaml.yaml @@ -5,7 +5,7 @@ metadata: annotations: workflows.argoproj.io/description: |- Takes a numerical input and returns - the remainder, output float, and output string to a json file + the remainder, output float, and output string to a json file workflows.argoproj.io/title: Division via hera test workflows.diamond.ac.uk/repository: https://github.com/DiamondLightSource/python-interface-to-workflows labels: diff --git a/src/python_interface_to_workflows/templates/example.yaml b/src/python_interface_to_workflows/templates/example.yaml index 69683b9..d851b6d 100644 --- a/src/python_interface_to_workflows/templates/example.yaml +++ b/src/python_interface_to_workflows/templates/example.yaml @@ -2,11 +2,10 @@ apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: generateName: hera-example- - namespace: None annotations: workflows.argoproj.io/description: |- Replicates the functionality of - example.yaml + example.yaml workflows.argoproj.io/title: example remade via hera workflows.diamond.ac.uk/repository: https://github.com/DiamondLightSource/python-interface-to-workflows labels: diff --git a/src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py b/src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py index d80824c..f481093 100644 --- a/src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py +++ b/src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py @@ -1,5 +1,4 @@ import json -import os from hera.workflows import ( Artifact, @@ -31,14 +30,13 @@ def do_division(a: int, b: int): with Workflow( generate_name="hera-division-", # when running on graphql this should be name entrypoint="divide", - namespace=os.environ.get("NAMESPACE"), api_version="argoproj.io/v1alpha1", kind="Workflow", # ClusterWorkflowTemplate", when on graphql labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, annotations={ "workflows.argoproj.io/title": "Division via hera test", "workflows.argoproj.io/description": """Takes a numerical input and returns - the remainder, output float, and output string to a json file""", + the remainder, output float, and output string to a json file""", "workflows.diamond.ac.uk/repository": "https://github.com/DiamondLightSource/python-interface-to-workflows", }, volumes=EmptyDirVolume(name="output-dir", mount_path="/output-dir"), diff --git a/src/python_interface_to_workflows/workflow_definitions/create_example_template.py b/src/python_interface_to_workflows/workflow_definitions/create_example_template.py index 814004a..5cece7d 100644 --- a/src/python_interface_to_workflows/workflow_definitions/create_example_template.py +++ b/src/python_interface_to_workflows/workflow_definitions/create_example_template.py @@ -152,14 +152,13 @@ def to_hdf5(paths: str): with Workflow( generate_name="hera-example-", # when running on graphql this should be name entrypoint="workflowentry", - namespace=str(os.environ.get("NAMESPACE")), api_version="argoproj.io/v1alpha1", kind="Workflow", # ClusterWorkflowTemplate", when on graphql labels={"workflows.diamond.ac.uk/science-group-examples": "true"}, annotations={ "workflows.argoproj.io/title": "example remade via hera", "workflows.argoproj.io/description": """Replicates the functionality of - example.yaml""", +example.yaml""", "workflows.diamond.ac.uk/repository": "https://github.com/DiamondLightSource/python-interface-to-workflows", }, volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), diff --git a/tests/test_prepare_to_submit.py b/tests/test_prepare_to_submit.py deleted file mode 100644 index 06485bd..0000000 --- a/tests/test_prepare_to_submit.py +++ /dev/null @@ -1,18 +0,0 @@ -from unittest.mock import MagicMock, call, patch - -from python_interface_to_workflows.prepare_to_submit import prepare_to_submit - - -@patch("python_interface_to_workflows.prepare_to_submit.os.environ.get") -@patch("python_interface_to_workflows.prepare_to_submit.dotenv.load_dotenv") -@patch("python_interface_to_workflows.prepare_to_submit.set_token_env_variable") -def test_prep_submit_a_workflow( - mock_key: MagicMock, mock_load_env: MagicMock, mock_get_env_var: MagicMock -): - - prepare_to_submit(True) - mock_key.assert_called_once_with(staging=True) - mock_load_env.assert_called_once_with(dotenv_path="src/.env", override=True) - mock_get_env_var.assert_has_calls( - [call("HOST"), call("IMAGE"), call("TOKEN")], any_order=True - ) diff --git a/tests/test_submit_to_argo.py b/tests/test_submit_to_argo.py new file mode 100644 index 0000000..b8be696 --- /dev/null +++ b/tests/test_submit_to_argo.py @@ -0,0 +1,20 @@ +from unittest.mock import MagicMock, call, patch + +from python_interface_to_workflows.submit_to_argo import submit_workflow_to_argo + + +@patch("python_interface_to_workflows.submit_to_argo.Workflow") +@patch("python_interface_to_workflows.submit_to_argo.os.environ.get") +@patch("python_interface_to_workflows.submit_to_argo.dotenv.load_dotenv") +@patch("python_interface_to_workflows.submit_to_argo.set_token_env_variable") +def test_submit_workflow_to_argo( + mock_key: MagicMock, + mock_load_env: MagicMock, + mock_get_env_var: MagicMock, + mock_workflow: MagicMock, +): + mock_workflow = MagicMock() + submit_workflow_to_argo(mock_workflow) + mock_key.assert_called_once_with(staging=True) + mock_load_env.assert_called_once_with(dotenv_path="src/.env", override=True) + mock_get_env_var.assert_has_calls([call("HOST"), call("TOKEN")], any_order=True) From 623cab0a3e135bacf8725531f533d43705e66819 Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Wed, 22 Jul 2026 09:48:14 +0000 Subject: [PATCH 5/7] feat(copier): adds instructions on how to submit a workflow to argo --- src/copier_template/src/README.md | 9 ------- src/copier_template/src/README.md.jinja | 26 +++++++++++++++++++ .../templates/example.yaml.jinja | 8 +++--- .../create_example_template.py.jinja | 6 ----- .../templates/example.yaml | 8 +++--- .../create_example_template.py | 6 ----- 6 files changed, 34 insertions(+), 29 deletions(-) delete mode 100644 src/copier_template/src/README.md create mode 100644 src/copier_template/src/README.md.jinja diff --git a/src/copier_template/src/README.md b/src/copier_template/src/README.md deleted file mode 100644 index 2c056f1..0000000 --- a/src/copier_template/src/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Setting up a development environment -1. Build the dev container -2. run "uv lock" to generate the uv.lock file -3. Create .env in this folder (with the path src/.env) containing the following variables: -HOST=https://argo-workflows.workflows.diamond.ac.uk/ (to submit to the production repo) -IMAGE= (usually python 3.10) -TOKEN= -NAMESPACE= (the cluster you wish to run the templates on) -AUTH= diff --git a/src/copier_template/src/README.md.jinja b/src/copier_template/src/README.md.jinja new file mode 100644 index 0000000..424f3dd --- /dev/null +++ b/src/copier_template/src/README.md.jinja @@ -0,0 +1,26 @@ +# Setting up a development environment +1. Build the dev container +2. run "uv lock" to generate the uv.lock file +3. Create .env in this folder (with the path src/.env) containing the following variables: + +HOST=https://argo-workflows.workflows.diamond.ac.uk/ (to submit to the production repo) +IMAGE= (usually python 3.10) +TOKEN= +NAMESPACE= (the cluster you wish to run the templates on) +AUTH= + + +# Submitting a workflow to Argo +1. At the start of your workflow definition file, add: + +```python +from {{project_name}}.submit_to_argo import submit_workflow_to_argo +``` + +2. At the end of a workflow definition file, once your workflow (w) is created, add: +```python +submit_workflow_to_argo(w) +``` + +NOTE: Be sure to remove this line upon commiting changes, as all workflow definition files are +by default, ran on pre-commit, to ensure that any yaml files they create are up to date. diff --git a/src/copier_template/src/{{ project_name }}/templates/example.yaml.jinja b/src/copier_template/src/{{ project_name }}/templates/example.yaml.jinja index 017bfb2..736759b 100644 --- a/src/copier_template/src/{{ project_name }}/templates/example.yaml.jinja +++ b/src/copier_template/src/{{ project_name }}/templates/example.yaml.jinja @@ -55,7 +55,7 @@ spec: value: '{{tasks.create-image.outputs.parameters.out-paths}}' - name: install-dependencies script: - image: None + image: python:3.10 source: |- import os import sys @@ -83,7 +83,7 @@ spec: valueFrom: path: /tmp/parameters.json script: - image: None + image: python:3.10 source: |- import os import sys @@ -128,7 +128,7 @@ spec: valueFrom: path: /tmp/{{inputs.parameters.extension}}-path.json script: - image: None + image: python:3.10 source: |- import os import sys @@ -177,7 +177,7 @@ spec: archive: none: {} script: - image: None + image: python:3.10 source: |- import os import sys diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja index 669cb23..6a6aa90 100644 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja @@ -1,5 +1,3 @@ -import os - from hera.workflows import ( DAG, Artifact, @@ -13,7 +11,6 @@ from hera.workflows.archive import NoneArchiveStrategy @script( - image=str(os.environ.get("IMAGE")), command=["python"], volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], ) @@ -30,7 +27,6 @@ def install_dependencies(): @script( command=["python"], - image=str(os.environ.get("IMAGE")), outputs=Parameter( name="out-parameters", value_from=m.ValueFrom(path="/tmp/parameters.json") ), @@ -71,7 +67,6 @@ def generate_parameters( @script( command=["/tmp/venv/bin/python"], - image=str(os.environ.get("IMAGE")), volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], outputs=[ Parameter( @@ -122,7 +117,6 @@ def create_image( @script( command=["/tmp/venv/bin/python"], - image=str(os.environ.get("IMAGE")), volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], outputs=Artifact( name="hdf5output", diff --git a/src/python_interface_to_workflows/templates/example.yaml b/src/python_interface_to_workflows/templates/example.yaml index d851b6d..1fbb1b7 100644 --- a/src/python_interface_to_workflows/templates/example.yaml +++ b/src/python_interface_to_workflows/templates/example.yaml @@ -55,7 +55,7 @@ spec: value: '{{tasks.create-image.outputs.parameters.out-paths}}' - name: install-dependencies script: - image: None + image: python:3.10 source: |- import os import sys @@ -83,7 +83,7 @@ spec: valueFrom: path: /tmp/parameters.json script: - image: None + image: python:3.10 source: |- import os import sys @@ -128,7 +128,7 @@ spec: valueFrom: path: /tmp/{{inputs.parameters.extension}}-path.json script: - image: None + image: python:3.10 source: |- import os import sys @@ -177,7 +177,7 @@ spec: archive: none: {} script: - image: None + image: python:3.10 source: |- import os import sys diff --git a/src/python_interface_to_workflows/workflow_definitions/create_example_template.py b/src/python_interface_to_workflows/workflow_definitions/create_example_template.py index 5cece7d..e718511 100644 --- a/src/python_interface_to_workflows/workflow_definitions/create_example_template.py +++ b/src/python_interface_to_workflows/workflow_definitions/create_example_template.py @@ -1,5 +1,3 @@ -import os - from hera.workflows import ( DAG, Artifact, @@ -13,7 +11,6 @@ @script( - image=str(os.environ.get("IMAGE")), command=["python"], volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], ) @@ -30,7 +27,6 @@ def install_dependencies(): @script( command=["python"], - image=str(os.environ.get("IMAGE")), outputs=Parameter( name="out-parameters", value_from=m.ValueFrom(path="/tmp/parameters.json") ), @@ -71,7 +67,6 @@ def generate_parameters( @script( command=["/tmp/venv/bin/python"], - image=str(os.environ.get("IMAGE")), volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], outputs=[ Parameter( @@ -122,7 +117,6 @@ def create_pattern( @script( command=["/tmp/venv/bin/python"], - image=str(os.environ.get("IMAGE")), volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")], outputs=Artifact( name="hdf5output", From ca94c9901fa452b8cbc166a8c485b329fcf53b2f Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Wed, 22 Jul 2026 12:40:22 +0000 Subject: [PATCH 6/7] feat(argo): added underscore in example name to avoid duplication --- scripts/makecopiercorrect.sh | 2 +- .../templates/{example.yaml.jinja => example.txt.jinja} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/copier_template/src/{{ project_name }}/templates/{example.yaml.jinja => example.txt.jinja} (100%) diff --git a/scripts/makecopiercorrect.sh b/scripts/makecopiercorrect.sh index 4669a44..0f2a966 100644 --- a/scripts/makecopiercorrect.sh +++ b/scripts/makecopiercorrect.sh @@ -2,7 +2,7 @@ cd "src/python_interface_to_workflows/workflow_definitions" cp *example*.py "../../copier_template/src/{{ project_name }}/workflow_definitions" -cp ../templates/*example*.yaml "../../copier_template/src/{{ project_name }}/templates" +cp ../templates/*example*.txt "../../copier_template/src/{{ project_name }}/templates" for file in "../../copier_template/src/{{ project_name }}/workflow_definitions"/* do diff --git a/src/copier_template/src/{{ project_name }}/templates/example.yaml.jinja b/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja similarity index 100% rename from src/copier_template/src/{{ project_name }}/templates/example.yaml.jinja rename to src/copier_template/src/{{ project_name }}/templates/example.txt.jinja From 5f1897e720e2ca2d122001aa46b2352152e9296d Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Fri, 24 Jul 2026 08:51:25 +0000 Subject: [PATCH 7/7] fix(api): fixes code based on comments --- .devcontainer/devcontainer.json | 8 -------- src/copier_template/.devcontainer/devcontainer.json | 3 --- src/copier_template/src/README.md.jinja | 12 +++++------- .../src/{{ project_name }}/submit_to_argo.py.jinja | 3 ++- .../auth/keycloak_checker.py | 2 +- src/python_interface_to_workflows/submit_to_argo.py | 2 +- 6 files changed, 9 insertions(+), 21 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 5173e8f..71710a6 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -53,14 +53,6 @@ "--env-file", "src/.env" ], - "forwardPorts": [ - 5173 - ], - "portsAttributes": { - "5173": { - "onAutoForward": "ignore" - } - }, "mounts": [ // Mount in the user terminal config folder so it can be edited { diff --git a/src/copier_template/.devcontainer/devcontainer.json b/src/copier_template/.devcontainer/devcontainer.json index 96d5d8e..71710a6 100644 --- a/src/copier_template/.devcontainer/devcontainer.json +++ b/src/copier_template/.devcontainer/devcontainer.json @@ -53,9 +53,6 @@ "--env-file", "src/.env" ], - "forwardPorts": [ - 5173 - ], "mounts": [ // Mount in the user terminal config folder so it can be edited { diff --git a/src/copier_template/src/README.md.jinja b/src/copier_template/src/README.md.jinja index 424f3dd..00f1d90 100644 --- a/src/copier_template/src/README.md.jinja +++ b/src/copier_template/src/README.md.jinja @@ -1,14 +1,12 @@ # Setting up a development environment -1. Build the dev container -2. run "uv lock" to generate the uv.lock file -3. Create .env in this folder (with the path src/.env) containing the following variables: +1. run "uv lock" to generate the uv.lock file +2. Create .env in this folder (with the path src/.env) containing the following variables: -HOST=https://argo-workflows.workflows.diamond.ac.uk/ (to submit to the production repo) -IMAGE= (usually python 3.10) -TOKEN= +HOST=https://argo-workflows.workflows.diamond.ac.uk/ (to submit to the production cluster) +DEFAULT_IMAGE= (usually python 3.10) NAMESPACE= (the cluster you wish to run the templates on) -AUTH= +3. Build the dev container # Submitting a workflow to Argo 1. At the start of your workflow definition file, add: diff --git a/src/copier_template/src/{{ project_name }}/submit_to_argo.py.jinja b/src/copier_template/src/{{ project_name }}/submit_to_argo.py.jinja index ff9bcfa..f0e49e4 100644 --- a/src/copier_template/src/{{ project_name }}/submit_to_argo.py.jinja +++ b/src/copier_template/src/{{ project_name }}/submit_to_argo.py.jinja @@ -10,13 +10,14 @@ from {{project_name}}.auth.keycloak_checker import set_token_env_variable def submit_workflow_to_argo(w: Workflow): - set_token_env_variable(staging=True) + set_token_env_variable() dotenv.load_dotenv(dotenv_path="src/.env", override=True) w.namespace = os.environ.get("NAMESPACE") w.workflows_service = WorkflowsService( host=str(os.environ.get("HOST")).strip("'"), token=str(os.environ.get("TOKEN")).strip("'"), ) + global_config.image = str(os.environ.get("DEFAULT_IMAGE")) submitted_w = cast(m.Workflow, w.create()) name = submitted_w.metadata.name namespace = submitted_w.metadata.namespace diff --git a/src/python_interface_to_workflows/auth/keycloak_checker.py b/src/python_interface_to_workflows/auth/keycloak_checker.py index 93cc21f..71390fb 100644 --- a/src/python_interface_to_workflows/auth/keycloak_checker.py +++ b/src/python_interface_to_workflows/auth/keycloak_checker.py @@ -20,7 +20,7 @@ def set_token_env_variable(staging: bool) -> str: port = 5173 case False: keycloak_openid = KeycloakOpenID( - client_id="workflows-dashboard", + client_id="workflows-cli", server_url="https://identity.diamond.ac.uk/", realm_name="dls", client_secret_key="", diff --git a/src/python_interface_to_workflows/submit_to_argo.py b/src/python_interface_to_workflows/submit_to_argo.py index 343d423..82b86db 100644 --- a/src/python_interface_to_workflows/submit_to_argo.py +++ b/src/python_interface_to_workflows/submit_to_argo.py @@ -17,7 +17,7 @@ def submit_workflow_to_argo(w: Workflow): host=str(os.environ.get("HOST")).strip("'"), token=str(os.environ.get("TOKEN")).strip("'"), ) - global_config.image = str(os.environ.get("IMAGE")) + global_config.image = str(os.environ.get("DEFAULT_IMAGE")) submitted_w = cast(m.Workflow, w.create()) name = submitted_w.metadata.name namespace = submitted_w.metadata.namespace