From 5108fc38a23f69dc91efa2e0c94cc07cf39ee4cf Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Wed, 22 Jul 2026 12:40:22 +0000 Subject: [PATCH 1/8] feat(argo): added underscore in example name to avoid duplication --- .../workflow_definitions/create_example_template.py.jinja | 2 +- .../templates/{example.yaml => _example.yaml} | 0 .../workflow_definitions/create_example_template.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename src/python_interface_to_workflows/templates/{example.yaml => _example.yaml} (100%) diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja index 6a6aa90..2657ce0 100644 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja @@ -178,5 +178,5 @@ example.yaml""", [install, params] >> makeimages >> makehdf5 # pyright: ignore -with open("example.yaml", "w") as div: +with open("_example.yaml", "w") as div: div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] diff --git a/src/python_interface_to_workflows/templates/example.yaml b/src/python_interface_to_workflows/templates/_example.yaml similarity index 100% rename from src/python_interface_to_workflows/templates/example.yaml rename to src/python_interface_to_workflows/templates/_example.yaml diff --git a/src/python_interface_to_workflows/workflow_definitions/create_example_template.py b/src/python_interface_to_workflows/workflow_definitions/create_example_template.py index e718511..8d3516a 100644 --- a/src/python_interface_to_workflows/workflow_definitions/create_example_template.py +++ b/src/python_interface_to_workflows/workflow_definitions/create_example_template.py @@ -178,5 +178,5 @@ def to_hdf5(paths: str): [install, params] >> makeimages >> makehdf5 # pyright: ignore -with open("example.yaml", "w") as div: +with open("_example.yaml", "w") as div: div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] From 872c2ab7fd19ec90f346b9b7061ad9d1a13ee144 Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Wed, 22 Jul 2026 13:16:44 +0000 Subject: [PATCH 2/8] feat(auth): removes opening site when not required --- .../auth/keycloak_checker.py | 28 ++++++++++---- .../auth/open_auth_url.py | 37 ++++++++++++------- 2 files changed, 43 insertions(+), 22 deletions(-) diff --git a/src/python_interface_to_workflows/auth/keycloak_checker.py b/src/python_interface_to_workflows/auth/keycloak_checker.py index 71390fb..44e486e 100644 --- a/src/python_interface_to_workflows/auth/keycloak_checker.py +++ b/src/python_interface_to_workflows/auth/keycloak_checker.py @@ -36,14 +36,26 @@ def set_token_env_variable(staging: bool) -> str: code_challenge=code_challenge, code_challenge_method=code_challenge_method, ) - open_auth_url(auth_url, port) - token: dict[str, str] = ( # pyright: ignore[reportUnknownVariableType] - keycloak_openid.token( # pyright: ignore[reportUnknownMemberType] - grant_type="authorization_code", - code=os.environ["AUTH"], - redirect_uri=f"http://localhost:{port}/", - code_verifier=code_verifier, - ) + match open_auth_url(auth_url, port): + case True: + token: dict[str, str] = ( # pyright: ignore[reportUnknownVariableType] + keycloak_openid.token( # pyright: ignore[reportUnknownMemberType] + grant_type="authorization_code", + code=os.environ.get("AUTH"), # pyright: ignore + redirect_uri=f"http://localhost:{port}/", + code_verifier=code_verifier, + ) + ) + case False: + token: dict[str, str] = keycloak_openid.refresh_token( # pyright: ignore + str(os.environ.get("REFRESHTOKEN")) # pyright: ignore + ) + token_info: dict[str, int | str | dict[str, list[str]]] = ( # pyright: ignore + keycloak_openid.decode_token(token["access_token"]) # pyright: ignore ) + expire_time = int(token_info["exp"]) + 1500 # pyright: ignore + dotenv.set_key("src/.env", "EXPIRY", str(expire_time).strip("'")) dotenv.set_key("src/.env", "TOKEN", token["access_token"].strip("'")) + dotenv.set_key("src/.env", "REFRESHTOKEN", token["refresh_token"].strip("'")) + dotenv.load_dotenv(dotenv_path="src/.env", override=True) return token["access_token"] # pyright: ignore[reportUnknownArgumentType] diff --git a/src/python_interface_to_workflows/auth/open_auth_url.py b/src/python_interface_to_workflows/auth/open_auth_url.py index 2f0dcf9..666094f 100644 --- a/src/python_interface_to_workflows/auth/open_auth_url.py +++ b/src/python_interface_to_workflows/auth/open_auth_url.py @@ -1,5 +1,6 @@ import os import socket +import time import urllib.parse import webbrowser from http.server import BaseHTTPRequestHandler, HTTPServer @@ -28,17 +29,25 @@ def do_GET(self): self.wfile.write(b"Missing authorization code.") -def open_auth_url(auth_url: str, port: int) -> None: - httpd = _ReusingHTTPServer(("localhost", port), CallbackHandler) - webbrowser.open(auth_url) - try: - httpd.handle_request() - os.environ["AUTH"] = httpd.auth_code - dotenv.set_key("src/.env", "AUTH", httpd.auth_code) - except OSError: - os.environ["AUTH"] = "" - print("ERROR: Port in use. Please restart your terminal.") - exit(1) - finally: - httpd.socket.shutdown(socket.SHUT_RDWR) - httpd.server_close() +def open_auth_url(auth_url: str, port: int) -> bool: + dotenv.load_dotenv(dotenv_path="src/.env", override=True) + expiry_str: str = os.environ.get("EXPIRY") # pyright: ignore + print(f"expiry_str: {expiry_str}", str(time.time())) + if (expiry_str == "" or int(expiry_str)) <= float(time.time()): + httpd = _ReusingHTTPServer(("localhost", port), CallbackHandler) + webbrowser.open(auth_url) + try: + httpd.handle_request() + os.environ["AUTH"] = httpd.auth_code + dotenv.set_key("src/.env", "AUTH", httpd.auth_code) + except OSError: + os.environ["AUTH"] = "" + print("ERROR: Port in use. Please restart your terminal.") + exit(1) + finally: + httpd.socket.shutdown(socket.SHUT_RDWR) + httpd.server_close() + return True + else: + print("using refresh token") + return False From 36229a78ef0fbfbe707bd5b6ebfcdf38fa22a03a Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Wed, 22 Jul 2026 15:01:29 +0000 Subject: [PATCH 3/8] feat(tests): updates unit tests --- .../auth/keycloak_checker.py.jinja | 57 ++++++++++++----- .../{{ project_name }}/auth/open_auth_url.py | 38 ++++++++---- .../tests/test_keycloak_checker.py.jinja | 62 +++++++++++++++---- .../tests/test_open_auth_url.py.jinja | 59 +++++++++++++++++- tests/test_keycloak_checker.py | 57 +++++++++++++---- tests/test_open_auth_url.py | 59 +++++++++++++++++- 6 files changed, 274 insertions(+), 58 deletions(-) diff --git a/src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py.jinja b/src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py.jinja index 04dd688..40c2362 100644 --- a/src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py.jinja +++ b/src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py.jinja @@ -7,15 +7,26 @@ from keycloak.pkce_utils import generate_code_challenge, generate_code_verifier from {{project_name}}.auth.open_auth_url import open_auth_url -def set_token_env_variable() -> str: - keycloak_openid = KeycloakOpenID( - client_id="workflows-dashboard", - server_url="https://identity.diamond.ac.uk/", - realm_name="dls", - client_secret_key="", - pool_maxsize=1, - ) - port = 8000 +def set_token_env_variable(staging: bool) -> str: + match staging: + case True: + keycloak_openid = KeycloakOpenID( + server_url="https://identity-test.diamond.ac.uk/", + client_id="workflows-ui-dev", + realm_name="dls", + client_secret_key="", + pool_maxsize=1, + ) + port = 5173 + case False: + keycloak_openid = KeycloakOpenID( + client_id="workflows-dashboard", + server_url="https://identity.diamond.ac.uk/", + realm_name="dls", + client_secret_key="", + pool_maxsize=1, + ) + port = 8000 code_verifier = generate_code_verifier() code_challenge, code_challenge_method = generate_code_challenge(code_verifier) auth_url = keycloak_openid.auth_url( @@ -25,14 +36,26 @@ def set_token_env_variable() -> str: code_challenge=code_challenge, code_challenge_method=code_challenge_method, ) - open_auth_url(auth_url) - token: dict[str, str] = ( # pyright: ignore[reportUnknownVariableType] - keycloak_openid.token( # pyright: ignore[reportUnknownMemberType] - grant_type="authorization_code", - code=os.environ["AUTH"], - redirect_uri=f"http://localhost:{port}/", - code_verifier=code_verifier, - ) + match open_auth_url(auth_url, port): + case True: + token: dict[str, str] = ( # pyright: ignore[reportUnknownVariableType] + keycloak_openid.token( # pyright: ignore[reportUnknownMemberType] + grant_type="authorization_code", + code=os.environ.get("AUTH"), # pyright: ignore + redirect_uri=f"http://localhost:{port}/", + code_verifier=code_verifier, + ) + ) + case False: + token: dict[str, str] = keycloak_openid.refresh_token( # pyright: ignore + str(os.environ.get("REFRESHTOKEN")) # pyright: ignore + ) + token_info: dict[str, int | str | dict[str, list[str]]] = ( # pyright: ignore + keycloak_openid.decode_token(token["access_token"]) # pyright: ignore ) + expire_time = int(token_info["exp"]) + 1500 # pyright: ignore + dotenv.set_key("src/.env", "EXPIRY", str(expire_time).strip("'")) dotenv.set_key("src/.env", "TOKEN", token["access_token"].strip("'")) + dotenv.set_key("src/.env", "REFRESHTOKEN", token["refresh_token"].strip("'")) + dotenv.load_dotenv(dotenv_path="src/.env", override=True) return token["access_token"] # pyright: ignore[reportUnknownArgumentType] diff --git a/src/copier_template/src/{{ project_name }}/auth/open_auth_url.py b/src/copier_template/src/{{ project_name }}/auth/open_auth_url.py index 5a18266..666094f 100644 --- a/src/copier_template/src/{{ project_name }}/auth/open_auth_url.py +++ b/src/copier_template/src/{{ project_name }}/auth/open_auth_url.py @@ -1,10 +1,13 @@ import os import socket +import time import urllib.parse import webbrowser from http.server import BaseHTTPRequestHandler, HTTPServer from typing import cast +import dotenv + class _ReusingHTTPServer(HTTPServer): allow_reuse_address = True @@ -26,16 +29,25 @@ def do_GET(self): self.wfile.write(b"Missing authorization code.") -def open_auth_url(auth_url: str) -> None: - httpd = _ReusingHTTPServer(("localhost", 8000), CallbackHandler) - webbrowser.open(auth_url) - try: - httpd.handle_request() - os.environ["AUTH"] = httpd.auth_code - except OSError: - os.environ["AUTH"] = "" - print("ERROR: Port in use. Please restart your terminal.") - exit(1) - finally: - httpd.socket.shutdown(socket.SHUT_RDWR) - httpd.server_close() +def open_auth_url(auth_url: str, port: int) -> bool: + dotenv.load_dotenv(dotenv_path="src/.env", override=True) + expiry_str: str = os.environ.get("EXPIRY") # pyright: ignore + print(f"expiry_str: {expiry_str}", str(time.time())) + if (expiry_str == "" or int(expiry_str)) <= float(time.time()): + httpd = _ReusingHTTPServer(("localhost", port), CallbackHandler) + webbrowser.open(auth_url) + try: + httpd.handle_request() + os.environ["AUTH"] = httpd.auth_code + dotenv.set_key("src/.env", "AUTH", httpd.auth_code) + except OSError: + os.environ["AUTH"] = "" + print("ERROR: Port in use. Please restart your terminal.") + exit(1) + finally: + httpd.socket.shutdown(socket.SHUT_RDWR) + httpd.server_close() + return True + else: + print("using refresh token") + return False diff --git a/src/copier_template/tests/test_keycloak_checker.py.jinja b/src/copier_template/tests/test_keycloak_checker.py.jinja index 2eda176..322b17b 100644 --- a/src/copier_template/tests/test_keycloak_checker.py.jinja +++ b/src/copier_template/tests/test_keycloak_checker.py.jinja @@ -1,35 +1,75 @@ import os -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, call, patch + +from pytest import mark from {{project_name}}.auth.keycloak_checker import set_token_env_variable +@mark.parametrize( + "staging,port,return_present", + [ + (True, 5173, True), + (False, 8000, False), + (True, 5173, False), + (False, 8000, True), + ], +) +@patch("{{project_name}}.auth.keycloak_checker.dotenv.load_dotenv") +@patch("{{project_name}}.auth.keycloak_checker.dotenv.set_key") @patch("{{project_name}}.auth.keycloak_checker.KeycloakOpenID") @patch("{{project_name}}.auth.keycloak_checker.generate_code_verifier") @patch("{{project_name}}.auth.keycloak_checker.generate_code_challenge") @patch("{{project_name}}.auth.keycloak_checker.open_auth_url") -def test_return_key( +def test_set_token_env_variable( mock_open_auth_url: MagicMock, mock_gen_code_challenge: MagicMock, mock_gen_code_verifier: MagicMock, mock_gen_keycloak_id: MagicMock, + mock_set_key: MagicMock, + mock_load_env: MagicMock, + staging: bool, + port: int, + return_present: bool, ): mock_gen_code_verifier.return_value = "verifier" mock_gen_code_challenge.return_value = ("challenge", "S256") os.environ["AUTH"] = "auth_url_code" + os.environ["REFRESHTOKEN"] = "refresh" + keycloak = MagicMock() mock_gen_keycloak_id.return_value = keycloak - + mock_open_auth_url.return_value = return_present keycloak.auth_url.return_value = "https://mock.site" - keycloak.token.return_value = { + token = { "access_token": "fake_token", "refresh_token": "fake_refresh", } - assert set_token_env_variable() == "fake_token" - mock_open_auth_url.assert_called_once_with("https://mock.site", 8000) - keycloak.token.assert_called_once_with( - grant_type="authorization_code", - code="auth_url_code", - redirect_uri="http://localhost:8000/", - code_verifier="verifier", + keycloak.token.return_value = token + keycloak.refresh_token.return_value = token + keycloak.decode_token.return_value = {"exp": 123456789} + assert set_token_env_variable(staging) == "fake_token" + + mock_open_auth_url.assert_called_once_with("https://mock.site", port) + if return_present: + keycloak.token.assert_called_once_with( + grant_type="authorization_code", + code="auth_url_code", + redirect_uri=f"http://localhost:{port}/", + code_verifier="verifier", + ) + keycloak.refresh_token.assert_not_called() + else: + keycloak.refresh_token.assert_called_once_with("refresh") + keycloak.token.assert_not_called() + mock_set_key.assert_has_calls( + [ + call("src/.env", "EXPIRY", str(123456789 + 1500)), + call("src/.env", "TOKEN", "fake_token"), + call("src/.env", "REFRESHTOKEN", "fake_refresh"), + ] + ) + mock_load_env.assert_called_once_with( + dotenv_path="src/.env", + override=True, ) diff --git a/src/copier_template/tests/test_open_auth_url.py.jinja b/src/copier_template/tests/test_open_auth_url.py.jinja index f6d3393..bc3799a 100644 --- a/src/copier_template/tests/test_open_auth_url.py.jinja +++ b/src/copier_template/tests/test_open_auth_url.py.jinja @@ -7,22 +7,45 @@ from {{project_name}}.auth.open_auth_url import ( ) +@patch("{{project_name}}.auth.open_auth_url.dotenv.set_key") +@patch("{{project_name}}.auth.open_auth_url.dotenv.load_dotenv") +@patch("{{project_name}}.auth.open_auth_url.time.time") @patch("{{project_name}}.auth.open_auth_url.webbrowser.open") @patch("{{project_name}}.auth.open_auth_url._ReusingHTTPServer") def test_open_auth_url_normal_function( mock_http_server: MagicMock, mock_open_browser: MagicMock, + mock_time: MagicMock, + mock_load_env: MagicMock, + mock_set_key: MagicMock, ): + mock_time.return_value = 100 + os.environ["EXPIRY"] = "" + server = mock_http_server.return_value server.auth_code = "this_is_your_code" - open_auth_url("url") + + assert open_auth_url("url", 5173) is True + + mock_load_env.assert_called_once_with( + dotenv_path="src/.env", + override=True, + ) server.handle_request.assert_called_once() mock_open_browser.assert_called_once_with("url") + mock_set_key.assert_called_once_with( + "src/.env", + "AUTH", + "this_is_your_code", + ) server.socket.shutdown.assert_called_once() server.server_close.assert_called_once() assert os.environ["AUTH"] == "this_is_your_code" +@patch("{{project_name}}.auth.open_auth_url.dotenv.set_key") +@patch("{{project_name}}.auth.open_auth_url.dotenv.load_dotenv") +@patch("{{project_name}}.auth.open_auth_url.time.time") @patch("{{project_name}}.auth.open_auth_url.exit") @patch("{{project_name}}.auth.open_auth_url.webbrowser.open") @patch("{{project_name}}.auth.open_auth_url._ReusingHTTPServer") @@ -30,18 +53,48 @@ def test_open_auth_url_raises_error( mock_http_server: MagicMock, mock_open_browser: MagicMock, mock_exit: MagicMock, + mock_time: MagicMock, + mock_load_env: MagicMock, + mock_set_key: MagicMock, ): + mock_time.return_value = 100 + os.environ["EXPIRY"] = "" + server = mock_http_server.return_value - server.auth_code = "this_is_your_code" server.handle_request.side_effect = OSError - open_auth_url("url") + + assert open_auth_url("url", 5173) is True mock_open_browser.assert_called_once_with("url") + mock_load_env.assert_called_once() assert os.environ["AUTH"] == "" mock_exit.assert_called_once_with(1) + mock_set_key.assert_not_called() server.socket.shutdown.assert_called_once() server.server_close.assert_called_once() +@patch("{{project_name}}.auth.open_auth_url.webbrowser.open") +@patch("{{project_name}}.auth.open_auth_url.dotenv.load_dotenv") +@patch("{{project_name}}.auth.open_auth_url.time.time") +@patch("{{project_name}}.auth.open_auth_url._ReusingHTTPServer") +def test_open_auth_url_refresh_token( + mock_http_server: MagicMock, + mock_time: MagicMock, + mock_load_env: MagicMock, + mock_open_browser: MagicMock, +): + mock_time.return_value = 100 + os.environ["EXPIRY"] = "200" + assert open_auth_url("url", 5173) is False + + mock_load_env.assert_called_once_with( + dotenv_path="src/.env", + override=True, + ) + mock_http_server.assert_not_called() + mock_open_browser.assert_not_called() + + def test_handler_normal_function(): handler = CallbackHandler.__new__(CallbackHandler) handler.path = "/?code=this_is_your_code" diff --git a/tests/test_keycloak_checker.py b/tests/test_keycloak_checker.py index bb5c776..6f6bc21 100644 --- a/tests/test_keycloak_checker.py +++ b/tests/test_keycloak_checker.py @@ -1,12 +1,22 @@ import os -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, call, patch from pytest import mark from python_interface_to_workflows.auth.keycloak_checker import set_token_env_variable -@mark.parametrize("dev,port", [(True, 5173), (False, 8000)]) +@mark.parametrize( + "staging,port,return_present", + [ + (True, 5173, True), + (False, 8000, False), + (True, 5173, False), + (False, 8000, True), + ], +) +@patch("python_interface_to_workflows.auth.keycloak_checker.dotenv.load_dotenv") +@patch("python_interface_to_workflows.auth.keycloak_checker.dotenv.set_key") @patch("python_interface_to_workflows.auth.keycloak_checker.KeycloakOpenID") @patch("python_interface_to_workflows.auth.keycloak_checker.generate_code_verifier") @patch("python_interface_to_workflows.auth.keycloak_checker.generate_code_challenge") @@ -16,25 +26,50 @@ def test_set_token_env_variable( mock_gen_code_challenge: MagicMock, mock_gen_code_verifier: MagicMock, mock_gen_keycloak_id: MagicMock, - dev: bool, + mock_set_key: MagicMock, + mock_load_env: MagicMock, + staging: bool, port: int, + return_present: bool, ): mock_gen_code_verifier.return_value = "verifier" mock_gen_code_challenge.return_value = ("challenge", "S256") os.environ["AUTH"] = "auth_url_code" + os.environ["REFRESHTOKEN"] = "refresh" + keycloak = MagicMock() mock_gen_keycloak_id.return_value = keycloak - + mock_open_auth_url.return_value = return_present keycloak.auth_url.return_value = "https://mock.site" - keycloak.token.return_value = { + token = { "access_token": "fake_token", "refresh_token": "fake_refresh", } - assert set_token_env_variable(dev) == "fake_token" + keycloak.token.return_value = token + keycloak.refresh_token.return_value = token + keycloak.decode_token.return_value = {"exp": 123456789} + assert set_token_env_variable(staging) == "fake_token" + mock_open_auth_url.assert_called_once_with("https://mock.site", port) - keycloak.token.assert_called_once_with( - grant_type="authorization_code", - code="auth_url_code", - redirect_uri=f"http://localhost:{port}/", - code_verifier="verifier", + if return_present: + keycloak.token.assert_called_once_with( + grant_type="authorization_code", + code="auth_url_code", + redirect_uri=f"http://localhost:{port}/", + code_verifier="verifier", + ) + keycloak.refresh_token.assert_not_called() + else: + keycloak.refresh_token.assert_called_once_with("refresh") + keycloak.token.assert_not_called() + mock_set_key.assert_has_calls( + [ + call("src/.env", "EXPIRY", str(123456789 + 1500)), + call("src/.env", "TOKEN", "fake_token"), + call("src/.env", "REFRESHTOKEN", "fake_refresh"), + ] + ) + mock_load_env.assert_called_once_with( + dotenv_path="src/.env", + override=True, ) diff --git a/tests/test_open_auth_url.py b/tests/test_open_auth_url.py index 59fa0d3..e05fb49 100644 --- a/tests/test_open_auth_url.py +++ b/tests/test_open_auth_url.py @@ -7,22 +7,45 @@ ) +@patch("python_interface_to_workflows.auth.open_auth_url.dotenv.set_key") +@patch("python_interface_to_workflows.auth.open_auth_url.dotenv.load_dotenv") +@patch("python_interface_to_workflows.auth.open_auth_url.time.time") @patch("python_interface_to_workflows.auth.open_auth_url.webbrowser.open") @patch("python_interface_to_workflows.auth.open_auth_url._ReusingHTTPServer") def test_open_auth_url_normal_function( mock_http_server: MagicMock, mock_open_browser: MagicMock, + mock_time: MagicMock, + mock_load_env: MagicMock, + mock_set_key: MagicMock, ): + mock_time.return_value = 100 + os.environ["EXPIRY"] = "" + server = mock_http_server.return_value server.auth_code = "this_is_your_code" - open_auth_url("url", 5173) + + assert open_auth_url("url", 5173) is True + + mock_load_env.assert_called_once_with( + dotenv_path="src/.env", + override=True, + ) server.handle_request.assert_called_once() mock_open_browser.assert_called_once_with("url") + mock_set_key.assert_called_once_with( + "src/.env", + "AUTH", + "this_is_your_code", + ) server.socket.shutdown.assert_called_once() server.server_close.assert_called_once() assert os.environ["AUTH"] == "this_is_your_code" +@patch("python_interface_to_workflows.auth.open_auth_url.dotenv.set_key") +@patch("python_interface_to_workflows.auth.open_auth_url.dotenv.load_dotenv") +@patch("python_interface_to_workflows.auth.open_auth_url.time.time") @patch("python_interface_to_workflows.auth.open_auth_url.exit") @patch("python_interface_to_workflows.auth.open_auth_url.webbrowser.open") @patch("python_interface_to_workflows.auth.open_auth_url._ReusingHTTPServer") @@ -30,18 +53,48 @@ def test_open_auth_url_raises_error( mock_http_server: MagicMock, mock_open_browser: MagicMock, mock_exit: MagicMock, + mock_time: MagicMock, + mock_load_env: MagicMock, + mock_set_key: MagicMock, ): + mock_time.return_value = 100 + os.environ["EXPIRY"] = "" + server = mock_http_server.return_value - server.auth_code = "this_is_your_code" server.handle_request.side_effect = OSError - open_auth_url("url", 5173) + + assert open_auth_url("url", 5173) is True mock_open_browser.assert_called_once_with("url") + mock_load_env.assert_called_once() assert os.environ["AUTH"] == "" mock_exit.assert_called_once_with(1) + mock_set_key.assert_not_called() server.socket.shutdown.assert_called_once() server.server_close.assert_called_once() +@patch("python_interface_to_workflows.auth.open_auth_url.webbrowser.open") +@patch("python_interface_to_workflows.auth.open_auth_url.dotenv.load_dotenv") +@patch("python_interface_to_workflows.auth.open_auth_url.time.time") +@patch("python_interface_to_workflows.auth.open_auth_url._ReusingHTTPServer") +def test_open_auth_url_refresh_token( + mock_http_server: MagicMock, + mock_time: MagicMock, + mock_load_env: MagicMock, + mock_open_browser: MagicMock, +): + mock_time.return_value = 100 + os.environ["EXPIRY"] = "200" + assert open_auth_url("url", 5173) is False + + mock_load_env.assert_called_once_with( + dotenv_path="src/.env", + override=True, + ) + mock_http_server.assert_not_called() + mock_open_browser.assert_not_called() + + def test_handler_normal_function(): handler = CallbackHandler.__new__(CallbackHandler) handler.path = "/?code=this_is_your_code" From 5418037a779a61225270831a078d7956ae3cdb57 Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Wed, 22 Jul 2026 15:14:47 +0000 Subject: [PATCH 4/8] feat(copier): updates README.md to include copying instructions --- README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.md b/README.md index fee2dff..0c5acd2 100644 --- a/README.md +++ b/README.md @@ -31,3 +31,24 @@ Or if it is a commandline tool then you might put some example commands here: ``` python -m python_interface_to_workflows --version ``` + +# Using Copier +```bash +mkdir new_directory_path +cd new_directory_path +git init +git remote add origin {origin ssh} +git branch -M main + +cd .. +``` +then either: +```bash +git clone git@github.com:DiamondLightSource/python-interface-to-workflows.git +copier copy {this_repo's_path} {new_directory_path} +``` +or: +```bash +copier copy git@github.com:DiamondLightSource/python-copier-template.git new directory path +``` +rebuild in dev container without cache From 30821fd210d56ef30e640073eabdeafd5cf5ca89 Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Wed, 22 Jul 2026 15:26:56 +0000 Subject: [PATCH 5/8] feat(copier): updates copier template and generator copy correctly --- scripts/lintyaml.sh | 2 +- scripts/makecopiercorrect.sh | 20 +++++++++++++++++++ scripts/runthefiles.sh | 1 + src/copier_template/pyproject.toml.jinja | 1 - src/copier_template/scripts/lintyaml.sh.jinja | 2 +- .../scripts/runthefiles.sh.jinja | 3 ++- .../auth/keycloak_checker.py.jinja | 19 ++++-------------- .../{{ project_name }}/auth/open_auth_url.py | 4 +--- .../templates/example.txt.jinja | 4 +++- .../create_example_template.py.jinja | 6 ++++-- .../auth/open_auth_url.py | 4 +--- .../templates/{_example.yaml => example.txt} | 0 .../create_example_template.py | 2 +- 13 files changed, 39 insertions(+), 29 deletions(-) rename src/python_interface_to_workflows/templates/{_example.yaml => example.txt} (100%) diff --git a/scripts/lintyaml.sh b/scripts/lintyaml.sh index 3de474e..9ebc381 100644 --- a/scripts/lintyaml.sh +++ b/scripts/lintyaml.sh @@ -1,6 +1,6 @@ #!/bin/bash cd src/python_interface_to_workflows/templates -for file in * +for file in *.yaml; do argo lint "$file" --offline SUCCESSFULLINT=$? diff --git a/scripts/makecopiercorrect.sh b/scripts/makecopiercorrect.sh index 0f2a966..973bd00 100644 --- a/scripts/makecopiercorrect.sh +++ b/scripts/makecopiercorrect.sh @@ -7,16 +7,36 @@ cp ../templates/*example*.txt "../../copier_template/src/{{ project_name }}/temp for file in "../../copier_template/src/{{ project_name }}/workflow_definitions"/* do [[ $file == *.jinja ]] && continue + sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file" sed -i 's/DiamondLightSource/{{github_org}}/g' "$file" + + sed -i '1i{% raw %}' "$file" + echo '{% endraw %}' >> "$file" + + sed -i \ + -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \ + -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \ + "$file" + mv "$file" "$file.jinja" done for file in "../../copier_template/src/{{ project_name }}/templates"/* do [[ $file == *.jinja ]] && continue + sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file" sed -i 's/DiamondLightSource/{{github_org}}/g' "$file" + + sed -i '1i{% raw %}' "$file" + echo '{% endraw %}' >> "$file" + + sed -i \ + -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \ + -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \ + "$file" + mv "$file" "$file.jinja" done git add "../../copier_template/src/{{ project_name }}/workflow_definitions"/* diff --git a/scripts/runthefiles.sh b/scripts/runthefiles.sh index a7453b4..960a7d8 100644 --- a/scripts/runthefiles.sh +++ b/scripts/runthefiles.sh @@ -5,4 +5,5 @@ do uv run "$file" done mv *.yaml ../templates/ +mv *.txt ../templates/ git add -u ../templates/ diff --git a/src/copier_template/pyproject.toml.jinja b/src/copier_template/pyproject.toml.jinja index cad7d38..9eec731 100644 --- a/src/copier_template/pyproject.toml.jinja +++ b/src/copier_template/pyproject.toml.jinja @@ -68,7 +68,6 @@ reportMissingImports = false # Ignore missing stubs in imported modules reportMissingTypeStubs = "none" reportMissingModuleSource="warning" reportInvalidTypeForm="warning" -reportMissingImports="warning" reportUndefinedVariable="warning" reportAbstractUsage="warning" reportArgumentType="warning" diff --git a/src/copier_template/scripts/lintyaml.sh.jinja b/src/copier_template/scripts/lintyaml.sh.jinja index 7b0b6e8..5c984df 100644 --- a/src/copier_template/scripts/lintyaml.sh.jinja +++ b/src/copier_template/scripts/lintyaml.sh.jinja @@ -1,6 +1,6 @@ #!/bin/bash cd src/{{project_name}}/templates -for file in * +for file in *.yaml; do argo lint "$file" --offline SUCCESSFULLINT=$? diff --git a/src/copier_template/scripts/runthefiles.sh.jinja b/src/copier_template/scripts/runthefiles.sh.jinja index ca0bea2..960a7d8 100644 --- a/src/copier_template/scripts/runthefiles.sh.jinja +++ b/src/copier_template/scripts/runthefiles.sh.jinja @@ -1,8 +1,9 @@ #!/bin/bash -cd src/{{project_name}}/workflow_definitions +cd src/python_interface_to_workflows/workflow_definitions for file in * do uv run "$file" done mv *.yaml ../templates/ +mv *.txt ../templates/ git add -u ../templates/ diff --git a/src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py.jinja b/src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py.jinja index 40c2362..2607840 100644 --- a/src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py.jinja +++ b/src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py.jinja @@ -7,26 +7,15 @@ from keycloak.pkce_utils import generate_code_challenge, generate_code_verifier from {{project_name}}.auth.open_auth_url import open_auth_url -def set_token_env_variable(staging: bool) -> str: - match staging: - case True: - keycloak_openid = KeycloakOpenID( - server_url="https://identity-test.diamond.ac.uk/", - client_id="workflows-ui-dev", - realm_name="dls", - client_secret_key="", - pool_maxsize=1, - ) - port = 5173 - case False: - keycloak_openid = KeycloakOpenID( - client_id="workflows-dashboard", +def set_token_env_variable() -> str: + keycloak_openid = KeycloakOpenID( + client_id="workflows-cli", server_url="https://identity.diamond.ac.uk/", realm_name="dls", client_secret_key="", pool_maxsize=1, ) - port = 8000 + port = 8000 code_verifier = generate_code_verifier() code_challenge, code_challenge_method = generate_code_challenge(code_verifier) auth_url = keycloak_openid.auth_url( diff --git a/src/copier_template/src/{{ project_name }}/auth/open_auth_url.py b/src/copier_template/src/{{ project_name }}/auth/open_auth_url.py index 666094f..9acae95 100644 --- a/src/copier_template/src/{{ project_name }}/auth/open_auth_url.py +++ b/src/copier_template/src/{{ project_name }}/auth/open_auth_url.py @@ -31,8 +31,7 @@ def do_GET(self): def open_auth_url(auth_url: str, port: int) -> bool: dotenv.load_dotenv(dotenv_path="src/.env", override=True) - expiry_str: str = os.environ.get("EXPIRY") # pyright: ignore - print(f"expiry_str: {expiry_str}", str(time.time())) + expiry_str: str = os.environ.get("EXPIRY").strip("'") # pyright: ignore if (expiry_str == "" or int(expiry_str)) <= float(time.time()): httpd = _ReusingHTTPServer(("localhost", port), CallbackHandler) webbrowser.open(auth_url) @@ -49,5 +48,4 @@ def open_auth_url(auth_url: str, port: int) -> bool: httpd.server_close() return True else: - print("using refresh token") return False diff --git a/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja b/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja index 736759b..3e74853 100644 --- a/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja +++ b/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja @@ -1,3 +1,4 @@ +{% raw %} apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: @@ -7,7 +8,7 @@ metadata: Replicates the functionality of example.yaml workflows.argoproj.io/title: example remade via hera - workflows.diamond.ac.uk/repository: https://github.com/{{github_org}}/{{repo_name}} + workflows.diamond.ac.uk/repository: https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %} labels: workflows.diamond.ac.uk/science-group-examples: 'true' spec: @@ -212,3 +213,4 @@ spec: resources: requests: storage: 1Gi +{% endraw %} diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja index 2657ce0..65be398 100644 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja @@ -1,3 +1,4 @@ +{% raw %} from hera.workflows import ( DAG, Artifact, @@ -153,7 +154,7 @@ with Workflow( "workflows.argoproj.io/title": "example remade via hera", "workflows.argoproj.io/description": """Replicates the functionality of example.yaml""", - "workflows.diamond.ac.uk/repository": "https://github.com/{{github_org}}/{{repo_name}}", + "workflows.diamond.ac.uk/repository": "https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %}", }, volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"), ) as w: @@ -178,5 +179,6 @@ example.yaml""", [install, params] >> makeimages >> makehdf5 # pyright: ignore -with open("_example.yaml", "w") as div: +with open("example.txt", "w") as div: div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] +{% endraw %} diff --git a/src/python_interface_to_workflows/auth/open_auth_url.py b/src/python_interface_to_workflows/auth/open_auth_url.py index 666094f..9acae95 100644 --- a/src/python_interface_to_workflows/auth/open_auth_url.py +++ b/src/python_interface_to_workflows/auth/open_auth_url.py @@ -31,8 +31,7 @@ def do_GET(self): def open_auth_url(auth_url: str, port: int) -> bool: dotenv.load_dotenv(dotenv_path="src/.env", override=True) - expiry_str: str = os.environ.get("EXPIRY") # pyright: ignore - print(f"expiry_str: {expiry_str}", str(time.time())) + expiry_str: str = os.environ.get("EXPIRY").strip("'") # pyright: ignore if (expiry_str == "" or int(expiry_str)) <= float(time.time()): httpd = _ReusingHTTPServer(("localhost", port), CallbackHandler) webbrowser.open(auth_url) @@ -49,5 +48,4 @@ def open_auth_url(auth_url: str, port: int) -> bool: httpd.server_close() return True else: - print("using refresh token") return False diff --git a/src/python_interface_to_workflows/templates/_example.yaml b/src/python_interface_to_workflows/templates/example.txt similarity index 100% rename from src/python_interface_to_workflows/templates/_example.yaml rename to src/python_interface_to_workflows/templates/example.txt diff --git a/src/python_interface_to_workflows/workflow_definitions/create_example_template.py b/src/python_interface_to_workflows/workflow_definitions/create_example_template.py index 8d3516a..c8f4ce7 100644 --- a/src/python_interface_to_workflows/workflow_definitions/create_example_template.py +++ b/src/python_interface_to_workflows/workflow_definitions/create_example_template.py @@ -178,5 +178,5 @@ def to_hdf5(paths: str): [install, params] >> makeimages >> makehdf5 # pyright: ignore -with open("_example.yaml", "w") as div: +with open("example.txt", "w") as div: div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType] From 4074446ac3c50a1a8aacade939221c323c876846 Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Wed, 22 Jul 2026 12:40:22 +0000 Subject: [PATCH 6/8] feat(argo): added underscore in example name to avoid duplication --- .../templates/_example.yaml.jinja | 214 ++++++++++++++++++ 1 file changed, 214 insertions(+) create mode 100644 src/copier_template/src/{{ project_name }}/templates/_example.yaml.jinja diff --git a/src/copier_template/src/{{ project_name }}/templates/_example.yaml.jinja b/src/copier_template/src/{{ project_name }}/templates/_example.yaml.jinja new file mode 100644 index 0000000..736759b --- /dev/null +++ b/src/copier_template/src/{{ project_name }}/templates/_example.yaml.jinja @@ -0,0 +1,214 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + generateName: hera-example- + annotations: + workflows.argoproj.io/description: |- + Replicates the functionality of + example.yaml + workflows.argoproj.io/title: example remade via hera + workflows.diamond.ac.uk/repository: https://github.com/{{github_org}}/{{repo_name}} + labels: + workflows.diamond.ac.uk/science-group-examples: 'true' +spec: + entrypoint: workflowentry + templates: + - name: workflowentry + dag: + tasks: + - name: install + template: install-dependencies + - name: params + template: generate-parameters + arguments: + parameters: + - name: png + value: 'True' + - name: jpg + value: 'True' + - name: jpeg + value: 'True' + - name: tif + value: 'True' + - name: tiff + value: 'True' + - name: create-image + depends: install && params + template: create-image + withParam: '{{tasks.params.outputs.parameters.out-parameters}}' + arguments: + parameters: + - name: width + value: '{{item.width}}' + - name: height + value: '{{item.height}}' + - name: weights + value: '{{item.weights}}' + - name: extension + value: '{{item.extension}}' + - name: to-hdf5 + depends: create-image + template: to-hdf5 + arguments: + parameters: + - name: paths + value: '{{tasks.create-image.outputs.parameters.out-paths}}' + - name: install-dependencies + script: + image: python:3.10 + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import subprocess + print('creating venv') + subprocess.check_call(['python', '-m', 'venv', '/tmp/venv']) + subprocess.check_call(['/tmp/venv/bin/pip', 'install', 'pillow', 'h5py', 'numpy', 'hera']) + command: + - python + volumeMounts: + - name: tmpdir + mountPath: /tmp + - name: generate-parameters + inputs: + parameters: + - name: png + - name: jpg + - name: jpeg + - name: tif + - name: tiff + outputs: + parameters: + - name: out-parameters + valueFrom: + path: /tmp/parameters.json + script: + image: python:3.10 + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: jpeg = json.loads(r'''{{inputs.parameters.jpeg}}''') + except: jpeg = r'''{{inputs.parameters.jpeg}}''' + try: jpg = json.loads(r'''{{inputs.parameters.jpg}}''') + except: jpg = r'''{{inputs.parameters.jpg}}''' + try: png = json.loads(r'''{{inputs.parameters.png}}''') + except: png = r'''{{inputs.parameters.png}}''' + try: tif = json.loads(r'''{{inputs.parameters.tif}}''') + except: tif = r'''{{inputs.parameters.tif}}''' + try: tiff = json.loads(r'''{{inputs.parameters.tiff}}''') + except: tiff = r'''{{inputs.parameters.tiff}}''' + + import json + params: list[dict[str, int | list[int] | str] | None] = [{'width': 500, 'height': 500, 'weights': [255, 1, 100], 'extension': 'png'} if png.lower() == 'true' else None, {'width': 600, 'height': 200, 'weights': [100, 150, 100], 'extension': 'jpg'} if jpg.lower() == 'true' else None, {'width': 300, 'height': 400, 'weights': [100, 150, 100], 'extension': 'jpeg'} if jpeg.lower() == 'true' else None, {'width': 300, 'height': 200, 'weights': [230, 100, 1], 'extension': 'tif'} if tif.lower() == 'true' else None, {'width': 200, 'height': 300, 'weights': [230, 100, 1], 'extension': 'tiff'} if tiff.lower() == 'true' else None] + params_to_write: list[dict[str, int | list[int] | str]] = [image_params for image_params in params if image_params is not None] + with open('/tmp/parameters.json', 'w') as f: + json.dump(params_to_write, f) + command: + - python + volumeMounts: + - name: tmpdir + mountPath: /tmp + - name: create-image + inputs: + parameters: + - name: width + - name: height + - name: weights + - name: extension + outputs: + artifacts: + - name: '{{inputs.parameters.extension}}-image' + path: /tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}} + archive: + none: {} + parameters: + - name: out-paths + valueFrom: + path: /tmp/{{inputs.parameters.extension}}-path.json + script: + image: python:3.10 + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: extension = json.loads(r'''{{inputs.parameters.extension}}''') + except: extension = r'''{{inputs.parameters.extension}}''' + try: height = json.loads(r'''{{inputs.parameters.height}}''') + except: height = r'''{{inputs.parameters.height}}''' + try: weights = json.loads(r'''{{inputs.parameters.weights}}''') + except: weights = r'''{{inputs.parameters.weights}}''' + try: width = json.loads(r'''{{inputs.parameters.width}}''') + except: width = r'''{{inputs.parameters.width}}''' + + import json + from PIL import Image + + def create_pattern(width: int, height: int, weights: tuple[int, int, int]) -> Image.Image: + print(f'width: {width}') + print(f'height: {height}') + print(f'RBG weights: {weights}') + image = Image.new('RGB', (width, height)) + pixels = image.load() + for i in range(width): + for j in range(height): + pixels[i, j] = ((i + j * 50) % weights[0], weights[1], (i * 300 + j) % weights[2]) + return image + image = create_pattern(width, height, weights) + path = f'/tmp/{extension}-image.{extension}' + image.save(path) + with open(f'/tmp/{extension}-path.json', 'w') as f: + json.dump(path, f) + command: + - /tmp/venv/bin/python + volumeMounts: + - name: tmpdir + mountPath: /tmp + - name: to-hdf5 + inputs: + parameters: + - name: paths + outputs: + artifacts: + - name: hdf5output + path: /tmp/images.hdf5 + archive: + none: {} + script: + image: python:3.10 + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: paths = json.loads(r'''{{inputs.parameters.paths}}''') + except: paths = r'''{{inputs.parameters.paths}}''' + + import h5py + import numpy as np + from PIL import Image + print('creating hdf5 file') + with h5py.File('/tmp/images.hdf5', 'w') as f: + for i, path in enumerate(paths): + path = path.strip('"') + print(f'Got {path}') + with Image.open(path) as image: + arr = np.array(image) + f.create_dataset(f'image_{i}', data=arr, dtype=arr.dtype) + print('done') + command: + - /tmp/venv/bin/python + volumeMounts: + - name: tmpdir + mountPath: /tmp + volumeClaimTemplates: + - metadata: + name: tmpdir + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi From e0d63ee2f8bb64d2fa112e78345196bb8ceb8c23 Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Wed, 22 Jul 2026 15:26:56 +0000 Subject: [PATCH 7/8] feat(copier): updates copier template and generator copy correctly --- .../src/{{ project_name }}/templates/_example.yaml.jinja | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/copier_template/src/{{ project_name }}/templates/_example.yaml.jinja b/src/copier_template/src/{{ project_name }}/templates/_example.yaml.jinja index 736759b..3e74853 100644 --- a/src/copier_template/src/{{ project_name }}/templates/_example.yaml.jinja +++ b/src/copier_template/src/{{ project_name }}/templates/_example.yaml.jinja @@ -1,3 +1,4 @@ +{% raw %} apiVersion: argoproj.io/v1alpha1 kind: Workflow metadata: @@ -7,7 +8,7 @@ metadata: Replicates the functionality of example.yaml workflows.argoproj.io/title: example remade via hera - workflows.diamond.ac.uk/repository: https://github.com/{{github_org}}/{{repo_name}} + workflows.diamond.ac.uk/repository: https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %} labels: workflows.diamond.ac.uk/science-group-examples: 'true' spec: @@ -212,3 +213,4 @@ spec: resources: requests: storage: 1Gi +{% endraw %} From 26c8f2516ce40e9e7a1cc52c379c5fb0eb099ede Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Thu, 23 Jul 2026 10:13:01 +0000 Subject: [PATCH 8/8] feat(ipynb): adds notebook w/ instructions on how to run --- .../templates/_example.yaml.jinja | 216 ------------------ .../notebook_division.ipynb | 138 +++++++++++ 2 files changed, 138 insertions(+), 216 deletions(-) delete mode 100644 src/copier_template/src/{{ project_name }}/templates/_example.yaml.jinja create mode 100644 src/python_interface_to_workflows/workflow_definitions/notebook_division.ipynb diff --git a/src/copier_template/src/{{ project_name }}/templates/_example.yaml.jinja b/src/copier_template/src/{{ project_name }}/templates/_example.yaml.jinja deleted file mode 100644 index 3e74853..0000000 --- a/src/copier_template/src/{{ project_name }}/templates/_example.yaml.jinja +++ /dev/null @@ -1,216 +0,0 @@ -{% raw %} -apiVersion: argoproj.io/v1alpha1 -kind: Workflow -metadata: - generateName: hera-example- - annotations: - workflows.argoproj.io/description: |- - Replicates the functionality of - example.yaml - workflows.argoproj.io/title: example remade via hera - workflows.diamond.ac.uk/repository: https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %} - labels: - workflows.diamond.ac.uk/science-group-examples: 'true' -spec: - entrypoint: workflowentry - templates: - - name: workflowentry - dag: - tasks: - - name: install - template: install-dependencies - - name: params - template: generate-parameters - arguments: - parameters: - - name: png - value: 'True' - - name: jpg - value: 'True' - - name: jpeg - value: 'True' - - name: tif - value: 'True' - - name: tiff - value: 'True' - - name: create-image - depends: install && params - template: create-image - withParam: '{{tasks.params.outputs.parameters.out-parameters}}' - arguments: - parameters: - - name: width - value: '{{item.width}}' - - name: height - value: '{{item.height}}' - - name: weights - value: '{{item.weights}}' - - name: extension - value: '{{item.extension}}' - - name: to-hdf5 - depends: create-image - template: to-hdf5 - arguments: - parameters: - - name: paths - value: '{{tasks.create-image.outputs.parameters.out-paths}}' - - name: install-dependencies - script: - image: python:3.10 - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import subprocess - print('creating venv') - subprocess.check_call(['python', '-m', 'venv', '/tmp/venv']) - subprocess.check_call(['/tmp/venv/bin/pip', 'install', 'pillow', 'h5py', 'numpy', 'hera']) - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - - name: generate-parameters - inputs: - parameters: - - name: png - - name: jpg - - name: jpeg - - name: tif - - name: tiff - outputs: - parameters: - - name: out-parameters - valueFrom: - path: /tmp/parameters.json - script: - image: python:3.10 - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: jpeg = json.loads(r'''{{inputs.parameters.jpeg}}''') - except: jpeg = r'''{{inputs.parameters.jpeg}}''' - try: jpg = json.loads(r'''{{inputs.parameters.jpg}}''') - except: jpg = r'''{{inputs.parameters.jpg}}''' - try: png = json.loads(r'''{{inputs.parameters.png}}''') - except: png = r'''{{inputs.parameters.png}}''' - try: tif = json.loads(r'''{{inputs.parameters.tif}}''') - except: tif = r'''{{inputs.parameters.tif}}''' - try: tiff = json.loads(r'''{{inputs.parameters.tiff}}''') - except: tiff = r'''{{inputs.parameters.tiff}}''' - - import json - params: list[dict[str, int | list[int] | str] | None] = [{'width': 500, 'height': 500, 'weights': [255, 1, 100], 'extension': 'png'} if png.lower() == 'true' else None, {'width': 600, 'height': 200, 'weights': [100, 150, 100], 'extension': 'jpg'} if jpg.lower() == 'true' else None, {'width': 300, 'height': 400, 'weights': [100, 150, 100], 'extension': 'jpeg'} if jpeg.lower() == 'true' else None, {'width': 300, 'height': 200, 'weights': [230, 100, 1], 'extension': 'tif'} if tif.lower() == 'true' else None, {'width': 200, 'height': 300, 'weights': [230, 100, 1], 'extension': 'tiff'} if tiff.lower() == 'true' else None] - params_to_write: list[dict[str, int | list[int] | str]] = [image_params for image_params in params if image_params is not None] - with open('/tmp/parameters.json', 'w') as f: - json.dump(params_to_write, f) - command: - - python - volumeMounts: - - name: tmpdir - mountPath: /tmp - - name: create-image - inputs: - parameters: - - name: width - - name: height - - name: weights - - name: extension - outputs: - artifacts: - - name: '{{inputs.parameters.extension}}-image' - path: /tmp/{{inputs.parameters.extension}}-image.{{inputs.parameters.extension}} - archive: - none: {} - parameters: - - name: out-paths - valueFrom: - path: /tmp/{{inputs.parameters.extension}}-path.json - script: - image: python:3.10 - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: extension = json.loads(r'''{{inputs.parameters.extension}}''') - except: extension = r'''{{inputs.parameters.extension}}''' - try: height = json.loads(r'''{{inputs.parameters.height}}''') - except: height = r'''{{inputs.parameters.height}}''' - try: weights = json.loads(r'''{{inputs.parameters.weights}}''') - except: weights = r'''{{inputs.parameters.weights}}''' - try: width = json.loads(r'''{{inputs.parameters.width}}''') - except: width = r'''{{inputs.parameters.width}}''' - - import json - from PIL import Image - - def create_pattern(width: int, height: int, weights: tuple[int, int, int]) -> Image.Image: - print(f'width: {width}') - print(f'height: {height}') - print(f'RBG weights: {weights}') - image = Image.new('RGB', (width, height)) - pixels = image.load() - for i in range(width): - for j in range(height): - pixels[i, j] = ((i + j * 50) % weights[0], weights[1], (i * 300 + j) % weights[2]) - return image - image = create_pattern(width, height, weights) - path = f'/tmp/{extension}-image.{extension}' - image.save(path) - with open(f'/tmp/{extension}-path.json', 'w') as f: - json.dump(path, f) - command: - - /tmp/venv/bin/python - volumeMounts: - - name: tmpdir - mountPath: /tmp - - name: to-hdf5 - inputs: - parameters: - - name: paths - outputs: - artifacts: - - name: hdf5output - path: /tmp/images.hdf5 - archive: - none: {} - script: - image: python:3.10 - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: paths = json.loads(r'''{{inputs.parameters.paths}}''') - except: paths = r'''{{inputs.parameters.paths}}''' - - import h5py - import numpy as np - from PIL import Image - print('creating hdf5 file') - with h5py.File('/tmp/images.hdf5', 'w') as f: - for i, path in enumerate(paths): - path = path.strip('"') - print(f'Got {path}') - with Image.open(path) as image: - arr = np.array(image) - f.create_dataset(f'image_{i}', data=arr, dtype=arr.dtype) - print('done') - command: - - /tmp/venv/bin/python - volumeMounts: - - name: tmpdir - mountPath: /tmp - volumeClaimTemplates: - - metadata: - name: tmpdir - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi -{% endraw %} diff --git a/src/python_interface_to_workflows/workflow_definitions/notebook_division.ipynb b/src/python_interface_to_workflows/workflow_definitions/notebook_division.ipynb new file mode 100644 index 0000000..6808de9 --- /dev/null +++ b/src/python_interface_to_workflows/workflow_definitions/notebook_division.ipynb @@ -0,0 +1,138 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "1e3146f1", + "metadata": {}, + "source": [ + "# Running in VSCode:\n", + "\n", + "1. Set kernal to python-interface-to-workflows 3.11.x\n", + "2. Hit F1, run Jupyter: Import Notebook to Script\n", + "3. Click notebook_division.ipynb\n", + "4. Run the cells sequentially" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "494174ef", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "from hera.workflows import (\n", + " Artifact,\n", + " EmptyDirVolume,\n", + " Steps,\n", + " Workflow,\n", + " script, # pyright: ignore[reportUnknownVariableType]\n", + ")\n", + "from hera.workflows import models as m\n", + "\n", + "\n", + "@script(\n", + " volume_mounts=[m.VolumeMount(name=\"output-dir\", mount_path=\"/output-dir/\")],\n", + " outputs=Artifact(name=\"json-output\", path=\"/output-dir/output.json\"),\n", + ")\n", + "def do_division(a: int, b: int):\n", + " div = a / b\n", + " intdiv = a // b\n", + " remain = a % b\n", + " dictionary_of_results = {\n", + " \"divide\": div,\n", + " \"quotient\": intdiv,\n", + " \"remainder\": remain,\n", + " }\n", + " with open(\"/output-dir/output.json\", \"w\") as otpt:\n", + " json.dump(dictionary_of_results, otpt)\n", + "\n", + "\n", + "with Workflow(\n", + " generate_name=\"hera-division-\", # when running on graphql this should be name\n", + " entrypoint=\"divide\",\n", + " api_version=\"argoproj.io/v1alpha1\",\n", + " kind=\"Workflow\", # ClusterWorkflowTemplate\", when on graphql\n", + " labels={\"workflows.diamond.ac.uk/science-group-examples\": \"true\"},\n", + " annotations={\n", + " \"workflows.argoproj.io/title\": \"Division via hera test\",\n", + " \"workflows.argoproj.io/description\": \"\"\"Takes a numerical input and returns\n", + " the remainder, output float, and output string to a json file\"\"\",\n", + " \"workflows.diamond.ac.uk/repository\": \"https://github.com/DiamondLightSource/python-interface-to-workflows\",\n", + " },\n", + " volumes=EmptyDirVolume(name=\"output-dir\", mount_path=\"/output-dir\"),\n", + ") as w:\n", + " with Steps(name=\"divide\"):\n", + " do_division(name=\"first\", arguments={\"a\": 2, \"b\": 5})\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04708f46", + "metadata": {}, + "outputs": [], + "source": [ + "with open(\"division_from_jupyter.yaml\", \"w\") as div:\n", + " div.write(w.to_yaml()) # pyright: ignore[reportUnknownMemberType]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7e9f88cb", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "expiry_str: '1784802224' 1784801151.7235122\n" + ] + }, + { + "ename": "ValueError", + "evalue": "invalid literal for int() with base 10: \"'1784802224'\"", + "output_type": "error", + "traceback": [ + "\u001b[31m---------------------------------------------------------------------------\u001b[39m", + "\u001b[31mValueError\u001b[39m Traceback (most recent call last)", + "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[3]\u001b[39m\u001b[32m, line 3\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m python_interface_to_workflows.submit_to_argo \u001b[38;5;28;01mimport\u001b[39;00m submit_workflow_to_argo\n\u001b[32m 2\u001b[39m \n\u001b[32m----> \u001b[39m\u001b[32m3\u001b[39m submit_workflow_to_argo(w)\n", + "\u001b[36mFile \u001b[39m\u001b[32m/workspaces/python-interface-to-workflows/src/python_interface_to_workflows/submit_to_argo.py:13\u001b[39m, in \u001b[36msubmit_workflow_to_argo\u001b[39m\u001b[34m(w)\u001b[39m\n\u001b[32m 12\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34msubmit_workflow_to_argo\u001b[39m(w: Workflow):\n\u001b[32m---> \u001b[39m\u001b[32m13\u001b[39m \u001b[30;43mset_token_env_variable\u001b[39;49m\u001b[30;43m(\u001b[39;49m\u001b[30;43mstaging\u001b[39;49m\u001b[30;43m=\u001b[39;49m\u001b[30;43;01mTrue\u001b[39;49;00m\u001b[30;43m)\u001b[39;49m\n\u001b[32m 14\u001b[39m dotenv.load_dotenv(dotenv_path=\u001b[33m\"\u001b[39m\u001b[33msrc/.env\u001b[39m\u001b[33m\"\u001b[39m, override=\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[32m 15\u001b[39m w.namespace = os.environ.get(\u001b[33m\"\u001b[39m\u001b[33mNAMESPACE\u001b[39m\u001b[33m\"\u001b[39m)\n", + "\u001b[36mFile \u001b[39m\u001b[32m/workspaces/python-interface-to-workflows/src/python_interface_to_workflows/auth/keycloak_checker.py:39\u001b[39m, in \u001b[36mset_token_env_variable\u001b[39m\u001b[34m(staging)\u001b[39m\n\u001b[32m 31\u001b[39m code_challenge, code_challenge_method = generate_code_challenge(code_verifier)\n\u001b[32m 32\u001b[39m auth_url = keycloak_openid.auth_url(\n\u001b[32m 33\u001b[39m redirect_uri=\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mhttp://localhost:\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mport\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m/\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m 34\u001b[39m scope=\u001b[33m\"\u001b[39m\u001b[33mopenid posix-uid profile email fedid\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m (...)\u001b[39m\u001b[32m 37\u001b[39m code_challenge_method=code_challenge_method,\n\u001b[32m 38\u001b[39m )\n\u001b[32m---> \u001b[39m\u001b[32m39\u001b[39m \u001b[38;5;28;01mmatch\u001b[39;00m \u001b[30;43mopen_auth_url\u001b[39;49m\u001b[30;43m(\u001b[39;49m\u001b[30;43mauth_url\u001b[39;49m\u001b[30;43m,\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43mport\u001b[39;49m\u001b[30;43m)\u001b[39;49m:\n\u001b[32m 40\u001b[39m \u001b[38;5;28;01mcase\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[32m 41\u001b[39m token: \u001b[38;5;28mdict\u001b[39m[\u001b[38;5;28mstr\u001b[39m, \u001b[38;5;28mstr\u001b[39m] = ( \u001b[38;5;66;03m# pyright: ignore[reportUnknownVariableType]\u001b[39;00m\n\u001b[32m 42\u001b[39m keycloak_openid.token( \u001b[38;5;66;03m# pyright: ignore[reportUnknownMemberType]\u001b[39;00m\n\u001b[32m 43\u001b[39m grant_type=\u001b[33m\"\u001b[39m\u001b[33mauthorization_code\u001b[39m\u001b[33m\"\u001b[39m,\n\u001b[32m (...)\u001b[39m\u001b[32m 47\u001b[39m )\n\u001b[32m 48\u001b[39m )\n", + "\u001b[36mFile \u001b[39m\u001b[32m/workspaces/python-interface-to-workflows/src/python_interface_to_workflows/auth/open_auth_url.py:36\u001b[39m, in \u001b[36mopen_auth_url\u001b[39m\u001b[34m(auth_url, port)\u001b[39m\n\u001b[32m 34\u001b[39m expiry_str: \u001b[38;5;28mstr\u001b[39m = os.environ.get(\u001b[33m\"\u001b[39m\u001b[33mEXPIRY\u001b[39m\u001b[33m\"\u001b[39m) \u001b[38;5;66;03m# pyright: ignore\u001b[39;00m\n\u001b[32m 35\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mexpiry_str: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mexpiry_str\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m, \u001b[38;5;28mstr\u001b[39m(time.time()))\n\u001b[32m---> \u001b[39m\u001b[32m36\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m (expiry_str == \u001b[33m\"\u001b[39m\u001b[33m\"\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m \u001b[30;43mint\u001b[39;49m\u001b[30;43m(\u001b[39;49m\u001b[30;43mexpiry_str\u001b[39;49m\u001b[30;43m)\u001b[39;49m) <= \u001b[38;5;28mfloat\u001b[39m(time.time()):\n\u001b[32m 37\u001b[39m httpd = _ReusingHTTPServer((\u001b[33m\"\u001b[39m\u001b[33mlocalhost\u001b[39m\u001b[33m\"\u001b[39m, port), CallbackHandler)\n\u001b[32m 38\u001b[39m webbrowser.open(auth_url)\n", + "\u001b[31mValueError\u001b[39m: invalid literal for int() with base 10: \"'1784802224'\"" + ] + } + ], + "source": [ + "from python_interface_to_workflows.submit_to_argo import submit_workflow_to_argo\n", + "\n", + "submit_workflow_to_argo(w)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "python-interface-to-workflows (3.11.x)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.15" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}