From ffd9f6740b630b20c1bcd755be6be5c9294f577e Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Mon, 27 Jul 2026 11:45:17 +0000 Subject: [PATCH 1/3] feat(graphql): reworks submission via graphql --- scripts/checkyamlcompliance.py | 2 + .../scripts/checkyamlcompliance.py.jinja | 2 + .../submit_to_graphql.py.jinja | 60 ++++++++----- .../templates/example.txt.jinja | 2 +- .../create_example_template.py.jinja | 2 +- .../tests/test_submit_to_graphql.py.jinja | 26 ++++-- .../tests/test_submit_workflow_to_argo.jinja | 29 ++++--- .../submit_to_graphql.py | 52 ++++++----- .../templates/divisionyaml.yaml | 86 +++++++++---------- .../templates/example.txt | 2 +- .../create_division_yaml.py | 2 +- .../create_example_template.py | 2 +- tests/test_submit_to_graphql.py | 24 ++++-- 13 files changed, 173 insertions(+), 118 deletions(-) diff --git a/scripts/checkyamlcompliance.py b/scripts/checkyamlcompliance.py index 834655d..2f64a09 100644 --- a/scripts/checkyamlcompliance.py +++ b/scripts/checkyamlcompliance.py @@ -24,6 +24,8 @@ api_ver = False if "kind" in yamldata.keys(): match yamldata["kind"]: + case "ClusterWorkflowTemplate": + clst_tmpt = True case "Workflow": clst_tmpt = True case "WorkflowTemplate": diff --git a/src/copier_template/scripts/checkyamlcompliance.py.jinja b/src/copier_template/scripts/checkyamlcompliance.py.jinja index 020b14a..b020cd6 100644 --- a/src/copier_template/scripts/checkyamlcompliance.py.jinja +++ b/src/copier_template/scripts/checkyamlcompliance.py.jinja @@ -24,6 +24,8 @@ for file in yamllist: api_ver = False if "kind" in yamldata.keys(): match yamldata["kind"]: + case "ClusterWorkflowTemplate": + clst_tmpt = True case "Workflow": clst_tmpt = True case _: diff --git a/src/copier_template/src/{{ project_name }}/submit_to_graphql.py.jinja b/src/copier_template/src/{{ project_name }}/submit_to_graphql.py.jinja index edfb8c9..828703b 100644 --- a/src/copier_template/src/{{ project_name }}/submit_to_graphql.py.jinja +++ b/src/copier_template/src/{{ project_name }}/submit_to_graphql.py.jinja @@ -1,11 +1,23 @@ +{% raw %} +import os + +import dotenv from gql import Client, gql from gql.transport.aiohttp import AIOHTTPTransport -import os -from {{project_name}}.auth.keycloak_checker import set_token_env_variable -def submit_to_graphql(): - token = set_token_env_variable() +from hera.workflows import Workflow + +from {% endraw %}{{project_name}}{% raw %}.auth.keycloak_checker import set_token_env_variable + + +def submit_workflow_to_graphql(w: Workflow): + yamlstr = w.to_yaml() # pyright:ignore + dotenv.load_dotenv(dotenv_path="src/.env", override=True) + token: str = set_token_env_variable(True) + host: str = "https://staging.workflows.diamond.ac.uk/graphql" + visit: str = os.environ.get("NAMESPACE") # pyright:ignore + transport = AIOHTTPTransport( - url="https://workflows.diamond.ac.uk/graphql", + url=host, headers={"Authorization": f"Bearer {token}"}, ) client = Client( @@ -13,24 +25,26 @@ def submit_to_graphql(): fetch_schema_from_transport=True, ) mutation = gql(""" -mutation SubmitDivision { - submitWorkflowTemplate( - name: "division" - visit: { - proposalCode: "ks", - proposalNumber: 10000, - number: 3 - } - parameters: { - numinput: "19", - numdivisor: "10" - } - ){ +mutation Submit($visit: VisitInput!, $manifest: String!) { + submitWorkflow( + visit: $visit + manifest: $manifest + ) { name -} + } } """) - result = client.execute(mutation) - visit = "ks10000-3/c" - name = str(result["submitWorkflowTemplate"]["name"]) - print(f"Job '{visit}-{name}' submitted.") + result = client.execute( + mutation, + variable_values={ + "visit": { + "proposalCode": str(visit[:2]), + "proposalNumber": int(visit[2:7]), + "number": int(visit[-1]), + }, + "manifest": yamlstr, + }, + ) + name = str(result["submitWorkflow"]["name"]) + print(f"Job '{name}' submitted to {visit}") +{% endraw %} diff --git a/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja b/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja index 387db1c..78cb885 100644 --- a/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja +++ b/src/copier_template/src/{{ project_name }}/templates/example.txt.jinja @@ -2,7 +2,7 @@ apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: - generateName: hera-example- + name: hera-example annotations: workflows.argoproj.io/description: |- Replicates the functionality of diff --git a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja index 42175cf..f438048 100644 --- a/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja +++ b/src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja @@ -145,7 +145,7 @@ def to_hdf5(paths: str): with Workflow( - generate_name="hera-example-", # when running on graphql this should be name + name="hera-example", # when running on argo this should be generate_name: ...- entrypoint="workflowentry", api_version="argoproj.io/v1alpha1", kind="WorkflowTemplate", # ClusterWorkflowTemplate", when on graphql diff --git a/src/copier_template/tests/test_submit_to_graphql.py.jinja b/src/copier_template/tests/test_submit_to_graphql.py.jinja index fcf376e..1d304ac 100644 --- a/src/copier_template/tests/test_submit_to_graphql.py.jinja +++ b/src/copier_template/tests/test_submit_to_graphql.py.jinja @@ -1,17 +1,27 @@ from unittest.mock import MagicMock, patch -from {{project_name}}.submit_to_graphql import submit_to_graphql +from {{project_name}}.submit_to_graphql import submit_workflow_to_graphql +@patch("{{project_name}}.submit_to_argo.os.environ.get") +@patch("{{project_name}}.submit_to_argo.dotenv.load_dotenv") +@patch("{{project_name}}.submit_to_argo.Workflow") @patch("{{project_name}}.submit_to_graphql.set_token_env_variable") @patch("{{project_name}}.submit_to_graphql.Client") -def test_submit_to_graphql(mock_client: MagicMock, mock_key: MagicMock): - mock_key.return_value = "fake_token" +def test_submit_workflow_to_graphql( + mock_client: MagicMock, + mock_key: MagicMock, + mock_workflow: MagicMock, + mock_load_env: MagicMock, + mock_os_get: MagicMock, +): + mock_instance = MagicMock() - mock_token.return_value = "token" + mock_key.return_value = "token" mock_client.return_value = mock_instance - mock_instance.execute.return_value = { - "submitWorkflowTemplate": {"name": "workflow123"} - } - submit_to_graphql() + mock_instance.execute.return_value = {"submitWorkflow": {"name": "workflow123"}} + submit_workflow_to_graphql(mock_workflow) + mock_load_env.assert_called_once_with(dotenv_path="src/.env", override=True) mock_instance.execute.assert_called_once() + mock_workflow.to_yaml.assert_called_once() + mock_os_get.assert_called_once_with("NAMESPACE") diff --git a/src/copier_template/tests/test_submit_workflow_to_argo.jinja b/src/copier_template/tests/test_submit_workflow_to_argo.jinja index c6eef98..1d304ac 100644 --- a/src/copier_template/tests/test_submit_workflow_to_argo.jinja +++ b/src/copier_template/tests/test_submit_workflow_to_argo.jinja @@ -1,20 +1,27 @@ -from unittest.mock import MagicMock, call, patch +from unittest.mock import MagicMock, patch -from {{project_name}}.submit_to_argo import submit_workflow_to_argo +from {{project_name}}.submit_to_graphql import submit_workflow_to_graphql -@patch("{{project_name}}.submit_to_argo.Workflow") @patch("{{project_name}}.submit_to_argo.os.environ.get") @patch("{{project_name}}.submit_to_argo.dotenv.load_dotenv") -@patch("{{project_name}}.submit_to_argo.set_token_env_variable") -def test_submit_workflow_to_argo( +@patch("{{project_name}}.submit_to_argo.Workflow") +@patch("{{project_name}}.submit_to_graphql.set_token_env_variable") +@patch("{{project_name}}.submit_to_graphql.Client") +def test_submit_workflow_to_graphql( + mock_client: MagicMock, mock_key: MagicMock, - mock_load_env: MagicMock, - mock_get_env_var: MagicMock, mock_workflow: MagicMock, + mock_load_env: MagicMock, + mock_os_get: MagicMock, ): - mock_workflow = MagicMock() - submit_workflow_to_argo(mock_workflow) - mock_key.assert_called_once_with(staging=True) + + mock_instance = MagicMock() + mock_key.return_value = "token" + mock_client.return_value = mock_instance + mock_instance.execute.return_value = {"submitWorkflow": {"name": "workflow123"}} + submit_workflow_to_graphql(mock_workflow) mock_load_env.assert_called_once_with(dotenv_path="src/.env", override=True) - mock_get_env_var.assert_has_calls([call("HOST"), call("TOKEN")], any_order=True) + mock_instance.execute.assert_called_once() + mock_workflow.to_yaml.assert_called_once() + mock_os_get.assert_called_once_with("NAMESPACE") diff --git a/src/python_interface_to_workflows/submit_to_graphql.py b/src/python_interface_to_workflows/submit_to_graphql.py index e449fc9..55abd72 100644 --- a/src/python_interface_to_workflows/submit_to_graphql.py +++ b/src/python_interface_to_workflows/submit_to_graphql.py @@ -1,13 +1,22 @@ +import os + +import dotenv from gql import Client, gql from gql.transport.aiohttp import AIOHTTPTransport +from hera.workflows import Workflow from python_interface_to_workflows.auth.keycloak_checker import set_token_env_variable -def submit_to_graphql(): - token = set_token_env_variable(True) +def submit_workflow_to_graphql(w: Workflow): + yamlstr = w.to_yaml() # pyright:ignore + dotenv.load_dotenv(dotenv_path="src/.env", override=True) + token: str = set_token_env_variable(True) + host: str = "https://staging.workflows.diamond.ac.uk/graphql" + visit: str = os.environ.get("NAMESPACE") # pyright:ignore + transport = AIOHTTPTransport( - url="https://staging.workflows.diamond.ac.uk/graphql", + url=host, headers={"Authorization": f"Bearer {token}"}, ) client = Client( @@ -15,24 +24,25 @@ def submit_to_graphql(): fetch_schema_from_transport=True, ) mutation = gql(""" -mutation SubmitDivision { - submitWorkflowTemplate( - name: "division" - visit: { - proposalCode: "ks", - proposalNumber: 10000, - number: 3 - } - parameters: { - numinput: "19", - numdivisor: "10" - } - ){ +mutation Submit($visit: VisitInput!, $manifest: String!) { + submitWorkflow( + visit: $visit + manifest: $manifest + ) { name -} + } } """) - result = client.execute(mutation) - visit = "ks10000-3/c" - name = str(result["submitWorkflowTemplate"]["name"]) - print(f"Job '{visit}-{name}' submitted.") + result = client.execute( + mutation, + variable_values={ + "visit": { + "proposalCode": str(visit[:2]), + "proposalNumber": int(visit[2:7]), + "number": int(visit[-1]), + }, + "manifest": yamlstr, + }, + ) + name = str(result["submitWorkflow"]["name"]) + print(f"Job '{name}' submitted to {visit}") diff --git a/src/python_interface_to_workflows/templates/divisionyaml.yaml b/src/python_interface_to_workflows/templates/divisionyaml.yaml index 79b810a..bf75377 100644 --- a/src/python_interface_to_workflows/templates/divisionyaml.yaml +++ b/src/python_interface_to_workflows/templates/divisionyaml.yaml @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: - generateName: hera-division- + name: hera-division annotations: workflows.argoproj.io/description: |- Takes a numerical input and returns @@ -9,52 +9,52 @@ metadata: workflows.argoproj.io/title: Division via hera test workflows.diamond.ac.uk/repository: https://github.com/DiamondLightSource/python-interface-to-workflows labels: - workflows.diamond.ac.uk/science-group-examples: 'true' + workflows.diamond.ac.uk/science-group-examples: "true" spec: entrypoint: divide templates: - - name: divide - steps: - - - name: first - template: do-division - arguments: - parameters: + - name: divide + steps: + - - name: first + template: do-division + arguments: + parameters: + - name: a + value: "2" + - name: b + value: "5" + - name: do-division + inputs: + parameters: - name: a - value: '2' - name: b - value: '5' - - name: do-division - inputs: - parameters: - - name: a - - name: b - outputs: - artifacts: - - name: json-output - path: /output-dir/output.json - script: - image: python:3.10 - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: a = json.loads(r'''{{inputs.parameters.a}}''') - except: a = r'''{{inputs.parameters.a}}''' - try: b = json.loads(r'''{{inputs.parameters.b}}''') - except: b = r'''{{inputs.parameters.b}}''' + outputs: + artifacts: + - name: json-output + path: /output-dir/output.json + script: + image: python:3.10 + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: a = json.loads(r'''{{inputs.parameters.a}}''') + except: a = r'''{{inputs.parameters.a}}''' + try: b = json.loads(r'''{{inputs.parameters.b}}''') + except: b = r'''{{inputs.parameters.b}}''' - div = a / b - intdiv = a // b - remain = a % b - dictionary_of_results = {'divide': div, 'quotient': intdiv, 'remainder': remain} - with open('/output-dir/output.json', 'w') as otpt: - json.dump(dictionary_of_results, otpt) - command: - - python - volumeMounts: - - name: output-dir - mountPath: /output-dir/ + div = a / b + intdiv = a // b + remain = a % b + dictionary_of_results = {'divide': div, 'quotient': intdiv, 'remainder': remain} + with open('/output-dir/output.json', 'w') as otpt: + json.dump(dictionary_of_results, otpt) + command: + - python + volumeMounts: + - name: output-dir + mountPath: /output-dir/ volumes: - - name: output-dir - emptyDir: {} + - name: output-dir + emptyDir: {} diff --git a/src/python_interface_to_workflows/templates/example.txt b/src/python_interface_to_workflows/templates/example.txt index 857fb41..1ad8e0e 100644 --- a/src/python_interface_to_workflows/templates/example.txt +++ b/src/python_interface_to_workflows/templates/example.txt @@ -1,7 +1,7 @@ apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate metadata: - generateName: hera-example- + name: hera-example annotations: workflows.argoproj.io/description: |- Replicates the functionality of diff --git a/src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py b/src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py index 3f063de..8a57ae1 100644 --- a/src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py +++ b/src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py @@ -28,7 +28,7 @@ def do_division(a: int, b: int): with Workflow( - generate_name="hera-division-", # when running on graphql this should be name + name="hera-division", # when running on argo this should be generate_name: ...- entrypoint="divide", api_version="argoproj.io/v1alpha1", kind="WorkflowTemplate", # ClusterWorkflowTemplate", when on graphql diff --git a/src/python_interface_to_workflows/workflow_definitions/create_example_template.py b/src/python_interface_to_workflows/workflow_definitions/create_example_template.py index 7b9de3c..013387a 100644 --- a/src/python_interface_to_workflows/workflow_definitions/create_example_template.py +++ b/src/python_interface_to_workflows/workflow_definitions/create_example_template.py @@ -144,7 +144,7 @@ def to_hdf5(paths: str): with Workflow( - generate_name="hera-example-", # when running on graphql this should be name + name="hera-example", # when running on argo this should be generate_name: ...- entrypoint="workflowentry", api_version="argoproj.io/v1alpha1", kind="WorkflowTemplate", # ClusterWorkflowTemplate", when on graphql diff --git a/tests/test_submit_to_graphql.py b/tests/test_submit_to_graphql.py index 37e6a11..f726299 100644 --- a/tests/test_submit_to_graphql.py +++ b/tests/test_submit_to_graphql.py @@ -1,17 +1,27 @@ from unittest.mock import MagicMock, patch -from python_interface_to_workflows.submit_to_graphql import submit_to_graphql +from python_interface_to_workflows.submit_to_graphql import submit_workflow_to_graphql +@patch("python_interface_to_workflows.submit_to_argo.os.environ.get") +@patch("python_interface_to_workflows.submit_to_argo.dotenv.load_dotenv") +@patch("python_interface_to_workflows.submit_to_argo.Workflow") @patch("python_interface_to_workflows.submit_to_graphql.set_token_env_variable") @patch("python_interface_to_workflows.submit_to_graphql.Client") -def test_submit_to_graphql(mock_client: MagicMock, mock_key: MagicMock): - mock_key.return_value = "fake_token" +def test_submit_workflow_to_graphql( + mock_client: MagicMock, + mock_key: MagicMock, + mock_workflow: MagicMock, + mock_load_env: MagicMock, + mock_os_get: MagicMock, +): + mock_instance = MagicMock() mock_key.return_value = "token" mock_client.return_value = mock_instance - mock_instance.execute.return_value = { - "submitWorkflowTemplate": {"name": "workflow123"} - } - submit_to_graphql() + mock_instance.execute.return_value = {"submitWorkflow": {"name": "workflow123"}} + submit_workflow_to_graphql(mock_workflow) + mock_load_env.assert_called_once_with(dotenv_path="src/.env", override=True) mock_instance.execute.assert_called_once() + mock_workflow.to_yaml.assert_called_once() + mock_os_get.assert_called_once_with("NAMESPACE") From 187e11e8515b45bf91e7cdcec7eba00cc8c96450 Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Tue, 28 Jul 2026 15:11:04 +0000 Subject: [PATCH 2/3] feat(graphql): removes argo, alters graphql to have clearer name --- src/copier_template/src/README.md.jinja | 6 +- .../submit_to_argo.py.jinja | 24 ------ ...phql.py.jinja => submit_workflow.py.jinja} | 4 +- .../tests/test_submit_to_graphql.py.jinja | 16 ++-- .../tests/test_submit_workflow_to_argo.jinja | 27 ------ .../submit_to_argo.py | 24 ------ ...ubmit_to_graphql.py => submit_workflow.py} | 4 +- .../templates/divisionyaml.yaml | 84 +++++++++---------- tests/test_submit_to_argo.py | 20 ----- tests/test_submit_to_graphql.py | 18 ++-- 10 files changed, 66 insertions(+), 161 deletions(-) delete mode 100644 src/copier_template/src/{{ project_name }}/submit_to_argo.py.jinja rename src/copier_template/src/{{ project_name }}/{submit_to_graphql.py.jinja => submit_workflow.py.jinja} (91%) delete mode 100644 src/copier_template/tests/test_submit_workflow_to_argo.jinja delete mode 100644 src/python_interface_to_workflows/submit_to_argo.py rename src/python_interface_to_workflows/{submit_to_graphql.py => submit_workflow.py} (91%) delete mode 100644 tests/test_submit_to_argo.py diff --git a/src/copier_template/src/README.md.jinja b/src/copier_template/src/README.md.jinja index 816c35d..c09fef6 100644 --- a/src/copier_template/src/README.md.jinja +++ b/src/copier_template/src/README.md.jinja @@ -11,16 +11,16 @@ AUTH= 3. Build the dev container -# Submitting a workflow to Argo +# Submitting a workflow 1. At the start of your workflow definition file, add: ```python -from {{project_name}}.submit_to_argo import submit_workflow_to_argo +from {{project_name}}.submit_to_graphql import submit_workflow ``` 2. At the end of a workflow definition file, once your workflow (w) is created, add: ```python -submit_workflow_to_argo(w) +submit_workflow(w) ``` NOTE: Be sure to remove this line upon commiting changes, as all workflow definition files are diff --git a/src/copier_template/src/{{ project_name }}/submit_to_argo.py.jinja b/src/copier_template/src/{{ project_name }}/submit_to_argo.py.jinja deleted file mode 100644 index 1b30683..0000000 --- a/src/copier_template/src/{{ project_name }}/submit_to_argo.py.jinja +++ /dev/null @@ -1,24 +0,0 @@ -import os -from typing import cast - -import dotenv -from hera.shared import global_config -from hera.workflows import Workflow, WorkflowsService -from hera.workflows import models as m - -from {{project_name}}.auth.keycloak_checker import set_token_env_variable - - -def submit_workflow_to_argo(w: Workflow): - set_token_env_variable() - dotenv.load_dotenv(dotenv_path="src/.env", override=True) - w.namespace = os.environ.get("VISIT") - w.workflows_service = WorkflowsService( - host=str(os.environ.get("HOST")).strip("'"), - token=str(os.environ.get("TOKEN")).strip("'"), - ) - global_config.image = str(os.environ.get("DEFAULT_IMAGE")) - submitted_w = cast(m.Workflow, w.create()) - name = submitted_w.metadata.name - namespace = submitted_w.metadata.namespace - print(f"submitted {name} to {namespace}") diff --git a/src/copier_template/src/{{ project_name }}/submit_to_graphql.py.jinja b/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja similarity index 91% rename from src/copier_template/src/{{ project_name }}/submit_to_graphql.py.jinja rename to src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja index 828703b..44a70ab 100644 --- a/src/copier_template/src/{{ project_name }}/submit_to_graphql.py.jinja +++ b/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja @@ -9,11 +9,11 @@ from hera.workflows import Workflow from {% endraw %}{{project_name}}{% raw %}.auth.keycloak_checker import set_token_env_variable -def submit_workflow_to_graphql(w: Workflow): +def submit_workflow(w: Workflow): yamlstr = w.to_yaml() # pyright:ignore dotenv.load_dotenv(dotenv_path="src/.env", override=True) token: str = set_token_env_variable(True) - host: str = "https://staging.workflows.diamond.ac.uk/graphql" + host: str = os.environ.get("HOST") # pyright:ignore visit: str = os.environ.get("NAMESPACE") # pyright:ignore transport = AIOHTTPTransport( diff --git a/src/copier_template/tests/test_submit_to_graphql.py.jinja b/src/copier_template/tests/test_submit_to_graphql.py.jinja index 1d304ac..961f6a6 100644 --- a/src/copier_template/tests/test_submit_to_graphql.py.jinja +++ b/src/copier_template/tests/test_submit_to_graphql.py.jinja @@ -1,13 +1,13 @@ from unittest.mock import MagicMock, patch -from {{project_name}}.submit_to_graphql import submit_workflow_to_graphql +from {{project_name}}.submit_workflow import submit_workflow -@patch("{{project_name}}.submit_to_argo.os.environ.get") -@patch("{{project_name}}.submit_to_argo.dotenv.load_dotenv") -@patch("{{project_name}}.submit_to_argo.Workflow") -@patch("{{project_name}}.submit_to_graphql.set_token_env_variable") -@patch("{{project_name}}.submit_to_graphql.Client") +@patch("{{project_name}}.submit_workflow.os.environ.get") +@patch("{{project_name}}.submit_workflow.dotenv.load_dotenv") +@patch("{{project_name}}.submit_workflow.Workflow") +@patch("{{project_name}}.submit_workflow.set_token_env_variable") +@patch("{{project_name}}.submit_workflow.Client") def test_submit_workflow_to_graphql( mock_client: MagicMock, mock_key: MagicMock, @@ -20,8 +20,8 @@ def test_submit_workflow_to_graphql( mock_key.return_value = "token" mock_client.return_value = mock_instance mock_instance.execute.return_value = {"submitWorkflow": {"name": "workflow123"}} - submit_workflow_to_graphql(mock_workflow) + submit_workflow(mock_workflow) mock_load_env.assert_called_once_with(dotenv_path="src/.env", override=True) mock_instance.execute.assert_called_once() mock_workflow.to_yaml.assert_called_once() - mock_os_get.assert_called_once_with("NAMESPACE") + mock_os_get.assert_has_calls([call("NAMESPACE"), call("HOST")], any_order=True) diff --git a/src/copier_template/tests/test_submit_workflow_to_argo.jinja b/src/copier_template/tests/test_submit_workflow_to_argo.jinja deleted file mode 100644 index 1d304ac..0000000 --- a/src/copier_template/tests/test_submit_workflow_to_argo.jinja +++ /dev/null @@ -1,27 +0,0 @@ -from unittest.mock import MagicMock, patch - -from {{project_name}}.submit_to_graphql import submit_workflow_to_graphql - - -@patch("{{project_name}}.submit_to_argo.os.environ.get") -@patch("{{project_name}}.submit_to_argo.dotenv.load_dotenv") -@patch("{{project_name}}.submit_to_argo.Workflow") -@patch("{{project_name}}.submit_to_graphql.set_token_env_variable") -@patch("{{project_name}}.submit_to_graphql.Client") -def test_submit_workflow_to_graphql( - mock_client: MagicMock, - mock_key: MagicMock, - mock_workflow: MagicMock, - mock_load_env: MagicMock, - mock_os_get: MagicMock, -): - - mock_instance = MagicMock() - mock_key.return_value = "token" - mock_client.return_value = mock_instance - mock_instance.execute.return_value = {"submitWorkflow": {"name": "workflow123"}} - submit_workflow_to_graphql(mock_workflow) - mock_load_env.assert_called_once_with(dotenv_path="src/.env", override=True) - mock_instance.execute.assert_called_once() - mock_workflow.to_yaml.assert_called_once() - mock_os_get.assert_called_once_with("NAMESPACE") diff --git a/src/python_interface_to_workflows/submit_to_argo.py b/src/python_interface_to_workflows/submit_to_argo.py deleted file mode 100644 index 0ec1233..0000000 --- a/src/python_interface_to_workflows/submit_to_argo.py +++ /dev/null @@ -1,24 +0,0 @@ -import os -from typing import cast - -import dotenv -from hera.shared import global_config -from hera.workflows import Workflow, WorkflowsService -from hera.workflows import models as m - -from python_interface_to_workflows.auth.keycloak_checker import set_token_env_variable - - -def submit_workflow_to_argo(w: Workflow): - set_token_env_variable(staging=True) - dotenv.load_dotenv(dotenv_path="src/.env", override=True) - w.namespace = os.environ.get("VISIT") - w.workflows_service = WorkflowsService( - host=str(os.environ.get("HOST")).strip("'"), - token=str(os.environ.get("TOKEN")).strip("'"), - ) - global_config.image = str(os.environ.get("DEFAULT_IMAGE")) - submitted_w = cast(m.Workflow, w.create()) - name = submitted_w.metadata.name - namespace = submitted_w.metadata.namespace - print(f"submitted {name} to {namespace}") diff --git a/src/python_interface_to_workflows/submit_to_graphql.py b/src/python_interface_to_workflows/submit_workflow.py similarity index 91% rename from src/python_interface_to_workflows/submit_to_graphql.py rename to src/python_interface_to_workflows/submit_workflow.py index 55abd72..8290e73 100644 --- a/src/python_interface_to_workflows/submit_to_graphql.py +++ b/src/python_interface_to_workflows/submit_workflow.py @@ -8,11 +8,11 @@ from python_interface_to_workflows.auth.keycloak_checker import set_token_env_variable -def submit_workflow_to_graphql(w: Workflow): +def submit_workflow(w: Workflow): yamlstr = w.to_yaml() # pyright:ignore dotenv.load_dotenv(dotenv_path="src/.env", override=True) token: str = set_token_env_variable(True) - host: str = "https://staging.workflows.diamond.ac.uk/graphql" + host: str = os.environ.get("HOST") # pyright:ignore visit: str = os.environ.get("NAMESPACE") # pyright:ignore transport = AIOHTTPTransport( diff --git a/src/python_interface_to_workflows/templates/divisionyaml.yaml b/src/python_interface_to_workflows/templates/divisionyaml.yaml index bf75377..c9ca1ea 100644 --- a/src/python_interface_to_workflows/templates/divisionyaml.yaml +++ b/src/python_interface_to_workflows/templates/divisionyaml.yaml @@ -9,52 +9,52 @@ metadata: workflows.argoproj.io/title: Division via hera test workflows.diamond.ac.uk/repository: https://github.com/DiamondLightSource/python-interface-to-workflows labels: - workflows.diamond.ac.uk/science-group-examples: "true" + workflows.diamond.ac.uk/science-group-examples: 'true' spec: entrypoint: divide templates: - - name: divide - steps: - - - name: first - template: do-division - arguments: - parameters: - - name: a - value: "2" - - name: b - value: "5" - - name: do-division - inputs: - parameters: + - name: divide + steps: + - - name: first + template: do-division + arguments: + parameters: - name: a + value: '2' - name: b - outputs: - artifacts: - - name: json-output - path: /output-dir/output.json - script: - image: python:3.10 - source: |- - import os - import sys - sys.path.append(os.getcwd()) - import json - try: a = json.loads(r'''{{inputs.parameters.a}}''') - except: a = r'''{{inputs.parameters.a}}''' - try: b = json.loads(r'''{{inputs.parameters.b}}''') - except: b = r'''{{inputs.parameters.b}}''' + value: '5' + - name: do-division + inputs: + parameters: + - name: a + - name: b + outputs: + artifacts: + - name: json-output + path: /output-dir/output.json + script: + image: python:3.10 + source: |- + import os + import sys + sys.path.append(os.getcwd()) + import json + try: a = json.loads(r'''{{inputs.parameters.a}}''') + except: a = r'''{{inputs.parameters.a}}''' + try: b = json.loads(r'''{{inputs.parameters.b}}''') + except: b = r'''{{inputs.parameters.b}}''' - div = a / b - intdiv = a // b - remain = a % b - dictionary_of_results = {'divide': div, 'quotient': intdiv, 'remainder': remain} - with open('/output-dir/output.json', 'w') as otpt: - json.dump(dictionary_of_results, otpt) - command: - - python - volumeMounts: - - name: output-dir - mountPath: /output-dir/ + div = a / b + intdiv = a // b + remain = a % b + dictionary_of_results = {'divide': div, 'quotient': intdiv, 'remainder': remain} + with open('/output-dir/output.json', 'w') as otpt: + json.dump(dictionary_of_results, otpt) + command: + - python + volumeMounts: + - name: output-dir + mountPath: /output-dir/ volumes: - - name: output-dir - emptyDir: {} + - name: output-dir + emptyDir: {} diff --git a/tests/test_submit_to_argo.py b/tests/test_submit_to_argo.py deleted file mode 100644 index b8be696..0000000 --- a/tests/test_submit_to_argo.py +++ /dev/null @@ -1,20 +0,0 @@ -from unittest.mock import MagicMock, call, patch - -from python_interface_to_workflows.submit_to_argo import submit_workflow_to_argo - - -@patch("python_interface_to_workflows.submit_to_argo.Workflow") -@patch("python_interface_to_workflows.submit_to_argo.os.environ.get") -@patch("python_interface_to_workflows.submit_to_argo.dotenv.load_dotenv") -@patch("python_interface_to_workflows.submit_to_argo.set_token_env_variable") -def test_submit_workflow_to_argo( - mock_key: MagicMock, - mock_load_env: MagicMock, - mock_get_env_var: MagicMock, - mock_workflow: MagicMock, -): - mock_workflow = MagicMock() - submit_workflow_to_argo(mock_workflow) - mock_key.assert_called_once_with(staging=True) - mock_load_env.assert_called_once_with(dotenv_path="src/.env", override=True) - mock_get_env_var.assert_has_calls([call("HOST"), call("TOKEN")], any_order=True) diff --git a/tests/test_submit_to_graphql.py b/tests/test_submit_to_graphql.py index f726299..f562fa5 100644 --- a/tests/test_submit_to_graphql.py +++ b/tests/test_submit_to_graphql.py @@ -1,13 +1,13 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, call, patch -from python_interface_to_workflows.submit_to_graphql import submit_workflow_to_graphql +from python_interface_to_workflows.submit_workflow import submit_workflow -@patch("python_interface_to_workflows.submit_to_argo.os.environ.get") -@patch("python_interface_to_workflows.submit_to_argo.dotenv.load_dotenv") -@patch("python_interface_to_workflows.submit_to_argo.Workflow") -@patch("python_interface_to_workflows.submit_to_graphql.set_token_env_variable") -@patch("python_interface_to_workflows.submit_to_graphql.Client") +@patch("python_interface_to_workflows.submit_workflow.os.environ.get") +@patch("python_interface_to_workflows.submit_workflow.dotenv.load_dotenv") +@patch("python_interface_to_workflows.submit_workflow.Workflow") +@patch("python_interface_to_workflows.submit_workflow.set_token_env_variable") +@patch("python_interface_to_workflows.submit_workflow.Client") def test_submit_workflow_to_graphql( mock_client: MagicMock, mock_key: MagicMock, @@ -20,8 +20,8 @@ def test_submit_workflow_to_graphql( mock_key.return_value = "token" mock_client.return_value = mock_instance mock_instance.execute.return_value = {"submitWorkflow": {"name": "workflow123"}} - submit_workflow_to_graphql(mock_workflow) + submit_workflow(mock_workflow) mock_load_env.assert_called_once_with(dotenv_path="src/.env", override=True) mock_instance.execute.assert_called_once() mock_workflow.to_yaml.assert_called_once() - mock_os_get.assert_called_once_with("NAMESPACE") + mock_os_get.assert_has_calls([call("NAMESPACE"), call("HOST")], any_order=True) From 5bc2c5a57bfe0420159f9fda1a9f7e5327148cdb Mon Sep 17 00:00:00 2001 From: Matthew Carre Date: Wed, 29 Jul 2026 08:26:31 +0000 Subject: [PATCH 3/3] feat(graphql): alters submission to take in VISIT not NAMESPACE --- .../src/{{ project_name }}/submit_workflow.py.jinja | 4 ++-- src/copier_template/tests/test_submit_to_graphql.py.jinja | 2 +- src/python_interface_to_workflows/submit_workflow.py | 4 ++-- tests/test_submit_to_graphql.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja b/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja index 44a70ab..597599a 100644 --- a/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja +++ b/src/copier_template/src/{{ project_name }}/submit_workflow.py.jinja @@ -14,7 +14,7 @@ def submit_workflow(w: Workflow): dotenv.load_dotenv(dotenv_path="src/.env", override=True) token: str = set_token_env_variable(True) host: str = os.environ.get("HOST") # pyright:ignore - visit: str = os.environ.get("NAMESPACE") # pyright:ignore + visit: str = os.environ.get("VISIT") # pyright:ignore transport = AIOHTTPTransport( url=host, @@ -42,7 +42,7 @@ mutation Submit($visit: VisitInput!, $manifest: String!) { "proposalNumber": int(visit[2:7]), "number": int(visit[-1]), }, - "manifest": yamlstr, + "manifest": f"""{yamlstr}""", }, ) name = str(result["submitWorkflow"]["name"]) diff --git a/src/copier_template/tests/test_submit_to_graphql.py.jinja b/src/copier_template/tests/test_submit_to_graphql.py.jinja index 961f6a6..25add52 100644 --- a/src/copier_template/tests/test_submit_to_graphql.py.jinja +++ b/src/copier_template/tests/test_submit_to_graphql.py.jinja @@ -24,4 +24,4 @@ def test_submit_workflow_to_graphql( mock_load_env.assert_called_once_with(dotenv_path="src/.env", override=True) mock_instance.execute.assert_called_once() mock_workflow.to_yaml.assert_called_once() - mock_os_get.assert_has_calls([call("NAMESPACE"), call("HOST")], any_order=True) + mock_os_get.assert_has_calls([call("VISIT"), call("HOST")], any_order=True) diff --git a/src/python_interface_to_workflows/submit_workflow.py b/src/python_interface_to_workflows/submit_workflow.py index 8290e73..da3422a 100644 --- a/src/python_interface_to_workflows/submit_workflow.py +++ b/src/python_interface_to_workflows/submit_workflow.py @@ -13,7 +13,7 @@ def submit_workflow(w: Workflow): dotenv.load_dotenv(dotenv_path="src/.env", override=True) token: str = set_token_env_variable(True) host: str = os.environ.get("HOST") # pyright:ignore - visit: str = os.environ.get("NAMESPACE") # pyright:ignore + visit: str = os.environ.get("VISIT") # pyright:ignore transport = AIOHTTPTransport( url=host, @@ -41,7 +41,7 @@ def submit_workflow(w: Workflow): "proposalNumber": int(visit[2:7]), "number": int(visit[-1]), }, - "manifest": yamlstr, + "manifest": f"""{yamlstr}""", }, ) name = str(result["submitWorkflow"]["name"]) diff --git a/tests/test_submit_to_graphql.py b/tests/test_submit_to_graphql.py index f562fa5..1eb8020 100644 --- a/tests/test_submit_to_graphql.py +++ b/tests/test_submit_to_graphql.py @@ -24,4 +24,4 @@ def test_submit_workflow_to_graphql( mock_load_env.assert_called_once_with(dotenv_path="src/.env", override=True) mock_instance.execute.assert_called_once() mock_workflow.to_yaml.assert_called_once() - mock_os_get.assert_has_calls([call("NAMESPACE"), call("HOST")], any_order=True) + mock_os_get.assert_has_calls([call("VISIT"), call("HOST")], any_order=True)