Submit to argo

.devcontainer/devcontainer.json

@@ -53,6 +53,9 @@
         "--env-file",
         "src/.env"
     ],
+    "forwardPorts": [
+        5173
+    ],
     "portsAttributes": {
         "5173": {
             "onAutoForward": "ignore"

scripts/makecopiercorrect.sh

@@ -2,7 +2,7 @@
 cd "src/python_interface_to_workflows/workflow_definitions"
 
 cp *example*.py "../../copier_template/src/{{ project_name }}/workflow_definitions"
-cp ../templates/*example*.yaml "../../copier_template/src/{{ project_name }}/templates"
+cp ../templates/*example*.txt "../../copier_template/src/{{ project_name }}/templates"
 
 for file in "../../copier_template/src/{{ project_name }}/workflow_definitions"/*
 do

src/copier_template/src/README.md.jinja

@@ -0,0 +1,26 @@
+# Setting up a development environment
+1. Build the dev container
+2. run "uv lock" to generate the uv.lock file
+3. Create .env in this folder (with the path src/.env) containing the following variables:
+
+HOST=https://argo-workflows.workflows.diamond.ac.uk/ (to submit to the production repo)
+IMAGE= (usually python 3.10)
+TOKEN=
+NAMESPACE= (the cluster you wish to run the templates on)
+AUTH=
+
+
+# Submitting a workflow to Argo
+1. At the start of your workflow definition file, add:
+
+```python
+from {{project_name}}.submit_to_argo import submit_workflow_to_argo
+```
+
+2. At the end of a workflow definition file, once your workflow (w) is created, add:
+```python
+submit_workflow_to_argo(w)
+```
+
+NOTE: Be sure to remove this line upon commiting changes, as all workflow definition files are 
+by default, ran on pre-commit, to ensure that any yaml files they create are up to date.

src/copier_template/src/{{ project_name }}/submit_to_argo.py.jinja

@@ -0,0 +1,23 @@
+import os
+from typing import cast
+
+import dotenv
+from hera.shared import global_config
+from hera.workflows import Workflow, WorkflowsService
+from hera.workflows import models as m
+
+from {{project_name}}.auth.keycloak_checker import set_token_env_variable
+
+
+def submit_workflow_to_argo(w: Workflow):
+    set_token_env_variable(staging=True)
+    dotenv.load_dotenv(dotenv_path="src/.env", override=True)
+    w.namespace = os.environ.get("NAMESPACE")
+    w.workflows_service = WorkflowsService(
+        host=str(os.environ.get("HOST")).strip("'"),
+        token=str(os.environ.get("TOKEN")).strip("'"),
+    )
+    submitted_w = cast(m.Workflow, w.create())
+    name = submitted_w.metadata.name
+    namespace = submitted_w.metadata.namespace
+    print(f"submitted {name} to {namespace}")

src/copier_template/src/{{ project_name }}/templates/example.yaml.jinja → src/copier_template/src/{{ project_name }}/templates/example.txt.jinja

@@ -2,11 +2,10 @@ apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
   generateName: hera-example-
-  namespace: None
   annotations:
     workflows.argoproj.io/description: |-
       Replicates the functionality of
-          example.yaml
+      example.yaml
     workflows.argoproj.io/title: example remade via hera
     workflows.diamond.ac.uk/repository: https://github.com/{{github_org}}/{{repo_name}}
   labels:
@@ -56,7 +55,7 @@ spec:
             value: '{{tasks.create-image.outputs.parameters.out-paths}}'
   - name: install-dependencies
     script:
-      image: None
+      image: python:3.10
       source: |-
         import os
         import sys
@@ -84,7 +83,7 @@ spec:
         valueFrom:
           path: /tmp/parameters.json
     script:
-      image: None
+      image: python:3.10
       source: |-
         import os
         import sys
@@ -129,7 +128,7 @@ spec:
         valueFrom:
           path: /tmp/{{inputs.parameters.extension}}-path.json
     script:
-      image: None
+      image: python:3.10
       source: |-
         import os
         import sys
@@ -178,7 +177,7 @@ spec:
         archive:
           none: {}
     script:
-      image: None
+      image: python:3.10
       source: |-
         import os
         import sys

src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja

@@ -1,5 +1,3 @@
-import os
-
 from hera.workflows import (
     DAG,
     Artifact,
@@ -13,7 +11,6 @@ from hera.workflows.archive import NoneArchiveStrategy
 
 
 @script(
-    image=str(os.environ.get("IMAGE")),
     command=["python"],
     volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")],
 )
@@ -30,7 +27,6 @@ def install_dependencies():
 
 @script(
     command=["python"],
-    image=str(os.environ.get("IMAGE")),
     outputs=Parameter(
         name="out-parameters", value_from=m.ValueFrom(path="/tmp/parameters.json")
     ),
@@ -71,7 +67,6 @@ def generate_parameters(
 
 @script(
     command=["/tmp/venv/bin/python"],
-    image=str(os.environ.get("IMAGE")),
     volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")],
     outputs=[
         Parameter(
@@ -122,7 +117,6 @@ def create_image(
 
 @script(
     command=["/tmp/venv/bin/python"],
-    image=str(os.environ.get("IMAGE")),
     volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")],
     outputs=Artifact(
         name="hdf5output",
@@ -152,14 +146,13 @@ def to_hdf5(paths: str):
 with Workflow(
     generate_name="hera-example-",  # when running on graphql this should be name
     entrypoint="workflowentry",
-    namespace=str(os.environ.get("NAMESPACE")),
     api_version="argoproj.io/v1alpha1",
     kind="Workflow",  # ClusterWorkflowTemplate", when on graphql
     labels={"workflows.diamond.ac.uk/science-group-examples": "true"},
     annotations={
         "workflows.argoproj.io/title": "example remade via hera",
         "workflows.argoproj.io/description": """Replicates the functionality of
-    example.yaml""",
+example.yaml""",
         "workflows.diamond.ac.uk/repository": "https://github.com/{{github_org}}/{{repo_name}}",
     },
     volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"),

src/copier_template/tests/test_keycloak_checker.py.jinja

@@ -1,8 +1,6 @@
 import os
 from unittest.mock import MagicMock, patch
 
-from pytest import mark
-
 from {{project_name}}.auth.keycloak_checker import set_token_env_variable
 
 
@@ -28,7 +26,7 @@ def test_return_key(
         "refresh_token": "fake_refresh",
     }
     assert set_token_env_variable() == "fake_token"
-    mock_open_auth_url.assert_called_once_with("https://mock.site", port)
+    mock_open_auth_url.assert_called_once_with("https://mock.site", 8000)
     keycloak.token.assert_called_once_with(
         grant_type="authorization_code",
         code="auth_url_code",

src/copier_template/tests/test_open_auth_url.py.jinja

@@ -15,7 +15,7 @@ def test_open_auth_url_normal_function(
 ):
     server = mock_http_server.return_value
     server.auth_code = "this_is_your_code"
-    open_auth_url("url", 5173)
+    open_auth_url("url")
     server.handle_request.assert_called_once()
     mock_open_browser.assert_called_once_with("url")
     server.socket.shutdown.assert_called_once()
@@ -34,7 +34,7 @@ def test_open_auth_url_raises_error(
     server = mock_http_server.return_value
     server.auth_code = "this_is_your_code"
     server.handle_request.side_effect = OSError
-    open_auth_url("url", 5173)
+    open_auth_url("url")
     mock_open_browser.assert_called_once_with("url")
     assert os.environ["AUTH"] == ""
     mock_exit.assert_called_once_with(1)

src/copier_template/tests/test_submit_workflow_to_argo.jinja

@@ -0,0 +1,20 @@
+from unittest.mock import MagicMock, call, patch
+
+from {{project_name}}.submit_to_argo import submit_workflow_to_argo
+
+
+@patch("{{project_name}}.submit_to_argo.Workflow")
+@patch("{{project_name}}.submit_to_argo.os.environ.get")
+@patch("{{project_name}}.submit_to_argo.dotenv.load_dotenv")
+@patch("{{project_name}}.submit_to_argo.set_token_env_variable")
+def test_submit_workflow_to_argo(
+    mock_key: MagicMock,
+    mock_load_env: MagicMock,
+    mock_get_env_var: MagicMock,
+    mock_workflow: MagicMock,
+):
+    mock_workflow = MagicMock()
+    submit_workflow_to_argo(mock_workflow)
+    mock_key.assert_called_once_with(staging=True)
+    mock_load_env.assert_called_once_with(dotenv_path="src/.env", override=True)
+    mock_get_env_var.assert_has_calls([call("HOST"), call("TOKEN")], any_order=True)

src/python_interface_to_workflows/auth/open_auth_url.py

@@ -5,6 +5,8 @@
 from http.server import BaseHTTPRequestHandler, HTTPServer
 from typing import cast
 
+import dotenv
+
 
 class _ReusingHTTPServer(HTTPServer):
     allow_reuse_address = True
@@ -32,6 +34,7 @@ def open_auth_url(auth_url: str, port: int) -> None:
     try:
         httpd.handle_request()
         os.environ["AUTH"] = httpd.auth_code
+        dotenv.set_key("src/.env", "AUTH", httpd.auth_code)
     except OSError:
         os.environ["AUTH"] = ""
         print("ERROR: Port in use. Please restart your terminal.")

src/python_interface_to_workflows/submit_to_argo.py

@@ -0,0 +1,24 @@
+import os
+from typing import cast
+
+import dotenv
+from hera.shared import global_config
+from hera.workflows import Workflow, WorkflowsService
+from hera.workflows import models as m
+
+from python_interface_to_workflows.auth.keycloak_checker import set_token_env_variable
+
+
+def submit_workflow_to_argo(w: Workflow):
+    set_token_env_variable(staging=True)
+    dotenv.load_dotenv(dotenv_path="src/.env", override=True)
+    w.namespace = os.environ.get("NAMESPACE")
+    w.workflows_service = WorkflowsService(
+        host=str(os.environ.get("HOST")).strip("'"),
+        token=str(os.environ.get("TOKEN")).strip("'"),
+    )
+    global_config.image = str(os.environ.get("IMAGE"))
+    submitted_w = cast(m.Workflow, w.create())
+    name = submitted_w.metadata.name
+    namespace = submitted_w.metadata.namespace
+    print(f"submitted {name} to {namespace}")

src/python_interface_to_workflows/templates/divisionyaml.yaml

@@ -5,7 +5,7 @@ metadata:
   annotations:
     workflows.argoproj.io/description: |-
       Takes a numerical input and returns
-              the remainder, output float, and output string to a json file
+          the remainder, output float, and output string to a json file
     workflows.argoproj.io/title: Division via hera test
     workflows.diamond.ac.uk/repository: https://github.com/DiamondLightSource/python-interface-to-workflows
   labels:

src/python_interface_to_workflows/templates/example.yaml

@@ -2,11 +2,10 @@ apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
   generateName: hera-example-
-  namespace: None
   annotations:
     workflows.argoproj.io/description: |-
       Replicates the functionality of
-          example.yaml
+      example.yaml
     workflows.argoproj.io/title: example remade via hera
     workflows.diamond.ac.uk/repository: https://github.com/DiamondLightSource/python-interface-to-workflows
   labels:
@@ -56,7 +55,7 @@ spec:
             value: '{{tasks.create-image.outputs.parameters.out-paths}}'
   - name: install-dependencies
     script:
-      image: None
+      image: python:3.10
       source: |-
         import os
         import sys
@@ -84,7 +83,7 @@ spec:
         valueFrom:
           path: /tmp/parameters.json
     script:
-      image: None
+      image: python:3.10
       source: |-
         import os
         import sys
@@ -129,7 +128,7 @@ spec:
         valueFrom:
           path: /tmp/{{inputs.parameters.extension}}-path.json
     script:
-      image: None
+      image: python:3.10
       source: |-
         import os
         import sys
@@ -178,7 +177,7 @@ spec:
         archive:
           none: {}
     script:
-      image: None
+      image: python:3.10
       source: |-
         import os
         import sys

src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py

@@ -1,5 +1,4 @@
 import json
-import os
 
 from hera.workflows import (
     Artifact,
@@ -31,14 +30,13 @@ def do_division(a: int, b: int):
 with Workflow(
     generate_name="hera-division-",  # when running on graphql this should be name
     entrypoint="divide",
-    namespace=os.environ.get("NAMESPACE"),
     api_version="argoproj.io/v1alpha1",
     kind="Workflow",  # ClusterWorkflowTemplate", when on graphql
     labels={"workflows.diamond.ac.uk/science-group-examples": "true"},
     annotations={
         "workflows.argoproj.io/title": "Division via hera test",
         "workflows.argoproj.io/description": """Takes a numerical input and returns
-        the remainder, output float, and output string to a json file""",
+    the remainder, output float, and output string to a json file""",
         "workflows.diamond.ac.uk/repository": "https://github.com/DiamondLightSource/python-interface-to-workflows",
     },
     volumes=EmptyDirVolume(name="output-dir", mount_path="/output-dir"),