Checks for refresh token properly

.devcontainer/devcontainer.json

@@ -53,6 +53,9 @@
         "--env-file",
         "src/.env"
     ],
+    "forwardPorts": [
+        5173
+    ],
     "portsAttributes": {
         "5173": {
             "onAutoForward": "ignore"

README.md

@@ -31,3 +31,24 @@ Or if it is a commandline tool then you might put some example commands here:
 ```
 python -m python_interface_to_workflows --version
 ```
+
+# Using Copier
+```bash
+mkdir new_directory_path
+cd new_directory_path
+git init 
+git remote add origin {origin ssh}
+git branch -M main
+
+cd ..
+```
+then either:
+```bash
+git clone git@github.com:DiamondLightSource/python-interface-to-workflows.git
+copier copy {this_repo's_path} {new_directory_path}
+```
+or:
+```bash
+copier copy git@github.com:DiamondLightSource/python-copier-template.git new directory path
+```
+rebuild in dev container without cache

scripts/makecopiercorrect.sh

@@ -7,16 +7,36 @@ cp ../templates/*example*.yaml "../../copier_template/src/{{ project_name }}/tem
 for file in "../../copier_template/src/{{ project_name }}/workflow_definitions"/*
 do
     [[ $file == *.jinja ]] && continue
+
     sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file"
     sed -i 's/DiamondLightSource/{{github_org}}/g' "$file"
+
+    sed -i '1i{% raw %}' "$file"
+    echo '{% endraw %}' >> "$file"
+
+    sed -i \
+        -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \
+        -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \
+        "$file"
+
     mv "$file" "$file.jinja"
 done
 
 for file in "../../copier_template/src/{{ project_name }}/templates"/*
 do
     [[ $file == *.jinja ]] && continue
+
     sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file"
     sed -i 's/DiamondLightSource/{{github_org}}/g' "$file"
+
+    sed -i '1i{% raw %}' "$file"
+    echo '{% endraw %}' >> "$file"
+
+    sed -i \
+        -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \
+        -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \
+        "$file"
+
     mv "$file" "$file.jinja"
 done
 git add "../../copier_template/src/{{ project_name }}/workflow_definitions"/*

src/copier_template/.devcontainer/devcontainer.json

@@ -56,6 +56,11 @@
     "forwardPorts": [
         5173
     ],
+    "portsAttributes": {
+        "5173": {
+            "onAutoForward": "ignore"
+        }
+    },
     "mounts": [
         // Mount in the user terminal config folder so it can be edited
         {

src/copier_template/pyproject.toml.jinja

@@ -68,7 +68,6 @@ reportMissingImports = false # Ignore missing stubs in imported modules
 reportMissingTypeStubs = "none"
 reportMissingModuleSource="warning"
 reportInvalidTypeForm="warning"
-reportMissingImports="warning"
 reportUndefinedVariable="warning"
 reportAbstractUsage="warning"
 reportArgumentType="warning"

src/copier_template/src/README.md.jinja

@@ -0,0 +1,26 @@
+# Setting up a development environment
+1. Build the dev container
+2. run "uv lock" to generate the uv.lock file
+3. Create .env in this folder (with the path src/.env) containing the following variables:
+
+HOST=https://argo-workflows.workflows.diamond.ac.uk/ (to submit to the production repo)
+IMAGE= (usually python 3.10)
+TOKEN=
+NAMESPACE= (the cluster you wish to run the templates on)
+AUTH=
+
+
+# Submitting a workflow to Argo
+1. At the start of your workflow definition file, add:
+
+```python
+from {{project_name}}.submit_to_argo import submit_workflow_to_argo
+```
+
+2. At the end of a workflow definition file, once your workflow (w) is created, add:
+```python
+submit_workflow_to_argo(w)
+```
+
+NOTE: Be sure to remove this line upon commiting changes, as all workflow definition files are 
+by default, ran on pre-commit, to ensure that any yaml files they create are up to date.

src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py.jinja

@@ -7,15 +7,26 @@ from keycloak.pkce_utils import generate_code_challenge, generate_code_verifier
 from {{project_name}}.auth.open_auth_url import open_auth_url
 
 
-def set_token_env_variable() -> str:
-    keycloak_openid = KeycloakOpenID(
-        client_id="workflows-dashboard",
-        server_url="https://identity.diamond.ac.uk/",
-        realm_name="dls",
-        client_secret_key="",
-        pool_maxsize=1,
-    )
-    port = 8000
+def set_token_env_variable(staging: bool) -> str:
+    match staging:
+        case True:
+            keycloak_openid = KeycloakOpenID(
+                server_url="https://identity-test.diamond.ac.uk/",
+                client_id="workflows-ui-dev",
+                realm_name="dls",
+                client_secret_key="",
+                pool_maxsize=1,
+            )
+            port = 5173
+        case False:
+            keycloak_openid = KeycloakOpenID(
+                client_id="workflows-dashboard",
+                server_url="https://identity.diamond.ac.uk/",
+                realm_name="dls",
+                client_secret_key="",
+                pool_maxsize=1,
+            )
+            port = 8000
     code_verifier = generate_code_verifier()
     code_challenge, code_challenge_method = generate_code_challenge(code_verifier)
     auth_url = keycloak_openid.auth_url(
@@ -25,14 +36,26 @@ def set_token_env_variable() -> str:
         code_challenge=code_challenge,
         code_challenge_method=code_challenge_method,
     )
-    open_auth_url(auth_url)
-    token: dict[str, str] = (  # pyright: ignore[reportUnknownVariableType]
-        keycloak_openid.token(  # pyright: ignore[reportUnknownMemberType]
-            grant_type="authorization_code",
-            code=os.environ["AUTH"],
-            redirect_uri=f"http://localhost:{port}/",
-            code_verifier=code_verifier,
-        )
+    match open_auth_url(auth_url, port):
+        case True:
+            token: dict[str, str] = (  # pyright: ignore[reportUnknownVariableType]
+                keycloak_openid.token(  # pyright: ignore[reportUnknownMemberType]
+                    grant_type="authorization_code",
+                    code=os.environ.get("AUTH"),  # pyright: ignore
+                    redirect_uri=f"http://localhost:{port}/",
+                    code_verifier=code_verifier,
+                )
+            )
+        case False:
+            token: dict[str, str] = keycloak_openid.refresh_token(  # pyright: ignore
+                str(os.environ.get("REFRESHTOKEN"))  # pyright: ignore
+            )
+    token_info: dict[str, int | str | dict[str, list[str]]] = (  # pyright: ignore
+        keycloak_openid.decode_token(token["access_token"])  # pyright: ignore
     )
+    expire_time = int(token_info["exp"]) + 1500  # pyright: ignore
+    dotenv.set_key("src/.env", "EXPIRY", str(expire_time).strip("'"))
     dotenv.set_key("src/.env", "TOKEN", token["access_token"].strip("'"))
+    dotenv.set_key("src/.env", "REFRESHTOKEN", token["refresh_token"].strip("'"))
+    dotenv.load_dotenv(dotenv_path="src/.env", override=True)
     return token["access_token"]  # pyright: ignore[reportUnknownArgumentType]

src/copier_template/src/{{ project_name }}/auth/open_auth_url.py

@@ -1,10 +1,13 @@
 import os
 import socket
+import time
 import urllib.parse
 import webbrowser
 from http.server import BaseHTTPRequestHandler, HTTPServer
 from typing import cast
 
+import dotenv
+
 
 class _ReusingHTTPServer(HTTPServer):
     allow_reuse_address = True
@@ -26,16 +29,23 @@ def do_GET(self):
             self.wfile.write(b"Missing authorization code.")
 
 
-def open_auth_url(auth_url: str) -> None:
-    httpd = _ReusingHTTPServer(("localhost", 8000), CallbackHandler)
-    webbrowser.open(auth_url)
-    try:
-        httpd.handle_request()
-        os.environ["AUTH"] = httpd.auth_code
-    except OSError:
-        os.environ["AUTH"] = ""
-        print("ERROR: Port in use. Please restart your terminal.")
-        exit(1)
-    finally:
-        httpd.socket.shutdown(socket.SHUT_RDWR)
-        httpd.server_close()
+def open_auth_url(auth_url: str, port: int) -> bool:
+    dotenv.load_dotenv(dotenv_path="src/.env", override=True)
+    expiry_str: str = os.environ.get("EXPIRY")  # pyright: ignore
+    if (expiry_str == "" or int(expiry_str)) <= float(time.time()):
+        httpd = _ReusingHTTPServer(("localhost", port), CallbackHandler)
+        webbrowser.open(auth_url)
+        try:
+            httpd.handle_request()
+            os.environ["AUTH"] = httpd.auth_code
+            dotenv.set_key("src/.env", "AUTH", httpd.auth_code)
+        except OSError:
+            os.environ["AUTH"] = ""
+            print("ERROR: Port in use. Please restart your terminal.")
+            exit(1)
+        finally:
+            httpd.socket.shutdown(socket.SHUT_RDWR)
+            httpd.server_close()
+        return True
+    else:
+        return False

src/copier_template/src/{{ project_name }}/submit_to_argo.py.jinja

@@ -0,0 +1,24 @@
+import os
+from typing import cast
+
+import dotenv
+from hera.shared import global_config
+from hera.workflows import Workflow, WorkflowsService
+from hera.workflows import models as m
+
+from {{project_name}}.auth.keycloak_checker import set_token_env_variable
+
+
+def submit_workflow_to_argo(w: Workflow):
+    set_token_env_variable(staging=True)
+    dotenv.load_dotenv(dotenv_path="src/.env", override=True)
+    w.namespace = os.environ.get("NAMESPACE")
+    w.workflows_service = WorkflowsService(
+        host=str(os.environ.get("HOST")).strip("'"),
+        token=str(os.environ.get("TOKEN")).strip("'"),
+    )
+    global_config.image = str(os.environ.get("IMAGE"))
+    submitted_w = cast(m.Workflow, w.create())
+    name = submitted_w.metadata.name
+    namespace = submitted_w.metadata.namespace
+    print(f"submitted {name} to {namespace}")

src/copier_template/src/{{ project_name }}/templates/example.yaml.jinja → src/copier_template/src/{{ project_name }}/templates/_example.yaml.jinja

@@ -1,14 +1,14 @@
+{% raw %}
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
   generateName: hera-example-
-  namespace: None
   annotations:
     workflows.argoproj.io/description: |-
       Replicates the functionality of
-          example.yaml
+      example.yaml
     workflows.argoproj.io/title: example remade via hera
-    workflows.diamond.ac.uk/repository: https://github.com/{{github_org}}/{{repo_name}}
+    workflows.diamond.ac.uk/repository: https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %}
   labels:
     workflows.diamond.ac.uk/science-group-examples: 'true'
 spec:
@@ -56,7 +56,7 @@ spec:
             value: '{{tasks.create-image.outputs.parameters.out-paths}}'
   - name: install-dependencies
     script:
-      image: None
+      image: python:3.10
       source: |-
         import os
         import sys
@@ -84,7 +84,7 @@ spec:
         valueFrom:
           path: /tmp/parameters.json
     script:
-      image: None
+      image: python:3.10
       source: |-
         import os
         import sys
@@ -129,7 +129,7 @@ spec:
         valueFrom:
           path: /tmp/{{inputs.parameters.extension}}-path.json
     script:
-      image: None
+      image: python:3.10
       source: |-
         import os
         import sys
@@ -178,7 +178,7 @@ spec:
         archive:
           none: {}
     script:
-      image: None
+      image: python:3.10
       source: |-
         import os
         import sys
@@ -213,3 +213,4 @@ spec:
       resources:
         requests:
           storage: 1Gi
+{% endraw %}

src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja

@@ -1,5 +1,4 @@
-import os
-
+{% raw %}
 from hera.workflows import (
     DAG,
     Artifact,
@@ -13,7 +12,6 @@ from hera.workflows.archive import NoneArchiveStrategy
 
 
 @script(
-    image=str(os.environ.get("IMAGE")),
     command=["python"],
     volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")],
 )
@@ -30,7 +28,6 @@ def install_dependencies():
 
 @script(
     command=["python"],
-    image=str(os.environ.get("IMAGE")),
     outputs=Parameter(
         name="out-parameters", value_from=m.ValueFrom(path="/tmp/parameters.json")
     ),
@@ -71,7 +68,6 @@ def generate_parameters(
 
 @script(
     command=["/tmp/venv/bin/python"],
-    image=str(os.environ.get("IMAGE")),
     volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")],
     outputs=[
         Parameter(
@@ -122,7 +118,6 @@ def create_image(
 
 @script(
     command=["/tmp/venv/bin/python"],
-    image=str(os.environ.get("IMAGE")),
     volume_mounts=[m.VolumeMount(name="tmpdir", mount_path="/tmp")],
     outputs=Artifact(
         name="hdf5output",
@@ -152,15 +147,14 @@ def to_hdf5(paths: str):
 with Workflow(
     generate_name="hera-example-",  # when running on graphql this should be name
     entrypoint="workflowentry",
-    namespace=str(os.environ.get("NAMESPACE")),
     api_version="argoproj.io/v1alpha1",
     kind="Workflow",  # ClusterWorkflowTemplate", when on graphql
     labels={"workflows.diamond.ac.uk/science-group-examples": "true"},
     annotations={
         "workflows.argoproj.io/title": "example remade via hera",
         "workflows.argoproj.io/description": """Replicates the functionality of
-    example.yaml""",
-        "workflows.diamond.ac.uk/repository": "https://github.com/{{github_org}}/{{repo_name}}",
+example.yaml""",
+        "workflows.diamond.ac.uk/repository": "https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %}",
     },
     volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"),
 ) as w:
@@ -185,5 +179,6 @@ with Workflow(
         [install, params] >> makeimages >> makehdf5  # pyright: ignore
 
 
-with open("example.yaml", "w") as div:
+with open("_example.yaml", "w") as div:
     div.write(w.to_yaml())  # pyright: ignore[reportUnknownMemberType]
+{% endraw %}