Adds a testing notebook to run hera files from a notebook

README.md

@@ -31,3 +31,24 @@ Or if it is a commandline tool then you might put some example commands here:
 ```
 python -m python_interface_to_workflows --version
 ```
+
+# Using Copier
+```bash
+mkdir new_directory_path
+cd new_directory_path
+git init 
+git remote add origin {origin ssh}
+git branch -M main
+
+cd ..
+```
+then either:
+```bash
+git clone git@github.com:DiamondLightSource/python-interface-to-workflows.git
+copier copy {this_repo's_path} {new_directory_path}
+```
+or:
+```bash
+copier copy git@github.com:DiamondLightSource/python-copier-template.git new directory path
+```
+rebuild in dev container without cache

scripts/lintyaml.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 cd src/python_interface_to_workflows/templates
-for file in *
+for file in *.yaml;
 do
 argo lint "$file" --offline
 SUCCESSFULLINT=$?

scripts/makecopiercorrect.sh

@@ -7,16 +7,36 @@ cp ../templates/*example*.txt "../../copier_template/src/{{ project_name }}/temp
 for file in "../../copier_template/src/{{ project_name }}/workflow_definitions"/*
 do
     [[ $file == *.jinja ]] && continue
+
     sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file"
     sed -i 's/DiamondLightSource/{{github_org}}/g' "$file"
+
+    sed -i '1i{% raw %}' "$file"
+    echo '{% endraw %}' >> "$file"
+
+    sed -i \
+        -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \
+        -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \
+        "$file"
+
     mv "$file" "$file.jinja"
 done
 
 for file in "../../copier_template/src/{{ project_name }}/templates"/*
 do
     [[ $file == *.jinja ]] && continue
+
     sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file"
     sed -i 's/DiamondLightSource/{{github_org}}/g' "$file"
+
+    sed -i '1i{% raw %}' "$file"
+    echo '{% endraw %}' >> "$file"
+
+    sed -i \
+        -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \
+        -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \
+        "$file"
+
     mv "$file" "$file.jinja"
 done
 git add "../../copier_template/src/{{ project_name }}/workflow_definitions"/*

scripts/runthefiles.sh

@@ -5,4 +5,5 @@ do
 uv run "$file"
 done
 mv *.yaml ../templates/
+mv *.txt ../templates/
 git add -u ../templates/

src/copier_template/pyproject.toml.jinja

@@ -68,7 +68,6 @@ reportMissingImports = false # Ignore missing stubs in imported modules
 reportMissingTypeStubs = "none"
 reportMissingModuleSource="warning"
 reportInvalidTypeForm="warning"
-reportMissingImports="warning"
 reportUndefinedVariable="warning"
 reportAbstractUsage="warning"
 reportArgumentType="warning"

src/copier_template/scripts/lintyaml.sh.jinja

@@ -1,6 +1,6 @@
 #!/bin/bash
 cd src/{{project_name}}/templates
-for file in *
+for file in *.yaml;
 do
 argo lint "$file" --offline
 SUCCESSFULLINT=$?

src/copier_template/scripts/runthefiles.sh.jinja

@@ -1,8 +1,9 @@
 #!/bin/bash
-cd src/{{project_name}}/workflow_definitions
+cd src/python_interface_to_workflows/workflow_definitions
 for file in *
 do
 uv run "$file"
 done
 mv *.yaml ../templates/
+mv *.txt ../templates/
 git add -u ../templates/

src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py.jinja

@@ -9,12 +9,12 @@ from {{project_name}}.auth.open_auth_url import open_auth_url
 
 def set_token_env_variable() -> str:
     keycloak_openid = KeycloakOpenID(
-        client_id="workflows-dashboard",
-        server_url="https://identity.diamond.ac.uk/",
-        realm_name="dls",
-        client_secret_key="",
-        pool_maxsize=1,
-    )
+                client_id="workflows-cli",
+                server_url="https://identity.diamond.ac.uk/",
+                realm_name="dls",
+                client_secret_key="",
+                pool_maxsize=1,
+            )
     port = 8000
     code_verifier = generate_code_verifier()
     code_challenge, code_challenge_method = generate_code_challenge(code_verifier)
@@ -25,14 +25,26 @@ def set_token_env_variable() -> str:
         code_challenge=code_challenge,
         code_challenge_method=code_challenge_method,
     )
-    open_auth_url(auth_url)
-    token: dict[str, str] = (  # pyright: ignore[reportUnknownVariableType]
-        keycloak_openid.token(  # pyright: ignore[reportUnknownMemberType]
-            grant_type="authorization_code",
-            code=os.environ["AUTH"],
-            redirect_uri=f"http://localhost:{port}/",
-            code_verifier=code_verifier,
-        )
+    match open_auth_url(auth_url, port):
+        case True:
+            token: dict[str, str] = (  # pyright: ignore[reportUnknownVariableType]
+                keycloak_openid.token(  # pyright: ignore[reportUnknownMemberType]
+                    grant_type="authorization_code",
+                    code=os.environ.get("AUTH"),  # pyright: ignore
+                    redirect_uri=f"http://localhost:{port}/",
+                    code_verifier=code_verifier,
+                )
+            )
+        case False:
+            token: dict[str, str] = keycloak_openid.refresh_token(  # pyright: ignore
+                str(os.environ.get("REFRESHTOKEN"))  # pyright: ignore
+            )
+    token_info: dict[str, int | str | dict[str, list[str]]] = (  # pyright: ignore
+        keycloak_openid.decode_token(token["access_token"])  # pyright: ignore
     )
+    expire_time = int(token_info["exp"]) + 1500  # pyright: ignore
+    dotenv.set_key("src/.env", "EXPIRY", str(expire_time).strip("'"))
     dotenv.set_key("src/.env", "TOKEN", token["access_token"].strip("'"))
+    dotenv.set_key("src/.env", "REFRESHTOKEN", token["refresh_token"].strip("'"))
+    dotenv.load_dotenv(dotenv_path="src/.env", override=True)
     return token["access_token"]  # pyright: ignore[reportUnknownArgumentType]

src/copier_template/src/{{ project_name }}/auth/open_auth_url.py

@@ -1,10 +1,13 @@
 import os
 import socket
+import time
 import urllib.parse
 import webbrowser
 from http.server import BaseHTTPRequestHandler, HTTPServer
 from typing import cast
 
+import dotenv
+
 
 class _ReusingHTTPServer(HTTPServer):
     allow_reuse_address = True
@@ -26,16 +29,23 @@ def do_GET(self):
             self.wfile.write(b"Missing authorization code.")
 
 
-def open_auth_url(auth_url: str) -> None:
-    httpd = _ReusingHTTPServer(("localhost", 8000), CallbackHandler)
-    webbrowser.open(auth_url)
-    try:
-        httpd.handle_request()
-        os.environ["AUTH"] = httpd.auth_code
-    except OSError:
-        os.environ["AUTH"] = ""
-        print("ERROR: Port in use. Please restart your terminal.")
-        exit(1)
-    finally:
-        httpd.socket.shutdown(socket.SHUT_RDWR)
-        httpd.server_close()
+def open_auth_url(auth_url: str, port: int) -> bool:
+    dotenv.load_dotenv(dotenv_path="src/.env", override=True)
+    expiry_str: str = os.environ.get("EXPIRY").strip("'")  # pyright: ignore
+    if (expiry_str == "" or int(expiry_str)) <= float(time.time()):
+        httpd = _ReusingHTTPServer(("localhost", port), CallbackHandler)
+        webbrowser.open(auth_url)
+        try:
+            httpd.handle_request()
+            os.environ["AUTH"] = httpd.auth_code
+            dotenv.set_key("src/.env", "AUTH", httpd.auth_code)
+        except OSError:
+            os.environ["AUTH"] = ""
+            print("ERROR: Port in use. Please restart your terminal.")
+            exit(1)
+        finally:
+            httpd.socket.shutdown(socket.SHUT_RDWR)
+            httpd.server_close()
+        return True
+    else:
+        return False

src/copier_template/src/{{ project_name }}/templates/example.txt.jinja

@@ -1,3 +1,4 @@
+{% raw %}
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
@@ -7,7 +8,7 @@ metadata:
       Replicates the functionality of
       example.yaml
     workflows.argoproj.io/title: example remade via hera
-    workflows.diamond.ac.uk/repository: https://github.com/{{github_org}}/{{repo_name}}
+    workflows.diamond.ac.uk/repository: https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %}
   labels:
     workflows.diamond.ac.uk/science-group-examples: 'true'
 spec:
@@ -212,3 +213,4 @@ spec:
       resources:
         requests:
           storage: 1Gi
+{% endraw %}

src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja

@@ -1,3 +1,4 @@
+{% raw %}
 from hera.workflows import (
     DAG,
     Artifact,
@@ -153,7 +154,7 @@ with Workflow(
         "workflows.argoproj.io/title": "example remade via hera",
         "workflows.argoproj.io/description": """Replicates the functionality of
 example.yaml""",
-        "workflows.diamond.ac.uk/repository": "https://github.com/{{github_org}}/{{repo_name}}",
+        "workflows.diamond.ac.uk/repository": "https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %}",
     },
     volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"),
 ) as w:
@@ -178,5 +179,6 @@ example.yaml""",
         [install, params] >> makeimages >> makehdf5  # pyright: ignore
 
 
-with open("example.yaml", "w") as div:
+with open("example.txt", "w") as div:
     div.write(w.to_yaml())  # pyright: ignore[reportUnknownMemberType]
+{% endraw %}

src/copier_template/tests/test_keycloak_checker.py.jinja

@@ -1,35 +1,75 @@
 import os
-from unittest.mock import MagicMock, patch
+from unittest.mock import MagicMock, call, patch
+
+from pytest import mark
 
 from {{project_name}}.auth.keycloak_checker import set_token_env_variable
 
 
+@mark.parametrize(
+    "staging,port,return_present",
+    [
+        (True, 5173, True),
+        (False, 8000, False),
+        (True, 5173, False),
+        (False, 8000, True),
+    ],
+)
+@patch("{{project_name}}.auth.keycloak_checker.dotenv.load_dotenv")
+@patch("{{project_name}}.auth.keycloak_checker.dotenv.set_key")
 @patch("{{project_name}}.auth.keycloak_checker.KeycloakOpenID")
 @patch("{{project_name}}.auth.keycloak_checker.generate_code_verifier")
 @patch("{{project_name}}.auth.keycloak_checker.generate_code_challenge")
 @patch("{{project_name}}.auth.keycloak_checker.open_auth_url")
-def test_return_key(
+def test_set_token_env_variable(
     mock_open_auth_url: MagicMock,
     mock_gen_code_challenge: MagicMock,
     mock_gen_code_verifier: MagicMock,
     mock_gen_keycloak_id: MagicMock,
+    mock_set_key: MagicMock,
+    mock_load_env: MagicMock,
+    staging: bool,
+    port: int,
+    return_present: bool,
 ):
     mock_gen_code_verifier.return_value = "verifier"
     mock_gen_code_challenge.return_value = ("challenge", "S256")
     os.environ["AUTH"] = "auth_url_code"
+    os.environ["REFRESHTOKEN"] = "refresh"
+
     keycloak = MagicMock()
     mock_gen_keycloak_id.return_value = keycloak
-
+    mock_open_auth_url.return_value = return_present
     keycloak.auth_url.return_value = "https://mock.site"
-    keycloak.token.return_value = {
+    token = {
         "access_token": "fake_token",
         "refresh_token": "fake_refresh",
     }
-    assert set_token_env_variable() == "fake_token"
-    mock_open_auth_url.assert_called_once_with("https://mock.site", 8000)
-    keycloak.token.assert_called_once_with(
-        grant_type="authorization_code",
-        code="auth_url_code",
-        redirect_uri="http://localhost:8000/",
-        code_verifier="verifier",
+    keycloak.token.return_value = token
+    keycloak.refresh_token.return_value = token
+    keycloak.decode_token.return_value = {"exp": 123456789}
+    assert set_token_env_variable(staging) == "fake_token"
+
+    mock_open_auth_url.assert_called_once_with("https://mock.site", port)
+    if return_present:
+        keycloak.token.assert_called_once_with(
+            grant_type="authorization_code",
+            code="auth_url_code",
+            redirect_uri=f"http://localhost:{port}/",
+            code_verifier="verifier",
+        )
+        keycloak.refresh_token.assert_not_called()
+    else:
+        keycloak.refresh_token.assert_called_once_with("refresh")
+        keycloak.token.assert_not_called()
+    mock_set_key.assert_has_calls(
+        [
+            call("src/.env", "EXPIRY", str(123456789 + 1500)),
+            call("src/.env", "TOKEN", "fake_token"),
+            call("src/.env", "REFRESHTOKEN", "fake_refresh"),
+        ]
+    )
+    mock_load_env.assert_called_once_with(
+        dotenv_path="src/.env",
+        override=True,
     )