From 5af62b30d31c08387638f77b8a38fcee46d10c99 Mon Sep 17 00:00:00 2001 From: Vornit Date: Tue, 3 Mar 2026 15:42:35 +0200 Subject: [PATCH 01/13] Add LUMI backend support --- src/q8s/bakefile.py | 1 + src/q8s/constants.py | 1 + src/q8s/enums.py | 1 + src/q8s/execution.py | 2 + src/q8s/plugins/hpc_job.py | 105 +++++++++++++++++++++++++++++++++++++ src/q8s/project.py | 1 + 6 files changed, 111 insertions(+) create mode 100644 src/q8s/plugins/hpc_job.py diff --git a/src/q8s/bakefile.py b/src/q8s/bakefile.py index 828d143..dd0793c 100644 --- a/src/q8s/bakefile.py +++ b/src/q8s/bakefile.py @@ -8,6 +8,7 @@ class BakeTargetName(str, Enum): cpu = "cpu" gpu = "gpu" qpu = "qpu" + hpc = "hpc" class BuildPlatform(str, Enum): diff --git a/src/q8s/constants.py b/src/q8s/constants.py index 4dd478c..88680a3 100644 --- a/src/q8s/constants.py +++ b/src/q8s/constants.py @@ -11,6 +11,7 @@ def get_base_images(python_version: str = "3.12") -> dict[str, str]: "cpu": f"python:{python_version}-slim", "gpu": f"ghcr.io/qubernetes-dev/cuda:12.8.1-r2-py{python_version}", "qpu": f"python:{python_version}-slim", + "hpc": f"python:{python_version}-slim" } diff --git a/src/q8s/enums.py b/src/q8s/enums.py index a86a39a..28781c9 100644 --- a/src/q8s/enums.py +++ b/src/q8s/enums.py @@ -5,3 +5,4 @@ class Target(str, Enum): cpu = "cpu" gpu = "gpu" qpu = "qpu" + hpc = "hpc" \ No newline at end of file diff --git a/src/q8s/execution.py b/src/q8s/execution.py index 108c22e..cf24bf2 100644 --- a/src/q8s/execution.py +++ b/src/q8s/execution.py @@ -15,6 +15,7 @@ from q8s.enums import Target from q8s.plugins.cpu_job import CPUJobTemplatePlugin from q8s.plugins.cuda_job import CUDAJobTemplatePlugin +from q8s.plugins.hpc_job import HPCJobTemplatePlugin from q8s.plugins.job_template_spec import JobTemplatePluginSpec from q8s.utils import extract_non_none_value @@ -116,6 +117,7 @@ def __init__(self, kubeconfig: str, logger=None, progress: Progress = None): self.jm.add_hookspecs(JobTemplatePluginSpec) self.jm.register(CPUJobTemplatePlugin()) self.jm.register(CUDAJobTemplatePlugin()) + self.jm.register(HPCJobTemplatePlugin()) task_config = self.__progress.add_task( "[cyan]Loading configuration...", total=1 diff --git a/src/q8s/plugins/hpc_job.py b/src/q8s/plugins/hpc_job.py new file mode 100644 index 0000000..deaf800 --- /dev/null +++ b/src/q8s/plugins/hpc_job.py @@ -0,0 +1,105 @@ + + +from kubernetes import client + +from q8s.constants import WORKSPACE +from q8s.enums import Target +from q8s.plugins.job import JobPlugin +from q8s.plugins.job_template_spec import hookimpl +from q8s.workload import Workload + +class HPCJobTemplatePlugin(JobPlugin): + @hookimpl + def makejob( + self, + name: str, + registry_pat: str | None, + registry_credentials_secret_name: str, + container_image: str, + workload: Workload, + env: list[client.V1EnvVar], + target: Target, + ) -> client.V1PodTemplateSpec: + + if target != Target.hpc: + return None + + volume_name = f"app-volume-{name}" + + env_var = list(env) + if workload.is_src_project: + env_var.append(client.V1EnvVar(name="PYTHONPATH", value=f"{WORKSPACE}/src")) + + self.patch_environment_with_git_info(env_var) + + container = client.V1Container( + name="quantum-routine", + image=container_image, + env=env_var, + command=["python"], + args=( + ["-m", workload.entry_module] + workload.args + if workload.is_src_project + else [f"{WORKSPACE}/{workload.entry_script}"] + workload.args + ), + image_pull_policy="Always", + + volume_mounts=[ + client.V1VolumeMount( + name=volume_name, + mount_path=WORKSPACE, + read_only=True, + ) + ], + ) + + template = client.V1PodTemplateSpec( + metadata=client.V1ObjectMeta( + labels={ + "app": name, + "interlink.cern.ch/provider": "remote-hpc", + } + ), + spec=client.V1PodSpec( + containers=[container], + + # automount_service_account_token=False, + + node_selector={ + "interlink.cern.ch/provider": "remote-hpc" + }, + + tolerations=[ + client.V1Toleration( + key="virtual-node.interlink/no-schedule", + operator="Exists" + ) + ], + + image_pull_secrets=( + [ + client.V1LocalObjectReference( + name=registry_credentials_secret_name + ) + ] + if registry_pat + else [] + ), + restart_policy="Never", + + volumes=[ + client.V1Volume( + name=volume_name, + config_map=client.V1ConfigMapVolumeSource( + name=name, + items=[ + client.V1KeyToPath(key=k, path=v) + for k, v in workload.mappings.items() + ], + ), + ) + ], + ), + ) + + return template diff --git a/src/q8s/project.py b/src/q8s/project.py index 2afa40a..d6d953c 100644 --- a/src/q8s/project.py +++ b/src/q8s/project.py @@ -88,6 +88,7 @@ class Q8STargets: cpu: Optional[Q8STarget] gpu: Optional[Q8STarget] qpu: Optional[Q8STarget] + hpc: Optional[Q8STarget] def keys(self): return [ From 02e105c472fff4d5cb4b1b0c17caabd6e4b7aad7 Mon Sep 17 00:00:00 2001 From: Vornit Date: Tue, 17 Mar 2026 20:26:22 +0200 Subject: [PATCH 02/13] Add support for dynamic Slurm resources from CLI --- src/q8s/cli.py | 11 +++++++++++ src/q8s/plugins/hpc_job.py | 21 ++++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/q8s/cli.py b/src/q8s/cli.py index 164563b..cd0abee 100644 --- a/src/q8s/cli.py +++ b/src/q8s/cli.py @@ -118,6 +118,11 @@ def execute( submit: Annotated[ bool, typer.Option(help="Submit job and exit without waiting for completion") ] = False, + + hpc_cpus: Annotated[int, typer.Option(help="SLURM CPUs per task")] = 1, + hpc_mem: Annotated[str, typer.Option(help="SLURM memory (e.g. 16G)")] = "6G", + hpc_time: Annotated[str, typer.Option(help="SLURM time (HH:MM:SS)")] = "00:04:00", + args: Annotated[list[str], typer.Argument(help="Additional arguments")] = None, ): project = Project() @@ -150,6 +155,12 @@ def execute( workload = Workload.from_entry_script(entry_script=file) workload.set_args(args or []) + workload.extra = { + "hpc_cpus": hpc_cpus, + "hpc_mem": hpc_mem, + "hpc_time": hpc_time, + } + output, stream_name = k8s_context.execute_workload( workload=workload, submit=submit ) diff --git a/src/q8s/plugins/hpc_job.py b/src/q8s/plugins/hpc_job.py index deaf800..74681bf 100644 --- a/src/q8s/plugins/hpc_job.py +++ b/src/q8s/plugins/hpc_job.py @@ -53,12 +53,31 @@ def makejob( ], ) + extra = getattr(workload, "extra", {}) + + cpus = extra.get("hpc_cpus", 1) + mem = extra.get("hpc_mem", "6G") + time = extra.get("hpc_time", "00:04:00") + + flags = [ + "--account=project_462001257", + "--partition=standard", + f"--cpus-per-task={cpus}", + f"--mem={mem}", + f"--time={time}", + ] + + annotations = { + "slurm-job.vk.io/flags": " ".join(flags) + } + template = client.V1PodTemplateSpec( metadata=client.V1ObjectMeta( labels={ "app": name, "interlink.cern.ch/provider": "remote-hpc", - } + }, + annotations=annotations, ), spec=client.V1PodSpec( containers=[container], From 3cbddf9bc494a8a9e13489a32b14cbbc76f048b6 Mon Sep 17 00:00:00 2001 From: EuenSkreuen Date: Tue, 24 Mar 2026 18:00:20 +0200 Subject: [PATCH 03/13] Change hpc target to use HpcConfig as a configuration file --- src/q8s/cli.py | 20 ++--- src/q8s/plugins/hpc_job.py | 156 +++++++++++++++++++++++++++++++------ 2 files changed, 142 insertions(+), 34 deletions(-) diff --git a/src/q8s/cli.py b/src/q8s/cli.py index cd0abee..bc3770e 100644 --- a/src/q8s/cli.py +++ b/src/q8s/cli.py @@ -118,11 +118,13 @@ def execute( submit: Annotated[ bool, typer.Option(help="Submit job and exit without waiting for completion") ] = False, - - hpc_cpus: Annotated[int, typer.Option(help="SLURM CPUs per task")] = 1, - hpc_mem: Annotated[str, typer.Option(help="SLURM memory (e.g. 16G)")] = "6G", - hpc_time: Annotated[str, typer.Option(help="SLURM time (HH:MM:SS)")] = "00:04:00", - + hpc_config: Annotated[ + Path | None, + typer.Option( + "--hpc-config", + help="Path to HPC config file. If omitted, q8s will look for 'HpcConfig' next to 'Q8Sproject'.", + ), + ] = None, args: Annotated[list[str], typer.Argument(help="Additional arguments")] = None, ): project = Project() @@ -155,11 +157,9 @@ def execute( workload = Workload.from_entry_script(entry_script=file) workload.set_args(args or []) - workload.extra = { - "hpc_cpus": hpc_cpus, - "hpc_mem": hpc_mem, - "hpc_time": hpc_time, - } + workload.extra = {} + if hpc_config is not None: + workload.extra["hpc_config"] = hpc_config.as_posix() output, stream_name = k8s_context.execute_workload( workload=workload, submit=submit diff --git a/src/q8s/plugins/hpc_job.py b/src/q8s/plugins/hpc_job.py index 74681bf..0478ed3 100644 --- a/src/q8s/plugins/hpc_job.py +++ b/src/q8s/plugins/hpc_job.py @@ -1,4 +1,7 @@ +from __future__ import annotations +import shlex +from pathlib import Path from kubernetes import client @@ -8,6 +11,130 @@ from q8s.plugins.job_template_spec import hookimpl from q8s.workload import Workload + +def _get_workload_extra(workload: Workload) -> dict: + """ + Gets the "extra" informatino of the workload. + """ + extra = getattr(workload, "extra", {}) + return extra if isinstance(extra, dict) else {} + + +def _find_project_root(start: Path) -> Path: + """ + Tries to find Q8Sproject file from current directory or a parent directory, + in order to locate the root directory of the project. + """ + current = start.resolve() + for candidate in [current, *current.parents]: + if (candidate / "Q8Sproject").exists(): + return candidate + return Path.cwd().resolve() + + +def resolve_hpc_config_path(workload: Workload) -> Path: + """ + Tries to find all necessary slurm settings by trying to + find HpcConfig file in the project root directory, and checking if + the workload already includes a path to it. + """ + extra = _get_workload_extra(workload) + explicit_path = extra.get("hpc_config") + + if explicit_path: + return Path(explicit_path).expanduser().resolve() + + project_root = _find_project_root(Path.cwd()) + return project_root / "HpcConfig" + + +def _parse_config_line(line: str, line_number: int) -> tuple[str, str] | None: + """ + Parses a single line of a HpcConfig file + + Accepts values formatted like these two options: + - --flag value + - --flag=value + + Blank lines and lines starting with '#' are ignored. + """ + stripped = line.strip() + + if not stripped or stripped.startswith("#"): + return None + + parts = shlex.split(stripped) + if not parts: + return None + + first = parts[0] + if not first.startswith("--"): + raise ValueError( + f"Invalid HPC config line {line_number}: expected a flag starting with '--', got: {line!r}" + ) + + if "=" in first: + if len(parts) != 1: + raise ValueError( + f"Invalid HPC config line {line_number}: do not mix '--flag=value' with extra tokens: {line!r}" + ) + flag, value = first.split("=", 1) + else: + if len(parts) != 2: + raise ValueError( + f"Invalid HPC config line {line_number}: expected '--flag value', got: {line!r}" + ) + flag, value = parts + + if value == "": + raise ValueError( + f"Invalid HPC config line {line_number}: missing value for flag {flag}" + ) + + return flag, value + + +def load_hpc_config(workload: Workload) -> dict: + """ + Load and parse the entire HpcCOnfig file + + Everything except q8s-node are treated as slurm flags. + """ + config_path = resolve_hpc_config_path(workload) + + if not config_path.exists(): + raise FileNotFoundError( + f"HPC target selected, but config file was not found: {config_path}" + ) + + q8s_node: str | None = None + slurm_flags: list[str] = [] + + with config_path.open("r", encoding="utf-8") as f: + for line_number, line in enumerate(f, start=1): + parsed = _parse_config_line(line, line_number) + if parsed is None: + continue + + flag, value = parsed + + if flag == "--q8s-node": + q8s_node = value + else: + slurm_flags.append(f"{flag}={value}") + + if q8s_node is None: + raise ValueError( + f"HPC config file '{config_path}' is missing required flag '--q8s-node'" + ) + + return { + "path": str(config_path), + "q8s_node": q8s_node, + "slurm_flags": slurm_flags, + } + + class HPCJobTemplatePlugin(JobPlugin): @hookimpl def makejob( @@ -24,6 +151,8 @@ def makejob( if target != Target.hpc: return None + hpc_config = load_hpc_config(workload) + volume_name = f"app-volume-{name}" env_var = list(env) @@ -43,7 +172,6 @@ def makejob( else [f"{WORKSPACE}/{workload.entry_script}"] + workload.args ), image_pull_policy="Always", - volume_mounts=[ client.V1VolumeMount( name=volume_name, @@ -53,22 +181,8 @@ def makejob( ], ) - extra = getattr(workload, "extra", {}) - - cpus = extra.get("hpc_cpus", 1) - mem = extra.get("hpc_mem", "6G") - time = extra.get("hpc_time", "00:04:00") - - flags = [ - "--account=project_462001257", - "--partition=standard", - f"--cpus-per-task={cpus}", - f"--mem={mem}", - f"--time={time}", - ] - annotations = { - "slurm-job.vk.io/flags": " ".join(flags) + "slurm-job.vk.io/flags": " ".join(hpc_config["slurm_flags"]) } template = client.V1PodTemplateSpec( @@ -81,20 +195,15 @@ def makejob( ), spec=client.V1PodSpec( containers=[container], - - # automount_service_account_token=False, - node_selector={ - "interlink.cern.ch/provider": "remote-hpc" + "kubernetes.io/hostname": hpc_config["q8s_node"] }, - tolerations=[ client.V1Toleration( key="virtual-node.interlink/no-schedule", operator="Exists" ) ], - image_pull_secrets=( [ client.V1LocalObjectReference( @@ -105,7 +214,6 @@ def makejob( else [] ), restart_policy="Never", - volumes=[ client.V1Volume( name=volume_name, @@ -121,4 +229,4 @@ def makejob( ), ) - return template + return template \ No newline at end of file From a6e7f2159ddf95a0fb293375c20f1064503ccf17 Mon Sep 17 00:00:00 2001 From: Vornit Date: Thu, 23 Apr 2026 17:51:53 +0300 Subject: [PATCH 04/13] Replace hardcoded targets with dynamically loaded plugin packages --- pyproject.toml | 3 +++ src/q8s/cli.py | 28 ++++++++++++++++++---------- src/q8s/execution.py | 32 +++++++++++++++++++++++--------- src/q8s/plugins/cpu_job.py | 3 ++- src/q8s/plugins/hpc_job.py | 4 +++- 5 files changed, 49 insertions(+), 21 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a0d4867..aab8250 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,3 +61,6 @@ q8sctl = "q8s.cli:app" testpaths = ["tests"] python_files = ["test_*.py"] addopts = "--cov=q8s --cov-report=term-missing" + +[project.entry-points."q8s.targets"] +cpu = "q8s.plugins.cpu_job:CPUJobTemplatePlugin" \ No newline at end of file diff --git a/src/q8s/cli.py b/src/q8s/cli.py index bc3770e..6e3db1b 100644 --- a/src/q8s/cli.py +++ b/src/q8s/cli.py @@ -7,7 +7,7 @@ from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn from typing_extensions import Annotated -from q8s.enums import Target +# from q8s.enums import Target from q8s.execution import K8sContext from q8s.install import install_my_kernel_spec from q8s.project import Project @@ -42,7 +42,7 @@ def init( def build( init: Annotated[bool, typer.Option(help="Initialize project")] = False, target: Annotated[ - Target, typer.Option(help="Execution target", case_sensitive=False) + str, typer.Option(help="Execution target", case_sensitive=False) ] = None, dry_run: Annotated[ bool, typer.Option(help="Dry run does not push images to the registry") @@ -77,7 +77,7 @@ def build( if target: project.build_container( - target=target.value, + target=target, progress=progress, push=(not dry_run), silent=silent, @@ -102,8 +102,8 @@ def build( def execute( file: Annotated[Path, typer.Argument(help="Python file to be executed")], target: Annotated[ - Target, typer.Option(help="Execution target", case_sensitive=False) - ] = Target.gpu, + str, typer.Option(help="Execution target", case_sensitive=False) + ] = "gpu", kubeconfig: Annotated[ Path, typer.Option(help="Kubernetes configuration", envvar="KUBECONFIG") ] = None, @@ -129,9 +129,6 @@ def execute( ): project = Project() - if image is None: - image = project.cached_images(target.value) - if kubeconfig is None: kubeconfig = project.kubeconfig @@ -150,6 +147,17 @@ def execute( expand=True, ) as progress: k8s_context = K8sContext(kubeconfig.as_posix(), progress=progress) + + if target not in k8s_context.available_targets(): + typer.echo( + f"Invalid target '{target}'. Available: {k8s_context.available_targets()}" + ) + raise typer.Exit(code=1) + + if image is None: + image = project.cached_images(target) + + k8s_context.set_target(target) k8s_context.set_registry_pat(registry_pat) k8s_context.set_container_image(image) @@ -178,8 +186,8 @@ def jupyter( ), ] = False, target: Annotated[ - Target, typer.Option(help="Execution target", case_sensitive=False) - ] = Target.gpu, + str, typer.Option(help="Execution target", case_sensitive=False) + ] = "gpu", kubeconfig: Annotated[ Path, typer.Option(help="Kubernetes configuration", envvar="KUBECONFIG") ] = None, diff --git a/src/q8s/execution.py b/src/q8s/execution.py index cf24bf2..5de9795 100644 --- a/src/q8s/execution.py +++ b/src/q8s/execution.py @@ -13,14 +13,13 @@ from rich.progress import Progress, TaskID from q8s.enums import Target -from q8s.plugins.cpu_job import CPUJobTemplatePlugin -from q8s.plugins.cuda_job import CUDAJobTemplatePlugin -from q8s.plugins.hpc_job import HPCJobTemplatePlugin from q8s.plugins.job_template_spec import JobTemplatePluginSpec from q8s.utils import extract_non_none_value from .workload import Workload +from importlib.metadata import entry_points + def load_env(): env = dotenv_values(".env.q8s") @@ -114,10 +113,12 @@ def __init__(self, kubeconfig: str, logger=None, progress: Progress = None): """ self.__progress = progress + # self.jm.register(CPUJobTemplatePlugin()) + # self.jm.register(CUDAJobTemplatePlugin()) + # self.jm.register(HPCJobTemplatePlugin()) + self.jm.add_hookspecs(JobTemplatePluginSpec) - self.jm.register(CPUJobTemplatePlugin()) - self.jm.register(CUDAJobTemplatePlugin()) - self.jm.register(HPCJobTemplatePlugin()) + self._load_plugins() task_config = self.__progress.add_task( "[cyan]Loading configuration...", total=1 @@ -151,15 +152,22 @@ def get_id(): return "".join(random.choices(string.ascii_lowercase + string.digits, k=6)) def set_container_image(self, image: str): - ContainerImageValidator.validate(image, self.registry_pat) + # ContainerImageValidator.validate(image, self.registry_pat) self.container_image = image def set_registry_pat(self, pat: str): self.registry_pat = pat - def set_target(self, target: Target): - self.target = target + def set_target(self, target): + self.target = target.value if hasattr(target, "value") else target + + def available_targets(self): + return [ + p.target_name + for p in self.jm.get_plugins() + if hasattr(p, "target_name") + ] def create_job_object(self, code: str) -> client.V1Job: return None @@ -531,3 +539,9 @@ def abort(self): Abort the execution. """ self.__delete_job() + + def _load_plugins(self): + for ep in entry_points(group="q8s.targets"): + plugin_cls = ep.load() + plugin = plugin_cls() + self.jm.register(plugin) \ No newline at end of file diff --git a/src/q8s/plugins/cpu_job.py b/src/q8s/plugins/cpu_job.py index 12807d8..412a03b 100644 --- a/src/q8s/plugins/cpu_job.py +++ b/src/q8s/plugins/cpu_job.py @@ -8,6 +8,7 @@ class CPUJobTemplatePlugin(JobPlugin): + target_name = "cpu" @hookimpl def makejob( @@ -21,7 +22,7 @@ def makejob( target: Target, ) -> client.V1PodTemplateSpec: - if target != Target.cpu: + if target != self.target_name: return None volume_name = f"app-volume-{name}" diff --git a/src/q8s/plugins/hpc_job.py b/src/q8s/plugins/hpc_job.py index 0478ed3..ff1c4c3 100644 --- a/src/q8s/plugins/hpc_job.py +++ b/src/q8s/plugins/hpc_job.py @@ -136,6 +136,8 @@ def load_hpc_config(workload: Workload) -> dict: class HPCJobTemplatePlugin(JobPlugin): + target_name = "hpc" + @hookimpl def makejob( self, @@ -148,7 +150,7 @@ def makejob( target: Target, ) -> client.V1PodTemplateSpec: - if target != Target.hpc: + if target != self.target_name: return None hpc_config = load_hpc_config(workload) From 8e2596a66010c34705f67618500479ff2b3379a8 Mon Sep 17 00:00:00 2001 From: Vornit Date: Thu, 30 Apr 2026 16:08:32 +0300 Subject: [PATCH 05/13] Replace enum-based targets with dynamic plugin system, add plugin packages --- plugins/q8s-cuda/pyproject.toml | 16 ++ plugins/q8s-cuda/src/q8s_cuda/__init__.py | 0 .../q8s-cuda/src/q8s_cuda}/cuda_job.py | 6 +- plugins/q8s-hpc/pyproject.toml | 16 ++ plugins/q8s-hpc/src/q8s_hpc/__init__.py | 0 .../q8s-hpc/src/q8s_hpc}/hpc_job.py | 7 +- plugins/q8s-rocm/pyproject.toml | 16 ++ plugins/q8s-rocm/src/q8s_hpc_rocm/__init__.py | 0 .../q8s-rocm/src/q8s_hpc_rocm/hpc_rocm_job.py | 234 ++++++++++++++++++ scripts/install-plugins.sh | 14 ++ src/q8s/bakefile.py | 19 +- src/q8s/cli.py | 28 ++- src/q8s/constants.py | 3 +- src/q8s/enums.py | 13 +- src/q8s/execution.py | 6 +- src/q8s/kernel.py | 13 +- src/q8s/plugins/cpu_job.py | 4 +- src/q8s/plugins/job_template_spec.py | 6 +- src/q8s/project.py | 42 ++-- src/q8s/utils.py | 24 +- 20 files changed, 407 insertions(+), 60 deletions(-) create mode 100644 plugins/q8s-cuda/pyproject.toml create mode 100644 plugins/q8s-cuda/src/q8s_cuda/__init__.py rename {src/q8s/plugins => plugins/q8s-cuda/src/q8s_cuda}/cuda_job.py (97%) create mode 100644 plugins/q8s-hpc/pyproject.toml create mode 100644 plugins/q8s-hpc/src/q8s_hpc/__init__.py rename {src/q8s/plugins => plugins/q8s-hpc/src/q8s_hpc}/hpc_job.py (98%) create mode 100644 plugins/q8s-rocm/pyproject.toml create mode 100644 plugins/q8s-rocm/src/q8s_hpc_rocm/__init__.py create mode 100644 plugins/q8s-rocm/src/q8s_hpc_rocm/hpc_rocm_job.py create mode 100755 scripts/install-plugins.sh diff --git a/plugins/q8s-cuda/pyproject.toml b/plugins/q8s-cuda/pyproject.toml new file mode 100644 index 0000000..3a6a7ff --- /dev/null +++ b/plugins/q8s-cuda/pyproject.toml @@ -0,0 +1,16 @@ +[project] +name = "q8s-cuda" +version = "0.1.0" +description = "CUDA execution target plugin for q8s" +requires-python = ">=3.10" +dependencies = ["q8s"] + +[build-system] +requires = ["setuptools >= 61.0"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["src"] + +[project.entry-points."q8s.targets"] +gpu = "q8s_cuda.cuda_job:CUDAJobTemplatePlugin" \ No newline at end of file diff --git a/plugins/q8s-cuda/src/q8s_cuda/__init__.py b/plugins/q8s-cuda/src/q8s_cuda/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/q8s/plugins/cuda_job.py b/plugins/q8s-cuda/src/q8s_cuda/cuda_job.py similarity index 97% rename from src/q8s/plugins/cuda_job.py rename to plugins/q8s-cuda/src/q8s_cuda/cuda_job.py index b68f25d..d6a13c5 100644 --- a/src/q8s/plugins/cuda_job.py +++ b/plugins/q8s-cuda/src/q8s_cuda/cuda_job.py @@ -3,7 +3,6 @@ from kubernetes import client from q8s.constants import WORKSPACE -from q8s.enums import Target from q8s.plugins.job import JobPlugin from q8s.plugins.job_template_spec import hookimpl from q8s.workload import Workload @@ -12,6 +11,7 @@ class CUDAJobTemplatePlugin(JobPlugin): + target_name = "gpu" """ This plugin is used to create a job template for a GPU job. """ @@ -25,10 +25,10 @@ def makejob( container_image: str, workload: Workload, env: list[client.V1EnvVar], - target: Target, + target: str, ) -> client.V1PodTemplateSpec: - if target != Target.gpu: + if target != self.target_name: return None volume_name = f"app-volume-{name}" diff --git a/plugins/q8s-hpc/pyproject.toml b/plugins/q8s-hpc/pyproject.toml new file mode 100644 index 0000000..4fe5b77 --- /dev/null +++ b/plugins/q8s-hpc/pyproject.toml @@ -0,0 +1,16 @@ +[project] +name = "q8s-hpc" +version = "0.1.0" +description = "HPC execution target plugin for q8s" +requires-python = ">=3.10" +dependencies = ["q8s"] + +[build-system] +requires = ["setuptools >= 61.0"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["src"] + +[project.entry-points."q8s.targets"] +hpc = "q8s_hpc.hpc_job:HPCJobTemplatePlugin" \ No newline at end of file diff --git a/plugins/q8s-hpc/src/q8s_hpc/__init__.py b/plugins/q8s-hpc/src/q8s_hpc/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/q8s/plugins/hpc_job.py b/plugins/q8s-hpc/src/q8s_hpc/hpc_job.py similarity index 98% rename from src/q8s/plugins/hpc_job.py rename to plugins/q8s-hpc/src/q8s_hpc/hpc_job.py index ff1c4c3..db1120e 100644 --- a/src/q8s/plugins/hpc_job.py +++ b/plugins/q8s-hpc/src/q8s_hpc/hpc_job.py @@ -6,12 +6,11 @@ from kubernetes import client from q8s.constants import WORKSPACE -from q8s.enums import Target +# from q8s.enums import Target from q8s.plugins.job import JobPlugin from q8s.plugins.job_template_spec import hookimpl from q8s.workload import Workload - def _get_workload_extra(workload: Workload) -> dict: """ Gets the "extra" informatino of the workload. @@ -136,7 +135,7 @@ def load_hpc_config(workload: Workload) -> dict: class HPCJobTemplatePlugin(JobPlugin): - target_name = "hpc" + target_name = "hpc-cpu" @hookimpl def makejob( @@ -147,7 +146,7 @@ def makejob( container_image: str, workload: Workload, env: list[client.V1EnvVar], - target: Target, + target: str, ) -> client.V1PodTemplateSpec: if target != self.target_name: diff --git a/plugins/q8s-rocm/pyproject.toml b/plugins/q8s-rocm/pyproject.toml new file mode 100644 index 0000000..47eb300 --- /dev/null +++ b/plugins/q8s-rocm/pyproject.toml @@ -0,0 +1,16 @@ +[project] +name = "q8s-hpc-rocm" +version = "0.1.0" +description = "ROCm execution target plugin for q8s" +requires-python = ">=3.10" +dependencies = ["q8s"] + +[build-system] +requires = ["setuptools >= 61.0"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["src"] + +[project.entry-points."q8s.targets"] +hpc-rocm = "q8s_hpc_rocm.hpc_rocm_job:HpcRocmJobTemplatePlugin" diff --git a/plugins/q8s-rocm/src/q8s_hpc_rocm/__init__.py b/plugins/q8s-rocm/src/q8s_hpc_rocm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/q8s-rocm/src/q8s_hpc_rocm/hpc_rocm_job.py b/plugins/q8s-rocm/src/q8s_hpc_rocm/hpc_rocm_job.py new file mode 100644 index 0000000..c8da42b --- /dev/null +++ b/plugins/q8s-rocm/src/q8s_hpc_rocm/hpc_rocm_job.py @@ -0,0 +1,234 @@ +from __future__ import annotations + +import shlex +from pathlib import Path + +from kubernetes import client + +from q8s.constants import WORKSPACE +# from q8s.enums import Target +from q8s.plugins.job import JobPlugin +from q8s.plugins.job_template_spec import hookimpl +from q8s.workload import Workload + + +def _get_workload_extra(workload: Workload) -> dict: + """ + Gets the "extra" information of the workload. + """ + extra = getattr(workload, "extra", {}) + return extra if isinstance(extra, dict) else {} + + +def _find_project_root(start: Path) -> Path: + """ + Tries to find Q8Sproject file from current directory or a parent directory, + in order to locate the root directory of the project. + """ + current = start.resolve() + for candidate in [current, *current.parents]: + if (candidate / "Q8Sproject").exists(): + return candidate + return Path.cwd().resolve() + + +def resolve_hpc_config_path(workload: Workload) -> Path: + """ + Tries to find all necessary slurm settings by trying to + find HpcConfig file in the project root directory, and checking if + the workload already includes a path to it. + """ + extra = _get_workload_extra(workload) + explicit_path = extra.get("hpc_config") + + if explicit_path: + return Path(explicit_path).expanduser().resolve() + + project_root = _find_project_root(Path.cwd()) + return project_root / "HpcConfig" + + +def _parse_config_line(line: str, line_number: int) -> tuple[str, str] | None: + """ + Parses a single line of a HpcConfig file + + Accepts values formatted like these two options: + - --flag value + - --flag=value + + Blank lines and lines starting with '#' are ignored. + """ + stripped = line.strip() + + if not stripped or stripped.startswith("#"): + return None + + parts = shlex.split(stripped) + if not parts: + return None + + first = parts[0] + if not first.startswith("--"): + raise ValueError( + f"Invalid HPC config line {line_number}: expected a flag starting with '--', got: {line!r}" + ) + + if "=" in first: + if len(parts) != 1: + raise ValueError( + f"Invalid HPC config line {line_number}: do not mix '--flag=value' with extra tokens: {line!r}" + ) + flag, value = first.split("=", 1) + else: + if len(parts) != 2: + raise ValueError( + f"Invalid HPC config line {line_number}: expected '--flag value', got: {line!r}" + ) + flag, value = parts + + if value == "": + raise ValueError( + f"Invalid HPC config line {line_number}: missing value for flag {flag}" + ) + + return flag, value + + +def load_hpc_config(workload: Workload) -> dict: + """ + Load and parse the entire HpcCOnfig file + + Everything except q8s-node are treated as slurm flags. + """ + config_path = resolve_hpc_config_path(workload) + + if not config_path.exists(): + raise FileNotFoundError( + f"HPC target selected, but config file was not found: {config_path}" + ) + + q8s_node: str | None = None + slurm_flags: list[str] = [] + + with config_path.open("r", encoding="utf-8") as f: + for line_number, line in enumerate(f, start=1): + parsed = _parse_config_line(line, line_number) + if parsed is None: + continue + + flag, value = parsed + + if flag == "--q8s-node": + q8s_node = value + else: + slurm_flags.append(f"{flag}={value}") + + if q8s_node is None: + raise ValueError( + f"HPC config file '{config_path}' is missing required flag '--q8s-node'" + ) + + return { + "path": str(config_path), + "q8s_node": q8s_node, + "slurm_flags": slurm_flags, + } + + +class HpcRocmJobTemplatePlugin(JobPlugin): + target_name = "hpc-rocm" + + @hookimpl + def makejob( + self, + name: str, + registry_pat: str | None, + registry_credentials_secret_name: str, + container_image: str, + workload: Workload, + env: list[client.V1EnvVar], + target: str, + ) -> client.V1PodTemplateSpec: + + if target != self.target_name: + return None + + hpc_config = load_hpc_config(workload) + + volume_name = f"app-volume-{name}" + + env_var = list(env) + if workload.is_src_project: + env_var.append(client.V1EnvVar(name="PYTHONPATH", value=f"{WORKSPACE}/src")) + + self.patch_environment_with_git_info(env_var) + + container = client.V1Container( + name="quantum-routine", + image=container_image, + env=env_var, + command=["python"], + args=( + ["-m", workload.entry_module] + workload.args + if workload.is_src_project + else [f"{WORKSPACE}/{workload.entry_script}"] + workload.args + ), + image_pull_policy="Always", + volume_mounts=[ + client.V1VolumeMount( + name=volume_name, + mount_path=WORKSPACE, + read_only=True, + ) + ], + ) + + annotations = { + "slurm-job.vk.io/flags": " ".join(hpc_config["slurm_flags"]) + } + + template = client.V1PodTemplateSpec( + metadata=client.V1ObjectMeta( + labels={ + "app": name, + "interlink.cern.ch/provider": "remote-hpc", + }, + annotations=annotations, + ), + spec=client.V1PodSpec( + containers=[container], + node_selector={ + "kubernetes.io/hostname": hpc_config["q8s_node"] + }, + tolerations=[ + client.V1Toleration( + key="virtual-node.interlink/no-schedule", + operator="Exists" + ) + ], + image_pull_secrets=( + [ + client.V1LocalObjectReference( + name=registry_credentials_secret_name + ) + ] + if registry_pat + else [] + ), + restart_policy="Never", + volumes=[ + client.V1Volume( + name=volume_name, + config_map=client.V1ConfigMapVolumeSource( + name=name, + items=[ + client.V1KeyToPath(key=k, path=v) + for k, v in workload.mappings.items() + ], + ), + ) + ], + ), + ) + + return template \ No newline at end of file diff --git a/scripts/install-plugins.sh b/scripts/install-plugins.sh new file mode 100755 index 0000000..94fea14 --- /dev/null +++ b/scripts/install-plugins.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -e + +echo "Installing q8s plugins..." + +for dir in plugins/*; do + if [ -f "$dir/pyproject.toml" ]; then + echo "→ Installing $dir" + pip install -e "$dir" + fi +done + +echo "Done." diff --git a/src/q8s/bakefile.py b/src/q8s/bakefile.py index dd0793c..44d4f65 100644 --- a/src/q8s/bakefile.py +++ b/src/q8s/bakefile.py @@ -4,11 +4,12 @@ # Dataclasses representing the structure of a Docker bake file -class BakeTargetName(str, Enum): - cpu = "cpu" - gpu = "gpu" - qpu = "qpu" - hpc = "hpc" +# class BakeTargetName(str, Enum): +# cpu = "cpu" +# gpu = "gpu" +# qpu = "qpu" +# hpc = "hpc" +# hpc_rocm = "hpc_rocm" class BuildPlatform(str, Enum): @@ -18,7 +19,7 @@ class BuildPlatform(str, Enum): @dataclass class Group: - targets: list[BakeTargetName] + targets: list[str] @dataclass @@ -37,7 +38,7 @@ class BakeTargetData: @dataclass class Bakefile: group: Groups - target: dict[BakeTargetName, BakeTargetData] + target: dict[str, BakeTargetData] def __init__(self): self.group = Groups(default=Group(targets=[])) @@ -49,10 +50,10 @@ def add_target( tags: list[str], platforms: list[BuildPlatform | str], ): - bake_target = BakeTargetName(name) + bake_target = name self.target[bake_target] = BakeTargetData( - context=f"./{bake_target.value}", + context=f"./{bake_target}", dockerfile="Dockerfile", tags=tags, platforms=[BuildPlatform(p) for p in platforms], diff --git a/src/q8s/cli.py b/src/q8s/cli.py index 6e3db1b..4f583e7 100644 --- a/src/q8s/cli.py +++ b/src/q8s/cli.py @@ -14,6 +14,8 @@ from q8s.utils import get_docker_image, get_kubeconfig from q8s.workload import Workload +from q8s.utils import get_available_targets + app = typer.Typer() @@ -75,9 +77,17 @@ def build( project.init_cache() progress.advance(task) + available = get_available_targets() + if target: + + if target not in available: + raise ValueError( + f"Invalid target '{target}'. Available plugin targets: {available}" + ) + project.build_container( - target=target, + target, progress=progress, push=(not dry_run), silent=silent, @@ -85,12 +95,21 @@ def build( else: for build in project.configuration.targets.keys(): + if build not in available: + raise ValueError( + f"Invalid target '{build}' in Q8Sproject. " + f"Available plugin targets: {available}" + ) + project.build_container( - build, progress=progress, push=(not dry_run), silent=silent + build, + progress=progress, + push=(not dry_run), + silent=silent, ) - print(f"Project {project.name} ready") - project.update_images_cache() + print(f"Project {project.name} ready") + project.update_images_cache() @app.command( @@ -157,7 +176,6 @@ def execute( if image is None: image = project.cached_images(target) - k8s_context.set_target(target) k8s_context.set_registry_pat(registry_pat) k8s_context.set_container_image(image) diff --git a/src/q8s/constants.py b/src/q8s/constants.py index 88680a3..27e3970 100644 --- a/src/q8s/constants.py +++ b/src/q8s/constants.py @@ -11,7 +11,8 @@ def get_base_images(python_version: str = "3.12") -> dict[str, str]: "cpu": f"python:{python_version}-slim", "gpu": f"ghcr.io/qubernetes-dev/cuda:12.8.1-r2-py{python_version}", "qpu": f"python:{python_version}-slim", - "hpc": f"python:{python_version}-slim" + "hpc-cpu": f"python:{python_version}-slim", + "hpc-rocm": "aapopeiponen/qiskit-lumi-rocm-singlenode" } diff --git a/src/q8s/enums.py b/src/q8s/enums.py index 28781c9..f485bc4 100644 --- a/src/q8s/enums.py +++ b/src/q8s/enums.py @@ -1,8 +1,9 @@ -from enum import Enum +# from enum import Enum -class Target(str, Enum): - cpu = "cpu" - gpu = "gpu" - qpu = "qpu" - hpc = "hpc" \ No newline at end of file +# class Target(str, Enum): +# cpu = "cpu" +# gpu = "gpu" +# qpu = "qpu" +# hpc = "hpc" +# hpc_rocm = "hpc_rocm" \ No newline at end of file diff --git a/src/q8s/execution.py b/src/q8s/execution.py index 5de9795..19957ad 100644 --- a/src/q8s/execution.py +++ b/src/q8s/execution.py @@ -12,7 +12,7 @@ from kubernetes import client, config, watch from rich.progress import Progress, TaskID -from q8s.enums import Target +# from q8s.enums import Target from q8s.plugins.job_template_spec import JobTemplatePluginSpec from q8s.utils import extract_non_none_value @@ -103,7 +103,7 @@ class K8sContext: container_image: str | None = None registry_pat: str | None = None jupyter_logger: None - target: Target = Target.gpu + target: str = "gpu" jm: pluggy.PluginManager = pluggy.PluginManager("q8s") __progress: Progress | None @@ -160,7 +160,7 @@ def set_registry_pat(self, pat: str): self.registry_pat = pat def set_target(self, target): - self.target = target.value if hasattr(target, "value") else target + self.target = target def available_targets(self): return [ diff --git a/src/q8s/kernel.py b/src/q8s/kernel.py index 164958d..f3bc725 100644 --- a/src/q8s/kernel.py +++ b/src/q8s/kernel.py @@ -5,11 +5,13 @@ from ipykernel.kernelbase import Kernel from rich.progress import Progress, SpinnerColumn, TextColumn -from q8s.enums import Target +# from q8s.enums import Target from q8s.execution import K8sContext from q8s.project import Project from q8s.workload import Workload +from q8s.utils import get_available_targets + FORMAT = "[%(levelname)s %(asctime)-15s q8s_kernel] %(message)s" logging.basicConfig(level=logging.INFO, format=FORMAT) @@ -161,8 +163,8 @@ def _on_comm_open(self, comm, msg): comm.send( { "command": "init", - "targets": Project().configuration.targets.keys(), - "selected_target": self.k8s_context.target.name, + "targets": get_available_targets(), + "selected_target": self.k8s_context.target, } ) @@ -173,9 +175,10 @@ def _recv(msg): # Respond to the frontend if data["command"] == "set_target": - self.k8s_context.set_target(Target(data["payload"]["target"])) + target = data["payload"]["target"] + self.k8s_context.set_target(target) project = Project() - image = project.cached_images(data["payload"]["target"]) + image = project.cached_images(target) self.k8s_context.set_container_image(image) logging.info(f"Updated execution target to {data['payload']['target']}") else: diff --git a/src/q8s/plugins/cpu_job.py b/src/q8s/plugins/cpu_job.py index 412a03b..c30e56f 100644 --- a/src/q8s/plugins/cpu_job.py +++ b/src/q8s/plugins/cpu_job.py @@ -1,7 +1,7 @@ from kubernetes import client from q8s.constants import WORKSPACE -from q8s.enums import Target +# from q8s.enums import Target from q8s.plugins.job import JobPlugin from q8s.plugins.job_template_spec import hookimpl from q8s.workload import Workload @@ -19,7 +19,7 @@ def makejob( container_image: str, workload: Workload, env: list[client.V1EnvVar], - target: Target, + target: str, ) -> client.V1PodTemplateSpec: if target != self.target_name: diff --git a/src/q8s/plugins/job_template_spec.py b/src/q8s/plugins/job_template_spec.py index 8145558..901e85b 100644 --- a/src/q8s/plugins/job_template_spec.py +++ b/src/q8s/plugins/job_template_spec.py @@ -3,7 +3,7 @@ import pluggy from kubernetes import client -from q8s.enums import Target +# from q8s.enums import Target from q8s.workload import Workload hookspec = pluggy.HookspecMarker("q8s") @@ -15,7 +15,7 @@ class JobTemplatePluginSpec: @hookspec def prepare( self, - target: Target, + target: str, name: str, namespace: str, env: Dict[ @@ -34,7 +34,7 @@ def makejob( container_image: str, workload: Workload, env: list[client.V1EnvVar], - target: Target, + target: str, ) -> client.V1PodTemplateSpec: return None diff --git a/src/q8s/project.py b/src/q8s/project.py index d6d953c..ce0029b 100644 --- a/src/q8s/project.py +++ b/src/q8s/project.py @@ -83,19 +83,19 @@ class Q8STarget: python_env: Q8SPythonEnv -@dataclass -class Q8STargets: - cpu: Optional[Q8STarget] - gpu: Optional[Q8STarget] - qpu: Optional[Q8STarget] - hpc: Optional[Q8STarget] +# @dataclass +# class Q8STargets: +# cpu: Optional[Q8STarget] +# gpu: Optional[Q8STarget] +# qpu: Optional[Q8STarget] +# hpc: Optional[Q8STarget] - def keys(self): - return [ - key - for key in self.__dataclass_fields__.keys() - if getattr(self, key) is not None - ] +# def keys(self): +# return [ +# key +# for key in self.__dataclass_fields__.keys() +# if getattr(self, key) is not None +# ] @dataclass @@ -108,7 +108,7 @@ class Q8SDocker: class Q8SProject: name: str python_env: Q8SPythonEnv - targets: Q8STargets + targets: dict[str, Q8STarget] docker: Q8SDocker kubeconfig: str @@ -203,7 +203,12 @@ def cached_images(self, target: str) -> str: ) with open(cachepath, "r") as f: - return yaml.safe_load(f)[target] + images = yaml.safe_load(f) + + key = target + + return images[key] + def build_container( self, target: str, progress: Progress, silent: bool, push: bool = True @@ -437,7 +442,12 @@ def __create_dockerfile(self, target: str, f): print("# Base image specifications are available at:", file=f) print("# https://github.com/qubernetes-dev/images/tree/main/cuda", file=f) + if target not in base_images: + print(f"Skipping target '{target}' (no base image defined)") + return + print(f"FROM {base_images[target]}", file=f) + print("", file=f) print( @@ -461,10 +471,10 @@ def __create_dockerfile(self, target: str, f): print("RUN pip install --no-cache -r requirements.txt", file=f) def __get_target(self, target: str) -> Q8STarget: - if hasattr(self.configuration.targets, target) is False: + if target not in self.configuration.targets: raise Exception(f"Target {target} not found") - return getattr(self.configuration.targets, target) + return self.configuration.targets[target] def __get_project_url(self) -> Optional[str]: """ diff --git a/src/q8s/utils.py b/src/q8s/utils.py index afd1072..16a8c98 100644 --- a/src/q8s/utils.py +++ b/src/q8s/utils.py @@ -1,8 +1,10 @@ import os -from q8s.enums import Target +# from q8s.enums import Target from q8s.project import CacheNotBuiltException, Project, ProjectNotFoundException +from importlib.metadata import entry_points + def extract_non_none_value(arr): """ @@ -15,11 +17,12 @@ def extract_non_none_value(arr): return non_none_values[0] if non_none_values else None -def get_docker_image(target: Target = None, logging=None): +def get_docker_image(target: str = None, logging=None): try: project = Project() + key = target + image = project.cached_images(target=key) - image = project.cached_images(target=target) except ProjectNotFoundException as e: if logging: logging.warning(e) @@ -46,3 +49,18 @@ def get_kubeconfig(kubeconfig=None): return project.kubeconfig except ProjectNotFoundException: return None + + +def get_available_targets(): + available = [] + + for ep in entry_points(group="q8s.targets"): + try: + plugin_cls = ep.load() + plugin = plugin_cls() + if hasattr(plugin, "target_name"): + available.append(plugin.target_name) + except Exception: + pass + + return available \ No newline at end of file From f6278d4bbd666d62475a2b2dabab9dd508975047 Mon Sep 17 00:00:00 2001 From: Vornit Date: Thu, 30 Apr 2026 16:18:05 +0300 Subject: [PATCH 06/13] Change entry point name --- plugins/q8s-hpc/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/q8s-hpc/pyproject.toml b/plugins/q8s-hpc/pyproject.toml index 4fe5b77..e3e0e39 100644 --- a/plugins/q8s-hpc/pyproject.toml +++ b/plugins/q8s-hpc/pyproject.toml @@ -13,4 +13,4 @@ build-backend = "setuptools.build_meta" where = ["src"] [project.entry-points."q8s.targets"] -hpc = "q8s_hpc.hpc_job:HPCJobTemplatePlugin" \ No newline at end of file +hpc-cpu = "q8s_hpc.hpc_job:HPCJobTemplatePlugin" \ No newline at end of file From e2b448de8c24eff22c9d09a5e7c9d6c20b383d0f Mon Sep 17 00:00:00 2001 From: Vornit Date: Mon, 11 May 2026 15:32:59 +0300 Subject: [PATCH 07/13] Move base image configuration to plugin packages and improve target validation --- plugins/q8s-cuda/src/q8s_cuda/cuda_job.py | 4 +++ plugins/q8s-hpc/src/q8s_hpc/hpc_job.py | 3 ++ .../q8s-rocm/src/q8s_hpc_rocm/hpc_rocm_job.py | 3 ++ src/q8s/plugins/cpu_job.py | 3 ++ src/q8s/project.py | 29 +++++++++++++------ 5 files changed, 33 insertions(+), 9 deletions(-) diff --git a/plugins/q8s-cuda/src/q8s_cuda/cuda_job.py b/plugins/q8s-cuda/src/q8s_cuda/cuda_job.py index d6a13c5..03692bf 100644 --- a/plugins/q8s-cuda/src/q8s_cuda/cuda_job.py +++ b/plugins/q8s-cuda/src/q8s_cuda/cuda_job.py @@ -12,10 +12,14 @@ class CUDAJobTemplatePlugin(JobPlugin): target_name = "gpu" + """ This plugin is used to create a job template for a GPU job. """ + def get_base_image(self, python_version: str) -> str: + return f"ghcr.io/qubernetes-dev/cuda:12.8.1-r2-py{python_version}" + @hookimpl def makejob( self, diff --git a/plugins/q8s-hpc/src/q8s_hpc/hpc_job.py b/plugins/q8s-hpc/src/q8s_hpc/hpc_job.py index db1120e..9009ddb 100644 --- a/plugins/q8s-hpc/src/q8s_hpc/hpc_job.py +++ b/plugins/q8s-hpc/src/q8s_hpc/hpc_job.py @@ -137,6 +137,9 @@ def load_hpc_config(workload: Workload) -> dict: class HPCJobTemplatePlugin(JobPlugin): target_name = "hpc-cpu" + def get_base_image(self, python_version: str) -> str: + return f"python:{python_version}-slim" + @hookimpl def makejob( self, diff --git a/plugins/q8s-rocm/src/q8s_hpc_rocm/hpc_rocm_job.py b/plugins/q8s-rocm/src/q8s_hpc_rocm/hpc_rocm_job.py index c8da42b..50b6877 100644 --- a/plugins/q8s-rocm/src/q8s_hpc_rocm/hpc_rocm_job.py +++ b/plugins/q8s-rocm/src/q8s_hpc_rocm/hpc_rocm_job.py @@ -138,6 +138,9 @@ def load_hpc_config(workload: Workload) -> dict: class HpcRocmJobTemplatePlugin(JobPlugin): target_name = "hpc-rocm" + def get_base_image(self, python_version: str) -> str: + return "aapopeiponen/qiskit-lumi-rocm-singlenode" + @hookimpl def makejob( self, diff --git a/src/q8s/plugins/cpu_job.py b/src/q8s/plugins/cpu_job.py index c30e56f..f171c3e 100644 --- a/src/q8s/plugins/cpu_job.py +++ b/src/q8s/plugins/cpu_job.py @@ -10,6 +10,9 @@ class CPUJobTemplatePlugin(JobPlugin): target_name = "cpu" + def get_base_image(self, python_version: str) -> str: + return f"python:{python_version}-slim" + @hookimpl def makejob( self, diff --git a/src/q8s/project.py b/src/q8s/project.py index ce0029b..b362feb 100644 --- a/src/q8s/project.py +++ b/src/q8s/project.py @@ -15,9 +15,11 @@ from rich.progress import Progress from q8s.bakefile import Bakefile, BuildPlatform -from q8s.constants import WORKSPACE, get_base_images +from q8s.constants import WORKSPACE from q8s.plugins.utils.git_info import get_git_info +from q8s.utils import get_available_targets + def load(path: str): """ @@ -431,22 +433,31 @@ def ___create_bakefile(self, f): json.dump(asdict(bakefile), f, indent=2) def __create_dockerfile(self, target: str, f): - base_images = get_base_images( - python_version=self.configuration.python_env.python_version - ) print("# This file is autogenerated by q8sctl", file=f) print("# Do not edit manually\n", file=f) + plugin = self._get_plugin_for_target(target) + available = get_available_targets() + if target == "gpu": print("# Base image specifications are available at:", file=f) print("# https://github.com/qubernetes-dev/images/tree/main/cuda", file=f) - if target not in base_images: - print(f"Skipping target '{target}' (no base image defined)") - return + if plugin is None: + if target == "qpu": + base_image = f"python:{self.configuration.python_env.python_version}-slim" + else: + raise ValueError( + f"No plugin found for target '{target}'. " + f"Available plugin targets: {available}" + ) + else: + base_image = plugin.get_base_image( + self.configuration.python_env.python_version + ) - print(f"FROM {base_images[target]}", file=f) + print(f"FROM {base_image}", file=f) print("", file=f) @@ -472,7 +483,7 @@ def __create_dockerfile(self, target: str, f): def __get_target(self, target: str) -> Q8STarget: if target not in self.configuration.targets: - raise Exception(f"Target {target} not found") + raise ValueError(f"Target '{target}' not found in Q8Sproject") return self.configuration.targets[target] From a9b421618656fc2c3fde1c89b5149e63ffe9b887 Mon Sep 17 00:00:00 2001 From: Vornit Date: Tue, 2 Jun 2026 18:43:38 +0300 Subject: [PATCH 08/13] Update tests for plugin-based targets --- plugins/q8s-cuda/pyproject.toml | 9 +- plugins/q8s-cuda/src/q8s_cuda/cuda_job.py | 24 +++ plugins/q8s-cuda/tests/test_cuda_job.py | 94 +++++++++++ plugins/q8s-cuda/tests/test_cuda_target.py | 126 +++++++++++++++ .../{q8s-hpc => q8s-hpc-cpu}/pyproject.toml | 2 +- .../src/q8s_hpc/__init__.py | 0 .../src/q8s_hpc/hpc_job.py | 18 ++- .../{q8s-rocm => q8s-hpc-rocm}/pyproject.toml | 0 .../src/q8s_hpc_rocm/__init__.py | 0 .../src/q8s_hpc_rocm/hpc_rocm_job.py | 16 ++ src/q8s/cli.py | 7 +- src/q8s/constants.py | 17 -- src/q8s/execution.py | 33 ++-- src/q8s/kernel.py | 3 +- src/q8s/plugins/cpu_job.py | 26 ++- src/q8s/project.py | 19 ++- src/q8s/targets.py | 18 +++ src/q8s/utils.py | 18 +-- tests/test_bakefile.py | 22 +-- tests/test_constants.py | 38 ----- tests/test_cpu_job.py | 116 +++++++++++++ tests/test_job_template_spec.py | 152 ------------------ tests/test_project.py | 80 ++------- tests/test_utils.py | 9 +- 24 files changed, 514 insertions(+), 333 deletions(-) create mode 100644 plugins/q8s-cuda/tests/test_cuda_job.py create mode 100644 plugins/q8s-cuda/tests/test_cuda_target.py rename plugins/{q8s-hpc => q8s-hpc-cpu}/pyproject.toml (94%) rename plugins/{q8s-hpc => q8s-hpc-cpu}/src/q8s_hpc/__init__.py (100%) rename plugins/{q8s-hpc => q8s-hpc-cpu}/src/q8s_hpc/hpc_job.py (95%) rename plugins/{q8s-rocm => q8s-hpc-rocm}/pyproject.toml (100%) rename plugins/{q8s-rocm => q8s-hpc-rocm}/src/q8s_hpc_rocm/__init__.py (100%) rename plugins/{q8s-rocm => q8s-hpc-rocm}/src/q8s_hpc_rocm/hpc_rocm_job.py (95%) create mode 100644 src/q8s/targets.py delete mode 100644 tests/test_constants.py create mode 100644 tests/test_cpu_job.py delete mode 100644 tests/test_job_template_spec.py diff --git a/plugins/q8s-cuda/pyproject.toml b/plugins/q8s-cuda/pyproject.toml index 3a6a7ff..a7767e0 100644 --- a/plugins/q8s-cuda/pyproject.toml +++ b/plugins/q8s-cuda/pyproject.toml @@ -5,6 +5,13 @@ description = "CUDA execution target plugin for q8s" requires-python = ">=3.10" dependencies = ["q8s"] +[project.optional-dependencies] +development = [ + "pytest", + "pytest-cov", + "dockerfile-parse", +] + [build-system] requires = ["setuptools >= 61.0"] build-backend = "setuptools.build_meta" @@ -13,4 +20,4 @@ build-backend = "setuptools.build_meta" where = ["src"] [project.entry-points."q8s.targets"] -gpu = "q8s_cuda.cuda_job:CUDAJobTemplatePlugin" \ No newline at end of file +gpu = "q8s_cuda.cuda_job:CUDAJobTemplatePlugin" diff --git a/plugins/q8s-cuda/src/q8s_cuda/cuda_job.py b/plugins/q8s-cuda/src/q8s_cuda/cuda_job.py index 03692bf..530a0b5 100644 --- a/plugins/q8s-cuda/src/q8s_cuda/cuda_job.py +++ b/plugins/q8s-cuda/src/q8s_cuda/cuda_job.py @@ -1,4 +1,5 @@ import os +import re from kubernetes import client @@ -7,6 +8,10 @@ from q8s.plugins.job_template_spec import hookimpl from q8s.workload import Workload +from importlib.metadata import version + +_PYTHON_VERSION_RE = re.compile(r"^3(\.\d+){1,2}$") + MEMORY = os.environ.get("MEMORY", "32Gi") @@ -18,6 +23,11 @@ class CUDAJobTemplatePlugin(JobPlugin): """ def get_base_image(self, python_version: str) -> str: + if not _PYTHON_VERSION_RE.fullmatch(python_version): + raise ValueError( + f"Invalid python_version format: {python_version!r}" + ) + return f"ghcr.io/qubernetes-dev/cuda:12.8.1-r2-py{python_version}" @hookimpl @@ -41,6 +51,20 @@ def makejob( if workload.is_src_project: env_var.append(client.V1EnvVar(name="PYTHONPATH", value=f"{WORKSPACE}/src")) + env_var.append( + client.V1EnvVar( + name="Q8S_VERSION", + value=version("q8s"), + ) + ) + + env_var.append( + client.V1EnvVar( + name="Q8S_PLUGIN_VERSION", + value=version("q8s-cuda"), + ) + ) + self.patch_environment_with_git_info(env_var) container = client.V1Container( diff --git a/plugins/q8s-cuda/tests/test_cuda_job.py b/plugins/q8s-cuda/tests/test_cuda_job.py new file mode 100644 index 0000000..df20a91 --- /dev/null +++ b/plugins/q8s-cuda/tests/test_cuda_job.py @@ -0,0 +1,94 @@ +import unittest +from unittest.mock import MagicMock, patch + +from kubernetes import client + +from q8s_cuda.cuda_job import CUDAJobTemplatePlugin + + +class TestCUDAJobTemplatePlugin(unittest.TestCase): + + @patch("q8s_cuda.cuda_job.client.V1Container") + @patch("q8s.plugins.job_template_spec.client.V1PodTemplateSpec") + @patch("q8s.plugins.job_template_spec.client.V1PodSpec") + @patch("q8s.plugins.job_template_spec.client.V1ObjectMeta") + @patch("q8s.plugins.job_template_spec.client.V1VolumeMount") + @patch("q8s.plugins.job_template_spec.client.V1Volume") + @patch("q8s.plugins.job_template_spec.client.V1ConfigMapVolumeSource") + @patch("q8s.plugins.job_template_spec.client.V1LocalObjectReference") + @patch("q8s.plugins.job_template_spec.client.V1ResourceRequirements") + + def test_makejob_gpu( + self, + mock_v1_resource_requirements, + mock_v1_local_object_reference, + mock_v1_config_map_volume_source, + mock_v1_volume, + mock_v1_volume_mount, + mock_v1_object_meta, + mock_v1_pod_spec, + mock_v1_pod_template_spec, + mock_v1_container, + ): + plugin = CUDAJobTemplatePlugin() + name = "test-job" + registry_pat = "test-pat" + registry_credentials_secret_name = "test-secret" + container_image = "test-image" + + env = [client.V1EnvVar(name="TEST_ENV", value="value")] + + workload = MagicMock() + workload.is_src_project = False + workload.entry_module = "module" + workload.entry_script = "script.py" + workload.args = [] + workload.mappings = {"main.py": "main.py"} + + target = "gpu" + + result = plugin.makejob( + name, + registry_pat, + registry_credentials_secret_name, + container_image, + workload, + env, + target, + ) + + self.assertIsNotNone(result) + mock_v1_container.assert_called_once() + mock_v1_pod_template_spec.assert_called_once() + mock_v1_pod_spec.assert_called_once() + mock_v1_object_meta.assert_called_once() + mock_v1_volume_mount.assert_called_once() + mock_v1_volume.assert_called_once() + mock_v1_config_map_volume_source.assert_called_once() + mock_v1_local_object_reference.assert_called_once() + mock_v1_resource_requirements.assert_called_once() + + def test_get_base_image_default(self): + plugin = CUDAJobTemplatePlugin() + + self.assertEqual( + plugin.get_base_image("3.12"), + "ghcr.io/qubernetes-dev/cuda:12.8.1-r2-py3.12", + ) + + def test_get_base_image_custom_version(self): + plugin = CUDAJobTemplatePlugin() + + self.assertEqual( + plugin.get_base_image("3.11"), + "ghcr.io/qubernetes-dev/cuda:12.8.1-r2-py3.11", + ) + + def test_get_base_image_invalid_version_raises(self): + plugin = CUDAJobTemplatePlugin() + + with self.assertRaises(ValueError): + plugin.get_base_image("invalid") + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/plugins/q8s-cuda/tests/test_cuda_target.py b/plugins/q8s-cuda/tests/test_cuda_target.py new file mode 100644 index 0000000..43e0df2 --- /dev/null +++ b/plugins/q8s-cuda/tests/test_cuda_target.py @@ -0,0 +1,126 @@ +import json +import unittest +from io import StringIO +from unittest.mock import Mock + +from dockerfile_parse import DockerfileParser + +from q8s.constants import WORKSPACE +from q8s.plugins.utils.git_info import GitInfo +from q8s.project import Project +from q8s_cuda.cuda_job import CUDAJobTemplatePlugin + +mocked_configuration = { + "name": "Example", + "python_env": {"dependencies": ["qiskit==1.1.0"]}, + "targets": { + "cpu": {"python_env": {"dependencies": ["qiskit-aer==0.15.1"]}}, + "gpu": {"python_env": {"dependencies": ["qiskit-aer-gpu==0.15.1"]}}, + }, + "docker": {"username": "vstirbu", "registry": None}, + "kubeconfig": "tests/fixtures/cache/kubeconfig", +} + +class TestProject(unittest.TestCase): + @unittest.mock.patch("q8s.project.datetime") + @unittest.mock.patch("q8s.project.load") + def test_create_dockerfile_gpu(self, mock_load: Mock, mock_datetime: Mock): + mock_load.return_value = mocked_configuration + mock_datetime.now.return_value.isoformat.return_value = "2024-01-01T00:00:00" + + project = Project("tests/fixtures/cache") + + dockerfile = StringIO() + project._Project__create_dockerfile("gpu", dockerfile) + + dfp = DockerfileParser(fileobj=dockerfile) + + lines = [line for line in dfp.content.splitlines()] + + plugin = CUDAJobTemplatePlugin() + gpu_base_image = plugin.get_base_image("3.12") + + self.assertEqual( + lines, + [ + "# This file is autogenerated by q8sctl", + "# Do not edit manually", + "", + "# Base image specifications are available at:", + "# https://github.com/qubernetes-dev/images/tree/main/cuda", + f"FROM {gpu_base_image}", + "", + "LABEL org.opencontainers.image.created=2024-01-01T00:00:00", + f"LABEL org.opencontainers.image.title={project.name}", + "", + f"WORKDIR {WORKSPACE}", + "COPY requirements.txt .", + "RUN pip install --no-cache -r requirements.txt", + ], + ) + + @unittest.mock.patch("q8s.project.get_git_info") + @unittest.mock.patch("q8s.project.load") + def test_create_bakefile_no_repo(self, mock_load: Mock, mock_get_git_info: Mock): + mock_load.return_value = mocked_configuration + mock_get_git_info.return_value = GitInfo( + commit=None, + branch=None, + remote_url=None, + extra={}, + ) + + project = Project("tests/fixtures/cache") + + bakefile = StringIO() + project._Project___create_bakefile(bakefile) + + bakefile_data = json.loads(bakefile.getvalue()) + + self.assertIn( + "gpu", + bakefile_data["group"]["default"]["targets"], + ) + + self.assertEqual( + bakefile_data["target"]["gpu"], + { + "context": "./gpu", + "dockerfile": "Dockerfile", + "tags": ["vstirbu/q8s-example:gpu"], + "platforms": ["linux/amd64"], + }, + ) + + @unittest.mock.patch("q8s.project.get_git_info") + @unittest.mock.patch("q8s.project.load") + def test_create_bakefile_in_repo(self, mock_load: Mock, mock_get_git_info: Mock): + mock_load.return_value = mocked_configuration + mock_get_git_info.return_value = GitInfo( + commit="abc123", + branch="main", + remote_url="https://example.com/repo.git", + extra={}, + ) + + project = Project("tests/fixtures/cache") + + bakefile = StringIO() + project._Project___create_bakefile(bakefile) + + bakefile_data = json.loads(bakefile.getvalue()) + + self.assertIn( + "gpu", + bakefile_data["group"]["default"]["targets"], + ) + + self.assertEqual( + bakefile_data["target"]["gpu"], + { + "context": "./gpu", + "dockerfile": "Dockerfile", + "tags": ["vstirbu/q8s-example:gpu-main"], + "platforms": ["linux/amd64"], + }, + ) \ No newline at end of file diff --git a/plugins/q8s-hpc/pyproject.toml b/plugins/q8s-hpc-cpu/pyproject.toml similarity index 94% rename from plugins/q8s-hpc/pyproject.toml rename to plugins/q8s-hpc-cpu/pyproject.toml index e3e0e39..c8dbf81 100644 --- a/plugins/q8s-hpc/pyproject.toml +++ b/plugins/q8s-hpc-cpu/pyproject.toml @@ -1,5 +1,5 @@ [project] -name = "q8s-hpc" +name = "q8s-hpc-cpu" version = "0.1.0" description = "HPC execution target plugin for q8s" requires-python = ">=3.10" diff --git a/plugins/q8s-hpc/src/q8s_hpc/__init__.py b/plugins/q8s-hpc-cpu/src/q8s_hpc/__init__.py similarity index 100% rename from plugins/q8s-hpc/src/q8s_hpc/__init__.py rename to plugins/q8s-hpc-cpu/src/q8s_hpc/__init__.py diff --git a/plugins/q8s-hpc/src/q8s_hpc/hpc_job.py b/plugins/q8s-hpc-cpu/src/q8s_hpc/hpc_job.py similarity index 95% rename from plugins/q8s-hpc/src/q8s_hpc/hpc_job.py rename to plugins/q8s-hpc-cpu/src/q8s_hpc/hpc_job.py index 9009ddb..4440f72 100644 --- a/plugins/q8s-hpc/src/q8s_hpc/hpc_job.py +++ b/plugins/q8s-hpc-cpu/src/q8s_hpc/hpc_job.py @@ -6,11 +6,12 @@ from kubernetes import client from q8s.constants import WORKSPACE -# from q8s.enums import Target from q8s.plugins.job import JobPlugin from q8s.plugins.job_template_spec import hookimpl from q8s.workload import Workload +from importlib.metadata import version + def _get_workload_extra(workload: Workload) -> dict: """ Gets the "extra" informatino of the workload. @@ -160,6 +161,21 @@ def makejob( volume_name = f"app-volume-{name}" env_var = list(env) + + env_var.append( + client.V1EnvVar( + name="Q8S_VERSION", + value=version("q8s"), + ) + ) + + env_var.append( + client.V1EnvVar( + name="Q8S_PLUGIN_VERSION", + value=version("q8s-hpc-cpu"), + ) + ) + if workload.is_src_project: env_var.append(client.V1EnvVar(name="PYTHONPATH", value=f"{WORKSPACE}/src")) diff --git a/plugins/q8s-rocm/pyproject.toml b/plugins/q8s-hpc-rocm/pyproject.toml similarity index 100% rename from plugins/q8s-rocm/pyproject.toml rename to plugins/q8s-hpc-rocm/pyproject.toml diff --git a/plugins/q8s-rocm/src/q8s_hpc_rocm/__init__.py b/plugins/q8s-hpc-rocm/src/q8s_hpc_rocm/__init__.py similarity index 100% rename from plugins/q8s-rocm/src/q8s_hpc_rocm/__init__.py rename to plugins/q8s-hpc-rocm/src/q8s_hpc_rocm/__init__.py diff --git a/plugins/q8s-rocm/src/q8s_hpc_rocm/hpc_rocm_job.py b/plugins/q8s-hpc-rocm/src/q8s_hpc_rocm/hpc_rocm_job.py similarity index 95% rename from plugins/q8s-rocm/src/q8s_hpc_rocm/hpc_rocm_job.py rename to plugins/q8s-hpc-rocm/src/q8s_hpc_rocm/hpc_rocm_job.py index 50b6877..5fe287a 100644 --- a/plugins/q8s-rocm/src/q8s_hpc_rocm/hpc_rocm_job.py +++ b/plugins/q8s-hpc-rocm/src/q8s_hpc_rocm/hpc_rocm_job.py @@ -11,6 +11,7 @@ from q8s.plugins.job_template_spec import hookimpl from q8s.workload import Workload +from importlib.metadata import version def _get_workload_extra(workload: Workload) -> dict: """ @@ -164,6 +165,21 @@ def makejob( if workload.is_src_project: env_var.append(client.V1EnvVar(name="PYTHONPATH", value=f"{WORKSPACE}/src")) + + env_var.append( + client.V1EnvVar( + name="Q8S_VERSION", + value=version("q8s"), + ) + ) + + env_var.append( + client.V1EnvVar( + name="Q8S_PLUGIN_VERSION", + value=version("q8s-hpc-rocm"), + ) + ) + self.patch_environment_with_git_info(env_var) container = client.V1Container( diff --git a/src/q8s/cli.py b/src/q8s/cli.py index 4f583e7..5335346 100644 --- a/src/q8s/cli.py +++ b/src/q8s/cli.py @@ -7,14 +7,13 @@ from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn from typing_extensions import Annotated -# from q8s.enums import Target from q8s.execution import K8sContext from q8s.install import install_my_kernel_spec from q8s.project import Project from q8s.utils import get_docker_image, get_kubeconfig from q8s.workload import Workload -from q8s.utils import get_available_targets +from q8s.targets import get_available_targets app = typer.Typer() @@ -122,7 +121,7 @@ def execute( file: Annotated[Path, typer.Argument(help="Python file to be executed")], target: Annotated[ str, typer.Option(help="Execution target", case_sensitive=False) - ] = "gpu", + ] = "cpu", kubeconfig: Annotated[ Path, typer.Option(help="Kubernetes configuration", envvar="KUBECONFIG") ] = None, @@ -205,7 +204,7 @@ def jupyter( ] = False, target: Annotated[ str, typer.Option(help="Execution target", case_sensitive=False) - ] = "gpu", + ] = "cpu", kubeconfig: Annotated[ Path, typer.Option(help="Kubernetes configuration", envvar="KUBECONFIG") ] = None, diff --git a/src/q8s/constants.py b/src/q8s/constants.py index 27e3970..949a465 100644 --- a/src/q8s/constants.py +++ b/src/q8s/constants.py @@ -2,21 +2,4 @@ _PYTHON_VERSION_RE = re.compile(r"^3(\.\d+){1,2}$") - -def get_base_images(python_version: str = "3.12") -> dict[str, str]: - if not _PYTHON_VERSION_RE.fullmatch(python_version): - raise ValueError(f"Invalid python_version format: {python_version!r}") - - return { - "cpu": f"python:{python_version}-slim", - "gpu": f"ghcr.io/qubernetes-dev/cuda:12.8.1-r2-py{python_version}", - "qpu": f"python:{python_version}-slim", - "hpc-cpu": f"python:{python_version}-slim", - "hpc-rocm": "aapopeiponen/qiskit-lumi-rocm-singlenode" - } - - -BASE_IMAGES = get_base_images() - - WORKSPACE = "/app" diff --git a/src/q8s/execution.py b/src/q8s/execution.py index 19957ad..e885336 100644 --- a/src/q8s/execution.py +++ b/src/q8s/execution.py @@ -12,13 +12,12 @@ from kubernetes import client, config, watch from rich.progress import Progress, TaskID -# from q8s.enums import Target from q8s.plugins.job_template_spec import JobTemplatePluginSpec from q8s.utils import extract_non_none_value from .workload import Workload -from importlib.metadata import entry_points +from importlib.metadata import entry_points, version def load_env(): @@ -103,7 +102,7 @@ class K8sContext: container_image: str | None = None registry_pat: str | None = None jupyter_logger: None - target: str = "gpu" + target: str = "cpu" jm: pluggy.PluginManager = pluggy.PluginManager("q8s") __progress: Progress | None @@ -113,10 +112,6 @@ def __init__(self, kubeconfig: str, logger=None, progress: Progress = None): """ self.__progress = progress - # self.jm.register(CPUJobTemplatePlugin()) - # self.jm.register(CUDAJobTemplatePlugin()) - # self.jm.register(HPCJobTemplatePlugin()) - self.jm.add_hookspecs(JobTemplatePluginSpec) self._load_plugins() @@ -162,6 +157,7 @@ def set_registry_pat(self, pat: str): def set_target(self, target): self.target = target + def available_targets(self): return [ p.target_name @@ -204,11 +200,30 @@ def __create_job_object_from_workload(self, workload: Workload) -> client.V1Job: spec = client.V1JobSpec( template=template, # cleanup job and associate resources - ttl_seconds_after_finished=10, + ttl_seconds_after_finished=100, # do not retry failed jobs backoff_limit=0, ) + plugin_version = None + + for ep in entry_points(group="q8s.targets"): + plugin_cls = ep.load() + plugin = plugin_cls() + + if ( + hasattr(plugin, "target_name") + and plugin.target_name == self.target + ): + plugin_version = ep.dist.version + break + + labels = { + "qubernetes.dev/job.type": "jupyter", + "qubernetes.dev/q8s-version": version("q8s"), + "qubernetes.dev/plugin-version": plugin_version, + } + # Instantiate the job object job_spec = client.V1Job( api_version="batch/v1", @@ -216,7 +231,7 @@ def __create_job_object_from_workload(self, workload: Workload) -> client.V1Job: metadata=client.V1ObjectMeta( name=self.name, namespace=self.namespace, - labels={"qubernetes.dev/job.type": "jupyter"}, + labels=labels, ), spec=spec, ) diff --git a/src/q8s/kernel.py b/src/q8s/kernel.py index f3bc725..a3926ea 100644 --- a/src/q8s/kernel.py +++ b/src/q8s/kernel.py @@ -5,12 +5,11 @@ from ipykernel.kernelbase import Kernel from rich.progress import Progress, SpinnerColumn, TextColumn -# from q8s.enums import Target from q8s.execution import K8sContext from q8s.project import Project from q8s.workload import Workload -from q8s.utils import get_available_targets +from q8s.targets import get_available_targets FORMAT = "[%(levelname)s %(asctime)-15s q8s_kernel] %(message)s" logging.basicConfig(level=logging.INFO, format=FORMAT) diff --git a/src/q8s/plugins/cpu_job.py b/src/q8s/plugins/cpu_job.py index f171c3e..14a022d 100644 --- a/src/q8s/plugins/cpu_job.py +++ b/src/q8s/plugins/cpu_job.py @@ -1,16 +1,24 @@ +from importlib.metadata import version +import re + from kubernetes import client from q8s.constants import WORKSPACE -# from q8s.enums import Target from q8s.plugins.job import JobPlugin from q8s.plugins.job_template_spec import hookimpl from q8s.workload import Workload +_PYTHON_VERSION_RE = re.compile(r"^3(\.\d+){1,2}$") class CPUJobTemplatePlugin(JobPlugin): target_name = "cpu" - + def get_base_image(self, python_version: str) -> str: + if not _PYTHON_VERSION_RE.fullmatch(python_version): + raise ValueError( + f"Invalid python_version format: {python_version!r}" + ) + return f"python:{python_version}-slim" @hookimpl @@ -34,6 +42,20 @@ def makejob( if workload.is_src_project: env_var.append(client.V1EnvVar(name="PYTHONPATH", value=f"{WORKSPACE}/src")) + env_var.append( + client.V1EnvVar( + name="Q8S_VERSION", + value=version("q8s"), + ) + ) + + env_var.append( + client.V1EnvVar( + name="Q8S_PLUGIN_VERSION", + value=version("q8s"), + ) + ) + self.patch_environment_with_git_info(env_var) container = client.V1Container( diff --git a/src/q8s/project.py b/src/q8s/project.py index b362feb..a6f5caa 100644 --- a/src/q8s/project.py +++ b/src/q8s/project.py @@ -18,7 +18,9 @@ from q8s.constants import WORKSPACE from q8s.plugins.utils.git_info import get_git_info -from q8s.utils import get_available_targets +from q8s.targets import get_available_targets +from importlib.metadata import entry_points + def load(path: str): @@ -210,6 +212,21 @@ def cached_images(self, target: str) -> str: key = target return images[key] + + def _get_plugin_for_target(self, target: str): + for ep in entry_points(group="q8s.targets"): + try: + plugin_cls = ep.load() + plugin = plugin_cls() + + if hasattr(plugin, "target_name"): + if plugin.target_name == target: + return plugin + + except Exception as e: + print(f"Failed to load plugin {ep.name}: {e}") + + return None def build_container( diff --git a/src/q8s/targets.py b/src/q8s/targets.py new file mode 100644 index 0000000..7d3b15c --- /dev/null +++ b/src/q8s/targets.py @@ -0,0 +1,18 @@ +from importlib.metadata import entry_points + + +def get_available_targets(): + available = [] + + for ep in entry_points(group="q8s.targets"): + try: + plugin_cls = ep.load() + plugin = plugin_cls() + + if hasattr(plugin, "target_name"): + available.append(plugin.target_name) + + except Exception: + pass + + return available \ No newline at end of file diff --git a/src/q8s/utils.py b/src/q8s/utils.py index 16a8c98..44d016b 100644 --- a/src/q8s/utils.py +++ b/src/q8s/utils.py @@ -1,6 +1,5 @@ import os -# from q8s.enums import Target from q8s.project import CacheNotBuiltException, Project, ProjectNotFoundException from importlib.metadata import entry_points @@ -48,19 +47,4 @@ def get_kubeconfig(kubeconfig=None): project = Project() return project.kubeconfig except ProjectNotFoundException: - return None - - -def get_available_targets(): - available = [] - - for ep in entry_points(group="q8s.targets"): - try: - plugin_cls = ep.load() - plugin = plugin_cls() - if hasattr(plugin, "target_name"): - available.append(plugin.target_name) - except Exception: - pass - - return available \ No newline at end of file + return None \ No newline at end of file diff --git a/tests/test_bakefile.py b/tests/test_bakefile.py index 8c87c14..0a16ce5 100644 --- a/tests/test_bakefile.py +++ b/tests/test_bakefile.py @@ -1,6 +1,6 @@ import unittest -from q8s.bakefile import Bakefile, BakeTargetName, BuildPlatform +from q8s.bakefile import Bakefile, BuildPlatform class TestBakefile(unittest.TestCase): @@ -15,9 +15,9 @@ def test_add_target_populates_structures(self): ) self.assertEqual(bakefile.group.default.targets, ["cpu"]) - self.assertIn(BakeTargetName.cpu, bakefile.target) + self.assertIn("cpu", bakefile.target) - target_data = bakefile.target[BakeTargetName.cpu] + target_data = bakefile.target["cpu"] self.assertEqual(target_data.context, "./cpu") self.assertEqual(target_data.dockerfile, "Dockerfile") self.assertEqual(target_data.tags, ["cpu-image:latest"]) @@ -39,23 +39,13 @@ def test_add_multiple_targets(self): self.assertEqual(bakefile.group.default.targets, ["cpu", "gpu"]) self.assertSetEqual( - set(bakefile.target.keys()), {BakeTargetName.cpu, BakeTargetName.gpu} + set(bakefile.target.keys()), {"cpu", "gpu"} ) - gpu_target = bakefile.target[BakeTargetName.gpu] + gpu_target = bakefile.target["gpu"] self.assertEqual(gpu_target.context, "./gpu") self.assertEqual(gpu_target.platforms, [BuildPlatform.linux_arm64]) - - def test_invalid_target_name_raises(self): - bakefile = Bakefile() - - with self.assertRaises(ValueError): - bakefile.add_target( - name="invalid", - tags=["tag"], - platforms=[BuildPlatform.linux_amd64.value], - ) - + def test_invalid_platform_raises(self): bakefile = Bakefile() diff --git a/tests/test_constants.py b/tests/test_constants.py deleted file mode 100644 index 4d083b8..0000000 --- a/tests/test_constants.py +++ /dev/null @@ -1,38 +0,0 @@ -import unittest - -from q8s.constants import get_base_images - - -class TestGetBaseImages(unittest.TestCase): - - def test_get_base_images_default(self): - images = get_base_images() - self.assertEqual( - images, - { - "cpu": "python:3.12-slim", - "gpu": "ghcr.io/qubernetes-dev/cuda:12.8.1-r2-py3.12", - "qpu": "python:3.12-slim", - }, - ) - - def test_get_base_images_custom_version(self): - images = get_base_images("3.11") - self.assertEqual( - images, - { - "cpu": "python:3.11-slim", - "gpu": "ghcr.io/qubernetes-dev/cuda:12.8.1-r2-py3.11", - "qpu": "python:3.11-slim", - }, - ) - - def test_get_base_images_invalid_version_raises(self): - for version in ("2.7", "3", "3.11.0.1", "3.11a"): - with self.subTest(version=version): - with self.assertRaises(ValueError): - get_base_images(version) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_cpu_job.py b/tests/test_cpu_job.py new file mode 100644 index 0000000..d1d9ebc --- /dev/null +++ b/tests/test_cpu_job.py @@ -0,0 +1,116 @@ +import unittest +from unittest.mock import MagicMock, patch + +from kubernetes import client + +from q8s.plugins.cpu_job import CPUJobTemplatePlugin + + +class TestCPUJobTemplatePlugin(unittest.TestCase): + + @patch("q8s.plugins.cpu_job.client.V1Container") + @patch("q8s.plugins.cpu_job.client.V1PodTemplateSpec") + @patch("q8s.plugins.cpu_job.client.V1PodSpec") + @patch("q8s.plugins.cpu_job.client.V1ObjectMeta") + @patch("q8s.plugins.cpu_job.client.V1VolumeMount") + @patch("q8s.plugins.cpu_job.client.V1Volume") + @patch("q8s.plugins.cpu_job.client.V1ConfigMapVolumeSource") + + def test_makejob_cpu( + self, + mock_v1_config_map_volume_source, + mock_v1_volume, + mock_v1_volume_mount, + mock_v1_object_meta, + mock_v1_pod_spec, + mock_v1_pod_template_spec, + mock_v1_container, + ): + plugin = CPUJobTemplatePlugin() + name = "test-job" + registry_pat = None + registry_credentials_secret_name = "test-secret" + container_image = "test-image" + env = [client.V1EnvVar(name="TEST_ENV", value="value")] + workload = MagicMock() + workload.is_src_project = False + workload.entry_module = "module" + workload.entry_script = "script.py" + workload.args = [] + workload.mappings = {"main.py": "main.py"} + target = "cpu" + + result = plugin.makejob( + name, + registry_pat, + registry_credentials_secret_name, + container_image, + workload, + env, + target, + ) + + self.assertIsNotNone(result) + mock_v1_container.assert_called_once() + mock_v1_pod_template_spec.assert_called_once() + mock_v1_pod_spec.assert_called_once() + mock_v1_object_meta.assert_called_once() + mock_v1_volume_mount.assert_called_once() + mock_v1_volume.assert_called_once() + mock_v1_config_map_volume_source.assert_called_once() + + + def test_makejob_invalid_target(self): + plugin = CPUJobTemplatePlugin() + name = "test-job" + registry_pat = None + registry_credentials_secret_name = "test-secret" + container_image = "test-image" + env = [client.V1EnvVar(name="TEST_ENV", value="value")] + workload = MagicMock() + workload.is_src_project = False + workload.entry_module = "module" + workload.entry_script = "script.py" + workload.args = [] + workload.mappings = {"main.py": "main.py"} + target = "invalid-target" + + result = plugin.makejob( + name, + registry_pat, + registry_credentials_secret_name, + container_image, + workload, + env, + target, + ) + + self.assertIsNone(result) + + + def test_get_base_image_default(self): + plugin = CPUJobTemplatePlugin() + + self.assertEqual( + plugin.get_base_image("3.12"), + "python:3.12-slim", + ) + + def test_get_base_image_custom_version(self): + plugin = CPUJobTemplatePlugin() + + self.assertEqual( + plugin.get_base_image("3.11"), + "python:3.11-slim", + ) + + def test_get_base_image_invalid_version_raises(self): + plugin = CPUJobTemplatePlugin() + + with self.assertRaises(ValueError): + plugin.get_base_image("invalid") + + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_job_template_spec.py b/tests/test_job_template_spec.py deleted file mode 100644 index 62c1488..0000000 --- a/tests/test_job_template_spec.py +++ /dev/null @@ -1,152 +0,0 @@ -import unittest -from unittest.mock import MagicMock, patch - -from kubernetes import client - -from q8s.enums import Target -from q8s.plugins.cpu_job import CPUJobTemplatePlugin -from q8s.plugins.cuda_job import CUDAJobTemplatePlugin - - -class TestCPUandGPUJobTemplatePlugin(unittest.TestCase): - - @patch("q8s.plugins.job_template_spec.client.V1Container") - @patch("q8s.plugins.job_template_spec.client.V1PodTemplateSpec") - @patch("q8s.plugins.job_template_spec.client.V1PodSpec") - @patch("q8s.plugins.job_template_spec.client.V1ObjectMeta") - @patch("q8s.plugins.job_template_spec.client.V1VolumeMount") - @patch("q8s.plugins.job_template_spec.client.V1Volume") - @patch("q8s.plugins.job_template_spec.client.V1ConfigMapVolumeSource") - @patch("q8s.plugins.job_template_spec.client.V1LocalObjectReference") - @patch("q8s.plugins.job_template_spec.client.V1ResourceRequirements") - def test_makejob_cpu( - self, - mock_v1_resource_requirements, - mock_v1_local_object_reference, - mock_v1_config_map_volume_source, - mock_v1_volume, - mock_v1_volume_mount, - mock_v1_object_meta, - mock_v1_pod_spec, - mock_v1_pod_template_spec, - mock_v1_container, - ): - plugin = CPUJobTemplatePlugin() - name = "test-job" - registry_pat = None - registry_credentials_secret_name = "test-secret" - container_image = "test-image" - env = [client.V1EnvVar(name="TEST_ENV", value="value")] - workload = MagicMock() - workload.is_src_project = False - workload.entry_module = "module" - workload.entry_script = "script.py" - workload.args = [] - workload.mappings = {"main.py": "main.py"} - target = Target.cpu - - result = plugin.makejob( - name, - registry_pat, - registry_credentials_secret_name, - container_image, - workload, - env, - target, - ) - - self.assertIsNotNone(result) - mock_v1_container.assert_called_once() - mock_v1_pod_template_spec.assert_called_once() - mock_v1_pod_spec.assert_called_once() - mock_v1_object_meta.assert_called_once() - mock_v1_volume_mount.assert_called_once() - mock_v1_volume.assert_called_once() - mock_v1_config_map_volume_source.assert_called_once() - - @patch("q8s.plugins.job_template_spec.client.V1Container") - @patch("q8s.plugins.job_template_spec.client.V1PodTemplateSpec") - @patch("q8s.plugins.job_template_spec.client.V1PodSpec") - @patch("q8s.plugins.job_template_spec.client.V1ObjectMeta") - @patch("q8s.plugins.job_template_spec.client.V1VolumeMount") - @patch("q8s.plugins.job_template_spec.client.V1Volume") - @patch("q8s.plugins.job_template_spec.client.V1ConfigMapVolumeSource") - @patch("q8s.plugins.job_template_spec.client.V1LocalObjectReference") - @patch("q8s.plugins.job_template_spec.client.V1ResourceRequirements") - def test_makejob_gpu( - self, - mock_v1_resource_requirements, - mock_v1_local_object_reference, - mock_v1_config_map_volume_source, - mock_v1_volume, - mock_v1_volume_mount, - mock_v1_object_meta, - mock_v1_pod_spec, - mock_v1_pod_template_spec, - mock_v1_container, - ): - plugin = CUDAJobTemplatePlugin() - name = "test-job" - registry_pat = "test-pat" - registry_credentials_secret_name = "test-secret" - container_image = "test-image" - env = [client.V1EnvVar(name="TEST_ENV", value="value")] - workload = MagicMock() - workload.is_src_project = False - workload.entry_module = "module" - workload.entry_script = "script.py" - workload.args = [] - workload.mappings = {"main.py": "main.py"} - target = Target.gpu - - result = plugin.makejob( - name, - registry_pat, - registry_credentials_secret_name, - container_image, - workload, - env, - target, - ) - - self.assertIsNotNone(result) - mock_v1_container.assert_called_once() - mock_v1_pod_template_spec.assert_called_once() - mock_v1_pod_spec.assert_called_once() - mock_v1_object_meta.assert_called_once() - mock_v1_volume_mount.assert_called_once() - mock_v1_volume.assert_called_once() - mock_v1_config_map_volume_source.assert_called_once() - mock_v1_local_object_reference.assert_called_once() - mock_v1_resource_requirements.assert_called_once() - - def test_makejob_invalid_target(self): - plugin = CPUJobTemplatePlugin() - name = "test-job" - registry_pat = None - registry_credentials_secret_name = "test-secret" - container_image = "test-image" - env = [client.V1EnvVar(name="TEST_ENV", value="value")] - workload = MagicMock() - workload.is_src_project = False - workload.entry_module = "module" - workload.entry_script = "script.py" - workload.args = [] - workload.mappings = {"main.py": "main.py"} - target = "invalid-target" - - result = plugin.makejob( - name, - registry_pat, - registry_credentials_secret_name, - container_image, - env, - workload, - target, - ) - - self.assertIsNone(result) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_project.py b/tests/test_project.py index a49242d..fce58f4 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -1,23 +1,22 @@ import json import unittest from io import StringIO -from os import remove from os.path import exists, join from unittest.mock import Mock from dockerfile_parse import DockerfileParser from rich.progress import Progress, SpinnerColumn -from q8s.constants import BASE_IMAGES, WORKSPACE +from q8s.constants import WORKSPACE from q8s.plugins.utils.git_info import GitInfo from q8s.project import Project +from q8s.plugins.cpu_job import CPUJobTemplatePlugin mocked_configuration = { "name": "Example", "python_env": {"dependencies": ["qiskit==1.1.0"]}, "targets": { - "cpu": {"python_env": {"dependencies": ["qiskit-aer==0.15.1"]}}, - "gpu": {"python_env": {"dependencies": ["qiskit-aer-gpu==0.15.1"]}}, + "cpu": {"python_env": {"dependencies": ["qiskit-aer==0.15.1"]}} }, "docker": {"username": "vstirbu", "registry": None}, "kubeconfig": "tests/fixtures/cache/kubeconfig", @@ -34,11 +33,7 @@ def _completed_process(): class TestProject(unittest.TestCase): - def before(self): - pass - def after(self): - remove("tests/fixtures/cache/.q8s_cache/cpu/requirements.txt", missing_ok=True) @unittest.mock.patch("q8s.project.load") def test_init(self, mock_load: Mock): @@ -49,7 +44,10 @@ def test_init(self, mock_load: Mock): self.assertEqual(project.name, "Example") self.assertIsNotNone(project.configuration.python_env) - self.assertEqual(project.configuration.targets.keys(), ["cpu", "gpu"]) + self.assertEqual( + list(project.configuration.targets.keys()), + ["cpu"], + ) @unittest.mock.patch("q8s.project.load") def test_init_cache(self, mock_load: Mock): @@ -59,12 +57,10 @@ def test_init_cache(self, mock_load: Mock): project = Project(project_path) self.assertFalse(exists(join(project_path, ".q8s_cache/cpu/requirements.txt"))) - self.assertFalse(exists(join(project_path, ".q8s_cache/gpu/requirements.txt"))) project.init_cache() self.assertTrue(exists(join(project_path, ".q8s_cache/cpu/requirements.txt"))) - self.assertTrue(exists(join(project_path, ".q8s_cache/gpu/requirements.txt"))) @unittest.mock.patch("q8s.project.load") def test_clear_cache(self, mock_load: Mock): @@ -76,12 +72,10 @@ def test_clear_cache(self, mock_load: Mock): project.init_cache() self.assertTrue(exists(join(project_path, ".q8s_cache/cpu/requirements.txt"))) - self.assertTrue(exists(join(project_path, ".q8s_cache/gpu/requirements.txt"))) project.clear_cache() self.assertFalse(exists(join(project_path, ".q8s_cache/cpu/requirements.txt"))) - self.assertFalse(exists(join(project_path, ".q8s_cache/gpu/requirements.txt"))) @unittest.mock.patch("q8s.project.load") def test_check_cache(self, mock_load: Mock): @@ -114,37 +108,8 @@ def test_create_dockerfile_cpu(self, mock_load: Mock, mock_datetime: Mock): lines = [line for line in dfp.content.splitlines()] - self.assertEqual( - lines, - [ - "# This file is autogenerated by q8sctl", - "# Do not edit manually", - "", - f"FROM {BASE_IMAGES['cpu']}", - "", - "LABEL org.opencontainers.image.created=2024-01-01T00:00:00", - f"LABEL org.opencontainers.image.title={project.name}", - "", - f"WORKDIR {WORKSPACE}", - "COPY requirements.txt .", - "RUN pip install --no-cache -r requirements.txt", - ], - ) - - @unittest.mock.patch("q8s.project.datetime") - @unittest.mock.patch("q8s.project.load") - def test_create_dockerfile_gpu(self, mock_load: Mock, mock_datetime: Mock): - mock_load.return_value = mocked_configuration - mock_datetime.now.return_value.isoformat.return_value = "2024-01-01T00:00:00" - - project = Project("tests/fixtures/cache") - - dockerfile = StringIO() - project._Project__create_dockerfile("gpu", dockerfile) - - dfp = DockerfileParser(fileobj=dockerfile) - - lines = [line for line in dfp.content.splitlines()] + plugin = CPUJobTemplatePlugin() + cpu_base_image = plugin.get_base_image("3.12") self.assertEqual( lines, @@ -152,9 +117,7 @@ def test_create_dockerfile_gpu(self, mock_load: Mock, mock_datetime: Mock): "# This file is autogenerated by q8sctl", "# Do not edit manually", "", - "# Base image specifications are available at:", - "# https://github.com/qubernetes-dev/images/tree/main/cuda", - f"FROM {BASE_IMAGES['gpu']}", + f"FROM {cpu_base_image}", "", "LABEL org.opencontainers.image.created=2024-01-01T00:00:00", f"LABEL org.opencontainers.image.title={project.name}", @@ -167,6 +130,7 @@ def test_create_dockerfile_gpu(self, mock_load: Mock, mock_datetime: Mock): @unittest.mock.patch("q8s.project.get_git_info") @unittest.mock.patch("q8s.project.load") + def test_create_bakefile_no_repo(self, mock_load: Mock, mock_get_git_info: Mock): mock_load.return_value = mocked_configuration mock_get_git_info.return_value = GitInfo( @@ -183,7 +147,7 @@ def test_create_bakefile_no_repo(self, mock_load: Mock, mock_get_git_info: Mock) bakefile_data = json.loads(bakefile.getvalue()) - self.assertEqual(bakefile_data["group"]["default"]["targets"], ["cpu", "gpu"]) + self.assertEqual(bakefile_data["group"]["default"]["targets"], ["cpu"]) self.assertEqual( bakefile_data["target"]["cpu"], { @@ -193,15 +157,6 @@ def test_create_bakefile_no_repo(self, mock_load: Mock, mock_get_git_info: Mock) "platforms": ["linux/amd64"], }, ) - self.assertEqual( - bakefile_data["target"]["gpu"], - { - "context": "./gpu", - "dockerfile": "Dockerfile", - "tags": ["vstirbu/q8s-example:gpu"], - "platforms": ["linux/amd64"], - }, - ) @unittest.mock.patch("q8s.project.get_git_info") @unittest.mock.patch("q8s.project.load") @@ -221,7 +176,7 @@ def test_create_bakefile_in_repo(self, mock_load: Mock, mock_get_git_info: Mock) bakefile_data = json.loads(bakefile.getvalue()) - self.assertEqual(bakefile_data["group"]["default"]["targets"], ["cpu", "gpu"]) + self.assertEqual(bakefile_data["group"]["default"]["targets"], ["cpu"]) self.assertEqual( bakefile_data["target"]["cpu"], { @@ -231,15 +186,6 @@ def test_create_bakefile_in_repo(self, mock_load: Mock, mock_get_git_info: Mock) "platforms": ["linux/amd64"], }, ) - self.assertEqual( - bakefile_data["target"]["gpu"], - { - "context": "./gpu", - "dockerfile": "Dockerfile", - "tags": ["vstirbu/q8s-example:gpu-main"], - "platforms": ["linux/amd64"], - }, - ) @unittest.mock.patch("q8s.project.get_git_info") @unittest.mock.patch("q8s.project.sys.platform", "win32") diff --git a/tests/test_utils.py b/tests/test_utils.py index c7d8c92..b32115f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,7 +1,6 @@ import unittest from unittest.mock import patch -from q8s.enums import Target from q8s.project import CacheNotBuiltException, ProjectNotFoundException from q8s.utils import get_docker_image @@ -12,7 +11,7 @@ class TestGetDockerImage(unittest.TestCase): def test_get_docker_image_success(self, MockProject): mock_project = MockProject.return_value mock_project.cached_images.return_value = "mock_image" - image = get_docker_image(target=Target.cpu) + image = get_docker_image(target="cpu") self.assertEqual(image, "mock_image") @patch("q8s.utils.Project") @@ -20,7 +19,7 @@ def test_get_docker_image_success(self, MockProject): def test_get_docker_image_project_not_found(self, mock_environ_get, MockProject): MockProject.side_effect = ProjectNotFoundException mock_environ_get.return_value = "env_image" - image = get_docker_image(target=Target.gpu) + image = get_docker_image(target="gpu") self.assertEqual(image, "env_image") @patch("q8s.utils.Project") @@ -28,7 +27,7 @@ def test_get_docker_image_project_not_found(self, mock_environ_get, MockProject) def test_get_docker_image_cache_not_built(self, mock_environ_get, MockProject): MockProject.return_value.cached_images.side_effect = CacheNotBuiltException mock_environ_get.return_value = "env_image" - image = get_docker_image(target=Target.gpu) + image = get_docker_image(target="gpu") self.assertEqual(image, "env_image") @patch("q8s.utils.Project") @@ -36,7 +35,7 @@ def test_get_docker_image_cache_not_built(self, mock_environ_get, MockProject): def test_get_docker_image_general_exception(self, mock_environ_get, MockProject): MockProject.side_effect = Exception("General error") mock_environ_get.return_value = "env_image" - image = get_docker_image(target=Target.cpu) + image = get_docker_image(target="cpu") self.assertEqual(image, "env_image") From b25804bcc0a02120584af645bcb984351f17415a Mon Sep 17 00:00:00 2001 From: Vornit Date: Tue, 2 Jun 2026 18:56:11 +0300 Subject: [PATCH 09/13] Improve exception handling --- src/q8s/targets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/q8s/targets.py b/src/q8s/targets.py index 7d3b15c..d58c0f0 100644 --- a/src/q8s/targets.py +++ b/src/q8s/targets.py @@ -12,7 +12,7 @@ def get_available_targets(): if hasattr(plugin, "target_name"): available.append(plugin.target_name) - except Exception: - pass + except Exception as e: + print(f"Failed to load plugin {ep.name}: {e}") return available \ No newline at end of file From 158625af93b173c08e372556bbbd4e595ea8295e Mon Sep 17 00:00:00 2001 From: Vornit Date: Tue, 2 Jun 2026 19:24:10 +0300 Subject: [PATCH 10/13] Add test coverage for plugin version labels --- tests/test_execution.py | 58 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/tests/test_execution.py b/tests/test_execution.py index 85a6e0d..c0ca5be 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -261,6 +261,64 @@ def test_complete_and_get_job_status_running_keeps_stdout(self): self.assertEqual(progress.update.call_count, 2) progress.advance.assert_called_once_with("task-id", 1) + @patch("q8s.execution.entry_points") + def test_create_job_object_sets_plugin_version_label(self, mock_entry_points): + ctx = K8sContext.__new__(K8sContext) + + ctx.target = "cpu" + ctx.name = "test-job" + ctx.namespace = "default" + ctx.registry_pat = None + + ctx._K8sContext__progress = Mock() + ctx._K8sContext__progress.add_task.return_value = "task-id" + ctx._K8sContext__progress.advance = Mock() + ctx._K8sContext__progress.console = Mock() + + ctx._K8sContext__prepare_environment = Mock(return_value=[]) + + ctx.jm = Mock() + + ctx.jm.hook.prepare = Mock() + ctx.jm.hook.prepare.return_value = [] + + ctx._K8sContext__env = {} + + plugin_template = Mock() + ctx.jm.hook.makejob.return_value = [plugin_template] + + ctx.batch_api_instance = Mock() + + created_job = Mock() + created_job.metadata.name = "test-job" + created_job.metadata.uid = "uid" + ctx.batch_api_instance.create_namespaced_job.return_value = created_job + + ctx._K8sContext__create_config_map_object_from_workload = Mock() + ctx._K8sContext__create_environment_secret = Mock() + + mock_ep = Mock() + mock_ep.dist.version = "1.2.3" + + plugin_cls = Mock() + plugin_instance = Mock() + plugin_instance.target_name = "cpu" + + plugin_cls.return_value = plugin_instance + mock_ep.load.return_value = plugin_cls + + mock_entry_points.return_value = [mock_ep] + + workload = Workload.from_code("print('hi')") + + ctx._K8sContext__create_job_object_from_workload(workload) + + created_spec = ctx.batch_api_instance.create_namespaced_job.call_args.kwargs["body"] + + self.assertEqual( + created_spec.metadata.labels["qubernetes.dev/plugin-version"], + "1.2.3", + ) if __name__ == "__main__": unittest.main() From 48841ec4e86c81254e82c2a0c9407064b4d3ed34 Mon Sep 17 00:00:00 2001 From: Vornit Date: Wed, 3 Jun 2026 13:33:35 +0300 Subject: [PATCH 11/13] Refactor repository into monorepo structure --- .github/workflows/publish.yml | 21 ++++++++--- {plugins => packages}/q8s-cuda/pyproject.toml | 0 .../q8s-cuda/src/q8s_cuda/__init__.py | 0 .../q8s-cuda/src/q8s_cuda/cuda_job.py | 0 .../q8s-cuda/tests/test_cuda_job.py | 0 .../q8s-cuda/tests/test_cuda_target.py | 0 .../q8s-hpc-cpu/pyproject.toml | 0 .../q8s-hpc-cpu/src/q8s_hpc/__init__.py | 0 .../q8s-hpc-cpu/src/q8s_hpc/hpc_job.py | 0 .../q8s-hpc-rocm/pyproject.toml | 0 .../q8s-hpc-rocm/src/q8s_hpc_rocm/__init__.py | 0 .../src/q8s_hpc_rocm/hpc_rocm_job.py | 0 pyproject.toml => packages/q8s/pyproject.toml | 0 {src => packages/q8s/src}/q8s/__init__.py | 0 {src => packages/q8s/src}/q8s/__main__.py | 0 {src => packages/q8s/src}/q8s/bakefile.py | 0 {src => packages/q8s/src}/q8s/cli.py | 0 {src => packages/q8s/src}/q8s/constants.py | 0 {src => packages/q8s/src}/q8s/enums.py | 0 {src => packages/q8s/src}/q8s/execution.py | 0 {src => packages/q8s/src}/q8s/install.py | 0 {src => packages/q8s/src}/q8s/kernel.py | 0 .../q8s/src}/q8s/matplotlib/backend.py | 0 {src => packages/q8s/src}/q8s/multifiles.py | 0 .../q8s/src}/q8s/plugins/__init__.py | 0 .../q8s/src}/q8s/plugins/cpu_job.py | 0 {src => packages/q8s/src}/q8s/plugins/job.py | 0 .../q8s/src}/q8s/plugins/job_template_spec.py | 0 .../q8s/src}/q8s/plugins/utils/__init__.py | 0 .../q8s/src}/q8s/plugins/utils/git_info.py | 0 {src => packages/q8s/src}/q8s/project.py | 0 {src => packages/q8s/src}/q8s/targets.py | 0 {src => packages/q8s/src}/q8s/utils.py | 0 {src => packages/q8s/src}/q8s/workload.py | 0 {tests => packages/q8s/tests}/__init__.py | 0 .../q8s/tests}/fixtures/cache/.gitignore | 0 .../q8s/tests}/fixtures/cache/Q8Sproject | 0 .../q8s/tests}/fixtures/cache/app.py | 0 .../q8s/tests}/fixtures/simple/.env.q8s | 0 .../q8s/tests}/fixtures/simple/qiskit.py | 0 {tests => packages/q8s/tests}/test_backend.py | 0 .../q8s/tests}/test_bakefile.py | 0 {tests => packages/q8s/tests}/test_cpu_job.py | 0 .../q8s/tests}/test_execution.py | 0 .../q8s/tests}/test_git_info.py | 0 .../q8s/tests}/test_job_plugin.py | 0 {tests => packages/q8s/tests}/test_kernel.py | 0 .../q8s/tests}/test_multifiles.py | 0 {tests => packages/q8s/tests}/test_project.py | 36 +++++++++---------- {tests => packages/q8s/tests}/test_utils.py | 0 .../q8s/tests}/test_workload.py | 0 {tests => packages/q8s/tests}/utils.py | 0 52 files changed, 35 insertions(+), 22 deletions(-) rename {plugins => packages}/q8s-cuda/pyproject.toml (100%) rename {plugins => packages}/q8s-cuda/src/q8s_cuda/__init__.py (100%) rename {plugins => packages}/q8s-cuda/src/q8s_cuda/cuda_job.py (100%) rename {plugins => packages}/q8s-cuda/tests/test_cuda_job.py (100%) rename {plugins => packages}/q8s-cuda/tests/test_cuda_target.py (100%) rename {plugins => packages}/q8s-hpc-cpu/pyproject.toml (100%) rename {plugins => packages}/q8s-hpc-cpu/src/q8s_hpc/__init__.py (100%) rename {plugins => packages}/q8s-hpc-cpu/src/q8s_hpc/hpc_job.py (100%) rename {plugins => packages}/q8s-hpc-rocm/pyproject.toml (100%) rename {plugins => packages}/q8s-hpc-rocm/src/q8s_hpc_rocm/__init__.py (100%) rename {plugins => packages}/q8s-hpc-rocm/src/q8s_hpc_rocm/hpc_rocm_job.py (100%) rename pyproject.toml => packages/q8s/pyproject.toml (100%) rename {src => packages/q8s/src}/q8s/__init__.py (100%) rename {src => packages/q8s/src}/q8s/__main__.py (100%) rename {src => packages/q8s/src}/q8s/bakefile.py (100%) rename {src => packages/q8s/src}/q8s/cli.py (100%) rename {src => packages/q8s/src}/q8s/constants.py (100%) rename {src => packages/q8s/src}/q8s/enums.py (100%) rename {src => packages/q8s/src}/q8s/execution.py (100%) rename {src => packages/q8s/src}/q8s/install.py (100%) rename {src => packages/q8s/src}/q8s/kernel.py (100%) rename {src => packages/q8s/src}/q8s/matplotlib/backend.py (100%) rename {src => packages/q8s/src}/q8s/multifiles.py (100%) rename {src => packages/q8s/src}/q8s/plugins/__init__.py (100%) rename {src => packages/q8s/src}/q8s/plugins/cpu_job.py (100%) rename {src => packages/q8s/src}/q8s/plugins/job.py (100%) rename {src => packages/q8s/src}/q8s/plugins/job_template_spec.py (100%) rename {src => packages/q8s/src}/q8s/plugins/utils/__init__.py (100%) rename {src => packages/q8s/src}/q8s/plugins/utils/git_info.py (100%) rename {src => packages/q8s/src}/q8s/project.py (100%) rename {src => packages/q8s/src}/q8s/targets.py (100%) rename {src => packages/q8s/src}/q8s/utils.py (100%) rename {src => packages/q8s/src}/q8s/workload.py (100%) rename {tests => packages/q8s/tests}/__init__.py (100%) rename {tests => packages/q8s/tests}/fixtures/cache/.gitignore (100%) rename {tests => packages/q8s/tests}/fixtures/cache/Q8Sproject (100%) rename {tests => packages/q8s/tests}/fixtures/cache/app.py (100%) rename {tests => packages/q8s/tests}/fixtures/simple/.env.q8s (100%) rename {tests => packages/q8s/tests}/fixtures/simple/qiskit.py (100%) rename {tests => packages/q8s/tests}/test_backend.py (100%) rename {tests => packages/q8s/tests}/test_bakefile.py (100%) rename {tests => packages/q8s/tests}/test_cpu_job.py (100%) rename {tests => packages/q8s/tests}/test_execution.py (100%) rename {tests => packages/q8s/tests}/test_git_info.py (100%) rename {tests => packages/q8s/tests}/test_job_plugin.py (100%) rename {tests => packages/q8s/tests}/test_kernel.py (100%) rename {tests => packages/q8s/tests}/test_multifiles.py (100%) rename {tests => packages/q8s/tests}/test_project.py (91%) rename {tests => packages/q8s/tests}/test_utils.py (100%) rename {tests => packages/q8s/tests}/test_workload.py (100%) rename {tests => packages/q8s/tests}/utils.py (100%) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e02729d..bdb80df 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -24,15 +24,28 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + + - name: Install q8s + working-directory: packages/q8s run: >- python3 -m pip install .[development] + + - name: Install q8s-cuda + working-directory: packages/q8s-cuda + run: pip install . + + # - name: Install dependencies + # run: >- + # python3 -m + # pip install + # .[development] + - name: Run tests with coverage run: >- python3 -m - pytest + pytest packages --cov=q8s --cov-report=xml - name: Upload coverage to Codecov @@ -62,13 +75,13 @@ jobs: build --user - name: Build a binary wheel and a source tarball + working-directory: packages/q8s run: python3 -m build - name: Store the distribution packages uses: actions/upload-artifact@v4 with: name: python-package-distributions - path: dist/ - + path: packages/q8s/dist/ publish-to-pypi: name: >- Publish Python 🐍 distribution 📦 to PyPI diff --git a/plugins/q8s-cuda/pyproject.toml b/packages/q8s-cuda/pyproject.toml similarity index 100% rename from plugins/q8s-cuda/pyproject.toml rename to packages/q8s-cuda/pyproject.toml diff --git a/plugins/q8s-cuda/src/q8s_cuda/__init__.py b/packages/q8s-cuda/src/q8s_cuda/__init__.py similarity index 100% rename from plugins/q8s-cuda/src/q8s_cuda/__init__.py rename to packages/q8s-cuda/src/q8s_cuda/__init__.py diff --git a/plugins/q8s-cuda/src/q8s_cuda/cuda_job.py b/packages/q8s-cuda/src/q8s_cuda/cuda_job.py similarity index 100% rename from plugins/q8s-cuda/src/q8s_cuda/cuda_job.py rename to packages/q8s-cuda/src/q8s_cuda/cuda_job.py diff --git a/plugins/q8s-cuda/tests/test_cuda_job.py b/packages/q8s-cuda/tests/test_cuda_job.py similarity index 100% rename from plugins/q8s-cuda/tests/test_cuda_job.py rename to packages/q8s-cuda/tests/test_cuda_job.py diff --git a/plugins/q8s-cuda/tests/test_cuda_target.py b/packages/q8s-cuda/tests/test_cuda_target.py similarity index 100% rename from plugins/q8s-cuda/tests/test_cuda_target.py rename to packages/q8s-cuda/tests/test_cuda_target.py diff --git a/plugins/q8s-hpc-cpu/pyproject.toml b/packages/q8s-hpc-cpu/pyproject.toml similarity index 100% rename from plugins/q8s-hpc-cpu/pyproject.toml rename to packages/q8s-hpc-cpu/pyproject.toml diff --git a/plugins/q8s-hpc-cpu/src/q8s_hpc/__init__.py b/packages/q8s-hpc-cpu/src/q8s_hpc/__init__.py similarity index 100% rename from plugins/q8s-hpc-cpu/src/q8s_hpc/__init__.py rename to packages/q8s-hpc-cpu/src/q8s_hpc/__init__.py diff --git a/plugins/q8s-hpc-cpu/src/q8s_hpc/hpc_job.py b/packages/q8s-hpc-cpu/src/q8s_hpc/hpc_job.py similarity index 100% rename from plugins/q8s-hpc-cpu/src/q8s_hpc/hpc_job.py rename to packages/q8s-hpc-cpu/src/q8s_hpc/hpc_job.py diff --git a/plugins/q8s-hpc-rocm/pyproject.toml b/packages/q8s-hpc-rocm/pyproject.toml similarity index 100% rename from plugins/q8s-hpc-rocm/pyproject.toml rename to packages/q8s-hpc-rocm/pyproject.toml diff --git a/plugins/q8s-hpc-rocm/src/q8s_hpc_rocm/__init__.py b/packages/q8s-hpc-rocm/src/q8s_hpc_rocm/__init__.py similarity index 100% rename from plugins/q8s-hpc-rocm/src/q8s_hpc_rocm/__init__.py rename to packages/q8s-hpc-rocm/src/q8s_hpc_rocm/__init__.py diff --git a/plugins/q8s-hpc-rocm/src/q8s_hpc_rocm/hpc_rocm_job.py b/packages/q8s-hpc-rocm/src/q8s_hpc_rocm/hpc_rocm_job.py similarity index 100% rename from plugins/q8s-hpc-rocm/src/q8s_hpc_rocm/hpc_rocm_job.py rename to packages/q8s-hpc-rocm/src/q8s_hpc_rocm/hpc_rocm_job.py diff --git a/pyproject.toml b/packages/q8s/pyproject.toml similarity index 100% rename from pyproject.toml rename to packages/q8s/pyproject.toml diff --git a/src/q8s/__init__.py b/packages/q8s/src/q8s/__init__.py similarity index 100% rename from src/q8s/__init__.py rename to packages/q8s/src/q8s/__init__.py diff --git a/src/q8s/__main__.py b/packages/q8s/src/q8s/__main__.py similarity index 100% rename from src/q8s/__main__.py rename to packages/q8s/src/q8s/__main__.py diff --git a/src/q8s/bakefile.py b/packages/q8s/src/q8s/bakefile.py similarity index 100% rename from src/q8s/bakefile.py rename to packages/q8s/src/q8s/bakefile.py diff --git a/src/q8s/cli.py b/packages/q8s/src/q8s/cli.py similarity index 100% rename from src/q8s/cli.py rename to packages/q8s/src/q8s/cli.py diff --git a/src/q8s/constants.py b/packages/q8s/src/q8s/constants.py similarity index 100% rename from src/q8s/constants.py rename to packages/q8s/src/q8s/constants.py diff --git a/src/q8s/enums.py b/packages/q8s/src/q8s/enums.py similarity index 100% rename from src/q8s/enums.py rename to packages/q8s/src/q8s/enums.py diff --git a/src/q8s/execution.py b/packages/q8s/src/q8s/execution.py similarity index 100% rename from src/q8s/execution.py rename to packages/q8s/src/q8s/execution.py diff --git a/src/q8s/install.py b/packages/q8s/src/q8s/install.py similarity index 100% rename from src/q8s/install.py rename to packages/q8s/src/q8s/install.py diff --git a/src/q8s/kernel.py b/packages/q8s/src/q8s/kernel.py similarity index 100% rename from src/q8s/kernel.py rename to packages/q8s/src/q8s/kernel.py diff --git a/src/q8s/matplotlib/backend.py b/packages/q8s/src/q8s/matplotlib/backend.py similarity index 100% rename from src/q8s/matplotlib/backend.py rename to packages/q8s/src/q8s/matplotlib/backend.py diff --git a/src/q8s/multifiles.py b/packages/q8s/src/q8s/multifiles.py similarity index 100% rename from src/q8s/multifiles.py rename to packages/q8s/src/q8s/multifiles.py diff --git a/src/q8s/plugins/__init__.py b/packages/q8s/src/q8s/plugins/__init__.py similarity index 100% rename from src/q8s/plugins/__init__.py rename to packages/q8s/src/q8s/plugins/__init__.py diff --git a/src/q8s/plugins/cpu_job.py b/packages/q8s/src/q8s/plugins/cpu_job.py similarity index 100% rename from src/q8s/plugins/cpu_job.py rename to packages/q8s/src/q8s/plugins/cpu_job.py diff --git a/src/q8s/plugins/job.py b/packages/q8s/src/q8s/plugins/job.py similarity index 100% rename from src/q8s/plugins/job.py rename to packages/q8s/src/q8s/plugins/job.py diff --git a/src/q8s/plugins/job_template_spec.py b/packages/q8s/src/q8s/plugins/job_template_spec.py similarity index 100% rename from src/q8s/plugins/job_template_spec.py rename to packages/q8s/src/q8s/plugins/job_template_spec.py diff --git a/src/q8s/plugins/utils/__init__.py b/packages/q8s/src/q8s/plugins/utils/__init__.py similarity index 100% rename from src/q8s/plugins/utils/__init__.py rename to packages/q8s/src/q8s/plugins/utils/__init__.py diff --git a/src/q8s/plugins/utils/git_info.py b/packages/q8s/src/q8s/plugins/utils/git_info.py similarity index 100% rename from src/q8s/plugins/utils/git_info.py rename to packages/q8s/src/q8s/plugins/utils/git_info.py diff --git a/src/q8s/project.py b/packages/q8s/src/q8s/project.py similarity index 100% rename from src/q8s/project.py rename to packages/q8s/src/q8s/project.py diff --git a/src/q8s/targets.py b/packages/q8s/src/q8s/targets.py similarity index 100% rename from src/q8s/targets.py rename to packages/q8s/src/q8s/targets.py diff --git a/src/q8s/utils.py b/packages/q8s/src/q8s/utils.py similarity index 100% rename from src/q8s/utils.py rename to packages/q8s/src/q8s/utils.py diff --git a/src/q8s/workload.py b/packages/q8s/src/q8s/workload.py similarity index 100% rename from src/q8s/workload.py rename to packages/q8s/src/q8s/workload.py diff --git a/tests/__init__.py b/packages/q8s/tests/__init__.py similarity index 100% rename from tests/__init__.py rename to packages/q8s/tests/__init__.py diff --git a/tests/fixtures/cache/.gitignore b/packages/q8s/tests/fixtures/cache/.gitignore similarity index 100% rename from tests/fixtures/cache/.gitignore rename to packages/q8s/tests/fixtures/cache/.gitignore diff --git a/tests/fixtures/cache/Q8Sproject b/packages/q8s/tests/fixtures/cache/Q8Sproject similarity index 100% rename from tests/fixtures/cache/Q8Sproject rename to packages/q8s/tests/fixtures/cache/Q8Sproject diff --git a/tests/fixtures/cache/app.py b/packages/q8s/tests/fixtures/cache/app.py similarity index 100% rename from tests/fixtures/cache/app.py rename to packages/q8s/tests/fixtures/cache/app.py diff --git a/tests/fixtures/simple/.env.q8s b/packages/q8s/tests/fixtures/simple/.env.q8s similarity index 100% rename from tests/fixtures/simple/.env.q8s rename to packages/q8s/tests/fixtures/simple/.env.q8s diff --git a/tests/fixtures/simple/qiskit.py b/packages/q8s/tests/fixtures/simple/qiskit.py similarity index 100% rename from tests/fixtures/simple/qiskit.py rename to packages/q8s/tests/fixtures/simple/qiskit.py diff --git a/tests/test_backend.py b/packages/q8s/tests/test_backend.py similarity index 100% rename from tests/test_backend.py rename to packages/q8s/tests/test_backend.py diff --git a/tests/test_bakefile.py b/packages/q8s/tests/test_bakefile.py similarity index 100% rename from tests/test_bakefile.py rename to packages/q8s/tests/test_bakefile.py diff --git a/tests/test_cpu_job.py b/packages/q8s/tests/test_cpu_job.py similarity index 100% rename from tests/test_cpu_job.py rename to packages/q8s/tests/test_cpu_job.py diff --git a/tests/test_execution.py b/packages/q8s/tests/test_execution.py similarity index 100% rename from tests/test_execution.py rename to packages/q8s/tests/test_execution.py diff --git a/tests/test_git_info.py b/packages/q8s/tests/test_git_info.py similarity index 100% rename from tests/test_git_info.py rename to packages/q8s/tests/test_git_info.py diff --git a/tests/test_job_plugin.py b/packages/q8s/tests/test_job_plugin.py similarity index 100% rename from tests/test_job_plugin.py rename to packages/q8s/tests/test_job_plugin.py diff --git a/tests/test_kernel.py b/packages/q8s/tests/test_kernel.py similarity index 100% rename from tests/test_kernel.py rename to packages/q8s/tests/test_kernel.py diff --git a/tests/test_multifiles.py b/packages/q8s/tests/test_multifiles.py similarity index 100% rename from tests/test_multifiles.py rename to packages/q8s/tests/test_multifiles.py diff --git a/tests/test_project.py b/packages/q8s/tests/test_project.py similarity index 91% rename from tests/test_project.py rename to packages/q8s/tests/test_project.py index fce58f4..fca0f1b 100644 --- a/tests/test_project.py +++ b/packages/q8s/tests/test_project.py @@ -19,7 +19,7 @@ "cpu": {"python_env": {"dependencies": ["qiskit-aer==0.15.1"]}} }, "docker": {"username": "vstirbu", "registry": None}, - "kubeconfig": "tests/fixtures/cache/kubeconfig", + "kubeconfig": "packages/q8s/tests/fixtures/cache/kubeconfig", } @@ -40,7 +40,7 @@ def test_init(self, mock_load: Mock): mock_load.return_value = mocked_configuration # TODO: mock reading configuration file - project = Project("tests/fixtures/cache") + project = Project("packages/q8s/tests/fixtures/cache") self.assertEqual(project.name, "Example") self.assertIsNotNone(project.configuration.python_env) @@ -53,7 +53,7 @@ def test_init(self, mock_load: Mock): def test_init_cache(self, mock_load: Mock): mock_load.return_value = mocked_configuration - project_path = "tests/fixtures/cache" + project_path = "packages/q8s/tests/fixtures/cache" project = Project(project_path) self.assertFalse(exists(join(project_path, ".q8s_cache/cpu/requirements.txt"))) @@ -66,7 +66,7 @@ def test_init_cache(self, mock_load: Mock): def test_clear_cache(self, mock_load: Mock): mock_load.return_value = mocked_configuration - project_path = "tests/fixtures/cache" + project_path = "packages/q8s/tests/fixtures/cache" project = Project(project_path) project.init_cache() @@ -81,7 +81,7 @@ def test_clear_cache(self, mock_load: Mock): def test_check_cache(self, mock_load: Mock): mock_load.return_value = mocked_configuration - project_path = "tests/fixtures/cache" + project_path = "packages/q8s/tests/fixtures/cache" project = Project(project_path) project.init_cache() @@ -99,7 +99,7 @@ def test_create_dockerfile_cpu(self, mock_load: Mock, mock_datetime: Mock): mock_load.return_value = mocked_configuration mock_datetime.now.return_value.isoformat.return_value = "2024-01-01T00:00:00" - project = Project("tests/fixtures/cache") + project = Project("packages/q8s/tests/fixtures/cache") dockerfile = StringIO() project._Project__create_dockerfile("cpu", dockerfile) @@ -140,7 +140,7 @@ def test_create_bakefile_no_repo(self, mock_load: Mock, mock_get_git_info: Mock) extra={}, ) - project = Project("tests/fixtures/cache") + project = Project("packages/q8s/tests/fixtures/cache") bakefile = StringIO() project._Project___create_bakefile(bakefile) @@ -169,7 +169,7 @@ def test_create_bakefile_in_repo(self, mock_load: Mock, mock_get_git_info: Mock) extra={}, ) - project = Project("tests/fixtures/cache") + project = Project("packages/q8s/tests/fixtures/cache") bakefile = StringIO() project._Project___create_bakefile(bakefile) @@ -202,7 +202,7 @@ def test_build_container( extra={}, ) - project_path = "tests/fixtures/cache" + project_path = "packages/q8s/tests/fixtures/cache" project = Project(project_path) mock_process = _completed_process() @@ -244,7 +244,7 @@ def test_build_container_in_repo( extra={}, ) - project_path = "tests/fixtures/cache" + project_path = "packages/q8s/tests/fixtures/cache" project = Project(project_path) mock_process = _completed_process() @@ -286,7 +286,7 @@ def test_push_container( extra={}, ) - project_path = "tests/fixtures/cache" + project_path = "packages/q8s/tests/fixtures/cache" project = Project(project_path) project.init_cache() @@ -322,7 +322,7 @@ def test_push_container_in_repo( extra={}, ) - project_path = "tests/fixtures/cache" + project_path = "packages/q8s/tests/fixtures/cache" project = Project(project_path) project.init_cache() @@ -354,11 +354,11 @@ def test_image_name_with_git_branch(self, mock_load: Mock, mock_get_git_info: Mo extra={}, ) - project = Project("tests/fixtures/cache") + project = Project("packages/q8s/tests/fixtures/cache") image_name = project._Project__image_name("cpu") self.assertEqual(image_name, "vstirbu/q8s-example:cpu-main") - mock_get_git_info.assert_called_once_with("tests/fixtures/cache") + mock_get_git_info.assert_called_once_with("packages/q8s/tests/fixtures/cache") @unittest.mock.patch("q8s.project.get_git_info") @unittest.mock.patch("q8s.project.load") @@ -373,11 +373,11 @@ def test_image_name_with_git_branch_sanitized( extra={}, ) - project = Project("tests/fixtures/cache") + project = Project("packages/q8s/tests/fixtures/cache") image_name = project._Project__image_name("cpu") self.assertEqual(image_name, "vstirbu/q8s-example:cpu-feature-1") - mock_get_git_info.assert_called_once_with("tests/fixtures/cache") + mock_get_git_info.assert_called_once_with("packages/q8s/tests/fixtures/cache") @unittest.mock.patch("q8s.project.get_git_info") @unittest.mock.patch("q8s.project.load") @@ -389,8 +389,8 @@ def test_image_name_without_git_branch( commit=None, branch=None, remote_url=None, extra={"reason": "no_repo"} ) - project = Project("tests/fixtures/cache") + project = Project("packages/q8s/tests/fixtures/cache") image_name = project._Project__image_name("cpu") self.assertEqual(image_name, "vstirbu/q8s-example:cpu") - mock_get_git_info.assert_called_once_with("tests/fixtures/cache") + mock_get_git_info.assert_called_once_with("packages/q8s/tests/fixtures/cache") diff --git a/tests/test_utils.py b/packages/q8s/tests/test_utils.py similarity index 100% rename from tests/test_utils.py rename to packages/q8s/tests/test_utils.py diff --git a/tests/test_workload.py b/packages/q8s/tests/test_workload.py similarity index 100% rename from tests/test_workload.py rename to packages/q8s/tests/test_workload.py diff --git a/tests/utils.py b/packages/q8s/tests/utils.py similarity index 100% rename from tests/utils.py rename to packages/q8s/tests/utils.py From 22f270b9b755e98ee95a0187c9401f8e21d7419c Mon Sep 17 00:00:00 2001 From: Vornit Date: Tue, 30 Jun 2026 15:47:37 +0300 Subject: [PATCH 12/13] Update package metadata and documentation --- .gitignore | 3 + README.md | 28 ++- ...004-entry-point-based-execution-targets.md | 31 +++ packages/q8s-cuda/LICENSE | 201 ++++++++++++++++++ packages/q8s-cuda/README.md | 19 ++ packages/q8s-cuda/pyproject.toml | 28 ++- packages/q8s-hpc-cpu/LICENSE | 201 ++++++++++++++++++ packages/q8s-hpc-cpu/README.md | 19 ++ packages/q8s-hpc-cpu/pyproject.toml | 22 +- packages/q8s-hpc-rocm/LICENSE | 201 ++++++++++++++++++ packages/q8s-hpc-rocm/README.md | 19 ++ packages/q8s-hpc-rocm/pyproject.toml | 24 ++- packages/q8s/LICENSE | 201 ++++++++++++++++++ packages/q8s/README.md | 94 ++++++++ packages/q8s/pyproject.toml | 4 +- 15 files changed, 1076 insertions(+), 19 deletions(-) create mode 100644 doc/adrs/0004-entry-point-based-execution-targets.md create mode 100644 packages/q8s-cuda/LICENSE create mode 100644 packages/q8s-cuda/README.md create mode 100644 packages/q8s-hpc-cpu/LICENSE create mode 100644 packages/q8s-hpc-cpu/README.md create mode 100644 packages/q8s-hpc-rocm/LICENSE create mode 100644 packages/q8s-hpc-rocm/README.md create mode 100644 packages/q8s/LICENSE create mode 100644 packages/q8s/README.md diff --git a/.gitignore b/.gitignore index f6dc10b..2d7b93d 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,9 @@ __pycache__/ *.pyo *.pyd *.pyc.so +dist/ +*.whl +*.tar.gz # Ignore Jupyter Notebook checkpoints .ipynb_checkpoints/ diff --git a/README.md b/README.md index bdc36d7..94cb2e2 100644 --- a/README.md +++ b/README.md @@ -9,17 +9,27 @@ Toolset for executing quantum jobs on [Qubernetes](https://www.qubernetes.dev). ## Installation -Install the for project folder: +Install the core package: ```bash pip install q8s ``` +The default installation includes the built-in `cpu` execution target. + +Additional execution targets are provided by plugin packages: + +```bash +pip install q8s-cuda +pip install q8s-hpc-cpu +pip install q8s-hpc-rocm +``` + ## Usage ### CLI -Sumbit a job to the Qubernetes cluster: +Submit a job to the Qubernetes cluster: ```bash q8sctl execute app.py --kubeconfig /path/to/kubeconfig @@ -59,14 +69,22 @@ Select the `Q8s kernel` when creating a new notebook. The development environment requires the following tools to be installed: -- [Docker](https://www.docker.com/get-started) +* [Docker](https://www.docker.com/get-started) ### Setup -Install the project in editable mode: +Install the core package in editable mode: + +```bash +pip install -e packages/q8s +``` + +Install plugin packages in editable mode as needed: ```bash -pip install -e . +pip install -e packages/q8s-cuda +pip install -e packages/q8s-hpc-cpu +pip install -e packages/q8s-hpc-rocm ``` If the project is installed in a virtual environment, the `q8s-kernel` can be installed by running the following command: diff --git a/doc/adrs/0004-entry-point-based-execution-targets.md b/doc/adrs/0004-entry-point-based-execution-targets.md new file mode 100644 index 0000000..52e01cb --- /dev/null +++ b/doc/adrs/0004-entry-point-based-execution-targets.md @@ -0,0 +1,31 @@ +# 4. Entry point based execution targets + +Date: 2026-06-30 + +## Status + +Proposed + +## Context + +Execution targets were previously registered by the core package. This made the core package responsible for knowing which targets are available and where their implementations are located. + +As new execution targets are added, this approach makes the core package harder to extend and maintain. + +## Decision + +Execution targets will be discovered through Python entry points. + +The core `q8s` package will provide the built-in `cpu` target. Additional targets will be provided by separate plugin packages and discovered through the `q8s.targets` entry point group. + +The initial external target packages are: + +* `q8s-cuda`, providing the `gpu` target +* `q8s-hpc-cpu`, providing the `hpc-cpu` target +* `q8s-hpc-rocm`, providing the `hpc-rocm` target + +## Consequences + +The core package no longer needs to hard-code all supported execution targets. + +New execution targets can be added by installing additional plugin packages. Users can install only the target plugins they need. diff --git a/packages/q8s-cuda/LICENSE b/packages/q8s-cuda/LICENSE new file mode 100644 index 0000000..5e79e67 --- /dev/null +++ b/packages/q8s-cuda/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [2024] [Qubernetes contributors] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/packages/q8s-cuda/README.md b/packages/q8s-cuda/README.md new file mode 100644 index 0000000..476e69c --- /dev/null +++ b/packages/q8s-cuda/README.md @@ -0,0 +1,19 @@ +# q8s-cuda + +CUDA-based GPU execution target plugin for q8s. + +This package provides the `gpu` execution target for q8s and depends on `q8s>=0.14.0`. + +## Installation + +```bash +pip install q8s-cuda +``` + +## Usage + +Submit a job using the `gpu` execution target: + +```bash +q8sctl execute app.py --target gpu --kubeconfig /path/to/kubeconfig +``` \ No newline at end of file diff --git a/packages/q8s-cuda/pyproject.toml b/packages/q8s-cuda/pyproject.toml index a7767e0..e81f0bc 100644 --- a/packages/q8s-cuda/pyproject.toml +++ b/packages/q8s-cuda/pyproject.toml @@ -1,16 +1,30 @@ [project] name = "q8s-cuda" version = "0.1.0" -description = "CUDA execution target plugin for q8s" +description = "CUDA-based GPU execution target plugin for q8s" +authors = [{ name = "Vlad Stirbu", email = "vstirbu@gmail.com" }] +readme = "README.md" +license = { file = "LICENSE" } +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", +] +dependencies = ["q8s>=0.14.0"] requires-python = ">=3.10" -dependencies = ["q8s"] + +[project.urls] +Homepage = "https://github.com/qubernetes-dev/q8s-kernel" +Issues = "https://github.com/qubernetes-dev/q8s-kernel/issues" [project.optional-dependencies] -development = [ - "pytest", - "pytest-cov", - "dockerfile-parse", -] +development = ["pytest", "pytest-cov", "dockerfile-parse"] [build-system] requires = ["setuptools >= 61.0"] diff --git a/packages/q8s-hpc-cpu/LICENSE b/packages/q8s-hpc-cpu/LICENSE new file mode 100644 index 0000000..5e79e67 --- /dev/null +++ b/packages/q8s-hpc-cpu/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [2024] [Qubernetes contributors] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/packages/q8s-hpc-cpu/README.md b/packages/q8s-hpc-cpu/README.md new file mode 100644 index 0000000..088bd41 --- /dev/null +++ b/packages/q8s-hpc-cpu/README.md @@ -0,0 +1,19 @@ +# q8s-hpc-cpu + +CPU execution target plugin for running q8s jobs on HPC nodes. + +This package provides the `hpc-cpu` execution target for q8s and depends on `q8s>=0.14.0`. + +## Installation + +```bash +pip install q8s-hpc-cpu +``` + +## Usage + +Submit a job using the `hpc-cpu` execution target: + +```bash +q8sctl execute app.py --target hpc-cpu --kubeconfig /path/to/kubeconfig +``` \ No newline at end of file diff --git a/packages/q8s-hpc-cpu/pyproject.toml b/packages/q8s-hpc-cpu/pyproject.toml index c8dbf81..783f878 100644 --- a/packages/q8s-hpc-cpu/pyproject.toml +++ b/packages/q8s-hpc-cpu/pyproject.toml @@ -1,9 +1,27 @@ [project] name = "q8s-hpc-cpu" version = "0.1.0" -description = "HPC execution target plugin for q8s" +description = "CPU execution target plugin for running q8s jobs on HPC nodes" +authors = [{ name = "Vlad Stirbu", email = "vstirbu@gmail.com" }] +readme = "README.md" +license = { file = "LICENSE" } +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", +] +dependencies = ["q8s>=0.14.0"] requires-python = ">=3.10" -dependencies = ["q8s"] + +[project.urls] +Homepage = "https://github.com/qubernetes-dev/q8s-kernel" +Issues = "https://github.com/qubernetes-dev/q8s-kernel/issues" [build-system] requires = ["setuptools >= 61.0"] diff --git a/packages/q8s-hpc-rocm/LICENSE b/packages/q8s-hpc-rocm/LICENSE new file mode 100644 index 0000000..5e79e67 --- /dev/null +++ b/packages/q8s-hpc-rocm/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [2024] [Qubernetes contributors] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/packages/q8s-hpc-rocm/README.md b/packages/q8s-hpc-rocm/README.md new file mode 100644 index 0000000..ed04ab4 --- /dev/null +++ b/packages/q8s-hpc-rocm/README.md @@ -0,0 +1,19 @@ +# q8s-hpc-rocm + +ROCm-based GPU execution target plugin for running q8s jobs on HPC nodes. + +This package provides the `hpc-rocm` execution target for q8s and depends on `q8s>=0.14.0`. + +## Installation + +```bash +pip install q8s-hpc-rocm +``` + +## Usage + +Submit a job using the `hpc-rocm` execution target: + +```bash +q8sctl execute app.py --target hpc-rocm --kubeconfig /path/to/kubeconfig +``` diff --git a/packages/q8s-hpc-rocm/pyproject.toml b/packages/q8s-hpc-rocm/pyproject.toml index 47eb300..3be3bb6 100644 --- a/packages/q8s-hpc-rocm/pyproject.toml +++ b/packages/q8s-hpc-rocm/pyproject.toml @@ -1,9 +1,27 @@ [project] name = "q8s-hpc-rocm" version = "0.1.0" -description = "ROCm execution target plugin for q8s" +description = "ROCm-based GPU execution target plugin for running q8s jobs on HPC nodes" +authors = [{ name = "Vlad Stirbu", email = "vstirbu@gmail.com" }] +readme = "README.md" +license = { file = "LICENSE" } +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", +] +dependencies = ["q8s>=0.14.0"] requires-python = ">=3.10" -dependencies = ["q8s"] + +[project.urls] +Homepage = "https://github.com/qubernetes-dev/q8s-kernel" +Issues = "https://github.com/qubernetes-dev/q8s-kernel/issues" [build-system] requires = ["setuptools >= 61.0"] @@ -13,4 +31,4 @@ build-backend = "setuptools.build_meta" where = ["src"] [project.entry-points."q8s.targets"] -hpc-rocm = "q8s_hpc_rocm.hpc_rocm_job:HpcRocmJobTemplatePlugin" +hpc-rocm = "q8s_hpc_rocm.hpc_rocm_job:HpcRocmJobTemplatePlugin" \ No newline at end of file diff --git a/packages/q8s/LICENSE b/packages/q8s/LICENSE new file mode 100644 index 0000000..5e79e67 --- /dev/null +++ b/packages/q8s/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [2024] [Qubernetes contributors] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/packages/q8s/README.md b/packages/q8s/README.md new file mode 100644 index 0000000..94cb2e2 --- /dev/null +++ b/packages/q8s/README.md @@ -0,0 +1,94 @@ +# q8s + +[![PyPI version](https://img.shields.io/pypi/v/q8s.svg)](https://pypi.org/project/q8s/) +[![Python versions](https://img.shields.io/pypi/pyversions/q8s.svg)](https://pypi.org/project/q8s/) +[![codecov](https://codecov.io/github/qubernetes-dev/q8s-kernel/graph/badge.svg?token=5JQ4ALSUZJ)](https://codecov.io/github/qubernetes-dev/q8s-kernel) +[![PyPI Downloads](https://static.pepy.tech/personalized-badge/q8s?period=total&units=INTERNATIONAL_SYSTEM&left_color=GREY&right_color=GREEN&left_text=downloads)](https://pepy.tech/projects/q8s) + +Toolset for executing quantum jobs on [Qubernetes](https://www.qubernetes.dev). + +## Installation + +Install the core package: + +```bash +pip install q8s +``` + +The default installation includes the built-in `cpu` execution target. + +Additional execution targets are provided by plugin packages: + +```bash +pip install q8s-cuda +pip install q8s-hpc-cpu +pip install q8s-hpc-rocm +``` + +## Usage + +### CLI + +Submit a job to the Qubernetes cluster: + +```bash +q8sctl execute app.py --kubeconfig /path/to/kubeconfig +``` + +For more options, run: + +```bash +q8sctl execute --help +``` + +### Jupyter Notebook + +Install the `q8s-kernel`: + +```bash +q8sctl jupyter --install +``` + +Start the jupyter notebook server: + +```bash +jupyter notebook +``` + +or the jupyter lab server: + +```bash +jupyter lab +``` + +Select the `Q8s kernel` when creating a new notebook. + +## Development + +### Prerequisites + +The development environment requires the following tools to be installed: + +* [Docker](https://www.docker.com/get-started) + +### Setup + +Install the core package in editable mode: + +```bash +pip install -e packages/q8s +``` + +Install plugin packages in editable mode as needed: + +```bash +pip install -e packages/q8s-cuda +pip install -e packages/q8s-hpc-cpu +pip install -e packages/q8s-hpc-rocm +``` + +If the project is installed in a virtual environment, the `q8s-kernel` can be installed by running the following command: + +```bash +q8sctl jupyter --install +``` diff --git a/packages/q8s/pyproject.toml b/packages/q8s/pyproject.toml index aab8250..bc14606 100644 --- a/packages/q8s/pyproject.toml +++ b/packages/q8s/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "q8s" -version = "0.13.0" +version = "0.14.0" description = "Kernel extension for executing quantum programs in simulators on q8s clusters" authors = [{ name = "Vlad Stirbu", email = "vstirbu@gmail.com" }] readme = "README.md" @@ -14,7 +14,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", - "License :: OSI Approved :: MIT License", + "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", ] dependencies = [ From 8f1fc89b112e493320edac9c31738c94be6ccb61 Mon Sep 17 00:00:00 2001 From: Vlad Stirbu Date: Wed, 1 Jul 2026 13:54:33 +0300 Subject: [PATCH 13/13] fixed pre-commit issues --- .github/workflows/publish.yml | 2 +- packages/q8s-cuda/README.md | 2 +- packages/q8s-cuda/src/q8s_cuda/cuda_job.py | 8 ++---- packages/q8s-cuda/tests/test_cuda_job.py | 5 ++-- packages/q8s-cuda/tests/test_cuda_target.py | 6 ++--- packages/q8s-hpc-cpu/README.md | 2 +- packages/q8s-hpc-cpu/pyproject.toml | 2 +- packages/q8s-hpc-cpu/src/q8s_hpc/hpc_job.py | 24 +++++++---------- packages/q8s-hpc-rocm/pyproject.toml | 2 +- .../src/q8s_hpc_rocm/hpc_rocm_job.py | 26 +++++++------------ packages/q8s/pyproject.toml | 2 +- packages/q8s/src/q8s/cli.py | 10 +++---- packages/q8s/src/q8s/enums.py | 2 +- packages/q8s/src/q8s/execution.py | 18 ++++--------- packages/q8s/src/q8s/kernel.py | 6 ++--- packages/q8s/src/q8s/plugins/cpu_job.py | 10 +++---- packages/q8s/src/q8s/plugins/job.py | 1 - packages/q8s/src/q8s/project.py | 14 +++++----- packages/q8s/src/q8s/targets.py | 2 +- packages/q8s/src/q8s/utils.py | 4 +-- packages/q8s/tests/fixtures/simple/qiskit.py | 5 ++-- packages/q8s/tests/test_backend.py | 1 - packages/q8s/tests/test_bakefile.py | 6 ++--- packages/q8s/tests/test_cpu_job.py | 5 ---- packages/q8s/tests/test_execution.py | 6 +++-- packages/q8s/tests/test_git_info.py | 1 - packages/q8s/tests/test_job_plugin.py | 1 - packages/q8s/tests/test_kernel.py | 1 - packages/q8s/tests/test_project.py | 11 +++----- 29 files changed, 68 insertions(+), 117 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index bdb80df..bf8148f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -35,7 +35,7 @@ jobs: - name: Install q8s-cuda working-directory: packages/q8s-cuda run: pip install . - + # - name: Install dependencies # run: >- # python3 -m diff --git a/packages/q8s-cuda/README.md b/packages/q8s-cuda/README.md index 476e69c..edb8506 100644 --- a/packages/q8s-cuda/README.md +++ b/packages/q8s-cuda/README.md @@ -16,4 +16,4 @@ Submit a job using the `gpu` execution target: ```bash q8sctl execute app.py --target gpu --kubeconfig /path/to/kubeconfig -``` \ No newline at end of file +``` diff --git a/packages/q8s-cuda/src/q8s_cuda/cuda_job.py b/packages/q8s-cuda/src/q8s_cuda/cuda_job.py index 530a0b5..e447021 100644 --- a/packages/q8s-cuda/src/q8s_cuda/cuda_job.py +++ b/packages/q8s-cuda/src/q8s_cuda/cuda_job.py @@ -1,15 +1,13 @@ import os import re +from importlib.metadata import version from kubernetes import client - from q8s.constants import WORKSPACE from q8s.plugins.job import JobPlugin from q8s.plugins.job_template_spec import hookimpl from q8s.workload import Workload -from importlib.metadata import version - _PYTHON_VERSION_RE = re.compile(r"^3(\.\d+){1,2}$") MEMORY = os.environ.get("MEMORY", "32Gi") @@ -24,9 +22,7 @@ class CUDAJobTemplatePlugin(JobPlugin): def get_base_image(self, python_version: str) -> str: if not _PYTHON_VERSION_RE.fullmatch(python_version): - raise ValueError( - f"Invalid python_version format: {python_version!r}" - ) + raise ValueError(f"Invalid python_version format: {python_version!r}") return f"ghcr.io/qubernetes-dev/cuda:12.8.1-r2-py{python_version}" diff --git a/packages/q8s-cuda/tests/test_cuda_job.py b/packages/q8s-cuda/tests/test_cuda_job.py index df20a91..417ec2d 100644 --- a/packages/q8s-cuda/tests/test_cuda_job.py +++ b/packages/q8s-cuda/tests/test_cuda_job.py @@ -2,7 +2,6 @@ from unittest.mock import MagicMock, patch from kubernetes import client - from q8s_cuda.cuda_job import CUDAJobTemplatePlugin @@ -17,7 +16,6 @@ class TestCUDAJobTemplatePlugin(unittest.TestCase): @patch("q8s.plugins.job_template_spec.client.V1ConfigMapVolumeSource") @patch("q8s.plugins.job_template_spec.client.V1LocalObjectReference") @patch("q8s.plugins.job_template_spec.client.V1ResourceRequirements") - def test_makejob_gpu( self, mock_v1_resource_requirements, @@ -90,5 +88,6 @@ def test_get_base_image_invalid_version_raises(self): with self.assertRaises(ValueError): plugin.get_base_image("invalid") + if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() diff --git a/packages/q8s-cuda/tests/test_cuda_target.py b/packages/q8s-cuda/tests/test_cuda_target.py index 43e0df2..464b57b 100644 --- a/packages/q8s-cuda/tests/test_cuda_target.py +++ b/packages/q8s-cuda/tests/test_cuda_target.py @@ -4,7 +4,6 @@ from unittest.mock import Mock from dockerfile_parse import DockerfileParser - from q8s.constants import WORKSPACE from q8s.plugins.utils.git_info import GitInfo from q8s.project import Project @@ -21,6 +20,7 @@ "kubeconfig": "tests/fixtures/cache/kubeconfig", } + class TestProject(unittest.TestCase): @unittest.mock.patch("q8s.project.datetime") @unittest.mock.patch("q8s.project.load") @@ -81,7 +81,7 @@ def test_create_bakefile_no_repo(self, mock_load: Mock, mock_get_git_info: Mock) "gpu", bakefile_data["group"]["default"]["targets"], ) - + self.assertEqual( bakefile_data["target"]["gpu"], { @@ -123,4 +123,4 @@ def test_create_bakefile_in_repo(self, mock_load: Mock, mock_get_git_info: Mock) "tags": ["vstirbu/q8s-example:gpu-main"], "platforms": ["linux/amd64"], }, - ) \ No newline at end of file + ) diff --git a/packages/q8s-hpc-cpu/README.md b/packages/q8s-hpc-cpu/README.md index 088bd41..a769419 100644 --- a/packages/q8s-hpc-cpu/README.md +++ b/packages/q8s-hpc-cpu/README.md @@ -16,4 +16,4 @@ Submit a job using the `hpc-cpu` execution target: ```bash q8sctl execute app.py --target hpc-cpu --kubeconfig /path/to/kubeconfig -``` \ No newline at end of file +``` diff --git a/packages/q8s-hpc-cpu/pyproject.toml b/packages/q8s-hpc-cpu/pyproject.toml index 783f878..dd35f0e 100644 --- a/packages/q8s-hpc-cpu/pyproject.toml +++ b/packages/q8s-hpc-cpu/pyproject.toml @@ -31,4 +31,4 @@ build-backend = "setuptools.build_meta" where = ["src"] [project.entry-points."q8s.targets"] -hpc-cpu = "q8s_hpc.hpc_job:HPCJobTemplatePlugin" \ No newline at end of file +hpc-cpu = "q8s_hpc.hpc_job:HPCJobTemplatePlugin" diff --git a/packages/q8s-hpc-cpu/src/q8s_hpc/hpc_job.py b/packages/q8s-hpc-cpu/src/q8s_hpc/hpc_job.py index 4440f72..18a4b16 100644 --- a/packages/q8s-hpc-cpu/src/q8s_hpc/hpc_job.py +++ b/packages/q8s-hpc-cpu/src/q8s_hpc/hpc_job.py @@ -1,16 +1,15 @@ from __future__ import annotations import shlex +from importlib.metadata import version from pathlib import Path from kubernetes import client - from q8s.constants import WORKSPACE from q8s.plugins.job import JobPlugin from q8s.plugins.job_template_spec import hookimpl from q8s.workload import Workload -from importlib.metadata import version def _get_workload_extra(workload: Workload) -> dict: """ @@ -22,7 +21,7 @@ def _get_workload_extra(workload: Workload) -> dict: def _find_project_root(start: Path) -> Path: """ - Tries to find Q8Sproject file from current directory or a parent directory, + Tries to find Q8Sproject file from current directory or a parent directory, in order to locate the root directory of the project. """ current = start.resolve() @@ -35,7 +34,7 @@ def _find_project_root(start: Path) -> Path: def resolve_hpc_config_path(workload: Workload) -> Path: """ Tries to find all necessary slurm settings by trying to - find HpcConfig file in the project root directory, and checking if + find HpcConfig file in the project root directory, and checking if the workload already includes a path to it. """ extra = _get_workload_extra(workload) @@ -70,13 +69,13 @@ def _parse_config_line(line: str, line_number: int) -> tuple[str, str] | None: first = parts[0] if not first.startswith("--"): raise ValueError( - f"Invalid HPC config line {line_number}: expected a flag starting with '--', got: {line!r}" + f"Invalid HPC config line {line_number}: expected a flag starting with '--', got: {line!r}" # noqa: E501 ) if "=" in first: if len(parts) != 1: raise ValueError( - f"Invalid HPC config line {line_number}: do not mix '--flag=value' with extra tokens: {line!r}" + f"Invalid HPC config line {line_number}: do not mix '--flag=value' with extra tokens: {line!r}" # noqa: E501 ) flag, value = first.split("=", 1) else: @@ -201,9 +200,7 @@ def makejob( ], ) - annotations = { - "slurm-job.vk.io/flags": " ".join(hpc_config["slurm_flags"]) - } + annotations = {"slurm-job.vk.io/flags": " ".join(hpc_config["slurm_flags"])} template = client.V1PodTemplateSpec( metadata=client.V1ObjectMeta( @@ -215,13 +212,10 @@ def makejob( ), spec=client.V1PodSpec( containers=[container], - node_selector={ - "kubernetes.io/hostname": hpc_config["q8s_node"] - }, + node_selector={"kubernetes.io/hostname": hpc_config["q8s_node"]}, tolerations=[ client.V1Toleration( - key="virtual-node.interlink/no-schedule", - operator="Exists" + key="virtual-node.interlink/no-schedule", operator="Exists" ) ], image_pull_secrets=( @@ -249,4 +243,4 @@ def makejob( ), ) - return template \ No newline at end of file + return template diff --git a/packages/q8s-hpc-rocm/pyproject.toml b/packages/q8s-hpc-rocm/pyproject.toml index 3be3bb6..009837f 100644 --- a/packages/q8s-hpc-rocm/pyproject.toml +++ b/packages/q8s-hpc-rocm/pyproject.toml @@ -31,4 +31,4 @@ build-backend = "setuptools.build_meta" where = ["src"] [project.entry-points."q8s.targets"] -hpc-rocm = "q8s_hpc_rocm.hpc_rocm_job:HpcRocmJobTemplatePlugin" \ No newline at end of file +hpc-rocm = "q8s_hpc_rocm.hpc_rocm_job:HpcRocmJobTemplatePlugin" diff --git a/packages/q8s-hpc-rocm/src/q8s_hpc_rocm/hpc_rocm_job.py b/packages/q8s-hpc-rocm/src/q8s_hpc_rocm/hpc_rocm_job.py index 5fe287a..09a0a98 100644 --- a/packages/q8s-hpc-rocm/src/q8s_hpc_rocm/hpc_rocm_job.py +++ b/packages/q8s-hpc-rocm/src/q8s_hpc_rocm/hpc_rocm_job.py @@ -1,17 +1,17 @@ from __future__ import annotations import shlex +from importlib.metadata import version from pathlib import Path from kubernetes import client - from q8s.constants import WORKSPACE + # from q8s.enums import Target from q8s.plugins.job import JobPlugin from q8s.plugins.job_template_spec import hookimpl from q8s.workload import Workload -from importlib.metadata import version def _get_workload_extra(workload: Workload) -> dict: """ @@ -23,7 +23,7 @@ def _get_workload_extra(workload: Workload) -> dict: def _find_project_root(start: Path) -> Path: """ - Tries to find Q8Sproject file from current directory or a parent directory, + Tries to find Q8Sproject file from current directory or a parent directory, in order to locate the root directory of the project. """ current = start.resolve() @@ -36,7 +36,7 @@ def _find_project_root(start: Path) -> Path: def resolve_hpc_config_path(workload: Workload) -> Path: """ Tries to find all necessary slurm settings by trying to - find HpcConfig file in the project root directory, and checking if + find HpcConfig file in the project root directory, and checking if the workload already includes a path to it. """ extra = _get_workload_extra(workload) @@ -71,13 +71,13 @@ def _parse_config_line(line: str, line_number: int) -> tuple[str, str] | None: first = parts[0] if not first.startswith("--"): raise ValueError( - f"Invalid HPC config line {line_number}: expected a flag starting with '--', got: {line!r}" + f"Invalid HPC config line {line_number}: expected a flag starting with '--', got: {line!r}" # noqa: E501 ) if "=" in first: if len(parts) != 1: raise ValueError( - f"Invalid HPC config line {line_number}: do not mix '--flag=value' with extra tokens: {line!r}" + f"Invalid HPC config line {line_number}: do not mix '--flag=value' with extra tokens: {line!r}" # noqa: E501 ) flag, value = first.split("=", 1) else: @@ -165,7 +165,6 @@ def makejob( if workload.is_src_project: env_var.append(client.V1EnvVar(name="PYTHONPATH", value=f"{WORKSPACE}/src")) - env_var.append( client.V1EnvVar( name="Q8S_VERSION", @@ -202,9 +201,7 @@ def makejob( ], ) - annotations = { - "slurm-job.vk.io/flags": " ".join(hpc_config["slurm_flags"]) - } + annotations = {"slurm-job.vk.io/flags": " ".join(hpc_config["slurm_flags"])} template = client.V1PodTemplateSpec( metadata=client.V1ObjectMeta( @@ -216,13 +213,10 @@ def makejob( ), spec=client.V1PodSpec( containers=[container], - node_selector={ - "kubernetes.io/hostname": hpc_config["q8s_node"] - }, + node_selector={"kubernetes.io/hostname": hpc_config["q8s_node"]}, tolerations=[ client.V1Toleration( - key="virtual-node.interlink/no-schedule", - operator="Exists" + key="virtual-node.interlink/no-schedule", operator="Exists" ) ], image_pull_secrets=( @@ -250,4 +244,4 @@ def makejob( ), ) - return template \ No newline at end of file + return template diff --git a/packages/q8s/pyproject.toml b/packages/q8s/pyproject.toml index bc14606..7977aa3 100644 --- a/packages/q8s/pyproject.toml +++ b/packages/q8s/pyproject.toml @@ -63,4 +63,4 @@ python_files = ["test_*.py"] addopts = "--cov=q8s --cov-report=term-missing" [project.entry-points."q8s.targets"] -cpu = "q8s.plugins.cpu_job:CPUJobTemplatePlugin" \ No newline at end of file +cpu = "q8s.plugins.cpu_job:CPUJobTemplatePlugin" diff --git a/packages/q8s/src/q8s/cli.py b/packages/q8s/src/q8s/cli.py index 5335346..5b5e23e 100644 --- a/packages/q8s/src/q8s/cli.py +++ b/packages/q8s/src/q8s/cli.py @@ -4,16 +4,14 @@ from subprocess import Popen import typer -from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn -from typing_extensions import Annotated - from q8s.execution import K8sContext from q8s.install import install_my_kernel_spec from q8s.project import Project +from q8s.targets import get_available_targets from q8s.utils import get_docker_image, get_kubeconfig from q8s.workload import Workload - -from q8s.targets import get_available_targets +from rich.progress import Progress, SpinnerColumn, TextColumn, TimeElapsedColumn +from typing_extensions import Annotated app = typer.Typer() @@ -140,7 +138,7 @@ def execute( Path | None, typer.Option( "--hpc-config", - help="Path to HPC config file. If omitted, q8s will look for 'HpcConfig' next to 'Q8Sproject'.", + help="Path to HPC config file. If omitted, q8s will look for 'HpcConfig' next to 'Q8Sproject'.", # noqa: E501 ), ] = None, args: Annotated[list[str], typer.Argument(help="Additional arguments")] = None, diff --git a/packages/q8s/src/q8s/enums.py b/packages/q8s/src/q8s/enums.py index f485bc4..1568915 100644 --- a/packages/q8s/src/q8s/enums.py +++ b/packages/q8s/src/q8s/enums.py @@ -6,4 +6,4 @@ # gpu = "gpu" # qpu = "qpu" # hpc = "hpc" -# hpc_rocm = "hpc_rocm" \ No newline at end of file +# hpc_rocm = "hpc_rocm" diff --git a/packages/q8s/src/q8s/execution.py b/packages/q8s/src/q8s/execution.py index e885336..7e17260 100644 --- a/packages/q8s/src/q8s/execution.py +++ b/packages/q8s/src/q8s/execution.py @@ -2,6 +2,7 @@ import logging import random import string +from importlib.metadata import entry_points, version from json import JSONEncoder, loads import pluggy @@ -10,15 +11,12 @@ from dxf import DXF from dxf.exceptions import DXFUnauthorizedError from kubernetes import client, config, watch -from rich.progress import Progress, TaskID - from q8s.plugins.job_template_spec import JobTemplatePluginSpec from q8s.utils import extract_non_none_value +from rich.progress import Progress, TaskID from .workload import Workload -from importlib.metadata import entry_points, version - def load_env(): env = dotenv_values(".env.q8s") @@ -157,12 +155,9 @@ def set_registry_pat(self, pat: str): def set_target(self, target): self.target = target - def available_targets(self): return [ - p.target_name - for p in self.jm.get_plugins() - if hasattr(p, "target_name") + p.target_name for p in self.jm.get_plugins() if hasattr(p, "target_name") ] def create_job_object(self, code: str) -> client.V1Job: @@ -211,10 +206,7 @@ def __create_job_object_from_workload(self, workload: Workload) -> client.V1Job: plugin_cls = ep.load() plugin = plugin_cls() - if ( - hasattr(plugin, "target_name") - and plugin.target_name == self.target - ): + if hasattr(plugin, "target_name") and plugin.target_name == self.target: plugin_version = ep.dist.version break @@ -559,4 +551,4 @@ def _load_plugins(self): for ep in entry_points(group="q8s.targets"): plugin_cls = ep.load() plugin = plugin_cls() - self.jm.register(plugin) \ No newline at end of file + self.jm.register(plugin) diff --git a/packages/q8s/src/q8s/kernel.py b/packages/q8s/src/q8s/kernel.py index a3926ea..3e4cdee 100644 --- a/packages/q8s/src/q8s/kernel.py +++ b/packages/q8s/src/q8s/kernel.py @@ -3,13 +3,11 @@ from ipykernel.comm import CommManager from ipykernel.kernelbase import Kernel -from rich.progress import Progress, SpinnerColumn, TextColumn - from q8s.execution import K8sContext from q8s.project import Project -from q8s.workload import Workload - from q8s.targets import get_available_targets +from q8s.workload import Workload +from rich.progress import Progress, SpinnerColumn, TextColumn FORMAT = "[%(levelname)s %(asctime)-15s q8s_kernel] %(message)s" logging.basicConfig(level=logging.INFO, format=FORMAT) diff --git a/packages/q8s/src/q8s/plugins/cpu_job.py b/packages/q8s/src/q8s/plugins/cpu_job.py index 14a022d..e37d193 100644 --- a/packages/q8s/src/q8s/plugins/cpu_job.py +++ b/packages/q8s/src/q8s/plugins/cpu_job.py @@ -1,8 +1,7 @@ -from importlib.metadata import version import re +from importlib.metadata import version from kubernetes import client - from q8s.constants import WORKSPACE from q8s.plugins.job import JobPlugin from q8s.plugins.job_template_spec import hookimpl @@ -10,14 +9,13 @@ _PYTHON_VERSION_RE = re.compile(r"^3(\.\d+){1,2}$") + class CPUJobTemplatePlugin(JobPlugin): target_name = "cpu" - + def get_base_image(self, python_version: str) -> str: if not _PYTHON_VERSION_RE.fullmatch(python_version): - raise ValueError( - f"Invalid python_version format: {python_version!r}" - ) + raise ValueError(f"Invalid python_version format: {python_version!r}") return f"python:{python_version}-slim" diff --git a/packages/q8s/src/q8s/plugins/job.py b/packages/q8s/src/q8s/plugins/job.py index 496cb0b..ef78beb 100644 --- a/packages/q8s/src/q8s/plugins/job.py +++ b/packages/q8s/src/q8s/plugins/job.py @@ -1,7 +1,6 @@ import os from kubernetes import client - from q8s.plugins.utils.git_info import get_git_info diff --git a/packages/q8s/src/q8s/project.py b/packages/q8s/src/q8s/project.py index a6f5caa..05b414c 100644 --- a/packages/q8s/src/q8s/project.py +++ b/packages/q8s/src/q8s/project.py @@ -4,6 +4,7 @@ import sys from dataclasses import asdict, dataclass from datetime import datetime +from importlib.metadata import entry_points from io import StringIO from os.path import join from pathlib import Path @@ -12,15 +13,11 @@ import yaml from dacite import from_dict -from rich.progress import Progress - from q8s.bakefile import Bakefile, BuildPlatform from q8s.constants import WORKSPACE from q8s.plugins.utils.git_info import get_git_info - from q8s.targets import get_available_targets -from importlib.metadata import entry_points - +from rich.progress import Progress def load(path: str): @@ -212,7 +209,7 @@ def cached_images(self, target: str) -> str: key = target return images[key] - + def _get_plugin_for_target(self, target: str): for ep in entry_points(group="q8s.targets"): try: @@ -227,7 +224,6 @@ def _get_plugin_for_target(self, target: str): print(f"Failed to load plugin {ep.name}: {e}") return None - def build_container( self, target: str, progress: Progress, silent: bool, push: bool = True @@ -463,7 +459,9 @@ def __create_dockerfile(self, target: str, f): if plugin is None: if target == "qpu": - base_image = f"python:{self.configuration.python_env.python_version}-slim" + base_image = ( + f"python:{self.configuration.python_env.python_version}-slim" + ) else: raise ValueError( f"No plugin found for target '{target}'. " diff --git a/packages/q8s/src/q8s/targets.py b/packages/q8s/src/q8s/targets.py index d58c0f0..960fe6b 100644 --- a/packages/q8s/src/q8s/targets.py +++ b/packages/q8s/src/q8s/targets.py @@ -15,4 +15,4 @@ def get_available_targets(): except Exception as e: print(f"Failed to load plugin {ep.name}: {e}") - return available \ No newline at end of file + return available diff --git a/packages/q8s/src/q8s/utils.py b/packages/q8s/src/q8s/utils.py index 44d016b..b3c81e7 100644 --- a/packages/q8s/src/q8s/utils.py +++ b/packages/q8s/src/q8s/utils.py @@ -2,8 +2,6 @@ from q8s.project import CacheNotBuiltException, Project, ProjectNotFoundException -from importlib.metadata import entry_points - def extract_non_none_value(arr): """ @@ -47,4 +45,4 @@ def get_kubeconfig(kubeconfig=None): project = Project() return project.kubeconfig except ProjectNotFoundException: - return None \ No newline at end of file + return None diff --git a/packages/q8s/tests/fixtures/simple/qiskit.py b/packages/q8s/tests/fixtures/simple/qiskit.py index 5104c7c..b1e77fb 100644 --- a/packages/q8s/tests/fixtures/simple/qiskit.py +++ b/packages/q8s/tests/fixtures/simple/qiskit.py @@ -1,7 +1,8 @@ -from qiskit import QuantumCircuit, transpile -from qiskit_aer import Aer, AerSimulator, AerError import os +from qiskit import QuantumCircuit, transpile +from qiskit_aer import AerSimulator + def demo_function(shotsAmount=1000): simulator = AerSimulator(method="statevector", device="GPU") diff --git a/packages/q8s/tests/test_backend.py b/packages/q8s/tests/test_backend.py index 0a8d57a..4c79140 100644 --- a/packages/q8s/tests/test_backend.py +++ b/packages/q8s/tests/test_backend.py @@ -2,7 +2,6 @@ from unittest.mock import patch import matplotlib.pyplot as plt - from q8s.matplotlib.backend import Q8SLoggerBackend diff --git a/packages/q8s/tests/test_bakefile.py b/packages/q8s/tests/test_bakefile.py index 0a16ce5..b30a528 100644 --- a/packages/q8s/tests/test_bakefile.py +++ b/packages/q8s/tests/test_bakefile.py @@ -38,14 +38,12 @@ def test_add_multiple_targets(self): ) self.assertEqual(bakefile.group.default.targets, ["cpu", "gpu"]) - self.assertSetEqual( - set(bakefile.target.keys()), {"cpu", "gpu"} - ) + self.assertSetEqual(set(bakefile.target.keys()), {"cpu", "gpu"}) gpu_target = bakefile.target["gpu"] self.assertEqual(gpu_target.context, "./gpu") self.assertEqual(gpu_target.platforms, [BuildPlatform.linux_arm64]) - + def test_invalid_platform_raises(self): bakefile = Bakefile() diff --git a/packages/q8s/tests/test_cpu_job.py b/packages/q8s/tests/test_cpu_job.py index d1d9ebc..8cdffa6 100644 --- a/packages/q8s/tests/test_cpu_job.py +++ b/packages/q8s/tests/test_cpu_job.py @@ -2,7 +2,6 @@ from unittest.mock import MagicMock, patch from kubernetes import client - from q8s.plugins.cpu_job import CPUJobTemplatePlugin @@ -15,7 +14,6 @@ class TestCPUJobTemplatePlugin(unittest.TestCase): @patch("q8s.plugins.cpu_job.client.V1VolumeMount") @patch("q8s.plugins.cpu_job.client.V1Volume") @patch("q8s.plugins.cpu_job.client.V1ConfigMapVolumeSource") - def test_makejob_cpu( self, mock_v1_config_map_volume_source, @@ -59,7 +57,6 @@ def test_makejob_cpu( mock_v1_volume.assert_called_once() mock_v1_config_map_volume_source.assert_called_once() - def test_makejob_invalid_target(self): plugin = CPUJobTemplatePlugin() name = "test-job" @@ -87,7 +84,6 @@ def test_makejob_invalid_target(self): self.assertIsNone(result) - def test_get_base_image_default(self): plugin = CPUJobTemplatePlugin() @@ -111,6 +107,5 @@ def test_get_base_image_invalid_version_raises(self): plugin.get_base_image("invalid") - if __name__ == "__main__": unittest.main() diff --git a/packages/q8s/tests/test_execution.py b/packages/q8s/tests/test_execution.py index c0ca5be..4298370 100644 --- a/packages/q8s/tests/test_execution.py +++ b/packages/q8s/tests/test_execution.py @@ -4,7 +4,6 @@ import requests from dxf.exceptions import DXFUnauthorizedError - from q8s.execution import ContainerImageValidator, K8sContext from q8s.workload import Workload @@ -313,12 +312,15 @@ def test_create_job_object_sets_plugin_version_label(self, mock_entry_points): ctx._K8sContext__create_job_object_from_workload(workload) - created_spec = ctx.batch_api_instance.create_namespaced_job.call_args.kwargs["body"] + created_spec = ctx.batch_api_instance.create_namespaced_job.call_args.kwargs[ + "body" + ] self.assertEqual( created_spec.metadata.labels["qubernetes.dev/plugin-version"], "1.2.3", ) + if __name__ == "__main__": unittest.main() diff --git a/packages/q8s/tests/test_git_info.py b/packages/q8s/tests/test_git_info.py index d254366..004b237 100644 --- a/packages/q8s/tests/test_git_info.py +++ b/packages/q8s/tests/test_git_info.py @@ -4,7 +4,6 @@ from pathlib import Path from git import Actor, Repo - from q8s.plugins.utils.git_info import ( GitExtraInfo, GitExtraReason, diff --git a/packages/q8s/tests/test_job_plugin.py b/packages/q8s/tests/test_job_plugin.py index fca200b..ea02d75 100644 --- a/packages/q8s/tests/test_job_plugin.py +++ b/packages/q8s/tests/test_job_plugin.py @@ -2,7 +2,6 @@ from unittest.mock import patch from kubernetes import client - from q8s.plugins.job import JobPlugin from q8s.plugins.utils.git_info import GitInfo diff --git a/packages/q8s/tests/test_kernel.py b/packages/q8s/tests/test_kernel.py index 2c81577..7f66fd5 100644 --- a/packages/q8s/tests/test_kernel.py +++ b/packages/q8s/tests/test_kernel.py @@ -2,7 +2,6 @@ from unittest.mock import MagicMock, patch import pytest - from q8s.kernel import Q8sKernel, kernel_comm_identifier diff --git a/packages/q8s/tests/test_project.py b/packages/q8s/tests/test_project.py index fca0f1b..ab233a0 100644 --- a/packages/q8s/tests/test_project.py +++ b/packages/q8s/tests/test_project.py @@ -5,19 +5,16 @@ from unittest.mock import Mock from dockerfile_parse import DockerfileParser -from rich.progress import Progress, SpinnerColumn - from q8s.constants import WORKSPACE +from q8s.plugins.cpu_job import CPUJobTemplatePlugin from q8s.plugins.utils.git_info import GitInfo from q8s.project import Project -from q8s.plugins.cpu_job import CPUJobTemplatePlugin +from rich.progress import Progress, SpinnerColumn mocked_configuration = { "name": "Example", "python_env": {"dependencies": ["qiskit==1.1.0"]}, - "targets": { - "cpu": {"python_env": {"dependencies": ["qiskit-aer==0.15.1"]}} - }, + "targets": {"cpu": {"python_env": {"dependencies": ["qiskit-aer==0.15.1"]}}}, "docker": {"username": "vstirbu", "registry": None}, "kubeconfig": "packages/q8s/tests/fixtures/cache/kubeconfig", } @@ -34,7 +31,6 @@ def _completed_process(): class TestProject(unittest.TestCase): - @unittest.mock.patch("q8s.project.load") def test_init(self, mock_load: Mock): mock_load.return_value = mocked_configuration @@ -130,7 +126,6 @@ def test_create_dockerfile_cpu(self, mock_load: Mock, mock_datetime: Mock): @unittest.mock.patch("q8s.project.get_git_info") @unittest.mock.patch("q8s.project.load") - def test_create_bakefile_no_repo(self, mock_load: Mock, mock_get_git_info: Mock): mock_load.return_value = mocked_configuration mock_get_git_info.return_value = GitInfo(