From ffc9ac8cb789e2dbd92e78ea19e637ba68bd9e7e Mon Sep 17 00:00:00 2001 From: frederick Date: Wed, 22 Jul 2026 11:10:03 +0100 Subject: [PATCH] Adding workflow to use the example notebook in /e02/notebooks --- e02/schema/notebook_Schema.json | 36 ++++++++++ e02/schema/notebook_ui.json | 39 +++++++++++ e02/templates/Notebook.yaml | 120 ++++++++++++++++++++++++++++++++ 3 files changed, 195 insertions(+) create mode 100644 e02/schema/notebook_Schema.json create mode 100644 e02/schema/notebook_ui.json create mode 100644 e02/templates/Notebook.yaml diff --git a/e02/schema/notebook_Schema.json b/e02/schema/notebook_Schema.json new file mode 100644 index 0000000..5cb0732 --- /dev/null +++ b/e02/schema/notebook_Schema.json @@ -0,0 +1,36 @@ +{ + "type": "object", + "description": "inputs for ePSIC notebook which calibrates 4D STEM data", + "properties": { + "sample": { + "default": null, + "maxLength": 128, + "minLength": 1, + "title": "Sample name", + "type": "string" + }, + "timestamp": { + "default": null, + "maxLength": 1, + "minLength": 128, + "title": "Time stamp", + "type": "string" + }, + "thresh_lower": { + "default": 0.05, + "maximum": 0.95, + "minimum": 0, + "title": "The lower threshold", + "type": "number" + }, + "thresh_upper": { + "default": 0.2, + "maximum": 1, + "minimum": 0.05, + "title": "The upper threshold", + "type": "number" + } + }, + "title": "Calibration of 4D STEM data notebook" +} + diff --git a/e02/schema/notebook_ui.json b/e02/schema/notebook_ui.json new file mode 100644 index 0000000..24c882d --- /dev/null +++ b/e02/schema/notebook_ui.json @@ -0,0 +1,39 @@ +{ + "type": "VerticalLayout", + "options": { + "formWidth": "80%" + }, + "elements": [ + { + "type": "Group", + "label": "ePSIC notebook which calibrates 4D STEM data", + "elements": [ + { + "type": "VerticalLayout", + "elements": [ + { + "type": "Control", + "label": "Name of the sample", + "scope": "#/properties/sample" + }, + { + "type": "Control", + "label": "Time stamp of the data to process", + "scope": "#/properties/timestamp" + }, + { + "type": "Control", + "label": "The lower threshold", + "scope": "#/properties/thresh_lower" + }, + { + "type": "Control", + "label": "The upper threshold", + "scope": "#/properties/thresh_upper" + } + ] + } + ] + } + ] +} diff --git a/e02/templates/Notebook.yaml b/e02/templates/Notebook.yaml new file mode 100644 index 0000000..f5e0d02 --- /dev/null +++ b/e02/templates/Notebook.yaml @@ -0,0 +1,120 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ClusterWorkflowTemplate +metadata: + name: Calibrate4DSTEM + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" + annotations: + workflows.argoproj.io/title: Calibrate 4D STEM data + workflows.argoproj.io/description: | + A utility which takes in a 4D STEM data set and outs the approximate real space pixel size, measured camera length and scan rotation (based off dpc). + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" + workflows.diamond.ac.uk/parameter-schema: | + {{- .Files.Get "schema/Calibrate4DSTEMSchema.json" | nindent 6 }} + workflows.diamond.ac.uk/ui-schema: | + {{- .Files.Get "schema/Calibrate4DSTEMUISchema.json" | nindent 6 }} +spec: + entrypoint: mount-files + arguments: + parameters: + - name: visitdir + valueFrom: + configMapKeyRef: + name: sessionspaces + key: data_directory + volumeClaimTemplates: + - metadata: + name: tmp + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + storageClassName: netapp + + templates: + - name: calibration-notebook + steps: + - - name: copy-notebook + template: mount-files + - - name: run-notebook + template: notebook + + - name: mount-files + script: + image: debian:stable-slim + volumeMounts: + - name: tmp + mountPath: /tmp + - name: session + mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" + command: [bash] + source: | + OUTPUT="{{`{{workflow.parameters.visitdir}}`}}"/processing/workflows/ + mkdir -p $OUTPUT + echo "something!" + echo '{{ .Files.Get "notebooks/Ptycho_calibration.ipynb" | b64enc }}' | base64 -d > $OUTPUT/Ptycho_calibration.ipynb + + - name: notebook + inputs: + parameters: + - name: sample + value: "{{`{{workflow.parameters.sample}}`}}" + - name: timestamp + value: "{{`{{workflow.parameters.timestamp}}`}}" + - name: outpath + value: "{{`{{workflow.parameters.outpath}}`}}" + - name: thresh_lower + value: "{{`{{workflow.parameters.thresh_lower}}`}}" + - name: thresh_upper + value: "{{`{{workflow.parameters.thresh_upper}}`}}" + script: + image: gitlab.diamond.ac.uk:5050/scisoft/ptychography/dimtools/mib2x + volumeMounts: + - name: session + mountPath: "{{`{{ workflows.parameters.visitdir }}`}}" + - name: tmp + mountPath: /tmp + - name: software + mountPath: /dls_sw/e02/medipix_mask + command: [bash] + source: | + ERROR_TXT="raise error" + ERROR_STR="An error occured within the notebook. Please open the output file to check" + OUTPUT="{{`{{workflow.parameters.visitdir}}`}}"/processing/workflows/ + python -m papermill "{{`{{workflow.parameters.visitdir}}`}}"/processing/workflows/Ptycho_calibration.ipynb $OUTPUT/Calibrate-notebook.ipynb \ + -p visit "{{`{{ workflow.parameters.visitdir }}`}}" + -p sample "{{`{{ workflows.parameters.sample }}`}}" + -p timestamp "{{`{{workflows.parameters.timestamp}}`}}" + -p maskpath /software/29042024_12bitmask2.h5 + -p outpath "{{`{{workflows.parameters.outpath}}`}}" + -p lower "{{`{{workflows.parameters.thresh_lower}}`}}" + -p upper "{{`{{workflows.parameters.thresh_upper}}`}}" + > $OUTPUT/papermill.log 2>&1 + + python -m jupyter nbconvert $OUTPUT/Calibrate-notebook.ipynb \ + --to html + + outputs: + artifacts: + - name: phase-output + path: "{{`{{ inputs.parameters.outpath }}`}}/phase-output.html" + archive: + none: {} + - name: notebook-log + path: "{{`{{ inputs.parameters.outpath }}`}}/papermill.log" + archive: + none: {} + podSpecPatch: | + containers: + - name: main + resources: + requests: + cpu: 10 + memory: 30Gi + limits: + cpu: 10 + memory: 30Gi + + \ No newline at end of file