TMA Alignment Pipeline

.github/workflows/ci.yml

@@ -8,41 +8,45 @@ on:
   workflow_dispatch:
 
 jobs:
+  lint:
+    name: "Black linting"
+    runs-on: "ubuntu-latest"
+    steps:
+     - uses: actions/checkout@v4
+
+     - name: Install the latest version of uv
+       uses: astral-sh/setup-uv@v5
+       with:
+        python-version: "3.12"
+
+     - name: Install the project
+       run: uv sync --extra dl
+
+     - name: Run black check
+       run: uv run black --check .
+
+     - name: Run ruff check
+       run: uv run ruff check src/valis tests examples
+
   tests:
-    name: "Python ${{ matrix.python-version }}"
+    name: "Python 3.13"
     runs-on: "ubuntu-latest"
-    env:
-      USING_COVERAGE: '3.9, 3.10, 3.11, 3.12, 3.13'
-
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version:
-          - "3.9"
-          - "3.10"
-          - "3.11"
-          - "3.12"
-          - "3.13"
-        os:
-          - "Ubuntu"
-          - "Windows"
-          - "macOS"
 
     steps:
      - uses: actions/checkout@v4
 
      - name: Install the latest version of uv and set the python version
        uses: astral-sh/setup-uv@v5
        with:
-        python-version: ${{ matrix.python-version }}
+        python-version: "3.13"
 
      - name: Install libvips
        run: |
           sudo apt-get update
           sudo apt-get install --no-install-recommends libvips
 
      - name: Install the project
-       run: uv sync
+       run: uv sync --extra dl
 
      - name: Run tests
        run: uv run pytest

.github/workflows/containers.yml

@@ -0,0 +1,31 @@
+name: Container Builds
+on:
+  push:
+    branches:
+      - 'main'
+jobs:
+  build-docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Build and push container
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: Dockerfile
+          push: true
+          tags: |
+            jeffquinnmsk/valis:latest
+            jeffquinnmsk/valis:${{ github.sha }}
+          cache-from: type=registry,ref=jeffquinnmsk/valis:latest
+          cache-to: type=inline

.gitignore

@@ -238,3 +238,6 @@ tests/test_tiler.py
 tests/test_warp_assoc_imgs.py
 
 tests/test_warp_fxns.py
+
+tests/test_output/
+tests/example_datasets/

.pre-commit-config.yaml

@@ -0,0 +1,24 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.2.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-toml
+      - id: sort-simple-yaml
+      - id: check-added-large-files
+      - id: check-merge-conflict
+  - repo: https://github.com/psf/black
+    rev: 24.1.1
+    hooks:
+      - id: black
+        # It is recommended to specify the latest version of Python
+        # supported by your project here, or alternatively use
+        # pre-commit's default_language_version, see
+        # https://pre-commit.com/#top_level-default_language_version
+        language_version: python3.11
+  - repo: https://github.com/conorfalvey/check_pdb_hook
+    rev: 0.0.9
+    hooks:
+      - id: check_pdb_hook

Dockerfile

@@ -1,114 +1,28 @@
-FROM ubuntu:noble AS builder
+FROM python:3.13-slim
 
-ARG WKDIR=/usr/local/src
-WORKDIR ${WKDIR}
+USER root
 
-ARG UV_VERSION=0.6.5
-ARG VIPS_VERSION=8.16.0
-ARG BF_VERSION=7.0.0
-ARG PYTORCH_VERSION=2.4.0
-ARG TORCHVISION_VERSION=0.20.1
-ARG OPENCV_VERSION=4.9.0.80
+RUN --mount=type=cache,target=/.cache/pip pip install --upgrade pip
+RUN --mount=type=cache,target=/.cache/pip pip install ipython ipdb memray
+RUN apt-get update && apt-get install --no-install-recommends -y \
+	libvips-tools \
+    libvips \
+    libvips-dev \
+    build-essential
 
-ENV PYTHONUNBUFFERED=1
-ENV PYTHONDONTWRITEBYTECODE=1
-
-# Get build dependencies
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update \
-	&& apt-get install -y \
-		build-essential \
-		software-properties-common \
-		ninja-build \
-		python3-pip \
-		bc \
-		wget \
-		ca-certificates \
-		git-all \
-		cmake \
-		libjxr-dev \
-    openjdk-11-jre
-
-# libvips dependencies for libvips build
-RUN apt-get install -y python3-venv
-RUN python3 -m venv ~/.local
-RUN echo $PATH
-ENV PATH="$PATH:~/.local/bin"
-RUN echo $PATH
-RUN ~/.local/bin/pip3 install meson
-
-RUN apt-get update
-RUN apt-get install --no-install-recommends -y \
-	libglib2.0-dev \
-	glib-2.0-dev \
-	libexpat1-dev \
-	libexpat-dev \
-	librsvg2-2 \
-	librsvg2-common \
-	librsvg2-dev \
-	libpng-dev \
-	libjpeg-turbo8-dev \
-	libopenjp2-7-dev \
-	libtiff-dev \
-	libexif-dev \
-	liblcms2-dev \
-	libheif-dev \
-	liborc-dev \
-  	libgirepository1.0-dev \
-	libopenslide-dev \
-	librsvg2-dev
-
-RUN update-ca-certificates
-### Install libvips from source to get latest version
-ENV LD_LIBRARY_PATH=/usr/local/lib
-ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig
+RUN mkdir -p /app
+COPY src/ /app/src/
 
-# build the head of the stable 8.14 branch
-ARG VIPS_URL=https://github.com/libvips/libvips/releases/download
-RUN wget ${VIPS_URL}/v${VIPS_VERSION}/vips-${VIPS_VERSION}.tar.xz --no-check-certificate \
-	&& tar xf vips-${VIPS_VERSION}.tar.xz \
-	&& cd vips-${VIPS_VERSION} \
-	&& ~/.local/bin/meson build --buildtype=release --libdir=lib \
-	&& cd build \
-	&& ninja \
-	&& ninja install
+COPY pyproject.toml setup.py LICENSE.txt README.rst /app/
 
-RUN rm vips-${VIPS_VERSION}.tar.xz
-RUN rm -r vips-${VIPS_VERSION}
+RUN --mount=type=cache,target=/.cache/pip cd /app && pip install '.[dev,test]'
 
-# Copy over necessary files
-COPY valis valis
-COPY pyproject.toml pyproject.toml
-COPY README.rst README.rst
-COPY LICENSE.txt LICENSE.txt
-COPY CITATION.cff CITATION.cff
-
-# Install python packages using UV
-ADD https://astral.sh/uv/install.sh /uv-installer.sh
-
-# Run the installer then remove it
-RUN sh /uv-installer.sh && rm /uv-installer.sh
-
-# Ensure the installed binary is on the `PATH`
-ENV PATH="/root/.local/bin/:$PATH"
-
-# RUN uv sync
-RUN uv pip install .
-# Set path to use .venv Python
-ENV PATH="${WKDIR}/.venv/bin:$PATH"
+ENV PYTHONUNBUFFERED=1
 
-# Install bioformats.jar in valis
-RUN wget https://downloads.openmicroscopy.org/bio-formats/${BF_VERSION}/artifacts/bioformats_package.jar -P valis
+WORKDIR /app
 
 # Download pytorch model weights
 COPY ./docker/docker_download_weights.py docker_download_weights.py
 RUN python3 docker_download_weights.py
 
-# Clean up
-RUN apt-get remove -y wget build-essential ninja-build && \
-  apt-get autoremove -y && \
-  apt-get autoclean && \
-  apt-get clean && \
-  rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
-  rm -rf /usr/local/lib/python*