diff --git a/.vscode/settings.json b/.vscode/settings.json index e5ceca1..a7d493f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,7 +2,7 @@ "maxNumberOfProblems": 100, "outputLevel": "debug", "editor.formatOnSave": false, - "python.defaultInterpreterPath": "/opt/poetry-venvs/packtly-builder-tooling-4uYNFs1t-py3.13/bin/python3", + "python.defaultInterpreterPath": "/opt/venv/bin/python3", "python.testing.autoTestDiscoverOnSaveEnabled": true, "python.testing.pytestEnabled": true, "python.testing.cwd": "${workspaceFolder}", diff --git a/.vscode/tasks.json b/.vscode/tasks.json index ce2a13d..18d15b8 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -3,12 +3,15 @@ "tasks": [ { "label": "packtly_builder_tooling", - "type": "vscode-just", - "task": "just", + "type": "shell", + "command": "just", "args": [ "--justfile=${workspaceFolder}/packtly-builder/tooling/justfile", "${input:justRecipe}" ], + "options": { + "cwd": "${workspaceFolder}/packtly-builder/tooling" + }, "problemMatcher": [] } ], diff --git a/CHANGELOG.md b/CHANGELOG.md index e37f700..ad9e43b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,16 +16,24 @@ RegEx for release version from file r"^\#\# \[\d{1,}[.]\d{1,}[.]\d{1,}\] \- \d{4}\-\d{2}-\d{2}$" --> -## [1.2.0] - 2026-06-26 +## [1.2.0] - 2026-07-04 ### Added - Source package build - Introduced automatic creation of .orig tarballs from the working tree when pristine-tar is not available - Added DebSourceBuilder to manage and automate upstream source archive generation. - Add of multi architecture support (x86, arm64 and armhf) - Add support for full build mode and enhance package existence checks +- `deb_source`: cache parsed changelog; invalidate after `git checkout` in `reset_source_tree` ### Changed - Improved SIGINT / KeyboardInterrupt handling +- `debuild`: pass `-sa` flag to always include orig tarball in `.changes` for source/full builds +- `apt`: improve binary and source package existence detection with per-file `[EXISTS]`/`[MISSING]` logging +- `AptManager`: reorganise class into logical sections (repository, installation, existence checks, private helpers) +- `log`: add error logging for failed package upload + +### Fixed +- Source package upload no longer fails when `.orig.tar.*` is absent from `.changes` on non-first revisions ## [1.1.0] - 2026-06-11 diff --git a/Readme.md b/Readme.md index 1534168..6ea09a3 100644 --- a/Readme.md +++ b/Readme.md @@ -1,65 +1,107 @@ # packtly-builder -A containerized build system for creating, signing, and publishing Debian packages to an [Aptly](https://www.aptly.info/) repository. +A containerized build system for **building, signing, and publishing Debian packages** to an [Aptly](https://www.aptly.info/) repository — reproducibly inside Podman containers, with no build tooling required on the host. + +--- + +## Table of contents + +- [Overview](#overview) +- [How it works](#how-it-works) +- [Prerequisites](#prerequisites) +- [Quick start](#quick-start) +- [Container images](#container-images) +- [`just` targets](#just-targets) +- [The `packtly_builder_tooling` CLI](#the-packtly_builder_tooling-cli) +- [Signing keys & credentials](#signing-keys--credentials) +- [Development](#development) +- [Versioning & releases](#versioning--releases) +- [Related projects](#related-projects) + +--- ## Overview -packtly-builder provides: +packtly-builder bundles three things: -- **Container images** — a layered set of Podman images based on Debian Trixie for building and running Debian packages -- **`packtly_builder_tooling`** — a Python CLI that orchestrates the full build-sign-publish pipeline -- **CI/CD pipeline** — GitLab CI configuration covering image builds, tooling tests, deployment, and versioned releases +- **Container images** — a layered set of Podman images based on Debian Trixie that carry the full Debian packaging toolchain (`debhelper`, `devscripts`, `gnupg2`, `git-buildpackage`, …). +- **`packtly_builder_tooling`** — a Python CLI that orchestrates the build → sign → publish pipeline for a single source tree. +- **CI/CD** — a GitLab CI pipeline that builds the images, runs the tooling test suite, and cuts versioned, multi-arch releases. -### Container stages +## How it works -| Image | Purpose | -|---|---| -| `packtly-builder-base` | Debian Trixie with `debhelper`, `devscripts`, `gnupg2`, etc. | -| `packtly-builder-builder` | Base + Poetry, `just`, tooling venv — used for CI builds | -| `packtly-builder` (runtime) | Base + installed `packtly_builder_tooling` wheel | -| `packtly-builder-devcontainer` | Builder image configured for VS Code devcontainer use | +```mermaid +flowchart LR + A[Debian source tree] --> B[debuild] + B --> C[debsign
GPG signing] + C --> D{--upload?} + D -- yes --> E[Aptly repo] + D -- no --> F[Local artifacts] +``` + +The CLI builds the package with `debuild`, signs the resulting `.changes`/`.dsc` with a configured GPG key, and — when `--upload` is given and an Aptly host is reachable — publishes the artifacts, skipping anything already deployed upstream. ## Prerequisites +On the **host** you only need a container runtime and a task runner: + - [Podman](https://podman.io/) and [podman-compose](https://github.com/containers/podman-compose) - [just](https://github.com/casey/just) +- [yq](https://github.com/mikefarah/yq) (used by some `just` targets) + +Everything else (the Debian toolchain, Python, Poetry) lives inside the images. -## Getting started +## Quick start ```bash -# Build all images, run tests, and build the tooling wheel +# Build the builder image, run tests, build the wheel, build the runtime image just all -# Or step by step: +# …or step by step: just build-builder # Build the builder image -just test-tooling-keys # Verify GPG key setup +just test-tooling-keys # Verify the GPG key setup just test-tooling # Run the Python test suite -just build-tooling # Build the Python wheel +just build-tooling # Build the packtly_builder_tooling wheel just build-runtime # Build the final runtime image ``` -## Available targets +List every available target at any time: +```bash +just # equivalent to `just --list` ``` -just # List all targets -``` + +## Container images + +| Image | Built from | Purpose | +|---|---|---| +| `packtly-builder-base` | Debian Trixie | Debian packaging toolchain (`debhelper`, `devscripts`, `gnupg2`, …) | +| `packtly-builder-builder` | base | + Poetry, `just`, Node, and the tooling venv — used for CI builds & tests | +| `packtly-builder` (runtime) | base | + the installed `packtly_builder_tooling` wheel; entrypoint for real builds | +| `packtly-builder-devcontainer` | builder | Builder image wired up for VS Code Dev Containers | + +Container build targets accept an optional **architecture** argument (`amd64` by default, `arm64` also supported), e.g. `just build-builder arm64`. + +## `just` targets ### Build | Target | Description | |---|---| -| `build-base` | Build the base container image | -| `build-builder` | Build the builder container image | -| `build-runtime` | Build the runtime container image | -| `build-devcontainer` | Build the devcontainer image | +| `build-base [arch]` | Build the base image | +| `build-builder [arch]` | Build the builder image | +| `build-runtime [arch]` | Build the runtime image | +| `build-devcontainer [arch]` | Build the devcontainer image | +| `build-builder-multiarch` | Build the builder image for amd64 + arm64 and assemble a manifest | +| `build-runtime-multiarch` | Build the runtime image for amd64 + arm64 and assemble a manifest | | `build-tooling` | Build the `packtly_builder_tooling` Python wheel | -### test +### Test | Target | Description | |---|---| -| `test-tooling` | Run the full pytest suite inside the builder container | -| `test-tooling-keys` | Verify GPG key availability before tests | +| `test-tooling [arch]` | Run the full pytest suite inside the builder container | +| `test-tooling-keys [arch]` | Verify GPG key availability before tests | ### Clean @@ -70,54 +112,76 @@ just # List all targets | `clean-runtime` | Remove the runtime image | | `clean-devcontainer` | Remove the devcontainer image | | `clean-containers` | Remove all container images | -| `clean` | Remove all images and built wheel artifacts | +| `clean` | Remove all images **and** built wheel artifacts | -### Utilities +### Utilities & pipeline | Target | Description | |---|---| | `shell` | Open an interactive bash shell in the builder container | -| `all` | Full pipeline: build-builder → test → build-tooling → build-runtime | +| `all` | Full pipeline: `build-builder` → `test-tooling` → `build-tooling` → `build-runtime-multiarch` | -## `packtly_builder_tooling` CLI +## The `packtly_builder_tooling` CLI -The CLI is the runtime entrypoint of the final container image. It builds, signs, and optionally uploads a Debian package. +The CLI is the entrypoint of the runtime image. It builds, signs, and optionally uploads a single Debian package. ``` packtly_builder_tooling [options] - -Options: - --aptlyhost URL Aptly REST API base URL (or set APTLYHOST env var) - --dist NAME Aptly publish distribution (e.g. trixie-apollo) - --component NAME Aptly component (e.g. main) - --upload Upload the built package to Aptly after signing - --verbose Enable debug logging ``` -GPG signing keys are expected at: +| Option | Description | +|---|---| +| `builddir` | Path to the Debian build directory (positional, required) | +| `--build-mode {binary,source,full}` | What to build: `binary` (default), `source`, or `full` (source + binary) | +| `--no-build` | Skip the build step (sign/upload existing artifacts) | +| `--aptlyhost URL` | Aptly REST API base URL (falls back to the `APTLYHOST` env var) | +| `--dist NAME` | Aptly publish distribution (e.g. `trixie-apollo`) | +| `--component NAME` | Aptly component (e.g. `main`) | +| `--credentials-file PATH` | Aptly credentials file (default: `/run/secrets/aptly-credentials`) | +| `--upload` | Upload the built package to Aptly after signing | +| `--force-upload` | Upload even if the package already exists upstream | +| `--log-file PATH` | Also write log output to this file | +| `--verbose`, `-v` | Enable debug logging | + +### Build modes + +| Mode | `debuild` flag | Produces | +|---|---|---| +| `binary` | `-b` | Binary packages only (no `.orig` tarball required) | +| `source` | `-S` | Source package only | +| `full` | `-F` | Source **and** binary packages | + +## Signing keys & credentials + +GPG signing keys are read from these fixed paths inside the container: + ``` -/opt/keys/gpg/repo_signing.key -/opt/keys/gpg/repo_signing_private.key -/opt/keys/gpg/repo_signing_private_pass +/opt/keys/gpg/repo_signing.key # public key +/opt/keys/gpg/repo_signing_private.key # private key +/opt/keys/gpg/repo_signing_private_pass # passphrase ``` -## Related -- **[packtly-infra](https://github.com/packtly/packtly-infra)** — Infrastructure automation for deploying packtly: a self-hosted Debian package repository based on [aptly](https://www.aptly.info/) and [nginx](https://nginx.org/), running as a rootless Podman container managed by systemd Quadlets. - -## Versioning - -The release version is driven by `CHANGELOG.md` (Keep a Changelog format). The CI reads the latest `## [x.y.z]` entry and applies it as the container image label and Git tag at release time. +Locally, the `just` targets mount `Keys/gpg/` into `/opt/keys/gpg`, so place your keys there before running builds. Aptly credentials are read from a simple `key = value` file (default `/run/secrets/aptly-credentials`) containing `username` and `password`. ## Development -The repository includes a VS Code devcontainer configuration. Open the project in VS Code and select **Reopen in Container** to get a fully configured Debian build environment. +The repository ships a VS Code Dev Container configuration. Open the project in VS Code and choose **Reopen in Container** for a fully configured Debian build environment. -For local development without the devcontainer: +For local development on the tooling without a devcontainer: ```bash cd packtly-builder/tooling -just prepare # Install Poetry dependencies -just pytest # Run tests +just prepare # Install Poetry dependencies (incl. dev tools) +just test # flake8 + mypy + pytest +just pytest # Run tests only just mypy # Type-check just flake8 # Lint ``` + +## Versioning & releases + +The release version is driven by `CHANGELOG.md` (Keep a Changelog format). At release time, CI reads the latest `## [x.y.z]` entry and applies it as the container image label and Git tag. + +## Related projects + +- **[packtly-infra](https://github.com/packtly/packtly-infra)** — Infrastructure automation for deploying packtly: a self-hosted Debian package repository based on [Aptly](https://www.aptly.info/) and [nginx](https://nginx.org/), running as a rootless Podman container managed by systemd Quadlets. diff --git a/packtly-builder/containers/Containerfile b/packtly-builder/containers/Containerfile index 51160c2..40105ce 100644 --- a/packtly-builder/containers/Containerfile +++ b/packtly-builder/containers/Containerfile @@ -44,6 +44,7 @@ RUN apt-get update && \ python3 \ python3-pip \ python3-apt \ + python3-venv \ dumb-init \ locales \ ca-certificates \ @@ -55,6 +56,10 @@ RUN apt-get update && \ locale-gen && \ rm -rf /var/lib/apt/lists/* +# Optional CA certificates to be added to the system store +COPY ${CONTEXT_ROOT}/conf/ca-certificates/ /usr/local/share/ca-certificates/ +RUN update-ca-certificates + RUN mkdir -p /etc/gnupg && chmod 700 /etc/gnupg # GPG config @@ -69,11 +74,13 @@ COPY ${CONTEXT_ROOT}/conf/rtprio.conf /etc/security/limits.d/ FROM base AS build ARG CONTEXT_ROOT=. -# Use a virtualenv in the build stage so Poetry is isolated from system packages -# but include system site packages so Debian-only modules like python3-apt are importable. -ENV POETRY_VIRTUALENVS_CREATE=true \ - POETRY_VIRTUALENVS_OPTIONS_SYSTEM_SITE_PACKAGES=true \ - POETRY_VIRTUALENVS_PATH=/opt/poetry-venvs +# Fixed-path venv (survives the ephemeral build mount into the devcontainer) with +# --system-site-packages for python3-apt; Poetry installs into it (create=false). +ENV VIRTUAL_ENV=/opt/venv \ + PATH=/opt/venv/bin:$PATH \ + POETRY_VIRTUALENVS_CREATE=false + +RUN python3 -m venv --system-site-packages /opt/venv RUN mkdir -p /etc/apt/keyrings && \ wget -qO- https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key \ diff --git a/packtly-builder/containers/conf/ca-certificates/.gitignore b/packtly-builder/containers/conf/ca-certificates/.gitignore new file mode 100644 index 0000000..a4e94ec --- /dev/null +++ b/packtly-builder/containers/conf/ca-certificates/.gitignore @@ -0,0 +1 @@ +*.crt diff --git a/packtly-builder/containers/conf/ca-certificates/.gitkeep b/packtly-builder/containers/conf/ca-certificates/.gitkeep new file mode 100644 index 0000000..c16af7f --- /dev/null +++ b/packtly-builder/containers/conf/ca-certificates/.gitkeep @@ -0,0 +1,3 @@ +# Drop corporate/proxy root CA files (*.crt, PEM format) into this folder. +# They are copied into the image and installed via update-ca-certificates. +# If empty, the build still works and no extra CAs are added. diff --git a/packtly-builder/tooling/justfile b/packtly-builder/tooling/justfile index e477c6a..a355482 100644 --- a/packtly-builder/tooling/justfile +++ b/packtly-builder/tooling/justfile @@ -8,7 +8,7 @@ prepare: set -eu if ! command -v poetry >/dev/null 2>&1; then - echo "Please install poetry first: scripts/setup_venv" >&2 + echo "Please install poetry first" >&2 exit 1 fi @@ -34,10 +34,11 @@ prepare: else echo "No pip credentials found — skipping private registry configuration" fi - - # Local project settings - poetry config virtualenvs.options.system-site-packages true --local - + if [ -z "${VIRTUAL_ENV:-}" ]; then + echo "No virtualenv active (VIRTUAL_ENV is unset). Create/activate one before running 'just prepare' (e.g. python3 -m venv .venv && . .venv/bin/activate)." >&2 + exit 1 + fi + rm -f poetry.lock || true poetry install --no-interaction --with dev [private] diff --git a/packtly-builder/tooling/packtly_builder_tooling/cli.py b/packtly-builder/tooling/packtly_builder_tooling/cli.py index a4e72fe..3067d26 100755 --- a/packtly-builder/tooling/packtly_builder_tooling/cli.py +++ b/packtly-builder/tooling/packtly_builder_tooling/cli.py @@ -68,11 +68,19 @@ def is_already_deployed( deb_files = [f for f in files if f.endswith(".deb")] if not deb_files: return False - if not all( - aptmanager.upstream_file_exists(Path(f), source_host=aptlyhost) - for f in deb_files - ): + + upstream_exists = True + for f in deb_files: + upstream_exists = True + if aptmanager.upstream_file_exists(Path(f), source_host=aptlyhost): + logger.info("[EXISTS] %s is already deployed upstream", Path(f).name) + else: + logger.warning("[MISSING] %s is not deployed upstream", Path(f).name) + upstream_exists = False + + if not upstream_exists: return False + if build_mode == BuildMode.BINARY: return True dsc_files = [f for f in files if f.endswith(".dsc")] diff --git a/packtly-builder/tooling/packtly_builder_tooling/parts/apt.py b/packtly-builder/tooling/packtly_builder_tooling/parts/apt.py index 44dd575..3caf75a 100644 --- a/packtly-builder/tooling/packtly_builder_tooling/parts/apt.py +++ b/packtly-builder/tooling/packtly_builder_tooling/parts/apt.py @@ -3,9 +3,10 @@ import base64 import apt import apt_pkg +import apt_inst +from debian.deb822 import Deb822 from pathlib import Path from typing import List, Optional, NamedTuple -import apt_inst from aptsources.sourceslist import SourcesList, Deb822SourceEntry from aptsources.distro import get_distro from packtly_builder_tooling.logging_setup import setup_logger @@ -69,6 +70,10 @@ def __init__(self) -> None: self.distro = get_distro() self.logger = setup_logger(__name__) + # ------------------------------------------------------------------ # + # Repository management # + # ------------------------------------------------------------------ # + def add_key(self, key_data: bytes | str, name: str) -> Path: """ Install an APT signing key into /usr/share/keyrings/.gpg. @@ -163,6 +168,10 @@ def update(self, sources_list: Optional[Path] = None) -> bool: ) return True + # ------------------------------------------------------------------ # + # Package installation # + # ------------------------------------------------------------------ # + def install_dependencies( self, package_name: str, @@ -265,6 +274,10 @@ def install_package( ) return False + # ------------------------------------------------------------------ # + # Package existence checks # + # ------------------------------------------------------------------ # + def package_exists( self, package_name: str, @@ -289,9 +302,51 @@ def package_exists( for candidate in pkg.versions ) + def upstream_file_exists( + self, + file_path: Path, + source_host: Optional[str] = None, + ) -> bool: + """Check whether the package represented by a .deb file path already + exists on *source_host*. + + Parses the .deb control data and delegates to :meth:`package_exists`. + Non-deb files (e.g. .buildinfo) that cannot be parsed are silently + ignored and return False. + """ + parsed = self._parse_deb_file(file_path) + if parsed is None: + return False + name, version, _arch = parsed + return self.package_exists( + name, + version=version, + source_host=source_host, + ) + + def deb_dsc_extra_files(self, dsc_path: Path) -> List[str]: + """Return all files listed in a .dsc. + + dpkg-buildpackage omits the orig tarball from the .changes on + non-first uploads. Aptly needs it to successfully import a source + package, so we surface those extra files here. + """ + with open(dsc_path, "r", encoding="utf-8") as fh: + dsc_info = Deb822(fh.read().split("\n")) + + dsc_files_str = dsc_info.get_as_string("Files") + if not dsc_files_str: + return [] + + return [line.split()[2] for line in dsc_files_str.strip().split("\n")] + def source_package_exists(self, dsc_path: Path) -> bool: - """Check whether the source package represented by a .dsc file path - already exists in the apt source cache. + """Check whether all source files listed in a .dsc are present in the + apt source cache. + + Looks up the source record by name + version in the apt SourceRecords + index. If a matching record is found, all files from the .dsc are + logged as [EXISTS]; otherwise each file is logged as [MISSING]. Requires ``deb-src`` to be listed in the apt sources entry so that aptly serves the Sources index and apt-get update downloads it. @@ -300,19 +355,33 @@ def source_package_exists(self, dsc_path: Path) -> bool: if parsed is None: return False name, version = parsed + + source_files = self.deb_dsc_extra_files(dsc_path) + if not source_files: + return False + try: records = apt_pkg.SourceRecords() + found = False while records.lookup(name): if records.version == version: - self.logger.info( - "Source package already present: %s %s", name, version - ) - return True - return False + found = True + break + + for f in source_files: + if found: + self.logger.info("[EXISTS] %s (source)", f) + else: + self.logger.warning("[MISSING] %s (source)", f) + return found except Exception: self.logger.exception("Failed source package lookup for %s", dsc_path) return False + # ------------------------------------------------------------------ # + # Private helpers # + # ------------------------------------------------------------------ # + def _parse_dsc_file(self, dsc_path: Path) -> Optional[tuple[str, str]]: """Parse a .dsc file and return (name, version), or None on failure.""" path = Path(dsc_path) @@ -321,7 +390,11 @@ def _parse_dsc_file(self, dsc_path: Path) -> Optional[tuple[str, str]]: try: fields: dict[str, str] = {} for line in path.read_text(encoding="utf-8").splitlines(): - if ":" in line and not line.startswith(" ") and not line.startswith("-"): + if ( + ":" in line + and not line.startswith(" ") + and not line.startswith("-") + ): key, _, value = line.partition(":") fields[key.strip().lower()] = value.strip() name = fields.get("source") or fields.get("package") @@ -332,31 +405,9 @@ def _parse_dsc_file(self, dsc_path: Path) -> Optional[tuple[str, str]]: pass return None - def upstream_file_exists( - self, - file_path: Path, - source_host: Optional[str] = None, - ) -> bool: - """Check whether the package represented by a .deb file path already - exists on *source_host*. - - Parses the filename with :func:`parse_deb_filename` and delegates to - :meth:`package_exists`. Non-deb files (e.g. .buildinfo) that cannot - be parsed are silently ignored and return False. - """ - parsed = self._parse_deb_file(file_path) - if parsed is None: - return False - name, version, _arch = parsed - return self.package_exists( - name, - version=version, - source_host=source_host, - ) - def _parse_deb_file(self, filepath: Path) -> Optional[DebFileInfo]: """ - Parse a Debian binary file into a DebFileInfo named tuple. + Parse a Debian binary package into a DebFileInfo named tuple. Uses apt_inst.DebFile which reads only the control.tar member. """ path = Path(filepath) diff --git a/packtly-builder/tooling/packtly_builder_tooling/parts/aptly.py b/packtly-builder/tooling/packtly_builder_tooling/parts/aptly.py index 03e2ff8..9b06f72 100644 --- a/packtly-builder/tooling/packtly_builder_tooling/parts/aptly.py +++ b/packtly-builder/tooling/packtly_builder_tooling/parts/aptly.py @@ -47,9 +47,7 @@ def __init__( RequestException, AptlyAPIException, ) as e: - raise ConnectionError( - f"Could not reach aptly server at {host}" - ) from e + raise ConnectionError(f"Could not reach aptly server at {host}") from e def get_publish_endpoint( self, @@ -198,6 +196,10 @@ def upload_deb_files( "Failed adding package: %s", uploaded_packages.failed_files, ) + logger.error( + "Failed adding package report: %s", + uploaded_packages.report.get("Warnings", []), + ) return False added_keys = set(uploaded_packages.report.get("Added", [])) diff --git a/packtly-builder/tooling/packtly_builder_tooling/parts/deb_source.py b/packtly-builder/tooling/packtly_builder_tooling/parts/deb_source.py index cc5c3e0..b9ce88d 100644 --- a/packtly-builder/tooling/packtly_builder_tooling/parts/deb_source.py +++ b/packtly-builder/tooling/packtly_builder_tooling/parts/deb_source.py @@ -1,7 +1,10 @@ import shutil import tarfile from pathlib import Path +from typing import Optional from debian.changelog import Changelog +from git import Remote, Repo +from git.exc import GitCommandNotFound, InvalidGitRepositoryError, NoSuchPathError from packtly_builder_tooling.logging_setup import setup_logger from packtly_builder_tooling.parts.utils import run_subprocess @@ -12,14 +15,11 @@ class DebSourceBuilder: def __init__(self, builddir: Path, outdir: Path) -> None: self._builddir = builddir self._outdir = outdir - self._git = shutil.which("git") self._gbp = shutil.which("gbp") self._dpkg_buildpackage = shutil.which("dpkg-buildpackage") - if not self._git: - logger.warning( - "git executable not found — cannot check for pristine-tar branch or " - "reset the source tree before a quilt source build" - ) + self._repo_loaded = False + self._repo_cache: Optional[Repo] = None + self._changelog_cache: Optional[Changelog] = None if not self._gbp: logger.warning( "gbp executable not found — cannot regenerate the upstream orig tarball " @@ -41,13 +41,6 @@ def is_quilt_format(self) -> bool: """True for non-native quilt packages that need a separate .orig tarball.""" return self.source_format().startswith("3.0 (quilt)") - def _changelog(self) -> Changelog: - changelog_file = self._builddir / "debian" / "changelog" - if not changelog_file.is_file(): - raise FileNotFoundError(f"Changelog file not found at {changelog_file}") - - return Changelog(changelog_file.read_text(encoding="utf-8")) - def source_name(self) -> str: return self._changelog().package or "" @@ -121,31 +114,29 @@ def reset_source_tree(self) -> None: """ if not self.is_quilt_format(): return - if not self._git: - return - result = run_subprocess( - [ - self._git, - "-C", - str(self._builddir), - "rev-parse", - "--is-inside-work-tree", - ], - cwd=self._builddir, - mode="capture", - check=False, - ) - if result.returncode != 0 or result.stdout.strip() != "true": + repo = self._repo() + if repo is None or repo.bare: return logger.info("Resetting source tree to committed state before source build") # Restore tracked files the clean target deleted or modified. - cmd_checkout = [self._git, "-C", str(self._builddir), "checkout", "--", "."] - run_subprocess(cmd_checkout, self._builddir) + repo.git.checkout("--", ".") + # Remove untracked and ignored build artifacts (symlinks, binaries, etc.), + # scoped to the build tree so a surrounding repo is never touched. + repo.git.clean("-fdx", "--", str(self._builddir)) + # Invalidate cached changelog — git checkout may have restored a different version. + self._changelog_cache = None + + def _changelog(self) -> Changelog: + changelog_file = self._builddir / "debian" / "changelog" + if not changelog_file.is_file(): + raise FileNotFoundError(f"Changelog file not found at {changelog_file}") - # Remove untracked and ignored build artifacts (symlinks, binaries, etc.). - cmd_clean = [self._git, "-C", str(self._builddir), "clean", "-fdx"] - run_subprocess(cmd_clean, self._builddir) + if self._changelog_cache is None: + self._changelog_cache = Changelog( + changelog_file.read_text(encoding="utf-8") + ) + return self._changelog_cache def _export_orig_via_pristine_tar(self) -> None: if not self._gbp: @@ -218,26 +209,15 @@ def _branch_exists(self, branch: str) -> bool: missing git binary or a non-git source tree simply means the branch is absent, so this never raises. """ - if not self._git: + repo = self._repo() + if repo is None: return False - for ref in (f"refs/heads/{branch}", f"refs/remotes/origin/{branch}"): - found = run_subprocess( - [ - self._git, - "-C", - str(self._builddir), - "show-ref", - "--verify", - "--quiet", - ref, - ], - self._builddir, - mode="silent", - check=False, - ) - if found.returncode == 0: - return True - return False + if branch in (head.name for head in repo.heads): + return True + origin = self._origin(repo) + if origin is None: + return False + return any(ref.remote_head == branch for ref in origin.refs) def _ensure_local_branch(self, branch: str) -> None: """Create a local branch tracking origin/ if it only exists remotely. @@ -245,52 +225,46 @@ def _ensure_local_branch(self, branch: str) -> None: After a plain ``git clone`` the pristine-tar data is a remote-tracking branch; pristine-tar/gbp operate on the local ``refs/heads`` ref. """ - if not self._git: - raise FileNotFoundError("git not found") - has_local = run_subprocess( - [ - self._git, - "-C", - str(self._builddir), - "show-ref", - "--verify", - "--quiet", - f"refs/heads/{branch}", - ], - self._builddir, - mode="capture", - check=False, - ) - if has_local.returncode == 0: + repo = self._repo() + if repo is None: + logger.error( + "Build directory %s is not a git repository; cannot create local branch %r", + self._builddir, + branch, + ) + raise FileNotFoundError(f"{self._builddir} is not a git repository") + + if branch in (head.name for head in repo.heads): return - has_remote = run_subprocess( - [ - self._git, - "-C", - str(self._builddir), - "show-ref", - "--verify", - "--quiet", - f"refs/remotes/origin/{branch}", - ], - self._builddir, - mode="capture", - check=False, - ) - if has_remote.returncode != 0: + origin = self._origin(repo) + if origin is None or branch not in (ref.remote_head for ref in origin.refs): raise FileNotFoundError( f"No '{branch}' branch found locally or on origin; cannot " "regenerate the upstream orig tarball" ) - cmd = [ - self._git, - "-C", - str(self._builddir), - "branch", - branch, - f"origin/{branch}", - ] + repo.create_head(branch, f"origin/{branch}") + + def _repo(self) -> Optional[Repo]: + """Return the git repository for the build tree, or ``None`` (cached). - run_subprocess(cmd, self._builddir) + Opens the build tree directly (no parent search) so a surrounding + repository is never mistaken for the source tree. Returns ``None`` + when git is unavailable or the directory is not a repository. + """ + if not self._repo_loaded: + self._repo_loaded = True + try: + self._repo_cache = Repo(self._builddir, search_parent_directories=False) + except (GitCommandNotFound, InvalidGitRepositoryError, NoSuchPathError): + self._repo_cache = None + return self._repo_cache + + @staticmethod + def _origin(repo: Repo) -> Optional[Remote]: + """Return the ``origin`` remote if it is configured, otherwise ``None``.""" + try: + return repo.remote("origin") + except ValueError: + return None diff --git a/packtly-builder/tooling/packtly_builder_tooling/parts/debuild.py b/packtly-builder/tooling/packtly_builder_tooling/parts/debuild.py index 5559c81..af4a974 100644 --- a/packtly-builder/tooling/packtly_builder_tooling/parts/debuild.py +++ b/packtly-builder/tooling/packtly_builder_tooling/parts/debuild.py @@ -74,17 +74,22 @@ def install_build_dependencies(self) -> None: logger.info("Build dependencies installed successfully.") def build(self, mode: BuildMode = BuildMode.BINARY) -> None: - # -b binary only — no .orig tarball required (3.0 quilt or native) - # -S source only — requires .orig.tar.* in parent dir for quilt - # -F full build — source + binary + # debuild flags: + # -uc / -us skip GPG signing of .changes and .dsc (we sign later via debsign) + # -b binary only — no .orig tarball required (3.0 quilt or native) + # -S source only — requires .orig.tar.* in parent dir for quilt + # -F full build — source + binary + # -sa always include the orig tarball in the .changes file. + # dpkg-buildpackage normally omits it for Debian revisions > -1 + # (assuming the archive already has it), but aptly has no such + # state and requires all files referenced by the .dsc to be + # present during import. debuild_cmd = [self._debuild, "-uc", "-us"] if mode in (BuildMode.SOURCE, BuildMode.FULL): self._orig_tarball.reset_source_tree() self._orig_tarball.ensure_orig_tarball() - # The build log is written into a ``logs/`` directory inside the - # source tree; tell dpkg-source to ignore it so the live log does - # not register as an unrepresentable upstream change. debuild_cmd.append("--source-option=--extend-diff-ignore=(^|/)logs/") + debuild_cmd.append("-sa") debuild_cmd.append(mode.value) run_subprocess(debuild_cmd, self._builddir, stdin_data="y\n") logger.info("Debian packages built successfully.") diff --git a/packtly-builder/tooling/packtly_builder_tooling/tests/test_deb_source.py b/packtly-builder/tooling/packtly_builder_tooling/tests/test_deb_source.py new file mode 100644 index 0000000..4e14e2b --- /dev/null +++ b/packtly-builder/tooling/packtly_builder_tooling/tests/test_deb_source.py @@ -0,0 +1,177 @@ +from pathlib import Path +from typing import Any, List, Optional + +import pytest +from git import Repo + +from packtly_builder_tooling.parts.deb_source import DebSourceBuilder + + +def _make_builder(builddir: Path, outdir: Path) -> DebSourceBuilder: + """Bypass __init__ so tests don't depend on which()-resolved binaries.""" + obj = DebSourceBuilder.__new__(DebSourceBuilder) + obj._builddir = builddir + obj._outdir = outdir + obj._gbp = "gbp" + obj._dpkg_buildpackage = "dpkg-buildpackage" + obj._repo_loaded = False + obj._repo_cache = None + obj._changelog_cache = None + return obj + + +def _write_source_format(builddir: Path, fmt: str) -> None: + src = builddir / "debian" / "source" + src.mkdir(parents=True, exist_ok=True) + (src / "format").write_text(fmt, encoding="utf-8") + + +def _write_changelog(builddir: Path, pkg: str, version: str) -> None: + deb = builddir / "debian" + deb.mkdir(parents=True, exist_ok=True) + (deb / "changelog").write_text( + f"{pkg} ({version}) unstable; urgency=low\n\n" + f" * test\n\n" + f" -- Tester Mon, 30 Jun 2026 00:00:00 +0000\n", + encoding="utf-8", + ) + + +# --- pure logic ------------------------------------------------------------- + + +def test_is_quilt_format_true(tmp_path: Path) -> None: + _write_source_format(tmp_path, "3.0 (quilt)\n") + builder = _make_builder(tmp_path, tmp_path.parent) + assert builder.is_quilt_format() is True + + +def test_is_quilt_format_native(tmp_path: Path) -> None: + _write_source_format(tmp_path, "3.0 (native)\n") + builder = _make_builder(tmp_path, tmp_path.parent) + assert builder.is_quilt_format() is False + + +def test_source_name_and_version(tmp_path: Path) -> None: + _write_changelog(tmp_path, "debhello", "1.0.0-1") + builder = _make_builder(tmp_path, tmp_path.parent) + assert builder.source_name() == "debhello" + assert builder.upstream_version() == "1.0.0" + + +def test_orig_tarball_exists_ignores_asc(tmp_path: Path) -> None: + out = tmp_path / "out" + out.mkdir() + _write_changelog(tmp_path, "debhello", "1.0.0-1") + builder = _make_builder(tmp_path, out) + + (out / "debhello_1.0.0.orig.tar.xz.asc").write_text("sig", encoding="utf-8") + assert builder.orig_tarball_exists() is False + + (out / "debhello_1.0.0.orig.tar.xz").write_bytes(b"data") + assert builder.orig_tarball_exists() is True + + +def test_remove_existing_orig(tmp_path: Path) -> None: + out = tmp_path / "out" + out.mkdir() + _write_changelog(tmp_path, "debhello", "1.0.0-1") + stale = out / "debhello_1.0.0.orig.tar.gz" + stale.write_bytes(b"old") + keep = out / "unrelated_2.0.0.orig.tar.gz" + keep.write_bytes(b"keep") + + builder = _make_builder(tmp_path, out) + builder._remove_existing_orig() + + assert not stale.exists() + assert keep.exists() + + +# --- git logic against a real temp repo ------------------------------------- + + +@pytest.fixture +def git_builddir(tmp_path: Path) -> Path: + builddir = tmp_path / "src" + builddir.mkdir() + repo = Repo.init(builddir) + with repo.config_writer() as cw: + cw.set_value("user", "name", "Tester") + cw.set_value("user", "email", "t@example.com") + (builddir / "file.txt").write_text("content", encoding="utf-8") + repo.index.add(["file.txt"]) + repo.index.commit("initial") + return builddir + + +def test_branch_exists_local(git_builddir: Path) -> None: + Repo(git_builddir).create_head("pristine-tar") + builder = _make_builder(git_builddir, git_builddir.parent) + assert builder._branch_exists("pristine-tar") is True + assert builder._branch_exists("nope") is False + + +def test_repo_none_for_non_git_tree(tmp_path: Path) -> None: + builder = _make_builder(tmp_path, tmp_path.parent) + assert builder._repo() is None + assert builder._branch_exists("pristine-tar") is False + + +def test_repo_does_not_climb_to_parent(tmp_path: Path) -> None: + """A non-git build tree nested in a parent repo must not resolve the parent.""" + Repo.init(tmp_path) + builddir = tmp_path / "src" + builddir.mkdir() + builder = _make_builder(builddir, tmp_path) + assert builder._repo() is None + + +def test_reset_source_tree_cleans_untracked(git_builddir: Path) -> None: + _write_source_format(git_builddir, "3.0 (quilt)\n") + artifact = git_builddir / "generated.bin" + artifact.write_bytes(b"junk") # untracked build artifact + + builder = _make_builder(git_builddir, git_builddir.parent) + builder.reset_source_tree() + + assert not artifact.exists() # git clean removed it + assert (git_builddir / "file.txt").exists() # tracked file preserved + + +def test_reset_source_tree_noop_when_native(git_builddir: Path) -> None: + _write_source_format(git_builddir, "3.0 (native)\n") + artifact = git_builddir / "generated.bin" + artifact.write_bytes(b"junk") + + builder = _make_builder(git_builddir, git_builddir.parent) + builder.reset_source_tree() + + assert artifact.exists() # native packages are not reset + + +# --- external tool call is delegated, not executed -------------------------- + + +def test_export_orig_invokes_gbp( + git_builddir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + Repo(git_builddir).create_head("pristine-tar") + captured: dict[str, Any] = {} + + def fake_run_subprocess( + cmd: List[str], cwd: Path, stdin_data: Optional[str] = None + ) -> None: + captured["cmd"] = cmd + captured["cwd"] = cwd + + monkeypatch.setattr( + "packtly_builder_tooling.parts.deb_source.run_subprocess", + fake_run_subprocess, + ) + + builder = _make_builder(git_builddir, git_builddir.parent) + builder._export_orig_via_pristine_tar() + + assert captured["cmd"][:2] == ["gbp", "export-orig"] + assert captured["cwd"] == git_builddir diff --git a/packtly-builder/tooling/poetry.toml b/packtly-builder/tooling/poetry.toml index 25758d2..088550b 100644 --- a/packtly-builder/tooling/poetry.toml +++ b/packtly-builder/tooling/poetry.toml @@ -1,2 +1,4 @@ -[virtualenvs.options] -system-site-packages = true +# Use the image's pre-activated venv (/opt/venv, via VIRTUAL_ENV) instead of +# letting Poetry create its own, so the interpreter path stays stable. +[virtualenvs] +create = false diff --git a/packtly-builder/tooling/pyproject.toml b/packtly-builder/tooling/pyproject.toml index 95f6330..744a5bc 100644 --- a/packtly-builder/tooling/pyproject.toml +++ b/packtly-builder/tooling/pyproject.toml @@ -27,6 +27,7 @@ aptly-api-client = "==0.3.0" python = ">=3.10,<4.0" python-gnupg = "^0.5.2" distro = "^1.9.0" +GitPython = "^3.1.43" [tool.poetry.group.dev.dependencies] mypy = "^1.14.0"