improve workflow files again (cache but not cache)

docker-bake.hcl

@@ -3,92 +3,77 @@ variable "NETDISCO_GIT_URL" {
 }
 
 variable "COMMITTISH" {
-  default = "latest"
+  default = "HEAD"
 }
 
 variable "BUILD_DATE" {
   default = ""
 }
 
 group "default" {
-  targets = ["netdisco-backend","netdisco-web"]
+  targets = ["netdisco"]
 }
 
 group "standalone" {
-  targets = ["netdisco-postgresql","netdisco-backend", "netdisco-web"]
+  targets = ["netdisco-postgresql","netdisco"]
 }
 
 group "all" {
-  targets = ["netdisco-postgresql","netdisco-postgresql-13","netdisco-backend", "netdisco-web"]
+  targets = ["netdisco-postgresql","netdisco-postgresql-13","netdisco"]
 }
 
 target "netdisco-base" {
-  context    = "./netdisco-base"
-  dockerfile = "Dockerfile"
-  tags = [
-    "localhost:5000/netdisco:${COMMITTISH}-base",
-  ]
+  context = "./netdisco-base"
   args = {
     COMMITTISH       = COMMITTISH
     NETDISCO_GIT_URL = NETDISCO_GIT_URL
-    BUILD_DATE       = BUILD_DATE
   }
-  output = ["type=docker"]
-}
-
-target "netdisco-web" {
-  context    = "./netdisco-web"
-  dockerfile = "Dockerfile"
   tags = [
-    "localhost:5000/netdisco:${COMMITTISH}-web",
+    "localhost:5000/netdisco:${COMMITTISH}-base",
   ]
-  args = {
-    COMMITTISH = COMMITTISH
-  }
-  contexts = {
-    "localhost:5000/netdisco:${COMMITTISH}-base" = "target:netdisco-base"
-  }
   output = ["type=docker"]
 }
 
-target "netdisco-backend" {
-  context    = "./netdisco-backend"
-  dockerfile = "Dockerfile"
-  tags = [
-    "localhost:5000/netdisco:${COMMITTISH}-backend",
-  ]
+target "netdisco" {
+  name = "netdisco-${tgt}"
+  matrix = {
+    tgt = ["backend", "web"]
+  }
+  context = "./netdisco-${tgt}"
   args = {
     COMMITTISH = COMMITTISH
+    BUILD_DATE = BUILD_DATE
   }
+  tags = [
+    "localhost:5000/netdisco:${COMMITTISH}-${tgt}",
+  ]
   contexts = {
     "localhost:5000/netdisco:${COMMITTISH}-base" = "target:netdisco-base"
   }
   output = ["type=docker"]
 }
 
 target "netdisco-postgresql" {
-  context    = "./netdisco-postgresql"
-  dockerfile = "Dockerfile"
-  tags = [
-    "localhost:5000/netdisco:${COMMITTISH}-postgresql",
-  ]
+  context = "./netdisco-postgresql"
   args = {
     COMMITTISH = COMMITTISH
     BUILD_DATE = BUILD_DATE
   }
+  tags = [
+    "localhost:5000/netdisco:${COMMITTISH}-postgresql",
+  ]
   output = ["type=docker"]
 }
 
 target "netdisco-postgresql-13" {
-  context    = "./netdisco-postgresql"
-  dockerfile = "Dockerfile"
-  tags = [
-    "localhost:5000/netdisco:${COMMITTISH}-postgresql-13",
-  ]
+  context = "./netdisco-postgresql"
   args = {
     PGVER      = "13.4"
     COMMITTISH = COMMITTISH
     BUILD_DATE = BUILD_DATE
   }
+  tags = [
+    "localhost:5000/netdisco:${COMMITTISH}-postgresql-13",
+  ]
   output = ["type=docker"]
 }

netdisco-backend/Dockerfile

@@ -1,33 +1,33 @@
 # vim: ft=Dockerfile
-ARG COMMITTISH=latest
-FROM localhost:5000/netdisco:${COMMITTISH}-base
-
-USER root
+ARG COMMITTISH=HEAD
+# ----------------------------------------------------------------------------
+FROM docker.io/alpine:latest AS netdisco-mibs
 
 RUN --mount=type=cache,target=/var/cache/apk \
-    apk add \
-      ca-certificates \
-      curl \
-      openssh-client \
-      bash \
-      tar
-
-WORKDIR /var/lib/postgresql/netdisco-sql
-RUN curl -sLO "https://github.com/netdisco/upstream-sources/raw/refs/heads/master/bootstrap/netdisco-lookup-tables.sql"
-
-USER netdisco:netdisco
-WORKDIR /home/netdisco
+    apk add curl
 
 # if Netdisco MIBs VERSION has not changed, Docker's cache remains valid.
 # there is a slight race, when the VERSION could have been updated in the repo before a release is made
 # in which case docker will simply re-download the previous version (the current release)
 ADD https://github.com/netdisco/netdisco-mibs/raw/refs/heads/master/VERSION /tmp/netdisco-mibs-version
 
-RUN curl -sL https://api.github.com/repos/netdisco/netdisco-mibs/releases/latest | \
-  grep browser_download_url | xargs printf "curl -sL %s\n" | tail -1 | \
-  sh | tar -zxf -
+WORKDIR /tmp/netdisco-mibs
+RUN `/usr/bin/curl -sL https://api.github.com/repos/netdisco/netdisco-mibs/releases/latest | \
+  grep browser_download_url | xargs printf "/usr/bin/curl -sL %s\n" | tail -1 | \
+  sh | tar --strip-components=1 -zxf -`
+
+# ----------------------------------------------------------------------------
+FROM docker.io/alpine:latest AS netdisco-sql
+
+# this will probably bust the cache as it updates weekly
+ADD https://github.com/netdisco/upstream-sources/raw/refs/heads/master/bootstrap/netdisco-lookup-tables.sql /tmp/
+
+# ----------------------------------------------------------------------------
+FROM localhost:5000/netdisco:${COMMITTISH}-base
+
+USER netdisco:netdisco
+WORKDIR /home/netdisco
 
-ARG COMMITTISH=HEAD
 RUN rm -f /home/netdisco/perl5/lib/perl5/auto/share/dist/App-Netdisco/python/netdisco/uv.lock
 RUN `/home/netdisco/bin/localenv perl -MAlien::ultraviolet -e 'print Alien::ultraviolet->uv'` \
       --no-cache --no-progress --quiet --project \
@@ -42,11 +42,25 @@ ENV UV_NO_MANAGED_PYTHON=true \
     UV_NO_CACHE=1 \
     UV_NO_DEV=1
 
-COPY --chown=netdisco:netdisco netdisco-updatedb.sh /home/netdisco/bin/
-RUN chmod +x /home/netdisco/bin/netdisco-updatedb.sh
+COPY --chown=netdisco:netdisco --chmod=755 netdisco-updatedb.sh /home/netdisco/bin/
+COPY --from=netdisco-mibs --chown=netdisco:netdisco /tmp/netdisco-mibs /home/netdisco/
+COPY --from=netdisco-sql --chown=root:root /tmp/netdisco-lookup-tables.sql /var/lib/postgresql/netdisco-sql/
+
+ARG COMMITTISH=HEAD
+ARG BUILD_DATE
 
-LABEL org.label-schema.name="netdisco-backend" \
-      org.label-schema.description="Polling Daemon for Netdisco"
+LABEL org.label-schema.docker.schema-version="1.0" \
+      org.label-schema.vendor="The Netdisco Project" \
+      org.label-schema.url="http://netdisco.org" \
+      org.label-schema.name="netdisco-backend" \
+      org.label-schema.description="Polling Daemon for Netdisco" \
+      org.label-schema.usage="https://github.com/netdisco/netdisco-docker/blob/master/README.md" \
+      org.label-schema.version=${COMMITTISH} \
+      org.label-schema.vcs-ref=${COMMITTISH} \
+      org.label-schema.vcs-url="git://github.com/netdisco/netdisco.git" \
+      org.label-schema.build-date=${BUILD_DATE} \
+      org.netdisco.maintainer="The Netdisco Project" \
+      org.netdisco.version=${COMMITTISH}
 
 ENTRYPOINT ["/home/netdisco/bin/netdisco-backend"]
 CMD ["foreground"]

netdisco-base/Dockerfile

@@ -8,21 +8,21 @@ FROM docker.io/alpine:latest AS fetcher
 RUN --mount=type=cache,target=/var/cache/apk \
     apk add git
 
+WORKDIR /fetch
+
 # NETDISCO_GIT_URL lets CI / local builds point at a fork (e.g. to test a PR)
 # without editing the Dockerfile. Defaults to upstream.
 
 ARG NETDISCO_GIT_URL=https://github.com/netdisco/netdisco.git
 ARG COMMITTISH=HEAD
 
-WORKDIR /fetch
-
 RUN git init \
   && git remote add origin "${NETDISCO_GIT_URL}" \
   && git fetch --depth 1 origin "${COMMITTISH}" \
   && git reset --hard FETCH_HEAD
 
 # ----------------------------------------------------------------------------
-FROM docker.io/alpine:latest AS netdisco-builder-image
+FROM docker.io/alpine:latest AS netdisco-common-base
 
 # temporary bloated image used for building Netdisco.
 # netdisco-base will copy the built Netdisco files from this image
@@ -31,31 +31,55 @@ ARG CACHE_BUST_OS=1
 RUN --mount=type=cache,target=/var/cache/apk \
     apk update && \
     apk add \
-      linux-headers \
+      bash \
       ca-certificates \
       curl \
+      expect \
+      expect-dev \
       findutils \
-      g++ \
-      gcc \
-      git \
+      icu-data-full \
       jq \
       libc6-compat \
       musl-dev \
       musl-obstack-dev \
-      make \
       net-snmp-perl \
-      openssl-dev \
+      openssh-client \
       perl-crypt-rijndael \
       perl-dbd-pg \
-      perl-dev \
+      perl-file-sharedir \
+      perl-file-slurp \
       perl-io-socket-ssl \
+      perl-json \
       perl-ldap \
+      perl-lwp-protocol-https \
       perl-netaddr-ip \
+      perl-regexp-common \
+      perl-regexp-common-net-cidr \
+      perl-regexp-ipv6 \
       postgresql18-client \
-      postgresql18-dev \
       python3 \
+      shadow \
+      tar \
       tzdata \
-      openssh-client \
+      zstd && \
+    apk fix --no-cache perl perl-dev
+
+# ----------------------------------------------------------------------------
+FROM netdisco-common-base AS netdisco-builder-image
+
+# temporary bloated image used for building Netdisco.
+# netdisco-base will copy the built Netdisco files from this image
+
+RUN --mount=type=cache,target=/var/cache/apk \
+    apk add \
+      g++ \
+      gcc \
+      git \
+      linux-headers \
+      make \
+      openssl-dev \
+      perl-dev \
+      postgresql18-dev \
       wget && \
     apk fix --no-cache perl perl-dev
 
@@ -82,7 +106,7 @@ RUN --mount=type=cache,target=${WORK_DIR}/.cpanm,sharing=locked \
       URL::Encode \
       REST::Client
 
-# grab the fetched file from the first stage.
+# grab the fetched file from the second stage.
 # if Build.PL has not changed, Docker's cache remains valid.
 COPY --from=fetcher /fetch/Build.PL /tmp/netdisco-tree/Build.PL
 COPY Netdisco.pm /tmp/netdisco-tree/lib/App/Netdisco.pm
@@ -112,39 +136,7 @@ RUN --mount=type=cache,target=${WORK_DIR}/.cpanm,sharing=locked \
 RUN mv /home/netdisco /home/netdisco-build
 
 # ----------------------------------------------------------------------------
-FROM docker.io/alpine:latest
-
-ARG CACHE_BUST_OS=1
-RUN --mount=type=cache,target=/var/cache/apk \
-    apk update && \
-    apk add \
-      shadow \
-      findutils \
-      libc6-compat \
-      musl-dev \
-      musl-obstack-dev \
-      net-snmp-perl \
-      expect \
-      expect-dev \
-      perl-crypt-rijndael \
-      perl-dbd-pg \
-      perl-io-socket-ssl \
-      perl-ldap \
-      perl-json \
-      perl-lwp-protocol-https \
-      perl-file-slurp \
-      perl-regexp-common \
-      perl-regexp-common-net-cidr \
-      perl-regexp-ipv6 \
-      perl-file-sharedir \
-      perl-netaddr-ip \
-      postgresql18-client \
-      python3 \
-      tzdata \
-      bash \
-      zstd \
-      icu-data-full \
-      jq
+FROM netdisco-common-base
 
 RUN groupadd -r netdisco -g 901 && \
     useradd -u 901 -r -p x -g netdisco -m -d /home/netdisco -s /bin/ash -c "netdisco user" netdisco
@@ -180,23 +172,7 @@ WORKDIR /home/netdisco
 ENV PATH="/home/netdisco/bin:$PATH"
 ENV SHELL=/bin/ash
 
-ARG BUILD_DATE
-ARG COMMITTISH=HEAD
-
-LABEL org.label-schema.docker.schema-version="1.0" \
-      org.label-schema.vendor="The Netdisco Project" \
-      org.label-schema.url="http://netdisco.org" \
-      org.label-schema.name="Netdisco Base" \
-      org.label-schema.description="Base Image for Netdisco" \
-      org.label-schema.usage="https://github.com/netdisco/netdisco-docker/blob/master/README.md" \
-      org.label-schema.version=${COMMITTISH} \
-      org.label-schema.vcs-ref=${COMMITTISH} \
-      org.label-schema.vcs-url="git://github.com/netdisco/netdisco.git" \
-      org.label-schema.build-date=${BUILD_DATE} \
-      org.netdisco.maintainer="The Netdisco Project" \
-      org.netdisco.version=${COMMITTISH}
-
-      ## volumes can be optional, and controlled by docker-compose file
+## volumes can be optional, and controlled by docker-compose file
 # VOLUME ["/home/netdisco/environments", "/home/netdisco/nd-site-local", "/home/netdisco/logs"]
 
 CMD ["ash"]

netdisco-web/Dockerfile

@@ -1,9 +1,22 @@
 # vim: ft=Dockerfile
-ARG COMMITTISH=latest
+ARG COMMITTISH=HEAD
 FROM localhost:5000/netdisco:${COMMITTISH}-base
 
-LABEL org.label-schema.name="netdisco-web" \
-      org.label-schema.description="Web Frontend for Netdisco"
+ARG COMMITTISH=HEAD
+ARG BUILD_DATE
+
+LABEL org.label-schema.docker.schema-version="1.0" \
+      org.label-schema.vendor="The Netdisco Project" \
+      org.label-schema.url="http://netdisco.org" \
+      org.label-schema.name="netdisco-web" \
+      org.label-schema.description="Web Frontend for Netdisco" \
+      org.label-schema.usage="https://github.com/netdisco/netdisco-docker/blob/master/README.md" \
+      org.label-schema.version=${COMMITTISH} \
+      org.label-schema.vcs-ref=${COMMITTISH} \
+      org.label-schema.vcs-url="git://github.com/netdisco/netdisco.git" \
+      org.label-schema.build-date=${BUILD_DATE} \
+      org.netdisco.maintainer="The Netdisco Project" \
+      org.netdisco.version=${COMMITTISH}
 
 EXPOSE 5000
 ENTRYPOINT ["/home/netdisco/bin/netdisco-web"]