diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3085f09ab..6e854bf65 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,7 +57,7 @@ jobs: - name: "Install Test Packages" run: | sudo apt-get update - sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage python3-etcd3 python3-plyvel flake8 faketime + sudo apt-get install -y --no-install-recommends graphviz gnupg2 gpgv2 git gcc make devscripts python3 python3-requests-unixsocket python3-termcolor python3-swiftclient python3-boto python3-azure-storage python3-etcd3 python3-plyvel flake8 faketime dput-ng - name: "Checkout Repository" uses: actions/checkout@v4 diff --git a/api/files.go b/api/files.go index e51ea0069..08aa6ac06 100644 --- a/api/files.go +++ b/api/files.go @@ -185,6 +185,69 @@ func apiFilesUpload(c *gin.Context) { c.JSON(200, stored) } +// @Summary Upload One File +// @Description **Upload one file to a directory** +// @Description +// @Description - file is uploaded +// @Description - existing uploaded are overwritten +// @Description +// @Description **Example:** +// @Description ``` +// @Description $ dput aptly aptly_0.9~dev+217+ge5d646c_i386.changes +// @Description ``` +// @Tags Files +// @Param dir path string true "Directory to upload files to. Created if does not exist" +// @Param file path string true "File to upload" +// @Produce json +// @Success 200 {array} string "Name of uploaded file" +// @Failure 400 {object} Error "Bad Request" +// @Failure 404 {object} Error "Not Found" +// @Failure 500 {object} Error "Internal Server Error" +// @Router /api/files/{dir}/{file} [put] +func apiFilesUploadOne(c *gin.Context) { + if !verifyDir(c) { + return + } + + fileName := c.Params.ByName("file") + if !verifyPath(fileName) { + AbortWithJSONError(c, 400, fmt.Errorf("wrong file")) + return + } + + path := filepath.Join(context.UploadPath(), utils.SanitizePath(c.Params.ByName("dir"))) + err := os.MkdirAll(path, 0777) + + if err != nil { + AbortWithJSONError(c, 500, err) + return + } + stored := []string{} + + destPath := filepath.Join(path, fileName) + dst, err := os.Create(destPath) + if err != nil { + AbortWithJSONError(c, 500, err) + return + } + defer dst.Close() + + if _, err = io.Copy(dst, c.Request.Body); err != nil { + AbortWithJSONError(c, 500, err) + return + } + + if err = syncFile(dst); err != nil { + AbortWithJSONError(c, 500, fmt.Errorf("error syncing file %s: %s", fileName, err)) + return + } + + stored = append(stored, filepath.Join(c.Params.ByName("dir"), fileName)) + + apiFilesUploadedCounter.WithLabelValues(c.Params.ByName("dir")).Inc() + c.JSON(200, stored) +} + // @Summary List Files // @Description **Show uploaded files in upload directory** // @Description diff --git a/api/router.go b/api/router.go index 62dd5c2f2..0cad19024 100644 --- a/api/router.go +++ b/api/router.go @@ -182,6 +182,7 @@ func Router(c *ctx.AptlyContext) http.Handler { { api.GET("/files", apiFilesListDirs) api.POST("/files/:dir", apiFilesUpload) + api.PUT("/files/:dir/:file", apiFilesUploadOne) api.GET("/files/:dir", apiFilesListFiles) api.DELETE("/files/:dir", apiFilesDeleteDir) api.DELETE("/files/:dir/:name", apiFilesDeleteFile) diff --git a/docker/test.Dockerfile b/docker/test.Dockerfile index 4ba65782c..5c8f1cdf8 100644 --- a/docker/test.Dockerfile +++ b/docker/test.Dockerfile @@ -1,5 +1,8 @@ FROM aptly-dev +RUN apt-get update -y && apt-get install -y --no-install-recommends dput-ng && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + ADD --chown=aptly:aptly . /work/src/ # Pre-populate the Go module cache so go mod verify works offline diff --git a/system/Dockerfile b/system/Dockerfile index 143569c00..59c86a520 100644 --- a/system/Dockerfile +++ b/system/Dockerfile @@ -7,7 +7,7 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends curl gnupg b binutils-arm-linux-gnueabihf bash-completion zip ruby-dev lintian npm \ libc6-dev-i386-cross libc6-dev-armhf-cross libc6-dev-arm64-cross \ gcc-i686-linux-gnu gcc-arm-linux-gnueabihf gcc-aarch64-linux-gnu \ - faketime && \ + faketime dput-ng && \ apt-get clean && rm -rf /var/lib/apt/lists/* RUN useradd -m --shell /bin/bash --home-dir /var/lib/aptly aptly diff --git a/system/t12_api/files.py b/system/t12_api/files.py index f1b9d52e1..60fa14aed 100644 --- a/system/t12_api/files.py +++ b/system/t12_api/files.py @@ -1,4 +1,10 @@ +import inspect +import os +import shutil +import tempfile + from api_lib import APITest +from lib import BaseTest class FilesAPITestUpload(APITest): @@ -97,3 +103,76 @@ def check(self): self.check_equal(self.delete("/api/files/../.").status_code, 404) self.check_equal(self.delete("/api/files/./..").status_code, 404) self.check_equal(self.delete("/api/files/dir/..").status_code, 404) + + +class FilesAPITestDputUpload(APITest): + """ + PUT /api/files/:dir/:file via dput, then POST /api/repos/:name/include/:dir + + Uses the real dput binary to upload a .changes file and all its referenced + files to the aptly API, then imports them into a local repo via include. + Skipped if dput is not installed. + """ + + def fixture_available(self): + return super().fixture_available() and shutil.which("dput") is not None + + def check(self): + d = self.random_name() + repo_name = self.random_name() + + # Create target repo + self.check_equal( + self.post("/api/repos", json={"Name": repo_name}).status_code, 201) + + changes_dir = os.path.join( + os.path.dirname(inspect.getsourcefile(BaseTest)), "changes") + changes_file = os.path.join(changes_dir, "hardlink_0.2.1_amd64.changes") + + # dput strips leading/trailing slashes from 'incoming' then prepends /, + # producing: PUT http://{fqdn}/api/files/{d}/{filename} + # fqdn includes host:port so dput connects directly to the test API server. + dput_cf = ( + "[aptly]\n" + f"fqdn = {self.base_url}\n" + "method = http\n" + f"incoming = api/files/{d}\n" + "login = *\n" + "allow_unsigned_uploads = 1\n" + "allow_dcut = 0\n" + ) + + tmpdir = tempfile.mkdtemp() + try: + dput_cf_path = os.path.join(tmpdir, "dput.cf") + with open(dput_cf_path, "w") as f: + f.write(dput_cf) + + # dput -U: allow unsigned uploads (skip local GPG check) + # dput reads the .changes and PUTs every file listed in Files: + the .changes itself + self.run_cmd(["dput", "-c", dput_cf_path, "-U", "aptly", changes_file]) + finally: + shutil.rmtree(tmpdir) + + # All files referenced in the .changes must now be present in the upload dir + self.check_exists(f"upload/{d}/hardlink_0.2.1_amd64.changes") + self.check_exists(f"upload/{d}/hardlink_0.2.1.dsc") + self.check_exists(f"upload/{d}/hardlink_0.2.1.tar.gz") + self.check_exists(f"upload/{d}/hardlink_0.2.1_amd64.deb") + + # Import via the .changes file into the repo + resp = self.post_task( + f"/api/repos/{repo_name}/include/{d}", + params={"ignoreSignature": 1}) + self.check_task(resp) + + output = self.get(f"/api/tasks/{resp.json()['ID']}/output") + self.check_in(b"Added: hardlink_0.2.1_source added, hardlink_0.2.1_amd64 added", output.content) + + # Packages must be in the repo + self.check_equal( + sorted(self.get(f"/api/repos/{repo_name}/packages").json()), + ["Pamd64 hardlink 0.2.1 daf8fcecbf8210ad", "Psource hardlink 0.2.1 8f72df429d7166e5"]) + + # include cleans up the upload dir + self.check_not_exists(f"upload/{d}")