diff --git a/internal/handler/pypi.go b/internal/handler/pypi.go index f5a0232..133d3cd 100644 --- a/internal/handler/pypi.go +++ b/internal/handler/pypi.go @@ -19,7 +19,11 @@ const ( minWheelParts = 5 // name + version + python + abi + platform minSubmatchParts = 2 // full match + first capture group minPyPIPathParts = 3 // hash_prefix + hash + filename - minPythonTagLen = 2 // minimum length for a python tag (e.g., "py") + minTaggedParts = 2 // name + version + + // pypiMetadataSuffix is the PEP 658 core-metadata sidecar suffix that pip + // appends to a distribution URL when the index advertises core metadata. + pypiMetadataSuffix = ".metadata" ) // PyPIHandler handles PyPI registry protocol requests. @@ -433,37 +437,59 @@ func (h *PyPIHandler) handleDownload(w http.ResponseWriter, r *http.Request) { ServeArtifact(w, result) } +// taggedExtensions are distribution formats whose filenames append build, +// interpreter and platform tags after the version. Their name and version +// fields are escaped so neither can contain a hyphen, which makes the first two +// hyphen-separated fields authoritative. minParts is the field count a +// well-formed filename of that format has at minimum. +var taggedExtensions = []struct { + ext string + minParts int +}{ + {".whl", minWheelParts}, + {".egg", minTaggedParts}, + {".exe", minTaggedParts}, + {".msi", minTaggedParts}, +} + +// archiveExtensions are sdist formats of the form {name}-{version}{ext}. Unlike +// the tagged formats these carry no trailing tags, but legacy sdist names may +// contain hyphens. +var archiveExtensions = []string{".tar.gz", ".tar.bz2", ".tar.xz", ".tar.Z", ".tgz", ".tar", ".zip"} + // parseFilename extracts package name and version from a PyPI filename. -// Handles both wheels and sdists: +// Handles wheels, sdists and legacy bdist formats: // - requests-2.31.0-py3-none-any.whl // - requests-2.31.0.tar.gz +// - numpy-1.8.0-py2.7-macosx-10.9-x86_64.egg func (h *PyPIHandler) parseFilename(filename string) (name, version string) { - // Try wheel format first: {name}-{version}(-{build})?-{python}-{abi}-{platform}.whl - if strings.HasSuffix(filename, ".whl") { - base := strings.TrimSuffix(filename, ".whl") - parts := strings.Split(base, "-") - if len(parts) >= minWheelParts { - // Find where version ends (version followed by python tag) - for i := 1; i < len(parts)-2; i++ { - // Check if this looks like a python tag (py2, py3, cp39, etc) - if isPythonTag(parts[i]) { - name = strings.Join(parts[:i-1], "-") - version = parts[i-1] - return - } - } + // PEP 658/714 core-metadata sidecars are the distribution filename plus + // ".metadata"; they describe the same name and version. Without this, pip's + // metadata-only fetches fall back to a hash-derived package identifier. + filename = strings.TrimSuffix(filename, pypiMetadataSuffix) + + // Wheels are {name}-{version}(-{build})?-{python}-{abi}-{platform}.whl; + // per PEP 427 the name and version fields have any hyphen escaped to '_'. + for _, format := range taggedExtensions { + if !strings.HasSuffix(filename, format.ext) { + continue } + parts := strings.Split(strings.TrimSuffix(filename, format.ext), "-") + if len(parts) < format.minParts { + return "", "" + } + return parts[0], parts[1] } - // Try sdist formats: {name}-{version}.tar.gz, {name}-{version}.zip - for _, ext := range []string{".tar.gz", ".tar.bz2", ".zip", ".tar"} { - if strings.HasSuffix(filename, ext) { - base := strings.TrimSuffix(filename, ext) - // Find last hyphen followed by version - for i := len(base) - 1; i >= 0; i-- { - if base[i] == '-' && i+1 < len(base) && isVersionStart(base[i+1]) { - return base[:i], base[i+1:] - } + for _, ext := range archiveExtensions { + if !strings.HasSuffix(filename, ext) { + continue + } + base := strings.TrimSuffix(filename, ext) + // Find last hyphen followed by version + for i := len(base) - 1; i >= 0; i-- { + if base[i] == '-' && i+1 < len(base) && isVersionStart(base[i+1]) { + return base[:i], base[i+1:] } } } @@ -471,20 +497,6 @@ func (h *PyPIHandler) parseFilename(filename string) (name, version string) { return "", "" } -func isPythonTag(s string) bool { - if len(s) < minPythonTagLen { - return false - } - // Python tags start with py, cp, pp, ip, jy - prefixes := []string{"py", "cp", "pp", "ip", "jy"} - for _, p := range prefixes { - if strings.HasPrefix(s, p) { - return true - } - } - return false -} - func isVersionStart(c byte) bool { return c >= '0' && c <= '9' } diff --git a/internal/handler/pypi_test.go b/internal/handler/pypi_test.go index 2b58960..61b9a17 100644 --- a/internal/handler/pypi_test.go +++ b/internal/handler/pypi_test.go @@ -28,13 +28,33 @@ func TestPyPIParseFilename(t *testing.T) { {"aws-sdk-1.0.0.tar.gz", "aws-sdk", "1.0.0"}, {"zipp-3.17.0.zip", "zipp", "3.17.0"}, + // Additional sdist archive formats + {"lxml-4.9.3.tar.xz", "lxml", "4.9.3"}, + {"docutils-0.20.1.tgz", "docutils", "0.20.1"}, + {"psycopg2-2.9.9.tar.bz2", "psycopg2", "2.9.9"}, + // Wheel formats {"requests-2.31.0-py3-none-any.whl", "requests", "2.31.0"}, {"numpy-1.26.2-cp311-cp311-manylinux_2_17_x86_64.whl", "numpy", "1.26.2"}, {"cryptography-41.0.5-cp37-abi3-manylinux_2_28_x86_64.whl", "cryptography", "41.0.5"}, + // Wheels with a build tag must not fold the tag into the version + {"foo-1.0-1-py3-none-any.whl", "foo", "1.0"}, + {"tensorflow-2.15.0-2-cp311-cp311-manylinux_2_17_x86_64.whl", "tensorflow", "2.15.0"}, + + // PEP 658 core-metadata sidecars resolve to the distribution they describe + {"backports_asyncio_runner-1.2.0-py3-none-any.whl.metadata", "backports_asyncio_runner", "1.2.0"}, + {"requests-2.31.0-py3-none-any.whl.metadata", "requests", "2.31.0"}, + {"requests-2.31.0.tar.gz.metadata", "requests", "2.31.0"}, + + // Legacy bdist formats + {"numpy-1.8.0-py2.7-macosx-10.9-x86_64.egg", "numpy", "1.8.0"}, + {"pywin32-223.win32-py2.7.exe", "pywin32", "223.win32"}, + // Invalid {"invalid", "", ""}, + {"invalid.metadata", "", ""}, + {"backports.ssl_match_hostname-3.4.0.2-py2.7.whl", "", ""}, } for _, tt := range tests { @@ -94,25 +114,24 @@ func TestPyPIRewriteJSONMetadataCooldown(t *testing.T) { } } -func TestIsPythonTag(t *testing.T) { - tests := []struct { - tag string - want bool - }{ - {"py3", true}, - {"py2", true}, - {"cp311", true}, - {"cp37", true}, - {"pp39", true}, - {"none", false}, - {"any", false}, - {"manylinux", false}, +// TestPyPIParseFilenameNoHashFallback guards the identifier used for caching: +// a filename that parses to an empty name makes handleDownload fall back to a +// "_hash_" package name, which surfaces as a bogus PURL in the package +// overview. +func TestPyPIParseFilenameNoHashFallback(t *testing.T) { + h := &PyPIHandler{proxy: &Proxy{Logger: slog.Default()}} + + filenames := []string{ + "backports_asyncio_runner-1.2.0-py3-none-any.whl", + "backports_asyncio_runner-1.2.0-py3-none-any.whl.metadata", + "backports_asyncio_runner-1.2.0.tar.gz", } - for _, tt := range tests { - got := isPythonTag(tt.tag) - if got != tt.want { - t.Errorf("isPythonTag(%q) = %v, want %v", tt.tag, got, tt.want) + for _, filename := range filenames { + name, version := h.parseFilename(filename) + if name != "backports_asyncio_runner" || version != "1.2.0" { + t.Errorf("parseFilename(%q) = (%q, %q), want (%q, %q)", + filename, name, version, "backports_asyncio_runner", "1.2.0") } } }