diff --git a/fetch/resolver.go b/fetch/resolver.go index bccf024..985fca5 100644 --- a/fetch/resolver.go +++ b/fetch/resolver.go @@ -16,6 +16,7 @@ import ( var ( ErrUnsupportedEcosystem = errors.New("unsupported ecosystem") ErrNoDownloadURL = errors.New("no download URL available") + ErrNoMatchingArtifact = errors.New("no artifact matches the requested file") ErrUnsafeURL = errors.New("unsafe download URL from registry metadata") ) @@ -49,25 +50,46 @@ type ArtifactInfo struct { URL string Filename string Integrity string // sha256-... or sha512-... + Size int64 // Zero when the registry does not publish a size. +} + +// ResolveOptions narrows versions that publish several artifact files. +type ResolveOptions struct { + Filename string + Integrity string } // Resolve returns the download URL and filename for a package artifact. func (r *Resolver) Resolve(ctx context.Context, ecosystem, name, version string) (*ArtifactInfo, error) { + return r.ResolveWithOptions(ctx, ecosystem, name, version, ResolveOptions{}) +} + +// ResolveWithOptions returns the download metadata matching filename and +// registry-native integrity constraints when they are populated. +func (r *Resolver) ResolveWithOptions( + ctx context.Context, + ecosystem string, + name string, + version string, + options ResolveOptions, +) (*ArtifactInfo, error) { reg, ok := r.registries[ecosystem] if !ok { - return r.resolveWithoutRegistry(ecosystem, name, version) + info, err := r.resolveWithoutRegistry(ecosystem, name, version) + return matchSingleArtifact(info, options, err) } // Try the simple URL builder first if url := reg.URLs().Download(name, version); url != "" { - return &ArtifactInfo{ + info := &ArtifactInfo{ URL: url, Filename: filenameFromURL(url), - }, nil + } + return matchSingleArtifact(info, options, nil) } // For ecosystems like PyPI, we need to fetch metadata to get the URL - return r.resolveFromMetadata(ctx, reg, name, version) + return r.resolveFromMetadata(ctx, reg, name, version, options) } // resolveWithoutRegistry handles ecosystems with predictable URLs @@ -132,7 +154,13 @@ func (r *Resolver) resolveWithoutRegistry(ecosystem, name, version string) (*Art } // resolveFromMetadata fetches version metadata to find download URL. -func (r *Resolver) resolveFromMetadata(ctx context.Context, reg Registry, name, version string) (*ArtifactInfo, error) { +func (r *Resolver) resolveFromMetadata( + ctx context.Context, + reg Registry, + name string, + version string, + options ResolveOptions, +) (*ArtifactInfo, error) { versions, err := reg.FetchVersions(ctx, name) if err != nil { return nil, fmt.Errorf("fetching versions: %w", err) @@ -143,15 +171,21 @@ func (r *Resolver) resolveFromMetadata(ctx context.Context, reg Registry, name, continue } + if len(v.Artifacts) > 0 { + return selectArtifact(v.Artifacts, options) + } + // Look for download URL in metadata. These come from the // registry's API response, not from us, so they need checking // before anyone fetches them. if v.Metadata != nil { if u, ok := v.Metadata["download_url"].(string); ok && u != "" { - return artifactFromMetadataURL(u, v.Integrity) + info, err := artifactFromMetadataURL(u, v.Integrity) + return matchSingleArtifact(info, options, err) } if u, ok := v.Metadata["tarball"].(string); ok && u != "" { - return artifactFromMetadataURL(u, v.Integrity) + info, err := artifactFromMetadataURL(u, v.Integrity) + return matchSingleArtifact(info, options, err) } } @@ -161,6 +195,54 @@ func (r *Resolver) resolveFromMetadata(ctx context.Context, reg Registry, name, return nil, ErrNotFound } +func selectArtifact(candidates []registries.Artifact, options ResolveOptions) (*ArtifactInfo, error) { + for _, candidate := range candidates { + filename := candidate.Filename + if filename == "" { + filename = filenameFromURL(candidate.URL) + } + if options.Filename != "" && filename != options.Filename { + continue + } + if options.Integrity != "" && !integrityMatches(candidate.Integrity, options.Integrity) { + continue + } + if err := checkMetadataURL(candidate.URL); err != nil { + return nil, err + } + return &ArtifactInfo{ + URL: candidate.URL, + Filename: filename, + Integrity: candidate.Integrity, + Size: candidate.Size, + }, nil + } + return nil, ErrNoMatchingArtifact +} + +func matchSingleArtifact(info *ArtifactInfo, options ResolveOptions, err error) (*ArtifactInfo, error) { + if err != nil { + return nil, err + } + if options.Filename != "" && info.Filename != options.Filename { + return nil, ErrNoMatchingArtifact + } + if options.Integrity != "" && info.Integrity != "" && + !integrityMatches(info.Integrity, options.Integrity) { + return nil, ErrNoMatchingArtifact + } + return info, nil +} + +func integrityMatches(candidate, requested string) bool { + for _, expected := range strings.Fields(requested) { + if candidate == expected { + return true + } + } + return false +} + func artifactFromMetadataURL(raw, integrity string) (*ArtifactInfo, error) { if err := checkMetadataURL(raw); err != nil { return nil, err diff --git a/fetch/resolver_test.go b/fetch/resolver_test.go index e7fc4b2..7cf03c3 100644 --- a/fetch/resolver_test.go +++ b/fetch/resolver_test.go @@ -230,6 +230,60 @@ func TestResolveFromMetadataAcceptsSafeURL(t *testing.T) { } } +func TestResolveFromMetadataSelectsPublishedArtifact(t *testing.T) { + reg := &fakeRegistry{ + versions: []registries.Version{{ + Number: "1.0.0", + Artifacts: []registries.Artifact{ + { + URL: "https://files.pythonhosted.org/example-1.0.0.tar.gz", + Filename: "example-1.0.0.tar.gz", + Integrity: "sha256-source", + }, + { + URL: "https://files.pythonhosted.org/example-1.0.0-py3-none-any.whl", + Filename: "example-1.0.0-py3-none-any.whl", + Integrity: "sha256-wheel", + Size: 1234, + }, + }, + }}, + } + r := NewResolver() + r.RegisterRegistry(reg) + + info, err := r.ResolveWithOptions(context.Background(), "fake", "example", "1.0.0", ResolveOptions{ + Integrity: "sha256-wheel", + }) + if err != nil { + t.Fatal(err) + } + if info.Filename != "example-1.0.0-py3-none-any.whl" || info.Size != 1234 { + t.Errorf("artifact = %#v", info) + } +} + +func TestResolveFromMetadataRejectsMissingPublishedArtifact(t *testing.T) { + reg := &fakeRegistry{ + versions: []registries.Version{{ + Number: "1.0.0", + Artifacts: []registries.Artifact{{ + URL: "https://files.pythonhosted.org/example-1.0.0.tar.gz", + Integrity: "sha256-source", + }}, + }}, + } + r := NewResolver() + r.RegisterRegistry(reg) + + _, err := r.ResolveWithOptions(context.Background(), "fake", "example", "1.0.0", ResolveOptions{ + Integrity: "sha256-wheel", + }) + if !errors.Is(err, ErrNoMatchingArtifact) { + t.Fatalf("ResolveWithOptions() error = %v, want ErrNoMatchingArtifact", err) + } +} + func TestFilenameFromURL(t *testing.T) { tests := []struct { url string diff --git a/internal/core/types.go b/internal/core/types.go index f7783bc..3e35046 100644 --- a/internal/core/types.go +++ b/internal/core/types.go @@ -23,9 +23,20 @@ type Version struct { Licenses string Integrity string // sha256-..., sha512-... Status VersionStatus // "", "yanked", "deprecated", "retracted" + Artifacts []Artifact Metadata map[string]any } +// Artifact describes one file published for a package version. +type Artifact struct { + URL string + Filename string + Integrity string + Size int64 // Zero when the registry does not publish a size. + MediaType string + Metadata map[string]any +} + // VersionStatus represents the status of a package version. type VersionStatus string diff --git a/internal/pypi/pypi.go b/internal/pypi/pypi.go index a750a33..000883c 100644 --- a/internal/pypi/pypi.go +++ b/internal/pypi/pypi.go @@ -79,7 +79,7 @@ type releaseFile struct { PackageType string `json:"packagetype"` PythonVersion string `json:"python_version"` RequiresPython string `json:"requires_python"` - Size int `json:"size"` + Size int64 `json:"size"` } type versionInfoResponse struct { @@ -229,11 +229,17 @@ func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Versi integrity = "sha256-" + sha256 } + artifacts := make([]core.Artifact, 0, len(files)) + for _, release := range files { + artifacts = append(artifacts, releaseArtifact(release)) + } + versions = append(versions, core.Version{ Number: num, PublishedAt: publishedAt, Integrity: integrity, Status: status, + Artifacts: artifacts, Metadata: map[string]any{ "download_url": file.URL, "requires_python": file.RequiresPython, @@ -247,6 +253,33 @@ func (r *Registry) FetchVersions(ctx context.Context, name string) ([]core.Versi return versions, nil } +func releaseArtifact(file releaseFile) core.Artifact { + var integrity string + if sha256, ok := file.Digests["sha256"]; ok { + integrity = "sha256-" + sha256 + } + return core.Artifact{ + URL: file.URL, + Filename: filenameFromURL(file.URL), + Integrity: integrity, + Size: file.Size, + Metadata: map[string]any{ + "requires_python": file.RequiresPython, + "yanked": file.Yanked, + "yanked_reason": file.YankedReason, + "packagetype": file.PackageType, + "python_version": file.PythonVersion, + }, + } +} + +func filenameFromURL(value string) string { + if index := strings.LastIndex(value, "/"); index >= 0 { + return value[index+1:] + } + return value +} + var pep508NameRegex = regexp.MustCompile(`^([A-Za-z0-9][-A-Za-z0-9._]*[A-Za-z0-9]|[A-Za-z0-9])(\s*\[.*?\])?`) func (r *Registry) FetchDependencies(ctx context.Context, name, version string) ([]core.Dependency, error) { diff --git a/internal/pypi/pypi_test.go b/internal/pypi/pypi_test.go index f3f1414..416670f 100644 --- a/internal/pypi/pypi_test.go +++ b/internal/pypi/pypi_test.go @@ -20,12 +20,12 @@ func TestFetchPackage(t *testing.T) { resp := packageResponse{ Info: infoBlock{ - Name: "requests", - Summary: "Python HTTP for Humans.", - License: "Apache 2.0", - HomePage: "https://requests.readthedocs.io", - Version: "2.31.0", - Keywords: "http,web,client", + Name: "requests", + Summary: "Python HTTP for Humans.", + License: "Apache 2.0", + HomePage: "https://requests.readthedocs.io", + Version: "2.31.0", + Keywords: "http,web,client", ProjectURLs: map[string]string{ "Source": "https://github.com/psf/requests", "Documentation": "https://requests.readthedocs.io", @@ -179,9 +179,15 @@ func TestFetchVersions(t *testing.T) { "2.31.0": { { Digests: map[string]string{"sha256": "abc123"}, + URL: "https://files.pythonhosted.org/requests-2.31.0.tar.gz", UploadTime: "2023-05-22T12:00:00", Yanked: false, }, + { + Digests: map[string]string{"sha256": "wheel456"}, + URL: "https://files.pythonhosted.org/requests-2.31.0-py3-none-any.whl", + Size: 1234, + }, }, "2.30.0": { { @@ -220,6 +226,19 @@ func TestFetchVersions(t *testing.T) { if yankedCount != 1 { t.Errorf("expected 1 yanked version, got %d", yankedCount) } + for _, version := range versions { + if version.Number != "2.31.0" { + continue + } + if len(version.Artifacts) != 2 { + t.Fatalf("artifacts = %#v, want two", version.Artifacts) + } + wheel := version.Artifacts[1] + if wheel.Filename != "requests-2.31.0-py3-none-any.whl" || + wheel.Integrity != "sha256-wheel456" || wheel.Size != 1234 { + t.Errorf("wheel = %#v", wheel) + } + } } func TestFetchDependencies(t *testing.T) { diff --git a/registries.go b/registries.go index b02a330..13250f0 100644 --- a/registries.go +++ b/registries.go @@ -52,6 +52,9 @@ type ( // Version represents a specific version of a package. Version = core.Version + // Artifact describes one file published for a package version. + Artifact = core.Artifact + // Dependency represents a package dependency. Dependency = core.Dependency