Skip to content
19 changes: 10 additions & 9 deletions vulnfeeds/cmd/combine-to-osv/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -136,18 +136,18 @@ func readVulnerability(ctx context.Context, client *storage.Client, fullPath str
}

func combineIntoOSV(cve5 *osvschema.Vulnerability, nvd *osvschema.Vulnerability) *osvschema.Vulnerability {
var baseOSV *osvschema.Vulnerability
if cve5 != nil && nvd != nil {
baseOSV = combineTwoOSVRecords(cve5, nvd)
} else if cve5 != nil {
baseOSV = cve5
} else if nvd != nil {
baseOSV = nvd
} else {
if (cve5.GetWithdrawn() != nil) || (nvd.GetWithdrawn() != nil) {
return nil
}

return baseOSV
if cve5 != nil && nvd != nil {
return combineTwoOSVRecords(cve5, nvd)
}
if cve5 != nil {
return cve5
}

return nvd
}

func readAndCombineWorker(ctx context.Context, client *storage.Client, workChan <-chan *CVEWorkItem, vulnChan chan<- *osvschema.Vulnerability) {
Expand Down Expand Up @@ -367,6 +367,7 @@ func main() {
// combineTwoOSVRecords takes two osv records and combines them into one
func combineTwoOSVRecords(cve5 *osvschema.Vulnerability, nvd *osvschema.Vulnerability) *osvschema.Vulnerability {
baseOSV := cve5

if baseOSV.GetDetails() == "" && nvd.GetDetails() != "" {
baseOSV.Details = nvd.GetDetails()
}
Expand Down
82 changes: 82 additions & 0 deletions vulnfeeds/cmd/combine-to-osv/main_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1075,6 +1075,88 @@ func TestCombineTwoOSVRecords(t *testing.T) {
}
}

func TestCombineIntoOSV(t *testing.T) {
cve5WithdrawnTime, _ := time.Parse(time.RFC3339, "2023-01-01T12:00:00Z")
nvdWithdrawnTime, _ := time.Parse(time.RFC3339, "2023-01-02T12:00:00Z")

validCVE5 := &osvschema.Vulnerability{Id: "CVE-2023-1234", Details: "CVE5 Details"}
validNVD := &osvschema.Vulnerability{Id: "CVE-2023-1234", Details: "NVD Details"}

withdrawnCVE5 := &osvschema.Vulnerability{
Id: "CVE-2023-1234",
Withdrawn: timestamppb.New(cve5WithdrawnTime),
}
withdrawnNVD := &osvschema.Vulnerability{
Id: "CVE-2023-1234",
Withdrawn: timestamppb.New(nvdWithdrawnTime),
}

tests := []struct {
name string
cve5 *osvschema.Vulnerability
nvd *osvschema.Vulnerability
want *osvschema.Vulnerability
}{
{
name: "CVE5 withdrawn, NVD valid",
cve5: withdrawnCVE5,
nvd: validNVD,
want: nil,
},
{
name: "CVE5 valid, NVD withdrawn",
cve5: validCVE5,
nvd: withdrawnNVD,
want: nil,
},
{
Comment thread
jess-lowe marked this conversation as resolved.
name: "Both withdrawn",
cve5: withdrawnCVE5,
nvd: withdrawnNVD,
want: nil,
},
{
name: "CVE5 withdrawn, NVD nil",
cve5: withdrawnCVE5,
nvd: nil,
want: nil,
},
{
name: "CVE5 nil, NVD withdrawn",
cve5: nil,
nvd: withdrawnNVD,
want: nil,
},
{
name: "CVE5 valid, NVD nil",
cve5: validCVE5,
nvd: nil,
want: validCVE5,
},
{
name: "CVE5 nil, NVD valid",
cve5: nil,
nvd: validNVD,
want: validNVD,
},
{
name: "Both nil",
cve5: nil,
nvd: nil,
want: nil,
},
}

for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
got := combineIntoOSV(tc.cve5, tc.nvd)
if diff := cmp.Diff(tc.want, got, protocmp.Transform()); diff != "" {
t.Errorf("combineIntoOSV() mismatch (-want +got):\n%s", diff)
}
})
}
}

func TestCombineTwoOSVRecords_ReferencesDeterminism(t *testing.T) {
cve5 := &osvschema.Vulnerability{
Id: "CVE-2023-1234",
Expand Down
18 changes: 13 additions & 5 deletions vulnfeeds/cmd/converters/cve/cve5/bulk-converter/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ func worker(wg *sync.WaitGroup, jobs <-chan string, gcsHelper *gcs.Helper, outDi
continue
}

if slices.Contains(cnas, cve.Metadata.AssignerShortName) || cve.Metadata.State != "PUBLISHED" {
if slices.Contains(cnas, cve.Metadata.AssignerShortName) || (cve.Metadata.State != "PUBLISHED" && cve.Metadata.State != "REJECTED") {
continue
}
cveID := cve.Metadata.CVEID
Expand All @@ -179,10 +179,14 @@ func worker(wg *sync.WaitGroup, jobs <-chan string, gcsHelper *gcs.Helper, outDi
if metrics.Outcome == models.Successful {
successfulConversionsCount.Add(1)
}
if rejectFailed && metrics.Outcome != models.Successful {
if !metrics.Outcome.ShouldEmit(rejectFailed) {
logger.Info("Rejecting failed OSV record", slog.String("cve", string(cveID)), slog.String("outcome", metrics.Outcome.String()))
} else {
logger.Info("Queueing OSV record for "+string(cveID), slog.String("cve", string(cveID)))
if metrics.Outcome == models.Rejected {
logger.Info("Queueing withdrawn OSV record for "+string(cveID), slog.String("cve", string(cveID)))
} else {
logger.Info("Queueing OSV record for "+string(cveID), slog.String("cve", string(cveID)))
}
if err := writer.UploadVulnIfChangedAsync(gcsHelper, *gcsPrefix, vuln.Vulnerability); err != nil {
logger.Error("Failed to queue vulnerability upload", slog.String("cve", string(cveID)), slog.Any("err", err))
}
Expand Down Expand Up @@ -216,12 +220,16 @@ func worker(wg *sync.WaitGroup, jobs <-chan string, gcsHelper *gcs.Helper, outDi
if metrics.Outcome == models.Successful {
successfulConversionsCount.Add(1)
}
if rejectFailed && metrics.Outcome != models.Successful {
if !metrics.Outcome.ShouldEmit(rejectFailed) {
logger.Info("Rejecting failed OSV record", slog.String("cve", string(cveID)), slog.String("outcome", metrics.Outcome.String()))
osvFile.Close()
os.Remove(osvFile.Name())
} else {
logger.Info("Generated OSV record for "+string(cveID), slog.String("cve", string(cveID)), slog.String("cna", cve.Metadata.AssignerShortName), slog.String("outcome", metrics.Outcome.String()))
if metrics.Outcome == models.Rejected {
logger.Info("Generated withdrawn OSV record for "+string(cveID), slog.String("cve", string(cveID)), slog.String("cna", cve.Metadata.AssignerShortName))
} else {
logger.Info("Generated OSV record for "+string(cveID), slog.String("cve", string(cveID)), slog.String("cna", cve.Metadata.AssignerShortName), slog.String("outcome", metrics.Outcome.String()))
}
}
}

Expand Down
16 changes: 10 additions & 6 deletions vulnfeeds/cmd/converters/cve/nvd-cve-osv/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -215,14 +215,18 @@ func worker(wg *sync.WaitGroup, jobs <-chan models.NVDCVE, gcsHelper *gcs.Helper
continue // Don't attempt to output files if there was an error
}

if outcome != models.Successful {
logger.Info("Failed to generate a successful OSV record", slog.String("cve", cveID), slog.String("outcome", outcome.String()))
if *rejectFailed {
continue // Skip outputting OSV file
}
} else {
if outcome == models.Successful {
logger.Info("Generated OSV record for "+cveID, slog.String("cve", cveID))
successfulConversionsCount.Add(1)
} else {
if outcome == models.Rejected {
logger.Info("Generated withdrawn OSV record for "+cveID, slog.String("cve", cveID))
} else {
logger.Info("Failed to generate a successful OSV record", slog.String("cve", cveID), slog.String("outcome", outcome.String()))
}
if !outcome.ShouldEmit(*rejectFailed) {
continue // Skip outputting OSV file
}
}

// Extract year from CVE ID to organize local outputs into subfolders
Expand Down
39 changes: 25 additions & 14 deletions vulnfeeds/conversion/cve5/converter.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,25 @@ func getCWEs(cna models.CNA, metrics *models.ConversionMetrics) []string {
// It populates the main fields of the OSV record, including ID, summary, details,
// references, timestamps, severity, and version information.
func FromCVE5(cve models.CVE5, refs []models.Reference, metrics *models.ConversionMetrics, sourceLink string) *vulns.Vulnerability {
Comment thread
jess-lowe marked this conversation as resolved.
published, err := models.ParseCVE5Timestamp(cve.Metadata.DatePublished)
if err != nil {
metrics.AddNote("[%s]: Published date failed to parse, falling back to Epoch", cve.Metadata.CVEID)
published = time.Unix(0, 0).UTC()
}

modified, err := models.ParseCVE5Timestamp(cve.Metadata.DateUpdated)
if err != nil {
metrics.AddNote("[%s]: Modified date failed to parse, falling back to Published time", cve.Metadata.CVEID)
modified = published
}

var withdrawnTime *timestamppb.Timestamp
if cve.Metadata.State == "REJECTED" {
withdrawnTime = timestamppb.New(modified)
}

aliases, related := vulns.ExtractReferencedVulns(cve.Metadata.CVEID, cve.Metadata.CVEID, refs)

v := vulns.Vulnerability{
Vulnerability: &osvschema.Vulnerability{
SchemaVersion: osvconstants.SchemaVersion,
Expand All @@ -80,22 +98,11 @@ func FromCVE5(cve models.CVE5, refs []models.Reference, metrics *models.Conversi
Aliases: aliases,
Related: related,
References: vulns.ClassifyReferences(refs),
Withdrawn: withdrawnTime,
Published: timestamppb.New(published),
Modified: timestamppb.New(modified),
}}

published, err := models.ParseCVE5Timestamp(cve.Metadata.DatePublished)
if err != nil {
metrics.AddNote("[%s]: Published date failed to parse, setting time to now", cve.Metadata.CVEID)
published = time.Now()
}
v.Published = timestamppb.New(published)

modified, err := models.ParseCVE5Timestamp(cve.Metadata.DateUpdated)
if err != nil {
metrics.AddNote("[%s]: Modified date failed to parse, setting time to now", cve.Metadata.CVEID)
modified = time.Now()
}
v.Modified = timestamppb.New(modified)

// Try to extract repository URLs from references.
repos := conversion.ReposFromReferencesCVEList(refs, models.RefTagDenyList, metrics)
metrics.Repos = repos
Expand Down Expand Up @@ -170,6 +177,10 @@ func CVEToOSV(cve models.CVE5, sourceLink string) (*vulns.Vulnerability, *models

models.DetermineOutcome(&metrics)

if cve.Metadata.State == "REJECTED" {
metrics.Outcome = models.Rejected
}

return v, &metrics
}

Expand Down
26 changes: 26 additions & 0 deletions vulnfeeds/conversion/cve5/converter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,32 @@ func TestFromCVE5(t *testing.T) {
},
},
},
{
name: "rejected record",
cve: models.CVE5{
Metadata: models.CVE5Metadata{
CVEID: "CVE-2025-8888",
State: "REJECTED",
DatePublished: "2025-05-04T07:20:46.575Z",
DateUpdated: "2025-05-04T07:20:46.575Z",
},
},
refs: []models.Reference{},
expectedVuln: &vulns.Vulnerability{
Vulnerability: &osvschema.Vulnerability{
Id: "CVE-2025-8888",
SchemaVersion: "1.7.5",
Published: timestamppb.New(cvePlaceholder),
Modified: timestamppb.New(cvePlaceholder),
Withdrawn: timestamppb.New(cvePlaceholder),
DatabaseSpecific: &structpb.Struct{
Fields: map[string]*structpb.Value{
"osv_generated_from": structpb.NewStringValue("unknown"),
},
},
},
},
},
{
name: "CVE-2025-1110",
cve: loadTestData(t, "CVE-2025-1110"),
Expand Down
11 changes: 11 additions & 0 deletions vulnfeeds/conversion/nvd/converter.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,17 @@ func CVEToOSV(cve models.NVDCVE, repos []string, vpRepoCache *c.VPRepoCache, cac
refs := c.DeduplicateRefs(cve.References)
// The vendor name and product name are used to construct the output `vulnDir` below, so need to be set to *something* to keep the output tidy.

if cve.VulnStatus != nil && *cve.VulnStatus == "Rejected" {
metrics.SetOutcome(models.Rejected)
v := vulns.FromNVDCVE(cve.ID, cve)
databaseSpecific, err := utility.NewStructpbFromMap(make(map[string]any))
if err == nil {
v.DatabaseSpecific = databaseSpecific
}

return v, metrics, models.Rejected
}

if len(CPEs) > 0 {
_, err := c.ParseCPE(CPEs[0]) // For naming the subdirectory used for output.
if err != nil {
Expand Down
32 changes: 32 additions & 0 deletions vulnfeeds/conversion/nvd/converter_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"path/filepath"
"sort"
"testing"
"time"

"github.com/gkampitakis/go-snaps/snaps"
"github.com/go-git/go-git/v5/plumbing/transport/client"
Expand Down Expand Up @@ -97,6 +98,37 @@ func TestCVEToOSV_429(t *testing.T) {
}
}

func TestCVEToOSV_Rejected(t *testing.T) {
cve := models.NVDCVE{
ID: "CVE-2025-54321",
VulnStatus: func() *string {
s := "Rejected"
return &s
}(),
LastModified: models.NVDTime{Time: func() time.Time {
t, _ := time.Parse(time.RFC3339, "2025-06-01T12:00:00Z")
return t
}()},
}

metrics := &models.ConversionMetrics{}
cache := &git.InMemoryRepoTagsCache{}

vuln, _, outcome := CVEToOSV(cve, nil, nil, cache, metrics)

if outcome != models.Rejected {
t.Errorf("Expected outcome models.Rejected, got %v", outcome)
}

if vuln == nil {
t.Fatal("Expected non-nil vulnerability for rejected CVE")
}

if vuln.GetWithdrawn() == nil {
t.Error("Expected Withdrawn timestamp to be set for rejected CVE")
}
}

func TestCVEToOSV_ReferencesDeterminism(t *testing.T) {
cve := models.NVDCVE{
ID: "CVE-2025-12345",
Expand Down
3 changes: 2 additions & 1 deletion vulnfeeds/conversion/writer/writer.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ var ErrUploadSkipped = errors.New("upload skipped")

// writeToDisk writes the vulnerability to a local file.
// It returns an error if the file could not be written.
// Writes out withdrawn records regardless of whether they don't already exist.
func writeToDisk(v *osvschema.Vulnerability, preModifiedBuf []byte, outputPrefix string) error {
filename := v.GetId() + ".json"
filePath := path.Join(outputPrefix, filename)
Expand Down Expand Up @@ -173,7 +174,7 @@ func handleOverride(ctx context.Context, v *osvschema.Vulnerability, overridesBk
func VulnWorker(ctx context.Context, vulnChan <-chan *osvschema.Vulnerability, outBkt, overridesBkt *storage.BucketHandle, gcsHelper *gcs.Helper, outputPrefix string, counter *atomic.Uint64) {
for v := range vulnChan {
vulnID := v.GetId()
if len(v.GetAffected()) == 0 {
if len(v.GetAffected()) == 0 && v.GetWithdrawn() == nil {
logger.Warn("Skipping OSV record as no affected versions found.", slog.String("id", vulnID))
continue
}
Expand Down
9 changes: 9 additions & 0 deletions vulnfeeds/models/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,15 @@ func (c ConversionOutcome) String() string {
return conversionOutcomeStrings[ConversionUnknown]
}

// ShouldEmit returns true if the record should be emitted based on its outcome and the rejectFailed flag.
func (c ConversionOutcome) ShouldEmit(rejectFailed bool) bool {
if !rejectFailed {
return true
}

return c == Successful || c == Rejected
}

func (c ConversionOutcome) MarshalJSON() ([]byte, error) {
return json.Marshal(c.String())
}
Expand Down
Loading
Loading