Skip to content

Commit deb5e1e

Browse files
jhutchings1jhutchings1
authored
Merge pull request #4 from jhutchings1/develop
Convert repository to support Conda
2 parents 6de9c73 + 28e1b2f commit deb5e1e

15 files changed

Lines changed: 28097 additions & 5907 deletions

‎.github/workflows/test.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ jobs:
1313
runs-on: ubuntu-latest
1414
steps:
1515
- uses: actions/checkout@v3
16-
- uses: ./
17-
with:
18-
filePath: "test"
16+
- run: |
17+
npm install
18+
node --experimental-vm-modules node_modules/jest/bin/jest.js
1919

‎README.md‎

Lines changed: 10 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,40 +1,23 @@
1-
# SPDX to Dependency Graph Action
1+
# Conda dependency submission action
22

3-
This repository makes it easy to upload an SPDX SBOM to GitHub's dependency submission API. This lets you quickly receive Dependabot alerts for package manifests which GitHub doesn't directly support like pnpm or Paket by using existing off-the-shelf SBOM generators.
3+
This repository scans Conda environment.yaml files and uploads the results to the dependency graph. While GitHub does not support alerting on OS-level dependencies, it will alert on any PyPI dependencies that are defined in the environment.yaml.
44

55

66
### Example workflow
7-
This workflow uses the [Microsoft sbom-tool](https://github.com/microsoft/sbom-tool).
7+
88
```yaml
99

10-
name: SBOM upload
10+
name: Conda dependency submission
1111

12-
on:
12+
on:
1313
workflow_dispatch:
14-
push:
15-
branches: ["main"]
14+
push:
1615

1716
jobs:
18-
SBOM-upload:
19-
17+
dependency-submission:
2018
runs-on: ubuntu-latest
21-
permissions:
22-
id-token: write
23-
contents: write
24-
2519
steps:
26-
- uses: actions/checkout@v3
27-
- name: Generate SBOM
28-
run: |
29-
curl -Lo $RUNNER_TEMP/sbom-tool https://github.com/microsoft/sbom-tool/releases/latest/download/sbom-tool-linux-x64
30-
chmod +x $RUNNER_TEMP/sbom-tool
31-
$RUNNER_TEMP/sbom-tool generate -b . -bc . -pn ${{ github.repository }} -pv 1.0.0 -ps OwnerName -nsb https://sbom.mycompany.com -V Verbose
32-
- uses: actions/upload-artifact@v3
33-
with:
34-
name: sbom
35-
path: _manifest/spdx_2.2
36-
- name: SBOM upload
37-
uses: jhutchings1/spdx-to-dependency-graph-action@v0.0.1
38-
with:
39-
filePath: "_manifest/spdx_2.2/"
20+
- uses: actions/checkout@v3
21+
- name: Conda dependency scanning
22+
uses: jhutchings1/conda-dependency-submission-action@v0.0.1
4023
```

‎action.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ inputs:
1212
filePattern:
1313
description: 'The file name pattern for environment files to upload'
1414
required: false
15-
default: '*environment.yaml'
15+
default: 'environment.yaml'
1616
runs:
1717
using: 'node16'
1818
main: 'dist/index.js'

‎condaParser.test.ts‎

Lines changed: 129 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,135 @@ test('Gets files', async () => {
55
expect(files.length).toEqual(1);
66
});
77

8-
test('Parses manifests', async() => {
8+
function roundTripJSON(obj: any): object {
9+
return JSON.parse(JSON.stringify(obj))
10+
}
11+
12+
test('Parses manifests', async () => {
913
var files = conda.searchFiles("test", "environment.yaml");
1014
var manifests = conda.getManifestsFromEnvironmentFiles(files);
1115
expect(manifests.length).toEqual(1);
12-
})
16+
expect(roundTripJSON(manifests[0])).toEqual(
17+
{
18+
"resolved": {
19+
"pkg:conda/python@3.8": {
20+
"package_url": "pkg:conda/python@3.8",
21+
"relationship": "direct",
22+
"dependencies": []
23+
},
24+
"pkg:conda/pytorch@1.10": {
25+
"package_url": "pkg:conda/pytorch@1.10",
26+
"relationship": "direct",
27+
"dependencies": []
28+
},
29+
"pkg:conda/torchvision": {
30+
"package_url": "pkg:conda/torchvision",
31+
"relationship": "direct",
32+
"dependencies": []
33+
},
34+
"pkg:conda/cudatoolkit@11.0": {
35+
"package_url": "pkg:conda/cudatoolkit@11.0",
36+
"relationship": "direct",
37+
"dependencies": []
38+
},
39+
"pkg:conda/pip": {
40+
"package_url": "pkg:conda/pip",
41+
"relationship": "direct",
42+
"dependencies": []
43+
},
44+
"pkg:pypi/pytorch-lightning@1.5.2": {
45+
"package_url": "pkg:pypi/pytorch-lightning@1.5.2",
46+
"relationship": "direct",
47+
"dependencies": []
48+
}, "pkg:pypi/einops@0.3.2": {
49+
"package_url": "pkg:pypi/einops@0.3.2",
50+
"relationship": "direct",
51+
"dependencies": []
52+
},
53+
"pkg:pypi/kornia@0.6.1": {
54+
"package_url": "pkg:pypi/kornia@0.6.1",
55+
"relationship": "direct",
56+
"dependencies": []
57+
},
58+
"pkg:pypi/opencv-python@4.5.4.58": {
59+
"package_url": "pkg:pypi/opencv-python@4.5.4.58",
60+
"relationship": "direct",
61+
"dependencies": []
62+
},
63+
"pkg:pypi/matplotlib@3.5.0": {
64+
"package_url": "pkg:pypi/matplotlib@3.5.0",
65+
"relationship": "direct",
66+
"dependencies": []
67+
},
68+
"pkg:pypi/imageio@2.10.4": {
69+
"package_url": "pkg:pypi/imageio@2.10.4",
70+
"relationship": "direct",
71+
"dependencies": []
72+
},
73+
"pkg:pypi/imageio-ffmpeg@0.4.5": {
74+
"package_url": "pkg:pypi/imageio-ffmpeg@0.4.5",
75+
"relationship": "direct",
76+
"dependencies": []
77+
},
78+
"pkg:pypi/torch-optimizer@0.3.0": {
79+
"package_url": "pkg:pypi/torch-optimizer@0.3.0",
80+
"relationship": "direct",
81+
"dependencies": []
82+
},
83+
"pkg:pypi/setuptools@58.2.0": {
84+
"package_url": "pkg:pypi/setuptools@58.2.0",
85+
"relationship": "direct",
86+
"dependencies": []
87+
},
88+
"pkg:pypi/pymcubes@0.1.2": {
89+
"package_url": "pkg:pypi/pymcubes@0.1.2",
90+
"relationship": "direct",
91+
"dependencies": []
92+
},
93+
"pkg:pypi/pycollada@0.7.1": {
94+
"package_url": "pkg:pypi/pycollada@0.7.1",
95+
"relationship": "direct",
96+
"dependencies": []
97+
},
98+
"pkg:pypi/trimesh@3.9.1": {
99+
"package_url": "pkg:pypi/trimesh@3.9.1",
100+
"relationship": "direct",
101+
"dependencies": []
102+
},
103+
"pkg:pypi/pyglet@1.5.10": {
104+
"package_url": "pkg:pypi/pyglet@1.5.10"
105+
, "relationship": "direct",
106+
"dependencies": []
107+
},
108+
"pkg:pypi/networkx@2.5": {
109+
"package_url": "pkg:pypi/networkx@2.5",
110+
"relationship": "direct",
111+
"dependencies": []
112+
},
113+
"pkg:pypi/plyfile@0.7.2": {
114+
"package_url": "pkg:pypi/plyfile@0.7.2",
115+
"relationship": "direct",
116+
"dependencies": []
117+
},
118+
"pkg:pypi/open3d@0.13.0": {
119+
"package_url": "pkg:pypi/open3d@0.13.0",
120+
"relationship": "direct",
121+
"dependencies": []
122+
},
123+
"pkg:pypi/configargparse@1.5.3": {
124+
"package_url": "pkg:pypi/configargparse@1.5.3",
125+
"relationship": "direct",
126+
"dependencies": []
127+
},
128+
"pkg:pypi/ninja": {
129+
"package_url": "pkg:pypi/ninja",
130+
"relationship": "direct",
131+
"dependencies": []
132+
}
133+
},
134+
"name": "test",
135+
"file": {
136+
"source_location": "test/environment.yaml"
137+
}
138+
})
139+
});

‎condaParser.ts‎

Lines changed: 51 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
1-
const core = require('@actions/core');
2-
const github = require('@actions/github');
3-
const fs = require('fs');
4-
const glob = require('glob');
5-
const yaml = require('yaml');
1+
import * as core from '@actions/core';
2+
import * as yaml from 'yaml';
3+
import * as glob from 'glob';
4+
import * as fs from 'fs';
65

76
import {
87
PackageCache,
@@ -12,75 +11,61 @@ import {
1211
Manifest,
1312
submitSnapshot
1413
} from '@github/dependency-submission-toolkit'
15-
import { YAMLMap } from 'yaml';
1614

17-
/**getManifestFromEnvironmentFile(document, fileName) {
18-
core.debug(`getManifestFromEnvironmentFile processing ${fileName}`);
15+
export default class CondaParser {
1916

20-
let manifest = new Manifest("Environment", fileName);
17+
static searchFiles(filePath = "", filePattern = "") {
18+
core.debug(`Searching for files in ${filePath} with pattern ${filePattern}`);
19+
return glob.sync(`${filePath}/${filePattern}`, {});
20+
}
2121

22+
static getManifestsFromEnvironmentFiles(files: string[]) {
23+
core.debug(`Processing ${files.length} files`);
24+
let manifests: any[] = [];
25+
files?.forEach(filePath => {
26+
core.debug(`Processing ${filePath}`);
27+
const contents = fs.readFileSync(filePath, 'utf8')
28+
manifests.push(CondaParser.getManifestFromYaml(yaml.parse(contents), filePath));
29+
});
30+
return manifests;
31+
}
2232

23-
/**
24-
let manifest = new Manifest(document.name, fileName);
25-
26-
core.debug(`Processing ${document.packages?.length} packages`);
27-
28-
document.packages?.forEach(pkg => {
29-
let packageName = pkg.name;
30-
let packageVersion = pkg.packageVersion;
31-
let referenceLocator = pkg.externalRefs?.find(ref => ref.referenceCategory === "PACKAGE-MANAGER" && ref.referenceType === "purl")?.referenceLocator;
32-
let genericPurl = `pkg:generic/${packageName}@${packageVersion}`;
33-
// SPDX 2.3 defines a purl field
34-
let purl;
35-
if (pkg.purl != undefined) {
36-
purl = pkg.purl;
37-
} else if (referenceLocator != undefined) {
38-
purl = referenceLocator;
39-
} else {
40-
purl = genericPurl;
41-
}
42-
43-
// Working around weird encoding issues from an SBOM generator
44-
// Find the last instance of %40 and replace it with @
45-
purl = replaceVersionEscape(purl);
33+
// Gets a Manifest object from an environment.yaml
34+
static getManifestFromYaml(yaml: any, filePath: string) {
35+
core.debug(`getManifestFromEnvironmentFile processing ${yaml}`);
4636

47-
let relationships = document.relationships?.find(rel => rel.relatedSpdxElement == pkg.SPDXID && rel.relationshipType == "DEPENDS_ON" && rel.spdxElementId != "SPDXRef-RootPackage");
48-
if (relationships != null && relationships.length > 0) {
49-
manifest.addIndirectDependency(new Package(purl));
50-
} else {
37+
let manifest = new Manifest(yaml.name, filePath);
38+
yaml.dependencies?.forEach((dependency: any) => {
39+
let purl = "";
40+
// If it's an object with the collection `pip`, then these are PyPI dependencies
41+
if (dependency instanceof Object && dependency.pip != null) {
42+
dependency.pip.forEach((pipDependency:string) => {
43+
purl = this.getPurlFromDependency(pipDependency, "pypi");
44+
manifest.addDirectDependency(new Package(purl));
45+
});
46+
} else {
47+
purl = this.getPurlFromDependency(dependency, "conda");
5148
manifest.addDirectDependency(new Package(purl));
52-
}
49+
}
5350
});
5451
return manifest;
55-
}*/
56-
57-
/***/
58-
59-
export default class CondaParser {
60-
61-
static searchFiles(filePath = "", filePattern = "") {
62-
if (filePath == "") {
63-
let filePath = core.getInput('filePath');
64-
}
65-
if (filePattern == "") {
66-
let filePattern = core.getInput('filePattern');
67-
}
68-
69-
return glob.sync(`${filePath}/${filePattern}`, {});
70-
}
52+
}
53+
54+
// Gets a purl string from an environment file's list of dependencies
55+
static getPurlFromDependency(dependency: string, ecosystem: string) {
56+
// Versions look like "python=3.8" or if nested in a pip section like "pytorch==1.0"
57+
// Split on the '=' to separate the packageName and version
58+
let delimiter = (ecosystem == "pypi") ? "==" : "=";
59+
let split = dependency.split(delimiter);
60+
let packageName = split[0];
61+
// If there is a version specified, use it, otherwise, leave it off.
62+
let version = (split.length > 1) ? `@${split[1]}` : "";
7163

72-
static getManifestsFromEnvironmentFiles(files:string[]) {
73-
core.debug(`Processing ${files.length} files`);
74-
let manifests: any[] = [];
75-
files?.forEach(filePath => {
76-
core.debug(`Processing ${filePath}`);
77-
const contents = fs.readFileSync(filePath, 'utf8')
78-
manifests.push(yaml.parse(contents));
79-
});
80-
return manifests;
81-
}
64+
// If it's a PyPI dependency, then normalize the package name
65+
if (ecosystem == "pypi") {
66+
packageName = packageName.toLowerCase().replace("_", "-");
67+
}
8268

83-
static getManifestFromYaml(yaml:any) {
84-
85-
}
69+
return `pkg:${ecosystem}/${packageName}${version}`;
70+
}
8671
}

‎dist/condaParser.d.ts‎

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎dist/index.d.ts‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)