diff --git a/lib/hypatia/spdx.ex b/lib/hypatia/spdx.ex new file mode 100644 index 00000000..07bd6535 --- /dev/null +++ b/lib/hypatia/spdx.ex @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: MPL-2.0 +# Copyright (c) 2026 Jonathan D.A. Jewell (hyperpolymath) + +defmodule Hypatia.SPDX do + @moduledoc """ + SPDX licence identifiers, from a vendored copy of the SPDX License List. + + The list lives in `priv/spdx/license-ids.txt` and is read at compile time, + so the escript carries it without needing `priv/` at run time. Refresh it + with `scripts/refresh-spdx-license-ids.sh ` when SPDX publishes a new + release; the first line of the file names the version in use. + """ + + @list_path Path.expand("../../priv/spdx/license-ids.txt", __DIR__) + @external_resource @list_path + + {header, ids} = + @list_path + |> File.read!() + |> String.split("\n", trim: true) + |> Enum.split_with(&String.starts_with?(&1, "#")) + + if length(ids) < 500 do + raise "#{@list_path} holds only #{length(ids)} identifiers; regenerate it" + end + + @ids MapSet.new(ids) + @version_line hd(header) + + @doc """ + Whether `id` is an SPDX licence identifier on the vendored list, or a + REUSE custom licence (`LicenseRef-` followed by letters, digits, `.` or + `-`). Matching is exact, as REUSE requires for `LICENSES/` file names. + Deprecated identifiers count: they still name a licence. + """ + def licence_id?(id) when is_binary(id) do + MapSet.member?(@ids, id) or Regex.match?(~r/\ALicenseRef-[A-Za-z0-9.\-]+\z/, id) + end + + @doc "The header line naming the SPDX License List version in use." + def list_version, do: @version_line +end diff --git a/lib/rules/root_hygiene.ex b/lib/rules/root_hygiene.ex index 48d25fc8..d43b27db 100644 --- a/lib/rules/root_hygiene.ex +++ b/lib/rules/root_hygiene.ex @@ -497,10 +497,16 @@ defmodule Hypatia.Rules.RootHygiene do end) end - # A REUSE `LICENSES/` directory counts only if it holds a licence text: a - # non-empty `.` file, per the REUSE spec. Dotfiles (such as - # `.gitkeep`) and READMEs do not count. - defp reuse_licenses?(repo_path) do + @doc """ + Whether `repo_path` has a REUSE `LICENSES/` directory holding a licence + text: a non-empty `.` file, per the REUSE spec. The stem must + be an identifier on the vendored SPDX License List or a `LicenseRef-` + (`Hypatia.SPDX.licence_id?/1`), so `notes.txt`, READMEs and dotfiles such + as `.gitkeep` do not count. + + Shared with `Hypatia.ScorecardIngestor`, so both licence checks agree. + """ + def reuse_licenses?(repo_path) do dir = Path.join(repo_path, "LICENSES") case File.ls(dir) do @@ -512,9 +518,8 @@ defmodule Hypatia.Rules.RootHygiene do defp reuse_licence_text?(dir, name) do path = Path.join(dir, name) - not String.starts_with?(name, ".") and - not String.starts_with?(String.downcase(name), "readme") and - Path.extname(name) != "" and + Path.extname(name) != "" and + Hypatia.SPDX.licence_id?(Path.rootname(name)) and File.regular?(path) and match?({:ok, %File.Stat{size: size}} when size > 0, File.stat(path)) end diff --git a/lib/rules/secret_scanning_alerts.ex b/lib/rules/secret_scanning_alerts.ex index 7121ede7..91aec1e5 100644 --- a/lib/rules/secret_scanning_alerts.ex +++ b/lib/rules/secret_scanning_alerts.ex @@ -36,6 +36,10 @@ defmodule Hypatia.Rules.SecretScanningAlerts do @github_api_base "https://api.github.com" @max_alerts_per_repo 100 + # Pages followed before giving up loudly (100 alerts per page). + @max_pages 50 + # Per-request transfer deadline, so one stalled repo cannot hang a sweep. + @request_timeout_seconds 30 # Stale thresholds (days). A revoked secret left in history is still a # finding -- but a fresh open alert is much more urgent. @@ -319,31 +323,111 @@ defmodule Hypatia.Rules.SecretScanningAlerts do "#{@github_api_base}/repos/#{owner}/#{repo}/secret-scanning/alerts" <> "?per_page=#{@max_alerts_per_repo}" - # No `-f`: on an HTTP error GitHub's JSON body carries the reason - # (for example "Secret scanning is disabled on this repository"), - # and `-f` would discard it. The status code is appended instead. - case System.cmd( - "curl", - [ - "-s", - "-w", - "\n%{http_code}", - "-H", - "Accept: application/vnd.github+json", - "-H", - "Authorization: Bearer #{token}", - "-H", - "X-GitHub-Api-Version: 2022-11-28", - url - ], - stderr_to_stdout: true - ) do - {output, 0} -> parse_response(output) - {error, _} -> {:error, "curl failed: #{String.slice(error, 0, 200)}"} - end + collect_pages(url, &request(&1, token)) + end + end + + @doc false + # Follow GitHub's `Link: rel="next"` pages until the last one. A list cut + # short at one page would report older open alerts as absent, so hitting + # the page cap is an error, never a truncated success. Every `next` URL + # must stay on the API host: the token goes with each request. + def collect_pages(url, request, acc \\ [], page \\ 1) + + def collect_pages(_url, _request, _acc, page) when page > @max_pages do + {:error, + "more than #{@max_pages * @max_alerts_per_repo} alerts -- " <> + "refusing to report a truncated list"} + end + + def collect_pages(url, request, acc, page) do + with {:ok, output} <- request.(url), + {:ok, alerts, next} <- parse_page(output) do + cond do + next == nil -> + {:ok, acc ++ alerts} + + String.starts_with?(next, @github_api_base <> "/") -> + collect_pages(next, request, acc ++ alerts, page + 1) + + true -> + {:error, "refusing to follow a pagination link off #{@github_api_base}"} + end end end + # No `-f`: on an HTTP error GitHub's JSON body carries the reason (for + # example "Secret scanning is disabled on this repository"), and `-f` would + # discard it. `-i` keeps the headers for the `Link` pagination header, and + # the status code is appended after the body. `--max-time` stops one stalled + # repository from hanging an estate-wide sweep. + defp request(url, token) do + case System.cmd( + "curl", + [ + "-s", + "-i", + "--max-time", + Integer.to_string(@request_timeout_seconds), + "-w", + "\n%{http_code}", + "-H", + "Accept: application/vnd.github+json", + "-H", + "Authorization: Bearer #{token}", + "-H", + "X-GitHub-Api-Version: 2022-11-28", + url + ], + stderr_to_stdout: true + ) do + {output, 0} -> + {:ok, output} + + {_, 28} -> + {:error, "GitHub API request timed out after #{@request_timeout_seconds}s"} + + {error, code} -> + {:error, "curl failed (exit #{code}): #{String.slice(error, 0, 200)}"} + end + end + + @doc false + # `output` is curl's `-i` output: one or more header blocks (an HTTPS proxy + # adds its own "Connection established" block first), the body, then a + # newline and the HTTP status. Returns the alerts and the next page's URL. + def parse_page(output) do + {headers, rest} = split_headers(output) + + with {:ok, alerts} <- parse_response(rest) do + {:ok, alerts, next_link(headers)} + end + end + + defp split_headers(output, headers \\ "") do + if String.starts_with?(output, "HTTP/") do + case String.split(output, ~r/\r?\n\r?\n/, parts: 2) do + [block, rest] -> split_headers(rest, block) + [block] -> {block, ""} + end + else + {headers, output} + end + end + + defp next_link(headers) do + headers + |> String.split(~r/\r?\n/) + |> Enum.find_value(fn line -> + with [_, value] <- Regex.run(~r/\Alink:\s*(.*)\z/i, line), + [_, next] <- Regex.run(~r/<([^>]+)>\s*;\s*rel="next"/, value) do + next + else + _ -> nil + end + end) + end + @doc false # `output` is the response body followed by a newline and the HTTP status. def parse_response(output) do diff --git a/lib/scorecard_ingestor.ex b/lib/scorecard_ingestor.ex index e3445ba2..e8c3a874 100644 --- a/lib/scorecard_ingestor.ex +++ b/lib/scorecard_ingestor.ex @@ -425,11 +425,14 @@ defmodule Hypatia.ScorecardIngestor do Path.join(repo_path, "LICENSE"), Path.join(repo_path, "LICENSE.txt"), Path.join(repo_path, "LICENSE.md"), + Path.join(repo_path, "LICENSE.adoc"), Path.join(repo_path, "LICENCE"), Path.join(repo_path, "COPYING") ] - unless Enum.any?(license_paths, &File.exists?/1) do + # Same REUSE rule as root hygiene, so the two never disagree. + unless Enum.any?(license_paths, &File.exists?/1) or + Hypatia.Rules.RootHygiene.reuse_licenses?(repo_path) do make_pattern("SC-010", "License", repo_name, "No LICENSE file found in #{repo_name}") end end diff --git a/priv/spdx/license-ids.txt b/priv/spdx/license-ids.txt new file mode 100644 index 00000000..ace1728c --- /dev/null +++ b/priv/spdx/license-ids.txt @@ -0,0 +1,743 @@ +# SPDX License List 3.29.0 (released 2026-09-16T00:00:00Z): licence identifiers, one per line. +# Source: https://raw.githubusercontent.com/spdx/license-list-data/v3.29.0/json/licenses.json +# Generated by scripts/refresh-spdx-license-ids.sh; do not edit by hand. +0BSD +3D-Slicer-1.0 +AAL +ADSL +AFL-1.1 +AFL-1.2 +AFL-2.0 +AFL-2.1 +AFL-3.0 +AGPL-1.0 +AGPL-1.0-only +AGPL-1.0-or-later +AGPL-3.0 +AGPL-3.0-only +AGPL-3.0-or-later +ALGLIB-Documentation +AMD-newlib +AMDPLPA +AML +AML-glslang +AMPAS +ANTLR-PD +ANTLR-PD-fallback +APAFML +APL-1.0 +APSL-1.0 +APSL-1.1 +APSL-1.2 +APSL-2.0 +ASWF-Digital-Assets-1.0 +ASWF-Digital-Assets-1.1 +Abstyles +AdaCore-doc +Adobe-2006 +Adobe-Display-PostScript +Adobe-Glyph +Adobe-Utopia +Advanced-Cryptics-Dictionary +Afmparse +Aladdin +Apache-1.0 +Apache-1.1 +Apache-2.0 +App-s2p +Arphic-1999 +Artistic-1.0 +Artistic-1.0-Perl +Artistic-1.0-cl8 +Artistic-2.0 +Artistic-dist +Aspell-RU +BOLA-1.1 +BSD-1-Clause +BSD-2-Clause +BSD-2-Clause-Darwin +BSD-2-Clause-FreeBSD +BSD-2-Clause-NetBSD +BSD-2-Clause-Patent +BSD-2-Clause-Views +BSD-2-Clause-first-lines +BSD-2-Clause-pkgconf-disclaimer +BSD-2-Clause-pos-unchanged +BSD-3-Clause +BSD-3-Clause-Attribution +BSD-3-Clause-Clear +BSD-3-Clause-HP +BSD-3-Clause-LBNL +BSD-3-Clause-Modification +BSD-3-Clause-No-Military-License +BSD-3-Clause-No-Nuclear-License +BSD-3-Clause-No-Nuclear-License-2014 +BSD-3-Clause-No-Nuclear-Warranty +BSD-3-Clause-Open-MPI +BSD-3-Clause-OpenWebUI +BSD-3-Clause-Sun +BSD-3-Clause-Tso +BSD-3-Clause-acpica +BSD-3-Clause-flex +BSD-4-Clause +BSD-4-Clause-Shortened +BSD-4-Clause-UC +BSD-4.3RENO +BSD-4.3TAHOE +BSD-Advertising-Acknowledgement +BSD-Attribution-HPND-disclaimer +BSD-Inferno-Nettverk +BSD-Mark-Modifications +BSD-Protection +BSD-Source-Code +BSD-Source-Code-no-disclaimer +BSD-Source-alt-GPL +BSD-Source-beginning-file +BSD-Systemics +BSD-Systemics-W3Works +BSD-ask-to-endorse +BSL-1.0 +BUSL-1.1 +Baekmuk +Bahyph +Barr +Beerware +BitTorrent-1.0 +BitTorrent-1.1 +Bitstream-Charter +Bitstream-Vera +BlueOak-1.0.0 +Boehm-GC +Boehm-GC-without-fee +Borceux +Brian-Gladman-2-Clause +Brian-Gladman-3-Clause +Brian-Gladman-3-Clause-no-conversion +Buddy +Bugroff +C-UDA-1.0 +CAL-1.0 +CAL-1.0-Combined-Work-Exception +CAPEC-tou +CATOSL-1.1 +CC-BY-1.0 +CC-BY-2.0 +CC-BY-2.5 +CC-BY-2.5-AU +CC-BY-3.0 +CC-BY-3.0-AT +CC-BY-3.0-AU +CC-BY-3.0-DE +CC-BY-3.0-IGO +CC-BY-3.0-NL +CC-BY-3.0-US +CC-BY-4.0 +CC-BY-NC-1.0 +CC-BY-NC-2.0 +CC-BY-NC-2.5 +CC-BY-NC-3.0 +CC-BY-NC-3.0-DE +CC-BY-NC-3.0-IGO +CC-BY-NC-4.0 +CC-BY-NC-ND-1.0 +CC-BY-NC-ND-2.0 +CC-BY-NC-ND-2.5 +CC-BY-NC-ND-3.0 +CC-BY-NC-ND-3.0-DE +CC-BY-NC-ND-3.0-IGO +CC-BY-NC-ND-4.0 +CC-BY-NC-SA-1.0 +CC-BY-NC-SA-2.0 +CC-BY-NC-SA-2.0-DE +CC-BY-NC-SA-2.0-FR +CC-BY-NC-SA-2.0-UK +CC-BY-NC-SA-2.5 +CC-BY-NC-SA-3.0 +CC-BY-NC-SA-3.0-DE +CC-BY-NC-SA-3.0-IGO +CC-BY-NC-SA-4.0 +CC-BY-ND-1.0 +CC-BY-ND-2.0 +CC-BY-ND-2.5 +CC-BY-ND-3.0 +CC-BY-ND-3.0-DE +CC-BY-ND-4.0 +CC-BY-SA-1.0 +CC-BY-SA-2.0 +CC-BY-SA-2.0-UK +CC-BY-SA-2.1-JP +CC-BY-SA-2.5 +CC-BY-SA-3.0 +CC-BY-SA-3.0-AT +CC-BY-SA-3.0-DE +CC-BY-SA-3.0-IGO +CC-BY-SA-4.0 +CC-PDDC +CC-PDM-1.0 +CC-SA-1.0 +CC0-1.0 +CDDL-1.0 +CDDL-1.1 +CDL-1.0 +CDLA-Permissive-1.0 +CDLA-Permissive-2.0 +CDLA-Sharing-1.0 +CECILL-1.0 +CECILL-1.1 +CECILL-2.0 +CECILL-2.1 +CECILL-B +CECILL-C +CERN-OHL-1.1 +CERN-OHL-1.2 +CERN-OHL-P-2.0 +CERN-OHL-S-2.0 +CERN-OHL-W-2.0 +CFITSIO +CMU-Mach +CMU-Mach-nodoc +CNRI-Jython +CNRI-Python +CNRI-Python-GPL-Compatible +COIL-1.0 +CPAL-1.0 +CPL-1.0 +CPOL-1.02 +CUA-OPL-1.0 +Caldera +Caldera-no-preamble +Catharon +ClArtistic +Clips +Community-Spec-1.0 +Condor-1.1 +Cornell-Lossless-JPEG +Cronyx +Crossword +CryptoSwift +CrystalStacker +Cube +D-FSL-1.0 +DEC-3-Clause +DL-DE-BY-2.0 +DL-DE-ZERO-2.0 +DOC +DRL-1.0 +DRL-1.1 +DSDP +DocBook-DTD +DocBook-Schema +DocBook-Stylesheet +DocBook-XML +Dotseqn +ECL-1.0 +ECL-2.0 +EFL-1.0 +EFL-2.0 +EPICS +EPL-1.0 +EPL-2.0 +ESA-PL-permissive-2.4 +ESA-PL-strong-copyleft-2.4 +ESA-PL-weak-copyleft-2.4 +EUDatagrid +EUPL-1.0 +EUPL-1.1 +EUPL-1.2 +Elastic-2.0 +Entessa +ErlPL-1.1 +Eurosym +FBM +FDK-AAC +FDK-MPEG-H +FSFAP +FSFAP-no-warranty-disclaimer +FSFUL +FSFULLR +FSFULLRSD +FSFULLRWD +FSL-1.1-ALv2 +FSL-1.1-MIT +FTL +Fair +Ferguson-Twofish +Frameworx-1.0 +FreeBSD-DOC +FreeImage +Furuseth +GCR-docs +GD +GFDL-1.1 +GFDL-1.1-invariants-only +GFDL-1.1-invariants-or-later +GFDL-1.1-no-invariants-only +GFDL-1.1-no-invariants-or-later +GFDL-1.1-only +GFDL-1.1-or-later +GFDL-1.2 +GFDL-1.2-invariants-only +GFDL-1.2-invariants-or-later +GFDL-1.2-no-invariants-only +GFDL-1.2-no-invariants-or-later +GFDL-1.2-only +GFDL-1.2-or-later +GFDL-1.3 +GFDL-1.3-invariants-only +GFDL-1.3-invariants-or-later +GFDL-1.3-no-invariants-only +GFDL-1.3-no-invariants-or-later +GFDL-1.3-only +GFDL-1.3-or-later +GL2PS +GLWTPL +GPL-1.0 +GPL-1.0+ +GPL-1.0-only +GPL-1.0-or-later +GPL-2.0 +GPL-2.0+ +GPL-2.0-only +GPL-2.0-or-later +GPL-2.0-with-GCC-exception +GPL-2.0-with-autoconf-exception +GPL-2.0-with-bison-exception +GPL-2.0-with-classpath-exception +GPL-2.0-with-font-exception +GPL-3.0 +GPL-3.0+ +GPL-3.0-only +GPL-3.0-or-later +GPL-3.0-with-GCC-exception +GPL-3.0-with-autoconf-exception +Game-Programming-Gems +Giftware +Glide +Glulxe +Graphics-Gems +Gutmann +HDF5 +HIDAPI +HP-1986 +HP-1989 +HPND +HPND-DEC +HPND-Fenneberg-Livingston +HPND-INRIA-IMAG +HPND-Intel +HPND-Kevlin-Henney +HPND-MIT-disclaimer +HPND-Markus-Kuhn +HPND-Netrek +HPND-Pbmplus +HPND-SMC +HPND-UC +HPND-UC-export-US +HPND-doc +HPND-doc-sell +HPND-export-US +HPND-export-US-acknowledgement +HPND-export-US-modify +HPND-export2-US +HPND-merchantability-variant +HPND-sell-MIT-disclaimer-xserver +HPND-sell-regexpr +HPND-sell-variant +HPND-sell-variant-MIT-disclaimer +HPND-sell-variant-MIT-disclaimer-rev +HPND-sell-variant-critical-systems +HTMLTIDY +HaskellReport +Hippocratic-2.1 +Hippocratic-3.0-core +IBM-pibs +ICU +IEC-Code-Components-EULA +IJG +IJG-short +IPA +IPL-1.0 +ISC +ISC-Veillard +ISO-permission +ImageMagick +Imlib2 +Info-ZIP +Informatica +Inner-Net-2.0 +InnoSetup +Intel +Intel-ACPI +Interbase-1.0 +JPL-image +JPNIC +JSON +Jam +JasPer-2.0 +Kastrup +Kazlib +Knuth-CTAN +LAL-1.2 +LAL-1.3 +LGPL-2.0 +LGPL-2.0+ +LGPL-2.0-only +LGPL-2.0-or-later +LGPL-2.1 +LGPL-2.1+ +LGPL-2.1-only +LGPL-2.1-or-later +LGPL-3.0 +LGPL-3.0+ +LGPL-3.0-only +LGPL-3.0-or-later +LGPLLR +LOOP +LPD-document +LPL-1.0 +LPL-1.02 +LPPL-1.0 +LPPL-1.1 +LPPL-1.2 +LPPL-1.3a +LPPL-1.3c +LZMA-SDK-9.11-to-9.20 +LZMA-SDK-9.22 +Latex2e +Latex2e-translated-notice +Leptonica +LiLiQ-P-1.1 +LiLiQ-R-1.1 +LiLiQ-Rplus-1.1 +Libpng +Linux-OpenIB +Linux-man-pages-1-para +Linux-man-pages-copyleft +Linux-man-pages-copyleft-2-para +Linux-man-pages-copyleft-var +Lucida-Bitmap-Fonts +MIPS +MIT +MIT-0 +MIT-CMU +MIT-Click +MIT-Festival +MIT-Khronos-old +MIT-Modern-Variant +MIT-STK +MIT-Wu +MIT-advertising +MIT-enna +MIT-feh +MIT-open-group +MIT-testregex +MITNFA +MMIXware +MMPL-1.0.1 +MPEG-SSG +MPL-1.0 +MPL-1.1 +MPL-2.0 +MPL-2.0-no-copyleft-exception +MS-LPL +MS-PL +MS-RL +MTLL +MVT-1.1 +Mackerras-3-Clause +Mackerras-3-Clause-acknowledgment +MakeIndex +Martin-Birgmeier +McPhee-slideshow +Minpack +MirOS +Motosoto +MulanPSL-1.0 +MulanPSL-2.0 +Multics +Mup +NAIST-2003 +NASA-1.3 +NBPL-1.0 +NCBI-PD +NCGL-UK-2.0 +NCL +NCSA +NGPL +NICTA-1.0 +NIST-PD +NIST-PD-TNT +NIST-PD-fallback +NIST-Software +NLOD-1.0 +NLOD-2.0 +NLPL +NOSL +NPL-1.0 +NPL-1.1 +NPOSL-3.0 +NRL +NTIA-PD +NTP +NTP-0 +Naumen +Net-SNMP +NetCDF +Newsletr +Nokia +Noweb +Nunit +O-UDA-1.0 +OAR +OCCT-PL +OCLC-2.0 +ODC-By-1.0 +ODbL-1.0 +OFFIS +OFL-1.0 +OFL-1.0-RFN +OFL-1.0-no-RFN +OFL-1.1 +OFL-1.1-RFN +OFL-1.1-no-RFN +OGC-1.0 +OGDL-Taiwan-1.0 +OGL-Canada-2.0 +OGL-UK-1.0 +OGL-UK-2.0 +OGL-UK-3.0 +OGTSL +OLDAP-1.1 +OLDAP-1.2 +OLDAP-1.3 +OLDAP-1.4 +OLDAP-2.0 +OLDAP-2.0.1 +OLDAP-2.1 +OLDAP-2.2 +OLDAP-2.2.1 +OLDAP-2.2.2 +OLDAP-2.3 +OLDAP-2.4 +OLDAP-2.5 +OLDAP-2.6 +OLDAP-2.7 +OLDAP-2.8 +OLFL-1.3 +OML +OPL-1.0 +OPL-UK-3.0 +OPUBL-1.0 +OSC-1.0 +OSET-PL-2.1 +OSL-1.0 +OSL-1.1 +OSL-2.0 +OSL-2.1 +OSL-3.0 +OSSP +OpenMDW-1.0 +OpenPBS-2.3 +OpenSSL +OpenSSL-standalone +OpenVision +PADL +PDDL-1.0 +PHP-3.0 +PHP-3.01 +PPL +PSF-2.0 +ParaType-Free-Font-1.3 +Parity-6.0.0 +Parity-7.0.0 +Pixar +Plexus +PolyForm-Noncommercial-1.0.0 +PolyForm-Small-Business-1.0.0 +PostgreSQL +Python-2.0 +Python-2.0.1 +QPL-1.0 +QPL-1.0-INRIA-2004 +Qhull +RHeCos-1.1 +RPL-1.1 +RPL-1.5 +RPSL-1.0 +RSA-MD +RSCPL +Rdisc +Ruby +Ruby-pty +SAX-PD +SAX-PD-2.0 +SCEA +SGI-B-1.0 +SGI-B-1.1 +SGI-B-2.0 +SGI-OpenGL +SGMLUG-PM +SGP4 +SHL-0.5 +SHL-0.51 +SISSL +SISSL-1.2 +SL +SMAIL-GPL +SMLNJ +SMPPL +SNIA +SOFA +SPL-1.0 +SSH-OpenSSH +SSH-short +SSLeay-standalone +SSPL-1.0 +SUL-1.0 +SWL +Saxpath +SchemeReport +Sendmail +Sendmail-8.23 +Sendmail-Open-Source-1.1 +SimPL-2.0 +Sleepycat +Soundex +Spencer-86 +Spencer-94 +Spencer-99 +StandardML-NJ +SugarCRM-1.1.3 +Sun-PPP +Sun-PPP-2000 +SunPro +Symlinks +TAPR-OHL-1.0 +TCL +TCP-wrappers +TGPPL-1.0 +TMate +TORQUE-1.1 +TOSL +TPDL +TPL-1.0 +TTWL +TTYP0 +TU-Berlin-1.0 +TU-Berlin-2.0 +TekHVC +TermReadKey +ThirdEye +TrustedQSL +UCAR +UCL-1.0 +UMich-Merit +UPL-1.0 +URT-RLE +Ubuntu-font-1.0 +UnRAR +Unicode-3.0 +Unicode-DFS-2015 +Unicode-DFS-2016 +Unicode-TOU +UnixCrypt +Unlicense +Unlicense-libtelnet +Unlicense-libwhirlpool +VOSTROM +VSL-1.0 +Vim +Vixie-Cron +W3C +W3C-19980720 +W3C-20150513 +WTFNMFPL +WTFPL +Watcom-1.0 +Widget-Workshop +WordNet +Wsuipa +X11 +X11-distribute-modifications-variant +X11-no-permit-persons +X11-swapped +XFree86-1.1 +XSkat +Xdebug-1.03 +Xerox +Xfig +Xnet +YPL-1.0 +YPL-1.1 +ZPL-1.1 +ZPL-2.0 +ZPL-2.1 +Zed +Zeeff +Zend-2.0 +Zimbra-1.3 +Zimbra-1.4 +Zlib +any-OSI +any-OSI-perl-modules +atc-game +bcrypt-Solar-Designer +blessing +bzip2-1.0.5 +bzip2-1.0.6 +check-cvs +checkmk +copyleft-next-0.3.0 +copyleft-next-0.3.1 +curl +cve-tou +diffmark +dtoa +dvipdfm +eCos-2.0 +eGenix +etalab-2.0 +fwlw +gSOAP-1.3b +generic-xts +gnuplot +gtkbook +hdparm +hyphen-bulgarian +iMatix +jove +libpng-1.6.35 +libpng-2.0 +libselinux-1.0 +libtiff +libutil-David-Nugent +lsof +magaz +mailprio +man2html +metamail +mpi-permissive +mpich2 +mplus +ngrep +pkgconf +pnmstitch +psfrag +psutils +python-ldap +radvd +snprintf +softSurfer +ssh-keyscan +swrule +threeparttable +ulem +w3m +wwl +wxWindows +xinetd +xkeyboard-config-Zinoviev +xlock +xpp +xzoom +zlib-acknowledgement diff --git a/scripts/refresh-spdx-license-ids.sh b/scripts/refresh-spdx-license-ids.sh new file mode 100755 index 00000000..60c9b9ff --- /dev/null +++ b/scripts/refresh-spdx-license-ids.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# refresh-spdx-license-ids.sh — regenerate priv/spdx/license-ids.txt from a +# tagged release of the SPDX License List (spdx/license-list-data). +# +# Hypatia.SPDX reads that file at compile time to decide whether a REUSE +# `LICENSES/.` file names a real licence. The list grows with each +# SPDX release, so refresh it when a new tag appears: +# +# scripts/refresh-spdx-license-ids.sh v3.29.0 +# +# Then rebuild (`mix compile`) and commit the regenerated file. Deprecated +# identifiers are kept: older repositories still ship `LICENSES/GPL-3.0.txt`. + +set -euo pipefail + +TAG="${1:?usage: $0 }" +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +OUT="${ROOT}/priv/spdx/license-ids.txt" +URL="https://raw.githubusercontent.com/spdx/license-list-data/${TAG}/json/licenses.json" + +TMP="$(mktemp)" +trap 'rm -f "$TMP"' EXIT + +curl -sSfL --max-time 60 -o "$TMP" "$URL" + +# Fail loudly on a malformed or unexpectedly small download rather than +# shipping a list that would reject real licences. +COUNT="$(jq '.licenses | length' "$TMP")" +if [ "$COUNT" -lt 500 ]; then + echo "error: ${URL} lists only ${COUNT} licences; refusing to write it" >&2 + exit 1 +fi + +VERSION="$(jq -r '.licenseListVersion' "$TMP")" +RELEASED="$(jq -r '.releaseDate' "$TMP")" + +mkdir -p "$(dirname "$OUT")" +{ + echo "# SPDX License List ${VERSION} (released ${RELEASED}): licence identifiers, one per line." + echo "# Source: ${URL}" + echo "# Generated by scripts/refresh-spdx-license-ids.sh; do not edit by hand." + jq -r '.licenses[].licenseId' "$TMP" | LC_ALL=C sort -u +} > "$OUT" + +echo "wrote ${OUT}: ${COUNT} identifiers from SPDX License List ${VERSION}" diff --git a/test/root_hygiene_test.exs b/test/root_hygiene_test.exs index 53c92088..4cf213ff 100644 --- a/test/root_hygiene_test.exs +++ b/test/root_hygiene_test.exs @@ -246,6 +246,8 @@ defmodule Hypatia.Rules.RootHygieneTest do File.write!(Path.join(licenses, "README.md"), "see the licence texts") File.write!(Path.join(licenses, "MPL-2.0.txt"), "") File.write!(Path.join(licenses, "NOTES"), "no extension") + # Named like a licence file, but `notes` is no SPDX identifier. + File.write!(Path.join(licenses, "notes.txt"), "build notes") result = RootHygiene.scan([".editorconfig", "demo_chora.deed"], repo) assert [%{file: "LICENSE", severity: :critical}] = result.findings diff --git a/test/scorecard_ingestor_license_test.exs b/test/scorecard_ingestor_license_test.exs new file mode 100644 index 00000000..286bf873 --- /dev/null +++ b/test/scorecard_ingestor_license_test.exs @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: MPL-2.0 +defmodule Hypatia.ScorecardIngestorLicenseTest do + # The estate template ships LICENSE.adoc plus REUSE LICENSES/. Root hygiene + # accepts both; the local Scorecard check still reported SC-010 "No LICENSE + # file found", so one scan gave contradictory licence results. The REUSE + # rule is now shared (RootHygiene.reuse_licenses?/1). + use ExUnit.Case, async: true + + setup do + dir = Path.join(System.tmp_dir!(), "scorecard-license-#{System.unique_integer([:positive])}") + File.mkdir_p!(dir) + on_exit(fn -> File.rm_rf!(dir) end) + %{dir: dir} + end + + defp license_findings(dir) do + {:ok, findings} = Hypatia.ScorecardIngestor.local_scan(dir, "fixture") + Enum.filter(findings, &(&1["scorecard_check"] == "License")) + end + + test "no licence is reported", %{dir: dir} do + assert [_] = license_findings(dir) + end + + for name <- ["LICENSE", "LICENSE.md", "LICENSE.adoc", "COPYING"] do + test "#{name} satisfies the check", %{dir: dir} do + File.write!(Path.join(dir, unquote(name)), "licence text") + assert [] = license_findings(dir) + end + end + + test "a REUSE LICENSES/ licence text satisfies the check", %{dir: dir} do + File.mkdir_p!(Path.join(dir, "LICENSES")) + File.write!(Path.join([dir, "LICENSES", "MPL-2.0.txt"]), "licence text") + assert [] = license_findings(dir) + end + + test "a LicenseRef- custom licence text satisfies the check", %{dir: dir} do + File.mkdir_p!(Path.join(dir, "LICENSES")) + File.write!(Path.join([dir, "LICENSES", "LicenseRef-Estate-1.0.txt"]), "licence text") + assert [] = license_findings(dir) + end + + test "a LICENSES/ file whose name is not an SPDX identifier does not", %{dir: dir} do + File.mkdir_p!(Path.join(dir, "LICENSES")) + File.write!(Path.join([dir, "LICENSES", "notes.txt"]), "build notes") + assert [_] = license_findings(dir) + end + + test "a LICENSES/ holding only a placeholder does not", %{dir: dir} do + File.mkdir_p!(Path.join(dir, "LICENSES")) + File.write!(Path.join([dir, "LICENSES", ".gitkeep"]), "") + assert [_] = license_findings(dir) + end +end diff --git a/test/secret_scanning_alerts_test.exs b/test/secret_scanning_alerts_test.exs index 572a0b6e..f3b2d5cb 100644 --- a/test/secret_scanning_alerts_test.exs +++ b/test/secret_scanning_alerts_test.exs @@ -116,6 +116,106 @@ defmodule Hypatia.Rules.SecretScanningAlertsTest do end end + describe "parse_page/1" do + test "reads the alerts and the next page from curl -i output" do + output = + "HTTP/2 200\r\n" <> + "content-type: application/json\r\n" <> + ~s(link: ; rel="next", ) <> + ~s(; rel="last"\r\n) <> + "\r\n" <> ~s([{"number":1}]) <> "\n200" + + assert {:ok, [%{"number" => 1}], + "https://api.github.com/repositories/1/secret-scanning/alerts?page=2"} = + SecretScanningAlerts.parse_page(output) + end + + test "the last page has no next link" do + output = "HTTP/2 200\r\ncontent-type: application/json\r\n\r\n[]\n200" + assert {:ok, [], nil} = SecretScanningAlerts.parse_page(output) + end + + test "skips an HTTPS proxy's own header block" do + output = + "HTTP/1.1 200 Connection established\r\n\r\n" <> + "HTTP/2 200\r\ncontent-type: application/json\r\n\r\n" <> + ~s([{"number":7}]) <> "\n200" + + assert {:ok, [%{"number" => 7}], nil} = SecretScanningAlerts.parse_page(output) + end + + test "an error page is an error" do + output = + "HTTP/2 404\r\ncontent-type: application/json\r\n\r\n" <> + ~s({"message":"Secret scanning is disabled on this repository."}) <> "\n404" + + assert {:error, "GitHub API 404: Secret scanning is disabled on this repository."} = + SecretScanningAlerts.parse_page(output) + end + end + + describe "collect_pages/2" do + @base "https://api.github.com/repos/acme/demo/secret-scanning/alerts?per_page=100" + + defp page(alerts, next) do + link = if next, do: ~s(link: <#{next}>; rel="next"\r\n), else: "" + "HTTP/2 200\r\n" <> link <> "\r\n" <> alerts <> "\n200" + end + + test "follows next links and returns every page's alerts" do + second = "https://api.github.com/repositories/1/secret-scanning/alerts?page=2" + + request = fn + @base -> {:ok, page(~s([{"number":1},{"number":2}]), second)} + ^second -> {:ok, page(~s([{"number":3}]), nil)} + end + + assert {:ok, alerts} = SecretScanningAlerts.collect_pages(@base, request) + assert Enum.map(alerts, & &1["number"]) == [1, 2, 3] + end + + test "never sends the token to a next link off the API host" do + request = fn @base -> {:ok, page("[]", "https://evil.example.com/steal")} end + + assert {:error, "refusing to follow a pagination link off https://api.github.com"} = + SecretScanningAlerts.collect_pages(@base, request) + end + + test "a failed later page fails the whole fetch" do + second = "https://api.github.com/repositories/1/secret-scanning/alerts?page=2" + + request = fn + @base -> {:ok, page(~s([{"number":1}]), second)} + ^second -> {:error, "GitHub API request timed out after 30s"} + end + + assert {:error, "GitHub API request timed out after 30s"} = + SecretScanningAlerts.collect_pages(@base, request) + end + + test "past the page cap it is an error, never a truncated list" do + request = fn _ -> flunk("must not request past the cap") end + + assert {:error, "more than 5000 alerts" <> _} = + SecretScanningAlerts.collect_pages(@base, request, [], 51) + end + + test "stops after exactly 50 requests when every page has a next link" do + {:ok, counter} = Agent.start_link(fn -> 0 end) + next = "https://api.github.com/repositories/1/secret-scanning/alerts?page=n" + + request = fn _url -> + Agent.update(counter, &(&1 + 1)) + {:ok, page("[]", next)} + end + + assert {:error, "more than 5000 alerts" <> _} = + SecretScanningAlerts.collect_pages(@base, request) + + assert Agent.get(counter, & &1) == 50 + end + end + describe "findings_from_alerts/3" do defp days_ago(n), do: DateTime.utc_now() |> DateTime.add(-n * 86_400) |> DateTime.to_iso8601() diff --git a/test/spdx_test.exs b/test/spdx_test.exs new file mode 100644 index 00000000..0275d1e5 --- /dev/null +++ b/test/spdx_test.exs @@ -0,0 +1,37 @@ +# SPDX-License-Identifier: MPL-2.0 +defmodule Hypatia.SPDXTest do + use ExUnit.Case, async: true + + alias Hypatia.SPDX + + test "identifiers on the SPDX License List are licence ids" do + for id <- ["MPL-2.0", "MIT", "Apache-2.0", "CC-BY-SA-4.0", "0BSD"] do + assert SPDX.licence_id?(id), id + end + end + + test "lowercase identifiers on the list count, so no case heuristic is needed" do + assert SPDX.licence_id?("curl") + assert SPDX.licence_id?("blessing") + end + + test "deprecated identifiers still name a licence" do + assert SPDX.licence_id?("GPL-3.0") + end + + test "REUSE custom licences are LicenseRef- ids" do + assert SPDX.licence_id?("LicenseRef-Estate-1.0") + refute SPDX.licence_id?("LicenseRef-") + refute SPDX.licence_id?("LicenseRef-has space") + end + + test "other names are not, and matching is exact" do + for id <- ["notes", "README", "LICENSE", "mpl-2.0", "MPL-2", ""] do + refute SPDX.licence_id?(id), id + end + end + + test "the list version is recorded" do + assert SPDX.list_version() =~ ~r/\A# SPDX License List \d+\.\d+/ + end +end