From 5b1d4130e9b8cb138f8ef0ce72c6dce095e47545 Mon Sep 17 00:00:00 2001 From: Maciek Date: Wed, 10 Jun 2026 14:18:02 +0200 Subject: [PATCH 01/15] feat(blocklists): add Security category, Hagezi NRD and new general lists Signed-off-by: Maciek --- blocklists/README.md | 7 +- blocklists/internal/extractor/domains.go | 33 +++++-- blocklists/internal/extractor/domains_test.go | 59 ++++++++++++ blocklists/internal/extractor/extractor.go | 6 ++ .../internal/extractor/extractor_test.go | 18 ++++ blocklists/service/testdata/real/README.md | 2 +- blocklists/sources/3rd_party/1hosts.json | 28 ++++++ blocklists/sources/3rd_party/peter_lowe.json | 15 +++ .../sources/3rd_party/someonewhocares.json | 15 +++ blocklists/sources/security/nrd.json | 92 +++++++++++++++++++ blocklists/sources/security/tif.json | 3 +- 11 files changed, 264 insertions(+), 14 deletions(-) create mode 100644 blocklists/sources/3rd_party/1hosts.json create mode 100644 blocklists/sources/3rd_party/peter_lowe.json create mode 100644 blocklists/sources/3rd_party/someonewhocares.json create mode 100644 blocklists/sources/security/nrd.json diff --git a/blocklists/README.md b/blocklists/README.md index e891c82a..dd05349e 100644 --- a/blocklists/README.md +++ b/blocklists/README.md @@ -63,11 +63,12 @@ The **extractor is selected by the `blocklist_id` prefix** | `hagezi*` | Hagezi | plain domains, `#` headers (strict metadata) | | `oisd*` | OISD | plain domains (`domainswild2`), `#` headers | | `steven_black*` | Steven Black | hosts file (`0.0.0.0 domain`) | -| `blp_*`, `ut1_*`, `shadowwhisperer_*` | Domains | plain domains, graceful metadata, hosts-tolerant | +| `blp_*`, `ut1_*`, `shadowwhisperer_*`, `someonewhocares_*`, `peter_lowe*`, `1hosts_*` | Domains | plain or hosts-format domains, graceful metadata (optional `Last modified`/`Last updated`/`Count` headers), hosts-tolerant | Key source fields: `blocklist_id` (required, routes the extractor), `name`, -`description`, `source_url`, `kind` (`general`/`category`/`security`), -`category`, `intensity`, `default`, `schedule` (cron). See any file under +`description`, `source_url`, `kind` (`general`/`category`), +`category` (e.g. `gambling`, `security`), `intensity`, `default`, +`schedule` (cron). See any file under `sources/` for a complete example. The authoritative behaviour spec is diff --git a/blocklists/internal/extractor/domains.go b/blocklists/internal/extractor/domains.go index e23c8548..3a04e751 100644 --- a/blocklists/internal/extractor/domains.go +++ b/blocklists/internal/extractor/domains.go @@ -11,9 +11,13 @@ import ( ) var ( - // Common header patterns across plain domain list formats - domainsReLastModified = regexp.MustCompile(`(?i)^[#!]\s*Last modified:\s*(.+)`) + // Common header patterns across plain domain list formats. These tolerate the + // variations seen across providers: "Last modified" (Block List Project) and + // "Last updated" (Dan Pollock / Peter Lowe); "Entries"/"Number of Entries" + // (yoyo) and "Count: N rules" with a decorative prefix (1Hosts). + domainsReLastModified = regexp.MustCompile(`(?i)^[#!]\s*Last (?:modified|updated):\s*(.+)`) domainsReEntries = regexp.MustCompile(`(?i)^[#!]\s*(?:Number of )?Entries:\s*([\d,]+)`) + domainsReCount = regexp.MustCompile(`(?i)^[#!].*\bCount:\s*([\d,]+)`) ) // DomainsExtractor implements the Extractor interface for plain domain-per-line @@ -88,11 +92,17 @@ func (e *DomainsExtractor) ExtractMetadata(blocklistBytes []byte) (time.Time, st } } - if matches := domainsReEntries.FindStringSubmatch(line); matches != nil && !foundEntries { - cleaned := strings.ReplaceAll(matches[1], ",", "") - if n, err := strconv.Atoi(cleaned); err == nil { - numEntries = n - foundEntries = true + if !foundEntries { + matches := domainsReEntries.FindStringSubmatch(line) + if matches == nil { + matches = domainsReCount.FindStringSubmatch(line) + } + if matches != nil { + cleaned := strings.ReplaceAll(matches[1], ",", "") + if n, err := strconv.Atoi(cleaned); err == nil { + numEntries = n + foundEntries = true + } } } @@ -124,7 +134,9 @@ func (e *DomainsExtractor) ProcessLine(line string) (string, error) { return stripHostsIP(line), nil } -// parseFlexibleDate tries multiple date formats commonly found in blocklist headers +// parseFlexibleDate tries multiple date formats commonly found in blocklist +// headers. Parsed values are normalized to UTC so callers can compare instants +// without location-pointer differences (e.g. a "GMT" zone vs time.UTC). func parseFlexibleDate(s string) (time.Time, bool) { formats := []string{ "2006-01-02 15:04:05 MST", @@ -135,10 +147,13 @@ func parseFlexibleDate(s string) (time.Time, bool) { "02 Jan 2006", "Jan 02, 2006", time.RFC3339, + time.RFC3339Nano, // 1Hosts: 2026-06-09T10:59:53.132Z + time.RFC1123, // Peter Lowe/yoyo: Tue, 09 Jun 2026 14:53:58 GMT + "Mon, 02 Jan 2006 at 15:04:05 MST", // someonewhocares: Wed, 10 Jun 2026 at 00:35:57 GMT } for _, fmt := range formats { if t, err := time.Parse(fmt, s); err == nil { - return t, true + return t.UTC(), true } } return time.Time{}, false diff --git a/blocklists/internal/extractor/domains_test.go b/blocklists/internal/extractor/domains_test.go index 4f9cae32..3a4ff4fa 100644 --- a/blocklists/internal/extractor/domains_test.go +++ b/blocklists/internal/extractor/domains_test.go @@ -149,6 +149,44 @@ domain3.com`, wantNumEntries: 3, wantErr: false, }, + { + // specRef: #C11 — someonewhocares "Last updated: , at