diff --git a/CHANGELOG.md b/CHANGELOG.md index 935f219..9c9a9e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ repository begins with a 1.1 import, so earlier releases are not reconstructed. ## Unreleased +### 2.3.0 + +- Reject non-string discovery inputs with `PolicyError` instead of + `NoMethodError` during HTTPS normalization. +- Reject overflowing JSON numbers instead of collapsing numeric identifiers to + `Infinity` or exposing extension data that cannot be serialized as JSON. +- Reject invalid UTF-8 in JSON Feed input before exposing strings that can fail + during serialization or downstream processing. +- Handle bodyless successful HTTP responses without `NoMethodError`: `fetch` + reports a malformed feed, while `discover` applies its usual empty-page or + parsing-error behavior. +- Reject out-of-range JSON Feed timestamp hours and offsets as normalization + issues instead of silently shifting dates or treating invalid offsets as UTC. + Use the Gregorian calendar consistently for historical JSON Feed timestamps. - Add `SimpleRSS.discover` for advertised RSS, Atom, and JSON Feed links, using optional Nokogiri HTML5 parsing. Return ordered, deduplicated candidates with titles, type hints, and verification status; recognize direct empty feeds. diff --git a/README.md b/README.md index ed6fd6d..c1c9951 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,22 @@ A simple, flexible, extensible, and liberal RSS, Atom, and JSON Feed reader for - Extensible tag definitions - No mandatory runtime gem dependencies; website discovery uses optional Nokogiri -## What's New in 2.x +## What's New in 2.3.0 + +- **Website discovery** - Find advertised RSS, Atom, and JSON feeds with + `SimpleRSS.discover("example.com")`. Bare domains default to HTTPS, and + requests have destination checks, timeouts, and size limits. Existing `fetch` + callers can opt into these request controls with `network_policy`. +- **Normalized entries** - Use `normalized_entries` for consistent URLs, dates, + content, authors, categories, and attachments while retaining raw feed data. +- **JSON Feed** - Parse JSON Feed 1.0 and 1.1 through the existing `parse` and + `fetch` APIs, with the same normalized entry interface as RSS and Atom. +- **Parser fixes** - Correct Atom category terms and relation links, handle + malformed dates during ordering, and accept self-closing empty feeds. + +See the [2.3.0 release notes](CHANGELOG.md#230) for compatibility details. + +## Earlier 2.x Features See the [changelog](CHANGELOG.md) for release history and unreleased changes. @@ -563,7 +578,9 @@ valid modification date when publication is invalid or absent. UTF-8 strings and readable IO accept ordinary leading JSON whitespace and one UTF-8 BOM at the very start, before whitespace. Embedded or repeated BOMs are -rejected. `source` preserves the original input. This is a parser, not a complete +rejected, as are invalid UTF-8 bytes and numbers that overflow Ruby's floating-point +range. Large integer IDs retain their full precision. `source` preserves the +original input. This is a parser, not a complete standards validator: it does not validate URL reachability, language tags, ID uniqueness across updates, or publisher extension schemas. `SimpleRSS.valid?(source)` reports parseability. A parsed JSON feed's instance diff --git a/lib/simple-rss.rb b/lib/simple-rss.rb index 92f0d2b..cb85db2 100644 --- a/lib/simple-rss.rb +++ b/lib/simple-rss.rb @@ -10,7 +10,7 @@ class SimpleRSS # rubocop:disable Metrics/ClassLength # @rbs! # include Enumerable[Hash[Symbol, untyped]] - VERSION = "2.2.0".freeze + VERSION = "2.3.0".freeze # @rbs @items: Array[Hash[Symbol, untyped]] # @rbs @source: String @@ -321,7 +321,7 @@ def fetch(url, options = {}) raise SimpleRSSError, "HTTP #{response.code}: #{response.message}" unless response.is_a?(Net::HTTPSuccess) - body = response.body.force_encoding(Encoding::UTF_8) + body = (response.body || "").force_encoding(Encoding::UTF_8) feed = parse(body, options.merge(source_url: final_uri.to_s)) feed.instance_variable_set(:@etag, response["ETag"]) feed.instance_variable_set(:@last_modified, response["Last-Modified"]) diff --git a/lib/simple-rss/discovery.rb b/lib/simple-rss/discovery.rb index c5bc205..b9159ac 100644 --- a/lib/simple-rss/discovery.rb +++ b/lib/simple-rss/discovery.rb @@ -25,12 +25,14 @@ def initialize(options) # @rbs (String) -> Array[Hash[Symbol, untyped]] def discover(url) + raise SimpleRSS::PolicyError, "Expected a website URL string" unless url.is_a?(String) + url = "https:#{url}" if url.start_with?("//") url = "https://#{url}" unless url.match?(/\A[a-z][a-z\d+.-]*:/i) response, uri = SimpleRSS::HTTPClient.new(@options).get(url) raise SimpleRSS::HTTPError, response.code.to_i unless response.is_a?(Net::HTTPSuccess) - candidates(response.body, uri, response.content_type, response.type_params["charset"]) + candidates(response.body.to_s, uri, response.content_type, response.type_params["charset"]) end private diff --git a/lib/simple-rss/json_entry_normalizer.rb b/lib/simple-rss/json_entry_normalizer.rb index 2992a46..4aa8fcc 100644 --- a/lib/simple-rss/json_entry_normalizer.rb +++ b/lib/simple-rss/json_entry_normalizer.rb @@ -4,6 +4,9 @@ require "date" class SimpleRSS::JsonEntryNormalizer + RFC3339_TIMESTAMP = /\A\d{4}-\d{2}-\d{2}[tT] + (?:[01]\d|2[0-3]):[0-5]\d:(?:[0-5]\d|60)(?:\.\d+)? + (?:[zZ]|[+-](?:[01]\d|2[0-3]):[0-5]\d)\z/x FIELDS = { title: "title", content_html: "content_html", content_text: "content_text", summary: "summary" }.freeze @@ -66,7 +69,7 @@ def issue(field, code, value, source) def read_date(source, field) value = @item[source] return if value.nil? - return DateTime.rfc3339(value).to_time if value.is_a?(String) && value.match?(/\A\d{4}-\d\d-\d\d[tT]\d\d:\d\d:\d\d(?:\.\d+)?(?:[zZ]|[+-]\d\d:\d\d)\z/) + return DateTime.rfc3339(value, Date::GREGORIAN).to_time if value.is_a?(String) && value.match?(RFC3339_TIMESTAMP) issue(field, :invalid_date, value, source) nil diff --git a/lib/simple-rss/json_feed.rb b/lib/simple-rss/json_feed.rb index 15eafbf..6f13c46 100644 --- a/lib/simple-rss/json_feed.rb +++ b/lib/simple-rss/json_feed.rb @@ -15,7 +15,10 @@ class SimpleRSS::JsonFeed # @rbs (String) -> void def initialize(source) - @document = JSON.parse(source.b.sub(/\A\xEF\xBB\xBF/n, "").force_encoding(Encoding::UTF_8)) + source = source.b.sub(/\A\xEF\xBB\xBF/n, "").force_encoding(Encoding::UTF_8) + raise SimpleRSSError, "Malformed JSON Feed: invalid UTF-8" unless source.valid_encoding? + + @document = JSON.parse(source) validate freeze_data(@document) @originals = {} #: Hash[Hash[Symbol, untyped], Hash[String, untyped]] @@ -40,6 +43,7 @@ def normalized_entry(item, source_url: nil) # @rbs () -> void def validate check_type(document, Hash, "feed") + validate_numbers(document) required_string(document, "version", "feed") raise SimpleRSSError, "Unsupported JSON Feed version: #{document["version"].inspect}" unless VERSIONS.include?(document["version"]) @@ -56,6 +60,16 @@ def validate document["items"].each_with_index { |item, index| validate_item(item, "items[#{index}]") } end + # @rbs (untyped) -> void + def validate_numbers(value) + case value + when Hash then value.each_value { |child| validate_numbers(child) } + when Array then value.each { |child| validate_numbers(child) } + when Float + raise SimpleRSSError, "JSON Feed number exceeds the supported range" unless value.finite? + end + end + # @rbs () -> void def validate_expiration return unless document.key?("expired") diff --git a/simple-rss.gemspec b/simple-rss.gemspec index e47f6db..641095f 100644 --- a/simple-rss.gemspec +++ b/simple-rss.gemspec @@ -1,12 +1,11 @@ Gem::Specification.new do |s| s.name = "simple-rss" - s.version = "2.2.0" - s.date = "2025-12-28" - s.summary = "A simple, flexible, extensible, and liberal RSS and Atom reader for Ruby. It is designed to be backwards compatible with the standard RSS parser, but will never do RSS generation." + s.version = "2.3.0" + s.summary = "A flexible RSS, Atom, and JSON Feed reader for Ruby." s.email = "lucas@rufy.com" s.homepage = "https://github.com/cardmagic/simple-rss" s.metadata["changelog_uri"] = "https://github.com/cardmagic/simple-rss/blob/master/CHANGELOG.md" - s.description = "A simple, flexible, extensible, and liberal RSS and Atom reader for Ruby. It is designed to be backwards compatible with the standard RSS parser, but will never do RSS generation." + s.description = "Parse RSS, Atom, and JSON Feed with normalized entries, HTTP fetching, website feed discovery, and JSON/XML serialization." s.authors = ["Lucas Carlson"] s.files = Dir["lib/**/*", "examples/**/*", "test/**/*", "LICENSE", "README.md", "CHANGELOG.md", "Rakefile", "simple-rss.gemspec"] s.required_ruby_version = ">= 3.1" diff --git a/test/base/discovery_transport_test.rb b/test/base/discovery_transport_test.rb index fc226cb..4ba6f1b 100644 --- a/test/base/discovery_transport_test.rb +++ b/test/base/discovery_transport_test.rb @@ -29,8 +29,8 @@ def test_public_policy_rejects_prohibited_ipv4_and_ipv6_before_connecting def test_malformed_urls_and_schemes_fail_before_connecting with_replaced_method(TCPSocket, :open, ->(*) { flunk "Invalid URL reached the socket" }) do - ["", "/relative", "ftp://example.com/feed", "file:///tmp/feed", "mailto:reader@example.com", "javascript:alert(1)", "http://", "http://[broken", "http://example.com:0", "http://example.com:65536", "https://user:password@example.com/", "user:password@example.com"].each do |url| - assert_raise(SimpleRSS::PolicyError, url) { SimpleRSS.discover(url) } + [nil, 123, [], {}, "", "/relative", "ftp://example.com/feed", "file:///tmp/feed", "mailto:reader@example.com", "javascript:alert(1)", "http://", "http://[broken", "http://example.com:0", "http://example.com:65536", "https://user:password@example.com/", "user:password@example.com"].each do |url| + assert_raise(SimpleRSS::PolicyError, url.inspect) { SimpleRSS.discover(url) } end end end @@ -342,6 +342,19 @@ def test_bounded_fetch_reuses_the_policy_and_conditional_get end end + def test_bodyless_success_responses_raise_feed_errors + with_server([[204, {}, ""]]) do |url, _requests| + assert_raise(SimpleRSS::DiscoveryError) { SimpleRSS.discover(url, network_policy: :unrestricted, timeout: 1) } + end + + [{}, { network_policy: :unrestricted }].each do |options| + with_server([[204, {}, ""]]) do |url, _requests| + error = assert_raise(SimpleRSSError) { SimpleRSS.fetch(url, options.merge(timeout: 1)) } + assert_equal "Poorly formatted feed", error.message + end + end + end + def test_invalid_options_and_controlled_headers_fail_before_connecting options = [ { network_policy: nil }, { network_policy: :unknown }, { timeout: 0 }, { timeout: nil }, { timeout: Float::INFINITY }, diff --git a/test/base/json_feed_test.rb b/test/base/json_feed_test.rb index 8b88bb6..5fd155f 100644 --- a/test/base/json_feed_test.rb +++ b/test/base/json_feed_test.rb @@ -191,6 +191,39 @@ def test_attachment_numeric_recovery_preserves_original_values assert_equal(["attachments[0].size_in_bytes", "attachments[0].duration_in_seconds"], entry.issues.map { |issue| issue[:source] }) end + def test_rfc3339_rejects_out_of_range_clock_and_offset_components + invalid = %w[ + 2026-09-01T24:00:00Z 2026-09-01T24:01:00Z 2026-09-01T12:60:00Z + 2026-09-01T00:00:00+25:00 2026-09-01T00:00:00-24:00 + 2026-09-01T00:00:00+00:60 2026-09-01T00:00:00-00:99 + ] + invalid.each do |value| + feed = parse_items([{ id: "1", content_text: "Hi", date_published: value, date_modified: value }]) + entry = feed.normalized_entries.first + assert_nil entry.published_at, value + assert_nil entry.updated_at, value + assert_equal(%w[date_published date_modified], entry.issues.map { |issue| issue[:source] }) + assert_equal value, entry.raw["date_published"] + assert_empty feed.items_since(Time.utc(2026)) + end + end + + def test_rfc3339_accepts_boundary_offsets_and_uses_gregorian_dates + expected = { + "2026-09-01T23:59:59.125+23:59" => Time.utc(2026, 9, 1, 0, 0, 59.125), + "2026-09-01t00:00:00-23:59" => Time.utc(2026, 9, 1, 23, 59), + "1582-10-10T00:00:00Z" => Time.utc(1582, 10, 10) + } + expected.each do |value, time| + entry = parse_items([{ id: "1", content_text: "Hi", date_published: value }]).normalized_entries.first + assert_equal time, entry.published_at + assert_empty entry.issues + end + entry = parse_items([{ id: "1", content_text: "Hi", date_published: "1500-02-29T00:00:00Z" }]).normalized_entries.first + assert_nil entry.published_at + assert_equal :invalid_date, entry.issues.first[:code] + end + def test_raw_and_serialized_representations_are_explicit_and_immutable feed = fixture("1_1") before = feed.to_json @@ -235,6 +268,36 @@ def test_utf8_bom_whitespace_and_readable_binary_io assert_raise(SimpleRSSError) { SimpleRSS.parse("\uFEFF\uFEFF#{source}") } end + def test_invalid_utf8_is_rejected_before_exposing_feed_data + source = JSON.generate(version: "https://jsonfeed.org/version/1.1", title: "Example", items: [{ id: "1", content_text: "payload" }]) + ["\xFF".b, "\xC0\x80".b, "\xE2\x82".b].each do |invalid| + body = source.b.sub("payload", invalid) + [body, StringIO.new(body)].each do |input| + error = assert_raise(SimpleRSSError) { SimpleRSS.parse(input) } + assert_include error.message, "invalid UTF-8" + end + assert_false SimpleRSS.valid?(body) + end + end + + def test_overflowing_json_numbers_cannot_collapse_identifiers_or_break_serialization + sources = [ + '{"version":"https://jsonfeed.org/version/1.1","title":"QA","items":[{"id":1e999,"content_text":"One"},{"id":2e999,"content_text":"Two"}]}', + '{"version":"https://jsonfeed.org/version/1.1","title":"QA","items":[],"_extension":{"numbers":[-1e999]}}', + '{"version":"https://jsonfeed.org/version/1.1","title":"QA","items":[{"id":"one","content_text":"One","attachments":[{"url":"https://example.com/audio","mime_type":"audio/mpeg","size_in_bytes":1e999}]}]}' + ] + sources.each do |source| + error = assert_raise(SimpleRSSError) { SimpleRSS.parse(source) } + assert_include error.message, "number exceeds the supported range" + assert_false SimpleRSS.valid?(source) + end + identifier = 10**100 + feed = parse_items([{ id: identifier, content_text: "Large integer" }], _number: 1.5) + assert_equal identifier.to_s, feed.normalized_entries.first.identifier + assert_equal identifier, feed.raw_json["items"].first["id"] + assert_equal 1.5, JSON.parse(feed.to_json)["_number"] + end + def test_invalid_json_and_structures_raise_library_errors ['{"version":', "[]", "null", "42", "true", '"hello"', '{"description":"not XML"}'].each do |source|