From 08cd53e27a92555b824f35695eb72713b4efd2d1 Mon Sep 17 00:00:00 2001 From: Lucas Carlson Date: Sun, 13 Sep 2026 06:55:47 -0700 Subject: [PATCH] fix: parse Atom category terms Read category terms from Atom attributes so scalar access, array collection, and category filtering agree. Preserve RSS text and CDATA while excluding nested categories and unusable Atom terms. Resolve namespaces from each entry's actual ancestors, including local overrides and resets. Share direct-child extraction with relation links and document the behavior in the README and changelog. Closes #56 --- CHANGELOG.md | 6 + README.md | 20 +++ lib/simple-rss.rb | 124 +++++++++++++++-- test/base/category_parsing_test.rb | 217 +++++++++++++++++++++++++++++ test/data/atom_categories.xml | 23 +++ test/data/rss_categories.xml | 15 ++ 6 files changed, 392 insertions(+), 13 deletions(-) create mode 100644 test/base/category_parsing_test.rb create mode 100644 test/data/atom_categories.xml create mode 100644 test/data/rss_categories.xml diff --git a/CHANGELOG.md b/CHANGELOG.md index 286baf1..4816595 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ repository begins with a 1.1 import, so earlier releases are not reconstructed. ## Unreleased +- Parse Atom 1.0 category terms from attributes, so scalar category access and + `array_tags: [:category]` work with `items_by_category`. Preserve source order + and duplicates, skip missing or blank terms, and resolve namespace declarations + within the current entry. Exclude categories in embedded content and source + metadata while preserving RSS category text and CDATA behavior. + ([#56](https://github.com/cardmagic/simple-rss/issues/56)) - Use the first valid `pubDate`, `updated`, or `published` timestamp in `latest`, `items_since`, and merge ordering. Malformed dates no longer make `latest` raise or hide a valid fallback. Preserve the original field values, diff --git a/README.md b/README.md index 749ceb8..4784a04 100644 --- a/README.md +++ b/README.md @@ -263,6 +263,26 @@ feed = SimpleRSS.parse(xml, array_tags: [:category]) item.category # => ["tech", "programming", "ruby"] ``` +RSS categories use element text, including CDATA: `ruby`. +Atom 1.0 categories use the `term` attribute: + +```xml + + +``` + +With `array_tags: [:category]`, those Atom categories become `["ruby", "rails"]` +in source order, preserving duplicates. Scalar mode returns the first usable +term. Both modes work with `feed.items_by_category("ruby")`. Missing or blank +Atom terms are skipped; labels and element content do not supply a fallback. +The original `label` and `scheme` attributes remain in `feed.source` for callers +that need the source metadata. + +Category extraction uses direct children of each entry or item and resolves +default and prefixed namespace declarations on the feed, entry, and category. +Categories inside content or source metadata are excluded. RSS categories keep +their existing scalar/array and CDATA behavior. + ## API Reference ### `SimpleRSS.parse(source, options = {})` diff --git a/lib/simple-rss.rb b/lib/simple-rss.rb index 5362efb..8255d95 100644 --- a/lib/simple-rss.rb +++ b/lib/simple-rss.rb @@ -352,6 +352,9 @@ def handle_redirect(response, options) DATE_TAGS = %i[pubDate lastBuildDate published updated expirationDate modified dc:date].freeze STRIP_HTML_TAGS = %i[author contributor skipHours skipDays].freeze + ATOM_NAMESPACE = "http://www.w3.org/2005/Atom".freeze + RSS_NAMESPACES = [nil, "", "http://purl.org/rss/1.0/", "http://my.netscape.com/rdf/simple/0.9/"].freeze + XML_TAG_PATTERN = %r{||<\?.*?\?>|<(/?)([\w:.-]+)((?:[^<>"']|"[^"]*"|'[^']*')*)>}m private @@ -397,12 +400,24 @@ def parse end # RSS items' title, link, and description - @source.scan(%r{<(rss:|atom:)?(item|entry)([\s][^>]*)?>(.*?)}mi) do |match| + namespace_contexts = entry_namespaces + entry_pattern = %r{<(rss:|atom:)?(item|entry)([\s][^>]*)?>(.*?)}mi + position = 0 + while (match = entry_pattern.match(@source, position)) + position = match.end(0) item = {} #: Hash[Symbol, untyped] + namespaces = namespace_contexts[match.begin(0)] @@item_tags.each do |tag| next if tag.to_s.strip.empty? - parse_item_tag(item, tag, match[3], match[2]) + if tag == :category + next unless namespaces + + parse_category_tag(item, match[4].to_s, namespaces, element_namespace("#{match[1]}#{match[2]}", namespaces)) + next + end + + parse_item_tag(item, tag, match[4], match[3]) end item.define_singleton_method(:method_missing) { |name, *_args| self[name] } add_item_media_helpers(item) @@ -474,26 +489,109 @@ def link_relation_href(content, relation) # @rbs (String) -> Array[Hash[String, String]] def entry_link_attributes(content) - links = [] #: Array[Hash[String, String]] + child_elements(content).filter_map do |tag, attributes, _body| + next unless %w[link atom:link rss:link].include?(tag.downcase) + + xml_attributes(attributes).transform_keys(&:downcase) + end + end + + # @rbs (String) -> Array[[String, String, String?]] + def child_elements(content) + elements = [] #: Array[[String, String, String?]] + current_element = nil #: [String, String, String?]? depth = 0 - tokens = %r{||<\?.*?\?>|<(/?)([\w:.-]+)((?:[^<>"']|"[^"]*"|'[^']*')*)>}m + body_start = 0 + position = 0 - content.scan(tokens) do - closing = Regexp.last_match(1) - tag = Regexp.last_match(2)&.downcase - attributes = Regexp.last_match(3) + while (token = XML_TAG_PATTERN.match(content, position)) + position = token.end(0) + attributes = token[3] next unless attributes - if closing == "/" + if token[1] == "/" depth = [depth - 1, 0].max + if depth.zero? && current_element + current_element[2] = content[body_start...token.begin(0)] + current_element = nil + end next end - links << xml_attributes(attributes) if depth.zero? && %w[link atom:link rss:link].include?(tag) - depth += 1 unless attributes.rstrip.end_with?("/") + self_closing = attributes.rstrip.end_with?("/") + if depth.zero? + element = [token[2].to_s, attributes, nil] #: [String, String, String?] + elements << element + current_element = element unless self_closing + body_start = token.end(0) + end + depth += 1 unless self_closing end - links + elements + end + + # @rbs (Hash[Symbol, untyped], String, Hash[String, String], String?) -> void + def parse_category_tag(item, content, namespaces, entry_namespace) + values = child_elements(content).filter_map do |tag, raw_attributes, body| + next unless tag.split(":").last&.casecmp?("category") + + attributes = xml_attributes(raw_attributes) + namespace = element_namespace(tag, namespaces.merge(namespace_attributes(raw_attributes))) + if namespace == ATOM_NAMESPACE + term = CGI.unescapeHTML(attributes["term"].to_s).strip + next term unless term.empty? + + next + end + + next if entry_namespace == ATOM_NAMESPACE || !RSS_NAMESPACES.include?(namespace) + next if tag.include?(":") && namespace.nil? + next if body.nil? && (array_tag?(:category) || !raw_attributes.rstrip.end_with?("/")) + + unescape(body.to_s) + end + return if values.empty? + + item[:category] = array_tag?(:category) ? values : values.first + end + + # @rbs () -> Hash[Integer, Hash[String, String]] + def entry_namespaces + contexts = {} #: Hash[Integer, Hash[String, String]] + scopes = [{}] #: Array[Hash[String, String]] + position = 0 + + while (token = XML_TAG_PATTERN.match(@source, position)) + position = token.end(0) + attributes = token[3] + next unless attributes + + if token[1] == "/" + scopes.pop if scopes.size > 1 + next + end + + namespaces = scopes.fetch(-1).merge(namespace_attributes(attributes)) + tag = token[2].to_s.split(":").last + contexts[token.begin(0).to_i] = namespaces if %w[item entry].include?(tag&.downcase) + scopes << namespaces unless attributes.rstrip.end_with?("/") + end + + contexts + end + + # @rbs (String) -> Hash[String, String] + def namespace_attributes(attributes) + xml_attributes(attributes) + .select { |name, _value| name == "xmlns" || name.start_with?("xmlns:") } + .transform_values { |value| CGI.unescapeHTML(value) } + end + + # @rbs (String, Hash[String, String]) -> String? + def element_namespace(tag, namespaces) + key = tag.include?(":") ? "xmlns:#{tag.split(":").first}" : "xmlns" + namespaces[key] end # @rbs (String) -> Hash[String, String] @@ -502,7 +600,7 @@ def xml_attributes(attributes) attributes.scan(/([\w:.-]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/m) do name = Regexp.last_match(1) value = Regexp.last_match(2) || Regexp.last_match(3) - values[name.downcase] = value if name && value + values[name] = value if name && value end values end diff --git a/test/base/category_parsing_test.rb b/test/base/category_parsing_test.rb new file mode 100644 index 0000000..a1962bc --- /dev/null +++ b/test/base/category_parsing_test.rb @@ -0,0 +1,217 @@ +require "test_helper" + +class CategoryParsingTest < Test::Unit::TestCase + def test_atom_terms_are_collected_and_filterable + source = <<~XML + + Categories + + Ruby news + + + + + XML + feed = SimpleRSS.parse(source, array_tags: [:category]) + + assert_equal %w[ruby rails], feed.first.category + assert_equal [feed.first], feed.items_by_category("ruby") + assert_equal "ruby", SimpleRSS.parse(source).first.category + end + + def test_atom_attribute_variants_preserve_order_duplicates_and_source + source = File.read(File.join(__dir__, "../data/atom_categories.xml")) + feed = SimpleRSS.parse(source, array_tags: [:category]) + + assert_equal ["ruby", "rails", "Ruby & RSS", "ruby"], feed.first.category + assert_equal [feed.first], feed.items_by_category("RAILS") + assert_equal [], feed.items_by_category("Human label") + assert_nil feed[1].category + assert_equal ["sports"], feed[2].category + assert_equal source, feed.source + assert_equal "ruby", SimpleRSS.parse(source).first.category + end + + def test_atom_scalar_mode_skips_missing_and_blank_terms + feed = parse_entry(<<~XML) + Not a term + + + + XML + + assert_equal "first", feed.first.category + assert_equal [feed.first], feed.items_by_category("first") + assert_equal [], feed.items_by_category("second") + end + + def test_categories_exclude_nested_metadata_content_comments_and_cdata + feed = parse_entry(<<~XML, array_tags: [:category]) + +
+ + + ]]> + ?> + + XML + + assert_equal ["direct"], feed.first.category + end + + def test_categories_do_not_borrow_terms_from_neighboring_elements_or_entries + source = <<~XML + + Boundaries + + + First + + + + Second + + XML + + [SimpleRSS.parse(source), SimpleRSS.parse(source, array_tags: [:category])].each do |feed| + assert_nil feed.first.category + assert_equal [], feed.items_by_category("neighbor") + assert_equal [feed[1]], feed.items_by_category("second") + end + end + + def test_namespace_prefixes_resolve_at_feed_entry_and_category_scope + source = <<~XML + + Namespace scopes + + First + + + + + Not Atom + + + XML + feed = SimpleRSS.parse(source, array_tags: [:category]) + + assert_equal %w[feed-scope entry-scope category-scope local-default], feed.first.category + end + + def test_namespace_rebinding_and_default_resets_do_not_leak_categories + source = <<~XML + + Rebinding + + First + + Foreign text + Unqualified text + + + Second + + XML + feed = SimpleRSS.parse(source, array_tags: [:category]) + + assert_equal ["restored"], feed.first.category + assert_equal ["inherited"], feed[1].category + end + + def test_exact_element_and_attribute_names_are_required + feed = parse_entry(<<~XML, array_tags: [:category]) + + + + + + + XML + + assert_equal ["correct"], feed.first.category + end + + def test_rss_text_cdata_scalar_and_array_filters_remain_compatible + source = File.read(File.join(__dir__, "../data/rss_categories.xml")) + scalar_feed = SimpleRSS.parse(source) + array_feed = SimpleRSS.parse(source, array_tags: [:category]) + + assert_equal "Technology", scalar_feed.first.category + assert_equal ["Technology", "Ruby & RSS", "Technology"], array_feed.first.category + assert_equal [scalar_feed.first], scalar_feed.items_by_category("tech") + assert_equal [array_feed.first], array_feed.items_by_category("ruby") + assert_equal ["Sports"], array_feed[1].category + end + + def test_rss_categories_exclude_embedded_markup_and_accept_atom_extensions + source = <<~XML + + + Mixed categories + + First + Embedded]]> + Nested + RSS text + + Foreign text + + + + XML + feed = SimpleRSS.parse(source, array_tags: [:category]) + + assert_equal ["RSS text", "Atom term"], feed.first.category + end + + def test_rss_namespaces_and_empty_category_behavior_are_preserved + source = <<~XML + + RSS 1.0 + FirstRuby + Second + + XML + feed = SimpleRSS.parse(source, array_tags: [:category]) + + assert_equal ["Ruby", ""], feed.first.category + assert_nil feed[1].category + assert_equal "", SimpleRSS.parse(source)[1].category + end + + def test_namespace_declarations_in_comments_do_not_override_the_feed + source = <<~XML + + + Actual feed + First + + XML + + assert_equal "actual", SimpleRSS.parse(source).first.category + end + + def test_rss_channel_namespaces_do_not_apply_to_sibling_rdf_items + source = <<~XML + + RSS + FirstRSS text + + XML + + assert_equal ["RSS text"], SimpleRSS.parse(source, array_tags: [:category]).first.category + end + + private + + def parse_entry(content, options = {}) + source = <<~XML + + Categories + Entry#{content} + + XML + SimpleRSS.parse(source, options) + end +end diff --git a/test/data/atom_categories.xml b/test/data/atom_categories.xml new file mode 100644 index 0000000..e323790 --- /dev/null +++ b/test/data/atom_categories.xml @@ -0,0 +1,23 @@ + + Atom categories + + Ruby news + + Ignored element content + + + + + No usable terms + Not a term + + + + + Other news + + + diff --git a/test/data/rss_categories.xml b/test/data/rss_categories.xml new file mode 100644 index 0000000..ca236ff --- /dev/null +++ b/test/data/rss_categories.xml @@ -0,0 +1,15 @@ + + + RSS categories + + Ruby news + Technology + + Technology + + + Other news + Sports + + +