diff --git a/CHANGELOG.md b/CHANGELOG.md
index 286baf1..4816595 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,12 @@ repository begins with a 1.1 import, so earlier releases are not reconstructed.
## Unreleased
+- Parse Atom 1.0 category terms from attributes, so scalar category access and
+ `array_tags: [:category]` work with `items_by_category`. Preserve source order
+ and duplicates, skip missing or blank terms, and resolve namespace declarations
+ within the current entry. Exclude categories in embedded content and source
+ metadata while preserving RSS category text and CDATA behavior.
+ ([#56](https://github.com/cardmagic/simple-rss/issues/56))
- Use the first valid `pubDate`, `updated`, or `published` timestamp in
`latest`, `items_since`, and merge ordering. Malformed dates no longer make
`latest` raise or hide a valid fallback. Preserve the original field values,
diff --git a/README.md b/README.md
index 749ceb8..4784a04 100644
--- a/README.md
+++ b/README.md
@@ -263,6 +263,26 @@ feed = SimpleRSS.parse(xml, array_tags: [:category])
item.category # => ["tech", "programming", "ruby"]
```
+RSS categories use element text, including CDATA: `ruby`.
+Atom 1.0 categories use the `term` attribute:
+
+```xml
+
+
+```
+
+With `array_tags: [:category]`, those Atom categories become `["ruby", "rails"]`
+in source order, preserving duplicates. Scalar mode returns the first usable
+term. Both modes work with `feed.items_by_category("ruby")`. Missing or blank
+Atom terms are skipped; labels and element content do not supply a fallback.
+The original `label` and `scheme` attributes remain in `feed.source` for callers
+that need the source metadata.
+
+Category extraction uses direct children of each entry or item and resolves
+default and prefixed namespace declarations on the feed, entry, and category.
+Categories inside content or source metadata are excluded. RSS categories keep
+their existing scalar/array and CDATA behavior.
+
## API Reference
### `SimpleRSS.parse(source, options = {})`
diff --git a/lib/simple-rss.rb b/lib/simple-rss.rb
index 5362efb..8255d95 100644
--- a/lib/simple-rss.rb
+++ b/lib/simple-rss.rb
@@ -352,6 +352,9 @@ def handle_redirect(response, options)
DATE_TAGS = %i[pubDate lastBuildDate published updated expirationDate modified dc:date].freeze
STRIP_HTML_TAGS = %i[author contributor skipHours skipDays].freeze
+ ATOM_NAMESPACE = "http://www.w3.org/2005/Atom".freeze
+ RSS_NAMESPACES = [nil, "", "http://purl.org/rss/1.0/", "http://my.netscape.com/rdf/simple/0.9/"].freeze
+ XML_TAG_PATTERN = %r{||<\?.*?\?>|<(/?)([\w:.-]+)((?:[^<>"']|"[^"]*"|'[^']*')*)>}m
private
@@ -397,12 +400,24 @@ def parse
end
# RSS items' title, link, and description
- @source.scan(%r{<(rss:|atom:)?(item|entry)([\s][^>]*)?>(.*?)}mi) do |match|
+ namespace_contexts = entry_namespaces
+ entry_pattern = %r{<(rss:|atom:)?(item|entry)([\s][^>]*)?>(.*?)}mi
+ position = 0
+ while (match = entry_pattern.match(@source, position))
+ position = match.end(0)
item = {} #: Hash[Symbol, untyped]
+ namespaces = namespace_contexts[match.begin(0)]
@@item_tags.each do |tag|
next if tag.to_s.strip.empty?
- parse_item_tag(item, tag, match[3], match[2])
+ if tag == :category
+ next unless namespaces
+
+ parse_category_tag(item, match[4].to_s, namespaces, element_namespace("#{match[1]}#{match[2]}", namespaces))
+ next
+ end
+
+ parse_item_tag(item, tag, match[4], match[3])
end
item.define_singleton_method(:method_missing) { |name, *_args| self[name] }
add_item_media_helpers(item)
@@ -474,26 +489,109 @@ def link_relation_href(content, relation)
# @rbs (String) -> Array[Hash[String, String]]
def entry_link_attributes(content)
- links = [] #: Array[Hash[String, String]]
+ child_elements(content).filter_map do |tag, attributes, _body|
+ next unless %w[link atom:link rss:link].include?(tag.downcase)
+
+ xml_attributes(attributes).transform_keys(&:downcase)
+ end
+ end
+
+ # @rbs (String) -> Array[[String, String, String?]]
+ def child_elements(content)
+ elements = [] #: Array[[String, String, String?]]
+ current_element = nil #: [String, String, String?]?
depth = 0
- tokens = %r{||<\?.*?\?>|<(/?)([\w:.-]+)((?:[^<>"']|"[^"]*"|'[^']*')*)>}m
+ body_start = 0
+ position = 0
- content.scan(tokens) do
- closing = Regexp.last_match(1)
- tag = Regexp.last_match(2)&.downcase
- attributes = Regexp.last_match(3)
+ while (token = XML_TAG_PATTERN.match(content, position))
+ position = token.end(0)
+ attributes = token[3]
next unless attributes
- if closing == "/"
+ if token[1] == "/"
depth = [depth - 1, 0].max
+ if depth.zero? && current_element
+ current_element[2] = content[body_start...token.begin(0)]
+ current_element = nil
+ end
next
end
- links << xml_attributes(attributes) if depth.zero? && %w[link atom:link rss:link].include?(tag)
- depth += 1 unless attributes.rstrip.end_with?("/")
+ self_closing = attributes.rstrip.end_with?("/")
+ if depth.zero?
+ element = [token[2].to_s, attributes, nil] #: [String, String, String?]
+ elements << element
+ current_element = element unless self_closing
+ body_start = token.end(0)
+ end
+ depth += 1 unless self_closing
end
- links
+ elements
+ end
+
+ # @rbs (Hash[Symbol, untyped], String, Hash[String, String], String?) -> void
+ def parse_category_tag(item, content, namespaces, entry_namespace)
+ values = child_elements(content).filter_map do |tag, raw_attributes, body|
+ next unless tag.split(":").last&.casecmp?("category")
+
+ attributes = xml_attributes(raw_attributes)
+ namespace = element_namespace(tag, namespaces.merge(namespace_attributes(raw_attributes)))
+ if namespace == ATOM_NAMESPACE
+ term = CGI.unescapeHTML(attributes["term"].to_s).strip
+ next term unless term.empty?
+
+ next
+ end
+
+ next if entry_namespace == ATOM_NAMESPACE || !RSS_NAMESPACES.include?(namespace)
+ next if tag.include?(":") && namespace.nil?
+ next if body.nil? && (array_tag?(:category) || !raw_attributes.rstrip.end_with?("/"))
+
+ unescape(body.to_s)
+ end
+ return if values.empty?
+
+ item[:category] = array_tag?(:category) ? values : values.first
+ end
+
+ # @rbs () -> Hash[Integer, Hash[String, String]]
+ def entry_namespaces
+ contexts = {} #: Hash[Integer, Hash[String, String]]
+ scopes = [{}] #: Array[Hash[String, String]]
+ position = 0
+
+ while (token = XML_TAG_PATTERN.match(@source, position))
+ position = token.end(0)
+ attributes = token[3]
+ next unless attributes
+
+ if token[1] == "/"
+ scopes.pop if scopes.size > 1
+ next
+ end
+
+ namespaces = scopes.fetch(-1).merge(namespace_attributes(attributes))
+ tag = token[2].to_s.split(":").last
+ contexts[token.begin(0).to_i] = namespaces if %w[item entry].include?(tag&.downcase)
+ scopes << namespaces unless attributes.rstrip.end_with?("/")
+ end
+
+ contexts
+ end
+
+ # @rbs (String) -> Hash[String, String]
+ def namespace_attributes(attributes)
+ xml_attributes(attributes)
+ .select { |name, _value| name == "xmlns" || name.start_with?("xmlns:") }
+ .transform_values { |value| CGI.unescapeHTML(value) }
+ end
+
+ # @rbs (String, Hash[String, String]) -> String?
+ def element_namespace(tag, namespaces)
+ key = tag.include?(":") ? "xmlns:#{tag.split(":").first}" : "xmlns"
+ namespaces[key]
end
# @rbs (String) -> Hash[String, String]
@@ -502,7 +600,7 @@ def xml_attributes(attributes)
attributes.scan(/([\w:.-]+)\s*=\s*(?:"([^"]*)"|'([^']*)')/m) do
name = Regexp.last_match(1)
value = Regexp.last_match(2) || Regexp.last_match(3)
- values[name.downcase] = value if name && value
+ values[name] = value if name && value
end
values
end
diff --git a/test/base/category_parsing_test.rb b/test/base/category_parsing_test.rb
new file mode 100644
index 0000000..a1962bc
--- /dev/null
+++ b/test/base/category_parsing_test.rb
@@ -0,0 +1,217 @@
+require "test_helper"
+
+class CategoryParsingTest < Test::Unit::TestCase
+ def test_atom_terms_are_collected_and_filterable
+ source = <<~XML
+
+ Categories
+
+ Ruby news
+
+
+
+
+ XML
+ feed = SimpleRSS.parse(source, array_tags: [:category])
+
+ assert_equal %w[ruby rails], feed.first.category
+ assert_equal [feed.first], feed.items_by_category("ruby")
+ assert_equal "ruby", SimpleRSS.parse(source).first.category
+ end
+
+ def test_atom_attribute_variants_preserve_order_duplicates_and_source
+ source = File.read(File.join(__dir__, "../data/atom_categories.xml"))
+ feed = SimpleRSS.parse(source, array_tags: [:category])
+
+ assert_equal ["ruby", "rails", "Ruby & RSS", "ruby"], feed.first.category
+ assert_equal [feed.first], feed.items_by_category("RAILS")
+ assert_equal [], feed.items_by_category("Human label")
+ assert_nil feed[1].category
+ assert_equal ["sports"], feed[2].category
+ assert_equal source, feed.source
+ assert_equal "ruby", SimpleRSS.parse(source).first.category
+ end
+
+ def test_atom_scalar_mode_skips_missing_and_blank_terms
+ feed = parse_entry(<<~XML)
+ Not a term
+
+
+
+ XML
+
+ assert_equal "first", feed.first.category
+ assert_equal [feed.first], feed.items_by_category("first")
+ assert_equal [], feed.items_by_category("second")
+ end
+
+ def test_categories_exclude_nested_metadata_content_comments_and_cdata
+ feed = parse_entry(<<~XML, array_tags: [:category])
+
+
+
+
+ ]]>
+ ?>
+
+ XML
+
+ assert_equal ["direct"], feed.first.category
+ end
+
+ def test_categories_do_not_borrow_terms_from_neighboring_elements_or_entries
+ source = <<~XML
+
+ Boundaries
+
+
+ First
+
+
+
+ Second
+
+ XML
+
+ [SimpleRSS.parse(source), SimpleRSS.parse(source, array_tags: [:category])].each do |feed|
+ assert_nil feed.first.category
+ assert_equal [], feed.items_by_category("neighbor")
+ assert_equal [feed[1]], feed.items_by_category("second")
+ end
+ end
+
+ def test_namespace_prefixes_resolve_at_feed_entry_and_category_scope
+ source = <<~XML
+
+ Namespace scopes
+
+ First
+
+
+
+
+ Not Atom
+
+
+ XML
+ feed = SimpleRSS.parse(source, array_tags: [:category])
+
+ assert_equal %w[feed-scope entry-scope category-scope local-default], feed.first.category
+ end
+
+ def test_namespace_rebinding_and_default_resets_do_not_leak_categories
+ source = <<~XML
+
+ Rebinding
+
+ First
+
+ Foreign text
+ Unqualified text
+
+
+ Second
+
+ XML
+ feed = SimpleRSS.parse(source, array_tags: [:category])
+
+ assert_equal ["restored"], feed.first.category
+ assert_equal ["inherited"], feed[1].category
+ end
+
+ def test_exact_element_and_attribute_names_are_required
+ feed = parse_entry(<<~XML, array_tags: [:category])
+
+
+
+
+
+
+ XML
+
+ assert_equal ["correct"], feed.first.category
+ end
+
+ def test_rss_text_cdata_scalar_and_array_filters_remain_compatible
+ source = File.read(File.join(__dir__, "../data/rss_categories.xml"))
+ scalar_feed = SimpleRSS.parse(source)
+ array_feed = SimpleRSS.parse(source, array_tags: [:category])
+
+ assert_equal "Technology", scalar_feed.first.category
+ assert_equal ["Technology", "Ruby & RSS", "Technology"], array_feed.first.category
+ assert_equal [scalar_feed.first], scalar_feed.items_by_category("tech")
+ assert_equal [array_feed.first], array_feed.items_by_category("ruby")
+ assert_equal ["Sports"], array_feed[1].category
+ end
+
+ def test_rss_categories_exclude_embedded_markup_and_accept_atom_extensions
+ source = <<~XML
+
+
+ Mixed categories
+ -
+ First
+ Embedded]]>
+ Nested
+ RSS text
+
+ Foreign text
+
+
+
+ XML
+ feed = SimpleRSS.parse(source, array_tags: [:category])
+
+ assert_equal ["RSS text", "Atom term"], feed.first.category
+ end
+
+ def test_rss_namespaces_and_empty_category_behavior_are_preserved
+ source = <<~XML
+
+ RSS 1.0
+ - FirstRuby
+ - Second
+
+ XML
+ feed = SimpleRSS.parse(source, array_tags: [:category])
+
+ assert_equal ["Ruby", ""], feed.first.category
+ assert_nil feed[1].category
+ assert_equal "", SimpleRSS.parse(source)[1].category
+ end
+
+ def test_namespace_declarations_in_comments_do_not_override_the_feed
+ source = <<~XML
+
+
+ Actual feed
+ First
+
+ XML
+
+ assert_equal "actual", SimpleRSS.parse(source).first.category
+ end
+
+ def test_rss_channel_namespaces_do_not_apply_to_sibling_rdf_items
+ source = <<~XML
+
+ RSS
+ - FirstRSS text
+
+ XML
+
+ assert_equal ["RSS text"], SimpleRSS.parse(source, array_tags: [:category]).first.category
+ end
+
+ private
+
+ def parse_entry(content, options = {})
+ source = <<~XML
+
+ Categories
+ Entry#{content}
+
+ XML
+ SimpleRSS.parse(source, options)
+ end
+end
diff --git a/test/data/atom_categories.xml b/test/data/atom_categories.xml
new file mode 100644
index 0000000..e323790
--- /dev/null
+++ b/test/data/atom_categories.xml
@@ -0,0 +1,23 @@
+
+ Atom categories
+
+ Ruby news
+
+ Ignored element content
+
+
+
+
+ No usable terms
+ Not a term
+
+
+
+
+ Other news
+
+
+
diff --git a/test/data/rss_categories.xml b/test/data/rss_categories.xml
new file mode 100644
index 0000000..ca236ff
--- /dev/null
+++ b/test/data/rss_categories.xml
@@ -0,0 +1,15 @@
+
+
+ RSS categories
+ -
+ Ruby news
+ Technology
+
+ Technology
+
+ -
+ Other news
+ Sports
+
+
+