diff --git a/lib/plaintext.rb b/lib/plaintext.rb index c6ddc21..4536329 100644 --- a/lib/plaintext.rb +++ b/lib/plaintext.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'active_support/core_ext/string' +require 'active_support/core_ext/object/blank' require 'plaintext/version' require 'plaintext/configuration' @@ -12,6 +13,7 @@ require 'plaintext/file_handler/external_command_handler/doc_handler' require 'plaintext/file_handler/external_command_handler/image_handler' require 'plaintext/file_handler/external_command_handler/pdf_handler' +require 'plaintext/pdf_url_extractor' require 'plaintext/file_handler/external_command_handler/ppt_handler' require 'plaintext/file_handler/external_command_handler/rtf_handler' require 'plaintext/file_handler/external_command_handler/xls_handler' diff --git a/lib/plaintext/file_handler/external_command_handler/pdf_handler.rb b/lib/plaintext/file_handler/external_command_handler/pdf_handler.rb index b1a823a..091492c 100644 --- a/lib/plaintext/file_handler/external_command_handler/pdf_handler.rb +++ b/lib/plaintext/file_handler/external_command_handler/pdf_handler.rb @@ -11,6 +11,19 @@ def initialize @command = Plaintext::Configuration['pdftotext'] || DEFAULT end + def text(file, options = {}) + base_text = super(file, options) + + if options[:extract_urls] + urls = Plaintext::PdfUrlExtractor.new(file).extract + if urls.any? + base_text += "\n\n" + urls.join("\n") + "\n" + end + end + + base_text + end + protected def utf8_stream? diff --git a/lib/plaintext/pdf_url_extractor.rb b/lib/plaintext/pdf_url_extractor.rb new file mode 100644 index 0000000..45de657 --- /dev/null +++ b/lib/plaintext/pdf_url_extractor.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'pathname' + +module Plaintext + class PdfUrlExtractor + def initialize(file_path, command = nil) + @file_path = Pathname(file_path).to_s + @command = command || default_command + end + + def extract + return [] unless available? + + xml_output = `#{@command} -xml -stdout "#{@file_path}" 2>/dev/null` + urls = xml_output.scan(//).flatten + + urls.compact.uniq + rescue StandardError + [] + end + + def available? + @command && File.executable?(@command) + end + + private + + def default_command + pdftotext_cmd = Plaintext::Configuration['pdftotext']&.first || '/usr/bin/pdftotext' + + # Determine pdftohtml path based on pdftotext path + pdftohtml_cmd = pdftotext_cmd.gsub('pdftotext', 'pdftohtml') + + return pdftohtml_cmd if File.executable?(pdftohtml_cmd) + + # Fallback to system PATH + pdftohtml_in_path = ENV['PATH'].split(File::PATH_SEPARATOR).map { |p| File.join(p, 'pdftohtml') }.find { |f| File.executable?(f) && !File.directory?(f) } + pdftohtml_in_path || 'pdftohtml' + end + end +end diff --git a/spec/fixtures/files/text-with-links.pdf b/spec/fixtures/files/text-with-links.pdf new file mode 100644 index 0000000..4f0f02c Binary files /dev/null and b/spec/fixtures/files/text-with-links.pdf differ diff --git a/spec/lib/file_handler/external_command_handler/pdf_handler_spec.rb b/spec/lib/file_handler/external_command_handler/pdf_handler_spec.rb index 001aa5c..1d53060 100644 --- a/spec/lib/file_handler/external_command_handler/pdf_handler_spec.rb +++ b/spec/lib/file_handler/external_command_handler/pdf_handler_spec.rb @@ -20,6 +20,21 @@ expect(subject.text(file)).to match /In der Küche hat es eine Kaffeemaschine/ expect(Plaintext::Resolver.new(file, 'application/pdf').text).to match /In der Küche hat es eine Kaffeemaschine/ end + + context 'when extracting URLs is requested' do + let(:file) { File.new('spec/fixtures/files/text-with-links.pdf', 'r') } + + it 'should NOT append URLs by default' do + extracted_text = subject.text(file) + expect(extracted_text).not_to match /http/ + end + + it 'should append URLs if extract_urls option is true' do + extracted_text = subject.text(file, extract_urls: true) + expect(extracted_text).to match /http/ + expect(extracted_text).to match /mailto:/ + end + end else warn "#{described_class.name} could not be tested as external program is not available." end diff --git a/spec/lib/pdf_url_extractor_spec.rb b/spec/lib/pdf_url_extractor_spec.rb new file mode 100644 index 0000000..965566a --- /dev/null +++ b/spec/lib/pdf_url_extractor_spec.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Plaintext::PdfUrlExtractor do + let(:file) { File.new('spec/fixtures/files/text-with-links.pdf', 'r') } + subject { described_class.new(file) } + + if described_class.new('dummy').available? + it 'should extract URLs from a PDF file' do + urls = subject.extract + expect(urls).to be_an(Array) + expect(urls.size).to be > 0 + expect(urls).to include(match(/http/)) + expect(urls).to include(match(/mailto:/)) + end + else + warn "#{described_class.name} could not be tested as external program pdftohtml is not available." + end +end \ No newline at end of file diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 167fd7b..2959a9c 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,5 +1,12 @@ # frozen_string_literal: true $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) +require 'active_support/all' require 'plaintext' require 'byebug' + +pdftotext_path = ENV['PATH'].split(File::PATH_SEPARATOR).map { |p| File.join(p, 'pdftotext') }.find { |f| File.executable?(f) && !File.directory?(f) } +if pdftotext_path + Plaintext::Configuration.config ||= {} + Plaintext::Configuration.config['pdftotext'] = [pdftotext_path, '-enc', 'UTF-8', '__FILE__', '-'] +end