From 1521c46d95b201a2fda6ce6a28046b498b095cd8 Mon Sep 17 00:00:00 2001 From: Ben Fairless Date: Wed, 12 Aug 2026 09:50:08 +0800 Subject: [PATCH] Add SimpleCov and expand test coverage Adds meaningful coverage for the parts of the gem that previously had none: the HTTP interaction with the Morph server and the Thor command line interface. - Add SimpleCov with branch coverage and a 90% minimum; the suite currently measures 97.5% line / 95.8% branch coverage - Add WebMock and disable all network access in specs - Extract the Thor CLI class from bin/morph into MorphCLI::CLI in lib/morph-cli/cli.rb so it is loadable (and therefore testable); bin/morph is now a thin wrapper with unchanged behaviour - Spec MorphCLI.execute against stubbed HTTP: streams run output, posts the API key and code as multipart form data, raises on 401, exits when no scraper file is present - Spec the CLI: version command, --dev config selection, API key prompting/saving, retry on rejected key, and exit paths for connection, server and client errors - Spec log stream routing, config save/load round-trip and permissions, tar creation/read-back, directory sizing and in_directory restoration - Exclude spec files from Metrics/BlockLength (standard practice) 29 examples, 0 failures on Ruby 3.4.10; RuboCop clean. Assisted-by: opencode/anthropic.claude-fable-5 Signed-off-by: Ben Fairless --- .rubocop.yml | 4 + Gemfile | 2 + bin/morph | 57 +----------- lib/morph-cli/cli.rb | 63 +++++++++++++ spec/morph_cli/cli_spec.rb | 106 ++++++++++++++++++++++ spec/morph_cli_spec.rb | 176 +++++++++++++++++++++++++++++++++++++ spec/spec_helper.rb | 10 +++ 7 files changed, 363 insertions(+), 55 deletions(-) create mode 100644 lib/morph-cli/cli.rb create mode 100644 spec/morph_cli/cli_spec.rb diff --git a/.rubocop.yml b/.rubocop.yml index 7095e99..c7a41a1 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -5,3 +5,7 @@ AllCops: NewCops: enable SuggestExtensions: false TargetRubyVersion: 3.2 + +Metrics/BlockLength: + Exclude: + - "spec/**/*" diff --git a/Gemfile b/Gemfile index 06de602..c06a024 100644 --- a/Gemfile +++ b/Gemfile @@ -8,4 +8,6 @@ group :development, :test do gem "rake", "~> 13.0" gem "rspec", "~> 3.0" gem "rubocop" + gem "simplecov", require: false + gem "webmock", "~> 3.0" end diff --git a/bin/morph b/bin/morph index c520557..4b8b856 100755 --- a/bin/morph +++ b/bin/morph @@ -1,62 +1,9 @@ #!/usr/bin/env ruby # Commandline client for controlling morph and running scrapers and things -require "thor" # TODO: Do compression on the tar file # require 'zlib' -require 'morph-cli' - -class MorphThor < Thor - class_option :dev, default: false, type: :boolean, desc: "Run against development Morph (for morph developers)" - - desc "[execute]", "execute morph scraper" - option :directory, default: Dir.getwd - - def execute - config = MorphCLI.load_config - env_config = if options[:dev] - config[:development] - else - config[:production] - end - - config = ask_and_save_api_key(env_config, config) if env_config[:api_key].nil? - - api_key_is_valid = false - until api_key_is_valid - begin - MorphCLI.execute(options[:directory], options[:dev], env_config) - api_key_is_valid = true - rescue Faraday::UnauthorizedError - puts "Your key isn't working. Let's try again." - config = ask_and_save_api_key(env_config, config) - rescue Faraday::ConnectionFailed => e - warn "Morph doesn't look to be running at #{env_config[:base_url]} (#{e})" - exit(1) - rescue Faraday::ServerError => e - warn "Uh oh. Something has gone wrong on the Morph server at #{env_config[:base_url]} (#{e})" - exit(1) - rescue Faraday::Error => e - warn "Request to #{env_config[:base_url]} failed (#{e})" - exit(1) - end - end - end - - desc "version", "Show Morph version number and quit" - def version - puts "Morph CLI #{MorphCLI::VERSION}" - exit - end - - no_commands do - def ask_and_save_api_key(env_config, config) - env_config[:api_key] = ask("What is your key? (Go to #{env_config[:base_url]}/settings)") - MorphCLI.save_config(config) - config - end - end -end +require "morph-cli/cli" # If morph is run without any parameters it's the same as "morph execute" -MorphThor.start(ARGV.empty? ? ["execute"] : ARGV) +MorphCLI::CLI.start(ARGV.empty? ? ["execute"] : ARGV) diff --git a/lib/morph-cli/cli.rb b/lib/morph-cli/cli.rb new file mode 100644 index 0000000..ad7b093 --- /dev/null +++ b/lib/morph-cli/cli.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require 'thor' +require 'morph-cli' + +module MorphCLI + # Thor command line interface for running Morph scrapers + class CLI < Thor + def self.exit_on_failure? + true + end + + class_option :dev, default: false, type: :boolean, desc: 'Run against development Morph (for morph developers)' + + desc '[execute]', 'execute morph scraper' + option :directory, default: Dir.getwd + + def execute + config = MorphCLI.load_config + env_config = if options[:dev] + config[:development] + else + config[:production] + end + + config = ask_and_save_api_key(env_config, config) if env_config[:api_key].nil? + + api_key_is_valid = false + until api_key_is_valid + begin + MorphCLI.execute(options[:directory], options[:dev], env_config) + api_key_is_valid = true + rescue Faraday::UnauthorizedError + puts "Your key isn't working. Let's try again." + config = ask_and_save_api_key(env_config, config) + rescue Faraday::ConnectionFailed => e + warn "Morph doesn't look to be running at #{env_config[:base_url]} (#{e})" + exit(1) + rescue Faraday::ServerError => e + warn "Uh oh. Something has gone wrong on the Morph server at #{env_config[:base_url]} (#{e})" + exit(1) + rescue Faraday::Error => e + warn "Request to #{env_config[:base_url]} failed (#{e})" + exit(1) + end + end + end + + desc 'version', 'Show Morph version number and quit' + def version + puts "Morph CLI #{MorphCLI::VERSION}" + exit + end + + no_commands do + def ask_and_save_api_key(env_config, config) + env_config[:api_key] = ask("What is your key? (Go to #{env_config[:base_url]}/settings)") + MorphCLI.save_config(config) + config + end + end + end +end diff --git a/spec/morph_cli/cli_spec.rb b/spec/morph_cli/cli_spec.rb new file mode 100644 index 0000000..324e559 --- /dev/null +++ b/spec/morph_cli/cli_spec.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe MorphCLI::CLI do + describe 'version' do + it 'prints the version and exits' do + expect do + expect { described_class.start(['version']) }.to raise_error(SystemExit) + end.to output("Morph CLI #{MorphCLI::VERSION}\n").to_stdout + end + end + + describe 'execute' do + let(:config) do + { + development: { base_url: 'http://127.0.0.1:3000', api_key: 'dev-key' }, + production: { base_url: 'https://morph.io', api_key: 'prod-key' } + } + end + + before do + allow(MorphCLI).to receive(:load_config).and_return(config) + allow(MorphCLI).to receive(:save_config) + end + + it 'runs the scraper with the production config by default' do + allow(MorphCLI).to receive(:execute) + + described_class.start(['execute', '--directory', '/somewhere']) + + expect(MorphCLI).to have_received(:execute) + .with('/somewhere', false, config[:production]) + end + + it 'runs the scraper with the development config when --dev is given' do + allow(MorphCLI).to receive(:execute) + + described_class.start(['execute', '--dev']) + + expect(MorphCLI).to have_received(:execute) + .with(anything, true, config[:development]) + end + + it 'asks for an API key and saves the config when none is set' do + config[:production].delete(:api_key) + allow(MorphCLI).to receive(:execute) + allow(Thor::LineEditor).to receive(:readline).and_return('shiny-new-key') + + described_class.start(['execute']) + + expect(Thor::LineEditor).to have_received(:readline) + .with(a_string_matching(/What is your key\?/), anything) + expect(config[:production][:api_key]).to eq('shiny-new-key') + expect(MorphCLI).to have_received(:save_config).with(config) + expect(MorphCLI).to have_received(:execute) + .with(anything, false, config[:production]) + end + + it 'asks for a new API key and retries when the server rejects it' do + attempts = 0 + allow(MorphCLI).to receive(:execute) do + attempts += 1 + raise Faraday::UnauthorizedError, '401' if attempts == 1 + end + allow(Thor::LineEditor).to receive(:readline).and_return('fresh-key') + + expect { described_class.start(['execute']) } + .to output(/Your key isn't working\. Let's try again\./).to_stdout + + expect(attempts).to eq(2) + expect(config[:production][:api_key]).to eq('fresh-key') + expect(MorphCLI).to have_received(:save_config).with(config) + end + + it 'exits with an error when morph is not reachable' do + allow(MorphCLI).to receive(:execute) + .and_raise(Faraday::ConnectionFailed, 'connection refused') + + expect do + expect { described_class.start(['execute']) } + .to raise_error(SystemExit) { |e| expect(e.status).to eq(1) } + end.to output(%r{Morph doesn't look to be running at https://morph\.io}).to_stderr + end + + it 'exits with an error when the morph server fails' do + allow(MorphCLI).to receive(:execute) + .and_raise(Faraday::ServerError, '500') + + expect do + expect { described_class.start(['execute']) } + .to raise_error(SystemExit) { |e| expect(e.status).to eq(1) } + end.to output(/Something has gone wrong on the Morph server/).to_stderr + end + + it 'exits with an error on any other request failure' do + allow(MorphCLI).to receive(:execute) + .and_raise(Faraday::BadRequestError, '400') + + expect do + expect { described_class.start(['execute']) } + .to raise_error(SystemExit) { |e| expect(e.status).to eq(1) } + end.to output(%r{Request to https://morph\.io failed}).to_stderr + end + end +end diff --git a/spec/morph_cli_spec.rb b/spec/morph_cli_spec.rb index 848f4f9..66111de 100644 --- a/spec/morph_cli_spec.rb +++ b/spec/morph_cli_spec.rb @@ -10,6 +10,182 @@ end end + describe ".execute" do + let(:env_config) { { base_url: "https://morph.io", api_key: "secret-key" } } + + def with_scraper_directory + Dir.mktmpdir do |dir| + File.write(File.join(dir, "scraper.rb"), "puts 'hi'\n") + yield dir + end + end + + it "uploads the scraper and streams the run output to stdout" do + stub_request(:post, "https://morph.io/run") + .to_return(status: 200, body: %({"stream":"stdout","text":"hello from morph"}\n)) + + with_scraper_directory do |dir| + expect { described_class.execute(dir, false, env_config) } + .to output(/\AUploading .*\nhello from morph\n\z/).to_stdout + end + end + + it "posts the API key and the code as multipart form data" do + stub_request(:post, "https://morph.io/run").to_return(status: 200, body: "") + + with_scraper_directory do |dir| + expect { described_class.execute(dir, false, env_config) } + .to output(/Uploading/).to_stdout + end + + expect(WebMock).to(have_requested(:post, "https://morph.io/run").with do |req| + req.headers["Content-Type"].start_with?("multipart/form-data") && + req.body.include?("secret-key") && + req.body.include?("scraper.rb") + end) + end + + it "raises Faraday::UnauthorizedError when the API key is rejected" do + stub_request(:post, "https://morph.io/run").to_return(status: 401, body: "") + + with_scraper_directory do |dir| + expect { described_class.execute(dir, false, env_config) } + .to raise_error(Faraday::UnauthorizedError) + .and output(/Uploading/).to_stdout + end + end + + it "exits with an error when there is no scraper to upload" do + Dir.mktmpdir do |dir| + expect do + expect { described_class.execute(dir, false, env_config) } + .to raise_error(SystemExit) { |e| expect(e.status).to eq(1) } + end.to output(/Can't find scraper to upload/).to_stderr + end + end + end + + describe ".log" do + it "writes stdout stream lines to stdout" do + expect { described_class.log(%({"stream":"stdout","text":"out"})) } + .to output("out\n").to_stdout + end + + it "writes internalout stream lines to stdout" do + expect { described_class.log(%({"stream":"internalout","text":"internal"})) } + .to output("internal\n").to_stdout + end + + it "writes stderr stream lines to stderr" do + expect { described_class.log(%({"stream":"stderr","text":"err"})) } + .to output("err\n").to_stderr + end + + it "ignores empty lines" do + expect { described_class.log("") }.not_to output.to_stdout + end + + it "raises on an unknown stream" do + expect { described_class.log(%({"stream":"mystery","text":"?"})) } + .to raise_error(/Unknown stream/) + end + end + + describe ".save_config / .load_config" do + let(:tmpdir) { Dir.mktmpdir } + let(:config_file) { File.join(tmpdir, ".morph") } + + before do + allow(described_class).to receive(:config_path).and_return(config_file) + end + + after do + FileUtils.remove_entry(tmpdir) + end + + it "round-trips a symbol-keyed config" do + config = { production: { api_key: "secret", base_url: "https://morph.io" } } + described_class.save_config(config) + + expect(described_class.load_config).to eq(config) + end + + it "writes the config file with 0600 permissions" do + described_class.save_config({ production: { api_key: "secret" } }) + + expect(File.stat(config_file).mode & 0o777).to eq(0o600) + end + + it "returns the default config when no file exists" do + expect(described_class.load_config).to eq(MorphCLI::DEFAULT_CONFIG) + end + end + + describe ".create_tar" do + it "packs the given paths into a readable tar" do + Dir.mktmpdir do |dir| + File.write(File.join(dir, "scraper.rb"), "puts 'hi'\n") + FileUtils.mkdir_p(File.join(dir, "lib")) + File.write(File.join(dir, "lib", "helper.rb"), "# helper\n") + paths = described_class.all_paths(dir) + + tar = described_class.create_tar(dir, paths) + + names = [] + Minitar::Input.open(tar.path) do |input| + input.each { |entry| names << entry.full_name } + end + expect(names).to contain_exactly("scraper.rb", "lib/helper.rb") + end + end + + it "returns an open file handle ready for reading" do + Dir.mktmpdir do |dir| + File.write(File.join(dir, "scraper.rb"), "puts 'hi'\n") + + tar = described_class.create_tar(dir, described_class.all_paths(dir)) + + expect(tar).not_to be_closed + expect(tar.pos).to eq(0) + expect(tar.read).to include("scraper.rb") + end + end + end + + describe ".get_dir_size" do + it "returns a human readable size of the given paths" do + Dir.mktmpdir do |dir| + File.write(File.join(dir, "scraper.rb"), "a" * 10) + + size = described_class.get_dir_size(dir, described_class.all_paths(dir)) + + expect(size).to eq("10.00 B") + end + end + end + + describe ".in_directory" do + it "runs the block in the given directory and restores the old one" do + original = Dir.pwd + Dir.mktmpdir do |dir| + described_class.in_directory(dir) do + expect(Dir.pwd).to eq(File.realpath(dir)) + end + expect(Dir.pwd).to eq(original) + end + end + + it "restores the working directory when the block raises" do + original = Dir.pwd + Dir.mktmpdir do |dir| + expect do + described_class.in_directory(dir) { raise "boom" } + end.to raise_error("boom") + expect(Dir.pwd).to eq(original) + end + end + end + describe ".all_paths" do it "excludes files inside dot-directories" do Dir.mktmpdir do |dir| diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index afc98ec..13167cb 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,4 +1,14 @@ +require "simplecov" +SimpleCov.start do + enable_coverage :branch + skip "/spec/" + minimum_coverage 90 +end + +require "webmock/rspec" + require "morph-cli" +require "morph-cli/cli" RSpec.configure do |config| config.disable_monkey_patching!