Fix integration exit codes and add more details to output

Author: Kevin Paulisse

spec/octocatalog-diff/integration/catalog_only_spec.rb

@@ -26,43 +26,36 @@
 
   it 'should compile the catalog' do
     expect(@result[:exitcode]).not_to eq(-1), OctocatalogDiff::Integration.format_exception(@result)
-    expect(@result[:exitcode]).to eq(2), "Runtime error: #{@result[:logs]}"
-  end
-
-  it 'should set the from-catalog to a no-op catalog type' do
-    pending 'catalog compilation failed' unless @result[:exitcode] == 2
-    from_catalog = @result[:diffs][0]
-    expect(from_catalog).to be_a_kind_of(OctocatalogDiff::API::V1::Catalog)
-    expect(from_catalog.builder).to eq('OctocatalogDiff::Catalog::Noop')
+    expect(@result[:exitcode]).to eq(0), "Runtime error: #{@result[:logs]}"
   end
 
   it 'should set the to-catalog to a computed catalog type' do
-    pending 'catalog compilation failed' unless @result[:exitcode] == 2
-    to_catalog = @result[:diffs][1]
+    pending 'catalog compilation failed' unless (@result[:exitcode]).zero?
+    to_catalog = @result.to
     expect(to_catalog).to be_a_kind_of(OctocatalogDiff::API::V1::Catalog)
     expect(to_catalog.builder).to eq('OctocatalogDiff::Catalog::Computed')
   end
 
   it 'should have log messages indicating catalog compilations' do
-    pending 'catalog compilation failed' unless @result[:exitcode] == 2
+    pending 'catalog compilation failed' unless (@result[:exitcode]).zero?
     logs = @result[:logs]
     expect(logs).to match(/Compiling catalog for rspec-node.github.net/)
     expect(logs).to match(/Initialized OctocatalogDiff::Catalog::Noop for from-catalog/)
     expect(logs).to match(/Initialized OctocatalogDiff::Catalog::Computed for to-catalog/)
   end
 
   it 'should produce a valid catalog' do
-    pending 'catalog compilation failed' unless @result[:exitcode] == 2
-    to_catalog = @result[:diffs][1]
+    pending 'catalog compilation failed' unless (@result[:exitcode]).zero?
+    to_catalog = @result.to
     expect(to_catalog.valid?).to eq(true)
     expect(to_catalog.to_h).to be_a_kind_of(Hash)
     expect(to_catalog.to_json).to be_a_kind_of(String)
     expect(to_catalog.error_message).to be(nil)
   end
 
   it 'should produce the expected catalog' do
-    pending 'catalog compilation failed' unless @result[:exitcode] == 2
-    to_catalog = @result[:diffs][1]
+    pending 'catalog compilation failed' unless @result.exitcode.zero?
+    to_catalog = @result.to
 
     param1 = { 'owner' => 'root', 'group' => 'root', 'mode' => '0644', 'content' => 'Testy McTesterson' }
     expect(to_catalog.resource(type: 'File', title: '/tmp/foo')['parameters']).to eq(param1)

spec/octocatalog-diff/integration/integration_helper.rb

@@ -113,39 +113,40 @@ def self.integration(options = {})
       options[:from_puppet_binary] ||= OctocatalogDiff::Spec::PUPPET_BINARY
       options[:to_puppet_binary] ||= OctocatalogDiff::Spec::PUPPET_BINARY
       options[:parallel] = false if ENV['COVERAGE']
+      options[:INTEGRATION] = true
 
       # Run octocatalog-diff CLI method. Capture stdout and stderr using 'strio'.
-      # Set options[:RETURN_DIFFS] so that the .cli method returns the JSON array
-      # of differences instead of the exit code.
       logger, logger_string = OctocatalogDiff::Spec.setup_logger
       begin
         # Capture stdout to a variable
         old_out = $stdout
         stdout_strio = StringIO.new
         $stdout = stdout_strio
 
-        # Tell OctocatalogDiff::Cli.cli to return the JSON differences and not a numeric exit code
-        # for a full catalog-diff.
-        options[:RETURN_DIFFS] = true
-
         # Run the OctocatalogDiff::Cli.cli and validate output format.
         result = OctocatalogDiff::Cli.cli(argv, logger, options)
         if result.is_a?(Fixnum)
-          return {
-            logs: logger_string.string,
-            output: stdout_strio.string,
-            exitcode: result
-          }
+          result = OpenStruct.new(exitcode: result, exception: nil, diffs: [], to: nil, from: nil)
+        elsif result.is_a?(Hash)
+          result = OpenStruct.new({ exitcode: nil, exception: nil, diffs: [], to: nil, from: nil }.merge(result))
+        elsif !result.is_a?(OpenStruct)
+          raise "Expected OpenStruct, got #{result.inspect} from OctocatalogDiff::Cli.cli!"
         end
 
-        raise "OctocatalogDiff::Cli.cli should return array, got #{result.inspect}" unless result.is_a?(Array)
+        exitcode = if result.exitcode.nil?
+          result.diffs.any? ? 2 : 0
+        else
+          result.exitcode
+        end
 
-        # Return hash
         OpenStruct.new(
           logs: logger_string.string,
+          log_messages: logger_string.string.split(/\n/).map { |x| OctocatalogDiff::Spec.strip_log_message(x) },
           output: stdout_strio.string,
-          diffs: result,
-          exitcode: result.any? ? 2 : 0,
+          diffs: result.diffs,
+          to: result.to,
+          from: result.from,
+          exitcode: exitcode,
           options: options
         )
       rescue => exc # Yes, rescue *everything*

spec/octocatalog-diff/integration/reference_validation_spec.rb

@@ -45,7 +45,7 @@ def self.catalog_contains_resource(result, type, title)
   end
 
   it 'should return the valid catalog' do
-    expect(@result.exitcode).to eq(2), OctocatalogDiff::Integration.format_exception(@result)
+    expect(@result.exitcode).to eq(0), OctocatalogDiff::Integration.format_exception(@result)
   end
 end
 
@@ -55,15 +55,15 @@ def self.catalog_contains_resource(result, type, title)
   end
 
   it 'should return the valid catalog' do
-    expect(@result.exitcode).to eq(2)
+    expect(@result.exitcode).to eq(0)
   end
 
   it 'should not raise any exceptions' do
     expect(@result.exception).to be_nil, OctocatalogDiff::Integration.format_exception(@result)
   end
 
   it 'should contain representative resources' do
-    pending 'Catalog failed' unless @result.exitcode == 2
+    pending 'Catalog failed' unless @result.exitcode.zero?
     expect(OctocatalogDiff::Spec.catalog_contains_resource(@result, 'File', '/tmp/test-main')).to eq(true)
   end
 end
@@ -75,15 +75,15 @@ def self.catalog_contains_resource(result, type, title)
     end
 
     it 'should succeed' do
-      expect(@result.exitcode).to eq(2)
+      expect(@result.exitcode).to eq(0)
     end
 
     it 'should not raise any exceptions' do
       expect(@result.exception).to be_nil, OctocatalogDiff::Integration.format_exception(@result)
     end
 
     it 'should contain representative resources' do
-      pending 'Catalog failed' unless @result.exitcode == 2
+      pending 'Catalog failed' unless @result.exitcode.zero?
       expect(OctocatalogDiff::Spec.catalog_contains_resource(@result, 'Exec', 'subscribe caller 1')).to eq(true)
       expect(OctocatalogDiff::Spec.catalog_contains_resource(@result, 'Exec', 'subscribe target')).to eq(true)
     end
@@ -179,7 +179,7 @@ def self.catalog_contains_resource(result, type, title)
     end
 
     it 'should succeed' do
-      expect(@result.exitcode).to eq(2), OctocatalogDiff::Integration.format_exception(@result)
+      expect(@result.exitcode).to eq(0), OctocatalogDiff::Integration.format_exception(@result)
     end
 
     it 'should not raise error' do
@@ -195,15 +195,15 @@ def self.catalog_contains_resource(result, type, title)
     end
 
     it 'should succeed' do
-      expect(@result.exitcode).to eq(2)
+      expect(@result.exitcode).to eq(0)
     end
 
     it 'should not raise any exceptions' do
       expect(@result.exception).to be_nil, OctocatalogDiff::Integration.format_exception(@result)
     end
 
     it 'should contain representative resources' do
-      pending 'Catalog failed' unless @result.exitcode == 2
+      pending 'Catalog failed' unless @result.exitcode.zero?
       expect(OctocatalogDiff::Spec.catalog_contains_resource(@result, 'Exec', 'before alias caller')).to eq(true)
       expect(OctocatalogDiff::Spec.catalog_contains_resource(@result, 'Exec', 'before alias target')).to eq(true)
       expect(OctocatalogDiff::Spec.catalog_contains_resource(@result, 'Exec', 'the before alias target')).to eq(true)