Smart backfill (#265)

* Smart backfill

* Update tests

Author: OsamaSayegh

app/services/commits_runner.rb

@@ -1,12 +1,13 @@
 module CommitsRunner
-  def self.run(trigger_source, commits, repo, pattern = '')
+  def self.run(trigger_source, commits, repo, pattern = '', smart: false)
     formatted_commits =
       if trigger_source == :webhook
         format_webhook(commits, repo)
       elsif trigger_source == :api
         format_api(commits, repo)
       end
 
+    formatted_commits = smart_reorder(formatted_commits) if smart
     formatted_commits.select { |commit| valid?(commit) }
                      .each { |commit| create_and_run(commit, pattern) }
                      .count
@@ -59,4 +60,32 @@ def self.create_and_run(commit, pattern)
       include_patterns: pattern
     )
   end
+
+  def self.smart_reorder(commits)
+    return commits if commits.size < 3
+
+    reordered = []
+    reordered << commits.first
+    reordered << commits[commits.size / 2]
+    reordered << commits.last
+
+    depth = 1
+    while (reorder_recursive(commits, 1, commits.size - 1, depth, reordered))
+      depth += 1
+    end
+    reordered
+  end
+
+  def self.reorder_recursive(commits, first, last, depth, reordered)
+    return false if last <= first
+    mid = first + (last - first) / 2
+    if (depth == 0)
+      reordered << commits[mid]
+      true
+    else
+      half1 = reorder_recursive(commits, first, mid, depth - 1, reordered)
+      half2 = reorder_recursive(commits, mid + 1, last, depth - 1, reordered)
+      half1 || half2
+    end
+  end
 end

app/services/manual_runner.rb

@@ -12,24 +12,25 @@ def run_releases(versions, pattern: '')
   end
 
   def run_last(commits_count, pattern: '')
-    if commits_count < 100
-      run_commits(per_page: commits_count, pattern: pattern)
-    else
-      run_paginated(commits_count, pattern: pattern)
-    end
+    commits = fetch_commits(commits_count)
+    CommitsRunner.run(:api, commits, @repo, pattern, smart: true)
   end
 
   private
 
-  def run_paginated(commits_count, page: 1, pattern: '')
-    unless (commits_count <= 0)
-      count_run = run_commits(page: page, pattern: pattern)
-      run_paginated(commits_count - count_run, page: page + 1, pattern: pattern)
+  def fetch_commits(commits_count)
+    commits = []
+
+    runs = commits_count.ceil(-2) / 100
+    per_page = [commits_count, 100].min
+
+    runs.times do |n|
+      page = n + 1
+      batch = @octokit.commits("#{@repo.organization.name}/#{@repo.name}", per_page: per_page, page: page)
+      commits.push(*batch)
     end
-  end
 
-  def run_commits(page: 1, per_page: 100, pattern: '')
-    commits = @octokit.commits("#{@repo.organization.name}/#{@repo.name}", per_page: per_page, page: page)
-    CommitsRunner.run(:api, commits, @repo, pattern)
+    commits.pop(commits.size - commits_count)
+    commits
   end
 end

test/services/commits_runner_test.rb

@@ -26,6 +26,35 @@ class CommitsRunnerTest < ActiveSupport::TestCase
     assert_enqueued_jobs(commits.count)
   end
 
+  test '#run can do smart reorder' do
+    commits = []
+    10.times do |n|
+      commits << {
+        'sha' => "abcd#{n}",
+        'commit' => {
+          'message' => 'My beautiful commit message',
+          'author' => {
+            'date' => 12345,
+            'name' => 'bmarkons'
+          }
+        },
+        'html_url' => 'https://github.com/commit'
+      }
+    end
+    CommitsRunner.run(:api, commits, @repo, '', smart: true)
+
+    expect_created(commits)
+    assert_enqueued_jobs(commits.count)
+    order = Commit
+            .unscope(:order)
+            .where(sha1: commits.map { |c| c['sha'] })
+            .order('created_at ASC')
+            .pluck(:sha1)
+
+    expected_order = [0, 5, 9, 3, 7, 2, 4, 6, 8, 1].map { |n| "abcd#{n}" }
+    assert_equal(expected_order, order)
+  end
+
   test '#run commits triggered manually' do
     commits = [
       {

test/services/manual_runner_test.rb

@@ -19,15 +19,15 @@ class ManualRunnerTest < ActiveSupport::TestCase
   end
 
   test 'run last 200 commits' do
-    expect_to_run count: 100, times: 2
+    expect_to_run count: 200, times: 1, smart: true
 
     VCR.use_cassette('github 200 commits') do
       ManualRunner.new(@repo).run_last(200)
     end
   end
 
   test 'run last 20 commits' do
-    expect_to_run count: 20, times: 1
+    expect_to_run count: 20, times: 1, smart: true
 
     VCR.use_cassette('github 20 commits') do
       ManualRunner.new(@repo).run_last(20)
@@ -44,8 +44,8 @@ class ManualRunnerTest < ActiveSupport::TestCase
 
   private
 
-  def expect_to_run(count:, times:)
-    CommitsRunner.expects(:run).times(times).returns(count).with do |source, commits, repo, pattern|
+  def expect_to_run(count:, times:, smart: false)
+    CommitsRunner.expects(:run).times(times).returns(count).with do |source, commits, repo, pattern, opts|
       commits.each do |commit|
         assert commit['sha']
         assert commit['commit']['message']
@@ -58,6 +58,7 @@ def expect_to_run(count:, times:)
       end
 
       assert_equal count, commits.count
+      assert_equal smart, opts[:smart]
     end
   end
 end