diff --git a/.github/actions/build-website/action.yml b/.github/actions/build-website/action.yml index e41e20bb4..0f6937ae0 100644 --- a/.github/actions/build-website/action.yml +++ b/.github/actions/build-website/action.yml @@ -31,7 +31,7 @@ runs: java-version: ${{ inputs.java-version }} - name: Setup mise - uses: jdx/mise-action@5228313ee0372e111a38da051671ca30fc5a96db # v3 + uses: jdx/mise-action@3c2e0cf82a5b2e5249f0d3635a4d83d0ae861518 # v4 - name: Setup JBang shell: bash diff --git a/.github/appstore.java b/.github/appstore.java index af44888be..22d6d9267 100755 --- a/.github/appstore.java +++ b/.github/appstore.java @@ -1,12 +1,12 @@ ///usr/bin/env jbang "$0" "$@" ; exit $? -//DEPS info.picocli:picocli:4.5.0 -//DEPS org.kohsuke:github-api:1.128 -//DEPS com.google.code.gson:gson:2.8.6 -//DEPS org.commonmark:commonmark:0.17.1 -//DEPS org.commonmark:commonmark-ext-gfm-tables:0.17.1 -//DEPS org.commonmark:commonmark-ext-autolink:0.17.1 -//DEPS org.commonmark:commonmark-ext-gfm-strikethrough:0.17.1 -//DEPS org.commonmark:commonmark-ext-task-list-items:0.17.1 +//DEPS info.picocli:picocli:4.7.7 +//DEPS org.kohsuke:github-api:1.330 +//DEPS com.google.code.gson:gson:2.14.0 +//DEPS org.commonmark:commonmark:0.30.0 +//DEPS org.commonmark:commonmark-ext-gfm-tables:0.30.0 +//DEPS org.commonmark:commonmark-ext-autolink:0.30.0 +//DEPS org.commonmark:commonmark-ext-gfm-strikethrough:0.30.0 +//DEPS org.commonmark:commonmark-ext-task-list-items:0.30.0 import static java.lang.System.out; @@ -37,13 +37,14 @@ import org.commonmark.ext.task.list.items.TaskListItemsExtension; import org.commonmark.parser.Parser; import org.commonmark.renderer.html.HtmlRenderer; -import org.kohsuke.github.AbuseLimitHandler; +import org.kohsuke.github.GitHubAbuseLimitHandler; import org.kohsuke.github.GHContent; import org.kohsuke.github.GHException; import org.kohsuke.github.GitHub; import org.kohsuke.github.GitHubBuilder; import org.kohsuke.github.PagedSearchIterable; -import org.kohsuke.github.RateLimitHandler; +import org.kohsuke.github.HttpException; +import org.kohsuke.github.GitHubRateLimitHandler; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -96,8 +97,8 @@ public static void printExceptionCauseChain(Throwable ex) { @Override public Integer call() throws Exception { gitHub = GitHubBuilder.fromEnvironment().withOAuthToken(ghToken) - .withAbuseLimitHandler(AbuseLimitHandler.WAIT) - .withRateLimitHandler(RateLimitHandler.WAIT).build(); + .withAbuseLimitHandler(GitHubAbuseLimitHandler.WAIT) + .withRateLimitHandler(GitHubRateLimitHandler.WAIT).build(); out.println(gitHub.getRateLimit().toString()); var gson = new GsonBuilder().setPrettyPrinting().create(); @@ -132,6 +133,29 @@ public Integer call() throws Exception { processed++; index++; retries = 5; + } catch (HttpException he) { + int code = he.getResponseCode(); + if (code == 400 || code == 404 || code == 422 || code == 451 || (code == 403 && !isRateLimit(he))) { + out.println("Skipping " + location + " due to HTTP " + code + " error."); + processed++; + index++; + retries = 5; + } else { + printExceptionCauseChain(he); + retries--; + if (retries <= 0) { + throw he; + } + Duration wait = backoffDelay(he); + out.println("GitHub returned HTTP " + code + " while processing " + location + ". Sleeping " + wait.toMinutes() + + "m before retrying (" + retries + " retries left)"); + TimeUnit.MILLISECONDS.sleep(wait.toMillis()); + } + } catch (IOException ioe) { + out.println("Skipping " + location + " due to IOException: " + ioe.getMessage()); + processed++; + index++; + retries = 5; } catch (GHException ghe) { printExceptionCauseChain(ghe); retries--; @@ -295,6 +319,16 @@ private boolean isSecondaryRateLimit(Throwable throwable) { } return false; } + private boolean isRateLimit(Throwable throwable) { + while (throwable != null) { + String message = throwable.getMessage(); + if (message != null && (message.contains("429") || message.toLowerCase().contains("rate limit"))) { + return true; + } + throwable = throwable.getCause(); + } + return false; + } private Catalog toJsonElement(Gson gson, GHContent catalogContent) throws IOException { if (catalogContent == null) diff --git a/.github/workflows/appstore-update.yml b/.github/workflows/appstore-update.yml index ee98e7796..f99a57767 100644 --- a/.github/workflows/appstore-update.yml +++ b/.github/workflows/appstore-update.yml @@ -9,8 +9,8 @@ jobs: appstore-update: runs-on: ubuntu-latest steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - uses: jdx/mise-action@5228313ee0372e111a38da051671ca30fc5a96db # v3 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: jdx/mise-action@3c2e0cf82a5b2e5249f0d3635a4d83d0ae861518 # v4 - run: jbang statsquery2.java --token ${{ secrets.CLOUDFLARE_TOKEN }} --account ${{ secrets.CLOUDFLARE_ACCOUNT }} env: GITHUB_TOKEN: ${{ secrets.APPSTORE_GH_TOKEN != '' && secrets.APPSTORE_GH_TOKEN || secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 22c85f62e..ba9faf981 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: fetch-depth: 5000 fetch-tags: false diff --git a/.github/workflows/github-pages.yml b/.github/workflows/github-pages.yml index 53937536d..a76f9caae 100644 --- a/.github/workflows/github-pages.yml +++ b/.github/workflows/github-pages.yml @@ -23,7 +23,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - name: Build Website uses: ./.github/actions/build-website