Skip to content

Commit a52d563

Browse files
OctomergerOctomerger
authored
repo sync
2 parents a31eaa3 + 4928b9d commit a52d563

15 files changed

Lines changed: 169 additions & 131 deletions

‎.github/allowed-actions.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ module.exports = [
3030
'rachmari/labeler@832d42ec5523f3c6d46e8168de71cd54363e3e2e',
3131
'repo-sync/github-sync@3832fe8e2be32372e1b3970bbae8e7079edeec88',
3232
'repo-sync/pull-request@33777245b1aace1a58c87a29c90321aa7a74bd7d',
33-
'rtCamp/action-slack-notify@e17352feaf9aee300bf0ebc1dfbf467d80438815',
33+
'someimportantcompany/github-actions-slack-message@0b470c14b39da4260ed9e3f9a4f1298a74ccdefd',
3434
'tjenkinson/gh-action-auto-merge-dependency-updates@cee2ac0',
3535
'EndBug/add-and-commit@9358097a71ad9fb9e2f9624c6098c89193d83575'
3636
]

‎.github/workflows/repo-freeze-reminders.yml‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,10 @@ jobs:
1414
if: github.repository == 'github/docs-internal'
1515
steps:
1616
- name: Send Slack notification if repo is frozen
17+
uses: someimportantcompany/github-actions-slack-message@0b470c14b39da4260ed9e3f9a4f1298a74ccdefd
1718
if: ${{ env.FREEZE == 'true' }}
18-
uses: rtCamp/action-slack-notify@e17352feaf9aee300bf0ebc1dfbf467d80438815
19-
env:
20-
SLACK_WEBHOOK: ${{ secrets.DOCS_ALERTS_SLACK_WEBHOOK }}
21-
SLACK_USERNAME: docs-repo-sync
22-
SLACK_ICON_EMOJI: ':freezing_face:'
23-
SLACK_COLOR: '#51A0D5' # Carolina Blue
24-
SLACK_MESSAGE: All repo-sync runs will fail for ${{ github.repository }} because the repo is currently frozen!
19+
with:
20+
channel: ${{ secrets.DOCS_ALERTS_SLACK_CHANNEL_ID }}
21+
bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }}
22+
color: info
23+
text: All repo-sync runs will fail for ${{ github.repository }} because the repo is currently frozen!

‎.github/workflows/repo-sync-stalls.yml‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,10 @@ jobs:
4545
}
4646
})
4747
- name: Send Slack notification if workflow fails
48-
uses: rtCamp/action-slack-notify@e17352feaf9aee300bf0ebc1dfbf467d80438815
49-
if: ${{ failure() }}
50-
env:
51-
SLACK_WEBHOOK: ${{ secrets.DOCS_ALERTS_SLACK_WEBHOOK }}
52-
SLACK_USERNAME: docs-repo-sync
53-
SLACK_ICON_EMOJI: ':ohno:'
54-
SLACK_COLOR: '#B90E0A' # Crimson
55-
SLACK_MESSAGE: Repo sync appears to be stalled for ${{github.repository}}. See https://github.com/${{github.repository}}/pulls?q=is%3Apr+is%3Aopen+repo+sync
48+
uses: someimportantcompany/github-actions-slack-message@0b470c14b39da4260ed9e3f9a4f1298a74ccdefd
49+
if: failure()
50+
with:
51+
channel: ${{ secrets.DOCS_ALERTS_SLACK_CHANNEL_ID }}
52+
bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }}
53+
color: failure
54+
text: Repo sync appears to be stalled for ${{github.repository}}. See https://github.com/${{github.repository}}/pulls?q=is%3Apr+is%3Aopen+repo+sync

‎.github/workflows/repo-sync.yml‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -70,11 +70,10 @@ jobs:
7070
number: ${{ steps.find-pull-request.outputs.number }}
7171

7272
- name: Send Slack notification if workflow fails
73-
uses: rtCamp/action-slack-notify@e17352feaf9aee300bf0ebc1dfbf467d80438815
74-
if: ${{ failure() }}
75-
env:
76-
SLACK_WEBHOOK: ${{ secrets.DOCS_ALERTS_SLACK_WEBHOOK }}
77-
SLACK_USERNAME: docs-repo-sync
78-
SLACK_ICON_EMOJI: ':ohno:'
79-
SLACK_COLOR: '#B90E0A' # Crimson
80-
SLACK_MESSAGE: The last repo-sync run for ${{github.repository}} failed. See https://github.com/${{github.repository}}/actions?query=workflow%3A%22Repo+Sync%22
73+
uses: someimportantcompany/github-actions-slack-message@0b470c14b39da4260ed9e3f9a4f1298a74ccdefd
74+
if: failure()
75+
with:
76+
channel: ${{ secrets.DOCS_ALERTS_SLACK_CHANNEL_ID }}
77+
bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }}
78+
color: failure
79+
text: The last repo-sync run for ${{github.repository}} failed. See https://github.com/${{github.repository}}/actions?query=workflow%3A%22Repo+Sync%22

‎.github/workflows/sync-algolia-search-indices.yml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,10 @@ jobs:
3333
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
3434
run: npm run sync-search
3535
- name: Send slack notification if workflow run fails
36-
uses: rtCamp/action-slack-notify@e17352feaf9aee300bf0ebc1dfbf467d80438815
36+
uses: someimportantcompany/github-actions-slack-message@0b470c14b39da4260ed9e3f9a4f1298a74ccdefd
3737
if: failure()
38-
env:
39-
SLACK_WEBHOOK: ${{ secrets.DOCS_ALERTS_SLACK_WEBHOOK }}
40-
SLACK_MESSAGE: The last Algolia workflow run for ${{github.repository}} failed. Search actions for `workflow:Algolia`
38+
with:
39+
channel: ${{ secrets.DOCS_ALERTS_SLACK_CHANNEL_ID }}
40+
bot-token: ${{ secrets.SLACK_DOCS_BOT_TOKEN }}
41+
color: failure
42+
text: The last Algolia workflow run for ${{github.repository}} failed. Search actions for `workflow:Algolia`

‎lib/page.js‎

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
const assert = require('assert')
2-
const fs = require('fs')
2+
const fs = require('fs').promises
33
const path = require('path')
44
const cheerio = require('cheerio')
55
const patterns = require('./patterns')
@@ -23,15 +23,30 @@ const slash = require('slash')
2323
const statsd = require('./statsd')
2424

2525
class Page {
26-
constructor (opts) {
26+
static async init (opts) {
2727
assert(opts.relativePath, 'relativePath is required')
2828
assert(opts.basePath, 'basePath is required')
29+
30+
const relativePath = slash(opts.relativePath)
31+
const fullPath = slash(path.join(opts.basePath, relativePath))
32+
const raw = await fs.readFile(fullPath, 'utf8')
33+
34+
return new Page({ ...opts, relativePath, fullPath, raw })
35+
}
36+
37+
static async exists (path) {
38+
try {
39+
return await fs.stat(path)
40+
} catch (err) {
41+
if (err.code === 'ENOENT') return false
42+
console.error(err)
43+
}
44+
}
45+
46+
constructor (opts) {
2947
assert(opts.languageCode, 'languageCode is required')
3048

3149
Object.assign(this, { ...opts })
32-
this.relativePath = slash(this.relativePath)
33-
this.fullPath = slash(path.join(this.basePath, this.relativePath))
34-
this.raw = fs.readFileSync(this.fullPath, 'utf8')
3550

3651
// TODO remove this when crowdin-support issue 66 has been resolved
3752
if (this.languageCode !== 'en' && this.raw.includes(': verdadero')) {

‎lib/pages.js‎

Lines changed: 31 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -2,42 +2,50 @@ const path = require('path')
22
const walk = require('walk-sync').entries
33
const Page = require('./page')
44
const languages = require('./languages')
5-
const fs = require('fs')
5+
const { mapLimit, filterLimit } = require('async')
6+
const FILE_READ_LIMIT = 500
67

78
async function loadPageList () {
89
const pageList = []
910

1011
// load english pages
1112
const englishPath = path.join(__dirname, '..', languages.en.dir, 'content')
12-
const englishPages = walk(englishPath)
13-
.filter(({ relativePath }) => {
14-
return relativePath.endsWith('.md') &&
15-
!relativePath.includes('README')
16-
})
17-
.map(fileData => new Page({ ...fileData, languageCode: languages.en.code }))
18-
13+
const englishPaths = walk(englishPath)
14+
.filter(({ relativePath }) =>
15+
relativePath.endsWith('.md') && !relativePath.includes('README')
16+
)
17+
const englishPages = await mapLimit(
18+
englishPaths,
19+
FILE_READ_LIMIT,
20+
async fileData => await Page.init({ ...fileData, languageCode: languages.en.code })
21+
)
1922
pageList.push(...englishPages)
2023

2124
// load matching pages in other languages
22-
for (const page of englishPages) {
23-
for (const language of Object.values(languages)) {
24-
if (language.code === 'en') continue
25-
25+
let localizedPaths = Object.values(languages)
26+
.filter(({ code }) => code !== 'en')
27+
.map(language => {
2628
const basePath = path.join(__dirname, '..', language.dir, 'content')
27-
const localizedPath = path.join(basePath, page.relativePath)
28-
try {
29-
fs.statSync(localizedPath)
30-
} catch (_) {
31-
continue
32-
}
33-
34-
pageList.push(new Page({
35-
relativePath: page.relativePath,
29+
return englishPages.map(page => ({
3630
basePath,
31+
relativePath: page.relativePath,
32+
localizedPath: path.join(basePath, page.relativePath),
3733
languageCode: language.code
3834
}))
39-
}
40-
}
35+
})
36+
.flat()
37+
localizedPaths = await filterLimit(
38+
localizedPaths,
39+
FILE_READ_LIMIT,
40+
async ({ localizedPath }) => Page.exists(localizedPath)
41+
)
42+
const localizedPages = await mapLimit(
43+
localizedPaths,
44+
FILE_READ_LIMIT,
45+
async ({ basePath, relativePath, languageCode }) =>
46+
await Page.init({ basePath, relativePath, languageCode })
47+
)
48+
pageList.push(...localizedPages)
4149

4250
return pageList
4351
}

‎tests/browser/browser.js‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,17 @@ const sleep = require('await-sleep')
33
const querystring = require('querystring')
44

55
describe('homepage', () => {
6+
jest.setTimeout(60 * 1000)
7+
68
test('should be titled "GitHub Documentation"', async () => {
79
await page.goto('http://localhost:4001')
810
await expect(page.title()).resolves.toMatch('GitHub Documentation')
911
})
1012
})
1113

1214
describe('algolia browser search', () => {
15+
jest.setTimeout(60 * 1000)
16+
1317
it('works on the homepage', async () => {
1418
await page.goto('http://localhost:4001/en')
1519
await page.click('#search-input-container input[type="search"]')

‎tests/content/crowdin-config.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ const ignoredPagePaths = config.files[0].ignore
44
const ignoredDataPaths = config.files[2].ignore
55

66
describe('crowdin.yml config file', () => {
7+
jest.setTimeout(60 * 1000)
8+
79
let pages
810
beforeAll(async (done) => {
911
pages = await loadPages()

‎tests/content/site-data-references.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ const fs = require('fs').promises
77
const path = require('path')
88

99
describe('data references', () => {
10+
jest.setTimeout(60 * 1000)
11+
1012
let data, pages
1113

1214
beforeAll(async (done) => {

0 commit comments

Comments
 (0)