Merge pull request #12292 from github/repo-sync

repo sync

Author: Octomerger

middleware/robots.js

@@ -8,14 +8,10 @@ export default function robots(req, res, next) {
 
   res.type('text/plain')
 
-  // remove subdomain from host
-  // docs-internal-12345--branch-name.herokuapp.com -> herokuapp.com
-  const rootDomain = req.hostname.split('.').slice(1).join('.')
-
-  // prevent crawlers from indexing staging apps
-  if (rootDomain === 'herokuapp.com') {
-    return res.send(disallowAll)
+  // only include robots.txt when it's our production domain and adding localhost for robots-txt.js test
+  if (req.hostname === 'docs.github.com' || req.hostname === '127.0.0.1') {
+    return res.send(defaultResponse)
   }
 
-  return res.send(defaultResponse)
+  return res.send(disallowAll)
 }

package.json

@@ -205,7 +205,7 @@
     "link-check-server": "cross-env NODE_ENV=production ENABLED_LANGUAGES='en' PORT=4002 node server.mjs",
     "link-check-test": "cross-env node script/check-internal-links.js",
     "lint": "eslint '**/*.{js,mjs,ts,tsx}'",
-    "lint-translation": "cross-env TEST_TRANSLATION=true jest content/lint-files",
+    "lint-translation": "cross-env NODE_OPTIONS=--experimental-vm-modules TEST_TRANSLATION=true jest tests/linting/lint-files.js",
     "pa11y-ci": "pa11y-ci",
     "pa11y-test": "start-server-and-test browser-test-server 4001 pa11y-ci",
     "prebrowser-test": "npm run build",

script/i18n/lint-translation-files.js

@@ -12,7 +12,7 @@ import program from 'commander'
 
 // Set up supported linting check types and their corresponding commands.
 const CHECK_COMMANDS = {
-  parsing: 'TEST_TRANSLATION=true npx jest linting/lint-files',
+  parsing: 'npm run lint-translation',
   rendering: 'script/i18n/test-render-translation.js',
 }
 const SUPPORTED_CHECK_TYPES = Object.keys(CHECK_COMMANDS)

tests/linting/lint-files.js

@@ -679,10 +679,18 @@ describe('lint yaml content', () => {
   if (ymlToLint.length < 1) return
   describe.each(ymlToLint)('%s', (yamlRelPath, yamlAbsPath) => {
     let dictionary, isEarlyAccess, ifversionConditionals, ifConditionals
+    // This variable is used to determine if the file was parsed successfully.
+    // When `yaml.load()` fails to parse the file, it is overwritten with the error message.
+    // `false` is intentionally chosen since `null` and `undefined` are valid return values.
+    let dictionaryError = false
 
     beforeAll(async () => {
       const fileContents = await readFileAsync(yamlAbsPath, 'utf8')
-      dictionary = yaml.load(fileContents, { filename: yamlRelPath })
+      try {
+        dictionary = yaml.load(fileContents, { filename: yamlRelPath })
+      } catch (error) {
+        dictionaryError = error
+      }
 
       isEarlyAccess = yamlRelPath.split('/').includes('early-access')
 
@@ -691,6 +699,10 @@ describe('lint yaml content', () => {
       ifConditionals = getLiquidConditionals(fileContents, 'if')
     })
 
+    test('it can be parsed as a single yaml document', () => {
+      expect(dictionaryError).toBe(false)
+    })
+
     test('ifversion conditionals are valid in yaml', async () => {
       const errors = validateIfversionConditionals(ifversionConditionals)
       expect(errors.length, errors.join('\n')).toBe(0)
@@ -907,10 +919,19 @@ describe('lint GHES release notes', () => {
   if (ghesReleaseNotesToLint.length < 1) return
   describe.each(ghesReleaseNotesToLint)('%s', (yamlRelPath, yamlAbsPath) => {
     let dictionary
+    let dictionaryError = false
 
     beforeAll(async () => {
       const fileContents = await readFileAsync(yamlAbsPath, 'utf8')
-      dictionary = yaml.load(fileContents, { filename: yamlRelPath })
+      try {
+        dictionary = yaml.load(fileContents, { filename: yamlRelPath })
+      } catch (error) {
+        dictionaryError = error
+      }
+    })
+
+    it('can be parsed as a single yaml document', () => {
+      expect(dictionaryError).toBe(false)
     })
 
     it('matches the schema', () => {
@@ -954,10 +975,19 @@ describe('lint GHAE release notes', () => {
   const currentWeeksFound = []
   describe.each(ghaeReleaseNotesToLint)('%s', (yamlRelPath, yamlAbsPath) => {
     let dictionary
+    let dictionaryError = false
 
     beforeAll(async () => {
       const fileContents = await readFileAsync(yamlAbsPath, 'utf8')
-      dictionary = yaml.load(fileContents, { filename: yamlRelPath })
+      try {
+        dictionary = yaml.load(fileContents, { filename: yamlRelPath })
+      } catch (error) {
+        dictionaryError = error
+      }
+    })
+
+    it('can be parsed as a single yaml document', () => {
+      expect(dictionaryError).toBe(false)
     })
 
     it('matches the schema', () => {
@@ -1008,10 +1038,19 @@ describe('lint learning tracks', () => {
   if (learningTracksToLint.length < 1) return
   describe.each(learningTracksToLint)('%s', (yamlRelPath, yamlAbsPath) => {
     let dictionary
+    let dictionaryError = false
 
     beforeAll(async () => {
       const fileContents = await readFileAsync(yamlAbsPath, 'utf8')
-      dictionary = yaml.load(fileContents, { filename: yamlRelPath })
+      try {
+        dictionary = yaml.load(fileContents, { filename: yamlRelPath })
+      } catch (error) {
+        dictionaryError = error
+      }
+    })
+
+    it('can be parsed as a single yaml document', () => {
+      expect(dictionaryError).toBe(false)
     })
 
     it('matches the schema', () => {
@@ -1083,10 +1122,19 @@ describe('lint feature versions', () => {
   if (featureVersionsToLint.length < 1) return
   describe.each(featureVersionsToLint)('%s', (yamlRelPath, yamlAbsPath) => {
     let dictionary
+    let dictionaryError = false
 
     beforeAll(async () => {
       const fileContents = await readFileAsync(yamlAbsPath, 'utf8')
-      dictionary = yaml.load(fileContents, { filename: yamlRelPath })
+      try {
+        dictionary = yaml.load(fileContents, { filename: yamlRelPath })
+      } catch (error) {
+        dictionaryError = error
+      }
+    })
+
+    it('can be parsed as a single yaml document', () => {
+      expect(dictionaryError).toBe(false)
     })
 
     it('matches the schema', () => {