Merge branch 'main' into patch-2

Author: lecoursen

.github/workflows/openapi-decorate.yml

@@ -10,6 +10,11 @@ jobs:
     if: github.event.pull_request.user.login == 'github-openapi-bot'
     runs-on: ubuntu-latest
     steps:
+      - name: Label pull requests with 'github-openapi-bot'
+        uses: rachmari/labeler@832d42ec5523f3c6d46e8168de71cd54363e3e2e
+        with:
+          repo-token: '${{ secrets.GITHUB_TOKEN }}'
+          add-labels: 'github-openapi-bot'
       - name: Checkout repository code
         uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
 

middleware/robots.js

@@ -4,6 +4,28 @@ const { deprecated } = require('../lib/enterprise-server-releases.js')
 
 let defaultResponse = 'User-agent: *'
 
+// Disallow crawling of WIP localized content
+Object.values(languages)
+  .filter(language => language.wip)
+  .forEach(language => {
+    defaultResponse = defaultResponse.concat(`\nDisallow: /${language.code}\nDisallow: /${language.code}/*\n`)
+  })
+
+// Disallow crawling of WIP products
+Object.values(products)
+  .filter(product => product.wip)
+  .forEach(product => {
+    defaultResponse = defaultResponse.concat(`\nDisallow: /*${product.href}\nDisallow: /*/enterprise/*/user${product.href}`)
+  })
+
+// Disallow crawling of Deprecated enterprise versions
+deprecated
+  .forEach(version => {
+    defaultResponse = defaultResponse
+      .concat(`\nDisallow: /*/enterprise-server@${version}/*`)
+      .concat(`\nDisallow: /*/enterprise/${version}/*`)
+  })
+
 const disallowAll = `User-agent: *
 Disallow: /`
 
@@ -21,27 +43,5 @@ module.exports = function (req, res, next) {
     return res.send(disallowAll)
   }
 
-  // Disallow crawling of WIP localized content
-  Object.values(languages)
-    .filter(language => language.wip)
-    .forEach(language => {
-      defaultResponse = defaultResponse.concat(`\nDisallow: /${language.code}\nDisallow: /${language.code}/*\n`)
-    })
-
-  // Disallow crawling of WIP products
-  Object.values(products)
-    .filter(product => product.wip)
-    .forEach(product => {
-      defaultResponse = defaultResponse.concat(`\nDisallow: /*${product.href}\nDisallow: /*/enterprise/*/user${product.href}`)
-    })
-
-  // Disallow crawling of Deprecated enterprise versions
-  deprecated
-    .forEach(version => {
-      defaultResponse = defaultResponse
-        .concat(`\nDisallow: /*/enterprise-server@${version}/*`)
-        .concat(`\nDisallow: /*/enterprise/${version}/*`)
-    })
-
   return res.send(defaultResponse)
 }

tests/rendering/robots-txt.js

@@ -110,4 +110,13 @@ describe('robots.txt', () => {
       })
     })
   })
+
+  it('does not have duplicate lines', () => {
+    const lines = new Set()
+    for (const line of res.text.split('\n')) {
+      if (/^\s*$/.test(line)) continue
+      expect(lines.has(line)).toBe(false)
+      lines.add(line)
+    }
+  })
 })