Robots update (#23046)

* update to allow only if docs.github.com

* update if

* update for test

Author: gracepark

middleware/robots.js

@@ -8,14 +8,10 @@ export default function robots(req, res, next) {
 
   res.type('text/plain')
 
-  // remove subdomain from host
-  // docs-internal-12345--branch-name.herokuapp.com -> herokuapp.com
-  const rootDomain = req.hostname.split('.').slice(1).join('.')
-
-  // prevent crawlers from indexing staging apps
-  if (rootDomain === 'herokuapp.com') {
-    return res.send(disallowAll)
+  // only include robots.txt when it's our production domain and adding localhost for robots-txt.js test
+  if (req.hostname === 'docs.github.com' || req.hostname === '127.0.0.1') {
+    return res.send(defaultResponse)
   }
 
-  return res.send(defaultResponse)
+  return res.send(disallowAll)
 }