Start an AB test with Lunr search results as treatment (#18993)

* Start search_lunr AB test

* Towards prewarming

* Towards prewarming

* Warm only english/dotcom index

* Update warm-server.js

Author: heiskr

javascripts/experiment.js

@@ -27,4 +27,13 @@ export default function () {
   // const x = document.querySelector(...)
   // x.addEventListener('click', () => { sendSuccess(testName) })
   // if (xbucket === TREATMENT) applyTreatment(x)
+
+  const testName = 'search_lunr'
+  const xbucket = bucket(testName)
+  document.addEventListener('click', evt => {
+    if (!evt.target.closest('.search-result > a')) return
+    console.log(testName, xbucket) // eslint-disable-line
+    sendSuccess(testName)
+  })
+  // Treatment code in middleware/search.js
 }

lib/search/lunr-search.js

@@ -1,5 +1,11 @@
 const path = require('path')
 const lunr = require('lunr')
+require('lunr-languages/lunr.stemmer.support')(lunr)
+require('lunr-languages/tinyseg')(lunr)
+require('lunr-languages/lunr.ja')(lunr)
+require('lunr-languages/lunr.es')(lunr)
+require('lunr-languages/lunr.pt')(lunr)
+require('lunr-languages/lunr.de')(lunr)
 const { get } = require('lodash')
 const readFileAsync = require('../readfile-async')
 const { namePrefix } = require('./config')
@@ -9,7 +15,7 @@ const LUNR_DIR = './indexes'
 const lunrIndexes = new Map()
 const lunrRecords = new Map()
 
-module.exports = async function loadLunrResults ({ version, language, query, limit }) {
+async function loadLunrResults ({ version, language, query, limit }) {
   const indexName = `${namePrefix}-${version}-${language}`
   if (!lunrIndexes.has(indexName) || !lunrRecords.has(indexName)) {
     lunrIndexes.set(indexName, await loadLunrIndex(indexName))
@@ -33,6 +39,15 @@ module.exports = async function loadLunrResults ({ version, language, query, lim
   return results
 }
 
+loadLunrResults.warmLunr = async function warmLunr () {
+  // Took about 60 seconds on local to warm them all...
+  // so doing just the most common for prewarming
+  const indexName = 'github-docs-dotcom-en'
+  lunrRecords.set(indexName, await loadLunrRecords(indexName))
+  lunrIndexes.set(indexName, await loadLunrIndex(indexName))
+  return true
+}
+
 async function loadLunrIndex (indexName) {
   const filePath = path.posix.join(__dirname, LUNR_DIR, `${indexName}.json.br`)
   // Do not set to 'utf8' on file reads
@@ -82,3 +97,5 @@ function field (result, record, name) {
 function mark (text) {
   return `<mark>${text}</mark>`
 }
+
+module.exports = loadLunrResults

lib/warm-server.js

@@ -3,6 +3,7 @@ const { loadPages, loadPageMap } = require('./pages')
 const loadRedirects = require('./redirects/precompile')
 const loadSiteData = require('./site-data')
 const loadSiteTree = require('./site-tree')
+const { warmLunr } = require('./search/lunr-search')
 
 // Instrument these functions so that
 // it's wrapped in a timer that reports to Datadog
@@ -11,15 +12,16 @@ const dog = {
   loadPageMap: statsd.asyncTimer(loadPageMap, 'load_page_map'),
   loadRedirects: statsd.asyncTimer(loadRedirects, 'load_redirects'),
   loadSiteData: statsd.timer(loadSiteData, 'load_site_data'),
-  loadSiteTree: statsd.asyncTimer(loadSiteTree, 'load_site_tree')
+  loadSiteTree: statsd.asyncTimer(loadSiteTree, 'load_site_tree'),
+  warmLunr: statsd.asyncTimer(warmLunr, 'warm_lunr')
 }
 
 // For local caching
-let pageList, pageMap, site, redirects, siteTree
+let pageList, pageMap, site, redirects, siteTree, isLunrWarmed
 
 function isFullyWarmed () {
   // NOTE: Yes, `pageList` is specifically excluded here as it is transient data
-  const fullyWarmed = !!(pageMap && site && redirects && siteTree)
+  const fullyWarmed = !!(pageMap && site && redirects && siteTree && isLunrWarmed)
   return fullyWarmed
 }
 
@@ -28,7 +30,8 @@ function getWarmedCache () {
     pages: pageMap,
     site,
     redirects,
-    siteTree
+    siteTree,
+    isLunrWarmed
   }
 }
 
@@ -59,6 +62,10 @@ async function warmServer () {
     siteTree = await dog.loadSiteTree(pageMap, site, redirects)
   }
 
+  if (!isLunrWarmed) {
+    isLunrWarmed = /* await */ dog.warmLunr()
+  }
+
   if (process.env.NODE_ENV !== 'test') {
     console.log(`Context primed in ${Date.now() - startTime} ms`)
   }

middleware/search.js

@@ -4,6 +4,10 @@ const versions = new Set(Object.values(require('../lib/search/versions')))
 const loadLunrResults = require('../lib/search/lunr-search')
 const loadAlgoliaResults = require('../lib/search/algolia-search')
 
+// temp
+const murmur = require('imurmurhash')
+// end temp
+
 const router = express.Router()
 
 router.get('/', async function postSearch (req, res, next) {
@@ -21,8 +25,16 @@ router.get('/', async function postSearch (req, res, next) {
     return res.status(200).json([])
   }
 
+  // temp
+  const userEventsId = req.cookies['_docs-events']
+  const experimentHash = userEventsId
+    ? murmur('search_lunr').hash(userEventsId).result()
+    : 0
+  const inTreatment = experimentHash % 2
+  // end temp
+
   try {
-    const results = process.env.AIRGAP || req.cookies.AIRGAP
+    const results = process.env.AIRGAP || req.cookies.AIRGAP || inTreatment
       ? await loadLunrResults({ version, language, query: `${query} ${filters || ''}`, limit })
       : await loadAlgoliaResults({ version, language, query, filters, limit })
     return res.status(200).json(results)