diff --git a/definitions/output/crawl/pages.js b/definitions/output/crawl/pages.js index 737b33a1..107ae6ed 100644 --- a/definitions/output/crawl/pages.js +++ b/definitions/output/crawl/pages.js @@ -32,6 +32,20 @@ ORDER BY cnt_pages DESC HAVING cnt_pages > 200 `) +assert('pages_per_client') + .tags(['crawl_complete']) + .query(ctx => ` +SELECT + client, + COUNT(DISTINCT page) AS cnt_pages +FROM ${ctx.ref('crawl_staging', 'pages')} +WHERE + date = '${constants.currentMonth}' +GROUP BY + client +HAVING cnt_pages < 20000000 + `) + publish('pages', { type: 'incremental', protected: true, diff --git a/definitions/output/reports/reports_html_elements_popularity.js b/definitions/output/reports/reports_html_elements_popularity.js index fe5f4f94..fec88a01 100644 --- a/definitions/output/reports/reports_html_elements_popularity.js +++ b/definitions/output/reports/reports_html_elements_popularity.js @@ -1,5 +1,3 @@ -const pastMonth = constants.fnPastMonth(constants.currentMonth) - publish('html_elements_popularity', { schema: 'reports', type: 'incremental', @@ -18,7 +16,7 @@ try { '''; DELETE FROM ${ctx.self()} -WHERE date = '${pastMonth}'; +WHERE date = '${constants.currentMonth}'; `).query(ctx => ` WITH pages_data AS ( SELECT @@ -29,7 +27,7 @@ WITH pages_data AS ( custom_metrics.element_count FROM ${ctx.ref('crawl', 'pages')} WHERE - date = '${pastMonth}' ${constants.devRankFilter} + date = '${constants.currentMonth}' ${constants.devRankFilter} ), totals AS ( @@ -69,9 +67,9 @@ SELECT "destination": "cloud_storage", "config": { "bucket": "${constants.bucket}", - "name": "${constants.storagePath}${pastMonth.replaceAll('-', '_')}/htmlElementPopularity.json" + "name": "${constants.storagePath}${constants.currentMonth.replaceAll('-', '_')}/htmlElementPopularity.json" }, - "query": "SELECT * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${pastMonth}'" + "query": "SELECT * EXCEPT(date) FROM ${ctx.self()} WHERE date = '${constants.currentMonth}'" }''' ); `)