diff --git a/.env.cleaner b/.env.cleaner index cf1cdd9..954918e 100644 --- a/.env.cleaner +++ b/.env.cleaner @@ -1,4 +1,4 @@ RELEASE=dev VERSION=1 BUILD=3 -FIX=7 +FIX=8 diff --git a/.env.fileproc b/.env.fileproc index 0c14c84..eca0603 100644 --- a/.env.fileproc +++ b/.env.fileproc @@ -1,4 +1,4 @@ RELEASE=dev VERSION=1 BUILD=3 -FIX=3 +FIX=4 diff --git a/.env.scrapper b/.env.scrapper index 82954df..cf1cdd9 100644 --- a/.env.scrapper +++ b/.env.scrapper @@ -1,4 +1,4 @@ RELEASE=dev VERSION=1 BUILD=3 -FIX=6 +FIX=7 diff --git a/.env.search b/.env.search index 276d92b..0c14c84 100644 --- a/.env.search +++ b/.env.search @@ -1,4 +1,4 @@ RELEASE=dev VERSION=1 BUILD=3 -FIX=2 +FIX=3 diff --git a/DSL/Resql/ckb/GET/source_file/get_source_file_exists_by_url.sql b/DSL/Resql/ckb/GET/source_file/get_source_file_exists_by_url.sql new file mode 100644 index 0000000..e2a2485 --- /dev/null +++ b/DSL/Resql/ckb/GET/source_file/get_source_file_exists_by_url.sql @@ -0,0 +1,31 @@ +/* +declaration: + version: 0.1 + description: "Check if a non-deleted source_file already exists for a source by URL" + method: get + namespace: source_file + returns: json + allowlist: + query: + - field: source_base_id + type: string + description: "Base ID of the source" + - field: url + type: string + description: "URL to check" + response: + fields: + - field: exists + type: boolean + description: "Whether a matching source_file exists" +*/ +SELECT count(*) > 0 AS exists +FROM data_collection.source_file +WHERE (base_id, updated_at) IN ( + SELECT base_id, max(updated_at) + FROM data_collection.source_file + WHERE source_base_id = :source_base_id::UUID + GROUP BY base_id +) AND source_base_id = :source_base_id::UUID + AND is_deleted = FALSE + AND url = :url; diff --git a/DSL/Resql/ckb/POST/source/update_source_is_running.sql b/DSL/Resql/ckb/POST/source/update_source_is_running.sql index aa0e78c..a619eeb 100644 --- a/DSL/Resql/ckb/POST/source/update_source_is_running.sql +++ b/DSL/Resql/ckb/POST/source/update_source_is_running.sql @@ -13,6 +13,9 @@ declaration: - field: agency_base_id type: string description: "agency base id" + - field: url + type: string + description: "url of the source" */ SELECT copy_row_with_modifications( 'data_collection.source', @@ -23,7 +26,7 @@ SELECT copy_row_with_modifications( 'status', '::SOURCE_STATUS_TYPE', 'running', 'updated_at', '::TIMESTAMP WITH TIME ZONE', NOW()::VARCHAR ]::VARCHAR[] -), base_id, agency_base_id, type +), base_id, agency_base_id, type, url FROM data_collection.source WHERE (base_id, updated_at) IN ( SELECT base_id, max(updated_at) FROM data_collection.source diff --git a/DSL/Ruuter.internal/ckb/GET/pipeline/trigger-pipeline-for-sceduled-sources.yml b/DSL/Ruuter.internal/ckb/GET/pipeline/trigger-pipeline-for-sceduled-sources.yml index decc729..a974b0b 100644 --- a/DSL/Ruuter.internal/ckb/GET/pipeline/trigger-pipeline-for-sceduled-sources.yml +++ b/DSL/Ruuter.internal/ckb/GET/pipeline/trigger-pipeline-for-sceduled-sources.yml @@ -91,6 +91,23 @@ triggerScrapperPipelineForSource: source_id: ${sourceToRun.response.body[0].baseId} agency_id: ${sourceToRun.response.body[0].agencyBaseId} result: res + next: triggerSitemapCollectForScheduledSource + +# Scheduled refresh also re-runs sitemap collection so that pages added to +# the site since the last crawl get discovered and registered as new +# source_files, matching manual refresh's behavior. Pages that already +# exist are skipped by the spider itself (see SitemapCollectSpider), so +# this only ever adds new pages here. +triggerSitemapCollectForScheduledSource: + maxRecursions: 999 + template: "[#CKB_PROJECT_LAYER]/pipeline/trigger-scrapper-sitemap-collect" + requestType: templates + body: + source_id: ${sourceToRun.response.body[0].baseId} + agency_id: ${sourceToRun.response.body[0].agencyBaseId} + url: ${sourceToRun.response.body[0].url} + is_initial_scrape: false + result: sitemapRes next: getOneSourceToRun returnSuccess: diff --git a/DSL/Ruuter.internal/ckb/GET/source-file/get-source-file-exists-by-url.yml b/DSL/Ruuter.internal/ckb/GET/source-file/get-source-file-exists-by-url.yml new file mode 100644 index 0000000..a53e45c --- /dev/null +++ b/DSL/Ruuter.internal/ckb/GET/source-file/get-source-file-exists-by-url.yml @@ -0,0 +1,29 @@ +declaration: + call: declare + version: 0.1 + description: "check if a source_file already exists for this source by url" + method: get + accepts: json + returns: json + namespace: ckb + allowlist: + params: + - field: source_id + type: string + description: "source base id" + - field: url + type: string + description: "url to check" + +checkSourceFileExists: + call: http.get + args: + url: "[#CKB_RESQL]/source_file/get_source_file_exists_by_url" + query: + source_base_id: ${incoming.params.source_id} + url: ${incoming.params.url} + result: existsResult + +returnResult: + return: ${existsResult.response.body[0].exists} + next: end diff --git a/DSL/Ruuter.internal/ckb/TEMPLATES/pipeline/trigger-scrapper-sitemap-collect.yml b/DSL/Ruuter.internal/ckb/TEMPLATES/pipeline/trigger-scrapper-sitemap-collect.yml new file mode 100644 index 0000000..87c8dfa --- /dev/null +++ b/DSL/Ruuter.internal/ckb/TEMPLATES/pipeline/trigger-scrapper-sitemap-collect.yml @@ -0,0 +1,22 @@ +extractRequestData: + assign: + agency_id: ${incoming.body.agency_id} + source_id: ${incoming.body.source_id} + url: ${incoming.body.url} + is_initial_scrape: ${incoming.body.is_initial_scrape} + + +triggerScrapper: + call: http.post + args: + url: "[#CKB_SCRAPPER_SERVICE]/sitemap-collect-scrapper-task" + contentType: json + body: + agency_id: ${agency_id} + source_id: ${source_id} + url: ${url} + is_initial_scrape: ${is_initial_scrape} + result: triggerScrapperResult + +returnResult: + return: "OK" diff --git a/DSL/Ruuter/ckb/POST/source/refresh.yml b/DSL/Ruuter/ckb/POST/source/refresh.yml index 57f2ff1..badc93f 100644 --- a/DSL/Ruuter/ckb/POST/source/refresh.yml +++ b/DSL/Ruuter/ckb/POST/source/refresh.yml @@ -85,6 +85,21 @@ triggerScrapperPipelineForSource: source_id: ${sourceToRun.response.body[0].baseId} agency_id: ${sourceToRun.response.body[0].agencyBaseId} result: res + next: triggerSitemapCollectForRefresh + +# Manual refresh also re-runs sitemap collection so that pages added to the +# site since the last crawl get discovered and registered as new +# source_files. Pages that already exist are skipped by the spider itself +# (see SitemapCollectSpider), so this only ever adds new pages here. +triggerSitemapCollectForRefresh: + template: "[#CKB_PROJECT_LAYER]/pipeline/trigger-scrapper-sitemap-collect" + requestType: templates + body: + source_id: ${sourceToRun.response.body[0].baseId} + agency_id: ${sourceToRun.response.body[0].agencyBaseId} + url: ${sourceToRun.response.body[0].url} + is_initial_scrape: false + result: sitemapRes next: returnResult returnResult: diff --git a/GUI/src/components/Dialog/Dialog.scss b/GUI/src/components/Dialog/Dialog.scss index 1aa954c..17b32c5 100644 --- a/GUI/src/components/Dialog/Dialog.scss +++ b/GUI/src/components/Dialog/Dialog.scss @@ -3,7 +3,7 @@ @import 'src/styles/settings/variables/other'; @import 'src/styles/settings/variables/typography'; -.dialog { +.dialog[role='dialog'] { background-color: get-color(white); box-shadow: 0 0 20px rgba(0, 0, 0, 0.25); border-radius: 4px; @@ -15,15 +15,18 @@ max-width: 600px; z-index: 101; max-height: 90vh; + box-sizing: border-box; + display: flex; + flex-direction: column; - &--large { + &.dialog--large { max-width: 800px; } - &--fullscreen { + &.dialog--fullscreen { max-width: unset; height: 100%; - + margin: 16px; box-sizing: border-box; max-height: unset; @@ -32,50 +35,53 @@ top: 0; left: 0; transform: unset; + .dialog__body { max-height: unset; - height: 100%; } } - &__overlay { - position: fixed; - inset: 0; - background-color: rgba(0, 0, 0, 0.54); - z-index: 100; - } - - &__header, - &__body, - &__footer { + .dialog__header, + .dialog__body, + .dialog__footer { padding: get-spacing(haapsalu); } - &__header { + .dialog__header { display: flex; align-items: center; gap: get-spacing(haapsalu); background-color: get-color(black-coral-0); border-bottom: 1px solid get-color(black-coral-2); + flex-shrink: 0; } - &__title { + .dialog__title { flex: 1; } - &__close { + .dialog__close { display: flex; align-items: center; justify-content: center; font-size: 20px; } - &__body { + .dialog__body { overflow: auto; - max-height: calc(90vh - 70px); + flex: 1 1 auto; + min-height: 0; } - &__footer { + .dialog__footer { border-top: 1px solid get-color(black-coral-2); + flex-shrink: 0; } } + +.dialog__overlay { + position: fixed; + inset: 0; + background-color: rgba(0, 0, 0, 0.54); + z-index: 100; +} diff --git a/GUI/src/pages/Agency/Agency.tsx b/GUI/src/pages/Agency/Agency.tsx index 84b3b5d..e6ce918 100644 --- a/GUI/src/pages/Agency/Agency.tsx +++ b/GUI/src/pages/Agency/Agency.tsx @@ -100,6 +100,11 @@ const Agency: FC = () => { const [sorting, setSorting] = useState([]); const [columnFilters, setColumnFilters] = useState([]); + // Set right after clicking Refresh; cleared once that source's "running" + // status is actually observed in a fetch. Lets refetchInterval keep polling + // through the gap before the backend has marked anything running yet. + const [awaitingRefreshStart, setAwaitingRefreshStart] = useState(false); + const [formData, setFormData] = useState( getInitialFormData() ); @@ -151,6 +156,22 @@ const Agency: FC = () => { queryFn: () => getSources(queryParams), enabled: !!agencyBaseId, keepPreviousData: true, + // Keep polling every 2s as long as some source is in a transient state + // (running, or just refreshed and not yet reflected as running), so both + // the idle->running and running->finished transitions show up without a + // manual reload. Stops on its own once nothing is in flight. + refetchInterval: (data) => { + const sources = data?.data ?? []; + const hasRunningSource = sources.some( + (source: Source) => source.status === 'running' + ); + + if (hasRunningSource && awaitingRefreshStart) { + setAwaitingRefreshStart(false); + } + + return hasRunningSource || awaitingRefreshStart ? 2000 : false; + }, }); // File upload mutation @@ -407,7 +428,17 @@ const Agency: FC = () => { title: t('global.notification'), message: t('knowledgeBase.refreshSuccess'), }); + + // The backend flips the source's status to "running" asynchronously + // (Celery task -> subprocess -> Scrapy spider startup), so a single + // refetch right after this call often lands before that happens. + // Polling (see refetchInterval above) picks up the change once it + // lands, and continues through running->finished afterwards. The + // timeout below is just a safety net in case the backend never marks + // it running (e.g. the job fails before reaching that point). queryClient.invalidateQueries(['sources']); + setAwaitingRefreshStart(true); + setTimeout(() => setAwaitingRefreshStart(false), 15000); }, onError: (error: any) => { toast.open({ @@ -814,12 +845,19 @@ const Agency: FC = () => { {uploadModal && ( !uploadProgress.isUploading && setUploadModal(false)} + onClose={() => { + if (uploadProgress.isUploading) return; + setUploadModal(false); + setFormData(getInitialFormData()); + }} footer={ @@ -1027,12 +1071,18 @@ const Agency: FC = () => { {addUrlListModal && ( setAddUrlListModal(false)} + onClose={() => { + setAddUrlListModal(false); + setFormData(getInitialFormData()); + }} footer={ diff --git a/GUI/src/pages/ScrapedFiles/index.tsx b/GUI/src/pages/ScrapedFiles/index.tsx index d638f08..80026be 100644 --- a/GUI/src/pages/ScrapedFiles/index.tsx +++ b/GUI/src/pages/ScrapedFiles/index.tsx @@ -181,10 +181,10 @@ const ScrapedFiles: FC = () => { refetchInterval: (data: any) => (sourceData?.status === 'running' || (sourceData?.status === 'in_review' && (data?.total === 0 || data == null))) ? 5000 : false, }); - // Clear row selection when data changes + // Clear row selection when data changes (including page/page-size changes) useEffect(() => { setRowSelection({}); - }, [scrapedFilesData?.total]); + }, [scrapedFilesData?.total, pagination.pageIndex, pagination.pageSize]); // Refresh file mutation const refreshMutation = useMutation({ diff --git a/GUI/src/pages/Settings/Settings.scss b/GUI/src/pages/Settings/Settings.scss index 74b9c7a..2da3c66 100644 --- a/GUI/src/pages/Settings/Settings.scss +++ b/GUI/src/pages/Settings/Settings.scss @@ -4,10 +4,19 @@ .knowledge-base-settings { height: 100%; + max-width: 100%; display: flex; flex-direction: column; padding: get-spacing(haapsalu); - + box-sizing: border-box; + overflow-x: auto; + + .card { + max-width: 100%; + box-sizing: border-box; + overflow-x: auto; + } + &__path { display: flex; align-items: center; @@ -35,11 +44,17 @@ &__content { flex: 1; + min-width: fit-content; display: flex; flex-direction: column; gap: get-spacing(haapsalu); } + &__url { + overflow-wrap: anywhere; + word-break: break-word; + } + &__section { display: flex; align-items: center; @@ -56,6 +71,10 @@ color: get-color(black-coral-15); } + &__section--indent { + margin-left: 318px; + } + &__repeat-group { display: flex; align-items: center; @@ -183,22 +202,17 @@ // Responsive design @media (max-width: 768px) { padding: get-spacing(paldiski); - + &__content { gap: get-spacing(paldiski); } - + &__days-grid { button { min-width: 50px; font-size: $veera-font-size-80; } } - - .track { - flex-wrap: wrap; - gap: get-spacing(paldiski); - } } } diff --git a/GUI/src/pages/Settings/index.tsx b/GUI/src/pages/Settings/index.tsx index 6ac4dc1..21c6f37 100644 --- a/GUI/src/pages/Settings/index.tsx +++ b/GUI/src/pages/Settings/index.tsx @@ -566,10 +566,9 @@ const SourceSettings: FC = () => { case 'months': return (
{ case 'years': return (