Pull from buerokratt/wip

.env.cleaner

@@ -1,4 +1,4 @@
 RELEASE=dev
 VERSION=1
 BUILD=3
-FIX=7
+FIX=8

.env.fileproc

@@ -1,4 +1,4 @@
 RELEASE=dev
 VERSION=1
 BUILD=3
-FIX=3
+FIX=4

.env.scrapper

@@ -1,4 +1,4 @@
 RELEASE=dev
 VERSION=1
 BUILD=3
-FIX=6
+FIX=7

.env.search

@@ -1,4 +1,4 @@
 RELEASE=dev
 VERSION=1
 BUILD=3
-FIX=2
+FIX=3

DSL/Resql/ckb/GET/source_file/get_source_file_exists_by_url.sql

@@ -0,0 +1,31 @@
+/*
+declaration:
+  version: 0.1
+  description: "Check if a non-deleted source_file already exists for a source by URL"
+  method: get
+  namespace: source_file
+  returns: json
+  allowlist:
+    query:
+      - field: source_base_id
+        type: string
+        description: "Base ID of the source"
+      - field: url
+        type: string
+        description: "URL to check"
+  response:
+    fields:
+      - field: exists
+        type: boolean
+        description: "Whether a matching source_file exists"
+*/
+SELECT count(*) > 0 AS exists
+FROM data_collection.source_file
+WHERE (base_id, updated_at) IN (
+    SELECT base_id, max(updated_at)
+    FROM data_collection.source_file
+    WHERE source_base_id = :source_base_id::UUID
+    GROUP BY base_id
+) AND source_base_id = :source_base_id::UUID
+    AND is_deleted = FALSE
+    AND url = :url;

DSL/Resql/ckb/POST/source/update_source_is_running.sql

@@ -13,6 +13,9 @@ declaration:
       - field: agency_base_id
         type: string
         description: "agency base id"
+      - field: url
+        type: string
+        description: "url of the source"
 */
 SELECT copy_row_with_modifications(
     'data_collection.source',
@@ -23,7 +26,7 @@ SELECT copy_row_with_modifications(
            'status', '::SOURCE_STATUS_TYPE', 'running',
            'updated_at', '::TIMESTAMP WITH TIME ZONE', NOW()::VARCHAR
        ]::VARCHAR[]
-), base_id, agency_base_id, type
+), base_id, agency_base_id, type, url
 FROM data_collection.source
 WHERE (base_id, updated_at) IN (
     SELECT base_id, max(updated_at) FROM data_collection.source

DSL/Ruuter.internal/ckb/GET/pipeline/trigger-pipeline-for-sceduled-sources.yml

@@ -91,6 +91,23 @@ triggerScrapperPipelineForSource:
     source_id: ${sourceToRun.response.body[0].baseId}
     agency_id: ${sourceToRun.response.body[0].agencyBaseId}
   result: res
+  next: triggerSitemapCollectForScheduledSource
+
+# Scheduled refresh also re-runs sitemap collection so that pages added to
+# the site since the last crawl get discovered and registered as new
+# source_files, matching manual refresh's behavior. Pages that already
+# exist are skipped by the spider itself (see SitemapCollectSpider), so
+# this only ever adds new pages here.
+triggerSitemapCollectForScheduledSource:
+  maxRecursions: 999
+  template: "[#CKB_PROJECT_LAYER]/pipeline/trigger-scrapper-sitemap-collect"
+  requestType: templates
+  body:
+    source_id: ${sourceToRun.response.body[0].baseId}
+    agency_id: ${sourceToRun.response.body[0].agencyBaseId}
+    url: ${sourceToRun.response.body[0].url}
+    is_initial_scrape: false
+  result: sitemapRes
   next: getOneSourceToRun
 
 returnSuccess:

DSL/Ruuter.internal/ckb/GET/source-file/get-source-file-exists-by-url.yml

@@ -0,0 +1,29 @@
+declaration:
+  call: declare
+  version: 0.1
+  description: "check if a source_file already exists for this source by url"
+  method: get
+  accepts: json
+  returns: json
+  namespace: ckb
+  allowlist:
+    params:
+      - field: source_id
+        type: string
+        description: "source base id"
+      - field: url
+        type: string
+        description: "url to check"
+
+checkSourceFileExists:
+  call: http.get
+  args:
+    url: "[#CKB_RESQL]/source_file/get_source_file_exists_by_url"
+    query:
+      source_base_id: ${incoming.params.source_id}
+      url: ${incoming.params.url}
+  result: existsResult
+
+returnResult:
+  return: ${existsResult.response.body[0].exists}
+  next: end

DSL/Ruuter.internal/ckb/TEMPLATES/pipeline/trigger-scrapper-sitemap-collect.yml

@@ -0,0 +1,22 @@
+extractRequestData:
+  assign:
+    agency_id: ${incoming.body.agency_id}
+    source_id: ${incoming.body.source_id}
+    url: ${incoming.body.url}
+    is_initial_scrape: ${incoming.body.is_initial_scrape}
+
+
+triggerScrapper:
+  call: http.post
+  args:
+    url: "[#CKB_SCRAPPER_SERVICE]/sitemap-collect-scrapper-task"
+    contentType: json
+    body:
+      agency_id: ${agency_id}
+      source_id: ${source_id}
+      url: ${url}
+      is_initial_scrape: ${is_initial_scrape}
+  result: triggerScrapperResult
+
+returnResult:
+  return: "OK"

DSL/Ruuter/ckb/POST/source/refresh.yml

@@ -85,6 +85,21 @@ triggerScrapperPipelineForSource:
     source_id: ${sourceToRun.response.body[0].baseId}
     agency_id: ${sourceToRun.response.body[0].agencyBaseId}
   result: res
+  next: triggerSitemapCollectForRefresh
+
+# Manual refresh also re-runs sitemap collection so that pages added to the
+# site since the last crawl get discovered and registered as new
+# source_files. Pages that already exist are skipped by the spider itself
+# (see SitemapCollectSpider), so this only ever adds new pages here.
+triggerSitemapCollectForRefresh:
+  template: "[#CKB_PROJECT_LAYER]/pipeline/trigger-scrapper-sitemap-collect"
+  requestType: templates
+  body:
+    source_id: ${sourceToRun.response.body[0].baseId}
+    agency_id: ${sourceToRun.response.body[0].agencyBaseId}
+    url: ${sourceToRun.response.body[0].url}
+    is_initial_scrape: false
+  result: sitemapRes
   next: returnResult
 
 returnResult:

GUI/src/components/Dialog/Dialog.scss

@@ -3,7 +3,7 @@
 @import 'src/styles/settings/variables/other';
 @import 'src/styles/settings/variables/typography';
 
-.dialog {
+.dialog[role='dialog'] {
   background-color: get-color(white);
   box-shadow: 0 0 20px rgba(0, 0, 0, 0.25);
   border-radius: 4px;
@@ -15,15 +15,18 @@
   max-width: 600px;
   z-index: 101;
   max-height: 90vh;
+  box-sizing: border-box;
+  display: flex;
+  flex-direction: column;
 
-  &--large {
+  &.dialog--large {
     max-width: 800px;
   }
 
-  &--fullscreen {
+  &.dialog--fullscreen {
     max-width: unset;
     height: 100%;
-    
+
     margin: 16px;
     box-sizing: border-box;
     max-height: unset;
@@ -32,50 +35,53 @@
     top: 0;
     left: 0;
     transform: unset;
+
     .dialog__body {
       max-height: unset;
-      height: 100%;
     }
   }
 
-  &__overlay {
-    position: fixed;
-    inset: 0;
-    background-color: rgba(0, 0, 0, 0.54);
-    z-index: 100;
-  }
-
-  &__header,
-  &__body,
-  &__footer {
+  .dialog__header,
+  .dialog__body,
+  .dialog__footer {
     padding: get-spacing(haapsalu);
   }
 
-  &__header {
+  .dialog__header {
     display: flex;
     align-items: center;
     gap: get-spacing(haapsalu);
     background-color: get-color(black-coral-0);
     border-bottom: 1px solid get-color(black-coral-2);
+    flex-shrink: 0;
   }
 
-  &__title {
+  .dialog__title {
     flex: 1;
   }
 
-  &__close {
+  .dialog__close {
     display: flex;
     align-items: center;
     justify-content: center;
     font-size: 20px;
   }
 
-  &__body {
+  .dialog__body {
     overflow: auto;
-    max-height: calc(90vh - 70px);
+    flex: 1 1 auto;
+    min-height: 0;
   }
 
-  &__footer {
+  .dialog__footer {
     border-top: 1px solid get-color(black-coral-2);
+    flex-shrink: 0;
   }
 }
+
+.dialog__overlay {
+  position: fixed;
+  inset: 0;
+  background-color: rgba(0, 0, 0, 0.54);
+  z-index: 100;
+}

GUI/src/pages/Agency/Agency.tsx

@@ -100,6 +100,11 @@ const Agency: FC = () => {
   const [sorting, setSorting] = useState<SortingState>([]);
   const [columnFilters, setColumnFilters] = useState<ColumnFiltersState>([]);
 
+  // Set right after clicking Refresh; cleared once that source's "running"
+  // status is actually observed in a fetch. Lets refetchInterval keep polling
+  // through the gap before the backend has marked anything running yet.
+  const [awaitingRefreshStart, setAwaitingRefreshStart] = useState(false);
+
    const [formData, setFormData] = useState<KnowledgeBaseFormData>(
     getInitialFormData()
   );
@@ -151,6 +156,22 @@ const Agency: FC = () => {
     queryFn: () => getSources(queryParams),
     enabled: !!agencyBaseId,
     keepPreviousData: true,
+    // Keep polling every 2s as long as some source is in a transient state
+    // (running, or just refreshed and not yet reflected as running), so both
+    // the idle->running and running->finished transitions show up without a
+    // manual reload. Stops on its own once nothing is in flight.
+    refetchInterval: (data) => {
+      const sources = data?.data ?? [];
+      const hasRunningSource = sources.some(
+        (source: Source) => source.status === 'running'
+      );
+
+      if (hasRunningSource && awaitingRefreshStart) {
+        setAwaitingRefreshStart(false);
+      }
+
+      return hasRunningSource || awaitingRefreshStart ? 2000 : false;
+    },
   });
 
   // File upload mutation
@@ -407,7 +428,17 @@ const Agency: FC = () => {
         title: t('global.notification'),
         message: t('knowledgeBase.refreshSuccess'),
       });
+
+      // The backend flips the source's status to "running" asynchronously
+      // (Celery task -> subprocess -> Scrapy spider startup), so a single
+      // refetch right after this call often lands before that happens.
+      // Polling (see refetchInterval above) picks up the change once it
+      // lands, and continues through running->finished afterwards. The
+      // timeout below is just a safety net in case the backend never marks
+      // it running (e.g. the job fails before reaching that point).
       queryClient.invalidateQueries(['sources']);
+      setAwaitingRefreshStart(true);
+      setTimeout(() => setAwaitingRefreshStart(false), 15000);
     },
     onError: (error: any) => {
       toast.open({
@@ -814,12 +845,19 @@ const Agency: FC = () => {
       {uploadModal && (
         <Dialog
           title={t('knowledgeBase.uploadFiles')}
-          onClose={() => !uploadProgress.isUploading && setUploadModal(false)}
+          onClose={() => {
+            if (uploadProgress.isUploading) return;
+            setUploadModal(false);
+            setFormData(getInitialFormData());
+          }}
           footer={
             <Track gap={16} justify="end">
               <Button
                 appearance="secondary"
-                onClick={() => setUploadModal(false)}
+                onClick={() => {
+                  setUploadModal(false);
+                  setFormData(getInitialFormData());
+                }}
                 disabled={uploadProgress.isUploading}
               >
                 {t('global.cancel')}
@@ -870,12 +908,18 @@ const Agency: FC = () => {
       {addUrlModal && (
         <Dialog
           title={t('knowledgeBase.addUrl')}
-          onClose={() => setAddUrlModal(false)}
+          onClose={() => {
+            setAddUrlModal(false);
+            setFormData(getInitialFormData());
+          }}
           footer={
             <Track gap={16} justify="end">
               <Button
                 appearance="secondary"
-                onClick={() => setAddUrlModal(false)}
+                onClick={() => {
+                  setAddUrlModal(false);
+                  setFormData(getInitialFormData());
+                }}
               >
                 {t('global.cancel')}
               </Button>
@@ -1027,12 +1071,18 @@ const Agency: FC = () => {
       {addUrlListModal && (
         <Dialog
           title={t('knowledgeBase.addUrlList')}
-          onClose={() => setAddUrlListModal(false)}
+          onClose={() => {
+            setAddUrlListModal(false);
+            setFormData(getInitialFormData());
+          }}
           footer={
             <Track gap={16} justify="end">
               <Button
                 appearance="secondary"
-                onClick={() => setAddUrlListModal(false)}
+                onClick={() => {
+                  setAddUrlListModal(false);
+                  setFormData(getInitialFormData());
+                }}
               >
                 {t('global.cancel')}
               </Button>