Pull from buerokratt/wip

.env.cleaner

@@ -1,4 +1,4 @@
 RELEASE=dev
 VERSION=1
 BUILD=3
-FIX=9
+FIX=10

.env.dev

@@ -1,4 +1,4 @@
 RELEASE=dev
 VERSION=1
 BUILD=2
-FIX=7
+FIX=9

.env.scrapper

@@ -1,4 +1,4 @@
 RELEASE=dev
 VERSION=1
 BUILD=3
-FIX=8
+FIX=9

.env.search

@@ -1,4 +1,4 @@
 RELEASE=dev
 VERSION=1
 BUILD=3
-FIX=4
+FIX=5

DSL/Resql/ckb/GET/source/list_agency_sources.sql

@@ -64,16 +64,19 @@ declaration:
       - field: has_finished_files
         type: boolean
         description: "True if source has at least one finished file"
+      - field: is_stopping
+        type: boolean
+        description: "True if a stop was requested and is still being processed"
 */
 WITH latest_sources AS (
     SELECT DISTINCT ON (base_id)
-        id, base_id, agency_base_id, url, subsector, status, last_scraped_at, type, is_deleted, updated_at
+        id, base_id, agency_base_id, url, subsector, status, last_scraped_at, type, is_deleted, updated_at, is_stopping
     FROM data_collection.source
     WHERE agency_base_id = :agency_base_id::UUID
     ORDER BY base_id, updated_at DESC
 )
 SELECT
-    ls.id, ls.base_id, ls.agency_base_id, ls.url, ls.subsector, ls.status, ls.last_scraped_at, ls.type,
+    ls.id, ls.base_id, ls.agency_base_id, ls.url, ls.subsector, ls.status, ls.last_scraped_at, ls.type, ls.is_stopping,
     :page as page,
     CEIL(COUNT(*) OVER () / :page_size::DECIMAL) AS total_pages,
     (COUNT(*) OVER ()) AS total,

GUI/src/pages/Agency/Agency.tsx

@@ -732,7 +732,11 @@ const Agency: FC = () => {
               appearance="text"
               size="s"
               onClick={() => handleStopScraping(row.original.baseId)}
-              disabled={stopScrapingMutation.isLoading}
+              disabled={
+                (stopScrapingMutation.isLoading &&
+                  stopScrapingMutation.variables === row.original.baseId) ||
+                row.original.isStopping
+              }
             >
               <Icon
                 icon={<MdOutlineStopCircle fontSize={20} />}
@@ -1265,43 +1269,72 @@ const Agency: FC = () => {
                 {t('knowledgeBase.contentExtractionQualityControlOptions')}
               </span>
               <div
-                className="quality-control-options__row"
-                style={{ flexDirection: 'column', alignItems: 'flex-start' }}
+                style={{ display: 'flex', flexDirection: 'column', alignItems: 'flex-start', gap: '8px', width: '100%' }}
               >
-                <label className="quality-control-options__item">
-                  <input
-                    type="radio"
-                    name="qualityControlLevelUrlList"
-                    checked={formData.qualityControlLevel === 'basic'}
-                    onClick={() =>
-                      setFormData((prev) => ({
-                        ...prev,
-                        qualityControlLevel:
-                          prev.qualityControlLevel === 'basic' ? '' : 'basic',
-                      }))
-                    }
-                    onChange={() => {}}
-                  />
-                  <span>{t('knowledgeBase.basicQualityControl')}</span>
-                </label>
-                <label className="quality-control-options__item">
-                  <input
-                    type="radio"
-                    name="qualityControlLevelUrlList"
-                    checked={formData.qualityControlLevel === 'comprehensive'}
-                    onClick={() =>
-                      setFormData((prev) => ({
-                        ...prev,
-                        qualityControlLevel:
-                          prev.qualityControlLevel === 'comprehensive'
-                            ? ''
-                            : 'comprehensive',
-                      }))
-                    }
-                    onChange={() => {}}
-                  />
-                  <span>{t('knowledgeBase.comprehensiveQualityControl')}</span>
-                </label>
+                <div className="quality-control-options__row">
+                  <label className="quality-control-options__item">
+                    <input
+                      type="radio"
+                      name="qualityControlLevelUrlList"
+                      checked={formData.qualityControlLevel === 'basic'}
+                      onClick={() =>
+                        setFormData((prev) => ({
+                          ...prev,
+                          qualityControlLevel:
+                            prev.qualityControlLevel === 'basic' ? '' : 'basic',
+                        }))
+                      }
+                      onChange={() => {}}
+                    />
+                    <span>{t('knowledgeBase.basicQualityControl')}</span>
+                  </label>
+                  <Tooltip content="Tooltip to be implemented">
+                    <button
+                      type="button"
+                      className="quality-control-options__info-btn"
+                      aria-label={t('knowledgeBase.basicQualityControlInfo') as string}
+                    >
+                      <Icon
+                        className="quality-control-options__info"
+                        icon={<MdInfoOutline fontSize={18} color="#005AA3" />}
+                        size="medium"
+                      />
+                    </button>
+                  </Tooltip>
+                </div>
+                <div className="quality-control-options__row">
+                  <label className="quality-control-options__item">
+                    <input
+                      type="radio"
+                      name="qualityControlLevelUrlList"
+                      checked={formData.qualityControlLevel === 'comprehensive'}
+                      onClick={() =>
+                        setFormData((prev) => ({
+                          ...prev,
+                          qualityControlLevel:
+                            prev.qualityControlLevel === 'comprehensive'
+                              ? ''
+                              : 'comprehensive',
+                        }))
+                      }
+                      onChange={() => {}}
+                    />
+                    <span>{t('knowledgeBase.comprehensiveQualityControl')}</span>
+                  </label>
+                  <Tooltip content="Tooltip to be implemented">
+                    <button
+                      type="button"
+                      className="quality-control-options__info-btn"
+                      aria-label={t('knowledgeBase.comprehensiveQualityControlInfo') as string}
+                    >
+                      <Icon
+                        className="quality-control-options__info"
+                        icon={<MdInfoOutline fontSize={18} color="#005AA3" />}
+                        size="medium"
+                      />
+                    </button>
+                  </Tooltip>
+                </div>
               </div>
             </div>
           </Track>

GUI/src/pages/Reports/Report.tsx

@@ -16,6 +16,7 @@ import {
   ReportPage,
   ReportPagesListParams,
 } from 'services/reports';
+import { getSanitizedErrorDetail } from 'utils/report-error-utils';
 import 'pages/Agency/AgencyList.scss';
 
 const Report: FC = () => {
@@ -168,19 +169,22 @@ const Report: FC = () => {
       accessorKey: 'errorMessage',
       header: t('reports.errorMessage'),
       enableColumnFilter: false,
-      cell: ({ row }) => (
-        <div
-          style={{
-            maxWidth: 300,
-            overflow: 'hidden',
-            textOverflow: 'ellipsis',
-          }}
-        >
-          <Tooltip content={row.original.errorMessage || '-'}>
-            <span>{row.original.errorMessage || '-'}</span>
-          </Tooltip>
-        </div>
-      ),
+      cell: ({ row }) => {
+        const errorDetail = getSanitizedErrorDetail(row.original.errorMessage);
+        return (
+          <div
+            style={{
+              maxWidth: 300,
+              overflow: 'hidden',
+              textOverflow: 'ellipsis',
+            }}
+          >
+            <Tooltip content={errorDetail}>
+              <span>{errorDetail}</span>
+            </Tooltip>
+          </div>
+        );
+      },
     },
     {
       accessorKey: 'scrapedAt',

GUI/src/services/sources.ts

@@ -24,6 +24,7 @@ export interface Source {
   type?: string;
   qualityControl?: 'basic' | 'comprehensive' | null;
   extractImages?: boolean;
+  isStopping?: boolean;
 }
 
 // API Integration interface - extends Source but with specific properties

GUI/src/utils/report-error-utils.ts

@@ -0,0 +1,28 @@
+// Redacts usernames/credentials from a raw scraper error message before it
+// is shown in the UI (both in the table cell and its hover tooltip), while
+// keeping the rest of the message intact.
+export function getSanitizedErrorDetail(rawMessage?: string | null): string {
+  if (!rawMessage) return '-';
+
+  let sanitized = rawMessage;
+
+  // Credentials embedded as URL userinfo, e.g. http://user:password@host.
+  sanitized = sanitized.replace(
+    /:\/\/[^\s/@]+:[^\s/@]+@/gi,
+    '://[redacted]@'
+  );
+
+  // Basic auth headers, e.g. Authorization: Basic dXNlcjpwYXNz.
+  sanitized = sanitized.replace(
+    /\b(authorization\s*:\s*(basic|bearer)\s+)\S+/gi,
+    '$1[redacted]'
+  );
+
+  // Common credential-style query params, e.g. ?token=..., &password=....
+  sanitized = sanitized.replace(
+    /([?&](?:token|api[_-]?key|password|passwd|secret|access[_-]?token|auth)=)[^&\s"'<>]+/gi,
+    '$1[redacted]'
+  );
+
+  return sanitized;
+}

GUI/translations/en/common.json

@@ -622,6 +622,7 @@
     "contentSaved": "File saved successfully",
     "failed": "Failed",
     "in_review": "In Review",
+    "not_found": "Not Found",
     "startCleaningNotification":"All pages have been scraped. Before continuing, delete all pages that you do not want to make available to the chatbot, like news and archived pages. Once you have made your selection, continue cleaning the pages by clicking the 'Start cleaning' button.",
     "cleaningStarted": "Cleaning started"
   },

GUI/translations/et/common.json

@@ -622,6 +622,7 @@
     "contentSaved": "Fail salvestati edukalt",
     "failed": "Ebaõnnestunud",
     "in_review": "Ülevaatamisel",
+    "not_found": "Ei leitud",
     "startCleaningNotification": "Kõik leheküljed on kraabitud. Enne jätkamist kustutage kõik leheküljed, mida te ei soovi vestlusrobotile kättesaadavaks teha, näiteks uudised ja arhiveeritud leheküljed. Kui olete oma valiku teinud, jätkake lehekülgede puhastamist, klõpsates nuppu \"Alusta puhastamist\".",
     "cleaningStarted": "Puhastamine algas"
 

cleaning/worker/utils.py

@@ -1,6 +1,7 @@
 import contextlib
 import datetime
 import logging
+import re
 import shutil
 from collections.abc import Iterator
 
@@ -11,6 +12,44 @@
 
 logger = logging.getLogger(__name__)
 
+# Query/form params commonly used to pass credentials or tokens.
+_SENSITIVE_PARAM_NAMES = (
+    "token",
+    "api_key",
+    "apikey",
+    "password",
+    "passwd",
+    "pwd",
+    "secret",
+    "access_token",
+    "auth",
+    "session",
+    "sessionid",
+    "sid",
+)
+
+_USERINFO_RE = re.compile(r"://[^\s/@]+:[^\s/@]+@")
+_AUTH_HEADER_RE = re.compile(
+    r"(authorization[\"']?\s*[:=]\s*[\"']?(basic|bearer)\s+)\S+", re.IGNORECASE
+)
+_SENSITIVE_PARAM_RE = re.compile(
+    r"([?&](?:" + "|".join(_SENSITIVE_PARAM_NAMES) + r")=)[^&\s\"'<>]+",
+    re.IGNORECASE,
+)
+
+
+def sanitize_sensitive_text(text: str) -> str:
+    """Redact credentials/tokens from a URL or error message before it is
+    logged or sent to the backend (e.g. userinfo in a URL, Authorization
+    headers appearing in exception text, credential-style query params)."""
+    if not text:
+        return text
+
+    sanitized = _USERINFO_RE.sub("://[redacted]@", text)
+    sanitized = _AUTH_HEADER_RE.sub(r"\1[redacted]", sanitized)
+    sanitized = _SENSITIVE_PARAM_RE.sub(r"\1[redacted]", sanitized)
+    return sanitized
+
 
 def send_error(
     url: str,
@@ -28,7 +67,7 @@ def send_error(
                 "url": url,
                 "scraped_at": scraped_at,
                 "error_type": error_type,
-                "error_message": error_message,
+                "error_message": sanitize_sensitive_text(error_message),
                 "source_base_id": source_base_id,
                 "agency_base_id": agency_base_id,
                 "source_run_report_base_id": source_run_report_base_id,

scrapper/scrapper/download_handler.py

@@ -8,6 +8,9 @@
 from twisted.internet.defer import Deferred
 
 
+PLAYWRIGHT_TIMEOUT_MAX_RETRIES = 3
+
+
 class DownloadHandler(ScrapyPlaywrightDownloadHandler):
     def __init__(self, crawler: Crawler) -> None:
         super().__init__(crawler)
@@ -36,10 +39,16 @@ def download_request(self, request: Request, spider: Spider) -> Deferred:
         spider.logger.info(f"Playwright download: {request.url}")
         return super().download_request(request, spider)
 
-    async def _download_request(self, request: Request, spider: Spider) -> Response:
+    async def _download_request(
+        self, request: Request, spider: Spider, _attempt: int = 1
+    ) -> Response:
         """
         Internal async download method with fallback for download errors.
         This is called by the parent's download_request when using Playwright.
+        Retries on Playwright timeout up to PLAYWRIGHT_TIMEOUT_MAX_RETRIES times
+        before giving up, so a single unresponsive URL can never wedge the
+        spider in an infinite retry loop and block it from ever reaching
+        parse() again (where the source's stop flag is checked).
         """
         try:
             spider.logger.info(f"Playwright request started: {request.url}")
@@ -48,13 +57,20 @@ async def _download_request(self, request: Request, spider: Spider) -> Response:
                 spider.logger.info(f"Playwright request finished: {request.url}")
                 return r
         except TimeoutError:
+            if _attempt >= PLAYWRIGHT_TIMEOUT_MAX_RETRIES:
+                spider.logger.error(
+                    f"request timed out due to playwright: {request.url}. "
+                    f"Giving up after {_attempt} attempts"
+                )
+                raise
             spider.logger.warning(
-                f"request timed out due to playwright: {request.url}. Try again"
+                f"request timed out due to playwright: {request.url}. "
+                f"Try again ({_attempt}/{PLAYWRIGHT_TIMEOUT_MAX_RETRIES})"
             )
             await self._close()
             super().__init__(self.crawler)  # Re-initialize with the same crawler
             await self._launch()
-            return await self._download_request(request, spider)
+            return await self._download_request(request, spider, _attempt + 1)
         except Exception as e:
             # Catch "Download is starting" and similar download errors as safety net
             if "Download is starting" in str(e) or "net::ERR_ABORTED" in str(e):