From 07198314e2b1787f69712023af2d2ee05a0ff374 Mon Sep 17 00:00:00 2001 From: ruwinirathnamalala Date: Tue, 4 Aug 2026 15:08:01 +0530 Subject: [PATCH 01/10] Fix for Agency Source page: raw status key shown to user --- GUI/translations/en/common.json | 1 + GUI/translations/et/common.json | 1 + 2 files changed, 2 insertions(+) diff --git a/GUI/translations/en/common.json b/GUI/translations/en/common.json index 3bfff3e..2122eae 100644 --- a/GUI/translations/en/common.json +++ b/GUI/translations/en/common.json @@ -621,6 +621,7 @@ "contentSaved": "File saved successfully", "failed": "Failed", "in_review": "In Review", + "not_found": "Not Found", "startCleaningNotification":"All pages have been scraped. Before continuing, delete all pages that you do not want to make available to the chatbot, like news and archived pages. Once you have made your selection, continue cleaning the pages by clicking the 'Start cleaning' button.", "cleaningStarted": "Cleaning started" }, diff --git a/GUI/translations/et/common.json b/GUI/translations/et/common.json index 7f1cbcc..bb424f8 100644 --- a/GUI/translations/et/common.json +++ b/GUI/translations/et/common.json @@ -621,6 +621,7 @@ "contentSaved": "Fail salvestati edukalt", "failed": "Ebaõnnestunud", "in_review": "Ülevaatamisel", + "not_found": "Ei leitud", "startCleaningNotification": "Kõik leheküljed on kraabitud. Enne jätkamist kustutage kõik leheküljed, mida te ei soovi vestlusrobotile kättesaadavaks teha, näiteks uudised ja arhiveeritud leheküljed. Kui olete oma valiku teinud, jätkake lehekülgede puhastamist, klõpsates nuppu \"Alusta puhastamist\".", "cleaningStarted": "Puhastamine algas" From 2485dde5b042ad61273fade70f9b03bf729577f9 Mon Sep 17 00:00:00 2001 From: ruwinirathnamalala Date: Tue, 11 Aug 2026 12:55:59 +0530 Subject: [PATCH 02/10] fix for CKB source page: deleting a URL in 'In Review' status fails with 500 --- search-service/index.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/search-service/index.js b/search-service/index.js index daab4f7..e38d093 100644 --- a/search-service/index.js +++ b/search-service/index.js @@ -424,9 +424,13 @@ app.delete("/documents/:sourceId/:sourceFileId", async (req, res) => { // Check if index exists const exists = await opensearch.indices.exists({ index: indexName }); if (!exists.body) { - return res.status(404).json({ - error: "Index not found", + console.warn(`⚠️ Index not found for source: ${sourceId}`); + return res.json({ + success: true, source_id: sourceId, + source_file_id: sourceFileId, + deleted_count: 0, + status: "index_not_found", }); } From 62ebc65a9b95a405e35fccc549df309c31a0a0ab Mon Sep 17 00:00:00 2001 From: ruwinirathnamalala Date: Wed, 12 Aug 2026 10:53:32 +0530 Subject: [PATCH 03/10] fix for search by part of the url for cleaned data --- search-service/index.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/search-service/index.js b/search-service/index.js index daab4f7..f066450 100644 --- a/search-service/index.js +++ b/search-service/index.js @@ -54,7 +54,12 @@ async function createSourceIndex(sourceId) { document_type: { type: "keyword" }, page_title: { type: "text", analyzer: "standard" }, file_name: { type: "text" }, - url: { type: "keyword" }, + url: { + type: "keyword", + fields: { + text: { type: "text", analyzer: "standard" }, + }, + }, subsector: { type: "keyword" }, content: { type: "text", analyzer: "standard" }, indexed_at: { type: "date" }, @@ -245,7 +250,7 @@ app.get("/search/:sourceId", async (req, res) => { multi_match: { query: q.trim(), fields: [ - "url^5", + "url.text^5", "content^3", "page_title^2", "file_name^2", @@ -313,7 +318,7 @@ app.get("/search/:sourceId", async (req, res) => { multi_match: { query: q.trim(), fields: [ - "url^5", + "url.text^5", "content^3", "page_title^2", "file_name^2", From 3229995f772162650cb3fa20f81ae57d04cda95c Mon Sep 17 00:00:00 2001 From: Varmo <101868197+varmoh@users.noreply.github.com> Date: Wed, 12 Aug 2026 09:48:21 +0300 Subject: [PATCH 04/10] Update .env.dev --- .env.dev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.dev b/.env.dev index 6f536cf..931fc9c 100644 --- a/.env.dev +++ b/.env.dev @@ -1,4 +1,4 @@ RELEASE=dev VERSION=1 BUILD=2 -FIX=7 +FIX=8 From 5ec5f93a06ba0cc4dd6b8e44ef12906b5e80241e Mon Sep 17 00:00:00 2001 From: ruwinirathnamalala Date: Wed, 12 Aug 2026 17:24:25 +0530 Subject: [PATCH 05/10] fix for Agency page: radio buttons in 'Add URL List' modal are missing tooltips --- GUI/src/pages/Agency/Agency.tsx | 101 ++++++++++++++++++++------------ 1 file changed, 65 insertions(+), 36 deletions(-) diff --git a/GUI/src/pages/Agency/Agency.tsx b/GUI/src/pages/Agency/Agency.tsx index 37a0b16..b24d627 100644 --- a/GUI/src/pages/Agency/Agency.tsx +++ b/GUI/src/pages/Agency/Agency.tsx @@ -1265,43 +1265,72 @@ const Agency: FC = () => { {t('knowledgeBase.contentExtractionQualityControlOptions')}
- - +
+ + + + +
+
+ + + + +
From 69b3a82851a8488e13ae1b2edfbb85f354d41759 Mon Sep 17 00:00:00 2001 From: ruwinirathnamalala Date: Fri, 14 Aug 2026 12:40:55 +0530 Subject: [PATCH 06/10] removing sensitive information from error messages in reports --- GUI/src/pages/Reports/Report.tsx | 30 ++++++++++++--------- GUI/src/utils/report-error-utils.ts | 28 +++++++++++++++++++ cleaning/worker/utils.py | 41 +++++++++++++++++++++++++++- scrapper/scrapper/utils.py | 42 ++++++++++++++++++++++++++++- 4 files changed, 126 insertions(+), 15 deletions(-) create mode 100644 GUI/src/utils/report-error-utils.ts diff --git a/GUI/src/pages/Reports/Report.tsx b/GUI/src/pages/Reports/Report.tsx index d59864f..e70739f 100644 --- a/GUI/src/pages/Reports/Report.tsx +++ b/GUI/src/pages/Reports/Report.tsx @@ -16,6 +16,7 @@ import { ReportPage, ReportPagesListParams, } from 'services/reports'; +import { getSanitizedErrorDetail } from 'utils/report-error-utils'; import 'pages/Agency/AgencyList.scss'; const Report: FC = () => { @@ -168,19 +169,22 @@ const Report: FC = () => { accessorKey: 'errorMessage', header: t('reports.errorMessage'), enableColumnFilter: false, - cell: ({ row }) => ( -
- - {row.original.errorMessage || '-'} - -
- ), + cell: ({ row }) => { + const errorDetail = getSanitizedErrorDetail(row.original.errorMessage); + return ( +
+ + {errorDetail} + +
+ ); + }, }, { accessorKey: 'scrapedAt', diff --git a/GUI/src/utils/report-error-utils.ts b/GUI/src/utils/report-error-utils.ts new file mode 100644 index 0000000..923dc50 --- /dev/null +++ b/GUI/src/utils/report-error-utils.ts @@ -0,0 +1,28 @@ +// Redacts usernames/credentials from a raw scraper error message before it +// is shown in the UI (both in the table cell and its hover tooltip), while +// keeping the rest of the message intact. +export function getSanitizedErrorDetail(rawMessage?: string | null): string { + if (!rawMessage) return '-'; + + let sanitized = rawMessage; + + // Credentials embedded as URL userinfo, e.g. http://user:password@host. + sanitized = sanitized.replace( + /:\/\/[^\s/@]+:[^\s/@]+@/gi, + '://[redacted]@' + ); + + // Basic auth headers, e.g. Authorization: Basic dXNlcjpwYXNz. + sanitized = sanitized.replace( + /\b(authorization\s*:\s*(basic|bearer)\s+)\S+/gi, + '$1[redacted]' + ); + + // Common credential-style query params, e.g. ?token=..., &password=.... + sanitized = sanitized.replace( + /([?&](?:token|api[_-]?key|password|passwd|secret|access[_-]?token|auth)=)[^&\s"'<>]+/gi, + '$1[redacted]' + ); + + return sanitized; +} diff --git a/cleaning/worker/utils.py b/cleaning/worker/utils.py index a6fc8c5..d2a00aa 100644 --- a/cleaning/worker/utils.py +++ b/cleaning/worker/utils.py @@ -1,6 +1,7 @@ import contextlib import datetime import logging +import re import shutil from collections.abc import Iterator @@ -11,6 +12,44 @@ logger = logging.getLogger(__name__) +# Query/form params commonly used to pass credentials or tokens. +_SENSITIVE_PARAM_NAMES = ( + "token", + "api_key", + "apikey", + "password", + "passwd", + "pwd", + "secret", + "access_token", + "auth", + "session", + "sessionid", + "sid", +) + +_USERINFO_RE = re.compile(r"://[^\s/@]+:[^\s/@]+@") +_AUTH_HEADER_RE = re.compile( + r"(authorization[\"']?\s*[:=]\s*[\"']?(basic|bearer)\s+)\S+", re.IGNORECASE +) +_SENSITIVE_PARAM_RE = re.compile( + r"([?&](?:" + "|".join(_SENSITIVE_PARAM_NAMES) + r")=)[^&\s\"'<>]+", + re.IGNORECASE, +) + + +def sanitize_sensitive_text(text: str) -> str: + """Redact credentials/tokens from a URL or error message before it is + logged or sent to the backend (e.g. userinfo in a URL, Authorization + headers appearing in exception text, credential-style query params).""" + if not text: + return text + + sanitized = _USERINFO_RE.sub("://[redacted]@", text) + sanitized = _AUTH_HEADER_RE.sub(r"\1[redacted]", sanitized) + sanitized = _SENSITIVE_PARAM_RE.sub(r"\1[redacted]", sanitized) + return sanitized + def send_error( url: str, @@ -28,7 +67,7 @@ def send_error( "url": url, "scraped_at": scraped_at, "error_type": error_type, - "error_message": error_message, + "error_message": sanitize_sensitive_text(error_message), "source_base_id": source_base_id, "agency_base_id": agency_base_id, "source_run_report_base_id": source_run_report_base_id, diff --git a/scrapper/scrapper/utils.py b/scrapper/scrapper/utils.py index 4c84e6d..e7aae45 100644 --- a/scrapper/scrapper/utils.py +++ b/scrapper/scrapper/utils.py @@ -1,6 +1,7 @@ import contextlib import datetime import functools +import re import typing from collections.abc import Callable, Iterator from urllib.parse import urlparse @@ -15,6 +16,45 @@ BaseSpider = object +# Query/form params commonly used to pass credentials or tokens. +_SENSITIVE_PARAM_NAMES = ( + "token", + "api_key", + "apikey", + "password", + "passwd", + "pwd", + "secret", + "access_token", + "auth", + "session", + "sessionid", + "sid", +) + +_USERINFO_RE = re.compile(r"://[^\s/@]+:[^\s/@]+@") +_AUTH_HEADER_RE = re.compile( + r"(authorization[\"']?\s*[:=]\s*[\"']?(basic|bearer)\s+)\S+", re.IGNORECASE +) +_SENSITIVE_PARAM_RE = re.compile( + r"([?&](?:" + "|".join(_SENSITIVE_PARAM_NAMES) + r")=)[^&\s\"'<>]+", + re.IGNORECASE, +) + + +def sanitize_sensitive_text(text: str) -> str: + """Redact credentials/tokens from a URL or error message before it is + logged or sent to the backend (e.g. userinfo in a URL, Authorization + headers appearing in exception text, credential-style query params).""" + if not text: + return text + + sanitized = _USERINFO_RE.sub("://[redacted]@", text) + sanitized = _AUTH_HEADER_RE.sub(r"\1[redacted]", sanitized) + sanitized = _SENSITIVE_PARAM_RE.sub(r"\1[redacted]", sanitized) + return sanitized + + def send_error( ruuter_internal: str, url: str, @@ -31,7 +71,7 @@ def send_error( "url": url, "scraped_at": scraped_at, "error_type": error_type, - "error_message": error_message, + "error_message": sanitize_sensitive_text(error_message), "source_base_id": source_base_id, "agency_base_id": agency_base_id, "source_run_report_base_id": source_run_report_base_id, From 7e9d55b66c745404298531075482f5a73b1bf859 Mon Sep 17 00:00:00 2001 From: ruwinirathnamalala Date: Mon, 17 Aug 2026 16:41:46 +0530 Subject: [PATCH 07/10] Disabling stop button when stoping is in progress --- DSL/Resql/ckb/GET/source/list_agency_sources.sql | 7 +++++-- GUI/src/pages/Agency/Agency.tsx | 6 +++++- GUI/src/services/sources.ts | 1 + 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/DSL/Resql/ckb/GET/source/list_agency_sources.sql b/DSL/Resql/ckb/GET/source/list_agency_sources.sql index 558acbc..5db747e 100644 --- a/DSL/Resql/ckb/GET/source/list_agency_sources.sql +++ b/DSL/Resql/ckb/GET/source/list_agency_sources.sql @@ -64,16 +64,19 @@ declaration: - field: has_finished_files type: boolean description: "True if source has at least one finished file" + - field: is_stopping + type: boolean + description: "True if a stop was requested and is still being processed" */ WITH latest_sources AS ( SELECT DISTINCT ON (base_id) - id, base_id, agency_base_id, url, subsector, status, last_scraped_at, type, is_deleted, updated_at + id, base_id, agency_base_id, url, subsector, status, last_scraped_at, type, is_deleted, updated_at, is_stopping FROM data_collection.source WHERE agency_base_id = :agency_base_id::UUID ORDER BY base_id, updated_at DESC ) SELECT - ls.id, ls.base_id, ls.agency_base_id, ls.url, ls.subsector, ls.status, ls.last_scraped_at, ls.type, + ls.id, ls.base_id, ls.agency_base_id, ls.url, ls.subsector, ls.status, ls.last_scraped_at, ls.type, ls.is_stopping, :page as page, CEIL(COUNT(*) OVER () / :page_size::DECIMAL) AS total_pages, (COUNT(*) OVER ()) AS total, diff --git a/GUI/src/pages/Agency/Agency.tsx b/GUI/src/pages/Agency/Agency.tsx index 37a0b16..46d095d 100644 --- a/GUI/src/pages/Agency/Agency.tsx +++ b/GUI/src/pages/Agency/Agency.tsx @@ -732,7 +732,11 @@ const Agency: FC = () => { appearance="text" size="s" onClick={() => handleStopScraping(row.original.baseId)} - disabled={stopScrapingMutation.isLoading} + disabled={ + (stopScrapingMutation.isLoading && + stopScrapingMutation.variables === row.original.baseId) || + row.original.isStopping + } > } diff --git a/GUI/src/services/sources.ts b/GUI/src/services/sources.ts index c8118ff..ccfbd71 100644 --- a/GUI/src/services/sources.ts +++ b/GUI/src/services/sources.ts @@ -24,6 +24,7 @@ export interface Source { type?: string; qualityControl?: 'basic' | 'comprehensive' | null; extractImages?: boolean; + isStopping?: boolean; } // API Integration interface - extends Source but with specific properties From a6a2d048c9b32a36e82b392ae38abf71138821bd Mon Sep 17 00:00:00 2001 From: ruwinirathnamalala Date: Mon, 17 Aug 2026 18:17:16 +0530 Subject: [PATCH 08/10] Limit Playwright timeout retries in scrapper download handler --- scrapper/scrapper/download_handler.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/scrapper/scrapper/download_handler.py b/scrapper/scrapper/download_handler.py index 3dda275..cb08e88 100644 --- a/scrapper/scrapper/download_handler.py +++ b/scrapper/scrapper/download_handler.py @@ -8,6 +8,9 @@ from twisted.internet.defer import Deferred +PLAYWRIGHT_TIMEOUT_MAX_RETRIES = 3 + + class DownloadHandler(ScrapyPlaywrightDownloadHandler): def __init__(self, crawler: Crawler) -> None: super().__init__(crawler) @@ -36,10 +39,16 @@ def download_request(self, request: Request, spider: Spider) -> Deferred: spider.logger.info(f"Playwright download: {request.url}") return super().download_request(request, spider) - async def _download_request(self, request: Request, spider: Spider) -> Response: + async def _download_request( + self, request: Request, spider: Spider, _attempt: int = 1 + ) -> Response: """ Internal async download method with fallback for download errors. This is called by the parent's download_request when using Playwright. + Retries on Playwright timeout up to PLAYWRIGHT_TIMEOUT_MAX_RETRIES times + before giving up, so a single unresponsive URL can never wedge the + spider in an infinite retry loop and block it from ever reaching + parse() again (where the source's stop flag is checked). """ try: spider.logger.info(f"Playwright request started: {request.url}") @@ -48,13 +57,20 @@ async def _download_request(self, request: Request, spider: Spider) -> Response: spider.logger.info(f"Playwright request finished: {request.url}") return r except TimeoutError: + if _attempt >= PLAYWRIGHT_TIMEOUT_MAX_RETRIES: + spider.logger.error( + f"request timed out due to playwright: {request.url}. " + f"Giving up after {_attempt} attempts" + ) + raise spider.logger.warning( - f"request timed out due to playwright: {request.url}. Try again" + f"request timed out due to playwright: {request.url}. " + f"Try again ({_attempt}/{PLAYWRIGHT_TIMEOUT_MAX_RETRIES})" ) await self._close() super().__init__(self.crawler) # Re-initialize with the same crawler await self._launch() - return await self._download_request(request, spider) + return await self._download_request(request, spider, _attempt + 1) except Exception as e: # Catch "Download is starting" and similar download errors as safety net if "Download is starting" in str(e) or "net::ERR_ABORTED" in str(e): From 3624984a177c3fa5994f14ce00214f3bdf1bb513 Mon Sep 17 00:00:00 2001 From: ruwinirathnamalala Date: Tue, 18 Aug 2026 14:02:13 +0530 Subject: [PATCH 09/10] Review comments resolved --- search-service/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search-service/index.js b/search-service/index.js index e38d093..aa55b9a 100644 --- a/search-service/index.js +++ b/search-service/index.js @@ -424,7 +424,7 @@ app.delete("/documents/:sourceId/:sourceFileId", async (req, res) => { // Check if index exists const exists = await opensearch.indices.exists({ index: indexName }); if (!exists.body) { - console.warn(`⚠️ Index not found for source: ${sourceId}`); + console.warn(`Index not found for source: ${sourceId}`); return res.json({ success: true, source_id: sourceId, From 7de50afc867706f96b7022a89561ea8f13a4a9df Mon Sep 17 00:00:00 2001 From: keitsria Date: Tue, 18 Aug 2026 13:28:47 +0300 Subject: [PATCH 10/10] chore: deploy --- .env.cleaner | 2 +- .env.dev | 2 +- .env.scrapper | 2 +- .env.search | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.env.cleaner b/.env.cleaner index f4c44ac..c539039 100644 --- a/.env.cleaner +++ b/.env.cleaner @@ -1,4 +1,4 @@ RELEASE=dev VERSION=1 BUILD=3 -FIX=9 +FIX=10 diff --git a/.env.dev b/.env.dev index 931fc9c..855224c 100644 --- a/.env.dev +++ b/.env.dev @@ -1,4 +1,4 @@ RELEASE=dev VERSION=1 BUILD=2 -FIX=8 +FIX=9 diff --git a/.env.scrapper b/.env.scrapper index 954918e..f4c44ac 100644 --- a/.env.scrapper +++ b/.env.scrapper @@ -1,4 +1,4 @@ RELEASE=dev VERSION=1 BUILD=3 -FIX=8 +FIX=9 diff --git a/.env.search b/.env.search index eca0603..5b2cf5e 100644 --- a/.env.search +++ b/.env.search @@ -1,4 +1,4 @@ RELEASE=dev VERSION=1 BUILD=3 -FIX=4 +FIX=5