diff --git a/.env.cleaner b/.env.cleaner index f4c44ac..c539039 100644 --- a/.env.cleaner +++ b/.env.cleaner @@ -1,4 +1,4 @@ RELEASE=dev VERSION=1 BUILD=3 -FIX=9 +FIX=10 diff --git a/.env.dev b/.env.dev index 6f536cf..855224c 100644 --- a/.env.dev +++ b/.env.dev @@ -1,4 +1,4 @@ RELEASE=dev VERSION=1 BUILD=2 -FIX=7 +FIX=9 diff --git a/.env.scrapper b/.env.scrapper index 954918e..f4c44ac 100644 --- a/.env.scrapper +++ b/.env.scrapper @@ -1,4 +1,4 @@ RELEASE=dev VERSION=1 BUILD=3 -FIX=8 +FIX=9 diff --git a/.env.search b/.env.search index eca0603..5b2cf5e 100644 --- a/.env.search +++ b/.env.search @@ -1,4 +1,4 @@ RELEASE=dev VERSION=1 BUILD=3 -FIX=4 +FIX=5 diff --git a/DSL/Resql/ckb/GET/source/list_agency_sources.sql b/DSL/Resql/ckb/GET/source/list_agency_sources.sql index 558acbc..5db747e 100644 --- a/DSL/Resql/ckb/GET/source/list_agency_sources.sql +++ b/DSL/Resql/ckb/GET/source/list_agency_sources.sql @@ -64,16 +64,19 @@ declaration: - field: has_finished_files type: boolean description: "True if source has at least one finished file" + - field: is_stopping + type: boolean + description: "True if a stop was requested and is still being processed" */ WITH latest_sources AS ( SELECT DISTINCT ON (base_id) - id, base_id, agency_base_id, url, subsector, status, last_scraped_at, type, is_deleted, updated_at + id, base_id, agency_base_id, url, subsector, status, last_scraped_at, type, is_deleted, updated_at, is_stopping FROM data_collection.source WHERE agency_base_id = :agency_base_id::UUID ORDER BY base_id, updated_at DESC ) SELECT - ls.id, ls.base_id, ls.agency_base_id, ls.url, ls.subsector, ls.status, ls.last_scraped_at, ls.type, + ls.id, ls.base_id, ls.agency_base_id, ls.url, ls.subsector, ls.status, ls.last_scraped_at, ls.type, ls.is_stopping, :page as page, CEIL(COUNT(*) OVER () / :page_size::DECIMAL) AS total_pages, (COUNT(*) OVER ()) AS total, diff --git a/GUI/src/pages/Agency/Agency.tsx b/GUI/src/pages/Agency/Agency.tsx index 37a0b16..ce43378 100644 --- a/GUI/src/pages/Agency/Agency.tsx +++ b/GUI/src/pages/Agency/Agency.tsx @@ -732,7 +732,11 @@ const Agency: FC = () => { appearance="text" size="s" onClick={() => handleStopScraping(row.original.baseId)} - disabled={stopScrapingMutation.isLoading} + disabled={ + (stopScrapingMutation.isLoading && + stopScrapingMutation.variables === row.original.baseId) || + row.original.isStopping + } > } @@ -1265,43 +1269,72 @@ const Agency: FC = () => { {t('knowledgeBase.contentExtractionQualityControlOptions')}
- - +
+ + + + +
+
+ + + + +
diff --git a/GUI/src/pages/Reports/Report.tsx b/GUI/src/pages/Reports/Report.tsx index d59864f..e70739f 100644 --- a/GUI/src/pages/Reports/Report.tsx +++ b/GUI/src/pages/Reports/Report.tsx @@ -16,6 +16,7 @@ import { ReportPage, ReportPagesListParams, } from 'services/reports'; +import { getSanitizedErrorDetail } from 'utils/report-error-utils'; import 'pages/Agency/AgencyList.scss'; const Report: FC = () => { @@ -168,19 +169,22 @@ const Report: FC = () => { accessorKey: 'errorMessage', header: t('reports.errorMessage'), enableColumnFilter: false, - cell: ({ row }) => ( -
- - {row.original.errorMessage || '-'} - -
- ), + cell: ({ row }) => { + const errorDetail = getSanitizedErrorDetail(row.original.errorMessage); + return ( +
+ + {errorDetail} + +
+ ); + }, }, { accessorKey: 'scrapedAt', diff --git a/GUI/src/services/sources.ts b/GUI/src/services/sources.ts index c8118ff..ccfbd71 100644 --- a/GUI/src/services/sources.ts +++ b/GUI/src/services/sources.ts @@ -24,6 +24,7 @@ export interface Source { type?: string; qualityControl?: 'basic' | 'comprehensive' | null; extractImages?: boolean; + isStopping?: boolean; } // API Integration interface - extends Source but with specific properties diff --git a/GUI/src/utils/report-error-utils.ts b/GUI/src/utils/report-error-utils.ts new file mode 100644 index 0000000..923dc50 --- /dev/null +++ b/GUI/src/utils/report-error-utils.ts @@ -0,0 +1,28 @@ +// Redacts usernames/credentials from a raw scraper error message before it +// is shown in the UI (both in the table cell and its hover tooltip), while +// keeping the rest of the message intact. +export function getSanitizedErrorDetail(rawMessage?: string | null): string { + if (!rawMessage) return '-'; + + let sanitized = rawMessage; + + // Credentials embedded as URL userinfo, e.g. http://user:password@host. + sanitized = sanitized.replace( + /:\/\/[^\s/@]+:[^\s/@]+@/gi, + '://[redacted]@' + ); + + // Basic auth headers, e.g. Authorization: Basic dXNlcjpwYXNz. + sanitized = sanitized.replace( + /\b(authorization\s*:\s*(basic|bearer)\s+)\S+/gi, + '$1[redacted]' + ); + + // Common credential-style query params, e.g. ?token=..., &password=.... + sanitized = sanitized.replace( + /([?&](?:token|api[_-]?key|password|passwd|secret|access[_-]?token|auth)=)[^&\s"'<>]+/gi, + '$1[redacted]' + ); + + return sanitized; +} diff --git a/GUI/translations/en/common.json b/GUI/translations/en/common.json index 67936b8..02362b0 100644 --- a/GUI/translations/en/common.json +++ b/GUI/translations/en/common.json @@ -622,6 +622,7 @@ "contentSaved": "File saved successfully", "failed": "Failed", "in_review": "In Review", + "not_found": "Not Found", "startCleaningNotification":"All pages have been scraped. Before continuing, delete all pages that you do not want to make available to the chatbot, like news and archived pages. Once you have made your selection, continue cleaning the pages by clicking the 'Start cleaning' button.", "cleaningStarted": "Cleaning started" }, diff --git a/GUI/translations/et/common.json b/GUI/translations/et/common.json index c0d7a90..f0e25fa 100644 --- a/GUI/translations/et/common.json +++ b/GUI/translations/et/common.json @@ -622,6 +622,7 @@ "contentSaved": "Fail salvestati edukalt", "failed": "Ebaõnnestunud", "in_review": "Ülevaatamisel", + "not_found": "Ei leitud", "startCleaningNotification": "Kõik leheküljed on kraabitud. Enne jätkamist kustutage kõik leheküljed, mida te ei soovi vestlusrobotile kättesaadavaks teha, näiteks uudised ja arhiveeritud leheküljed. Kui olete oma valiku teinud, jätkake lehekülgede puhastamist, klõpsates nuppu \"Alusta puhastamist\".", "cleaningStarted": "Puhastamine algas" diff --git a/cleaning/worker/utils.py b/cleaning/worker/utils.py index a6fc8c5..d2a00aa 100644 --- a/cleaning/worker/utils.py +++ b/cleaning/worker/utils.py @@ -1,6 +1,7 @@ import contextlib import datetime import logging +import re import shutil from collections.abc import Iterator @@ -11,6 +12,44 @@ logger = logging.getLogger(__name__) +# Query/form params commonly used to pass credentials or tokens. +_SENSITIVE_PARAM_NAMES = ( + "token", + "api_key", + "apikey", + "password", + "passwd", + "pwd", + "secret", + "access_token", + "auth", + "session", + "sessionid", + "sid", +) + +_USERINFO_RE = re.compile(r"://[^\s/@]+:[^\s/@]+@") +_AUTH_HEADER_RE = re.compile( + r"(authorization[\"']?\s*[:=]\s*[\"']?(basic|bearer)\s+)\S+", re.IGNORECASE +) +_SENSITIVE_PARAM_RE = re.compile( + r"([?&](?:" + "|".join(_SENSITIVE_PARAM_NAMES) + r")=)[^&\s\"'<>]+", + re.IGNORECASE, +) + + +def sanitize_sensitive_text(text: str) -> str: + """Redact credentials/tokens from a URL or error message before it is + logged or sent to the backend (e.g. userinfo in a URL, Authorization + headers appearing in exception text, credential-style query params).""" + if not text: + return text + + sanitized = _USERINFO_RE.sub("://[redacted]@", text) + sanitized = _AUTH_HEADER_RE.sub(r"\1[redacted]", sanitized) + sanitized = _SENSITIVE_PARAM_RE.sub(r"\1[redacted]", sanitized) + return sanitized + def send_error( url: str, @@ -28,7 +67,7 @@ def send_error( "url": url, "scraped_at": scraped_at, "error_type": error_type, - "error_message": error_message, + "error_message": sanitize_sensitive_text(error_message), "source_base_id": source_base_id, "agency_base_id": agency_base_id, "source_run_report_base_id": source_run_report_base_id, diff --git a/scrapper/scrapper/download_handler.py b/scrapper/scrapper/download_handler.py index 3dda275..cb08e88 100644 --- a/scrapper/scrapper/download_handler.py +++ b/scrapper/scrapper/download_handler.py @@ -8,6 +8,9 @@ from twisted.internet.defer import Deferred +PLAYWRIGHT_TIMEOUT_MAX_RETRIES = 3 + + class DownloadHandler(ScrapyPlaywrightDownloadHandler): def __init__(self, crawler: Crawler) -> None: super().__init__(crawler) @@ -36,10 +39,16 @@ def download_request(self, request: Request, spider: Spider) -> Deferred: spider.logger.info(f"Playwright download: {request.url}") return super().download_request(request, spider) - async def _download_request(self, request: Request, spider: Spider) -> Response: + async def _download_request( + self, request: Request, spider: Spider, _attempt: int = 1 + ) -> Response: """ Internal async download method with fallback for download errors. This is called by the parent's download_request when using Playwright. + Retries on Playwright timeout up to PLAYWRIGHT_TIMEOUT_MAX_RETRIES times + before giving up, so a single unresponsive URL can never wedge the + spider in an infinite retry loop and block it from ever reaching + parse() again (where the source's stop flag is checked). """ try: spider.logger.info(f"Playwright request started: {request.url}") @@ -48,13 +57,20 @@ async def _download_request(self, request: Request, spider: Spider) -> Response: spider.logger.info(f"Playwright request finished: {request.url}") return r except TimeoutError: + if _attempt >= PLAYWRIGHT_TIMEOUT_MAX_RETRIES: + spider.logger.error( + f"request timed out due to playwright: {request.url}. " + f"Giving up after {_attempt} attempts" + ) + raise spider.logger.warning( - f"request timed out due to playwright: {request.url}. Try again" + f"request timed out due to playwright: {request.url}. " + f"Try again ({_attempt}/{PLAYWRIGHT_TIMEOUT_MAX_RETRIES})" ) await self._close() super().__init__(self.crawler) # Re-initialize with the same crawler await self._launch() - return await self._download_request(request, spider) + return await self._download_request(request, spider, _attempt + 1) except Exception as e: # Catch "Download is starting" and similar download errors as safety net if "Download is starting" in str(e) or "net::ERR_ABORTED" in str(e): diff --git a/scrapper/scrapper/utils.py b/scrapper/scrapper/utils.py index 4c84e6d..e7aae45 100644 --- a/scrapper/scrapper/utils.py +++ b/scrapper/scrapper/utils.py @@ -1,6 +1,7 @@ import contextlib import datetime import functools +import re import typing from collections.abc import Callable, Iterator from urllib.parse import urlparse @@ -15,6 +16,45 @@ BaseSpider = object +# Query/form params commonly used to pass credentials or tokens. +_SENSITIVE_PARAM_NAMES = ( + "token", + "api_key", + "apikey", + "password", + "passwd", + "pwd", + "secret", + "access_token", + "auth", + "session", + "sessionid", + "sid", +) + +_USERINFO_RE = re.compile(r"://[^\s/@]+:[^\s/@]+@") +_AUTH_HEADER_RE = re.compile( + r"(authorization[\"']?\s*[:=]\s*[\"']?(basic|bearer)\s+)\S+", re.IGNORECASE +) +_SENSITIVE_PARAM_RE = re.compile( + r"([?&](?:" + "|".join(_SENSITIVE_PARAM_NAMES) + r")=)[^&\s\"'<>]+", + re.IGNORECASE, +) + + +def sanitize_sensitive_text(text: str) -> str: + """Redact credentials/tokens from a URL or error message before it is + logged or sent to the backend (e.g. userinfo in a URL, Authorization + headers appearing in exception text, credential-style query params).""" + if not text: + return text + + sanitized = _USERINFO_RE.sub("://[redacted]@", text) + sanitized = _AUTH_HEADER_RE.sub(r"\1[redacted]", sanitized) + sanitized = _SENSITIVE_PARAM_RE.sub(r"\1[redacted]", sanitized) + return sanitized + + def send_error( ruuter_internal: str, url: str, @@ -31,7 +71,7 @@ def send_error( "url": url, "scraped_at": scraped_at, "error_type": error_type, - "error_message": error_message, + "error_message": sanitize_sensitive_text(error_message), "source_base_id": source_base_id, "agency_base_id": agency_base_id, "source_run_report_base_id": source_run_report_base_id, diff --git a/search-service/index.js b/search-service/index.js index daab4f7..7c4a4cc 100644 --- a/search-service/index.js +++ b/search-service/index.js @@ -54,7 +54,12 @@ async function createSourceIndex(sourceId) { document_type: { type: "keyword" }, page_title: { type: "text", analyzer: "standard" }, file_name: { type: "text" }, - url: { type: "keyword" }, + url: { + type: "keyword", + fields: { + text: { type: "text", analyzer: "standard" }, + }, + }, subsector: { type: "keyword" }, content: { type: "text", analyzer: "standard" }, indexed_at: { type: "date" }, @@ -245,7 +250,7 @@ app.get("/search/:sourceId", async (req, res) => { multi_match: { query: q.trim(), fields: [ - "url^5", + "url.text^5", "content^3", "page_title^2", "file_name^2", @@ -313,7 +318,7 @@ app.get("/search/:sourceId", async (req, res) => { multi_match: { query: q.trim(), fields: [ - "url^5", + "url.text^5", "content^3", "page_title^2", "file_name^2", @@ -424,9 +429,13 @@ app.delete("/documents/:sourceId/:sourceFileId", async (req, res) => { // Check if index exists const exists = await opensearch.indices.exists({ index: indexName }); if (!exists.body) { - return res.status(404).json({ - error: "Index not found", + console.warn(`Index not found for source: ${sourceId}`); + return res.json({ + success: true, source_id: sourceId, + source_file_id: sourceFileId, + deleted_count: 0, + status: "index_not_found", }); }