diff --git a/.gitignore b/.gitignore index b1f98f47..8d2503bb 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,5 @@ Postman/ .env.* !.env.example env.local +*.bak scripts/private/* diff --git a/docker/local/docker-compose.local.yml b/docker/local/docker-compose.local.yml new file mode 100644 index 00000000..21df7bab --- /dev/null +++ b/docker/local/docker-compose.local.yml @@ -0,0 +1,273 @@ +name: tap_lms_local + +# ── Changes from the original docker-compose.yml ───────────────────────────── +# +# Added/Updated services: +# tap_plg_worker + tap_plg_api + tap_plg_postgres — real ML service +# llm-stub — fake OpenAI/TogetherAI/VertexAI (no real LLM calls) +# glific-stub — fake Glific GraphQL API (no real WhatsApp messages) +# +# Container count: 10 total +# dev-lms, postgres, redis-cache, redis-queue, rabbitmq, +# tap_plg_postgres, tap_plg_worker, tap_plg_api, llm-stub, glific-stub + +services: + # ── Frappe dev container (tap_lms) ────────────────────────── + + dev-lms: + user: "${UID:-1000}:${GID:-1000}" + build: + context: ../.. + dockerfile: docker/local/Dockerfile + container_name: tap_lms_dev + # Runs the idempotent bootstrap, then bench + both consumers, so a + # plain `podman-compose up` is enough on its own — no more manual + # `exec ... bash -lc '...'` steps required after the containers exist. + # Override with `command: sleep infinity` if you'd rather exec in + # and drive things by hand. + command: ["bash", "/workspace/frappe_tap/scripts/entrypoint.sh"] + restart: on-failure + depends_on: + postgres: + condition: service_healthy + redis-cache: + condition: service_started + redis-queue: + condition: service_started + rabbitmq: + condition: service_started + glific-stub: + condition: service_started + tap_plg_api: + condition: service_started + llm-stub: + condition: service_started + env_file: + - ../../env.local + environment: + SHELL: /bin/bash + GLIFIC_API_URL: http://glific-stub:4000 + APP_ENV: dev + STUB_MODE: "0" + # rag_service's isolated venv reuses downloaded wheels across + # rebuilds via this cache, so even a forced reinstall (changed + # requirements.txt) doesn't re-download numpy/opencv/etc. from + # the network every time. + PIP_CACHE_DIR: /home/frappe/.cache/pip + ports: + - "${WEB_PORT:-8000}:8000" + - "${SOCKETIO_PORT:-9000}:9000" + volumes: + - bench-data:/home/frappe/frappe-bench + - ../..:/workspace/frappe_tap:cached,z + - ../../../rag_service:/workspace/rag_service:cached,z + # Named volumes so the rag_service venv and its pip download + # cache survive container recreation (down/up, --build, etc.) + # instead of being rebuilt from scratch every time. + - rag-venv-data:/home/frappe/rag_venv + # Mount the local folder for pip cache so it survives + # volume deletions. the :U flag changes the UID/GID to + # that of the container. + - ~/.cache/pip-docker-cache:/home/frappe/.cache/pip:z,U + working_dir: /home/frappe + + # ── PostgreSQL (Frappe) ──────────────────────────────────────────────────── + postgres: + image: postgres:15 + container_name: tap_lms_postgres + environment: + POSTGRES_USER: "${POSTGRES_USER:-postgres}" + POSTGRES_PASSWORD: "${POSTGRES_PASSWORD:-postgres}" + POSTGRES_DB: "${POSTGRES_DB:-postgres}" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-postgres}"] + interval: 5s + timeout: 5s + retries: 20 + ports: + - "${POSTGRES_PORT:-5432}:5432" + volumes: + - postgres-data:/var/lib/postgresql/data + + # ── Redis cache ──────────────────────────────────────────────────────────── + redis-cache: + image: redis:7-alpine + container_name: tap_lms_redis_cache + ports: + - "${REDIS_CACHE_PORT:-6379}:6379" + + # ── Redis queue ──────────────────────────────────────────────────────────── + redis-queue: + image: redis:7-alpine + container_name: tap_lms_redis_queue + ports: + - "${REDIS_QUEUE_PORT:-6380}:6379" + + # ── RabbitMQ ─────────────────────────────────────────────────────────────── + rabbitmq: + image: docker.io/library/rabbitmq:4-management + container_name: tap_lms_rabbitmq + ports: + - "5672:5672" + - "15672:15672" + environment: + RABBITMQ_DEFAULT_USER: guest + RABBITMQ_DEFAULT_PASS: guest + + # ── tap_plg PostgreSQL (pgvector) ────────────────────────────────────────── + tap_plg_postgres: + image: pgvector/pgvector:pg16 + container_name: tap_lms_plg_postgres + environment: + POSTGRES_DB: plagiarism_db + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d plagiarism_db"] + interval: 5s + timeout: 5s + retries: 20 + ports: + - "5433:5432" + volumes: + - tap_plg_postgres_data:/var/lib/postgresql/data + + # ── tap_plg worker (Real ML Service) ────────────────────────────────────── + tap_plg_worker: + build: + context: ../../../tap_plg + dockerfile: Dockerfile + container_name: tap_lms_plg_worker + environment: + RABBITMQ_HOST: "${RABBITMQ_HOST:-rabbitmq}" + RABBITMQ_PORT: "${RABBITMQ_PORT:-5672}" + RABBITMQ_USER: "${RABBITMQ_USERNAME:-guest}" + RABBITMQ_PASS: "${RABBITMQ_PASSWORD:-guest}" + RABBITMQ_VHOST: "${RABBITMQ_VIRTUAL_HOST:-/}" + SUBMISSION_QUEUE: "${RABBITMQ_SUBMISSION_QUEUE:-submission_queue}" + FEEDBACK_QUEUE: "${RABBITMQ_PLAGIARISM_RESULTS_QUEUE:-plagiarism_feedback}" + POSTGRES_HOST: tap_plg_postgres + POSTGRES_PORT: 5432 + POSTGRES_DB: plagiarism_db + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + CLIP_MODEL: "${CLIP_MODEL:-ViT-B/32}" + CLIP_PRETRAINED: "${CLIP_PRETRAINED:-openai}" + USE_PGVECTOR: "true" + MOCK_GLIFIC: "true" + APP_ENV: dev + LOG_LEVEL: INFO + # Ensure all ML libraries use the persistent cache volume + HF_HOME: /root/.cache/huggingface + TORCH_HOME: /root/.cache/torch + XDG_CACHE_HOME: /root/.cache + HF_HUB_DISABLE_SYMLINKS: "1" + depends_on: + rabbitmq: + condition: service_started + tap_plg_postgres: + condition: service_started + volumes: + - ../../../tap_plg:/app:cached,z + - tap_plg_model_cache:/root/.cache:z + + # ── tap_plg API ──────────────────────────────────────────────────────────── + tap_plg_api: + build: + context: ../../../tap_plg + dockerfile: Dockerfile.api + container_name: tap_lms_plg_api + command: + [ + "uvicorn", + "api.api:app", + "--host", + "0.0.0.0", + "--port", + "8000", + "--reload", + ] + environment: + RABBITMQ_HOST: "${RABBITMQ_HOST:-rabbitmq}" + RABBITMQ_PORT: "${RABBITMQ_PORT:-5672}" + RABBITMQ_USER: "${RABBITMQ_USERNAME:-guest}" + RABBITMQ_PASS: "${RABBITMQ_PASSWORD:-guest}" + RABBITMQ_VHOST: "${RABBITMQ_VIRTUAL_HOST:-/}" + SUBMISSION_QUEUE: "${RABBITMQ_SUBMISSION_QUEUE:-submission_queue}" + FEEDBACK_QUEUE: "${RABBITMQ_PLAGIARISM_RESULTS_QUEUE:-plagiarism_feedback}" + APP_ENV: dev + LOG_LEVEL: INFO + ports: + - "${TAP_PLG_API_PORT:-8080}:8000" + depends_on: + rabbitmq: + condition: service_started + volumes: + - ../../../tap_plg:/app:cached,z + healthcheck: + test: + ["CMD-SHELL", "curl -f http://localhost:8000/health || exit 1"] + interval: 30s + timeout: 5s + retries: 5 + + # ── LLM stub ────────────────────────────────────────────────────────────── + # Replaces OpenAI / TogetherAI / Vertex AI. + # Returns realistic randomised feedback JSON in OpenAI-compatible format. + # Deterministic per submission_id. Simulates 0.5–2s latency. + # Point rag_service LLM Settings → base_url at http://llm-stub:8001 + llm-stub: + build: + context: ./llm_stub + dockerfile: Dockerfile + container_name: tap_lms_llm_stub + command: + [ + "uvicorn", + "main:app", + "--host", + "0.0.0.0", + "--port", + "8001", + "--reload", + ] + ports: + - "${LLM_STUB_PORT:-8001}:8001" + volumes: + - ./llm_stub:/app:z + healthcheck: + test: + ["CMD-SHELL", "curl -f http://localhost:8001/health || exit 1"] + interval: 30s + timeout: 5s + retries: 3 + + # ── Glific stub ──────────────────────────────────────────────────────────── + # Replaces the Glific GraphQL API. No real WhatsApp messages sent. + # Stubs all 8 operations used by tap_lms/glific_integration.py. + # + # Developer audit endpoints: + # GET http://localhost:4000/stub/flow-calls — all startContactFlow calls + # GET http://localhost:4000/stub/contacts — all contacts created + # GET http://localhost:4000/stub/reset — clear state between tests + glific-stub: + build: + context: ./glific_stub + dockerfile: Dockerfile + container_name: tap_lms_glific_stub + ports: + - "${GLIFIC_STUB_PORT:-4000}:4000" + healthcheck: + test: + ["CMD-SHELL", "curl -f http://localhost:4000/health || exit 1"] + interval: 30s + timeout: 5s + retries: 3 + +volumes: + bench-data: + postgres-data: + tap_plg_postgres_data: + tap_plg_model_cache: + rag-venv-data: + pip-cache: diff --git a/docker/local/docker-compose.yml b/docker/local/docker-compose.yml index 24abb6d6..c35d7e88 100644 --- a/docker/local/docker-compose.yml +++ b/docker/local/docker-compose.yml @@ -14,6 +14,8 @@ services: condition: service_started redis-queue: condition: service_started + rabbitmq: + condition: service_started env_file: - ../../env.local environment: @@ -23,7 +25,7 @@ services: - "${SOCKETIO_PORT:-9000}:9000" volumes: - bench-data:/home/frappe/frappe-bench - - ../..:/workspace/frappe_tap:cached + - ../..:/workspace/frappe_tap:cached,z working_dir: /home/frappe postgres: @@ -55,6 +57,15 @@ services: ports: - "${REDIS_QUEUE_PORT:-6380}:6379" + rabbitmq: + image: docker.io/library/rabbitmq:4-management + ports: + - "5672:5672" # AMQP protocol port + - "15672:15672" # Web Management Dashboard + environment: + RABBITMQ_DEFAULT_USER: guest + RABBITMQ_DEFAULT_PASS: guest + volumes: bench-data: postgres-data: diff --git a/docker/local/glific_stub/Dockerfile b/docker/local/glific_stub/Dockerfile new file mode 100644 index 00000000..8e80e787 --- /dev/null +++ b/docker/local/glific_stub/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.11-slim + +WORKDIR /app + +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY main.py . + +EXPOSE 4000 + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "4000", "--log-level", "info"] diff --git a/docker/local/glific_stub/main.py b/docker/local/glific_stub/main.py new file mode 100644 index 00000000..3048248f --- /dev/null +++ b/docker/local/glific_stub/main.py @@ -0,0 +1,430 @@ +""" +infra/docker/glific_stub/main.py + +FastAPI stub that mimics the Glific GraphQL API used by tap_lms. + +Replaces the real Glific API in local development so: + - No real WhatsApp messages are sent to students + - No real Glific account credentials are needed + - All GraphQL operations return realistic responses immediately + - Every call is logged so developers can verify the pipeline reached + the notification step + +Glific uses a single GraphQL endpoint: POST /api +Authentication: POST /api/v1/session + +Operations stubbed (all found in glific_integration.py): + Mutations: + createContact — returns a fake contact with a deterministic ID + updateContact — returns success + optinContact — returns success + startContactFlow — returns success (the feedback delivery step) + createGroup — returns a fake group + updateGroupContacts — returns success + + Queries: + contactByPhone — returns a fake contact or null + contact(id) — returns a fake contact + groups(filter) — returns matching fake groups + +Usage: + Configured in docker-compose.local.yml as service "glific-stub". + Set GLIFIC_API_URL=http://glific-stub:4000 in the Frappe dev container's + environment — this overrides what is stored in the Glific Settings DocType. + (Or seed the DocType with this URL during local setup.) +""" + +import json +import logging +import time +import uuid +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Optional + +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse + + +# remove health check pings from logs +class _HealthCheckFilter(logging.Filter): + def filter(self, record): + return "/health" not in record.getMessage() + + +# ── Setup ───────────────────────────────────────────────────────────────────── +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("glific-stub").addFilter(_HealthCheckFilter()) + +app = FastAPI( + title="Glific Stub", description="Local Glific API stub for TAP LMS development" +) + +# ── In-memory state ─────────────────────────────────────────────────────────── +# Keeps fake contacts and groups alive for the duration of the process. +# Resets on container restart — intentional for a dev stub. + +_contacts: Dict[str, Dict] = {} # phone → contact +_groups: Dict[str, Dict] = {} # label → group +_flow_calls: list = [] # audit log of all startContactFlow calls + + +def _make_contact_id(phone: str) -> str: + """Deterministic fake contact ID from phone number.""" + return str(abs(hash(phone)) % 9_000_000 + 1_000_000) + + +def _make_contact(phone: str, name: str, language_id: int = 1) -> Dict: + return { + "id": _make_contact_id(phone), + "name": name, + "phone": phone, + "languageId": language_id, + "optinTime": datetime.now(timezone.utc).isoformat(), + "optoutTime": None, + "bspStatus": "SESSION_AND_HSM", + "status": "VALID", + "lastMessageAt": datetime.now(timezone.utc).isoformat(), + "fields": "{}", + "settings": "{}", + } + + +def _make_group(label: str, description: str = "") -> Dict: + return { + "id": str(abs(hash(label)) % 900_000 + 100_000), + "label": label, + "description": description, + } + + +# ── Authentication endpoint ─────────────────────────────────────────────────── + + +@app.post("/api/v1/session") +async def session(request: Request): + """ + Glific auth endpoint. Returns a stub token that never expires + (well, expires in 100 years — effectively never for local dev). + tap_lms caches the token in the Glific Settings DocType and + refreshes when it expires, so a long-lived token avoids noise. + """ + logger.info("AUTH: token requested") + expiry = datetime.now(timezone.utc) + timedelta(days=36500) + return JSONResponse( + content={ + "data": { + "access_token": "stub-access-token-local-dev", + "renewal_token": "stub-renewal-token-local-dev", + "token_expiry_time": expiry.isoformat(), + } + } + ) + + +# ── GraphQL endpoint ────────────────────────────────────────────────────────── + + +@app.post("/api") +async def graphql(request: Request): + """ + Single GraphQL endpoint. Routes by operation name extracted from the query. + Returns realistic stubbed responses for all operations used by tap_lms. + """ + body = await request.json() + query: str = body.get("query", "") + variables: Dict = body.get("variables", {}) + + # Route by operation name / first keyword in the query + query_lower = query.lower().strip() + + # ── Mutations ───────────────────────────────────────────────────────────── + + if "startcontactflow" in query_lower: + return _start_contact_flow(variables) + + if "createcontact" in query_lower: + return _create_contact(variables) + + if "updatecontact" in query_lower: + return _update_contact(variables) + + if "optincontact" in query_lower: + return _optin_contact(variables) + + if "creategroup" in query_lower: + return _create_group(variables) + + if "updategroupcontacts" in query_lower: + return _update_group_contacts(variables) + + # ── Queries ─────────────────────────────────────────────────────────────── + + if "contactbyphone" in query_lower: + return _contact_by_phone(variables) + + if "contact(" in query_lower or "contact(id" in query_lower: + return _get_contact(variables) + + if "groups(" in query_lower: + return _list_groups(variables) + + # Unknown operation — return empty success so tap_lms doesn't crash + logger.warning( + f"UNKNOWN GraphQL operation — returning empty success. Query: {query[:120]}" + ) + return JSONResponse(content={"data": {}}) + + +# ── Mutation handlers ───────────────────────────────────────────────────────── + + +def _start_contact_flow(variables: Dict) -> JSONResponse: + """ + The most important stub — this is what fires when a student receives + feedback via WhatsApp. Log every call with full context so developers + can verify the pipeline reached the notification step. + """ + flow_id = variables.get("flowId") + contact_id = variables.get("contactId") + default_results_raw = variables.get("defaultResults", "{}") + + try: + default_results = ( + json.loads(default_results_raw) + if isinstance(default_results_raw, str) + else default_results_raw + ) + except Exception: + default_results = {} + + call_record = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "flow_id": flow_id, + "contact_id": contact_id, + "submission_id": default_results.get("submission_id"), + "feedback_preview": str(default_results.get("feedback", ""))[:120], + } + _flow_calls.append(call_record) + + logger.info( + f"FLOW TRIGGERED ✓ | flow_id={flow_id} contact_id={contact_id} " + f"submission_id={default_results.get('submission_id')} | " + f"[WhatsApp message would be sent here in production]" + ) + + return JSONResponse( + content={"data": {"startContactFlow": {"success": True, "errors": []}}} + ) + + +def _create_contact(variables: Dict) -> JSONResponse: + inp = variables.get("input", {}) + phone = inp.get("phone", f"stub_{uuid.uuid4().hex[:8]}") + name = inp.get("name", "Stub Student") + language_id = inp.get("languageId", 1) + + contact = _make_contact(phone, name, language_id) + _contacts[phone] = contact + + logger.info(f"CREATE CONTACT | name={name} phone={phone} id={contact['id']}") + + return JSONResponse( + content={ + "data": { + "createContact": { + "contact": { + "id": contact["id"], + "name": contact["name"], + "phone": contact["phone"], + }, + "errors": [], + } + } + } + ) + + +def _update_contact(variables: Dict) -> JSONResponse: + contact_id = variables.get("id") + inp = variables.get("input", {}) + + # Update fields in our in-memory store if contact exists + for phone, contact in _contacts.items(): + if contact["id"] == str(contact_id): + if "fields" in inp: + contact["fields"] = inp["fields"] + break + + logger.info(f"UPDATE CONTACT | id={contact_id}") + + return JSONResponse( + content={ + "data": { + "updateContact": { + "contact": {"id": contact_id, "fields": inp.get("fields", "{}")}, + "errors": [], + } + } + } + ) + + +def _optin_contact(variables: Dict) -> JSONResponse: + phone = variables.get("phone", "") + name = variables.get("name", "") + + # Create contact if not already in store + if phone not in _contacts: + _contacts[phone] = _make_contact(phone, name) + + contact = _contacts[phone] + logger.info(f"OPTIN CONTACT | phone={phone} name={name} id={contact['id']}") + + return JSONResponse( + content={ + "data": { + "optinContact": { + "contact": { + "id": contact["id"], + "phone": contact["phone"], + "name": contact["name"], + "lastMessageAt": contact["lastMessageAt"], + "optinTime": contact["optinTime"], + "bspStatus": contact["bspStatus"], + }, + "errors": [], + } + } + } + ) + + +def _create_group(variables: Dict) -> JSONResponse: + inp = variables.get("input", {}) + label = inp.get("label", f"stub-group-{uuid.uuid4().hex[:6]}") + description = inp.get("description", "") + + group = _make_group(label, description) + _groups[label] = group + + logger.info(f"CREATE GROUP | label={label} id={group['id']}") + + return JSONResponse( + content={"data": {"createGroup": {"group": group, "errors": []}}} + ) + + +def _update_group_contacts(variables: Dict) -> JSONResponse: + inp = variables.get("input", {}) + group_id = inp.get("groupId") + add_ids = inp.get("addContactIds", []) + + logger.info(f"ADD TO GROUP | group_id={group_id} contact_ids={add_ids}") + + return JSONResponse( + content={ + "data": { + "updateGroupContacts": { + "groupContacts": [{"id": str(uuid.uuid4())} for _ in add_ids], + "numberDeleted": 0, + } + } + } + ) + + +# ── Query handlers ──────────────────────────────────────────────────────────── + + +def _contact_by_phone(variables: Dict) -> JSONResponse: + phone = variables.get("phone", "") + contact = _contacts.get(phone) + + if contact: + logger.info(f"CONTACT BY PHONE | phone={phone} → found id={contact['id']}") + else: + logger.info(f"CONTACT BY PHONE | phone={phone} → not found") + + return JSONResponse(content={"data": {"contactByPhone": {"contact": contact}}}) + + +def _get_contact(variables: Dict) -> JSONResponse: + contact_id = str(variables.get("id", "")) + + # Find by ID in our store + found = None + for contact in _contacts.values(): + if contact["id"] == contact_id: + found = contact + break + + return JSONResponse(content={"data": {"contact": {"contact": found}}}) + + +def _list_groups(variables: Dict) -> JSONResponse: + label_filter = variables.get("filter", {}).get("label", "") + + if label_filter: + matching = [ + g for label, g in _groups.items() if label_filter.lower() in label.lower() + ] + else: + matching = list(_groups.values()) + + return JSONResponse(content={"data": {"groups": matching}}) + + +# ── Audit log endpoint (bonus — useful for dev inspection) ─────────────────── + + +@app.get("/stub/flow-calls") +def get_flow_calls(): + """ + Returns a log of all startContactFlow calls made during this session. + Use this to verify the pipeline reached the WhatsApp notification step + for a given submission_id: + + curl http://localhost:4000/stub/flow-calls | python3 -m json.tool + """ + return JSONResponse( + content={ + "total": len(_flow_calls), + "calls": _flow_calls, + } + ) + + +@app.get("/stub/contacts") +def get_contacts(): + """Returns all contacts created during this session.""" + return JSONResponse( + content={"total": len(_contacts), "contacts": list(_contacts.values())} + ) + + +@app.get("/stub/reset") +def reset(): + """Clears all in-memory state. Useful between test runs.""" + _contacts.clear() + _groups.clear() + _flow_calls.clear() + logger.info("STUB STATE RESET") + return JSONResponse(content={"status": "reset"}) + + +@app.get("/health") +def health(): + return JSONResponse( + content={ + "status": "ok", + "service": "glific-stub", + "flow_calls_this_session": len(_flow_calls), + "contacts_this_session": len(_contacts), + } + ) + + +if __name__ == "__main__": + import uvicorn + + uvicorn.run(app, host="0.0.0.0", port=4000, log_level="info") diff --git a/docker/local/glific_stub/requirements.txt b/docker/local/glific_stub/requirements.txt new file mode 100644 index 00000000..475e08f8 --- /dev/null +++ b/docker/local/glific_stub/requirements.txt @@ -0,0 +1,2 @@ +fastapi==0.111.0 +uvicorn==0.30.1 diff --git a/docker/local/llm_stub/Dockerfile b/docker/local/llm_stub/Dockerfile new file mode 100644 index 00000000..4ca220a3 --- /dev/null +++ b/docker/local/llm_stub/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.11-slim + +WORKDIR /app + +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* + + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY main.py . + +EXPOSE 8001 + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8001", "--log-level", "info"] diff --git a/docker/local/llm_stub/main.py b/docker/local/llm_stub/main.py new file mode 100644 index 00000000..ee0eb486 --- /dev/null +++ b/docker/local/llm_stub/main.py @@ -0,0 +1,264 @@ +""" +infra/docker/llm_stub/main.py + +Lightweight FastAPI stub that mimics the OpenAI / TogetherAI / Vertex AI +response format used by rag_service's EvaluationGenerator. + +Replaces real LLM API calls in local development so the full pipeline +can be tested end-to-end without incurring API costs or needing +production credentials. + +Returns a realistic but randomised feedback JSON that matches the exact +schema tap_lms's feedback_consumer.py expects. + +Usage: + Configured in docker-compose.local.yml as service "llm-stub". + Point rag_service's LLM provider base URL at http://llm-stub:8001. + +Endpoints: + POST /v1/chat/completions — OpenAI-compatible chat endpoint + POST /v1/completions — legacy completions endpoint + GET /health — health check +""" + +import json +import logging +import random +import time +import uuid +from typing import Any, Dict, List + +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse + + +# remove health check pings from logs +class _HealthCheckFilter(logging.Filter): + def filter(self, record): + return "/health" not in record.getMessage() + + +logging.getLogger("uvicorn.access").addFilter(_HealthCheckFilter()) + +app = FastAPI(title="LLM Stub", description="Local LLM stub for pipeline testing") + +# ── Canned feedback content ─────────────────────────────────────────────────── +# Varied enough that repeated test submissions feel distinct, but structured +# exactly as rag_service's EvaluationGenerator produces. + +_STRENGTHS = [ + "Strong use of colour contrast to create visual interest", + "Confident line work that demonstrates good motor control", + "Creative composition that fills the page effectively", + "Good understanding of light and shadow relationships", + "Expressive use of texture to add depth and character", + "Clear focal point that draws the viewer's attention", + "Thoughtful use of negative space in the composition", + "Consistent and controlled brushwork throughout the piece", +] + +_IMPROVEMENTS = [ + "Try varying line thickness to add more dynamism", + "Consider adding more detail to the background elements", + "Experiment with mixing colours directly on the canvas", + "Work on proportions by observing the subject more carefully", + "Add a mid-tone layer between the highlights and shadows", +] + +_ENCOURAGEMENTS = [ + "Keep experimenting — every artwork teaches you something new!", + "You are developing a unique artistic voice. Keep going!", + "Great effort this week. Practice makes progress!", + "Your creativity shines through in this piece. Well done!", + "You are growing as an artist with every submission!", +] + +_OVERALL_TEMPLATES = [ + "This is a {quality} piece of work that shows {quality2} understanding of the assignment. " + "Your use of {technique} is particularly noteworthy.", + "A {quality} submission that demonstrates {quality2} progress. " + "The {technique} in this piece is well-executed.", + "This artwork shows {quality} creativity and {quality2} technical skill, " + "especially in the way you have handled {technique}.", +] + +_QUALITIES = ["good", "strong", "impressive", "solid", "creative"] +_TECHNIQUES = ["colour", "composition", "line work", "shading", "texture"] + +_RUBRIC_SKILLS = [ + "Content Knowledge", + "Creativity", + "Technical Skill", + "Composition", + "Use of Colour", +] + + +def _random_feedback(submission_id: str) -> Dict[str, Any]: + """ + Generate a randomised but structurally valid feedback object. + The submission_id is seeded into the random selection so the same + submission always gets the same stub feedback (deterministic per run). + """ + rng = random.Random(submission_id) + + quality = rng.choice(_QUALITIES) + quality2 = rng.choice([q for q in _QUALITIES if q != quality]) + technique = rng.choice(_TECHNIQUES) + + overall = rng.choice(_OVERALL_TEMPLATES).format( + quality=quality, quality2=quality2, technique=technique + ) + + strengths = rng.sample(_STRENGTHS, k=rng.randint(2, 3)) + improvements = rng.sample(_IMPROVEMENTS, k=rng.randint(1, 2)) + encouragement = rng.choice(_ENCOURAGEMENTS) + final_grade = rng.randint(60, 95) + + rubric_evaluations = [ + { + "Skill": skill, + "grade_value": rng.randint(2, 4), + "observation": f"Student demonstrated {rng.choice(_QUALITIES)} ability in {skill.lower()}.", + } + for skill in _RUBRIC_SKILLS + ] + + return { + "overall_feedback": overall, + "overall_feedback_translated": f"[STUB TRANSLATION] {overall}", + "strengths": strengths, + "areas_for_improvement": improvements, + "encouragement": encouragement, + "rubric_evaluations": rubric_evaluations, + "learning_objectives_feedback": [ + f"Objective met: {rng.choice(_TECHNIQUES)} was applied effectively." + ], + "final_grade": final_grade, + "translation_language": "English", + } + + +def _wrap_as_openai_response(content: str, model: str = "stub-gpt-4") -> Dict[str, Any]: + """Wrap the feedback JSON string in an OpenAI-compatible chat completion response.""" + return { + "id": f"chatcmpl-stub-{uuid.uuid4().hex[:12]}", + "object": "chat.completion", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": content, + }, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": random.randint(800, 1200), + "completion_tokens": random.randint(200, 400), + "total_tokens": random.randint(1000, 1600), + }, + } + + +# ── Endpoints ───────────────────────────────────────────────────────────────── + + +@app.get("/health") +def health(): + return {"status": "ok", "service": "llm-stub"} + + +@app.post("/v1/chat/completions") +async def chat_completions(request: Request): + """ + OpenAI-compatible chat completions endpoint. + Reads the submission_id from the prompt if present (for deterministic + responses), otherwise uses a random ID. + """ + body = await request.json() + + # Attempt to extract submission_id from the prompt for deterministic output + submission_id = _extract_submission_id(body) + + # Simulate realistic LLM latency (0.5–2s locally) + await _simulate_latency() + + feedback = _random_feedback(submission_id) + content = json.dumps(feedback, ensure_ascii=False) + + model = body.get("model", "stub-gpt-4") + return JSONResponse(content=_wrap_as_openai_response(content, model=model)) + + +@app.post("/v1/completions") +async def completions(request: Request): + """Legacy completions endpoint — same stub response.""" + body = await request.json() + submission_id = _extract_submission_id(body) + await _simulate_latency() + feedback = _random_feedback(submission_id) + content = json.dumps(feedback, ensure_ascii=False) + return JSONResponse( + content={ + "id": f"cmpl-stub-{uuid.uuid4().hex[:12]}", + "object": "text_completion", + "created": int(time.time()), + "model": body.get("model", "stub-gpt-4"), + "choices": [{"text": content, "index": 0, "finish_reason": "stop"}], + "usage": { + "prompt_tokens": 900, + "completion_tokens": 300, + "total_tokens": 1200, + }, + } + ) + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def _extract_submission_id(body: Dict) -> str: + """ + Try to find a submission_id in the prompt messages. + Falls back to a random UUID so the stub always returns something valid. + """ + try: + messages: List[Dict] = body.get("messages", []) + for msg in messages: + content = msg.get("content", "") + if isinstance(content, str) and "submission_id" in content.lower(): + # Simple extraction — look for "SUB-" prefix pattern + import re + + match = re.search(r"SUB-[\w-]+", content, re.IGNORECASE) + if match: + return match.group(0) + elif isinstance(content, list): + for part in content: + if isinstance(part, dict) and "submission_id" in str(part).lower(): + import re + + match = re.search(r"SUB-[\w-]+", str(part), re.IGNORECASE) + if match: + return match.group(0) + except Exception: + pass + return str(uuid.uuid4()) + + +async def _simulate_latency(): + """Simulate realistic LLM response latency.""" + import asyncio + + delay = random.uniform(0.5, 2.0) + await asyncio.sleep(delay) + + +if __name__ == "__main__": + import uvicorn + + uvicorn.run(app, host="0.0.0.0", port=8001, log_level="info") diff --git a/docker/local/llm_stub/requirements.txt b/docker/local/llm_stub/requirements.txt new file mode 100644 index 00000000..475e08f8 --- /dev/null +++ b/docker/local/llm_stub/requirements.txt @@ -0,0 +1,2 @@ +fastapi==0.111.0 +uvicorn==0.30.1 diff --git a/docs/TAP_LMS_System_Understanding.md b/docs/TAP_LMS_System_Understanding.md new file mode 100644 index 00000000..8891d6d6 --- /dev/null +++ b/docs/TAP_LMS_System_Understanding.md @@ -0,0 +1,938 @@ +# TAP LMS — Full System Understanding + +**Version:** 1.7 +**Date:** July 2026 +**Purpose:** System architecture, user flows, and observability analysis across all five services — prepared for client validation. + +--- + +## Table of Contents + +1. [System Overview](#1-system-overview) +2. [Infrastructure](#2-infrastructure) +3. [The Five Services](#3-the-five-services) +4. [The Summer Program — Architecture Deep Dive](#4-the-summer-program--architecture-deep-dive) +5. [Complete User Flow — Student Submission](#5-complete-user-flow--student-submission) +6. [Complete User Flow — Quiz Assessment](#6-complete-user-flow--quiz-assessment) +7. [Full Pipeline — End to End](#7-full-pipeline--end-to-end) +8. [External Integrations Map](#8-external-integrations-map) +9. [Current Observability Gaps — Per Service](#9-current-observability-gaps--per-service) +10. [What a Stuck Submission Looks Like Today](#10-what-a-stuck-submission-looks-like-today) +11. [What a Stuck Submission Will Look Like After Monitoring](#11-what-a-stuck-submission-will-look-like-after-monitoring) +12. [Risks and Notable Code Issues Found](#12-risks-and-notable-code-issues-found) +13. [Monitoring Implementation Plan — All Three Services](#13-monitoring-implementation-plan--all-three-services) +14. [Glific ↔ tap_lms API Visibility — New Monitoring Scope](#14-glific--tap_lms-api-visibility--new-monitoring-scope) +15. [Local Development Environment and Testing Strategy](#15-local-development-environment-and-testing-strategy) +16. [Open Questions](#16-open-questions) +17. [DLQ Monitoring — Detailed Design](#17-dlq-monitoring--detailed-design) +18. [Error Classification — Retryable vs Non-Retryable Failures](#18-error-classification--retryable-vs-non-retryable-failures) + +--- + +## 1. System Overview + +TAP LMS is an online Learning Management System serving students in structured programs (including a Summer Program). Students submit artwork, text, audio, or video assignments via WhatsApp (through the Glific platform). Their submissions are automatically graded for plagiarism and AI-generated content, then evaluated by an AI model, with feedback delivered back to the student via WhatsApp in their local language with an audio component. + +The system is composed of **five distinct services**. Three are built and maintained by the TAP team, one is a third-party open-source platform, and one is a separately hosted AI assistant engine. + +``` + Student (WhatsApp) + │ + Glific ──────── API calls ──────── tap_lms + (3rd Party) + │ + tap_lms ────────<──────────────────────┐ + (Frappe) │ + │ │ + [submission_queue] HTTP API calls: + │ - assignment context + tap_plg ───────────────────>─ - reference images + (Plagiarism Service) - student details + │ │ + [plagiarism_feedback queue] │ + │ │ + rag_service ──────────────────>─────────┘ + (RAG / LLM Feedback) + │ + [feedback_results_queue] + │ + tap_lms + (Frappe) + │ + ElevenLabs (TTS audio) + │ + Glific + │ + Student (WhatsApp) + +---------------------------------------------------------------------------------------------- + ┌────────────────────────┐ + Teacher / Student ───▶│ tap_ai │ + (via Telegram or │ (Conversational AI │ + tap_lms integration) │ Engine — Frappe) │ + └──────────┬─────────────┘ + │ + RabbitMQ Workers + Pinecone + PostgreSQL + (local + tap_lms) +``` + +--- + +## 2. Infrastructure + +### Hosting + +The three services: `tap_lms`, `tap_plg`, `rag_service`, are hosted on **Google Cloud Platform (GCP)** using Compute Engine VMs. The development environment is an *e2-medium VM* (2 vCPUs, 4 GB RAM). Each service runs on its own dedicated VM in production (refer [Open Questions](#15-open-questions)). The fourth service, `tap_ai` is not directly called from `tap_lms` but it queries `tap_lms`'s postgres DB for required information. + +### Deployment model per service + +| Service | Runtime | Deployment | Database | +|---|---|---|---| +| `tap_lms` | Frappe 14.29.0 / Python | Frappe bench v5.16.2 on VM | PostgreSQL (Frappe-managed) | +| `rag_service` | Frappe 14.29.0 / Python | Frappe bench v5.16.2 on VM | PostgreSQL (Frappe-managed) | +| `tap_plg` | Standalone Python | Docker containers (docker-compose) | Dedicated PostgreSQL with pgvector extension + FAISS index on disk | +| `tap_ai` | Frappe 14.29.0 / Python | Frappe bench v5.16.2 on VM | PostgreSQL (local, tap_lms) + Pinecone + Redis | + +### Message broker + +All three services share a single **RabbitMQ instance hosted on CloudAMQP**. Three queues are in use: + +| Queue name | Direction | Publisher | Consumer | Config method | +|---|---|---|---|---| +| `submission_queue` | tap_lms → tap_plg | tap_lms | tap_plg | DocType in tap_lms; `.env` in tap_plg | +| `plagiarism_feedback` | tap_plg → rag_service | tap_plg | rag_service | `.env` in tap_plg; DocType in rag_service | +| `feedback_results_queue` | rag_service → tap_lms | rag_service | tap_lms | DocType in both | + +> **Note:** tap_lms and rag_service store queue names in the `RabbitMQ Settings` DocType (configurable via the Frappe UI). tap_plg stores them as environment variables with hardcoded defaults (`plagiarism_submissions` and `plagiarism_feedback`). These names must be kept in sync across all three services — a mismatch will cause submissions to silently disappear (refer [Open Questions](#15-open-questions)). + +--- + +## 3. The Five Services + +### 3.1 tap_lms (Core LMS — Frappe app) + +**What it is:** The primary application. Manages all student data, assignments, submissions, program state, and external integrations. Built on the Frappe framework (Python), which provides DocTypes (database-backed data models), a REST API layer, a scheduler, and RQ-based background job processing. + +**Key components:** + +- **`summer_program/save_submission.py`** — **Active entry point for all student submissions.** Receives student submissions via API (`POST /api/method/tap_lms.summer_program.save_submission.save_submission`), uploads media to Google Cloud Storage, creates a `Submission` DocType record, and publishes the submission to RabbitMQ. Supports four media types: **image** (jpg, png, gif, webp, bmp, svg), **video** (mp4, mov, avi, mkv, webm), **audio** (mp3, wav, ogg, opus, m4a, aac, flac), and **text** (inline, no GCS upload). Media type is auto-detected from the file extension. Also provides `get_submission_feedback` (poll for feedback status) and `ready_to_receive_feedback` (trigger feedback delivery flow) endpoints. +- **`imgana/submission.py`** — **Deprecated.** The original submission entry point (`POST /api/method/tap_lms.imgana.submission.assignment_submission`). No longer in active use; all Glific flows should call `save_submission` instead. +- **`summer_program/student_progression_sp.py`** — Manages the quiz assessment flow. Key whitelisted endpoints: `start_quiz` (initialise a `StudentQuizAttempt`), `submit_answer` (one call per question — records answer, returns next question or final result), and the private `_complete_quiz_sp` (auto-triggered on the last answer — computes score, determines pass/fail, awards points). See Section 6 for the full quiz flow. +- **`feedback_handler/feedback_consumer.py`** — A long-running RabbitMQ consumer that receives graded feedback results, updates the `Submission` record, triggers the ElevenLabs TTS call (for audio feedback), and sends a Glific WhatsApp notification to the student. +- **`summer_program/pe_dispatcher.py`** — A scheduled job running **every 1 minute** that drives a state machine for every active Summer Program student. This is the most performance-critical background process in the system — it processes up to 100,000 students per cycle. +- **`summer_program/escalation_runner.py`** — Runs every 2 hours to handle students whose program progression has stalled past a threshold. +- **`summer_program/scheduler.py`** — Daily batch job for admin-level program actions. +- **`glific_integration.py`** — HTTP client for the Glific API (WhatsApp messaging). Manages OAuth token refresh, contact creation, and flow triggers. + +**Scheduled jobs summary:** + +| Job | Frequency | What it does | +|---|---|---| +| `pe_dispatcher` | Every 1 minute | Drives Summer Program student state machine | +| `escalation_runner` | Every 2 hours | Handles stalled student progressions | +| `run_daily_actions` | Daily | Batch admin actions | +| `dlq_monitor` | Every 5 minutes | Polls CloudAMQP Management API for DLQ depths; emits structured log per queue | + +**External calls made by tap_lms:** + +| External service | What for | +|---|---| +| RabbitMQ (CloudAMQP) | Publish submissions; consume feedback results | +| CloudAMQP Management API | Poll DLQ depths every 5 minutes (HTTPS REST — not AMQP) | +| Google Cloud Storage | Store submitted images, audio, video | +| Glific API | Send WhatsApp messages and feedback to students | +| ElevenLabs API | Generate multilingual audio feedback (TTS) | + +--- + +### 3.2 tap_plg (Plagiarism Detection Service — standalone Python) + +**What it is:** A completely independent Python service (not Frappe) that performs multi-method plagiarism and AI-content detection on student image submissions. It is the most computationally intensive service in the system. Runs in Docker containers with two processes: a background worker and a FastAPI HTTP API. + +**Key components:** + +- **`app.py`** — Main async entry point. Initialises the shared database connection pool, the `ImageWorker` (loads ML models), the `SubmissionChecker`, and the RabbitMQ consumer. Handles graceful shutdown on SIGTERM/SIGINT. +- **`image_worker/worker.py` (`ImageWorker`)** — The core processing engine. Runs five detection methods sequentially on each submission. +- **`plag_checker/submissions_checker.py` (`SubmissionChecker`)** — Orchestrates message consumption, calls the `ImageProcessor`, handles ACK/NACK logic with the `MessageAckManager`, and manages retries. +- **`api/api.py`** — FastAPI HTTP server. Provides REST endpoints for direct submission creation, result polling, and a health check endpoint at `GET /health`. +- **`mq/rmq_client.py` (`RabbitMQClient`)** — Async RabbitMQ client using `aio_pika`. Handles connection, queue declaration, message publishing, retry with exponential backoff, and dead letter queue support. +- **`database/db_manager.py`** — `asyncpg`-based PostgreSQL connection pool manager. + +**External calls made by tap_plg:** + +| External service | What for | Auth method | +|---|---|---| +| RabbitMQ (CloudAMQP) | Consume from `submission_queue`; publish to `plagiarism_feedback` | AMQP credentials | +| Google Cloud Storage | Download submitted images for processing | GCS credentials | +| PostgreSQL (own DB) | Store and query submission hashes and CLIP embeddings | DB credentials | +| **tap_lms HTTP API** | **Fetch assignment context and reference images per assignment** | `FRAPPE_API_KEY` + `FRAPPE_API_SECRET` | + +--- + +### 3.3 rag_service (RAG Feedback Generation — Frappe app) + +**What it is:** A Frappe application that receives the plagiarism-checked submission result from tap_plg and uses a Large Language Model (LLM) to generate personalised, rubric-based feedback for the student. + +**Key components:** + +- **`core/feedback_handler.py` (`FeedbackHandler`)** — Orchestrates the full feedback generation flow. +- **`core/feedback_service.py` (`FeedbackService`)** — Generates feedback; routes by plagiarism result. +- **`core/assignment_context_manager.py` (`AssignmentContextManager`)** — Makes HTTP calls back to tap_lms; implements caching. +- **`core/llm_providers.py`** — Three LLM provider implementations (OpenAI, TogetherAI, Vertex AI). Runtime configurable via `LLM Settings` DocType. + +**External calls made by rag_service:** + +| External service | What for | +|---|---| +| RabbitMQ (CloudAMQP) | Consume from `plagiarism_feedback`; publish to `feedback_results_queue` | +| tap_lms HTTP API | Fetch assignment context and student details | +| OpenAI / TogetherAI / Vertex AI | LLM feedback generation | + +--- + +### 3.4 Glific (WhatsApp Communication Platform — third-party open source) + +**What it is:** Glific is an open-source two-way WhatsApp communication platform. TAP uses a hosted instance as the student-facing communication layer. TAP does not own or modify Glific's codebase. + +**How TAP uses Glific:** Students interact via WhatsApp; Glific flows call tap_lms API endpoints for data operations. When tap_lms needs to notify a student, it calls Glific's GraphQL API. Glific logs stream to BigQuery in real time — no additional export is needed for monitoring. + +--- + +### 3.5 tap_ai (Conversational AI Engine — Frappe app) + +**What it is:** A separate Frappe application that provides a conversational AI layer over TAP LMS data, supporting text and voice queries via intelligent routing (Knowledge Bank, Text-to-SQL, Vector RAG, Direct LLM). Hosted at `ai.evalix.xyz`. Uses RabbitMQ workers, Pinecone, and a remote PostgreSQL at `data.evalix.xyz`. + +--- + +## 4. The Summer Program — Architecture Deep Dive + +### 4.1 What it is and why it exists separately + +The Summer Program is a time-bounded intensive program that must **proactively drive each student** through a week-by-week journey: deliver content on a schedule, escalate with nudges if the student goes silent, track whether they are keeping up or falling behind, eventually drop students who never engage, and graduate those who complete — all automatically, at the individual student level, across potentially 100,000 concurrent students. + +### 4.2 The ProgramEnrollment state machine + +Every Summer Program student gets a `ProgramEnrollment` (PE) Frappe document. Core fields: + +| Field | What it holds | +|---|---| +| `resolved_flow_state` | Which stage of the week the student is in (11 possible states) | +| `current_week` | Which week of the program they are on | +| `next_action_at` | **When** the system should next act on this student | +| `next_action_type` | **What** the system should do at that time | +| `program_status` | `active`, `paused`, `completed`, or `dropped` | +| `grace_window_end_at` | Deadline for a late submission before auto-drop | + +The 11 states form a directed graph within each week, with 25 named transitions (T0–T25) in `state_machine.py`. + +### 4.3 The 1-minute dispatcher (pe_dispatcher.py) + +`pe_dispatcher.py` runs **every 1 minute**. It queries all PEs where `next_action_at <= now` and routes each to one of six action handlers: `content_delivery`, `escalation`, `week_advancement`, `feedback_notification`, `grace_check`, `pause_check`. Uses `FOR UPDATE SKIP LOCKED` for parallel safety. Batch size of 1,000 PEs per cycle handles 100K student bursts. + +### 4.4 The escalation chain + +When a student does not respond to content delivery, they enter an escalation sequence. Steps have types (`help_note_a`, `help_note_b`, `voice_note`, `parent_call`) and `hours_after_previous` delays. `parent_call` steps route through `vocallabs.py` for automated phone calls to parents. + +### 4.5 The archetype and A/B experiment system + +Each student is assigned an **archetype** and **experiment arm** at enrollment, creating 8 Glific collections per batch. Collection-level flow triggers reduce API calls from 100,000 per cycle to 8. + +### 4.6 The BatchProgramRun lifecycle + +A `BatchProgramRun` (BPR) manages cohort setup through states: `draft → importing → enrolling → collections_ready → active → completed`. + +### 4.7 Gamification + +Each PE carries points, streaks, and gems synced to Glific contact fields on every transition. + +### 4.8 The flow callback bridge (update_flow_status) + +Every Glific SP flow calls `update_flow_status` on completion, bridging Glific's stateless flow engine to the tap_lms state machine. + +--- + +## 5. Complete User Flow — Student Submission + +The system supports **four submission types**: image, video, audio, and text. All four types are published to the same `submission_queue` — tap_plg processes image submissions; video, audio, and text bypass plagiarism checking by design. + +> **Active endpoint:** `POST /api/method/tap_lms.summer_program.save_submission.save_submission` +> The legacy endpoint `POST /api/method/tap_lms.imgana.submission.assignment_submission` (`imgana/submission.py`) is **deprecated** and should not be used in new Glific flows. + +``` +Step 1 — Student sends artwork +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Student sends image via WhatsApp + └── Glific receives the message + └── Glific calls tap_lms API: + POST /api/method/tap_lms.summer_program.save_submission.save_submission + { assignment_id, student_id, submission } + +Step 2 — tap_lms processes the submission +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +tap_lms (save_submission.py): + ├── Authenticates the API key (Authorization header + active ProgramEnrollment check) + ├── Validates student and assignment; guards against Glific placeholder strings + ├── Creates a new Submission DocType record (status: "Pending") + ├── Downloads media from Glific; uploads to Google Cloud Storage + └── Calls enqueue_submission(submission.name) + └── Publishes JSON message to [submission_queue] + └── Returns { submission_id, student_id, status } to Glific + +Step 3 — tap_plg detects plagiarism +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +tap_plg (SubmissionChecker): + ├── Consumes message from [submission_queue] + └── ImageWorker runs 5 detection steps sequentially + └── Publishes result to [plagiarism_feedback queue] + +Step 4 — rag_service generates feedback +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +rag_service (FeedbackHandler): + ├── Consumes message from [plagiarism_feedback queue] + ├── Routes by result: AI-generated / plagiarised → stock feedback; original → LLM call + └── Publishes to [feedback_results_queue] + +Step 5 — tap_lms delivers feedback to student +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +tap_lms (feedback_consumer.py): + ├── Consumes message from [feedback_results_queue] + ├── Updates Submission DocType (status: "Completed") + ├── Sends Glific WhatsApp notification + ├── Advances Summer Program state machine (T12 transition) + └── Calls ElevenLabs TTS; uploads audio to GCS + +Step 6 — Student requests and receives feedback +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Glific calls: + POST /api/method/tap_lms.summer_program.save_submission.ready_to_receive_feedback + └── Marks feedback as requested; triggers FeedbackConsumer flow if ready + GET /api/method/tap_lms.summer_program.save_submission.get_submission_feedback + └── Returns { status, overall_feedback, audio_feedback_url } + +Student receives on WhatsApp: + ├── Text feedback in their local language + └── Audio feedback (voice message) +``` + +--- + +## 6. Complete User Flow — Quiz Assessment + +Quizzes are short assessments (typically 3–5 questions) delivered inline through the WhatsApp/Glific flow as part of a learning unit. Unlike submissions, quizzes are **entirely synchronous and self-contained within tap_lms** — no RabbitMQ, no GCS, no tap_plg or rag_service involvement. + +**Active file:** `tap_lms/summer_program/student_progression_sp.py` +**Deprecated file:** `tap_lms/journey/student_progression.py` — contains one-line shims pointing to the above; do not use. + +**Doctypes involved:** `Quiz`, `QuizQuestion`, `QuizOption` (+ translation variants), `StudentQuizAttempt`, `StudentQuizAnswer` + +``` +Step 1 — Quiz initiated +━━━━━━━━━━━━━━━━━━━━━━━ +Glific calls: + POST /api/method/tap_lms.summer_program.student_progression_sp.start_quiz + { student_id, course_level, quiz_id, language } + +tap_lms (start_quiz): + ├── Resolves student_id → Student doc + ├── Fetches active ProgramEnrollment (must be active or paused) + ├── Creates StudentQuizAttempt (status: "in_progress") + │ attempt_number tracks re-attempts for the same quiz + ├── Loads all questions via _get_quiz_questions(quiz_doc) + └── Returns first question as flat key/value response (Glific Rule 2): + { quiz_attempt_id, total_questions, question_index=1, + question_text, option_a, option_b, option_c, option_d } + ✦ emit: quiz_started + + If a prior in-progress attempt exists → _resume_quiz(): + └── Returns the next unanswered question + ✦ emit: quiz_resumed + +Step 2 — Student answers each question (one call per question) +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +Glific calls for each answer: + POST /api/method/tap_lms.summer_program.student_progression_sp.submit_answer + { student_id, quiz_attempt_id, question_index, answer } + answer: one of A, B, C, D + +tap_lms (submit_answer): + ├── Validates attempt ownership and status + ├── Looks up correct_option from cached QuizQuestion + ├── Records StudentQuizAnswer child row on the attempt + │ fields: selected_option, correct_option, is_correct, + │ started_at, answered_at, time_spent_seconds + ├── Updates attempt.correct_answers (running total) + │ + ├── If more questions remain: + │ └── Returns next question in same response + │ { status: "next_question", question_index, question_text, + │ option_a..d, progress_answered, progress_correct } + │ ✦ emit: quiz_answer_submitted (with was_correct, time_spent_seconds) + │ + └── If last question (question_index == total_questions): + └── Calls _complete_quiz_sp() inline — no separate API call needed + +Step 3 — Quiz completion (auto-triggered on last answer) +━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +tap_lms (_complete_quiz_sp): + ├── Computes score = correct_answers / total_questions × 100 + ├── Determines passed = score >= quiz.passing_score + ├── Sets attempt.status = "passed" or "failed" + ├── Awards points via gamification hook (stored as attempt.points_earned) + ├── Advances Summer Program state machine: + │ Core quiz FAIL → advance to next content (no remedial switch) + │ Remedial quiz FAIL → restart or continue remedial LU + │ Remedial quiz PASS → advance (exit remedial for next week) + └── Returns final result as flat response: + { status: "quiz_passed" | "quiz_failed", + score, correct_answers, total_questions, + points_earned, feedback_message } + ✦ emit: quiz_completed (score, passed, correct_answers, points_earned, + time_spent_seconds, attempt_number) +``` + +**Key design decisions:** +- **One API call per question** — Glific sends answers one at a time as the student responds in WhatsApp. There is no "batch submit all answers" path. +- **Server-side time tracking** — `started_at` / `answered_at` are recorded by tap_lms, not sent by Glific, so time-per-question is tamper-proof. +- **In-process question cache** (`cached_question_details`) — `QuizQuestion` docs are immutable after publishing; they are cached per process lifetime to avoid repeated DB reads during a quiz session. +- **Re-attempt support** — `start_quiz` creates a new `StudentQuizAttempt` with an incremented `attempt_number` if a prior completed attempt exists. If a prior *in-progress* attempt exists, it resumes from the last unanswered question. +- **Structured logging** — All four key events (`quiz_started`, `quiz_resumed`, `quiz_answer_submitted`, `quiz_completed`) emit structured JSON logs via `tap_lms/monitoring.py`, consistent with the rest of the Summer Program module. + +--- + +### Timeline view of a single submission + +``` +T+0s Student sends image on WhatsApp +T+1s Glific calls tap_lms save_submission API +T+4s tap_lms: Message published to [submission_queue] +T+5s tap_plg: Message consumed +T+25s tap_plg: All 5 detection steps complete +T+26s tap_plg: Result published to [plagiarism_feedback queue] +T+27s rag_service: Message consumed +T+60s rag_service: LLM feedback generated +T+62s rag_service: Result published to [feedback_results_queue] +T+63s tap_lms: Message consumed by feedback_consumer +T+65s tap_lms: Glific notification triggered +T+85s tap_lms: Audio uploaded to GCS, Submission updated +``` + +### What can go wrong at each handoff + +| Handoff point | Common failure modes | +|---|---| +| Glific → tap_lms | API key invalid; student not found; network timeout | +| tap_lms → GCS | Credentials expired; bucket permissions; large file timeout | +| tap_lms → RabbitMQ | Connection limit hit; queue full; credentials expired | +| RabbitMQ → tap_plg | Consumer not running; worker crashed; CLIP model OOM | +| tap_plg → RabbitMQ | Publish failure after long processing; connection dropped | +| RabbitMQ → rag_service | Consumer not running | +| rag_service → LLM provider | Rate limit; API key expired; model timeout | +| RabbitMQ → tap_lms consumer | Consumer not running; message rejected → DLQ | +| tap_lms → Glific | Token expired; student not in Glific; flow not found | + +--- + +## 7. External Integrations Map + +| Integration | Used by | Protocol | Risk if down | +|---|---|---|---| +| RabbitMQ (CloudAMQP) | All three services | AMQP | **Critical** — entire pipeline stalls | +| CloudAMQP Management API | tap_lms (DLQ monitor) | HTTPS REST | Low — monitoring only; does not affect pipeline | +| Google Cloud Storage | tap_lms, tap_plg | HTTPS | **Critical** — submissions cannot be stored or processed | +| Glific API | tap_lms | HTTPS (GraphQL) | **High** — students receive no notifications or feedback | +| ElevenLabs API | tap_lms | HTTPS | **Medium** — audio feedback unavailable; text feedback still delivered | +| OpenAI / TogetherAI / Vertex AI | rag_service | HTTPS | **High** — feedback generation fails for original submissions | +| tap_lms HTTP API | rag_service, tap_plg | HTTPS | **High** — assignment context unavailable | +| PostgreSQL (tap_plg's own) | tap_plg | TCP | **Critical** — plagiarism check cannot run | + +--- + +## 8. Current Observability Gaps — Per Service + +### 7.1 tap_lms + +| What is happening | Is it visible? | Where logs go | +|---|---|---| +| Submission received from Glific | No structured log | — | +| Image uploaded to GCS | Plain text via `frappe.logger()` | Rotating file on disk | +| Message published to RabbitMQ | Plain text via `frappe.logger()` | Rotating file on disk | +| Feedback result consumed from RabbitMQ | Plain text via `frappe.logger()` | Rotating file on disk | +| Glific notification sent / failed | Plain text via `frappe.logger()` | Rotating file on disk | +| ElevenLabs TTS called | Plain text via `frappe.logger()` | Rotating file on disk | +| pe_dispatcher cycle ran | No structured log | — | +| pe_dispatcher cycle errors | No structured log | — | +| HTTP request latency | No log | — | +| Unhandled exceptions | Frappe Error Log (DB only) | Frappe's own DB table | +| **DLQ depth** | **Not monitored** | **—** | + +**Root cause:** `frappe.logger()` writes to rotating `.log` files under `frappe-bench/logs/`. These are not shipped to GCP Cloud Logging. They are readable only via SSH on the VM and cannot be queried, alerted on, or visualised. + +### 7.2 rag_service + +| What is happening | Is it visible? | Where logs go | +|---|---|---| +| Submission message consumed | `print()` to stdout | Unstructured, not shipped | +| LLM call started / completed | `print()` to stdout | Unstructured, not shipped | +| Feedback generated successfully | `print()` to stdout | Unstructured, not shipped | +| Result published to tap_lms queue | `print()` to stdout | Unstructured, not shipped | + +**Root cause:** The codebase uses `print()` throughout rather than structured logging. + +### 7.3 tap_plg + +| What is happening | Is it visible? | Where logs go | +|---|---|---| +| Submission consumed from queue | `logging.info()` — plain text | Docker container stdout | +| Per-step processing duration | **No timing logged** | — | +| DB pool exhaustion | `logging.error()` — plain text | Docker container stdout | + +**Root cause:** tap_plg has the best logging foundation (Python `logging` module) but output is plain text, not JSON. + +### 7.4 Glific + +Glific logs stream to BigQuery in real time. The gap is on the tap_lms side — there is no corresponding structured export from tap_lms, so the two sides cannot currently be joined. + +### 7.5 tap_ai + +tap_ai has a `ai_query_log` DocType but it is not exported to GCP Cloud Logging. Worker processes use print statements. + +--- + +## 9. What a Stuck Submission Looks Like Today + +A support team member receives a complaint: "Student ST-00123 submitted their artwork 2 hours ago and has not received any feedback." + +**Current investigation process:** + +1. SSH into the VM +2. Open Frappe desk, search for the Submission record — status shows "Pending" +3. `grep "ST-00123" /home/frappe/frappe-bench/logs/*.log` — likely returns nothing +4. Check CloudAMQP management UI — see queue depths but no per-message tracking +5. SSH into the tap_plg container — check Docker logs — unstructured, hard to search +6. Check rag_service Frappe Bench logs — similar problem +7. No way to determine which of the 6 handoff points the submission is stuck at + +**Time to identify the root cause: 30–60 minutes minimum.** + +--- + +## 10. What a Stuck Submission Looks Like After Monitoring + +**Cloud Logging query (takes 5 seconds):** + +``` +jsonPayload.submission_id="SUB-2024-00123" +``` + +**Result — all events in chronological order:** + +``` +2024-11-15 10:00:01 INFO tap_lms submission_published student_id=ST-00123 +2024-11-15 10:00:03 INFO tap_plg plg_submission_received student_id=ST-00123 +2024-11-15 10:00:09 ERROR tap_plg detection_step_failed step=ai_detection error="CUDA OOM" +``` + +**Conclusion in 5 seconds:** The CLIP/AI detection step crashed with an OOM error. The submission is stuck at step 3. Restart the tap_plg worker container; the message will be requeued automatically. + +--- + +## 11. Risks and Notable Code Issues Found + +### 10.1 Hardcoded student ID — Resolved (Internal only) + +**File:** `tap_lms/imgana/submission.py` — `submit_artwork_internal()` + +Confirmed by the client: `submit_artwork_internal()` is an **internal testing endpoint only** and is not exposed in production. No action required. + +### 10.2 Versioned API files — Resolved (Legacy code) + +Date-stamped files (`api_19_11_2025.py`, `api_28_11_25.py`) are legacy code. Only `api.py` is in use. + +### 10.3 RabbitMQ consumer for rag_service is a CLI command — Resolved (Testing only) + +`bench execute start-rag-consumer` is used for testing purposes only. Production consumer mechanism to be confirmed (OQ-10). + +### 10.4 CLIP model memory footprint — Resolved (Separate higher-RAM VM) + +tap_plg runs on a dedicated VM with higher RAM. Memory pressure from the CLIP model is not a concern in production. + +### 10.5 Dead letter queue exists but is not monitored — Addressed in Section 16 + +Both tap_lms and tap_plg implement DLQs. Messages can accumulate silently indefinitely. This is addressed by the DLQ monitoring implementation in Section 16. + +### 10.6 CloudAMQP connection limits + +The free/low tier of CloudAMQP has hard connection limits. The health check endpoint opens an AMQP connection on every GCP Uptime Check (once per minute). The DLQ poller in Section 16 uses the HTTPS Management API — not an AMQP connection — and does not consume from the connection limit. + +### 10.7 Non-image submissions routed through tap_plg (By design) + +Image-only plagiarism checking is by design. However, all four submission types are currently published unconditionally to `submission_queue`. The routing logic in `enqueue_submission()` should branch by `submission_type` so non-image submissions bypass tap_plg. + +### 10.8 Error classification in feedback_consumer — Addressed in Section 17 + +The current `is_retryable_error()` method uses string pattern matching to decide whether a failed message is retried or sent to the DLQ. This is fragile: unrecognised exception types are silently classified as retryable without any log record of why. See Section 17 for the full analysis and the `classify_error()` replacement design. + +--- + +## 12. Monitoring Implementation Plan — All Three Services + +### What will be added + +#### tap_lms (4 new files, 7 modified files) + +| File | Action | What it adds | +|---|---|---| +| `tap_lms/monitoring.py` | Create | Structured JSON logging core — all emit functions | +| `tap_lms/health.py` | Create | Health endpoint: DB + Redis + RQ workers + RabbitMQ checks | +| `tap_lms/middleware.py` | Create | HTTP request latency, error rate, exception hooks | +| `tap_lms/summer_program/dlq_monitor.py` | Create | CloudAMQP Management API poller; structured log per DLQ | +| `tap_lms/hooks.py` | +4 lines | Register before_request / after_request / on_exception + DLQ monitor cron | +| `tap_lms/summer_program/save_submission.py` | Already instrumented (27 emit calls) | `save_submission_called`, `save_submission_success`, `save_submission_*_error`, `feedback_fetched`, `feedback_requested`, `feedback_flow_triggered` — complete | +| `tap_lms/imgana/submission.py` | Deprecated — not instrumented | Legacy entry point; no new monitoring work required | +| `tap_lms/feedback_handler/feedback_consumer.py` | +4 emit calls | `feedback_result_received`, `feedback_processing_complete`, `feedback_processing_failed` (with failure_reason), `glific_notification_sent` | +| `tap_lms/feedback_handler/feedback_processor.py` | Replace method | Replace `is_retryable_error()` with `classify_error()` returning (bool, failure_reason) | +| `tap_lms/summer_program/pe_dispatcher.py` | Wrap entry point | `dispatcher_cycle` metric: processed / skipped / errors / duration | +| `tap_lms/summer_program/scheduler.py` | Wrap entry point | `background_job` success/error/duration | +| `tap_lms/summer_program/escalation_runner.py` | Wrap entry point | `background_job` success/error/duration | + +#### tap_plg (1 file modified, 1 endpoint exposed) + +| File | Action | What it adds | +|---|---|---| +| `tap_plg/app.py` | Replace log formatter | `StructuredJsonFormatter` — makes all existing `logger.*` calls emit JSON automatically | +| `tap_plg/api/api.py` | Expose existing health check | `GET /health` | +| `tap_plg/plag_checker/submissions_checker.py` | +2 emit calls | `plg_submission_received`, `plg_result_published` | +| `tap_plg/image_worker/worker.py` | +per-step timing | `detection_step_complete` / `detection_step_failed` for each of 5 steps | + +#### rag_service (3 new files, 2 modified files) + +| File | Action | What it adds | +|---|---|---| +| `rag_service/monitoring.py` | **Already exists** | Structured logging core — complete | +| `rag_service/middleware.py` | **Already exists** | HTTP request hooks — complete | +| `rag_service/hooks.py` | **Already registered** | Middleware hooks — complete | +| `rag_service/core/feedback_handler.py` | **Already instrumented** | `rag_submission_received`, `rag_feedback_complete`, `rag_feedback_failed` — complete | +| `rag_service/health.py` | Create | Health endpoint | +| `rag_service/core/feedback_service.py` | +2 emit calls | `llm_call_complete` / `llm_call_failed` | +| `rag_service/core/assignment_context_manager.py` | +1 emit call | `tap_lms_api_call` | + +### The complete traceable log after implementation + +``` +[tap_lms] submission_published ← T+4s +[tap_plg] plg_submission_received ← T+5s +[tap_plg] detection_step_complete ← T+10s hash check: 80ms +[tap_plg] detection_step_complete ← T+12s ai_detection: 120ms +[tap_plg] detection_step_complete ← T+25s pgvector_search: 14000ms +[tap_plg] plg_result_published ← T+26s +[rag_service] rag_submission_received ← T+27s +[rag_service] llm_call_complete ← T+60s provider=gemini, 32000ms +[rag_service] rag_feedback_complete ← T+61s +[tap_lms] feedback_result_received ← T+63s +[tap_lms] feedback_processing_complete ← T+64s +[tap_lms] glific_notification_sent ← T+65s success=true +``` + +### GCP infrastructure changes + +| Change | What it does | +|---|---| +| Install Cloud Ops Agent on each VM | Ships CPU, memory, disk, network metrics automatically | +| Configure Ops Agent to tail Frappe logs | Ships tap_lms and rag_service on-disk logs to Cloud Logging | +| Configure Ops Agent for Docker logs | Ships tap_plg container logs to Cloud Logging | +| GCP Uptime Check (3 endpoints) | Polls health endpoints every 1 minute | +| 5 log-based metrics | `http_error_rate`, `http_latency_ms`, `dispatcher_errors`, `job_failures`, `dlq_depth` | +| 12 alerting policies (Terraform) | Automated alerts for all critical failure modes | + +--- + +## 13. Glific ↔ tap_lms API Visibility — New Monitoring Scope + +### The problem in detail + +Every time a student or teacher interacts with the WhatsApp bot, Glific executes one or more flows. Each flow node that requires LMS data calls a whitelisted tap_lms API endpoint. Currently: + +- tap_lms logs these incoming HTTP requests only as plain text — not queryable, not correlated to `student_id` +- Glific logs are already streaming to BigQuery in real time (confirmed by client) +- There is no single place to see "Glific called tap_lms function X with student_id Y at time T, and tap_lms returned status Z in N milliseconds" + +### What needs to change + +The `before_request` / `after_request` middleware covers all 42+ Glific → tap_lms calls automatically. Log fields must include `endpoint`, `student_id`, `glific_id`, `http_status`, `duration_ms`, and `error_detail`. + +### Recommended single-view correlation approach + +Since Glific logs are already in BigQuery and tap_lms logs can be routed via a Cloud Logging sink, the cleanest single-view correlation is a **BigQuery joined view** joining on `student_id` / `glific_id` with a ±5-second timestamp window. + +### Log retention + +| Table / partition | Retention | +|---|---| +| tap_lms request logs — errors and warnings only | 90 days | +| tap_lms request logs — INFO (200 OK, fast) | 7 days | +| tap_lms pipeline milestones | 365 days | +| Glific BigQuery logs | Per client's existing Glific retention policy | + +--- + +## 14. Local Development Environment and Testing Strategy + +### 14.1 Current local environment (8 containers) + +The `docker-compose.local.yml` defines a complete local development environment: + +| Container | What it replaces | Notes | +|---|---|---| +| `dev-lms` | The Frappe LMS application | tap_lms runs here | +| `postgres` | PostgreSQL for Frappe | | +| `redis-cache`, `redis-queue` | Redis for Frappe cache and RQ | | +| `rabbitmq` | CloudAMQP | Local RabbitMQ with management UI on port 15672 | +| `tap_plg_stub` | Both tap_plg_worker and tap_plg_api | No CLIP/FAISS/GCS — marks every submission "original" | +| `llm-stub` | OpenAI / TogetherAI / Vertex AI | Returns realistic deterministic feedback JSON | +| `glific-stub` | Glific GraphQL API | All 8 Glific operations stubbed | + +### 14.2 The tap_plg stub's current limitation + +`tap_plg_stub` currently marks every submission as `is_plagiarized: false, is_ai_generated: false`. This means integration tests never exercise the plagiarised or AI-generated branches. + +**Recommended improvement:** Add env vars to control result distribution: + +```yaml +tap_plg_stub: + environment: + STUB_PLAGIARISM_RATE: "0.1" + STUB_AI_GENERATED_RATE: "0.05" +``` + +### 14.3 Integrating tap_ai into the local environment + +Full tap_ai local integration requires: an `OPENAI_BASE_URL` redirect to the existing `llm-stub`, a new `pinecone-stub` FastAPI service, a `postgres-ai` container seeded from `seed_local.py`, and tap-ai-workers started inside `dev-lms`. Estimated effort: ~2.75 days. Pending client confirmation. + +--- + +## 15. Open Questions + +Items marked **Resolved** have been answered by TAP. Items marked **Open** are still pending. + +### Infrastructure + +| # | Status | Question | Client Response / Notes | +|---|---|---|---| +| OQ-1 | ✅ Resolved | Does each service run on its own dedicated VM? | Yes | +| OQ-2 | ✅ Resolved | Is the same VM used for production? | No — production uses a different larger configuration | +| OQ-3 | 🔲 Open | Is there a reverse proxy (Nginx, Caddy, or similar) in front of each service? | Client is checking | +| OQ-4 | ✅ Resolved | Does tap_plg have enough RAM for the CLIP model? | Yes — dedicated higher-RAM VM | + +### Queue Configuration + +| # | Status | Question | Client Response / Notes | +|---|---|---|---| +| OQ-5 | 🔲 Open | tap_plg's default queue name is `plagiarism_submissions` — does this match what tap_lms publishes to? | Pending verification | +| OQ-6 | 🔲 Open | Are queue names consistent across all environments? | Pending verification | +| OQ-21 | 🔲 Open | What are the exact DLQ names in each environment? | Required before DLQ name standardisation; verify against CloudAMQP console | + +### Submission Type Routing — Action Required + +| # | Status | Finding | Notes | +|---|---|---|---| +| OQ-7 | ✅ Partially resolved | Confirmed that Image-only plagiarism checking is by design | Code shows all four submission types published unconditionally to `submission_queue`. Routing logic in `enqueue_submission()` needs to branch by `submission_type`. | + +### pe_dispatcher Architecture + +| # | Status | Question | Notes | +|---|---|---|---| +| OQ-8 | 🔲 Open | Has the team considered a trigger-based approach for pe_dispatcher? | 1-minute cron was tuned for 100K-student burst handling. Worth discussing once monitoring data is available. | + +### tap_plg Production Environment — Action Required + +| # | Status | Finding | Notes | +|---|---|---|---| +| OQ-9 | 🔲 Open | `FRAPPE_API_KEY`, `FRAPPE_API_SECRET`, and `FRAPPE_API_BASE_URL` missing from tap_plg env docs | Required by `assigment_ref_images.py`. Must be added to production environment or reference image comparison silently fails. | + +### rag_service Consumer Process Management + +| # | Status | Question | Notes | +|---|---|---|---| +| OQ-10 | 🔲 Resolved | How is the rag_service consumer kept running in production? | If no restart policy exists, a consumer crash goes undetected indefinitely. | + +### Data and Compliance + +| # | Status | Question | Client Response / Notes | +|---|---|---|---| +| OQ-12 | ✅ Resolved | Are there privacy constraints on logging student IDs? | Yes — Cloud Logging access restricted via GCP IAM | +| OQ-13 | 🔲 Open | What is the required log retention period per category? | Recommendation: error logs 30–90 days; pipeline milestone logs 1 year | +| OQ-18 | 🔲 Open | Are there PII fields in Glific's BigQuery webhook log table? | Client to confirm schema | +| OQ-20 | 🔲 Open | What is the exact BigQuery dataset ID for Glific's logs? | Required before the correlation view can be written | + +--- + +## 16. DLQ Monitoring — Detailed Design + +### 16.1 Current State + +Dead letter queues are implemented but completely dark. The three DLQs in the system can accumulate failed messages indefinitely with no alert, no visibility in any dashboard, and no structured log record of depth over time. + +The two monitoring mechanisms that currently exist are inadequate: `feedback_consumer.py` calls `frappe.logger().info()` when it declares the DLQ at startup (plain text, not shipped to Cloud Logging), and `rmq_client.py` calls `publish_to_dlq()` when a message exceeds retries (no structured log, no alert). + +A message in a DLQ represents a student whose submission is permanently stuck. Without monitoring, the first indication is typically a student or teacher reporting that feedback never arrived. + +### 16.2 The Three DLQs + +| DLQ | Declared by | Source of messages | +|---|---|---| +| `{feedback_results_queue}_dead_letter` | `tap_lms/feedback_handler/feedback_consumer.py` | Feedback results that tap_lms could not handle after retries | +| `{plagiarism_results_queue}.dead_letter` | `rag_service/rag_service/utils/rabbitmq_consumer.py` | Plagiarism results rag_service could not process | +| `$DEAD_LETTER_QUEUE` (env var) | `tap_plg/mq/rmq_client.py` | Student submissions tap_plg could not process | + +The naming convention is inconsistent across the three. All should be standardised to `{source_queue}.dead_letter` in a coordinated deploy across all three services. + +> **Note on naming standardisation:** This requires a coordinated deploy. If any service is deployed ahead of the others, old and new queue names temporarily diverge and messages may land on an unconsumed queue. Plan as a single release across all three services. + +### 16.3 Two Complementary Monitoring Mechanisms + +| Mechanism | What it catches | Latency | +|---|---|---| +| Event-driven: `feedback_processing_failed` with `failure_reason` | Individual messages rejected by the consumer, with full classification context | Immediate | +| Periodic depth poll (5 min): CloudAMQP Management API | Depth from before monitoring existed; messages from services not yet emitting structured logs; broker-side routing failures | Up to 5 minutes | + +Together these two mechanisms ensure no DLQ accumulation goes undetected for more than 5 minutes. + +### 16.4 What Happens When a Message Is in a DLQ + +A message in a DLQ is not automatically retried. The operator must act. The three options are: + +**Replay** — move the message back to the source queue. Appropriate if the failure was transient (temporary Glific outage, DB deadlock). Use the `failure_reason` from the event-driven log to confirm this is safe. The consumer must be idempotent — `feedback_consumer.py`'s `process_message()` is largely idempotent but verify before replaying. + +**Fix and replay** — if the failure was caused by a bug, deploy the fix first, then replay. Replaying into broken code causes the same failure again. + +**Inspect and discard** — if the message is malformed or refers to a submission that no longer exists (`failure_reason="not_found"` or `"invalid_payload"`), inspect the message body in the CloudAMQP console and discard. The affected student will need to be identified and manually re-triggered. + +The `failure_reason` field in the event-driven log (Section 17) tells the operator which path is appropriate without requiring them to inspect the message body first. + +### 16.5 The CloudAMQP Management API + +The poller uses the RabbitMQ Management Plugin HTTP API, enabled on all CloudAMQP plans including the free tier. + +**Base URL:** Available in the CloudAMQP console under "Details". Format: `https://your-instance.cloudamqp.com`. This is an HTTPS endpoint on port 443 — different from the AMQP connection string. One new field (`management_api_url`) is required in the `RabbitMQ Settings` DocType to store this URL. + +**Per-queue endpoint:** +``` +GET https://{management_api_url}/api/queues/{vhost}/{queue_name} +Authorization: Basic {base64(username:password)} +``` + +The same `username` and `password` from `RabbitMQ Settings` are reused. The virtual host must be URL-encoded (`/` becomes `%2F`). + +**Key response fields:** +```json +{ + "messages": 3, + "consumers": 0 +} +``` + +`consumers` being 0 on a DLQ is expected — nothing actively consumes the DLQ by design. A non-zero `messages` value is the alert signal. + +The DLQ poller makes three HTTPS requests per 5-minute cycle. These are REST calls, not AMQP connections, and do not consume from the CloudAMQP connection limit. + +### 16.6 Section 11.5 — Resolved + +The risk noted in Section 10.5 (dead letter queue exists but is not monitored) is addressed by this implementation. Once `dlq_monitor.py` is deployed and the `dlq_depth` log-based metric and alert are configured, the risk is resolved. + +--- + +## 17. Error Classification — Retryable vs Non-Retryable Failures + +### 17.1 Current State and the Problem + +When `feedback_consumer.py` fails to process a message, it calls `feedback_processor.is_retryable_error()` to decide whether to requeue the message (retryable) or route it to the dead letter queue (non-retryable). + +The current implementation: + +```python +non_retryable_patterns = [ + "does not exist", "not found", "invalid", "permission denied", + "duplicate", "constraint violation", "missing submission_id", + "missing feedback data", "validation error", +] +return not any(pattern in error_str for pattern in non_retryable_patterns) +``` + +This has three problems: + +**Problem 1 — Misclassification risk.** The matching is substring-based against the exception message string. An exception containing "invalid" anywhere — even incidentally, like `"invalid state transition after database reconnect"` — is classified as non-retryable and goes straight to the DLQ, even if the root cause is transient. Conversely, an exception with an unusual message (e.g. a PostgreSQL `FATAL: remaining connection slots are reserved`) doesn't match any pattern and is classified as retryable — which is the right call, but it's accidental, not intentional. + +**Problem 2 — No diagnostic information in the log.** The `feedback_processing_failed` structured log includes `retryable=true/false` but gives no information about *why* it was classified that way. When the alert fires, the operator must inspect the CloudAMQP message body to understand what happened — a slower path than reading it from the log. + +**Problem 3 — Unknown failures are invisible.** Any exception whose message doesn't match a known pattern is silently classified as retryable. There is no way to know from the logs that novel failure modes are accumulating. + +### 17.2 The Fix: `classify_error()` + +`is_retryable_error()` is replaced by `classify_error()`, which returns both the retryable boolean and a `failure_reason` string. The `failure_reason` is emitted in the structured log alongside the error and `retryable` flag, making every failure immediately actionable from the log alone. + +The method maintains an explicit non-retryable section (same patterns as today, now organised by named reason) and an explicit retryable section. Anything that doesn't match either becomes `failure_reason="unknown"` and defaults to retryable — the same safe default as today, but now visible. + +### 17.3 What `failure_reason` Values Mean in Practice + +This table is the operator's decision guide when a `feedback_processing_failed` alert fires: + +| `failure_reason` | `retryable` | What it means | Right operator response | +|---|---|---|---| +| `not_found` | false | Submission doc was deleted or never created in tap_lms | Investigate data integrity; do not replay | +| `invalid_payload` | false | Message from rag_service is malformed JSON or missing required fields | Bug in rag_service serialisation; fix and redeploy before replaying | +| `validation_error` | false | Frappe validation rejected the Submission doc update | Schema mismatch or bad data; inspect message body | +| `db_constraint` | false | Duplicate write attempt on a unique field | Message likely already processed; safe to discard | +| `db_connection` | true | PostgreSQL was temporarily unreachable | Transient; will requeue automatically; confirm DB health | +| `timeout` | true | External call (Glific, ElevenLabs) timed out | Transient; will requeue; verify external service status | +| `broker_error` | true | RabbitMQ channel or connection error during ack/nack | Transient; check broker health | +| `unknown` | true | Exception message did not match any known pattern | **Investigate before deciding whether to replay** — this label signals the classification list needs updating | + +### 17.4 The `unknown` Bucket as a Continuous Improvement Signal + +The `unknown` bucket is not a failure of the classification system — it is a deliberate signal. A Cloud Logging query for `failure_reason="unknown"` surfaces exception types that have appeared in production but are not yet categorised. Over time, the operator reviews these, adds the appropriate patterns to the non-retryable or retryable sections of `classify_error()`, and the classification becomes more precise. + +This is preferable to expanding the non-retryable pattern list aggressively upfront, which risks misclassifying legitimate transient failures as permanent. + +### 17.5 Impact on the `feedback_processing_failed` Structured Log + +The full structured log payload for a non-retryable failure after this change: + +```json +{ + "severity": "ERROR", + "message": "feedback_processing_failed", + "app": "tap_lms", + "submission_id": "SUB-2024-00123", + "student_id": "ST-00456", + "error": "Submission SUB-2024-00123 does not exist", + "error_type": "ValueError", + "retryable": false, + "failure_reason": "not_found", + "retry_count": 1, + "timestamp": "2024-11-15T10:05:32Z" +} +``` + +For a transient retryable failure: + +```json +{ + "severity": "ERROR", + "message": "feedback_processing_failed", + "app": "tap_lms", + "submission_id": "SUB-2024-00456", + "student_id": "ST-00789", + "error": "could not connect to server: Connection refused", + "error_type": "OperationalError", + "retryable": true, + "failure_reason": "db_connection", + "retry_count": 2, + "timestamp": "2024-11-15T10:06:14Z" +} +``` + +The operator can read the second log and know immediately: this is a DB connection error, it will requeue automatically, check the PostgreSQL service health. No CloudAMQP console inspection required. + +--- + +*Document prepared for client validation. Version 1.6. All findings are based on static code analysis of the five provided codebases and client responses received through June 2026.* diff --git a/docs/local_docker_setup.md b/docs/local_docker_setup.md index 1bab8e57..e00ba6ab 100644 --- a/docs/local_docker_setup.md +++ b/docs/local_docker_setup.md @@ -187,3 +187,136 @@ docker compose --env-file env.local -f docker/local/docker-compose.yml down -v ``` Only use the reset command when you are comfortable deleting the local Docker database and bench volumes. + +## 8. Manual testing cheat sheet + +These are the commands needed for a typical round of manual Summer Program testing: creating an assignment, enrolling a student in a program, resetting a prior submission, and making sure the student's batch is actually active. + +Open a bench console first: + +```sh +docker compose --env-file env.local -f docker/local/docker-compose.yml exec dev bash -lc "cd /home/frappe/frappe-bench && bench --site tap_lms.localhost console" +``` + +Everything below is pasted into that console. + +### 8.1 Create an assignment (if it doesn't already exist) + +`Assignment.autoname` is `format:{assignment_name}-{difficulty_tier}`, so the doc name is predictable and you can check existence directly by name: + +```python +assignment_name = "MockAssign" +difficulty_tier = "Basic" # Remedial | Basic | Intermediate | Advanced +assign_id = f"{assignment_name}-{difficulty_tier}" + +if not frappe.db.exists("Assignment", assign_id): + a = frappe.new_doc("Assignment") + a.assignment_name = assignment_name + a.difficulty_tier = difficulty_tier + a.assignment_type = "Written" # Written | Practical | Performance | Collaborative + a.max_score = "10" + a.insert(ignore_permissions=True) + frappe.db.commit() + print(f"created {a.name}") +else: + print(f"already exists: {assign_id}") +``` + +### 8.2 Enroll a student in a program + +Use `create_test_student_with_pe` from `tap_lms.summer_program.dev_tools` rather than creating `Student`/`ProgramEnrollment` docs by hand. It's idempotent (reuses an existing `Student` on `phone`+`name1`, and an existing active/paused `ProgramEnrollment` on `student`+`batch`), and by default skips real Glific HTTP calls: + +```python +from tap_lms.summer_program.dev_tools import create_test_student_with_pe + +result = create_test_student_with_pe( + name="Test Student", + phone="9876543210", + batch="palv2-test-BT52231", # Batch doc name -- must already exist + archetype="submitter", # must be in ALL_ARCHETYPES + experiment_arm="default", # must be in ALL_ARMS +) +print(result) # {"student_id": ..., "pe_name": ..., "created_student": bool, "created_pe": bool, ...} +``` + +### 8.3 Reset a prior submission + +Use `reset_pe_to_state_0`, not a hand-written `frappe.db.set_value` call -- it resets every state-machine field, counter, grace window, and gamification field the real reset needs (a partial manual reset has caused real drift bugs in the past). Try `dry_run=True` first to see the diff before writing: + +```python +from tap_lms.summer_program.dev_tools import reset_pe_to_state_0 + +reset_pe_to_state_0("ST00062543", dry_run=True) +reset_pe_to_state_0("ST00062543", push_to_glific=False) +``` + +The reset deliberately **preserves** `Submission` rows (so feedback quality can be compared across reset cycles), so delete those separately for a true clean slate: + +```python +frappe.db.delete("Submission", {"student_id": "ST00062543", "assign_id": "MockAssign-Basic"}) +frappe.db.commit() +``` + +### 8.4 Check the student's batch is active + +A `BatchProgramRun` (BPR) that isn't `status = "active"` won't deliver content -- if a student stops progressing for no obvious reason, check this before anything else: + +```python +status = frappe.db.get_value("BatchProgramRun", "hsugtupp28", "status") +print(status) +``` + +If it isn't `"active"`, check `validation_status`, then activate: + +```python +validation_status = frappe.db.get_value("BatchProgramRun", "hsugtupp28", "validation_status") +print(validation_status) # must be "passed" for the next call to work + +from tap_lms.summer_program.batch_activation import activate_bpr +result = activate_bpr("hsugtupp28") +print(result) +``` + +If `validation_status` isn't `"passed"`, don't call `validate_bpr()` blindly -- it requires `status == "collections_ready"`, which a previously-active BPR won't have, so it will just fail with a confusing status error. Inspect the BPR's `validation_report` field manually first. + +Activation doesn't fire content delivery immediately -- that only happens via the Tuesday 09:00 IST cron, or by triggering it manually for just this batch (see below). It also enqueues a background job to populate the `main` Glific collection, so check that finished before assuming a student will actually receive anything: + +```python +frappe.db.sql(""" + SELECT collection_label, glific_group_id, member_count + FROM "tabPGCollection" + WHERE parent = %s AND kind = 'main' +""", ("hsugtupp28",), as_dict=True) +``` + +### 8.5 Make sure the student is actually in the batch's Glific group + +`create_test_student_with_pe` defaults to `skip_glific_sync=True`, so a freshly created test student is **not** added to any Glific group, including the batch's `main` collection. Check and fix if needed: + +```python +student_id = "ST00062543" +glific_id = frappe.db.get_value("Student", student_id, "glific_id") +print(glific_id) + +from tap_lms.glific_integration import add_contact_to_group +add_contact_to_group(contact_id=glific_id, group_id="20529") # main collection's glific_group_id +``` + +### 8.6 Trigger content delivery outside the Tuesday cron (optional) + +Fire delivery for just this batch, rather than `weekly_content_delivery_trigger()`, which loops over **every** active BPR on the site and would affect other people's test batches too: + +```python +from tap_lms.summer_program.glific_extensions import start_group_flow + +flow_id = frappe.db.get_value("BatchProgramRun", "hsugtupp28", "content_delivery_flow") +start_group_flow(flow_id=str(flow_id), group_id="20529") +``` + +### 8.7 All of the above in one script + +`tap_lms/summer_program/test_setup_script.py` wraps 8.1-8.5 into a single `run_test_setup(...)` call. Run it via `bench execute` without opening a console at all: + +```sh +docker compose --env-file env.local -f docker/local/docker-compose.yml exec dev bash -lc "cd /home/frappe/frappe-bench && bench --site tap_lms.localhost execute tap_lms.summer_program.test_setup_script.run_test_setup --kwargs '{\"assignment_name\": \"MockAssign\", \"difficulty_tier\": \"Basic\", \"student_name\": \"Test Student\", \"student_phone\": \"9876543210\", \"batch\": \"palv2-test-BT52231\", \"bpr_name\": \"hsugtupp28\", \"reset_existing\": true}'" +``` diff --git a/docs/log-config.md b/docs/log-config.md new file mode 100644 index 00000000..abf573ed --- /dev/null +++ b/docs/log-config.md @@ -0,0 +1,141 @@ +# System Operations: Logging, Rotation & Security Hardening + +This document outlines the setup, architecture, and maintenance operations for application logging, cloud log harvesting, and gateway security controls. + +--- + +## 1. Structured Logging Setup (GCP Cloud Logging) + +To trace issues effectively using Google Cloud Platform (GCP) monitoring dashboards, application logs must be emitted as raw, un-prefixed JSON objects. Writing to standard output (`stdout`) via Gunicorn introduces process tracking prefixes that break GCP's JSON parsing engine [STEM]. + +### Implementation +We utilize a dedicated, isolated logging channel that bypasses Gunicorn streams and writes directly to disk under the active bench's `logs/` directory. It automatically extracts unique request context identifiers from Frappe’s thread-local memory to allow cross-log correlation [STEM]. Refer [monitoring.py](../tap_lms/monitoring.py) + + +### Usage Example +Developers can call this function from **any** nesting depth without altering intermediate function parameters or passing request arguments manually: +```python +@frappe.whitelist(allow_guest=True) +def save_submission(submission_id): + # Business logic execution... + emit_structured_log("INFO", "submission_processed", submission_id=submission_id) +``` + +--- + +## 2. Automated Log File Rotation + +We use RotationLogger from the logger library to rotate the log files once they reach 10MB and to keep last 5 files available incase required. An alternative to this approach, suggested by Gemini, is to use the below (NOTE: Do not use both these methods at the same time): + +Because the structured log grows continuously in production, we use Linux's native `logrotate` engine with a `copytruncate` directive [STEM]. This ensures logs are safely truncated without forcing a restart of Gunicorn or dropping active TCP/HTTP client connections [STEM]. + +By omitting explicit user and group names from the `create` directive, `logrotate` dynamically inspects who owns the existing log file and duplicates those exact permissions on the new, empty file [STEM]. This prevents the file from being hijacked by the `root` user context. + +### Configuration Procedure +1. Create a dedicated rotation configuration file: + ```bash + sudo nano /etc/logrotate.d/frappe-gcp-structured + ``` + +2. Paste the following user-agnostic configuration rules: + ```text + /home/*/frappe-bench/logs/gcp_structured.log { + daily + missingok + rotate 14 + compress + delaycompress + notifempty + copytruncate + create 0664 + } + ``` + +3. Validate the layout format configuration using a dry-run test: + ```bash + sudo logrotate -d /etc/logrotate.d/frappe-gcp-structured + ``` + +--- + +## 3. Security Hardening (Nginx Access Control) + +Automated vulnerability scanners routinely query paths looking for standard environment configurations (e.g., `/aws_credentials.env`, `.env.staging`). Frappe catches these requests and returns a custom webpage with an HTTP `200 OK` network status code, triggering false alarms in security monitoring agents [STEM]. We drop these requests instantly at the firewall boundary layer. + +### Backup Rule Location +Always keep a backup of the Nginx configuration snippet inside your app repository directory so it can be easily recovered if a deployment overwrites the active web server files: +`apps/tap_lms/deployment_configs/nginx_security.conf` + +```nginx +# ========================================================================= +# SECURITY HARDENING: Explicitly block credential harvesting bot requests +# ========================================================================= +location ~* \.(env|env\..*|aws_credentials|git|bak|sql)\$ { + log_not_found off; + access_log off; + return 404; +} +``` + +### Active System Configuration Procedure +1. Open the primary active site Nginx configuration file: + ```bash + sudo nano /home/gcp-data/frappe-bench/config/nginx.conf + ``` + +2. Paste the security rule block inside the main `server { ... }` block block matching your application routing directives. + +3. Verify that Nginx passes down the correlation ID header to Gunicorn within the `location /` section: + ```nginx + proxy_set_header X-Request-Id \$request_id; + ``` + +4. Audit the file syntax rules and cycle the daemon live: + ```bash + sudo nginx -t + sudo systemctl reload nginx + ``` + +--- + +## 4. Manual Production Deployment Runbook + +Because this environment relies on manual deployments without an automated CI/CD pipeline, follow this exact sequence whenever pulling new code updates to ensure security configurations and logging systems are not broken or overwritten. Also refer to [Ops Agent Setup doc](./ops_agent_setup.md) for more details. + +### Step-by-Step Manual Release Sequence: + +1. **Pull the latest repository updates:** + ```bash + cd /home/gcp-data/rjs/frappe_tap + git fetch --all + git checkout main # Or your active target branch + git pull origin main + ``` + +2. **Re-sync system environment structures:** + ```bash + cd /home/gcp-data/frappe-bench + bench setup requirements + bench --site your_site_name migrate + ``` + +3. **CRITICAL: Restore Nginx Security Configuration:** + If a teammate ran `bench setup nginx` during this deployment window, your gateway security rules were overwritten. Open `/home/gcp-data/frappe-bench/config/nginx.conf` and ensure your custom `location` security block is manually appended back inside the primary `server { ... }` configuration space. + +4. **Verify and Reload Gateway Operations:** + ```bash + sudo nginx -t + sudo systemctl reload nginx + ``` + +5. **Clear Application Context & Restart Workers:** + ```bash + bench clear-cache + sudo supervisorctl restart frappe-bench-web:* + ``` + +6. **Verify Live Log Streams:** + Confirm that your tracking pipelines are clean and streaming: + ```bash + tail -n 20 -f /home/gcp-data/frappe-bench/logs/gcp_structured.log + ``` diff --git a/docs/ops_agent_setup.md b/docs/ops_agent_setup.md new file mode 100644 index 00000000..9466f86f --- /dev/null +++ b/docs/ops_agent_setup.md @@ -0,0 +1,584 @@ +# GCP Ops Agent Setup — TAP LMS Server + +This document covers installing and configuring the GCP Ops Agent on the +TAP LMS server to ship structured application logs to Cloud Logging and +hardware metrics (CPU, memory, disk, network) to Cloud Monitoring. + +--- + +## Overview + +The Ops Agent is Google Cloud's unified telemetry agent for Compute Engine. +It uses Fluent Bit for log collection and OpenTelemetry for metrics — both +managed via a single YAML config file. + +**What this setup covers:** + +| Source | What gets shipped | +|---|---| +| `gcp_structured.log` | Structured JSON business events (submissions, feedback, errors) | +| `rag_gcp_structured.log` | rag_service structured events (if co-located) | +| `frappe.log`, `worker.log` | Frappe application and worker logs | +| nginx access/error logs | HTTP access and nginx errors | +| `feedback-consumer.log` | Feedback consumer stdout/stderr | +| `supervisor/supervisord.log` | Supervisor process management events | +| Host metrics | CPU, memory, disk, network — automatic, no config needed | + +--- + +## Prerequisites + +- GCP Compute Engine VM (Ubuntu 22.04) +- The VM's service account must have these IAM roles: + - `roles/logging.logWriter` — to write logs to Cloud Logging + - `roles/monitoring.metricWriter` — to write metrics to Cloud Monitoring + +Verify in GCP Console → IAM & Admin → Service Accounts → find the VM's +service account → check roles. If missing, add them. + +--- + +## 1. Install the Ops Agent + +```bash +curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh +sudo bash add-google-cloud-ops-agent-repo.sh --also-install +``` + +Verify it's running: + +```bash +sudo systemctl status google-cloud-ops-agent +``` + +All three sub-services should show `active (running)`: +- `google-cloud-ops-agent.service` +- `google-cloud-ops-agent-fluent-bit.service` +- `google-cloud-ops-agent-otel-collector.service` + +--- + +## 2. Deploy the Config File + +Copy the config from the repo to the Ops Agent config location: + +```bash +sudo cp /home/lms-dev/frappe-bench/apps/tap_lms/deployment_configs/ops_agent_config.yaml \ + /etc/google-cloud-ops-agent/config.yaml +``` + +Validate the config syntax: + +```bash +sudo google-cloud-ops-agent --config /etc/google-cloud-ops-agent/config.yaml --dryrun +``` + +Restart the agent to apply: + +```bash +sudo systemctl restart google-cloud-ops-agent +sudo systemctl status google-cloud-ops-agent +``` + +--- + +## 3. Key Config Decisions + +### severity field promotion + +`monitoring.py` emits `"severity": "ERROR"` as a plain JSON field. The Ops +Agent only promotes a field to `LogEntry.severity` if it is named +`logging.googleapis.com/severity`. Without promotion: + +- Log Explorer shows all entries as grey (no severity colour) +- Alert policies can't filter on `severity=ERROR` +- The severity histogram in the sidebar is empty + +The `promote_severity` processor in the config moves `jsonPayload.severity` +to `logging.googleapis.com/severity` automatically. No changes to +`monitoring.py` are needed. + +### timestamp parsing + +Our logs emit `"timestamp": "2026-07-28T11:25:22.536652"`. The +`parse_tap_lms_json` processor extracts this via `time_key: timestamp` so it +becomes `LogEntry.timestamp` in Cloud Logging rather than the agent's +ingestion time. + +### INFO http_request exclusion + +Every API call emits an `http_request` INFO log (used for P95 latency and +error rate metrics). These are high volume and don't need long-term storage. +The `exclude_http_info` processor drops them before ingestion. + +To keep all HTTP logs, comment out `exclude_http_info` from the pipeline: + +```yaml +tap_lms_pipeline: + receivers: [tap_lms_structured] + processors: [parse_tap_lms_json, promote_severity] # exclude_http_info removed +``` + +--- + +## 4. Hardware Metrics + +Hardware metrics are collected automatically by the built-in `hostmetrics` +receiver — no additional configuration needed. Available immediately after +install. + +View in GCP Console → **Monitoring → Metrics Explorer**: + +| Metric | Filter | +|---|---| +| CPU utilization | `agent.googleapis.com/cpu/utilization` | +| Memory used | `agent.googleapis.com/memory/usage` + `state=used` | +| Disk usage | `agent.googleapis.com/disk/usage` + `state=used` | +| Network bytes | `agent.googleapis.com/interface/traffic` | +| Process count | `agent.googleapis.com/processes/count` | + +To get faster metric resolution (e.g. for sensitive CPU alerts), increase +collection frequency by adding to `config.yaml`: + +```yaml +metrics: + receivers: + hostmetrics: + type: hostmetrics + collection_interval: 30s +``` + +--- + +## 5. Verify Logs Are Arriving + +Wait ~2 minutes after restart, then check: + +```bash +# Check agent is ingesting the file +sudo journalctl -u google-cloud-ops-agent-fluent-bit -n 50 + +# Check via gcloud +gcloud logging read \ + 'logName=~"tap_lms_structured"' \ + --limit=5 \ + --project= \ + --format=json +``` + +In Log Explorer, use this filter to see structured events: + +``` +logName=~"tap_lms_structured" +severity=ERROR +``` + +Or to see a specific submission end-to-end: + +``` +logName=~"tap_lms_structured" +jsonPayload.submission_id="" +``` + +--- + +## 6. Complete Message Catalogue + +Every structured log line has a `message` field. The full list below shows +the message name, which file emits it, and what it means operationally. + +### Infrastructure & HTTP + +| Message | Source | Notes | +|---|---|---| +| `http_request` | `monitoring.py:235` | Every API call — high volume, excluded from storage by default | +| `background_job` | `monitoring.py:321` | Every RQ/scheduler job boundary (v15 only via before/after_job hooks) | +| `unhandled_exception` | `monitoring.py:493` | Genuine unhandled exception — page immediately | +| `watchdog_alert` | `monitoring.py:482` | Intentional operator alert from watchdog/watcher jobs | + +### Submission pipeline + +| Message | Source | Notes | +|---|---|---| +| `save_submission_called` | `save_submission.py:114` | Entry point — Glific webhook received | +| `save_submission_success` | `save_submission.py:539` | Submission saved to DB | +| `save_submission_empty_payload` | `save_submission.py:132` | Request body missing | +| `save_submission_validation_error` | `save_submission.py:194` | Schema validation failed | +| `save_submission_not_found_error` | `save_submission.py:215` | Submission doc not found | +| `save_submission_internal_error` | `save_submission.py:245` | Unexpected exception | +| `save_submission_serialization_failure` | `save_submission.py:266` | PostgreSQL serialization conflict | +| `save_submission_retry_exhausted` | `save_submission.py:284` | Max retries hit | +| `save_submission_missing_assignment` | `save_submission.py:325` | Assignment not found | +| `save_submission_student_not_resolved` | `save_submission.py:342` | Student ID not in DB | +| `save_submission_no_active_pe` | `save_submission.py:359` | No active ProgramEnrollment | +| `save_submission_terminal_state` | `save_submission.py:378` | PE already in terminal state | +| `save_submission_placeholder_detected` | `save_submission.py:165` | Submission text is a placeholder | +| `save_submission_insert_failed` | `save_submission.py:437` | DB insert failed | +| `save_submission_processing_queued` | `save_submission.py:1211` | Background job enqueued | +| `student_duplicate_submission` | `state_machine.py:1181` | PE already submitted this week | +| `student_delivery_failure` | `state_machine.py:1263` | State machine delivery error | +| `student_state_transition` | `state_machine.py:130` | PE journey_label changed | +| `enqueue_submission_start` | `save_submission.py:1335` | RabbitMQ publish starting | +| `enqueue_submission_retry` | `save_submission.py:1457` | Retrying RabbitMQ publish | +| `enqueue_submission_failed` | `save_submission.py:1440` | RabbitMQ publish failed | +| `enqueue_submission_dlq` | `save_submission.py:1501` | Submission sent to DLQ | +| `submission_published` | `monitoring.py:378` | Successfully published to RabbitMQ | +| `process_submission_async_start` | `save_submission.py:1239` | Async processing started | +| `process_submission_prepared` | `save_submission.py:1281` | Submission prepared for GCS | +| `process_submission_uploading_gcs` | `save_submission.py:1255` | GCS upload in progress | +| `process_submission_async_failed` | `save_submission.py:1305` | Async processing failed | +| `process_submission_failed_status_update_failed` | `save_submission.py:1323` | Both processing and status update failed | +| `submission_background_processing_failed` | `imgana/submission.py:93` | Legacy endpoint — background job failed | +| `submission_enqueued_raw` | `imgana/submission.py:317` | Legacy endpoint — raw enqueue | +| `submission_enqueued` | `imgana/submission.py:343` | Legacy endpoint — enqueued | +| `submission_enqueue_failed` | `imgana/submission.py:350` | Legacy endpoint — enqueue failed | +| `submission_prepared` | `imgana/submission.py:78` | Legacy endpoint — prepared | +| `submission_status_check_failed` | `imgana/submission.py:393` | Legacy endpoint — status check failed | +| `submission_status_update_failed` | `imgana/submission.py:107` | Legacy endpoint — status update failed | +| `assignment_submission_failed` | `imgana/submission.py:253` | Legacy endpoint — submission failed | +| `assignment_submission_internal_failed` | `imgana/submission.py:208` | Legacy endpoint — internal error | +| `get_assignment_context_failed` | `imgana/submission.py:518` | Assignment context lookup failed | + +### GCS (image/audio uploads) + +| Message | Source | Notes | +|---|---|---| +| `gcs_upload_success` | `imgana/gcs_client.py:140` | Image uploaded to GCS | +| `gcs_upload_failed` | `imgana/gcs_client.py:162` | GCS upload failed — submission image lost | +| `gcs_download_failed` | `imgana/gcs_client.py:152` | GCS download failed | + +### Feedback pipeline + +| Message | Source | Notes | +|---|---|---| +| `feedback_result_received` | `monitoring.py:393` | Result arrived from plagiarism queue | +| `feedback_processing_complete` | `monitoring.py:402` | Feedback fully processed | +| `feedback_processing_failed` | `monitoring.py:418` | Processing failed — check `retryable` field | +| `feedback_requested` | `save_submission.py:636` | Feedback requested from student | +| `feedback_fetched` | `save_submission.py:578` | Feedback doc fetched | +| `feedback_not_ready` | `save_submission.py:593` | Feedback not ready yet | +| `feedback_fetch_failed` | `save_submission.py:612` | Fetch failed | +| `feedback_fetch_submission_not_found` | `save_submission.py:603` | Submission missing | +| `feedback_flow_triggered` | `save_submission.py:684` | Glific feedback flow triggered | +| `feedback_flow_already_triggered` | `save_submission.py:645` | Duplicate trigger prevented | +| `feedback_flow_claim_lost` | `save_submission.py:692` | Race — another process claimed | +| `feedback_flow_trigger_failed` | `save_submission.py:722` | Flow trigger failed | +| `feedback_ready_submission_not_found` | `save_submission.py:706` | Submission missing at trigger time | +| `glific_notification_sent` | `monitoring.py:512` | Glific notification send result | + +### Feedback audio generation + +| Message | Source | Notes | +|---|---|---| +| `feedback_audio_generation_start` | `audio_creation.py:77` | Audio generation starting | +| `feedback_audio_language_defaulted` | `audio_creation.py:46` | Language fallback applied | +| `feedback_audio_speech_generation_start` | `audio_creation.py:124` | TTS starting | +| `feedback_audio_speech_generated` | `audio_creation.py:132` | TTS complete | +| `feedback_audio_gcs_upload_start` | `audio_creation.py:142` | Uploading to GCS | +| `feedback_audio_generation_success` | `audio_creation.py:154` | Audio ready | +| `feedback_audio_generation_failed` | `audio_creation.py:86` | Audio generation failed | + +### Dispatcher (pe_dispatcher.py) + +| Message | Source | Notes | +|---|---|---| +| `dispatcher_cycle` | `monitoring.py:340` | Every 1-min cycle summary | +| `dispatcher_content_delivery` | `pe_dispatcher.py:299` | Content delivered to student | +| `dispatcher_escalation_no_config` | `pe_dispatcher.py:353` | Missing archetype config — student stuck | +| `dispatcher_escalation_steps_exhausted` | `pe_dispatcher.py:377` | All escalation steps done | +| `dispatcher_escalation_flow` | `pe_dispatcher.py:480` | Escalation flow triggered | +| `dispatcher_escalation_parent_call` | `pe_dispatcher.py:454` | Parent call initiated | +| `dispatcher_feedback_timeout_stale` | `pe_dispatcher.py:521` | Feedback timeout PE is stale | +| `dispatcher_feedback_timeout_resolved` | `pe_dispatcher.py:543` | Feedback timeout resolved | +| `dispatcher_feedback_timeout_retry` | `pe_dispatcher.py:560` | Feedback timeout retry | +| `dispatcher_feedback_timeout_exhausted` | `pe_dispatcher.py:585` | Feedback timeout exhausted | +| `dispatcher_week_advancement_stale` | `pe_dispatcher.py:616` | Week advancement PE is stale | +| `dispatcher_program_completed` | `pe_dispatcher.py:631` | Student completed program | +| `dispatcher_binge_paused` | `pe_dispatcher.py:647` | Binge protection triggered | +| `dispatcher_week_advanced` | `pe_dispatcher.py:663` | Week advanced for student | +| `dispatcher_binge_resumed` | `pe_dispatcher.py:797` | Binge protection lifted | +| `dispatcher_binge_paused_remaining` | `pe_dispatcher.py:813` | Still in binge pause | +| `dispatcher_grace_check_stale` | `pe_dispatcher.py:698` | Grace check PE is stale | +| `dispatcher_grace_check_submitted` | `pe_dispatcher.py:710` | Grace period submission received | +| `dispatcher_grace_check_rescheduled` | `pe_dispatcher.py:727` | Grace check rescheduled | +| `dispatcher_grace_expired_dropped` | `pe_dispatcher.py:745` | Grace expired — dropped | +| `dispatcher_pause_check_stale` | `pe_dispatcher.py:773` | Pause check PE is stale | + +### Glific integration + +| Message | Source | Notes | +|---|---|---| +| `glific_token_health` | `glific_integration.py:1583` | Token probe result — check `token_status` field | +| `glific_contact_created` | `glific_integration.py:257` | Contact created in Glific | +| `glific_create_contact_failed` | `glific_integration.py:242` | Contact creation failed | +| `glific_contact_fields_updated` | `glific_integration.py:437` | Contact fields updated | +| `glific_contact_fields_update_failed` | `api.py:1576` | Contact fields update failed | +| `glific_update_contact_failed` | `glific_integration.py:448` | Contact update failed | +| `glific_flow_started` | `glific_integration.py:828` | Flow triggered successfully | +| `glific_flow_start_failed` | `glific_integration.py:837` | Flow trigger failed | +| `glific_start_group_flow_success` | `glific_extensions.py:86` | Group flow started | +| `glific_start_group_flow_failed` | `glific_extensions.py:95` | Group flow failed | +| `glific_start_group_flow_api_error` | `glific_extensions.py:72` | API error on group flow | +| `glific_start_group_flow_exception` | `glific_extensions.py:114` | Exception on group flow | +| `glific_add_contacts_bulk_success` | `glific_extensions.py:186` | Bulk add succeeded | +| `glific_add_contacts_bulk_failed` | `glific_extensions.py:195` | Bulk add failed | +| `glific_add_contacts_bulk_api_error` | `glific_extensions.py:172` | Bulk add API error | +| `glific_add_contacts_bulk_exception` | `glific_extensions.py:215` | Bulk add exception | +| `glific_bulk_add_circuit_tripped` | `glific_extensions.py:482` | Circuit breaker open on bulk add | +| `glific_bulk_add_complete` | `glific_extensions.py:512` | Bulk add batch complete | +| `glific_remove_contacts_bulk_success` | `glific_extensions.py:293` | Bulk remove succeeded | +| `glific_remove_contacts_bulk_failed` | `glific_extensions.py:304` | Bulk remove failed | +| `glific_remove_contacts_bulk_api_error` | `glific_extensions.py:279` | Bulk remove API error | +| `glific_remove_contacts_bulk_exception` | `glific_extensions.py:324` | Bulk remove exception | +| `glific_bulk_remove_circuit_tripped` | `glific_extensions.py:628` | Circuit breaker open on bulk remove | +| `glific_bulk_remove_complete` | `glific_extensions.py:658` | Bulk remove batch complete | + +### Vocallabs (parent calls) + +| Message | Source | Notes | +|---|---|---| +| `vocallabs_initiating_call` | `vocallabs.py:191` | Call being placed | +| `vocallabs_call_success` | `vocallabs.py:225` | Call succeeded | +| `vocallabs_call_transient_failure` | `vocallabs.py:1249` | Transient failure — will retry | +| `vocallabs_call_duplicate_prospect_no_retry` | `vocallabs.py:1218` | Duplicate prospect — not retried | +| `vocallabs_call_double_fault` | `vocallabs.py:1283` | Double fault — check manually | +| `vocallabs_call_dlq_exhausted` | `vocallabs.py:1311` | DLQ exhausted — call permanently failed | +| `vocallabs_disabled` | `vocallabs.py:131` | Vocallabs disabled in settings | +| `vocallabs_dormant_skipped` | `vocallabs.py:177` | Dormant PE skipped | +| `vocallabs_pe_not_found` | `vocallabs.py:103` | PE not found | +| `vocallabs_phone_missing` | `vocallabs.py:165` | Student phone number missing | +| `vocallabs_config_missing` | `vocallabs.py:149` | Vocallabs config not set up | +| `vocallabs_settings_missing` | `vocallabs.py:117` | Settings doc missing | + +### Teacher onboarding + +| Message | Source | Notes | +|---|---|---| +| `teacher_linked_to_glific` | `api.py:1498` | Teacher linked to Glific contact | +| `teacher_missing_glific_id` | `api.py:1478` | Teacher has no Glific ID | +| `teacher_glific_contact_created` | `api.py:1550` | Glific contact created for teacher | +| `teacher_glific_contact_creation_failed` | `api.py:1557` | Contact creation failed | +| `teacher_still_missing_glific_id` | `api.py:1583` | Still no Glific ID after retry | +| `teacher_added_to_batch_group` | `api.py:1607` | Added to batch Glific group | +| `teacher_group_addition_failed` | `api.py:1615` | Group addition failed | +| `teacher_batch_history_creation_failed` | `api.py:1635` | Batch history insert failed | +| `teacher_batch_update_exception` | `api.py:1676` | Batch update exception | +| `teacher_optin_failed` | `background_jobs.py:45` | Teacher opt-in failed | +| `teacher_glific_id_missing_in_background_job` | `background_jobs.py:58` | Glific ID missing in background | +| `teacher_added_to_group_background` | `background_jobs.py:88` | Added to group in background | +| `teacher_group_addition_failed_background` | `background_jobs.py:96` | Group addition failed in background | +| `teacher_group_creation_failed_background` | `background_jobs.py:104` | Group creation failed | +| `teacher_group_management_error` | `background_jobs.py:113` | General group management error | +| `teacher_group_skipped_no_batch` | `background_jobs.py:120` | No batch found for teacher | +| `teacher_onboarding_flow_started_background` | `background_jobs.py:140` | Onboarding flow started | +| `teacher_onboarding_flow_failed_background` | `background_jobs.py:148` | Onboarding flow failed | +| `teacher_onboarding_flow_not_found` | `background_jobs.py:158` | Flow not configured | + +### Quiz + +| Message | Source | Notes | +|---|---|---| +| `quiz_started` | `student_progression_sp.py:1381` | Student started quiz | +| `quiz_resumed` | `student_progression_sp.py:1449` | Student resumed quiz | +| `quiz_answer_submitted` | `student_progression_sp.py:1631` | Answer submitted | +| `quiz_completed` | `student_progression_sp.py:1685` | Quiz completed | + +### API / misc + +| Message | Source | Notes | +|---|---|---| +| `active_batch_not_found` | `api.py:64` | No active batch for school | +| `active_batch_not_found_create_teacher` | `api.py:1802` | No batch when creating teacher | +| `api_list_cities_exception` | `api.py:140` | City list API exception | +| `api_list_districts_exception` | `api.py:101` | District list API exception | +| `create_teacher_web_exception` | `api.py:1948` | Teacher creation exception | +| `verify_otp_exception` | `api.py:1708` | OTP verification exception | +| `gupshup_settings_missing` | `api.py:154` | Gupshup settings not configured | +| `gupshup_settings_incomplete` | `api.py:166` | Gupshup settings incomplete | +| `gupshup_send_failed` | `api.py:193` | Gupshup message send failed | +| `no_english_language_found` | `api.py:1523` | No English language record | +| `model_name_not_found` | `api.py:2123` | Model name resolution failed | +| `model_resolved_from_batch_onboarding` | `api.py:2104` | Model from batch onboarding | +| `model_resolved_from_school_default` | `api.py:2113` | Model from school default | +| `process_glific_actions_exception` | `background_jobs.py:171` | Glific actions background job failed | + +--- + +## 7. Recommended Cloud Monitoring Alerts + +Set these up in GCP Console → **Monitoring → Alerting → Create Policy**. + +### Page immediately (ERROR severity) + +``` +# Unhandled exception +jsonPayload.message = "unhandled_exception" +severity = "ERROR" +``` + +``` +# Glific token stale +jsonPayload.message = "glific_token_health" +jsonPayload.token_status = "stale" +severity = "ERROR" +``` + +``` +# Non-retryable feedback failure → check DLQ +jsonPayload.message = "feedback_processing_failed" +jsonPayload.retryable = false +severity = "ERROR" +``` + +``` +# GCS upload failed — submission image lost +jsonPayload.message = "gcs_upload_failed" +severity = "ERROR" +``` + +``` +# Vocallabs DLQ exhausted — call permanently failed +jsonPayload.message = "vocallabs_call_dlq_exhausted" +severity = "ERROR" +``` + +### Notify Slack (WARNING — investigate) + +``` +# Watchdog alerts (stuck PEs, DLQ depth etc.) +jsonPayload.message = "watchdog_alert" +severity = "WARNING" +``` + +``` +# Glific flow not triggering for student +jsonPayload.message = "glific_flow_start_failed" +``` + +``` +# Feedback not being delivered after processing +jsonPayload.message = "feedback_flow_trigger_failed" +``` + +``` +# Dispatcher missing archetype config — student stuck +jsonPayload.message = "dispatcher_escalation_no_config" +``` + +``` +# Retryable feedback failure — watch for repeated occurrences +jsonPayload.message = "feedback_processing_failed" +jsonPayload.retryable = true +severity = "WARNING" +``` + +``` +# Submission sent to DLQ +jsonPayload.message = "enqueue_submission_dlq" +``` + +### Hardware metric alerts + +| Metric | Threshold | Action | +|---|---|---| +| `agent.googleapis.com/cpu/utilization` | > 80% for 5 min | Slack | +| `agent.googleapis.com/memory/percent_used` | > 85% for 5 min | Slack | +| `agent.googleapis.com/disk/percent_used` | > 80% | Slack + email | + +--- + +## 8. Useful Log Explorer Queries + +**Trace a submission end-to-end:** +``` +jsonPayload.submission_id="" +``` + +**All errors in the last hour:** +``` +severity=ERROR +timestamp >= "2026-07-29T01:00:00Z" +``` + +**Which students had feedback failures today:** +``` +jsonPayload.message="feedback_processing_failed" +jsonPayload.retryable=false +``` + +**Dispatcher stuck PEs:** +``` +jsonPayload.message="dispatcher_escalation_no_config" +``` + +**Glific bulk operation failures:** +``` +jsonPayload.message=~"glific_(add|remove)_contacts_bulk_failed" +``` + +**All Vocallabs failures:** +``` +jsonPayload.message=~"vocallabs_call_(transient_failure|double_fault|dlq_exhausted)" +``` + +--- + +## 9. Troubleshooting + +**Agent not starting:** +```bash +sudo journalctl -u google-cloud-ops-agent -n 50 +sudo journalctl -u google-cloud-ops-agent-fluent-bit -n 50 +``` + +**Logs not appearing in Cloud Logging:** +```bash +# Check the agent can read the log file +sudo -u root cat /home/lms-dev/frappe-bench/logs/gcp_structured.log | head -5 + +# Check IAM permissions +gcloud projects get-iam-policy \ + --flatten="bindings[].members" \ + --filter="bindings.members:" +``` + +**severity still showing as grey in Log Explorer:** +Check the `promote_severity` processor is in the pipeline and the config +was reloaded: +```bash +sudo systemctl restart google-cloud-ops-agent +``` + +**Config syntax error:** +```bash +sudo google-cloud-ops-agent --config /etc/google-cloud-ops-agent/config.yaml --dryrun +``` + +**After `bench setup nginx` overwrites nginx.conf:** +The Ops Agent config at `/etc/google-cloud-ops-agent/config.yaml` is +unaffected — only nginx's config is overwritten. No Ops Agent action needed. + +--- + +## 10. Adding to Deployment Runbook + +After any code deploy that adds new log message types or changes field names +in `monitoring.py`, check: + +1. The new message type appears in Log Explorer +2. The `severity` field is being promoted correctly +3. Any new intentional `frappe.log_error()` calls have their titles added + to `_OPERATOR_ALERT_TITLES` in `monitoring.py` +4. If the new message type needs a Cloud Monitoring alert, add it + +--- + +## 11. File Locations + +| File | Purpose | +|---|---| +| `/etc/google-cloud-ops-agent/config.yaml` | Active Ops Agent config | +| `deployment_configs/ops_agent_config.yaml` | Config stored in repo (source of truth) | +| `/home/lms-dev/frappe-bench/logs/gcp_structured.log` | tap_lms structured log | +| `/home/lms-dev/frappe-bench/logs/rag_gcp_structured.log` | rag_service structured log | +| `/var/log/google-cloud-ops-agent/` | Ops Agent self logs | diff --git a/docs/server_setup.md b/docs/server_setup.md new file mode 100644 index 00000000..65b99a8d --- /dev/null +++ b/docs/server_setup.md @@ -0,0 +1,584 @@ +# TAP LMS — Server Setup & Restore Guide + +This document covers setting up a fresh Ubuntu server and restoring a TAP LMS +instance from a backup. Tested on Ubuntu 22.04 / GCP Compute Engine. + +--- + +## Prerequisites + +- Ubuntu 22.04 VM (GCP or equivalent) +- A bench user account (e.g. `lms-dev`) — do **not** run bench as root +- SSH access to the server +- Backup files from the old server: + - `-database.sql.gz` + - `-files.tar` (public files) + - `-private-files.tar` (private files) + +--- + +## 1. Install System Dependencies + +```bash +sudo apt update && sudo apt upgrade -y +sudo apt install -y git python3-pip python3-venv redis-server \ + postgresql postgresql-contrib nginx supervisor \ + libpq-dev wkhtmltopdf cron +``` + +### Node.js (via nvm — must be Node 16) + +```bash +curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.0/install.sh | bash +source ~/.bashrc +nvm install 16 +nvm use 16 +nvm alias default 16 +``` + +> **Important:** Frappe v14's `socketio.js` requires Node 16. Node 18+ will cause +> a spawn error in supervisor. + +### Add bench to PATH + +`pip install frappe-bench` installs bench to `~/.local/bin` which is not in +PATH by default: + +```bash +echo 'export PATH=$HOME/.local/bin:$PATH' >> ~/.bashrc +source ~/.bashrc +bench --version # verify +``` + +### Install yarn + +Frappe requires yarn for building frontend assets: + +```bash +npm install -g yarn +``` + +### Make node visible to supervisor (runs as root) + +```bash +sudo ln -sf ~/.nvm/versions/node/v16.*/bin/node /usr/local/bin/node +sudo ln -sf ~/.nvm/versions/node/v16.*/bin/npm /usr/local/bin/npm +``` + +--- + +## 2. Install bench + +```bash +pip3 install frappe-bench +``` + +--- + +## 3. Initialise the Bench + +```bash +bench init frappe-bench --frappe-branch version-14 +cd frappe-bench +``` + +--- + +## 4. Get Apps + +```bash +bench get-app https://github.com//tap_lms +bench get-app https://github.com//business_theme_v14 +# add any other custom apps here +``` + +--- + +## 5. PostgreSQL Setup + +Frappe on the old server uses `frappe_db` as the database role. Mirror this +on the new server so the backup restores without ownership errors. + +```bash +sudo -u postgres psql +``` + +```sql +CREATE ROLE frappe_db LOGIN PASSWORD ''; +ALTER ROLE frappe_db CREATEDB; +CREATE DATABASE frappe_db OWNER frappe_db; +GRANT frappe_db TO postgres; +\q +``` + +--- + +## 6. Create the Site + +```bash +bench new-site tap_lms.dev \ + --db-type postgres \ + --db-host localhost +# enter the frappe_db password when prompted +# set an admin password when prompted +``` + +After site creation, update the site config to use the `frappe_db` user: + +```bash +bench --site tap_lms.dev set-config db_name frappe_db +bench --site tap_lms.dev set-config db_password '' +bench use tap_lms.dev +``` + +--- + +## 7. Restore the Database + +Bypass `bench restore` (it tries to read `tabSingles` before the DB is +populated, causing an error). Restore directly via psql instead: + +```bash +# drop the empty database bench just created and restore the backup +sudo -u postgres psql -c "DROP DATABASE frappe_db;" +sudo -u postgres psql -c "CREATE DATABASE frappe_db OWNER frappe_db;" + +gunzip -c /path/to/-database.sql.gz | sudo -u postgres psql -d frappe_db +``` + +--- + +## 7a. Copy the Encryption Key + +Frappe encrypts sensitive fields (passwords, API tokens, credentials) using an +encryption key stored in `site_config.json`. The restored database contains +data encrypted with the **old server's key** — if the new server has a +different key, all encrypted fields will be unreadable. + +On the **old server**: + +```bash +cat ~/frappe-bench/sites/tap_lms.dev/site_config.json | grep encryption_key +``` + +On the **new server**: + +```bash +bench --site tap_lms.dev set-config encryption_key +``` + +> This affects RabbitMQ Settings, Glific Settings, and any other DocType +> that stores passwords or tokens. Skipping this step will cause silent +> failures when the app tries to connect to external services. + +--- + +## 8. Restore Files + +The backup tars contain the full site path internally (e.g. +`./tap_lms.dev/public/files/...`). Extract to `sites/`: + +```bash +cd ~/frappe-bench + +# public files +tar -xf /path/to/-files.tar \ + --transform='s|tap_lms\.dev|tap_lms.dev|' \ + -C sites/ + +# private files +tar -xf /path/to/-private-files.tar \ + --transform='s|tap_lms\.dev|tap_lms.dev|' \ + -C sites/ + +# fix ownership +sudo chown -R lms-dev:lms-dev sites/tap_lms.dev/public/files +sudo chown -R lms-dev:lms-dev sites/tap_lms.dev/private/files +``` + +> If you are restoring to a site with a **different name** (e.g. old server +> was `tap_lms.dev`, new server is `tap_lms.localhost`), update the +> `--transform` pattern accordingly. + +--- + +## 9. Install Apps into the Site + +```bash +bench --site tap_lms.dev install-app tap_lms +bench --site tap_lms.dev install-app business_theme_v14 +``` + +--- + +## 10. Run Migrations + +Start Redis first (supervisor isn't set up yet): + +```bash +redis-server ~/frappe-bench/config/redis_cache.conf & +redis-server ~/frappe-bench/config/redis_queue.conf & + +bench --site tap_lms.dev migrate +``` + +--- + +## 11. Set Up Supervisor + +Supervisor manages Redis, Gunicorn, workers, and socketio so they start +automatically on reboot without needing `bench start`. + +```bash +bench setup supervisor +sudo ln -sf ~/frappe-bench/config/supervisor.conf \ + /etc/supervisor/conf.d/frappe-bench.conf +``` + +### Fix socketio — use full node path + +Supervisor runs as root and can't find nvm-installed node via PATH. Update +the socketio command to use the full node path: + +```bash +sed -i 's|command=.*bench socketio|command=/home/lms-dev/.nvm/versions/node/v16.20.2/bin/node /home/lms-dev/frappe-bench/apps/frappe/socketio.js|' \ + ~/frappe-bench/config/supervisor.conf +``` + +Verify the change: + +```bash +grep -A4 "node-socketio\]" ~/frappe-bench/config/supervisor.conf +``` + +### Kill any manually started Redis instances before starting supervisor + +```bash +sudo pkill -f "redis-server.*frappe-bench" +``` + +### Start supervisor + +```bash +sudo supervisorctl reread +sudo supervisorctl update +sudo supervisorctl start all +sudo systemctl enable supervisor +sudo supervisorctl status +``` + +All processes should show `RUNNING`. If `node-socketio` shows `BACKOFF`, +check `~/frappe-bench/logs/node-socketio.error.log`. + +--- + +## 11a. Start the Feedback Consumer + +The feedback consumer listens to the `plagiarism_feedback` RabbitMQ queue and +processes feedback results. It is **not** started by supervisor automatically — +it must be added as a separate supervisor program. + +Add the following to `~/frappe-bench/config/supervisor.conf` (placement within the file doesn't matter — supervisor processes all `[program:...]` blocks regardless of order): + +```ini +[program:frappe-bench-feedback-consumer] +command=/home/lms-dev/frappe-bench/env/bin/python /home/lms-dev/frappe-bench/apps/tap_lms/scripts/console_consumer.py +directory=/home/lms-dev/frappe-bench/sites +environment=SITE_NAME="tap_lms.dev" +user=lms-dev +autostart=true +autorestart=true +stdout_logfile=/home/lms-dev/frappe-bench/logs/feedback-consumer.log +stderr_logfile=/home/lms-dev/frappe-bench/logs/feedback-consumer.error.log +``` + +> **Note on log files:** Structured business logic logs (feedback events, +> errors) go to `gcp_structured.log` via `monitoring.py`. The supervisor +> `stdout_logfile` and `stderr_logfile` are a safety net for raw stdout/stderr +> output — startup crashes, import errors, and anything that bypasses +> structured logging. Both are needed. + +> **Working directory:** The `directory=/home/lms-dev/frappe-bench/sites` +> setting is critical — the consumer script must be invoked from the `sites/` +> folder for Frappe to resolve the site name correctly. Supervisor `cd`s to +> this directory before executing the command, equivalent to running: +> ```bash +> cd ~/frappe-bench/sites +> ../env/bin/python ../apps/tap_lms/scripts/console_consumer.py +> ``` + +Then reload supervisor: + +```bash +sudo supervisorctl reread +sudo supervisorctl update +sudo supervisorctl start frappe-bench-feedback-consumer +sudo supervisorctl status +``` + +Verify it started correctly: + +```bash +cat ~/frappe-bench/logs/feedback-consumer.log +``` + +If you see `tap_lms.dev does not exist`, the `SITE_NAME` env var is not being +picked up or the `directory` is wrong. If you see the consumer connecting to +RabbitMQ, it's running correctly. + +> **Warning:** If you are also running a local podman setup connected to the +> same CloudAMQP instance, stop the local consumer first to avoid race +> conditions — two consumers competing for the same queue will cause +> intermittent failures and messages going to the DLQ. Check active consumers +> in CloudAMQP Manager → Queues → `plagiarism_feedback` → Consumers before +> starting. + +--- + +## 12. Set Up nginx + +```bash +bench setup nginx +sudo rm /etc/nginx/sites-enabled/default # remove default page + +sudo ln -sf ~/frappe-bench/config/nginx.conf \ + /etc/nginx/conf.d/frappe-bench.conf +``` + +### Add log_format (not included by default on Ubuntu) + +```bash +sudo sed -i '/http {/a\\n\tlog_format main '"'"'$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for"'"'"';' \ + /etc/nginx/nginx.conf +``` + +### Set server_name + +If you have a domain name, use it directly. If not, use a temporary value +and update it when the domain is ready (see Section 14a — SSL/HTTPS): + +```bash +# with domain name +sed -i 's|server_name .*;|server_name lms-dev.theapprenticeproject.org;|' \ + ~/frappe-bench/config/nginx.conf + +# without domain (temporary — IP access only) +sed -i 's|server_name ;|server_name tap_lms.dev;|' \ + ~/frappe-bench/config/nginx.conf +``` + +> **Note:** If `server_name` spans multiple lines in the nginx config (e.g. +> after `bench setup nginx`), use `nano` to edit it directly rather than `sed`. + +### Fix asset permissions + +nginx runs as `www-data` and cannot read files in the home directory by +default. Grant traversal and read access to the assets only: + +```bash +chmod o+x /home/lms-dev +chmod o+x /home/lms-dev/frappe-bench +chmod o+x /home/lms-dev/frappe-bench/sites +chmod -R o+rX /home/lms-dev/frappe-bench/sites/assets +``` + +### Start nginx + +```bash +sudo nginx -t +sudo systemctl enable nginx +sudo systemctl start nginx +``` + +--- + +## 13. Build Assets + +```bash +bench build +``` + +--- + +## 14. GCP Firewall + +Open port 80 in the GCP Console: + +**Compute Engine → VM Instances → click instance → Edit → Firewalls → +check "Allow HTTP traffic"** + +Or via gcloud: + +```bash +gcloud compute firewall-rules create allow-http \ + --allow tcp:80 \ + --target-tags http-server \ + --description "Allow HTTP traffic" +``` + +--- + +## 14a. SSL/HTTPS Setup (Let's Encrypt) + +Skip this section if you don't have a domain name yet. Come back once the +client has set up an A record pointing the domain to your server's external IP. + +### Prerequisites + +- A domain name with an A record pointing to the server's external IP +- Port 80 open in GCP firewall (Section 14) +- nginx running and serving the site on HTTP + +### Verify DNS has propagated + +```bash +dig +short +# should return your server's external IP +curl -s ifconfig.me +# should return the same IP +``` + +### Update server_name to use the domain + +```bash +# edit nginx.conf directly (server_name may span multiple lines) +nano ~/frappe-bench/config/nginx.conf +# find server_name block and change to: +# server_name ; + +sudo nginx -t +sudo systemctl reload nginx +``` + +### Install Certbot and obtain certificate + +```bash +sudo apt install -y certbot python3-certbot-nginx + +# obtain and install certificate automatically +sudo certbot --nginx -d +``` + +Certbot will: +- Obtain the SSL certificate from Let's Encrypt +- Automatically update nginx config with HTTPS settings +- Set up HTTP → HTTPS redirect + +If certbot can't find the server block automatically: + +```bash +# install certificate manually after updating server_name +sudo certbot install --cert-name +``` + +### Update Frappe host_name + +```bash +bench --site tap_lms.dev set-config host_name "https://" +bench --site tap_lms.dev clear-cache +sudo supervisorctl restart all +``` + +### Verify + +```bash +# HTTPS should return 200 +curl -s -o /dev/null -w "%{http_code}" https:// + +# HTTP should redirect to HTTPS (301) +curl -s -o /dev/null -w "%{http_code}" http:// +``` + +### Auto-renewal + +Certbot sets up automatic renewal via a systemd timer. Verify it's active: + +```bash +sudo systemctl status certbot.timer +``` + +Certificates renew automatically every 90 days. No manual action needed. + +### Open HTTPS port in GCP firewall + +``` +GCP Console → Compute Engine → VM Instances → click instance → +Edit → Firewalls → check "Allow HTTPS traffic" +``` + +--- + +## 15. Final Steps + +```bash +bench --site tap_lms.dev clear-cache +bench --site tap_lms.dev clear-website-cache +sudo supervisorctl restart all +sudo systemctl restart nginx +``` + +Add the server IP to your local `/etc/hosts` to resolve the site name: + +```bash +# on your LOCAL machine +echo " tap_lms.dev" | sudo tee -a /etc/hosts +``` + +Then browse to `http://tap_lms.dev` and log in with: + +- **Username:** `Administrator` +- **Password:** the Administrator password from the old server (restored from DB) + +To reset the admin password if needed: + +```bash +bench --site tap_lms.dev set-admin-password +``` + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +|---|---|---| +| `Could not automatically find a matching server block` | certbot can't find domain in nginx config | Edit `server_name` in nginx.conf manually, reload nginx, then `certbot install --cert-name ` | +| `NXDOMAIN` from dig | Including `https://` in dig command | Run `dig +short` without protocol | +| Certificate obtained but HTTPS not working | Port 443 not open in GCP | Check "Allow HTTPS traffic" in GCP Console → VM → Edit | +| `bench: command not found` after pip install | `~/.local/bin` not in PATH | `echo 'export PATH=$HOME/.local/bin:$PATH' >> ~/.bashrc && source ~/.bashrc` | +| `FileNotFoundError: /usr/bin/crontab` during bench init | cron not installed | `sudo apt install -y cron` then remove partial bench and retry | +| `engine "node" is incompatible, Expected version ">=18"` | Wrong Node version for Frappe v15 | Install Node 18 via nvm; Node 16 is for v14 only | +| `must be member of role "frappe_db"` | Role doesn't exist on new server | Section 5 — create `frappe_db` role | +| `relation "tabSingles" does not exist` | bench restore pre-flight on empty DB | Section 7 — restore via psql directly | +| `ModuleNotFoundError: No module named 'business_theme_v14'` | App not installed | `bench get-app` + `install-app` | +| `Service redis_cache is not running` during migrate | Redis not started | Start Redis manually before migrate | +| `node-socketio BACKOFF` in supervisor | Wrong node version or path | Use Node 16 full nvm path in supervisor.conf | +| `Error: Cannot start socketio: node not found` | supervisor can't find node in PATH | Use full nvm path in supervisor.conf command | +| Assets 404 in browser | Bundle hash mismatch or permissions | Run `bench build`, fix `o+rX` on assets | +| nginx serves default page instead of Frappe | Default site enabled | `sudo rm /etc/nginx/sites-enabled/default` | +| `unknown log format "main"` in nginx | log_format not defined | Add log_format to `/etc/nginx/nginx.conf` http block | +| Permission denied on assets | www-data can't read home dir | `chmod o+x` on path, `o+rX` on assets | +| Encrypted fields unreadable / external service auth failures | Encryption key mismatch | Section 7a — copy encryption key from old server | +| `SerializationFailure: could not serialize access` in console | Concurrent DB write from pe_dispatcher | Run `frappe.db.rollback()` then retry | +| `InFailedSqlTransaction: current transaction is aborted` | Previous statement failed, transaction broken | Run `frappe.db.rollback()` then retry | +| Messages going to DLQ intermittently | Two consumers competing for same queue | Stop local podman consumer; check CloudAMQP Manager → Consumers | +| `tap_lms.localhost does not exist` in consumer | `SITE_NAME` env var not set or wrong | Set `export SITE_NAME=tap_lms.dev` before starting consumer | + +--- + +## Notes + +- **Encryption key** must be copied from the old server (Section 7a) — without + it, RabbitMQ, Glific, and all other stored credentials will silently fail. +- **Feedback consumer** must be added to supervisor manually (Section 11a) — + it is not included in `bench setup supervisor` output. Without it, plagiarism + feedback results will pile up in the `plagiarism_feedback` queue unprocessed. +- **Never run two consumers** against the same CloudAMQP queue simultaneously + (e.g. local podman + dev server) — messages will be split between them, + causing intermittent failures and DLQ buildup. Always check CloudAMQP + Manager → Queues → Consumers before starting the consumer on a new server. +- GCS credentials (`GOOGLE_APPLICATION_CREDENTIALS`) must be configured + separately for submission image uploads to work. +- After any code deploy, restart workers: + `sudo supervisorctl restart frappe-bench-workers:` +- After any hooks.py change, restart everything: + `sudo supervisorctl restart all` diff --git a/env.example b/env.example new file mode 100644 index 00000000..06033bfa --- /dev/null +++ b/env.example @@ -0,0 +1,71 @@ +# Copy this file to env.local before running scripts/start_local_docker.sh. + +# Frappe / site +SITE_NAME=tap_lms.localhost +FRAPPE_BRANCH=version-16 +ADMIN_PASSWORD=admin +WEB_PORT=8000 +SOCKETIO_PORT=9000 +BUSINESS_THEME_REPO=https://github.com/Midocean-Technologies/business_theme_v14.git + +# rag_service site +# Mirrors dev/prod, where rag_service runs as its own Frappe site with its +# own database, separate from tap_lms. They talk over HTTP (RAG Settings.base_url) +# and RabbitMQ, never by sharing a DB connection. +RAG_SITE_NAME=rag.localhost +RAG_POSTGRES_DB=rag_lms + +# Local dev API credentials +# Single source of truth — used to seed both the tap_lms Administrator user +# (api_key/api_secret) and the rag_service site's RAG Settings (api_key/ +# api_secret), since rag_service authenticates against tap_lms with these. +LOCAL_API_KEY=local-dev-api-key-001 +LOCAL_API_SECRET=local-secret-key + +# Optional: only needed if you want the sample curl's "Authorization: token +# ..." header to use different credentials than LOCAL_API_KEY/LOCAL_API_SECRET +# (e.g. testing with a different/rotated API user). If unset, they default to +# LOCAL_API_KEY/LOCAL_API_SECRET, so leaving these commented out is fine. +# AUTH_KEY= +# AUTH_SECRET= + +# Postgres +# Single Postgres *instance* (container) hosts both databases above: +# POSTGRES_DB -> tap_lms site's database +# RAG_POSTGRES_DB -> rag_service site's database +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +POSTGRES_DB=postgres +POSTGRES_PORT=5432 + +# Redis host ports. Container-internal ports stay 6379. +REDIS_CACHE_PORT=6379 +REDIS_QUEUE_PORT=6380 + +# CloudAMQP / RabbitMQ. +# In CloudAMQP, the AMQP URL usually has this shape: +# amqps://USERNAME:PASSWORD@HOST/VIRTUAL_HOST +# Split that URL into the fields below. +RABBITMQ_HOST= +RABBITMQ_PORT=5671 +RABBITMQ_VIRTUAL_HOST= +RABBITMQ_USERNAME= +RABBITMQ_PASSWORD= +RABBITMQ_SUBMISSION_QUEUE= +RABBITMQ_PLAGIARISM_RESULTS_QUEUE= +RABBITMQ_FEEDBACK_RESULTS_QUEUE= + +# Optional integrations. Keep these disabled for a basic local site. +GCS_ENABLED=0 +GCS_BUCKET_NAME= +GCS_PROJECT_ID= +GCS_CREDENTIALS_JSON={} +ELEVENLABS_ENABLED=0 +ELEVENLABS_API_KEY= +VOICE_AGENT_ENABLED=0 +VOICE_AGENT_SERVICE_URL= +VOICE_AGENT_CLIENT_ID= +VOICE_AGENT_CLIENT_SECRET= +VOICE_AGENT_DEFAULT_CONTACT_GROUP_ID= +VOICE_AGENT_AGENT_ID= +VOICE_AGENT_AUTH_TOKEN_CACHE_TTL=3600 diff --git a/ops_config/ops_agent_config.yaml b/ops_config/ops_agent_config.yaml new file mode 100644 index 00000000..3eff86cd --- /dev/null +++ b/ops_config/ops_agent_config.yaml @@ -0,0 +1,177 @@ +# /etc/google-cloud-ops-agent/config.yaml +# +# GCP Ops Agent configuration for TAP LMS server. +# +# Covers: +# 1. Structured JSON logs from tap_lms (gcp_structured.log) +# 2. Structured JSON logs from rag_service (if co-located) +# 3. Frappe/nginx/supervisor system logs +# 4. Hardware metrics (CPU, memory, disk, network) — built-in, no config needed +# +# After editing, restart the agent: +# sudo systemctl restart google-cloud-ops-agent +# +# Verify logs are arriving: +# gcloud logging read 'logName=~"tap_lms_structured"' --limit=5 --project= +# +# ── IMPORTANT: severity field mapping ───────────────────────────────────────── +# Our monitoring.py emits `"severity": "ERROR"` as a plain JSON field. +# The Ops Agent only promotes a field to LogEntry.severity if it is named +# `logging.googleapis.com/severity`. The modify_fields processor below +# renames our `severity` field so Cloud Logging displays the correct +# severity level and colour-coding in Log Explorer, and so log-based +# alert policies can filter on `severity=ERROR` directly. +# +# Similarly, our `timestamp` field is promoted to LogEntry.timestamp via +# the parse_json time_key setting. +# ───────────────────────────────────────────────────────────────────────────── + +logging: + receivers: + + # ── tap_lms structured log ───────────────────────────────────────────── + tap_lms_structured: + type: files + include_paths: + - /home/*/frappe-bench/logs/gcp_structured.log + # wildcard_refresh_interval handles log rotation — after copytruncate, + # a new file appears and the agent picks it up within this interval. + wildcard_refresh_interval: 30s + + # ── rag_service structured log (if co-located on same VM) ───────────── + rag_service_structured: + type: files + include_paths: + - /home/*/frappe-bench/logs/rag_gcp_structured.log + wildcard_refresh_interval: 30s + + # ── Frappe application log ───────────────────────────────────────────── + frappe_app: + type: files + include_paths: + - /home/*/frappe-bench/logs/frappe.log + - /home/*/frappe-bench/logs/worker.log + wildcard_refresh_interval: 30s + + # ── nginx access + error logs ────────────────────────────────────────── + nginx_access: + type: files + include_paths: + - /var/log/nginx/access.log + - /home/*/frappe-bench/logs/access.log + + nginx_error: + type: files + include_paths: + - /var/log/nginx/error.log + + # ── Feedback consumer log ────────────────────────────────────────────── + feedback_consumer: + type: files + include_paths: + - /home/*/frappe-bench/logs/feedback-consumer.log + - /home/*/frappe-bench/logs/feedback-consumer.error.log + wildcard_refresh_interval: 30s + + # ── Supervisor logs ──────────────────────────────────────────────────── + supervisor: + type: files + include_paths: + - /var/log/supervisor/supervisord.log + + processors: + + # ── Parse tap_lms structured JSON ───────────────────────────────────── + # Parses each line of gcp_structured.log as JSON and extracts the + # timestamp field so it becomes LogEntry.timestamp in Cloud Logging. + # Format matches monitoring.py output: 2026-07-31T10:24:40.313895 + # Note: Fluent Bit uses strptime — %L is milliseconds (3 digits), + # %f is NOT supported. Use %s.%N for full nanosecond precision, + # or omit time_key entirely and let Cloud Logging use ingestion time. + # The safest cross-version approach is to let parse_json handle the + # JSON structure and use modify_fields to map the timestamp separately. + parse_tap_lms_json: + type: parse_json + time_key: timestamp + time_format: "%Y-%m-%dT%H:%M:%S.%f" + + # ── Promote severity field to LogEntry.severity ──────────────────────── + # monitoring.py emits `"severity": "ERROR"` as a plain JSON field. + # This processor moves it to `logging.googleapis.com/severity` so + # Cloud Logging promotes it to the top-level severity field, enabling: + # - Correct colour-coding in Log Explorer + # - Filtering with `severity=ERROR` in alert policies + # - The severity histogram in the Logs Explorer sidebar + # + # Also maps the timestamp field. Fluent Bit's strptime does not support + # %f (microseconds) so we move the raw timestamp string into the special + # logging.googleapis.com field and let Cloud Logging parse it — it + # handles ISO8601 with microseconds natively. + promote_severity: + type: modify_fields + fields: + severity: + move_from: jsonPayload.severity + + # ── Exclude high-volume INFO http_request logs from storage ─────────── + # http_request logs are emitted for every API call (needed for P95 + # latency and error rate metrics) but don't need to be stored long-term. + # This reduces Cloud Logging ingestion costs without losing error coverage. + # Remove or comment out this processor if you want full HTTP logs stored. + exclude_http_info: + type: exclude_logs + match_any: + - 'jsonPayload.message = "http_request" AND jsonPayload.severity = "INFO"' + + service: + pipelines: + + # tap_lms structured logs pipeline + tap_lms_pipeline: + receivers: [tap_lms_structured] + processors: [parse_tap_lms_json, promote_severity, exclude_http_info] + + # rag_service structured logs pipeline + rag_service_pipeline: + receivers: [rag_service_structured] + processors: [parse_tap_lms_json, promote_severity] + + # Frappe application logs (plain text — no JSON parsing) + frappe_pipeline: + receivers: [frappe_app] + + # nginx logs + nginx_pipeline: + receivers: [nginx_access, nginx_error] + + # Feedback consumer logs + feedback_consumer_pipeline: + receivers: [feedback_consumer] + + # Supervisor logs + supervisor_pipeline: + receivers: [supervisor] + +# ── Hardware metrics ─────────────────────────────────────────────────────────── +# The Ops Agent collects CPU, memory, disk, network, and process metrics +# automatically via the built-in hostmetrics receiver. No additional +# configuration is needed — these are already active by default. +# +# Metrics are visible in: +# GCP Console → Monitoring → Metrics Explorer +# Filter: resource.type = "gce_instance" +# Metric prefix: agent.googleapis.com/ +# +# Key metrics available out of the box: +# agent.googleapis.com/cpu/utilization +# agent.googleapis.com/memory/usage (state: used/free/cached/buffered) +# agent.googleapis.com/disk/usage (device: sda1 etc.) +# agent.googleapis.com/network/tcp_connections +# agent.googleapis.com/processes/count +# +# To increase collection frequency (default is 60s), uncomment below: +# metrics: +# receivers: +# hostmetrics: +# type: hostmetrics +# collection_interval: 30s diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000..28274ae2 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,215 @@ +# scripts/ + +Utility scripts for local development and testing. All scripts run inside the +Frappe bench environment (`frappe.connect()` is called internally). + +```bash +cd /path/to/frappe-bench +python apps/frappe_tap/scripts/