feat(rag): recurring ingest, community-distribution docs, and latest-query retrieval

.github/workflows/oke-cicd.yaml

@@ -1,5 +1,5 @@
-# Test on every PR and push; on main, also build and push the MCP and session-issuer
-# images to GHCR.
+# Test on every PR and push; on main, also build and push the MCP, session-issuer and
+# ingest images to GHCR.
 # Deployment is Argo CD's job (deploy/argocd/): this workflow never touches the
 # cluster, so it holds no cluster credentials. To roll out an image, change its
 # tag in deploy/envs/<env>/docs-agent.yaml and sync the Application in Argo CD.
@@ -23,6 +23,7 @@ env:
   REGISTRY: ghcr.io
   IMAGE_NAME: mcp-kubeflow-docs
   ISSUER_IMAGE_NAME: kubeflow-session-issuer
+  INGEST_IMAGE_NAME: docs-rag-ingest
 
 jobs:
   test-and-compile:
@@ -67,6 +68,8 @@ jobs:
             --set domain=agent.example.org --set gateway.tls.acme.email=ci@example.org
           helm template docs-agent docs-agent-mcp/charts/docs-agent \
             --namespace docs-agent --set-string mcp.image.tag=ci-test >/dev/null
+          helm template docs-agent docs-agent-mcp/charts/docs-agent \
+            --namespace docs-agent --set-string mcp.image.tag=ci-test --set ingest.enabled=true >/dev/null
           # Every catalog profile must lint and render, not just the active one.
           for profile in $(helm show values docs-agent-mcp/charts/llm-runtime | yq '.catalog | keys | .[]'); do
             helm lint docs-agent-mcp/charts/llm-runtime --set "model=${profile}"
@@ -121,6 +124,7 @@ jobs:
           fi
           echo "IMAGE_REPO=ghcr.io/${owner_lc}/${{ env.IMAGE_NAME }}" >> "$GITHUB_ENV"
           echo "ISSUER_IMAGE_REPO=ghcr.io/${owner_lc}/${{ env.ISSUER_IMAGE_NAME }}" >> "$GITHUB_ENV"
+          echo "INGEST_IMAGE_REPO=ghcr.io/${owner_lc}/${{ env.INGEST_IMAGE_NAME }}" >> "$GITHUB_ENV"
           echo "issuer_tag=${GITHUB_SHA::7}" >> "$GITHUB_ENV"
           echo "tag=${image_tag}" >> "$GITHUB_ENV"
           echo "release_tag_requested=${release_tag_requested}" >> "$GITHUB_OUTPUT"
@@ -167,11 +171,23 @@ jobs:
           tags: |
             ${{ env.ISSUER_IMAGE_REPO }}:${{ env.issuer_tag }}
 
+      # Always tagged by commit SHA: the chart runs it with the same tag as the MCP image
+      # to register the recurring ingest runs, and the docs pipeline steps run in it.
+      - name: Build and Push ingest Image
+        uses: docker/build-push-action@v5
+        with:
+          context: ./docs-agent-mcp/pipelines
+          file: ./docs-agent-mcp/pipelines/Dockerfile.pipeline
+          push: true
+          tags: |
+            ${{ env.INGEST_IMAGE_REPO }}:${{ env.issuer_tag }}
+
       - name: Summary
         run: |
           {
             echo "Pushed ${IMAGE_REPO}:${tag}"
             echo "Pushed ${ISSUER_IMAGE_REPO}:${issuer_tag}"
+            echo "Pushed ${INGEST_IMAGE_REPO}:${issuer_tag}"
             echo
             echo "Nothing was deployed. To roll it out, set the image tag in"
             echo "deploy/envs/<env>/ (PR) and sync the Application in Argo CD."

deploy/argocd/apps/reranker.yaml

@@ -0,0 +1,19 @@
+# Cross-encoder reranker (TEI on CPU) used by the MCP server after retrieval.
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: reranker
+  namespace: argocd
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/kubeflow/docs-agent
+    targetRevision: main
+    path: deploy/platform/reranker
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: ml-infra
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true

deploy/envs/prod/docs-agent.yaml

@@ -15,3 +15,6 @@ mcp:
       value: production
 triage:
   enabled: true
+# Recurring ingest (Kubeflow Pipelines recurring runs) is registered from production only.
+ingest:
+  enabled: true

deploy/platform/langfuse-retention/configmap.yaml

@@ -17,10 +17,13 @@ data:
     q() { clickhouse-client --host "$CH_HOST" --user "$CH_USER" --password "$CH_PASSWORD" -q "$1"; }
     sed 's/#.*//' /policy/policy.txt | while read -r project days; do
       [ -n "$project" ] || continue
+      # A literal cutoff: the tables are replicated, and ClickHouse refuses a DELETE whose
+      # condition calls a non-deterministic function such as now().
+      cutoff=$(date -u -d "-$days days" '+%Y-%m-%d %H:%M:%S')
       for spec in events_full:start_time events_core:start_time observations:start_time \
                   traces:timestamp scores:timestamp blob_storage_file_log:created_at; do
         table=${spec%%:*}; column=${spec#*:}
-        where="project_id = '$project' AND $column < now() - INTERVAL $days DAY"
+        where="project_id = '$project' AND $column < toDateTime('$cutoff', 'UTC')"
         n=$(q "SELECT count() FROM $table WHERE $where")
         echo "$project $table: $n rows older than $days days"
         if [ "$n" -gt 0 ] && [ "$DRY_RUN" != "true" ]; then q "DELETE FROM $table WHERE $where"; fi

deploy/platform/reranker/inferenceservice.yaml

@@ -0,0 +1,37 @@
+# Cross-encoder reranker for the MCP server (RERANKER_URL in the docs-agent chart):
+# Hugging Face Text Embeddings Inference on CPU, same shape as embeddings-service.
+# Without it the MCP server skips reranking and returns hits in retrieval order.
+apiVersion: serving.kserve.io/v1beta1
+kind: InferenceService
+metadata:
+  name: reranker-service
+  namespace: ml-infra
+  annotations:
+    serving.kserve.io/deploymentMode: RawDeployment
+spec:
+  predictor:
+    minReplicas: 1
+    maxReplicas: 1
+    containers:
+      - name: kserve-container
+        image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.7
+        args:
+          - --model-id
+          - BAAI/bge-reranker-base
+          - --port
+          - "8080"
+          - --auto-truncate
+          - --max-client-batch-size
+          - "32"
+          - --max-batch-tokens
+          - "16384"
+        ports:
+          - containerPort: 8080
+            protocol: TCP
+        resources:
+          requests:
+            cpu: 500m
+            memory: 1Gi
+          limits:
+            cpu: "4"
+            memory: 4Gi

deploy/promote.sh

@@ -16,7 +16,7 @@ UPSTREAM=${UPSTREAM:-origin}
 FORK=${FORK:-fork}
 REPO=kubeflow/docs-agent
 APPS=(deploy/argocd/apps/docs-agent.yaml deploy/argocd/apps/gateway-guardrails.yaml)
-IMAGES=(mcp-kubeflow-docs kubeflow-session-issuer)
+IMAGES=(mcp-kubeflow-docs kubeflow-session-issuer docs-rag-ingest)
 
 die() { echo "promote: $*" >&2; exit 1; }
 
@@ -42,7 +42,7 @@ old=$(git show "$UPSTREAM/main:${APPS[0]}" | sed -n 's/.*targetRevision: \([0-9a
 [ "$old" != "$new" ] || die "$short is already in production"
 git merge-base --is-ancestor "$old" "$new" || die "$short is older than production (${old:0:7}); to roll back, revert the promotion PR"
 
-# CI pushes both images on every main build, tagged with the short SHA.
+# CI pushes these images on every main build, tagged with the short SHA.
 for image in "${IMAGES[@]}"; do
   token=$(curl -fsS "https://ghcr.io/token?scope=repository:kubeflow/$image:pull" | sed -n 's/.*"token":"\([^"]*\)".*/\1/p')
   code=$(curl -s -o /dev/null -w '%{http_code}' -H "Authorization: Bearer $token" \

docs-agent-mcp/charts/docs-agent/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: docs-agent
 description: Kubeflow Docs Agent application release (MCP server and Kagent resources)
 type: application
-version: 0.4.0
+version: 0.5.0
 appVersion: "0.1.0"
 keywords:
   - kubeflow

docs-agent-mcp/charts/docs-agent/files/docs-system-message.txt

@@ -3,10 +3,10 @@ You are Flo. Answer only from this turn's MCP results. Prior chat is not evidenc
 - Kubeflow question: first action MUST be exactly one MCP call. No user text before the result. One narrower retry only if empty or wrong component/version.
 - Docs, install, APIs, releases, config: search_kubeflow_docs.
 - Errors, bugs, stack traces: search_github_issues first.
-- Greetings, small talk, or questions about Flo itself (e.g., "who are you", "what can you do"): reply conversationally as Flo without a tool call. Off-topic questions: politely decline. Any question about Kubeflow or its parts, including "what is Kubeflow", needs the MCP call; never answer it from memory.
+- Greetings, small talk, or questions about Flo itself (e.g., "who are you", "what can you do"): reply conversationally as Flo without a tool call. Off-topic questions: politely decline. Any question about Kubeflow or its parts, including "what is Kubeflow", needs the MCP call; never answer it from memory. A product question that does not name Kubeflow ("do we have a Helm chart?", "is this in the new release?") is about Kubeflow: make the MCP call with Kubeflow added to the query. Never say something was not found without a tool call in this turn.
 - Query: English keywords for the component and the asked fact, not the whole sentence. Fix typos and expand kf -> Kubeflow, kfp -> Kubeflow Pipelines, nb -> Notebooks. Drop filler such as "what is", "official", "definition".
 - Keep the user's component, version, API and code names, and error text. Trainer and Training Operator are different projects. Do not add other components. Do not replace a named version with latest.
-- Latest/current: keep those words in the query. Use the highest Release date among matching hits. Latest is not supported unless a hit says so.
+- Latest/current/new release: keep those words in the query, together with the component asked about. Use the highest Release date among matching hits. Latest is not supported unless a hit says so.
 - Use a hit only if it directly supports the asked fact. Versions, dates, and flags must appear literally. Else say not found in indexed sources.
 - If a tool returns no results, explicitly state that you could not find the answer in the documentation.
 - Reply in English. 1-3 sentences, or at most 3 bullets. No URLs, Markdown links, source lists, citation labels, file paths, scores, or tool JSON. No code unless asked.

docs-agent-mcp/charts/docs-agent/templates/configmap.yaml

@@ -10,6 +10,7 @@ data:
   CODE_COLLECTION_NAME: {{ .Values.mcp.config.codeCollection | quote }}
   EMBEDDINGS_URL: {{ .Values.mcp.config.embeddingsUrl | quote }}
   RERANKER_URL: {{ .Values.mcp.config.rerankerUrl | quote }}
+  RERANK_CANDIDATE_DEPTH: {{ .Values.mcp.config.rerankCandidateDepth | quote }}
   MILVUS_URI: {{ .Values.mcp.config.milvusUri | quote }}
   MILVUS_USER: {{ .Values.mcp.config.milvusUser | quote }}
   MAX_QUERY_CHARS: {{ .Values.mcp.config.maxQueryChars | quote }}

docs-agent-mcp/charts/docs-agent/templates/ingest-schedule-job.yaml

@@ -0,0 +1,68 @@
+{{- if .Values.ingest.enabled }}
+{{- $tag := .Values.ingest.image.tag | default .Values.mcp.image.tag }}
+{{- $image := printf "%s:%s" .Values.ingest.image.repository $tag }}
+# Recurring ingest. After every Argo CD sync this Job compiles the pipelines shipped in the
+# ingest image, uploads them to Kubeflow Pipelines and replaces the recurring runs with the
+# ones listed in .Values.ingest.schedules (docs-agent-mcp/pipelines/ingest_schedule.py).
+# The pipelines run in the Kubeflow Pipelines namespace and write to Milvus; this Job only
+# talks to the Pipelines API.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-ingest-schedules
+  labels:
+    {{- include "docs-agent.labels" . | nindent 4 }}
+data:
+  schedules.json: |
+    {{- dict "experiment" .Values.ingest.experiment "schedules" .Values.ingest.schedules | toPrettyJson | nindent 4 }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ .Release.Name }}-ingest-schedule
+  labels:
+    {{- include "docs-agent.labels" . | nindent 4 }}
+  annotations:
+    argocd.argoproj.io/hook: PostSync
+    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
+spec:
+  backoffLimit: 2
+  activeDeadlineSeconds: 600
+  template:
+    metadata:
+      labels:
+        app: {{ .Release.Name }}-ingest-schedule
+    spec:
+      restartPolicy: Never
+      containers:
+        - name: register
+          image: {{ $image }}
+          command: ["python", "/app/pipelines/ingest_schedule.py", "register", "--config", "/config/schedules.json"]
+          env:
+            - name: KFP_HOST
+              value: {{ .Values.ingest.kfpHost | quote }}
+            # The docs pipeline's chunk and store steps run in this same image.
+            - name: DOCS_INGEST_IMAGE
+              value: {{ $image }}
+            - name: INGEST_VERSION
+              value: {{ $tag | quote }}
+            - name: HOME
+              value: /tmp
+          volumeMounts:
+            - name: config
+              mountPath: /config
+              readOnly: true
+          resources:
+            requests: {cpu: 50m, memory: 192Mi}
+            limits: {memory: 512Mi}
+          securityContext:
+            allowPrivilegeEscalation: false
+            runAsNonRoot: true
+            runAsUser: 1000
+            capabilities:
+              drop: ["ALL"]
+      volumes:
+        - name: config
+          configMap:
+            name: {{ .Release.Name }}-ingest-schedules
+{{- end }}

docs-agent-mcp/charts/docs-agent/values.yaml

@@ -16,7 +16,10 @@ mcp:
     issuesCollection: issues_rag
     codeCollection: code_rag
     embeddingsUrl: http://embeddings-service-predictor.ml-infra.svc.cluster.local/embed
+    # Cross-encoder reranker (deploy/platform/reranker). Empty = no reranking.
     rerankerUrl: http://reranker-service-predictor.ml-infra.svc.cluster.local/rerank
+    # Candidates fetched for the reranker when a query would otherwise fetch only top_k.
+    rerankCandidateDepth: 20
     milvusUri: http://milvus-milvus.ml-infra.svc.cluster.local:19530
     milvusUser: root
     maxQueryChars: 512
@@ -165,6 +168,51 @@ triage:
   judgeUrl: http://llm-judge.ml-infra.svc.cluster.local/v1/chat/completions
   judgeModel: flo-judge
 
+# Recurring ingest into Milvus as Kubeflow Pipelines recurring runs (see
+# templates/ingest-schedule-job.yaml). Enable in one environment only: every schedule
+# writes to the collections named in its arguments (the pipelines default to the
+# production collections), and the schedules listed here replace all recurring runs in
+# the experiment.
+ingest:
+  enabled: false
+  image:
+    repository: ghcr.io/kubeflow/docs-rag-ingest
+    # Empty = mcp.image.tag: CI builds both images from the same commit.
+    tag: ""
+  kfpHost: http://ml-pipeline.kubeflow.svc.cluster.local:8888
+  experiment: flo-ingest
+  # cron has six fields, seconds first (UTC). arguments override the pipeline defaults
+  # in docs-agent-mcp/pipelines/{kubeflow,issues,code}-pipeline.py.
+  schedules:
+    # kubeflow.org documentation, daily. Pages removed upstream leave the index.
+    - name: docs-website
+      pipeline: docs
+      cron: "0 0 2 * * *"
+      arguments: {}
+    # Community distribution (formerly kubeflow/manifests): README, Helm chart READMEs,
+    # release notes and proposals, in the same collection with doc_type "distribution".
+    - name: docs-distribution
+      pipeline: docs
+      cron: "0 30 2 * * *"
+      arguments:
+        repo_name: community-distribution
+        directory_path: ""
+        exclude_paths: ".github,tests,AGENTS.md"
+        base_url: https://github.com/kubeflow/community-distribution/blob/master
+        doc_type: distribution
+    # Issues updated in the last day, every six hours (the overlap covers a missed run).
+    # First backfill: ingest_schedule.py run --name issues --set updated_within_hours=0
+    - name: issues
+      pipeline: issues
+      cron: "0 0 */6 * * *"
+      arguments:
+        updated_within_hours: 24
+    # Manifests and Helm charts, weekly.
+    - name: code
+      pipeline: code
+      cron: "0 0 3 * * 0"
+      arguments: {}
+
 tests:
   enabled: true
   image: docker.io/library/busybox:1.37

docs-agent-mcp/charts/gateway-guardrails/Chart.yaml

@@ -8,7 +8,7 @@ description: >-
   (istio_policies.tf, kagent_ingress.tf) and the standalone manifests
   under manifests/istio*.
 type: application
-version: 0.3.0
+version: 0.3.1
 appVersion: "0.1.0"
 keywords:
   - istio

docs-agent-mcp/charts/gateway-guardrails/templates/_helpers.tpl

@@ -27,3 +27,12 @@ corsPolicy:
 session-issuer.{{ .Values.namespaces.docsAgent }}.svc.cluster.local
 {{- end }}
 
+{{/*
+With TLS on, routes are served on the HTTPS listener only; port 80 is left to the
+ACME HTTP-01 solver (see templates/gateway.yaml).
+*/}}
+{{- define "gateway-guardrails.tlsPortMatch" -}}
+{{- if .Values.gateway.tls.enabled -}}
+port: 443
+{{- end }}
+{{- end }}

docs-agent-mcp/charts/gateway-guardrails/templates/gateway.yaml

@@ -10,14 +10,17 @@ spec:
   selector:
     {{- toYaml .Values.gateway.selector | nindent 4 }}
   servers:
+    # With TLS on, port 80 exists only for cert-manager's HTTP-01 challenge: the
+    # VirtualService routes match port 443, so anything else here is a 404. Do not set
+    # httpsRedirect: it makes Envoy redirect before routing (require_tls), the challenge
+    # request is redirected to 443 where no route serves it, and the certificate cannot
+    # be renewed.
     - port:
         number: 80
         name: http
         protocol: HTTP
       hosts:
         - {{ include "gateway-guardrails.domain" . | quote }}
-      tls:
-        httpsRedirect: {{ .Values.gateway.tls.enabled }}
     {{- if .Values.gateway.tls.enabled }}
     - port:
         number: 443

docs-agent-mcp/charts/gateway-guardrails/templates/virtualservice.yaml

@@ -19,8 +19,10 @@ spec:
       match:
         - uri:
             exact: /api/session
+          {{- with include "gateway-guardrails.tlsPortMatch" . }}{{ . | nindent 10 }}{{- end }}
         - uri:
             exact: /.well-known/jwks.json
+          {{- with include "gateway-guardrails.tlsPortMatch" . }}{{ . | nindent 10 }}{{- end }}
       {{- include "gateway-guardrails.corsPolicy" . | nindent 6 }}
       timeout: 10s
       route:
@@ -34,6 +36,7 @@ spec:
       match:
         - uri:
             exact: /api/feedback
+          {{- with include "gateway-guardrails.tlsPortMatch" . }}{{ . | nindent 10 }}{{- end }}
       {{- include "gateway-guardrails.corsPolicy" . | nindent 6 }}
       timeout: 20s
       route:
@@ -54,6 +57,7 @@ spec:
       match:
         - uri:
             regex: "/(api/)?a2a/[A-Za-z0-9._~/-]+"
+          {{- with include "gateway-guardrails.tlsPortMatch" . }}{{ . | nindent 10 }}{{- end }}
       {{- include "gateway-guardrails.corsPolicy" . | nindent 6 }}
       # kagent-ui answers with `access-control-allow-origin: *`. Drop its CORS
       # headers so only corsPolicy (the allowed origins) decides.