Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions examples/airflow-k8s-pipeline/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,8 @@ aws --endpoint-url=http://localhost:9000 s3 cp NRM_<...>FDR_<...>.VIC s3://ids-p
│ ├── listener/
│ │ ├── Dockerfile
│ │ ├── listener.py # RabbitMQ consumer + Airflow trigger
│ │ ├── requirements.in # Direct dependencies (pika, requests)
│ │ ├── requirements.txt # Hash-locked pins (pip-compile --generate-hashes)
│ │ └── deployment.yaml
│ ├── worker/
│ │ └── Dockerfile # tig-worker (TIG base + AWS CLI)
Expand Down
11 changes: 8 additions & 3 deletions examples/airflow-k8s-pipeline/k8s/listener/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,16 @@ FROM python:3.11-slim

WORKDIR /app

# Install dependencies
RUN pip install --no-cache-dir pika requests
# Install locked, hash-pinned dependencies from wheels only (regenerate with
# `pip-compile --generate-hashes --strip-extras requirements.in` on Python 3.11)
COPY requirements.txt /app/
RUN pip install --no-cache-dir --only-binary :all: --require-hashes -r requirements.txt

# Copy listener script
COPY listener.py /app/

# Run listener
# Run listener as an unprivileged user
RUN useradd --uid 1000 --user-group --no-create-home listener
USER 1000:1000

CMD ["python", "-u", "listener.py"]
1 change: 1 addition & 0 deletions examples/airflow-k8s-pipeline/k8s/listener/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ spec:
labels:
app: ids-listener
spec:
automountServiceAccountToken: false
containers:
- name: listener
image: ids-listener:latest
Expand Down
2 changes: 2 additions & 0 deletions examples/airflow-k8s-pipeline/k8s/listener/requirements.in
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
pika
requests
200 changes: 200 additions & 0 deletions examples/airflow-k8s-pipeline/k8s/listener/requirements.txt

Large diffs are not rendered by default.

8 changes: 7 additions & 1 deletion examples/airflow-k8s-pipeline/k8s/minio/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ spec:
labels:
app: minio
spec:
automountServiceAccountToken: false
# MinIO validates the AMQP target at startup, so wait for the broker.
initContainers:
- name: wait-for-rabbitmq
Expand Down Expand Up @@ -73,9 +74,14 @@ spec:
requests:
memory: "256Mi"
cpu: "100m"
ephemeral-storage: "1Gi"
limits:
memory: "1Gi"
cpu: "500m"
# Covers the /data emptyDir, which holds every object in the demo bucket
# (~260 MiB per stereo-pair run); half of minikube's default 20 GB disk.
ephemeral-storage: "10Gi"
volumes:
- name: data
emptyDir: {}
emptyDir:
sizeLimit: 10Gi
3 changes: 3 additions & 0 deletions examples/airflow-k8s-pipeline/k8s/rabbitmq/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ spec:
labels:
app: rabbitmq
spec:
automountServiceAccountToken: false
containers:
- name: rabbitmq
image: rabbitmq:3.13-management
Expand All @@ -79,6 +80,8 @@ spec:
command: ["rabbitmq-diagnostics", "-q", "check_port_connectivity"]
initialDelaySeconds: 10
periodSeconds: 10
# The Erlang CLI takes >1s (the default) to start under the 500m CPU limit.
timeoutSeconds: 5
resources:
requests:
memory: "256Mi"
Expand Down
8 changes: 8 additions & 0 deletions examples/airflow-k8s-pipeline/k8s/seed/job.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ spec:
template:
spec:
restartPolicy: OnFailure
automountServiceAccountToken: false
containers:
- name: seed
image: minio/mc:RELEASE.2024-11-21T17-21-54Z
Expand All @@ -19,6 +20,13 @@ spec:
value: "minioadmin"
- name: MINIO_ROOT_PASSWORD
value: "minioadmin"
resources:
requests:
memory: "64Mi"
cpu: "50m"
limits:
memory: "128Mi"
cpu: "200m"
volumeMounts:
- name: seed-script
mountPath: /scripts
Expand Down
6 changes: 5 additions & 1 deletion examples/airflow-k8s-pipeline/k8s/worker/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,13 @@ FROM ghcr.io/nasa-ammos/tig/terrain-intelligence-generator:opensource
USER root

# Install AWS CLI v2 (official bundle, no python dependency)
RUN curl -sSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip \
RUN curl -sSL --proto '=https' --tlsv1.2 "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip \
&& (command -v unzip >/dev/null 2>&1 || (microdnf install -y unzip || dnf install -y unzip || yum install -y unzip)) \
&& unzip -q /tmp/awscliv2.zip -d /tmp \
&& /tmp/aws/install \
&& rm -rf /tmp/awscliv2.zip /tmp/aws \
&& aws --version

# Wrappers only write under /tmp, so tasks run as an unprivileged user.
RUN useradd --uid 1000 --user-group --create-home tig
USER 1000:1000
Loading