From 1e36eb0ce76bed0eb458247fa1861e4619ef6638 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:57:08 +0000 Subject: [PATCH 1/2] examples(airflow): address SonarCloud findings in k8s manifests and Dockerfiles - Disable service-account token automount on listener, MinIO, RabbitMQ and seed pods - Add seed job resource limits and MinIO ephemeral-storage limit (+ emptyDir sizeLimit) - Run listener and tig-worker images as uid 1000 - Restrict the AWS CLI download to HTTPS/TLS1.2 - Install listener deps from a hash-locked requirements.txt, wheels only - Give the RabbitMQ readiness probe a 5s timeout (CLI exceeds the 1s default) Co-Authored-By: jason.han --- examples/airflow-k8s-pipeline/README.md | 2 + .../k8s/listener/Dockerfile | 11 +- .../k8s/listener/deployment.yaml | 1 + .../k8s/listener/requirements.in | 2 + .../k8s/listener/requirements.txt | 200 ++++++++++++++++++ .../k8s/minio/deployment.yaml | 7 +- .../k8s/rabbitmq/deployment.yaml | 3 + .../airflow-k8s-pipeline/k8s/seed/job.yaml | 8 + .../k8s/worker/Dockerfile | 6 +- 9 files changed, 235 insertions(+), 5 deletions(-) create mode 100644 examples/airflow-k8s-pipeline/k8s/listener/requirements.in create mode 100644 examples/airflow-k8s-pipeline/k8s/listener/requirements.txt diff --git a/examples/airflow-k8s-pipeline/README.md b/examples/airflow-k8s-pipeline/README.md index 7dce330..ab09a5b 100644 --- a/examples/airflow-k8s-pipeline/README.md +++ b/examples/airflow-k8s-pipeline/README.md @@ -279,6 +279,8 @@ aws --endpoint-url=http://localhost:9000 s3 cp NRM_<...>FDR_<...>.VIC s3://ids-p │ ├── listener/ │ │ ├── Dockerfile │ │ ├── listener.py # RabbitMQ consumer + Airflow trigger +│ │ ├── requirements.in # Direct dependencies (pika, requests) +│ │ ├── requirements.txt # Hash-locked pins (pip-compile --generate-hashes) │ │ └── deployment.yaml │ ├── worker/ │ │ └── Dockerfile # tig-worker (TIG base + AWS CLI) diff --git a/examples/airflow-k8s-pipeline/k8s/listener/Dockerfile b/examples/airflow-k8s-pipeline/k8s/listener/Dockerfile index e560a37..3bb3526 100644 --- a/examples/airflow-k8s-pipeline/k8s/listener/Dockerfile +++ b/examples/airflow-k8s-pipeline/k8s/listener/Dockerfile @@ -2,11 +2,16 @@ FROM python:3.11-slim WORKDIR /app -# Install dependencies -RUN pip install --no-cache-dir pika requests +# Install locked, hash-pinned dependencies from wheels only (regenerate with +# `pip-compile --generate-hashes --strip-extras requirements.in` on Python 3.11) +COPY requirements.txt /app/ +RUN pip install --no-cache-dir --only-binary :all: --require-hashes -r requirements.txt # Copy listener script COPY listener.py /app/ -# Run listener +# Run listener as an unprivileged user +RUN useradd --uid 1000 --user-group --no-create-home listener +USER 1000:1000 + CMD ["python", "-u", "listener.py"] diff --git a/examples/airflow-k8s-pipeline/k8s/listener/deployment.yaml b/examples/airflow-k8s-pipeline/k8s/listener/deployment.yaml index 7809046..d6abc80 100644 --- a/examples/airflow-k8s-pipeline/k8s/listener/deployment.yaml +++ b/examples/airflow-k8s-pipeline/k8s/listener/deployment.yaml @@ -13,6 +13,7 @@ spec: labels: app: ids-listener spec: + automountServiceAccountToken: false containers: - name: listener image: ids-listener:latest diff --git a/examples/airflow-k8s-pipeline/k8s/listener/requirements.in b/examples/airflow-k8s-pipeline/k8s/listener/requirements.in new file mode 100644 index 0000000..0f665cd --- /dev/null +++ b/examples/airflow-k8s-pipeline/k8s/listener/requirements.in @@ -0,0 +1,2 @@ +pika +requests diff --git a/examples/airflow-k8s-pipeline/k8s/listener/requirements.txt b/examples/airflow-k8s-pipeline/k8s/listener/requirements.txt new file mode 100644 index 0000000..7aff08c --- /dev/null +++ b/examples/airflow-k8s-pipeline/k8s/listener/requirements.txt @@ -0,0 +1,200 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --generate-hashes --no-emit-index-url --output-file=requirements.txt --strip-extras requirements.in +# +certifi==2026.10.23 \ + --hash=sha256:a9895e6e2758b371e0545e321a3a8fd1fcfd9829ada8824a936dfb16aaaf36cd \ + --hash=sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1 + # via requests +charset-normalizer==3.5.2 \ + --hash=sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e \ + --hash=sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf \ + --hash=sha256:04851f73ae72b8413dddadb16a49dfee95263553741fd42d546f7d66907e6be5 \ + --hash=sha256:0521c5665880b33d603717defa76c094048900010897909952397feb3039da56 \ + --hash=sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26 \ + --hash=sha256:0891b9d3903c5571c03771ca669a4b0ec5618ca722a5c957d3d29cd4e5062848 \ + --hash=sha256:0c951d5e6dd9c2ff60609476752bee49da4206adde960ebc247766937f72e718 \ + --hash=sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93 \ + --hash=sha256:114e4d0c92d618409ed82a99e22b5c5e768fe995f2973f78265f4524f49d4640 \ + --hash=sha256:11912e4bb14baae7c5d8791aa55ba0a3a03ec6729073307b0f57270abaa713d3 \ + --hash=sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875 \ + --hash=sha256:124fbf1a8ff966d87ae05bb8bd45a71f966055ed8bba320d0c7cf450bc5f4d0e \ + --hash=sha256:1461ac396c4fdb983a675f20aa555624f0ee18ac83d832b9244ffff3d8055275 \ + --hash=sha256:1503bccbeb36d5527790c3930327704c39af22de3112f1b1666a9f3ce15ee204 \ + --hash=sha256:15bb4005af6320d259dc7593ca84a38d7fe06a421dbcf7b910ae23979101e787 \ + --hash=sha256:15c44f7edfd477b06f517a5cc317fc1707edb9de2c865f43d4b6513907473234 \ + --hash=sha256:16fa0eccf81304b79c5cd87f9271c3b85dd9dd99245e4422ae9c0dd45e0f99d3 \ + --hash=sha256:183b88127acdb4fabe59d951ab424faf1af7b63cdbb5f776186c1ea2ffcaed98 \ + --hash=sha256:195c26fb65950f8fce54e26349852b7bdd7c5f120aeefbcc440b8a20faaed4a3 \ + --hash=sha256:1afb975bd5d68d5ce9f6b6d44fdf2f7e34b895a35e95708a7a91b20a3b51d187 \ + --hash=sha256:1b4cbc7c3491ccb4aa17fcd8165649d01cf39f76de1696da8631b5f71b85401d \ + --hash=sha256:1bc0baf5ef96b6ede57d47f4b8fe4d9d84019c3bfcbeb20a41edc6a6ee341f1f \ + --hash=sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7 \ + --hash=sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011 \ + --hash=sha256:211d5a3eb6af8f513b8d4ca19a8c1b7accab1b5f0d3175f9826b03c1a920dc1f \ + --hash=sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869 \ + --hash=sha256:254eb48b9fa5ee9898a3c445825a1f340fe53712a098904b39b0bddba8ea3cb1 \ + --hash=sha256:2625388c6c754520c37abaf3b41eb34d1cc4a373f457898f08606c8e362b891d \ + --hash=sha256:281cb91036248400f4cc957495cccd44c275c2e0c5854f7e45ac5cf7dc193847 \ + --hash=sha256:28a15fdad492a99b6eccfaaed66ef3f74050680545ea61ec8b2f4c538f1f1320 \ + --hash=sha256:28b4f0d66fb834ff90f28209ac7bce77868c45d8c93e26f906709d9b7c2e1af9 \ + --hash=sha256:2a925889534b3748302dae5dead07cc13480de1dac3aea80a941b729b471ef93 \ + --hash=sha256:2b7b3bbfb4fe8ef40600792d762fbaa9057559f9d3fad209525b7a22b99e91fd \ + --hash=sha256:2c9ad19a6cfcd5ea5c0d41161d22f9df1dcc277e9bef2751391334546a314c00 \ + --hash=sha256:2cc961b171b3f3440f410489ab3573e86aea8736134ebbb40ea1338b7f0831bc \ + --hash=sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0 \ + --hash=sha256:2e06a3a98f916dd41d27f3105e02e7a40181c98c94b9158733d03a6f80506c09 \ + --hash=sha256:304d5463e65a35d7bb0850550e0780395395f6fcf452f04db7d5ca7cecc425ac \ + --hash=sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621 \ + --hash=sha256:30fcd120b732aa79317f08dee04d7de0847822e4cf7ee0e9f445bb958832252c \ + --hash=sha256:31f3930700408d211f13378ccbe1c40845d8da54bd0681fac3a9b5aae81c7aa8 \ + --hash=sha256:34276fd796040bf0993ab33a369aa572e6979c7aab225a88893667ad8eac8f7a \ + --hash=sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51 \ + --hash=sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0 \ + --hash=sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef \ + --hash=sha256:3d14b50de6bf4d0edf857a9386836846f982b8f524e188e2e68b96d702bcf4aa \ + --hash=sha256:3d21b8b13c7592db2ac5e544a6d83187b995257472b0c9e8351b6d507ae37ed6 \ + --hash=sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649 \ + --hash=sha256:3ddacd27458c45bdacd6bd6db644bfb730efbf9e830310186e3045c9c5be8fb2 \ + --hash=sha256:3df041de8887954562c9b261cba85ca0e9ded74048daf125f45edcfaa4832229 \ + --hash=sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e \ + --hash=sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd \ + --hash=sha256:443eae2bf318abeaf6f15d785138f71fd6de770e99a92158b8b814265e079115 \ + --hash=sha256:447441e76ec720b15e64418d32e092297340387053047c7c694f579efb0ee1d9 \ + --hash=sha256:4495c5002a7b28557e7e222e77e0b661183e432b7d6d2e788101e3f240e05b8c \ + --hash=sha256:44bd4fbb29dfbeba60e7d2bd000c59e4b21ddb3cc53912b14048d37092706d7c \ + --hash=sha256:4685902cf26edf013ed7a3da0f426ebba7a00ebb9541386d835afbf002c11cab \ + --hash=sha256:498dc3188ca05a68231ac3fdbfc7f57eb67e1343c30e0fea17f8218c1599b253 \ + --hash=sha256:4c2b5031f63e331e3839b40aed2dd6f191e9c07edbde303e7876846ea1946995 \ + --hash=sha256:4d48f2d08b9de5864e2c8744d4461b862fb149a18274abc8b698c45975573438 \ + --hash=sha256:4f87960d57feabfb618e4e0af6e7371645fa26a277860739d6e5d6e0012c92f0 \ + --hash=sha256:50e3adfb96fc189eb27b1cf62d3b598b89b4bb0420d93a3d3e42e137409011be \ + --hash=sha256:51cf45226a9b588d0d2b4880c62d686934b63ab0bd79ca23ab0e9762eb27441b \ + --hash=sha256:52aa6992700996af31f375de0c6bacd402b0097fe40b53c426b9f51a90ebabc7 \ + --hash=sha256:55ea99acb17b9325618de155a0cd6a2e8f5d10be008113e1d433bbb58db543b2 \ + --hash=sha256:56bc200a365efb37383b7852e4cc5898d3b2da5987289b543956cf8cad71018a \ + --hash=sha256:588461c2e8384d309bd63e5826019b6977bc66d629b99ac8737bb795d7b2cb5a \ + --hash=sha256:58ca3755ee7ff7f59b57789ec9833c9de9ea275405cdd240eda1f193112e398a \ + --hash=sha256:58f361dcbab699cf8f42db3f47c8e7fd1036f138c23a5d08de9fde5f425a730c \ + --hash=sha256:598a11a2c7ebaa5334bf698bf29568c9c390abac6a154d8170fedecd1cea38c5 \ + --hash=sha256:59f63901b0031c3136cf64704dcb21de0bbae62ce2c9529bc39d27665463de37 \ + --hash=sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e \ + --hash=sha256:5e2b6b57e9733d39f0c9fd3185efa6b8e29652c4cd8fe94180272cf6ed9a78c4 \ + --hash=sha256:5fb29fb8cd1a46c27a1bf9613ad5ec2599310d46b4025d9556404a6b6a292800 \ + --hash=sha256:6045373d5a89a5ec71afde535db987ca28e76dfa276c2d4c818265b375d4b055 \ + --hash=sha256:619799369eeef6366ed3e8755a5670f4f2f0fb6b30a0fd7264dc0fdc2357058e \ + --hash=sha256:62588a277bfb59def052abd940703fa35107152bf479781a878617d60faf8fb5 \ + --hash=sha256:62603db9a7caa0802eaa28c1c46fecd7b3a263a774069c24c3c28c302448721c \ + --hash=sha256:65cd72beeeca9d3aaea1201e5923859f308f952f9c71de93f06063c79f0f7a3b \ + --hash=sha256:68eb192d85ab8e5f6ec69c2bc6ac0179fbf04a5ac1569d12fbef74883fe102d0 \ + --hash=sha256:6bd128f206a7752ae1f2ab6c61bf8a24ba28913a10df8b14c2637b973ff97a80 \ + --hash=sha256:6be488a102b8cf28d0391d8c4ba7748938ae28b78ad901f8585520fca33ead1a \ + --hash=sha256:7218e8f32b0956cfcd048fd42d9d5779809745ca1d86113ca56f66e7ae1549c4 \ + --hash=sha256:7441d755b7ab94f8d4eb3e43ec05482d760842fd263d003a99102d742cd835e2 \ + --hash=sha256:749e97e1b32313717a565abbe321bc2190bc8b35f1a67e4cdbc7c56c8d8ffe58 \ + --hash=sha256:75a3ceed0724d625d64b86ca20aba182e4df462e04c2414fc941c0f523f06aac \ + --hash=sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc \ + --hash=sha256:78456a747de8dc58360ffa581f30a002baf5aa28cb262536545e91f113ed7639 \ + --hash=sha256:7967d08cf06dee78443b874f98c98036f624f3a4e73e11f9f64f5be4d25393cf \ + --hash=sha256:7a881931aa470808df94a8c380eed2bbbc76cd9dc622310f99665658c821eb6d \ + --hash=sha256:7dcd882da75ef9adf94903b1e3b9419e8aa8fb4c7396822b834b9ef7fb96954f \ + --hash=sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c \ + --hash=sha256:7fdde2c9fd9e3eca40631e024664cf2584272cc8f96308cbe5fdfc930f51d8bc \ + --hash=sha256:8024d00c3faf3fc0c16e07a69f4405e8eac7cc0ab15f65fe6cf43827c4cf72b4 \ + --hash=sha256:80d02b6f04e92601a081dd97b23d3128033098bff5d35d392ddcc0476ea11253 \ + --hash=sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade \ + --hash=sha256:849df64e889b2e17230d58410a03dba311a65b163508fd33679b2b737d4b7858 \ + --hash=sha256:87475fabc8d9996fd9c27debb395e642e8c838d78a00b6e932227a0e06b81e26 \ + --hash=sha256:87e50a3e7cb90af586b6c5faf23e302a970415ac73bd7bd90a515a04b427ef96 \ + --hash=sha256:89b53f3cda69831909888e0494f4fa0bcd3537e3e138dabeb620bd6ad946bae8 \ + --hash=sha256:8a893cc101149f80a653f82062ebc95b34525a2614382e1da5458fe7c6997249 \ + --hash=sha256:8b2bfab86aa71ae13aa41a6a26aab338e0db2b8bc75434b05aea89e011ff35a4 \ + --hash=sha256:8d86d6fc60743dc916eb79e2eb1ec4818e21e427731543af40a3021851174a13 \ + --hash=sha256:915563965d418f986e7e145accc592eae9e1a1be3566ff98a05d7a9ec42a76e1 \ + --hash=sha256:92888bb3187c5ba50500b00b3b310c9f2c651709d28036077680cb5255450a03 \ + --hash=sha256:93223adc95033dd47133a46ccfc316a0139176fd79085762e27202ec56018f03 \ + --hash=sha256:9373ad13ef0d2c0fb761e04e55bfdee5a08b52cef2c882c8fbe9935b1517152e \ + --hash=sha256:9409a8bf35cf78353942504b24a57de3d75b708997a1e4bd8db71ac8633ce364 \ + --hash=sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4 \ + --hash=sha256:9bde855991b7e362c146535e3136a50bfaffc0487d38b33ca7e5edefc6e23849 \ + --hash=sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0 \ + --hash=sha256:9cf9b1a857e25c4baceeb3624e92a56df3668f398c4acba74e174d81fb4d1d3a \ + --hash=sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036 \ + --hash=sha256:a090bb2c68df85450502e3e20d665e3a5af9c65a84d6508ed477badd49166fd3 \ + --hash=sha256:a192e2c40070d92c3ccf777e3a5c4ff515573cd2bb7ed0c537fdadbbec5bbf21 \ + --hash=sha256:a19a731138fc27d5682277d3b9df22855cea1239bce7fcec5f78f42ef2d1f3c3 \ + --hash=sha256:a66c3bc5ab1f0ff2164fc9965ddd611ff0802173f4b9d24554c563f6ab7e1d6e \ + --hash=sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413 \ + --hash=sha256:a89012d6d5476ee112d20d998570ed58df2260a852afb1758809cd6900411d21 \ + --hash=sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346 \ + --hash=sha256:b6856554c4f44d79fc2307d5768854310a8f0096e501c75637542c82292b0429 \ + --hash=sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685 \ + --hash=sha256:b736353c0a625bbd5fcec108576e2385db3496f4f771f785ff32e108d3c3bc45 \ + --hash=sha256:b7fd005a73d9e657273b7a10dc71a9e03c8fb9ee6999798d6918ce095b81ac7f \ + --hash=sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c \ + --hash=sha256:ba0b1d2620edf869789c3879223f52bf2afc5d31b3cb47cc57b3a12c05e2aa9d \ + --hash=sha256:bbbfc8e28816f19d7c0f1816664980c0a9875d01b27cdf8eedddb639d9e108ad \ + --hash=sha256:bd16aabe4a02a297c23417aa17ac6299dbd8c49f673bcd645b4929b11f5a4400 \ + --hash=sha256:c0afc6800ba57ccc350374c5bd6150419915d95ce93cdbab2d783d75eaf30ecb \ + --hash=sha256:c6708715abcf3c73b99508253e961a9967f02fe536532834149574eda6de0d1c \ + --hash=sha256:c7c9ab723cde841fefb34efbad91e87f00a674b1fe1cd0784fde742bf2c154dc \ + --hash=sha256:c8f3d67aeaf55f017982b73683f0e7342ba2f6635a78f69ce89ebb26aa411e5c \ + --hash=sha256:c9790464842f85f437dbbb54417eda1e0e6bfc52dd8d22d6fd1c994b73b2dc74 \ + --hash=sha256:ca403d7e4798f525fdfc78e258820419cbbd0f0ecbab9de7840e3c017cf6b8cf \ + --hash=sha256:d008d90a7f2471519aef0c90dfbe73b3e6e4d5e66ac48e19154c17e89e98b604 \ + --hash=sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f \ + --hash=sha256:d1befeed746d247c81127bb14de9dc3d30edb6e5976d34f83f86ed262b1d9105 \ + --hash=sha256:d2374b62878abb00cd8309b32af6c0b715cd02dec0ca74ef12e5069bdc64144a \ + --hash=sha256:d376bbd28b3a8999db1a103b3b388aee6f1ddeb3e51bc2172993efdcd86e064d \ + --hash=sha256:d4a7319f304a774bed22115bc891618e45f85065ab44ea6acd07d274e750519a \ + --hash=sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1 \ + --hash=sha256:d760fe2a4d7c3b226cb9026d6a842868d52a7901bd98420e1baf14e80da85cf5 \ + --hash=sha256:d913de495d90407cd859d263bee2e5d1a4ed3eb6573c04e70d9ec619a7cbed7f \ + --hash=sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e \ + --hash=sha256:dca9ab98072a5a54ebacebdc45f53e645336b320c667410b061be1ca588ae709 \ + --hash=sha256:ddc7dacc8ece3a182e7f15cb862d1fd616b46d076cb1ae9dd232b2c38b655874 \ + --hash=sha256:ddf19c062bea7a0cc80f519243d2c01dd091be0cf952a0750d4ad576709559f5 \ + --hash=sha256:def79fa35ef0cef8d2accec024f4fdc7ead3012ff02f5215c783f39f03ef8cfc \ + --hash=sha256:df29a0a7107f7011e77f4eebdddec4c7331e24d787a0b21a46d63bdf7445da95 \ + --hash=sha256:e09a3942ecbdee5cce73ea9d42da82b81b72ac1bf031ce069b93b5adf4eac8cd \ + --hash=sha256:e242bb1c5e76e97dfa9e7f209a71e93a01d7f19ffdd5cfbb2e2d55b4f08f8ab0 \ + --hash=sha256:e243bd13217235fc7290c621941c3f5cc8b66e4872495be821d7436ba2fb838d \ + --hash=sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3 \ + --hash=sha256:e4e81e09c1578b8df602e3db08b0b3ea0a6947ad612f52bf8dc5ea8d47691f0c \ + --hash=sha256:e54da4baf05720032d527874d40b65fa4d7e5c6c6a43d0c3adbeffcaf275a2b3 \ + --hash=sha256:e80e6c2f55656b4824d72065abb4ddd6a525c74bd78a0aab5d9fc2cf4fb5af50 \ + --hash=sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491 \ + --hash=sha256:ed905975ab14056a2e5eb1c376cb2e1ebc5396baf84163939c518556fccde9f5 \ + --hash=sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5 \ + --hash=sha256:ee43c17b173d46a3212baa6ead3ae258eeabdae48c263a01ccf0218c366dd655 \ + --hash=sha256:ef4fcbf3327382cd4c9f540babd61248208af7b93eec4de397b4d5f58a09e288 \ + --hash=sha256:eff0ac9dbe711a4aee69bf04a83896aa9b85f19641264053a9f6d48573abb7dd \ + --hash=sha256:f0aa869112ef88429ae17820d99c3dd9504c9e9c671d3c246f3d7442cb051084 \ + --hash=sha256:f3c96f633825733f735c5a9cf21d21a257d8e1edf0b1cee0a064b9c424ca0f7d \ + --hash=sha256:f5833ad231be5eb6553de524a70f48d71b2c8563101750531e0b80184e175cd4 \ + --hash=sha256:f5ec61164adcec446f8969a3358ec3f9b26bbda3b9213e5586d219afa8df2915 \ + --hash=sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1 \ + --hash=sha256:fb9e68df06293761f9fe66ade60a9bc6d0f5e42b8acf2939a9158af86ab0e5bd \ + --hash=sha256:fc14a032f813bf5fe624d991960ea83e9715adc27e4c1830a2361eb1d02ac341 \ + --hash=sha256:fcff63213e8e6e47770541a4607175404f47cbb3ebea7b6058cc82d524a0e424 \ + --hash=sha256:fd1fbe0f116b6e55da77aca2c6ddcddcfac2186cbf78bdebf40fc156efca389d \ + --hash=sha256:fe9753dfee015c570d73df76f899f18444d41388bffcde097deba51c4fadbb9f + # via requests +idna==3.20 \ + --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \ + --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c + # via requests +pika==1.4.4 \ + --hash=sha256:48de960c97a93b55db06b8be4c53eb977c9c8a2754c57cdae9097abcbd70ce04 \ + --hash=sha256:8cfc8b33a5cb16e733bd60cffca9732c0d1d761ecd80a89f34ed7df2cd38d6d6 + # via -r requirements.in +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed + # via -r requirements.in +urllib3==2.8.0 \ + --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \ + --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63 + # via requests diff --git a/examples/airflow-k8s-pipeline/k8s/minio/deployment.yaml b/examples/airflow-k8s-pipeline/k8s/minio/deployment.yaml index 9f21eaa..4ed9e95 100644 --- a/examples/airflow-k8s-pipeline/k8s/minio/deployment.yaml +++ b/examples/airflow-k8s-pipeline/k8s/minio/deployment.yaml @@ -30,6 +30,7 @@ spec: labels: app: minio spec: + automountServiceAccountToken: false # MinIO validates the AMQP target at startup, so wait for the broker. initContainers: - name: wait-for-rabbitmq @@ -73,9 +74,13 @@ spec: requests: memory: "256Mi" cpu: "100m" + ephemeral-storage: "1Gi" limits: memory: "1Gi" cpu: "500m" + # Covers the /data emptyDir, which holds every object in the demo bucket. + ephemeral-storage: "4Gi" volumes: - name: data - emptyDir: {} + emptyDir: + sizeLimit: 4Gi diff --git a/examples/airflow-k8s-pipeline/k8s/rabbitmq/deployment.yaml b/examples/airflow-k8s-pipeline/k8s/rabbitmq/deployment.yaml index 8be2fce..c0d8f50 100644 --- a/examples/airflow-k8s-pipeline/k8s/rabbitmq/deployment.yaml +++ b/examples/airflow-k8s-pipeline/k8s/rabbitmq/deployment.yaml @@ -61,6 +61,7 @@ spec: labels: app: rabbitmq spec: + automountServiceAccountToken: false containers: - name: rabbitmq image: rabbitmq:3.13-management @@ -79,6 +80,8 @@ spec: command: ["rabbitmq-diagnostics", "-q", "check_port_connectivity"] initialDelaySeconds: 10 periodSeconds: 10 + # The Erlang CLI takes >1s (the default) to start under the 500m CPU limit. + timeoutSeconds: 5 resources: requests: memory: "256Mi" diff --git a/examples/airflow-k8s-pipeline/k8s/seed/job.yaml b/examples/airflow-k8s-pipeline/k8s/seed/job.yaml index 047b836..9e4e8e5 100644 --- a/examples/airflow-k8s-pipeline/k8s/seed/job.yaml +++ b/examples/airflow-k8s-pipeline/k8s/seed/job.yaml @@ -8,6 +8,7 @@ spec: template: spec: restartPolicy: OnFailure + automountServiceAccountToken: false containers: - name: seed image: minio/mc:RELEASE.2024-11-21T17-21-54Z @@ -19,6 +20,13 @@ spec: value: "minioadmin" - name: MINIO_ROOT_PASSWORD value: "minioadmin" + resources: + requests: + memory: "64Mi" + cpu: "50m" + limits: + memory: "128Mi" + cpu: "200m" volumeMounts: - name: seed-script mountPath: /scripts diff --git a/examples/airflow-k8s-pipeline/k8s/worker/Dockerfile b/examples/airflow-k8s-pipeline/k8s/worker/Dockerfile index be16fec..597d6f3 100644 --- a/examples/airflow-k8s-pipeline/k8s/worker/Dockerfile +++ b/examples/airflow-k8s-pipeline/k8s/worker/Dockerfile @@ -6,9 +6,13 @@ FROM ghcr.io/nasa-ammos/tig/terrain-intelligence-generator:opensource USER root # Install AWS CLI v2 (official bundle, no python dependency) -RUN curl -sSL "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip \ +RUN curl -sSL --proto '=https' --tlsv1.2 "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip \ && (command -v unzip >/dev/null 2>&1 || (microdnf install -y unzip || dnf install -y unzip || yum install -y unzip)) \ && unzip -q /tmp/awscliv2.zip -d /tmp \ && /tmp/aws/install \ && rm -rf /tmp/awscliv2.zip /tmp/aws \ && aws --version + +# Wrappers only write under /tmp, so tasks run as an unprivileged user. +RUN useradd --uid 1000 --user-group --create-home tig +USER 1000:1000 From d6786b50ab1e48aa40d3dca35d45a4608963470a Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:58:34 +0000 Subject: [PATCH 2/2] examples(airflow): raise MinIO storage limit to 10Gi so repeated runs fit Co-Authored-By: jason.han --- examples/airflow-k8s-pipeline/k8s/minio/deployment.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/airflow-k8s-pipeline/k8s/minio/deployment.yaml b/examples/airflow-k8s-pipeline/k8s/minio/deployment.yaml index 4ed9e95..71eddac 100644 --- a/examples/airflow-k8s-pipeline/k8s/minio/deployment.yaml +++ b/examples/airflow-k8s-pipeline/k8s/minio/deployment.yaml @@ -78,9 +78,10 @@ spec: limits: memory: "1Gi" cpu: "500m" - # Covers the /data emptyDir, which holds every object in the demo bucket. - ephemeral-storage: "4Gi" + # Covers the /data emptyDir, which holds every object in the demo bucket + # (~260 MiB per stereo-pair run); half of minikube's default 20 GB disk. + ephemeral-storage: "10Gi" volumes: - name: data emptyDir: - sizeLimit: 4Gi + sizeLimit: 10Gi