diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 7b48e1e..abfce14 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -88,3 +88,41 @@ jobs: - name: Take everything down, volumes included if: always() run: docker compose down -v -t 60 + + chart: + runs-on: ubuntu-latest + env: + KUBECONFORM_VERSION: v0.7.0 + KUBECONFORM_SHA256: c31518ddd122663b3f3aa874cfe8178cb0988de944f29c74a0b9260920d115d3 + steps: + - uses: actions/checkout@v7 + - uses: azure/setup-helm@v4 + with: + version: v3.19.0 + - name: kubeconform, pinned by version and checksum + run: | + curl -fsSL -o kubeconform.tar.gz \ + "https://github.com/yannh/kubeconform/releases/download/${KUBECONFORM_VERSION}/kubeconform-linux-amd64.tar.gz" + echo "${KUBECONFORM_SHA256} kubeconform.tar.gz" | sha256sum -c - + mkdir -p "${RUNNER_TEMP}/bin" + tar -C "${RUNNER_TEMP}/bin" -xzf kubeconform.tar.gz kubeconform + echo "${RUNNER_TEMP}/bin" >> "${GITHUB_PATH}" + - name: The images in compose.yml, values.yaml and Chart.yaml are the same versions + run: scripts/check-versions.sh + - name: helm lint with every set of values under ci/ + run: | + for values in charts/pyxis/ci/*-values.yaml; do + echo "== ${values}" + helm lint --strict charts/pyxis -f "${values}" + done + - name: helm template and kubeconform with every set of values under ci/, IngressRoute checked against the Traefik CRD schema + run: | + for values in charts/pyxis/ci/*-values.yaml; do + echo "== ${values}" + helm template carina charts/pyxis -f "${values}" \ + | kubeconform -strict -summary \ + -schema-location default \ + -schema-location 'https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json' + done + - name: No chart version in the Pod templates, the checksum on the driver only, Service names from the release name + run: scripts/check-rendering.sh diff --git a/.yamllint b/.yamllint index b181b81..7c2e602 100644 --- a/.yamllint +++ b/.yamllint @@ -6,3 +6,7 @@ rules: max: 160 truthy: check-keys: false + +# Helm のテンプレートは描く前の YAML ではない +ignore: + - charts/pyxis/templates/ diff --git a/README.md b/README.md index 03ab121..d7ff2eb 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,8 @@ 日本のテレビ放送を録画して観るサーバを、Docker の入った Linux 1 台に立てる。\ 録画と配信は [Carina](https://github.com/NewWorldOrg/Carina)、画面は [Vela](https://github.com/NewWorldOrg/Vela) が受け持つ。\ -Pyxis はその 2 つを、動作を確かめた版の組で固定して配る。 +Pyxis はその 2 つを、動作を確かめた版の組で固定して配る。\ +Kubernetes に立てるなら、同じ版の組を固定した Helm チャートが `charts/pyxis` にある。 立てると、ブラウザから次のことをする。 @@ -174,6 +175,208 @@ services: あわせて `app` の `environment` に `CARINA_KNOWN_NETWORKS` を足し、前段のアドレスの範囲を書く。\ これが無いと `X-Forwarded-*` が読まれず、https で開いてもセッション Cookie に Secure が付かない。 +## Helm で立てる + +Kubernetes に立てるなら、`charts/pyxis` のチャートを使う。\ +compose と同じイメージの版を固定していて、driver・app・画面・データベースを部品ごとに有効と無効を切り替えられる。\ +チャートはまだ公開先に置いていないので、リポジトリを取って `charts/pyxis` のパスを渡す。 + +driver は特権で動き、チューナーと pcscd のソケットをホストから受けるので、名前空間の Pod Security を privileged にしておく。 + +```bash +kubectl create namespace tv +kubectl label namespace tv pod-security.kubernetes.io/enforce=privileged +``` + +driver と app はソケットのディレクトリをホストのパスで共有するので、同じノードに置く。\ +ノードが複数なら `nodeSelector` にチューナーのあるノードを書く。 + +値のファイルに、チューナー・置き場・前段・ブラウザから届くアドレスを書く。\ +データベースのパスワードは Secret に置き、その名前を `postgres.auth.existingSecret` に書く。 + +```yaml +# values.yaml +nodeSelector: + kubernetes.io/hostname: node-1 +driver: + config: + outputRoots: + - name: primary + path: /srv/recordings + tuner: + backend: dvb + devices: + - id: adapter0.frontend0 + kind: terrestrial + devicePath: /dev/dvb/adapter0/frontend0 + hostDevices: + - /dev/dvb/adapter0 + pcscd: + enabled: true +app: + publicOrigin: https://tv.example.com + knownNetworks: + - 10.42.0.0/16 +postgres: + auth: + existingSecret: pyxis-db +route: + kind: ingressRoute + host: tv.example.com +persistence: + recordings: + type: hostPath + hostPath: + path: /srv/pyxis/recordings +``` + +```bash +kubectl -n tv create secret generic pyxis-db --from-literal=password=... +helm install carina charts/pyxis -n tv -f values.yaml +``` + +リリース名は資源の名前の頭になり、`carina` なら `carina-driver`・`carina-app`・`carina-db`・`carina-vela` になる。\ +app は起動のたびに初期化コンテナでデータベースのスキーマを当て、古い app が止まってから当たる。\ +ローカルアカウントのパスワードは app のログに 1 度だけ出る。 + +```bash +kubectl -n tv logs deploy/carina-app -c app | grep 'Sign in as' +``` + +何も書かずに入れると、合成のチューナー・同梱のデータベース・GPU なし・標準の Ingress で動く。\ +そのときもデータベースのパスワードだけは要り、無ければチャートは描画を止める。 + +### 値 + +主な値は次のとおりで、全部の値と説明は `charts/pyxis/values.yaml` にある。 + +| 値 | 用途 | 既定 | +| --- | --- | --- | +| `nodeSelector` | 全部の Pod に当てる nodeSelector。部品ごとの `driver.nodeSelector` などで上書きできる | 空 | +| `carina.uid` / `carina.gid` | app を動かす番号。gid は driver の `socketGroupId` にも入る | `10001` | +| `driver.enabled` / `app.enabled` / `vela.enabled` / `postgres.enabled` | その部品を作るか | `true` | +| `driver.image.tag` / `app.image.tag` / `vela.image.tag` / `postgres.image.tag` | イメージの版。compose と同じ数字で固定してある | 版の表のとおり | +| `driver.config` | `driver.json` の中身をそのまま書く。`socketGroupId` はチャートが入れる | 合成のチューナー | +| `driver.hostDevices` | ホストの装置のディレクトリの一覧。同じパスで driver に渡す | 空 | +| `driver.pcscd.enabled` | ホストの `/run/pcscd` を読み取り専用で渡すか | `false` | +| `driver.socket.hostPath` | driver と app が共有するソケットのディレクトリのホスト側のパス | `/run/pyxis` | +| `driver.hostNetwork` | ホストのネットワークで動かすか。カードを貸す driver は `true` | `false` | +| `driver.lendingKey.existingSecret` | カードの貸し借りの鍵を持つ Secret。`/etc/carina/lending.key` に所有者だけが読める形で置く | 空 | +| `driver.terminationGracePeriodSeconds` | 止めるときに録画の終わりを待つ秒数。`shutdownGraceHours` より長くする | `25200` | +| `app.publicOrigin` | ブラウザから届くアドレス。OIDC でサインインするときに要る | 空 | +| `app.knownNetworks` | `X-Forwarded-*` を信じる前段の網の一覧。Ingress のコントローラの Pod がいる網を書く | 空 | +| `app.encodingsPrefer` / `app.transcodingPrefer` | 変換器。`Software` か `Vaapi` | `Software` | +| `app.gpu.mode` | GPU の渡し方。`none`、`devicePlugin`、`hostPath` | `none` | +| `app.gpu.resource` | device plugin に要求する資源の名前 | `gpu.intel.com/i915` | +| `app.gpu.renderNode` | VAAPI で使う GPU の描画ノード | `/dev/dri/renderD128` | +| `app.dataProtection.seedSecret` | 鍵の種を持つ Secret。鍵の置き場が空のときだけ写す | 空 | +| `app.extraEnv` | app に足す環境変数の一覧。Carina の README の `app` の節の設定に使う | 空 | +| `postgres.auth.existingSecret` / `postgres.auth.passwordKey` | パスワードを持つ Secret とそのキー | 空 / `password` | +| `postgres.auth.password` | Secret を名指ししないときに、チャートが作る Secret に入れるパスワード | 空 | +| `externalDatabase.connectionSecret` / `externalDatabase.connectionKey` | 同梱のデータベースを使わないときの、接続文字列を持つ Secret とそのキー | 空 / `connection` | +| `route.kind` | 前段の種類。`ingress`、`ingressRoute`、`none` | `ingress` | +| `route.host` | ブラウザが開くホスト名。空なら全部を受ける | 空 | +| `route.ingress.className` / `route.ingress.annotations` / `route.ingress.tls` | 標準の Ingress の設定 | 空 | +| `route.ingressRoute.entryPoints` / `route.ingressRoute.tls` | Traefik の IngressRoute の設定。`tls` を `null` にすると http | `websecure` / `{}` | +| `persistence.<置き場>.type` | `recordings`・`encodes`・`thumbnails`・`captions`・`keys`・`db` の置き方。`pvc`、`existingClaim`、`hostPath` | `pvc` | +| `<部品>.resources` | 部品ごとの resources | 空 | + +### 置き場 + +置き場ごとに `type` を選ぶ。\ +`pvc` は PVC を作り、リリースを消しても残す。\ +`existingClaim` は手で作った PVC を使う。\ +`hostPath` はホストのディレクトリで、無ければ起動しない。\ +サムネイルと字幕は `emptyDir` も選べ、Pod と一緒に消えて作り直される。 + +録画は driver が書き、app は読み取り専用で同じ実体を見る。\ +2 つは同じノードに置くので、ReadWriteOnce の PVC でよい。\ +鍵の置き場はデータベースとは別のバックアップの対象にする。 + +### Helm での GPU + +Intel の GPU をクラスタの device plugin で渡すなら `app.gpu.mode` を `devicePlugin` にする。\ +device plugin はクラスタ全体のものなので、チャートには入っていない。\ +device plugin が作る装置は app のユーザーが開けない。\ +そのため app は root で始まり、装置のグループを app のものに替えてから権限を落とす。\ +`hostPath` は `/dev/dri` をディレクトリごと渡す形で、装置を開くために app を特権にするので、device plugin のほうを勧める。\ +どちらも渡しただけでは変換はソフトウェアのままで、`app.encodingsPrefer` と `app.transcodingPrefer` を `Vaapi` にする。 + +### Helm での前段 + +`ingress` は Ingress を 1 つ作り、`/api` を app に、それ以外を画面に送る。\ +`ingressRoute` は Traefik の IngressRoute を作り、同じ振り分けをする。\ +Traefik は SSE と WebSocket を既定で溜めずに通す。\ +ingress-nginx なら `route.ingress.annotations` に `nginx.ingress.kubernetes.io/proxy-buffering: "off"` と、`proxy-read-timeout` を長くする行を書く。 + +`none` は何も作らず、自前の前段に app の Service の 8080 番と画面の Service の 3000 番を向ける。\ +その前段には「前段を自分で持つ」の 5 つを持たせる。\ +どの形でも、前段の Pod がいる網を `app.knownNetworks` に書く。 + +### driver だけを立てる + +カードを貸す機械には driver だけを入れる。\ +app・画面・データベースを作らず、ホストのネットワークで口を開き、鍵を Secret から渡す。 + +```yaml +driver: + hostNetwork: true + config: + outputRoots: + - name: primary + path: /srv/recordings + tuner: + backend: dvb + devices: + - id: adapter0.frontend0 + kind: terrestrial + devicePath: /dev/dvb/adapter0/frontend0 + cardLending: + lend: + address: 0.0.0.0 + hostNames: + - lender.example.com + keyFile: /etc/carina/lending.key + hostDevices: + - /dev/dvb/adapter0 + pcscd: + enabled: true + lendingKey: + existingSecret: pyxis-lending-key +app: + enabled: false +vela: + enabled: false +postgres: + enabled: false +route: + kind: none +``` + +```bash +openssl rand -base64 48 > lending.key +kubectl -n tv create secret generic pyxis-lending-key --from-file=key=lending.key +``` + +借りる側の driver は `config` に `cardLending.borrow` を書き、同じ Secret を渡す。\ +貸し借りのキーは Carina の README の `driver` の節にある。 + +### Helm での更新と戻し方 + +新しい版のチャートを `helm upgrade carina charts/pyxis -n tv -f values.yaml` で当てる。\ +app は古いものが止まってから新しいものが上がり、そのときにスキーマを当てるので、列や表を消す変更でも古い app を壊さない。\ +driver のイメージの版が前と同じなら driver は作り直されず、録画は続く。\ +driver の版が変わる更新は、「更新」の節と同じく録画中とライブの視聴中を避ける。 + +控えは compose と同じ `pg_dump` で取る。 + +```bash +kubectl -n tv exec sts/carina-db -- pg_dump -U carina -Fc carina > carina-$(date +%Y%m%d).dump +``` + +戻すときは `helm rollback` で前の版のチャートに戻してから、「戻し方」の節と同じ手順でデータベースを入れ直す。\ +PostgreSQL のメジャーの版上げは `pg_dump` と `pg_restore` が要り、チャートは自動ではやらない。 + ## 更新 Pyxis の新しい版は、`compose.yml` のイメージの版を上げたリリースとして出る。\ @@ -254,7 +457,8 @@ docker compose logs driver | --- | --- | --- | --- | --- | --- | | 未リリース | v1.0.0 | v1.0.0 | v1.0.0 | 18.6 | 2.11.7 | -イメージの版は `compose.yml` に数字で書いてあり、`latest` は使わない。\ +イメージの版は `compose.yml` と `charts/pyxis/values.yaml` に数字で書いてあり、`latest` は使わない。\ +2 か所が同じ版であることは CI が確かめる。\ Pyxis の版は、その組で起動・録画・再生を確かめてから付ける。 ## ライセンス diff --git a/charts/pyxis/Chart.yaml b/charts/pyxis/Chart.yaml new file mode 100644 index 0000000..ad64c23 --- /dev/null +++ b/charts/pyxis/Chart.yaml @@ -0,0 +1,23 @@ +apiVersion: v2 +name: pyxis +description: A recording server for Japanese digital broadcasting, with Carina and Vela pinned to a verified set of image versions +type: application +version: 0.1.0 +appVersion: 0.1.0 +kubeVersion: ">=1.25.0-0" +home: https://github.com/NewWorldOrg/Pyxis +sources: + - https://github.com/NewWorldOrg/Pyxis + - https://github.com/NewWorldOrg/Carina + - https://github.com/NewWorldOrg/Vela +keywords: + - dtv + - recording + - carina + - vela +annotations: + # この版が固定するイメージの組。compose.yml と values.yaml の image と同じ値にする + images/carina-app: ghcr.io/newworldorg/carina:app-v1.0.0 + images/carina-driver: ghcr.io/newworldorg/carina:driver-v1.0.0 + images/vela: ghcr.io/newworldorg/vela:v1.0.0 + images/postgres: postgres:18.6-alpine diff --git a/charts/pyxis/ci/default-values.yaml b/charts/pyxis/ci/default-values.yaml new file mode 100644 index 0000000..d41ddfa --- /dev/null +++ b/charts/pyxis/ci/default-values.yaml @@ -0,0 +1,4 @@ +# 既定の形。合成のチューナー、同梱のデータベース、GPU なし、標準の Ingress +postgres: + auth: + password: check diff --git a/charts/pyxis/ci/driver-only-values.yaml b/charts/pyxis/ci/driver-only-values.yaml new file mode 100644 index 0000000..bce43ec --- /dev/null +++ b/charts/pyxis/ci/driver-only-values.yaml @@ -0,0 +1,39 @@ +# driver だけを別の機械に立てる形。ホストのネットワークでカードを貸し、app・画面・DB・前段は作らない +driver: + hostNetwork: true + config: + outputRoots: + - name: primary + path: /srv/recordings + tuner: + backend: dvb + devices: + - id: adapter0.frontend0 + kind: terrestrial + devicePath: /dev/dvb/adapter0/frontend0 + enabled: true + cardLending: + lend: + address: 0.0.0.0 + hostNames: + - lender.example.com + keyFile: /etc/carina/lending.key + hostDevices: + - /dev/dvb/adapter0 + pcscd: + enabled: true + lendingKey: + existingSecret: pyxis-lending-key +app: + enabled: false +vela: + enabled: false +postgres: + enabled: false +route: + kind: none +persistence: + recordings: + type: hostPath + hostPath: + path: /srv/pyxis/recordings diff --git a/charts/pyxis/ci/external-db-values.yaml b/charts/pyxis/ci/external-db-values.yaml new file mode 100644 index 0000000..aee0ab1 --- /dev/null +++ b/charts/pyxis/ci/external-db-values.yaml @@ -0,0 +1,32 @@ +# 外のデータベース。同梱の DB を作らず、接続文字列を Secret から読む。GPU は hostPath、サムネイルと字幕は emptyDir +postgres: + enabled: false +externalDatabase: + connectionSecret: pyxis-db + connectionKey: connection +app: + gpu: + mode: hostPath +route: + kind: ingress + host: tv.example.com + ingress: + className: nginx + annotations: + nginx.ingress.kubernetes.io/proxy-buffering: "off" + nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" + tls: + - hosts: + - tv.example.com + secretName: tv-example-com-tls +persistence: + recordings: + type: existingClaim + existingClaim: recordings + encodes: + type: pvc + storageClass: fast + thumbnails: + type: emptyDir + captions: + type: emptyDir diff --git a/charts/pyxis/ci/ingressroute-gpu-values.yaml b/charts/pyxis/ci/ingressroute-gpu-values.yaml new file mode 100644 index 0000000..2c2f01d --- /dev/null +++ b/charts/pyxis/ci/ingressroute-gpu-values.yaml @@ -0,0 +1,79 @@ +# Traefik の IngressRoute と Intel の device plugin。DVB のチューナーと pcscd を渡し、置き場は hostPath +nodeSelector: + kubernetes.io/hostname: node-1 +driver: + config: + outputRoots: + - name: primary + path: /srv/recordings + shutdownGraceHours: 6 + tuner: + backend: dvb + devices: + - id: adapter0.frontend0 + kind: terrestrial + devicePath: /dev/dvb/adapter0/frontend0 + enabled: true + hostDevices: + - /dev/dvb/adapter0 + pcscd: + enabled: true + lendingKey: + existingSecret: pyxis-lending-key + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 2Gi +app: + publicOrigin: https://tv.example.com + knownNetworks: + - 10.42.0.0/16 + encodingsPrefer: Vaapi + transcodingPrefer: Vaapi + gpu: + mode: devicePlugin + dataProtection: + seedSecret: pyxis-data-protection-seed + extraEnv: + - name: Encodings__MostCores + value: "4" + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + memory: 8Gi +postgres: + auth: + existingSecret: pyxis-db + passwordKey: password +route: + kind: ingressRoute + host: tv.example.com +persistence: + recordings: + type: hostPath + hostPath: + path: /srv/pyxis/recordings + encodes: + type: hostPath + hostPath: + path: /srv/pyxis/encodes + thumbnails: + type: hostPath + hostPath: + path: /srv/pyxis/thumbnails + captions: + type: hostPath + hostPath: + path: /srv/pyxis/captions + keys: + type: hostPath + hostPath: + path: /srv/pyxis/keys + db: + type: hostPath + hostPath: + path: /srv/pyxis/postgres diff --git a/charts/pyxis/templates/NOTES.txt b/charts/pyxis/templates/NOTES.txt new file mode 100644 index 0000000..b23dc48 --- /dev/null +++ b/charts/pyxis/templates/NOTES.txt @@ -0,0 +1,22 @@ +{{ .Release.Name }} を名前空間 {{ .Release.Namespace }} に入れた。 +{{- if .Values.driver.enabled }} + +driver は特権で動く。名前空間の Pod Security が privileged でないと driver は上がらない。 + + kubectl label namespace {{ .Release.Namespace }} pod-security.kubernetes.io/enforce=privileged +{{- end }} +{{- if and .Values.driver.enabled .Values.app.enabled (not .Values.nodeSelector) (not .Values.driver.nodeSelector) (not .Values.app.nodeSelector) }} + +nodeSelector が無い。ノードが複数のクラスタでは driver と app が別のノードに載り、ソケットでつながらない。 +値の nodeSelector に、チューナーのあるノードを書く。 +{{- end }} +{{- if .Values.app.enabled }} + +最初の起動で app がローカルアカウントを作り、ユーザー名とパスワードをログに 1 度だけ書く。 + + kubectl -n {{ .Release.Namespace }} logs deploy/{{ include "pyxis.partName" (dict "root" . "part" "app") }} -c app | grep 'Sign in as' +{{- end }} +{{- if and .Values.vela.enabled (ne .Values.route.kind "none") }} + +ブラウザで {{ if .Values.route.host }}https://{{ .Values.route.host }}/{{ else }}前段のアドレス{{ end }} を開く。 +{{- end }} diff --git a/charts/pyxis/templates/_helpers.tpl b/charts/pyxis/templates/_helpers.tpl new file mode 100644 index 0000000..29874f5 --- /dev/null +++ b/charts/pyxis/templates/_helpers.tpl @@ -0,0 +1,98 @@ +{{/* +部品の名前。リリース名に部品の名前をつなぐ (carina-driver、carina-app、carina-db、carina-vela) +*/}} +{{- define "pyxis.partName" -}} +{{- printf "%s-%s" .root.Release.Name .part | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +資源の metadata に付けるラベル。チャートの版はここにだけ入れ、Pod のテンプレートには入れない +*/}} +{{- define "pyxis.labels" -}} +helm.sh/chart: {{ printf "%s-%s" .root.Chart.Name .root.Chart.Version | replace "+" "_" }} +app.kubernetes.io/version: {{ .root.Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .root.Release.Service }} +{{ include "pyxis.selectorLabels" . }} +{{- end -}} + +{{/* +selector と Pod のテンプレートに使うラベル。作ったあとは変えられない +*/}} +{{- define "pyxis.selectorLabels" -}} +app.kubernetes.io/name: {{ .root.Chart.Name }} +app.kubernetes.io/instance: {{ .root.Release.Name }} +app.kubernetes.io/component: {{ .part }} +{{- end -}} + +{{/* +部品の nodeSelector。部品のものがあればそれを、無ければ共通のものを使う +*/}} +{{- define "pyxis.nodeSelector" -}} +{{- $selector := .own | default .root.Values.nodeSelector -}} +{{- if $selector }} +nodeSelector: + {{- toYaml $selector | nindent 2 }} +{{- end }} +{{- end -}} + +{{/* +Pod の imagePullSecrets +*/}} +{{- define "pyxis.imagePullSecrets" -}} +{{- with .Values.imagePullSecrets }} +imagePullSecrets: + {{- range . }} + - name: {{ . }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +置き場の volume の中身。type が pvc、existingClaim、hostPath、emptyDir のどれかで形が変わる +*/}} +{{- define "pyxis.volumeSource" -}} +{{- $spec := index .root.Values.persistence .name -}} +{{- if eq $spec.type "pvc" }} +persistentVolumeClaim: + claimName: {{ include "pyxis.partName" (dict "root" .root "part" .name) }} +{{- else if eq $spec.type "existingClaim" }} +persistentVolumeClaim: + claimName: {{ required (printf "persistence.%s.existingClaim に PVC の名前を書く" .name) $spec.existingClaim }} +{{- else if eq $spec.type "hostPath" }} +hostPath: + path: {{ required (printf "persistence.%s.hostPath.path にホスト側のパスを書く" .name) $spec.hostPath.path }} + type: {{ $spec.hostPath.type | default "Directory" }} +{{- else if eq $spec.type "emptyDir" }} +emptyDir: {} +{{- else }} +{{- fail (printf "persistence.%s.type は pvc、existingClaim、hostPath のどれかにする" .name) }} +{{- end }} +{{- end -}} + +{{/* +データベースのパスワードを持つ Secret の名前。existingSecret が無ければチャートが作る Secret +*/}} +{{- define "pyxis.dbSecretName" -}} +{{- .Values.postgres.auth.existingSecret | default (include "pyxis.partName" (dict "root" . "part" "db")) -}} +{{- end -}} + +{{/* +app と migrate に渡す接続文字列の環境変数。同梱のときは Service 名とパスワードから組み、外のときは Secret から読む +*/}} +{{- define "pyxis.connectionStringEnv" -}} +{{- if .Values.postgres.enabled }} +- name: PGPASSWORD + valueFrom: + secretKeyRef: + name: {{ include "pyxis.dbSecretName" . }} + key: {{ .Values.postgres.auth.passwordKey }} +- name: ConnectionStrings__Carina + value: {{ printf "Host=%s;Port=5432;Database=%s;Username=%s;Password=$(PGPASSWORD)" (include "pyxis.partName" (dict "root" . "part" "db")) .Values.postgres.auth.database .Values.postgres.auth.username | quote }} +{{- else }} +- name: ConnectionStrings__Carina + valueFrom: + secretKeyRef: + name: {{ required "postgres.enabled が false のときは externalDatabase.connectionSecret に接続文字列を持つ Secret の名前を書く" .Values.externalDatabase.connectionSecret }} + key: {{ .Values.externalDatabase.connectionKey }} +{{- end }} +{{- end -}} diff --git a/charts/pyxis/templates/app-deployment.yaml b/charts/pyxis/templates/app-deployment.yaml new file mode 100644 index 0000000..91ad39e --- /dev/null +++ b/charts/pyxis/templates/app-deployment.yaml @@ -0,0 +1,181 @@ +{{- if .Values.app.enabled }} +{{- $app := .Values.app }} +{{- $image := printf "%s:%s" $app.image.repository $app.image.tag }} +{{- $gid := int .Values.carina.gid }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "pyxis.partName" (dict "root" . "part" "app") }} + labels: + {{- include "pyxis.labels" (dict "root" . "part" "app") | nindent 4 }} +spec: + # 予約の確認やエンコードの列をプロセスの中に持つので 1 つだけ。古い app が止まってから migrate を当てる + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + {{- include "pyxis.selectorLabels" (dict "root" . "part" "app") | nindent 6 }} + template: + metadata: + labels: + {{- include "pyxis.selectorLabels" (dict "root" . "part" "app") | nindent 8 }} + spec: + {{- with include "pyxis.imagePullSecrets" . }}{{ . | trim | nindent 6 }}{{- end }} + {{- with include "pyxis.nodeSelector" (dict "root" . "own" $app.nodeSelector) }}{{ . | trim | nindent 6 }}{{- end }} + securityContext: + fsGroup: {{ $gid }} + {{- if eq $app.gpu.mode "none" }} + runAsUser: {{ int .Values.carina.uid }} + runAsGroup: {{ $gid }} + {{- end }} + initContainers: + {{- if $app.dataProtection.seedSecret }} + # client secret を封じる鍵の種を、置き場に無いときだけ写す。あとから増えた鍵には触らない + - name: seed-keys + image: {{ $image | quote }} + imagePullPolicy: {{ $app.image.pullPolicy }} + command: + - bash + - -c + - | + set -eu + for seed in /seed/*; do + [ -f "$seed" ] || continue + kept=/var/lib/carina/keys/$(basename "$seed") + [ -e "$kept" ] || install -o {{ int .Values.carina.uid }} -g {{ $gid }} -m 0600 "$seed" "$kept" + done + securityContext: + runAsUser: 0 + runAsGroup: 0 + volumeMounts: + - name: keys + mountPath: /var/lib/carina/keys + - name: keys-seed + mountPath: /seed + readOnly: true + {{- end }} + - name: migrate + image: {{ $image | quote }} + imagePullPolicy: {{ $app.image.pullPolicy }} + env: + - name: CARINA_ROLE + value: migrate + {{- include "pyxis.connectionStringEnv" . | trim | nindent 12 }} + containers: + - name: app + image: {{ $image | quote }} + imagePullPolicy: {{ $app.image.pullPolicy }} + {{- if eq $app.gpu.mode "devicePlugin" }} + # device plugin が作る装置は root:root 0660 で app が開けない。グループを app のものに替えてから entrypoint に渡す + command: + - bash + - -c + - chgrp {{ $gid }} /dev/dri/* && exec /usr/local/bin/carina app + {{- end }} + {{- if eq $app.gpu.mode "hostPath" }} + # hostPath で渡した装置は device cgroup が開かせないので、特権にする + securityContext: + privileged: true + {{- end }} + env: + - name: CARINA_ROLE + value: app + {{- include "pyxis.connectionStringEnv" . | trim | nindent 12 }} + - name: CARINA_DATA_PROTECTION_KEYS + value: /var/lib/carina/keys + {{- with $app.publicOrigin }} + - name: CARINA_PUBLIC_ORIGIN + value: {{ . | quote }} + {{- end }} + {{- with $app.knownNetworks }} + - name: CARINA_KNOWN_NETWORKS + value: {{ join "," . | quote }} + {{- end }} + - name: Integrity__OutputRoots + value: primary=/srv/recordings + - name: Thumbnails__WrittenTo + value: /srv/thumbnails + - name: Captions__WrittenTo + value: /srv/captions + - name: Encodings__OutputRoots + value: encodes=/srv/encodes + - name: Encodings__Prefer + value: {{ $app.encodingsPrefer }} + - name: Transcoding__Prefer + value: {{ $app.transcodingPrefer }} + - name: Machine__RenderNode + value: {{ $app.gpu.renderNode }} + {{- with $app.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http + containerPort: 8080 + volumeMounts: + - name: socket + mountPath: /run/carina + - name: recordings + mountPath: /srv/recordings + readOnly: true + - name: encodes + mountPath: /srv/encodes + - name: thumbnails + mountPath: /srv/thumbnails + - name: captions + mountPath: /srv/captions + - name: keys + mountPath: /var/lib/carina/keys + {{- if eq $app.gpu.mode "hostPath" }} + - name: dri + mountPath: /dev/dri + {{- end }} + startupProbe: + httpGet: + path: /api/health + port: http + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 60 + readinessProbe: + httpGet: + path: /api/health + port: http + periodSeconds: 10 + timeoutSeconds: 5 + {{- $resources := deepCopy $app.resources }} + {{- if eq $app.gpu.mode "devicePlugin" }} + {{- $_ := set $resources "requests" (merge (dict $app.gpu.resource "1") ($resources.requests | default dict)) }} + {{- $_ := set $resources "limits" (merge (dict $app.gpu.resource "1") ($resources.limits | default dict)) }} + {{- end }} + {{- with $resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: socket + hostPath: + path: {{ .Values.driver.socket.hostPath }} + type: DirectoryOrCreate + - name: recordings + {{- include "pyxis.volumeSource" (dict "root" . "name" "recordings") | trim | nindent 10 }} + - name: encodes + {{- include "pyxis.volumeSource" (dict "root" . "name" "encodes") | trim | nindent 10 }} + - name: thumbnails + {{- include "pyxis.volumeSource" (dict "root" . "name" "thumbnails") | trim | nindent 10 }} + - name: captions + {{- include "pyxis.volumeSource" (dict "root" . "name" "captions") | trim | nindent 10 }} + - name: keys + {{- include "pyxis.volumeSource" (dict "root" . "name" "keys") | trim | nindent 10 }} + {{- if $app.dataProtection.seedSecret }} + - name: keys-seed + secret: + secretName: {{ $app.dataProtection.seedSecret }} + {{- end }} + {{- if eq $app.gpu.mode "hostPath" }} + - name: dri + hostPath: + path: {{ $app.gpu.hostPath }} + type: Directory + {{- end }} +{{- end }} diff --git a/charts/pyxis/templates/app-service.yaml b/charts/pyxis/templates/app-service.yaml new file mode 100644 index 0000000..f2dac6a --- /dev/null +++ b/charts/pyxis/templates/app-service.yaml @@ -0,0 +1,16 @@ +{{- if .Values.app.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "pyxis.partName" (dict "root" . "part" "app") }} + labels: + {{- include "pyxis.labels" (dict "root" . "part" "app") | nindent 4 }} +spec: + type: ClusterIP + selector: + {{- include "pyxis.selectorLabels" (dict "root" . "part" "app") | nindent 4 }} + ports: + - name: http + port: 8080 + targetPort: http +{{- end }} diff --git a/charts/pyxis/templates/db-secret.yaml b/charts/pyxis/templates/db-secret.yaml new file mode 100644 index 0000000..9f6a933 --- /dev/null +++ b/charts/pyxis/templates/db-secret.yaml @@ -0,0 +1,14 @@ +{{- if and .Values.postgres.enabled (not .Values.postgres.auth.existingSecret) }} +{{- if not .Values.postgres.auth.password }} +{{- fail "postgres.auth.existingSecret か postgres.auth.password のどちらかにデータベースのパスワードを書く" }} +{{- end }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "pyxis.partName" (dict "root" . "part" "db") }} + labels: + {{- include "pyxis.labels" (dict "root" . "part" "db") | nindent 4 }} +type: Opaque +stringData: + {{ .Values.postgres.auth.passwordKey }}: {{ .Values.postgres.auth.password | quote }} +{{- end }} diff --git a/charts/pyxis/templates/db-service.yaml b/charts/pyxis/templates/db-service.yaml new file mode 100644 index 0000000..9a27712 --- /dev/null +++ b/charts/pyxis/templates/db-service.yaml @@ -0,0 +1,16 @@ +{{- if .Values.postgres.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "pyxis.partName" (dict "root" . "part" "db") }} + labels: + {{- include "pyxis.labels" (dict "root" . "part" "db") | nindent 4 }} +spec: + type: ClusterIP + selector: + {{- include "pyxis.selectorLabels" (dict "root" . "part" "db") | nindent 4 }} + ports: + - name: postgres + port: 5432 + targetPort: postgres +{{- end }} diff --git a/charts/pyxis/templates/db-statefulset.yaml b/charts/pyxis/templates/db-statefulset.yaml new file mode 100644 index 0000000..3347e49 --- /dev/null +++ b/charts/pyxis/templates/db-statefulset.yaml @@ -0,0 +1,59 @@ +{{- if .Values.postgres.enabled }} +{{- $db := .Values.postgres }} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "pyxis.partName" (dict "root" . "part" "db") }} + labels: + {{- include "pyxis.labels" (dict "root" . "part" "db") | nindent 4 }} +spec: + replicas: 1 + serviceName: {{ include "pyxis.partName" (dict "root" . "part" "db") }} + selector: + matchLabels: + {{- include "pyxis.selectorLabels" (dict "root" . "part" "db") | nindent 6 }} + template: + metadata: + labels: + {{- include "pyxis.selectorLabels" (dict "root" . "part" "db") | nindent 8 }} + spec: + {{- with include "pyxis.imagePullSecrets" . }}{{ . | trim | nindent 6 }}{{- end }} + {{- with include "pyxis.nodeSelector" (dict "root" . "own" $db.nodeSelector) }}{{ . | trim | nindent 6 }}{{- end }} + containers: + - name: postgres + image: {{ printf "%s:%s" $db.image.repository $db.image.tag | quote }} + imagePullPolicy: {{ $db.image.pullPolicy }} + env: + - name: POSTGRES_DB + value: {{ $db.auth.database | quote }} + - name: POSTGRES_USER + value: {{ $db.auth.username | quote }} + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "pyxis.dbSecretName" . }} + key: {{ $db.auth.passwordKey }} + ports: + - name: postgres + containerPort: 5432 + volumeMounts: + - name: data + mountPath: /var/lib/postgresql + readinessProbe: + exec: + command: + - pg_isready + - -U + - {{ $db.auth.username | quote }} + - -d + - {{ $db.auth.database | quote }} + periodSeconds: 10 + timeoutSeconds: 5 + {{- with $db.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: data + {{- include "pyxis.volumeSource" (dict "root" . "name" "db") | trim | nindent 10 }} +{{- end }} diff --git a/charts/pyxis/templates/driver-configmap.yaml b/charts/pyxis/templates/driver-configmap.yaml new file mode 100644 index 0000000..24c64aa --- /dev/null +++ b/charts/pyxis/templates/driver-configmap.yaml @@ -0,0 +1,12 @@ +{{- if .Values.driver.enabled }} +{{- $config := merge (dict "socketGroupId" (int .Values.carina.gid)) .Values.driver.config }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "pyxis.partName" (dict "root" . "part" "driver") }} + labels: + {{- include "pyxis.labels" (dict "root" . "part" "driver") | nindent 4 }} +data: + driver.json: | + {{- toPrettyJson $config | nindent 4 }} +{{- end }} diff --git a/charts/pyxis/templates/driver-deployment.yaml b/charts/pyxis/templates/driver-deployment.yaml new file mode 100644 index 0000000..c1d9c32 --- /dev/null +++ b/charts/pyxis/templates/driver-deployment.yaml @@ -0,0 +1,98 @@ +{{- if .Values.driver.enabled }} +{{- $driver := .Values.driver }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "pyxis.partName" (dict "root" . "part" "driver") }} + labels: + {{- include "pyxis.labels" (dict "root" . "part" "driver") | nindent 4 }} +spec: + # チューナーを占有するので 1 つだけ動かし、入れ替えも古いものが止まってから + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + {{- include "pyxis.selectorLabels" (dict "root" . "part" "driver") | nindent 6 }} + template: + metadata: + labels: + {{- include "pyxis.selectorLabels" (dict "root" . "part" "driver") | nindent 8 }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/driver-configmap.yaml") . | sha256sum }} + spec: + {{- with include "pyxis.imagePullSecrets" . }}{{ . | trim | nindent 6 }}{{- end }} + {{- with include "pyxis.nodeSelector" (dict "root" . "own" $driver.nodeSelector) }}{{ . | trim | nindent 6 }}{{- end }} + {{- if $driver.hostNetwork }} + hostNetwork: true + {{- end }} + # 録画中に止められると録画の終わりまで待つ。shutdownGraceHours より長くする + terminationGracePeriodSeconds: {{ $driver.terminationGracePeriodSeconds }} + containers: + - name: driver + image: {{ printf "%s:%s" $driver.image.repository $driver.image.tag | quote }} + imagePullPolicy: {{ $driver.image.pullPolicy }} + env: + - name: CARINA_ROLE + value: driver + - name: CARINA_DRIVER_CONFIG + value: /etc/carina/driver.json + securityContext: + privileged: true + volumeMounts: + - name: config + mountPath: /etc/carina/driver.json + subPath: driver.json + readOnly: true + - name: socket + mountPath: /run/carina + - name: recordings + mountPath: /srv/recordings + {{- if $driver.pcscd.enabled }} + - name: pcscd + mountPath: /run/pcscd + readOnly: true + {{- end }} + {{- if $driver.lendingKey.existingSecret }} + - name: lending-key + mountPath: /etc/carina/lending.key + subPath: {{ $driver.lendingKey.key }} + readOnly: true + {{- end }} + {{- range $i, $device := $driver.hostDevices }} + - name: device-{{ $i }} + mountPath: {{ $device }} + {{- end }} + {{- with $driver.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumes: + - name: config + configMap: + name: {{ include "pyxis.partName" (dict "root" . "part" "driver") }} + - name: socket + hostPath: + path: {{ $driver.socket.hostPath }} + type: DirectoryOrCreate + - name: recordings + {{- include "pyxis.volumeSource" (dict "root" . "name" "recordings") | trim | nindent 10 }} + {{- if $driver.pcscd.enabled }} + - name: pcscd + hostPath: + path: /run/pcscd + type: Directory + {{- end }} + {{- if $driver.lendingKey.existingSecret }} + - name: lending-key + secret: + secretName: {{ $driver.lendingKey.existingSecret }} + defaultMode: 0400 + {{- end }} + {{- range $i, $device := $driver.hostDevices }} + - name: device-{{ $i }} + hostPath: + path: {{ $device }} + type: Directory + {{- end }} +{{- end }} diff --git a/charts/pyxis/templates/pvc.yaml b/charts/pyxis/templates/pvc.yaml new file mode 100644 index 0000000..057bbfc --- /dev/null +++ b/charts/pyxis/templates/pvc.yaml @@ -0,0 +1,32 @@ +{{- $needs := dict + "recordings" (or .Values.driver.enabled .Values.app.enabled) + "encodes" .Values.app.enabled + "thumbnails" .Values.app.enabled + "captions" .Values.app.enabled + "keys" .Values.app.enabled + "db" .Values.postgres.enabled +}} +{{- range $name, $needed := $needs }} +{{- $spec := index $.Values.persistence $name }} +{{- if and $needed (eq $spec.type "pvc") }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "pyxis.partName" (dict "root" $ "part" $name) }} + labels: + {{- include "pyxis.labels" (dict "root" $ "part" $name) | nindent 4 }} + annotations: + # リリースを消しても残す + helm.sh/resource-policy: keep +spec: + accessModes: + {{- toYaml $spec.accessModes | nindent 4 }} + {{- if $spec.storageClass }} + storageClassName: {{ if eq $spec.storageClass "-" }}""{{ else }}{{ $spec.storageClass }}{{ end }} + {{- end }} + resources: + requests: + storage: {{ $spec.size }} +{{- end }} +{{- end }} diff --git a/charts/pyxis/templates/route.yaml b/charts/pyxis/templates/route.yaml new file mode 100644 index 0000000..d0873ae --- /dev/null +++ b/charts/pyxis/templates/route.yaml @@ -0,0 +1,80 @@ +{{- $route := .Values.route }} +{{- $app := include "pyxis.partName" (dict "root" . "part" "app") }} +{{- $vela := include "pyxis.partName" (dict "root" . "part" "vela") }} +{{- if and (ne $route.kind "none") (or .Values.app.enabled .Values.vela.enabled) }} +{{- if eq $route.kind "ingress" }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ .Release.Name }} + labels: + {{- include "pyxis.labels" (dict "root" . "part" "route") | nindent 4 }} + {{- with $route.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with $route.ingress.className }} + ingressClassName: {{ . }} + {{- end }} + {{- with $route.ingress.tls }} + tls: + {{- toYaml . | nindent 4 }} + {{- end }} + rules: + - {{ with $route.host }}host: {{ . | quote }} + {{ end }}http: + paths: + {{- if .Values.app.enabled }} + - path: /api + pathType: Prefix + backend: + service: + name: {{ $app }} + port: + name: http + {{- end }} + {{- if .Values.vela.enabled }} + - path: / + pathType: Prefix + backend: + service: + name: {{ $vela }} + port: + name: http + {{- end }} +{{- else if eq $route.kind "ingressRoute" }} +{{- $host := "" }} +{{- with $route.host }}{{ $host = printf "Host(`%s`) && " . }}{{ end }} +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: {{ .Release.Name }} + labels: + {{- include "pyxis.labels" (dict "root" . "part" "route") | nindent 4 }} +spec: + entryPoints: + {{- toYaml $route.ingressRoute.entryPoints | nindent 4 }} + routes: + {{- if .Values.app.enabled }} + - match: {{ printf "%sPathPrefix(`/api`)" $host | quote }} + kind: Rule + services: + - name: {{ $app }} + port: 8080 + {{- end }} + {{- if .Values.vela.enabled }} + - match: {{ printf "%sPathPrefix(`/`)" $host | quote }} + kind: Rule + services: + - name: {{ $vela }} + port: 3000 + {{- end }} + {{- if not (kindIs "invalid" $route.ingressRoute.tls) }} + tls: + {{- toYaml $route.ingressRoute.tls | nindent 4 }} + {{- end }} +{{- else }} +{{- fail "route.kind は ingress、ingressRoute、none のどれかにする" }} +{{- end }} +{{- end }} diff --git a/charts/pyxis/templates/vela-deployment.yaml b/charts/pyxis/templates/vela-deployment.yaml new file mode 100644 index 0000000..d9f9a69 --- /dev/null +++ b/charts/pyxis/templates/vela-deployment.yaml @@ -0,0 +1,51 @@ +{{- if .Values.vela.enabled }} +{{- $vela := .Values.vela }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "pyxis.partName" (dict "root" . "part" "vela") }} + labels: + {{- include "pyxis.labels" (dict "root" . "part" "vela") | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "pyxis.selectorLabels" (dict "root" . "part" "vela") | nindent 6 }} + template: + metadata: + labels: + {{- include "pyxis.selectorLabels" (dict "root" . "part" "vela") | nindent 8 }} + spec: + {{- with include "pyxis.imagePullSecrets" . }}{{ . | trim | nindent 6 }}{{- end }} + {{- with include "pyxis.nodeSelector" (dict "root" . "own" $vela.nodeSelector) }}{{ . | trim | nindent 6 }}{{- end }} + containers: + - name: vela + image: {{ printf "%s:%s" $vela.image.repository $vela.image.tag | quote }} + imagePullPolicy: {{ $vela.image.pullPolicy }} + env: + - name: CARINA_API_BASE_URL + value: {{ printf "http://%s:8080" (include "pyxis.partName" (dict "root" . "part" "app")) | quote }} + # next の standalone は HOSTNAME で待ち受ける。k8s は Pod 名を入れるので全部で待ち受けさせる + - name: HOSTNAME + value: 0.0.0.0 + ports: + - name: http + containerPort: 3000 + startupProbe: + httpGet: + path: /login + port: http + periodSeconds: 5 + timeoutSeconds: 5 + failureThreshold: 24 + readinessProbe: + httpGet: + path: /login + port: http + periodSeconds: 10 + timeoutSeconds: 5 + {{- with $vela.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} +{{- end }} diff --git a/charts/pyxis/templates/vela-service.yaml b/charts/pyxis/templates/vela-service.yaml new file mode 100644 index 0000000..8ac6f6e --- /dev/null +++ b/charts/pyxis/templates/vela-service.yaml @@ -0,0 +1,16 @@ +{{- if .Values.vela.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "pyxis.partName" (dict "root" . "part" "vela") }} + labels: + {{- include "pyxis.labels" (dict "root" . "part" "vela") | nindent 4 }} +spec: + type: ClusterIP + selector: + {{- include "pyxis.selectorLabels" (dict "root" . "part" "vela") | nindent 4 }} + ports: + - name: http + port: 3000 + targetPort: http +{{- end }} diff --git a/charts/pyxis/values.schema.json b/charts/pyxis/values.schema.json new file mode 100644 index 0000000..2de843c --- /dev/null +++ b/charts/pyxis/values.schema.json @@ -0,0 +1,300 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "pyxis の値", + "type": "object", + "additionalProperties": false, + "properties": { + "global": { "type": "object" }, + "nodeSelector": { "$ref": "#/$defs/stringMap" }, + "imagePullSecrets": { "type": "array", "items": { "type": "string", "minLength": 1 } }, + "carina": { + "type": "object", + "additionalProperties": false, + "required": ["uid", "gid"], + "properties": { + "uid": { "type": "integer", "minimum": 1 }, + "gid": { "type": "integer", "minimum": 1 } + } + }, + "driver": { + "type": "object", + "additionalProperties": false, + "required": ["enabled", "image", "config", "hostDevices", "pcscd", "socket", "hostNetwork", "lendingKey", "terminationGracePeriodSeconds"], + "properties": { + "enabled": { "type": "boolean" }, + "image": { "$ref": "#/$defs/image" }, + "config": { + "type": "object", + "required": ["outputRoots", "tuner", "devices"], + "properties": { + "outputRoots": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["name", "path"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "path": { "$ref": "#/$defs/absolutePath" } + } + } + }, + "shutdownGraceHours": { "type": "integer", "minimum": 1, "maximum": 168 }, + "tuner": { + "type": "object", + "required": ["backend"], + "properties": { + "backend": { "type": "string", "enum": ["dvb", "usb", "fake"] } + } + }, + "devices": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["id", "kind"], + "properties": { + "id": { "type": "string", "minLength": 1 }, + "kind": { "type": "string", "enum": ["terrestrial", "satellite"] }, + "devicePath": { "$ref": "#/$defs/absolutePath" }, + "enabled": { "type": "boolean" } + } + } + } + } + }, + "hostDevices": { "type": "array", "items": { "$ref": "#/$defs/absolutePath" } }, + "pcscd": { + "type": "object", + "additionalProperties": false, + "required": ["enabled"], + "properties": { "enabled": { "type": "boolean" } } + }, + "socket": { + "type": "object", + "additionalProperties": false, + "required": ["hostPath"], + "properties": { "hostPath": { "$ref": "#/$defs/absolutePath" } } + }, + "hostNetwork": { "type": "boolean" }, + "lendingKey": { + "type": "object", + "additionalProperties": false, + "required": ["existingSecret", "key"], + "properties": { + "existingSecret": { "type": "string" }, + "key": { "type": "string", "minLength": 1 } + } + }, + "terminationGracePeriodSeconds": { "type": "integer", "minimum": 0 }, + "nodeSelector": { "$ref": "#/$defs/stringMap" }, + "resources": { "$ref": "#/$defs/resources" } + } + }, + "app": { + "type": "object", + "additionalProperties": false, + "required": ["enabled", "image", "publicOrigin", "knownNetworks", "encodingsPrefer", "transcodingPrefer", "gpu", "dataProtection", "extraEnv"], + "properties": { + "enabled": { "type": "boolean" }, + "image": { "$ref": "#/$defs/image" }, + "publicOrigin": { "type": "string", "pattern": "^(https?://[^/]+)?$" }, + "knownNetworks": { "type": "array", "items": { "type": "string", "minLength": 1 } }, + "encodingsPrefer": { "$ref": "#/$defs/prefer" }, + "transcodingPrefer": { "$ref": "#/$defs/prefer" }, + "gpu": { + "type": "object", + "additionalProperties": false, + "required": ["mode", "resource", "renderNode", "hostPath"], + "properties": { + "mode": { "type": "string", "enum": ["none", "devicePlugin", "hostPath"] }, + "resource": { "type": "string", "minLength": 1 }, + "renderNode": { "$ref": "#/$defs/absolutePath" }, + "hostPath": { "$ref": "#/$defs/absolutePath" } + } + }, + "dataProtection": { + "type": "object", + "additionalProperties": false, + "required": ["seedSecret"], + "properties": { "seedSecret": { "type": "string" } } + }, + "extraEnv": { + "type": "array", + "items": { + "type": "object", + "required": ["name"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "value": { "type": "string" }, + "valueFrom": { "type": "object" } + } + } + }, + "nodeSelector": { "$ref": "#/$defs/stringMap" }, + "resources": { "$ref": "#/$defs/resources" } + } + }, + "postgres": { + "type": "object", + "additionalProperties": false, + "required": ["enabled", "image", "auth"], + "properties": { + "enabled": { "type": "boolean" }, + "image": { "$ref": "#/$defs/image" }, + "auth": { + "type": "object", + "additionalProperties": false, + "required": ["database", "username", "existingSecret", "passwordKey", "password"], + "properties": { + "database": { "type": "string", "minLength": 1 }, + "username": { "type": "string", "minLength": 1 }, + "existingSecret": { "type": "string" }, + "passwordKey": { "type": "string", "minLength": 1 }, + "password": { "type": "string", "pattern": "^[^;]*$" } + } + }, + "nodeSelector": { "$ref": "#/$defs/stringMap" }, + "resources": { "$ref": "#/$defs/resources" } + } + }, + "externalDatabase": { + "type": "object", + "additionalProperties": false, + "required": ["connectionSecret", "connectionKey"], + "properties": { + "connectionSecret": { "type": "string" }, + "connectionKey": { "type": "string", "minLength": 1 } + } + }, + "vela": { + "type": "object", + "additionalProperties": false, + "required": ["enabled", "image"], + "properties": { + "enabled": { "type": "boolean" }, + "image": { "$ref": "#/$defs/image" }, + "nodeSelector": { "$ref": "#/$defs/stringMap" }, + "resources": { "$ref": "#/$defs/resources" } + } + }, + "route": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "host", "ingress", "ingressRoute"], + "properties": { + "kind": { "type": "string", "enum": ["ingress", "ingressRoute", "none"] }, + "host": { "type": "string", "pattern": "^[^/\\s]*$" }, + "ingress": { + "type": "object", + "additionalProperties": false, + "required": ["className", "annotations", "tls"], + "properties": { + "className": { "type": "string" }, + "annotations": { "$ref": "#/$defs/stringMap" }, + "tls": { + "type": "array", + "items": { + "type": "object", + "properties": { + "hosts": { "type": "array", "items": { "type": "string" } }, + "secretName": { "type": "string" } + } + } + } + } + }, + "ingressRoute": { + "type": "object", + "additionalProperties": false, + "required": ["entryPoints"], + "properties": { + "entryPoints": { "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 } }, + "tls": { "type": ["object", "null"] } + } + } + } + }, + "persistence": { + "type": "object", + "additionalProperties": false, + "required": ["recordings", "encodes", "thumbnails", "captions", "keys", "db"], + "properties": { + "recordings": { "$ref": "#/$defs/storage" }, + "encodes": { "$ref": "#/$defs/storage" }, + "thumbnails": { "$ref": "#/$defs/storageOrEmptyDir" }, + "captions": { "$ref": "#/$defs/storageOrEmptyDir" }, + "keys": { "$ref": "#/$defs/storage" }, + "db": { "$ref": "#/$defs/storage" } + } + } + }, + "$defs": { + "absolutePath": { "type": "string", "pattern": "^/" }, + "stringMap": { "type": "object", "additionalProperties": { "type": "string" } }, + "prefer": { "type": "string", "enum": ["Software", "Vaapi"] }, + "resources": { + "type": "object", + "additionalProperties": false, + "properties": { + "requests": { "type": "object", "additionalProperties": { "type": ["string", "number"] } }, + "limits": { "type": "object", "additionalProperties": { "type": ["string", "number"] } } + } + }, + "image": { + "type": "object", + "additionalProperties": false, + "required": ["repository", "tag", "pullPolicy"], + "properties": { + "repository": { "type": "string", "minLength": 1 }, + "tag": { "type": "string", "minLength": 1 }, + "pullPolicy": { "type": "string", "enum": ["Always", "IfNotPresent", "Never"] } + } + }, + "storageFields": { + "type": "object", + "required": ["type", "size", "storageClass", "accessModes", "existingClaim", "hostPath"], + "properties": { + "size": { "type": "string", "pattern": "^[0-9]+(\\.[0-9]+)?(Ki|Mi|Gi|Ti|Pi|Ei|k|M|G|T|P|E)?$" }, + "storageClass": { "type": "string" }, + "accessModes": { "type": "array", "minItems": 1, "items": { "type": "string", "enum": ["ReadWriteOnce", "ReadOnlyMany", "ReadWriteMany", "ReadWriteOncePod"] } }, + "existingClaim": { "type": "string" }, + "hostPath": { + "type": "object", + "additionalProperties": false, + "required": ["path", "type"], + "properties": { + "path": { "type": "string", "pattern": "^(/.*)?$" }, + "type": { "type": "string", "enum": ["Directory", "DirectoryOrCreate"] } + } + } + } + }, + "storage": { + "allOf": [ + { "$ref": "#/$defs/storageFields" }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { "type": "string", "enum": ["pvc", "existingClaim", "hostPath"] }, + "size": true, "storageClass": true, "accessModes": true, "existingClaim": true, "hostPath": true + } + } + ] + }, + "storageOrEmptyDir": { + "allOf": [ + { "$ref": "#/$defs/storageFields" }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { "type": "string", "enum": ["pvc", "existingClaim", "hostPath", "emptyDir"] }, + "size": true, "storageClass": true, "accessModes": true, "existingClaim": true, "hostPath": true + } + } + ] + } + } +} diff --git a/charts/pyxis/values.yaml b/charts/pyxis/values.yaml new file mode 100644 index 0000000..22ef197 --- /dev/null +++ b/charts/pyxis/values.yaml @@ -0,0 +1,269 @@ +# 全部の Pod に当てる nodeSelector。driver と app はソケットのディレクトリを hostPath で共有するので、ノードが複数なら同じノードを指す +nodeSelector: {} +# イメージを引くときに使う Secret の名前の一覧。GHCR の公開イメージなら要らない +imagePullSecrets: [] + +carina: + # app を動かす uid。鍵の置き場やエンコード済みファイルの持ち主になる + uid: 10001 + # app を動かす gid。driver のソケットの所有グループ (driver.json の socketGroupId) にも同じ値を入れる + gid: 10001 + +driver: + # driver の Deployment と ConfigMap を作るか + enabled: true + image: + # driver のイメージ + repository: ghcr.io/newworldorg/carina + # driver のイメージのタグ。compose.yml と同じ版にする + tag: driver-v1.0.0 + # イメージを引く条件 + pullPolicy: IfNotPresent + # driver.json の中身をそのまま書く。socketGroupId はチャートが carina.gid で入れ、socketPath は既定 (/run/carina/driver.sock) のままにする + config: + # 録画の保存先。app の Integrity__OutputRoots と同じ名前とパスにする + outputRoots: + - name: primary + path: /srv/recordings + # 録画中に止められたとき、録画の終わりを待つ上限の時間 + shutdownGraceHours: 6 + tuner: + # dvb、usb か fake。置いてあるのは合成のチューナー + backend: fake + # チューナーの一覧。dvb と usb なら devicePath も書く + devices: + - id: fake-terrestrial + kind: terrestrial + enabled: true + - id: fake-satellite + kind: satellite + enabled: true + # ホストの装置のディレクトリの一覧。同じパスで hostPath で渡す (/dev/dvb/adapter0 や /dev/bus/usb) + hostDevices: [] + pcscd: + # ホストの pcscd の /run/pcscd をディレクトリごと読み取り専用で渡すか。tuner.backend が dvb なら要る + enabled: false + socket: + # driver と app が共有するソケットのディレクトリのホスト側のパス。無ければ作る + hostPath: /run/pyxis + # ホストのネットワークで動かすか。カードを貸す driver は true にして、口を Pod の網の外に開く + hostNetwork: false + lendingKey: + # カードの貸し借りの鍵を持つ Secret の名前。空なら渡さない + existingSecret: "" + # その Secret の中の鍵のキー。/etc/carina/lending.key に所有者だけが読める形で置く + key: key + # 止めるときに録画の終わりを待つ秒数。config.shutdownGraceHours より長くする + terminationGracePeriodSeconds: 25200 + # driver だけに当てる nodeSelector。空なら上の nodeSelector を使う + nodeSelector: {} + # driver の resources + resources: {} + +app: + # app の Deployment と Service を作るか + enabled: true + image: + # app と migrate のイメージ + repository: ghcr.io/newworldorg/carina + # app のイメージのタグ。compose.yml と同じ版にする + tag: app-v1.0.0 + # イメージを引く条件 + pullPolicy: IfNotPresent + # ブラウザからこのサーバに届くアドレス (https://tv.example.com)。OIDC の redirect URI の元になる + publicOrigin: "" + # X-Forwarded-* を信じる前段の網の一覧 (10.42.0.0/16 など)。Ingress のコントローラの Pod がいる網を書く + knownNetworks: [] + # 録画をあとからエンコードするときの変換器。Software か Vaapi + encodingsPrefer: Software + # ライブと録画再生をその場で変換するときの変換器。Software か Vaapi + transcodingPrefer: Software + gpu: + # GPU の渡し方。none、devicePlugin (Intel の device plugin)、hostPath (/dev/dri を渡し、app を特権にする) + mode: none + # devicePlugin のときに要求する資源の名前 + resource: gpu.intel.com/i915 + # VAAPI で使う GPU の描画ノード + renderNode: /dev/dri/renderD128 + # hostPath のときにホストから渡す装置のディレクトリ + hostPath: /dev/dri + dataProtection: + # OIDC の client secret を封じる鍵の種を持つ Secret の名前。鍵の置き場が空のときだけ写す。空なら app が初回に作る + seedSecret: "" + # app に足す環境変数の一覧 (name と value)。Encodings__MostCores などの細かい設定に使う + extraEnv: [] + # app だけに当てる nodeSelector。空なら上の nodeSelector を使う + nodeSelector: {} + # app の resources。devicePlugin のときは GPU の資源をチャートが足す + resources: {} + +postgres: + # 同梱のデータベース (StatefulSet と Service) を作るか。false なら externalDatabase を使う + enabled: true + image: + # PostgreSQL のイメージ + repository: postgres + # PostgreSQL のイメージのタグ。compose.yml と同じ版にする + tag: 18.6-alpine + # イメージを引く条件 + pullPolicy: IfNotPresent + auth: + # データベースの名前 + database: carina + # データベースのユーザ名 + username: carina + # パスワードを持つ Secret の名前。空なら password から Secret を作る + existingSecret: "" + # その Secret の中のパスワードのキー + passwordKey: password + # existingSecret が空のときに Secret に入れるパスワード。`;` は使えない。どちらも空だと描画が止まる + password: "" + # データベースだけに当てる nodeSelector。空なら上の nodeSelector を使う + nodeSelector: {} + # データベースの resources + resources: {} + +externalDatabase: + # postgres.enabled が false のとき、接続文字列を持つ Secret の名前 + connectionSecret: "" + # その Secret の中の接続文字列のキー + connectionKey: connection + +vela: + # 画面の Deployment と Service を作るか + enabled: true + image: + # 画面のイメージ + repository: ghcr.io/newworldorg/vela + # 画面のイメージのタグ。compose.yml と同じ版にする + tag: v1.0.0 + # イメージを引く条件 + pullPolicy: IfNotPresent + # 画面だけに当てる nodeSelector。空なら上の nodeSelector を使う + nodeSelector: {} + # 画面の resources + resources: {} + +route: + # 前段の種類。ingress (標準の Ingress)、ingressRoute (Traefik の IngressRoute)、none (自前の前段) + kind: ingress + # ブラウザが開くホスト名 (tv.example.com)。空ならホスト名を見ずに全部を受ける + host: "" + ingress: + # Ingress の ingressClassName。空なら書かない + className: "" + # Ingress の annotations。SSE と WebSocket を溜めない設定はコントローラごとに違う + annotations: {} + # Ingress の tls の一覧 (hosts と secretName) + tls: [] + ingressRoute: + # IngressRoute の entryPoints + entryPoints: + - websecure + # IngressRoute の tls。{} なら Traefik の既定の証明書、null なら http + tls: {} + +# 置き場ごとに type を pvc、existingClaim、hostPath から選ぶ。thumbnails と captions は emptyDir も選べる +persistence: + recordings: + # 録画。driver が書き、app は読むだけ + type: pvc + # pvc のときの大きさ + size: 500Gi + # pvc のときの storageClassName。空なら既定、"-" なら無し + storageClass: "" + # pvc のときの accessModes + accessModes: + - ReadWriteOnce + # existingClaim のときの PVC の名前 + existingClaim: "" + hostPath: + # hostPath のときのホスト側のパス + path: "" + # hostPath のときの type。Directory なら無いときに起動しない + type: Directory + encodes: + # エンコード済みのファイル。app が書く + type: pvc + # pvc のときの大きさ + size: 200Gi + # pvc のときの storageClassName。空なら既定、"-" なら無し + storageClass: "" + # pvc のときの accessModes + accessModes: + - ReadWriteOnce + # existingClaim のときの PVC の名前 + existingClaim: "" + hostPath: + # hostPath のときのホスト側のパス + path: "" + # hostPath のときの type + type: Directory + thumbnails: + # サムネイル。app が書く。emptyDir なら Pod と一緒に消え、作り直される + type: pvc + # pvc のときの大きさ + size: 5Gi + # pvc のときの storageClassName。空なら既定、"-" なら無し + storageClass: "" + # pvc のときの accessModes + accessModes: + - ReadWriteOnce + # existingClaim のときの PVC の名前 + existingClaim: "" + hostPath: + # hostPath のときのホスト側のパス + path: "" + # hostPath のときの type + type: Directory + captions: + # 録画から取り出した字幕とデータ放送の記録。app が書く。emptyDir なら Pod と一緒に消え、取り出し直される + type: pvc + # pvc のときの大きさ + size: 5Gi + # pvc のときの storageClassName。空なら既定、"-" なら無し + storageClass: "" + # pvc のときの accessModes + accessModes: + - ReadWriteOnce + # existingClaim のときの PVC の名前 + existingClaim: "" + hostPath: + # hostPath のときのホスト側のパス + path: "" + # hostPath のときの type + type: Directory + keys: + # OIDC の client secret を封じる鍵。データベースとは別に残す + type: pvc + # pvc のときの大きさ + size: 100Mi + # pvc のときの storageClassName。空なら既定、"-" なら無し + storageClass: "" + # pvc のときの accessModes + accessModes: + - ReadWriteOnce + # existingClaim のときの PVC の名前 + existingClaim: "" + hostPath: + # hostPath のときのホスト側のパス + path: "" + # hostPath のときの type + type: Directory + db: + # 同梱のデータベースのデータ + type: pvc + # pvc のときの大きさ + size: 20Gi + # pvc のときの storageClassName。空なら既定、"-" なら無し + storageClass: "" + # pvc のときの accessModes + accessModes: + - ReadWriteOnce + # existingClaim のときの PVC の名前 + existingClaim: "" + hostPath: + # hostPath のときのホスト側のパス + path: "" + # hostPath のときの type + type: Directory diff --git a/scripts/check-rendering.sh b/scripts/check-rendering.sh new file mode 100755 index 0000000..7dffb40 --- /dev/null +++ b/scripts/check-rendering.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# チャートの描き方の決まりを、既定の値で描いた結果に対して確かめる。 +# Pod のテンプレートにチャートの版 (helm.sh/chart) を入れない。ConfigMap の checksum は driver の Pod にだけ付ける。 +# リリース名 carina で描くと Service が carina-app・carina-db・carina-vela になる +set -euo pipefail + +cd "$(dirname "$0")/.." + +rendered=$(helm template carina charts/pyxis -f charts/pyxis/ci/default-values.yaml) + +status=0 + +in_pod_templates() { + awk '/^---/ { inside = 0 } /^ template:/ { inside = 1 } inside' +} + +if printf '%s\n' "$rendered" | in_pod_templates | grep -q 'helm.sh/chart'; then + echo 'Pod のテンプレートに helm.sh/chart のラベルがある' >&2 + status=1 +fi + +checksum_owners=$(printf '%s\n' "$rendered" | awk ' + /^---/ { name = "" } + /^ name: / && name == "" { name = $2 } + /checksum\/config/ { print name } +') +if [ "$checksum_owners" != 'carina-driver' ]; then + echo "ConfigMap の checksum を持つのは carina-driver だけのはずが: ${checksum_owners:-(無し)}" >&2 + status=1 +fi + +services=$(printf '%s\n' "$rendered" | awk ' + /^---/ { kind = ""; name = "" } + /^kind: / { kind = $2 } + /^ name: / && name == "" { name = $2; if (kind == "Service") print name } +' | sort | tr '\n' ' ') +if [ "$services" != 'carina-app carina-db carina-vela ' ]; then + echo "Service の名前が違う: $services" >&2 + status=1 +fi + +[ "$status" = 0 ] && echo '描き方の決まりは守られている' +exit "$status" diff --git a/scripts/check-versions.sh b/scripts/check-versions.sh new file mode 100755 index 0000000..44c50ee --- /dev/null +++ b/scripts/check-versions.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# compose.yml、charts/pyxis/values.yaml、charts/pyxis/Chart.yaml の 3 か所に書いたイメージの版が同じかを確かめる。 +# values.yaml と Chart.yaml のイメージはどれも compose.yml に無ければならず、compose.yml の carina・vela・postgres もその中に無ければならない +set -euo pipefail + +cd "$(dirname "$0")/.." + +compose_images() { + sed -n 's/^ *image: *\(.*\)$/\1/p' compose.yml | tr -d "'\"" | sort -u +} + +values_images() { + awk ' + /^ *repository: / { repository = $2 } + /^ *tag: / { print repository ":" $2 } + ' charts/pyxis/values.yaml | sort -u +} + +chart_images() { + sed -n 's/^ *images\/[a-z-]*: *\(.*\)$/\1/p' charts/pyxis/Chart.yaml | sort -u +} + +status=0 + +check_subset() { + local label=$1 + local images=$2 + local image + while IFS= read -r image; do + [ -n "$image" ] || continue + if ! compose_images | grep -qxF "$image"; then + echo "$label の $image は compose.yml に無い" >&2 + status=1 + fi + done <<<"$images" +} + +check_subset 'values.yaml' "$(values_images)" +check_subset 'Chart.yaml' "$(chart_images)" + +while IFS= read -r image; do + case "$image" in + */carina:* | */vela:* | postgres:*) ;; + *) continue ;; + esac + if ! values_images | grep -qxF "$image"; then + echo "compose.yml の $image は values.yaml に無い" >&2 + status=1 + fi + if ! chart_images | grep -qxF "$image"; then + echo "compose.yml の $image は Chart.yaml に無い" >&2 + status=1 + fi +done < <(compose_images) + +if [ "$status" = 0 ]; then + echo 'compose.yml、values.yaml、Chart.yaml のイメージの版は同じ' + values_images +fi +exit "$status"