diff --git a/.hazyforge/clusters/anvil-primaris/namespace/chat2db/deploy.yaml b/.hazyforge/clusters/anvil-primaris/namespace/chat2db/deploy.yaml index 2a52a34f3b..192d68549c 100644 --- a/.hazyforge/clusters/anvil-primaris/namespace/chat2db/deploy.yaml +++ b/.hazyforge/clusters/anvil-primaris/namespace/chat2db/deploy.yaml @@ -31,6 +31,23 @@ ai: models: - gpt-5.4 +datasourceBootstrap: + enabled: true + rolloutRevision: "1" + managementKey: hazytrade-production-readonly + alias: Hazy Trade Production (read only) + type: POSTGRESQL + host: 5.161.253.191 + port: 5432 + database: hazytrade + user: chat2db_hazytrade_ro + authenticationType: "1" + environmentId: 2 + sslMode: verify-full + existingSecret: chat2db-hazytrade-datasource + passwordKey: password + caKey: ca.crt + resources: requests: cpu: 500m diff --git a/.hazyforge/clusters/anvil-primaris/namespace/chat2db/manifests/hazytrade-datasource-external-secret.yaml b/.hazyforge/clusters/anvil-primaris/namespace/chat2db/manifests/hazytrade-datasource-external-secret.yaml new file mode 100644 index 0000000000..e5a8d9caf3 --- /dev/null +++ b/.hazyforge/clusters/anvil-primaris/namespace/chat2db/manifests/hazytrade-datasource-external-secret.yaml @@ -0,0 +1,21 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: chat2db-hazytrade-datasource + namespace: chat2db +spec: + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: azurekv-cluster-secret-store + target: + name: chat2db-hazytrade-datasource + creationPolicy: Owner + deletionPolicy: Delete + data: + - secretKey: password + remoteRef: + key: secret/chat2db-hazytrade-ro-password + - secretKey: ca.crt + remoteRef: + key: secret/hazy-trade-postgres-ca-crt diff --git a/HAZYFORGE.md b/HAZYFORGE.md index 550a80b599..dbe9225e8d 100644 --- a/HAZYFORGE.md +++ b/HAZYFORGE.md @@ -31,6 +31,27 @@ identity allowlists are not supported by this deployment. - The dedicated ZITADEL client ID and secret, oauth2-proxy cookie key, and exact allowed subject are projected from Azure Key Vault by External Secrets. +## Managed PostgreSQL datasource + +The chart can reconcile one PostgreSQL datasource at application startup with +`datasourceBootstrap.enabled`. Non-secret connection identity is supplied in +Helm values. The password and CA certificate are read from files in a dedicated +existing Kubernetes Secret; they must never be placed directly in values. + +The datasource is owned by an explicit management key stored with its local +record. Reconciliation never adopts an unmarked datasource, never treats an +alias as ownership, and refuses an alias collision rather than changing a +user-created connection. New and changed configuration is connection-tested +before it is persisted, and the password uses the same AES-GCM encryption key +as manually created Community datasources. + +The PostgreSQL account is the write-safety boundary. Give it only `CONNECT`, +schema `USAGE`, and the required `SELECT` grants; Chat2DB does not impose a +query-level read-only policy. Because this cluster has no Secret-reload +controller, rotate the mounted credential together with a declarative pod +rollout by bumping `datasourceBootstrap.rolloutRevision`, so startup +reconciliation applies the new password. + ## Image `.github/workflows/hazyforge-image.yml` builds the frontend and backend from diff --git a/charts/chat2db/templates/deployment.yaml b/charts/chat2db/templates/deployment.yaml index b7391f96c3..17ac821295 100644 --- a/charts/chat2db/templates/deployment.yaml +++ b/charts/chat2db/templates/deployment.yaml @@ -15,6 +15,10 @@ spec: metadata: labels: {{- include "chat2db.selectorLabels" . | nindent 8 }} + {{- with .Values.datasourceBootstrap.rolloutRevision }} + annotations: + hazyforge.io/datasource-rollout-revision: {{ . | quote }} + {{- end }} spec: automountServiceAccountToken: {{ .Values.automountServiceAccountToken }} securityContext: @@ -38,6 +42,34 @@ spec: value: {{ .Values.encryptionKey.mountPath | quote }} - name: JAVA_TOOL_OPTIONS value: "-XX:MaxRAMPercentage=75.0 -XX:+ExitOnOutOfMemoryError" + {{- if .Values.datasourceBootstrap.enabled }} + - name: CHAT2DB_BOOTSTRAP_DATASOURCE_ENABLED + value: "true" + - name: CHAT2DB_BOOTSTRAP_DATASOURCE_MANAGEMENT_KEY + value: {{ required "datasourceBootstrap.managementKey is required when datasource bootstrap is enabled" .Values.datasourceBootstrap.managementKey | quote }} + - name: CHAT2DB_BOOTSTRAP_DATASOURCE_ALIAS + value: {{ required "datasourceBootstrap.alias is required when datasource bootstrap is enabled" .Values.datasourceBootstrap.alias | quote }} + - name: CHAT2DB_BOOTSTRAP_DATASOURCE_TYPE + value: {{ required "datasourceBootstrap.type is required when datasource bootstrap is enabled" .Values.datasourceBootstrap.type | quote }} + - name: CHAT2DB_BOOTSTRAP_DATASOURCE_HOST + value: {{ required "datasourceBootstrap.host is required when datasource bootstrap is enabled" .Values.datasourceBootstrap.host | quote }} + - name: CHAT2DB_BOOTSTRAP_DATASOURCE_PORT + value: {{ .Values.datasourceBootstrap.port | quote }} + - name: CHAT2DB_BOOTSTRAP_DATASOURCE_DATABASE + value: {{ required "datasourceBootstrap.database is required when datasource bootstrap is enabled" .Values.datasourceBootstrap.database | quote }} + - name: CHAT2DB_BOOTSTRAP_DATASOURCE_USER + value: {{ required "datasourceBootstrap.user is required when datasource bootstrap is enabled" .Values.datasourceBootstrap.user | quote }} + - name: CHAT2DB_BOOTSTRAP_DATASOURCE_AUTHENTICATION_TYPE + value: {{ .Values.datasourceBootstrap.authenticationType | quote }} + - name: CHAT2DB_BOOTSTRAP_DATASOURCE_ENVIRONMENT_ID + value: {{ .Values.datasourceBootstrap.environmentId | quote }} + - name: CHAT2DB_BOOTSTRAP_DATASOURCE_SSL_MODE + value: {{ .Values.datasourceBootstrap.sslMode | quote }} + - name: CHAT2DB_BOOTSTRAP_DATASOURCE_PASSWORD_FILE + value: {{ .Values.datasourceBootstrap.passwordMountPath | quote }} + - name: CHAT2DB_BOOTSTRAP_DATASOURCE_CA_FILE + value: {{ .Values.datasourceBootstrap.caMountPath | quote }} + {{- end }} {{- with .Values.cors.allowedOrigin }} - name: CHAT2DB_WEB_CORS_ALLOWED_ORIGIN value: {{ . | quote }} @@ -95,6 +127,16 @@ spec: mountPath: {{ .Values.encryptionKey.mountPath | quote }} subPath: {{ .Values.encryptionKey.secretKey | quote }} readOnly: true + {{- if .Values.datasourceBootstrap.enabled }} + - name: datasource-bootstrap + mountPath: {{ .Values.datasourceBootstrap.passwordMountPath | quote }} + subPath: password + readOnly: true + - name: datasource-bootstrap + mountPath: {{ .Values.datasourceBootstrap.caMountPath | quote }} + subPath: ca.crt + readOnly: true + {{- end }} - name: tmp mountPath: /tmp resources: @@ -107,6 +149,17 @@ spec: secret: secretName: {{ .Values.encryptionKey.existingSecret }} defaultMode: 0440 + {{- if .Values.datasourceBootstrap.enabled }} + - name: datasource-bootstrap + secret: + secretName: {{ required "datasourceBootstrap.existingSecret is required when datasource bootstrap is enabled" .Values.datasourceBootstrap.existingSecret }} + defaultMode: 0440 + items: + - key: {{ .Values.datasourceBootstrap.passwordKey | quote }} + path: password + - key: {{ .Values.datasourceBootstrap.caKey | quote }} + path: ca.crt + {{- end }} - name: tmp emptyDir: sizeLimit: 1Gi diff --git a/charts/chat2db/test/render-managed-datasource.sh b/charts/chat2db/test/render-managed-datasource.sh new file mode 100755 index 0000000000..6f847d1a98 --- /dev/null +++ b/charts/chat2db/test/render-managed-datasource.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail + +chart_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +render_dir="$(mktemp -d)" +trap 'rm -rf "${render_dir}"' EXIT + +helm lint "${chart_dir}" +helm template chat2db "${chart_dir}" >"${render_dir}/default.yaml" + +if grep -q 'CHAT2DB_BOOTSTRAP_DATASOURCE' "${render_dir}/default.yaml"; then + echo "disabled datasource bootstrap unexpectedly rendered configuration" >&2 + exit 1 +fi + +helm template chat2db "${chart_dir}" \ + --set datasourceBootstrap.enabled=true \ + --set datasourceBootstrap.rolloutRevision=1 \ + --set datasourceBootstrap.managementKey=test-readonly \ + --set 'datasourceBootstrap.alias=Test Read Only' \ + --set datasourceBootstrap.host=postgres.example.internal \ + --set datasourceBootstrap.database=app \ + --set datasourceBootstrap.user=reader \ + --set datasourceBootstrap.existingSecret=chat2db-database \ + >"${render_dir}/enabled.yaml" + +grep -q 'name: CHAT2DB_BOOTSTRAP_DATASOURCE_ENABLED' "${render_dir}/enabled.yaml" +grep -q 'hazyforge.io/datasource-rollout-revision: "1"' "${render_dir}/enabled.yaml" +grep -q 'secretName: chat2db-database' "${render_dir}/enabled.yaml" +grep -q 'defaultMode: 0440' "${render_dir}/enabled.yaml" +grep -q 'mountPath: "/run/secrets/chat2db-datasource/password"' "${render_dir}/enabled.yaml" +grep -q 'mountPath: "/run/secrets/chat2db-datasource/ca.crt"' "${render_dir}/enabled.yaml" + +if helm template chat2db "${chart_dir}" \ + --set datasourceBootstrap.enabled=true \ + >"${render_dir}/invalid.yaml" 2>"${render_dir}/invalid.err"; then + echo "enabled datasource bootstrap rendered without required values" >&2 + exit 1 +fi diff --git a/charts/chat2db/values.yaml b/charts/chat2db/values.yaml index 3b8d65e403..06e15816ea 100644 --- a/charts/chat2db/values.yaml +++ b/charts/chat2db/values.yaml @@ -33,6 +33,29 @@ encryptionKey: secretKey: encryption.key mountPath: /run/secrets/chat2db-community-encryption.key +# Optional, controller-owned PostgreSQL datasource reconciled at application +# startup. Credentials and the trust root must come from an existing Secret. +datasourceBootstrap: + enabled: false + # Bump whenever the externally managed password or CA changes so Kubernetes + # replaces the pod and startup reconciliation reads the new secret files. + rolloutRevision: "" + managementKey: "" + alias: "" + type: POSTGRESQL + host: "" + port: 5432 + database: "" + user: "" + authenticationType: "1" + environmentId: 2 + sslMode: verify-full + existingSecret: "" + passwordKey: password + caKey: ca.crt + passwordMountPath: /run/secrets/chat2db-datasource/password + caMountPath: /run/secrets/chat2db-datasource/ca.crt + resources: requests: cpu: 500m diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/datasource/DataSource.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/datasource/DataSource.java index 270a1153ef..9859003a4a 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/datasource/DataSource.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/datasource/DataSource.java @@ -92,4 +92,10 @@ public class DataSource { private Long spaceId; + + /** Identifies the controller that owns this datasource, when any. */ + private String managedBy; + + /** Stable controller-scoped identity used for idempotent reconciliation. */ + private String managedKey; } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/datasource/ManagedDataSourceReconcileResult.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/datasource/ManagedDataSourceReconcileResult.java new file mode 100644 index 0000000000..02572153de --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/datasource/ManagedDataSourceReconcileResult.java @@ -0,0 +1,19 @@ +package ai.chat2db.community.domain.api.model.datasource; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor(staticName = "of") +public class ManagedDataSourceReconcileResult { + + public enum Action { + CREATED, + UPDATED, + UNCHANGED + } + + private Action action; + + private Long dataSourceId; +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/datasource/ManagedDataSourceSpec.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/datasource/ManagedDataSourceSpec.java new file mode 100644 index 0000000000..f3b53b5252 --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/datasource/ManagedDataSourceSpec.java @@ -0,0 +1,30 @@ +package ai.chat2db.community.domain.api.model.datasource; + +import lombok.Data; + +/** Desired configuration for one controller-managed datasource. */ +@Data +public class ManagedDataSourceSpec { + + private String managementKey; + + private String alias; + + private String type; + + private String host; + + private String port; + + private String database; + + private String user; + + private String password; + + private String authenticationType; + + private Long environmentId; + + private String url; +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/storage/WorkspaceDataSource.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/storage/WorkspaceDataSource.java index ef07df4318..aa153f7085 100644 --- a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/storage/WorkspaceDataSource.java +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/model/storage/WorkspaceDataSource.java @@ -71,4 +71,8 @@ public class WorkspaceDataSource { private String storageType; private Long spaceId; + + private String managedBy; + + private String managedKey; } diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/service/db/IManagedDataSourceReconciler.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/service/db/IManagedDataSourceReconciler.java new file mode 100644 index 0000000000..fe95e981cd --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-api/src/main/java/ai/chat2db/community/domain/api/service/db/IManagedDataSourceReconciler.java @@ -0,0 +1,9 @@ +package ai.chat2db.community.domain.api.service.db; + +import ai.chat2db.community.domain.api.model.datasource.ManagedDataSourceReconcileResult; +import ai.chat2db.community.domain.api.model.datasource.ManagedDataSourceSpec; + +public interface IManagedDataSourceReconciler { + + ManagedDataSourceReconcileResult reconcile(ManagedDataSourceSpec spec); +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/db/ManagedDataSourceReconcilerImpl.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/db/ManagedDataSourceReconcilerImpl.java new file mode 100644 index 0000000000..03ec14907d --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/main/java/ai/chat2db/community/domain/core/impl/db/ManagedDataSourceReconcilerImpl.java @@ -0,0 +1,174 @@ +package ai.chat2db.community.domain.core.impl.db; + +import ai.chat2db.community.domain.api.model.PageResponse; +import ai.chat2db.community.domain.api.model.datasource.ManagedDataSourceReconcileResult; +import ai.chat2db.community.domain.api.model.datasource.ManagedDataSourceSpec; +import ai.chat2db.community.domain.api.model.request.datasource.DbDataSourcePageQueryRequest; +import ai.chat2db.community.domain.api.model.request.datasource.DbDataSourcePreConnectRequest; +import ai.chat2db.community.domain.api.model.storage.WorkspaceDataSource; +import ai.chat2db.community.domain.api.service.db.IDbWorkspaceDataSourceService; +import ai.chat2db.community.domain.api.service.db.IManagedDataSourceReconciler; +import ai.chat2db.community.domain.api.service.storage.IWorkspaceStorageFacade; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Service; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +@Service +public class ManagedDataSourceReconcilerImpl implements IManagedDataSourceReconciler { + + public static final String MANAGED_BY = "KUBERNETES_BOOTSTRAP"; + + private final IWorkspaceStorageFacade workspaceStorageFacade; + private final IDbWorkspaceDataSourceService workspaceDataSourceService; + + public ManagedDataSourceReconcilerImpl(IWorkspaceStorageFacade workspaceStorageFacade, + IDbWorkspaceDataSourceService workspaceDataSourceService) { + this.workspaceStorageFacade = workspaceStorageFacade; + this.workspaceDataSourceService = workspaceDataSourceService; + } + + @Override + public ManagedDataSourceReconcileResult reconcile(ManagedDataSourceSpec spec) { + validate(spec); + List dataSources = listDataSources(); + List managed = dataSources.stream() + .filter(dataSource -> MANAGED_BY.equals(dataSource.getManagedBy())) + .filter(dataSource -> spec.getManagementKey().equals(dataSource.getManagedKey())) + .toList(); + if (managed.size() > 1) { + throw new IllegalStateException("Multiple datasources use managed key: " + spec.getManagementKey()); + } + if (managed.isEmpty()) { + boolean aliasExists = dataSources.stream() + .anyMatch(dataSource -> spec.getAlias().equals(dataSource.getAlias())); + if (aliasExists) { + throw new IllegalStateException("Datasource alias is already owned outside this bootstrap: " + + spec.getAlias()); + } + WorkspaceDataSource desired = desired(spec, null, true); + workspaceDataSourceService.preConnect(preConnect(desired)); + WorkspaceDataSource created = workspaceDataSourceService.createDataSource(desired); + return ManagedDataSourceReconcileResult.of( + ManagedDataSourceReconcileResult.Action.CREATED, created.getId()); + } + + Long dataSourceId = managed.get(0).getId(); + boolean aliasOwnedByAnotherDataSource = dataSources.stream() + .anyMatch(dataSource -> spec.getAlias().equals(dataSource.getAlias()) + && !Objects.equals(dataSourceId, dataSource.getId())); + if (aliasOwnedByAnotherDataSource) { + throw new IllegalStateException("Datasource alias is already owned by another datasource: " + + spec.getAlias()); + } + WorkspaceDataSource existing = workspaceDataSourceService.queryDisplayDataSourceById(dataSourceId, true); + if (existing == null) { + throw new IllegalStateException("Managed datasource disappeared during reconciliation: " + dataSourceId); + } + boolean passwordChanged = !passwordEquals(existing.getPassword(), spec.getPassword()); + WorkspaceDataSource desired = desired(spec, dataSourceId, passwordChanged); + preserveUnmanagedUpdateFields(existing, desired); + if (!declarativeFieldsChanged(existing, desired) && !passwordChanged) { + return ManagedDataSourceReconcileResult.of( + ManagedDataSourceReconcileResult.Action.UNCHANGED, dataSourceId); + } + + WorkspaceDataSource testCandidate = desired(spec, dataSourceId, true); + testCandidate.setDriverConfig(existing.getDriverConfig()); + workspaceDataSourceService.preConnect(preConnect(testCandidate)); + WorkspaceDataSource updated = workspaceDataSourceService.updateDataSource(desired); + return ManagedDataSourceReconcileResult.of( + ManagedDataSourceReconcileResult.Action.UPDATED, updated.getId()); + } + + private void preserveUnmanagedUpdateFields(WorkspaceDataSource existing, WorkspaceDataSource desired) { + // Local storage merges nullable fields, but these booleans are primitive and + // the workspace service supplies a default driver when it is absent. + desired.setSupportDatabase(existing.isSupportDatabase()); + desired.setSupportSchema(existing.isSupportSchema()); + desired.setDriverConfig(existing.getDriverConfig()); + } + + private List listDataSources() { + DbDataSourcePageQueryRequest request = new DbDataSourcePageQueryRequest(); + request.setPageNo(1); + request.setPageSize(10_000); + PageResponse page = workspaceStorageFacade.listDataSources(request); + return page == null || page.getData() == null ? Collections.emptyList() : page.getData(); + } + + private WorkspaceDataSource desired(ManagedDataSourceSpec spec, Long id, boolean includePassword) { + WorkspaceDataSource desired = new WorkspaceDataSource(); + desired.setId(id); + desired.setAlias(spec.getAlias()); + desired.setType(spec.getType()); + desired.setHost(spec.getHost()); + desired.setPort(spec.getPort()); + desired.setServiceName(spec.getDatabase()); + desired.setUrl(spec.getUrl()); + desired.setUser(spec.getUser()); + desired.setPassword(includePassword ? spec.getPassword() : null); + desired.setAuthenticationType(spec.getAuthenticationType()); + desired.setEnvironmentId(spec.getEnvironmentId()); + desired.setManagedBy(MANAGED_BY); + desired.setManagedKey(spec.getManagementKey()); + return desired; + } + + private DbDataSourcePreConnectRequest preConnect(WorkspaceDataSource dataSource) { + DbDataSourcePreConnectRequest request = new DbDataSourcePreConnectRequest(); + request.setId(null); + request.setAlias(dataSource.getAlias()); + request.setType(dataSource.getType()); + request.setUrl(dataSource.getUrl()); + request.setUser(dataSource.getUser()); + request.setPassword(dataSource.getPassword()); + request.setAuthenticationType(dataSource.getAuthenticationType()); + request.setHost(dataSource.getHost()); + request.setPort(dataSource.getPort()); + return request; + } + + private boolean declarativeFieldsChanged(WorkspaceDataSource existing, WorkspaceDataSource desired) { + return !Objects.equals(existing.getAlias(), desired.getAlias()) + || !Objects.equals(existing.getType(), desired.getType()) + || !Objects.equals(existing.getHost(), desired.getHost()) + || !Objects.equals(existing.getPort(), desired.getPort()) + || !Objects.equals(existing.getServiceName(), desired.getServiceName()) + || !Objects.equals(existing.getUrl(), desired.getUrl()) + || !Objects.equals(existing.getUser(), desired.getUser()) + || !Objects.equals(existing.getAuthenticationType(), desired.getAuthenticationType()) + || !Objects.equals(existing.getEnvironmentId(), desired.getEnvironmentId()) + || !Objects.equals(existing.getManagedBy(), desired.getManagedBy()) + || !Objects.equals(existing.getManagedKey(), desired.getManagedKey()); + } + + private boolean passwordEquals(String existing, String desired) { + byte[] existingBytes = existing == null ? new byte[0] : existing.getBytes(StandardCharsets.UTF_8); + byte[] desiredBytes = desired == null ? new byte[0] : desired.getBytes(StandardCharsets.UTF_8); + try { + return MessageDigest.isEqual(existingBytes, desiredBytes); + } finally { + Arrays.fill(existingBytes, (byte) 0); + Arrays.fill(desiredBytes, (byte) 0); + } + } + + private void validate(ManagedDataSourceSpec spec) { + if (spec == null + || StringUtils.isAnyBlank(spec.getManagementKey(), spec.getAlias(), spec.getType(), spec.getHost(), + spec.getPort(), spec.getDatabase(), spec.getUser(), spec.getPassword(), spec.getAuthenticationType(), + spec.getUrl()) + || spec.getEnvironmentId() == null) { + throw new IllegalArgumentException("Managed datasource configuration is incomplete"); + } + if (!"POSTGRESQL".equalsIgnoreCase(spec.getType())) { + throw new IllegalArgumentException("Managed datasource bootstrap currently supports POSTGRESQL only"); + } + } +} diff --git a/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/db/ManagedDataSourceReconcilerImplTest.java b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/db/ManagedDataSourceReconcilerImplTest.java new file mode 100644 index 0000000000..0b03b9a0b5 --- /dev/null +++ b/chat2db-community-server/chat2db-community-domain/chat2db-community-domain-core/src/test/java/ai/chat2db/community/domain/core/impl/db/ManagedDataSourceReconcilerImplTest.java @@ -0,0 +1,226 @@ +package ai.chat2db.community.domain.core.impl.db; + +import ai.chat2db.community.domain.api.model.PageResponse; +import ai.chat2db.community.domain.api.config.DriverConfig; +import ai.chat2db.community.domain.api.model.datasource.ManagedDataSourceReconcileResult; +import ai.chat2db.community.domain.api.model.datasource.ManagedDataSourceSpec; +import ai.chat2db.community.domain.api.model.storage.WorkspaceDataSource; +import ai.chat2db.community.domain.api.service.db.IDbWorkspaceDataSourceService; +import ai.chat2db.community.domain.api.service.storage.IWorkspaceStorageFacade; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Proxy; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class ManagedDataSourceReconcilerImplTest { + + private final List stored = new ArrayList<>(); + private final List calls = new ArrayList<>(); + private WorkspaceDataSource updateRequest; + private boolean failPreConnect; + private ManagedDataSourceReconcilerImpl reconciler; + + @BeforeEach + void setUp() { + stored.clear(); + calls.clear(); + updateRequest = null; + failPreConnect = false; + reconciler = new ManagedDataSourceReconcilerImpl(storageFacade(), workspaceService()); + } + + @Test + void createsManagedDatasourceAfterConnectionTest() { + ManagedDataSourceReconcileResult result = reconciler.reconcile(spec()); + + assertEquals(ManagedDataSourceReconcileResult.Action.CREATED, result.getAction()); + assertEquals(101L, result.getDataSourceId()); + assertEquals(List.of("preConnect", "create"), calls); + assertEquals(ManagedDataSourceReconcilerImpl.MANAGED_BY, stored.get(0).getManagedBy()); + assertEquals("prod-readonly", stored.get(0).getManagedKey()); + } + + @Test + void leavesIdenticalManagedDatasourceUnchanged() { + stored.add(existing("secret")); + + ManagedDataSourceReconcileResult result = reconciler.reconcile(spec()); + + assertEquals(ManagedDataSourceReconcileResult.Action.UNCHANGED, result.getAction()); + assertEquals(List.of("query:7"), calls); + } + + @Test + void testsThenUpdatesChangedPasswordWithoutPopulatingUnmanagedFields() { + WorkspaceDataSource existing = existing("old-secret"); + existing.setKind("user-owned-kind"); + existing.setSupportDatabase(true); + existing.setSupportSchema(true); + DriverConfig customDriver = new DriverConfig(); + customDriver.setJdbcDriverClass("custom.Driver"); + existing.setDriverConfig(customDriver); + stored.add(existing); + + ManagedDataSourceReconcileResult result = reconciler.reconcile(spec()); + + assertEquals(ManagedDataSourceReconcileResult.Action.UPDATED, result.getAction()); + assertEquals(List.of("query:7", "preConnect", "update:7"), calls); + assertEquals("secret", updateRequest.getPassword()); + assertNull(updateRequest.getKind()); + assertEquals(true, updateRequest.isSupportDatabase()); + assertEquals(true, updateRequest.isSupportSchema()); + assertEquals("custom.Driver", updateRequest.getDriverConfig().getJdbcDriverClass()); + } + + @Test + void doesNotPersistWhenConnectionTestFails() { + failPreConnect = true; + + assertThrows(IllegalStateException.class, () -> reconciler.reconcile(spec())); + + assertEquals(List.of("preConnect"), calls); + assertEquals(0, stored.size()); + } + + @Test + void refusesToAdoptUserOwnedAlias() { + WorkspaceDataSource userOwned = existing("secret"); + userOwned.setManagedBy(null); + userOwned.setManagedKey(null); + stored.add(userOwned); + + assertThrows(IllegalStateException.class, () -> reconciler.reconcile(spec())); + + assertEquals(List.of(), calls); + assertNull(stored.get(0).getManagedBy()); + } + + @Test + void refusesDuplicateManagedKeys() { + stored.add(existing("secret")); + WorkspaceDataSource duplicate = existing("secret"); + duplicate.setId(8L); + duplicate.setAlias("other alias"); + stored.add(duplicate); + + assertThrows(IllegalStateException.class, () -> reconciler.reconcile(spec())); + + assertEquals(List.of(), calls); + } + + @Test + void refusesToRenameManagedDatasourceOntoAnotherAlias() { + WorkspaceDataSource managed = existing("secret"); + managed.setAlias("Previous managed alias"); + stored.add(managed); + WorkspaceDataSource userOwned = existing("unrelated-secret"); + userOwned.setId(8L); + userOwned.setManagedBy(null); + userOwned.setManagedKey(null); + stored.add(userOwned); + + assertThrows(IllegalStateException.class, () -> reconciler.reconcile(spec())); + + assertEquals(List.of(), calls); + assertEquals("Previous managed alias", stored.get(0).getAlias()); + } + + private IWorkspaceStorageFacade storageFacade() { + return (IWorkspaceStorageFacade) Proxy.newProxyInstance( + IWorkspaceStorageFacade.class.getClassLoader(), new Class[]{IWorkspaceStorageFacade.class}, + (proxy, method, args) -> { + if ("listDataSources".equals(method.getName())) { + return PageResponse.of(new ArrayList<>(stored), (long) stored.size(), 1, 10_000); + } + return defaultValue(method.getReturnType()); + }); + } + + private IDbWorkspaceDataSourceService workspaceService() { + return (IDbWorkspaceDataSourceService) Proxy.newProxyInstance( + IDbWorkspaceDataSourceService.class.getClassLoader(), + new Class[]{IDbWorkspaceDataSourceService.class}, (proxy, method, args) -> switch (method.getName()) { + case "queryDisplayDataSourceById" -> { + calls.add("query:" + args[0]); + yield stored.stream().filter(item -> item.getId().equals(args[0])).findFirst().orElse(null); + } + case "preConnect" -> { + calls.add("preConnect"); + if (failPreConnect) { + throw new IllegalStateException("connection failed"); + } + yield null; + } + case "createDataSource" -> { + calls.add("create"); + WorkspaceDataSource created = (WorkspaceDataSource) args[0]; + created.setId(101L); + stored.add(created); + yield created; + } + case "updateDataSource" -> { + updateRequest = (WorkspaceDataSource) args[0]; + calls.add("update:" + updateRequest.getId()); + yield updateRequest; + } + default -> defaultValue(method.getReturnType()); + }); + } + + private ManagedDataSourceSpec spec() { + ManagedDataSourceSpec spec = new ManagedDataSourceSpec(); + spec.setManagementKey("prod-readonly"); + spec.setAlias("Production (read only)"); + spec.setType("POSTGRESQL"); + spec.setHost("postgres.example.internal"); + spec.setPort("5432"); + spec.setDatabase("app"); + spec.setUser("chat2db_reader"); + spec.setPassword("secret"); + spec.setAuthenticationType("1"); + spec.setEnvironmentId(2L); + spec.setUrl("jdbc:postgresql://postgres.example.internal:5432/app?sslmode=verify-full"); + return spec; + } + + private WorkspaceDataSource existing(String password) { + ManagedDataSourceSpec spec = spec(); + WorkspaceDataSource dataSource = new WorkspaceDataSource(); + dataSource.setId(7L); + dataSource.setAlias(spec.getAlias()); + dataSource.setType(spec.getType()); + dataSource.setHost(spec.getHost()); + dataSource.setPort(spec.getPort()); + dataSource.setServiceName(spec.getDatabase()); + dataSource.setUrl(spec.getUrl()); + dataSource.setUser(spec.getUser()); + dataSource.setPassword(password); + dataSource.setAuthenticationType(spec.getAuthenticationType()); + dataSource.setEnvironmentId(spec.getEnvironmentId()); + dataSource.setManagedBy(ManagedDataSourceReconcilerImpl.MANAGED_BY); + dataSource.setManagedKey(spec.getManagementKey()); + return dataSource; + } + + private static Object defaultValue(Class type) { + if (!type.isPrimitive()) { + return null; + } + if (type == boolean.class) { + return false; + } + if (type == int.class) { + return 0; + } + if (type == long.class) { + return 0L; + } + return null; + } +} diff --git a/chat2db-community-server/chat2db-community-start/src/main/java/ai/chat2db/community/start/bootstrap/ManagedDataSourceBootstrapProperties.java b/chat2db-community-server/chat2db-community-start/src/main/java/ai/chat2db/community/start/bootstrap/ManagedDataSourceBootstrapProperties.java new file mode 100644 index 0000000000..77bc88400d --- /dev/null +++ b/chat2db-community-server/chat2db-community-start/src/main/java/ai/chat2db/community/start/bootstrap/ManagedDataSourceBootstrapProperties.java @@ -0,0 +1,37 @@ +package ai.chat2db.community.start.bootstrap; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Data +@Component +@ConfigurationProperties(prefix = "chat2db.bootstrap.datasource") +public class ManagedDataSourceBootstrapProperties { + + private boolean enabled; + + private String managementKey; + + private String alias; + + private String type = "POSTGRESQL"; + + private String host; + + private int port = 5432; + + private String database; + + private String user; + + private String authenticationType = "1"; + + private Long environmentId = 2L; + + private String sslMode = "verify-full"; + + private String passwordFile; + + private String caFile; +} diff --git a/chat2db-community-server/chat2db-community-start/src/main/java/ai/chat2db/community/start/bootstrap/ManagedDataSourceBootstrapRunner.java b/chat2db-community-server/chat2db-community-start/src/main/java/ai/chat2db/community/start/bootstrap/ManagedDataSourceBootstrapRunner.java new file mode 100644 index 0000000000..6b129a8cf3 --- /dev/null +++ b/chat2db-community-server/chat2db-community-start/src/main/java/ai/chat2db/community/start/bootstrap/ManagedDataSourceBootstrapRunner.java @@ -0,0 +1,148 @@ +package ai.chat2db.community.start.bootstrap; + +import ai.chat2db.community.domain.api.model.datasource.ManagedDataSourceReconcileResult; +import ai.chat2db.community.domain.api.model.datasource.ManagedDataSourceSpec; +import ai.chat2db.community.domain.api.service.db.IManagedDataSourceReconciler; +import ai.chat2db.community.tools.annotation.CommunityRuntimeOnly; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; + +@Slf4j +@Component +@CommunityRuntimeOnly +@ConditionalOnProperty(name = "chat2db.bootstrap.datasource.enabled", havingValue = "true") +public class ManagedDataSourceBootstrapRunner implements ApplicationRunner { + + private static final long MAX_PASSWORD_BYTES = 64 * 1024; + private static final long MAX_CA_BYTES = 1024 * 1024; + + private final ManagedDataSourceBootstrapProperties properties; + private final IManagedDataSourceReconciler reconciler; + + public ManagedDataSourceBootstrapRunner(ManagedDataSourceBootstrapProperties properties, + IManagedDataSourceReconciler reconciler) { + this.properties = properties; + this.reconciler = reconciler; + } + + @Override + public void run(ApplicationArguments args) { + try { + ManagedDataSourceSpec spec = buildSpec(); + ManagedDataSourceReconcileResult result = reconciler.reconcile(spec); + log.info("Managed datasource reconciliation action={} id={} key={}", + result.getAction(), result.getDataSourceId(), properties.getManagementKey()); + } catch (RuntimeException exception) { + log.error("Managed datasource reconciliation failed for key={}", safeManagementKey()); + throw new IllegalStateException("Managed datasource bootstrap failed"); + } + } + + ManagedDataSourceSpec buildSpec() { + validateProperties(); + Path passwordPath = validatedPath(properties.getPasswordFile(), MAX_PASSWORD_BYTES, "password"); + Path caPath = validatedPath(properties.getCaFile(), MAX_CA_BYTES, "CA certificate"); + String password = readPassword(passwordPath); + requireNonBlankFile(caPath, "CA certificate"); + + ManagedDataSourceSpec spec = new ManagedDataSourceSpec(); + spec.setManagementKey(properties.getManagementKey()); + spec.setAlias(properties.getAlias()); + spec.setType("POSTGRESQL"); + spec.setHost(properties.getHost()); + spec.setPort(Integer.toString(properties.getPort())); + spec.setDatabase(properties.getDatabase()); + spec.setUser(properties.getUser()); + spec.setPassword(password); + spec.setAuthenticationType(properties.getAuthenticationType()); + spec.setEnvironmentId(properties.getEnvironmentId()); + spec.setUrl(buildJdbcUrl(caPath)); + return spec; + } + + private String buildJdbcUrl(Path caPath) { + return "jdbc:postgresql://" + properties.getHost() + ":" + properties.getPort() + "/" + + properties.getDatabase() + "?sslmode=verify-full&sslrootcert=" + caPath + + "&ApplicationName=Chat2DB"; + } + + private void validateProperties() { + if (StringUtils.isAnyBlank(properties.getManagementKey(), properties.getAlias(), properties.getType(), + properties.getHost(), properties.getDatabase(), properties.getUser(), + properties.getAuthenticationType(), properties.getPasswordFile(), properties.getCaFile())) { + throw new IllegalArgumentException("Managed datasource bootstrap configuration is incomplete"); + } + if (!"POSTGRESQL".equalsIgnoreCase(properties.getType())) { + throw new IllegalArgumentException("Managed datasource bootstrap currently supports POSTGRESQL only"); + } + if (!"verify-full".equals(properties.getSslMode())) { + throw new IllegalArgumentException("Managed PostgreSQL datasource requires sslMode=verify-full"); + } + if (!properties.getHost().matches("[A-Za-z0-9.-]+") + || !properties.getDatabase().matches("[A-Za-z0-9_-]+")) { + throw new IllegalArgumentException("Managed datasource host or database contains unsupported characters"); + } + if (properties.getPort() < 1 || properties.getPort() > 65535 || properties.getEnvironmentId() == null) { + throw new IllegalArgumentException("Managed datasource port or environmentId is invalid"); + } + } + + private Path validatedPath(String configuredPath, long maxBytes, String description) { + Path path = Path.of(configuredPath).toAbsolutePath().normalize(); + if (!Path.of(configuredPath).isAbsolute() + || Files.isSymbolicLink(path) + || !Files.isRegularFile(path, LinkOption.NOFOLLOW_LINKS)) { + throw new IllegalArgumentException("Managed datasource " + description + " file is invalid"); + } + try { + long size = Files.size(path); + if (size < 1 || size > maxBytes) { + throw new IllegalArgumentException("Managed datasource " + description + " file has invalid size"); + } + } catch (IOException exception) { + throw new IllegalArgumentException("Unable to inspect managed datasource " + description + " file"); + } + return path; + } + + private String readPassword(Path path) { + try { + String value = Files.readString(path, StandardCharsets.UTF_8); + if (value.endsWith("\r\n")) { + value = value.substring(0, value.length() - 2); + } else if (value.endsWith("\n")) { + value = value.substring(0, value.length() - 1); + } + if (value.isBlank()) { + throw new IllegalArgumentException("Managed datasource password file is blank"); + } + return value; + } catch (IOException exception) { + throw new IllegalArgumentException("Unable to read managed datasource password file"); + } + } + + private void requireNonBlankFile(Path path, String description) { + try { + if (Files.readString(path, StandardCharsets.UTF_8).isBlank()) { + throw new IllegalArgumentException("Managed datasource " + description + " file is blank"); + } + } catch (IOException exception) { + throw new IllegalArgumentException("Unable to read managed datasource " + description + " file"); + } + } + + private String safeManagementKey() { + return StringUtils.defaultIfBlank(properties.getManagementKey(), ""); + } +} diff --git a/chat2db-community-server/chat2db-community-start/src/test/java/ai/chat2db/community/start/bootstrap/ManagedDataSourceBootstrapRunnerTest.java b/chat2db-community-server/chat2db-community-start/src/test/java/ai/chat2db/community/start/bootstrap/ManagedDataSourceBootstrapRunnerTest.java new file mode 100644 index 0000000000..60fc7be197 --- /dev/null +++ b/chat2db-community-server/chat2db-community-start/src/test/java/ai/chat2db/community/start/bootstrap/ManagedDataSourceBootstrapRunnerTest.java @@ -0,0 +1,116 @@ +package ai.chat2db.community.start.bootstrap; + +import ai.chat2db.community.domain.api.model.datasource.ManagedDataSourceReconcileResult; +import ai.chat2db.community.domain.api.model.datasource.ManagedDataSourceSpec; +import ai.chat2db.community.domain.api.service.db.IManagedDataSourceReconciler; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.boot.DefaultApplicationArguments; + +import java.io.IOException; +import java.lang.reflect.Proxy; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class ManagedDataSourceBootstrapRunnerTest { + + @TempDir + Path tempDir; + + @Test + void buildsVerifyFullPostgresqlSpecFromMountedFiles() throws IOException { + Path password = write("password", "db-password\n"); + Path ca = write("ca.crt", "-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----\n"); + ManagedDataSourceBootstrapRunner runner = new ManagedDataSourceBootstrapRunner( + properties(password, ca), noOpReconciler()); + + ManagedDataSourceSpec spec = runner.buildSpec(); + + assertEquals("db-password", spec.getPassword()); + assertEquals("POSTGRESQL", spec.getType()); + assertEquals("jdbc:postgresql://postgres.example.internal:5432/app?sslmode=verify-full&sslrootcert=" + + ca + "&ApplicationName=Chat2DB", spec.getUrl()); + } + + @Test + void rejectsBlankPassword() throws IOException { + Path password = write("password", "\n"); + Path ca = write("ca.crt", "certificate"); + ManagedDataSourceBootstrapRunner runner = new ManagedDataSourceBootstrapRunner( + properties(password, ca), noOpReconciler()); + + assertThrows(IllegalArgumentException.class, runner::buildSpec); + } + + @Test + void removesOnlyTheSecretFileLineEndingFromPassword() throws IOException { + Path password = write("password", " leading-and-trailing-space \r\n"); + Path ca = write("ca.crt", "certificate"); + ManagedDataSourceBootstrapRunner runner = new ManagedDataSourceBootstrapRunner( + properties(password, ca), noOpReconciler()); + + ManagedDataSourceSpec spec = runner.buildSpec(); + + assertEquals(" leading-and-trailing-space ", spec.getPassword()); + } + + @Test + void rejectsSymlinkedSecretFile() throws IOException { + Path target = write("password-target", "db-password"); + Path password = tempDir.resolve("password"); + Files.createSymbolicLink(password, target); + Path ca = write("ca.crt", "certificate"); + ManagedDataSourceBootstrapRunner runner = new ManagedDataSourceBootstrapRunner( + properties(password, ca), noOpReconciler()); + + assertThrows(IllegalArgumentException.class, runner::buildSpec); + } + + @Test + void publicFailureDoesNotContainPassword() throws IOException { + Path password = write("password", "highly-sensitive-password"); + Path ca = write("ca.crt", "certificate"); + IManagedDataSourceReconciler failing = spec -> { + throw new IllegalStateException("driver echoed " + spec.getPassword()); + }; + ManagedDataSourceBootstrapRunner runner = new ManagedDataSourceBootstrapRunner( + properties(password, ca), failing); + + IllegalStateException exception = assertThrows(IllegalStateException.class, + () -> runner.run(new DefaultApplicationArguments())); + + assertFalse(exception.getMessage().contains("highly-sensitive-password")); + assertEquals(null, exception.getCause()); + } + + private ManagedDataSourceBootstrapProperties properties(Path password, Path ca) { + ManagedDataSourceBootstrapProperties properties = new ManagedDataSourceBootstrapProperties(); + properties.setEnabled(true); + properties.setManagementKey("prod-readonly"); + properties.setAlias("Production (read only)"); + properties.setHost("postgres.example.internal"); + properties.setDatabase("app"); + properties.setUser("chat2db_reader"); + properties.setPasswordFile(password.toString()); + properties.setCaFile(ca.toString()); + return properties; + } + + private IManagedDataSourceReconciler noOpReconciler() { + return (IManagedDataSourceReconciler) Proxy.newProxyInstance( + IManagedDataSourceReconciler.class.getClassLoader(), + new Class[]{IManagedDataSourceReconciler.class}, + (proxy, method, args) -> ManagedDataSourceReconcileResult.of( + ManagedDataSourceReconcileResult.Action.UNCHANGED, 7L)); + } + + private Path write(String name, String value) throws IOException { + Path path = tempDir.resolve(name); + Files.writeString(path, value); + return path; + } +}