diff --git a/helm/gandalf/ncats-images-meta.yaml b/helm/gandalf/ncats-images-meta.yaml index dcac7f458..2c7836832 100644 --- a/helm/gandalf/ncats-images-meta.yaml +++ b/helm/gandalf/ncats-images-meta.yaml @@ -1,3 +1,9 @@ +plater: + image: "ghcr.io/translatorsri/plater-clustered" + version: "v2.1.4" +neo4j: + image: "ghcr.io/translatorsri/plater-neo4j" + version: "neo4j-5.26" busybox: image: "busybox" version: "1.35" diff --git a/helm/gandalf/templates/configmap.yaml b/helm/gandalf/templates/configmap.yaml index 984353c52..c2b33ac49 100644 --- a/helm/gandalf/templates/configmap.yaml +++ b/helm/gandalf/templates/configmap.yaml @@ -8,10 +8,10 @@ data: GRAPH_DB: "{{ .Values.graph.backend }}" GRAPH_QUERY_TIMEOUT: "{{ .Values.app.GRAPH_QUERY_TIMEOUT }}" - # turn off automat logic - # GANDALF_SERVICE_ADDRESS: "{{ include "gandalf.fullname" . }}-service" - # GANDALF_AUTOMAT_HOST: {{ .Values.app.automatAddress }} - # GANDALF_PLATER_TITLE: {{ .Release.Name }} + # Enable automat registration + GANDALF_SERVICE_ADDRESS: "{{ include "gandalf.fullname" . }}-service" + GANDALF_AUTOMAT_HOST: {{ .Values.app.automatAddress }} + GANDALF_PLATER_TITLE: {{ .Release.Name }} GANDALF_INFORES: "{{ .Values.datasetDesc.provenanceTag }}" BL_VERSION: "{{ .Values.app.bl_version }}" GANDALF_SERVER_MATURITY: "{{ .Values.x_trapi.maturity.value }}" diff --git a/helm/gandalf/templates/gandalf-loader.yaml b/helm/gandalf/templates/gandalf-loader.yaml index bf3d17964..00c6d766a 100644 --- a/helm/gandalf/templates/gandalf-loader.yaml +++ b/helm/gandalf/templates/gandalf-loader.yaml @@ -97,6 +97,21 @@ spec: nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.app.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- else }} + {{- if and (eq .Values.graph.backend "gandalf") (not .Values.graph.external) }} + # Force loader to run on same node as the PVC (RWO volume constraint) + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + {{- include "gandalf.selectorLabels" . | nindent 18 }} + topologyKey: kubernetes.io/hostname + {{- end }} + {{- end }} {{- with .Values.app.tolerations }} tolerations: {{- toYaml . | nindent 8 }} diff --git a/helm/gandalf/templates/ingress.yaml b/helm/gandalf/templates/ingress.yaml index 552eb405b..ed4101bc8 100644 --- a/helm/gandalf/templates/ingress.yaml +++ b/helm/gandalf/templates/ingress.yaml @@ -1,3 +1,4 @@ +{{- if .Values.ingress.enabled -}} {{- if and .Values.ingress.class (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} {{- if not (hasKey .Values.ingress.gandalf.annotations "kubernetes.io/ingress.class") }} {{- $_ := set .Values.ingress.gandalf.annotations "kubernetes.io/ingress.class" .Values.ingress.class }} @@ -48,3 +49,4 @@ spec: serviceName: {{ include "gandalf.fullname" . }}-service servicePort: {{ .Values.app.gandalf.service.port }} {{- end }} +{{- end -}} diff --git a/helm/gandalf/values.yaml b/helm/gandalf/values.yaml index 90018cd63..1cffddeee 100644 --- a/helm/gandalf/values.yaml +++ b/helm/gandalf/values.yaml @@ -24,7 +24,7 @@ image: tag: 0.3.3 pullPolicy: Always -nameOverride: "" +nameOverride: "plater" fullnameOverride: "" app: @@ -81,7 +81,7 @@ app: ephemeral-storage: "6Gi" persistentVolume: accessModes: - - ReadWriteMany + - ReadWriteOnce size: "5Gi" openapi_config: diff --git a/helm/gandalf/values/cam-kp/values-populated.yaml b/helm/gandalf/values/cam-kp/values-populated.yaml index 079341a5a..1b21f0fbb 100644 Binary files a/helm/gandalf/values/cam-kp/values-populated.yaml and b/helm/gandalf/values/cam-kp/values-populated.yaml differ diff --git a/helm/gandalf/values/hgnc/values-populated.yaml b/helm/gandalf/values/hgnc/values-populated.yaml index 9464a9d2d..ed77b1c5b 100644 Binary files a/helm/gandalf/values/hgnc/values-populated.yaml and b/helm/gandalf/values/hgnc/values-populated.yaml differ diff --git a/helm/gandalf/values/icees-kg/values-populated.yaml b/helm/gandalf/values/icees-kg/values-populated.yaml index 4a15a6934..b9a7c52c7 100644 Binary files a/helm/gandalf/values/icees-kg/values-populated.yaml and b/helm/gandalf/values/icees-kg/values-populated.yaml differ diff --git a/helm/gandalf/values/monarch-kg/values-populated.yaml b/helm/gandalf/values/monarch-kg/values-populated.yaml index c2507b93f..451773665 100644 Binary files a/helm/gandalf/values/monarch-kg/values-populated.yaml and b/helm/gandalf/values/monarch-kg/values-populated.yaml differ diff --git a/helm/gandalf/values/robokopkg/ncats-dev-values.yaml b/helm/gandalf/values/robokopkg/ncats-dev-values.yaml index 910a7f589..77e529d3c 100644 Binary files a/helm/gandalf/values/robokopkg/ncats-dev-values.yaml and b/helm/gandalf/values/robokopkg/ncats-dev-values.yaml differ diff --git a/helm/gandalf/values/robokopkg/ncats-prod-values.yaml b/helm/gandalf/values/robokopkg/ncats-prod-values.yaml index be6d96670..5a8529b20 100644 Binary files a/helm/gandalf/values/robokopkg/ncats-prod-values.yaml and b/helm/gandalf/values/robokopkg/ncats-prod-values.yaml differ diff --git a/helm/gandalf/values/robokopkg/ncats-test-values.yaml b/helm/gandalf/values/robokopkg/ncats-test-values.yaml index 8e0fcd452..70969fa5a 100644 Binary files a/helm/gandalf/values/robokopkg/ncats-test-values.yaml and b/helm/gandalf/values/robokopkg/ncats-test-values.yaml differ diff --git a/helm/gandalf/values/robokopkg/values-populated.yaml b/helm/gandalf/values/robokopkg/values-populated.yaml index 15a7a8dcd..03de3cb32 100644 Binary files a/helm/gandalf/values/robokopkg/values-populated.yaml and b/helm/gandalf/values/robokopkg/values-populated.yaml differ diff --git a/helm/jaeger/ncats-dev-values.yaml b/helm/jaeger/ncats-dev-values.yaml index 5bcca1662..c0818e40a 100644 Binary files a/helm/jaeger/ncats-dev-values.yaml and b/helm/jaeger/ncats-dev-values.yaml differ diff --git a/helm/jaeger/ncats-prod-values.yaml b/helm/jaeger/ncats-prod-values.yaml index 502925360..d2ee5a03b 100644 Binary files a/helm/jaeger/ncats-prod-values.yaml and b/helm/jaeger/ncats-prod-values.yaml differ diff --git a/helm/jaeger/ncats-test-values.yaml b/helm/jaeger/ncats-test-values.yaml index c919dc38d..173c7677f 100644 Binary files a/helm/jaeger/ncats-test-values.yaml and b/helm/jaeger/ncats-test-values.yaml differ diff --git a/jenkins/gandalf/Jenkinsfile b/jenkins/gandalf/Jenkinsfile new file mode 100644 index 000000000..fef642bb5 --- /dev/null +++ b/jenkins/gandalf/Jenkinsfile @@ -0,0 +1,171 @@ +pipeline { + options { + timestamps() + skipDefaultCheckout() + disableConcurrentBuilds() + } + agent { + node { label 'translator && aws && build' } + } + parameters { + string(name: 'BUILD_VERSION', defaultValue: '', description: 'The build version to deploy (optional)') + choice(name: 'PLATERS', choices: 'all\nhgnc\ncam-kp\nicees-kg\nmonarch-kg\nrobokopkg\n', description: 'Platers to deploy') + choice(name: 'ACTION', choices: 'install\ndelete', description: 'Install or delete, default install') + string(name: 'AWS_REGION', defaultValue: 'us-east-1', description: 'AWS Region to deploy') + string(name: 'KUBERNETES_CLUSTER_NAME', defaultValue: 'translator-eks-ci-blue-cluster', description: 'AWS EKS that will host this application') + } + triggers { + pollSCM('H/2 * * * *') + } + environment { + DEPLOY_ENV = "dev" + } + stages { + stage('Clean') { + steps { + cleanWs() + checkout scm + } + } + stage('Build Version') { + when { + allOf { + expression { + return !params.BUILD_VERSION + } + anyOf { + changeset "helm/gandalf/**" + triggeredBy 'UserIdCause' + } + } + } + steps{ + script { + def imageMeta = readYaml file: 'helm/gandalf/ncats-images-meta.yaml' + env.ORIGINAL_REPO = imageMeta.plater.image + env.BUILD_VERSION = imageMeta.plater.version + env.ORIGINAL_REPO_NEO4J = imageMeta.neo4j.image + env.BUILD_VERSION_NEO4J = imageMeta.neo4j.version + env.ORIGINAL_REPO_BUSYBOX = imageMeta.busybox.image + env.BUILD_VERSION_BUSYBOX = imageMeta.busybox.version + currentBuild.displayName = imageMeta.plater.version + "#" + imageMeta.neo4j.version + } + } + } + stage('build image') { + when { + allOf { + expression { + return !params.BUILD_VERSION + } + anyOf { + changeset "helm/gandalf/**" + triggeredBy 'UserIdCause' + } + } + } + steps { + withEnv([ + "IMAGE_NAME=translator-sri-plater-clustered", + "BUILD_VERSION=" + (params.BUILD_VERSION ? params.BUILD_VERSION.tokenize("#")[0] : env.BUILD_VERSION) + ]) { + dir("helm/gandalf") { + script { + + sh ''' + docker pull $ORIGINAL_REPO:$BUILD_VERSION + docker tag $ORIGINAL_REPO:$BUILD_VERSION 853771734544.dkr.ecr.us-east-1.amazonaws.com/$IMAGE_NAME:$BUILD_VERSION + ''' + def image = docker.image("853771734544.dkr.ecr.us-east-1.amazonaws.com/$IMAGE_NAME:$BUILD_VERSION") + sh ''' + aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 853771734544.dkr.ecr.us-east-1.amazonaws.com + ''' + image.push() + } + } + } + withEnv([ + "IMAGE_NAME=translator-sri-neo4j-42-apoc-gds", + "BUILD_VERSION=" + (params.BUILD_VERSION ? params.BUILD_VERSION.tokenize("#")[1] : env.BUILD_VERSION_NEO4J) + ]) { + dir("helm/gandalf") { + script { + + sh ''' + docker pull $ORIGINAL_REPO_NEO4J:$BUILD_VERSION_NEO4J + docker tag $ORIGINAL_REPO_NEO4J:$BUILD_VERSION_NEO4J 853771734544.dkr.ecr.us-east-1.amazonaws.com/$IMAGE_NAME:$BUILD_VERSION + ''' + def image = docker.image("853771734544.dkr.ecr.us-east-1.amazonaws.com/$IMAGE_NAME:$BUILD_VERSION") + sh ''' + aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 853771734544.dkr.ecr.us-east-1.amazonaws.com + ''' + image.push() + } + } + } + withEnv([ + "IMAGE_NAME=translator-busybox" + ]) { + dir("helm/gandalf") { + script { + + sh ''' + docker pull $ORIGINAL_REPO_BUSYBOX:stable + docker pull $ORIGINAL_REPO_BUSYBOX:latest + docker tag $ORIGINAL_REPO_BUSYBOX:stable 853771734544.dkr.ecr.us-east-1.amazonaws.com/$IMAGE_NAME:stable + docker tag $ORIGINAL_REPO_BUSYBOX:latest 853771734544.dkr.ecr.us-east-1.amazonaws.com/$IMAGE_NAME:latest + ''' + def image = docker.image("853771734544.dkr.ecr.us-east-1.amazonaws.com/$IMAGE_NAME:stable") + def imageLatest = docker.image("853771734544.dkr.ecr.us-east-1.amazonaws.com/$IMAGE_NAME:latest") + + sh ''' + aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 853771734544.dkr.ecr.us-east-1.amazonaws.com + ''' + image.push() + imageLatest.push() + } + } + } + } + } + stage('Deploy') { + agent { + label 'translator && ci && deploy' + } + when { + anyOf { + changeset "helm/gandalf/**" + triggeredBy 'UserIdCause' + } + } + steps { + checkout scm + dir('helm/gandalf') { + configFileProvider([ + configFile(fileId: 'gandalf-plater.sh', targetLocation: 'prepare.sh'), + configFile(fileId: 'values-plater.yaml', targetLocation: 'values-plater.yaml'), + configFile(fileId: 'gandalf-deploy.sh', targetLocation: 'deploy.sh') + ]){ + withEnv([ + "BUILD_VERSION_PLATER=" + (params.BUILD_VERSION ? params.BUILD_VERSION.tokenize("#")[0] : env.BUILD_VERSION), + "BUILD_VERSION_NEO4J=" + (params.BUILD_VERSION ? params.BUILD_VERSION.tokenize("#")[1] : env.BUILD_VERSION_NEO4J), + "PLATERS=" + params.PLATERS, + "ACTION=" + params.ACTION + ]) { + sh ''' + aws --region ${AWS_REGION} eks update-kubeconfig --name ${KUBERNETES_CLUSTER_NAME} + /bin/bash prepare.sh + ''' + } + } + } + } + post { + always { + echo " Clean up the workspace in deploy node!" + cleanWs() + } + } + } + } +}