Extract postgresql chart (#2568)

* Extract postgresql chart * Apply migrations with DB dump * Apply migrations with DB dump remove commented out code * Introduce separate chart for Local DB * Add test namespace * Create DB Dump apply job that do not use schema migrator * Remove DB Dump handling logic from schema migrator * Fix job hang when no dump * Remove old workarounds * Update README.md review readme text * Address comments * One more place for Documentation update Co-authored-by: nkanchev <44201396+nkanchev@users.noreply.github.com>
kyma-incubator · Aug 29, 2022 · cd8797a · cd8797a
1 parent 9a5d196
commit cd8797a
Show file tree

Hide file tree

Showing 39 changed files with 296 additions and 55 deletions.
diff --git a/chart/compass/charts/ns-adapter/templates/deployment.yaml b/chart/compass/charts/ns-adapter/templates/deployment.yaml
@@ -7,11 +7,6 @@ metadata:
     app: {{ .Chart.Name }}
     reqlimit: big
     release: {{ .Release.Name }}
-  {{- if .Values.global.isLocalEnv }}
-  annotations:
-    "helm.sh/hook": post-install,post-upgrade
-    "helm.sh/hook-weight": "0"
-  {{- end }}
 spec:
   replicas: {{ .Values.deployment.minReplicas }}
   selector:

diff --git a/chart/compass/requirements.yaml b/chart/compass/requirements.yaml
@@ -1,6 +1,4 @@
 dependencies:
-- name: postgresql
-  condition: global.database.embedded.enabled
 - name: pairing-adapter
   condition: pairing-adapter.enabled
 - name: external-services-mock

diff --git a/chart/compass/templates/ord-aggregator-job.yaml b/chart/compass/templates/ord-aggregator-job.yaml
@@ -4,11 +4,6 @@ apiVersion: batch/v1beta1
 kind: CronJob
 metadata:
   name: "{{ $.Chart.Name }}-ord-aggregator"
-  {{ if .Values.global.isLocalEnv }} # ord-aggregator job needs fully initialized DB, so it is installed with helm post hook
-  annotations:
-    "helm.sh/hook": post-install,post-upgrade
-    "helm.sh/hook-weight": "0"
-  {{ end }}
 spec:
   schedule: "{{ .Values.global.ordAggregator.schedule }}"
   failedJobsHistoryLimit: 5

diff --git a/chart/compass/values.yaml b/chart/compass/values.yaml
@@ -143,7 +143,7 @@ global:
       name: compass-ord-service
     schema_migrator:
       dir:
-      version: "PR-2582"
+      version: "PR-2568"
       name: compass-schema-migrator
     system_broker:
       dir:

diff --git a/chart/localdb/.helmignore b/chart/localdb/.helmignore
@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+OWNERS
diff --git a/chart/localdb/Chart.yaml b/chart/localdb/Chart.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+description: Local DB
+name: localdb
+version: 0.1.0
diff --git a/chart/localdb/README.md b/chart/localdb/README.md
@@ -0,0 +1,15 @@
+# LocalDB
+
+## Overview
+
+LocalDB is used only in a local env scenario to install and apply the DB dump if needed. LocalDB comprises the following sub-charts:
+- `postgresql` - installing the PostgreSQL database
+- `dbdump` - applying the DB dump
+
+## Details
+
+### Configuration
+
+LocalDB has a standard Helm chart configuration. You can check all available configurations in the chart and sub-charts's `values.yaml` files.
+
+The values from those files can be overridden during installation. 
diff --git a/chart/localdb/charts/dbdump/.helmignore b/chart/localdb/charts/dbdump/.helmignore
@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+OWNERS
diff --git a/chart/localdb/charts/dbdump/Chart.yaml b/chart/localdb/charts/dbdump/Chart.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+description: DB Dump
+name: dbdump
+version: 0.1.0
diff --git a/chart/localdb/charts/dbdump/templates/dbdump-job.yaml b/chart/localdb/charts/dbdump/templates/dbdump-job.yaml
@@ -0,0 +1,79 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: compass-dbdump
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    app: {{ .Chart.Name }}
+    release: {{ .Release.Name }}
+spec:
+  template:
+    metadata:
+      labels:
+        app: {{ .Chart.Name }}
+        release: {{ .Release.Name }}
+      annotations:
+        cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
+    spec:
+        serviceAccountName: {{ $.Chart.Name }}-dbdump-job
+        restartPolicy: Never
+        shareProcessNamespace: true
+        containers:
+          - name: dbdump
+            image: bitnami/postgresql:11.15.0-debian-10-r93
+            imagePullPolicy: IfNotPresent
+            env:
+            - name: DB_USER
+              valueFrom:
+                secretKeyRef:
+                  name: compass-postgresql
+                  key: postgresql-director-username
+            - name: DB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: compass-postgresql
+                  key: postgresql-director-password
+            - name: DB_HOST
+              valueFrom:
+                secretKeyRef:
+                  name: compass-postgresql
+                  key: postgresql-serviceName
+            - name: DB_PORT
+              valueFrom:
+                secretKeyRef:
+                  name: compass-postgresql
+                  key: postgresql-servicePort
+            - name: DB_NAME
+              valueFrom:
+                secretKeyRef:
+                  name: compass-postgresql
+                  key: postgresql-director-db-name
+            - name: DB_SSL
+              valueFrom:
+                secretKeyRef:
+                  name: compass-postgresql
+                  key: postgresql-sslMode
+            command:
+              - "/bin/bash"
+            args:
+              - "-c"
+              - "[ -f /dbdata/dump.sql ] && PGPASSWORD=${DB_PASSWORD} psql --username=${DB_USER} --host=${DB_HOST} --port=${DB_PORT} --dbname=${DB_NAME} --single-transaction -f /dbdata/dump.sql --set ON_ERROR_STOP=on --set sslmode=${DB_SSL} || echo 'There is no dump.sql to be applied.' ; exit_code=$?; echo '# KILLING PILOT-AGENT #'; pkill -INT cloud_sql_proxy; curl -XPOST http://127.0.0.1:15020/quitquitquit; sleep 5; exit $exit_code;"
+            volumeMounts:
+              - name: dump-folder
+                mountPath: "/dbdata"
+        volumes:
+          - name: dump-folder
+            persistentVolumeClaim:
+              claimName: local-pvc
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ $.Chart.Name }}-dbdump-job
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    app: {{ $.Chart.Name }}
+    release: {{ $.Release.Name }}
+    helm.sh/chart: {{ $.Chart.Name }}-{{ $.Chart.Version | replace "+" "_" }}
+    app.kubernetes.io/managed-by: {{ $.Release.Service }}
+    app.kubernetes.io/instance: {{ $.Release.Name }}
diff --git a/chart/localdb/charts/dbdump/templates/dbdump-pv.yaml b/chart/localdb/charts/dbdump/templates/dbdump-pv.yaml
@@ -0,0 +1,35 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: local-pv
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    type: local
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app: {{ .Chart.Name }}
+    release: {{ .Release.Name }}
+spec:
+  storageClassName: manual
+  capacity:
+    storage: 1Gi
+  accessModes:
+    - ReadWriteOnce
+  hostPath:
+    path: "/dbdata"
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: local-pvc
+  namespace: {{ $.Release.Namespace }}
+  labels:
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app: {{ .Chart.Name }}
+    release: {{ .Release.Name }}
+spec:
+  storageClassName: manual
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
diff --git a/chart/localdb/charts/dbdump/values.yaml b/chart/localdb/charts/dbdump/values.yaml
diff --git a/chart/compass/charts/postgresql/.helmignore → chart/localdb/charts/postgresql/.helmignore b/chart/compass/charts/postgresql/.helmignore → chart/localdb/charts/postgresql/.helmignore
diff --git a/chart/compass/charts/postgresql/Chart.yaml → chart/localdb/charts/postgresql/Chart.yaml b/chart/compass/charts/postgresql/Chart.yaml → chart/localdb/charts/postgresql/Chart.yaml
diff --git a/chart/compass/charts/postgresql/README.md → chart/localdb/charts/postgresql/README.md b/chart/compass/charts/postgresql/README.md → chart/localdb/charts/postgresql/README.md
diff --git a/...ass/charts/postgresql/templates/NOTES.txt → ...ldb/charts/postgresql/templates/NOTES.txt b/...ass/charts/postgresql/templates/NOTES.txt → ...ldb/charts/postgresql/templates/NOTES.txt
diff --git a/.../charts/postgresql/templates/_helpers.tpl → .../charts/postgresql/templates/_helpers.tpl b/.../charts/postgresql/templates/_helpers.tpl → .../charts/postgresql/templates/_helpers.tpl
@@ -14,8 +14,7 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- if .Values.fullnameOverride -}}
 {{- printf .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
 {{- else -}}
-{{- $name := default .Chart.Name .Values.nameOverride -}}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- printf "compass-postgresql" -}}
 {{- end -}}
 {{- end -}}
 

diff --git a/...harts/postgresql/templates/configmap.yaml → ...harts/postgresql/templates/configmap.yaml b/...harts/postgresql/templates/configmap.yaml → ...harts/postgresql/templates/configmap.yaml
diff --git a/.../templates/extended-config-configmap.yaml → .../templates/extended-config-configmap.yaml b/.../templates/extended-config-configmap.yaml → .../templates/extended-config-configmap.yaml
diff --git a/...tgresql/templates/init-db-config-map.yaml → ...tgresql/templates/init-db-config-map.yaml b/...tgresql/templates/init-db-config-map.yaml → ...tgresql/templates/init-db-config-map.yaml
@@ -2,7 +2,7 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ template "fullname" . }}-db-init
+  name: {{ template "postgresql.fullname" . }}-db-init
 data:
   init-director-db.sql: |
-    CREATE DATABASE {{ .Values.global.database.embedded.director.name }};
+    CREATE DATABASE postgres;
diff --git a/...l/templates/initialization-configmap.yaml → ...l/templates/initialization-configmap.yaml b/...l/templates/initialization-configmap.yaml → ...l/templates/initialization-configmap.yaml
diff --git a/...rts/postgresql/templates/metrics-svc.yaml → ...rts/postgresql/templates/metrics-svc.yaml b/...rts/postgresql/templates/metrics-svc.yaml → ...rts/postgresql/templates/metrics-svc.yaml
diff --git a/...s/postgresql/templates/networkpolicy.yaml → ...s/postgresql/templates/networkpolicy.yaml b/...s/postgresql/templates/networkpolicy.yaml → ...s/postgresql/templates/networkpolicy.yaml
diff --git a/.../charts/postgresql/templates/secrets.yaml → .../charts/postgresql/templates/secrets.yaml b/.../charts/postgresql/templates/secrets.yaml → .../charts/postgresql/templates/secrets.yaml
diff --git a/.../postgresql/templates/serviceaccount.yaml → .../postgresql/templates/serviceaccount.yaml b/.../postgresql/templates/serviceaccount.yaml → .../postgresql/templates/serviceaccount.yaml
diff --git a/.../postgresql/templates/servicemonitor.yaml → .../postgresql/templates/servicemonitor.yaml b/.../postgresql/templates/servicemonitor.yaml → .../postgresql/templates/servicemonitor.yaml
diff --git a/...tgresql/templates/statefulset-slaves.yaml → ...tgresql/templates/statefulset-slaves.yaml b/...tgresql/templates/statefulset-slaves.yaml → ...tgresql/templates/statefulset-slaves.yaml
diff --git a/...rts/postgresql/templates/statefulset.yaml → ...rts/postgresql/templates/statefulset.yaml b/...rts/postgresql/templates/statefulset.yaml → ...rts/postgresql/templates/statefulset.yaml
diff --git a/...ts/postgresql/templates/svc-headless.yaml → ...ts/postgresql/templates/svc-headless.yaml b/...ts/postgresql/templates/svc-headless.yaml → ...ts/postgresql/templates/svc-headless.yaml
diff --git a/...charts/postgresql/templates/svc-read.yaml → ...charts/postgresql/templates/svc-read.yaml b/...charts/postgresql/templates/svc-read.yaml → ...charts/postgresql/templates/svc-read.yaml
diff --git a/...pass/charts/postgresql/templates/svc.yaml → ...aldb/charts/postgresql/templates/svc.yaml b/...pass/charts/postgresql/templates/svc.yaml → ...aldb/charts/postgresql/templates/svc.yaml
diff --git a/.../charts/postgresql/values-production.yaml → .../charts/postgresql/values-production.yaml b/.../charts/postgresql/values-production.yaml → .../charts/postgresql/values-production.yaml
diff --git a/chart/compass/charts/postgresql/values.yaml → chart/localdb/charts/postgresql/values.yaml b/chart/compass/charts/postgresql/values.yaml → chart/localdb/charts/postgresql/values.yaml
@@ -8,6 +8,7 @@ global:
 #   imagePullSecrets:
 #     - myRegistryKeySecretName
 
+
 ## Bitnami PostgreSQL image version
 ## ref: https://hub.docker.com/r/bitnami/postgresql/tags/
 ##

diff --git a/chart/localdb/values.yaml b/chart/localdb/values.yaml
@@ -0,0 +1,8 @@
+global:
+  tests:
+    namespace: kyma-system
+  database:
+    embedded:
+      director:
+        name: "postgres"
+      directorDBName: "postgres"
diff --git a/components/schema-migrator/run.sh b/components/schema-migrator/run.sh
@@ -104,23 +104,6 @@ fi
 
 set -e
 
-DUMP_FILE=$(find seeds -type f -name "dump-*.sql")
-if [[ $DUMP_FILE ]] && [[ "${DIRECTION}" == "up" ]]; then
-    echo "Will reuse existing dump - $DUMP_FILE"
-    PGPASSWORD="${DB_PASSWORD}" psql --username="${DB_USER}" --host="${DB_HOST}" --port="${DB_PORT}" --dbname="${DB_NAME}" --single-transaction -f "$DUMP_FILE" --set ON_ERROR_STOP=on --set "${SSL_OPTION}"
-
-    REMOTE_MIGRATION_VERSION=$(PGPASSWORD="${DB_PASSWORD}" psql -qtAX -U "${DB_USER}" -h "${DB_HOST}" -p "${DB_PORT}" -d "${DB_NAME}" -c "SELECT version FROM schema_migrations")
-    LOCAL_MIGRATION_VERSION=$(echo $(ls migrations/director | tail -n 1) | grep -o -E '^[0-9]+' | head -1 | sed -e 's/^0\+//')
-
-    if [[ ${REMOTE_MIGRATION_VERSION} = ${LOCAL_MIGRATION_VERSION} ]]; then
-        echo -e "${GREEN}Both remote and local migrations are at the same version.${NC}"
-    else
-        echo -e "${RED}Remote and local migrations are at different versions.${NC}"
-        echo -e "${YELLOW}REMOTE: $REMOTE_MIGRATION_VERSION${NC}"
-        echo -e "${YELLOW}LOCAL: $LOCAL_MIGRATION_VERSION${NC}"
-    fi
-fi
-
 CONNECTION_STRING="postgres://$DB_USER:$DB_PASSWORD@$DB_HOST:$DB_PORT/$DB_NAME_SSL"
 function currentVersion {
   echo $(migrate -path ${MIGRATION_STORAGE_PATH} -database "$CONNECTION_STRING" version 2>&1)

diff --git a/docs/compass/04-01-installation.md b/docs/compass/04-01-installation.md
@@ -119,7 +119,12 @@ global:
 #      tlsKey: ${TLS_KEY}
 ```
 
-And then start the Compass installation by using the following command:
+Start the Database installation by using the following command:
+
+```bash
+<script from ../../installation/scripts/install-db.sh> --overrides-file <file from ../../installation/resources/compass-overrides-local.yaml> --overrides-file <file from above step - e.g. additionalCompassOverrides.yaml> --timeout <e.g: 30m0s>
+```
+Once the Database is provisioned procced and start the Compass installation by using the following command:
 
 ```bash
 <script from ../../installation/scripts/install-compass.sh> --overrides-file <file from ../../installation/resources/compass-overrides-local.yaml> --overrides-file <file from above step - e.g. additionalCompassOverrides.yaml> --timeout <e.g: 30m0s>
@@ -375,7 +380,11 @@ global:
 #      tlsCrt: ${TLS_CERT}
 #      tlsKey: ${TLS_KEY}
 ```
-
+Start Database installation:
+```bash
+<script from ../../installation/scripts/install-db.sh> --overrides-file <file from ../../installation/resources/compass-overrides-local.yaml> --overrides-file <file from above step - e.g. additionalCompassOverrides.yaml> --timeout <e.g: 30m0s>
+```
+Then, install compass component:
 ```bash
 <script from ../../installation/scripts/install-compass.sh> --overrides-file <file from ../../installation/resources/compass-overrides-local.yaml> --overrides-file <file from above step - e.g. additionalCompassOverrides.yaml> --timeout <e.g: 30m0s>
 ```