test: adds E2E tests for the cnpg plugin hibernate command. (cloudnative-pg#855)

Closes cloudnative-pg#941

Signed-off-by: Jitendra Wadle <jitendra.wadle@enterprisedb.com>
Signed-off-by: Danish Khan <danish.khan@enterprisedb.com>
Signed-off-by: Armando Ruocco <armando.ruocco@enterprisedb.com>
Signed-off-by: Hai He <hai.he@enterprisedb.com>
Signed-off-by: Tao Li <tao.li@enterprisedb.com>
Co-authored-by: Danish Khan <danish.khan@enterprisedb.com>
Co-authored-by: Armando Ruocco <armando.ruocco@enterprisedb.com>
Co-authored-by: Hai He <hai.he@enterprisedb.com>
Co-authored-by: Tao Li <tao.li@enterprisedb.com>

Author: 5 people

tests/e2e/fixtures/hibernate/cluster-storage-class-without-wal.yaml.template

@@ -0,0 +1,33 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: postgresql-hibernate
+spec:
+  instances: 3
+
+  postgresql:
+    parameters:
+      log_checkpoints: "on"
+      log_lock_waits: "on"
+      log_min_duration_statement: '1000'
+      log_statement: 'ddl'
+      log_temp_files: '1024'
+      log_autovacuum_min_duration: '1s'
+      log_replication_commands: 'on'
+
+  # Example of rolling update strategy:
+  # - unsupervised: automated update of the primary once all
+  #                 replicas have been upgraded (default)
+  # - supervised: requires manual supervision to perform
+  #               the switchover of the primary
+  primaryUpdateStrategy: unsupervised
+
+  bootstrap:
+    initdb:
+      database: app
+      owner: app
+
+  # Persistent storage configuration
+  storage:
+    storageClass: ${E2E_DEFAULT_STORAGE_CLASS}
+    size: 1Gi

tests/e2e/hibernation_test.go

@@ -0,0 +1,354 @@
+package e2e
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/api/rbac/v1"
+	"k8s.io/apimachinery/pkg/types"
+	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
+
+	apiv1 "github.com/cloudnative-pg/cloudnative-pg/api/v1"
+	"github.com/cloudnative-pg/cloudnative-pg/pkg/specs"
+	"github.com/cloudnative-pg/cloudnative-pg/pkg/utils"
+	"github.com/cloudnative-pg/cloudnative-pg/tests"
+	testsUtils "github.com/cloudnative-pg/cloudnative-pg/tests/utils"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("Cluster Hibernation with plugin", func() {
+	type mode string
+	type hibernateSatusMessage string
+	type expectedKeysInStatus string
+	const (
+		sampleFileClusterWithPGWalVolume    = fixturesDir + "/base/cluster-storage-class.yaml.template"
+		sampleFileClusterWithOutPGWalVolume = fixturesDir + "/hibernate/" +
+			"cluster-storage-class-without-wal.yaml.template"
+		level                                         = tests.Medium
+		HibernateOn             mode                  = "on"
+		HibernateOff            mode                  = "off"
+		HibernateStatus         mode                  = "status"
+		clusterOffStatusMessage hibernateSatusMessage = "No Hibernation. Cluster Deployed."
+		clusterOnStatusMessage  hibernateSatusMessage = "Cluster Hibernated"
+		summaryInStatus         expectedKeysInStatus  = "summary"
+		tableName                                     = "test"
+	)
+	var namespace string
+	BeforeEach(func() {
+		if testLevelEnv.Depth < int(level) {
+			Skip("Test depth is lower than the amount requested for this test")
+		}
+	})
+
+	JustAfterEach(func() {
+		if CurrentSpecReport().Failed() {
+			env.DumpNamespaceObjects(namespace, "out/"+CurrentSpecReport().FullText()+".log")
+		}
+	})
+
+	AfterEach(func() {
+		err := env.DeleteNamespace(namespace)
+		Expect(err).ToNot(HaveOccurred())
+	})
+
+	Context("hibernate", func() {
+		var err error
+		getPrimaryAndClusterManifest := func(namespace, clusterName string) ([]byte, *apiv1.Cluster, string) {
+			var beforeHibernationClusterInfo *apiv1.Cluster
+			var clusterManifest []byte
+			var beforeHibernationCurrentPrimary string
+			By("collecting current primary details", func() {
+				beforeHibernationClusterInfo, err = env.GetCluster(namespace, clusterName)
+				Expect(err).ToNot(HaveOccurred())
+				beforeHibernationCurrentPrimary = beforeHibernationClusterInfo.Status.CurrentPrimary
+				// collect expected cluster manifesto info
+				clusterManifest, err = json.Marshal(&beforeHibernationClusterInfo)
+				Expect(err).ToNot(HaveOccurred())
+			})
+			return clusterManifest, beforeHibernationClusterInfo, beforeHibernationCurrentPrimary
+		}
+		getPvc := func(role utils.PVCRole, clusterInfo *apiv1.Cluster, instanceName string) corev1.PersistentVolumeClaim {
+			pvcName := specs.GetPVCName(*clusterInfo, instanceName, role)
+			pvcInfo := corev1.PersistentVolumeClaim{}
+			err = testsUtils.GetObject(env, ctrlclient.ObjectKey{Namespace: namespace, Name: pvcName}, &pvcInfo)
+			Expect(err).ToNot(HaveOccurred())
+			return pvcInfo
+		}
+		performHibernation := func(mode mode, namespace, clusterName string) {
+			By(fmt.Sprintf("performing hibernation %v", mode), func() {
+				_, _, err := testsUtils.Run(fmt.Sprintf("kubectl cnpg hibernate %v %v -n %v",
+					mode, clusterName, namespace))
+				Expect(err).ToNot(HaveOccurred())
+			})
+			By(fmt.Sprintf("verifying cluster %v pods are removed", clusterName), func() {
+				Eventually(func(g Gomega) {
+					podList, _ := env.GetClusterPodList(namespace, clusterName)
+					g.Expect(len(podList.Items)).Should(BeEquivalentTo(0))
+				}, 300).Should(Succeed())
+			})
+		}
+
+		getHibernationStatusInJSON := func(namespace, clusterName string) map[string]interface{} {
+			var data map[string]interface{}
+			By("getting hibernation status", func() {
+				stdOut, _, err := testsUtils.Run(fmt.Sprintf("kubectl cnpg hibernate %v %v -n %v -ojson",
+					HibernateStatus, clusterName, namespace))
+				Expect(err).ToNot(HaveOccurred(), stdOut)
+				err = json.Unmarshal([]byte(stdOut), &data)
+				Expect(err).ToNot(HaveOccurred())
+			})
+			return data
+		}
+
+		verifySummaryInHibernationStatus := func(clusterName string, message hibernateSatusMessage) {
+			statusOut := getHibernationStatusInJSON(namespace, clusterName)
+			actualStatus := statusOut[string(summaryInStatus)].(map[string]interface{})["status"].(string)
+			Expect(strings.Contains(string(message), actualStatus)).Should(BeEquivalentTo(true), statusOut)
+		}
+		verifyClusterResources := func(namespace, clusterName string, roles []utils.PVCRole) {
+			By(fmt.Sprintf("verifying cluster resources are removed "+
+				"post hibernation where roles %v", roles), func() {
+				timeout := 120
+
+				By(fmt.Sprintf("verifying cluster %v is removed", clusterName), func() {
+					Eventually(func() (bool, apiv1.Cluster) {
+						cluster := &apiv1.Cluster{}
+						err := env.Client.Get(env.Ctx,
+							ctrlclient.ObjectKey{Namespace: namespace, Name: clusterName},
+							cluster)
+						if err != nil {
+							return true, apiv1.Cluster{}
+						}
+						return false, *cluster
+					}, timeout).Should(BeTrue())
+				})
+
+				By(fmt.Sprintf("verifying cluster %v PVCs are removed", clusterName), func() {
+					Eventually(func() (int, error) {
+						pvcList, err := env.GetPVCList(namespace)
+						if err != nil {
+							return -1, err
+						}
+						return len(pvcList.Items), nil
+					}, timeout).Should(BeEquivalentTo(len(roles)))
+				})
+
+				By(fmt.Sprintf("verifying cluster %v configMap is removed", clusterName), func() {
+					Eventually(func() (bool, corev1.ConfigMap) {
+						configMap := corev1.ConfigMap{}
+						err = env.Client.Get(env.Ctx,
+							ctrlclient.ObjectKey{Namespace: namespace, Name: apiv1.DefaultMonitoringConfigMapName},
+							&configMap)
+						if err != nil {
+							return true, corev1.ConfigMap{}
+						}
+						return false, configMap
+					}, timeout).Should(BeTrue())
+				})
+
+				By(fmt.Sprintf("verifying cluster %v secrets are removed", clusterName), func() {
+					Eventually(func() (bool, corev1.SecretList, error) {
+						secretList := corev1.SecretList{}
+						err = env.Client.List(env.Ctx, &secretList, ctrlclient.InNamespace(namespace))
+						if err != nil {
+							return false, corev1.SecretList{}, err
+						}
+						var getClusterSecrets []string
+						for _, secret := range secretList.Items {
+							if strings.HasPrefix(secret.GetName(), clusterName) {
+								getClusterSecrets = append(getClusterSecrets, secret.GetName())
+							}
+						}
+						if len(getClusterSecrets) == 0 {
+							return true, corev1.SecretList{}, nil
+						}
+						return false, secretList, nil
+					}, timeout).Should(BeTrue())
+				})
+
+				By(fmt.Sprintf("verifying cluster %v role is removed", clusterName), func() {
+					Eventually(func() (bool, v1.Role) {
+						role := v1.Role{}
+						err = env.Client.Get(env.Ctx,
+							ctrlclient.ObjectKey{Namespace: namespace, Name: clusterName},
+							&role)
+						if err != nil {
+							return true, v1.Role{}
+						}
+						return false, role
+					}, timeout).Should(BeTrue())
+				})
+
+				By(fmt.Sprintf("verifying cluster %v rolebinding is removed", clusterName), func() {
+					Eventually(func() (bool, v1.RoleBinding) {
+						roleBinding := v1.RoleBinding{}
+						err = env.Client.Get(env.Ctx,
+							ctrlclient.ObjectKey{Namespace: namespace, Name: clusterName},
+							&roleBinding)
+						if err != nil {
+							return true, v1.RoleBinding{}
+						}
+						return false, roleBinding
+					}, timeout).Should(BeTrue())
+				})
+			})
+		}
+		verifyPvc := func(role utils.PVCRole, pvcUid types.UID, clusterInfo *apiv1.Cluster, clusterManifest []byte,
+			instanceName string,
+		) {
+			pvcInfo := getPvc(role, clusterInfo, instanceName)
+			Expect(pvcUid).Should(BeEquivalentTo(pvcInfo.GetUID()))
+			// pvc should be attached annotation with pgControlData and Cluster manifesto
+			expectedAnnotationKeyPresent := []string{
+				utils.HibernatePgControlDataAnnotationName,
+				utils.HibernateClusterManifestAnnotationName,
+			}
+			testsUtils.ObjectHasAnnotations(&pvcInfo, expectedAnnotationKeyPresent)
+			expectedAnnotation := map[string]string{
+				utils.HibernateClusterManifestAnnotationName: string(clusterManifest),
+			}
+			testsUtils.ObjectMatchesAnnotations(&pvcInfo, expectedAnnotation)
+		}
+
+		assertHibernation := func(namespace, clusterName, tableName string) {
+			var beforeHibernationPgWalPvcUID types.UID
+			var beforeHibernationPgDataPvcUID types.UID
+
+			// Write a table and some data on the "app" database
+			AssertCreateTestData(namespace, clusterName, tableName)
+			clusterManifest, beforeHibernationClusterInfo, currentPrimary := getPrimaryAndClusterManifest(namespace, clusterName)
+
+			By("collecting pgWal pvc details of current primary", func() {
+				pvcInfo := getPvc(utils.PVCRolePgWal, beforeHibernationClusterInfo, currentPrimary)
+				beforeHibernationPgWalPvcUID = pvcInfo.GetUID()
+			})
+
+			By("collecting pgData pvc details of current primary", func() {
+				pvcInfo := getPvc(utils.PVCRolePgData, beforeHibernationClusterInfo, currentPrimary)
+				beforeHibernationPgDataPvcUID = pvcInfo.GetUID()
+			})
+
+			By(fmt.Sprintf("verifying hibernation status"+
+				" before hibernate on cluster %v", clusterName), func() {
+				verifySummaryInHibernationStatus(clusterName, clusterOffStatusMessage)
+			})
+
+			performHibernation(HibernateOn, namespace, clusterName)
+
+			By(fmt.Sprintf("verifying hibernation status"+
+				" after hibernate on cluster %v", clusterName), func() {
+				verifySummaryInHibernationStatus(clusterName, clusterOnStatusMessage)
+			})
+
+			// After hibernation, it will destroy all the resources generated by the cluster,
+			// except the PVCs that belong to the PostgreSQL primary instance.
+			verifyClusterResources(namespace, clusterName, []utils.PVCRole{utils.PVCRolePgWal, utils.PVCRolePgData})
+
+			By("verifying primary pgWal pvc info", func() {
+				verifyPvc(utils.PVCRolePgWal, beforeHibernationPgWalPvcUID, beforeHibernationClusterInfo,
+					clusterManifest, currentPrimary)
+			})
+
+			By("verifying primary pgData pvc info", func() {
+				verifyPvc(utils.PVCRolePgData, beforeHibernationPgDataPvcUID, beforeHibernationClusterInfo,
+					clusterManifest, currentPrimary)
+			})
+
+			// verifying hibernation off
+			performHibernation(HibernateOff, namespace, clusterName)
+
+			By(fmt.Sprintf("verifying hibernation status after "+
+				"perform hibernation off on cluster %v", clusterName), func() {
+				verifySummaryInHibernationStatus(clusterName, clusterOffStatusMessage)
+			})
+
+			AssertClusterIsReady(namespace, clusterName, 600, env)
+			// Test data should be present after hibernation off
+			AssertDataExpectedCount(namespace, currentPrimary, tableName, 2)
+		}
+
+		When("cluster setup with PG-WAL volume", func() {
+			It("hibernation process should work", func() {
+				namespace = "hibernation-on-with-pg-wal"
+				clusterName, err := env.GetResourceNameFromYAML(sampleFileClusterWithPGWalVolume)
+				Expect(err).ToNot(HaveOccurred())
+				// Create a cluster in a namespace we'll delete after the test
+				err = env.CreateNamespace(namespace)
+				Expect(err).ToNot(HaveOccurred())
+				AssertCreateCluster(namespace, clusterName, sampleFileClusterWithPGWalVolume, env)
+				assertHibernation(namespace, clusterName, tableName)
+			})
+		})
+		When("cluster setup without PG-WAL volume", func() {
+			It("hibernation process should work", func() {
+				var beforeHibernationPgDataPvcUID types.UID
+
+				namespace = "hibernation-without-pg-wal"
+				clusterName, err := env.GetResourceNameFromYAML(sampleFileClusterWithOutPGWalVolume)
+				Expect(err).ToNot(HaveOccurred())
+				// Create a cluster in a namespace we'll delete after the test
+				err = env.CreateNamespace(namespace)
+				Expect(err).ToNot(HaveOccurred())
+				AssertCreateCluster(namespace, clusterName, sampleFileClusterWithOutPGWalVolume, env)
+				// Write a table and some data on the "app" database
+				AssertCreateTestData(namespace, clusterName, tableName)
+				clusterManifest, beforeHibernationClusterInfo, currentPrimary := getPrimaryAndClusterManifest(namespace,
+					clusterName)
+
+				By("collecting pgData pvc details of current primary", func() {
+					pvcInfo := getPvc(utils.PVCRolePgData, beforeHibernationClusterInfo, currentPrimary)
+					beforeHibernationPgDataPvcUID = pvcInfo.GetUID()
+				})
+
+				By(fmt.Sprintf("verifying hibernation status"+
+					" before hibernate on cluster %v", clusterName), func() {
+					verifySummaryInHibernationStatus(clusterName, clusterOffStatusMessage)
+				})
+
+				performHibernation(HibernateOn, namespace, clusterName)
+
+				By(fmt.Sprintf("verifying hibernation status"+
+					" after hibernate on cluster %v", clusterName), func() {
+					verifySummaryInHibernationStatus(clusterName, clusterOnStatusMessage)
+				})
+
+				// After hibernation, it will destroy all the resources generated by the cluster,
+				// except the PVCs that belong to the PostgreSQL primary instance.
+				verifyClusterResources(namespace, clusterName, []utils.PVCRole{utils.PVCRolePgData})
+
+				By("verifying primary pgData pvc info", func() {
+					verifyPvc(utils.PVCRolePgData, beforeHibernationPgDataPvcUID, beforeHibernationClusterInfo,
+						clusterManifest, currentPrimary)
+				})
+
+				// verifying hibernation off
+				performHibernation(HibernateOff, namespace, clusterName)
+				By(fmt.Sprintf("verifying hibernation status"+
+					" before hibernate on cluster %v", clusterName), func() {
+					verifySummaryInHibernationStatus(clusterName, clusterOffStatusMessage)
+				})
+
+				AssertClusterIsReady(namespace, clusterName, 600, env)
+				// Test data should be present after hibernation off
+				AssertDataExpectedCount(namespace, currentPrimary, tableName, 2)
+			})
+		})
+		When("cluster hibernation after switchover", func() {
+			It("hibernation process should work", func() {
+				namespace = "hibernation-with-switchover"
+				clusterName, err := env.GetResourceNameFromYAML(sampleFileClusterWithPGWalVolume)
+				Expect(err).ToNot(HaveOccurred())
+				// Create a cluster in a namespace we'll delete after the test
+				err = env.CreateNamespace(namespace)
+				Expect(err).ToNot(HaveOccurred())
+				AssertCreateCluster(namespace, clusterName, sampleFileClusterWithPGWalVolume, env)
+				AssertSwitchover(namespace, clusterName, env)
+				assertHibernation(namespace, clusterName, tableName)
+			})
+		})
+	})
+})