test: E2E on monitoring queries against the app db in a replica cluster (cloudnative-pg#372)

Co-authored-by: Danish Khan <danish.khan@enterprisedb.com>
Co-authored-by: Leonardo Cecchi <leonardo.cecchi@enterprisedb.com>

Author: 3 people

tests/e2e/fixtures/metrics/.gitignore

@@ -2,3 +2,4 @@ cluster-metrics.yaml
 cluster-metrics-with-target-databases.yaml
 cluster-disable-default-metrics.yaml
 cluster-default-metrics.yaml
+cluster-replica-tls-with-metrics.yaml

tests/e2e/fixtures/metrics/cluster-replica-tls-with-metrics.yaml.template

@@ -0,0 +1,43 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: cluster-replica-tls
+spec:
+  instances: 2
+
+  bootstrap:
+    pg_basebackup:
+      source: cluster-replica-src
+      database: appSrc
+      owner: userSrc
+
+  replica:
+    enabled: true
+    source: cluster-replica-src
+
+  storage:
+    size: 1Gi
+    storageClass: ${E2E_DEFAULT_STORAGE_CLASS}
+
+  monitoring:
+    customQueriesConfigMap:
+      - name: monitoring-replica-cluster
+        key: queries.yaml
+
+  externalClusters:
+  - name: cluster-replica-src
+    connectionParameters:
+      host: cluster-replica-src-rw
+      user: streaming_replica
+      sslmode: verify-full
+      dbname: postgres
+      port: "5432"
+    sslKey:
+      name: cluster-replica-src-replication
+      key: tls.key
+    sslCert:
+      name: cluster-replica-src-replication
+      key: tls.crt
+    sslRootCert:
+      name: cluster-replica-src-ca
+      key: ca.crt

tests/e2e/fixtures/metrics/custom-queries-for-replica-cluster.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: monitoring-replica-cluster
+  labels:
+    e2e: metrics
+data:
+  queries.yaml: |
+    replica_test:
+      query: |
+        SELECT count(*) as row_count FROM test_replica
+      primary: false
+      metrics:
+        - row_count:
+            usage: "GAUGE"
+            description: "Number of rows present in test_replica table"

tests/e2e/metrics_test.go

@@ -18,10 +18,13 @@ package e2e
 
 import (
 	"regexp"
+	"strings"
+	"time"
 
 	corev1 "k8s.io/api/core/v1"
 	ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
 
+	"github.com/cloudnative-pg/cloudnative-pg/pkg/specs"
 	"github.com/cloudnative-pg/cloudnative-pg/tests"
 	"github.com/cloudnative-pg/cloudnative-pg/tests/utils"
 
@@ -71,6 +74,7 @@ var _ = Describe("Metrics", func() {
 			env.DumpNamespaceObjects(namespace, "out/"+CurrentSpecReport().LeafNodeText+".log")
 		}
 	})
+
 	AfterEach(func() {
 		err := env.DeleteNamespace(namespace)
 		Expect(err).ToNot(HaveOccurred())
@@ -190,4 +194,65 @@ var _ = Describe("Metrics", func() {
 
 		collectAndAssertDefaultMetricsPresentOnEachPod(namespace, metricsClusterName, curlPodName, false)
 	})
+
+	It("execute custom queries against the application database on replica clusters", func() {
+		const (
+			replicaModeClusterDir    = "/replica_mode_cluster/"
+			replicaClusterSampleFile = fixturesDir + "/metrics/cluster-replica-tls-with-metrics.yaml"
+			srcClusterSampleFile     = fixturesDir + replicaModeClusterDir + "cluster-replica-src.yaml"
+			configMapFIle            = fixturesDir + "/metrics/custom-queries-for-replica-cluster.yaml"
+			checkQuery               = "SELECT count(*) FROM test_replica"
+		)
+
+		namespace = "metrics-with-replica-mode"
+
+		// Fetching the source cluster name
+		srcClusterName, err := env.GetResourceNameFromYAML(srcClusterSampleFile)
+		Expect(err).ToNot(HaveOccurred())
+
+		// Fetching replica cluster name
+		replicaClusterName, err := env.GetResourceNameFromYAML(replicaClusterSampleFile)
+		Expect(err).ToNot(HaveOccurred())
+
+		// create namespace
+		err = env.CreateNamespace(namespace)
+		Expect(err).ToNot(HaveOccurred())
+
+		// Creating and verifying custom queries configmap
+		AssertCustomMetricsResourcesExist(namespace, configMapFIle, 1, 0)
+
+		// Create the curl client pod and wait for it to be ready
+		By("setting up curl client pod", func() {
+			curlClient := utils.CurlClient(namespace)
+			err := utils.PodCreateAndWaitForReady(env, &curlClient, 240)
+			Expect(err).ToNot(HaveOccurred())
+			curlPodName = curlClient.GetName()
+		})
+
+		AssertReplicaModeCluster(namespace, srcClusterName, srcClusterSampleFile, replicaClusterName,
+			replicaClusterSampleFile, checkQuery)
+
+		By("grant select permission for test_replica table to pg_monitor", func() {
+			primarySrcCluster, err := env.GetClusterPrimary(namespace, srcClusterName)
+			Expect(err).ToNot(HaveOccurred())
+			commandTimeout := time.Second * 5
+			cmd := "GRANT SELECT ON test_replica TO pg_monitor"
+			_, _, err = env.EventuallyExecCommand(env.Ctx, *primarySrcCluster, specs.PostgresContainerName,
+				&commandTimeout, "psql", "-U", "postgres", "appSrc", "-tAc", cmd)
+			Expect(err).ToNot(HaveOccurred())
+		})
+
+		By("collecting metrics on each pod and checking that the table has been found", func() {
+			podList, err := env.GetClusterPodList(namespace, replicaClusterName)
+			Expect(err).ToNot(HaveOccurred())
+			// Gather metrics in each pod
+			for _, pod := range podList.Items {
+				podIP := pod.Status.PodIP
+				out, err := utils.CurlGetMetrics(namespace, curlPodName, podIP, 9187)
+				Expect(err).Should(Not(HaveOccurred()))
+				Expect(strings.Split(out, "\n")).Should(ContainElement("cnpg_replica_test_row_count 3"))
+			}
+		})
+		collectAndAssertDefaultMetricsPresentOnEachPod(namespace, replicaClusterName, curlPodName, true)
+	})
 })