Skip to content

Commit 1ac4703

Browse files
wadlejitendradanishedbDanish Khanphisco
authored
test: wrap API server calls with Eventually clauses in E2E tests
Wrap all direct interactions with the API server with `Eventually` clauses in E2E tests, in order to make tests more resilient to cloud providers' transient failures. Co-authored-by: Danish Khan <danish.khan@enterprisedb.com> Co-authored-by: Danish Khan <danish.khan@Danishs-MacBook-Pro.local> Co-authored-by: Philippe Scorsolini <philippe.scorsolini@enterprisedb.com>
1 parent 1e386b8 commit 1ac4703

11 files changed

+190
-160
lines changed

‎tests/e2e/asserts_test.go

Lines changed: 53 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ func AssertSwitchover(namespace string, clusterName string, env *testsUtils.Test
155155

156156
// AssertCreateCluster tests that the pods that should have been created by the sample
157157
// exist and are in ready state
158-
func AssertCreateCluster(namespace string, clusterName string, sample string, env *testsUtils.TestingEnvironment) {
158+
func AssertCreateCluster(namespace string, clusterName string, sampleFile string, env *testsUtils.TestingEnvironment) {
159159
By(fmt.Sprintf("having a %v namespace", namespace), func() {
160160
// Creating a namespace should be quick
161161
timeout := 20
@@ -172,8 +172,7 @@ func AssertCreateCluster(namespace string, clusterName string, sample string, en
172172
})
173173

174174
By(fmt.Sprintf("creating a Cluster in the %v namespace", namespace), func() {
175-
_, _, err := testsUtils.Run("kubectl create -n " + namespace + " -f " + sample)
176-
Expect(err).ToNot(HaveOccurred())
175+
CreateResourceFromFile(namespace, sampleFile)
177176
})
178177
// Setting up a cluster with three pods is slow, usually 200-600s
179178
AssertClusterIsReady(namespace, clusterName, 600, env)
@@ -582,11 +581,7 @@ func AssertArchiveWalOnMinio(namespace, clusterName string) {
582581
}
583582

584583
func AssertScheduledBackupsAreScheduled(namespace string, backupYAMLPath string, timeout int) {
585-
_, _, err := testsUtils.Run(fmt.Sprintf(
586-
"kubectl apply -n %v -f %v",
587-
namespace, backupYAMLPath))
588-
Expect(err).NotTo(HaveOccurred())
589-
584+
CreateResourceFromFile(namespace, backupYAMLPath)
590585
scheduledBackupName, err := env.GetResourceNameFromYAML(backupYAMLPath)
591586
Expect(err).NotTo(HaveOccurred())
592587

@@ -801,11 +796,8 @@ func AssertFastFailOver(
801796
}, timeout).Should(BeEquivalentTo(namespace))
802797
})
803798

804-
By(fmt.Sprintf("creating a Cluster in the %v namespace",
805-
namespace), func() {
806-
_, _, err = testsUtils.Run(
807-
"kubectl create -n " + namespace + " -f " + sampleFile)
808-
Expect(err).ToNot(HaveOccurred())
799+
By(fmt.Sprintf("creating a Cluster in the %v namespace", namespace), func() {
800+
CreateResourceFromFile(namespace, sampleFile)
809801
})
810802

811803
By("having a Cluster with three instances ready", func() {
@@ -1132,11 +1124,7 @@ func AssertClusterAsyncReplica(namespace, sourceClusterFile, restoreClusterFile,
11321124
By("Async Replication into external cluster", func() {
11331125
restoredClusterName, err := env.GetResourceNameFromYAML(restoreClusterFile)
11341126
Expect(err).ToNot(HaveOccurred())
1135-
_, _, err = testsUtils.Run(fmt.Sprintf(
1136-
"kubectl apply -n %v -f %v",
1137-
namespace, restoreClusterFile))
1138-
Expect(err).ToNot(HaveOccurred())
1139-
1127+
CreateResourceFromFile(namespace, restoreClusterFile)
11401128
// We give more time than the usual 600s, since the recovery is slower
11411129
AssertClusterIsReady(namespace, restoredClusterName, 800, env)
11421130

@@ -1179,10 +1167,7 @@ func AssertClusterRestore(namespace, restoreClusterFile, tableName string) {
11791167
By("Restoring a backup in a new cluster", func() {
11801168
restoredClusterName, err := env.GetResourceNameFromYAML(restoreClusterFile)
11811169
Expect(err).ToNot(HaveOccurred())
1182-
_, _, err = testsUtils.Run(fmt.Sprintf(
1183-
"kubectl apply -n %v -f %v",
1184-
namespace, restoreClusterFile))
1185-
Expect(err).ToNot(HaveOccurred())
1170+
CreateResourceFromFile(namespace, restoreClusterFile)
11861171

11871172
// We give more time than the usual 600s, since the recovery is slower
11881173
AssertClusterIsReady(namespace, restoredClusterName, 800, env)
@@ -1209,10 +1194,7 @@ func AssertScheduledBackupsImmediate(namespace, backupYAMLPath, scheduledBackupN
12091194
By("scheduling immediate backups", func() {
12101195
var err error
12111196
// Create the ScheduledBackup
1212-
_, _, err = testsUtils.Run(fmt.Sprintf(
1213-
"kubectl apply -n %v -f %v",
1214-
namespace, backupYAMLPath))
1215-
Expect(err).NotTo(HaveOccurred())
1197+
CreateResourceFromFile(namespace, backupYAMLPath)
12161198

12171199
// We expect the scheduled backup to be scheduled after creation
12181200
scheduledBackupNamespacedName := types.NamespacedName{
@@ -1243,10 +1225,15 @@ func AssertSuspendScheduleBackups(namespace, scheduledBackupName string) {
12431225
var err error
12441226
By("suspending the scheduled backup", func() {
12451227
// update suspend status to true
1246-
cmd := fmt.Sprintf("kubectl patch ScheduledBackup %v -n %v -p '{\"spec\":{\"suspend\":true}}' "+
1247-
"--type='merge'", scheduledBackupName, namespace)
1248-
_, _, err = testsUtils.Run(cmd)
1249-
Expect(err).ToNot(HaveOccurred())
1228+
Eventually(func() error {
1229+
cmd := fmt.Sprintf("kubectl patch ScheduledBackup %v -n %v -p '{\"spec\":{\"suspend\":true}}' "+
1230+
"--type='merge'", scheduledBackupName, namespace)
1231+
_, _, err = testsUtils.RunUnchecked(cmd)
1232+
if err != nil {
1233+
return err
1234+
}
1235+
return nil
1236+
}, 60, 5).Should(BeNil())
12501237
scheduledBackupNamespacedName := types.NamespacedName{
12511238
Namespace: namespace,
12521239
Name: scheduledBackupName,
@@ -1286,11 +1273,15 @@ func AssertSuspendScheduleBackups(namespace, scheduledBackupName string) {
12861273
// take current backup count before suspend the schedule backup
12871274
completedBackupsCount, err = getScheduledBackupCompleteBackupsCount(namespace, scheduledBackupName)
12881275
Expect(err).ToNot(HaveOccurred())
1289-
1290-
cmd := fmt.Sprintf("kubectl patch ScheduledBackup %v -n %v -p '{\"spec\":{\"suspend\":false}}' "+
1291-
"--type='merge'", scheduledBackupName, namespace)
1292-
_, _, err = testsUtils.Run(cmd)
1293-
Expect(err).ToNot(HaveOccurred())
1276+
Eventually(func() error {
1277+
cmd := fmt.Sprintf("kubectl patch ScheduledBackup %v -n %v -p '{\"spec\":{\"suspend\":false}}' "+
1278+
"--type='merge'", scheduledBackupName, namespace)
1279+
_, _, err = testsUtils.RunUnchecked(cmd)
1280+
if err != nil {
1281+
return err
1282+
}
1283+
return nil
1284+
}, 60, 5).Should(BeNil())
12941285
})
12951286
By("verifying backup has resumed", func() {
12961287
Eventually(func() (int, error) {
@@ -1585,9 +1576,7 @@ func installAzCli(namespace string) {
15851576
}
15861577

15871578
func createAndAssertPgBouncerPoolerIsSetUp(namespace, poolerYamlFilePath string, expectedInstanceCount int) {
1588-
_, _, err := testsUtils.Run("kubectl create -n " + namespace + " -f " + poolerYamlFilePath)
1589-
Expect(err).ToNot(HaveOccurred())
1590-
1579+
CreateResourceFromFile(namespace, poolerYamlFilePath)
15911580
Eventually(func() (int32, error) {
15921581
poolerName, err := env.GetResourceNameFromYAML(poolerYamlFilePath)
15931582
Expect(err).ToNot(HaveOccurred())
@@ -1837,9 +1826,14 @@ func OnlineResizePVC(namespace, clusterName string) {
18371826
})
18381827
By("expanding Cluster storage", func() {
18391828
// Patching cluster to expand storage size from 1Gi to 2Gi
1840-
_, _, err := testsUtils.Run("kubectl patch cluster " + clusterName + " -n " + namespace +
1841-
" -p '{\"spec\":{\"storage\":{\"size\":\"2Gi\"}}}' --type=merge")
1842-
Expect(err).ToNot(HaveOccurred())
1829+
Eventually(func() error {
1830+
_, _, err := testsUtils.RunUnchecked("kubectl patch cluster " + clusterName + " -n " + namespace +
1831+
" -p '{\"spec\":{\"storage\":{\"size\":\"2Gi\"}}}' --type=merge")
1832+
if err != nil {
1833+
return err
1834+
}
1835+
return nil
1836+
}, 60, 5).Should(BeNil())
18431837
})
18441838
By("verifying Cluster storage is expanded", func() {
18451839
// Gathering and verifying the new size of PVC after update on cluster
@@ -1874,9 +1868,14 @@ func OfflineResizePVC(namespace, clusterName string, timeout int) {
18741868
})
18751869
By("expanding Cluster storage", func() {
18761870
// Expanding cluster storage
1877-
_, _, err := testsUtils.Run("kubectl patch cluster " + clusterName + " -n " + namespace +
1878-
" -p '{\"spec\":{\"storage\":{\"size\":\"2Gi\"}}}' --type=merge")
1879-
Expect(err).ToNot(HaveOccurred())
1871+
Eventually(func() error {
1872+
_, _, err := testsUtils.RunUnchecked("kubectl patch cluster " + clusterName + " -n " + namespace +
1873+
" -p '{\"spec\":{\"storage\":{\"size\":\"2Gi\"}}}' --type=merge")
1874+
if err != nil {
1875+
return err
1876+
}
1877+
return nil
1878+
}, 60, 5).Should(BeNil())
18801879
})
18811880
By("deleting Pod and pPVC", func() {
18821881
// Gathering cluster primary
@@ -2005,3 +2004,13 @@ func collectAndAssertDefaultMetricsPresentOnEachPod(namespace, clusterName strin
20052004
}
20062005
})
20072006
}
2007+
2008+
func CreateResourceFromFile(namespace, sampleFilePath string) {
2009+
Eventually(func() error {
2010+
_, _, err := testsUtils.RunUnchecked("kubectl apply -n " + namespace + " -f " + sampleFilePath)
2011+
if err != nil {
2012+
return err
2013+
}
2014+
return nil
2015+
}, 60, 5).Should(BeNil())
2016+
}

‎tests/e2e/cnp_certificates_test.go

Lines changed: 46 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -78,8 +78,7 @@ var _ = Describe("Certificates", func() {
7878
})
7979
AfterEach(func() {
8080
// deleting root CA certificates
81-
_, _, err := utils.Run(fmt.Sprintf("kubectl apply -n %v -f %v", namespace, sampleFile))
82-
Expect(err).ToNot(HaveOccurred())
81+
CreateResourceFromFile(namespace, sampleFile)
8382
})
8483

8584
It("can authenticate using a Certificate that is generated from the 'kubectl-cnp' plugin", func() {
@@ -114,12 +113,17 @@ var _ = Describe("Certificates", func() {
114113
)
115114
// Updating defaults certificates entries with user provided certificates,
116115
// i.e server CA and TLS secrets inside the cluster
117-
_, _, err = utils.Run(fmt.Sprintf(
118-
"kubectl patch cluster %v -n %v -p "+
119-
"'{\"spec\":{\"certificates\":{\"serverCASecret\":\"%v\","+
120-
"\"serverTLSSecret\":\"%v\"}}}'"+
121-
" --type='merge'", clusterName, namespace, serverCASecretName, serverCertSecretName))
122-
Expect(err).ToNot(HaveOccurred())
116+
Eventually(func() error {
117+
_, _, err = utils.RunUnchecked(fmt.Sprintf(
118+
"kubectl patch cluster %v -n %v -p "+
119+
"'{\"spec\":{\"certificates\":{\"serverCASecret\":\"%v\","+
120+
"\"serverTLSSecret\":\"%v\"}}}'"+
121+
" --type='merge'", clusterName, namespace, serverCASecretName, serverCertSecretName))
122+
if err != nil {
123+
return err
124+
}
125+
return nil
126+
}, 60, 5).Should(BeNil())
123127
Eventually(func() (bool, error) {
124128
certUpdateStatus := false
125129
cluster := &apiv1.Cluster{}
@@ -154,48 +158,58 @@ var _ = Describe("Certificates", func() {
154158

155159
// Updating defaults certificates entries with user provided certificates,
156160
// i.e client CA and TLS secrets inside the cluster
157-
_, _, err := utils.Run(fmt.Sprintf(
158-
"kubectl patch cluster %v -n %v -p "+
159-
"'{\"spec\":{\"certificates\":{\"clientCASecret\":\"%v\","+
160-
"\"replicationTLSSecret\":\"%v\"}}}'"+
161-
" --type='merge'", clusterName, namespace, clientCASecretName, replicaCertSecretName))
162-
Expect(err).ToNot(HaveOccurred())
161+
Eventually(func() error {
162+
_, _, err := utils.RunUnchecked(fmt.Sprintf(
163+
"kubectl patch cluster %v -n %v -p "+
164+
"'{\"spec\":{\"certificates\":{\"clientCASecret\":\"%v\","+
165+
"\"replicationTLSSecret\":\"%v\"}}}'"+
166+
" --type='merge'", clusterName, namespace, clientCASecretName, replicaCertSecretName))
167+
if err != nil {
168+
return err
169+
}
170+
return nil
171+
}, 60, 5).Should(BeNil())
163172

164173
Eventually(func() (bool, error) {
165174
cluster := &apiv1.Cluster{}
166-
err = env.Client.Get(env.Ctx, ctrlclient.ObjectKey{Namespace: namespace, Name: clusterName}, cluster)
175+
err := env.Client.Get(env.Ctx, ctrlclient.ObjectKey{Namespace: namespace, Name: clusterName}, cluster)
167176

168177
return cluster.Spec.Certificates.ClientCASecret == clientCASecretName &&
169178
cluster.Status.Certificates.ReplicationTLSSecret == replicaCertSecretName, err
170179
}, 120, 5).Should(BeTrue())
171180

172181
pod := utils.DefaultWebapp(namespace, "app-pod-cert-3", defaultCASecretName, clientCertSecretName)
173-
err = utils.PodCreateAndWaitForReady(env, &pod, 240)
182+
err := utils.PodCreateAndWaitForReady(env, &pod, 240)
174183
Expect(err).ToNot(HaveOccurred())
175184
AssertSSLVerifyFullDBConnectionFromAppPod(namespace, clusterName, pod)
176185
})
177186

178187
It("can connect after switching both server and client certificates to user-supplied mode", func() {
179188
// Updating defaults certificates entries with user provided certificates,
180189
// i.e server and client CA and TLS secrets inside the cluster
181-
_, _, err := utils.Run(fmt.Sprintf(
182-
"kubectl patch cluster %v -n %v -p "+
183-
"'{\"spec\":{\"certificates\":{\"serverCASecret\":\"%v\","+
184-
"\"serverTLSSecret\":\"%v\",\"clientCASecret\":\"%v\","+
185-
"\"replicationTLSSecret\":\"%v\"}}}'"+
186-
" --type='merge'",
187-
clusterName,
188-
namespace,
189-
serverCASecretName,
190-
serverCertSecretName,
191-
clientCASecretName,
192-
replicaCertSecretName,
193-
))
194-
Expect(err).ToNot(HaveOccurred())
190+
Eventually(func() error {
191+
_, _, err := utils.RunUnchecked(fmt.Sprintf(
192+
"kubectl patch cluster %v -n %v -p "+
193+
"'{\"spec\":{\"certificates\":{\"serverCASecret\":\"%v\","+
194+
"\"serverTLSSecret\":\"%v\",\"clientCASecret\":\"%v\","+
195+
"\"replicationTLSSecret\":\"%v\"}}}'"+
196+
" --type='merge'",
197+
clusterName,
198+
namespace,
199+
serverCASecretName,
200+
serverCertSecretName,
201+
clientCASecretName,
202+
replicaCertSecretName,
203+
))
204+
if err != nil {
205+
return err
206+
}
207+
return nil
208+
}, 60, 5).Should(BeNil())
195209

196210
Eventually(func() (bool, error) {
197211
cluster := &apiv1.Cluster{}
198-
err = env.Client.Get(env.Ctx, ctrlclient.ObjectKey{Namespace: namespace, Name: clusterName}, cluster)
212+
err := env.Client.Get(env.Ctx, ctrlclient.ObjectKey{Namespace: namespace, Name: clusterName}, cluster)
199213

200214
return cluster.Status.Certificates.ServerCASecret == serverCASecretName &&
201215
cluster.Status.Certificates.ClientCASecret == clientCASecretName &&
@@ -204,7 +218,7 @@ var _ = Describe("Certificates", func() {
204218
}, 120, 5).Should(BeTrue())
205219

206220
pod := utils.DefaultWebapp(namespace, "app-pod-cert-4", serverCASecretName, clientCertSecretName)
207-
err = utils.PodCreateAndWaitForReady(env, &pod, 240)
221+
err := utils.PodCreateAndWaitForReady(env, &pod, 240)
208222
Expect(err).ToNot(HaveOccurred())
209223
AssertSSLVerifyFullDBConnectionFromAppPod(namespace, clusterName, pod)
210224
})

‎tests/e2e/configuration_update_test.go

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,7 @@ var _ = Describe("Configuration update", func() {
5858
podList, err := env.GetClusterPodList(namespace, clusterName)
5959
Expect(err).ToNot(HaveOccurred())
6060
// Update the configuration
61-
_, _, err = utils.Run("kubectl apply -n " + namespace + " -f " + sample)
62-
Expect(err).ToNot(HaveOccurred())
61+
CreateResourceFromFile(namespace, sample)
6362
timeout := 60
6463
commandtimeout := time.Second * 2
6564
// Check that the parameter has been modified in every pod
@@ -85,8 +84,7 @@ var _ = Describe("Configuration update", func() {
8584
"psql", "-U", "postgres", "-h", endpointName, "-tAc", "select 1")
8685
Expect(err).To(HaveOccurred())
8786
// Update the configuration
88-
_, _, err = utils.Run("kubectl apply -n " + namespace + " -f " + sample)
89-
Expect(err).ToNot(HaveOccurred())
87+
CreateResourceFromFile(namespace, sample)
9088
// The new pg_hba rule should be present in every pod
9189
for _, pod := range podList.Items {
9290
pod := pod // pin the variable
@@ -119,8 +117,7 @@ var _ = Describe("Configuration update", func() {
119117
Expect(cluster.Status.CurrentPrimary, err).To(BeEquivalentTo(cluster.Status.TargetPrimary))
120118
oldPrimary := cluster.Status.CurrentPrimary
121119
// Update the configuration
122-
_, _, err = utils.Run("kubectl apply -n " + namespace + " -f " + sample)
123-
Expect(err).ToNot(HaveOccurred())
120+
CreateResourceFromFile(namespace, sample)
124121
timeout := 300
125122
commandtimeout := time.Second * 2
126123
// Check that the new parameter has been modified in every pod
@@ -154,8 +151,7 @@ var _ = Describe("Configuration update", func() {
154151
Expect(cluster.Status.CurrentPrimary, err).To(BeEquivalentTo(cluster.Status.TargetPrimary))
155152
oldPrimary := cluster.Status.CurrentPrimary
156153
// Update the configuration
157-
_, _, err = utils.Run("kubectl apply -n " + namespace + " -f " + sample)
158-
Expect(err).ToNot(HaveOccurred())
154+
CreateResourceFromFile(namespace, sample)
159155
timeout := 300
160156
commandtimeout := time.Second * 2
161157
// Check that both parameters have been modified in each pod

‎tests/e2e/fastswitchover_test.go

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -79,11 +79,8 @@ var _ = Describe("Fast switchover", Serial, Label(tests.LabelPerformance), func(
7979
return namespaceResource.GetName(), err
8080
}, timeout).Should(BeEquivalentTo(namespace))
8181
})
82-
By(fmt.Sprintf("creating a Cluster in the %v namespace",
83-
namespace), func() {
84-
_, _, err := utils.Run(
85-
"kubectl create -n " + namespace + " -f " + sampleFile)
86-
Expect(err).ToNot(HaveOccurred())
82+
By(fmt.Sprintf("creating a Cluster in the %v namespace", namespace), func() {
83+
CreateResourceFromFile(namespace, sampleFile)
8784
})
8885
By("having a Cluster with three instances ready", func() {
8986
AssertClusterIsReady(namespace, clusterName, 600, env)

‎tests/e2e/nodeselector_test.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,7 @@ var _ = Describe("nodeSelector", func() {
6363
})
6464

6565
By(fmt.Sprintf("creating a cluster in the %v namespace", namespace), func() {
66-
_, _, err := utils.Run("kubectl create -n " + namespace + " -f " + sampleFile)
67-
Expect(err).ToNot(HaveOccurred())
66+
CreateResourceFromFile(namespace, sampleFile)
6867
})
6968

7069
// The cluster should be created but the pods shouldn't be scheduled

‎tests/e2e/replica_mode_cluster_test.go

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -85,11 +85,16 @@ var _ = Describe("Replica Mode", func() {
8585
replicaClusterName, replicaClusterSampleBasicAuth, checkQuery)
8686

8787
By("disabling the replica mode", func() {
88-
_, _, err = utils.Run(fmt.Sprintf(
89-
"kubectl patch cluster %v -n %v -p '{\"spec\":{\"replica\":{\"enabled\":false}}}'"+
90-
" --type='merge'",
91-
replicaClusterName, replicaNamespace))
92-
Expect(err).ToNot(HaveOccurred())
88+
Eventually(func() error {
89+
_, _, err = utils.RunUnchecked(fmt.Sprintf(
90+
"kubectl patch cluster %v -n %v -p '{\"spec\":{\"replica\":{\"enabled\":false}}}'"+
91+
" --type='merge'",
92+
replicaClusterName, replicaNamespace))
93+
if err != nil {
94+
return err
95+
}
96+
return nil
97+
}, 60, 5).Should(BeNil())
9398
})
9499

95100
By("verifying write operation on the replica cluster primary pod", func() {

0 commit comments

Comments
 (0)