Skip to content

Commit 1435c94

Browse files
wadlejitendraJitendra Wadle
wadlejitendra
and
Jitendra Wadle
authored
test: add E2E test for backup and restore using endpoint CA
Added test case for backup and restore with object storage using TLS, only covering `.spec.bootstrap.recovery.backup.endpointCA` for recovery, still to be validated through an E2E test: - endpointCA specified in Backup Status - endpointCA specified in externalCluster Co-authored-by: Jitendra Wadle <jitendrawadle@Laptop566-pn-in.local>
1 parent e2afcd1 commit 1435c94

8 files changed

+268
-19
lines changed

‎tests/e2e/asserts_test.go

+6-7
Original file line numberDiff line numberDiff line change
@@ -477,12 +477,11 @@ func AssertStorageCredentialsAreCreated(namespace string, name string, id string
477477
}
478478

479479
// InstallMinio installs minio to verify the backup and archive walls
480-
func InstallMinio(namespace string) {
480+
func InstallMinio(namespace string, deploymentFile string) {
481481
// Create a PVC-based deployment for the minio version
482482
// minio/minio:RELEASE.2020-04-23T00-58-49Z
483483
minioPVCFile := fixturesDir + "/backup/minio/minio-pvc.yaml"
484-
minioDeploymentFile := fixturesDir +
485-
"/backup/minio/minio-deployment.yaml"
484+
minioDeploymentFile := fixturesDir + deploymentFile
486485

487486
_, _, err := testsUtils.Run(fmt.Sprintf("kubectl apply -n %v -f %v",
488487
namespace, minioPVCFile))
@@ -509,8 +508,8 @@ func InstallMinio(namespace string) {
509508
}
510509

511510
// InstallMinioClient installs minio client to verify the backup and archive walls
512-
func InstallMinioClient(namespace string) {
513-
clientFile := fixturesDir + "/backup/minio/minio-client.yaml"
511+
func InstallMinioClient(namespace string, minioClientPodFile string) {
512+
clientFile := fixturesDir + minioClientPodFile
514513
_, _, err := testsUtils.Run(fmt.Sprintf(
515514
"kubectl apply -n %v -f %v",
516515
namespace, clientFile))
@@ -1403,13 +1402,13 @@ func prepareClusterForPITROnMinio(
14031402
})
14041403

14051404
By("setting up minio", func() {
1406-
InstallMinio(namespace)
1405+
InstallMinio(namespace, "/backup/minio/minio-deployment.yaml")
14071406
})
14081407

14091408
// Create the minio client pod and wait for it to be ready.
14101409
// We'll use it to check if everything is archived correctly
14111410
By("setting up minio client pod", func() {
1412-
InstallMinioClient(namespace)
1411+
InstallMinioClient(namespace, "/backup/minio/minio-client.yaml")
14131412
})
14141413

14151414
// Create the cluster

‎tests/e2e/backup_restore_test.go

+94-10
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
ctrlclient "sigs.k8s.io/controller-runtime/pkg/client"
1313

1414
apiv1 "github.com/EnterpriseDB/cloud-native-postgresql/api/v1"
15+
"github.com/EnterpriseDB/cloud-native-postgresql/pkg/certs"
1516
"github.com/EnterpriseDB/cloud-native-postgresql/tests"
1617
"github.com/EnterpriseDB/cloud-native-postgresql/tests/utils"
1718

@@ -84,13 +85,13 @@ var _ = Describe("Backup and restore", func() {
8485
})
8586

8687
By("setting up minio", func() {
87-
InstallMinio(namespace)
88+
InstallMinio(namespace, "/backup/minio/minio-deployment.yaml")
8889
})
8990

9091
// Create the minio client pod and wait for it to be ready.
9192
// We'll use it to check if everything is archived correctly
9293
By("setting up minio client pod", func() {
93-
InstallMinioClient(namespace)
94+
InstallMinioClient(namespace, "/backup/minio/minio-client.yaml")
9495
})
9596

9697
// Create ConfigMap and secrets to verify metrics for target database after backup restore
@@ -168,13 +169,13 @@ var _ = Describe("Backup and restore", func() {
168169
})
169170

170171
By("setting up minio", func() {
171-
InstallMinio(namespace)
172+
InstallMinio(namespace, "/backup/minio/minio-deployment.yaml")
172173
})
173174

174175
// Create the minio client pod and wait for it to be ready.
175176
// We'll use it to check if everything is archived correctly
176177
By("setting up minio client pod", func() {
177-
InstallMinioClient(namespace)
178+
InstallMinioClient(namespace, "/backup/minio/minio-client.yaml")
178179
})
179180

180181
AssertCreateCluster(namespace, clusterName, clusterWithMinioSampleFile, env)
@@ -207,13 +208,13 @@ var _ = Describe("Backup and restore", func() {
207208
})
208209

209210
By("setting up minio", func() {
210-
InstallMinio(namespace)
211+
InstallMinio(namespace, "/backup/minio/minio-deployment.yaml")
211212
})
212213

213214
// Create the minio client pod and wait for it to be ready.
214215
// We'll use it to check if everything is archived correctly
215216
By("setting up minio client pod", func() {
216-
InstallMinioClient(namespace)
217+
InstallMinioClient(namespace, "/backup/minio/minio-client.yaml")
217218
})
218219

219220
AssertCreateCluster(namespace, clusterName, clusterWithMinioSampleFile, env)
@@ -246,6 +247,89 @@ var _ = Describe("Backup and restore", func() {
246247

247248
AssertClusterRestorePITR(namespace, restoredClusterName, tableName)
248249
})
250+
251+
It("backup and restore with endpoint ca and tls connection", func() {
252+
const (
253+
clusterWithMinioSampleFile = fixturesDir + "/backup/minio-with-tls/cluster-with-backup-minio.yaml"
254+
clusterRestoreSampleFile = fixturesDir + "/backup/minio-with-tls/cluster-from-restore.yaml"
255+
caSecName = "minio-server-ca-secret"
256+
tlsSecName = "minio-server-tls-secret"
257+
)
258+
namespace = "backup-minio-endpoint-ca"
259+
clusterName, err := env.GetResourceNameFromYAML(clusterWithMinioSampleFile)
260+
Expect(err).ToNot(HaveOccurred())
261+
// create namespace
262+
err = env.CreateNamespace(namespace)
263+
Expect(err).ToNot(HaveOccurred())
264+
265+
// create CA certificates
266+
_, caPair := utils.CreateSecretCA(namespace, clusterName, caSecName, true, env)
267+
268+
// sign and create secret using CA certificate and key
269+
serverPair, err := caPair.CreateAndSignPair("minio-service", certs.CertTypeServer,
270+
[]string{"minio-service.internal.mydomain.net, minio-service.default.svc, minio-service.default,"},
271+
)
272+
Expect(err).ToNot(HaveOccurred())
273+
serverSecret := serverPair.GenerateCertificateSecret(namespace, tlsSecName)
274+
err = env.Client.Create(env.Ctx, serverSecret)
275+
Expect(err).ToNot(HaveOccurred())
276+
277+
By("creating the credentials for minio", func() {
278+
AssertStorageCredentialsAreCreated(namespace, "backup-storage-creds", "minio", "minio123")
279+
})
280+
281+
By("setting up minio", func() {
282+
InstallMinio(namespace, "/backup/minio-with-tls/minio-deployment.yaml")
283+
})
284+
285+
// Create the minio client pod and wait for it to be ready.
286+
// We'll use it to check if everything is archived correctly
287+
By("setting up minio client pod", func() {
288+
InstallMinioClient(namespace, "/backup/minio-with-tls/minio-client.yaml")
289+
})
290+
291+
// Create the cluster
292+
AssertCreateCluster(namespace, clusterName, clusterWithMinioSampleFile, env)
293+
294+
// Write a table and some data on the "app" database
295+
AssertCreateTestData(namespace, clusterName, "test_table")
296+
297+
AssertArchiveWalOnMinio(namespace, clusterName)
298+
299+
// There should be a backup resource and
300+
By("backing up a cluster and verifying it exists on minio", func() {
301+
utils.ExecuteBackup(namespace, backupFile, env)
302+
303+
Eventually(func() (int, error) {
304+
return CountFilesOnMinio(namespace, "data.tar")
305+
}, 30).Should(BeEquivalentTo(1))
306+
Eventually(func() (string, error) {
307+
cluster := &apiv1.Cluster{}
308+
err := env.Client.Get(env.Ctx,
309+
ctrlclient.ObjectKey{Namespace: namespace, Name: clusterName},
310+
cluster)
311+
return cluster.Status.FirstRecoverabilityPoint, err
312+
}, 30).ShouldNot(BeEmpty())
313+
})
314+
315+
// Restore backup in a new cluster
316+
AssertClusterRestore(namespace, clusterRestoreSampleFile, "test_table")
317+
318+
previous := 0
319+
320+
By("checking the previous number of .history files in minio", func() {
321+
previous, err = CountFilesOnMinio(namespace, "*.history.gz")
322+
Expect(err).ToNot(HaveOccurred())
323+
})
324+
325+
AssertSwitchover(namespace, clusterName, env)
326+
327+
By("checking the number of .history after switchover", func() {
328+
Eventually(func() (int, error) {
329+
return CountFilesOnMinio(namespace, "*.history.gz")
330+
}, 60).Should(BeNumerically(">", previous))
331+
})
332+
})
249333
})
250334

251335
Context("using azure blobs as object storage with storage account access authentication", func() {
@@ -589,13 +673,13 @@ var _ = Describe("Clusters Recovery From Barman Object Store", func() {
589673
AssertStorageCredentialsAreCreated(namespace, "backup-storage-creds", "minio", "minio123")
590674
})
591675
By("setting up minio", func() {
592-
InstallMinio(namespace)
676+
InstallMinio(namespace, "/backup/minio/minio-deployment.yaml")
593677
})
594678

595679
// Create the minio client pod and wait for it to be ready.
596680
// We'll use it to check if everything is archived correctly
597681
By("setting up minio client pod", func() {
598-
InstallMinioClient(namespace)
682+
InstallMinioClient(namespace, "/backup/minio/minio-client.yaml")
599683
})
600684

601685
// Create the cluster
@@ -659,13 +743,13 @@ var _ = Describe("Clusters Recovery From Barman Object Store", func() {
659743
AssertStorageCredentialsAreCreated(namespace, "backup-storage-creds", "minio", "minio123")
660744
})
661745
By("setting up minio", func() {
662-
InstallMinio(namespace)
746+
InstallMinio(namespace, "/backup/minio/minio-deployment.yaml")
663747
})
664748

665749
// Create the minio client pod and wait for it to be ready.
666750
// We'll use it to check if everything is archived correctly.
667751
By("setting up minio client pod", func() {
668-
InstallMinioClient(namespace)
752+
InstallMinioClient(namespace, "/backup/minio/minio-client.yaml")
669753
})
670754

671755
// Create the Cluster

‎tests/e2e/fixtures/backup/minio-with-tls/.gitignore

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
cluster-with-backup-minio.yaml
2+
cluster-from-restore.yaml

‎tests/e2e/fixtures/backup/minio-with-tls/cluster-from-restore.yaml.template

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
apiVersion: postgresql.k8s.enterprisedb.io/v1
2+
kind: Cluster
3+
metadata:
4+
name: cluster-restore
5+
spec:
6+
instances: 2
7+
8+
postgresql:
9+
parameters:
10+
log_checkpoints: "on"
11+
log_lock_waits: "on"
12+
log_min_duration_statement: '1000'
13+
log_statement: 'ddl'
14+
log_temp_files: '1024'
15+
log_autovacuum_min_duration: '1s'
16+
log_replication_commands: 'on'
17+
18+
storage:
19+
size: 1Gi
20+
storageClass: ${E2E_DEFAULT_STORAGE_CLASS}
21+
22+
bootstrap:
23+
recovery:
24+
backup:
25+
name: cluster-backup
26+
endpointCA:
27+
key: ca.crt
28+
name: minio-server-ca-secret

‎tests/e2e/fixtures/backup/minio-with-tls/cluster-with-backup-minio.yaml.template

+54
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
apiVersion: postgresql.k8s.enterprisedb.io/v1
2+
kind: Cluster
3+
metadata:
4+
name: pg-backup-minio
5+
spec:
6+
instances: 2
7+
8+
postgresql:
9+
parameters:
10+
log_checkpoints: "on"
11+
log_lock_waits: "on"
12+
log_min_duration_statement: '1000'
13+
log_statement: 'ddl'
14+
log_temp_files: '1024'
15+
log_autovacuum_min_duration: '1s'
16+
log_replication_commands: 'on'
17+
18+
# Example of rolling update strategy:
19+
# - unsupervised: automated update of the primary once all
20+
# replicas have been upgraded (default)
21+
# - supervised: requires manual supervision to perform
22+
# the switchover of the primary
23+
primaryUpdateStrategy: unsupervised
24+
25+
# Persistent storage configuration
26+
storage:
27+
storageClass: ${E2E_DEFAULT_STORAGE_CLASS}
28+
size: 1Gi
29+
30+
bootstrap:
31+
initdb:
32+
database: app
33+
owner: app
34+
35+
backup:
36+
barmanObjectStore:
37+
destinationPath: s3://cluster-backups/
38+
endpointURL: https://minio-service:9000
39+
# Name field will be tls secret name
40+
endpointCA:
41+
key: ca.crt
42+
name: minio-server-ca-secret
43+
s3Credentials:
44+
accessKeyId:
45+
name: backup-storage-creds
46+
key: ID
47+
secretAccessKey:
48+
name: backup-storage-creds
49+
key: KEY
50+
wal:
51+
compression: gzip
52+
data:
53+
immediateCheckpoint: true
54+
retentionPolicy: "30d"

‎tests/e2e/fixtures/backup/minio-with-tls/minio-client.yaml

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
apiVersion: v1
2+
kind: Pod
3+
metadata:
4+
labels:
5+
run: mc
6+
name: mc
7+
spec:
8+
containers:
9+
- env:
10+
- name: MC_HOST_minio
11+
value: https://minio:minio123@minio-service:9000
12+
image: minio/mc
13+
name: mc
14+
volumeMounts:
15+
# ca certificate path
16+
- name: secret-volume
17+
mountPath: /root/.mc/certs/CAs
18+
resources: {}
19+
# Keep the pod up to exec stuff on it
20+
command:
21+
- sleep
22+
- "3600"
23+
dnsPolicy: ClusterFirst
24+
restartPolicy: Always
25+
volumes:
26+
- name: secret-volume
27+
secret:
28+
defaultMode: 0600
29+
secretName: minio-server-ca-secret

‎tests/e2e/fixtures/backup/minio-with-tls/minio-deployment.yaml

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: minio
5+
labels:
6+
app: minio
7+
spec:
8+
replicas: 1
9+
selector:
10+
matchLabels:
11+
app: minio
12+
template:
13+
metadata:
14+
labels:
15+
app: minio
16+
spec:
17+
containers:
18+
- name: minio
19+
image: minio/minio:RELEASE.2020-04-23T00-58-49Z
20+
ports:
21+
- containerPort: 9000
22+
volumeMounts:
23+
- name: data
24+
mountPath: /data
25+
- name: secret-volume
26+
mountPath: /etc/secrets/certs
27+
command: ["minio"]
28+
args: ["--certs-dir", "/etc/secrets/certs", "server", "/data"]
29+
env:
30+
- name: MINIO_ACCESS_KEY
31+
value: "minio"
32+
- name: MINIO_SECRET_KEY
33+
value: "minio123"
34+
volumes:
35+
- name: data
36+
persistentVolumeClaim:
37+
claimName: minio-pv-claim
38+
- name: secret-volume
39+
projected:
40+
defaultMode: 0600
41+
sources:
42+
- secret:
43+
name: minio-server-tls-secret
44+
items:
45+
- key: tls.crt
46+
path: public.crt
47+
- key: tls.key
48+
path: private.key
49+
- secret:
50+
name: minio-server-ca-secret
51+
items:
52+
- key: ca.crt
53+
path: CAs/public.crt

0 commit comments

Comments
 (0)