Skip to content

Commit dd79ff4

Browse files
wadlejitendraarmrugabriele-wolfoxfcanovai
authored
test(pgbouncer tls connection): add E2E test for PgBouncer and TLS connections
Connect to Postgres via PgBouncer against a cluster with different client and server CA. Co-authored-by: Armando Ruocco <armando.ruocco@enterprisedb.com> Co-authored-by: Gabriele Quaresima <gabriele.quaresima@enterprisedb.com> Co-authored-by: Francesco Canovai <francesco.canovai@enterprisedb.com>
1 parent 05163da commit dd79ff4

File tree

6 files changed

+149
-24
lines changed

6 files changed

+149
-24
lines changed

‎tests/e2e/cnp_certificates_test.go

Lines changed: 43 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"time"
1212

1313
corev1 "k8s.io/api/core/v1"
14+
apierrors "k8s.io/apimachinery/pkg/api/errors"
1415
"k8s.io/apimachinery/pkg/types"
1516
"sigs.k8s.io/controller-runtime/pkg/client"
1617

@@ -85,7 +86,7 @@ var _ = Describe("Certificates", func() {
8586

8687
AssertDBConnectionFromAppPod(namespace, clusterName, sampleAppFile, appPod)
8788

88-
AssertCertificatesSecrets(namespace, clusterName, caSecName, tlsSecName, certs.CertTypeServer)
89+
createAndAssertCertificatesSecrets(namespace, clusterName, caSecName, tlsSecName, certs.CertTypeServer, false)
8990

9091
By("switching to user-supplied server certificates", func() {
9192
// Updating defaults certificates entries with user provided certificates,
@@ -124,7 +125,8 @@ var _ = Describe("Certificates", func() {
124125
AssertCreateCluster(namespace, clusterName, sampleFile, env)
125126

126127
// Create certificates secret for client
127-
AssertCertificatesSecrets(namespace, clusterName, caSecNameClient, tlsSecNameClient, certs.CertTypeClient)
128+
createAndAssertCertificatesSecrets(namespace, clusterName, caSecNameClient, tlsSecNameClient,
129+
certs.CertTypeClient, false)
128130

129131
By("switching to user-supplied client certificates", func() {
130132
// Updating defaults certificates entries with user provided certificates,
@@ -162,9 +164,10 @@ var _ = Describe("Certificates", func() {
162164
// Create cluster
163165
AssertCreateCluster(namespace, clusterName, sampleFile, env)
164166
// Create certificates secret for server
165-
AssertCertificatesSecrets(namespace, clusterName, caSecName, tlsSecName, certs.CertTypeServer)
167+
createAndAssertCertificatesSecrets(namespace, clusterName, caSecName, tlsSecName, certs.CertTypeServer, false)
166168
// Create certificates secret for client
167-
AssertCertificatesSecrets(namespace, clusterName, caSecNameClient, tlsSecNameClient, certs.CertTypeClient)
169+
createAndAssertCertificatesSecrets(namespace, clusterName, caSecNameClient, tlsSecNameClient,
170+
certs.CertTypeClient, false)
168171

169172
By("switching to user-supplied server and client certificates", func() {
170173
// Updating defaults certificates entries with user provided certificates,
@@ -219,7 +222,7 @@ var _ = Describe("Certificates", func() {
219222
// Create a cluster in a namespace that will be deleted after the test
220223
err := env.CreateNamespace(namespace)
221224
Expect(err).ToNot(HaveOccurred())
222-
AssertCertificatesSecrets(namespace, clusterName, caSecName, tlsSecName, certs.CertTypeServer)
225+
createAndAssertCertificatesSecrets(namespace, clusterName, caSecName, tlsSecName, certs.CertTypeServer, false)
223226
AssertCreateCluster(namespace, clusterName, sampleFile, env)
224227
AssertClientCertificatesSecretsUsingCnpPlugin(namespace, clusterName)
225228
AssertDBConnectionFromAppPod(namespace, clusterName, sampleAppFileUserSuppliedCert, appPodUserSuppliedCert)
@@ -251,7 +254,8 @@ var _ = Describe("Certificates", func() {
251254
Expect(err).ToNot(HaveOccurred())
252255

253256
// Create certificates secret for client
254-
AssertCertificatesSecrets(namespace, clusterName, caSecNameClient, tlsSecNameClient, certs.CertTypeClient)
257+
createAndAssertCertificatesSecrets(namespace, clusterName, caSecNameClient, tlsSecNameClient,
258+
certs.CertTypeClient, false)
255259
AssertCreateCluster(namespace, clusterName, sampleFile, env)
256260
AssertDBConnectionFromAppPod(namespace, clusterName, sampleAppFileUserSuppliedCertClient, appPodUserSuppliedCert)
257261
})
@@ -282,34 +286,26 @@ var _ = Describe("Certificates", func() {
282286
Expect(err).ToNot(HaveOccurred())
283287

284288
// Create certificates secret for server
285-
AssertCertificatesSecrets(namespace, clusterName, caSecName, tlsSecName, certs.CertTypeServer)
289+
createAndAssertCertificatesSecrets(namespace, clusterName, caSecName, tlsSecName, certs.CertTypeServer, false)
286290

287291
// Create certificates secret for client
288-
AssertCertificatesSecrets(namespace, clusterName, caSecNameClient, tlsSecNameClient, certs.CertTypeClient)
292+
createAndAssertCertificatesSecrets(namespace, clusterName, caSecNameClient, tlsSecNameClient,
293+
certs.CertTypeClient, false)
289294
AssertCreateCluster(namespace, clusterName, sampleFile, env)
290295
AssertDBConnectionFromAppPod(namespace, clusterName, sampleUserSuppliedCertClientServer, appPodUserSuppliedCert)
291296
})
292297
})
293298
})
294299

295-
func AssertCertificatesSecrets(namespace, clusterName, caSecName, tlsSecName, certType string) {
296-
// creating root CA certificates
297-
cluster := &apiv1.Cluster{}
298-
cluster.Namespace = namespace
299-
cluster.Name = clusterName
300-
secret := &corev1.Secret{}
301-
err := env.Client.Get(env.Ctx, client.ObjectKey{Namespace: namespace, Name: caSecName}, secret)
302-
Expect(err).To(HaveOccurred())
303-
304-
caPair, err := certs.CreateRootCA(cluster.Name, namespace)
305-
Expect(err).ToNot(HaveOccurred())
300+
func createAndAssertCertificatesSecrets(
301+
namespace, clusterName, caSecName, tlsSecName, certType string, includeCAPrivateKey bool) {
302+
cluster, caPair := createSecretCA(namespace, clusterName, caSecName, includeCAPrivateKey)
306303

307-
caSecret := caPair.GenerateCASecret(namespace, caSecName)
308-
// delete the key from the CA, as it is not needed in this case
309-
delete(caSecret.Data, certs.CAPrivateKeyKey)
310-
err = env.Client.Create(env.Ctx, caSecret)
311-
Expect(err).ToNot(HaveOccurred())
304+
assertCACertificateCreation(namespace, certType, caPair, cluster, tlsSecName)
305+
}
312306

307+
func assertCACertificateCreation(namespace string, certType string, caPair *certs.KeyPair,
308+
cluster *apiv1.Cluster, tlsSecName string) {
313309
if certType == certs.CertTypeServer {
314310
By("creating server TLS certificate", func() {
315311
serverPair, err := caPair.CreateAndSignPair(cluster.GetServiceReadWriteName(), certs.CertTypeServer,
@@ -342,6 +338,29 @@ func AssertCertificatesSecrets(namespace, clusterName, caSecName, tlsSecName, ce
342338
}
343339
}
344340

341+
func createSecretCA(namespace string, clusterName string, caSecName string, includeCAPrivateKey bool) (
342+
*apiv1.Cluster, *certs.KeyPair) {
343+
// creating root CA certificates
344+
cluster := &apiv1.Cluster{}
345+
cluster.Namespace = namespace
346+
cluster.Name = clusterName
347+
secret := &corev1.Secret{}
348+
err := env.Client.Get(env.Ctx, client.ObjectKey{Namespace: namespace, Name: caSecName}, secret)
349+
Expect(apierrors.IsNotFound(err)).To(BeTrue())
350+
351+
caPair, err := certs.CreateRootCA(cluster.Name, namespace)
352+
Expect(err).ToNot(HaveOccurred())
353+
354+
caSecret := caPair.GenerateCASecret(namespace, caSecName)
355+
// delete the key from the CA, as it is not needed in this case
356+
if !includeCAPrivateKey {
357+
delete(caSecret.Data, certs.CAPrivateKeyKey)
358+
}
359+
err = env.Client.Create(env.Ctx, caSecret)
360+
Expect(err).ToNot(HaveOccurred())
361+
return cluster, caPair
362+
}
363+
345364
func AssertClientCertificatesSecretsUsingCnpPlugin(namespace, clusterName string) {
346365
clientCertName := "cluster-cert"
347366
By("creating a client Certificate using the 'kubectl-cnp' plugin", func() {

‎tests/e2e/fixtures/pgbouncer/pgbouncer_separate_client_server_ca/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
cluster-user-supplied-client-server-certificates.yaml

‎tests/e2e/fixtures/pgbouncer/pgbouncer_separate_client_server_ca/cluster-user-supplied-client-server-certificates.yaml.template

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
apiVersion: postgresql.k8s.enterprisedb.io/v1
2+
kind: Cluster
3+
metadata:
4+
name: pgbouncer-separate-client-server-ca
5+
spec:
6+
instances: 3
7+
8+
postgresql:
9+
parameters:
10+
work_mem: "8MB"
11+
log_checkpoints: "on"
12+
log_lock_waits: "on"
13+
log_min_duration_statement: '1000'
14+
log_statement: 'ddl'
15+
log_temp_files: '1024'
16+
log_autovacuum_min_duration: '1s'
17+
log_replication_commands: 'on'
18+
19+
certificates:
20+
serverCASecret: my-postgresql-server-ca
21+
serverTLSSecret: my-postgresql-server
22+
clientCASecret: my-postgresql-client-ca
23+
replicationTLSSecret: my-postgresql-client
24+
25+
bootstrap:
26+
initdb:
27+
database: app
28+
owner: app
29+
30+
# Persistent storage configuration
31+
storage:
32+
storageClass: ${E2E_DEFAULT_STORAGE_CLASS}
33+
size: 1Gi

‎tests/e2e/fixtures/pgbouncer/pgbouncer_separate_client_server_ca/pgbouncer-pooler-tls-ro.yaml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: postgresql.k8s.enterprisedb.io/v1
2+
kind: Pooler
3+
metadata:
4+
name: pooler-connection-tls-ro
5+
spec:
6+
cluster:
7+
name: pgbouncer-separate-client-server-ca
8+
9+
instances: 1
10+
type: ro
11+
pgbouncer:
12+
poolMode: session

‎tests/e2e/fixtures/pgbouncer/pgbouncer_separate_client_server_ca/pgbouncer-pooler-tls-rw.yaml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: postgresql.k8s.enterprisedb.io/v1
2+
kind: Pooler
3+
metadata:
4+
name: pooler-connection-tls-rw
5+
spec:
6+
cluster:
7+
name: pgbouncer-separate-client-server-ca
8+
9+
instances: 1
10+
type: rw
11+
pgbouncer:
12+
poolMode: session

‎tests/e2e/pgbouncer_test.go

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
appsv1 "k8s.io/api/apps/v1"
1313
"sigs.k8s.io/controller-runtime/pkg/client"
1414

15+
"github.com/EnterpriseDB/cloud-native-postgresql/pkg/certs"
1516
"github.com/EnterpriseDB/cloud-native-postgresql/pkg/specs/pgbouncer"
1617
"github.com/EnterpriseDB/cloud-native-postgresql/pkg/utils"
1718
"github.com/EnterpriseDB/cloud-native-postgresql/tests"
@@ -104,6 +105,53 @@ var _ = Describe("PGBouncer Connections", func() {
104105
poolerCertificateROSampleFile, false)
105106
})
106107
})
108+
109+
It("can connect to Postgres via pgbouncer against a cluster with separate client and server CA", func() {
110+
const (
111+
folderPath = fixturesDir + "/pgbouncer/pgbouncer_separate_client_server_ca/"
112+
sampleFileWithCertificate = folderPath + "cluster-user-supplied-client-server-certificates.yaml"
113+
poolerCertificateROSampleFile = folderPath + "pgbouncer-pooler-tls-ro.yaml"
114+
poolerCertificateRWSampleFile = folderPath + "pgbouncer-pooler-tls-rw.yaml"
115+
caSecName = "my-postgresql-server-ca"
116+
tlsSecName = "my-postgresql-server"
117+
tlsSecNameClient = "my-postgresql-client"
118+
caSecNameClient = "my-postgresql-client-ca"
119+
)
120+
// Create a cluster in a namespace that will be deleted after the test
121+
namespace = "pgbouncer-separate-certificates"
122+
err := env.CreateNamespace(namespace)
123+
Expect(err).ToNot(HaveOccurred())
124+
clusterName, err = env.GetResourceNameFromYAML(sampleFileWithCertificate)
125+
Expect(err).ToNot(HaveOccurred())
126+
127+
// Create certificates secret for server
128+
createAndAssertCertificatesSecrets(namespace, clusterName,
129+
caSecName, tlsSecName, certs.CertTypeServer, true)
130+
131+
// Create certificates secret for client
132+
createAndAssertCertificatesSecrets(namespace, clusterName,
133+
caSecNameClient, tlsSecNameClient, certs.CertTypeClient, true)
134+
135+
AssertCreateCluster(namespace, clusterName, sampleFileWithCertificate, env)
136+
137+
By("setting up read write type pgbouncer pooler", func() {
138+
createAndAssertPgBouncerPoolerIsSetUp(namespace, poolerCertificateRWSampleFile, 1)
139+
})
140+
141+
By("setting up read only type pgbouncer pooler", func() {
142+
createAndAssertPgBouncerPoolerIsSetUp(namespace, poolerCertificateROSampleFile, 1)
143+
})
144+
145+
By("verifying read and write connections using pgbouncer service", func() {
146+
assertReadWriteConnectionUsingPgBouncerService(namespace, clusterName,
147+
poolerCertificateRWSampleFile, true)
148+
})
149+
150+
By("verifying read connections using pgbouncer service", func() {
151+
assertReadWriteConnectionUsingPgBouncerService(namespace, clusterName,
152+
poolerCertificateROSampleFile, false)
153+
})
154+
})
107155
})
108156

109157
var _ = Describe("PgBouncer Pooler Resources", func() {

0 commit comments

Comments
 (0)