feat: instance fencing through annotation

It's now possible to fence one or more instances in the cluster by
setting the content of the "k8s.enterprisedb.io/fencedInstances"
annotation to a JSON list of instances (e.g.
["cluster-example-1","cluster-example-2"]) or using
the "kubectl cnp fence on/off" subcommand.

Currently, fencing an instance will result in:
- switchovers/failovers are not to happen while at least an instance in the
  cluster is fenced
- fenced instances will be shut down, but the pod will be kept running,
  but not considered ready, Postgres will be restarted once the fencing
  is removed for that specific instance
- in case a primary instance is fenced, as soon as the fencing is
  removed a failover to another instance will be performed

Co-authored-by: Armando Ruocco <armando.ruocco@enterprisedb.com>
Co-authored-by: Leonardo Cecchi <leonardo.cecchi@enterprisedb.com>
Co-authored-by: Gabriele Bartolini <gabriele.bartolini@enterprisedb.com>
Co-authored-by: Marco Nenciarini <marco.nenciarini@enterprisedb.com>

Author: 5 people

api/v1/cluster_types.go

@@ -1316,6 +1316,19 @@ func (cluster *Cluster) IsReusePVCEnabled() bool {
 	return reusePVC
 }
 
+// IsInstanceFenced check if in a given instance should be fenced
+func (cluster *Cluster) IsInstanceFenced(instance string) bool {
+	fencedInstances, err := utils.GetFencedInstances(cluster.Annotations)
+	if err != nil {
+		return false
+	}
+
+	if fencedInstances.Has(utils.FenceAllServers) {
+		return true
+	}
+	return fencedInstances.Has(instance)
+}
+
 // ShouldResizeInUseVolumes is true when we should resize PVC we already
 // created
 func (cluster *Cluster) ShouldResizeInUseVolumes() bool {

api/v1/cluster_types_test.go

@@ -9,6 +9,8 @@ package v1
 import (
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	"github.com/EnterpriseDB/cloud-native-postgresql/pkg/utils"
+
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 )
@@ -584,3 +586,51 @@ var _ = Describe("Barman Endpoint CA for replica cluster", func() {
 		Expect(cluster3.GetBarmanEndpointCAForReplicaCluster()).To(Not(BeNil()))
 	})
 })
+
+var _ = Describe("Fencing annotation", func() {
+	When("one instance is fenced", func() {
+		cluster := Cluster{
+			ObjectMeta: v1.ObjectMeta{
+				Annotations: map[string]string{
+					utils.FencedInstanceAnnotation: "[\"one\"]",
+				},
+			},
+		}
+
+		It("detect when an instance is fenced", func() {
+			Expect(cluster.IsInstanceFenced("one")).To(BeTrue())
+		})
+
+		It("detect when an instance is not fenced", func() {
+			Expect(cluster.IsInstanceFenced("two")).To(BeFalse())
+		})
+	})
+
+	When("the whole cluster is fenced", func() {
+		cluster := Cluster{
+			ObjectMeta: v1.ObjectMeta{
+				Annotations: map[string]string{
+					utils.FencedInstanceAnnotation: "[\"*\"]",
+				},
+			},
+		}
+
+		It("detect when an instance is fenced", func() {
+			Expect(cluster.IsInstanceFenced("one")).To(BeTrue())
+			Expect(cluster.IsInstanceFenced("two")).To(BeTrue())
+			Expect(cluster.IsInstanceFenced("three")).To(BeTrue())
+		})
+	})
+
+	When("the annotation doesn't exist", func() {
+		cluster := Cluster{
+			ObjectMeta: v1.ObjectMeta{
+				Annotations: map[string]string{},
+			},
+		}
+
+		It("ensure no instances are fenced", func() {
+			Expect(cluster.IsInstanceFenced("one")).To(BeFalse())
+		})
+	})
+})

cmd/kubectl-cnp/main.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/EnterpriseDB/cloud-native-postgresql/internal/cmd/plugin"
 	"github.com/EnterpriseDB/cloud-native-postgresql/internal/cmd/plugin/certificate"
+	"github.com/EnterpriseDB/cloud-native-postgresql/internal/cmd/plugin/fence"
 	"github.com/EnterpriseDB/cloud-native-postgresql/internal/cmd/plugin/promote"
 	"github.com/EnterpriseDB/cloud-native-postgresql/internal/cmd/plugin/reload"
 	"github.com/EnterpriseDB/cloud-native-postgresql/internal/cmd/plugin/report"
@@ -42,6 +43,7 @@ func main() {
 	rootCmd.AddCommand(status.NewCmd())
 	rootCmd.AddCommand(promote.NewCmd())
 	rootCmd.AddCommand(certificate.NewCmd())
+	rootCmd.AddCommand(fence.NewCmd())
 	rootCmd.AddCommand(restart.NewCmd())
 	rootCmd.AddCommand(reload.NewCmd())
 	rootCmd.AddCommand(versions.NewCmd())

controllers/cluster_status.go

@@ -684,7 +684,10 @@ func (r *ClusterReconciler) extractInstancesStatus(
 
 	for idx := range filteredPods {
 		instanceStatus := r.getReplicaStatusFromPodViaHTTP(ctx, filteredPods[idx])
-		instanceStatus.IsReady = utils.IsPodReady(filteredPods[idx])
+
+		// Here we need to have pod marked as ready even when they are fenced. We want this
+		// to avoid a fenced primary to cause a failover to a different Pod.
+		instanceStatus.IsReady = instanceStatus.IsReady || utils.IsPodReady(filteredPods[idx])
 		instanceStatus.Node = filteredPods[idx].Spec.NodeName
 		instanceStatus.Pod = filteredPods[idx]
 

docs/mkdocs.yml

@@ -41,6 +41,7 @@ nav:
   - openshift.md
   - failover.md
   - troubleshooting.md
+  - fencing.md
   - e2e.md
   - container_images.md
   - operator_capability_levels.md

docs/src/failover.md

@@ -73,4 +73,3 @@ Failover may result in the service being impacted and/or data being lost:
     level. On the contrary, setting it to a high value, might remove the risk of
     data loss while leaving the cluster without an active primary for a longer time
     during the switchover.
-

docs/src/fencing.md

@@ -0,0 +1,115 @@
+# Fencing
+
+Fencing in Cloud Native PostgreSQL is the ultimate process of protecting the
+data in one, more, or even all instances of a PostgreSQL cluster when they
+appear to be malfunctioning. When an instance is fenced, the PostgreSQL server
+process (`postmaster`) is guaranteed to be shut down, while the pod is kept running.
+This makes sure that, until the fence is lifted, data on the pod is not modified by
+PostgreSQL and that the file system can be investigated for debugging and
+troubleshooting purposes.
+
+## How to fence instances
+
+In Cloud Native PostgreSQL you can fence:
+
+- a specific instance
+- a list of instances
+- an entire Postgres `Cluster`
+
+Fencing is controlled through the content of the `k8s.enterprisedb.io/fencedInstances`
+annotation, which expects a JSON formatted list of instance names.
+If the annotation is set to `'["*"]'`, a singleton list with a wildcard, the
+whole cluster is fenced.
+If the annotation is set to an empty JSON list, the operator behaves as if the
+annotation was not set.
+
+For example:
+
+- `k8s.enterprisedb.io/fencedInstances: '["cluster-example-1"]'` will fence just
+  the `cluster-example-1` instance
+
+- `k8s.enterprisedb.io/fencedInstances: '["cluster-example-1","cluster-example-2"]'`
+  will fence the `cluster-example-1` and `cluster-example-2` instances
+
+- `k8s.enterprisedb.io/fencedInstances: '["*"]'` will fence every instance in
+  the cluster.
+
+The annotation can be manually set on the Kubernetes object, for example via
+the `kubectl annotate` command, or in a transparent way using the
+`kubectl cnp fencing on` subcommand:
+
+```shell
+# to fence only one instance
+kubectl cnp fencing on cluster-example 1
+
+# to fence all the instances in a Cluster
+kubectl cnp fencing on cluster-example "*"
+```
+
+Here is an example of a `Cluster` with an instance that was previously fenced:
+
+```yaml
+apiVersion: postgresql.k8s.enterprisedb.io/v1
+kind: Cluster
+metadata:
+    annotations:
+      k8s.enterprisedb.io/fencedInstances: '["cluster-example-1"]'
+[...]
+```
+
+## How to lift fencing
+
+Fencing can be lifted by clearing the annotation, or set it to a different value.
+
+As for fencing, this can be done either manually with `kubectl annotate`, or
+using the `kubectl cnp fencing` subcommand as follows:
+
+```shell
+# to lift the fencing only for one instance
+# N.B.: at the moment this won't work if the whole cluster was fenced previously,
+#       in that case you will have to manually set the annotation as explained above
+kubectl cnp fencing off cluster-example 1
+
+# to lift the fencing for all the instances in a Cluster
+kubectl cnp fencing off cluster-example "*"
+```
+
+## How fencing works
+
+Once an instance is set for fencing, the procedure to shut down the
+`postmaster` process is initiated. This consists of an initial smart shutdown
+with a timeout set to `.spec.stopDelay`, followed by a fast shutdown if
+required. Then:
+
+- the Pod will be kept alive
+
+- the Pod won't be marked as *Ready*
+
+- all the changes that don't require the Postgres instance to be up will be
+  reconciled, including:
+    - configuration files
+    - certificates and all the cryptographic material
+
+- metrics will not be collected, except `cnp_collector_fencing_on` which will be
+  set to 1
+
+!!! Warning
+    When at least one instance in a `Cluster` is fenced, failovers/switchovers for that
+    `Cluster` will be blocked until the fence is lifted, as the status of the `Cluster`
+    cannot be considered stable.
+
+    In particular, if a **primary instance** will be fenced, the postmaster process
+    will be shut down but no failover will happen, interrupting the operativity of
+    the applications. When the fence will be lifted, the primary instance will be
+    started up again without any failover happening.
+
+    Given that, we advise the user to fence only replica instances when possible.
+
+!!! Warning
+    If the primary is the only fenced instance in a `Cluster` and the pod is deleted, a
+    failover will be performed. When the fence on the old primary is lifted, that instance
+    is restarted as a standby (follower of the new primary).
+
+If a fenced instance is deleted, the pod will be recreated normally, but the
+postmaster won't be started. This can be extremely helpful when instances
+are `Crashlooping`.

docs/src/index.md

@@ -100,6 +100,7 @@ architectures on OpenShift only.
 * Standard output logging of PostgreSQL error messages in JSON format
 * Support for the `restricted` security context constraint (SCC) in Red Hat OpenShift
 * `cnp` plugin for `kubectl`
+* Fencing of an entire PostgreSQL cluster, or a subset of the instances
 * Multi-arch format container images
 
 ## About this guide

docs/src/operator_capability_levels.md

@@ -399,6 +399,16 @@ in the maintenance window section enables to specify the strategy to be used:
 allocate new storage in a different PVC for the evicted instance or wait
 for the underlying node to be available again.
 
+### Fencing
+
+Fencing is the process of protecting the data in one, more, or even all
+instances of a PostgreSQL cluster when they appear to be malfunctioning.
+When an instance is fenced, the PostgreSQL server process is
+guaranteed to be shut down, while the pod is kept running. This makes sure
+that, until the fence is lifted, data on the pod is not modified by PostgreSQL
+and that the file system can be investigated for debugging and troubleshooting
+purposes.
+
 ### Reuse of Persistent Volumes storage in Pods
 
 When the operator needs to create a pod that has been deleted by the user or