Multi-Region Backup and Restore

docs/operator-public-documentation/preview/backup-and-restore.md

@@ -7,7 +7,7 @@
 1. Run the CSI driver deployment script **before** installing the documentdb-operator:
 
 ```bash
-scripts/test-scripts/deploy-csi-driver.sh
+./operator/src/scripts/test-scripts/deploy-csi-driver.sh
 ```
 
 2. Validate storage and snapshot components:

documentdb-playground/aks-fleet-deployment/README.md

@@ -255,6 +255,79 @@ az network vnet peering list --resource-group $RESOURCE_GROUP \
   --vnet-name member-westus3-vnet --output table
 ```
 
+## Backup and Restore
+### Backup
+
+Create a one-time backup:
+```bash
+kubectl --context hub apply -f - <<EOF
+apiVersion: db.microsoft.com/preview
+kind: Backup
+metadata:
+  name: backup-documentdb
+  namespace: documentdb-preview-ns
+spec:
+  cluster:
+    name: documentdb-preview
+EOF
+```
+
+Create automatic backups on a schedule:
+```bash
+kubectl --context hub apply -f - <<EOF
+apiVersion: db.microsoft.com/preview
+kind: ScheduledBackup
+metadata:
+  name: scheduled-backup
+  namespace: documentdb-preview-ns
+spec:
+  cluster:
+    name: documentdb-preview
+  schedule: "0 2 * * *" # Daily at 2 AM
+EOF
+```
+
+Backups will be created on the primary cluster. 
+
+### Restore
+
+Step 1: Identify Available Backups
+```bash
+PRIMARY_CLUSTER=$(kubectl --context hub get documentdb documentdb-preview -n documentdb-preview-ns -o jsonpath='{.spec.clusterReplication.primary}')
+
+kubectl --context $PRIMARY_CLUSTER get backups -n documentdb-preview-ns
+```
+
+Step 2: Modify multi-region.yaml for Restore
+
+Important: Restores must be to a new DocumentDB resource with a different name.
+
+Edit `./multi-region.yaml` and change:
+1. The DocumentDB resource name (e.g., documentdb-preview-restore)
+2. Add the bootstrap section with backup reference
+
+Example:
+```yaml
+apiVersion: db.microsoft.com/preview
+kind: DocumentDB
+metadata:
+  name: documentdb-preview-restore  # New name, different from original
+  namespace: documentdb-preview-ns
+spec:
+  ...
+  bootstrap:
+    recovery:
+      backup:
+        name: scheduled-backup-xxxxxx  # Name of the backup to restore from
+```
+
+Step 3: Deploy the Restored Cluster
+
+Run the deployment script:
+```bash
+./deploy-multi-region.sh "${DOCUMENTDB_PASSWORD}"
+```
+
 ## Troubleshooting
 
 ### Authentication Issues

documentdb-playground/aks-fleet-deployment/documentdb-base.yaml

@@ -32,14 +32,22 @@ spec:
       version: v1
       kind: CustomResourceDefinition
       name: imagecatalogs.postgresql.cnpg.io
+    - group: "apiextensions.k8s.io"
+      version: v1
+      kind: CustomResourceDefinition
+      name: backups.postgresql.cnpg.io
     - group: "apiextensions.k8s.io"
       version: v1
       kind: CustomResourceDefinition
       name: scheduledbackups.postgresql.cnpg.io
     - group: "apiextensions.k8s.io"
       version: v1
       kind: CustomResourceDefinition
-      name: backups.postgresql.cnpg.io
+      name: backups.db.microsoft.com
+    - group: "apiextensions.k8s.io"
+      version: v1
+      kind: CustomResourceDefinition
+      name: scheduledbackups.db.microsoft.com
     - group: "apiextensions.k8s.io"
       version: v1
       kind: CustomResourceDefinition

documentdb-playground/aks-fleet-deployment/multi-region.yaml

@@ -31,6 +31,7 @@ spec:
   resource:
     storage:
       pvcSize: 10Gi
+  environment: aks
   clusterReplication:
     highAvailability: true
     crossCloudNetworkingStrategy: AzureFleet

operator/documentdb-helm-chart/crds/db.microsoft.com_backups.yaml

@@ -35,9 +35,9 @@ spec:
       jsonPath: .status.expiredAt
       name: ExpiredAt
       type: string
-    - description: Backup error information
-      jsonPath: .status.error
-      name: Error
+    - description: Backup status message
+      jsonPath: .status.message
+      name: Message
       type: string
     name: preview
     schema:
@@ -88,14 +88,17 @@ spec:
           status:
             description: BackupStatus defines the observed state of Backup.
             properties:
-              error:
-                description: Error contains error information if the backup failed.
-                type: string
               expiredAt:
                 description: ExpiredAt is the time when the backup is considered expired
                   and can be deleted.
                 format: date-time
                 type: string
+              message:
+                description: |-
+                  Message contains additional information about the backup status.
+                  For failed backups, this contains the error message.
+                  For skipped backups, this explains why the backup was skipped.
+                type: string
               phase:
                 description: Phase represents the current phase of the backup operation.
                 type: string

operator/src/api/preview/backup_funcs.go

@@ -13,7 +13,7 @@ import (
 )
 
 // CreateCNPGBackup creates a CNPG Backup resource based on the DocumentDB Backup spec.
-func (backup *Backup) CreateCNPGBackup(scheme *runtime.Scheme) (*cnpgv1.Backup, error) {
+func (backup *Backup) CreateCNPGBackup(scheme *runtime.Scheme, clusterName string) (*cnpgv1.Backup, error) {
 	cnpgBackup := &cnpgv1.Backup{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      backup.Name,
@@ -22,7 +22,7 @@ func (backup *Backup) CreateCNPGBackup(scheme *runtime.Scheme) (*cnpgv1.Backup,
 		Spec: cnpgv1.BackupSpec{
 			Method: cnpgv1.BackupMethodVolumeSnapshot,
 			Cluster: cnpgv1.LocalObjectReference{
-				Name: backup.Spec.Cluster.Name,
+				Name: clusterName,
 			},
 		},
 	}
@@ -52,8 +52,8 @@ func (backup *Backup) UpdateStatus(cnpgBackup *cnpgv1.Backup, backupConfiguratio
 		needsUpdate = true
 	}
 
-	if backup.Status.Error != cnpgBackup.Status.Error {
-		backup.Status.Error = cnpgBackup.Status.Error
+	if backup.Status.Message != cnpgBackup.Status.Error {
+		backup.Status.Message = cnpgBackup.Status.Error
 		needsUpdate = true
 	}
 
@@ -106,7 +106,7 @@ func areTimesEqual(t1, t2 *metav1.Time) bool {
 
 // IsDone returns true if the backup operation is completed or failed.
 func (backupStatus *BackupStatus) IsDone() bool {
-	return backupStatus.Phase == cnpgv1.BackupPhaseCompleted || backupStatus.Phase == cnpgv1.BackupPhaseFailed
+	return backupStatus.Phase == cnpgv1.BackupPhaseCompleted || backupStatus.Phase == cnpgv1.BackupPhaseFailed || backupStatus.Phase == BackupPhaseSkipped
 }
 
 // IsExpired returns true if the backup has expired based on the current time.

operator/src/api/preview/backup_funcs_test.go

@@ -35,15 +35,15 @@ var _ = Describe("Backup", func() {
 				},
 			}
 
-			cnpg, err := backup.CreateCNPGBackup(scheme)
+			cnpg, err := backup.CreateCNPGBackup(scheme, "my-cluster-x")
 			Expect(err).To(BeNil())
 			Expect(cnpg).ToNot(BeNil())
 
 			// metadata and spec checks
 			Expect(cnpg.Name).To(Equal("my-backup"))
 			Expect(cnpg.Namespace).To(Equal("my-ns"))
 			Expect(cnpg.Spec.Method).To(Equal(cnpgv1.BackupMethodVolumeSnapshot))
-			Expect(cnpg.Spec.Cluster.Name).To(Equal("my-cluster"))
+			Expect(cnpg.Spec.Cluster.Name).To(Equal("my-cluster-x"))
 
 			// owner reference set by SetControllerReference
 			Expect(len(cnpg.OwnerReferences)).To(BeNumerically(">", 0))
@@ -79,7 +79,7 @@ var _ = Describe("Backup", func() {
 					Phase:     cnpgv1.BackupPhase(""),
 					StartedAt: nil,
 					StoppedAt: nil,
-					Error:     "",
+					Message:   "",
 				},
 			}
 
@@ -88,7 +88,7 @@ var _ = Describe("Backup", func() {
 			Expect(string(backup.Status.Phase)).To(Equal(cnpgv1.BackupPhaseCompleted))
 			Expect(backup.Status.StartedAt).To(Equal(&startedAt))
 			Expect(backup.Status.StoppedAt).To(Equal(&stoppedAt))
-			Expect(backup.Status.Error).To(Equal("none"))
+			Expect(backup.Status.Message).To(Equal("none"))
 			// ExpiredAt should be StoppedAt + 30 days (default)
 			Expect(backup.Status.ExpiredAt).ToNot(BeNil())
 			Expect(backup.Status.ExpiredAt.Time.Equal(stoppedAt.Time.Add(30 * 24 * time.Hour))).To(BeTrue())
@@ -114,7 +114,7 @@ var _ = Describe("Backup", func() {
 					Phase:     cnpgv1.BackupPhaseCompleted,
 					StartedAt: &startedAt,
 					StoppedAt: &stoppedAt,
-					Error:     "none",
+					Message:   "none",
 					ExpiredAt: &expiredAt,
 				},
 			}
@@ -225,6 +225,13 @@ var _ = Describe("Backup", func() {
 			Expect(status.IsDone()).To(BeTrue())
 		})
 
+		It("returns true when phase is Skipped", func() {
+			status := &BackupStatus{
+				Phase: BackupPhaseSkipped,
+			}
+			Expect(status.IsDone()).To(BeTrue())
+		})
+
 		It("returns false when phase is Running", func() {
 			status := &BackupStatus{
 				Phase: cnpgv1.BackupPhaseRunning,

operator/src/api/preview/backup_types.go

@@ -43,9 +43,11 @@ type BackupStatus struct {
 	// +optional
 	ExpiredAt *metav1.Time `json:"expiredAt,omitempty"`
 
-	// Error contains error information if the backup failed.
+	// Message contains additional information about the backup status.
+	// For failed backups, this contains the error message.
+	// For skipped backups, this explains why the backup was skipped.
 	// +optional
-	Error string `json:"error,omitempty"`
+	Message string `json:"message,omitempty"`
 }
 
 // +kubebuilder:object:root=true
@@ -55,7 +57,7 @@ type BackupStatus struct {
 // +kubebuilder:printcolumn:name="StartedAt",type=string,JSONPath=".status.startedAt",description="Backup start time"
 // +kubebuilder:printcolumn:name="StoppedAt",type=string,JSONPath=".status.stoppedAt",description="Backup completion time"
 // +kubebuilder:printcolumn:name="ExpiredAt",type=string,JSONPath=".status.expiredAt",description="Backup expiration time"
-// +kubebuilder:printcolumn:name="Error",type=string,JSONPath=".status.error",description="Backup error information"
+// +kubebuilder:printcolumn:name="Message",type=string,JSONPath=".status.message",description="Backup status message"
 type Backup struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata,omitempty"`

operator/src/config/crd/bases/db.microsoft.com_backups.yaml

@@ -35,9 +35,9 @@ spec:
       jsonPath: .status.expiredAt
       name: ExpiredAt
       type: string
-    - description: Backup error information
-      jsonPath: .status.error
-      name: Error
+    - description: Backup status message
+      jsonPath: .status.message
+      name: Message
       type: string
     name: preview
     schema:
@@ -88,14 +88,17 @@ spec:
           status:
             description: BackupStatus defines the observed state of Backup.
             properties:
-              error:
-                description: Error contains error information if the backup failed.
-                type: string
               expiredAt:
                 description: ExpiredAt is the time when the backup is considered expired
                   and can be deleted.
                 format: date-time
                 type: string
+              message:
+                description: |-
+                  Message contains additional information about the backup status.
+                  For failed backups, this contains the error message.
+                  For skipped backups, this explains why the backup was skipped.
+                type: string
               phase:
                 description: Phase represents the current phase of the backup operation.
                 type: string

operator/src/internal/cnpg/cnpg_cluster.go

@@ -16,7 +16,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 )
 
-func GetCnpgClusterSpec(req ctrl.Request, documentdb *dbpreview.DocumentDB, documentdb_image, serviceAccountName, storageClass string, log logr.Logger) *cnpgv1.Cluster {
+func GetCnpgClusterSpec(req ctrl.Request, documentdb *dbpreview.DocumentDB, documentdb_image, serviceAccountName, storageClass string, isPrimaryRegion bool, log logr.Logger) *cnpgv1.Cluster {
 	sidecarPluginName := documentdb.Spec.SidecarInjectorPluginName
 	if sidecarPluginName == "" {
 		sidecarPluginName = util.DEFAULT_SIDECAR_INJECTOR_PLUGIN
@@ -88,7 +88,7 @@ func GetCnpgClusterSpec(req ctrl.Request, documentdb *dbpreview.DocumentDB, docu
 						"host replication all all trust",
 					},
 				},
-				Bootstrap: getBootstrapConfiguration(documentdb, log),
+				Bootstrap: getBootstrapConfiguration(documentdb, isPrimaryRegion, log),
 				LogLevel:  cmp.Or(documentdb.Spec.LogLevel, "info"),
 				Backup: &cnpgv1.BackupConfiguration{
 					VolumeSnapshot: &cnpgv1.VolumeSnapshotConfiguration{
@@ -112,8 +112,8 @@ func getInheritedMetadataLabels(appName string) *cnpgv1.EmbeddedObjectMetadata {
 	}
 }
 
-func getBootstrapConfiguration(documentdb *dbpreview.DocumentDB, log logr.Logger) *cnpgv1.BootstrapConfiguration {
-	if documentdb.Spec.Bootstrap != nil && documentdb.Spec.Bootstrap.Recovery != nil && documentdb.Spec.Bootstrap.Recovery.Backup.Name != "" {
+func getBootstrapConfiguration(documentdb *dbpreview.DocumentDB, isPrimaryRegion bool, log logr.Logger) *cnpgv1.BootstrapConfiguration {
+	if isPrimaryRegion && documentdb.Spec.Bootstrap != nil && documentdb.Spec.Bootstrap.Recovery != nil && documentdb.Spec.Bootstrap.Recovery.Backup.Name != "" {
 		backupName := documentdb.Spec.Bootstrap.Recovery.Backup.Name
 		log.Info("DocumentDB cluster will be bootstrapped from backup", "backupName", backupName)
 		return &cnpgv1.BootstrapConfiguration{