diff --git a/api/v1alpha1/backupsession_types.go b/api/v1alpha1/backupsession_types.go index 7bab396..03f69cd 100644 --- a/api/v1alpha1/backupsession_types.go +++ b/api/v1alpha1/backupsession_types.go @@ -24,14 +24,15 @@ import ( type SessionState string const ( - New SessionState = "New" - Init SessionState = "Initializing" - Running SessionState = "Running" - Waiting SessionState = "Waiting" - Finalize SessionState = "Finalizing" - Success SessionState = "Success" - Failure SessionState = "Failure" - Deleted SessionState = "Deleted" + New SessionState = "New" + Initializing SessionState = "Initializing" + Initialized SessionState = "Initialized" + Running SessionState = "Running" + Waiting SessionState = "Waiting" + Finalize SessionState = "Finalize" + Success SessionState = "Success" + Failure SessionState = "Failure" + Deleted SessionState = "Deleted" ) type TargetStatus struct { diff --git a/controllers/backupsession_controller.go b/controllers/backupsession_controller.go index f48861b..6ebd11f 100644 --- a/controllers/backupsession_controller.go +++ b/controllers/backupsession_controller.go @@ -91,72 +91,49 @@ func (r *BackupSessionReconciler) Reconcile(ctx context.Context, req ctrl.Reques return ctrl.Result{}, err } + var newSessionState formolv1alpha1.SessionState switch backupSession.Status.SessionState { case formolv1alpha1.New: + // Go through the Targets and create the corresponding TargetStatus. Move to Initializing. if r.isBackupOngoing(backupConf) { r.Log.V(0).Info("there is an ongoing backup. Let's reschedule this operation") return ctrl.Result{ RequeueAfter: 30 * time.Second, }, nil } - if nextTargetStatus := r.startNextTask(&backupSession, backupConf); nextTargetStatus != nil { - r.Log.V(0).Info("New backup. Start the first task", "task", nextTargetStatus) - backupSession.Status.SessionState = formolv1alpha1.Running - if err := r.Status().Update(ctx, &backupSession); err != nil { - r.Log.Error(err, "unable to update BackupSession status") - } - return ctrl.Result{}, err - } else { - r.Log.V(0).Info("No first target? That should not happen. Mark the backup has failed") - backupSession.Status.SessionState = formolv1alpha1.Failure - if err := r.Status().Update(ctx, &backupSession); err != nil { - r.Log.Error(err, "unable to update BackupSession status") - } - return ctrl.Result{}, err - } + newSessionState = r.initBackup(&backupSession, backupConf) + case formolv1alpha1.Initializing: + // Wait for all the Targets to be in the Initialized state then move them to Running and move to Running myself. + // if one of the Target fails to initialize, move it back to New state and decrement Try. + // if try reaches 0, move all the Targets to Finalize and move myself to Failure. + newSessionState = r.checkInitialized(&backupSession, backupConf) case formolv1alpha1.Running: - // Backup ongoing. Check the status of the last backup task and decide what to do next. - currentTargetStatus := &(backupSession.Status.Targets[len(backupSession.Status.Targets)-1]) - switch currentTargetStatus.SessionState { - case formolv1alpha1.Running: - r.Log.V(0).Info("Current task is still running. Wait until it's finished") - case formolv1alpha1.Success: - r.Log.V(0).Info("Last backup task was a success. Start a new one") - if nextTargetStatus := r.startNextTask(&backupSession, backupConf); nextTargetStatus != nil { - r.Log.V(0).Info("Starting a new task", "task", nextTargetStatus) - } else { - r.Log.V(0).Info("No more tasks to start. The backup is a success. Let's do some cleanup") - backupSession.Status.SessionState = formolv1alpha1.Success - } - if err := r.Status().Update(ctx, &backupSession); err != nil { - r.Log.Error(err, "unable to update BackupSession") - } - return ctrl.Result{}, err - case formolv1alpha1.Failure: - // Last task failed. Try to run it again - if currentTargetStatus.Try < backupConf.Spec.Targets[len(backupSession.Status.Targets)-1].Retry { - r.Log.V(0).Info("Last task failed. Try to run it again") - currentTargetStatus.Try++ - currentTargetStatus.SessionState = formolv1alpha1.New - currentTargetStatus.StartTime = &metav1.Time{Time: time.Now()} - } else { - r.Log.V(0).Info("Task failed again and for the last time") - backupSession.Status.SessionState = formolv1alpha1.Failure - } - if err := r.Status().Update(ctx, &backupSession); err != nil { - r.Log.Error(err, "unable to update BackupSession") - } - return ctrl.Result{}, err + // Wait for all the target to be in Waiting state then move them to the Finalize state. Move myself to Finalize. + // if one of the Target fails the backup, move it back to Running state and decrement Try. + // if try reaches 0, move all the Targets to Finalize and move myself to Failure. + newSessionState = r.checkWaiting(&backupSession, backupConf) + case formolv1alpha1.Finalize: + // Check the TargetStatus of all the Targets. If they are all Success then move myself to Success. + // if one of the Target fails to Finalize, move it back to Finalize state and decrement Try. + // if try reaches 0, move myself to Success because the backup was a Success even if the Finalize failed. + if newSessionState = r.checkSuccess(&backupSession, backupConf); newSessionState == formolv1alpha1.Failure { + r.Log.V(0).Info("One of the target did not manage to Finalize but the backup is still a Success") + newSessionState = formolv1alpha1.Success } + case formolv1alpha1.Success: + r.Log.V(0).Info("Backup was a success") case formolv1alpha1.Failure: - // Failed backup. Don't do anything anymore - case formolv1alpha1.Success: - // Backup was a success + r.Log.V(0).Info("Backup failed") + default: // BackupSession has just been created - backupSession.Status.SessionState = formolv1alpha1.New + newSessionState = formolv1alpha1.New backupSession.Status.StartTime = &metav1.Time{Time: time.Now()} + } + if newSessionState != "" { + r.Log.V(0).Info("BackupSession needs a status update", "newSessionState", newSessionState, "backupSession", backupSession) + backupSession.Status.SessionState = newSessionState if err := r.Status().Update(ctx, &backupSession); err != nil { r.Log.Error(err, "unable to update BackupSession.Status") return ctrl.Result{}, err diff --git a/controllers/backupsession_controller_helpers.go b/controllers/backupsession_controller_helpers.go index 86d678c..0ef20db 100644 --- a/controllers/backupsession_controller_helpers.go +++ b/controllers/backupsession_controller_helpers.go @@ -38,29 +38,67 @@ func (r *BackupSessionReconciler) isBackupOngoing(backupConf formolv1alpha1.Back return len(backupSessionList.Items) > 0 } -func (r *BackupSessionReconciler) startNextTask(backupSession *formolv1alpha1.BackupSession, backupConf formolv1alpha1.BackupConfiguration) *formolv1alpha1.TargetStatus { - nextTargetIndex := len(backupSession.Status.Targets) - if nextTargetIndex < len(backupConf.Spec.Targets) { - nextTarget := backupConf.Spec.Targets[nextTargetIndex] - nextTargetStatus := formolv1alpha1.TargetStatus{ - BackupType: nextTarget.BackupType, - TargetName: nextTarget.TargetName, - TargetKind: nextTarget.TargetKind, - SessionState: formolv1alpha1.New, +func (r *BackupSessionReconciler) initBackup(backupSession *formolv1alpha1.BackupSession, backupConf formolv1alpha1.BackupConfiguration) formolv1alpha1.SessionState { + for _, target := range backupConf.Spec.Targets { + r.Log.V(0).Info("Creating new target", "target", target.TargetName) + backupSession.Status.Targets = append(backupSession.Status.Targets, formolv1alpha1.TargetStatus{ + BackupType: target.BackupType, + TargetName: target.TargetName, + TargetKind: target.TargetKind, + SessionState: "", StartTime: &metav1.Time{Time: time.Now()}, Try: 1, - } - switch nextTarget.BackupType { - case formolv1alpha1.OnlineKind: - r.Log.V(0).Info("Starts a new OnlineKind task", "target", nextTarget) - case formolv1alpha1.JobKind: - r.Log.V(0).Info("Starts a new JobKind task", "target", nextTarget) - case formolv1alpha1.SnapshotKind: - r.Log.V(0).Info("Starts a new SnapshotKind task", "target", nextTarget) - } - backupSession.Status.Targets = append(backupSession.Status.Targets, nextTargetStatus) - return &nextTargetStatus - } else { - return nil + }) } + return formolv1alpha1.Initializing +} + +func (r *BackupSessionReconciler) checkSessionState( + backupSession *formolv1alpha1.BackupSession, + backupConf formolv1alpha1.BackupConfiguration, + currentState formolv1alpha1.SessionState, + waitState formolv1alpha1.SessionState, + nextState formolv1alpha1.SessionState) formolv1alpha1.SessionState { + for i, targetStatus := range backupSession.Status.Targets { + r.Log.V(0).Info("Target status", "target", targetStatus.TargetName, "session state", targetStatus.SessionState) + switch targetStatus.SessionState { + case currentState: + r.Log.V(0).Info("Move target to waitState", "target", targetStatus.TargetName, "waitState", waitState) + backupSession.Status.Targets[i].SessionState = waitState + return waitState + case formolv1alpha1.Failure: + if targetStatus.Try < backupConf.Spec.Targets[i].Retry { + r.Log.V(0).Info("Target failed. Try one more time", "target", targetStatus.TargetName, "waitState", waitState) + backupSession.Status.Targets[i].SessionState = waitState + backupSession.Status.Targets[i].Try++ + backupSession.Status.Targets[i].StartTime = &metav1.Time{Time: time.Now()} + return waitState + } else { + r.Log.V(0).Info("Target failed for the last time", "target", targetStatus.TargetName) + return formolv1alpha1.Failure + } + case waitState: + // target is still busy with its current state. Wait until it is done. + r.Log.V(0).Info("Waiting for one target to finish", "waitState", waitState) + return "" + default: + if i == len(backupSession.Status.Targets)-1 { + r.Log.V(0).Info("Moving to next state", "nextState", nextState) + return nextState + } + } + } + return "" +} + +func (r *BackupSessionReconciler) checkInitialized(backupSession *formolv1alpha1.BackupSession, backupConf formolv1alpha1.BackupConfiguration) formolv1alpha1.SessionState { + return r.checkSessionState(backupSession, backupConf, "", formolv1alpha1.Initializing, formolv1alpha1.Running) +} + +func (r *BackupSessionReconciler) checkWaiting(backupSession *formolv1alpha1.BackupSession, backupConf formolv1alpha1.BackupConfiguration) formolv1alpha1.SessionState { + return r.checkSessionState(backupSession, backupConf, formolv1alpha1.Initialized, formolv1alpha1.Running, formolv1alpha1.Finalize) +} + +func (r *BackupSessionReconciler) checkSuccess(backupSession *formolv1alpha1.BackupSession, backupConf formolv1alpha1.BackupConfiguration) formolv1alpha1.SessionState { + return r.checkSessionState(backupSession, backupConf, formolv1alpha1.Waiting, formolv1alpha1.Finalize, formolv1alpha1.Success) }