jandre/formol
1
1
Fork 0
Code Issues 6 Pull Requests Projects Releases 1 Wiki Activity

Reworked the scheduling of the tasks. We want the init/backup/finalize tasks to be run for all the targets one after the other.

Browse Source
This commit is contained in:
Jean-Marc ANDRE 2023-02-27 00:52:01 +01:00
parent b42bd46efe
commit b5a217bc3a
3 changed files with 97 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
api/v1alpha1/backupsession_types.go
View File

@ -24,14 +24,15 @@ import (
type SessionState string
const (
New SessionState = "New"
Init SessionState = "Initializing"
Running SessionState = "Running"
Waiting SessionState = "Waiting"
Finalize SessionState = "Finalizing"
Success SessionState = "Success"
Failure SessionState = "Failure"
Deleted SessionState = "Deleted"
New SessionState = "New"
Initializing SessionState = "Initializing"
Initialized SessionState = "Initialized"
Running SessionState = "Running"
Waiting SessionState = "Waiting"
Finalize SessionState = "Finalize"
Success SessionState = "Success"
Failure SessionState = "Failure"
Deleted SessionState = "Deleted"
)
type TargetStatus struct {

79
controllers/backupsession_controller.go
View File

@ -91,72 +91,49 @@ func (r *BackupSessionReconciler) Reconcile(ctx context.Context, req ctrl.Reques
return ctrl.Result{}, err
}
var newSessionState formolv1alpha1.SessionState
switch backupSession.Status.SessionState {
case formolv1alpha1.New:
// Go through the Targets and create the corresponding TargetStatus. Move to Initializing.
if r.isBackupOngoing(backupConf) {
r.Log.V(0).Info("there is an ongoing backup. Let's reschedule this operation")
return ctrl.Result{
RequeueAfter: 30 * time.Second,
}, nil
}
if nextTargetStatus := r.startNextTask(&backupSession, backupConf); nextTargetStatus != nil {
r.Log.V(0).Info("New backup. Start the first task", "task", nextTargetStatus)
backupSession.Status.SessionState = formolv1alpha1.Running
if err := r.Status().Update(ctx, &backupSession); err != nil {
r.Log.Error(err, "unable to update BackupSession status")
}
return ctrl.Result{}, err
} else {
r.Log.V(0).Info("No first target? That should not happen. Mark the backup has failed")
backupSession.Status.SessionState = formolv1alpha1.Failure
if err := r.Status().Update(ctx, &backupSession); err != nil {
r.Log.Error(err, "unable to update BackupSession status")
}
return ctrl.Result{}, err
}
newSessionState = r.initBackup(&backupSession, backupConf)
case formolv1alpha1.Initializing:
// Wait for all the Targets to be in the Initialized state then move them to Running and move to Running myself.
// if one of the Target fails to initialize, move it back to New state and decrement Try.
// if try reaches 0, move all the Targets to Finalize and move myself to Failure.
newSessionState = r.checkInitialized(&backupSession, backupConf)
case formolv1alpha1.Running:
// Backup ongoing. Check the status of the last backup task and decide what to do next.
currentTargetStatus := &(backupSession.Status.Targets[len(backupSession.Status.Targets)-1])
switch currentTargetStatus.SessionState {
case formolv1alpha1.Running:
r.Log.V(0).Info("Current task is still running. Wait until it's finished")
case formolv1alpha1.Success:
r.Log.V(0).Info("Last backup task was a success. Start a new one")
if nextTargetStatus := r.startNextTask(&backupSession, backupConf); nextTargetStatus != nil {
r.Log.V(0).Info("Starting a new task", "task", nextTargetStatus)
} else {
r.Log.V(0).Info("No more tasks to start. The backup is a success. Let's do some cleanup")
backupSession.Status.SessionState = formolv1alpha1.Success
}
if err := r.Status().Update(ctx, &backupSession); err != nil {
r.Log.Error(err, "unable to update BackupSession")
}
return ctrl.Result{}, err
case formolv1alpha1.Failure:
// Last task failed. Try to run it again
if currentTargetStatus.Try < backupConf.Spec.Targets[len(backupSession.Status.Targets)-1].Retry {
r.Log.V(0).Info("Last task failed. Try to run it again")
currentTargetStatus.Try++
currentTargetStatus.SessionState = formolv1alpha1.New
currentTargetStatus.StartTime = &metav1.Time{Time: time.Now()}
} else {
r.Log.V(0).Info("Task failed again and for the last time")
backupSession.Status.SessionState = formolv1alpha1.Failure
}
if err := r.Status().Update(ctx, &backupSession); err != nil {
r.Log.Error(err, "unable to update BackupSession")
}
return ctrl.Result{}, err
// Wait for all the target to be in Waiting state then move them to the Finalize state. Move myself to Finalize.
// if one of the Target fails the backup, move it back to Running state and decrement Try.
// if try reaches 0, move all the Targets to Finalize and move myself to Failure.
newSessionState = r.checkWaiting(&backupSession, backupConf)
case formolv1alpha1.Finalize:
// Check the TargetStatus of all the Targets. If they are all Success then move myself to Success.
// if one of the Target fails to Finalize, move it back to Finalize state and decrement Try.
// if try reaches 0, move myself to Success because the backup was a Success even if the Finalize failed.
if newSessionState = r.checkSuccess(&backupSession, backupConf); newSessionState == formolv1alpha1.Failure {
r.Log.V(0).Info("One of the target did not manage to Finalize but the backup is still a Success")
newSessionState = formolv1alpha1.Success
}
case formolv1alpha1.Success:
r.Log.V(0).Info("Backup was a success")
case formolv1alpha1.Failure:
// Failed backup. Don't do anything anymore
case formolv1alpha1.Success:
// Backup was a success
r.Log.V(0).Info("Backup failed")
default:
// BackupSession has just been created
backupSession.Status.SessionState = formolv1alpha1.New
newSessionState = formolv1alpha1.New
backupSession.Status.StartTime = &metav1.Time{Time: time.Now()}
}
if newSessionState != "" {
r.Log.V(0).Info("BackupSession needs a status update", "newSessionState", newSessionState, "backupSession", backupSession)
backupSession.Status.SessionState = newSessionState
if err := r.Status().Update(ctx, &backupSession); err != nil {
r.Log.Error(err, "unable to update BackupSession.Status")
return ctrl.Result{}, err

82
controllers/backupsession_controller_helpers.go
View File

@ -38,29 +38,67 @@ func (r *BackupSessionReconciler) isBackupOngoing(backupConf formolv1alpha1.Back
return len(backupSessionList.Items) > 0
}
func (r *BackupSessionReconciler) startNextTask(backupSession *formolv1alpha1.BackupSession, backupConf formolv1alpha1.BackupConfiguration) *formolv1alpha1.TargetStatus {
nextTargetIndex := len(backupSession.Status.Targets)
if nextTargetIndex < len(backupConf.Spec.Targets) {
nextTarget := backupConf.Spec.Targets[nextTargetIndex]
nextTargetStatus := formolv1alpha1.TargetStatus{
BackupType: nextTarget.BackupType,
TargetName: nextTarget.TargetName,
TargetKind: nextTarget.TargetKind,
SessionState: formolv1alpha1.New,
func (r *BackupSessionReconciler) initBackup(backupSession *formolv1alpha1.BackupSession, backupConf formolv1alpha1.BackupConfiguration) formolv1alpha1.SessionState {
for _, target := range backupConf.Spec.Targets {
r.Log.V(0).Info("Creating new target", "target", target.TargetName)
backupSession.Status.Targets = append(backupSession.Status.Targets, formolv1alpha1.TargetStatus{
BackupType: target.BackupType,
TargetName: target.TargetName,
TargetKind: target.TargetKind,
SessionState: "",
StartTime: &metav1.Time{Time: time.Now()},
Try: 1,
}
switch nextTarget.BackupType {
case formolv1alpha1.OnlineKind:
r.Log.V(0).Info("Starts a new OnlineKind task", "target", nextTarget)
case formolv1alpha1.JobKind:
r.Log.V(0).Info("Starts a new JobKind task", "target", nextTarget)
case formolv1alpha1.SnapshotKind:
r.Log.V(0).Info("Starts a new SnapshotKind task", "target", nextTarget)
}
backupSession.Status.Targets = append(backupSession.Status.Targets, nextTargetStatus)
return &nextTargetStatus
} else {
return nil
})
}
return formolv1alpha1.Initializing
}
func (r *BackupSessionReconciler) checkSessionState(
backupSession *formolv1alpha1.BackupSession,
backupConf formolv1alpha1.BackupConfiguration,
currentState formolv1alpha1.SessionState,
waitState formolv1alpha1.SessionState,
nextState formolv1alpha1.SessionState) formolv1alpha1.SessionState {
for i, targetStatus := range backupSession.Status.Targets {
r.Log.V(0).Info("Target status", "target", targetStatus.TargetName, "session state", targetStatus.SessionState)
switch targetStatus.SessionState {
case currentState:
r.Log.V(0).Info("Move target to waitState", "target", targetStatus.TargetName, "waitState", waitState)
backupSession.Status.Targets[i].SessionState = waitState
return waitState
case formolv1alpha1.Failure:
if targetStatus.Try < backupConf.Spec.Targets[i].Retry {
r.Log.V(0).Info("Target failed. Try one more time", "target", targetStatus.TargetName, "waitState", waitState)
backupSession.Status.Targets[i].SessionState = waitState
backupSession.Status.Targets[i].Try++
backupSession.Status.Targets[i].StartTime = &metav1.Time{Time: time.Now()}
return waitState
} else {
r.Log.V(0).Info("Target failed for the last time", "target", targetStatus.TargetName)
return formolv1alpha1.Failure
}
case waitState:
// target is still busy with its current state. Wait until it is done.
r.Log.V(0).Info("Waiting for one target to finish", "waitState", waitState)
return ""
default:
if i == len(backupSession.Status.Targets)-1 {
r.Log.V(0).Info("Moving to next state", "nextState", nextState)
return nextState
}
}
}
return ""
}
func (r *BackupSessionReconciler) checkInitialized(backupSession *formolv1alpha1.BackupSession, backupConf formolv1alpha1.BackupConfiguration) formolv1alpha1.SessionState {
return r.checkSessionState(backupSession, backupConf, "", formolv1alpha1.Initializing, formolv1alpha1.Running)
}
func (r *BackupSessionReconciler) checkWaiting(backupSession *formolv1alpha1.BackupSession, backupConf formolv1alpha1.BackupConfiguration) formolv1alpha1.SessionState {
return r.checkSessionState(backupSession, backupConf, formolv1alpha1.Initialized, formolv1alpha1.Running, formolv1alpha1.Finalize)
}
func (r *BackupSessionReconciler) checkSuccess(backupSession *formolv1alpha1.BackupSession, backupConf formolv1alpha1.BackupConfiguration) formolv1alpha1.SessionState {
return r.checkSessionState(backupSession, backupConf, formolv1alpha1.Waiting, formolv1alpha1.Finalize, formolv1alpha1.Success)
}