Restore session with post restore steps works

Makefile

@@ -5,6 +5,9 @@ IMG ?= desmo999r/formolcli:latest
 formolcli: fmt vet
 	GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -o bin/formolcli main.go
 
+test: fmt vet
+	go test ./... -coverprofile cover.out
+
 fmt:
 	go fmt ./...
 

pkg/controllers/backupsession_controller.go

@@ -44,52 +44,98 @@ func (r *BackupSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
-	deploymentName := os.Getenv("POD_DEPLOYMENT")
+	deploymentName := os.Getenv(formolv1alpha1.TARGET_NAME)
 	for _, target := range backupConf.Spec.Targets {
 		switch target.Kind {
-		case "Deployment":
+		case formolv1alpha1.SidecarKind:
 			if target.Name == deploymentName {
-				for i, status := range backupSession.Status.Targets {
+				// We are involved in that Backup, let's see if it's our turn
-					if status.Name == target.Name {
+				status := &(backupSession.Status.Targets[len(backupSession.Status.Targets)-1])
-						log.V(0).Info("It's for us!", "target", target)
+				if status.Name == deploymentName {
-						switch status.SessionState {
+					log.V(0).Info("It's for us!", "target", target)
-						case formolv1alpha1.New:
+					switch status.SessionState {
-							// TODO: Run beforeBackup
+					case formolv1alpha1.New:
-							log.V(0).Info("New session, run the beforeBackup hooks if any")
+						log.V(0).Info("New session, move to Initializing state")
-							result := formolv1alpha1.Running
+						status.SessionState = formolv1alpha1.Init
-							if err := formolcliutils.RunBeforeBackup(target); err != nil {
+						if err := r.Status().Update(ctx, backupSession); err != nil {
-								result = formolv1alpha1.Failure
+							log.Error(err, "unable to update backupsession status")
-							}
+							return ctrl.Result{}, err
-							backupSession.Status.Targets[i].SessionState = result
-							log.V(1).Info("current backupSession status", "status", backupSession.Status)
-							if err := r.Status().Update(ctx, backupSession); err != nil {
-								log.Error(err, "unable to update backupsession status")
-								return ctrl.Result{}, err
-							}
-						case formolv1alpha1.Running:
-							log.V(0).Info("Running session. Do the backup")
-							result := formolv1alpha1.Success
-							status.StartTime = &metav1.Time{Time: time.Now()}
-							output, err := restic.BackupPaths(backupSession.Name, target.Paths)
-							if err != nil {
-								log.Error(err, "unable to backup deployment", "output", string(output))
-								result = formolv1alpha1.Failure
-							} else {
-								snapshotId := restic.GetBackupResults(output)
-								backupSession.Status.Targets[i].SnapshotId = snapshotId
-								backupSession.Status.Targets[i].Duration = &metav1.Duration{Duration: time.Now().Sub(backupSession.Status.Targets[i].StartTime.Time)}
-							}
-							backupSession.Status.Targets[i].SessionState = result
-							log.V(1).Info("current backupSession status", "status", backupSession.Status)
-							if err := r.Status().Update(ctx, backupSession); err != nil {
-								log.Error(err, "unable to update backupsession status")
-								return ctrl.Result{}, err
-							}
-						case formolv1alpha1.Success, formolv1alpha1.Failure:
-							// I decided not to flag the backup as a failure if the AfterBackup command fail. But maybe I'm wrong
-							log.V(0).Info("Backup is over, run the afterBackup hooks if any")
-							formolcliutils.RunAfterBackup(target)
 						}
+					case formolv1alpha1.Init:
+						log.V(0).Info("Start to run the backup initializing steps if any")
+						result := formolv1alpha1.Running
+						for _, step := range target.Steps {
+							if step.Finalize != nil && *step.Finalize == true {
+								continue
+							}
+							function := &formolv1alpha1.Function{}
+							if err := r.Get(ctx, client.ObjectKey{
+								Name:      step.Name,
+								Namespace: backupConf.Namespace,
+							}, function); err != nil {
+								log.Error(err, "unable to get function", "function", step.Name)
+								return ctrl.Result{}, err
+							}
+							if err := formolcliutils.RunChroot(function.Spec.Command[0], function.Spec.Command[1:]...); err != nil {
+								log.Error(err, "unable to run function command", "command", function.Spec.Command)
+								result = formolv1alpha1.Failure
+								break
+							}
+						}
+						status.SessionState = result
+
+						if err := r.Status().Update(ctx, backupSession); err != nil {
+							log.Error(err, "unable to update backupsession status")
+							return ctrl.Result{}, err
+						}
+					case formolv1alpha1.Running:
+						log.V(0).Info("Running session. Do the backup")
+						result := formolv1alpha1.Finalize
+						status.StartTime = &metav1.Time{Time: time.Now()}
+						output, err := restic.BackupPaths(backupSession.Name, target.Paths)
+						if err != nil {
+							log.Error(err, "unable to backup deployment", "output", string(output))
+							result = formolv1alpha1.Failure
+						} else {
+							snapshotId := restic.GetBackupResults(output)
+							status.SnapshotId = snapshotId
+							status.Duration = &metav1.Duration{Duration: time.Now().Sub(status.StartTime.Time)}
+						}
+						status.SessionState = result
+						log.V(1).Info("current backupSession status", "status", backupSession.Status)
+						if err := r.Status().Update(ctx, backupSession); err != nil {
+							log.Error(err, "unable to update backupsession status")
+							return ctrl.Result{}, err
+						}
+					case formolv1alpha1.Finalize:
+						log.V(0).Info("Start to run the backup finalizing steps if any")
+						result := formolv1alpha1.Success
+						for _, step := range target.Steps {
+							if step.Finalize != nil && *step.Finalize == true {
+								function := &formolv1alpha1.Function{}
+								if err := r.Get(ctx, client.ObjectKey{
+									Name:      step.Name,
+									Namespace: backupConf.Namespace,
+								}, function); err != nil {
+									log.Error(err, "unable to get function", "function", step.Name)
+									return ctrl.Result{}, err
+								}
+								if err := formolcliutils.RunChroot(function.Spec.Command[0], function.Spec.Command[1:]...); err != nil {
+									log.Error(err, "unable to run function command", "command", function.Spec.Command)
+									result = formolv1alpha1.Failure
+									break
+								}
+							}
+						}
+						status.SessionState = result
+
+						if err := r.Status().Update(ctx, backupSession); err != nil {
+							log.Error(err, "unable to update backupsession status")
+							return ctrl.Result{}, err
+						}
+
+					case formolv1alpha1.Success, formolv1alpha1.Failure:
+						log.V(0).Info("Backup is over")
 					}
 				}
 			}

pkg/controllers/restoresession_controller.go

@@ -0,0 +1,153 @@
+package controllers
+
+import (
+	"context"
+	formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1"
+	formolcliutils "github.com/desmo999r/formolcli/pkg/utils"
+	"github.com/go-logr/logr"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"os"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"strings"
+	"time"
+)
+
+type RestoreSessionReconciler struct {
+	client.Client
+	Log    logr.Logger
+	Scheme *runtime.Scheme
+}
+
+func (r *RestoreSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
+	ctx := context.Background()
+	log := r.Log.WithValues("restoresession", req.NamespacedName)
+
+	restoreSession := &formolv1alpha1.RestoreSession{}
+	if err := r.Get(ctx, req.NamespacedName, restoreSession); err != nil {
+		log.Error(err, "unable to get restoresession")
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+	backupSession := &formolv1alpha1.BackupSession{}
+	if err := r.Get(ctx, client.ObjectKey{
+		Namespace: restoreSession.Namespace,
+		Name:      restoreSession.Spec.BackupSessionRef.Ref.Name,
+	}, backupSession); err != nil {
+		if errors.IsNotFound(err) {
+			backupSession = &formolv1alpha1.BackupSession{
+				Spec:   restoreSession.Spec.BackupSessionRef.Spec,
+				Status: restoreSession.Spec.BackupSessionRef.Status,
+			}
+			log.V(1).Info("generated backupsession", "backupsession", backupSession)
+		} else {
+			log.Error(err, "unable to get backupsession", "restoresession", restoreSession.Spec)
+			return ctrl.Result{}, client.IgnoreNotFound(err)
+		}
+	}
+	backupConf := &formolv1alpha1.BackupConfiguration{}
+	if err := r.Get(ctx, client.ObjectKey{
+		Namespace: backupSession.Spec.Ref.Namespace,
+		Name:      backupSession.Spec.Ref.Name,
+	}, backupConf); err != nil {
+		log.Error(err, "unable to get backupConfiguration")
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+	deploymentName := os.Getenv(formolv1alpha1.TARGET_NAME)
+	currentTargetStatus := &(restoreSession.Status.Targets[len(restoreSession.Status.Targets)-1])
+	currentTarget := backupConf.Spec.Targets[len(restoreSession.Status.Targets)-1]
+	switch currentTarget.Kind {
+	case formolv1alpha1.SidecarKind:
+		if currentTarget.Name == deploymentName {
+			switch currentTargetStatus.SessionState {
+			case formolv1alpha1.Finalize:
+				log.V(0).Info("It's for us!")
+				podName := os.Getenv(formolv1alpha1.POD_NAME)
+				podNamespace := os.Getenv(formolv1alpha1.POD_NAMESPACE)
+				pod := &corev1.Pod{}
+				if err := r.Get(ctx, client.ObjectKey{
+					Namespace: podNamespace,
+					Name:      podName,
+				}, pod); err != nil {
+					log.Error(err, "unable to get pod", "name", podName, "namespace", podNamespace)
+					return ctrl.Result{}, err
+				}
+				for _, containerStatus := range pod.Status.ContainerStatuses {
+					if !containerStatus.Ready {
+						log.V(0).Info("Not all the containers in the pod are ready. Reschedule", "name", containerStatus.Name)
+						return ctrl.Result{RequeueAfter: 10 * time.Second}, nil
+					}
+				}
+				log.V(0).Info("All the containers in the pod are ready. Time to run the restore steps (in reverse order)")
+				// We iterate through the steps in reverse order
+				result := formolv1alpha1.Success
+				for i := range currentTarget.Steps {
+					step := currentTarget.Steps[len(currentTarget.Steps)-1-i]
+					backupFunction := &formolv1alpha1.Function{}
+					if err := r.Get(ctx, client.ObjectKey{
+						Namespace: backupConf.Namespace,
+						Name:      step.Name,
+					}, backupFunction); err != nil {
+						log.Error(err, "unable to get backup function")
+						return ctrl.Result{}, err
+					}
+					// We got the backup function corresponding to the step from the BackupConfiguration
+					// Now let's try to get the restore function is there is one
+					restoreFunction := &formolv1alpha1.Function{}
+					if restoreFunctionName, exists := backupFunction.Annotations[formolv1alpha1.RESTORE_ANNOTATION]; exists {
+						log.V(0).Info("got restore function", "name", restoreFunctionName)
+						if err := r.Get(ctx, client.ObjectKey{
+							Namespace: backupConf.Namespace,
+							Name:      restoreFunctionName,
+						}, restoreFunction); err != nil {
+							log.Error(err, "unable to get restore function")
+							continue
+						}
+					} else {
+						if strings.HasPrefix(backupFunction.Name, "backup-") {
+							log.V(0).Info("backupFunction starts with 'backup-'", "name", backupFunction.Name)
+							if err := r.Get(ctx, client.ObjectKey{
+								Namespace: backupConf.Namespace,
+								Name:      strings.Replace(backupFunction.Name, "backup-", "restore-", 1),
+							}, restoreFunction); err != nil {
+								log.Error(err, "unable to get restore function")
+								continue
+							}
+						}
+					}
+					if len(restoreFunction.Spec.Command) > 1 {
+						log.V(0).Info("Running the restore function", "name", restoreFunction.Name, "command", restoreFunction.Spec.Command)
+						if err := formolcliutils.RunChroot(restoreFunction.Spec.Command[0], restoreFunction.Spec.Command[1:]...); err != nil {
+							log.Error(err, "unable to run function command", "command", restoreFunction.Spec.Command)
+							result = formolv1alpha1.Failure
+							break
+						} else {
+							log.V(0).Info("Restore command is successful")
+						}
+					}
+				}
+				// We are done with the restore of this target. We flag it as success of failure
+				// so that we can move to the next step
+				currentTargetStatus.SessionState = result
+				if err := r.Status().Update(ctx, restoreSession); err != nil {
+					log.Error(err, "unable to update restoresession")
+				}
+			}
+		}
+	}
+
+	return ctrl.Result{}, nil
+}
+
+func (r *RestoreSessionReconciler) SetupWithManager(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&formolv1alpha1.RestoreSession{}).
+		WithEventFilter(predicate.Funcs{
+			CreateFunc: func(e event.CreateEvent) bool { return false },
+			DeleteFunc: func(e event.DeleteEvent) bool { return false },
+		}).
+		Complete(r)
+}

pkg/restore/root.go

@@ -26,10 +26,10 @@ func init() {
 
 func RestoreVolume(snapshotId string) error {
 	log := logger.WithName("restore-volume")
-	if err := session.RestoreSessionUpdateTargetStatus(formolv1alpha1.Running); err != nil {
+	if err := session.RestoreSessionUpdateTargetStatus(formolv1alpha1.Init); err != nil {
 		return err
 	}
-	state := formolv1alpha1.Success
+	state := formolv1alpha1.Waiting
 	output, err := restic.RestorePaths(snapshotId)
 	if err != nil {
 		log.Error(err, "unable to restore volume", "output", string(output))

pkg/server/root.go

@@ -1,7 +1,7 @@
 package server
 
 import (
-	"flag"
+	//	"flag"
 	formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1"
 	"github.com/desmo999r/formolcli/pkg/controllers"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -22,30 +22,31 @@ func init() {
 }
 
 func Server() {
-	var metricsAddr string
+	//	var metricsAddr string
-	var enableLeaderElection bool
+	//	var enableLeaderElection bool
-	flag.StringVar(&metricsAddr, "metrics-addr", ":8082", "The address the metric endpoint binds to.")
+	//	flag.StringVar(&metricsAddr, "metrics-addr", ":8082", "The address the metric endpoint binds to.")
-	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
+	//	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
-		"Enable leader election for controller manager. "+
+	//		"Enable leader election for controller manager. "+
-			"Enabling this will ensure there is only one active controller manager.")
+	//			"Enabling this will ensure there is only one active controller manager.")
-	flag.Parse()
+	//	flag.Parse()
 
 	ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
 
 	config, err := ctrl.GetConfig()
 	mgr, err := ctrl.NewManager(config, ctrl.Options{
-		Scheme:             scheme,
+		Scheme: scheme,
-		MetricsBindAddress: metricsAddr,
+		//		MetricsBindAddress: metricsAddr,
-		Port:               9443,
+		//		Port:               9443,
-		LeaderElection:     enableLeaderElection,
+		//		LeaderElection:     enableLeaderElection,
-		LeaderElectionID:   "12345.desmojim.fr",
+		//		LeaderElectionID:   "12345.desmojim.fr",
-		Namespace:          os.Getenv("POD_NAMESPACE"),
+		Namespace: os.Getenv("POD_NAMESPACE"),
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to create manager")
 		os.Exit(1)
 	}
 
+	// BackupSession controller
 	if err = (&controllers.BackupSessionReconciler{
 		Client: mgr.GetClient(),
 		Log:    ctrl.Log.WithName("controllers").WithName("BackupSession"),
@@ -55,6 +56,16 @@ func Server() {
 		os.Exit(1)
 	}
 
+	// RestoreSession controller
+	if err = (&controllers.RestoreSessionReconciler{
+		Client: mgr.GetClient(),
+		Log:    ctrl.Log.WithName("controllers").WithName("RestoreSession"),
+		Scheme: mgr.GetScheme(),
+	}).SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "RestoreSession")
+		os.Exit(1)
+	}
+
 	setupLog.Info("starting manager")
 	if err = mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "problem running the manager")

pkg/session/root.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1"
 	"github.com/go-logr/logr"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
@@ -61,12 +62,11 @@ func BackupSessionUpdateTargetStatus(state formolv1alpha1.SessionState, snapshot
 		log.Error(err, "unable to get backupsession", "BACKUPSESSION_NAME", os.Getenv("BACKUPSESSION_NAME"), "BACKUPSESSION_NAMESPACE", os.Getenv("BACKUPSESSION_NAMESPACE"))
 		return err
 	}
-	for i, target := range backupSession.Status.Targets {
+	target := &(backupSession.Status.Targets[len(backupSession.Status.Targets)-1])
-		if target.Name == targetName {
+	if target.Name == targetName {
-			backupSession.Status.Targets[i].SessionState = state
+		target.SessionState = state
-			backupSession.Status.Targets[i].SnapshotId = snapshotId
+		target.SnapshotId = snapshotId
-			backupSession.Status.Targets[i].Duration = &metav1.Duration{Duration: time.Now().Sub(backupSession.Status.Targets[i].StartTime.Time)}
+		target.Duration = &metav1.Duration{Duration: time.Now().Sub(target.StartTime.Time)}
-		}
 	}
 
 	if err := cl.Status().Update(context.Background(), backupSession); err != nil {
@@ -128,8 +128,9 @@ func CreateBackupSession(name string, namespace string) {
 			Namespace: namespace,
 		},
 		Spec: formolv1alpha1.BackupSessionSpec{
-			Ref: formolv1alpha1.Ref{
+			Ref: corev1.ObjectReference{
-				Name: name,
+				Name:      name,
+				Namespace: namespace,
 			},
 		},
 	}

pkg/utils/root.go

@@ -19,9 +19,7 @@ func init() {
 	logger = zapr.NewLogger(zapLog)
 }
 
-func runHook(hooks []formolv1alpha1.Hook, label string) error {
+func RunHooks(hooks []formolv1alpha1.Hook) error {
-	log := logger.WithName(label)
-	log.V(0).Info("Run commands")
 	for _, hook := range hooks {
 		err := RunChroot(hook.Cmd, hook.Args...)
 		if err != nil {
@@ -78,11 +76,3 @@ func RunChroot(runCmd string, args ...string) error {
 	}
 	return nil
 }
-
-func RunBeforeBackup(target formolv1alpha1.Target) error {
-	return runHook(target.BeforeBackup, "runBeforeBackup")
-}
-
-func RunAfterBackup(target formolv1alpha1.Target) error {
-	return runHook(target.AfterBackup, "runAfterBackup")
-}