From 077a54079fbd2907dff37dcb209bb1fa86325670 Mon Sep 17 00:00:00 2001 From: Jean-Marc Andre Date: Thu, 11 Feb 2021 22:07:27 +0100 Subject: [PATCH] Added restore functionality. Needs more testing --- controllers/backupconfiguration_controller.go | 31 ++--- controllers/backupsession_controller.go | 6 +- controllers/restoresession_controller.go | 125 +++++++++++++++++- pkg/rbac/backupconfiguration.go | 108 ++++++++++++++- test/00-setup.yaml | 2 +- test/02-backupconf.yaml | 2 +- 6 files changed, 245 insertions(+), 29 deletions(-) diff --git a/controllers/backupconfiguration_controller.go b/controllers/backupconfiguration_controller.go index cbaf3b0..787b965 100644 --- a/controllers/backupconfiguration_controller.go +++ b/controllers/backupconfiguration_controller.go @@ -18,6 +18,7 @@ package controllers import ( "context" + "time" formolrbac "github.com/desmo999r/formol/pkg/rbac" formolutils "github.com/desmo999r/formol/pkg/utils" @@ -54,16 +55,15 @@ func (r *BackupConfigurationReconciler) getDeployment(namespace string, name str return deployment, err } -// +kubebuilder:rbac:groups=formol.desmojim.fr,resources=backupconfigurations,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=formol.desmojim.fr,resources=backupconfigurations/status,verbs=get;list;watch;update;patch -// +kubebuilder:rbac:groups=formol.desmojim.fr,resources=backupsessions/status,verbs=get;list;watch -// +kubebuilder:rbac:groups=formol.desmojim.fr,resources=repoes,verbs=get;list;watch +// +kubebuilder:rbac:groups=formol.desmojim.fr,resources=*,verbs=* // +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=apps,resources=replicasets,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch // +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterroles,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterrolebindings,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=batch,resources=cronjobs,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=batch,resources=cronjobs/status,verbs=get @@ -135,7 +135,7 @@ func (r *BackupConfigurationReconciler) addSidecarContainer(backupConf *formolv1 sidecar := corev1.Container{ Name: "backup", Image: "desmo999r/formolcli:latest", - Args: []string{"create", "server"}, + Args: []string{"backupsession", "server"}, //Image: "busybox", //Command: []string{ // "sh", @@ -190,7 +190,7 @@ func (r *BackupConfigurationReconciler) addSidecarContainer(backupConf *formolv1 } log.V(1).Info("getting pods matching label", "label", selector) pods := &corev1.PodList{} - err = r.List(context.Background(), pods, client.MatchingLabels(selector)) + err = r.List(context.Background(), pods, client.InNamespace(backupConf.Namespace), client.MatchingLabels(selector)) if err != nil { log.Error(err, "unable to get deployment pods") return nil @@ -220,14 +220,10 @@ func (r *BackupConfigurationReconciler) addSidecarContainer(backupConf *formolv1 deployment.Spec.Template.Spec.Containers = append(deployment.Spec.Template.Spec.Containers, sidecar) deployment.Spec.Template.Spec.ShareProcessNamespace = func() *bool { b := true; return &b }() - if err := formolrbac.CreateBackupSessionListenerRBAC(r.Client, deployment.Spec.Template.Spec.ServiceAccountName, deployment.Namespace); err != nil { + if err := formolrbac.CreateFormolRBAC(r.Client, deployment.Spec.Template.Spec.ServiceAccountName, deployment.Namespace); err != nil { log.Error(err, "unable to create backupsessionlistener RBAC") return nil } - if err := formolrbac.CreateBackupSessionStatusUpdaterRBAC(r.Client, "default", backupConf.Namespace); err != nil { - log.Error(err, "unable to create backupsession-statusupdater RBAC") - return nil - } log.V(0).Info("Adding a sicar container") if err := r.Update(context.Background(), deployment); err != nil { @@ -266,10 +262,11 @@ func (r *BackupConfigurationReconciler) addCronJob(backupConf *formolv1alpha1.Ba return err } - if err := formolrbac.CreateBackupSessionCreatorRBAC(r.Client, backupConf.Namespace); err != nil { - log.Error(err, "unable to create backupsession-creator RBAC") + if err := formolrbac.CreateFormolRBAC(r.Client, "default", backupConf.Namespace); err != nil { + log.Error(err, "unable to create backupsessionlistener RBAC") return nil } + if err := formolrbac.CreateBackupSessionStatusUpdaterRBAC(r.Client, "default", backupConf.Namespace); err != nil { log.Error(err, "unable to create backupsession-statusupdater RBAC") return nil @@ -293,8 +290,8 @@ func (r *BackupConfigurationReconciler) addCronJob(backupConf *formolv1alpha1.Ba Name: "job-createbackupsession-" + backupConf.Name, Image: "desmo999r/formolcli:latest", Args: []string{ - "create", "backupsession", + "create", "--namespace", backupConf.Namespace, "--name", @@ -323,6 +320,7 @@ func (r *BackupConfigurationReconciler) addCronJob(backupConf *formolv1alpha1.Ba func (r *BackupConfigurationReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { ctx := context.Background() log := r.Log.WithValues("backupconfiguration", req.NamespacedName) + time.Sleep(300 * time.Millisecond) log.V(1).Info("Enter Reconcile with req", "req", req) @@ -394,7 +392,7 @@ func (r *BackupConfigurationReconciler) deleteExternalResources(backupConf *form if err != nil { return err } - if err := formolrbac.DeleteBackupSessionListenerRBAC(r.Client, deployment.Spec.Template.Spec.ServiceAccountName, deployment.Namespace); err != nil { + if err := formolrbac.DeleteFormolRBAC(r.Client, deployment.Spec.Template.Spec.ServiceAccountName, deployment.Namespace); err != nil { return err } if err := r.deleteSidecarContainer(backupConf, target); err != nil { @@ -405,6 +403,9 @@ func (r *BackupConfigurationReconciler) deleteExternalResources(backupConf *form } } // TODO: remove the hardcoded "default" + if err := formolrbac.DeleteFormolRBAC(r.Client, "default", backupConf.Namespace); err != nil { + return err + } if err := formolrbac.DeleteBackupSessionStatusUpdaterRBAC(r.Client, "default", backupConf.Namespace); err != nil { return err } diff --git a/controllers/backupsession_controller.go b/controllers/backupsession_controller.go index df51a99..a763b3e 100644 --- a/controllers/backupsession_controller.go +++ b/controllers/backupsession_controller.go @@ -63,6 +63,7 @@ func (r *BackupSessionReconciler) StatusUpdate() error { Name: target.Name, Kind: target.Kind, SessionState: formolv1alpha1.New, + StartTime: &metav1.Time{Time: time.Now()}, } r.BackupSession.Status.Targets = append(r.BackupSession.Status.Targets, targetStatus) switch target.Kind { @@ -247,7 +248,6 @@ func (r *BackupSessionReconciler) IsBackupOngoing() bool { // +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;create;update;patch;delete;watch func (r *BackupSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { - time.Sleep(100 * time.Millisecond) log := r.Log.WithValues("backupsession", req.NamespacedName) ctx := context.Background() @@ -338,7 +338,7 @@ func (r *BackupSessionReconciler) CreateBackupJob(target formolv1alpha1.Target) restic := corev1.Container{ Name: "restic", Image: "desmo999r/formolcli:latest", - Args: []string{"backup", "volume", "--tag", r.BackupSession.Name, "--path", "/output"}, + Args: []string{"volume", "backup", "--tag", r.BackupSession.Name, "--path", "/output"}, VolumeMounts: []corev1.VolumeMount{output}, Env: backupSessionEnv, } @@ -418,7 +418,7 @@ func (r *BackupSessionReconciler) deleteExternalResources() error { deleteSnapshots = append(deleteSnapshots, corev1.Container{ Name: target.Name, Image: "desmo999r/formolcli:latest", - Args: []string{"delete", "snapshot", "--snapshot", target.SnapshotId}, + Args: []string{"snapshot", "delete", "--snapshot-id", target.SnapshotId}, Env: env, }) } diff --git a/controllers/restoresession_controller.go b/controllers/restoresession_controller.go index 9153195..807917e 100644 --- a/controllers/restoresession_controller.go +++ b/controllers/restoresession_controller.go @@ -18,16 +18,20 @@ package controllers import ( "context" + "fmt" + "strings" "time" "github.com/go-logr/logr" batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1" + formolutils "github.com/desmo999r/formol/pkg/utils" ) // RestoreSessionReconciler reconciles a RestoreSession object @@ -41,6 +45,97 @@ type RestoreSessionReconciler struct { } func (r *RestoreSessionReconciler) CreateRestoreJob(target formolv1alpha1.Target) error { + log := r.Log.WithValues("createrestorejob", target.Name) + ctx := context.Background() + restoreSessionEnv := []corev1.EnvVar{ + corev1.EnvVar{ + Name: "TARGET_NAME", + Value: target.Name, + }, + corev1.EnvVar{ + Name: "RESTORESESSION_NAME", + Value: r.RestoreSession.Name, + }, + corev1.EnvVar{ + Name: "RESTORESESSION_NAMESPACE", + Value: r.RestoreSession.Namespace, + }, + } + + output := corev1.VolumeMount{ + Name: "output", + MountPath: "/output", + } + for _, targetStatus := range r.BackupSession.Status.Targets { + if targetStatus.Name == target.Name { + snapshotId := targetStatus.SnapshotId + restic := corev1.Container{ + Name: "restic", + Image: "desmo999r/formolcli:latest", + Args: []string{"volume", "restore", "--snapshot-id", snapshotId}, + VolumeMounts: []corev1.VolumeMount{output}, + Env: restoreSessionEnv, + } + finalizer := corev1.Container{ + Name: "finalizer", + Image: "desmo999r/formolcli:latest", + Args: []string{"target", "finalize"}, + VolumeMounts: []corev1.VolumeMount{output}, + Env: restoreSessionEnv, + } + repo := &formolv1alpha1.Repo{} + if err := r.Get(ctx, client.ObjectKey{ + Namespace: r.BackupConf.Namespace, + Name: r.BackupConf.Spec.Repository.Name, + }, repo); err != nil { + log.Error(err, "unable to get Repo from BackupConfiguration") + return err + } + // S3 backing storage + var ttl int32 = 300 + restic.Env = append(restic.Env, formolutils.ConfigureResticEnvVar(r.BackupConf, repo)...) + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: fmt.Sprintf("%s-%s-", r.RestoreSession.Name, target.Name), + Namespace: r.RestoreSession.Namespace, + }, + Spec: batchv1.JobSpec{ + TTLSecondsAfterFinished: &ttl, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{restic}, + Containers: []corev1.Container{finalizer}, + Volumes: []corev1.Volume{ + corev1.Volume{Name: "output"}, + }, + RestartPolicy: corev1.RestartPolicyOnFailure, + }, + }, + }, + } + for _, step := range target.Steps { + function := &formolv1alpha1.Function{} + if err := r.Get(ctx, client.ObjectKey{ + Namespace: r.RestoreSession.Namespace, + Name: strings.Replace(step.Name, "backup", "restore", 1)}, function); err != nil { + log.Error(err, "unable to get function", "function", step) + return err + } + function.Spec.Env = append(step.Env, restoreSessionEnv...) + function.Spec.VolumeMounts = append(function.Spec.VolumeMounts, output) + job.Spec.Template.Spec.InitContainers = append(job.Spec.Template.Spec.InitContainers, function.Spec) + } + if err := ctrl.SetControllerReference(r.RestoreSession, job, r.Scheme); err != nil { + log.Error(err, "unable to set controller on job", "job", job, "restoresession", r.RestoreSession) + return err + } + log.V(0).Info("creating a restore job", "target", target.Name) + if err := r.Create(ctx, job); err != nil { + log.Error(err, "unable to create job", "job", job) + return err + } + } + } return nil } @@ -55,6 +150,7 @@ func (r *RestoreSessionReconciler) StatusUpdate() error { Name: target.Name, Kind: target.Kind, SessionState: formolv1alpha1.New, + StartTime: &metav1.Time{Time: time.Now()}, } r.RestoreSession.Status.Targets = append(r.RestoreSession.Status.Targets, targetStatus) switch target.Kind { @@ -70,7 +166,6 @@ func (r *RestoreSessionReconciler) StatusUpdate() error { return nil, nil } } - var ret error switch r.RestoreSession.Status.SessionState { case formolv1alpha1.New: r.RestoreSession.Status.SessionState = formolv1alpha1.Running @@ -79,18 +174,38 @@ func (r *RestoreSessionReconciler) StatusUpdate() error { return err } log.V(0).Info("New restore. Start the first task", "task", targetStatus.Name) + case formolv1alpha1.Running: + currentTargetStatus := r.RestoreSession.Status.Targets[len(r.RestoreSession.Status.Targets)-1] + switch currentTargetStatus.SessionState { + case formolv1alpha1.Failure: + log.V(0).Info("last restore task failed. Stop here", "target", currentTargetStatus.Name) + r.RestoreSession.Status.SessionState = formolv1alpha1.Failure + case formolv1alpha1.Running: + log.V(0).Info("task is still running", "target", currentTargetStatus.Name) + return nil + case formolv1alpha1.Success: + log.V(0).Info("last task was a success. start a new one", "target", currentTargetStatus) + targetStatus, err := startNextTask() + if err != nil { + return err + } + if targetStatus == nil { + // No more task to start. The restore is over + r.RestoreSession.Status.SessionState = formolv1alpha1.Success + } + } } - if ret = r.Status().Update(ctx, r.RestoreSession); ret != nil { - log.Error(ret, "unable to update restoresession") + if err := r.Status().Update(ctx, r.RestoreSession); err != nil { + log.Error(err, "unable to update restoresession") + return err } - return ret + return nil } // +kubebuilder:rbac:groups=formol.desmojim.fr,resources=restoresessions,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=formol.desmojim.fr,resources=restoresessions/status,verbs=get;update;patch func (r *RestoreSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { - time.Sleep(100 * time.Millisecond) ctx := context.Background() log := r.Log.WithValues("restoresession", req.NamespacedName) diff --git a/pkg/rbac/backupconfiguration.go b/pkg/rbac/backupconfiguration.go index d1583e5..224317f 100644 --- a/pkg/rbac/backupconfiguration.go +++ b/pkg/rbac/backupconfiguration.go @@ -10,6 +10,7 @@ import ( ) const ( + formolRole = "formol-sidecar-role" backupListenerRole = "backup-listener-role" backupListenerRoleBinding = "backup-listener-rolebinding" backupSessionCreatorSA = "backupsession-creator" @@ -162,6 +163,104 @@ func DeleteBackupSessionListenerRBAC(cl client.Client, saName string, namespace return nil } +func DeleteFormolRBAC(cl client.Client, saName string, namespace string) error { + if saName == "" { + saName = "default" + } + formolRoleBinding := namespace + "-" + saName + "-formol-sidecar-rolebinding" + clusterRoleBinding := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: formolRoleBinding, + }, + Subjects: []rbacv1.Subject{ + rbacv1.Subject{ + Kind: "ServiceAccount", + Namespace: namespace, + Name: saName, + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: formolRole, + }, + } + if err := cl.Delete(context.Background(), clusterRoleBinding); err != nil { + return client.IgnoreNotFound(err) + } + return nil +} + +func CreateFormolRBAC(cl client.Client, saName string, namespace string) error { + if saName == "" { + saName = "default" + } + sa := &corev1.ServiceAccount{} + if err := cl.Get(context.Background(), client.ObjectKey{ + Namespace: namespace, + Name: saName, + }, sa); err != nil { + return err + } + clusterRole := &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: formolRole, + }, + Rules: []rbacv1.PolicyRule{ + rbacv1.PolicyRule{ + Verbs: []string{"*"}, + APIGroups: []string{"formol.desmojim.fr"}, + Resources: []string{"*"}, + //APIGroups: []string{"formol.desmojim.fr"}, + //Resources: []string{"restoresessions", "backupsessions", "backupconfigurations"}, + }, + rbacv1.PolicyRule{ + Verbs: []string{"get", "list", "watch"}, + APIGroups: []string{""}, + Resources: []string{"pods"}, + }, + rbacv1.PolicyRule{ + Verbs: []string{"get", "list", "watch"}, + APIGroups: []string{"apps"}, + Resources: []string{"deployments", "replicasets"}, + }, + }, + } + if err := cl.Get(context.Background(), client.ObjectKey{ + Name: formolRole, + }, clusterRole); err != nil && errors.IsNotFound(err) { + if err = cl.Create(context.Background(), clusterRole); err != nil { + return err + } + } + formolRoleBinding := namespace + "-" + saName + "-formol-rolebinding" + clusterRoleBinding := &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: formolRoleBinding, + }, + Subjects: []rbacv1.Subject{ + rbacv1.Subject{ + Kind: "ServiceAccount", + Namespace: namespace, + Name: saName, + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: formolRole, + }, + } + if err := cl.Get(context.Background(), client.ObjectKey{ + Name: formolRoleBinding, + }, clusterRoleBinding); err != nil && errors.IsNotFound(err) { + if err = cl.Create(context.Background(), clusterRoleBinding); err != nil { + return err + } + } + return nil +} + func CreateBackupSessionListenerRBAC(cl client.Client, saName string, namespace string) error { if saName == "" { saName = "default" @@ -192,12 +291,12 @@ func CreateBackupSessionListenerRBAC(cl client.Client, saName string, namespace rbacv1.PolicyRule{ Verbs: []string{"get", "list", "watch"}, APIGroups: []string{"formol.desmojim.fr"}, - Resources: []string{"backupsessions", "backupconfigurations"}, + Resources: []string{"restoresessions", "backupsessions", "backupconfigurations"}, }, rbacv1.PolicyRule{ Verbs: []string{"update", "delete"}, APIGroups: []string{"formol.desmojim.fr"}, - Resources: []string{"backupsessions"}, + Resources: []string{"restoresessions", "backupsessions"}, }, }, } @@ -234,6 +333,7 @@ func CreateBackupSessionListenerRBAC(cl client.Client, saName string, namespace return err } } + return nil } @@ -292,12 +392,12 @@ func CreateBackupSessionStatusUpdaterRBAC(cl client.Client, saName string, names rbacv1.PolicyRule{ Verbs: []string{"get", "list", "watch", "patch", "update"}, APIGroups: []string{"formol.desmojim.fr"}, - Resources: []string{"backupsessions/status"}, + Resources: []string{"restoresessions/status", "backupsessions/status"}, }, rbacv1.PolicyRule{ Verbs: []string{"get", "list", "watch"}, APIGroups: []string{"formol.desmojim.fr"}, - Resources: []string{"backupsessions"}, + Resources: []string{"restoresessions", "backupsessions"}, }, }, } diff --git a/test/00-setup.yaml b/test/00-setup.yaml index bfb592a..2e88742 100644 --- a/test/00-setup.yaml +++ b/test/00-setup.yaml @@ -81,4 +81,4 @@ metadata: spec: name: backup-pg image: desmo999r/formolcli:latest - args: ["backup", "postgres", "--hostname", $(PGHOST), "--database", $(PGDATABASE), "--username", $(PGUSER), "--password", $(PGPASSWD), "--file", "/output/backup-pg.sql"] + args: ["postgres", "backup", "--hostname", $(PGHOST), "--database", $(PGDATABASE), "--username", $(PGUSER), "--password", $(PGPASSWD), "--file", "/output/backup-pg.sql"] diff --git a/test/02-backupconf.yaml b/test/02-backupconf.yaml index 848e709..a758593 100644 --- a/test/02-backupconf.yaml +++ b/test/02-backupconf.yaml @@ -7,7 +7,7 @@ metadata: spec: repository: name: repo-minio - schedule: "1 * * * *" + schedule: "15 * * * *" targets: - kind: Deployment apiVersion: v1