jandre/formol
1
1
Fork 0
Code Issues 6 Pull Requests Projects Releases 1 Wiki Activity

Added restore functionality. Needs more testing

Browse Source
This commit is contained in:
jandre 2021-02-11 22:07:27 +01:00
parent 38d6877962
commit 19db817b0c
6 changed files with 355 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
controllers/backupconfiguration_controller.go
View File

@ -18,6 +18,7 @@ package controllers
import (
"context"
"time"
formolrbac "github.com/desmo999r/formol/pkg/rbac"
formolutils "github.com/desmo999r/formol/pkg/utils"
@ -54,16 +55,15 @@ func (r *BackupConfigurationReconciler) getDeployment(namespace string, name str
return deployment, err
}
// +kubebuilder:rbac:groups=formol.desmojim.fr,resources=backupconfigurations,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=formol.desmojim.fr,resources=backupconfigurations/status,verbs=get;list;watch;update;patch
// +kubebuilder:rbac:groups=formol.desmojim.fr,resources=backupsessions/status,verbs=get;list;watch
// +kubebuilder:rbac:groups=formol.desmojim.fr,resources=repoes,verbs=get;list;watch
// +kubebuilder:rbac:groups=formol.desmojim.fr,resources=*,verbs=*
// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=apps,resources=replicasets,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch
// +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterroles,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=clusterrolebindings,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=batch,resources=cronjobs,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=batch,resources=cronjobs/status,verbs=get
@ -135,7 +135,7 @@ func (r *BackupConfigurationReconciler) addSidecarContainer(backupConf *formolv1
sidecar := corev1.Container{
Name: "backup",
Image: "desmo999r/formolcli:latest",
Args: []string{"create", "server"},
Args: []string{"backupsession", "server"},
//Image: "busybox",
//Command: []string{
// "sh",
@ -190,7 +190,7 @@ func (r *BackupConfigurationReconciler) addSidecarContainer(backupConf *formolv1
}
log.V(1).Info("getting pods matching label", "label", selector)
pods := &corev1.PodList{}
err = r.List(context.Background(), pods, client.MatchingLabels(selector))
err = r.List(context.Background(), pods, client.InNamespace(backupConf.Namespace), client.MatchingLabels(selector))
if err != nil {
log.Error(err, "unable to get deployment pods")
return nil
@ -220,7 +220,7 @@ func (r *BackupConfigurationReconciler) addSidecarContainer(backupConf *formolv1
deployment.Spec.Template.Spec.Containers = append(deployment.Spec.Template.Spec.Containers, sidecar)
deployment.Spec.Template.Spec.ShareProcessNamespace = func() *bool { b := true; return &b }()
if err := formolrbac.CreateBackupSessionListenerRBAC(r.Client, deployment.Spec.Template.Spec.ServiceAccountName, deployment.Namespace); err != nil {
if err := formolrbac.CreateFormolRBAC(r.Client, deployment.Spec.Template.Spec.ServiceAccountName, deployment.Namespace); err != nil {
log.Error(err, "unable to create backupsessionlistener RBAC")
return nil
}
@ -242,6 +242,11 @@ func (r *BackupConfigurationReconciler) addSidecarContainer(backupConf *formolv1
func (r *BackupConfigurationReconciler) addCronJob(backupConf *formolv1alpha1.BackupConfiguration) error {
log := r.Log.WithValues("addCronJob", backupConf.Name)
if err := formolrbac.CreateFormolRBAC(r.Client, "default", backupConf.Namespace); err != nil {
log.Error(err, "unable to create backupsessionlistener RBAC")
return nil
}
if err := formolrbac.CreateBackupSessionCreatorRBAC(r.Client, backupConf.Namespace); err != nil {
log.Error(err, "unable to create backupsession-creator RBAC")
return nil
@ -295,8 +300,8 @@ func (r *BackupConfigurationReconciler) addCronJob(backupConf *formolv1alpha1.Ba
Name: "job-createbackupsession-" + backupConf.Name,
Image: "desmo999r/formolcli:latest",
Args: []string{
"create",
"backupsession",
"create",
"--namespace",
backupConf.Namespace,
"--name",
@ -325,6 +330,7 @@ func (r *BackupConfigurationReconciler) addCronJob(backupConf *formolv1alpha1.Ba
func (r *BackupConfigurationReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
ctx := context.Background()
log := r.Log.WithValues("backupconfiguration", req.NamespacedName)
time.Sleep(300 * time.Millisecond)
log.V(1).Info("Enter Reconcile with req", "req", req)
@ -396,7 +402,7 @@ func (r *BackupConfigurationReconciler) deleteExternalResources(backupConf *form
if err != nil {
return err
}
if err := formolrbac.DeleteBackupSessionListenerRBAC(r.Client, deployment.Spec.Template.Spec.ServiceAccountName, deployment.Namespace); err != nil {
if err := formolrbac.DeleteFormolRBAC(r.Client, deployment.Spec.Template.Spec.ServiceAccountName, deployment.Namespace); err != nil {
return err
}
if err := formolrbac.DeleteBackupSessionCreatorRBAC(r.Client, backupConf.Namespace); err != nil {
@ -407,6 +413,16 @@ func (r *BackupConfigurationReconciler) deleteExternalResources(backupConf *form
}
}
}
// TODO: remove the hardcoded "default"
if err := formolrbac.DeleteFormolRBAC(r.Client, "default", backupConf.Namespace); err != nil {
return err
}
if err := formolrbac.DeleteBackupSessionStatusUpdaterRBAC(r.Client, "default", backupConf.Namespace); err != nil {
return err
}
if err := formolrbac.DeleteBackupSessionCreatorRBAC(r.Client, backupConf.Namespace); err != nil {
return err
}
return nil
}

6
controllers/backupsession_controller.go
View File

@ -63,6 +63,7 @@ func (r *BackupSessionReconciler) StatusUpdate() error {
Name: target.Name,
Kind: target.Kind,
SessionState: formolv1alpha1.New,
StartTime: &metav1.Time{Time: time.Now()},
}
r.BackupSession.Status.Targets = append(r.BackupSession.Status.Targets, targetStatus)
switch target.Kind {
@ -247,7 +248,6 @@ func (r *BackupSessionReconciler) IsBackupOngoing() bool {
// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;create;update;patch;delete;watch
func (r *BackupSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
time.Sleep(100 * time.Millisecond)
log := r.Log.WithValues("backupsession", req.NamespacedName)
ctx := context.Background()
@ -338,7 +338,7 @@ func (r *BackupSessionReconciler) CreateBackupJob(target formolv1alpha1.Target)
restic := corev1.Container{
Name: "restic",
Image: "desmo999r/formolcli:latest",
Args: []string{"backup", "volume", "--tag", r.BackupSession.Name, "--path", "/output"},
Args: []string{"volume", "backup", "--tag", r.BackupSession.Name, "--path", "/output"},
VolumeMounts: []corev1.VolumeMount{output},
Env: backupSessionEnv,
}
@ -418,7 +418,7 @@ func (r *BackupSessionReconciler) deleteExternalResources() error {
deleteSnapshots = append(deleteSnapshots, corev1.Container{
Name: target.Name,
Image: "desmo999r/formolcli:latest",
Args: []string{"delete", "snapshot", "--snapshot", target.SnapshotId},
Args: []string{"snapshot", "delete", "--snapshot-id", target.SnapshotId},
Env: env,
})
}

124
controllers/restoresession_controller.go
View File

@ -18,16 +18,20 @@ package controllers
import (
"context"
"fmt"
"strings"
"time"
"github.com/go-logr/logr"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1"
formolutils "github.com/desmo999r/formol/pkg/utils"
)
// RestoreSessionReconciler reconciles a RestoreSession object
@ -41,6 +45,97 @@ type RestoreSessionReconciler struct {
}
func (r *RestoreSessionReconciler) CreateRestoreJob(target formolv1alpha1.Target) error {
log := r.Log.WithValues("createrestorejob", target.Name)
ctx := context.Background()
restoreSessionEnv := []corev1.EnvVar{
corev1.EnvVar{
Name: "TARGET_NAME",
Value: target.Name,
},
corev1.EnvVar{
Name: "RESTORESESSION_NAME",
Value: r.RestoreSession.Name,
},
corev1.EnvVar{
Name: "RESTORESESSION_NAMESPACE",
Value: r.RestoreSession.Namespace,
},
}
output := corev1.VolumeMount{
Name: "output",
MountPath: "/output",
}
for _, targetStatus := range r.BackupSession.Status.Targets {
if targetStatus.Name == target.Name {
snapshotId := targetStatus.SnapshotId
restic := corev1.Container{
Name: "restic",
Image: "desmo999r/formolcli:latest",
Args: []string{"volume", "restore", "--snapshot-id", snapshotId},
VolumeMounts: []corev1.VolumeMount{output},
Env: restoreSessionEnv,
}
finalizer := corev1.Container{
Name: "finalizer",
Image: "desmo999r/formolcli:latest",
Args: []string{"target", "finalize"},
VolumeMounts: []corev1.VolumeMount{output},
Env: restoreSessionEnv,
}
repo := &formolv1alpha1.Repo{}
if err := r.Get(ctx, client.ObjectKey{
Namespace: r.BackupConf.Namespace,
Name: r.BackupConf.Spec.Repository.Name,
}, repo); err != nil {
log.Error(err, "unable to get Repo from BackupConfiguration")
return err
}
// S3 backing storage
var ttl int32 = 300
restic.Env = append(restic.Env, formolutils.ConfigureResticEnvVar(r.BackupConf, repo)...)
job := &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
GenerateName: fmt.Sprintf("%s-%s-", r.RestoreSession.Name, target.Name),
Namespace: r.RestoreSession.Namespace,
},
Spec: batchv1.JobSpec{
TTLSecondsAfterFinished: &ttl,
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
InitContainers: []corev1.Container{restic},
Containers: []corev1.Container{finalizer},
Volumes: []corev1.Volume{
corev1.Volume{Name: "output"},
},
RestartPolicy: corev1.RestartPolicyOnFailure,
},
},
},
}
for _, step := range target.Steps {
function := &formolv1alpha1.Function{}
if err := r.Get(ctx, client.ObjectKey{
Namespace: r.RestoreSession.Namespace,
Name: strings.Replace(step.Name, "backup", "restore", 1)}, function); err != nil {
log.Error(err, "unable to get function", "function", step)
return err
}
function.Spec.Env = append(step.Env, restoreSessionEnv...)
function.Spec.VolumeMounts = append(function.Spec.VolumeMounts, output)
job.Spec.Template.Spec.InitContainers = append(job.Spec.Template.Spec.InitContainers, function.Spec)
}
if err := ctrl.SetControllerReference(r.RestoreSession, job, r.Scheme); err != nil {
log.Error(err, "unable to set controller on job", "job", job, "restoresession", r.RestoreSession)
return err
}
log.V(0).Info("creating a restore job", "target", target.Name)
if err := r.Create(ctx, job); err != nil {
log.Error(err, "unable to create job", "job", job)
return err
}
}
}
return nil
}
@ -59,6 +154,7 @@ func (r *RestoreSessionReconciler) StatusUpdate() error {
Name: target.Name,
Kind: target.Kind,
SessionState: formolv1alpha1.New,
StartTime: &metav1.Time{Time: time.Now()},
}
r.RestoreSession.Status.Targets = append(r.RestoreSession.Status.Targets, targetStatus)
switch target.Kind {
@ -80,7 +176,6 @@ func (r *RestoreSessionReconciler) StatusUpdate() error {
return nil, nil
}
}
var ret error
switch r.RestoreSession.Status.SessionState {
case formolv1alpha1.New:
r.RestoreSession.Status.SessionState = formolv1alpha1.Running
@ -89,11 +184,32 @@ func (r *RestoreSessionReconciler) StatusUpdate() error {
return err
}
log.V(0).Info("New restore. Start the first task", "task", targetStatus.Name)
case formolv1alpha1.Running:
currentTargetStatus := r.RestoreSession.Status.Targets[len(r.RestoreSession.Status.Targets)-1]
switch currentTargetStatus.SessionState {
case formolv1alpha1.Failure:
log.V(0).Info("last restore task failed. Stop here", "target", currentTargetStatus.Name)
r.RestoreSession.Status.SessionState = formolv1alpha1.Failure
case formolv1alpha1.Running:
log.V(0).Info("task is still running", "target", currentTargetStatus.Name)
return nil
case formolv1alpha1.Success:
log.V(0).Info("last task was a success. start a new one", "target", currentTargetStatus)
targetStatus, err := startNextTask()
if err != nil {
return err
}
if targetStatus == nil {
// No more task to start. The restore is over
r.RestoreSession.Status.SessionState = formolv1alpha1.Success
}
}
}
if ret = r.Status().Update(ctx, r.RestoreSession); ret != nil {
log.Error(ret, "unable to update restoresession")
if err := r.Status().Update(ctx, r.RestoreSession); err != nil {
log.Error(err, "unable to update restoresession")
return err
}
return ret
return nil
}
// +kubebuilder:rbac:groups=formol.desmojim.fr,resources=restoresessions,verbs=get;list;watch;create;update;patch;delete

213
pkg/rbac/backupconfiguration.go
View File

@ -10,11 +10,14 @@ import (
)
const (
backupListenerRole = "backup-listener-role"
backupListenerRoleBinding = "backup-listener-rolebinding"
backupSessionCreatorSA = "backupsession-creator"
backupSessionCreatorRole = "backupsession-creator-role"
backupSessionCreatorRoleBinding = "backupsession-creator-rolebinding"
formolRole = "formol-sidecar-role"
backupListenerRole = "backup-listener-role"
backupListenerRoleBinding = "backup-listener-rolebinding"
backupSessionCreatorSA = "backupsession-creator"
backupSessionCreatorRole = "backupsession-creator-role"
backupSessionCreatorRoleBinding = "backupsession-creator-rolebinding"
backupSessionStatusUpdaterRole = "backupsession-statusupdater-role"
backupSessionStatusUpdaterRoleBinding = "backupsession-statusupdater-rolebinding"
)
func DeleteBackupSessionCreatorRBAC(cl client.Client, namespace string) error {
@ -160,6 +163,104 @@ func DeleteBackupSessionListenerRBAC(cl client.Client, saName string, namespace
return nil
}
func DeleteFormolRBAC(cl client.Client, saName string, namespace string) error {
if saName == "" {
saName = "default"
}
formolRoleBinding := namespace + "-" + saName + "-formol-sidecar-rolebinding"
clusterRoleBinding := &rbacv1.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: formolRoleBinding,
},
Subjects: []rbacv1.Subject{
rbacv1.Subject{
Kind: "ServiceAccount",
Namespace: namespace,
Name: saName,
},
},
RoleRef: rbacv1.RoleRef{
APIGroup: "rbac.authorization.k8s.io",
Kind: "ClusterRole",
Name: formolRole,
},
}
if err := cl.Delete(context.Background(), clusterRoleBinding); err != nil {
return client.IgnoreNotFound(err)
}
return nil
}
func CreateFormolRBAC(cl client.Client, saName string, namespace string) error {
if saName == "" {
saName = "default"
}
sa := &corev1.ServiceAccount{}
if err := cl.Get(context.Background(), client.ObjectKey{
Namespace: namespace,
Name: saName,
}, sa); err != nil {
return err
}
clusterRole := &rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{
Name: formolRole,
},
Rules: []rbacv1.PolicyRule{
rbacv1.PolicyRule{
Verbs: []string{"*"},
APIGroups: []string{"formol.desmojim.fr"},
Resources: []string{"*"},
//APIGroups: []string{"formol.desmojim.fr"},
//Resources: []string{"restoresessions", "backupsessions", "backupconfigurations"},
},
rbacv1.PolicyRule{
Verbs: []string{"get", "list", "watch"},
APIGroups: []string{""},
Resources: []string{"pods"},
},
rbacv1.PolicyRule{
Verbs: []string{"get", "list", "watch"},
APIGroups: []string{"apps"},
Resources: []string{"deployments", "replicasets"},
},
},
}
if err := cl.Get(context.Background(), client.ObjectKey{
Name: formolRole,
}, clusterRole); err != nil && errors.IsNotFound(err) {
if err = cl.Create(context.Background(), clusterRole); err != nil {
return err
}
}
formolRoleBinding := namespace + "-" + saName + "-formol-rolebinding"
clusterRoleBinding := &rbacv1.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: formolRoleBinding,
},
Subjects: []rbacv1.Subject{
rbacv1.Subject{
Kind: "ServiceAccount",
Namespace: namespace,
Name: saName,
},
},
RoleRef: rbacv1.RoleRef{
APIGroup: "rbac.authorization.k8s.io",
Kind: "ClusterRole",
Name: formolRole,
},
}
if err := cl.Get(context.Background(), client.ObjectKey{
Name: formolRoleBinding,
}, clusterRoleBinding); err != nil && errors.IsNotFound(err) {
if err = cl.Create(context.Background(), clusterRoleBinding); err != nil {
return err
}
}
return nil
}
func CreateBackupSessionListenerRBAC(cl client.Client, saName string, namespace string) error {
if saName == "" {
saName = "default"
@ -190,17 +291,113 @@ func CreateBackupSessionListenerRBAC(cl client.Client, saName string, namespace
rbacv1.PolicyRule{
Verbs: []string{"get", "list", "watch"},
APIGroups: []string{"formol.desmojim.fr"},
Resources: []string{"backupsessions", "backupconfigurations"},
Resources: []string{"restoresessions", "backupsessions", "backupconfigurations"},
},
rbacv1.PolicyRule{
Verbs: []string{"update", "delete"},
APIGroups: []string{"formol.desmojim.fr"},
Resources: []string{"backupsessions"},
Resources: []string{"restoresessions", "backupsessions"},
},
},
}
if err := cl.Get(context.Background(), client.ObjectKey{
Namespace: namespace,
Name: backupListenerRole,
}, role); err != nil && errors.IsNotFound(err) {
if err = cl.Create(context.Background(), role); err != nil {
return err
}
}
rolebinding := &rbacv1.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Name: backupListenerRoleBinding,
},
Subjects: []rbacv1.Subject{
rbacv1.Subject{
Kind: "ServiceAccount",
Name: saName,
},
},
RoleRef: rbacv1.RoleRef{
APIGroup: "rbac.authorization.k8s.io",
Kind: "Role",
Name: backupListenerRole,
},
}
if err := cl.Get(context.Background(), client.ObjectKey{
Namespace: namespace,
Name: backupListenerRoleBinding,
}, rolebinding); err != nil && errors.IsNotFound(err) {
if err = cl.Create(context.Background(), rolebinding); err != nil {
return err
}
}
return nil
}
func DeleteBackupSessionStatusUpdaterRBAC(cl client.Client, saName string, namespace string) error {
if saName == "" {
saName = "default"
}
sa := &corev1.ServiceAccount{}
if err := cl.Get(context.Background(), client.ObjectKey{
Namespace: namespace,
Name: saName,
}, sa); err != nil {
return err
}
role := &rbacv1.Role{}
if err := cl.Get(context.Background(), client.ObjectKey{
Namespace: namespace,
Name: backupSessionStatusUpdaterRole,
}, role); err == nil {
if err = cl.Delete(context.Background(), role); err != nil {
return err
}
}
rolebinding := &rbacv1.RoleBinding{}
if err := cl.Get(context.Background(), client.ObjectKey{
Namespace: namespace,
Name: backupSessionStatusUpdaterRoleBinding,
}, rolebinding); err == nil {
if err = cl.Delete(context.Background(), rolebinding); err != nil {
return err
}
}
return nil
}
func CreateBackupSessionStatusUpdaterRBAC(cl client.Client, saName string, namespace string) error {
if saName == "" {
saName = "default"
}
sa := &corev1.ServiceAccount{}
if err := cl.Get(context.Background(), client.ObjectKey{
Namespace: namespace,
Name: saName,
}, sa); err != nil {
return err
}
role := &rbacv1.Role{
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Name: backupSessionStatusUpdaterRole,
},
Rules: []rbacv1.PolicyRule{
rbacv1.PolicyRule{
Verbs: []string{"get", "list", "watch", "patch", "update"},
APIGroups: []string{"formol.desmojim.fr"},
Resources: []string{"backupsessions/status"},
Resources: []string{"restoresessions/status", "backupsessions/status"},
},
rbacv1.PolicyRule{
Verbs: []string{"get", "list", "watch"},
APIGroups: []string{"formol.desmojim.fr"},
Resources: []string{"restoresessions", "backupsessions"},
},
},
}

2
test/00-setup.yaml
View File

@ -81,4 +81,4 @@ metadata:
spec:
name: backup-pg
image: desmo999r/formolcli:latest
args: ["backup", "postgres", "--hostname", $(PGHOST), "--database", $(PGDATABASE), "--username", $(PGUSER), "--password", $(PGPASSWD), "--file", "/output/backup-pg.sql"]
args: ["postgres", "backup", "--hostname", $(PGHOST), "--database", $(PGDATABASE), "--username", $(PGUSER), "--password", $(PGPASSWD), "--file", "/output/backup-pg.sql"]

2
test/02-backupconf.yaml
View File

@ -7,7 +7,7 @@ metadata:
spec:
repository:
name: repo-minio
schedule: "1 * * * *"
schedule: "15 * * * *"
targets:
- kind: Deployment
apiVersion: v1