jandre/formol
1
1
Fork 0
Code Issues 6 Pull Requests Projects Releases 1 Wiki Activity

Refactored since reconciler are not reentrant

Browse Source
This commit is contained in:
jandre 2021-04-17 16:21:26 +02:00
parent 393465300a
commit 2901f9aa1a
26 changed files with 1428 additions and 1072 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Makefile
View File

@ -2,7 +2,8 @@
# Image URL to use all building/pushing image targets
IMG ?= desmo999r/formolcontroller:latest
# Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
CRD_OPTIONS ?= "crd:trivialVersions=true"
#CRD_OPTIONS ?= "crd:trivialVersions=true"
CRD_OPTIONS ?= "crd:trivialVersions=true,crdVersions=v1"
# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
ifeq (,$(shell go env GOBIN))

22
api/v1alpha1/backupconfiguration_types.go
View File

@ -21,10 +21,18 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
const (
SidecarKind string = "Sidecar"
JobKind string = "Job"
BackupVolumes string = "Volumes"
)
type Step struct {
Name string `json:"name"`
Namespace string `json:"namespace"`
Env []corev1.EnvVar `json:"env"`
Name string `json:"name"`
// +optional
Env []corev1.EnvVar `json:"env,omitempty"`
// +optional
Finalize *bool `json:"finalize,omitempty"`
}
type Hook struct {
@ -34,14 +42,10 @@ type Hook struct {
}
type Target struct {
// +kubebuilder:validation:Enum=Deployment;Task
// +kubebuilder:validation:Enum=Sidecar;Job
Kind string `json:"kind"`
Name string `json:"name"`
// +optional
BeforeBackup []Hook `json:"beforeBackup,omitempty"`
// +optional
AfterBackup []Hook `json:"afterBackup,omitempty"`
// +optional
ApiVersion string `json:"apiVersion,omitempty"`
// +optional
VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"`
@ -50,6 +54,8 @@ type Target struct {
// +optional
// +kubebuilder:validation:MinItems=1
Steps []Step `json:"steps,omitempty"`
// +kubebuilder:default:=2
Retry int `json:"retry,omitempty"`
}
type Keep struct {

12
api/v1alpha1/backupsession_types.go
View File

@ -23,17 +23,9 @@ import (
// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN!
// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized.
type Ref struct {
Name string `json:"name"`
}
// BackupSessionSpec defines the desired state of BackupSession
type BackupSessionSpec struct {
// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
// Important: Run "make" to regenerate code after modifying this file
// Foo is an example field of BackupSession. Edit BackupSession_types.go to remove/update
Ref `json:"ref"`
Ref string `json:"ref"`
}
// BackupSessionStatus defines the observed state of BackupSession
@ -41,8 +33,6 @@ type BackupSessionStatus struct {
// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
// Important: Run "make" to regenerate code after modifying this file
// +optional
ObservedGeneration int64 `json:"observedGeneration,omitempty"`
// +optional
SessionState `json:"state,omitempty"`
// +optional
StartTime *metav1.Time `json:"startTime,omitempty"`

28
api/v1alpha1/common.go
View File

@ -7,14 +7,24 @@ import (
type SessionState string
const (
New SessionState = "New"
Running SessionState = "Running"
Success SessionState = "Success"
Failure SessionState = "Failure"
Deleted SessionState = "Deleted"
TARGET_NAME string = "TARGET_NAME"
RESTORESESSION_NAMESPACE string = "RESTORESESSION_NAMESPACE"
RESTORESESSION_NAME string = "RESTORESESSION_NAME"
New SessionState = "New"
Init SessionState = "Initializing"
Running SessionState = "Running"
Finalize SessionState = "Finalizing"
Success SessionState = "Success"
Failure SessionState = "Failure"
Deleted SessionState = "Deleted"
// Environment variables used by the sidecar container
// the name of the sidecar container
SIDECARCONTAINER_NAME string = "formol"
// Used by both the backupsession and restoresession controllers to identified the target deployment
TARGET_NAME string = "TARGET_NAME"
// Used by restoresession controller
RESTORESESSION_NAMESPACE string = "RESTORESESSION_NAMESPACE"
RESTORESESSION_NAME string = "RESTORESESSION_NAME"
// Used by the backupsession controller
POD_NAME string = "POD_NAME"
POD_NAMESPACE string = "POD_NAMESPACE"
)
type TargetStatus struct {
@ -28,4 +38,6 @@ type TargetStatus struct {
StartTime *metav1.Time `json:"startTime,omitempty"`
// +optional
Duration *metav1.Duration `json:"duration,omitempty"`
// +optional
Try int `json:"try,omitemmpty"`
}

76
api/v1alpha1/function_webhook.go Normal file
View File

@ -0,0 +1,76 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1alpha1
import (
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/webhook"
)
// log is for logging in this package.
var functionlog = logf.Log.WithName("function-resource")
func (r *Function) SetupWebhookWithManager(mgr ctrl.Manager) error {
return ctrl.NewWebhookManagedBy(mgr).
For(r).
Complete()
}
// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN!
// +kubebuilder:webhook:path=/mutate-formol-desmojim-fr-v1alpha1-function,mutating=true,failurePolicy=fail,groups=formol.desmojim.fr,resources=functions,verbs=create;update,versions=v1alpha1,name=mfunction.kb.io
var _ webhook.Defaulter = &Function{}
// Default implements webhook.Defaulter so a webhook will be registered for the type
func (r *Function) Default() {
functionlog.Info("default", "name", r.Name)
// TODO(user): fill in your defaulting logic.
r.Spec.Name = r.Name
}
// TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation.
// +kubebuilder:webhook:verbs=create;update,path=/validate-formol-desmojim-fr-v1alpha1-function,mutating=false,failurePolicy=fail,groups=formol.desmojim.fr,resources=functions,versions=v1alpha1,name=vfunction.kb.io
var _ webhook.Validator = &Function{}
// ValidateCreate implements webhook.Validator so a webhook will be registered for the type
func (r *Function) ValidateCreate() error {
functionlog.Info("validate create", "name", r.Name)
// TODO(user): fill in your validation logic upon object creation.
return nil
}
// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type
func (r *Function) ValidateUpdate(old runtime.Object) error {
functionlog.Info("validate update", "name", r.Name)
// TODO(user): fill in your validation logic upon object update.
return nil
}
// ValidateDelete implements webhook.Validator so a webhook will be registered for the type
func (r *Function) ValidateDelete() error {
functionlog.Info("validate delete", "name", r.Name)
// TODO(user): fill in your validation logic upon object deletion.
return nil
}

3
api/v1alpha1/restoresession_types.go
View File

@ -18,6 +18,7 @@ package v1alpha1
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
//"k8s.io/apimachinery/pkg/types"
)
// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN!
@ -28,7 +29,7 @@ type RestoreSessionSpec struct {
// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
// Important: Run "make" to regenerate code after modifying this file
BackupSessionRef metav1.ObjectMeta `json:"backupSessionRef"`
Ref string `json:"backupSessionRef"`
}
// RestoreSessionStatus defines the observed state of RestoreSession

38
api/v1alpha1/zz_generated.deepcopy.go
View File

@ -210,7 +210,6 @@ func (in *BackupSessionList) DeepCopyObject() runtime.Object {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *BackupSessionSpec) DeepCopyInto(out *BackupSessionSpec) {
*out = *in
out.Ref = in.Ref
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupSessionSpec.
@ -342,21 +341,6 @@ func (in *Keep) DeepCopy() *Keep {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Ref) DeepCopyInto(out *Ref) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Ref.
func (in *Ref) DeepCopy() *Ref {
if in == nil {
return nil
}
out := new(Ref)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Repo) DeepCopyInto(out *Repo) {
*out = *in
@ -452,7 +436,7 @@ func (in *RestoreSession) DeepCopyInto(out *RestoreSession) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
in.Spec.DeepCopyInto(&out.Spec)
out.Spec = in.Spec
in.Status.DeepCopyInto(&out.Status)
}
@ -509,7 +493,6 @@ func (in *RestoreSessionList) DeepCopyObject() runtime.Object {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *RestoreSessionSpec) DeepCopyInto(out *RestoreSessionSpec) {
*out = *in
in.BackupSessionRef.DeepCopyInto(&out.BackupSessionRef)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestoreSessionSpec.
@ -573,6 +556,11 @@ func (in *Step) DeepCopyInto(out *Step) {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
if in.Finalize != nil {
in, out := &in.Finalize, &out.Finalize
*out = new(bool)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Step.
@ -588,20 +576,6 @@ func (in *Step) DeepCopy() *Step {
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Target) DeepCopyInto(out *Target) {
*out = *in
if in.BeforeBackup != nil {
in, out := &in.BeforeBackup, &out.BeforeBackup
*out = make([]Hook, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
if in.AfterBackup != nil {
in, out := &in.AfterBackup, &out.AfterBackup
*out = make([]Hook, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
if in.VolumeMounts != nil {
in, out := &in.VolumeMounts, &out.VolumeMounts
*out = make([]v1.VolumeMount, len(*in))

10
config/crd/kustomization.yaml
View File

@ -14,17 +14,17 @@ patchesStrategicMerge:
# patches here are for enabling the conversion webhook for each CRD
#- patches/webhook_in_tasks.yaml
#- patches/webhook_in_functions.yaml
#- patches/webhook_in_backupconfigurations.yaml
- patches/webhook_in_backupsessions.yaml
- patches/webhook_in_backupconfigurations.yaml
#- patches/webhook_in_backupsessions.yaml
#- patches/webhook_in_repoes.yaml
#- patches/webhook_in_restoresessions.yaml
# +kubebuilder:scaffold:crdkustomizewebhookpatch
# [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix.
# patches here are for enabling the CA injection for each CRD
- patches/cainjection_in_functions.yaml
- patches/cainjection_in_backupconfigurations.yaml
- patches/cainjection_in_backupsessions.yaml
#- patches/cainjection_in_functions.yaml
#- patches/cainjection_in_backupconfigurations.yaml
#- patches/cainjection_in_backupsessions.yaml
#- patches/cainjection_in_repoes.yaml
#- patches/cainjection_in_restoresessions.yaml
# +kubebuilder:scaffold:crdkustomizecainjectionpatch

2
config/crd/patches/cainjection_in_backupconfigurations.yaml
View File

@ -1,6 +1,6 @@
# The following patch adds a directive for certmanager to inject CA into the CRD
# CRD conversion requires k8s 1.13 or later.
apiVersion: apiextensions.k8s.io/v1beta1
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:

2
config/crd/patches/cainjection_in_backupsessions.yaml
View File

@ -1,6 +1,6 @@
# The following patch adds a directive for certmanager to inject CA into the CRD
# CRD conversion requires k8s 1.13 or later.
apiVersion: apiextensions.k8s.io/v1beta1
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:

2
config/crd/patches/cainjection_in_functions.yaml
View File

@ -1,6 +1,6 @@
# The following patch adds a directive for certmanager to inject CA into the CRD
# CRD conversion requires k8s 1.13 or later.
apiVersion: apiextensions.k8s.io/v1beta1
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:

2
config/crd/patches/cainjection_in_restoresessions.yaml
View File

@ -1,6 +1,6 @@
# The following patch adds a directive for certmanager to inject CA into the CRD
# CRD conversion requires k8s 1.13 or later.
apiVersion: apiextensions.k8s.io/v1beta1
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:

17
config/crd/patches/webhook_in_backupconfigurations.yaml
View File

@ -1,17 +1,20 @@
# The following patch enables conversion webhook for CRD
# CRD conversion requires k8s 1.13 or later.
apiVersion: apiextensions.k8s.io/v1beta1
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: backupconfigurations.formol.desmojim.fr
spec:
preserveUnknownFields: false
conversion:
strategy: Webhook
webhookClientConfig:
webhook:
conversionReviewVersions: ["v1", "v1beta1", "v1alpha1"]
clientConfig:
# this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank,
# but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager)
caBundle: Cg==
service:
namespace: system
name: webhook-service
path: /convert
caBundle: Cg==
service:
namespace: system
name: webhook-service
path: /convert

2
config/crd/patches/webhook_in_backupsessions.yaml
View File

@ -1,6 +1,6 @@
# The following patch enables conversion webhook for CRD
# CRD conversion requires k8s 1.13 or later.
apiVersion: apiextensions.k8s.io/v1beta1
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: backupsessions.formol.desmojim.fr

2
config/crd/patches/webhook_in_functions.yaml
View File

@ -1,6 +1,6 @@
# The following patch enables conversion webhook for CRD
# CRD conversion requires k8s 1.13 or later.
apiVersion: apiextensions.k8s.io/v1beta1
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: functions.formol.desmojim.fr

2
config/crd/patches/webhook_in_restoresessions.yaml
View File

@ -1,6 +1,6 @@
# The following patch enables conversion webhook for CRD
# CRD conversion requires k8s 1.13 or later.
apiVersion: apiextensions.k8s.io/v1beta1
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: restoresessions.formol.desmojim.fr.desmojim.fr

549
controllers/backupconfiguration_controller.go
View File

@ -18,7 +18,7 @@ package controllers
import (
"context"
"time"
//"time"
formolrbac "github.com/desmo999r/formol/pkg/rbac"
formolutils "github.com/desmo999r/formol/pkg/utils"
@ -33,7 +33,7 @@ import (
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/predicate"
//"sigs.k8s.io/controller-runtime/pkg/predicate"
formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1"
)
@ -45,16 +45,6 @@ type BackupConfigurationReconciler struct {
Scheme *runtime.Scheme
}
func (r *BackupConfigurationReconciler) getDeployment(namespace string, name string) (*appsv1.Deployment, error) {
deployment := &appsv1.Deployment{}
err := r.Get(context.Background(), client.ObjectKey{
Namespace: namespace,
Name: name,
}, deployment)
return deployment, err
}
// +kubebuilder:rbac:groups=formol.desmojim.fr,resources=*,verbs=*
// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=apps,resources=replicasets,verbs=get;list;watch;create;update;patch;delete
@ -67,263 +57,108 @@ func (r *BackupConfigurationReconciler) getDeployment(namespace string, name str
// +kubebuilder:rbac:groups=batch,resources=cronjobs,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=batch,resources=cronjobs/status,verbs=get
func (r *BackupConfigurationReconciler) deleteSidecarContainer(backupConf *formolv1alpha1.BackupConfiguration, target formolv1alpha1.Target) error {
deployment, err := r.getDeployment(backupConf.Namespace, target.Name)
if err != nil {
return err
}
restorecontainers := []corev1.Container{}
for _, container := range deployment.Spec.Template.Spec.Containers {
if container.Name == "backup" {
continue
}
restorecontainers = append(restorecontainers, container)
}
deployment.Spec.Template.Spec.Containers = restorecontainers
if err := r.Update(context.Background(), deployment); err != nil {
return err
}
if err := formolrbac.DeleteFormolRBAC(r.Client, deployment.Spec.Template.Spec.ServiceAccountName, deployment.Namespace); err != nil {
return err
}
selector, err := metav1.LabelSelectorAsMap(deployment.Spec.Selector)
if err != nil {
return nil
}
pods := &corev1.PodList{}
err = r.List(context.Background(), pods, client.MatchingLabels(selector))
if err != nil {
return nil
}
replicasToDelete := []appsv1.ReplicaSet{}
for _, pod := range pods.Items {
for _, podRef := range pod.OwnerReferences {
rs := &appsv1.ReplicaSet{}
if err := r.Get(context.Background(), client.ObjectKey{
Name: podRef.Name,
Namespace: pod.Namespace,
}, rs); err != nil {
return nil
}
for _, rsRef := range rs.OwnerReferences {
if rsRef.Kind == deployment.Kind && rsRef.Name == deployment.Name {
replicasToDelete = append(replicasToDelete, *rs)
}
}
}
func (r *BackupConfigurationReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
ctx := context.Background()
log := r.Log.WithValues("backupconfiguration", req.NamespacedName)
//time.Sleep(300 * time.Millisecond)
log.V(1).Info("Enter Reconcile with req", "req", req, "reconciler", r)
backupConf := &formolv1alpha1.BackupConfiguration{}
if err := r.Get(ctx, req.NamespacedName, backupConf); err != nil {
return ctrl.Result{}, client.IgnoreNotFound(err)
}
for _, replica := range replicasToDelete {
if err := r.Delete(context.TODO(), &replica); err != nil {
return nil
}
}
return nil
}
func (r *BackupConfigurationReconciler) addSidecarContainer(backupConf *formolv1alpha1.BackupConfiguration, target formolv1alpha1.Target) error {
log := r.Log.WithValues("backupconf", backupConf.Name)
deployment, err := r.getDeployment(backupConf.Namespace, target.Name)
if err != nil {
log.Error(err, "unable to get Deployment")
return err
}
log.V(1).Info("got deployment", "Deployment", deployment)
for _, container := range deployment.Spec.Template.Spec.Containers {
if container.Name == "backup" {
log.V(0).Info("There is already a backup sidecar container. Skipping", "container", container)
return nil
}
}
sidecar := corev1.Container{
Name: "backup",
Image: "desmo999r/formolcli:latest",
Args: []string{"backupsession", "server"},
//Image: "busybox",
//Command: []string{
// "sh",
// "-c",
// "sleep 3600; echo done",
//},
Env: []corev1.EnvVar{
corev1.EnvVar{
Name: "POD_NAME",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "metadata.name",
},
},
},
corev1.EnvVar{
Name: "POD_NAMESPACE",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "metadata.namespace",
},
},
},
corev1.EnvVar{
Name: "POD_DEPLOYMENT",
Value: target.Name,
},
},
VolumeMounts: []corev1.VolumeMount{},
getDeployment := func(namespace string, name string) (*appsv1.Deployment, error) {
deployment := &appsv1.Deployment{}
err := r.Get(context.Background(), client.ObjectKey{
Namespace: namespace,
Name: name,
}, deployment)
return deployment, err
}
// Gather information from the repo
repo := &formolv1alpha1.Repo{}
if err := r.Get(context.Background(), client.ObjectKey{
Namespace: backupConf.Namespace,
Name: backupConf.Spec.Repository,
}, repo); err != nil {
log.Error(err, "unable to get Repo from BackupConfiguration")
return err
}
sidecar.Env = append(sidecar.Env, formolutils.ConfigureResticEnvVar(backupConf, repo)...)
for _, volumemount := range target.VolumeMounts {
log.V(1).Info("mounts", "volumemount", volumemount)
volumemount.ReadOnly = true
sidecar.VolumeMounts = append(sidecar.VolumeMounts, *volumemount.DeepCopy())
}
selector, err := metav1.LabelSelectorAsMap(deployment.Spec.Selector)
if err != nil {
log.Error(err, "unable to get LableSelector for deployment", "label", deployment.Spec.Selector)
return nil
}
log.V(1).Info("getting pods matching label", "label", selector)
pods := &corev1.PodList{}
err = r.List(context.Background(), pods, client.InNamespace(backupConf.Namespace), client.MatchingLabels(selector))
if err != nil {
log.Error(err, "unable to get deployment pods")
return nil
}
replicasToDelete := []appsv1.ReplicaSet{}
log.V(1).Info("got that list of pods", "pods", len(pods.Items))
for _, pod := range pods.Items {
log.V(1).Info("checking pod", "pod", pod)
for _, podRef := range pod.OwnerReferences {
rs := &appsv1.ReplicaSet{}
if err := r.Get(context.Background(), client.ObjectKey{
Name: podRef.Name,
Namespace: pod.Namespace,
}, rs); err != nil {
log.Error(err, "unable to get replicaset", "replicaset", podRef.Name)
return nil
}
log.V(1).Info("got a replicaset", "rs", rs.Name)
for _, rsRef := range rs.OwnerReferences {
if rsRef.Kind == deployment.Kind && rsRef.Name == deployment.Name {
log.V(0).Info("Adding pod to the list of pods to be restarted", "pod", pod.Name)
replicasToDelete = append(replicasToDelete, *rs)
}
}
}
}
deployment.Spec.Template.Spec.Containers = append(deployment.Spec.Template.Spec.Containers, sidecar)
deployment.Spec.Template.Spec.ShareProcessNamespace = func() *bool { b := true; return &b }()
if err := formolrbac.CreateFormolRBAC(r.Client, deployment.Spec.Template.Spec.ServiceAccountName, deployment.Namespace); err != nil {
log.Error(err, "unable to create backupsessionlistener RBAC")
return nil
}
log.V(0).Info("Adding a sicar container")
if err := r.Update(context.Background(), deployment); err != nil {
log.Error(err, "unable to update the Deployment")
return err
}
for _, replica := range replicasToDelete {
if err := r.Delete(context.TODO(), &replica); err != nil {
log.Error(err, "unable to delete replica", "replica", replica.Name)
return nil
}
}
return nil
}
func (r *BackupConfigurationReconciler) deleteCronJob(backupConf *formolv1alpha1.BackupConfiguration) error {
log := r.Log.WithValues("deleteCronJob", backupConf.Name)
_ = formolrbac.DeleteFormolRBAC(r.Client, "default", backupConf.Namespace)
_ = formolrbac.DeleteBackupSessionCreatorRBAC(r.Client, backupConf.Namespace)
cronjob := &kbatch_beta1.CronJob{}
if err := r.Get(context.Background(), client.ObjectKey{
Namespace: backupConf.Namespace,
Name: "backup-" + backupConf.Name,
}, cronjob); err == nil {
log.V(0).Info("Deleting cronjob", "cronjob", cronjob.Name)
return r.Delete(context.TODO(), cronjob)
} else {
return err
}
}
func (r *BackupConfigurationReconciler) addCronJob(backupConf *formolv1alpha1.BackupConfiguration) error {
log := r.Log.WithValues("addCronJob", backupConf.Name)
if err := formolrbac.CreateFormolRBAC(r.Client, "default", backupConf.Namespace); err != nil {
log.Error(err, "unable to create backupsessionlistener RBAC")
return nil
}
if err := formolrbac.CreateBackupSessionCreatorRBAC(r.Client, backupConf.Namespace); err != nil {
log.Error(err, "unable to create backupsession-creator RBAC")
return nil
}
cronjob := &kbatch_beta1.CronJob{}
if err := r.Get(context.Background(), client.ObjectKey{
Namespace: backupConf.Namespace,
Name: "backup-" + backupConf.Name,
}, cronjob); err == nil {
log.V(0).Info("there is already a cronjob")
var changed bool
if backupConf.Spec.Schedule != cronjob.Spec.Schedule {
log.V(0).Info("cronjob schedule has changed", "old schedule", cronjob.Spec.Schedule, "new schedule", backupConf.Spec.Schedule)
cronjob.Spec.Schedule = backupConf.Spec.Schedule
changed = true
}
if backupConf.Spec.Suspend != cronjob.Spec.Suspend {
log.V(0).Info("cronjob suspend has changed", "before", cronjob.Spec.Suspend, "new", backupConf.Spec.Suspend)
cronjob.Spec.Suspend = backupConf.Spec.Suspend
changed = true
}
if changed == true {
if err := r.Update(context.TODO(), cronjob); err != nil {
log.Error(err, "unable to update cronjob definition")
return err
}
}
return nil
} else if errors.IsNotFound(err) == false {
log.Error(err, "something went wrong")
return err
}
cronjob = &kbatch_beta1.CronJob{
ObjectMeta: metav1.ObjectMeta{
Name: "backup-" + backupConf.Name,
deleteCronJob := func() error {
_ = formolrbac.DeleteFormolRBAC(r.Client, "default", backupConf.Namespace)
_ = formolrbac.DeleteBackupSessionCreatorRBAC(r.Client, backupConf.Namespace)
cronjob := &kbatch_beta1.CronJob{}
if err := r.Get(context.Background(), client.ObjectKey{
Namespace: backupConf.Namespace,
},
Spec: kbatch_beta1.CronJobSpec{
Suspend: backupConf.Spec.Suspend,
Schedule: backupConf.Spec.Schedule,
JobTemplate: kbatch_beta1.JobTemplateSpec{
Spec: batchv1.JobSpec{
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
RestartPolicy: corev1.RestartPolicyOnFailure,
ServiceAccountName: "backupsession-creator",
Containers: []corev1.Container{
corev1.Container{
Name: "job-createbackupsession-" + backupConf.Name,
Image: "desmo999r/formolcli:latest",
Args: []string{
"backupsession",
"create",
"--namespace",
backupConf.Namespace,
"--name",
backupConf.Name,
Name: "backup-" + backupConf.Name,
}, cronjob); err == nil {
log.V(0).Info("Deleting cronjob", "cronjob", cronjob.Name)
return r.Delete(context.TODO(), cronjob)
} else {
return err
}
}
addCronJob := func() error {
if err := formolrbac.CreateFormolRBAC(r.Client, "default", backupConf.Namespace); err != nil {
log.Error(err, "unable to create backupsessionlistener RBAC")
return nil
}
if err := formolrbac.CreateBackupSessionCreatorRBAC(r.Client, backupConf.Namespace); err != nil {
log.Error(err, "unable to create backupsession-creator RBAC")
return nil
}
cronjob := &kbatch_beta1.CronJob{}
if err := r.Get(context.Background(), client.ObjectKey{
Namespace: backupConf.Namespace,
Name: "backup-" + backupConf.Name,
}, cronjob); err == nil {
log.V(0).Info("there is already a cronjob")
var changed bool
if backupConf.Spec.Schedule != cronjob.Spec.Schedule {
log.V(0).Info("cronjob schedule has changed", "old schedule", cronjob.Spec.Schedule, "new schedule", backupConf.Spec.Schedule)
cronjob.Spec.Schedule = backupConf.Spec.Schedule
changed = true
}
if backupConf.Spec.Suspend != cronjob.Spec.Suspend {
log.V(0).Info("cronjob suspend has changed", "before", cronjob.Spec.Suspend, "new", backupConf.Spec.Suspend)
cronjob.Spec.Suspend = backupConf.Spec.Suspend
changed = true
}
if changed == true {
if err := r.Update(context.TODO(), cronjob); err != nil {
log.Error(err, "unable to update cronjob definition")
return err
}
}
return nil
} else if errors.IsNotFound(err) == false {
log.Error(err, "something went wrong")
return err
}
cronjob = &kbatch_beta1.CronJob{
ObjectMeta: metav1.ObjectMeta{
Name: "backup-" + backupConf.Name,
Namespace: backupConf.Namespace,
},
Spec: kbatch_beta1.CronJobSpec{
Suspend: backupConf.Spec.Suspend,
Schedule: backupConf.Spec.Schedule,
JobTemplate: kbatch_beta1.JobTemplateSpec{
Spec: batchv1.JobSpec{
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
RestartPolicy: corev1.RestartPolicyOnFailure,
ServiceAccountName: "backupsession-creator",
Containers: []corev1.Container{
corev1.Container{
Name: "job-createbackupsession-" + backupConf.Name,
Image: "desmo999r/formolcli:latest",
Args: []string{
"backupsession",
"create",
"--namespace",
backupConf.Namespace,
"--name",
backupConf.Name,
},
},
},
},
@ -331,46 +166,140 @@ func (r *BackupConfigurationReconciler) addCronJob(backupConf *formolv1alpha1.Ba
},
},
},
},
}
if err := ctrl.SetControllerReference(backupConf, cronjob, r.Scheme); err != nil {
log.Error(err, "unable to set controller on job", "cronjob", cronjob, "backupconf", backupConf)
return err
}
log.V(0).Info("creating the cronjob")
if err := r.Create(context.Background(), cronjob); err != nil {
log.Error(err, "unable to create the cronjob", "cronjob", cronjob)
return err
}
return nil
}
if err := ctrl.SetControllerReference(backupConf, cronjob, r.Scheme); err != nil {
log.Error(err, "unable to set controller on job", "cronjob", cronjob, "backupconf", backupConf)
return err
deleteSidecarContainer := func(target formolv1alpha1.Target) error {
deployment, err := getDeployment(backupConf.Namespace, target.Name)
if err != nil {
return err
}
restorecontainers := []corev1.Container{}
for _, container := range deployment.Spec.Template.Spec.Containers {
if container.Name == formolv1alpha1.SIDECARCONTAINER_NAME {
continue
}
restorecontainers = append(restorecontainers, container)
}
deployment.Spec.Template.Spec.Containers = restorecontainers
if err := r.Update(context.Background(), deployment); err != nil {
return err
}
if err := formolrbac.DeleteFormolRBAC(r.Client, deployment.Spec.Template.Spec.ServiceAccountName, deployment.Namespace); err != nil {
return err
}
return nil
}
log.V(0).Info("creating the cronjob")
if err := r.Create(context.Background(), cronjob); err != nil {
log.Error(err, "unable to create the cronjob", "cronjob", cronjob)
return err
addSidecarContainer := func(target formolv1alpha1.Target) error {
deployment, err := getDeployment(backupConf.Namespace, target.Name)
if err != nil {
log.Error(err, "unable to get Deployment")
return err
}
log.V(1).Info("got deployment", "Deployment", deployment)
for _, container := range deployment.Spec.Template.Spec.Containers {
if container.Name == formolv1alpha1.SIDECARCONTAINER_NAME {
log.V(0).Info("There is already a backup sidecar container. Skipping", "container", container)
return nil
}
}
sidecar := corev1.Container{
Name: formolv1alpha1.SIDECARCONTAINER_NAME,
// TODO: Put the image in the BackupConfiguration YAML file
Image: "desmo999r/formolcli:latest",
Args: []string{"backupsession", "server"},
//Image: "busybox",
//Command: []string{
// "sh",
// "-c",
// "sleep 3600; echo done",
//},
Env: []corev1.EnvVar{
corev1.EnvVar{
Name: formolv1alpha1.POD_NAME,
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "metadata.name",
},
},
},
corev1.EnvVar{
Name: formolv1alpha1.POD_NAMESPACE,
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
FieldPath: "metadata.namespace",
},
},
},
corev1.EnvVar{
Name: formolv1alpha1.TARGET_NAME,
Value: target.Name,
},
},
VolumeMounts: []corev1.VolumeMount{},
}
// Gather information from the repo
repo := &formolv1alpha1.Repo{}
if err := r.Get(context.Background(), client.ObjectKey{
Namespace: backupConf.Namespace,
Name: backupConf.Spec.Repository,
}, repo); err != nil {
log.Error(err, "unable to get Repo from BackupConfiguration")
return err
}
sidecar.Env = append(sidecar.Env, formolutils.ConfigureResticEnvVar(backupConf, repo)...)
for _, volumemount := range target.VolumeMounts {
log.V(1).Info("mounts", "volumemount", volumemount)
volumemount.ReadOnly = true
sidecar.VolumeMounts = append(sidecar.VolumeMounts, *volumemount.DeepCopy())
}
deployment.Spec.Template.Spec.Containers = append(deployment.Spec.Template.Spec.Containers, sidecar)
deployment.Spec.Template.Spec.ShareProcessNamespace = func() *bool { b := true; return &b }()
if err := formolrbac.CreateFormolRBAC(r.Client, deployment.Spec.Template.Spec.ServiceAccountName, deployment.Namespace); err != nil {
log.Error(err, "unable to create backupsessionlistener RBAC")
return nil
}
log.V(0).Info("Adding a sicar container")
if err := r.Update(context.Background(), deployment); err != nil {
log.Error(err, "unable to update the Deployment")
return err
}
return nil
}
return nil
}
func (r *BackupConfigurationReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
ctx := context.Background()
log := r.Log.WithValues("backupconfiguration", req.NamespacedName)
time.Sleep(300 * time.Millisecond)
log.V(1).Info("Enter Reconcile with req", "req", req)
backupConf := &formolv1alpha1.BackupConfiguration{}
if err := r.Get(ctx, req.NamespacedName, backupConf); err != nil {
return ctrl.Result{}, client.IgnoreNotFound(err)
deleteExternalResources := func() error {
for _, target := range backupConf.Spec.Targets {
switch target.Kind {
case formolv1alpha1.SidecarKind:
_ = deleteSidecarContainer(target)
}
}
// TODO: remove the hardcoded "default"
_ = deleteCronJob()
return nil
}
finalizerName := "finalizer.backupconfiguration.formol.desmojim.fr"
if backupConf.ObjectMeta.DeletionTimestamp.IsZero() {
if !formolutils.ContainsString(backupConf.ObjectMeta.Finalizers, finalizerName) {
backupConf.ObjectMeta.Finalizers = append(backupConf.ObjectMeta.Finalizers, finalizerName)
if err := r.Update(context.Background(), backupConf); err != nil {
log.Error(err, "unable to append finalizer")
return ctrl.Result{}, err
}
}
} else {
if !backupConf.ObjectMeta.DeletionTimestamp.IsZero() {
log.V(0).Info("backupconf being deleted", "backupconf", backupConf.Name)
if formolutils.ContainsString(backupConf.ObjectMeta.Finalizers, finalizerName) {
_ = r.deleteExternalResources(backupConf)
_ = deleteExternalResources()
}
backupConf.ObjectMeta.Finalizers = formolutils.RemoveString(backupConf.ObjectMeta.Finalizers, finalizerName)
if err := r.Update(context.Background(), backupConf); err != nil {
@ -382,21 +311,27 @@ func (r *BackupConfigurationReconciler) Reconcile(req ctrl.Request) (ctrl.Result
return ctrl.Result{}, nil
}
if err := r.addCronJob(backupConf); err != nil {
if !formolutils.ContainsString(backupConf.ObjectMeta.Finalizers, finalizerName) {
backupConf.ObjectMeta.Finalizers = append(backupConf.ObjectMeta.Finalizers, finalizerName)
err := r.Update(context.Background(), backupConf)
if err != nil {
log.Error(err, "unable to append finalizer")
}
return ctrl.Result{}, err
}
if err := addCronJob(); err != nil {
return ctrl.Result{}, nil
}
backupConf.Status.ActiveCronJob = true
for _, target := range backupConf.Spec.Targets {
switch target.Kind {
case "Deployment":
if err := r.addSidecarContainer(backupConf, target); err != nil {
case formolv1alpha1.SidecarKind:
if err := addSidecarContainer(target); err != nil {
return ctrl.Result{}, client.IgnoreNotFound(err)
}
backupConf.Status.ActiveSidecar = true
case "PersistentVolumeClaim":
log.V(0).Info("TODO backup PVC")
return ctrl.Result{}, nil
}
}
@ -410,23 +345,11 @@ func (r *BackupConfigurationReconciler) Reconcile(req ctrl.Request) (ctrl.Result
return ctrl.Result{}, nil
}
func (r *BackupConfigurationReconciler) deleteExternalResources(backupConf *formolv1alpha1.BackupConfiguration) error {
for _, target := range backupConf.Spec.Targets {
switch target.Kind {
case "Deployment":
_ = r.deleteSidecarContainer(backupConf, target)
}
}
// TODO: remove the hardcoded "default"
_ = r.deleteCronJob(backupConf)
return nil
}
func (r *BackupConfigurationReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&formolv1alpha1.BackupConfiguration{}).
WithOptions(controller.Options{MaxConcurrentReconciles: 3}).
WithEventFilter(predicate.GenerationChangedPredicate{}). // Don't reconcile when status gets updated
//WithEventFilter(predicate.GenerationChangedPredicate{}). // Don't reconcile when status gets updated
//Owns(&formolv1alpha1.BackupSession{}).
Owns(&kbatch_beta1.CronJob{}).
Complete(r)

49
controllers/backupconfiguration_controller_test.go
View File

@ -5,7 +5,7 @@ import (
//"k8s.io/apimachinery/pkg/types"
//"reflect"
//"fmt"
//"time"
"time"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@ -21,11 +21,11 @@ import (
var _ = Describe("Testing BackupConf controller", func() {
const (
BackupConfName = "test-backupconf"
BCBackupConfName = "test-backupconf-controller"
)
var (
key = types.NamespacedName{
Name: BackupConfName,
Name: BCBackupConfName,
Namespace: TestNamespace,
}
ctx = context.Background()
@ -35,24 +35,32 @@ var _ = Describe("Testing BackupConf controller", func() {
BeforeEach(func() {
backupConf = &formolv1alpha1.BackupConfiguration{
ObjectMeta: metav1.ObjectMeta{
Name: BackupConfName,
Name: BCBackupConfName,
Namespace: TestNamespace,
},
Spec: formolv1alpha1.BackupConfigurationSpec{
Repository: RepoName,
Repository: TestRepoName,
Schedule: "1 * * * *",
Targets: []formolv1alpha1.Target{
formolv1alpha1.Target{
Kind: "Deployment",
Name: DeploymentName,
Kind: formolv1alpha1.SidecarKind,
Name: TestDeploymentName,
VolumeMounts: []corev1.VolumeMount{
corev1.VolumeMount{
Name: TestDataVolume,
MountPath: TestDataMountPath,
},
},
Paths: []string{
TestDataMountPath,
},
},
formolv1alpha1.Target{
Kind: "Task",
Name: BackupFuncName,
Kind: formolv1alpha1.JobKind,
Name: TestBackupFuncName,
Steps: []formolv1alpha1.Step{
formolv1alpha1.Step{
Name: BackupFuncName,
Namespace: TestNamespace,
Name: TestBackupFuncName,
Env: []corev1.EnvVar{
corev1.EnvVar{
Name: "foo",
@ -66,7 +74,7 @@ var _ = Describe("Testing BackupConf controller", func() {
},
}
})
Context("There is a backupconf", func() {
Context("Creating a backupconf", func() {
JustBeforeEach(func() {
Eventually(func() error {
return k8sClient.Create(ctx, backupConf)
@ -85,18 +93,16 @@ var _ = Describe("Testing BackupConf controller", func() {
return true
}, timeout, interval).Should(BeTrue())
Expect(realBackupConf.Spec.Schedule).Should(Equal("1 * * * *"))
Expect(realBackupConf.Spec.Targets[0].Retry).Should(Equal(2))
})
It("Should also create a CronJob", func() {
cronJob := &batchv1beta1.CronJob{}
Eventually(func() bool {
err := k8sClient.Get(ctx, types.NamespacedName{
Name: "backup-" + BackupConfName,
Name: "backup-" + BCBackupConfName,
Namespace: TestNamespace,
}, cronJob)
if err != nil {
return false
}
return true
return err == nil
}, timeout, interval).Should(BeTrue())
Expect(cronJob.Spec.Schedule).Should(Equal("1 * * * *"))
})
@ -104,7 +110,7 @@ var _ = Describe("Testing BackupConf controller", func() {
realDeployment := &appsv1.Deployment{}
Eventually(func() (int, error) {
err := k8sClient.Get(ctx, types.NamespacedName{
Name: DeploymentName,
Name: TestDeploymentName,
Namespace: TestNamespace,
}, realDeployment)
if err != nil {
@ -115,6 +121,7 @@ var _ = Describe("Testing BackupConf controller", func() {
})
It("Should also update the CronJob", func() {
realBackupConf := &formolv1alpha1.BackupConfiguration{}
time.Sleep(300 * time.Millisecond)
Eventually(func() bool {
err := k8sClient.Get(ctx, key, realBackupConf)
if err != nil {
@ -129,7 +136,7 @@ var _ = Describe("Testing BackupConf controller", func() {
cronJob := &batchv1beta1.CronJob{}
Eventually(func() (string, error) {
err := k8sClient.Get(ctx, types.NamespacedName{
Name: "backup-" + BackupConfName,
Name: "backup-" + BCBackupConfName,
Namespace: TestNamespace,
}, cronJob)
if err != nil {
@ -139,7 +146,7 @@ var _ = Describe("Testing BackupConf controller", func() {
}, timeout, interval).Should(Equal("1 0 * * *"))
Eventually(func() (bool, error) {
err := k8sClient.Get(ctx, types.NamespacedName{
Name: "backup-" + BackupConfName,
Name: "backup-" + BCBackupConfName,
Namespace: TestNamespace,
}, cronJob)
if err != nil {
@ -160,7 +167,7 @@ var _ = Describe("Testing BackupConf controller", func() {
realDeployment := &appsv1.Deployment{}
Eventually(func() (int, error) {
err := k8sClient.Get(ctx, types.NamespacedName{
Name: DeploymentName,
Name: TestDeploymentName,
Namespace: TestNamespace,
}, realDeployment)
if err != nil {

765
controllers/backupsession_controller.go
View File

@ -46,214 +46,8 @@ const (
// BackupSessionReconciler reconciles a BackupSession object
type BackupSessionReconciler struct {
client.Client
Log logr.Logger
Scheme *runtime.Scheme
BackupSession *formolv1alpha1.BackupSession
BackupConf *formolv1alpha1.BackupConfiguration
}
func (r *BackupSessionReconciler) StatusUpdate() error {
log := r.Log.WithValues("backupsession-statusupdate", r.BackupSession)
ctx := context.Background()
r.BackupConf = &formolv1alpha1.BackupConfiguration{}
if err := r.Get(ctx, client.ObjectKey{
Namespace: r.BackupSession.Namespace,
Name: r.BackupSession.Spec.Ref.Name}, r.BackupConf); err != nil {
log.Error(err, "unable to get backupConfiguration")
return client.IgnoreNotFound(err)
}
// start the next task
startNextTask := func() (*formolv1alpha1.TargetStatus, error) {
nextTarget := len(r.BackupSession.Status.Targets)
if nextTarget < len(r.BackupConf.Spec.Targets) {
target := r.BackupConf.Spec.Targets[nextTarget]
targetStatus := formolv1alpha1.TargetStatus{
Name: target.Name,
Kind: target.Kind,
SessionState: formolv1alpha1.New,
StartTime: &metav1.Time{Time: time.Now()},
}
r.BackupSession.Status.Targets = append(r.BackupSession.Status.Targets, targetStatus)
switch target.Kind {
case "Task":
if err := r.CreateBackupJob(target); err != nil {
log.V(0).Info("unable to create task", "task", target)
targetStatus.SessionState = formolv1alpha1.Failure
return nil, err
}
}
return &targetStatus, nil
} else {
return nil, nil
}
}
// Test the backupsession backupstate to decide what to do
switch r.BackupSession.Status.SessionState {
case formolv1alpha1.New:
// Brand new backupsession; start the first task
r.BackupSession.Status.SessionState = formolv1alpha1.Running
targetStatus, err := startNextTask()
if err != nil {
return err
}
log.V(0).Info("New backup. Start the first task", "task", targetStatus)
if err := r.Status().Update(ctx, r.BackupSession); err != nil {
log.Error(err, "unable to update BackupSession status")
return err
}
case formolv1alpha1.Running:
// Backup ongoing. Check the status of the last task to decide what to do
currentTargetStatus := r.BackupSession.Status.Targets[len(r.BackupSession.Status.Targets)-1]
switch currentTargetStatus.SessionState {
case formolv1alpha1.Failure:
// The last task failed. We mark the backupsession as failed and we stop here.
log.V(0).Info("last backup task failed. Stop here", "targetStatus", currentTargetStatus)
r.BackupSession.Status.SessionState = formolv1alpha1.Failure
log.V(1).Info("New BackupSession status", "status", r.BackupSession.Status.SessionState)
if err := r.Status().Update(ctx, r.BackupSession); err != nil {
log.Error(err, "unable to update BackupSession status")
return err
}
case formolv1alpha1.Running:
// The current task is still running. Nothing to do
log.V(0).Info("task is still running", "targetStatus", currentTargetStatus)
case formolv1alpha1.Success:
// The last task successed. Let's try to start the next one
log.V(0).Info("last task was a success. start a new one", "currentTargetStatus", currentTargetStatus, "targetStatus", currentTargetStatus)
targetStatus, err := startNextTask()
if err != nil {
return err
}
if targetStatus == nil {
// No more task to start. The backup is a success
r.BackupSession.Status.SessionState = formolv1alpha1.Success
log.V(0).Info("Backup is successful. Let's try to do some cleanup")
backupSessionList := &formolv1alpha1.BackupSessionList{}
if err := r.List(ctx, backupSessionList, client.InNamespace(r.BackupConf.Namespace), client.MatchingFieldsSelector{Selector: fields.SelectorFromSet(fields.Set{sessionState: "Success"})}); err != nil {
log.Error(err, "unable to get backupsessionlist")
return nil
}
if len(backupSessionList.Items) < 2 {
// Not enough backupSession to proceed
log.V(1).Info("Not enough successful backup jobs")
break
}
sort.Slice(backupSessionList.Items, func(i, j int) bool {
return backupSessionList.Items[i].Status.StartTime.Time.Unix() > backupSessionList.Items[j].Status.StartTime.Time.Unix()
})
type KeepBackup struct {
Counter int32
Last time.Time
}
var lastBackups, dailyBackups, weeklyBackups, monthlyBackups, yearlyBackups KeepBackup
lastBackups.Counter = r.BackupConf.Spec.Keep.Last
dailyBackups.Counter = r.BackupConf.Spec.Keep.Daily
weeklyBackups.Counter = r.BackupConf.Spec.Keep.Weekly
monthlyBackups.Counter = r.BackupConf.Spec.Keep.Monthly
yearlyBackups.Counter = r.BackupConf.Spec.Keep.Yearly
for _, session := range backupSessionList.Items {
if session.Spec.Ref.Name != r.BackupConf.Name {
continue
}
deleteSession := true
keep := []string{}
if lastBackups.Counter > 0 {
log.V(1).Info("Keep backup", "last", session.Status.StartTime)
lastBackups.Counter--
keep = append(keep, "last")
deleteSession = false
}
if dailyBackups.Counter > 0 {
if session.Status.StartTime.Time.YearDay() != dailyBackups.Last.YearDay() {
log.V(1).Info("Keep backup", "daily", session.Status.StartTime)
dailyBackups.Counter--
dailyBackups.Last = session.Status.StartTime.Time
keep = append(keep, "daily")
deleteSession = false
}
}
if weeklyBackups.Counter > 0 {
if session.Status.StartTime.Time.Weekday().String() == "Sunday" && session.Status.StartTime.Time.YearDay() != weeklyBackups.Last.YearDay() {
log.V(1).Info("Keep backup", "weekly", session.Status.StartTime)
weeklyBackups.Counter--
weeklyBackups.Last = session.Status.StartTime.Time
keep = append(keep, "weekly")
deleteSession = false
}
}
if monthlyBackups.Counter > 0 {
if session.Status.StartTime.Time.Day() == 1 && session.Status.StartTime.Time.Month() != monthlyBackups.Last.Month() {
log.V(1).Info("Keep backup", "monthly", session.Status.StartTime)
monthlyBackups.Counter--
monthlyBackups.Last = session.Status.StartTime.Time
keep = append(keep, "monthly")
deleteSession = false
}
}
if yearlyBackups.Counter > 0 {
if session.Status.StartTime.Time.YearDay() == 1 && session.Status.StartTime.Time.Year() != yearlyBackups.Last.Year() {
log.V(1).Info("Keep backup", "yearly", session.Status.StartTime)
yearlyBackups.Counter--
yearlyBackups.Last = session.Status.StartTime.Time
keep = append(keep, "yearly")
deleteSession = false
}
}
if deleteSession {
log.V(1).Info("Delete session", "delete", session.Status.StartTime)
if err := r.Delete(ctx, &session); err != nil {
log.Error(err, "unable to delete backupsession", "session", session.Name)
// we don't return anything, we keep going
}
} else {
session.Status.Keep = strings.Join(keep, ",") // + " " + time.Now().Format("2006 Jan 02 15:04:05 -0700 MST")
if err := r.Status().Update(ctx, &session); err != nil {
log.Error(err, "unable to update session status", "session", session)
}
}
}
}
log.V(1).Info("New BackupSession status", "status", r.BackupSession.Status.SessionState)
if err := r.Status().Update(ctx, r.BackupSession); err != nil {
log.Error(err, "unable to update BackupSession status")
return err
}
}
case formolv1alpha1.Deleted:
for _, target := range r.BackupSession.Status.Targets {
if target.SessionState != formolv1alpha1.Deleted {
log.V(1).Info("snaphot has not been deleted. won't delete the backupsession", "target", target)
return nil
}
}
log.V(1).Info("all the snapshots have been deleted. deleting the backupsession")
controllerutil.RemoveFinalizer(r.BackupSession, finalizerName)
if err := r.Update(ctx, r.BackupSession); err != nil {
log.Error(err, "unable to remove finalizer")
return err
}
}
return nil
}
func (r *BackupSessionReconciler) IsBackupOngoing() bool {
log := r.Log.WithName("IsBackupOngoing")
ctx := context.Background()
backupSessionList := &formolv1alpha1.BackupSessionList{}
if err := r.List(ctx, backupSessionList, client.InNamespace(r.BackupConf.Namespace), client.MatchingFieldsSelector{Selector: fields.SelectorFromSet(fields.Set{sessionState: "Running"})}); err != nil {
log.Error(err, "unable to get backupsessionlist")
return true
}
if len(backupSessionList.Items) > 0 {
return true
} else {
return false
}
Log logr.Logger
Scheme *runtime.Scheme
}
// +kubebuilder:rbac:groups=formol.desmojim.fr,resources=backupsessions,verbs=get;list;watch;create;update;patch;delete
@ -265,55 +59,394 @@ func (r *BackupSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
log := r.Log.WithValues("backupsession", req.NamespacedName)
ctx := context.Background()
// your logic here
//time.Sleep(300 * time.Millisecond)
r.BackupSession = &formolv1alpha1.BackupSession{}
if err := r.Get(ctx, req.NamespacedName, r.BackupSession); err != nil {
backupSession := &formolv1alpha1.BackupSession{}
if err := r.Get(ctx, req.NamespacedName, backupSession); err != nil {
log.Error(err, "unable to get backupsession")
return ctrl.Result{}, client.IgnoreNotFound(err)
}
log.V(0).Info("backupSession", "backupSession.ObjectMeta", r.BackupSession.ObjectMeta, "backupSession.Status", r.BackupSession.Status)
if r.BackupSession.Status.ObservedGeneration == r.BackupSession.ObjectMeta.Generation {
// status update
log.V(0).Info("status update")
return ctrl.Result{}, r.StatusUpdate()
backupConf := &formolv1alpha1.BackupConfiguration{}
if err := r.Get(ctx, client.ObjectKey{
Namespace: backupSession.Namespace,
Name: backupSession.Spec.Ref,
}, backupConf); err != nil {
log.Error(err, "unable to get backupConfiguration")
return ctrl.Result{}, client.IgnoreNotFound(err)
}
if r.IsBackupOngoing() {
// There is already a backup ongoing. We don't do anything and we reschedule
log.V(0).Info("there is an ongoing backup. let's reschedule this operation")
return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
} else if r.BackupSession.ObjectMeta.DeletionTimestamp.IsZero() {
// Check if the finalizer has been registered
if !controllerutil.ContainsFinalizer(r.BackupSession, finalizerName) {
controllerutil.AddFinalizer(r.BackupSession, finalizerName)
// We update the BackupSession to add the finalizer
// Reconcile will be called again
// return now
err := r.Update(ctx, r.BackupSession)
if err != nil {
log.Error(err, "unable to add finalizer")
}
return ctrl.Result{}, err
// helper functions
// is there a backup operation ongoing
isBackupOngoing := func() bool {
backupSessionList := &formolv1alpha1.BackupSessionList{}
if err := r.List(ctx, backupSessionList, client.InNamespace(backupConf.Namespace), client.MatchingFieldsSelector{Selector: fields.SelectorFromSet(fields.Set{sessionState: "Running"})}); err != nil {
log.Error(err, "unable to get backupsessionlist")
return true
}
// All signals are green
// We start the backup process
r.BackupSession.Status.ObservedGeneration = r.BackupSession.ObjectMeta.Generation
r.BackupSession.Status.SessionState = formolv1alpha1.New
r.BackupSession.Status.StartTime = &metav1.Time{Time: time.Now()}
if err := r.Status().Update(ctx, r.BackupSession); err != nil {
log.Error(err, "unable to update backupSession")
return ctrl.Result{}, err
return len(backupSessionList.Items) > 0
}
// delete session specific backup resources
deleteExternalResources := func() error {
log := r.Log.WithValues("deleteExternalResources", backupSession.Name)
// Gather information from the repo
repo := &formolv1alpha1.Repo{}
if err := r.Get(ctx, client.ObjectKey{
Namespace: backupConf.Namespace,
Name: backupConf.Spec.Repository,
}, repo); err != nil {
log.Error(err, "unable to get Repo from BackupConfiguration")
return err
}
env := formolutils.ConfigureResticEnvVar(backupConf, repo)
// container that will delete the restic snapshot(s) matching the backupsession
deleteSnapshots := []corev1.Container{}
for _, target := range backupSession.Status.Targets {
if target.SessionState == formolv1alpha1.Success {
deleteSnapshots = append(deleteSnapshots, corev1.Container{
Name: target.Name,
Image: "desmo999r/formolcli:latest",
Args: []string{"snapshot", "delete", "--snapshot-id", target.SnapshotId},
Env: env,
})
}
}
// create a job to delete the restic snapshot(s) with the backupsession name tag
if len(deleteSnapshots) > 0 {
jobTtl := JOBTTL
job := &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
GenerateName: fmt.Sprintf("delete-%s-", backupSession.Name),
Namespace: backupSession.Namespace,
},
Spec: batchv1.JobSpec{
TTLSecondsAfterFinished: &jobTtl,
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
InitContainers: []corev1.Container{},
Containers: deleteSnapshots,
RestartPolicy: corev1.RestartPolicyOnFailure,
},
},
},
}
log.V(0).Info("creating a job to delete restic snapshots")
if err := r.Create(ctx, job); err != nil {
log.Error(err, "unable to delete job", "job", job)
return err
}
}
return nil
}
// create a backup job
createBackupJob := func(target formolv1alpha1.Target) error {
log := r.Log.WithValues("createbackupjob", target.Name)
ctx := context.Background()
backupSessionEnv := []corev1.EnvVar{
corev1.EnvVar{
Name: "TARGET_NAME",
Value: target.Name,
},
corev1.EnvVar{
Name: "BACKUPSESSION_NAME",
Value: backupSession.Name,
},
corev1.EnvVar{
Name: "BACKUPSESSION_NAMESPACE",
Value: backupSession.Namespace,
},
}
} else {
log.V(0).Info("backupsession being deleted", "backupsession", r.BackupSession.Name)
if controllerutil.ContainsFinalizer(r.BackupSession, finalizerName) {
if err := r.deleteExternalResources(); err != nil {
output := corev1.VolumeMount{
Name: "output",
MountPath: "/output",
}
restic := corev1.Container{
Name: "restic",
Image: "desmo999r/formolcli:latest",
Args: []string{"volume", "backup", "--tag", backupSession.Name, "--path", "/output"},
VolumeMounts: []corev1.VolumeMount{output},
Env: backupSessionEnv,
}
log.V(1).Info("creating a tagged backup job", "container", restic)
// Gather information from the repo
repo := &formolv1alpha1.Repo{}
if err := r.Get(ctx, client.ObjectKey{
Namespace: backupConf.Namespace,
Name: backupConf.Spec.Repository,
}, repo); err != nil {
log.Error(err, "unable to get Repo from BackupConfiguration")
return err
}
// S3 backing storage
restic.Env = append(restic.Env, formolutils.ConfigureResticEnvVar(backupConf, repo)...)
jobTtl := JOBTTL
job := &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
GenerateName: fmt.Sprintf("%s-%s-", backupSession.Name, target.Name),
Namespace: backupConf.Namespace,
},
Spec: batchv1.JobSpec{
TTLSecondsAfterFinished: &jobTtl,
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
InitContainers: []corev1.Container{},
Containers: []corev1.Container{restic},
Volumes: []corev1.Volume{
corev1.Volume{Name: "output"},
},
RestartPolicy: corev1.RestartPolicyOnFailure,
},
},
},
}
for _, step := range target.Steps {
function := &formolv1alpha1.Function{}
if err := r.Get(ctx, client.ObjectKey{
Namespace: backupConf.Namespace,
Name: step.Name,
}, function); err != nil {
log.Error(err, "unable to get function", "Function", step)
return err
}
function.Spec.Name = function.Name
function.Spec.Env = append(step.Env, backupSessionEnv...)
function.Spec.VolumeMounts = append(function.Spec.VolumeMounts, output)
job.Spec.Template.Spec.InitContainers = append(job.Spec.Template.Spec.InitContainers, function.Spec)
}
if err := ctrl.SetControllerReference(backupConf, job, r.Scheme); err != nil {
log.Error(err, "unable to set controller on job", "job", job, "backupconf", backupConf)
return err
}
log.V(0).Info("creating a backup job", "target", target)
if err := r.Create(ctx, job); err != nil {
log.Error(err, "unable to create job", "job", job)
return err
}
return nil
}
// start the next task
startNextTask := func() (*formolv1alpha1.TargetStatus, error) {
nextTarget := len(backupSession.Status.Targets)
if nextTarget < len(backupConf.Spec.Targets) {
target := backupConf.Spec.Targets[nextTarget]
targetStatus := formolv1alpha1.TargetStatus{
Name: target.Name,
Kind: target.Kind,
SessionState: formolv1alpha1.New,
StartTime: &metav1.Time{Time: time.Now()},
Try: 1,
}
backupSession.Status.Targets = append(backupSession.Status.Targets, targetStatus)
switch target.Kind {
case formolv1alpha1.JobKind:
if err := createBackupJob(target); err != nil {
log.V(0).Info("unable to create task", "task", target)
targetStatus.SessionState = formolv1alpha1.Failure
return nil, err
}
}
return &targetStatus, nil
} else {
return nil, nil
}
}
// cleanup existing backupsessions
cleanupSessions := func() {
backupSessionList := &formolv1alpha1.BackupSessionList{}
if err := r.List(ctx, backupSessionList, client.InNamespace(backupConf.Namespace), client.MatchingFieldsSelector{Selector: fields.SelectorFromSet(fields.Set{sessionState: string(formolv1alpha1.Success)})}); err != nil {
log.Error(err, "unable to get backupsessionlist")
return
}
if len(backupSessionList.Items) < 2 {
// Not enough backupSession to proceed
log.V(1).Info("Not enough successful backup jobs")
return
}
sort.Slice(backupSessionList.Items, func(i, j int) bool {
return backupSessionList.Items[i].Status.StartTime.Time.Unix() > backupSessionList.Items[j].Status.StartTime.Time.Unix()
})
type KeepBackup struct {
Counter int32
Last time.Time
}
var lastBackups, dailyBackups, weeklyBackups, monthlyBackups, yearlyBackups KeepBackup
lastBackups.Counter = backupConf.Spec.Keep.Last
dailyBackups.Counter = backupConf.Spec.Keep.Daily
weeklyBackups.Counter = backupConf.Spec.Keep.Weekly
monthlyBackups.Counter = backupConf.Spec.Keep.Monthly
yearlyBackups.Counter = backupConf.Spec.Keep.Yearly
for _, session := range backupSessionList.Items {
if session.Spec.Ref != backupConf.Name {
continue
}
deleteSession := true
keep := []string{}
if lastBackups.Counter > 0 {
log.V(1).Info("Keep backup", "last", session.Status.StartTime)
lastBackups.Counter--
keep = append(keep, "last")
deleteSession = false
}
if dailyBackups.Counter > 0 {
if session.Status.StartTime.Time.YearDay() != dailyBackups.Last.YearDay() {
log.V(1).Info("Keep backup", "daily", session.Status.StartTime)
dailyBackups.Counter--
dailyBackups.Last = session.Status.StartTime.Time
keep = append(keep, "daily")
deleteSession = false
}
}
if weeklyBackups.Counter > 0 {
if session.Status.StartTime.Time.Weekday().String() == "Sunday" && session.Status.StartTime.Time.YearDay() != weeklyBackups.Last.YearDay() {
log.V(1).Info("Keep backup", "weekly", session.Status.StartTime)
weeklyBackups.Counter--
weeklyBackups.Last = session.Status.StartTime.Time
keep = append(keep, "weekly")
deleteSession = false
}
}
if monthlyBackups.Counter > 0 {
if session.Status.StartTime.Time.Day() == 1 && session.Status.StartTime.Time.Month() != monthlyBackups.Last.Month() {
log.V(1).Info("Keep backup", "monthly", session.Status.StartTime)
monthlyBackups.Counter--
monthlyBackups.Last = session.Status.StartTime.Time
keep = append(keep, "monthly")
deleteSession = false
}
}
if yearlyBackups.Counter > 0 {
if session.Status.StartTime.Time.YearDay() == 1 && session.Status.StartTime.Time.Year() != yearlyBackups.Last.Year() {
log.V(1).Info("Keep backup", "yearly", session.Status.StartTime)
yearlyBackups.Counter--
yearlyBackups.Last = session.Status.StartTime.Time
keep = append(keep, "yearly")
deleteSession = false
}
}
if deleteSession {
log.V(1).Info("Delete session", "delete", session.Status.StartTime)
if err := r.Delete(ctx, &session); err != nil {
log.Error(err, "unable to delete backupsession", "session", session.Name)
// we don't return anything, we keep going
}
} else {
session.Status.Keep = strings.Join(keep, ",") // + " " + time.Now().Format("2006 Jan 02 15:04:05 -0700 MST")
if err := r.Status().Update(ctx, &session); err != nil {
log.Error(err, "unable to update session status", "session", session)
}
}
}
}
// end helper functions
log.V(0).Info("backupSession", "backupSession.ObjectMeta", backupSession.ObjectMeta, "backupSession.Status", backupSession.Status)
if backupSession.ObjectMeta.DeletionTimestamp.IsZero() {
switch backupSession.Status.SessionState {
case formolv1alpha1.New:
// Check if the finalizer has been registered
if !controllerutil.ContainsFinalizer(backupSession, finalizerName) {
controllerutil.AddFinalizer(backupSession, finalizerName)
// We update the BackupSession to add the finalizer
// Reconcile will be called again
// return now
err := r.Update(ctx, backupSession)
if err != nil {
log.Error(err, "unable to add finalizer")
}
return ctrl.Result{}, err
}
// Brand new backupsession
if isBackupOngoing() {
log.V(0).Info("There is an ongoing backup. Let's reschedule this operation")
return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
}
// start the first task
backupSession.Status.SessionState = formolv1alpha1.Running
targetStatus, err := startNextTask()
if err != nil {
return ctrl.Result{}, err
}
log.V(0).Info("New backup. Start the first task", "task", targetStatus)
if err := r.Status().Update(ctx, backupSession); err != nil {
log.Error(err, "unable to update BackupSession status")
return ctrl.Result{}, err
}
case formolv1alpha1.Running:
// Backup ongoing. Check the status of the last task to decide what to do
currentTargetStatus := &backupSession.Status.Targets[len(backupSession.Status.Targets)-1]
switch currentTargetStatus.SessionState {
case formolv1alpha1.Running:
// The current task is still running. Nothing to do
log.V(0).Info("task is still running", "targetStatus", currentTargetStatus)
case formolv1alpha1.Success:
// The last task succeed. Let's try to start the next one
targetStatus, err := startNextTask()
log.V(0).Info("last task was a success. start a new one", "currentTargetStatus", currentTargetStatus, "targetStatus", targetStatus)
if err != nil {
return ctrl.Result{}, err
}
if targetStatus == nil {
// No more task to start. The backup is a success
backupSession.Status.SessionState = formolv1alpha1.Success
log.V(0).Info("Backup is successful. Let's try to do some cleanup")
cleanupSessions()
}
if err := r.Status().Update(ctx, backupSession); err != nil {
log.Error(err, "unable to update BackupSession status")
return ctrl.Result{}, err
}
case formolv1alpha1.Failure:
// last task failed. Try to run it again
currentTarget := backupConf.Spec.Targets[len(backupSession.Status.Targets)-1]
if currentTargetStatus.Try < currentTarget.Retry {
log.V(0).Info("last task was a failure. try again", "currentTargetStatus", currentTargetStatus)
currentTargetStatus.Try++
currentTargetStatus.SessionState = formolv1alpha1.New
currentTargetStatus.StartTime = &metav1.Time{Time: time.Now()}
switch currentTarget.Kind {
case formolv1alpha1.JobKind:
if err := createBackupJob(currentTarget); err != nil {
log.V(0).Info("unable to create task", "task", currentTarget)
currentTargetStatus.SessionState = formolv1alpha1.Failure
return ctrl.Result{}, err
}
}
} else {
log.V(0).Info("task failed again and for the last time", "currentTargetStatus", currentTargetStatus)
backupSession.Status.SessionState = formolv1alpha1.Failure
}
if err := r.Status().Update(ctx, backupSession); err != nil {
log.Error(err, "unable to update BackupSession status")
return ctrl.Result{}, err
}
}
case formolv1alpha1.Success:
// Should never go there
case formolv1alpha1.Failure:
// The backup failed
case "":
// BackupSession has just been created
backupSession.Status.SessionState = formolv1alpha1.New
backupSession.Status.StartTime = &metav1.Time{Time: time.Now()}
if err := r.Status().Update(ctx, backupSession); err != nil {
log.Error(err, "unable to update backupSession")
return ctrl.Result{}, err
}
}
controllerutil.RemoveFinalizer(r.BackupSession, finalizerName)
if err := r.Update(ctx, r.BackupSession); err != nil {
} else {
log.V(0).Info("backupsession being deleted", "backupsession", backupSession.Name)
if controllerutil.ContainsFinalizer(backupSession, finalizerName) {
if err := deleteExternalResources(); err != nil {
return ctrl.Result{}, err
}
}
controllerutil.RemoveFinalizer(backupSession, finalizerName)
if err := r.Update(ctx, backupSession); err != nil {
log.Error(err, "unable to remove finalizer")
return ctrl.Result{}, err
}
@ -323,144 +456,6 @@ func (r *BackupSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
return ctrl.Result{}, nil
}
func (r *BackupSessionReconciler) CreateBackupJob(target formolv1alpha1.Target) error {
log := r.Log.WithValues("createbackupjob", target.Name)
ctx := context.Background()
backupSessionEnv := []corev1.EnvVar{
corev1.EnvVar{
Name: "TARGET_NAME",
Value: target.Name,
},
corev1.EnvVar{
Name: "BACKUPSESSION_NAME",
Value: r.BackupSession.Name,
},
corev1.EnvVar{
Name: "BACKUPSESSION_NAMESPACE",
Value: r.BackupSession.Namespace,
},
}
output := corev1.VolumeMount{
Name: "output",
MountPath: "/output",
}
restic := corev1.Container{
Name: "restic",
Image: "desmo999r/formolcli:latest",
Args: []string{"volume", "backup", "--tag", r.BackupSession.Name, "--path", "/output"},
VolumeMounts: []corev1.VolumeMount{output},
Env: backupSessionEnv,
}
log.V(1).Info("creating a tagged backup job", "container", restic)
// Gather information from the repo
repo := &formolv1alpha1.Repo{}
if err := r.Get(ctx, client.ObjectKey{
Namespace: r.BackupConf.Namespace,
Name: r.BackupConf.Spec.Repository,
}, repo); err != nil {
log.Error(err, "unable to get Repo from BackupConfiguration")
return err
}
// S3 backing storage
restic.Env = append(restic.Env, formolutils.ConfigureResticEnvVar(r.BackupConf, repo)...)
jobTtl := JOBTTL
job := &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
GenerateName: fmt.Sprintf("%s-%s-", r.BackupSession.Name, target.Name),
Namespace: r.BackupConf.Namespace,
},
Spec: batchv1.JobSpec{
TTLSecondsAfterFinished: &jobTtl,
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
InitContainers: []corev1.Container{},
Containers: []corev1.Container{restic},
Volumes: []corev1.Volume{
corev1.Volume{Name: "output"},
},
RestartPolicy: corev1.RestartPolicyOnFailure,
},
},
},
}
for _, step := range target.Steps {
function := &formolv1alpha1.Function{}
if err := r.Get(ctx, client.ObjectKey{
Namespace: step.Namespace,
Name: step.Name}, function); err != nil {
log.Error(err, "unable to get function", "Function", step)
return err
}
function.Spec.Env = append(step.Env, backupSessionEnv...)
function.Spec.VolumeMounts = append(function.Spec.VolumeMounts, output)
job.Spec.Template.Spec.InitContainers = append(job.Spec.Template.Spec.InitContainers, function.Spec)
}
if err := ctrl.SetControllerReference(r.BackupConf, job, r.Scheme); err != nil {
log.Error(err, "unable to set controller on job", "job", job, "backupconf", r.BackupConf)
return err
}
log.V(0).Info("creating a backup job", "target", target)
if err := r.Create(ctx, job); err != nil {
log.Error(err, "unable to create job", "job", job)
return err
}
return nil
}
func (r *BackupSessionReconciler) deleteExternalResources() error {
ctx := context.Background()
log := r.Log.WithValues("deleteExternalResources", r.BackupSession.Name)
// Gather information from the repo
repo := &formolv1alpha1.Repo{}
if err := r.Get(ctx, client.ObjectKey{
Namespace: r.BackupConf.Namespace,
Name: r.BackupConf.Spec.Repository,
}, repo); err != nil {
log.Error(err, "unable to get Repo from BackupConfiguration")
return err
}
env := formolutils.ConfigureResticEnvVar(r.BackupConf, repo)
// container that will delete the restic snapshot(s) matching the backupsession
deleteSnapshots := []corev1.Container{}
for _, target := range r.BackupSession.Status.Targets {
if target.SessionState == formolv1alpha1.Success {
deleteSnapshots = append(deleteSnapshots, corev1.Container{
Name: target.Name,
Image: "desmo999r/formolcli:latest",
Args: []string{"snapshot", "delete", "--snapshot-id", target.SnapshotId},
Env: env,
})
}
}
// create a job to delete the restic snapshot(s) with the backupsession name tag
if len(deleteSnapshots) > 0 {
jobTtl := JOBTTL
job := &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
GenerateName: fmt.Sprintf("delete-%s-", r.BackupSession.Name),
Namespace: r.BackupSession.Namespace,
},
Spec: batchv1.JobSpec{
TTLSecondsAfterFinished: &jobTtl,
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
InitContainers: []corev1.Container{},
Containers: deleteSnapshots,
RestartPolicy: corev1.RestartPolicyOnFailure,
},
},
},
}
log.V(0).Info("creating a job to delete restic snapshots")
if err := r.Create(ctx, job); err != nil {
log.Error(err, "unable to delete job", "job", job)
return err
}
}
return nil
}
func (r *BackupSessionReconciler) SetupWithManager(mgr ctrl.Manager) error {
if err := mgr.GetFieldIndexer().IndexField(context.TODO(), &formolv1alpha1.BackupSession{}, sessionState, func(rawObj runtime.Object) []string {
session := rawObj.(*formolv1alpha1.BackupSession)

144
controllers/backupsession_controller_test.go Normal file
View File

@ -0,0 +1,144 @@
package controllers
import (
"context"
formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
//corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)
var _ = Describe("Testing BackupSession controller", func() {
const (
BSBackupSessionName = "test-backupsession-controller"
)
var (
ctx = context.Background()
key = types.NamespacedName{
Name: BSBackupSessionName,
Namespace: TestNamespace,
}
backupSession = &formolv1alpha1.BackupSession{}
)
BeforeEach(func() {
backupSession = &formolv1alpha1.BackupSession{
ObjectMeta: metav1.ObjectMeta{
Name: BSBackupSessionName,
Namespace: TestNamespace,
},
Spec: formolv1alpha1.BackupSessionSpec{
Ref: TestBackupConfName,
},
}
})
Context("Creating a backupsession", func() {
JustBeforeEach(func() {
Eventually(func() error {
return k8sClient.Create(ctx, backupSession)
}, timeout, interval).Should(Succeed())
realBackupSession := &formolv1alpha1.BackupSession{}
Eventually(func() error {
err := k8sClient.Get(ctx, key, realBackupSession)
return err
}, timeout, interval).Should(Succeed())
Eventually(func() formolv1alpha1.SessionState {
if err := k8sClient.Get(ctx, key, realBackupSession); err != nil {
return ""
} else {
return realBackupSession.Status.SessionState
}
}, timeout, interval).Should(Equal(formolv1alpha1.Running))
})
AfterEach(func() {
Expect(k8sClient.Delete(ctx, backupSession)).Should(Succeed())
})
It("Should have a new task", func() {
realBackupSession := &formolv1alpha1.BackupSession{}
_ = k8sClient.Get(ctx, key, realBackupSession)
Expect(realBackupSession.Status.Targets[0].Name).Should(Equal(TestDeploymentName))
Expect(realBackupSession.Status.Targets[0].SessionState).Should(Equal(formolv1alpha1.New))
Expect(realBackupSession.Status.Targets[0].Kind).Should(Equal(formolv1alpha1.SidecarKind))
Expect(realBackupSession.Status.Targets[0].Try).Should(Equal(1))
})
It("Should move to the next task when the first one is a success", func() {
realBackupSession := &formolv1alpha1.BackupSession{}
Expect(k8sClient.Get(ctx, key, realBackupSession)).Should(Succeed())
realBackupSession.Status.Targets[0].SessionState = formolv1alpha1.Success
Expect(k8sClient.Status().Update(ctx, realBackupSession)).Should(Succeed())
Eventually(func() int {
_ = k8sClient.Get(ctx, key, realBackupSession)
return len(realBackupSession.Status.Targets)
}, timeout, interval).Should(Equal(2))
Expect(k8sClient.Get(ctx, key, realBackupSession)).Should(Succeed())
Expect(realBackupSession.Status.Targets[1].Name).Should(Equal(TestBackupFuncName))
Expect(realBackupSession.Status.Targets[1].SessionState).Should(Equal(formolv1alpha1.New))
Expect(realBackupSession.Status.Targets[1].Kind).Should(Equal(formolv1alpha1.JobKind))
})
It("Should be a success when the last task is a success", func() {
realBackupSession := &formolv1alpha1.BackupSession{}
Expect(k8sClient.Get(ctx, key, realBackupSession)).Should(Succeed())
realBackupSession.Status.Targets[0].SessionState = formolv1alpha1.Success
Expect(k8sClient.Status().Update(ctx, realBackupSession)).Should(Succeed())
Eventually(func() int {
_ = k8sClient.Get(ctx, key, realBackupSession)
return len(realBackupSession.Status.Targets)
}, timeout, interval).Should(Equal(2))
Expect(k8sClient.Get(ctx, key, realBackupSession)).Should(Succeed())
realBackupSession.Status.Targets[1].SessionState = formolv1alpha1.Success
Expect(k8sClient.Status().Update(ctx, realBackupSession)).Should(Succeed())
Expect(k8sClient.Get(ctx, key, realBackupSession)).Should(Succeed())
Eventually(func() formolv1alpha1.SessionState {
_ = k8sClient.Get(ctx, key, realBackupSession)
return realBackupSession.Status.SessionState
}, timeout, interval).Should(Equal(formolv1alpha1.Success))
})
It("Should retry when the task is a failure", func() {
realBackupSession := &formolv1alpha1.BackupSession{}
Expect(k8sClient.Get(ctx, key, realBackupSession)).Should(Succeed())
realBackupSession.Status.Targets[0].SessionState = formolv1alpha1.Success
Expect(k8sClient.Status().Update(ctx, realBackupSession)).Should(Succeed())
Eventually(func() int {
_ = k8sClient.Get(ctx, key, realBackupSession)
return len(realBackupSession.Status.Targets)
}, timeout, interval).Should(Equal(2))
Expect(k8sClient.Get(ctx, key, realBackupSession)).Should(Succeed())
realBackupSession.Status.Targets[1].SessionState = formolv1alpha1.Failure
Expect(k8sClient.Status().Update(ctx, realBackupSession)).Should(Succeed())
Eventually(func() int {
_ = k8sClient.Get(ctx, key, realBackupSession)
return realBackupSession.Status.Targets[1].Try
}, timeout, interval).Should(Equal(2))
Expect(k8sClient.Get(ctx, key, realBackupSession)).Should(Succeed())
Expect(realBackupSession.Status.Targets[1].SessionState).Should(Equal(formolv1alpha1.New))
realBackupSession.Status.Targets[1].SessionState = formolv1alpha1.Failure
Expect(k8sClient.Status().Update(ctx, realBackupSession)).Should(Succeed())
Eventually(func() formolv1alpha1.SessionState {
_ = k8sClient.Get(ctx, key, realBackupSession)
return realBackupSession.Status.SessionState
}, timeout, interval).Should(Equal(formolv1alpha1.Failure))
})
It("should create a backup job", func() {
})
})
Context("When other BackupSession exist", func() {
const (
bs1Name = "test-backupsession-controller1"
bs2Name = "test-backupsession-controller2"
bs3Name = "test-backupsession-controller3"
)
var ()
BeforeEach(func() {
})
JustBeforeEach(func() {
})
It("Should clean up old sessions", func() {
})
})
})

518
controllers/restoresession_controller.go
View File

@ -43,224 +43,259 @@ const (
// RestoreSessionReconciler reconciles a RestoreSession object
type RestoreSessionReconciler struct {
client.Client
Log logr.Logger
Scheme *runtime.Scheme
RestoreSession *formolv1alpha1.RestoreSession
BackupSession *formolv1alpha1.BackupSession
BackupConf *formolv1alpha1.BackupConfiguration
Log logr.Logger
Scheme *runtime.Scheme
}
func (r *RestoreSessionReconciler) CreateRestoreJob(target formolv1alpha1.Target) error {
log := r.Log.WithValues("createrestorejob", target.Name)
// +kubebuilder:rbac:groups=formol.desmojim.fr,resources=restoresessions,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=formol.desmojim.fr,resources=restoresessions/status,verbs=get;update;patch
func (r *RestoreSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
ctx := context.Background()
restoreSessionEnv := []corev1.EnvVar{
corev1.EnvVar{
Name: "TARGET_NAME",
Value: target.Name,
},
corev1.EnvVar{
Name: "RESTORESESSION_NAME",
Value: r.RestoreSession.Name,
},
corev1.EnvVar{
Name: "RESTORESESSION_NAMESPACE",
Value: r.RestoreSession.Namespace,
},
log := r.Log.WithValues("restoresession", req.NamespacedName)
// Get the RestoreSession
restoreSession := &formolv1alpha1.RestoreSession{}
if err := r.Get(ctx, req.NamespacedName, restoreSession); err != nil {
log.Error(err, "unable to get restoresession")
return ctrl.Result{}, client.IgnoreNotFound(err)
}
// Get the BackupSession the RestoreSession references
backupSession := &formolv1alpha1.BackupSession{}
if err := r.Get(ctx, client.ObjectKey{
Namespace: restoreSession.Namespace,
Name: restoreSession.Spec.Ref}, backupSession); err != nil {
log.Error(err, "unable to get backupsession", "restoresession", restoreSession.Spec)
return ctrl.Result{}, client.IgnoreNotFound(err)
}
// Get the BackupConfiguration linked to the BackupSession
backupConf := &formolv1alpha1.BackupConfiguration{}
if err := r.Get(ctx, client.ObjectKey{
Namespace: backupSession.Namespace,
Name: backupSession.Spec.Ref}, backupConf); err != nil {
log.Error(err, "unable to get backupConfiguration")
return ctrl.Result{}, client.IgnoreNotFound(err)
}
output := corev1.VolumeMount{
Name: "output",
MountPath: "/output",
}
for _, targetStatus := range r.BackupSession.Status.Targets {
if targetStatus.Name == target.Name {
snapshotId := targetStatus.SnapshotId
restic := corev1.Container{
Name: "restic",
Image: "desmo999r/formolcli:latest",
Args: []string{"volume", "restore", "--snapshot-id", snapshotId},
VolumeMounts: []corev1.VolumeMount{output},
Env: restoreSessionEnv,
}
finalizer := corev1.Container{
Name: "finalizer",
Image: "desmo999r/formolcli:latest",
Args: []string{"target", "finalize"},
VolumeMounts: []corev1.VolumeMount{output},
Env: restoreSessionEnv,
}
repo := &formolv1alpha1.Repo{}
if err := r.Get(ctx, client.ObjectKey{
Namespace: r.BackupConf.Namespace,
Name: r.BackupConf.Spec.Repository,
}, repo); err != nil {
log.Error(err, "unable to get Repo from BackupConfiguration")
return err
}
// S3 backing storage
var ttl int32 = 300
restic.Env = append(restic.Env, formolutils.ConfigureResticEnvVar(r.BackupConf, repo)...)
job := &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
GenerateName: fmt.Sprintf("%s-%s-", r.RestoreSession.Name, target.Name),
Namespace: r.RestoreSession.Namespace,
},
Spec: batchv1.JobSpec{
TTLSecondsAfterFinished: &ttl,
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
InitContainers: []corev1.Container{restic},
Containers: []corev1.Container{finalizer},
Volumes: []corev1.Volume{
corev1.Volume{Name: "output"},
},
RestartPolicy: corev1.RestartPolicyOnFailure,
},
},
},
}
for _, step := range target.Steps {
function := &formolv1alpha1.Function{}
// Helper functions
createRestoreJob := func(target formolv1alpha1.Target) error {
restoreSessionEnv := []corev1.EnvVar{
corev1.EnvVar{
Name: "TARGET_NAME",
Value: target.Name,
},
corev1.EnvVar{
Name: "RESTORESESSION_NAME",
Value: restoreSession.Name,
},
corev1.EnvVar{
Name: "RESTORESESSION_NAMESPACE",
Value: restoreSession.Namespace,
},
}
output := corev1.VolumeMount{
Name: "output",
MountPath: "/output",
}
for _, targetStatus := range backupSession.Status.Targets {
if targetStatus.Name == target.Name {
snapshotId := targetStatus.SnapshotId
restic := corev1.Container{
Name: "restic",
Image: "desmo999r/formolcli:latest",
Args: []string{"volume", "restore", "--snapshot-id", snapshotId},
VolumeMounts: []corev1.VolumeMount{output},
Env: restoreSessionEnv,
}
finalizer := corev1.Container{
Name: "finalizer",
Image: "desmo999r/formolcli:latest",
Args: []string{"target", "finalize"},
VolumeMounts: []corev1.VolumeMount{output},
Env: restoreSessionEnv,
}
repo := &formolv1alpha1.Repo{}
if err := r.Get(ctx, client.ObjectKey{
Namespace: r.RestoreSession.Namespace,
Name: strings.Replace(step.Name, "backup", "restore", 1)}, function); err != nil {
log.Error(err, "unable to get function", "function", step)
Namespace: backupConf.Namespace,
Name: backupConf.Spec.Repository,
}, repo); err != nil {
log.Error(err, "unable to get Repo from BackupConfiguration")
return err
}
// S3 backing storage
var ttl int32 = 300
restic.Env = append(restic.Env, formolutils.ConfigureResticEnvVar(backupConf, repo)...)
job := &batchv1.Job{
ObjectMeta: metav1.ObjectMeta{
GenerateName: fmt.Sprintf("%s-%s-", restoreSession.Name, target.Name),
Namespace: restoreSession.Namespace,
},
Spec: batchv1.JobSpec{
TTLSecondsAfterFinished: &ttl,
Template: corev1.PodTemplateSpec{
Spec: corev1.PodSpec{
InitContainers: []corev1.Container{restic},
Containers: []corev1.Container{finalizer},
Volumes: []corev1.Volume{
corev1.Volume{Name: "output"},
},
RestartPolicy: corev1.RestartPolicyOnFailure,
},
},
},
}
for _, step := range target.Steps {
function := &formolv1alpha1.Function{}
// get the backup function
if err := r.Get(ctx, client.ObjectKey{
Namespace: restoreSession.Namespace,
Name: step.Name,
}, function); err != nil {
log.Error(err, "unable to get backup function", "name", step.Name)
return err
}
var restoreName string
if function.Annotations["restoreFunction"] != "" {
restoreName = function.Annotations["restoreFunction"]
} else {
restoreName = strings.Replace(step.Name, "backup", "restore", 1)
}
if err := r.Get(ctx, client.ObjectKey{
Namespace: restoreSession.Namespace,
Name: restoreName,
}, function); err != nil {
log.Error(err, "unable to get function", "function", step)
return err
}
function.Spec.Name = function.Name
function.Spec.Env = append(step.Env, restoreSessionEnv...)
function.Spec.VolumeMounts = append(function.Spec.VolumeMounts, output)
job.Spec.Template.Spec.InitContainers = append(job.Spec.Template.Spec.InitContainers, function.Spec)
}
if err := ctrl.SetControllerReference(restoreSession, job, r.Scheme); err != nil {
log.Error(err, "unable to set controller on job", "job", job, "restoresession", restoreSession)
return err
}
log.V(0).Info("creating a restore job", "target", target.Name)
if err := r.Create(ctx, job); err != nil {
log.Error(err, "unable to create job", "job", job)
return err
}
function.Spec.Env = append(step.Env, restoreSessionEnv...)
function.Spec.VolumeMounts = append(function.Spec.VolumeMounts, output)
job.Spec.Template.Spec.InitContainers = append(job.Spec.Template.Spec.InitContainers, function.Spec)
}
if err := ctrl.SetControllerReference(r.RestoreSession, job, r.Scheme); err != nil {
log.Error(err, "unable to set controller on job", "job", job, "restoresession", r.RestoreSession)
return err
}
log.V(0).Info("creating a restore job", "target", target.Name)
if err := r.Create(ctx, job); err != nil {
log.Error(err, "unable to create job", "job", job)
return err
}
}
return nil
}
return nil
}
func (r *RestoreSessionReconciler) DeleteRestoreInitContainer(target formolv1alpha1.Target) error {
log := r.Log.WithValues("createrestoreinitcontainer", target.Name)
ctx := context.Background()
deployment := &appsv1.Deployment{}
if err := r.Get(context.Background(), client.ObjectKey{
Namespace: r.BackupConf.Namespace,
Name: target.Name,
}, deployment); err != nil {
log.Error(err, "unable to get deployment")
return err
}
log.V(1).Info("got deployment", "namespace", deployment.Namespace, "name", deployment.Name)
newInitContainers := []corev1.Container{}
for _, initContainer := range deployment.Spec.Template.Spec.InitContainers {
if initContainer.Name == RESTORESESSION {
log.V(0).Info("Found our restoresession container. Removing it from the list of init containers", "container", initContainer)
} else {
newInitContainers = append(newInitContainers, initContainer)
deleteRestoreInitContainer := func(target formolv1alpha1.Target) error {
deployment := &appsv1.Deployment{}
if err := r.Get(context.Background(), client.ObjectKey{
Namespace: backupConf.Namespace,
Name: target.Name,
}, deployment); err != nil {
log.Error(err, "unable to get deployment")
return err
}
}
deployment.Spec.Template.Spec.InitContainers = newInitContainers
if err := r.Update(ctx, deployment); err != nil {
log.Error(err, "unable to update deployment")
return err
}
return nil
}
func (r *RestoreSessionReconciler) CreateRestoreInitContainer(target formolv1alpha1.Target) error {
log := r.Log.WithValues("createrestoreinitcontainer", target.Name)
ctx := context.Background()
deployment := &appsv1.Deployment{}
if err := r.Get(context.Background(), client.ObjectKey{
Namespace: r.RestoreSession.Namespace,
Name: target.Name,
}, deployment); err != nil {
log.Error(err, "unable to get deployment")
return err
}
log.V(1).Info("got deployment", "namespace", deployment.Namespace, "name", deployment.Name)
for _, initContainer := range deployment.Spec.Template.Spec.InitContainers {
if initContainer.Name == RESTORESESSION {
log.V(0).Info("there is already a restoresession initcontainer", "deployment", deployment.Spec.Template.Spec.InitContainers)
return nil
log.V(1).Info("got deployment", "namespace", deployment.Namespace, "name", deployment.Name)
newInitContainers := []corev1.Container{}
for _, initContainer := range deployment.Spec.Template.Spec.InitContainers {
if initContainer.Name == RESTORESESSION {
log.V(0).Info("Found our restoresession container. Removing it from the list of init containers", "container", initContainer)
} else {
newInitContainers = append(newInitContainers, initContainer)
}
}
}
var snapshotId string
for _, targetStatus := range r.BackupSession.Status.Targets {
if targetStatus.Name == target.Name && targetStatus.Kind == target.Kind {
snapshotId = targetStatus.SnapshotId
deployment.Spec.Template.Spec.InitContainers = newInitContainers
if err := r.Update(ctx, deployment); err != nil {
log.Error(err, "unable to update deployment")
return err
}
}
restoreSessionEnv := []corev1.EnvVar{
corev1.EnvVar{
Name: formolv1alpha1.TARGET_NAME,
Value: target.Name,
},
corev1.EnvVar{
Name: formolv1alpha1.RESTORESESSION_NAME,
Value: r.RestoreSession.Name,
},
corev1.EnvVar{
Name: formolv1alpha1.RESTORESESSION_NAMESPACE,
Value: r.RestoreSession.Namespace,
},
}
initContainer := corev1.Container{
Name: RESTORESESSION,
Image: formolutils.FORMOLCLI,
Args: []string{"volume", "restore", "--snapshot-id", snapshotId},
VolumeMounts: target.VolumeMounts,
Env: restoreSessionEnv,
}
repo := &formolv1alpha1.Repo{}
if err := r.Get(ctx, client.ObjectKey{
Namespace: r.BackupConf.Namespace,
Name: r.BackupConf.Spec.Repository,
}, repo); err != nil {
log.Error(err, "unable to get Repo from BackupConfiguration")
return err
}
// S3 backing storage
initContainer.Env = append(initContainer.Env, formolutils.ConfigureResticEnvVar(r.BackupConf, repo)...)
deployment.Spec.Template.Spec.InitContainers = append([]corev1.Container{initContainer},
deployment.Spec.Template.Spec.InitContainers...)
if err := r.Update(ctx, deployment); err != nil {
log.Error(err, "unable to update deployment")
return err
return nil
}
return nil
}
createRestoreInitContainer := func(target formolv1alpha1.Target) error {
deployment := &appsv1.Deployment{}
if err := r.Get(context.Background(), client.ObjectKey{
Namespace: restoreSession.Namespace,
Name: target.Name,
}, deployment); err != nil {
log.Error(err, "unable to get deployment")
return err
}
log.V(1).Info("got deployment", "namespace", deployment.Namespace, "name", deployment.Name)
for _, initContainer := range deployment.Spec.Template.Spec.InitContainers {
if initContainer.Name == RESTORESESSION {
log.V(0).Info("there is already a restoresession initcontainer", "deployment", deployment.Spec.Template.Spec.InitContainers)
return nil
}
}
var snapshotId string
for _, targetStatus := range backupSession.Status.Targets {
if targetStatus.Name == target.Name && targetStatus.Kind == target.Kind {
snapshotId = targetStatus.SnapshotId
}
}
restoreSessionEnv := []corev1.EnvVar{
corev1.EnvVar{
Name: formolv1alpha1.TARGET_NAME,
Value: target.Name,
},
corev1.EnvVar{
Name: formolv1alpha1.RESTORESESSION_NAME,
Value: restoreSession.Name,
},
corev1.EnvVar{
Name: formolv1alpha1.RESTORESESSION_NAMESPACE,
Value: restoreSession.Namespace,
},
}
initContainer := corev1.Container{
Name: RESTORESESSION,
Image: formolutils.FORMOLCLI,
Args: []string{"volume", "restore", "--snapshot-id", snapshotId},
VolumeMounts: target.VolumeMounts,
Env: restoreSessionEnv,
}
repo := &formolv1alpha1.Repo{}
if err := r.Get(ctx, client.ObjectKey{
Namespace: backupConf.Namespace,
Name: backupConf.Spec.Repository,
}, repo); err != nil {
log.Error(err, "unable to get Repo from BackupConfiguration")
return err
}
// S3 backing storage
initContainer.Env = append(initContainer.Env, formolutils.ConfigureResticEnvVar(backupConf, repo)...)
deployment.Spec.Template.Spec.InitContainers = append([]corev1.Container{initContainer},
deployment.Spec.Template.Spec.InitContainers...)
if err := r.Update(ctx, deployment); err != nil {
log.Error(err, "unable to update deployment")
return err
}
return nil
}
func (r *RestoreSessionReconciler) StatusUpdate() error {
log := r.Log.WithValues("statusupdate", r.RestoreSession.Name)
ctx := context.Background()
startNextTask := func() (*formolv1alpha1.TargetStatus, error) {
nextTarget := len(r.RestoreSession.Status.Targets)
if nextTarget < len(r.BackupConf.Spec.Targets) {
target := r.BackupConf.Spec.Targets[nextTarget]
nextTarget := len(restoreSession.Status.Targets)
if nextTarget < len(backupConf.Spec.Targets) {
target := backupConf.Spec.Targets[nextTarget]
targetStatus := formolv1alpha1.TargetStatus{
Name: target.Name,
Kind: target.Kind,
SessionState: formolv1alpha1.New,
StartTime: &metav1.Time{Time: time.Now()},
}
r.RestoreSession.Status.Targets = append(r.RestoreSession.Status.Targets, targetStatus)
restoreSession.Status.Targets = append(restoreSession.Status.Targets, targetStatus)
switch target.Kind {
case "Deployment":
if err := r.CreateRestoreInitContainer(target); err != nil {
case formolv1alpha1.SidecarKind:
if err := createRestoreInitContainer(target); err != nil {
log.V(0).Info("unable to create restore init container", "task", target)
targetStatus.SessionState = formolv1alpha1.Failure
return nil, err
}
case "Task":
if err := r.CreateRestoreJob(target); err != nil {
case formolv1alpha1.JobKind:
if err := createRestoreJob(target); err != nil {
log.V(0).Info("unable to create restore job", "task", target)
targetStatus.SessionState = formolv1alpha1.Failure
return nil, err
@ -271,110 +306,71 @@ func (r *RestoreSessionReconciler) StatusUpdate() error {
return nil, nil
}
}
endTask := func() error {
target := r.BackupConf.Spec.Targets[len(r.RestoreSession.Status.Targets)-1]
target := backupConf.Spec.Targets[len(restoreSession.Status.Targets)-1]
switch target.Kind {
case "Deployment":
if err := r.DeleteRestoreInitContainer(target); err != nil {
case formolv1alpha1.SidecarKind:
if err := deleteRestoreInitContainer(target); err != nil {
log.Error(err, "unable to delete restore init container")
return err
}
}
return nil
}
switch r.RestoreSession.Status.SessionState {
switch restoreSession.Status.SessionState {
case formolv1alpha1.New:
r.RestoreSession.Status.SessionState = formolv1alpha1.Running
targetStatus, err := startNextTask()
if err != nil {
return err
}
log.V(0).Info("New restore. Start the first task", "task", targetStatus.Name)
if err := r.Status().Update(ctx, r.RestoreSession); err != nil {
log.Error(err, "unable to update restoresession")
return err
restoreSession.Status.SessionState = formolv1alpha1.Running
if targetStatus, err := startNextTask(); err != nil {
log.Error(err, "unable to start next restore task")
return ctrl.Result{}, err
} else {
log.V(0).Info("New restore. Start the first task", "task", targetStatus.Name)
if err := r.Status().Update(ctx, restoreSession); err != nil {
log.Error(err, "unable to update restoresession")
return ctrl.Result{}, err
}
}
case formolv1alpha1.Running:
currentTargetStatus := r.RestoreSession.Status.Targets[len(r.RestoreSession.Status.Targets)-1]
currentTargetStatus := restoreSession.Status.Targets[len(restoreSession.Status.Targets)-1]
switch currentTargetStatus.SessionState {
case formolv1alpha1.Failure:
log.V(0).Info("last restore task failed. Stop here", "target", currentTargetStatus.Name)
r.RestoreSession.Status.SessionState = formolv1alpha1.Failure
if err := r.Status().Update(ctx, r.RestoreSession); err != nil {
restoreSession.Status.SessionState = formolv1alpha1.Failure
if err := r.Status().Update(ctx, restoreSession); err != nil {
log.Error(err, "unable to update restoresession")
return err
return ctrl.Result{}, err
}
case formolv1alpha1.Running:
log.V(0).Info("task is still running", "target", currentTargetStatus.Name)
return nil
return ctrl.Result{}, nil
case formolv1alpha1.Success:
_ = endTask()
log.V(0).Info("last task was a success. start a new one", "target", currentTargetStatus)
targetStatus, err := startNextTask()
if err != nil {
return err
return ctrl.Result{}, err
}
if targetStatus == nil {
// No more task to start. The restore is over
r.RestoreSession.Status.SessionState = formolv1alpha1.Success
if err := r.Status().Update(ctx, r.RestoreSession); err != nil {
log.Error(err, "unable to update restoresession")
return err
}
restoreSession.Status.SessionState = formolv1alpha1.Success
}
if err := r.Status().Update(ctx, r.RestoreSession); err != nil {
if err := r.Status().Update(ctx, restoreSession); err != nil {
log.Error(err, "unable to update restoresession")
return err
return ctrl.Result{}, err
}
}
case "":
// Restore session has just been created
restoreSession.Status.SessionState = formolv1alpha1.New
restoreSession.Status.StartTime = &metav1.Time{Time: time.Now()}
if err := r.Status().Update(ctx, restoreSession); err != nil {
log.Error(err, "unable to update restoreSession")
return ctrl.Result{}, err
}
}
return nil
}
// +kubebuilder:rbac:groups=formol.desmojim.fr,resources=restoresessions,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=formol.desmojim.fr,resources=restoresessions/status,verbs=get;update;patch
func (r *RestoreSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
time.Sleep(500 * time.Millisecond)
ctx := context.Background()
log := r.Log.WithValues("restoresession", req.NamespacedName)
r.RestoreSession = &formolv1alpha1.RestoreSession{}
if err := r.Get(ctx, req.NamespacedName, r.RestoreSession); err != nil {
log.Error(err, "unable to get restoresession")
return ctrl.Result{}, client.IgnoreNotFound(err)
}
r.BackupSession = &formolv1alpha1.BackupSession{}
if err := r.Get(ctx, client.ObjectKey{
Namespace: r.RestoreSession.Spec.BackupSessionRef.Namespace,
Name: r.RestoreSession.Spec.BackupSessionRef.Name}, r.BackupSession); err != nil {
log.Error(err, "unable to get backupsession")
return ctrl.Result{}, client.IgnoreNotFound(err)
}
r.BackupConf = &formolv1alpha1.BackupConfiguration{}
if err := r.Get(ctx, client.ObjectKey{
Namespace: r.BackupSession.Namespace,
Name: r.BackupSession.Spec.Ref.Name}, r.BackupConf); err != nil {
log.Error(err, "unable to get backupConfiguration")
return ctrl.Result{}, client.IgnoreNotFound(err)
}
if r.RestoreSession.Status.ObservedGeneration == r.RestoreSession.ObjectMeta.Generation {
// status update
log.V(0).Info("status update")
return ctrl.Result{}, r.StatusUpdate()
}
r.RestoreSession.Status.ObservedGeneration = r.RestoreSession.ObjectMeta.Generation
r.RestoreSession.Status.SessionState = formolv1alpha1.New
r.RestoreSession.Status.StartTime = &metav1.Time{Time: time.Now()}
reschedule := ctrl.Result{RequeueAfter: 5 * time.Second}
if err := r.Status().Update(ctx, r.RestoreSession); err != nil {
log.Error(err, "unable to update restoresession")
return ctrl.Result{}, err
}
return reschedule, nil
return ctrl.Result{}, nil
}
func (r *RestoreSessionReconciler) SetupWithManager(mgr ctrl.Manager) error {

90
controllers/restoresession_controller_test.go Normal file
View File

@ -0,0 +1,90 @@
package controllers
import (
"context"
formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)
var _ = Describe("Testing RestoreSession controller", func() {
const (
RSRestoreSessionName = "test-restoresession-controller"
)
var (
ctx = context.Background()
key = types.NamespacedName{
Name: RSRestoreSessionName,
Namespace: TestNamespace,
}
restoreSession = &formolv1alpha1.RestoreSession{}
)
BeforeEach(func() {
restoreSession = &formolv1alpha1.RestoreSession{
ObjectMeta: metav1.ObjectMeta{
Name: RSRestoreSessionName,
Namespace: TestNamespace,
},
Spec: formolv1alpha1.RestoreSessionSpec{
Ref: TestBackupSessionName,
},
}
})
Context("Creating a RestoreSession", func() {
JustBeforeEach(func() {
Eventually(func() error {
return k8sClient.Create(ctx, restoreSession)
}, timeout, interval).Should(Succeed())
realRestoreSession := &formolv1alpha1.RestoreSession{}
Eventually(func() error {
return k8sClient.Get(ctx, key, realRestoreSession)
}, timeout, interval).Should(Succeed())
Eventually(func() formolv1alpha1.SessionState {
_ = k8sClient.Get(ctx, key, realRestoreSession)
return realRestoreSession.Status.SessionState
}, timeout, interval).Should(Equal(formolv1alpha1.Running))
})
AfterEach(func() {
Expect(k8sClient.Delete(ctx, restoreSession)).Should(Succeed())
})
It("Should have a new task and should fail if the task fails", func() {
restoreSession := &formolv1alpha1.RestoreSession{}
Expect(k8sClient.Get(ctx, key, restoreSession)).Should(Succeed())
Expect(len(restoreSession.Status.Targets)).Should(Equal(1))
Expect(restoreSession.Status.Targets[0].SessionState).Should(Equal(formolv1alpha1.New))
restoreSession.Status.Targets[0].SessionState = formolv1alpha1.Running
Expect(k8sClient.Status().Update(ctx, restoreSession)).Should(Succeed())
Expect(k8sClient.Get(ctx, key, restoreSession)).Should(Succeed())
Eventually(func() formolv1alpha1.SessionState {
_ = k8sClient.Get(ctx, key, restoreSession)
return restoreSession.Status.Targets[0].SessionState
}, timeout, interval).Should(Equal(formolv1alpha1.Running))
restoreSession.Status.Targets[0].SessionState = formolv1alpha1.Failure
Expect(k8sClient.Status().Update(ctx, restoreSession)).Should(Succeed())
Expect(k8sClient.Get(ctx, key, restoreSession)).Should(Succeed())
Eventually(func() formolv1alpha1.SessionState {
_ = k8sClient.Get(ctx, key, restoreSession)
return restoreSession.Status.SessionState
}, timeout, interval).Should(Equal(formolv1alpha1.Failure))
})
It("Should move to the new task if the first one is a success and be a success if all the tasks succeed", func() {
restoreSession := &formolv1alpha1.RestoreSession{}
Expect(k8sClient.Get(ctx, key, restoreSession)).Should(Succeed())
Expect(len(restoreSession.Status.Targets)).Should(Equal(1))
restoreSession.Status.Targets[0].SessionState = formolv1alpha1.Success
Expect(k8sClient.Status().Update(ctx, restoreSession)).Should(Succeed())
Eventually(func() int {
_ = k8sClient.Get(ctx, key, restoreSession)
return len(restoreSession.Status.Targets)
}, timeout, interval).Should(Equal(2))
restoreSession.Status.Targets[1].SessionState = formolv1alpha1.Success
Expect(k8sClient.Status().Update(ctx, restoreSession)).Should(Succeed())
Eventually(func() formolv1alpha1.SessionState {
_ = k8sClient.Get(ctx, key, restoreSession)
return restoreSession.Status.SessionState
}, timeout, interval).Should(Equal(formolv1alpha1.Success))
})
})
})

151
controllers/suite_test.go
View File

@ -44,12 +44,18 @@ import (
// These tests use Ginkgo (BDD-style Go testing framework). Refer to
// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
const (
BackupFuncName = "test-backup-func"
TestNamespace = "test-namespace"
RepoName = "test-repo"
DeploymentName = "test-deployment"
timeout = time.Second * 10
interval = time.Millisecond * 250
TestBackupFuncName = "test-backup-func"
TestFunc = "test-norestore-func"
TestRestoreFuncName = "test-restore-func"
TestNamespace = "test-namespace"
TestRepoName = "test-repo"
TestDeploymentName = "test-deployment"
TestBackupConfName = "test-backupconf"
TestBackupSessionName = "test-backupsession"
TestDataVolume = "data"
TestDataMountPath = "/data"
timeout = time.Second * 10
interval = time.Millisecond * 250
)
var cfg *rest.Config
@ -64,7 +70,7 @@ var (
}
deployment = &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: DeploymentName,
Name: TestDeploymentName,
Namespace: TestNamespace,
},
Spec: appsv1.DeploymentSpec{
@ -82,6 +88,11 @@ var (
Image: "test-image",
},
},
Volumes: []corev1.Volume{
corev1.Volume{
Name: TestDataVolume,
},
},
},
},
},
@ -105,7 +116,7 @@ var (
}
repo = &formolv1alpha1.Repo{
ObjectMeta: metav1.ObjectMeta{
Name: RepoName,
Name: TestRepoName,
Namespace: TestNamespace,
},
Spec: formolv1alpha1.RepoSpec{
@ -120,7 +131,29 @@ var (
}
function = &formolv1alpha1.Function{
ObjectMeta: metav1.ObjectMeta{
Name: BackupFuncName,
Name: TestFunc,
Namespace: TestNamespace,
},
Spec: corev1.Container{
Name: "norestore-func",
Image: "myimage",
Args: []string{"a", "set", "of", "args"},
},
}
backupFunc = &formolv1alpha1.Function{
ObjectMeta: metav1.ObjectMeta{
Name: TestRestoreFuncName,
Namespace: TestNamespace,
},
Spec: corev1.Container{
Name: "restore-func",
Image: "myimage",
Args: []string{"a", "set", "of", "args"},
},
}
restoreFunc = &formolv1alpha1.Function{
ObjectMeta: metav1.ObjectMeta{
Name: TestBackupFuncName,
Namespace: TestNamespace,
},
Spec: corev1.Container{
@ -129,6 +162,66 @@ var (
Args: []string{"a", "set", "of", "args"},
},
}
testBackupConf = &formolv1alpha1.BackupConfiguration{
ObjectMeta: metav1.ObjectMeta{
Name: TestBackupConfName,
Namespace: TestNamespace,
},
Spec: formolv1alpha1.BackupConfigurationSpec{
Repository: TestRepoName,
Schedule: "1 * * * *",
Keep: formolv1alpha1.Keep{
Last: 2,
},
Targets: []formolv1alpha1.Target{
formolv1alpha1.Target{
Kind: formolv1alpha1.SidecarKind,
Name: TestDeploymentName,
Steps: []formolv1alpha1.Step{
formolv1alpha1.Step{
Name: TestFunc,
},
},
Paths: []string{
TestDataMountPath,
},
VolumeMounts: []corev1.VolumeMount{
corev1.VolumeMount{
Name: TestDataVolume,
MountPath: TestDataMountPath,
},
},
},
formolv1alpha1.Target{
Kind: formolv1alpha1.JobKind,
Name: TestBackupFuncName,
Steps: []formolv1alpha1.Step{
formolv1alpha1.Step{
Name: TestFunc,
},
formolv1alpha1.Step{
Name: TestBackupFuncName,
Env: []corev1.EnvVar{
corev1.EnvVar{
Name: "foo",
Value: "bar",
},
},
},
},
},
},
},
}
testBackupSession = &formolv1alpha1.BackupSession{
ObjectMeta: metav1.ObjectMeta{
Name: TestBackupSessionName,
Namespace: TestNamespace,
},
Spec: formolv1alpha1.BackupSessionSpec{
Ref: TestBackupConfName,
},
}
)
func TestAPIs(t *testing.T) {
@ -168,6 +261,20 @@ var _ = BeforeSuite(func() {
}).SetupWithManager(k8sManager)
Expect(err).ToNot(HaveOccurred())
err = (&BackupSessionReconciler{
Client: k8sManager.GetClient(),
Scheme: k8sManager.GetScheme(),
Log: ctrl.Log.WithName("controllers").WithName("BackupSession"),
}).SetupWithManager(k8sManager)
Expect(err).ToNot(HaveOccurred())
err = (&RestoreSessionReconciler{
Client: k8sManager.GetClient(),
Scheme: k8sManager.GetScheme(),
Log: ctrl.Log.WithName("controllers").WithName("RestoreSession"),
}).SetupWithManager(k8sManager)
Expect(err).ToNot(HaveOccurred())
go func() {
err = k8sManager.Start(ctrl.SetupSignalHandler())
Expect(err).ToNot(HaveOccurred())
@ -182,6 +289,32 @@ var _ = BeforeSuite(func() {
Expect(k8sClient.Create(ctx, repo)).Should(Succeed())
Expect(k8sClient.Create(ctx, deployment)).Should(Succeed())
Expect(k8sClient.Create(ctx, function)).Should(Succeed())
Expect(k8sClient.Create(ctx, backupFunc)).Should(Succeed())
Expect(k8sClient.Create(ctx, restoreFunc)).Should(Succeed())
Expect(k8sClient.Create(ctx, testBackupConf)).Should(Succeed())
Expect(k8sClient.Create(ctx, testBackupSession)).Should(Succeed())
Eventually(func() error {
return k8sClient.Get(ctx, client.ObjectKey{
Name: TestBackupSessionName,
Namespace: TestNamespace,
}, testBackupSession)
}, timeout, interval).Should(Succeed())
testBackupSession.Status.SessionState = formolv1alpha1.Success
testBackupSession.Status.Targets = []formolv1alpha1.TargetStatus{
formolv1alpha1.TargetStatus{
Name: TestDeploymentName,
Kind: formolv1alpha1.SidecarKind,
SessionState: formolv1alpha1.Success,
SnapshotId: "12345abcdef",
},
formolv1alpha1.TargetStatus{
Name: TestBackupFuncName,
Kind: formolv1alpha1.JobKind,
SessionState: formolv1alpha1.Success,
SnapshotId: "67890ghijk",
},
}
Expect(k8sClient.Status().Update(ctx, testBackupSession)).Should(Succeed())
}, 60)
var _ = AfterSuite(func() {

4
main.go
View File

@ -102,6 +102,10 @@ func main() {
setupLog.Error(err, "unable to create controller", "controller", "RestoreSession")
os.Exit(1)
}
if err = (&formoldesmojimfrv1alpha1.Function{}).SetupWebhookWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create webhook", "webhook", "Function")
os.Exit(1)
}
// +kubebuilder:scaffold:builder
setupLog.Info("starting manager")

2
test/01-deployment.yaml
View File

@ -8,6 +8,8 @@ metadata:
app: nginx
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app: nginx

5
test/02-backupconf.yaml
View File

@ -8,7 +8,7 @@ spec:
repository: repo-minio
schedule: "15 * * * *"
targets:
- kind: Deployment
- kind: Sidecar
apiVersion: v1
name: nginx-deployment
volumeMounts:
@ -16,11 +16,10 @@ spec:
mountPath: /data
paths:
- /data
- kind: Task
- kind: Job
name: backup-pg
steps:
- name: backup-pg
namespace: demo
env:
- name: PGHOST
value: postgres