Merge pull request 'steps' (#8) from steps into master

Reviewed-on: http://git.desmojim.fr/jandre/formol/pulls/8

Makefile

@@ -1,6 +1,6 @@
 
 # Image URL to use all building/pushing image targets
-IMG ?= desmo999r/formolcontroller:latest
+IMG ?= desmo999r/formolcontroller:0.2.0
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 #CRD_OPTIONS ?= "crd:trivialVersions=true"
 CRD_OPTIONS ?= "crd:trivialVersions=true,crdVersions=v1"

api/v1alpha1/backupconfiguration_types.go

@@ -30,8 +30,6 @@ const (
 type Step struct {
 	Name string `json:"name"`
 	// +optional
-	Env []corev1.EnvVar `json:"env,omitempty"`
-	// +optional
 	Finalize *bool `json:"finalize,omitempty"`
 }
 
@@ -46,6 +44,8 @@ type Target struct {
 	Kind string `json:"kind"`
 	Name string `json:"name"`
 	// +optional
+	ContainerName string `json:"containerName"`
+	// +optional
 	ApiVersion string `json:"apiVersion,omitempty"`
 	// +optional
 	VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"`

api/v1alpha1/common.go

@@ -19,6 +19,8 @@ const (
 	RESTORE_ANNOTATION = "restore"
 	// the name of the sidecar container
 	SIDECARCONTAINER_NAME string = "formol"
+	// the name of the container we backup when there are more than 1 container in the pod
+	TARGETCONTAINER_TAG string = "FORMOL_TARGET"
 	// Used by both the backupsession and restoresession controllers to identified the target deployment
 	TARGET_NAME string = "TARGET_NAME"
 	// Used by restoresession controller

api/v1alpha1/zz_generated.deepcopy.go

@@ -569,13 +569,6 @@ func (in *S3) DeepCopy() *S3 {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Step) DeepCopyInto(out *Step) {
 	*out = *in
-	if in.Env != nil {
-		in, out := &in.Env, &out.Env
-		*out = make([]v1.EnvVar, len(*in))
-		for i := range *in {
-			(*in)[i].DeepCopyInto(&(*out)[i])
-		}
-	}
 	if in.Finalize != nil {
 		in, out := &in.Finalize, &out.Finalize
 		*out = new(bool)

config/manager/kustomization.yaml

@@ -5,4 +5,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: desmo999r/formolcontroller
-  newTag: latest
+  newTag: 0.2.0

controllers/backupconfiguration_controller.go

@@ -51,6 +51,8 @@ var _ reconcile.Reconciler = &BackupConfigurationReconciler{}
 // +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=apps,resources=replicasets,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch
+// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch
+// +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch
 // +kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=get;list;watch;create;update;patch;delete
@@ -212,11 +214,19 @@ func (r *BackupConfigurationReconciler) Reconcile(ctx context.Context, req recon
 			return err
 		}
 		log.V(1).Info("got deployment", "Deployment", deployment)
-		for _, container := range deployment.Spec.Template.Spec.Containers {
+		for i, container := range deployment.Spec.Template.Spec.Containers {
 			if container.Name == formolv1alpha1.SIDECARCONTAINER_NAME {
 				log.V(0).Info("There is already a backup sidecar container. Skipping", "container", container)
 				return nil
 			}
+			if target.ContainerName != "" && target.ContainerName == container.Name {
+				// Put a tag so we can find what container we are supposed to backup
+				// and what process we are supposed to chroot to run the init steps
+				deployment.Spec.Template.Spec.Containers[i].Env = append(container.Env, corev1.EnvVar{
+					Name:  formolv1alpha1.TARGETCONTAINER_TAG,
+					Value: "True",
+				})
+			}
 		}
 		sidecar := corev1.Container{
 			Name: formolv1alpha1.SIDECARCONTAINER_NAME,

controllers/backupconfiguration_controller_test.go

@@ -62,12 +62,6 @@ var _ = Describe("Testing BackupConf controller", func() {
 						Steps: []formolv1alpha1.Step{
 							formolv1alpha1.Step{
 								Name: TestBackupFuncName,
-								Env: []corev1.EnvVar{
-									corev1.EnvVar{
-										Name:  "foo",
-										Value: "bar",
-									},
-								},
 							},
 						},
 					},

controllers/backupsession_controller.go

@@ -124,8 +124,8 @@ func (r *BackupSessionReconciler) Reconcile(ctx context.Context, req reconcile.R
 					TTLSecondsAfterFinished: &jobTtl,
 					Template: corev1.PodTemplateSpec{
 						Spec: corev1.PodSpec{
-							InitContainers: []corev1.Container{},
+							InitContainers: deleteSnapshots[1:],
-							Containers:     deleteSnapshots,
+							Containers:     []corev1.Container{deleteSnapshots[0]},
 							RestartPolicy:  corev1.RestartPolicyOnFailure,
 						},
 					},
@@ -212,7 +212,7 @@ func (r *BackupSessionReconciler) Reconcile(ctx context.Context, req reconcile.R
 				return err
 			}
 			function.Spec.Name = function.Name
-			function.Spec.Env = append(step.Env, backupSessionEnv...)
+			function.Spec.Env = append(function.Spec.Env, backupSessionEnv...)
 			function.Spec.VolumeMounts = append(function.Spec.VolumeMounts, output)
 			job.Spec.Template.Spec.InitContainers = append(job.Spec.Template.Spec.InitContainers, function.Spec)
 		}

controllers/restoresession_controller.go

@@ -200,7 +200,7 @@ func (r *RestoreSessionReconciler) Reconcile(ctx context.Context, req reconcile.
 				return err
 			}
 			function.Spec.Name = function.Name
-			function.Spec.Env = append(step.Env, restoreSessionEnv...)
+			function.Spec.Env = append(function.Spec.Env, restoreSessionEnv...)
 			function.Spec.VolumeMounts = append(function.Spec.VolumeMounts, output)
 			job.Spec.Template.Spec.InitContainers = append(job.Spec.Template.Spec.InitContainers, function.Spec)
 		}

controllers/suite_test.go

@@ -159,6 +159,12 @@ var (
 			Name:  "backup-func",
 			Image: "myimage",
 			Args:  []string{"a", "set", "of", "args"},
+			Env: []corev1.EnvVar{
+				corev1.EnvVar{
+					Name:  "foo",
+					Value: "bar",
+				},
+			},
 		},
 	}
 	testBackupConf = &formolv1alpha1.BackupConfiguration{
@@ -201,12 +207,6 @@ var (
 						},
 						formolv1alpha1.Step{
 							Name: TestBackupFuncName,
-							Env: []corev1.EnvVar{
-								corev1.EnvVar{
-									Name:  "foo",
-									Value: "bar",
-								},
-							},
 						},
 					},
 				},

pkg/rbac/backupconfiguration.go

@@ -217,7 +217,7 @@ func CreateFormolRBAC(cl client.Client, saName string, namespace string) error {
 			rbacv1.PolicyRule{
 				Verbs:     []string{"get", "list", "watch"},
 				APIGroups: []string{""},
-				Resources: []string{"pods"},
+				Resources: []string{"pods", "secrets", "configmaps"},
 			},
 			rbacv1.PolicyRule{
 				Verbs:     []string{"get", "list", "watch"},
@@ -281,7 +281,7 @@ func CreateBackupSessionListenerRBAC(cl client.Client, saName string, namespace
 			rbacv1.PolicyRule{
 				Verbs:     []string{"get", "list", "watch"},
 				APIGroups: []string{""},
-				Resources: []string{"pods"},
+				Resources: []string{"pods", "secrets", "configmaps"},
 			},
 			rbacv1.PolicyRule{
 				Verbs:     []string{"get", "list", "watch"},

test/00-setup.yaml

@@ -17,6 +17,14 @@ data:
   node.session.auth.password: VHJtK1lZaXZvMUNZSGszcGFGVWMrcTdCMmdJPQo=
 ---
 apiVersion: v1
+kind: Secret
+metadata:
+  namespace: demo
+  name: with-envfrom-secret
+data:
+  title: dmVyeXNlY3JldA==
+---
+apiVersion: v1
 kind: PersistentVolume
 metadata:
   name: demo-pv
@@ -82,6 +90,39 @@ spec:
   name: restore-pg
   image: desmo999r/formolcli:latest
   args: ["postgres", "restore", "--hostname", $(PGHOST), "--database", $(PGDATABASE), "--username", $(PGUSER), "--password", $(PGPASSWD), "--file", "/output/backup-pg.sql"]
+  env:
+    - name: PGHOST
+      value: postgres
+    - name: PGDATABASE
+      value: demopostgres
+    - name: PGUSER
+      value: demopostgres
+    - name: PGPASSWD
+      value: password123!
+---
+apiVersion: formol.desmojim.fr/v1alpha1
+kind: Function
+metadata:
+  name: with-envfrom
+  namespace: demo
+spec:
+  name: with-envfrom
+  command: ["touch", $(title)]
+  envFrom:
+    - secretRef:
+        name: with-envfrom-secret
+---
+apiVersion: formol.desmojim.fr/v1alpha1
+kind: Function
+metadata:
+  name: with-env
+  namespace: demo
+spec:
+  name: with-env
+  command: ["touch", $(TESTFILE)]
+  env:
+    - name: TESTFILE
+      value: /data/testfile
 ---
 apiVersion: formol.desmojim.fr/v1alpha1
 kind: Function
@@ -92,6 +133,15 @@ spec:
   name: backup-pg
   image: desmo999r/formolcli:latest
   args: ["postgres", "backup", "--hostname", $(PGHOST), "--database", $(PGDATABASE), "--username", $(PGUSER), "--password", $(PGPASSWD), "--file", "/output/backup-pg.sql"]
+  env:
+    - name: PGHOST
+      value: postgres
+    - name: PGDATABASE
+      value: demopostgres
+    - name: PGUSER
+      value: demopostgres
+    - name: PGPASSWD
+      value: password123!
 ---
 apiVersion: formol.desmojim.fr/v1alpha1
 kind: Function

test/02-backupconf.yaml

@@ -6,6 +6,7 @@ metadata:
   namespace: demo
 spec:
   suspend: true
+  image: desmo999r/formolcli:latest
   repository: repo-minio
   schedule: "15 * * * *"
   targets:
@@ -14,6 +15,8 @@ spec:
     name: nginx-deployment
     steps:
       - name: maintenance-on
+      - name: with-env
+      - name: with-envfrom
       - name: maintenance-off
         finalize: true
     volumeMounts:
@@ -21,19 +24,10 @@ spec:
         mountPath: /data
     paths:
       - /data
-  - kind: Job
+#  - kind: Job
-    name: backup-pg
+#    name: backup-pg
-    steps:
+#    steps:
-      - name: backup-pg
+#      - name: backup-pg
-        env:
-          - name: PGHOST
-            value: postgres
-          - name: PGDATABASE
-            value: demopostgres
-          - name: PGUSER
-            value: demopostgres
-          - name: PGPASSWD
-            value: password123!
   keep:
     last: 5
     daily: 2

test/README

@@ -1 +1,2 @@
 NAMESPACE=demo; for i in $(kubectl -n $NAMESPACE get bs | awk 'NR>1 { print $1 }'); do kubectl -n $NAMESPACE get bs -o json $i | sed '/finalizers/,+2d' | curl -vvv -X PUT -H 'Content-type: application/json' -d @- http://127.0.0.1:8001/apis/formol.desmojim.fr/v1alpha1/namespaces/$NAMESPACE/backupsessions/$i; kubectl -n $NAMESPACE delete bs $i; done
+NAMESPACE=demo; for i in $(kubectl -n $NAMESPACE get bs | awk 'NR>1 { if ($3 == "New") { print $1 }}'); do kubectl -n $NAMESPACE get bs -o json $i | sed '/finalizers/,+2d' | curl -vvv -X PUT -H 'Content-type: application/json' -d @- http://127.0.0.1:8001/apis/formol.desmojim.fr/v1alpha1/namespaces/$NAMESPACE/backupsessions/$i; kubectl -n $NAMESPACE delete bs $i; done