jandre/formolcli
1
0
Fork 0
Code Issues 4 Pull Requests Projects Releases 1 Wiki Activity

Preparing the common code between BackupSession and RestoreSession

Browse Source
This commit is contained in:
jandre 2023-03-17 16:42:49 +01:00
parent 319e226a30
commit d4231768d7
4 changed files with 240 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
controllers/backupsession_controller.go
View File

@ -2,10 +2,8 @@ package controllers
import (
"context"
"github.com/go-logr/logr"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"os"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
@ -17,11 +15,7 @@ import (
)
type BackupSessionReconciler struct {
client.Client
Log logr.Logger
Scheme *runtime.Scheme
context.Context
Namespace string
Session
}
func (r *BackupSessionReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
@ -85,7 +79,7 @@ func (r *BackupSessionReconciler) Reconcile(ctx context.Context, req ctrl.Reques
// Run the initializing Steps and then move to Initialized or Failure
r.Log.V(0).Info("Start to run the backup initializing steps is any")
// Runs the Steps functions in chroot env
if err := r.runInitializeBackupSteps(target); err != nil {
if err := r.runInitializeSteps(target); err != nil {
r.Log.Error(err, "unable to run the initialization steps")
newSessionState = formolv1alpha1.Failure
} else {
@ -124,7 +118,7 @@ func (r *BackupSessionReconciler) Reconcile(ctx context.Context, req ctrl.Reques
// Run the finalize Steps and move to Success or Failure
r.Log.V(0).Info("Backup is over. Run the finalize steps is any")
// Runs the finalize Steps functions in chroot env
if result = r.runFinalizeBackupSteps(target); result != nil {
if result = r.runFinalizeSteps(target); result != nil {
r.Log.Error(err, "unable to run finalize steps")
}
if targetStatus.SnapshotId == "" {

212
controllers/backupsession_controller_helpers.go
View File

@ -2,20 +2,13 @@ package controllers
import (
"bufio"
"bytes"
"encoding/json"
"fmt"
formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1"
"io"
"io/fs"
"io/ioutil"
corev1 "k8s.io/api/core/v1"
"os"
"os/exec"
"path/filepath"
"regexp"
"sigs.k8s.io/controller-runtime/pkg/client"
"strconv"
"strings"
)
@ -23,89 +16,6 @@ const (
RESTIC_EXEC = "/usr/bin/restic"
)
func (r *BackupSessionReconciler) getSecretData(name string) map[string][]byte {
secret := corev1.Secret{}
namespace := os.Getenv(formolv1alpha1.POD_NAMESPACE)
if err := r.Get(r.Context, client.ObjectKey{
Namespace: namespace,
Name: name,
}, &secret); err != nil {
r.Log.Error(err, "unable to get Secret", "Secret", name)
return nil
}
return secret.Data
}
func (r *BackupSessionReconciler) getEnvFromSecretKeyRef(name string, key string) string {
if data := r.getSecretData(name); data != nil {
return string(data[key])
}
return ""
}
func (r *BackupSessionReconciler) getConfigMapData(name string) map[string]string {
configMap := corev1.ConfigMap{}
namespace := os.Getenv(formolv1alpha1.POD_NAMESPACE)
if err := r.Get(r.Context, client.ObjectKey{
Namespace: namespace,
Name: name,
}, &configMap); err != nil {
r.Log.Error(err, "unable to get ConfigMap", "configmap", name)
return nil
}
return configMap.Data
}
func (r *BackupSessionReconciler) getEnvFromConfigMapKeyRef(name string, key string) string {
if data := r.getConfigMapData(name); data != nil {
return string(data[key])
}
return ""
}
func (r *BackupSessionReconciler) getFuncEnv(vars map[string]string, envVars []corev1.EnvVar) {
for _, env := range envVars {
if env.ValueFrom != nil {
if env.ValueFrom.ConfigMapKeyRef != nil {
vars[env.Name] = r.getEnvFromConfigMapKeyRef(env.ValueFrom.ConfigMapKeyRef.LocalObjectReference.Name, env.ValueFrom.ConfigMapKeyRef.Key)
}
if env.ValueFrom.SecretKeyRef != nil {
vars[env.Name] = r.getEnvFromSecretKeyRef(env.ValueFrom.SecretKeyRef.LocalObjectReference.Name, env.ValueFrom.SecretKeyRef.Key)
}
} else {
vars[env.Name] = env.Value
}
}
}
func (r *BackupSessionReconciler) getEnvFromSecretEnvSource(vars map[string]string, name string) {
for key, value := range r.getSecretData(name) {
vars[key] = string(value)
}
}
func (r *BackupSessionReconciler) getEnvFromConfigMapEnvSource(vars map[string]string, name string) {
for key, value := range r.getConfigMapData(name) {
vars[key] = value
}
}
func (r *BackupSessionReconciler) getFuncEnvFrom(vars map[string]string, envVars []corev1.EnvFromSource) {
for _, env := range envVars {
if env.ConfigMapRef != nil {
r.getEnvFromConfigMapEnvSource(vars, env.ConfigMapRef.LocalObjectReference.Name)
}
if env.SecretRef != nil {
r.getEnvFromSecretEnvSource(vars, env.SecretRef.LocalObjectReference.Name)
}
}
}
func (r *BackupSessionReconciler) getFuncVars(function formolv1alpha1.Function, vars map[string]string) {
r.getFuncEnvFrom(vars, function.Spec.EnvFrom)
r.getFuncEnv(vars, function.Spec.Env)
}
func (r *BackupSessionReconciler) setResticEnv(backupConf formolv1alpha1.BackupConfiguration) error {
repo := formolv1alpha1.Repo{}
if err := r.Get(r.Context, client.ObjectKey{
@ -129,128 +39,6 @@ func (r *BackupSessionReconciler) setResticEnv(backupConf formolv1alpha1.BackupC
return nil
}
func (r *BackupSessionReconciler) runFunction(name string) error {
namespace := os.Getenv(formolv1alpha1.POD_NAMESPACE)
function := formolv1alpha1.Function{}
if err := r.Get(r.Context, client.ObjectKey{
Namespace: namespace,
Name: name,
}, &function); err != nil {
r.Log.Error(err, "unable to get Function", "Function", name)
return err
}
vars := make(map[string]string)
r.getFuncVars(function, vars)
r.Log.V(0).Info("function vars", "vars", vars)
// Loop through the function.Spec.Command arguments to replace ${ARG}|$(ARG)|$ARG
// with the environment variable value
pattern := regexp.MustCompile(`^\$\((?P<env>\w+)\)$`)
for i, arg := range function.Spec.Args {
if pattern.MatchString(arg) {
r.Log.V(0).Info("arg matches $()", "arg", arg)
arg = pattern.ReplaceAllString(arg, "$env")
function.Spec.Args[i] = vars[arg]
}
}
r.Log.V(1).Info("about to run Function", "Function", name, "command", function.Spec.Command, "args", function.Spec.Args)
if err := r.runTargetContainerChroot(function.Spec.Command[0],
function.Spec.Args...); err != nil {
r.Log.Error(err, "unable to run command", "command", function.Spec.Command)
return err
}
return nil
}
func (r *BackupSessionReconciler) runBackupSteps(initializeSteps bool, target formolv1alpha1.Target) error {
r.Log.V(0).Info("start to run the backup steps it any")
// For every container listed in the target, run the initialization steps
for _, container := range target.Containers {
// Runs the steps one after the other
for _, step := range container.Steps {
if (initializeSteps == true && step.Finalize != nil && *step.Finalize == true) || (initializeSteps == false && (step.Finalize == nil || step.Finalize != nil && *step.Finalize == false)) {
continue
}
return r.runFunction(step.Name)
}
}
return nil
}
// Run the initializing steps in the INITIALIZING state of the controller
// before actualy doing the backup in the RUNNING state
func (r *BackupSessionReconciler) runFinalizeBackupSteps(target formolv1alpha1.Target) error {
return r.runBackupSteps(false, target)
}
// Run the finalizing steps in the FINALIZE state of the controller
// after the backup in the RUNNING state.
// The finalize happens whatever the result of the backup.
func (r *BackupSessionReconciler) runInitializeBackupSteps(target formolv1alpha1.Target) error {
return r.runBackupSteps(true, target)
}
// Runs the given command in the target container chroot
func (r *BackupSessionReconciler) runTargetContainerChroot(runCmd string, args ...string) error {
env := regexp.MustCompile(`/proc/[0-9]+/environ`)
if err := filepath.WalkDir("/proc", func(path string, info fs.DirEntry, err error) error {
if err != nil {
return nil
}
// Skip process 1 and ourself
if info.IsDir() && (info.Name() == "1" || info.Name() == strconv.Itoa(os.Getpid())) {
return filepath.SkipDir
}
// Found an environ file. Start looking for TARGETCONTAINER_TAG
if env.MatchString(path) {
content, err := ioutil.ReadFile(path)
// cannot read environ file. not the process we want to backup
if err != nil {
return fs.SkipDir
}
// Loops over the process environement variable looking for TARGETCONTAINER_TAG
for _, env := range bytes.Split(content, []byte{'\000'}) {
matched, err := regexp.Match(formolv1alpha1.TARGETCONTAINER_TAG, env)
if err != nil {
r.Log.Error(err, "unable to regexp", "env", string(env))
return err
}
if matched {
// Found the right process. Now run the command in its 'root'
r.Log.V(0).Info("Found the tag", "file", path)
root := filepath.Join(filepath.Dir(path), "root")
if _, err := filepath.EvalSymlinks(root); err != nil {
r.Log.Error(err, "cannot EvalSymlink.")
return err
}
r.Log.V(0).Info("running cmd in chroot", "path", root)
cmd := exec.Command("chroot", append([]string{root, runCmd}, args...)...)
stdout, _ := cmd.StdoutPipe()
stderr, _ := cmd.StderrPipe()
_ = cmd.Start()
scanner := bufio.NewScanner(io.MultiReader(stdout, stderr))
scanner.Split(bufio.ScanLines)
for scanner.Scan() {
r.Log.V(0).Info("cmd output", "output", scanner.Text())
}
if err := cmd.Wait(); err != nil {
return err
} else {
return filepath.SkipAll
}
}
}
}
return nil
}); err != nil {
r.Log.Error(err, "cannot walk /proc")
return err
}
return nil
}
func (r *BackupSessionReconciler) checkRepo() error {
r.Log.V(0).Info("Checking repo")
if err := exec.Command(RESTIC_EXEC, "unlock").Run(); err != nil {

6
controllers/server.go
View File

@ -38,8 +38,10 @@ func StartServer() {
os.Exit(1)
}
if err = (&BackupSessionReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Session: Session{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
},
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "BackupSession")
os.Exit(1)

233
controllers/session.go Normal file
View File

@ -0,0 +1,233 @@
package controllers
import (
"bufio"
"bytes"
"context"
formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1"
"github.com/go-logr/logr"
"io"
"io/fs"
"io/ioutil"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"os"
"os/exec"
"path/filepath"
"regexp"
"sigs.k8s.io/controller-runtime/pkg/client"
"strconv"
)
type Session struct {
client.Client
Log logr.Logger
Scheme *runtime.Scheme
context.Context
Namespace string
}
func (s Session) getSecretData(name string) map[string][]byte {
secret := corev1.Secret{}
namespace := os.Getenv(formolv1alpha1.POD_NAMESPACE)
if err := s.Get(s.Context, client.ObjectKey{
Namespace: namespace,
Name: name,
}, &secret); err != nil {
s.Log.Error(err, "unable to get Secret", "Secret", name)
return nil
}
return secret.Data
}
func (s Session) getEnvFromSecretKeyRef(name string, key string) string {
if data := s.getSecretData(name); data != nil {
return string(data[key])
}
return ""
}
func (s Session) getConfigMapData(name string) map[string]string {
configMap := corev1.ConfigMap{}
namespace := os.Getenv(formolv1alpha1.POD_NAMESPACE)
if err := s.Get(s.Context, client.ObjectKey{
Namespace: namespace,
Name: name,
}, &configMap); err != nil {
s.Log.Error(err, "unable to get ConfigMap", "configmap", name)
return nil
}
return configMap.Data
}
func (s Session) getEnvFromConfigMapKeyRef(name string, key string) string {
if data := s.getConfigMapData(name); data != nil {
return string(data[key])
}
return ""
}
func (s Session) getFuncEnv(vars map[string]string, envVars []corev1.EnvVar) {
for _, env := range envVars {
if env.ValueFrom != nil {
if env.ValueFrom.ConfigMapKeyRef != nil {
vars[env.Name] = s.getEnvFromConfigMapKeyRef(env.ValueFrom.ConfigMapKeyRef.LocalObjectReference.Name, env.ValueFrom.ConfigMapKeyRef.Key)
}
if env.ValueFrom.SecretKeyRef != nil {
vars[env.Name] = s.getEnvFromSecretKeyRef(env.ValueFrom.SecretKeyRef.LocalObjectReference.Name, env.ValueFrom.SecretKeyRef.Key)
}
} else {
vars[env.Name] = env.Value
}
}
}
func (s Session) getEnvFromSecretEnvSource(vars map[string]string, name string) {
for key, value := range s.getSecretData(name) {
vars[key] = string(value)
}
}
func (s Session) getEnvFromConfigMapEnvSource(vars map[string]string, name string) {
for key, value := range s.getConfigMapData(name) {
vars[key] = value
}
}
func (s Session) getFuncEnvFrom(vars map[string]string, envVars []corev1.EnvFromSource) {
for _, env := range envVars {
if env.ConfigMapRef != nil {
s.getEnvFromConfigMapEnvSource(vars, env.ConfigMapRef.LocalObjectReference.Name)
}
if env.SecretRef != nil {
s.getEnvFromSecretEnvSource(vars, env.SecretRef.LocalObjectReference.Name)
}
}
}
func (s Session) getFuncVars(function formolv1alpha1.Function, vars map[string]string) {
s.getFuncEnvFrom(vars, function.Spec.EnvFrom)
s.getFuncEnv(vars, function.Spec.Env)
}
func (s Session) runFunction(name string) error {
namespace := os.Getenv(formolv1alpha1.POD_NAMESPACE)
function := formolv1alpha1.Function{}
if err := s.Get(s.Context, client.ObjectKey{
Namespace: namespace,
Name: name,
}, &function); err != nil {
s.Log.Error(err, "unable to get Function", "Function", name)
return err
}
vars := make(map[string]string)
s.getFuncVars(function, vars)
s.Log.V(0).Info("function vars", "vars", vars)
// Loop through the function.Spec.Command arguments to replace ${ARG}|$(ARG)|$ARG
// with the environment variable value
pattern := regexp.MustCompile(`^\$\((?P<env>\w+)\)$`)
for i, arg := range function.Spec.Args {
if pattern.MatchString(arg) {
s.Log.V(0).Info("arg matches $()", "arg", arg)
arg = pattern.ReplaceAllString(arg, "$env")
function.Spec.Args[i] = vars[arg]
}
}
s.Log.V(1).Info("about to run Function", "Function", name, "command", function.Spec.Command, "args", function.Spec.Args)
if err := s.runTargetContainerChroot(function.Spec.Command[0],
function.Spec.Args...); err != nil {
s.Log.Error(err, "unable to run command", "command", function.Spec.Command)
return err
}
return nil
}
// Runs the given command in the target container chroot
func (s Session) runTargetContainerChroot(runCmd string, args ...string) error {
env := regexp.MustCompile(`/proc/[0-9]+/environ`)
if err := filepath.WalkDir("/proc", func(path string, info fs.DirEntry, err error) error {
if err != nil {
return nil
}
// Skip process 1 and ourself
if info.IsDir() && (info.Name() == "1" || info.Name() == strconv.Itoa(os.Getpid())) {
return filepath.SkipDir
}
// Found an environ file. Start looking for TARGETCONTAINER_TAG
if env.MatchString(path) {
content, err := ioutil.ReadFile(path)
// cannot read environ file. not the process we want to backup
if err != nil {
return fs.SkipDir
}
// Loops over the process environement variable looking for TARGETCONTAINER_TAG
for _, env := range bytes.Split(content, []byte{'\000'}) {
matched, err := regexp.Match(formolv1alpha1.TARGETCONTAINER_TAG, env)
if err != nil {
s.Log.Error(err, "unable to regexp", "env", string(env))
return err
}
if matched {
// Found the right process. Now run the command in its 'root'
s.Log.V(0).Info("Found the tag", "file", path)
root := filepath.Join(filepath.Dir(path), "root")
if _, err := filepath.EvalSymlinks(root); err != nil {
s.Log.Error(err, "cannot EvalSymlink.")
return err
}
s.Log.V(0).Info("running cmd in chroot", "path", root)
cmd := exec.Command("chroot", append([]string{root, runCmd}, args...)...)
stdout, _ := cmd.StdoutPipe()
stderr, _ := cmd.StderrPipe()
_ = cmd.Start()
scanner := bufio.NewScanner(io.MultiReader(stdout, stderr))
scanner.Split(bufio.ScanLines)
for scanner.Scan() {
s.Log.V(0).Info("cmd output", "output", scanner.Text())
}
if err := cmd.Wait(); err != nil {
return err
} else {
return filepath.SkipAll
}
}
}
}
return nil
}); err != nil {
s.Log.Error(err, "cannot walk /proc")
return err
}
return nil
}
func (s Session) runSteps(initializeSteps bool, target formolv1alpha1.Target) error {
s.Log.V(0).Info("start to run the backup steps it any")
// For every container listed in the target, run the initialization steps
for _, container := range target.Containers {
// Runs the steps one after the other
for _, step := range container.Steps {
if (initializeSteps == true && step.Finalize != nil && *step.Finalize == true) || (initializeSteps == false && (step.Finalize == nil || step.Finalize != nil && *step.Finalize == false)) {
continue
}
return s.runFunction(step.Name)
}
}
return nil
}
// Run the initializing steps in the INITIALIZING state of the controller
// before actualy doing the backup in the RUNNING state
func (s Session) runFinalizeSteps(target formolv1alpha1.Target) error {
return s.runSteps(false, target)
}
// Run the finalizing steps in the FINALIZE state of the controller
// after the backup in the RUNNING state.
// The finalize happens whatever the result of the backup.
func (s Session) runInitializeSteps(target formolv1alpha1.Target) error {
return s.runSteps(true, target)
}