From 7f227c1ec4d73ec9e7239b94b048584348f046b9 Mon Sep 17 00:00:00 2001 From: Jean-Marc Andre Date: Mon, 20 Feb 2023 10:30:31 +0100 Subject: [PATCH] Should backup paths --- Dockerfile.amd64 | 2 +- Makefile | 2 +- controllers/backupsession_controller.go | 85 +++++---- .../backupsession_controller_helpers.go | 167 +++++++++++++++++- 4 files changed, 215 insertions(+), 41 deletions(-) diff --git a/Dockerfile.amd64 b/Dockerfile.amd64 index b603055..1df3e77 100644 --- a/Dockerfile.amd64 +++ b/Dockerfile.amd64 @@ -1,5 +1,5 @@ # Build a small image -FROM alpine:3.17 +FROM --platform=linux/amd64 alpine:3 RUN apk add --no-cache su-exec restic postgresql-client COPY ./bin/formolcli /usr/local/bin diff --git a/Makefile b/Makefile index cd9fa58..aeece64 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ vet: .PHONY: docker-build docker-build: formolcli - buildah bud --disable-compression --format=docker --platform $(GOOS)/$(GOARCH) --manifest $(IMG) Dockerfile.$(GOARCH) + buildah bud --tag $(IMG) Dockerfile.$(GOARCH) .PHONY: docker-push docker-push: docker-build diff --git a/controllers/backupsession_controller.go b/controllers/backupsession_controller.go index 38e407d..0bbd590 100644 --- a/controllers/backupsession_controller.go +++ b/controllers/backupsession_controller.go @@ -4,11 +4,14 @@ import ( "context" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "os" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" + "strings" + "time" formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1" ) @@ -23,7 +26,6 @@ type BackupSessionReconciler struct { func (r *BackupSessionReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { r.Log = log.FromContext(ctx) r.Context = ctx - r.Log.V(0).Info("Enter Reconcile with req", "req", req) backupSession := formolv1alpha1.BackupSession{} err := r.Get(ctx, req.NamespacedName, &backupSession) @@ -39,70 +41,79 @@ func (r *BackupSessionReconciler) Reconcile(ctx context.Context, req ctrl.Reques r.Log.V(0).Info("No task has been assigned yet. Wait for the next update...") return ctrl.Result{}, nil } - // backupConf := formolv1alpha1.BackupConfiguration{} - // err := r.Get(ctx, client.ObjectKey { - // Namespace: backupSession.Spec.Ref.Namespace, - // Name: backupSession.Spec.Ref.Name, - // }, &backupConf) - // if err != nil { - // if errors.IsNotFound(err) { - // return ctrl.Result{}, nil - // } - // return ctrl.Result{}, err - // } + backupConf := formolv1alpha1.BackupConfiguration{} + err = r.Get(ctx, client.ObjectKey{ + Namespace: backupSession.Spec.Ref.Namespace, + Name: backupSession.Spec.Ref.Name, + }, &backupConf) + if err != nil { + if errors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } targetName := os.Getenv(formolv1alpha1.TARGET_NAME) // we don't want a copy because we will modify and update it. currentTargetStatus := &(backupSession.Status.Targets[len(backupSession.Status.Targets)-1]) + currentTarget := backupConf.Spec.Targets[len(backupSession.Status.Targets)-1] + var result error if currentTargetStatus.TargetName == targetName { // The current task is for us + var newSessionState formolv1alpha1.SessionState switch currentTargetStatus.SessionState { case formolv1alpha1.New: r.Log.V(0).Info("New session, move to Initializing state") - currentTargetStatus.SessionState = formolv1alpha1.Init - err := r.Status().Update(ctx, &backupSession) - if err != nil { - r.Log.Error(err, "unable to update BackupSession status") - } - return ctrl.Result{}, err + newSessionState = formolv1alpha1.Init case formolv1alpha1.Init: r.Log.V(0).Info("Start to run the backup initializing steps is any") // Runs the Steps functions in chroot env - result := formolv1alpha1.Running - currentTargetStatus.SessionState = result - err := r.Status().Update(ctx, &backupSession) - if err != nil { - r.Log.Error(err, "unable to update BackupSession status") + if result = r.runInitializeBackupSteps(currentTarget); result != nil { + r.Log.Error(result, "unable to run the initialization steps") + newSessionState = formolv1alpha1.Finalize + } else { + newSessionState = formolv1alpha1.Running } - return ctrl.Result{}, err case formolv1alpha1.Running: r.Log.V(0).Info("Running state. Do the backup") // Actually do the backup with restic - currentTargetStatus.SessionState = formolv1alpha1.Finalize - err := r.Status().Update(ctx, &backupSession) - if err != nil { - r.Log.Error(err, "unable to update BackupSession status") + backupPaths := strings.Split(os.Getenv(formolv1alpha1.BACKUP_PATHS), string(os.PathListSeparator)) + if backupResult, result := r.backupPaths(backupSession.Name, backupPaths); result != nil { + r.Log.Error(result, "unable to backup paths", "target name", targetName, "paths", backupPaths) + } else { + r.Log.V(0).Info("Backup of the paths is over", "target name", targetName, "paths", backupPaths, + "snapshotID", backupResult.SnapshotId, "duration", backupResult.Duration) + currentTargetStatus.SnapshotId = backupResult.SnapshotId + currentTargetStatus.Duration = &metav1.Duration{Duration: time.Now().Sub(currentTargetStatus.StartTime.Time)} } - return ctrl.Result{}, err + newSessionState = formolv1alpha1.Finalize case formolv1alpha1.Finalize: r.Log.V(0).Info("Backup is over. Run the finalize steps is any") // Runs the finalize Steps functions in chroot env - if currentTargetStatus.SnapshotId == "" { - currentTargetStatus.SessionState = formolv1alpha1.Failure - } else { - currentTargetStatus.SessionState = formolv1alpha1.Success + if result = r.runFinalizeBackupSteps(currentTarget); result != nil { + r.Log.Error(err, "unable to run finalize steps") } + if currentTargetStatus.SnapshotId == "" { + newSessionState = formolv1alpha1.Failure + } else { + newSessionState = formolv1alpha1.Success + } + case formolv1alpha1.Success: + r.Log.V(0).Info("Backup is over") + case formolv1alpha1.Failure: + r.Log.V(0).Info("Backup is over") + } + if newSessionState != "" { + currentTargetStatus.SessionState = newSessionState err := r.Status().Update(ctx, &backupSession) if err != nil { r.Log.Error(err, "unable to update BackupSession status") } return ctrl.Result{}, err - case formolv1alpha1.Success: - case formolv1alpha1.Failure: - r.Log.V(0).Info("Backup is over") + } } - return ctrl.Result{}, nil + return ctrl.Result{}, result } // SetupWithManager sets up the controller with the Manager. diff --git a/controllers/backupsession_controller_helpers.go b/controllers/backupsession_controller_helpers.go index e35f505..5a8f61b 100644 --- a/controllers/backupsession_controller_helpers.go +++ b/controllers/backupsession_controller_helpers.go @@ -1,12 +1,39 @@ package controllers import ( + "bufio" + "bytes" + "encoding/json" formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1" + "io" + "io/ioutil" corev1 "k8s.io/api/core/v1" "os" + "os/exec" + "path/filepath" + "regexp" "sigs.k8s.io/controller-runtime/pkg/client" + "strconv" ) +var ( + REPOSITORY string + PASSWORD_FILE string + AWS_ACCESS_KEY_ID string + AWS_SECRET_ACCESS_KEY string +) + +const ( + RESTIC_EXEC = "/usr/bin/restic" +) + +func init() { + REPOSITORY = os.Getenv(formolv1alpha1.RESTIC_REPOSITORY) + PASSWORD_FILE = os.Getenv(formolv1alpha1.RESTIC_PASSWORD) + AWS_ACCESS_KEY_ID = os.Getenv(formolv1alpha1.AWS_ACCESS_KEY_ID) + AWS_SECRET_ACCESS_KEY = os.Getenv(formolv1alpha1.AWS_SECRET_ACCESS_KEY) +} + func (r *BackupSessionReconciler) getSecretData(name string) map[string][]byte { secret := corev1.Secret{} namespace := os.Getenv(formolv1alpha1.POD_NAMESPACE) @@ -88,12 +115,14 @@ func (r *BackupSessionReconciler) getFuncVars(function formolv1alpha1.Function, r.getFuncEnv(vars, function.Spec.Env) } -func (r *BackupSessionReconciler) runInitBackupSteps(target formolv1alpha1.Target) error { +func (r *BackupSessionReconciler) runBackupSteps(initializeSteps bool, target formolv1alpha1.Target) error { namespace := os.Getenv(formolv1alpha1.POD_NAMESPACE) r.Log.V(0).Info("start to run the backup initializing steps it any") + // For every container listed in the target, run the initialization steps for _, container := range target.Containers { + // Runs the steps one after the other for _, step := range container.Steps { - if step.Finalize != nil && *step.Finalize == true { + if step.Finalize != nil && *step.Finalize == true && initializeSteps { continue } function := formolv1alpha1.Function{} @@ -107,7 +136,141 @@ func (r *BackupSessionReconciler) runInitBackupSteps(target formolv1alpha1.Targe vars := make(map[string]string) r.getFuncVars(function, vars) + // Loop through the function.Spec.Command arguments to replace ${ARG}|$(ARG)|$ARG + // with the environment variable value + pattern := regexp.MustCompile(`^\$(\{(?P\w+)\}|^\$\((?P\w+)\)|(?P\w+))$`) + for i, arg := range function.Spec.Command[1:] { + if pattern.MatchString(arg) { + arg = pattern.ReplaceAllString(arg, "$env") + function.Spec.Command[i] = vars[arg] + } + } + if err := r.runTargetContainerChroot(function.Spec.Command[0], + function.Spec.Command[1:]...); err != nil { + r.Log.Error(err, "unable to run command", "command", function.Spec.Command) + return err + } } } return nil } + +func (r *BackupSessionReconciler) runFinalizeBackupSteps(target formolv1alpha1.Target) error { + return r.runBackupSteps(true, target) +} + +func (r *BackupSessionReconciler) runInitializeBackupSteps(target formolv1alpha1.Target) error { + return r.runBackupSteps(true, target) +} + +func (r *BackupSessionReconciler) runTargetContainerChroot(runCmd string, args ...string) error { + env := regexp.MustCompile(`/proc/[0-9]+/env`) + if err := filepath.Walk("/proc", func(path string, info os.FileInfo, err error) error { + if err != nil { + return nil + } + // Skip process 1 and ourself + if info.IsDir() && (info.Name() == "1" || info.Name() == strconv.Itoa(os.Getpid())) { + return filepath.SkipDir + } + // Found an environ file. Start looking for TARGETCONTAINER_TAG + if env.MatchString(path) { + r.Log.V(0).Info("Looking for tag", "file", path, "TARGETCONTAINER_TAG", formolv1alpha1.TARGETCONTAINER_TAG) + content, err := ioutil.ReadFile(path) + // cannot read environ file. not the process we want to backup + if err != nil { + r.Log.Error(err, "unable to read file", "file", path) + return filepath.SkipDir + } + // Loops over the process environement variable looking for TARGETCONTAINER_TAG + for _, env := range bytes.Split(content, []byte{'\000'}) { + matched, err := regexp.Match(formolv1alpha1.TARGETCONTAINER_TAG, env) + if err != nil { + r.Log.Error(err, "unable to regexp", "env", string(env)) + return err + } + if matched { + // Found the right process. Now run the command in its 'root' + r.Log.V(0).Info("Found the tag", "file", path) + root := filepath.Join(filepath.Dir(path), "root") + if _, err := filepath.EvalSymlinks(root); err != nil { + r.Log.Error(err, "cannot EvalSymlink.") + return err + } + r.Log.V(0).Info("running cmd in chroot", "path", root) + cmd := exec.Command("chroot", append([]string{root, runCmd}, args...)...) + stdout, _ := cmd.StdoutPipe() + stderr, _ := cmd.StderrPipe() + _ = cmd.Start() + + scanner := bufio.NewScanner(io.MultiReader(stdout, stderr)) + scanner.Split(bufio.ScanLines) + for scanner.Scan() { + r.Log.V(0).Info("cmd output", "output", scanner.Text()) + } + + return cmd.Wait() + } + } + } + return nil + }); err != nil { + r.Log.Error(err, "cannot walk /proc") + return err + } + return nil +} + +func (r *BackupSessionReconciler) checkRepo(repo string) error { + r.Log.V(0).Info("Checking repo", "repo", repo) + if err := exec.Command(RESTIC_EXEC, "unlock", "-r", repo).Run(); err != nil { + r.Log.Error(err, "unable to unlock repo", "repo", repo) + return err + } + output, err := exec.Command(RESTIC_EXEC, "check", "-r", repo).CombinedOutput() + if err != nil { + r.Log.V(0).Info("Initializing new repo", "repo", repo) + output, err = exec.Command(RESTIC_EXEC, "init", "-r", repo).CombinedOutput() + if err != nil { + r.Log.Error(err, "something went wrong during repo init", "output", output) + } + } + return err +} + +type BackupResult struct { + SnapshotId string + Duration float64 +} + +func (r *BackupSessionReconciler) backupPaths(tag string, paths []string) (result BackupResult, err error) { + if err = r.checkRepo(REPOSITORY); err != nil { + r.Log.Error(err, "unable to setup repo", "repo", REPOSITORY) + return + } + r.Log.V(0).Info("backing up paths", "paths", paths) + cmd := exec.Command(RESTIC_EXEC, append([]string{"backup", "--json", "--tag", tag, "-r", REPOSITORY}, paths...)...) + stdout, _ := cmd.StdoutPipe() + stderr, _ := cmd.StderrPipe() + _ = cmd.Start() + + scanner := bufio.NewScanner(io.MultiReader(stdout, stderr)) + scanner.Split(bufio.ScanLines) + var data map[string]interface{} + for scanner.Scan() { + if err := json.Unmarshal(scanner.Bytes(), &data); err != nil { + r.Log.Error(err, "unable to unmarshal json", "data", scanner.Text()) + continue + } + switch data["message_type"].(string) { + case "summary": + result.SnapshotId = data["snapshot_id"].(string) + result.Duration = data["total_duration"].(float64) + case "status": + r.Log.V(0).Info("backup running", "percent done", data["percent_done"].(float64)) + } + } + + err = cmd.Wait() + return +}