From 16640f34eb9a0eec8b8c743923e5bb73439eaf32 Mon Sep 17 00:00:00 2001 From: Jean-Marc Andre Date: Sun, 3 Jan 2021 00:58:13 +0100 Subject: [PATCH] backupsession status --- .gitignore | 4 +- Dockerfile | 9 + Dockerfile.deployment | 2 +- Makefile | 26 +- go.mod | 2 +- main.go | 2 +- pkg/backup/root.go | 129 +++++----- pkg/backupsession/root.go | 57 ++--- pkg/cmd/backupsession.go | 23 +- pkg/cmd/postgres.go | 68 ++++++ pkg/cmd/server.go | 2 +- pkg/cmd/snapshot.go | 56 +++++ pkg/cmd/volume.go | 15 +- pkg/controllers/backupsession_controller.go | 256 ++++++-------------- pkg/server/root.go | 22 +- 15 files changed, 360 insertions(+), 313 deletions(-) create mode 100644 Dockerfile create mode 100644 pkg/cmd/postgres.go create mode 100644 pkg/cmd/snapshot.go diff --git a/.gitignore b/.gitignore index 4024132..c0e88cc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ -src/go.sum -src/formolcli +go.sum +bin diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..68500fb --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +# Build a small image +FROM arm32v7/alpine:3.12 + +RUN apk add --no-cache restic postgresql-client +COPY bin/formolcli /usr/local/bin + +# Command to run +ENTRYPOINT ["/usr/local/bin/formolcli"] +CMD ["--help"] diff --git a/Dockerfile.deployment b/Dockerfile.deployment index c471e4a..3e7b6e8 100644 --- a/Dockerfile.deployment +++ b/Dockerfile.deployment @@ -30,7 +30,7 @@ RUN cp /build/formolcli . # Build a small image FROM arm32v7/alpine:3.12 -RUN apk add --no-cache restic +RUN apk add --no-cache restic postgresql-client #COPY bin/restic /usr/local/bin COPY --from=builder /dist/formolcli /usr/local/bin diff --git a/Makefile b/Makefile index f87e4f3..e0f0c5b 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,22 @@ -IMG-deployment ?= desmo999r/formolcli:latest -docker-build-deployment: - podman build --disable-compression --format=docker --file Dockerfile.deployment -t ${IMG-deployment} +.PHONY: all formolcli docker docker-build docker-push -docker-push-deployment: - podman push ${IMG-deployment} +IMG ?= desmo999r/formolcli:latest -deployment: docker-build-deployment docker-push-deployment +formolcli: fmt vet + GO111MODULE=on CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -o bin/formolcli main.go -all: deployment +fmt: + go fmt ./... + +vet: + go vet ./... + +docker-build: + podman build --disable-compression --format=docker -t ${IMG} . + +docker-push: + podman push ${IMG} + +docker: formolcli docker-build docker-push + +all: docker diff --git a/go.mod b/go.mod index 22b7a08..74605c4 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/desmo999r/formolcli go 1.14 require ( - github.com/desmo999r/formol v0.6.2-0.20201228110830-3aaf127e5835 + github.com/desmo999r/formol v0.6.2-0.20210102135435-1be5004eadf3 github.com/go-logr/logr v0.1.0 github.com/go-logr/zapr v0.1.0 github.com/mitchellh/go-homedir v1.1.0 diff --git a/main.go b/main.go index ea2eace..81f4c1a 100644 --- a/main.go +++ b/main.go @@ -15,7 +15,7 @@ limitations under the License. */ package main -import "github.com/desmo999r/formolcli/cmd" +import "github.com/desmo999r/formolcli/pkg/cmd" func main() { cmd.Execute() diff --git a/pkg/backup/root.go b/pkg/backup/root.go index 0668f5c..888122d 100644 --- a/pkg/backup/root.go +++ b/pkg/backup/root.go @@ -1,22 +1,29 @@ package backup import ( - "strings" "bufio" - "os" - "os/exec" - "go.uber.org/zap" + "bytes" + "encoding/json" + "fmt" + formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1" + "github.com/desmo999r/formolcli/pkg/backupsession" "github.com/go-logr/logr" "github.com/go-logr/zapr" + "go.uber.org/zap" + "io/ioutil" + "os" + "os/exec" + "time" ) var ( - repository string - passwordFile string - aws_access_key_id string + repository string + passwordFile string + aws_access_key_id string aws_secret_access_key string - resticExec = "/usr/bin/restic" - logger logr.Logger + resticExec = "/usr/bin/restic" + pg_dumpExec = "/usr/bin/pg_dump" + logger logr.Logger ) func init() { @@ -47,7 +54,7 @@ func checkRepo(repo string) error { log.Error(err, "cannot start repo init") return err } - go func(){ + go func() { scanner := bufio.NewScanner(stderr) for scanner.Scan() { log.V(0).Info("and error happened", "stderr", scanner.Text()) @@ -61,63 +68,77 @@ func checkRepo(repo string) error { return err } -func BackupVolume(path string) error { - return nil +func GetBackupResults(output []byte) (snapshotId string, duration time.Duration) { + log := logger.WithName("backup-getbackupresults") + scanner := bufio.NewScanner(bytes.NewReader(output)) + var dat map[string]interface{} + for scanner.Scan() { + if err := json.Unmarshal(scanner.Bytes(), &dat); err != nil { + log.Error(err, "unable to unmarshal json", "msg", string(scanner.Bytes())) + continue + } + log.V(1).Info("message on stdout", "stdout", dat) + if message_type, ok := dat["message_type"]; ok && message_type == "summary" { + snapshotId = dat["snapshot_id"].(string) + duration = time.Duration(dat["total_duration"].(float64)*1000) * time.Millisecond + } + } + return } -func BackupDeployment(prefix string, paths []string, c chan []byte) (error) { - log := logger.WithName("backup-deployment") - newrepo := repository - if prefix != "" { - newrepo = repository + "/" + prefix - } - if err := checkRepo(newrepo); err != nil { - log.Error(err, "unable to setup newrepo", "newrepo", newrepo) - return err - } - cmd := exec.Command(resticExec, "backup", "--json", "-r", newrepo, strings.Join(paths, " ")) - stderr, err := cmd.StderrPipe() +func BackupVolume(tag string, paths []string) error { + log := logger.WithName("backup-volume") + state := formolv1alpha1.Success + output, err := BackupPaths(tag, paths) + var snapshotId string + var duration time.Duration if err != nil { - log.Error(err, "unable to pipe stderr") + log.Error(err, "unable to backup volume", "output", string(output)) + state = formolv1alpha1.Failure + } else { + snapshotId, duration = GetBackupResults(output) + } + backupsession.BackupSessionUpdateStatus(state, snapshotId, duration) + return err +} + +func BackupPostgres(file string, hostname string, database string, username string, password string) error { + log := logger.WithName("backup-postgres") + pgpass := []byte(fmt.Sprintf("%s:*:%s:%s:%s", hostname, database, username, password)) + if err := ioutil.WriteFile("/output/.pgpass", pgpass, 0600); err != nil { + log.Error(err, "unable to write password to /output/.pgpass") return err } - stdout, err := cmd.StdoutPipe() + defer os.Remove("/output/.pgpass") + cmd := exec.Command(pg_dumpExec, "--clean", "--create", "--file", file, "--host", hostname, "--dbname", database, "--username", username, "--no-password") + cmd.Env = append(os.Environ(), "PGPASSFILE=/output/.pgpass") + output, err := cmd.CombinedOutput() + log.V(1).Info("postgres backup output", "output", string(output)) if err != nil { - log.Error(err, "unable to pipe stdout") - return err - } - if err := cmd.Start(); err != nil { - log.Error(err, "cannot start backup") - return err - } - go func(){ - scanner := bufio.NewScanner(stderr) - for scanner.Scan() { - log.V(0).Info("and error happened", "stderr", scanner.Text()) - } - }() - go func(c chan []byte){ - scanner := bufio.NewScanner(stdout) - for scanner.Scan() { - c <- scanner.Bytes() - } - }(c) - if err := cmd.Wait(); err != nil { log.Error(err, "something went wrong during the backup") return err } - return nil } -func DeleteSnapshot(prefix string, snapshotId string) error { - log := logger.WithValues("delete-snapshot", snapshotId) - newrepo := repository - if prefix != "" { - newrepo = repository + "/" + prefix +func BackupPaths(tag string, paths []string) ([]byte, error) { + log := logger.WithName("backup-deployment") + if err := checkRepo(repository); err != nil { + log.Error(err, "unable to setup newrepo", "newrepo", repository) + return []byte{}, err } - cmd := exec.Command(resticExec, "forget", "-r", newrepo, snapshotId) - if err := cmd.Run(); err != nil { + cmd := exec.Command(resticExec, append([]string{"backup", "--json", "--tag", tag, "-r", repository}, paths...)...) + output, err := cmd.CombinedOutput() + return output, err +} + +func DeleteSnapshot(snapshot string) error { + log := logger.WithValues("delete-snapshot", snapshot) + cmd := exec.Command(resticExec, "forget", "-r", repository, "--prune", snapshot) + log.V(0).Info("deleting snapshot", "snapshot", snapshot) + output, err := cmd.CombinedOutput() + log.V(1).Info("delete snapshot output", "output", string(output)) + if err != nil { log.Error(err, "unable to delete the snapshot") return err } diff --git a/pkg/backupsession/root.go b/pkg/backupsession/root.go index 51b953c..b5b29d8 100644 --- a/pkg/backupsession/root.go +++ b/pkg/backupsession/root.go @@ -1,29 +1,30 @@ package backupsession import ( - "strings" - "time" "context" - "os" - "strconv" - "path/filepath" + formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1" + "github.com/go-logr/logr" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + "os" + "path/filepath" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log/zap" - ctrl "sigs.k8s.io/controller-runtime" - formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1" - "github.com/go-logr/logr" + "strconv" + "strings" + "time" ) var ( config *rest.Config scheme *runtime.Scheme - cl client.Client + cl client.Client logger logr.Logger + //backupSession *formolv1alpha1.BackupSession ) func init() { @@ -32,7 +33,7 @@ func init() { ctrl.SetLogger(logger) config, err := rest.InClusterConfig() if err != nil { - config, err = clientcmd.BuildConfigFromFlags("", filepath.Join(os.Getenv("HOME"), ".kube", "config",)) + config, err = clientcmd.BuildConfigFromFlags("", filepath.Join(os.Getenv("HOME"), ".kube", "config")) if err != nil { log.Error(err, "unable to get config") os.Exit(1) @@ -48,19 +49,24 @@ func init() { } } -func DeleteBackupSession(name string, namespace string) error { - log := logger.WithName("CreateBackupSession") - log.V(0).Info("CreateBackupSession called") +func BackupSessionUpdateStatus(state formolv1alpha1.BackupState, snapshotId string, duration time.Duration) error { + log := logger.WithName("BackupSessionUpdateStatus") + targetName := os.Getenv("TARGET_NAME") backupSession := &formolv1alpha1.BackupSession{} - if err := cl.Get(context.TODO(), client.ObjectKey{ - Namespace: namespace, - Name: name, - }, backupSession); err != nil { - log.Error(err, "unable to get backupsession", "backupsession", name) - return err + cl.Get(context.Background(), client.ObjectKey{ + Namespace: os.Getenv("BACKUPSESSION_NAMESPACE"), + Name: os.Getenv("BACKUPSESSION_NAME"), + }, backupSession) + for i, target := range backupSession.Status.Targets { + if target.Name == targetName { + backupSession.Status.Targets[i].BackupState = state + backupSession.Status.Targets[i].SnapshotId = snapshotId + backupSession.Status.Targets[i].Duration = &metav1.Duration{Duration: duration} + } } - if err := cl.Delete(context.TODO(), backupSession); err != nil { - log.Error(err, "unable to delete backupsession", "backupsession", name) + + if err := cl.Status().Update(context.Background(), backupSession); err != nil { + log.Error(err, "unable to update status", "backupsession", backupSession) return err } return nil @@ -80,10 +86,11 @@ func CreateBackupSession(name string, namespace string) { *backupConf = bc } } + log.V(0).Info("got backupConf", "backupConf", backupConf) backupSession := &formolv1alpha1.BackupSession{ ObjectMeta: metav1.ObjectMeta{ - Name: strings.Join([]string{"backupsession",name,strconv.FormatInt(time.Now().Unix(), 10)}, "-"), + Name: strings.Join([]string{"backupsession", name, strconv.FormatInt(time.Now().Unix(), 10)}, "-"), Namespace: namespace, }, Spec: formolv1alpha1.BackupSessionSpec{ @@ -91,12 +98,8 @@ func CreateBackupSession(name string, namespace string) { Name: name, }, }, - Status: formolv1alpha1.BackupSessionStatus{}, - } - if err := ctrl.SetControllerReference(backupConf, backupSession, scheme); err != nil { - log.Error(err, "unable to set controller reference") - os.Exit(1) } + log.V(1).Info("create backupsession", "backupSession", backupSession) if err := cl.Create(context.TODO(), backupSession); err != nil { log.Error(err, "unable to create backupsession") os.Exit(1) diff --git a/pkg/cmd/backupsession.go b/pkg/cmd/backupsession.go index 36f1e32..644a474 100644 --- a/pkg/cmd/backupsession.go +++ b/pkg/cmd/backupsession.go @@ -16,8 +16,8 @@ limitations under the License. package cmd import ( - "github.com/spf13/cobra" "github.com/desmo999r/formolcli/pkg/backupsession" + "github.com/spf13/cobra" ) // backupsessionCmd represents the backupsession command @@ -37,25 +37,8 @@ to quickly create a Cobra application.`, }, } -var deleteBackupsessionCmd = &cobra.Command{ - Use: "backupsession", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples -and usage of using your command. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, - Run: func(cmd *cobra.Command, args []string) { - name, _ := cmd.Flags().GetString("name") - namespace, _ := cmd.Flags().GetString("namespace") - backupsession.DeleteBackupSession(name, namespace) - }, -} - func init() { createCmd.AddCommand(createBackupsessionCmd) - deleteCmd.AddCommand(deleteBackupsessionCmd) // Here you will define your flags and configuration settings. @@ -70,8 +53,4 @@ func init() { createBackupsessionCmd.Flags().String("name", "", "The referenced BackupSessionConfiguration name") createBackupsessionCmd.MarkFlagRequired("namespace") createBackupsessionCmd.MarkFlagRequired("name") - deleteBackupsessionCmd.Flags().String("namespace", "", "The referenced BackupSessionConfiguration namespace") - deleteBackupsessionCmd.Flags().String("name", "", "The referenced BackupSessionConfiguration name") - deleteBackupsessionCmd.MarkFlagRequired("namespace") - deleteBackupsessionCmd.MarkFlagRequired("name") } diff --git a/pkg/cmd/postgres.go b/pkg/cmd/postgres.go new file mode 100644 index 0000000..07f5a86 --- /dev/null +++ b/pkg/cmd/postgres.go @@ -0,0 +1,68 @@ +/* +Copyright © 2020 NAME HERE + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package cmd + +import ( + "fmt" + + "github.com/desmo999r/formolcli/pkg/backup" + "github.com/spf13/cobra" +) + +// postgresCmd represents the postgres command +var postgresCmd = &cobra.Command{ + Use: "postgres", + Short: "backup a PostgreSQL database", + Long: `A longer description that spans multiple lines and likely contains examples +and usage of using your command. For example: + +Cobra is a CLI library for Go that empowers applications. +This application is a tool to generate the needed files +to quickly create a Cobra application.`, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("postgres called") + file, _ := cmd.Flags().GetString("file") + hostname, _ := cmd.Flags().GetString("hostname") + database, _ := cmd.Flags().GetString("database") + username, _ := cmd.Flags().GetString("username") + password, _ := cmd.Flags().GetString("password") + _ = backup.BackupPostgres(file, hostname, database, username, password) + }, +} + +func init() { + backupCmd.AddCommand(postgresCmd) + + // Here you will define your flags and configuration settings. + + // Cobra supports Persistent Flags which will work for this command + // and all subcommands, e.g.: + // postgresCmd.PersistentFlags().String("foo", "", "A help for foo") + + // Cobra supports local flags which will only run when this command + // is called directly, e.g.: + // postgresCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") + postgresCmd.Flags().String("file", "", "The file the backup will be stored") + postgresCmd.Flags().String("hostname", "", "The postgresql server host") + postgresCmd.Flags().String("database", "", "The postgresql database") + postgresCmd.Flags().String("username", "", "The postgresql username") + postgresCmd.Flags().String("password", "", "The postgresql password") + postgresCmd.MarkFlagRequired("path") + postgresCmd.MarkFlagRequired("hostname") + postgresCmd.MarkFlagRequired("database") + postgresCmd.MarkFlagRequired("username") + postgresCmd.MarkFlagRequired("password") +} diff --git a/pkg/cmd/server.go b/pkg/cmd/server.go index 1784d7f..246d104 100644 --- a/pkg/cmd/server.go +++ b/pkg/cmd/server.go @@ -18,8 +18,8 @@ package cmd import ( "fmt" - "github.com/spf13/cobra" "github.com/desmo999r/formolcli/pkg/server" + "github.com/spf13/cobra" ) // serverCmd represents the server command diff --git a/pkg/cmd/snapshot.go b/pkg/cmd/snapshot.go new file mode 100644 index 0000000..ea4ea2c --- /dev/null +++ b/pkg/cmd/snapshot.go @@ -0,0 +1,56 @@ +/* Copyright © 2020 NAME HERE + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package cmd + +import ( + "os" + + "github.com/desmo999r/formolcli/pkg/backup" + "github.com/spf13/cobra" +) + +// snapshotCmd represents the snapshot command +var snapshotCmd = &cobra.Command{ + Use: "snapshot", + Short: "A brief description of your command", + Long: `A longer description that spans multiple lines and likely contains examples +and usage of using your command. For example: + +Cobra is a CLI library for Go that empowers applications. +This application is a tool to generate the needed files +to quickly create a Cobra application.`, + Run: func(cmd *cobra.Command, args []string) { + snapshot, _ := cmd.Flags().GetString("snapshot") + if err := backup.DeleteSnapshot(snapshot); err != nil { + os.Exit(1) + } + }, +} + +func init() { + deleteCmd.AddCommand(snapshotCmd) + + // Here you will define your flags and configuration settings. + + // Cobra supports Persistent Flags which will work for this command + // and all subcommands, e.g.: + // snapshotCmd.PersistentFlags().String("foo", "", "A help for foo") + + // Cobra supports local flags which will only run when this command + // is called directly, e.g.: + // snapshotCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") + snapshotCmd.Flags().String("snapshot", "", "The snapshot to delete") + snapshotCmd.MarkFlagRequired("snapshot") +} diff --git a/pkg/cmd/volume.go b/pkg/cmd/volume.go index b786b3a..3209824 100644 --- a/pkg/cmd/volume.go +++ b/pkg/cmd/volume.go @@ -16,10 +16,10 @@ limitations under the License. package cmd import ( - "fmt" + "os" - "github.com/spf13/cobra" "github.com/desmo999r/formolcli/pkg/backup" + "github.com/spf13/cobra" ) // pvcCmd represents the pvc command @@ -33,9 +33,11 @@ Cobra is a CLI library for Go that empowers applications. This application is a tool to generate the needed files to quickly create a Cobra application.`, Run: func(cmd *cobra.Command, args []string) { - fmt.Println("volume called") - path, _ := cmd.Flags().GetString("path") - _ = backup.BackupVolume(path) + paths, _ := cmd.Flags().GetStringSlice("path") + tag, _ := cmd.Flags().GetString("tag") + if err := backup.BackupVolume(tag, paths); err != nil { + os.Exit(1) + } }, } @@ -51,6 +53,7 @@ func init() { // Cobra supports local flags which will only run when this command // is called directly, e.g.: // pvcCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") - volumeCmd.Flags().String("path", "", "Path to the data to backup") + volumeCmd.Flags().StringSlice("path", nil, "Path to the data to backup") + volumeCmd.Flags().String("tag", "", "Tag associated to the backup") volumeCmd.MarkFlagRequired("path") } diff --git a/pkg/controllers/backupsession_controller.go b/pkg/controllers/backupsession_controller.go index dc9c61a..d437063 100644 --- a/pkg/controllers/backupsession_controller.go +++ b/pkg/controllers/backupsession_controller.go @@ -1,32 +1,28 @@ package controllers import ( - "time" - "sort" - "encoding/json" "context" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/runtime" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/predicate" - "sigs.k8s.io/controller-runtime/pkg/log/zap" "os" "path/filepath" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "time" formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1" + "github.com/desmo999r/formolcli/pkg/backup" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" - "github.com/desmo999r/formolcli/pkg/backup" - formolutils "github.com/desmo999r/formol/pkg/utils" ) var ( deploymentName = "" - sessionState = ".metadata.state" ) func init() { @@ -38,7 +34,7 @@ func init() { } config, err := rest.InClusterConfig() if err != nil { - config, err = clientcmd.BuildConfigFromFlags("", filepath.Join(os.Getenv("HOME"), ".kube", "config",)) + config, err = clientcmd.BuildConfigFromFlags("", filepath.Join(os.Getenv("HOME"), ".kube", "config")) if err != nil { log.Error(err, "unable to get config") panic(err.Error()) @@ -90,38 +86,6 @@ func (r *BackupSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro log.Error(err, "unable to get backupsession") return ctrl.Result{}, client.IgnoreNotFound(err) } - finalizerName := "finalizer.backupsession.formol.desmojim.fr" - - if backupSession.ObjectMeta.DeletionTimestamp.IsZero() { - if !formolutils.ContainsString(backupSession.ObjectMeta.Finalizers, finalizerName) { - backupSession.ObjectMeta.Finalizers = append(backupSession.ObjectMeta.Finalizers, finalizerName) - if err := r.Update(context.Background(), backupSession); err != nil { - log.Error(err, "unable to append finalizer") - return ctrl.Result{}, err - } - } - } else { - log.V(0).Info("backupsession being deleted", "backupsession", backupSession.Name) - if formolutils.ContainsString(backupSession.ObjectMeta.Finalizers, finalizerName) { - if err := r.deleteExternalResources(backupSession); err != nil { - return ctrl.Result{}, err - } - } - backupSession.ObjectMeta.Finalizers = formolutils.RemoveString(backupSession.ObjectMeta.Finalizers, finalizerName) - if err := r.Update(context.Background(), backupSession); err != nil { - log.Error(err, "unable to remove finalizer") - return ctrl.Result{}, err - } - // We have been deleted. Return here - return ctrl.Result{}, nil - } - - log.V(1).Info("backupSession.Namespace", "namespace", backupSession.Namespace) - log.V(1).Info("backupSession.Spec.Ref.Name", "name", backupSession.Spec.Ref.Name) - if backupSession.Status.BackupSessionState != "" { - log.V(0).Info("State is not null. Skipping", "state", backupSession.Status.BackupSessionState) - return ctrl.Result{}, nil - } backupConf := &formolv1alpha1.BackupConfiguration{} if err := r.Get(ctx, client.ObjectKey{ Namespace: backupSession.Namespace, @@ -132,155 +96,87 @@ func (r *BackupSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro } // Found the BackupConfiguration. - log.V(1).Info("Found BackupConfiguration", "BackupConfiguration", backupConf.Name) - - backupDeployment := func(target formolv1alpha1.Target) error { - log.V(0).Info("before", "backupsession", backupSession) - backupSession.Status.BackupSessionState = formolv1alpha1.Running - if err := r.Client.Status().Update(ctx, backupSession); err != nil { - log.Error(err, "unable to update status", "backupsession", backupSession) - return err - } - // Preparing for backup - c := make(chan []byte) - - go func(){ - for msg := range c { - var dat map[string]interface{} - if err := json.Unmarshal(msg, &dat); err != nil { - log.Error(err, "unable to unmarshal json", "msg", msg) - continue - } - log.V(1).Info("message on stdout", "stdout", dat) - if message_type, ok := dat["message_type"]; ok && message_type == "summary"{ - backupSession.Status.SnapshotId = dat["snapshot_id"].(string) - backupSession.Status.Duration = &metav1.Duration{Duration : time.Duration(dat["total_duration"].(float64) * 1000) * time.Millisecond} - } - } - }() - result := formolv1alpha1.Failure - defer func() { - close(c) - backupSession.Status.BackupSessionState = result - if err := r.Status().Update(ctx, backupSession); err != nil { - log.Error(err, "unable to update status") - } - }() - // do the backup - backupSession.Status.StartTime = &metav1.Time {Time: time.Now()} - if err := backup.BackupDeployment("", target.Paths, c); err != nil { - log.Error(err, "unable to backup deployment") - return err - } - result = formolv1alpha1.Success - - // cleanup old backups - backupSessionList := &formolv1alpha1.BackupSessionList{} - if err := r.List(ctx, backupSessionList, client.InNamespace(backupConf.Namespace), client.MatchingFieldsSelector{fields.SelectorFromSet(fields.Set{sessionState: "Success"})}); err != nil { - return nil - } - if len(backupSessionList.Items) < 2 { - // Not enough backupSession to proceed - return nil - } - - sort.Slice(backupSessionList.Items, func(i, j int) bool { - return backupSessionList.Items[i].Status.StartTime.Time.Unix() > backupSessionList.Items[j].Status.StartTime.Time.Unix() - }) - - type KeepBackup struct { - Counter int32 - Last time.Time - } - - var lastBackups, dailyBackups, weeklyBackups, monthlyBackups, yearlyBackups KeepBackup - lastBackups.Counter = backupConf.Spec.Keep.Last - dailyBackups.Counter = backupConf.Spec.Keep.Daily - weeklyBackups.Counter = backupConf.Spec.Keep.Weekly - monthlyBackups.Counter = backupConf.Spec.Keep.Monthly - yearlyBackups.Counter = backupConf.Spec.Keep.Yearly - for _, session := range backupSessionList.Items { - if session.Spec.Ref.Name != backupConf.Name { - continue - } - deleteSession := true - if lastBackups.Counter > 0 { - log.V(1).Info("Keep backup", "last", session.Status.StartTime) - lastBackups.Counter-- - deleteSession = false - } - if dailyBackups.Counter > 0 { - if session.Status.StartTime.Time.YearDay() != dailyBackups.Last.YearDay() { - log.V(1).Info("Keep backup", "daily", session.Status.StartTime) - dailyBackups.Counter-- - dailyBackups.Last = session.Status.StartTime.Time - deleteSession = false - } - } - if weeklyBackups.Counter > 0 { - if session.Status.StartTime.Time.Weekday().String() == "Sunday" && session.Status.StartTime.Time.YearDay() != weeklyBackups.Last.YearDay() { - log.V(1).Info("Keep backup", "weekly", session.Status.StartTime) - weeklyBackups.Counter-- - weeklyBackups.Last = session.Status.StartTime.Time - deleteSession = false - } - } - if monthlyBackups.Counter > 0 { - if session.Status.StartTime.Time.Day() == 1 && session.Status.StartTime.Time.Month() != monthlyBackups.Last.Month() { - log.V(1).Info("Keep backup", "monthly", session.Status.StartTime) - monthlyBackups.Counter-- - monthlyBackups.Last = session.Status.StartTime.Time - deleteSession = false - } - } - if yearlyBackups.Counter > 0 { - if session.Status.StartTime.Time.YearDay() == 1 && session.Status.StartTime.Time.Year() != yearlyBackups.Last.Year() { - log.V(1).Info("Keep backup", "yearly", session.Status.StartTime) - yearlyBackups.Counter-- - yearlyBackups.Last = session.Status.StartTime.Time - deleteSession = false - } - } - if deleteSession { - log.V(1).Info("Delete session", "delete", session.Status.StartTime) - if err := r.Delete(ctx, &session); err != nil { - log.Error(err, "unable to delete backupsession", "session", session.Name) - // we don't return anything, we keep going - } - } - } - return nil - } + // backupDeployment := func(target formolv1alpha1.Target) error { + // //backupSession.Status.BackupSessionState = formolv1alpha1.Running + // //if err := r.Client.Status().Update(ctx, backupSession); err != nil { + // // log.Error(err, "unable to update status", "backupsession", backupSession) + // // return err + // //} + // // Preparing for backup + // c := make(chan []byte) + // + // go func() { + // for msg := range c { + // var dat map[string]interface{} + // if err := json.Unmarshal(msg, &dat); err != nil { + // log.Error(err, "unable to unmarshal json", "msg", msg) + // continue + // } + // log.V(1).Info("message on stdout", "stdout", dat) + // //if message_type, ok := dat["message_type"]; ok && message_type == "summary" { + // // backupSession.Status.SnapshotId = dat["snapshot_id"].(string) + // // backupSession.Status.Duration = &metav1.Duration{Duration: time.Duration(dat["total_duration"].(float64)*1000) * time.Millisecond} + // //} + // } + // }() + // //result := formolv1alpha1.Failure + // defer func() { + // close(c) + // //backupSession.Status.BackupSessionState = result + // //if err := r.Status().Update(ctx, backupSession); err != nil { + // // log.Error(err, "unable to update status") + // //} + // }() + // // do the backup + // //backupSession.Status.StartTime = &metav1.Time{Time: time.Now()} + // if err := backup.BackupPaths(backupSession.Name, target.Paths, c); err != nil { + // log.Error(err, "unable to backup deployment") + // return err + // } + // //result = formolv1alpha1.Success + // + // return nil + // } for _, target := range backupConf.Spec.Targets { switch target.Kind { case "Deployment": if target.Name == deploymentName { - log.V(0).Info("It's for us!", "target", target) - return ctrl.Result{}, backupDeployment(target) + for i, status := range backupSession.Status.Targets { + if status.Name == target.Name && status.BackupState == formolv1alpha1.New { + log.V(0).Info("It's for us!", "target", target) + result := formolv1alpha1.Success + status.StartTime = &metav1.Time{Time: time.Now()} + output, err := backup.BackupPaths(backupSession.Name, target.Paths) + if err != nil { + log.Error(err, "unable to backup deployment", "output", string(output)) + result = formolv1alpha1.Failure + } else { + snapshotId, duration := backup.GetBackupResults(output) + backupSession.Status.Targets[i].SnapshotId = snapshotId + backupSession.Status.Targets[i].Duration = &metav1.Duration{Duration: duration} + } + backupSession.Status.Targets[i].BackupState = result + log.V(1).Info("current backupSession status", "status", backupSession.Status) + if err := r.Status().Update(ctx, backupSession); err != nil { + log.Error(err, "unable to update backupsession status") + return ctrl.Result{}, err + } + } + } } } } return ctrl.Result{}, nil } -func (r *BackupSessionReconciler) deleteExternalResources(backupSession *formolv1alpha1.BackupSession) error { - if err := backup.DeleteSnapshot("", backupSession.Status.SnapshotId); err != nil { - return err - } - return nil -} - func (r *BackupSessionReconciler) SetupWithManager(mgr ctrl.Manager) error { - if err := mgr.GetFieldIndexer().IndexField(context.TODO(), &formolv1alpha1.BackupSession{}, sessionState, func(rawObj runtime.Object) []string { - session := rawObj.(*formolv1alpha1.BackupSession) - return []string{string(session.Status.BackupSessionState)} - }); err != nil { - return err - } return ctrl.NewControllerManagedBy(mgr). For(&formolv1alpha1.BackupSession{}). - WithEventFilter(predicate.GenerationChangedPredicate{}). // Don't reconcile when status gets updated + //WithEventFilter(predicate.GenerationChangedPredicate{}). // Don't reconcile when status gets updated + WithEventFilter(predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { return false }, + DeleteFunc: func(e event.DeleteEvent) bool { return false }, + }). // Don't reconcile when status gets updated Complete(r) } - diff --git a/pkg/server/root.go b/pkg/server/root.go index 1617f59..1635e2f 100644 --- a/pkg/server/root.go +++ b/pkg/server/root.go @@ -1,18 +1,18 @@ package server import ( - "os" "flag" + formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1" + "github.com/desmo999r/formolcli/pkg/controllers" "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "os" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/log/zap" - "github.com/desmo999r/formolcli/pkg/controllers" - formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1" ) var ( - scheme = runtime.NewScheme() + scheme = runtime.NewScheme() setupLog = ctrl.Log.WithName("server") ) @@ -21,7 +21,7 @@ func init() { _ = formolv1alpha1.AddToScheme(scheme) } -func Server(){ +func Server() { var metricsAddr string var enableLeaderElection bool flag.StringVar(&metricsAddr, "metrics-addr", ":8082", "The address the metric endpoint binds to.") @@ -34,12 +34,12 @@ func Server(){ config, err := ctrl.GetConfig() mgr, err := ctrl.NewManager(config, ctrl.Options{ - Scheme: scheme, + Scheme: scheme, MetricsBindAddress: metricsAddr, - Port: 9443, - LeaderElection: enableLeaderElection, - LeaderElectionID: "12345.desmojim.fr", - Namespace: os.Getenv("POD_NAMESPACE"), + Port: 9443, + LeaderElection: enableLeaderElection, + LeaderElectionID: "12345.desmojim.fr", + Namespace: os.Getenv("POD_NAMESPACE"), }) if err != nil { setupLog.Error(err, "unable to create manager") @@ -48,7 +48,7 @@ func Server(){ if err = (&controllers.BackupSessionReconciler{ Client: mgr.GetClient(), - Log: ctrl.Log.WithName("controllers").WithName("BackupSession"), + Log: ctrl.Log.WithName("controllers").WithName("BackupSession"), Scheme: mgr.GetScheme(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "BackupSession")