From fba08693cda57d5f67e46b4aff510716b39025aa Mon Sep 17 00:00:00 2001 From: Jean-Marc Andre Date: Thu, 11 Feb 2021 22:04:58 +0100 Subject: [PATCH] Re-organized the formolcli CLI commands and added the restore functionality --- go.mod | 2 + pkg/backup/root.go | 91 ++-------------- pkg/cmd/backup.go | 51 --------- pkg/cmd/backupsession.go | 53 +++++---- pkg/cmd/command.go | 54 --------- pkg/cmd/create.go | 51 --------- pkg/cmd/delete.go | 51 --------- pkg/cmd/postgres.go | 68 +++++++++--- pkg/cmd/run.go | 51 --------- pkg/cmd/server.go | 53 --------- pkg/cmd/snapshot.go | 32 ++---- pkg/cmd/target.go | 44 ++++++++ pkg/cmd/volume.go | 48 +++++--- pkg/controllers/backupsession_controller.go | 9 +- pkg/controllers/restoresession_controller.go | 84 +++++++++++--- pkg/restic/root.go | 109 +++++++++++++++++++ pkg/restore/root.go | 56 ++++++++++ pkg/server/root.go | 11 +- pkg/{backupsession => session}/root.go | 31 +++++- 19 files changed, 461 insertions(+), 488 deletions(-) delete mode 100644 pkg/cmd/backup.go delete mode 100644 pkg/cmd/command.go delete mode 100644 pkg/cmd/create.go delete mode 100644 pkg/cmd/delete.go delete mode 100644 pkg/cmd/run.go delete mode 100644 pkg/cmd/server.go create mode 100644 pkg/cmd/target.go create mode 100644 pkg/restic/root.go create mode 100644 pkg/restore/root.go rename pkg/{backupsession => session}/root.go (71%) diff --git a/go.mod b/go.mod index 684f3a9..f35c682 100644 --- a/go.mod +++ b/go.mod @@ -17,3 +17,5 @@ require ( k8s.io/client-go v0.18.6 sigs.k8s.io/controller-runtime v0.6.4 ) + +replace github.com/desmo999r/formol => /home/jandre/devel/golang/formol diff --git a/pkg/backup/root.go b/pkg/backup/root.go index 7801170..5004803 100644 --- a/pkg/backup/root.go +++ b/pkg/backup/root.go @@ -1,93 +1,40 @@ package backup import ( - "bufio" - "bytes" - "encoding/json" "fmt" formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1" - "github.com/desmo999r/formolcli/pkg/backupsession" + "github.com/desmo999r/formolcli/pkg/restic" + "github.com/desmo999r/formolcli/pkg/session" "github.com/go-logr/logr" "github.com/go-logr/zapr" "go.uber.org/zap" "io/ioutil" "os" "os/exec" - "time" ) var ( - repository string - passwordFile string - aws_access_key_id string - aws_secret_access_key string - resticExec = "/usr/bin/restic" - pg_dumpExec = "/usr/bin/pg_dump" - logger logr.Logger + pg_dumpExec = "/usr/bin/pg_dump" + logger logr.Logger ) func init() { zapLog, _ := zap.NewDevelopment() logger = zapr.NewLogger(zapLog) - repository = os.Getenv("RESTIC_REPOSITORY") - passwordFile = os.Getenv("RESTIC_PASSWORD") - aws_access_key_id = os.Getenv("AWS_ACCESS_KEY_ID") - aws_secret_access_key = os.Getenv("AWS_SECRET_ACCESS_KEY") -} - -func checkRepo(repo string) error { - log := logger.WithValues("backup-checkrepo", repo) - cmd := exec.Command(resticExec, "unlock", "-r", repo) - if err := cmd.Run(); err != nil { - log.Error(err, "unable to unlock repo", "repo", repo) - } - cmd = exec.Command(resticExec, "check", "-r", repo) - output, err := cmd.CombinedOutput() - log.V(1).Info("restic check output", "output", string(output)) - if err != nil { - log.V(0).Info("initializing new repo", "repo", repo) - cmd = exec.Command(resticExec, "init", "-r", repo) - output, err = cmd.CombinedOutput() - log.V(1).Info("restic init repo", "output", string(output)) - if err != nil { - log.Error(err, "something went wrong during repo init") - return err - } - } - return err -} - -func GetBackupResults(output []byte) (snapshotId string, duration time.Duration) { - log := logger.WithName("backup-getbackupresults") - scanner := bufio.NewScanner(bytes.NewReader(output)) - var dat map[string]interface{} - for scanner.Scan() { - if err := json.Unmarshal(scanner.Bytes(), &dat); err != nil { - log.Error(err, "unable to unmarshal json", "msg", string(scanner.Bytes())) - continue - } - log.V(1).Info("message on stdout", "stdout", dat) - if message_type, ok := dat["message_type"]; ok && message_type == "summary" { - snapshotId = dat["snapshot_id"].(string) - duration = time.Duration(dat["total_duration"].(float64)*1000) * time.Millisecond - } - } - return } func BackupVolume(tag string, paths []string) error { log := logger.WithName("backup-volume") state := formolv1alpha1.Success - output, err := BackupPaths(tag, paths) + output, err := restic.BackupPaths(tag, paths) var snapshotId string - var duration time.Duration if err != nil { log.Error(err, "unable to backup volume", "output", string(output)) state = formolv1alpha1.Failure } else { - snapshotId, duration = GetBackupResults(output) + snapshotId = restic.GetBackupResults(output) } - backupsession.BackupSessionUpdateStatus(state, snapshotId, duration) + session.BackupSessionUpdateTargetStatus(state, snapshotId) return err } @@ -109,27 +56,3 @@ func BackupPostgres(file string, hostname string, database string, username stri } return nil } - -func BackupPaths(tag string, paths []string) ([]byte, error) { - log := logger.WithName("backup-deployment") - if err := checkRepo(repository); err != nil { - log.Error(err, "unable to setup newrepo", "newrepo", repository) - return []byte{}, err - } - cmd := exec.Command(resticExec, append([]string{"backup", "--json", "--tag", tag, "-r", repository}, paths...)...) - output, err := cmd.CombinedOutput() - return output, err -} - -func DeleteSnapshot(snapshot string) error { - log := logger.WithValues("delete-snapshot", snapshot) - cmd := exec.Command(resticExec, "forget", "-r", repository, "--prune", snapshot) - log.V(0).Info("deleting snapshot", "snapshot", snapshot) - output, err := cmd.CombinedOutput() - log.V(1).Info("delete snapshot output", "output", string(output)) - if err != nil { - log.Error(err, "unable to delete the snapshot") - return err - } - return nil -} diff --git a/pkg/cmd/backup.go b/pkg/cmd/backup.go deleted file mode 100644 index af23db7..0000000 --- a/pkg/cmd/backup.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright © 2020 NAME HERE - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package cmd - -import ( - "fmt" - - "github.com/spf13/cobra" -) - -// backupCmd represents the backup command -var backupCmd = &cobra.Command{ - Use: "backup", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples -and usage of using your command. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, - Run: func(cmd *cobra.Command, args []string) { - fmt.Println("backup called") - }, -} - -func init() { - rootCmd.AddCommand(backupCmd) - - // Here you will define your flags and configuration settings. - - // Cobra supports Persistent Flags which will work for this command - // and all subcommands, e.g.: - // backupCmd.PersistentFlags().String("foo", "", "A help for foo") - - // Cobra supports local flags which will only run when this command - // is called directly, e.g.: - // backupCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") -} diff --git a/pkg/cmd/backupsession.go b/pkg/cmd/backupsession.go index 644a474..12e3a26 100644 --- a/pkg/cmd/backupsession.go +++ b/pkg/cmd/backupsession.go @@ -16,41 +16,52 @@ limitations under the License. package cmd import ( - "github.com/desmo999r/formolcli/pkg/backupsession" + "github.com/desmo999r/formolcli/pkg/server" + "github.com/desmo999r/formolcli/pkg/session" "github.com/spf13/cobra" ) -// backupsessionCmd represents the backupsession command -var createBackupsessionCmd = &cobra.Command{ - Use: "backupsession", +var serverBackupSessionCmd = &cobra.Command{ + Use: "server", Short: "A brief description of your command", Long: `A longer description that spans multiple lines and likely contains examples and usage of using your command. For example: +Cobra is a CLI library for Go that empowers applications. +This application is a tool to generate the needed files +to quickly create a Cobra application.`, + Run: func(cmd *cobra.Command, args []string) { + server.Server() + }, +} + +var createBackupSessionCmd = &cobra.Command{ + Use: "create", + Short: "Creates a backupsession", + Long: `A longer description that spans multiple lines and likely contains examples +and usage of using your command. For example: + Cobra is a CLI library for Go that empowers applications. This application is a tool to generate the needed files to quickly create a Cobra application.`, Run: func(cmd *cobra.Command, args []string) { name, _ := cmd.Flags().GetString("name") namespace, _ := cmd.Flags().GetString("namespace") - backupsession.CreateBackupSession(name, namespace) + session.CreateBackupSession(name, namespace) }, } -func init() { - createCmd.AddCommand(createBackupsessionCmd) - - // Here you will define your flags and configuration settings. - - // Cobra supports Persistent Flags which will work for this command - // and all subcommands, e.g.: - // backupsessionCmd.PersistentFlags().String("foo", "", "A help for foo") - - // Cobra supports local flags which will only run when this command - // is called directly, e.g.: - // backupsessionCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") - createBackupsessionCmd.Flags().String("namespace", "", "The referenced BackupSessionConfiguration namespace") - createBackupsessionCmd.Flags().String("name", "", "The referenced BackupSessionConfiguration name") - createBackupsessionCmd.MarkFlagRequired("namespace") - createBackupsessionCmd.MarkFlagRequired("name") +var backupSessionCmd = &cobra.Command{ + Use: "backupsession", + Short: "backupsession related commands", +} + +func init() { + rootCmd.AddCommand(backupSessionCmd) + backupSessionCmd.AddCommand(createBackupSessionCmd) + backupSessionCmd.AddCommand(serverBackupSessionCmd) + createBackupSessionCmd.Flags().String("namespace", "", "The referenced BackupSessionConfiguration namespace") + createBackupSessionCmd.Flags().String("name", "", "The referenced BackupSessionConfiguration name") + createBackupSessionCmd.MarkFlagRequired("namespace") + createBackupSessionCmd.MarkFlagRequired("name") } diff --git a/pkg/cmd/command.go b/pkg/cmd/command.go deleted file mode 100644 index 3975666..0000000 --- a/pkg/cmd/command.go +++ /dev/null @@ -1,54 +0,0 @@ -/* -Copyright © 2021 NAME HERE - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package cmd - -import ( - formolcliutils "github.com/desmo999r/formolcli/pkg/utils" - "github.com/spf13/cobra" - "os" -) - -// commandCmd represents the command command -var commandCmd = &cobra.Command{ - Use: "command [cmd] ([args]...)", - Args: cobra.MinimumNArgs(1), - Short: "Run a specific command with args", - Long: `A longer description that spans multiple lines and likely contains examples -and usage of using your command. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, - Run: func(cmd *cobra.Command, args []string) { - if err := formolcliutils.RunChroot(args[0], args[1:]...); err != nil { - os.Exit(1) - } - }, -} - -func init() { - runCmd.AddCommand(commandCmd) - - // Here you will define your flags and configuration settings. - - // Cobra supports Persistent Flags which will work for this command - // and all subcommands, e.g.: - // commandCmd.PersistentFlags().String("foo", "", "A help for foo") - - // Cobra supports local flags which will only run when this command - // is called directly, e.g.: - // commandCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") -} diff --git a/pkg/cmd/create.go b/pkg/cmd/create.go deleted file mode 100644 index f161df6..0000000 --- a/pkg/cmd/create.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright © 2020 NAME HERE - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package cmd - -import ( - "fmt" - - "github.com/spf13/cobra" -) - -// createCmd represents the create command -var createCmd = &cobra.Command{ - Use: "create", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples -and usage of using your command. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, - Run: func(cmd *cobra.Command, args []string) { - fmt.Println("create called") - }, -} - -func init() { - rootCmd.AddCommand(createCmd) - - // Here you will define your flags and configuration settings. - - // Cobra supports Persistent Flags which will work for this command - // and all subcommands, e.g.: - // createCmd.PersistentFlags().String("foo", "", "A help for foo") - - // Cobra supports local flags which will only run when this command - // is called directly, e.g.: - // createCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") -} diff --git a/pkg/cmd/delete.go b/pkg/cmd/delete.go deleted file mode 100644 index 61a8350..0000000 --- a/pkg/cmd/delete.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright © 2020 NAME HERE - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package cmd - -import ( - "fmt" - - "github.com/spf13/cobra" -) - -// deleteCmd represents the delete command -var deleteCmd = &cobra.Command{ - Use: "delete", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples -and usage of using your command. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, - Run: func(cmd *cobra.Command, args []string) { - fmt.Println("delete called") - }, -} - -func init() { - rootCmd.AddCommand(deleteCmd) - - // Here you will define your flags and configuration settings. - - // Cobra supports Persistent Flags which will work for this command - // and all subcommands, e.g.: - // deleteCmd.PersistentFlags().String("foo", "", "A help for foo") - - // Cobra supports local flags which will only run when this command - // is called directly, e.g.: - // deleteCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") -} diff --git a/pkg/cmd/postgres.go b/pkg/cmd/postgres.go index 07f5a86..91bf264 100644 --- a/pkg/cmd/postgres.go +++ b/pkg/cmd/postgres.go @@ -19,12 +19,33 @@ import ( "fmt" "github.com/desmo999r/formolcli/pkg/backup" + "github.com/desmo999r/formolcli/pkg/restore" "github.com/spf13/cobra" ) // postgresCmd represents the postgres command -var postgresCmd = &cobra.Command{ - Use: "postgres", +var postgresRestoreCmd = &cobra.Command{ + Use: "restore", + Short: "restore a PostgreSQL database", + Long: `A longer description that spans multiple lines and likely contains examples +and usage of using your command. For example: + +Cobra is a CLI library for Go that empowers applications. +This application is a tool to generate the needed files +to quickly create a Cobra application.`, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("postgres called") + file, _ := cmd.Flags().GetString("file") + hostname, _ := cmd.Flags().GetString("hostname") + database, _ := cmd.Flags().GetString("database") + username, _ := cmd.Flags().GetString("username") + password, _ := cmd.Flags().GetString("password") + _ = restore.RestorePostgres(file, hostname, database, username, password) + }, +} + +var postgresBackupCmd = &cobra.Command{ + Use: "backup", Short: "backup a PostgreSQL database", Long: `A longer description that spans multiple lines and likely contains examples and usage of using your command. For example: @@ -43,26 +64,43 @@ to quickly create a Cobra application.`, }, } +var postgresCmd = &cobra.Command{ + Use: "postgres", + Short: "postgres actions", +} + func init() { - backupCmd.AddCommand(postgresCmd) + rootCmd.AddCommand(postgresCmd) + postgresCmd.AddCommand(postgresBackupCmd) + postgresCmd.AddCommand(postgresRestoreCmd) // Here you will define your flags and configuration settings. // Cobra supports Persistent Flags which will work for this command // and all subcommands, e.g.: - // postgresCmd.PersistentFlags().String("foo", "", "A help for foo") + // backupPostgresCmd.PersistentFlags().String("foo", "", "A help for foo") // Cobra supports local flags which will only run when this command // is called directly, e.g.: - // postgresCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") - postgresCmd.Flags().String("file", "", "The file the backup will be stored") - postgresCmd.Flags().String("hostname", "", "The postgresql server host") - postgresCmd.Flags().String("database", "", "The postgresql database") - postgresCmd.Flags().String("username", "", "The postgresql username") - postgresCmd.Flags().String("password", "", "The postgresql password") - postgresCmd.MarkFlagRequired("path") - postgresCmd.MarkFlagRequired("hostname") - postgresCmd.MarkFlagRequired("database") - postgresCmd.MarkFlagRequired("username") - postgresCmd.MarkFlagRequired("password") + // backupPostgresCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") + postgresBackupCmd.Flags().String("file", "", "The file the backup will be stored") + postgresBackupCmd.Flags().String("hostname", "", "The postgresql server host") + postgresBackupCmd.Flags().String("database", "", "The postgresql database") + postgresBackupCmd.Flags().String("username", "", "The postgresql username") + postgresBackupCmd.Flags().String("password", "", "The postgresql password") + postgresBackupCmd.MarkFlagRequired("path") + postgresBackupCmd.MarkFlagRequired("hostname") + postgresBackupCmd.MarkFlagRequired("database") + postgresBackupCmd.MarkFlagRequired("username") + postgresBackupCmd.MarkFlagRequired("password") + postgresRestoreCmd.Flags().String("file", "", "The file the database will be restored from") + postgresRestoreCmd.Flags().String("hostname", "", "The postgresql server host") + postgresRestoreCmd.Flags().String("database", "", "The postgresql database") + postgresRestoreCmd.Flags().String("username", "", "The postgresql username") + postgresRestoreCmd.Flags().String("password", "", "The postgresql password") + postgresRestoreCmd.MarkFlagRequired("path") + postgresRestoreCmd.MarkFlagRequired("hostname") + postgresRestoreCmd.MarkFlagRequired("database") + postgresRestoreCmd.MarkFlagRequired("username") + postgresRestoreCmd.MarkFlagRequired("password") } diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go deleted file mode 100644 index 7bfde8c..0000000 --- a/pkg/cmd/run.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright © 2021 NAME HERE - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package cmd - -import ( - "fmt" - - "github.com/spf13/cobra" -) - -// runCmd represents the run command -var runCmd = &cobra.Command{ - Use: "run", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples -and usage of using your command. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, - Run: func(cmd *cobra.Command, args []string) { - fmt.Println("run called") - }, -} - -func init() { - rootCmd.AddCommand(runCmd) - - // Here you will define your flags and configuration settings. - - // Cobra supports Persistent Flags which will work for this command - // and all subcommands, e.g.: - // runCmd.PersistentFlags().String("foo", "", "A help for foo") - - // Cobra supports local flags which will only run when this command - // is called directly, e.g.: - // runCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") -} diff --git a/pkg/cmd/server.go b/pkg/cmd/server.go deleted file mode 100644 index 246d104..0000000 --- a/pkg/cmd/server.go +++ /dev/null @@ -1,53 +0,0 @@ -/* -Copyright © 2020 NAME HERE - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ -package cmd - -import ( - "fmt" - - "github.com/desmo999r/formolcli/pkg/server" - "github.com/spf13/cobra" -) - -// serverCmd represents the server command -var serverCmd = &cobra.Command{ - Use: "server", - Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples -and usage of using your command. For example: - -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, - Run: func(cmd *cobra.Command, args []string) { - fmt.Println("server called") - server.Server() - }, -} - -func init() { - createCmd.AddCommand(serverCmd) - - // Here you will define your flags and configuration settings. - - // Cobra supports Persistent Flags which will work for this command - // and all subcommands, e.g.: - // serverCmd.PersistentFlags().String("foo", "", "A help for foo") - - // Cobra supports local flags which will only run when this command - // is called directly, e.g.: - // serverCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") -} diff --git a/pkg/cmd/snapshot.go b/pkg/cmd/snapshot.go index ea4ea2c..0cadb76 100644 --- a/pkg/cmd/snapshot.go +++ b/pkg/cmd/snapshot.go @@ -17,7 +17,7 @@ package cmd import ( "os" - "github.com/desmo999r/formolcli/pkg/backup" + "github.com/desmo999r/formolcli/pkg/restic" "github.com/spf13/cobra" ) @@ -25,32 +25,22 @@ import ( var snapshotCmd = &cobra.Command{ Use: "snapshot", Short: "A brief description of your command", - Long: `A longer description that spans multiple lines and likely contains examples -and usage of using your command. For example: +} -Cobra is a CLI library for Go that empowers applications. -This application is a tool to generate the needed files -to quickly create a Cobra application.`, +var snapshotDeleteCmd = &cobra.Command{ + Use: "delete", + Short: "delete a snapshot", Run: func(cmd *cobra.Command, args []string) { - snapshot, _ := cmd.Flags().GetString("snapshot") - if err := backup.DeleteSnapshot(snapshot); err != nil { + snapshot, _ := cmd.Flags().GetString("snapshot-id") + if err := restic.DeleteSnapshot(snapshot); err != nil { os.Exit(1) } }, } func init() { - deleteCmd.AddCommand(snapshotCmd) - - // Here you will define your flags and configuration settings. - - // Cobra supports Persistent Flags which will work for this command - // and all subcommands, e.g.: - // snapshotCmd.PersistentFlags().String("foo", "", "A help for foo") - - // Cobra supports local flags which will only run when this command - // is called directly, e.g.: - // snapshotCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") - snapshotCmd.Flags().String("snapshot", "", "The snapshot to delete") - snapshotCmd.MarkFlagRequired("snapshot") + rootCmd.AddCommand(snapshotCmd) + snapshotCmd.AddCommand(snapshotDeleteCmd) + snapshotDeleteCmd.Flags().String("snapshot-id", "", "The snapshot to delete") + snapshotDeleteCmd.MarkFlagRequired("snapshot-id") } diff --git a/pkg/cmd/target.go b/pkg/cmd/target.go new file mode 100644 index 0000000..f7a63c7 --- /dev/null +++ b/pkg/cmd/target.go @@ -0,0 +1,44 @@ +/* +Copyright © 2021 NAME HERE + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package cmd + +import ( + "fmt" + + formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1" + "github.com/desmo999r/formolcli/pkg/session" + "github.com/spf13/cobra" +) + +// targetCmd represents the target command +var targetFinalizeCmd = &cobra.Command{ + Use: "finalize", + Short: "Update the session target status", + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("target called") + session.RestoreSessionUpdateTargetStatus(formolv1alpha1.Success) + }, +} + +var targetCmd = &cobra.Command{ + Use: "target", + Short: "A brief description of your command", +} + +func init() { + rootCmd.AddCommand(targetCmd) + targetCmd.AddCommand(targetFinalizeCmd) +} diff --git a/pkg/cmd/volume.go b/pkg/cmd/volume.go index 3209824..b2a70ff 100644 --- a/pkg/cmd/volume.go +++ b/pkg/cmd/volume.go @@ -19,13 +19,36 @@ import ( "os" "github.com/desmo999r/formolcli/pkg/backup" + "github.com/desmo999r/formolcli/pkg/restore" "github.com/spf13/cobra" ) -// pvcCmd represents the pvc command +var volumeRestoreCmd = &cobra.Command{ + Use: "restore", + Short: "restore a volume", + Run: func(cmd *cobra.Command, args []string) { + snapshotId, _ := cmd.Flags().GetString("snapshot-id") + if err := restore.RestoreVolume(snapshotId); err != nil { + os.Exit(1) + } + }, +} + +var volumeBackupCmd = &cobra.Command{ + Use: "backup", + Short: "backup a volume", + Run: func(cmd *cobra.Command, args []string) { + paths, _ := cmd.Flags().GetStringSlice("path") + tag, _ := cmd.Flags().GetString("tag") + if err := backup.BackupVolume(tag, paths); err != nil { + os.Exit(1) + } + }, +} + var volumeCmd = &cobra.Command{ Use: "volume", - Short: "A brief description of your command", + Short: "volume actions", Long: `A longer description that spans multiple lines and likely contains examples and usage of using your command. For example: @@ -42,18 +65,13 @@ to quickly create a Cobra application.`, } func init() { - backupCmd.AddCommand(volumeCmd) + rootCmd.AddCommand(volumeCmd) + volumeCmd.AddCommand(volumeBackupCmd) + volumeCmd.AddCommand(volumeRestoreCmd) - // Here you will define your flags and configuration settings. - - // Cobra supports Persistent Flags which will work for this command - // and all subcommands, e.g.: - // pvcCmd.PersistentFlags().String("foo", "", "A help for foo") - - // Cobra supports local flags which will only run when this command - // is called directly, e.g.: - // pvcCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") - volumeCmd.Flags().StringSlice("path", nil, "Path to the data to backup") - volumeCmd.Flags().String("tag", "", "Tag associated to the backup") - volumeCmd.MarkFlagRequired("path") + volumeBackupCmd.Flags().StringSlice("path", nil, "Path to the data to backup") + volumeBackupCmd.Flags().String("tag", "", "Tag associated to the backup") + volumeBackupCmd.MarkFlagRequired("path") + volumeRestoreCmd.Flags().String("snapshot-id", "", "snapshot id associated to the backup") + volumeRestoreCmd.MarkFlagRequired("snapshot-id") } diff --git a/pkg/controllers/backupsession_controller.go b/pkg/controllers/backupsession_controller.go index 6a62d40..8413f22 100644 --- a/pkg/controllers/backupsession_controller.go +++ b/pkg/controllers/backupsession_controller.go @@ -12,7 +12,7 @@ import ( "time" formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1" - "github.com/desmo999r/formolcli/pkg/backup" + "github.com/desmo999r/formolcli/pkg/restic" formolcliutils "github.com/desmo999r/formolcli/pkg/utils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -25,6 +25,7 @@ type BackupSessionReconciler struct { } func (r *BackupSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { + time.Sleep(2 * time.Second) ctx := context.Background() log := r.Log.WithValues("backupsession", req.NamespacedName) @@ -69,14 +70,14 @@ func (r *BackupSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro log.V(0).Info("Running session. Do the backup") result := formolv1alpha1.Success status.StartTime = &metav1.Time{Time: time.Now()} - output, err := backup.BackupPaths(backupSession.Name, target.Paths) + output, err := restic.BackupPaths(backupSession.Name, target.Paths) if err != nil { log.Error(err, "unable to backup deployment", "output", string(output)) result = formolv1alpha1.Failure } else { - snapshotId, duration := backup.GetBackupResults(output) + snapshotId := restic.GetBackupResults(output) backupSession.Status.Targets[i].SnapshotId = snapshotId - backupSession.Status.Targets[i].Duration = &metav1.Duration{Duration: duration} + backupSession.Status.Targets[i].Duration = &metav1.Duration{Duration: time.Now().Sub(backupSession.Status.Targets[i].StartTime.Time)} } backupSession.Status.Targets[i].SessionState = result log.V(1).Info("current backupSession status", "status", backupSession.Status) diff --git a/pkg/controllers/restoresession_controller.go b/pkg/controllers/restoresession_controller.go index 48a2d8d..231040f 100644 --- a/pkg/controllers/restoresession_controller.go +++ b/pkg/controllers/restoresession_controller.go @@ -2,26 +2,33 @@ package controllers import ( "context" + "fmt" formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1" + "github.com/desmo999r/formolcli/pkg/restic" + formolcliutils "github.com/desmo999r/formolcli/pkg/utils" "github.com/go-logr/logr" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "os" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/predicate" + "strings" + "time" ) type RestoreSessionReconciler struct { client.Client - Log logr.Logger - Scheme *runtime.Scheme - RestoreSession *formolv1alpha1.RestoreSession - BackupSession *formolv1alpha1.BackupSession - BackupConfiguration *formolv1alpha1.BackupConfiguration + Log logr.Logger + Scheme *runtime.Scheme + RestoreSession *formolv1alpha1.RestoreSession + BackupSession *formolv1alpha1.BackupSession + BackupConf *formolv1alpha1.BackupConfiguration } func (r *RestoreSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { + time.Sleep(2 * time.Second) ctx := context.Background() log := r.Log.WithValues("restoresession", req.NamespacedName) r.RestoreSession = &formolv1alpha1.RestoreSession{} @@ -30,24 +37,75 @@ func (r *RestoreSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, err return ctrl.Result{}, client.IgnoreNotFound(err) } r.BackupSession = &formolv1alpha1.BackupSession{} - if err := r.Get(ctx, client.ObjectKey{Namespace: r.RestoreSession.Spec.BackupSessionRef.Namespace, - Name: r.RestoreSession.Spec.BackupSessionRef.Name}, r.BackupSession); err != nil { - log.Error(err, "unable to get backupsession") + if err := r.Get(ctx, client.ObjectKey{ + Namespace: r.RestoreSession.Spec.BackupSessionRef.Namespace, + Name: r.RestoreSession.Spec.BackupSessionRef.Name}, r.BackupSession); err != nil { + log.Error(err, "unable to get backupsession", "namespace", r.RestoreSession.Spec.BackupSessionRef.Namespace, "name", r.RestoreSession.Spec.BackupSessionRef.Name) return ctrl.Result{}, client.IgnoreNotFound(err) } - r.BackupConfiguration = &formolv1alpha1.BackupConfiguration{} + r.BackupConf = &formolv1alpha1.BackupConfiguration{} if err := r.Get(ctx, client.ObjectKey{Namespace: r.BackupSession.Namespace, - Name: r.BackupSession.Spec.Ref.Name}, r.BackupConfiguration); err != nil { + Name: r.BackupSession.Spec.Ref.Name}, r.BackupConf); err != nil { log.Error(err, "unable to get backupconfiguration") return ctrl.Result{}, client.IgnoreNotFound(err) } deploymentName := os.Getenv("POD_DEPLOYMENT") - for i := len(r.BackupConfiguration.Spec.Targets) - 1; i >= 0; i-- { - target := r.BackupConfiguration.Spec.Targets[i] + // we go reverse order compared to the backup session + for i := len(r.BackupConf.Spec.Targets) - 1; i >= 0; i-- { + target := r.BackupConf.Spec.Targets[i] switch target.Kind { case "Deployment": if target.Name == deploymentName { - log.V(0).Info("It's for us!", "target", target.Name) + for i, status := range r.RestoreSession.Status.Targets { + if status.Name == target.Name { + log.V(0).Info("It's for us!", "target", target.Name) + switch status.SessionState { + case formolv1alpha1.New: + log.V(0).Info("New session, run the beforeBackup hooks if any") + result := formolv1alpha1.Running + if err := formolcliutils.RunBeforeBackup(target); err != nil { + result = formolv1alpha1.Failure + } + r.RestoreSession.Status.Targets[i].SessionState = result + if err := r.Status().Update(ctx, r.RestoreSession); err != nil { + log.Error(err, " unable to update restoresession status") + return ctrl.Result{}, err + } + case formolv1alpha1.Running: + log.V(0).Info("Running session. Do the restore") + status.StartTime = &metav1.Time{Time: time.Now()} + result := formolv1alpha1.Success + + repo := &formolv1alpha1.Repo{} + if err := r.Get(ctx, client.ObjectKey{ + Namespace: r.BackupConf.Namespace, + Name: r.BackupConf.Spec.Repository.Name, + }, repo); err != nil { + log.Error(err, "unable to get Repo from BackupConf") + return ctrl.Result{}, err + } + url := fmt.Sprintf("s3:http://%s/%s/%s-%s", repo.Spec.Backend.S3.Server, repo.Spec.Backend.S3.Bucket, strings.ToUpper(r.BackupConf.Namespace), strings.ToLower(r.BackupConf.Name)) + output, err := restic.RestorePaths(url, r.BackupSession.Status.Targets[i].SnapshotId) + if err != nil { + log.Error(err, "unable to restore deployment", "output", string(output)) + result = formolv1alpha1.Failure + } else { + duration := restic.GetRestoreResults(output) + r.RestoreSession.Status.Targets[i].Duration = &metav1.Duration{Duration: duration} + } + r.RestoreSession.Status.Targets[i].SessionState = result + log.V(1).Info("current restoresession status", "status", result) + if err := r.Status().Update(ctx, r.RestoreSession); err != nil { + log.Error(err, "unable to update restoresession status") + return ctrl.Result{}, err + } + + case formolv1alpha1.Failure, formolv1alpha1.Success: + log.V(0).Info("Restore is over, run afterBackup hooks if any") + formolcliutils.RunAfterBackup(target) + } + } + } } } diff --git a/pkg/restic/root.go b/pkg/restic/root.go new file mode 100644 index 0000000..df6aec1 --- /dev/null +++ b/pkg/restic/root.go @@ -0,0 +1,109 @@ +package restic + +import ( + "bufio" + "bytes" + "encoding/json" + "github.com/go-logr/logr" + "github.com/go-logr/zapr" + "go.uber.org/zap" + "os" + "os/exec" + "time" +) + +var ( + repository string + passwordFile string + aws_access_key_id string + aws_secret_access_key string + resticExec = "/usr/bin/restic" + logger logr.Logger +) + +func init() { + zapLog, _ := zap.NewDevelopment() + logger = zapr.NewLogger(zapLog) + repository = os.Getenv("RESTIC_REPOSITORY") + passwordFile = os.Getenv("RESTIC_PASSWORD") + aws_access_key_id = os.Getenv("AWS_ACCESS_KEY_ID") + aws_secret_access_key = os.Getenv("AWS_SECRET_ACCESS_KEY") +} + +func checkRepo(repo string) error { + log := logger.WithValues("backup-checkrepo", repo) + cmd := exec.Command(resticExec, "unlock", "-r", repo) + if err := cmd.Run(); err != nil { + log.Error(err, "unable to unlock repo", "repo", repo) + } + cmd = exec.Command(resticExec, "check", "-r", repo) + output, err := cmd.CombinedOutput() + log.V(1).Info("restic check output", "output", string(output)) + if err != nil { + log.V(0).Info("initializing new repo", "repo", repo) + cmd = exec.Command(resticExec, "init", "-r", repo) + output, err = cmd.CombinedOutput() + log.V(1).Info("restic init repo", "output", string(output)) + if err != nil { + log.Error(err, "something went wrong during repo init") + return err + } + } + return err +} + +func GetBackupResults(output []byte) (snapshotId string) { + log := logger.WithName("backup-getbackupresults") + scanner := bufio.NewScanner(bytes.NewReader(output)) + var dat map[string]interface{} + for scanner.Scan() { + if err := json.Unmarshal(scanner.Bytes(), &dat); err != nil { + log.Error(err, "unable to unmarshal json", "msg", string(scanner.Bytes())) + continue + } + log.V(1).Info("message on stdout", "stdout", dat) + if message_type, ok := dat["message_type"]; ok && message_type == "summary" { + snapshotId = dat["snapshot_id"].(string) + //duration = time.Duration(dat["total_duration"].(float64)*1000) * time.Millisecond + } + } + return +} + +func GetRestoreResults(output []byte) time.Duration { + return 0 * time.Millisecond +} + +func BackupPaths(tag string, paths []string) ([]byte, error) { + log := logger.WithName("backup-deployment") + if err := checkRepo(repository); err != nil { + log.Error(err, "unable to setup newrepo", "newrepo", repository) + return []byte{}, err + } + cmd := exec.Command(resticExec, append([]string{"backup", "--json", "--tag", tag, "-r", repository}, paths...)...) + output, err := cmd.CombinedOutput() + return output, err +} + +func RestorePaths(repository string, snapshotId string) ([]byte, error) { + log := logger.WithName("restore-deployment") + if err := checkRepo(repository); err != nil { + log.Error(err, "unable to setup repo", "repo", repository) + return []byte{}, err + } + cmd := exec.Command(resticExec, "restore", "-r", repository, snapshotId, "--target", "/") + return cmd.CombinedOutput() +} + +func DeleteSnapshot(snapshot string) error { + log := logger.WithValues("delete-snapshot", snapshot) + cmd := exec.Command(resticExec, "forget", "-r", repository, "--prune", snapshot) + log.V(0).Info("deleting snapshot", "snapshot", snapshot) + output, err := cmd.CombinedOutput() + log.V(1).Info("delete snapshot output", "output", string(output)) + if err != nil { + log.Error(err, "unable to delete the snapshot") + return err + } + return nil +} diff --git a/pkg/restore/root.go b/pkg/restore/root.go new file mode 100644 index 0000000..f597ce9 --- /dev/null +++ b/pkg/restore/root.go @@ -0,0 +1,56 @@ +package restore + +import ( + "fmt" + formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1" + "github.com/desmo999r/formolcli/pkg/restic" + "github.com/desmo999r/formolcli/pkg/session" + "github.com/go-logr/logr" + "github.com/go-logr/zapr" + "go.uber.org/zap" + "io/ioutil" + "os" + "os/exec" +) + +var ( + pg_restoreExec = "/usr/bin/pg_restore" + logger logr.Logger +) + +func init() { + zapLog, _ := zap.NewDevelopment() + logger = zapr.NewLogger(zapLog) +} + +func RestoreVolume(snapshotId string) error { + log := logger.WithName("restore-volume") + state := formolv1alpha1.Success + repository := os.Getenv("RESTIC_REPOSITORY") + output, err := restic.RestorePaths(repository, snapshotId) + if err != nil { + log.Error(err, "unable to restore volume", "output", string(output)) + state = formolv1alpha1.Failure + } + session.RestoreSessionUpdateTargetStatus(state) + return err +} + +func RestorePostgres(file string, hostname string, database string, username string, password string) error { + log := logger.WithName("restore-postgres") + pgpass := []byte(fmt.Sprintf("%s:*:%s:%s:%s", hostname, database, username, password)) + if err := ioutil.WriteFile("/output/.pgpass", pgpass, 0600); err != nil { + log.Error(err, "unable to write password to /output/.pgpass") + return err + } + defer os.Remove("/output/.pgpass") + cmd := exec.Command(pg_restoreExec, "--clean", "--create", "--file", file, "--host", hostname, "--dbname", database, "--username", username, "--no-password") + cmd.Env = append(os.Environ(), "PGPASSFILE=/output/.pgpass") + output, err := cmd.CombinedOutput() + log.V(1).Info("postgres restore output", "output", string(output)) + if err != nil { + log.Error(err, "something went wrong during the restore") + return err + } + return nil +} diff --git a/pkg/server/root.go b/pkg/server/root.go index 1635e2f..5173cdf 100644 --- a/pkg/server/root.go +++ b/pkg/server/root.go @@ -39,7 +39,7 @@ func Server() { Port: 9443, LeaderElection: enableLeaderElection, LeaderElectionID: "12345.desmojim.fr", - Namespace: os.Getenv("POD_NAMESPACE"), + //Namespace: os.Getenv("POD_NAMESPACE"), }) if err != nil { setupLog.Error(err, "unable to create manager") @@ -55,6 +55,15 @@ func Server() { os.Exit(1) } + if err = (&controllers.RestoreSessionReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("RestoreSession"), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "RestoreSession") + os.Exit(1) + } + setupLog.Info("starting manager") if err = mgr.Start(ctrl.SetupSignalHandler()); err != nil { setupLog.Error(err, "problem running the manager") diff --git a/pkg/backupsession/root.go b/pkg/session/root.go similarity index 71% rename from pkg/backupsession/root.go rename to pkg/session/root.go index 1c1f1b7..d35ae78 100644 --- a/pkg/backupsession/root.go +++ b/pkg/session/root.go @@ -1,4 +1,4 @@ -package backupsession +package session import ( "context" @@ -49,7 +49,7 @@ func init() { } } -func BackupSessionUpdateStatus(state formolv1alpha1.SessionState, snapshotId string, duration time.Duration) error { +func BackupSessionUpdateTargetStatus(state formolv1alpha1.SessionState, snapshotId string) error { log := logger.WithName("BackupSessionUpdateStatus") targetName := os.Getenv("TARGET_NAME") backupSession := &formolv1alpha1.BackupSession{} @@ -64,7 +64,7 @@ func BackupSessionUpdateStatus(state formolv1alpha1.SessionState, snapshotId str if target.Name == targetName { backupSession.Status.Targets[i].SessionState = state backupSession.Status.Targets[i].SnapshotId = snapshotId - backupSession.Status.Targets[i].Duration = &metav1.Duration{Duration: duration} + backupSession.Status.Targets[i].Duration = &metav1.Duration{Duration: time.Now().Sub(backupSession.Status.Targets[i].StartTime.Time)} } } @@ -75,6 +75,31 @@ func BackupSessionUpdateStatus(state formolv1alpha1.SessionState, snapshotId str return nil } +func RestoreSessionUpdateTargetStatus(state formolv1alpha1.SessionState) error { + log := logger.WithName("RestoreSessionUpdateStatus") + targetName := os.Getenv("TARGET_NAME") + restoreSession := &formolv1alpha1.RestoreSession{} + if err := cl.Get(context.Background(), client.ObjectKey{ + Namespace: os.Getenv("RESTORESESSION_NAMESPACE"), + Name: os.Getenv("RESTORESESSION_NAME"), + }, restoreSession); err != nil { + log.Error(err, "unable to get backupsession", "RESTORESESSION_NAME", os.Getenv("RESTORESESSION_NAME"), "RESTORESESSION_NAMESPACE", os.Getenv("RESTORESESSION_NAMESPACE")) + return err + } + for i, target := range restoreSession.Status.Targets { + if target.Name == targetName { + restoreSession.Status.Targets[i].SessionState = state + restoreSession.Status.Targets[i].Duration = &metav1.Duration{Duration: time.Now().Sub(restoreSession.Status.Targets[i].StartTime.Time)} + } + } + + if err := cl.Status().Update(context.Background(), restoreSession); err != nil { + log.Error(err, "unable to update restoresession status", "restoresession", restoreSession) + return err + } + return nil +} + func CreateBackupSession(name string, namespace string) { log := logger.WithName("CreateBackupSession") log.V(0).Info("CreateBackupSession called")