Compare commits

...

16 Commits

Author SHA1 Message Date
956c726a63 Skip the restore if it has already been done 2021-02-21 14:27:54 +01:00
b7468cbe2b Don't need to pass repo as arg anymore 2021-02-17 22:15:21 +01:00
7f3ee66b1f cleaned volume command 2021-02-17 22:15:21 +01:00
bc36f5a2b7 fine tuning and logging 2021-02-17 22:15:21 +01:00
8e138819a6 Use binary store for postgres backups 2021-02-17 22:15:21 +01:00
147b70a26d restoresession finalize 2021-02-17 22:15:21 +01:00
697c09c2c5 restore functions 2021-02-17 22:15:21 +01:00
cce762cb78 reworked snapshot 2021-02-17 22:15:21 +01:00
fbdc69f168 reworked volume 2021-02-17 22:15:21 +01:00
5b1a5827f6 reworked postgres 2021-02-17 22:15:21 +01:00
680c23460b reworked backupsession 2021-02-17 22:15:21 +01:00
022180febb removed commands 2021-02-17 22:15:21 +01:00
827536c1c7 Restic operations in restic module, session in session module 2021-02-17 22:15:21 +01:00
b52d876815 replace github.com/desmo999r/formol => /home/jandre/devel/golang/formol 2021-02-17 22:15:21 +01:00
93710e9341 renamed BackupState into SessionState so it can be used for the restore as well
renamed BackupState into SessionState so it can be used for the restore as well
2021-02-16 21:53:34 +01:00
a8faed3306 Get the deployment name from an ENV var instead of all that mess 2021-02-16 21:53:23 +01:00
17 changed files with 406 additions and 544 deletions

2
go.mod
View File

@ -17,3 +17,5 @@ require (
k8s.io/client-go v0.18.6
sigs.k8s.io/controller-runtime v0.6.4
)
replace github.com/desmo999r/formol => /home/jandre/devel/golang/formol

View File

@ -1,93 +1,40 @@
package backup
import (
"bufio"
"bytes"
"encoding/json"
"fmt"
formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1"
"github.com/desmo999r/formolcli/pkg/backupsession"
"github.com/desmo999r/formolcli/pkg/restic"
"github.com/desmo999r/formolcli/pkg/session"
"github.com/go-logr/logr"
"github.com/go-logr/zapr"
"go.uber.org/zap"
"io/ioutil"
"os"
"os/exec"
"time"
)
var (
repository string
passwordFile string
aws_access_key_id string
aws_secret_access_key string
resticExec = "/usr/bin/restic"
pg_dumpExec = "/usr/bin/pg_dump"
logger logr.Logger
pg_dumpExec = "/usr/bin/pg_dump"
logger logr.Logger
)
func init() {
zapLog, _ := zap.NewDevelopment()
logger = zapr.NewLogger(zapLog)
repository = os.Getenv("RESTIC_REPOSITORY")
passwordFile = os.Getenv("RESTIC_PASSWORD")
aws_access_key_id = os.Getenv("AWS_ACCESS_KEY_ID")
aws_secret_access_key = os.Getenv("AWS_SECRET_ACCESS_KEY")
}
func checkRepo(repo string) error {
log := logger.WithValues("backup-checkrepo", repo)
cmd := exec.Command(resticExec, "unlock", "-r", repo)
if err := cmd.Run(); err != nil {
log.Error(err, "unable to unlock repo", "repo", repo)
}
cmd = exec.Command(resticExec, "check", "-r", repo)
output, err := cmd.CombinedOutput()
log.V(1).Info("restic check output", "output", string(output))
if err != nil {
log.V(0).Info("initializing new repo", "repo", repo)
cmd = exec.Command(resticExec, "init", "-r", repo)
output, err = cmd.CombinedOutput()
log.V(1).Info("restic init repo", "output", string(output))
if err != nil {
log.Error(err, "something went wrong during repo init")
return err
}
}
return err
}
func GetBackupResults(output []byte) (snapshotId string, duration time.Duration) {
log := logger.WithName("backup-getbackupresults")
scanner := bufio.NewScanner(bytes.NewReader(output))
var dat map[string]interface{}
for scanner.Scan() {
if err := json.Unmarshal(scanner.Bytes(), &dat); err != nil {
log.Error(err, "unable to unmarshal json", "msg", string(scanner.Bytes()))
continue
}
log.V(1).Info("message on stdout", "stdout", dat)
if message_type, ok := dat["message_type"]; ok && message_type == "summary" {
snapshotId = dat["snapshot_id"].(string)
duration = time.Duration(dat["total_duration"].(float64)*1000) * time.Millisecond
}
}
return
}
func BackupVolume(tag string, paths []string) error {
log := logger.WithName("backup-volume")
state := formolv1alpha1.Success
output, err := BackupPaths(tag, paths)
output, err := restic.BackupPaths(tag, paths)
var snapshotId string
var duration time.Duration
if err != nil {
log.Error(err, "unable to backup volume", "output", string(output))
state = formolv1alpha1.Failure
} else {
snapshotId, duration = GetBackupResults(output)
snapshotId = restic.GetBackupResults(output)
}
backupsession.BackupSessionUpdateStatus(state, snapshotId, duration)
session.BackupSessionUpdateTargetStatus(state, snapshotId)
return err
}
@ -99,7 +46,7 @@ func BackupPostgres(file string, hostname string, database string, username stri
return err
}
defer os.Remove("/output/.pgpass")
cmd := exec.Command(pg_dumpExec, "--clean", "--create", "--file", file, "--host", hostname, "--dbname", database, "--username", username, "--no-password")
cmd := exec.Command(pg_dumpExec, "--format=custom", "--clean", "--create", "--file", file, "--host", hostname, "--dbname", database, "--username", username, "--no-password")
cmd.Env = append(os.Environ(), "PGPASSFILE=/output/.pgpass")
output, err := cmd.CombinedOutput()
log.V(1).Info("postgres backup output", "output", string(output))
@ -109,27 +56,3 @@ func BackupPostgres(file string, hostname string, database string, username stri
}
return nil
}
func BackupPaths(tag string, paths []string) ([]byte, error) {
log := logger.WithName("backup-deployment")
if err := checkRepo(repository); err != nil {
log.Error(err, "unable to setup newrepo", "newrepo", repository)
return []byte{}, err
}
cmd := exec.Command(resticExec, append([]string{"backup", "--json", "--tag", tag, "-r", repository}, paths...)...)
output, err := cmd.CombinedOutput()
return output, err
}
func DeleteSnapshot(snapshot string) error {
log := logger.WithValues("delete-snapshot", snapshot)
cmd := exec.Command(resticExec, "forget", "-r", repository, "--prune", snapshot)
log.V(0).Info("deleting snapshot", "snapshot", snapshot)
output, err := cmd.CombinedOutput()
log.V(1).Info("delete snapshot output", "output", string(output))
if err != nil {
log.Error(err, "unable to delete the snapshot")
return err
}
return nil
}

View File

@ -1,51 +0,0 @@
/*
Copyright © 2020 NAME HERE <EMAIL ADDRESS>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
"github.com/spf13/cobra"
)
// backupCmd represents the backup command
var backupCmd = &cobra.Command{
Use: "backup",
Short: "A brief description of your command",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
Run: func(cmd *cobra.Command, args []string) {
fmt.Println("backup called")
},
}
func init() {
rootCmd.AddCommand(backupCmd)
// Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// backupCmd.PersistentFlags().String("foo", "", "A help for foo")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// backupCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}

View File

@ -16,41 +16,52 @@ limitations under the License.
package cmd
import (
"github.com/desmo999r/formolcli/pkg/backupsession"
"github.com/desmo999r/formolcli/pkg/server"
"github.com/desmo999r/formolcli/pkg/session"
"github.com/spf13/cobra"
)
// backupsessionCmd represents the backupsession command
var createBackupsessionCmd = &cobra.Command{
Use: "backupsession",
var serverBackupSessionCmd = &cobra.Command{
Use: "server",
Short: "A brief description of your command",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
Run: func(cmd *cobra.Command, args []string) {
server.Server()
},
}
var createBackupSessionCmd = &cobra.Command{
Use: "create",
Short: "Creates a backupsession",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
Run: func(cmd *cobra.Command, args []string) {
name, _ := cmd.Flags().GetString("name")
namespace, _ := cmd.Flags().GetString("namespace")
backupsession.CreateBackupSession(name, namespace)
session.CreateBackupSession(name, namespace)
},
}
func init() {
createCmd.AddCommand(createBackupsessionCmd)
// Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// backupsessionCmd.PersistentFlags().String("foo", "", "A help for foo")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// backupsessionCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
createBackupsessionCmd.Flags().String("namespace", "", "The referenced BackupSessionConfiguration namespace")
createBackupsessionCmd.Flags().String("name", "", "The referenced BackupSessionConfiguration name")
createBackupsessionCmd.MarkFlagRequired("namespace")
createBackupsessionCmd.MarkFlagRequired("name")
var backupSessionCmd = &cobra.Command{
Use: "backupsession",
Short: "backupsession related commands",
}
func init() {
rootCmd.AddCommand(backupSessionCmd)
backupSessionCmd.AddCommand(createBackupSessionCmd)
backupSessionCmd.AddCommand(serverBackupSessionCmd)
createBackupSessionCmd.Flags().String("namespace", "", "The referenced BackupSessionConfiguration namespace")
createBackupSessionCmd.Flags().String("name", "", "The referenced BackupSessionConfiguration name")
createBackupSessionCmd.MarkFlagRequired("namespace")
createBackupSessionCmd.MarkFlagRequired("name")
}

View File

@ -1,54 +0,0 @@
/*
Copyright © 2021 NAME HERE <EMAIL ADDRESS>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
formolcliutils "github.com/desmo999r/formolcli/pkg/utils"
"github.com/spf13/cobra"
"os"
)
// commandCmd represents the command command
var commandCmd = &cobra.Command{
Use: "command [cmd] ([args]...)",
Args: cobra.MinimumNArgs(1),
Short: "Run a specific command with args",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
Run: func(cmd *cobra.Command, args []string) {
if err := formolcliutils.RunChroot(args[0], args[1:]...); err != nil {
os.Exit(1)
}
},
}
func init() {
runCmd.AddCommand(commandCmd)
// Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// commandCmd.PersistentFlags().String("foo", "", "A help for foo")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// commandCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}

View File

@ -1,51 +0,0 @@
/*
Copyright © 2020 NAME HERE <EMAIL ADDRESS>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
"github.com/spf13/cobra"
)
// createCmd represents the create command
var createCmd = &cobra.Command{
Use: "create",
Short: "A brief description of your command",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
Run: func(cmd *cobra.Command, args []string) {
fmt.Println("create called")
},
}
func init() {
rootCmd.AddCommand(createCmd)
// Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// createCmd.PersistentFlags().String("foo", "", "A help for foo")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// createCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}

View File

@ -1,51 +0,0 @@
/*
Copyright © 2020 NAME HERE <EMAIL ADDRESS>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
"github.com/spf13/cobra"
)
// deleteCmd represents the delete command
var deleteCmd = &cobra.Command{
Use: "delete",
Short: "A brief description of your command",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
Run: func(cmd *cobra.Command, args []string) {
fmt.Println("delete called")
},
}
func init() {
rootCmd.AddCommand(deleteCmd)
// Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// deleteCmd.PersistentFlags().String("foo", "", "A help for foo")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// deleteCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}

View File

@ -19,12 +19,33 @@ import (
"fmt"
"github.com/desmo999r/formolcli/pkg/backup"
"github.com/desmo999r/formolcli/pkg/restore"
"github.com/spf13/cobra"
)
// postgresCmd represents the postgres command
var postgresCmd = &cobra.Command{
Use: "postgres",
var postgresRestoreCmd = &cobra.Command{
Use: "restore",
Short: "restore a PostgreSQL database",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
Run: func(cmd *cobra.Command, args []string) {
fmt.Println("postgres called")
file, _ := cmd.Flags().GetString("file")
hostname, _ := cmd.Flags().GetString("hostname")
database, _ := cmd.Flags().GetString("database")
username, _ := cmd.Flags().GetString("username")
password, _ := cmd.Flags().GetString("password")
_ = restore.RestorePostgres(file, hostname, database, username, password)
},
}
var postgresBackupCmd = &cobra.Command{
Use: "backup",
Short: "backup a PostgreSQL database",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
@ -43,26 +64,43 @@ to quickly create a Cobra application.`,
},
}
var postgresCmd = &cobra.Command{
Use: "postgres",
Short: "postgres actions",
}
func init() {
backupCmd.AddCommand(postgresCmd)
rootCmd.AddCommand(postgresCmd)
postgresCmd.AddCommand(postgresBackupCmd)
postgresCmd.AddCommand(postgresRestoreCmd)
// Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// postgresCmd.PersistentFlags().String("foo", "", "A help for foo")
// backupPostgresCmd.PersistentFlags().String("foo", "", "A help for foo")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// postgresCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
postgresCmd.Flags().String("file", "", "The file the backup will be stored")
postgresCmd.Flags().String("hostname", "", "The postgresql server host")
postgresCmd.Flags().String("database", "", "The postgresql database")
postgresCmd.Flags().String("username", "", "The postgresql username")
postgresCmd.Flags().String("password", "", "The postgresql password")
postgresCmd.MarkFlagRequired("path")
postgresCmd.MarkFlagRequired("hostname")
postgresCmd.MarkFlagRequired("database")
postgresCmd.MarkFlagRequired("username")
postgresCmd.MarkFlagRequired("password")
// backupPostgresCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
postgresBackupCmd.Flags().String("file", "", "The file the backup will be stored")
postgresBackupCmd.Flags().String("hostname", "", "The postgresql server host")
postgresBackupCmd.Flags().String("database", "", "The postgresql database")
postgresBackupCmd.Flags().String("username", "", "The postgresql username")
postgresBackupCmd.Flags().String("password", "", "The postgresql password")
postgresBackupCmd.MarkFlagRequired("path")
postgresBackupCmd.MarkFlagRequired("hostname")
postgresBackupCmd.MarkFlagRequired("database")
postgresBackupCmd.MarkFlagRequired("username")
postgresBackupCmd.MarkFlagRequired("password")
postgresRestoreCmd.Flags().String("file", "", "The file the database will be restored from")
postgresRestoreCmd.Flags().String("hostname", "", "The postgresql server host")
postgresRestoreCmd.Flags().String("database", "", "The postgresql database")
postgresRestoreCmd.Flags().String("username", "", "The postgresql username")
postgresRestoreCmd.Flags().String("password", "", "The postgresql password")
postgresRestoreCmd.MarkFlagRequired("path")
postgresRestoreCmd.MarkFlagRequired("hostname")
postgresRestoreCmd.MarkFlagRequired("database")
postgresRestoreCmd.MarkFlagRequired("username")
postgresRestoreCmd.MarkFlagRequired("password")
}

View File

@ -1,51 +0,0 @@
/*
Copyright © 2021 NAME HERE <EMAIL ADDRESS>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
"github.com/spf13/cobra"
)
// runCmd represents the run command
var runCmd = &cobra.Command{
Use: "run",
Short: "A brief description of your command",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
Run: func(cmd *cobra.Command, args []string) {
fmt.Println("run called")
},
}
func init() {
rootCmd.AddCommand(runCmd)
// Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// runCmd.PersistentFlags().String("foo", "", "A help for foo")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// runCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}

View File

@ -1,53 +0,0 @@
/*
Copyright © 2020 NAME HERE <EMAIL ADDRESS>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
"github.com/desmo999r/formolcli/pkg/server"
"github.com/spf13/cobra"
)
// serverCmd represents the server command
var serverCmd = &cobra.Command{
Use: "server",
Short: "A brief description of your command",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
Run: func(cmd *cobra.Command, args []string) {
fmt.Println("server called")
server.Server()
},
}
func init() {
createCmd.AddCommand(serverCmd)
// Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// serverCmd.PersistentFlags().String("foo", "", "A help for foo")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// serverCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}

View File

@ -17,7 +17,7 @@ package cmd
import (
"os"
"github.com/desmo999r/formolcli/pkg/backup"
"github.com/desmo999r/formolcli/pkg/restic"
"github.com/spf13/cobra"
)
@ -25,32 +25,22 @@ import (
var snapshotCmd = &cobra.Command{
Use: "snapshot",
Short: "A brief description of your command",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
}
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
var snapshotDeleteCmd = &cobra.Command{
Use: "delete",
Short: "delete a snapshot",
Run: func(cmd *cobra.Command, args []string) {
snapshot, _ := cmd.Flags().GetString("snapshot")
if err := backup.DeleteSnapshot(snapshot); err != nil {
snapshot, _ := cmd.Flags().GetString("snapshot-id")
if err := restic.DeleteSnapshot(snapshot); err != nil {
os.Exit(1)
}
},
}
func init() {
deleteCmd.AddCommand(snapshotCmd)
// Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// snapshotCmd.PersistentFlags().String("foo", "", "A help for foo")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// snapshotCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
snapshotCmd.Flags().String("snapshot", "", "The snapshot to delete")
snapshotCmd.MarkFlagRequired("snapshot")
rootCmd.AddCommand(snapshotCmd)
snapshotCmd.AddCommand(snapshotDeleteCmd)
snapshotDeleteCmd.Flags().String("snapshot-id", "", "The snapshot to delete")
snapshotDeleteCmd.MarkFlagRequired("snapshot-id")
}

44
pkg/cmd/target.go Normal file
View File

@ -0,0 +1,44 @@
/*
Copyright © 2021 NAME HERE <EMAIL ADDRESS>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1"
"github.com/desmo999r/formolcli/pkg/session"
"github.com/spf13/cobra"
)
// targetCmd represents the target command
var targetFinalizeCmd = &cobra.Command{
Use: "finalize",
Short: "Update the session target status",
Run: func(cmd *cobra.Command, args []string) {
fmt.Println("target called")
session.RestoreSessionUpdateTargetStatus(formolv1alpha1.Success)
},
}
var targetCmd = &cobra.Command{
Use: "target",
Short: "A brief description of your command",
}
func init() {
rootCmd.AddCommand(targetCmd)
targetCmd.AddCommand(targetFinalizeCmd)
}

View File

@ -19,19 +19,24 @@ import (
"os"
"github.com/desmo999r/formolcli/pkg/backup"
"github.com/desmo999r/formolcli/pkg/restore"
"github.com/spf13/cobra"
)
// pvcCmd represents the pvc command
var volumeCmd = &cobra.Command{
Use: "volume",
Short: "A brief description of your command",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
var volumeRestoreCmd = &cobra.Command{
Use: "restore",
Short: "restore a volume",
Run: func(cmd *cobra.Command, args []string) {
snapshotId, _ := cmd.Flags().GetString("snapshot-id")
if err := restore.RestoreVolume(snapshotId); err != nil {
os.Exit(1)
}
},
}
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
var volumeBackupCmd = &cobra.Command{
Use: "backup",
Short: "backup a volume",
Run: func(cmd *cobra.Command, args []string) {
paths, _ := cmd.Flags().GetStringSlice("path")
tag, _ := cmd.Flags().GetString("tag")
@ -41,19 +46,25 @@ to quickly create a Cobra application.`,
},
}
func init() {
backupCmd.AddCommand(volumeCmd)
var volumeCmd = &cobra.Command{
Use: "volume",
Short: "volume actions",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
// Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// pvcCmd.PersistentFlags().String("foo", "", "A help for foo")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// pvcCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
volumeCmd.Flags().StringSlice("path", nil, "Path to the data to backup")
volumeCmd.Flags().String("tag", "", "Tag associated to the backup")
volumeCmd.MarkFlagRequired("path")
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
}
func init() {
rootCmd.AddCommand(volumeCmd)
volumeCmd.AddCommand(volumeBackupCmd)
volumeCmd.AddCommand(volumeRestoreCmd)
volumeBackupCmd.Flags().StringSlice("path", nil, "Path to the data to backup")
volumeBackupCmd.Flags().String("tag", "", "Tag associated to the backup")
volumeBackupCmd.MarkFlagRequired("path")
volumeRestoreCmd.Flags().String("snapshot-id", "", "snapshot id associated to the backup")
volumeRestoreCmd.MarkFlagRequired("snapshot-id")
}

View File

@ -5,71 +5,18 @@ import (
"github.com/go-logr/logr"
"k8s.io/apimachinery/pkg/runtime"
"os"
"path/filepath"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/controller-runtime/pkg/predicate"
"time"
formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1"
"github.com/desmo999r/formolcli/pkg/backup"
"github.com/desmo999r/formolcli/pkg/restic"
formolcliutils "github.com/desmo999r/formolcli/pkg/utils"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
)
var (
deploymentName = ""
)
func init() {
log := zap.New(zap.UseDevMode(true)).WithName("init")
namespace := os.Getenv("POD_NAMESPACE")
if namespace == "" {
return
}
config, err := rest.InClusterConfig()
if err != nil {
config, err = clientcmd.BuildConfigFromFlags("", filepath.Join(os.Getenv("HOME"), ".kube", "config"))
if err != nil {
log.Error(err, "unable to get config")
panic(err.Error())
}
}
clientset, err := kubernetes.NewForConfig(config)
if err != nil {
panic("unable to get clientset")
}
hostname := os.Getenv("POD_NAME")
if hostname == "" {
panic("unable to get hostname")
}
pod, err := clientset.CoreV1().Pods(namespace).Get(context.Background(), hostname, metav1.GetOptions{})
if err != nil {
log.Error(err, "unable to get pod")
panic("unable to get pod")
}
podOwner := metav1.GetControllerOf(pod)
replicasetList, err := clientset.AppsV1().ReplicaSets(namespace).List(context.Background(), metav1.ListOptions{
FieldSelector: "metadata.name=" + string(podOwner.Name),
})
if err != nil {
panic("unable to get replicaset" + err.Error())
}
for _, replicaset := range replicasetList.Items {
replicasetOwner := metav1.GetControllerOf(&replicaset)
deploymentName = replicasetOwner.Name
}
}
// BackupSessionReconciler reconciles a BackupSession object
type BackupSessionReconciler struct {
client.Client
@ -78,6 +25,7 @@ type BackupSessionReconciler struct {
}
func (r *BackupSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
time.Sleep(2 * time.Second)
ctx := context.Background()
log := r.Log.WithValues("backupsession", req.NamespacedName)
@ -96,6 +44,7 @@ func (r *BackupSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
return ctrl.Result{}, client.IgnoreNotFound(err)
}
deploymentName := os.Getenv("POD_DEPLOYMENT")
for _, target := range backupConf.Spec.Targets {
switch target.Kind {
case "Deployment":
@ -103,7 +52,7 @@ func (r *BackupSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
for i, status := range backupSession.Status.Targets {
if status.Name == target.Name {
log.V(0).Info("It's for us!", "target", target)
switch status.BackupState {
switch status.SessionState {
case formolv1alpha1.New:
// TODO: Run beforeBackup
log.V(0).Info("New session, run the beforeBackup hooks if any")
@ -111,7 +60,7 @@ func (r *BackupSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
if err := formolcliutils.RunBeforeBackup(target); err != nil {
result = formolv1alpha1.Failure
}
backupSession.Status.Targets[i].BackupState = result
backupSession.Status.Targets[i].SessionState = result
log.V(1).Info("current backupSession status", "status", backupSession.Status)
if err := r.Status().Update(ctx, backupSession); err != nil {
log.Error(err, "unable to update backupsession status")
@ -121,16 +70,16 @@ func (r *BackupSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
log.V(0).Info("Running session. Do the backup")
result := formolv1alpha1.Success
status.StartTime = &metav1.Time{Time: time.Now()}
output, err := backup.BackupPaths(backupSession.Name, target.Paths)
output, err := restic.BackupPaths(backupSession.Name, target.Paths)
if err != nil {
log.Error(err, "unable to backup deployment", "output", string(output))
result = formolv1alpha1.Failure
} else {
snapshotId, duration := backup.GetBackupResults(output)
snapshotId := restic.GetBackupResults(output)
backupSession.Status.Targets[i].SnapshotId = snapshotId
backupSession.Status.Targets[i].Duration = &metav1.Duration{Duration: duration}
backupSession.Status.Targets[i].Duration = &metav1.Duration{Duration: time.Now().Sub(backupSession.Status.Targets[i].StartTime.Time)}
}
backupSession.Status.Targets[i].BackupState = result
backupSession.Status.Targets[i].SessionState = result
log.V(1).Info("current backupSession status", "status", backupSession.Status)
if err := r.Status().Update(ctx, backupSession); err != nil {
log.Error(err, "unable to update backupsession status")
@ -152,10 +101,9 @@ func (r *BackupSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
func (r *BackupSessionReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&formolv1alpha1.BackupSession{}).
//WithEventFilter(predicate.GenerationChangedPredicate{}). // Don't reconcile when status gets updated
WithEventFilter(predicate.Funcs{
CreateFunc: func(e event.CreateEvent) bool { return false },
DeleteFunc: func(e event.DeleteEvent) bool { return false },
}). // Don't reconcile when status gets updated
}).
Complete(r)
}

111
pkg/restic/root.go Normal file
View File

@ -0,0 +1,111 @@
package restic
import (
"bufio"
"bytes"
"encoding/json"
"github.com/go-logr/logr"
"github.com/go-logr/zapr"
"go.uber.org/zap"
"os"
"os/exec"
"time"
)
var (
repository string
passwordFile string
aws_access_key_id string
aws_secret_access_key string
resticExec = "/usr/bin/restic"
logger logr.Logger
)
func init() {
zapLog, _ := zap.NewDevelopment()
logger = zapr.NewLogger(zapLog)
repository = os.Getenv("RESTIC_REPOSITORY")
passwordFile = os.Getenv("RESTIC_PASSWORD")
aws_access_key_id = os.Getenv("AWS_ACCESS_KEY_ID")
aws_secret_access_key = os.Getenv("AWS_SECRET_ACCESS_KEY")
}
func checkRepo(repo string) error {
log := logger.WithValues("backup-checkrepo", repo)
cmd := exec.Command(resticExec, "unlock", "-r", repo)
if err := cmd.Run(); err != nil {
log.Error(err, "unable to unlock repo", "repo", repo)
}
cmd = exec.Command(resticExec, "check", "-r", repo)
output, err := cmd.CombinedOutput()
log.V(1).Info("restic check output", "output", string(output))
if err != nil {
log.V(0).Info("initializing new repo", "repo", repo)
cmd = exec.Command(resticExec, "init", "-r", repo)
output, err = cmd.CombinedOutput()
log.V(1).Info("restic init repo", "output", string(output))
if err != nil {
log.Error(err, "something went wrong during repo init")
return err
}
}
return err
}
func GetBackupResults(output []byte) (snapshotId string) {
log := logger.WithName("backup-getbackupresults")
scanner := bufio.NewScanner(bytes.NewReader(output))
var dat map[string]interface{}
for scanner.Scan() {
if err := json.Unmarshal(scanner.Bytes(), &dat); err != nil {
log.Error(err, "unable to unmarshal json", "msg", string(scanner.Bytes()))
continue
}
log.V(1).Info("message on stdout", "stdout", dat)
if message_type, ok := dat["message_type"]; ok && message_type == "summary" {
snapshotId = dat["snapshot_id"].(string)
//duration = time.Duration(dat["total_duration"].(float64)*1000) * time.Millisecond
}
}
return
}
func GetRestoreResults(output []byte) time.Duration {
return 0 * time.Millisecond
}
func BackupPaths(tag string, paths []string) ([]byte, error) {
log := logger.WithName("backup-deployment")
if err := checkRepo(repository); err != nil {
log.Error(err, "unable to setup newrepo", "newrepo", repository)
return []byte{}, err
}
cmd := exec.Command(resticExec, append([]string{"backup", "--json", "--tag", tag, "-r", repository}, paths...)...)
output, err := cmd.CombinedOutput()
return output, err
}
func RestorePaths(snapshotId string) ([]byte, error) {
log := logger.WithName("restore-deployment")
if err := checkRepo(repository); err != nil {
log.Error(err, "unable to setup repo", "repo", repository)
return []byte{}, err
}
cmd := exec.Command(resticExec, "restore", "-r", repository, snapshotId, "--target", "/")
output, err := cmd.CombinedOutput()
log.V(1).Info("restic restore output", "output", string(output))
return output, err
}
func DeleteSnapshot(snapshot string) error {
log := logger.WithValues("delete-snapshot", snapshot)
cmd := exec.Command(resticExec, "forget", "-r", repository, "--prune", snapshot)
log.V(0).Info("deleting snapshot", "snapshot", snapshot)
output, err := cmd.CombinedOutput()
log.V(1).Info("delete snapshot output", "output", string(output))
if err != nil {
log.Error(err, "unable to delete the snapshot")
return err
}
return nil
}

61
pkg/restore/root.go Normal file
View File

@ -0,0 +1,61 @@
package restore
import (
"fmt"
formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1"
"github.com/desmo999r/formolcli/pkg/restic"
"github.com/desmo999r/formolcli/pkg/session"
"github.com/go-logr/logr"
"github.com/go-logr/zapr"
"go.uber.org/zap"
"io/ioutil"
"os"
"os/exec"
)
var (
//psqlExec = "/usr/bin/psql"
pg_restoreExec = "/usr/bin/pg_restore"
logger logr.Logger
)
func init() {
zapLog, _ := zap.NewDevelopment()
logger = zapr.NewLogger(zapLog)
}
func RestoreVolume(snapshotId string) error {
log := logger.WithName("restore-volume")
if err := session.RestoreSessionUpdateTargetStatus(formolv1alpha1.Running); err != nil {
return err
}
state := formolv1alpha1.Success
output, err := restic.RestorePaths(snapshotId)
if err != nil {
log.Error(err, "unable to restore volume", "output", string(output))
state = formolv1alpha1.Failure
}
log.V(1).Info("restic restore output", "output", string(output))
session.RestoreSessionUpdateTargetStatus(state)
return err
}
func RestorePostgres(file string, hostname string, database string, username string, password string) error {
log := logger.WithName("restore-postgres")
pgpass := []byte(fmt.Sprintf("%s:*:%s:%s:%s", hostname, database, username, password))
if err := ioutil.WriteFile("/output/.pgpass", pgpass, 0600); err != nil {
log.Error(err, "unable to write password to /output/.pgpass")
return err
}
defer os.Remove("/output/.pgpass")
//cmd := exec.Command(psqlExec, "--file", file, "--host", hostname, "--dbname", database, "--username", username, "--no-password")
cmd := exec.Command(pg_restoreExec, "--format=custom", "--clean", "--host", hostname, "--dbname", database, "--username", username, "--no-password", file)
cmd.Env = append(os.Environ(), "PGPASSFILE=/output/.pgpass")
output, err := cmd.CombinedOutput()
log.V(1).Info("postgres restore output", "output", string(output))
if err != nil {
log.Error(err, "something went wrong during the restore")
return err
}
return nil
}

View File

@ -1,7 +1,8 @@
package backupsession
package session
import (
"context"
"errors"
formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1"
"github.com/go-logr/logr"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -49,19 +50,22 @@ func init() {
}
}
func BackupSessionUpdateStatus(state formolv1alpha1.BackupState, snapshotId string, duration time.Duration) error {
func BackupSessionUpdateTargetStatus(state formolv1alpha1.SessionState, snapshotId string) error {
log := logger.WithName("BackupSessionUpdateStatus")
targetName := os.Getenv("TARGET_NAME")
backupSession := &formolv1alpha1.BackupSession{}
cl.Get(context.Background(), client.ObjectKey{
if err := cl.Get(context.Background(), client.ObjectKey{
Namespace: os.Getenv("BACKUPSESSION_NAMESPACE"),
Name: os.Getenv("BACKUPSESSION_NAME"),
}, backupSession)
}, backupSession); err != nil {
log.Error(err, "unable to get backupsession", "BACKUPSESSION_NAME", os.Getenv("BACKUPSESSION_NAME"), "BACKUPSESSION_NAMESPACE", os.Getenv("BACKUPSESSION_NAMESPACE"))
return err
}
for i, target := range backupSession.Status.Targets {
if target.Name == targetName {
backupSession.Status.Targets[i].BackupState = state
backupSession.Status.Targets[i].SessionState = state
backupSession.Status.Targets[i].SnapshotId = snapshotId
backupSession.Status.Targets[i].Duration = &metav1.Duration{Duration: duration}
backupSession.Status.Targets[i].Duration = &metav1.Duration{Duration: time.Now().Sub(backupSession.Status.Targets[i].StartTime.Time)}
}
}
@ -72,6 +76,36 @@ func BackupSessionUpdateStatus(state formolv1alpha1.BackupState, snapshotId stri
return nil
}
func RestoreSessionUpdateTargetStatus(state formolv1alpha1.SessionState) error {
log := logger.WithName("RestoreSessionUpdateStatus")
targetName := os.Getenv("TARGET_NAME")
restoreSession := &formolv1alpha1.RestoreSession{}
if err := cl.Get(context.Background(), client.ObjectKey{
Namespace: os.Getenv("RESTORESESSION_NAMESPACE"),
Name: os.Getenv("RESTORESESSION_NAME"),
}, restoreSession); err != nil {
log.Error(err, "unable to get backupsession", "RESTORESESSION_NAME", os.Getenv("RESTORESESSION_NAME"), "RESTORESESSION_NAMESPACE", os.Getenv("RESTORESESSION_NAMESPACE"))
return err
}
for i, target := range restoreSession.Status.Targets {
if target.Name == targetName {
if target.SessionState == formolv1alpha1.Success {
return errors.New("the restore has already been done. Skipping")
}
restoreSession.Status.Targets[i].SessionState = state
if state == formolv1alpha1.Success || state == formolv1alpha1.Failure {
restoreSession.Status.Targets[i].Duration = &metav1.Duration{Duration: time.Now().Sub(restoreSession.Status.Targets[i].StartTime.Time)}
}
}
}
if err := cl.Status().Update(context.Background(), restoreSession); err != nil {
log.Error(err, "unable to update restoresession status", "restoresession", restoreSession)
return err
}
return nil
}
func CreateBackupSession(name string, namespace string) {
log := logger.WithName("CreateBackupSession")
log.V(0).Info("CreateBackupSession called")