Compare commits

...

6 Commits

19 changed files with 530 additions and 537 deletions

2
go.mod
View File

@ -17,3 +17,5 @@ require (
k8s.io/client-go v0.18.6
sigs.k8s.io/controller-runtime v0.6.4
)
replace github.com/desmo999r/formol => /home/jandre/devel/golang/formol

View File

@ -1,93 +1,40 @@
package backup
import (
"bufio"
"bytes"
"encoding/json"
"fmt"
formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1"
"github.com/desmo999r/formolcli/pkg/backupsession"
"github.com/desmo999r/formolcli/pkg/restic"
"github.com/desmo999r/formolcli/pkg/session"
"github.com/go-logr/logr"
"github.com/go-logr/zapr"
"go.uber.org/zap"
"io/ioutil"
"os"
"os/exec"
"time"
)
var (
repository string
passwordFile string
aws_access_key_id string
aws_secret_access_key string
resticExec = "/usr/bin/restic"
pg_dumpExec = "/usr/bin/pg_dump"
logger logr.Logger
pg_dumpExec = "/usr/bin/pg_dump"
logger logr.Logger
)
func init() {
zapLog, _ := zap.NewDevelopment()
logger = zapr.NewLogger(zapLog)
repository = os.Getenv("RESTIC_REPOSITORY")
passwordFile = os.Getenv("RESTIC_PASSWORD")
aws_access_key_id = os.Getenv("AWS_ACCESS_KEY_ID")
aws_secret_access_key = os.Getenv("AWS_SECRET_ACCESS_KEY")
}
func checkRepo(repo string) error {
log := logger.WithValues("backup-checkrepo", repo)
cmd := exec.Command(resticExec, "unlock", "-r", repo)
if err := cmd.Run(); err != nil {
log.Error(err, "unable to unlock repo", "repo", repo)
}
cmd = exec.Command(resticExec, "check", "-r", repo)
output, err := cmd.CombinedOutput()
log.V(1).Info("restic check output", "output", string(output))
if err != nil {
log.V(0).Info("initializing new repo", "repo", repo)
cmd = exec.Command(resticExec, "init", "-r", repo)
output, err = cmd.CombinedOutput()
log.V(1).Info("restic init repo", "output", string(output))
if err != nil {
log.Error(err, "something went wrong during repo init")
return err
}
}
return err
}
func GetBackupResults(output []byte) (snapshotId string, duration time.Duration) {
log := logger.WithName("backup-getbackupresults")
scanner := bufio.NewScanner(bytes.NewReader(output))
var dat map[string]interface{}
for scanner.Scan() {
if err := json.Unmarshal(scanner.Bytes(), &dat); err != nil {
log.Error(err, "unable to unmarshal json", "msg", string(scanner.Bytes()))
continue
}
log.V(1).Info("message on stdout", "stdout", dat)
if message_type, ok := dat["message_type"]; ok && message_type == "summary" {
snapshotId = dat["snapshot_id"].(string)
duration = time.Duration(dat["total_duration"].(float64)*1000) * time.Millisecond
}
}
return
}
func BackupVolume(tag string, paths []string) error {
log := logger.WithName("backup-volume")
state := formolv1alpha1.Success
output, err := BackupPaths(tag, paths)
output, err := restic.BackupPaths(tag, paths)
var snapshotId string
var duration time.Duration
if err != nil {
log.Error(err, "unable to backup volume", "output", string(output))
state = formolv1alpha1.Failure
} else {
snapshotId, duration = GetBackupResults(output)
snapshotId = restic.GetBackupResults(output)
}
backupsession.BackupSessionUpdateStatus(state, snapshotId, duration)
session.BackupSessionUpdateTargetStatus(state, snapshotId)
return err
}
@ -99,7 +46,7 @@ func BackupPostgres(file string, hostname string, database string, username stri
return err
}
defer os.Remove("/output/.pgpass")
cmd := exec.Command(pg_dumpExec, "--clean", "--create", "--file", file, "--host", hostname, "--dbname", database, "--username", username, "--no-password")
cmd := exec.Command(pg_dumpExec, "--format=custom", "--clean", "--create", "--file", file, "--host", hostname, "--dbname", database, "--username", username, "--no-password")
cmd.Env = append(os.Environ(), "PGPASSFILE=/output/.pgpass")
output, err := cmd.CombinedOutput()
log.V(1).Info("postgres backup output", "output", string(output))
@ -109,27 +56,3 @@ func BackupPostgres(file string, hostname string, database string, username stri
}
return nil
}
func BackupPaths(tag string, paths []string) ([]byte, error) {
log := logger.WithName("backup-deployment")
if err := checkRepo(repository); err != nil {
log.Error(err, "unable to setup newrepo", "newrepo", repository)
return []byte{}, err
}
cmd := exec.Command(resticExec, append([]string{"backup", "--json", "--tag", tag, "-r", repository}, paths...)...)
output, err := cmd.CombinedOutput()
return output, err
}
func DeleteSnapshot(snapshot string) error {
log := logger.WithValues("delete-snapshot", snapshot)
cmd := exec.Command(resticExec, "forget", "-r", repository, "--prune", snapshot)
log.V(0).Info("deleting snapshot", "snapshot", snapshot)
output, err := cmd.CombinedOutput()
log.V(1).Info("delete snapshot output", "output", string(output))
if err != nil {
log.Error(err, "unable to delete the snapshot")
return err
}
return nil
}

View File

@ -1,51 +0,0 @@
/*
Copyright © 2020 NAME HERE <EMAIL ADDRESS>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
"github.com/spf13/cobra"
)
// backupCmd represents the backup command
var backupCmd = &cobra.Command{
Use: "backup",
Short: "A brief description of your command",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
Run: func(cmd *cobra.Command, args []string) {
fmt.Println("backup called")
},
}
func init() {
rootCmd.AddCommand(backupCmd)
// Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// backupCmd.PersistentFlags().String("foo", "", "A help for foo")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// backupCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}

View File

@ -16,41 +16,52 @@ limitations under the License.
package cmd
import (
"github.com/desmo999r/formolcli/pkg/backupsession"
"github.com/desmo999r/formolcli/pkg/server"
"github.com/desmo999r/formolcli/pkg/session"
"github.com/spf13/cobra"
)
// backupsessionCmd represents the backupsession command
var createBackupsessionCmd = &cobra.Command{
Use: "backupsession",
var serverBackupSessionCmd = &cobra.Command{
Use: "server",
Short: "A brief description of your command",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
Run: func(cmd *cobra.Command, args []string) {
server.Server()
},
}
var createBackupSessionCmd = &cobra.Command{
Use: "create",
Short: "Creates a backupsession",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
Run: func(cmd *cobra.Command, args []string) {
name, _ := cmd.Flags().GetString("name")
namespace, _ := cmd.Flags().GetString("namespace")
backupsession.CreateBackupSession(name, namespace)
session.CreateBackupSession(name, namespace)
},
}
func init() {
createCmd.AddCommand(createBackupsessionCmd)
// Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// backupsessionCmd.PersistentFlags().String("foo", "", "A help for foo")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// backupsessionCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
createBackupsessionCmd.Flags().String("namespace", "", "The referenced BackupSessionConfiguration namespace")
createBackupsessionCmd.Flags().String("name", "", "The referenced BackupSessionConfiguration name")
createBackupsessionCmd.MarkFlagRequired("namespace")
createBackupsessionCmd.MarkFlagRequired("name")
var backupSessionCmd = &cobra.Command{
Use: "backupsession",
Short: "backupsession related commands",
}
func init() {
rootCmd.AddCommand(backupSessionCmd)
backupSessionCmd.AddCommand(createBackupSessionCmd)
backupSessionCmd.AddCommand(serverBackupSessionCmd)
createBackupSessionCmd.Flags().String("namespace", "", "The referenced BackupSessionConfiguration namespace")
createBackupSessionCmd.Flags().String("name", "", "The referenced BackupSessionConfiguration name")
createBackupSessionCmd.MarkFlagRequired("namespace")
createBackupSessionCmd.MarkFlagRequired("name")
}

View File

@ -1,54 +0,0 @@
/*
Copyright © 2021 NAME HERE <EMAIL ADDRESS>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
formolcliutils "github.com/desmo999r/formolcli/pkg/utils"
"github.com/spf13/cobra"
"os"
)
// commandCmd represents the command command
var commandCmd = &cobra.Command{
Use: "command [cmd] ([args]...)",
Args: cobra.MinimumNArgs(1),
Short: "Run a specific command with args",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
Run: func(cmd *cobra.Command, args []string) {
if err := formolcliutils.RunChroot(args[0], args[1:]...); err != nil {
os.Exit(1)
}
},
}
func init() {
runCmd.AddCommand(commandCmd)
// Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// commandCmd.PersistentFlags().String("foo", "", "A help for foo")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// commandCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}

View File

@ -1,51 +0,0 @@
/*
Copyright © 2020 NAME HERE <EMAIL ADDRESS>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
"github.com/spf13/cobra"
)
// createCmd represents the create command
var createCmd = &cobra.Command{
Use: "create",
Short: "A brief description of your command",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
Run: func(cmd *cobra.Command, args []string) {
fmt.Println("create called")
},
}
func init() {
rootCmd.AddCommand(createCmd)
// Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// createCmd.PersistentFlags().String("foo", "", "A help for foo")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// createCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}

View File

@ -1,51 +0,0 @@
/*
Copyright © 2020 NAME HERE <EMAIL ADDRESS>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
"github.com/spf13/cobra"
)
// deleteCmd represents the delete command
var deleteCmd = &cobra.Command{
Use: "delete",
Short: "A brief description of your command",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
Run: func(cmd *cobra.Command, args []string) {
fmt.Println("delete called")
},
}
func init() {
rootCmd.AddCommand(deleteCmd)
// Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// deleteCmd.PersistentFlags().String("foo", "", "A help for foo")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// deleteCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}

View File

@ -19,12 +19,33 @@ import (
"fmt"
"github.com/desmo999r/formolcli/pkg/backup"
"github.com/desmo999r/formolcli/pkg/restore"
"github.com/spf13/cobra"
)
// postgresCmd represents the postgres command
var postgresCmd = &cobra.Command{
Use: "postgres",
var postgresRestoreCmd = &cobra.Command{
Use: "restore",
Short: "restore a PostgreSQL database",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
Run: func(cmd *cobra.Command, args []string) {
fmt.Println("postgres called")
file, _ := cmd.Flags().GetString("file")
hostname, _ := cmd.Flags().GetString("hostname")
database, _ := cmd.Flags().GetString("database")
username, _ := cmd.Flags().GetString("username")
password, _ := cmd.Flags().GetString("password")
_ = restore.RestorePostgres(file, hostname, database, username, password)
},
}
var postgresBackupCmd = &cobra.Command{
Use: "backup",
Short: "backup a PostgreSQL database",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
@ -43,26 +64,43 @@ to quickly create a Cobra application.`,
},
}
var postgresCmd = &cobra.Command{
Use: "postgres",
Short: "postgres actions",
}
func init() {
backupCmd.AddCommand(postgresCmd)
rootCmd.AddCommand(postgresCmd)
postgresCmd.AddCommand(postgresBackupCmd)
postgresCmd.AddCommand(postgresRestoreCmd)
// Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// postgresCmd.PersistentFlags().String("foo", "", "A help for foo")
// backupPostgresCmd.PersistentFlags().String("foo", "", "A help for foo")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// postgresCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
postgresCmd.Flags().String("file", "", "The file the backup will be stored")
postgresCmd.Flags().String("hostname", "", "The postgresql server host")
postgresCmd.Flags().String("database", "", "The postgresql database")
postgresCmd.Flags().String("username", "", "The postgresql username")
postgresCmd.Flags().String("password", "", "The postgresql password")
postgresCmd.MarkFlagRequired("path")
postgresCmd.MarkFlagRequired("hostname")
postgresCmd.MarkFlagRequired("database")
postgresCmd.MarkFlagRequired("username")
postgresCmd.MarkFlagRequired("password")
// backupPostgresCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
postgresBackupCmd.Flags().String("file", "", "The file the backup will be stored")
postgresBackupCmd.Flags().String("hostname", "", "The postgresql server host")
postgresBackupCmd.Flags().String("database", "", "The postgresql database")
postgresBackupCmd.Flags().String("username", "", "The postgresql username")
postgresBackupCmd.Flags().String("password", "", "The postgresql password")
postgresBackupCmd.MarkFlagRequired("path")
postgresBackupCmd.MarkFlagRequired("hostname")
postgresBackupCmd.MarkFlagRequired("database")
postgresBackupCmd.MarkFlagRequired("username")
postgresBackupCmd.MarkFlagRequired("password")
postgresRestoreCmd.Flags().String("file", "", "The file the database will be restored from")
postgresRestoreCmd.Flags().String("hostname", "", "The postgresql server host")
postgresRestoreCmd.Flags().String("database", "", "The postgresql database")
postgresRestoreCmd.Flags().String("username", "", "The postgresql username")
postgresRestoreCmd.Flags().String("password", "", "The postgresql password")
postgresRestoreCmd.MarkFlagRequired("path")
postgresRestoreCmd.MarkFlagRequired("hostname")
postgresRestoreCmd.MarkFlagRequired("database")
postgresRestoreCmd.MarkFlagRequired("username")
postgresRestoreCmd.MarkFlagRequired("password")
}

View File

@ -1,51 +0,0 @@
/*
Copyright © 2021 NAME HERE <EMAIL ADDRESS>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
"github.com/spf13/cobra"
)
// runCmd represents the run command
var runCmd = &cobra.Command{
Use: "run",
Short: "A brief description of your command",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
Run: func(cmd *cobra.Command, args []string) {
fmt.Println("run called")
},
}
func init() {
rootCmd.AddCommand(runCmd)
// Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// runCmd.PersistentFlags().String("foo", "", "A help for foo")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// runCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}

View File

@ -1,53 +0,0 @@
/*
Copyright © 2020 NAME HERE <EMAIL ADDRESS>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
"github.com/desmo999r/formolcli/pkg/server"
"github.com/spf13/cobra"
)
// serverCmd represents the server command
var serverCmd = &cobra.Command{
Use: "server",
Short: "A brief description of your command",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
Run: func(cmd *cobra.Command, args []string) {
fmt.Println("server called")
server.Server()
},
}
func init() {
createCmd.AddCommand(serverCmd)
// Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// serverCmd.PersistentFlags().String("foo", "", "A help for foo")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// serverCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}

View File

@ -17,7 +17,7 @@ package cmd
import (
"os"
"github.com/desmo999r/formolcli/pkg/backup"
"github.com/desmo999r/formolcli/pkg/restic"
"github.com/spf13/cobra"
)
@ -25,32 +25,22 @@ import (
var snapshotCmd = &cobra.Command{
Use: "snapshot",
Short: "A brief description of your command",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
}
Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
var snapshotDeleteCmd = &cobra.Command{
Use: "delete",
Short: "delete a snapshot",
Run: func(cmd *cobra.Command, args []string) {
snapshot, _ := cmd.Flags().GetString("snapshot")
if err := backup.DeleteSnapshot(snapshot); err != nil {
snapshot, _ := cmd.Flags().GetString("snapshot-id")
if err := restic.DeleteSnapshot(snapshot); err != nil {
os.Exit(1)
}
},
}
func init() {
deleteCmd.AddCommand(snapshotCmd)
// Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// snapshotCmd.PersistentFlags().String("foo", "", "A help for foo")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// snapshotCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
snapshotCmd.Flags().String("snapshot", "", "The snapshot to delete")
snapshotCmd.MarkFlagRequired("snapshot")
rootCmd.AddCommand(snapshotCmd)
snapshotCmd.AddCommand(snapshotDeleteCmd)
snapshotDeleteCmd.Flags().String("snapshot-id", "", "The snapshot to delete")
snapshotDeleteCmd.MarkFlagRequired("snapshot-id")
}

44
pkg/cmd/target.go Normal file
View File

@ -0,0 +1,44 @@
/*
Copyright © 2021 NAME HERE <EMAIL ADDRESS>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cmd
import (
"fmt"
formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1"
"github.com/desmo999r/formolcli/pkg/session"
"github.com/spf13/cobra"
)
// targetCmd represents the target command
var targetFinalizeCmd = &cobra.Command{
Use: "finalize",
Short: "Update the session target status",
Run: func(cmd *cobra.Command, args []string) {
fmt.Println("target called")
session.RestoreSessionUpdateTargetStatus(formolv1alpha1.Success)
},
}
var targetCmd = &cobra.Command{
Use: "target",
Short: "A brief description of your command",
}
func init() {
rootCmd.AddCommand(targetCmd)
targetCmd.AddCommand(targetFinalizeCmd)
}

View File

@ -19,13 +19,36 @@ import (
"os"
"github.com/desmo999r/formolcli/pkg/backup"
"github.com/desmo999r/formolcli/pkg/restore"
"github.com/spf13/cobra"
)
// pvcCmd represents the pvc command
var volumeRestoreCmd = &cobra.Command{
Use: "restore",
Short: "restore a volume",
Run: func(cmd *cobra.Command, args []string) {
snapshotId, _ := cmd.Flags().GetString("snapshot-id")
if err := restore.RestoreVolume(snapshotId); err != nil {
os.Exit(1)
}
},
}
var volumeBackupCmd = &cobra.Command{
Use: "backup",
Short: "backup a volume",
Run: func(cmd *cobra.Command, args []string) {
paths, _ := cmd.Flags().GetStringSlice("path")
tag, _ := cmd.Flags().GetString("tag")
if err := backup.BackupVolume(tag, paths); err != nil {
os.Exit(1)
}
},
}
var volumeCmd = &cobra.Command{
Use: "volume",
Short: "A brief description of your command",
Short: "volume actions",
Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:
@ -42,18 +65,13 @@ to quickly create a Cobra application.`,
}
func init() {
backupCmd.AddCommand(volumeCmd)
rootCmd.AddCommand(volumeCmd)
volumeCmd.AddCommand(volumeBackupCmd)
volumeCmd.AddCommand(volumeRestoreCmd)
// Here you will define your flags and configuration settings.
// Cobra supports Persistent Flags which will work for this command
// and all subcommands, e.g.:
// pvcCmd.PersistentFlags().String("foo", "", "A help for foo")
// Cobra supports local flags which will only run when this command
// is called directly, e.g.:
// pvcCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
volumeCmd.Flags().StringSlice("path", nil, "Path to the data to backup")
volumeCmd.Flags().String("tag", "", "Tag associated to the backup")
volumeCmd.MarkFlagRequired("path")
volumeBackupCmd.Flags().StringSlice("path", nil, "Path to the data to backup")
volumeBackupCmd.Flags().String("tag", "", "Tag associated to the backup")
volumeBackupCmd.MarkFlagRequired("path")
volumeRestoreCmd.Flags().String("snapshot-id", "", "snapshot id associated to the backup")
volumeRestoreCmd.MarkFlagRequired("snapshot-id")
}

View File

@ -5,71 +5,18 @@ import (
"github.com/go-logr/logr"
"k8s.io/apimachinery/pkg/runtime"
"os"
"path/filepath"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/controller-runtime/pkg/predicate"
"time"
formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1"
"github.com/desmo999r/formolcli/pkg/backup"
"github.com/desmo999r/formolcli/pkg/restic"
formolcliutils "github.com/desmo999r/formolcli/pkg/utils"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
)
var (
deploymentName = ""
)
func init() {
log := zap.New(zap.UseDevMode(true)).WithName("init")
namespace := os.Getenv("POD_NAMESPACE")
if namespace == "" {
return
}
config, err := rest.InClusterConfig()
if err != nil {
config, err = clientcmd.BuildConfigFromFlags("", filepath.Join(os.Getenv("HOME"), ".kube", "config"))
if err != nil {
log.Error(err, "unable to get config")
panic(err.Error())
}
}
clientset, err := kubernetes.NewForConfig(config)
if err != nil {
panic("unable to get clientset")
}
hostname := os.Getenv("POD_NAME")
if hostname == "" {
panic("unable to get hostname")
}
pod, err := clientset.CoreV1().Pods(namespace).Get(context.Background(), hostname, metav1.GetOptions{})
if err != nil {
log.Error(err, "unable to get pod")
panic("unable to get pod")
}
podOwner := metav1.GetControllerOf(pod)
replicasetList, err := clientset.AppsV1().ReplicaSets(namespace).List(context.Background(), metav1.ListOptions{
FieldSelector: "metadata.name=" + string(podOwner.Name),
})
if err != nil {
panic("unable to get replicaset" + err.Error())
}
for _, replicaset := range replicasetList.Items {
replicasetOwner := metav1.GetControllerOf(&replicaset)
deploymentName = replicasetOwner.Name
}
}
// BackupSessionReconciler reconciles a BackupSession object
type BackupSessionReconciler struct {
client.Client
@ -78,6 +25,7 @@ type BackupSessionReconciler struct {
}
func (r *BackupSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
time.Sleep(2 * time.Second)
ctx := context.Background()
log := r.Log.WithValues("backupsession", req.NamespacedName)
@ -96,6 +44,7 @@ func (r *BackupSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
return ctrl.Result{}, client.IgnoreNotFound(err)
}
deploymentName := os.Getenv("POD_DEPLOYMENT")
for _, target := range backupConf.Spec.Targets {
switch target.Kind {
case "Deployment":
@ -103,7 +52,7 @@ func (r *BackupSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
for i, status := range backupSession.Status.Targets {
if status.Name == target.Name {
log.V(0).Info("It's for us!", "target", target)
switch status.BackupState {
switch status.SessionState {
case formolv1alpha1.New:
// TODO: Run beforeBackup
log.V(0).Info("New session, run the beforeBackup hooks if any")
@ -111,7 +60,7 @@ func (r *BackupSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
if err := formolcliutils.RunBeforeBackup(target); err != nil {
result = formolv1alpha1.Failure
}
backupSession.Status.Targets[i].BackupState = result
backupSession.Status.Targets[i].SessionState = result
log.V(1).Info("current backupSession status", "status", backupSession.Status)
if err := r.Status().Update(ctx, backupSession); err != nil {
log.Error(err, "unable to update backupsession status")
@ -121,16 +70,16 @@ func (r *BackupSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
log.V(0).Info("Running session. Do the backup")
result := formolv1alpha1.Success
status.StartTime = &metav1.Time{Time: time.Now()}
output, err := backup.BackupPaths(backupSession.Name, target.Paths)
output, err := restic.BackupPaths(backupSession.Name, target.Paths)
if err != nil {
log.Error(err, "unable to backup deployment", "output", string(output))
result = formolv1alpha1.Failure
} else {
snapshotId, duration := backup.GetBackupResults(output)
snapshotId := restic.GetBackupResults(output)
backupSession.Status.Targets[i].SnapshotId = snapshotId
backupSession.Status.Targets[i].Duration = &metav1.Duration{Duration: duration}
backupSession.Status.Targets[i].Duration = &metav1.Duration{Duration: time.Now().Sub(backupSession.Status.Targets[i].StartTime.Time)}
}
backupSession.Status.Targets[i].BackupState = result
backupSession.Status.Targets[i].SessionState = result
log.V(1).Info("current backupSession status", "status", backupSession.Status)
if err := r.Status().Update(ctx, backupSession); err != nil {
log.Error(err, "unable to update backupsession status")
@ -152,10 +101,9 @@ func (r *BackupSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
func (r *BackupSessionReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&formolv1alpha1.BackupSession{}).
//WithEventFilter(predicate.GenerationChangedPredicate{}). // Don't reconcile when status gets updated
WithEventFilter(predicate.Funcs{
CreateFunc: func(e event.CreateEvent) bool { return false },
DeleteFunc: func(e event.DeleteEvent) bool { return false },
}). // Don't reconcile when status gets updated
}).
Complete(r)
}

View File

@ -0,0 +1,123 @@
package controllers
import (
"context"
"fmt"
formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1"
"github.com/desmo999r/formolcli/pkg/restic"
formolcliutils "github.com/desmo999r/formolcli/pkg/utils"
"github.com/go-logr/logr"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"os"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/predicate"
"strings"
"time"
)
type RestoreSessionReconciler struct {
client.Client
Log logr.Logger
Scheme *runtime.Scheme
RestoreSession *formolv1alpha1.RestoreSession
BackupSession *formolv1alpha1.BackupSession
BackupConf *formolv1alpha1.BackupConfiguration
}
func (r *RestoreSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
time.Sleep(300 * time.Millisecond)
ctx := context.Background()
log := r.Log.WithValues("restoresession", req.NamespacedName)
r.RestoreSession = &formolv1alpha1.RestoreSession{}
if err := r.Get(ctx, req.NamespacedName, r.RestoreSession); err != nil {
log.Error(err, "unable to get restoresession")
return ctrl.Result{}, client.IgnoreNotFound(err)
}
r.BackupSession = &formolv1alpha1.BackupSession{}
if err := r.Get(ctx, client.ObjectKey{
Namespace: r.RestoreSession.Spec.BackupSessionRef.Namespace,
Name: r.RestoreSession.Spec.BackupSessionRef.Name}, r.BackupSession); err != nil {
log.Error(err, "unable to get backupsession", "namespace", r.RestoreSession.Spec.BackupSessionRef.Namespace, "name", r.RestoreSession.Spec.BackupSessionRef.Name)
return ctrl.Result{}, client.IgnoreNotFound(err)
}
r.BackupConf = &formolv1alpha1.BackupConfiguration{}
if err := r.Get(ctx, client.ObjectKey{Namespace: r.BackupSession.Namespace,
Name: r.BackupSession.Spec.Ref.Name}, r.BackupConf); err != nil {
log.Error(err, "unable to get backupconfiguration")
return ctrl.Result{}, client.IgnoreNotFound(err)
}
deploymentName := os.Getenv("POD_DEPLOYMENT")
for _, target := range r.BackupConf.Spec.Targets {
switch target.Kind {
case "Deployment":
if target.Name == deploymentName {
for i, status := range r.RestoreSession.Status.Targets {
if status.Name == target.Name {
log.V(0).Info("It's for us!", "target", target.Name)
switch status.SessionState {
case formolv1alpha1.New:
log.V(0).Info("New session, run the beforeBackup hooks if any")
result := formolv1alpha1.Running
if err := formolcliutils.RunBeforeBackup(target); err != nil {
result = formolv1alpha1.Failure
}
r.RestoreSession.Status.Targets[i].SessionState = result
if err := r.Status().Update(ctx, r.RestoreSession); err != nil {
log.Error(err, " unable to update restoresession status")
return ctrl.Result{RequeueAfter: 30 * time.Second}, err
}
case formolv1alpha1.Running:
log.V(0).Info("Running session. Do the restore")
status.StartTime = &metav1.Time{Time: time.Now()}
result := formolv1alpha1.Success
repo := &formolv1alpha1.Repo{}
if err := r.Get(ctx, client.ObjectKey{
Namespace: r.BackupConf.Namespace,
Name: r.BackupConf.Spec.Repository.Name,
}, repo); err != nil {
log.Error(err, "unable to get Repo from BackupConf")
return ctrl.Result{}, err
}
url := fmt.Sprintf("s3:http://%s/%s/%s-%s", repo.Spec.Backend.S3.Server, repo.Spec.Backend.S3.Bucket, strings.ToUpper(r.BackupConf.Namespace), strings.ToLower(r.BackupConf.Name))
log.V(0).Info("restoring", "url", url, "snapshot", r.BackupSession.Status.Targets[i].SnapshotId)
output, err := restic.RestorePaths(url, r.BackupSession.Status.Targets[i].SnapshotId)
if err != nil {
log.Error(err, "unable to restore deployment", "output", string(output))
result = formolv1alpha1.Failure
} else {
duration := restic.GetRestoreResults(output)
r.RestoreSession.Status.Targets[i].Duration = &metav1.Duration{Duration: duration}
}
r.RestoreSession.Status.Targets[i].SessionState = result
log.V(1).Info("current restoresession status", "status", result)
if err := r.Status().Update(ctx, r.RestoreSession); err != nil {
log.Error(err, "unable to update restoresession status")
return ctrl.Result{RequeueAfter: 30 * time.Second}, err
}
case formolv1alpha1.Failure, formolv1alpha1.Success:
log.V(0).Info("Restore is over, run afterBackup hooks if any")
formolcliutils.RunAfterBackup(target)
}
}
}
}
}
}
return ctrl.Result{}, nil
}
func (r *RestoreSessionReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&formolv1alpha1.RestoreSession{}).
WithEventFilter(predicate.Funcs{
CreateFunc: func(e event.CreateEvent) bool { return false },
DeleteFunc: func(e event.DeleteEvent) bool { return false },
}).
Complete(r)
}

111
pkg/restic/root.go Normal file
View File

@ -0,0 +1,111 @@
package restic
import (
"bufio"
"bytes"
"encoding/json"
"github.com/go-logr/logr"
"github.com/go-logr/zapr"
"go.uber.org/zap"
"os"
"os/exec"
"time"
)
var (
repository string
passwordFile string
aws_access_key_id string
aws_secret_access_key string
resticExec = "/usr/bin/restic"
logger logr.Logger
)
func init() {
zapLog, _ := zap.NewDevelopment()
logger = zapr.NewLogger(zapLog)
repository = os.Getenv("RESTIC_REPOSITORY")
passwordFile = os.Getenv("RESTIC_PASSWORD")
aws_access_key_id = os.Getenv("AWS_ACCESS_KEY_ID")
aws_secret_access_key = os.Getenv("AWS_SECRET_ACCESS_KEY")
}
func checkRepo(repo string) error {
log := logger.WithValues("backup-checkrepo", repo)
cmd := exec.Command(resticExec, "unlock", "-r", repo)
if err := cmd.Run(); err != nil {
log.Error(err, "unable to unlock repo", "repo", repo)
}
cmd = exec.Command(resticExec, "check", "-r", repo)
output, err := cmd.CombinedOutput()
log.V(1).Info("restic check output", "output", string(output))
if err != nil {
log.V(0).Info("initializing new repo", "repo", repo)
cmd = exec.Command(resticExec, "init", "-r", repo)
output, err = cmd.CombinedOutput()
log.V(1).Info("restic init repo", "output", string(output))
if err != nil {
log.Error(err, "something went wrong during repo init")
return err
}
}
return err
}
func GetBackupResults(output []byte) (snapshotId string) {
log := logger.WithName("backup-getbackupresults")
scanner := bufio.NewScanner(bytes.NewReader(output))
var dat map[string]interface{}
for scanner.Scan() {
if err := json.Unmarshal(scanner.Bytes(), &dat); err != nil {
log.Error(err, "unable to unmarshal json", "msg", string(scanner.Bytes()))
continue
}
log.V(1).Info("message on stdout", "stdout", dat)
if message_type, ok := dat["message_type"]; ok && message_type == "summary" {
snapshotId = dat["snapshot_id"].(string)
//duration = time.Duration(dat["total_duration"].(float64)*1000) * time.Millisecond
}
}
return
}
func GetRestoreResults(output []byte) time.Duration {
return 0 * time.Millisecond
}
func BackupPaths(tag string, paths []string) ([]byte, error) {
log := logger.WithName("backup-deployment")
if err := checkRepo(repository); err != nil {
log.Error(err, "unable to setup newrepo", "newrepo", repository)
return []byte{}, err
}
cmd := exec.Command(resticExec, append([]string{"backup", "--json", "--tag", tag, "-r", repository}, paths...)...)
output, err := cmd.CombinedOutput()
return output, err
}
func RestorePaths(repository string, snapshotId string) ([]byte, error) {
log := logger.WithName("restore-deployment")
if err := checkRepo(repository); err != nil {
log.Error(err, "unable to setup repo", "repo", repository)
return []byte{}, err
}
cmd := exec.Command(resticExec, "restore", "-r", repository, snapshotId, "--target", "/")
output, err := cmd.CombinedOutput()
log.V(1).Info("restic restore output", "output", string(output))
return output, err
}
func DeleteSnapshot(snapshot string) error {
log := logger.WithValues("delete-snapshot", snapshot)
cmd := exec.Command(resticExec, "forget", "-r", repository, "--prune", snapshot)
log.V(0).Info("deleting snapshot", "snapshot", snapshot)
output, err := cmd.CombinedOutput()
log.V(1).Info("delete snapshot output", "output", string(output))
if err != nil {
log.Error(err, "unable to delete the snapshot")
return err
}
return nil
}

59
pkg/restore/root.go Normal file
View File

@ -0,0 +1,59 @@
package restore
import (
"fmt"
formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1"
"github.com/desmo999r/formolcli/pkg/restic"
"github.com/desmo999r/formolcli/pkg/session"
"github.com/go-logr/logr"
"github.com/go-logr/zapr"
"go.uber.org/zap"
"io/ioutil"
"os"
"os/exec"
)
var (
//psqlExec = "/usr/bin/psql"
pg_restoreExec = "/usr/bin/pg_restore"
logger logr.Logger
)
func init() {
zapLog, _ := zap.NewDevelopment()
logger = zapr.NewLogger(zapLog)
}
func RestoreVolume(snapshotId string) error {
log := logger.WithName("restore-volume")
state := formolv1alpha1.Success
repository := os.Getenv("RESTIC_REPOSITORY")
output, err := restic.RestorePaths(repository, snapshotId)
if err != nil {
log.Error(err, "unable to restore volume", "output", string(output))
state = formolv1alpha1.Failure
}
log.V(1).Info("restic restore output", "output", string(output))
session.RestoreSessionUpdateTargetStatus(state)
return err
}
func RestorePostgres(file string, hostname string, database string, username string, password string) error {
log := logger.WithName("restore-postgres")
pgpass := []byte(fmt.Sprintf("%s:*:%s:%s:%s", hostname, database, username, password))
if err := ioutil.WriteFile("/output/.pgpass", pgpass, 0600); err != nil {
log.Error(err, "unable to write password to /output/.pgpass")
return err
}
defer os.Remove("/output/.pgpass")
//cmd := exec.Command(psqlExec, "--file", file, "--host", hostname, "--dbname", database, "--username", username, "--no-password")
cmd := exec.Command(pg_restoreExec, "--format=custom", "--clean", "--host", hostname, "--dbname", database, "--username", username, "--no-password", file)
cmd.Env = append(os.Environ(), "PGPASSFILE=/output/.pgpass")
output, err := cmd.CombinedOutput()
log.V(1).Info("postgres restore output", "output", string(output))
if err != nil {
log.Error(err, "something went wrong during the restore")
return err
}
return nil
}

View File

@ -39,7 +39,7 @@ func Server() {
Port: 9443,
LeaderElection: enableLeaderElection,
LeaderElectionID: "12345.desmojim.fr",
Namespace: os.Getenv("POD_NAMESPACE"),
//Namespace: os.Getenv("POD_NAMESPACE"),
})
if err != nil {
setupLog.Error(err, "unable to create manager")
@ -55,6 +55,15 @@ func Server() {
os.Exit(1)
}
if err = (&controllers.RestoreSessionReconciler{
Client: mgr.GetClient(),
Log: ctrl.Log.WithName("controllers").WithName("RestoreSession"),
Scheme: mgr.GetScheme(),
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "RestoreSession")
os.Exit(1)
}
setupLog.Info("starting manager")
if err = mgr.Start(ctrl.SetupSignalHandler()); err != nil {
setupLog.Error(err, "problem running the manager")

View File

@ -1,4 +1,4 @@
package backupsession
package session
import (
"context"
@ -49,19 +49,22 @@ func init() {
}
}
func BackupSessionUpdateStatus(state formolv1alpha1.BackupState, snapshotId string, duration time.Duration) error {
func BackupSessionUpdateTargetStatus(state formolv1alpha1.SessionState, snapshotId string) error {
log := logger.WithName("BackupSessionUpdateStatus")
targetName := os.Getenv("TARGET_NAME")
backupSession := &formolv1alpha1.BackupSession{}
cl.Get(context.Background(), client.ObjectKey{
if err := cl.Get(context.Background(), client.ObjectKey{
Namespace: os.Getenv("BACKUPSESSION_NAMESPACE"),
Name: os.Getenv("BACKUPSESSION_NAME"),
}, backupSession)
}, backupSession); err != nil {
log.Error(err, "unable to get backupsession", "BACKUPSESSION_NAME", os.Getenv("BACKUPSESSION_NAME"), "BACKUPSESSION_NAMESPACE", os.Getenv("BACKUPSESSION_NAMESPACE"))
return err
}
for i, target := range backupSession.Status.Targets {
if target.Name == targetName {
backupSession.Status.Targets[i].BackupState = state
backupSession.Status.Targets[i].SessionState = state
backupSession.Status.Targets[i].SnapshotId = snapshotId
backupSession.Status.Targets[i].Duration = &metav1.Duration{Duration: duration}
backupSession.Status.Targets[i].Duration = &metav1.Duration{Duration: time.Now().Sub(backupSession.Status.Targets[i].StartTime.Time)}
}
}
@ -72,6 +75,31 @@ func BackupSessionUpdateStatus(state formolv1alpha1.BackupState, snapshotId stri
return nil
}
func RestoreSessionUpdateTargetStatus(state formolv1alpha1.SessionState) error {
log := logger.WithName("RestoreSessionUpdateStatus")
targetName := os.Getenv("TARGET_NAME")
restoreSession := &formolv1alpha1.RestoreSession{}
if err := cl.Get(context.Background(), client.ObjectKey{
Namespace: os.Getenv("RESTORESESSION_NAMESPACE"),
Name: os.Getenv("RESTORESESSION_NAME"),
}, restoreSession); err != nil {
log.Error(err, "unable to get backupsession", "RESTORESESSION_NAME", os.Getenv("RESTORESESSION_NAME"), "RESTORESESSION_NAMESPACE", os.Getenv("RESTORESESSION_NAMESPACE"))
return err
}
for i, target := range restoreSession.Status.Targets {
if target.Name == targetName {
restoreSession.Status.Targets[i].SessionState = state
restoreSession.Status.Targets[i].Duration = &metav1.Duration{Duration: time.Now().Sub(restoreSession.Status.Targets[i].StartTime.Time)}
}
}
if err := cl.Status().Update(context.Background(), restoreSession); err != nil {
log.Error(err, "unable to update restoresession status", "restoresession", restoreSession)
return err
}
return nil
}
func CreateBackupSession(name string, namespace string) {
log := logger.WithName("CreateBackupSession")
log.V(0).Info("CreateBackupSession called")