Restic operations in restic module, session in session module
This commit is contained in:
parent
b52d876815
commit
827536c1c7
@ -1,93 +1,40 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1"
|
||||
"github.com/desmo999r/formolcli/pkg/backupsession"
|
||||
"github.com/desmo999r/formolcli/pkg/restic"
|
||||
"github.com/desmo999r/formolcli/pkg/session"
|
||||
"github.com/go-logr/logr"
|
||||
"github.com/go-logr/zapr"
|
||||
"go.uber.org/zap"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
repository string
|
||||
passwordFile string
|
||||
aws_access_key_id string
|
||||
aws_secret_access_key string
|
||||
resticExec = "/usr/bin/restic"
|
||||
pg_dumpExec = "/usr/bin/pg_dump"
|
||||
logger logr.Logger
|
||||
pg_dumpExec = "/usr/bin/pg_dump"
|
||||
logger logr.Logger
|
||||
)
|
||||
|
||||
func init() {
|
||||
zapLog, _ := zap.NewDevelopment()
|
||||
logger = zapr.NewLogger(zapLog)
|
||||
repository = os.Getenv("RESTIC_REPOSITORY")
|
||||
passwordFile = os.Getenv("RESTIC_PASSWORD")
|
||||
aws_access_key_id = os.Getenv("AWS_ACCESS_KEY_ID")
|
||||
aws_secret_access_key = os.Getenv("AWS_SECRET_ACCESS_KEY")
|
||||
}
|
||||
|
||||
func checkRepo(repo string) error {
|
||||
log := logger.WithValues("backup-checkrepo", repo)
|
||||
cmd := exec.Command(resticExec, "unlock", "-r", repo)
|
||||
if err := cmd.Run(); err != nil {
|
||||
log.Error(err, "unable to unlock repo", "repo", repo)
|
||||
}
|
||||
cmd = exec.Command(resticExec, "check", "-r", repo)
|
||||
output, err := cmd.CombinedOutput()
|
||||
log.V(1).Info("restic check output", "output", string(output))
|
||||
if err != nil {
|
||||
log.V(0).Info("initializing new repo", "repo", repo)
|
||||
cmd = exec.Command(resticExec, "init", "-r", repo)
|
||||
output, err = cmd.CombinedOutput()
|
||||
log.V(1).Info("restic init repo", "output", string(output))
|
||||
if err != nil {
|
||||
log.Error(err, "something went wrong during repo init")
|
||||
return err
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func GetBackupResults(output []byte) (snapshotId string, duration time.Duration) {
|
||||
log := logger.WithName("backup-getbackupresults")
|
||||
scanner := bufio.NewScanner(bytes.NewReader(output))
|
||||
var dat map[string]interface{}
|
||||
for scanner.Scan() {
|
||||
if err := json.Unmarshal(scanner.Bytes(), &dat); err != nil {
|
||||
log.Error(err, "unable to unmarshal json", "msg", string(scanner.Bytes()))
|
||||
continue
|
||||
}
|
||||
log.V(1).Info("message on stdout", "stdout", dat)
|
||||
if message_type, ok := dat["message_type"]; ok && message_type == "summary" {
|
||||
snapshotId = dat["snapshot_id"].(string)
|
||||
duration = time.Duration(dat["total_duration"].(float64)*1000) * time.Millisecond
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func BackupVolume(tag string, paths []string) error {
|
||||
log := logger.WithName("backup-volume")
|
||||
state := formolv1alpha1.Success
|
||||
output, err := BackupPaths(tag, paths)
|
||||
output, err := restic.BackupPaths(tag, paths)
|
||||
var snapshotId string
|
||||
var duration time.Duration
|
||||
if err != nil {
|
||||
log.Error(err, "unable to backup volume", "output", string(output))
|
||||
state = formolv1alpha1.Failure
|
||||
} else {
|
||||
snapshotId, duration = GetBackupResults(output)
|
||||
snapshotId = restic.GetBackupResults(output)
|
||||
}
|
||||
backupsession.BackupSessionUpdateStatus(state, snapshotId, duration)
|
||||
session.BackupSessionUpdateTargetStatus(state, snapshotId)
|
||||
return err
|
||||
}
|
||||
|
||||
@ -109,27 +56,3 @@ func BackupPostgres(file string, hostname string, database string, username stri
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func BackupPaths(tag string, paths []string) ([]byte, error) {
|
||||
log := logger.WithName("backup-deployment")
|
||||
if err := checkRepo(repository); err != nil {
|
||||
log.Error(err, "unable to setup newrepo", "newrepo", repository)
|
||||
return []byte{}, err
|
||||
}
|
||||
cmd := exec.Command(resticExec, append([]string{"backup", "--json", "--tag", tag, "-r", repository}, paths...)...)
|
||||
output, err := cmd.CombinedOutput()
|
||||
return output, err
|
||||
}
|
||||
|
||||
func DeleteSnapshot(snapshot string) error {
|
||||
log := logger.WithValues("delete-snapshot", snapshot)
|
||||
cmd := exec.Command(resticExec, "forget", "-r", repository, "--prune", snapshot)
|
||||
log.V(0).Info("deleting snapshot", "snapshot", snapshot)
|
||||
output, err := cmd.CombinedOutput()
|
||||
log.V(1).Info("delete snapshot output", "output", string(output))
|
||||
if err != nil {
|
||||
log.Error(err, "unable to delete the snapshot")
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -12,7 +12,7 @@ import (
|
||||
"time"
|
||||
|
||||
formolv1alpha1 "github.com/desmo999r/formol/api/v1alpha1"
|
||||
"github.com/desmo999r/formolcli/pkg/backup"
|
||||
"github.com/desmo999r/formolcli/pkg/restic"
|
||||
formolcliutils "github.com/desmo999r/formolcli/pkg/utils"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
@ -25,6 +25,7 @@ type BackupSessionReconciler struct {
|
||||
}
|
||||
|
||||
func (r *BackupSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
|
||||
time.Sleep(2 * time.Second)
|
||||
ctx := context.Background()
|
||||
log := r.Log.WithValues("backupsession", req.NamespacedName)
|
||||
|
||||
@ -69,14 +70,14 @@ func (r *BackupSessionReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro
|
||||
log.V(0).Info("Running session. Do the backup")
|
||||
result := formolv1alpha1.Success
|
||||
status.StartTime = &metav1.Time{Time: time.Now()}
|
||||
output, err := backup.BackupPaths(backupSession.Name, target.Paths)
|
||||
output, err := restic.BackupPaths(backupSession.Name, target.Paths)
|
||||
if err != nil {
|
||||
log.Error(err, "unable to backup deployment", "output", string(output))
|
||||
result = formolv1alpha1.Failure
|
||||
} else {
|
||||
snapshotId, duration := backup.GetBackupResults(output)
|
||||
snapshotId := restic.GetBackupResults(output)
|
||||
backupSession.Status.Targets[i].SnapshotId = snapshotId
|
||||
backupSession.Status.Targets[i].Duration = &metav1.Duration{Duration: duration}
|
||||
backupSession.Status.Targets[i].Duration = &metav1.Duration{Duration: time.Now().Sub(backupSession.Status.Targets[i].StartTime.Time)}
|
||||
}
|
||||
backupSession.Status.Targets[i].SessionState = result
|
||||
log.V(1).Info("current backupSession status", "status", backupSession.Status)
|
||||
|
||||
109
pkg/restic/root.go
Normal file
109
pkg/restic/root.go
Normal file
@ -0,0 +1,109 @@
|
||||
package restic
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"github.com/go-logr/logr"
|
||||
"github.com/go-logr/zapr"
|
||||
"go.uber.org/zap"
|
||||
"os"
|
||||
"os/exec"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
repository string
|
||||
passwordFile string
|
||||
aws_access_key_id string
|
||||
aws_secret_access_key string
|
||||
resticExec = "/usr/bin/restic"
|
||||
logger logr.Logger
|
||||
)
|
||||
|
||||
func init() {
|
||||
zapLog, _ := zap.NewDevelopment()
|
||||
logger = zapr.NewLogger(zapLog)
|
||||
repository = os.Getenv("RESTIC_REPOSITORY")
|
||||
passwordFile = os.Getenv("RESTIC_PASSWORD")
|
||||
aws_access_key_id = os.Getenv("AWS_ACCESS_KEY_ID")
|
||||
aws_secret_access_key = os.Getenv("AWS_SECRET_ACCESS_KEY")
|
||||
}
|
||||
|
||||
func checkRepo(repo string) error {
|
||||
log := logger.WithValues("backup-checkrepo", repo)
|
||||
cmd := exec.Command(resticExec, "unlock", "-r", repo)
|
||||
if err := cmd.Run(); err != nil {
|
||||
log.Error(err, "unable to unlock repo", "repo", repo)
|
||||
}
|
||||
cmd = exec.Command(resticExec, "check", "-r", repo)
|
||||
output, err := cmd.CombinedOutput()
|
||||
log.V(1).Info("restic check output", "output", string(output))
|
||||
if err != nil {
|
||||
log.V(0).Info("initializing new repo", "repo", repo)
|
||||
cmd = exec.Command(resticExec, "init", "-r", repo)
|
||||
output, err = cmd.CombinedOutput()
|
||||
log.V(1).Info("restic init repo", "output", string(output))
|
||||
if err != nil {
|
||||
log.Error(err, "something went wrong during repo init")
|
||||
return err
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func GetBackupResults(output []byte) (snapshotId string) {
|
||||
log := logger.WithName("backup-getbackupresults")
|
||||
scanner := bufio.NewScanner(bytes.NewReader(output))
|
||||
var dat map[string]interface{}
|
||||
for scanner.Scan() {
|
||||
if err := json.Unmarshal(scanner.Bytes(), &dat); err != nil {
|
||||
log.Error(err, "unable to unmarshal json", "msg", string(scanner.Bytes()))
|
||||
continue
|
||||
}
|
||||
log.V(1).Info("message on stdout", "stdout", dat)
|
||||
if message_type, ok := dat["message_type"]; ok && message_type == "summary" {
|
||||
snapshotId = dat["snapshot_id"].(string)
|
||||
//duration = time.Duration(dat["total_duration"].(float64)*1000) * time.Millisecond
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func GetRestoreResults(output []byte) time.Duration {
|
||||
return 0 * time.Millisecond
|
||||
}
|
||||
|
||||
func BackupPaths(tag string, paths []string) ([]byte, error) {
|
||||
log := logger.WithName("backup-deployment")
|
||||
if err := checkRepo(repository); err != nil {
|
||||
log.Error(err, "unable to setup newrepo", "newrepo", repository)
|
||||
return []byte{}, err
|
||||
}
|
||||
cmd := exec.Command(resticExec, append([]string{"backup", "--json", "--tag", tag, "-r", repository}, paths...)...)
|
||||
output, err := cmd.CombinedOutput()
|
||||
return output, err
|
||||
}
|
||||
|
||||
func RestorePaths(repository string, snapshotId string) ([]byte, error) {
|
||||
log := logger.WithName("restore-deployment")
|
||||
if err := checkRepo(repository); err != nil {
|
||||
log.Error(err, "unable to setup repo", "repo", repository)
|
||||
return []byte{}, err
|
||||
}
|
||||
cmd := exec.Command(resticExec, "restore", "-r", repository, snapshotId, "--target", "/")
|
||||
return cmd.CombinedOutput()
|
||||
}
|
||||
|
||||
func DeleteSnapshot(snapshot string) error {
|
||||
log := logger.WithValues("delete-snapshot", snapshot)
|
||||
cmd := exec.Command(resticExec, "forget", "-r", repository, "--prune", snapshot)
|
||||
log.V(0).Info("deleting snapshot", "snapshot", snapshot)
|
||||
output, err := cmd.CombinedOutput()
|
||||
log.V(1).Info("delete snapshot output", "output", string(output))
|
||||
if err != nil {
|
||||
log.Error(err, "unable to delete the snapshot")
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@ -1,4 +1,4 @@
|
||||
package backupsession
|
||||
package session
|
||||
|
||||
import (
|
||||
"context"
|
||||
@ -49,7 +49,7 @@ func init() {
|
||||
}
|
||||
}
|
||||
|
||||
func BackupSessionUpdateStatus(state formolv1alpha1.SessionState, snapshotId string, duration time.Duration) error {
|
||||
func BackupSessionUpdateTargetStatus(state formolv1alpha1.SessionState, snapshotId string) error {
|
||||
log := logger.WithName("BackupSessionUpdateStatus")
|
||||
targetName := os.Getenv("TARGET_NAME")
|
||||
backupSession := &formolv1alpha1.BackupSession{}
|
||||
@ -64,7 +64,7 @@ func BackupSessionUpdateStatus(state formolv1alpha1.SessionState, snapshotId str
|
||||
if target.Name == targetName {
|
||||
backupSession.Status.Targets[i].SessionState = state
|
||||
backupSession.Status.Targets[i].SnapshotId = snapshotId
|
||||
backupSession.Status.Targets[i].Duration = &metav1.Duration{Duration: duration}
|
||||
backupSession.Status.Targets[i].Duration = &metav1.Duration{Duration: time.Now().Sub(backupSession.Status.Targets[i].StartTime.Time)}
|
||||
}
|
||||
}
|
||||
|
||||
@ -75,6 +75,31 @@ func BackupSessionUpdateStatus(state formolv1alpha1.SessionState, snapshotId str
|
||||
return nil
|
||||
}
|
||||
|
||||
func RestoreSessionUpdateTargetStatus(state formolv1alpha1.SessionState) error {
|
||||
log := logger.WithName("RestoreSessionUpdateStatus")
|
||||
targetName := os.Getenv("TARGET_NAME")
|
||||
restoreSession := &formolv1alpha1.RestoreSession{}
|
||||
if err := cl.Get(context.Background(), client.ObjectKey{
|
||||
Namespace: os.Getenv("RESTORESESSION_NAMESPACE"),
|
||||
Name: os.Getenv("RESTORESESSION_NAME"),
|
||||
}, restoreSession); err != nil {
|
||||
log.Error(err, "unable to get backupsession", "RESTORESESSION_NAME", os.Getenv("RESTORESESSION_NAME"), "RESTORESESSION_NAMESPACE", os.Getenv("RESTORESESSION_NAMESPACE"))
|
||||
return err
|
||||
}
|
||||
for i, target := range restoreSession.Status.Targets {
|
||||
if target.Name == targetName {
|
||||
restoreSession.Status.Targets[i].SessionState = state
|
||||
restoreSession.Status.Targets[i].Duration = &metav1.Duration{Duration: time.Now().Sub(restoreSession.Status.Targets[i].StartTime.Time)}
|
||||
}
|
||||
}
|
||||
|
||||
if err := cl.Status().Update(context.Background(), restoreSession); err != nil {
|
||||
log.Error(err, "unable to update restoresession status", "restoresession", restoreSession)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func CreateBackupSession(name string, namespace string) {
|
||||
log := logger.WithName("CreateBackupSession")
|
||||
log.V(0).Info("CreateBackupSession called")
|
||||
Loading…
Reference in New Issue
Block a user