Skip to content

Commit

Permalink
feat: implement forget and prune support in restic pkg
Browse files Browse the repository at this point in the history
  • Loading branch information
@garethgeorge
garethgeorge committed Nov 25, 2023
1 parent b507d42 commit ffb4573
Show file tree
Hide file tree
Showing 3 changed files with 258 additions and 82 deletions.
61 changes: 33 additions & 28 deletions pkg/restic/outputs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,34 +13,6 @@ import (
v1 "github.com/garethgeorge/resticui/gen/go/v1"
)

type LsEntry struct {
Name string `json:"name"`
Type string `json:"type"`
Path string `json:"path"`
Uid int `json:"uid"`
Gid int `json:"gid"`
Size int `json:"size"`
Mode int `json:"mode"`
Mtime string `json:"mtime"`
Atime string `json:"atime"`
Ctime string `json:"ctime"`
}

func (e *LsEntry) ToProto() *v1.LsEntry {
return &v1.LsEntry{
Name: e.Name,
Type: e.Type,
Path: e.Path,
Uid: int64(e.Uid),
Gid: int64(e.Gid),
Size: int64(e.Size),
Mode: int64(e.Mode),
Mtime: e.Mtime,
Atime: e.Atime,
Ctime: e.Ctime,
}
}

type Snapshot struct {
Id string `json:"id"`
Time string `json:"time"`
Expand Down Expand Up @@ -189,6 +161,34 @@ func readBackupProgressEntries(cmd *exec.Cmd, output io.Reader, callback func(ev
return summary, nil
}

type LsEntry struct {
Name string `json:"name"`
Type string `json:"type"`
Path string `json:"path"`
Uid int `json:"uid"`
Gid int `json:"gid"`
Size int `json:"size"`
Mode int `json:"mode"`
Mtime string `json:"mtime"`
Atime string `json:"atime"`
Ctime string `json:"ctime"`
}

func (e *LsEntry) ToProto() *v1.LsEntry {
return &v1.LsEntry{
Name: e.Name,
Type: e.Type,
Path: e.Path,
Uid: int64(e.Uid),
Gid: int64(e.Gid),
Size: int64(e.Size),
Mode: int64(e.Mode),
Mtime: e.Mtime,
Atime: e.Atime,
Ctime: e.Ctime,
}
}

func readLs(output io.Reader) (*Snapshot, []*LsEntry, error) {
scanner := bufio.NewScanner(output)
scanner.Split(bufio.ScanLines)
Expand All @@ -212,3 +212,8 @@ func readLs(output io.Reader) (*Snapshot, []*LsEntry, error) {
}
return snapshot, entries, nil
}

type ForgetResult struct {
Keep []Snapshot `json:"keep"`
Remove []Snapshot `json:"remove"`
}
140 changes: 116 additions & 24 deletions pkg/restic/restic.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@ import (
)

type Repo struct {
mu sync.Mutex
cmd string
repo *v1.Repo
mu sync.Mutex
cmd string
repo *v1.Repo
initialized bool

extraArgs []string
extraEnv []string
extraEnv []string
}

// NewRepo instantiates a new repository. TODO: should not accept a v1.Repo, should instead be configured by parameters.
Expand All @@ -34,11 +34,11 @@ func NewRepo(repo *v1.Repo, opts ...GenericOption) *Repo {
}

return &Repo{
cmd: "restic", // TODO: configurable binary path
repo: repo,
cmd: "restic", // TODO: configurable binary path
repo: repo,
initialized: false,
extraArgs: opt.extraArgs,
extraEnv: opt.extraEnv,
extraArgs: opt.extraArgs,
extraEnv: opt.extraEnv,
}
}

Expand Down Expand Up @@ -85,10 +85,6 @@ func (r *Repo) Backup(ctx context.Context, progressCallback func(*BackupProgress
r.mu.Lock()
defer r.mu.Unlock()

if err := r.init(ctx); err != nil {
return nil, fmt.Errorf("failed to initialize repo: %w", err)
}

opt := &BackupOpts{}
for _, o := range opts {
o(opt)
Expand All @@ -115,12 +111,12 @@ func (r *Repo) Backup(ctx context.Context, progressCallback func(*BackupProgress
if err := cmd.Start(); err != nil {
return nil, NewCmdError(cmd, nil, err)
}

var wg sync.WaitGroup
var summary *BackupProgressEntry
var cmdErr error
var cmdErr error
var readErr error

wg.Add(1)
go func() {
defer wg.Done()
Expand All @@ -141,7 +137,7 @@ func (r *Repo) Backup(ctx context.Context, progressCallback func(*BackupProgress
}()

wg.Wait()

var err error
if cmdErr != nil || readErr != nil {
err = multierror.Append(nil, cmdErr, readErr)
Expand All @@ -153,10 +149,6 @@ func (r *Repo) Snapshots(ctx context.Context, opts ...GenericOption) ([]*Snapsho
r.mu.Lock()
defer r.mu.Unlock()

if err := r.init(ctx); err != nil {
return nil, fmt.Errorf("failed to initialize repo: %w", err)
}

opt := resolveOpts(opts)

args := []string{"snapshots", "--json"}
Expand All @@ -180,6 +172,60 @@ func (r *Repo) Snapshots(ctx context.Context, opts ...GenericOption) ([]*Snapsho
return snapshots, nil
}

func (r *Repo) Forget(ctx context.Context, policy RetentionPolicy, pruneOutput io.Writer, opts ...GenericOption) (*ForgetResult, error) {
r.mu.Lock()
defer r.mu.Unlock()

// first run the forget command
opt := resolveOpts(opts)

args := []string{"forget", "--json"}
args = append(args, r.extraArgs...)
args = append(args, opt.extraArgs...)
args = append(args, policy.toForgetFlags()...)

cmd := exec.CommandContext(ctx, r.cmd, args...)
cmd.Env = append(cmd.Env, r.buildEnv()...)
cmd.Env = append(cmd.Env, opt.extraEnv...)

output, err := cmd.CombinedOutput()
if err != nil {
return nil, NewCmdError(cmd, output, err)
}

var result []ForgetResult
if err := json.Unmarshal(output, &result); err != nil {
return nil, NewCmdError(cmd, output, fmt.Errorf("command output is not valid JSON: %w", err))
}
if len(result) != 1 {
return nil, fmt.Errorf("expected 1 output from forget, got %v", len(result))
}

// then run the prune command
args = []string{"prune", "--json"}
args = append(args, r.extraArgs...)
args = append(args, opt.extraArgs...)
args = append(args, policy.toPruneFlags()...)

cmd = exec.CommandContext(ctx, r.cmd, args...)
cmd.Env = append(cmd.Env, r.buildEnv()...)
cmd.Env = append(cmd.Env, opt.extraEnv...)

buf := bytes.NewBuffer(nil)
var writer io.Writer = buf
if pruneOutput != nil {
writer = io.MultiWriter(pruneOutput, buf)
}
cmd.Stdout = writer
cmd.Stderr = writer

if err := cmd.Run(); err != nil {
return nil, NewCmdError(cmd, buf.Bytes(), err)
}

return &result[0], nil
}

func (r *Repo) ListDirectory(ctx context.Context, snapshot string, path string, opts ...GenericOption) (*Snapshot, []*LsEntry, error) {
r.mu.Lock()
defer r.mu.Unlock()
Expand Down Expand Up @@ -216,8 +262,53 @@ func (r *Repo) ListDirectory(ctx context.Context, snapshot string, path string,
return snapshots, entries, nil
}

type RetentionPolicy struct {
MaxUnused string // e.g. a percentage i.e. 25% or a number of megabytes.
KeepLastN int // keep the last n snapshots.
KeepHourly int // keep the last n hourly snapshots.
KeepDaily int // keep the last n daily snapshots.
KeepWeekly int // keep the last n weekly snapshots.
KeepMonthly int // keep the last n monthly snapshots.
KeepYearly int // keep the last n yearly snapshots.
KeepWithinDuration string // keep snapshots within a duration e.g. 1y2m3d4h5m6s
}

func (r *RetentionPolicy) toForgetFlags() []string {
flags := []string{}
if r.KeepLastN != 0 {
flags = append(flags, "--keep-last", fmt.Sprintf("%d", r.KeepLastN))
}
if r.KeepHourly != 0 {
flags = append(flags, "--keep-hourly", fmt.Sprintf("%d", r.KeepHourly))
}
if r.KeepDaily != 0 {
flags = append(flags, "--keep-daily", fmt.Sprintf("%d", r.KeepDaily))
}
if r.KeepWeekly != 0 {
flags = append(flags, "--keep-weekly", fmt.Sprintf("%d", r.KeepWeekly))
}
if r.KeepMonthly != 0 {
flags = append(flags, "--keep-monthly", fmt.Sprintf("%d", r.KeepMonthly))
}
if r.KeepYearly != 0 {
flags = append(flags, "--keep-yearly", fmt.Sprintf("%d", r.KeepYearly))
}
if r.KeepWithinDuration != "" {
flags = append(flags, "--keep-within", r.KeepWithinDuration)
}
return flags
}

func (r *RetentionPolicy) toPruneFlags() []string {
flags := []string{}
if r.MaxUnused != "" {
flags = append(flags, "--max-unused", r.MaxUnused)
}
return flags
}

type BackupOpts struct {
paths []string
paths []string
extraArgs []string
}

Expand Down Expand Up @@ -253,7 +344,7 @@ func WithBackupParent(parent string) BackupOption {

type GenericOpts struct {
extraArgs []string
extraEnv []string
extraEnv []string
}

func resolveOpts(opts []GenericOption) *GenericOpts {
Expand Down Expand Up @@ -287,14 +378,15 @@ func WithEnv(env ...string) GenericOption {
}

var EnvToPropagate = []string{"PATH", "HOME", "XDG_CACHE_HOME"}

func WithPropagatedEnvVars(extras ...string) GenericOption {
var extension []string

for _, env := range EnvToPropagate {
if val, ok := os.LookupEnv(env); ok {
extension = append(extension, env + "=" + val)
extension = append(extension, env+"="+val)
}
}

return WithEnv(extension...)
}
}
Loading

0 comments on commit ffb4573

Please sign in to comment.