Skip to content

Commit

Permalink
fix: support more versions of restic
Browse files Browse the repository at this point in the history
  • Loading branch information
@garethgeorge
garethgeorge committed Nov 18, 2023
1 parent 667d766 commit 0cdfd11
Show file tree
Hide file tree
Showing 3 changed files with 1,171 additions and 3,033 deletions.
97 changes: 50 additions & 47 deletions internal/database/oplog/oplog_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@ import (
)

func TestCreate(t *testing.T) {
t.Parallel()
// t.Parallel()
log, err := NewOpLog(t.TempDir() + "/test.boltdb")
t.Cleanup(func() { log.Close() })
if err != nil {
t.Fatalf("error creating oplog: %s", err)
}
Expand All @@ -19,16 +20,16 @@ func TestCreate(t *testing.T) {
}

func TestAddOperation(t *testing.T) {
t.Parallel()
// t.Parallel()
log, err := NewOpLog(t.TempDir() + "/test.boltdb")
if err != nil {
t.Fatalf("error creating oplog: %s", err)
}
defer log.Close()
t.Cleanup(func() { log.Close() })

var tests = []struct {
name string
op *v1.Operation
name string
op *v1.Operation
wantErr bool
}{
{
Expand Down Expand Up @@ -65,8 +66,7 @@ func TestAddOperation(t *testing.T) {
op: &v1.Operation{
Id: 0,
Op: &v1.Operation_OperationIndexSnapshot{
OperationIndexSnapshot: &v1.OperationIndexSnapshot{
},
OperationIndexSnapshot: &v1.OperationIndexSnapshot{},
},
},
wantErr: true,
Expand All @@ -82,17 +82,17 @@ func TestAddOperation(t *testing.T) {
{
name: "operation with repo",
op: &v1.Operation{
Id: 0,
Id: 0,
RepoId: "testrepo",
Op: &v1.Operation_OperationBackup{},
Op: &v1.Operation_OperationBackup{},
},
},
{
name: "operation with plan",
op: &v1.Operation{
Id: 0,
Id: 0,
PlanId: "testplan",
Op: &v1.Operation_OperationBackup{},
Op: &v1.Operation_OperationBackup{},
},
},
}
Expand All @@ -112,31 +112,32 @@ func TestAddOperation(t *testing.T) {
}

func TestListOperation(t *testing.T) {
t.Parallel()
// t.Parallel()
log, err := NewOpLog(t.TempDir() + "/test.boltdb")
if err != nil {
t.Fatalf("error creating oplog: %s", err)
}
t.Cleanup(func() { log.Close() })

// these should get assigned IDs 1-3 respectively by the oplog
ops := []*v1.Operation{
{
PlanId: "plan1",
RepoId: "repo1",
PlanId: "plan1",
RepoId: "repo1",
DisplayMessage: "op1",
Op: &v1.Operation_OperationBackup{},
Op: &v1.Operation_OperationBackup{},
},
{
PlanId: "plan1",
RepoId: "repo2",
PlanId: "plan1",
RepoId: "repo2",
DisplayMessage: "op2",
Op: &v1.Operation_OperationBackup{},
Op: &v1.Operation_OperationBackup{},
},
{
PlanId: "plan2",
RepoId: "repo2",
PlanId: "plan2",
RepoId: "repo2",
DisplayMessage: "op3",
Op: &v1.Operation_OperationBackup{},
Op: &v1.Operation_OperationBackup{},
},
}

Expand All @@ -147,42 +148,42 @@ func TestListOperation(t *testing.T) {
}

tests := []struct {
name string
byPlan bool
byRepo bool
id string
name string
byPlan bool
byRepo bool
id string
expected []string
}{
{
name: "list plan1",
byPlan: true,
id: "plan1",
name: "list plan1",
byPlan: true,
id: "plan1",
expected: []string{"op1", "op2"},
},
{
name: "list plan2",
byPlan: true,
id: "plan2",
name: "list plan2",
byPlan: true,
id: "plan2",
expected: []string{"op3"},
},
{
name: "list repo1",
byRepo: true,
id: "repo1",
name: "list repo1",
byRepo: true,
id: "repo1",
expected: []string{"op1"},
},
{
name: "list repo2",
byRepo: true,
id: "repo2",
name: "list repo2",
byRepo: true,
id: "repo2",
expected: []string{"op2", "op3"},
},
}

for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
// t.Parallel()
var ops []*v1.Operation
var err error
if tc.byPlan {
Expand All @@ -204,17 +205,18 @@ func TestListOperation(t *testing.T) {
}

func TestBigIO(t *testing.T) {
t.Parallel()
// t.Parallel()
log, err := NewOpLog(t.TempDir() + "/test.boltdb")
if err != nil {
t.Fatalf("error creating oplog: %s", err)
}
t.Cleanup(func() { log.Close() })

for i := 0; i < 1000; i++ {
for i := 0; i < 100; i++ {
if err := log.Add(&v1.Operation{
PlanId: "plan1",
RepoId: "repo1",
Op: &v1.Operation_OperationBackup{},
Op: &v1.Operation_OperationBackup{},
}); err != nil {
t.Fatalf("error adding operation: %s", err)
}
Expand All @@ -224,16 +226,16 @@ func TestBigIO(t *testing.T) {
if err != nil {
t.Fatalf("error listing operations: %s", err)
}
if len(ops) != 1000 {
t.Errorf("want 1000 operations, got %d", len(ops))
if len(ops) != 100 {
t.Errorf("want 100 operations, got %d", len(ops))
}

ops, err = log.GetByRepo("repo1", FilterKeepAll())
if err != nil {
t.Fatalf("error listing operations: %s", err)
}
if len(ops) != 1000 {
t.Errorf("want 1000 operations, got %d", len(ops))
if len(ops) != 100 {
t.Errorf("want 100 operations, got %d", len(ops))
}
}

Expand All @@ -243,6 +245,7 @@ func TestIndexSnapshot(t *testing.T) {
if err != nil {
t.Fatalf("error creating oplog: %s", err)
}
t.Cleanup(func() { log.Close() })

op := &v1.Operation{
PlanId: "plan1",
Expand All @@ -266,7 +269,7 @@ func TestIndexSnapshot(t *testing.T) {
if id != op.Id {
t.Fatalf("want id %d, got %d", op.Id, id)
}

id, err = log.HasIndexedSnapshot("notfound")
if err != nil {
t.Fatalf("error checking for snapshot: %s", err)
Expand All @@ -282,4 +285,4 @@ func collectMessages(ops []*v1.Operation) []string {
messages = append(messages, op.DisplayMessage)
}
return messages
}
}
47 changes: 16 additions & 31 deletions internal/orchestrator/repo.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package orchestrator
import (
"context"
"fmt"
"slices"
"sort"
"sync"
"time"
Expand All @@ -18,36 +17,32 @@ type RepoOrchestrator struct {
mu sync.Mutex

repoConfig *v1.Repo
repo *restic.Repo
repo *restic.Repo
}

func newRepoOrchestrator(repoConfig *v1.Repo, repo *restic.Repo) *RepoOrchestrator {
return &RepoOrchestrator{
repoConfig: repoConfig,
repo: repo,
repo: repo,
}
}

func (r *RepoOrchestrator) Snapshots(ctx context.Context) ([]*restic.Snapshot, error) {
snapshots, err := r.repo.Snapshots(ctx, restic.WithFlags("--latest", "1000"))
snapshots, err := r.repo.Snapshots(ctx)
if err != nil {
return nil, fmt.Errorf("restic.Snapshots: %w", err)
return nil, fmt.Errorf("get snapshots for repo %v: %w", r.repoConfig.Id, err)
}

sort.SliceStable(snapshots, func(i, j int) bool {
return snapshots[i].UnixTimeMs() < snapshots[j].UnixTimeMs()
})

sortSnapshotsByTime(snapshots)
return snapshots, nil
}

func (r *RepoOrchestrator) SnapshotsForPlan(ctx context.Context, plan *v1.Plan) ([]*restic.Snapshot, error) {
snapshots, err := r.Snapshots(ctx)
snapshots, err := r.repo.Snapshots(ctx, restic.WithFlags("--tag", tagForPlan(plan)))
if err != nil {
return nil, err
return nil, fmt.Errorf("get snapshots for plan %q: %w", plan.Id, err)
}

return filterSnapshotsForPlan(snapshots, plan), nil
sortSnapshotsByTime(snapshots)
return snapshots, nil
}

func (r *RepoOrchestrator) Backup(ctx context.Context, plan *v1.Plan, progressCallback func(event *restic.BackupProgressEntry)) (*restic.BackupProgressEntry, error) {
Expand All @@ -71,7 +66,7 @@ func (r *RepoOrchestrator) Backup(ctx context.Context, plan *v1.Plan, progressCa

if len(snapshots) > 0 {
// TODO: design a test strategy to verify that the backup parent is used correctly.
opts = append(opts, restic.WithBackupParent(snapshots[len(snapshots) - 1].Id))
opts = append(opts, restic.WithBackupParent(snapshots[len(snapshots)-1].Id))
}

summary, err := r.repo.Backup(ctx, progressCallback, opts...)
Expand Down Expand Up @@ -100,22 +95,12 @@ func (r *RepoOrchestrator) ListSnapshotFiles(ctx context.Context, snapshotId str
return lsEnts, nil
}

func filterSnapshotsForPlan(snapshots []*restic.Snapshot, plan *v1.Plan) []*restic.Snapshot {
wantTag := tagForPlan(plan)
var filtered []*restic.Snapshot
for _, snapshot := range snapshots {
if snapshot.Tags == nil {
continue
}

if slices.Contains(snapshot.Tags, wantTag) {
filtered = append(filtered, snapshot)
}
}

return filtered
}

func tagForPlan(plan *v1.Plan) string {
return fmt.Sprintf("plan:%s", plan.Id)
}

func sortSnapshotsByTime(snapshots []*restic.Snapshot) {
sort.SliceStable(snapshots, func(i, j int) bool {
return snapshots[i].UnixTimeMs() < snapshots[j].UnixTimeMs()
})
}
Loading

0 comments on commit 0cdfd11

Please sign in to comment.