Skip to content

Commit

Permalink
(DRON-392) cascade deletes on purge (#3243)
Browse files Browse the repository at this point in the history 
* (DRON-392) cascade deletes on purge
* move cascade purge to build.go
TP Honey authored Sep 8, 2022
1 parent 8b6564f commit 730425f
Showing 5 changed files with 133 additions and 83 deletions.
2 changes: 1 addition & 1 deletion mock/mockscm/mock_gen.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

33 changes: 32 additions & 1 deletion store/build/build.go
View file
Original file line number Diff line number Diff line change
@@ -416,7 +416,7 @@ func (s *buildStore) Purge(ctx context.Context, repo, number int64) error {
RepoID: repo,
Number: number,
}
return s.db.Lock(func(execer db.Execer, binder db.Binder) error {
stageErr := s.db.Lock(func(execer db.Execer, binder db.Binder) error {
params := toParams(build)
stmt, args, err := binder.BindNamed(stmtPurge, params)
if err != nil {
@@ -425,6 +425,26 @@ func (s *buildStore) Purge(ctx context.Context, repo, number int64) error {
_, err = execer.Exec(stmt, args...)
return err
})
if stageErr != nil {
return stageErr
}
if s.db.Driver() == db.Postgres || s.db.Driver() == db.Mysql {
// purge orphaned stages
err := s.db.Update(func(execer db.Execer, binder db.Binder) error {
_, err := execer.Exec(stmtStagePurge)
return err
})
if err != nil {
return err
}
// purge orphaned steps
err = s.db.Update(func(execer db.Execer, binder db.Binder) error {
_, err := execer.Exec(stmtStepPurge)
return err
})
return err
}
return nil
}

// Count returns a count of builds.
@@ -727,6 +747,17 @@ DELETE FROM builds
WHERE build_repo_id = :build_repo_id
AND build_number < :build_number
`
const stmtStagePurge = `
DELETE FROM stages
WHERE stage_build_id NOT IN (
SELECT build_id FROM builds
)`

const stmtStepPurge = `
DELETE FROM steps
WHERE step_stage_id NOT IN (
SELECT stage_id FROM stages
)`

//
// latest builds index
126 changes: 88 additions & 38 deletions store/build/build_test.go
View file
Original file line number Diff line number Diff line change
@@ -7,6 +7,7 @@ package build
import (
"context"
"database/sql"
"fmt"
"testing"

"github.com/drone/drone/core"
@@ -25,7 +26,7 @@ func TestBuild(t *testing.T) {
}
defer func() {
dbtest.Reset(conn)
dbtest.Disconnect(conn)
_ = dbtest.Disconnect(conn)
}()

store := New(conn).(*buildStore)
@@ -201,14 +202,29 @@ func testBuildDelete(store *buildStore, build *core.Build) func(t *testing.T) {

func testBuildPurge(store *buildStore) func(t *testing.T) {
return func(t *testing.T) {
store.db.Update(func(execer db.Execer, binder db.Binder) error {
_, err := execer.Exec("DELETE FROM builds")
return err
_ = store.db.Update(func(execer db.Execer, binder db.Binder) error {
_, _ = execer.Exec("DELETE FROM builds")
_, _ = execer.Exec("DELETE FROM stages")
_, _ = execer.Exec("DELETE FROM steps")
return nil
})
_ = store.Create(noContext, &core.Build{RepoID: 1, Number: 98}, []*core.Stage{{RepoID: 1, Number: 1, Status: core.StatusPending}})
_ = store.Create(noContext, &core.Build{RepoID: 1, Number: 99}, []*core.Stage{{RepoID: 1, Number: 1, Status: core.StatusPending}})
_ = store.Create(noContext, &core.Build{RepoID: 1, Number: 100}, []*core.Stage{{RepoID: 1, Number: 1, Status: core.StatusPending}})
_ = store.Create(noContext, &core.Build{RepoID: 1, Number: 101}, []*core.Stage{{RepoID: 1, Number: 1, Status: core.StatusPending}})
// get the first stageid
var startingStageID int64
_ = store.db.View(func(queryer db.Queryer, binder db.Binder) error {
return queryer.QueryRow("SELECT stage_id FROM stages limit 1").Scan(&startingStageID)
})
// lets add steps to the builds
_ = store.db.Update(func(execer db.Execer, binder db.Binder) error {
_, _ = execer.Exec(fmt.Sprintf("INSERT INTO steps (step_stage_id, step_number, step_status) VALUES (%d, 1, 'pending')", startingStageID))
_, _ = execer.Exec(fmt.Sprintf("INSERT INTO steps (step_stage_id, step_number, step_status) VALUES (%d, 1, 'pending')", startingStageID+1))
_, _ = execer.Exec(fmt.Sprintf("INSERT INTO steps (step_stage_id, step_number, step_status) VALUES (%d, 1, 'pending')", startingStageID+2))
_, _ = execer.Exec(fmt.Sprintf("INSERT INTO steps (step_stage_id, step_number, step_status) VALUES (%d, 1, 'pending')", startingStageID+3))
return nil
})
store.Create(noContext, &core.Build{RepoID: 1, Number: 98}, nil)
store.Create(noContext, &core.Build{RepoID: 1, Number: 99}, nil)
store.Create(noContext, &core.Build{RepoID: 1, Number: 100}, nil)
store.Create(noContext, &core.Build{RepoID: 1, Number: 101}, nil)

before, err := store.List(noContext, 1, 100, 0)
if err != nil {
@@ -217,6 +233,25 @@ func testBuildPurge(store *buildStore) func(t *testing.T) {
if got, want := len(before), 4; got != want {
t.Errorf("Want build count %d, got %d", want, got)
}
// count the number of stages
countOfStages := 4
_ = store.db.View(func(queryer db.Queryer, binder db.Binder) error {
return queryer.QueryRow("SELECT count(*) FROM stages").Scan(&countOfStages)
})
want := 4
if want != countOfStages {
t.Errorf("Want stage count %d, got %d", want, countOfStages)
}
// count the number of steps
countOfSteps := 4
_ = store.db.View(func(queryer db.Queryer, binder db.Binder) error {
return queryer.QueryRow("SELECT count(*) FROM steps").Scan(&countOfSteps)
})
want = 4
if want != countOfSteps {
t.Errorf("Want step count %d, got %d", want, countOfSteps)
}
// purge the builds
err = store.Purge(noContext, 1, 100)
if err != nil {
t.Error(err)
@@ -225,27 +260,48 @@ func testBuildPurge(store *buildStore) func(t *testing.T) {
if err != nil {
t.Error(err)
}
if got, want := len(after), 2; got != want {
// we want 2 builds
want = 2
got := len(after)
if got != want {
t.Errorf("Want build count %d, got %d", want, got)
}
for _, build := range after {
if build.Number < 100 {
t.Errorf("Expect purge if build number is less than 100")
}
}
// check that orphaned stages are deleted
_ = store.db.View(func(queryer db.Queryer, binder db.Binder) error {
return queryer.QueryRow("SELECT count(*) FROM stages").Scan(&countOfStages)
})
want = 2
if want != countOfStages {
t.Errorf("Want stage count %d, got %d", want, countOfStages)
}
// check that orphaned steps are deleted
// count the number of steps
countOfSteps = 2
_ = store.db.View(func(queryer db.Queryer, binder db.Binder) error {
return queryer.QueryRow("SELECT count(*) FROM steps").Scan(&countOfSteps)
})
want = 2
if want != countOfSteps {
t.Errorf("Want step count %d, got %d", want, countOfSteps)
}
}
}

func testBuildCount(store *buildStore) func(t *testing.T) {
return func(t *testing.T) {
store.db.Update(func(execer db.Execer, binder db.Binder) error {
_ = store.db.Update(func(execer db.Execer, binder db.Binder) error {
_, err := execer.Exec("DELETE FROM builds")
return err
})
store.Create(noContext, &core.Build{RepoID: 1, Number: 98}, nil)
store.Create(noContext, &core.Build{RepoID: 1, Number: 99}, nil)
store.Create(noContext, &core.Build{RepoID: 1, Number: 100}, nil)
store.Create(noContext, &core.Build{RepoID: 1, Number: 101}, nil)
_ = store.Create(noContext, &core.Build{RepoID: 1, Number: 98}, nil)
_ = store.Create(noContext, &core.Build{RepoID: 1, Number: 99}, nil)
_ = store.Create(noContext, &core.Build{RepoID: 1, Number: 100}, nil)
_ = store.Create(noContext, &core.Build{RepoID: 1, Number: 101}, nil)

count, err := store.Count(noContext)
if err != nil {
@@ -258,15 +314,15 @@ func testBuildCount(store *buildStore) func(t *testing.T) {

func testBuildPending(store *buildStore) func(t *testing.T) {
return func(t *testing.T) {
store.db.Update(func(execer db.Execer, binder db.Binder) error {
execer.Exec("DELETE FROM builds")
execer.Exec("DELETE FROM stages")
_ = store.db.Update(func(execer db.Execer, binder db.Binder) error {
_, _ = execer.Exec("DELETE FROM builds")
_, _ = execer.Exec("DELETE FROM stages")
return nil
})
store.Create(noContext, &core.Build{RepoID: 1, Number: 98, Status: core.StatusPending}, []*core.Stage{&core.Stage{RepoID: 1, Number: 1, Status: core.StatusPending}})
store.Create(noContext, &core.Build{RepoID: 1, Number: 99, Status: core.StatusPending}, []*core.Stage{&core.Stage{RepoID: 1, Number: 1, Status: core.StatusPending}})
store.Create(noContext, &core.Build{RepoID: 1, Number: 100, Status: core.StatusRunning}, []*core.Stage{&core.Stage{RepoID: 1, Number: 1, Status: core.StatusRunning}})
store.Create(noContext, &core.Build{RepoID: 1, Number: 101, Status: core.StatusPassing}, []*core.Stage{&core.Stage{RepoID: 1, Number: 1, Status: core.StatusPassing}})
_ = store.Create(noContext, &core.Build{RepoID: 1, Number: 98, Status: core.StatusPending}, []*core.Stage{{RepoID: 1, Number: 1, Status: core.StatusPending}})
_ = store.Create(noContext, &core.Build{RepoID: 1, Number: 99, Status: core.StatusPending}, []*core.Stage{{RepoID: 1, Number: 1, Status: core.StatusPending}})
_ = store.Create(noContext, &core.Build{RepoID: 1, Number: 100, Status: core.StatusRunning}, []*core.Stage{{RepoID: 1, Number: 1, Status: core.StatusRunning}})
_ = store.Create(noContext, &core.Build{RepoID: 1, Number: 101, Status: core.StatusPassing}, []*core.Stage{{RepoID: 1, Number: 1, Status: core.StatusPassing}})

count, err := store.Count(noContext)
if err != nil {
@@ -285,15 +341,15 @@ func testBuildPending(store *buildStore) func(t *testing.T) {

func testBuildRunning(store *buildStore) func(t *testing.T) {
return func(t *testing.T) {
store.db.Update(func(execer db.Execer, binder db.Binder) error {
execer.Exec("DELETE FROM builds")
execer.Exec("DELETE FROM stages")
_ = store.db.Update(func(execer db.Execer, binder db.Binder) error {
_, _ = execer.Exec("DELETE FROM builds")
_, _ = execer.Exec("DELETE FROM stages")
return nil
})
store.Create(noContext, &core.Build{RepoID: 1, Number: 98, Status: core.StatusRunning}, []*core.Stage{&core.Stage{RepoID: 1, Number: 1, Status: core.StatusRunning}})
store.Create(noContext, &core.Build{RepoID: 1, Number: 99, Status: core.StatusRunning}, []*core.Stage{&core.Stage{RepoID: 1, Number: 1, Status: core.StatusRunning}})
store.Create(noContext, &core.Build{RepoID: 1, Number: 100, Status: core.StatusBlocked}, []*core.Stage{&core.Stage{RepoID: 1, Number: 1, Status: core.StatusBlocked}})
store.Create(noContext, &core.Build{RepoID: 1, Number: 101, Status: core.StatusPassing}, []*core.Stage{&core.Stage{RepoID: 1, Number: 1, Status: core.StatusPassing}})
_ = store.Create(noContext, &core.Build{RepoID: 1, Number: 98, Status: core.StatusRunning}, []*core.Stage{{RepoID: 1, Number: 1, Status: core.StatusRunning}})
_ = store.Create(noContext, &core.Build{RepoID: 1, Number: 99, Status: core.StatusRunning}, []*core.Stage{{RepoID: 1, Number: 1, Status: core.StatusRunning}})
_ = store.Create(noContext, &core.Build{RepoID: 1, Number: 100, Status: core.StatusBlocked}, []*core.Stage{{RepoID: 1, Number: 1, Status: core.StatusBlocked}})
_ = store.Create(noContext, &core.Build{RepoID: 1, Number: 101, Status: core.StatusPassing}, []*core.Stage{{RepoID: 1, Number: 1, Status: core.StatusPassing}})

count, err := store.Count(noContext)
if err != nil {
@@ -312,17 +368,14 @@ func testBuildRunning(store *buildStore) func(t *testing.T) {

func testBuildLatest(store *buildStore) func(t *testing.T) {
return func(t *testing.T) {
store.db.Update(func(execer db.Execer, binder db.Binder) error {
execer.Exec("DELETE FROM stages")
execer.Exec("DELETE FROM latest")
execer.Exec("DELETE FROM builds")
_ = store.db.Update(func(execer db.Execer, binder db.Binder) error {
_, _ = execer.Exec("DELETE FROM stages")
_, _ = execer.Exec("DELETE FROM latest")
_, _ = execer.Exec("DELETE FROM builds")
return nil
})

//
// step 1: insert the initial builds
//

build := &core.Build{
RepoID: 1,
Number: 99,
@@ -363,10 +416,7 @@ func testBuildLatest(store *buildStore) func(t *testing.T) {
return
}

//
// step 2: verify the latest build number was captured
//

latest, _ := store.LatestBranches(noContext, build.RepoID)
if len(latest) != 2 {
t.Errorf("Expect latest branch list == 1, got %d", len(latest))
36 changes: 2 additions & 34 deletions store/stage/stage.go
View file
Original file line number Diff line number Diff line change
@@ -22,8 +22,8 @@ import (
)

// New returns a new StageStore.
func New(db *db.DB) core.StageStore {
return &stageStore{db}
func New(database *db.DB) core.StageStore {
return &stageStore{database}
}

type stageStore struct {
@@ -422,35 +422,3 @@ INSERT INTO stages (
const stmtInsertPg = stmtInsert + `
RETURNING stage_id
`

const stmtInsertStep = `
INSERT INTO steps (
step_stage_id
,step_number
,step_name
,step_status
,step_error
,step_errignore
,step_exit_code
,step_started
,step_stopped
,step_version
,step_depends_on
,step_image
,step_detached
) VALUES (
:step_stage_id
,:step_number
,:step_name
,:step_status
,:step_error
,:step_errignore
,:step_exit_code
,:step_started
,:step_stopped
,:step_version
,:step_depends_on
,:step_image
,:step_detached
)
`
19 changes: 10 additions & 9 deletions store/stage/stage_test.go
View file
Original file line number Diff line number Diff line change
@@ -2,6 +2,7 @@
// Use of this source code is governed by the Drone Non-Commercial License
// that can be found in the LICENSE file.

//go:build !oss
// +build !oss

package stage
@@ -45,11 +46,11 @@ func TestStage(t *testing.T) {
t.Run("ListState", testStageListStatus(store, abuild))
}

func testStageCreate(store *stageStore, build *core.Build) func(t *testing.T) {
func testStageCreate(store *stageStore, abuild *core.Build) func(t *testing.T) {
return func(t *testing.T) {
item := &core.Stage{
RepoID: 42,
BuildID: build.ID,
BuildID: abuild.ID,
Number: 2,
Name: "clone",
Status: core.StatusRunning,
@@ -194,16 +195,16 @@ func testStageLocking(store *stageStore, stage *core.Stage) func(t *testing.T) {
}
}

func testStageListStatus(store *stageStore, build *core.Build) func(t *testing.T) {
func testStageListStatus(store *stageStore, abuild *core.Build) func(t *testing.T) {
return func(t *testing.T) {
store.db.Update(func(execer db.Execer, binder db.Binder) error {
execer.Exec("DELETE FROM stages_unfinished")
execer.Exec("DELETE FROM stages")
_ = store.db.Update(func(execer db.Execer, binder db.Binder) error {
_, _ = execer.Exec("DELETE FROM stages_unfinished")
_, _ = execer.Exec("DELETE FROM stages")
return nil
})
store.Create(noContext, &core.Stage{Number: 1, BuildID: build.ID, Status: core.StatusPending})
store.Create(noContext, &core.Stage{Number: 2, BuildID: build.ID, Status: core.StatusRunning})
store.Create(noContext, &core.Stage{Number: 3, BuildID: build.ID, Status: core.StatusFailing})
_ = store.Create(noContext, &core.Stage{Number: 1, BuildID: abuild.ID, Status: core.StatusPending})
_ = store.Create(noContext, &core.Stage{Number: 2, BuildID: abuild.ID, Status: core.StatusRunning})
_ = store.Create(noContext, &core.Stage{Number: 3, BuildID: abuild.ID, Status: core.StatusFailing})
list, err := store.ListState(noContext, core.StatusPending)
if err != nil {
t.Error(err)

0 comments on commit 730425f

Please sign in to comment.