feat: add extends capability to engine (#515)

* feat: validate url * feat: loader supports extends * test: add unit test for extends * feat: update extends algorithm * test: add more unit tests to increase coverage
reviewpad · Dec 9, 2022 · c5218ab · c5218ab
1 parent c6f7b00
commit c5218ab
Show file tree

Hide file tree

Showing 18 changed files with 579 additions and 53 deletions.
diff --git a/cli/cmd/check.go b/cli/cmd/check.go
@@ -6,9 +6,11 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"os"
 
 	"github.com/reviewpad/reviewpad/v3"
+	gh "github.com/reviewpad/reviewpad/v3/codehost/github"
 	"github.com/reviewpad/reviewpad/v3/lang/aladino"
 	"github.com/spf13/cobra"
 )
@@ -21,13 +23,16 @@ var checkCmd = &cobra.Command{
 	Use:   "check",
 	Short: "Check if input reviewpad file is valid",
 	RunE: func(cmd *cobra.Command, args []string) error {
+		ctx := context.Background()
+		githubClient := gh.NewGithubClientFromToken(ctx, gitHubToken)
+
 		data, err := os.ReadFile(reviewpadFile)
 		if err != nil {
 			return err
 		}
 
 		buf := bytes.NewBuffer(data)
-		reviewpadFile, err := reviewpad.Load(buf)
+		reviewpadFile, err := reviewpad.Load(ctx, githubClient, buf)
 		if err != nil {
 			return err
 		}

diff --git a/cli/cmd/run.go b/cli/cmd/run.go
@@ -113,7 +113,7 @@ func run() error {
 	}
 
 	buf := bytes.NewBuffer(data)
-	file, err := reviewpad.Load(buf)
+	file, err := reviewpad.Load(ctx, githubClient, buf)
 	if err != nil {
 		return fmt.Errorf("error running reviewpad team edition. Details %v", err.Error())
 	}

diff --git a/engine/exec_test.go b/engine/exec_test.go
@@ -5,6 +5,7 @@
 package engine_test
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"testing"
@@ -13,7 +14,6 @@ import (
 	"github.com/migueleliasweb/go-github-mock/src/mock"
 	gh "github.com/reviewpad/reviewpad/v3/codehost/github"
 	"github.com/reviewpad/reviewpad/v3/engine"
-	"github.com/reviewpad/reviewpad/v3/engine/testutils"
 	"github.com/reviewpad/reviewpad/v3/handler"
 	"github.com/reviewpad/reviewpad/v3/lang/aladino"
 	"github.com/reviewpad/reviewpad/v3/utils"
@@ -23,6 +23,8 @@ import (
 func TestEval_WhenGitHubRequestsFail(t *testing.T) {
 	tests := map[string]struct {
 		inputReviewpadFilePath string
+		inputContext           context.Context
+		inputGitHubClient      *gh.GithubClient
 		clientOptions          []mock.MockBackendOption
 		wantErr                string
 	}{
@@ -90,12 +92,12 @@ func TestEval_WhenGitHubRequestsFail(t *testing.T) {
 				assert.FailNow(t, fmt.Sprintf("engine MockEnvWith: %v", err))
 			}
 
-			reviewpadFileData, err := utils.LoadFile(test.inputReviewpadFilePath)
+			reviewpadFileData, err := utils.ReadFile(test.inputReviewpadFilePath)
 			if err != nil {
 				assert.FailNow(t, fmt.Sprintf("Error reading reviewpad file: %v", err))
 			}
 
-			reviewpadFile, err := testutils.ParseReviewpadFile(reviewpadFileData)
+			reviewpadFile, err := engine.Load(test.inputContext, test.inputGitHubClient, reviewpadFileData)
 			if err != nil {
 				assert.FailNow(t, "Error parsing reviewpad file: %v", err)
 			}
@@ -111,6 +113,8 @@ func TestEval_WhenGitHubRequestsFail(t *testing.T) {
 func TestEval(t *testing.T) {
 	tests := map[string]struct {
 		inputReviewpadFilePath string
+		inputContext           context.Context
+		inputGitHubClient      *gh.GithubClient
 		clientOptions          []mock.MockBackendOption
 		targetEntity           *handler.TargetEntity
 		eventData              *handler.EventData
@@ -371,12 +375,12 @@ func TestEval(t *testing.T) {
 				assert.FailNow(t, fmt.Sprintf("engine MockEnvWith: %v", err))
 			}
 
-			reviewpadFileData, err := utils.LoadFile(test.inputReviewpadFilePath)
+			reviewpadFileData, err := utils.ReadFile(test.inputReviewpadFilePath)
 			if err != nil {
 				assert.FailNow(t, fmt.Sprintf("Error reading reviewpad file: %v", err))
 			}
 
-			reviewpadFile, err := testutils.ParseReviewpadFile(reviewpadFileData)
+			reviewpadFile, err := engine.Load(test.inputContext, test.inputGitHubClient, reviewpadFileData)
 			if err != nil {
 				assert.FailNow(t, fmt.Sprintf("Error parsing reviewpad file: %v", err))
 			}

diff --git a/engine/lang.go b/engine/lang.go
@@ -198,6 +198,7 @@ type ReviewpadFile struct {
 	Mode         string              `yaml:"mode"`
 	IgnoreErrors bool                `yaml:"ignore-errors"`
 	Imports      []PadImport         `yaml:"imports"`
+	Extends      []string            `yaml:"extends"`
 	Groups       []PadGroup          `yaml:"groups"`
 	Rules        []PadRule           `yaml:"rules"`
 	Labels       map[string]PadLabel `yaml:"labels"`
@@ -244,6 +245,16 @@ func (r *ReviewpadFile) equals(o *ReviewpadFile) bool {
 		}
 	}
 
+	if len(r.Extends) != len(o.Extends) {
+		return false
+	}
+	for i, rE := range r.Extends {
+		oE := o.Extends[i]
+		if rE != oE {
+			return false
+		}
+	}
+
 	if len(r.Rules) != len(o.Rules) {
 		return false
 	}
@@ -298,28 +309,60 @@ func (r *ReviewpadFile) appendLabels(o *ReviewpadFile) {
 	}
 }
 
-func (r *ReviewpadFile) appendRules(o *ReviewpadFile) {
-	if r.Rules == nil {
-		r.Rules = make([]PadRule, 0)
+func (r *ReviewpadFile) appendGroups(o *ReviewpadFile) {
+	updatedGroups := make([]PadGroup, 0)
+
+	for _, group := range r.Groups {
+		if _, ok := findGroup(o.Groups, group.Name); !ok {
+			updatedGroups = append(updatedGroups, group)
+		}
 	}
 
-	r.Rules = append(r.Rules, o.Rules...)
+	r.Groups = append(updatedGroups, o.Groups...)
 }
 
-func (r *ReviewpadFile) appendGroups(o *ReviewpadFile) {
-	if r.Groups == nil {
-		r.Groups = make([]PadGroup, 0)
+func (r *ReviewpadFile) appendRules(o *ReviewpadFile) {
+	updatedRules := make([]PadRule, 0)
+
+	for _, rule := range r.Rules {
+		if _, ok := findRule(o.Rules, rule.Name); !ok {
+			updatedRules = append(updatedRules, rule)
+		}
 	}
 
-	r.Groups = append(r.Groups, o.Groups...)
+	r.Rules = append(updatedRules, o.Rules...)
 }
 
 func (r *ReviewpadFile) appendWorkflows(o *ReviewpadFile) {
-	if r.Workflows == nil {
-		r.Workflows = make([]PadWorkflow, 0)
+	updatedWorkflows := make([]PadWorkflow, 0)
+
+	for _, workflow := range r.Workflows {
+		if _, ok := findWorkflow(o.Workflows, workflow.Name); !ok {
+			updatedWorkflows = append(updatedWorkflows, workflow)
+		}
+	}
+
+	r.Workflows = append(updatedWorkflows, o.Workflows...)
+}
+
+func (r *ReviewpadFile) appendPipelines(o *ReviewpadFile) {
+	updatedPipelines := make([]PadPipeline, 0)
+
+	for _, pipeline := range r.Pipelines {
+		if _, ok := findPipeline(o.Pipelines, pipeline.Name); !ok {
+			updatedPipelines = append(updatedPipelines, pipeline)
+		}
 	}
 
-	r.Workflows = append(r.Workflows, o.Workflows...)
+	r.Pipelines = append(updatedPipelines, o.Pipelines...)
+}
+
+func (r *ReviewpadFile) extend(o *ReviewpadFile) {
+	r.appendLabels(o)
+	r.appendGroups(o)
+	r.appendRules(o)
+	r.appendWorkflows(o)
+	r.appendPipelines(o)
 }
 
 func findGroup(groups []PadGroup, name string) (*PadGroup, bool) {
@@ -341,3 +384,23 @@ func findRule(rules []PadRule, name string) (*PadRule, bool) {
 
 	return nil, false
 }
+
+func findWorkflow(workflows []PadWorkflow, name string) (*PadWorkflow, bool) {
+	for _, workflow := range workflows {
+		if workflow.Name == name {
+			return &workflow, true
+		}
+	}
+
+	return nil, false
+}
+
+func findPipeline(pipelines []PadPipeline, name string) (*PadPipeline, bool) {
+	for _, pipeline := range pipelines {
+		if pipeline.Name == name {
+			return &pipeline, true
+		}
+	}
+
+	return nil, false
+}
diff --git a/engine/loader.go b/engine/loader.go
@@ -5,12 +5,15 @@
 package engine
 
 import (
+	"context"
 	"crypto/sha256"
 	"fmt"
 	"io"
 	"net/http"
 
+	gh "github.com/reviewpad/reviewpad/v3/codehost/github"
 	"github.com/reviewpad/reviewpad/v3/handler"
+	"github.com/reviewpad/reviewpad/v3/utils"
 	"gopkg.in/yaml.v3"
 )
 
@@ -25,7 +28,7 @@ func hash(data []byte) string {
 	return dHash
 }
 
-func Load(data []byte) (*ReviewpadFile, error) {
+func Load(ctx context.Context, githubClient *gh.GithubClient, data []byte) (*ReviewpadFile, error) {
 	file, err := parse(data)
 	if err != nil {
 		return nil, err
@@ -48,6 +51,11 @@ func Load(data []byte) (*ReviewpadFile, error) {
 		return nil, err
 	}
 
+	file, err = processExtends(ctx, githubClient, file, env)
+	if err != nil {
+		return nil, err
+	}
+
 	file, err = normalize(file, inlineRulesNormalizer())
 	if err != nil {
 		return nil, err
@@ -147,6 +155,7 @@ func transform(file *ReviewpadFile) *ReviewpadFile {
 		Mode:         file.Mode,
 		IgnoreErrors: file.IgnoreErrors,
 		Imports:      file.Imports,
+		Extends:      file.Extends,
 		Groups:       file.Groups,
 		Rules:        transformedRules,
 		Labels:       file.Labels,
@@ -213,10 +222,66 @@ func processImports(file *ReviewpadFile, env *LoadEnv) (*ReviewpadFile, error) {
 		file.appendGroups(subTreeFile)
 		file.appendRules(subTreeFile)
 		file.appendWorkflows(subTreeFile)
+		file.appendPipelines(subTreeFile)
 	}
 
 	// reset all imports
 	file.Imports = []PadImport{}
 
 	return file, nil
 }
+
+func loadExtension(ctx context.Context, githubClient *gh.GithubClient, extension string) (*ReviewpadFile, string, error) {
+	branch, filePath, err := utils.ValidateUrl(extension)
+	if err != nil {
+		return nil, "", err
+	}
+
+	content, err := githubClient.DownloadContents(ctx, filePath, branch)
+	if err != nil {
+		return nil, "", err
+	}
+
+	file, err := parse(content)
+	if err != nil {
+		return nil, "", err
+	}
+
+	return file, hash(content), nil
+}
+
+// processExtends inlines files into the current reviewpad file
+// precedence: current file > extends file
+// Post-condition: ReviewpadFile without extends statements
+func processExtends(ctx context.Context, githubClient *gh.GithubClient, file *ReviewpadFile, env *LoadEnv) (*ReviewpadFile, error) {
+	extendedFile := &ReviewpadFile{}
+	for _, extensionUri := range file.Extends {
+		eFile, eHash, err := loadExtension(ctx, githubClient, extensionUri)
+		if err != nil {
+			return nil, err
+		}
+
+		if _, ok := env.Stack[eHash]; ok {
+			return nil, fmt.Errorf("loader: cyclic extends dependency")
+		}
+
+		env.Stack[eHash] = true
+
+		extensionFile, err := processExtends(ctx, githubClient, eFile, env)
+		if err != nil {
+			return nil, err
+		}
+
+		// remove from the stack
+		delete(env.Stack, eHash)
+
+		extendedFile.extend(extensionFile)
+	}
+
+	extendedFile.extend(file)
+
+	// reset all extends
+	extendedFile.Extends = []string{}
+
+	return extendedFile, nil
+}