feat: loader supports extends

reviewpad · marcelosousa · Dec 9, 2022 · Dec 8, 2022 · Dec 8, 2022 · Dec 8, 2022
commit 4754fd35094e976c21b3cd8120783878a676672d
diff --git a/cli/cmd/check.go b/cli/cmd/check.go
@@ -6,9 +6,11 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"os"
 
 	"github.com/reviewpad/reviewpad/v3"
+	gh "github.com/reviewpad/reviewpad/v3/codehost/github"
 	"github.com/reviewpad/reviewpad/v3/lang/aladino"
 	"github.com/spf13/cobra"
 )
@@ -21,13 +23,16 @@ var checkCmd = &cobra.Command{
 	Use:   "check",
 	Short: "Check if input reviewpad file is valid",
 	RunE: func(cmd *cobra.Command, args []string) error {
+		ctx := context.Background()
+		githubClient := gh.NewGithubClientFromToken(ctx, gitHubToken)
+
 		data, err := os.ReadFile(reviewpadFile)
 		if err != nil {
 			return err
 		}
 
 		buf := bytes.NewBuffer(data)
-		reviewpadFile, err := reviewpad.Load(buf)
+		reviewpadFile, err := reviewpad.Load(ctx, githubClient, buf)
 		if err != nil {
 			return err
 		}

diff --git a/cli/cmd/run.go b/cli/cmd/run.go
@@ -113,7 +113,7 @@ func run() error {
 	}
 
 	buf := bytes.NewBuffer(data)
-	file, err := reviewpad.Load(buf)
+	file, err := reviewpad.Load(ctx, githubClient, buf)
 	if err != nil {
 		return fmt.Errorf("error running reviewpad team edition. Details %v", err.Error())
 	}

diff --git a/engine/exec_test.go b/engine/exec_test.go
@@ -5,6 +5,7 @@
 package engine_test
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"testing"
@@ -95,7 +96,7 @@ func TestEval_WhenGitHubRequestsFail(t *testing.T) {
 				assert.FailNow(t, fmt.Sprintf("Error reading reviewpad file: %v", err))
 			}
 
-			reviewpadFile, err := testutils.ParseReviewpadFile(reviewpadFileData)
+			reviewpadFile, err := testutils.ParseReviewpadFile(context.Background(), nil, reviewpadFileData)
 			if err != nil {
 				assert.FailNow(t, "Error parsing reviewpad file: %v", err)
 			}
@@ -376,7 +377,7 @@ func TestEval(t *testing.T) {
 				assert.FailNow(t, fmt.Sprintf("Error reading reviewpad file: %v", err))
 			}
 
-			reviewpadFile, err := testutils.ParseReviewpadFile(reviewpadFileData)
+			reviewpadFile, err := testutils.ParseReviewpadFile(context.Background(), nil, reviewpadFileData)
 			if err != nil {
 				assert.FailNow(t, fmt.Sprintf("Error parsing reviewpad file: %v", err))
 			}

diff --git a/engine/lang.go b/engine/lang.go
@@ -198,6 +198,7 @@ type ReviewpadFile struct {
 	Mode         string              `yaml:"mode"`
 	IgnoreErrors bool                `yaml:"ignore-errors"`
 	Imports      []PadImport         `yaml:"imports"`
+	Extends      []string            `yaml:"extends"`
 	Groups       []PadGroup          `yaml:"groups"`
 	Rules        []PadRule           `yaml:"rules"`
 	Labels       map[string]PadLabel `yaml:"labels"`
@@ -244,6 +245,16 @@ func (r *ReviewpadFile) equals(o *ReviewpadFile) bool {
 		}
 	}
 
+	if len(r.Extends) != len(o.Extends) {
+		return false
+	}
+	for i, rE := range r.Extends {
+		oE := o.Extends[i]
+		if rE != oE {
+			return false
+		}
+	}
+
 	if len(r.Rules) != len(o.Rules) {
 		return false
 	}
@@ -322,6 +333,87 @@ func (r *ReviewpadFile) appendWorkflows(o *ReviewpadFile) {
 	r.Workflows = append(r.Workflows, o.Workflows...)
 }
 
+func (r *ReviewpadFile) appendPipelines(o *ReviewpadFile) {
+	if r.Pipelines == nil {
+		r.Pipelines = make([]PadPipeline, 0)
+	}
+
+	r.Pipelines = append(r.Pipelines, o.Pipelines...)
+}
+
+func (r *ReviewpadFile) extends(o *ReviewpadFile) {
+	// extend labels
+	if r.Labels == nil {
+		r.Labels = make(map[string]PadLabel)
+	}
+
+	for labelName, label := range o.Labels {
+		if _, ok := r.Labels[labelName]; !ok {
+			r.Labels[labelName] = label
+		}
+	}
+
+	// extend groups
+	if r.Groups == nil {
+		r.Groups = make([]PadGroup, 0)
+	}
+
+	filteredGroups := make([]PadGroup, 0)
+
+	for _, group := range o.Groups {
+		if _, ok := findGroup(r.Groups, group.Name); !ok {
+			filteredGroups = append(filteredGroups, group)
+		}
+	}
+
+	r.Groups = append(filteredGroups, r.Groups...)
+
+	// extend rules
+	if r.Rules == nil {
+		r.Rules = make([]PadRule, 0)
+	}
+
+	filteredRules := make([]PadRule, 0)
+
+	for _, rule := range o.Rules {
+		if _, ok := findRule(r.Rules, rule.Name); !ok {
+			filteredRules = append(filteredRules, rule)
+		}
+	}
+
+	r.Rules = append(filteredRules, r.Rules...)
+
+	// extend workflows
+	if r.Workflows == nil {
+		r.Workflows = make([]PadWorkflow, 0)
+	}
+
+	filteredWorkflows := make([]PadWorkflow, 0)
+
+	for _, workflow := range o.Workflows {
+		if _, ok := findWorkflow(r.Workflows, workflow.Name); !ok {
+			filteredWorkflows = append(filteredWorkflows, workflow)
+		}
+	}
+
+	r.Workflows = append(filteredWorkflows, r.Workflows...)
+
+	// extend pipelines
+	if r.Pipelines == nil {
+		r.Pipelines = make([]PadPipeline, 0)
+	}
+
+	filteredPipelines := make([]PadPipeline, 0)
+
+	for _, pipeline := range o.Pipelines {
+		if _, ok := findPipeline(r.Pipelines, pipeline.Name); !ok {
+			filteredPipelines = append(filteredPipelines, pipeline)
+		}
+	}
+
+	r.Pipelines = append(filteredPipelines, r.Pipelines...)
+}
+
 func findGroup(groups []PadGroup, name string) (*PadGroup, bool) {
 	for _, group := range groups {
 		if group.Name == name {
@@ -341,3 +433,23 @@ func findRule(rules []PadRule, name string) (*PadRule, bool) {
 
 	return nil, false
 }
+
+func findWorkflow(workflows []PadWorkflow, name string) (*PadWorkflow, bool) {
+	for _, workflow := range workflows {
+		if workflow.Name == name {
+			return &workflow, true
+		}
+	}
+
+	return nil, false
+}
+
+func findPipeline(pipelines []PadPipeline, name string) (*PadPipeline, bool) {
+	for _, pipeline := range pipelines {
+		if pipeline.Name == name {
+			return &pipeline, true
+		}
+	}
+
+	return nil, false
+}
diff --git a/engine/loader.go b/engine/loader.go
@@ -5,12 +5,15 @@
 package engine
 
 import (
+	"context"
 	"crypto/sha256"
 	"fmt"
 	"io"
 	"net/http"
 
+	gh "github.com/reviewpad/reviewpad/v3/codehost/github"
 	"github.com/reviewpad/reviewpad/v3/handler"
+	"github.com/reviewpad/reviewpad/v3/utils"
 	"gopkg.in/yaml.v3"
 )
 
@@ -25,7 +28,7 @@ func hash(data []byte) string {
 	return dHash
 }
 
-func Load(data []byte) (*ReviewpadFile, error) {
+func Load(ctx context.Context, githubClient *gh.GithubClient, data []byte) (*ReviewpadFile, error) {
 	file, err := parse(data)
 	if err != nil {
 		return nil, err
@@ -48,6 +51,11 @@ func Load(data []byte) (*ReviewpadFile, error) {
 		return nil, err
 	}
 
+	file, err = processExtends(ctx, githubClient, file, env)
+	if err != nil {
+		return nil, err
+	}
+
 	file, err = normalize(file, inlineRulesNormalizer())
 	if err != nil {
 		return nil, err
@@ -213,10 +221,74 @@ func processImports(file *ReviewpadFile, env *LoadEnv) (*ReviewpadFile, error) {
 		file.appendGroups(subTreeFile)
 		file.appendRules(subTreeFile)
 		file.appendWorkflows(subTreeFile)
+		file.appendPipelines(subTreeFile)
 	}
 
 	// reset all imports
 	file.Imports = []PadImport{}
 
 	return file, nil
 }
+
+func loadExtension(ctx context.Context, githubClient *gh.GithubClient, extension string) (*ReviewpadFile, string, error) {
+	branch, filePath, err := utils.ValidateUrl(extension)
+	if err != nil {
+		return nil, "", err
+	}
+
+	content, err := githubClient.DownloadContents(ctx, filePath, branch)
+	if err != nil {
+		return nil, "", err
+	}
+
+	file, err := parse(content)
+	if err != nil {
+		return nil, "", err
+	}
+
+	return file, hash(content), nil
+}
+
+// processExtends inlines files into the current reviewpad file
+// precedence: current file > extends file
+// Post-condition: ReviewpadFile without extends statements
+func processExtends(ctx context.Context, githubClient *gh.GithubClient, file *ReviewpadFile, env *LoadEnv) (*ReviewpadFile, error) {
+	for i := len(file.Extends) - 1; i >= 0; i-- {
+		extension := file.Extends[i]
+
+		eFile, eHash, err := loadExtension(ctx, githubClient, extension)
+		if err != nil {
+			return nil, err
+		}
+
+		// check for cycles
+		if _, ok := env.Stack[eHash]; ok {
+			return nil, fmt.Errorf("loader: cyclic extends dependency")
+		}
+
+		// optimize visits
+		if _, ok := env.Visited[eHash]; ok {
+			continue
+		}
+
+		// DFS call inline imports
+		// update the environment
+		env.Stack[eHash] = true
+		env.Visited[eHash] = true
+
+		extensionFile, err := processExtends(ctx, githubClient, eFile, env)
+		if err != nil {
+			return nil, err
+		}
+
+		// remove from the stack
+		delete(env.Stack, eHash)
+
+		file.extends(extensionFile)
+	}
+
+	// reset all extends
+	file.Extends = []string{}
+
+	return file, nil
+}
diff --git a/engine/loader_test.go b/engine/loader_test.go
@@ -5,6 +5,7 @@
 package engine_test
 
 import (
+	"context"
 	"fmt"
 	"testing"
 
@@ -133,7 +134,7 @@ func TestLoad(t *testing.T) {
 					assert.FailNow(t, "Error reading reviewpad file: %v", err)
 				}
 
-				wantReviewpadFile, err = testutils.ParseReviewpadFile(wantReviewpadFileData)
+				wantReviewpadFile, err = testutils.ParseReviewpadFile(context.Background(), nil, wantReviewpadFileData)
 				if err != nil {
 					assert.FailNow(t, "Error parsing reviewpad file: %v", err)
 				}
@@ -144,7 +145,7 @@ func TestLoad(t *testing.T) {
 				assert.FailNow(t, "Error reading reviewpad file: %v", err)
 			}
 
-			gotReviewpadFile, gotErr := engine.Load(reviewpadFileData)
+			gotReviewpadFile, gotErr := engine.Load(context.Background(), nil, reviewpadFileData)
 
 			if gotErr != nil && gotErr.Error() != test.wantErr {
 				assert.FailNow(t, "Load() error = %v, wantErr %v", gotErr, test.wantErr)

diff --git a/engine/testutils/parse.go b/engine/testutils/parse.go
@@ -5,11 +5,14 @@
 package testutils
 
 import (
+	"context"
+
+	gh "github.com/reviewpad/reviewpad/v3/codehost/github"
 	"github.com/reviewpad/reviewpad/v3/engine"
 )
 
-func ParseReviewpadFile(data []byte) (*engine.ReviewpadFile, error) {
-	reviewpadFile, err := engine.Load(data)
+func ParseReviewpadFile(ctx context.Context, githubClient *gh.GithubClient, data []byte) (*engine.ReviewpadFile, error) {
+	reviewpadFile, err := engine.Load(ctx, githubClient, data)
 	if err != nil {
 		return nil, err
 	}

diff --git a/runner.go b/runner.go
@@ -20,8 +20,8 @@ import (
 	"github.com/reviewpad/reviewpad/v3/utils/fmtio"
 )
 
-func Load(buf *bytes.Buffer) (*engine.ReviewpadFile, error) {
-	file, err := engine.Load(buf.Bytes())
+func Load(ctx context.Context, githubClient *gh.GithubClient, buf *bytes.Buffer) (*engine.ReviewpadFile, error) {
+	file, err := engine.Load(ctx, githubClient, buf.Bytes())
 	if err != nil {
 		return nil, err
 	}