Skip to content

Commit

Permalink
Add SCM webhook to trigger Pipelines (#597)
Browse files Browse the repository at this point in the history
* Add scm webhook support

* Add scm webhook support

* Support to scan multi-branch pipeline via webhook

* Add unit test for scm webhook

* Support to trigger a regular pipeline via webhook

* Add missing license header

* Add documentation about the webhook

* Fix the unit tests error
  • Loading branch information
LinuxSuRen authored Jun 7, 2022
1 parent 2561eb9 commit 058ff4d
Show file tree
Hide file tree
Showing 18 changed files with 1,018 additions and 39 deletions.
26 changes: 25 additions & 1 deletion docs/webhook.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,31 @@ There are several ways to trigger Pipeline via [webhook](https://en.wikipedia.or

## SCM Webhook

TODO
Supported SCM providers:
* GitHub
* Gitlab
* Bitbucket

There are two types of Jenkins based Pipelines: regular or multi-branch Pipeline. When a SCM webhook request received,
the server will search all Pipelines by the Git URL, then trigger the scan action if it's a multi-branch Pipeline,
or create a new PipelineRun if there is an annotation key-value likes the following one:
```
scm.devops.kubesphere.io=https=https://github.com/linuxsuren/tools
```

In case you only want some Pipelines to be triggered when specific branches changed. You can add an annotation:
```
scm.devops.kubesphere.io/ref='["master","fea-.*"]'
```

The webhook address is:
```
http://ip:port/v1alpha3/webhooks/scm
```

### Using webhook locally

It's also possible to use webhook feature locally. You just need to start a proyx with [ngrok](https://ngrok.com/).

## Automatic webhook

Expand Down
47 changes: 47 additions & 0 deletions pkg/api/devops/v1alpha3/last_changes.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
// Copyright 2022 KubeSphere Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package v1alpha3

import "encoding/json"

// +kubebuilder:object:generate=false

// LastChanges represents a set of last SCM changes
type LastChanges map[string]string

// GetLastChanges returns the last changes
func GetLastChanges(jsonText string) (lastChange LastChanges, err error) {
lastChange = map[string]string{}
err = json.Unmarshal([]byte(jsonText), &lastChange)
return
}

// Update updates hash by ref
func (l LastChanges) Update(ref, hash string) LastChanges {
l[ref] = hash
return l
}

// LastHash return last hash value
func (l LastChanges) LastHash(ref string) (hash string) {
return l[ref]
}

// String returns the string JSON format
func (l LastChanges) String() string {
data, _ := json.Marshal(l)
return string(data)
}
136 changes: 136 additions & 0 deletions pkg/api/devops/v1alpha3/last_changes_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
/*
Copyright 2022 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha3

import (
"fmt"
"github.com/stretchr/testify/assert"
"testing"
)

func TestGetLastChanges(t *testing.T) {
type args struct {
jsonText string
}
tests := []struct {
name string
args args
wantLastChange LastChanges
wantErr assert.ErrorAssertionFunc
}{{
name: "normal JSON data with map format",
args: args{jsonText: `{"master":"1234"}`},
wantLastChange: map[string]string{
"master": "1234",
},
wantErr: func(t assert.TestingT, err error, i ...interface{}) bool {
return false
},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotLastChange, err := GetLastChanges(tt.args.jsonText)
if !tt.wantErr(t, err, fmt.Sprintf("GetLastChanges(%v)", tt.args.jsonText)) {
return
}
assert.Equalf(t, tt.wantLastChange, gotLastChange, "GetLastChanges(%v)", tt.args.jsonText)
})
}
}

func TestLastChanges_Update(t *testing.T) {
type args struct {
ref string
hash string
}
tests := []struct {
name string
l LastChanges
args args
want LastChanges
}{{
name: "update the not existing value",
l: map[string]string{},
args: args{
ref: "master",
hash: "2345",
},
want: map[string]string{
"master": "2345",
},
}, {
name: "update the existing value",
l: map[string]string{
"master": "1234",
},
args: args{
ref: "master",
hash: "2345",
},
want: map[string]string{
"master": "2345",
},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equalf(t, tt.want, tt.l.Update(tt.args.ref, tt.args.hash), "Update(%v, %v)", tt.args.ref, tt.args.hash)
})
}
}

func TestLastChanges_LastHash(t *testing.T) {
type args struct {
ref string
}
tests := []struct {
name string
l LastChanges
args args
wantHash string
}{{
name: "normal case",
l: map[string]string{
"master": "1234",
},
args: args{ref: "master"},
wantHash: "1234",
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equalf(t, tt.wantHash, tt.l.LastHash(tt.args.ref), "LastHash(%v)", tt.args.ref)
})
}
}

func TestLastChanges_String(t *testing.T) {
tests := []struct {
name string
l LastChanges
want string
}{{
name: "normal case",
l: map[string]string{
"master": "1234",
},
want: `{"master":"1234"}`,
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
assert.Equalf(t, tt.want, tt.l.String(), "String()")
})
}
}
25 changes: 24 additions & 1 deletion pkg/api/devops/v1alpha3/pipeline_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package v1alpha3

import (
"fmt"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

Expand All @@ -27,13 +28,13 @@ const PipelineFinalizerName = "pipeline.finalizers.kubesphere.io"

const (
ResourceKindPipeline = "Pipeline"
ResourceSingularPipeline = "pipeline"
ResourcePluralPipeline = "pipelines"
PipelinePrefix = "pipeline.devops.kubesphere.io/"
PipelineSpecHash = PipelinePrefix + "spechash"
PipelineSyncStatusAnnoKey = PipelinePrefix + "syncstatus"
PipelineSyncTimeAnnoKey = PipelinePrefix + "synctime"
PipelineSyncMsgAnnoKey = PipelinePrefix + "syncmsg"
PipelineLastChanges = PipelinePrefix + "last-changes"
// PipelineJenkinsMetadataAnnoKey is the annotation key of Jenkins Pipeline data.
PipelineJenkinsMetadataAnnoKey = PipelinePrefix + "jenkins-metadata"
// PipelineJenkinsBranchesAnnoKey is the annotation key of Jenkins Pipeline branches.
Expand Down Expand Up @@ -132,6 +133,28 @@ type MultiBranchPipeline struct {
MultiBranchJobTrigger *MultiBranchJobTrigger `json:"multibranch_job_trigger,omitempty" mapstructure:"multibranch_job_trigger" description:"Pipeline tasks that need to be triggered when branch creation/deletion"`
}

func (b *MultiBranchPipeline) GetGitURL() string {
switch b.SourceType {
case SourceTypeGit:
if b.GitSource != nil {
return b.GitSource.Url
}
case SourceTypeGithub:
if b.GitHubSource != nil {
return fmt.Sprintf("https://github.com/%s/%s", b.GitHubSource.Owner, b.GitHubSource.Repo)
}
case SourceTypeGitlab:
if b.GitlabSource != nil {
return fmt.Sprintf("https://gitlab.com/%s/%s", b.GitlabSource.Owner, b.GitlabSource.Repo)
}
case SourceTypeBitbucket:
if b.BitbucketServerSource != nil {
return fmt.Sprintf("https://bitbucket.org/%s/%s", b.BitbucketServerSource.Owner, b.BitbucketServerSource.Repo)
}
}
return ""
}

type GitSource struct {
ScmId string `json:"scm_id,omitempty" description:"uid of scm"`
Url string `json:"url,omitempty" mapstructure:"url" description:"url of git source"`
Expand Down
63 changes: 63 additions & 0 deletions pkg/api/devops/v1alpha3/pipeline_types_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package v1alpha3

import (
"github.com/stretchr/testify/assert"
"testing"
)

Expand Down Expand Up @@ -61,3 +62,65 @@ func TestPipeline_IsMultiBranch(t *testing.T) {
})
}
}

func TestMultiBranchPipeline_GetGitURL(t *testing.T) {
type fields struct {
SourceType string
GitSource *GitSource
GitHubSource *GithubSource
GitlabSource *GitlabSource
BitbucketServerSource *BitbucketServerSource
}
tests := []struct {
name string
fields fields
want string
}{{
name: "github",
fields: fields{
SourceType: SourceTypeGithub,
GitHubSource: &GithubSource{Owner: "linuxsuren", Repo: "tools"},
},
want: "https://github.com/linuxsuren/tools",
}, {
name: "gitlab",
fields: fields{
SourceType: SourceTypeGitlab,
GitlabSource: &GitlabSource{Owner: "linuxsuren", Repo: "tools"},
},
want: "https://gitlab.com/linuxsuren/tools",
}, {
name: "git",
fields: fields{
SourceType: SourceTypeGit,
GitSource: &GitSource{Url: "https://fake.com"},
},
want: "https://fake.com",
}, {
name: "bitbucket",
fields: fields{
SourceType: SourceTypeBitbucket,
BitbucketServerSource: &BitbucketServerSource{Owner: "linuxsuren", Repo: "tools"},
},
want: "https://bitbucket.org/linuxsuren/tools",
}, {
name: "fake",
fields: fields{
SourceType: "fake",
GitSource: &GitSource{Url: "https://fake.com"},
},
want: "",
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
b := &MultiBranchPipeline{
SourceType: tt.fields.SourceType,
GitSource: tt.fields.GitSource,
GitHubSource: tt.fields.GitHubSource,
GitlabSource: tt.fields.GitlabSource,
BitbucketServerSource: tt.fields.BitbucketServerSource,
}
assert.Equalf(t, tt.want, b.GetGitURL(), "GetGitURL()")
})
}
}
8 changes: 7 additions & 1 deletion pkg/apiserver/apiserver.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"bytes"
"context"
"fmt"
"kubesphere.io/devops/pkg/jwt/token"
"kubesphere.io/devops/pkg/kapis/common"
"kubesphere.io/devops/pkg/kapis/doc"
gitops "kubesphere.io/devops/pkg/kapis/gitops/v1alpha1"
Expand Down Expand Up @@ -141,6 +142,7 @@ func (s *APIServer) installKubeSphereAPIs() {
}

var wss []*restful.WebService
tokenIssue := getTokenIssue(s.Config)

v1alpha2WSS, err := devopsv1alpha2.AddToContainer(s.container,
s.InformerFactory.KubeSphereSharedInformerFactory(),
Expand All @@ -153,7 +155,7 @@ func (s *APIServer) installKubeSphereAPIs() {
jenkinsCore)
utilruntime.Must(err)
wss = append(wss, v1alpha2WSS...)
wss = append(wss, devopsv1alpha3.AddToContainer(s.container, s.DevopsClient, s.KubernetesClient, s.Client)...)
wss = append(wss, devopsv1alpha3.AddToContainer(s.container, s.DevopsClient, s.KubernetesClient, s.Client, tokenIssue, jenkinsCore)...)
wss = append(wss, oauth.AddToContainer(s.container,
auth.NewTokenOperator(
s.CacheClient,
Expand All @@ -165,6 +167,10 @@ func (s *APIServer) installKubeSphereAPIs() {
doc.AddSwaggerService(wss, s.container)
}

func getTokenIssue(config *apiserverconfig.Config) token.Issuer {
return token.NewTokenIssuer(config.AuthenticationOptions.JwtSecret, config.AuthenticationOptions.MaximumClockSkew)
}

func (s *APIServer) Run(stopCh <-chan struct{}) (err error) {
if err := indexers.CreatePipelineRunSCMRefNameIndexer(s.RuntimeCache); err != nil {
return err
Expand Down
Loading

0 comments on commit 058ff4d

Please sign in to comment.