Skip to content

Commit

Permalink
Merge pull request #95 from rails/rm-refactor-bin
Browse files Browse the repository at this point in the history
Organize bin and initial pipeline files into sub-folders
  • Loading branch information
rafaelfranca authored Apr 15, 2024
2 parents 046b28c + 2713f15 commit f9c90fe
Show file tree
Hide file tree
Showing 18 changed files with 467 additions and 282 deletions.
File renamed without changes.
File renamed without changes.
File renamed without changes.
40 changes: 40 additions & 0 deletions bin/trigger-pipeline
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
#!/usr/bin/env ruby
# frozen_string_literal: true

# Example usage:
# bin/trigger-pipeline [options] org pipeline [slug]

require "buildkit"
require "optparse"

def buildkite_token
ENV.fetch("BUILDKITE_TOKEN") {
raise "BUILDKITE_TOKEN undefined!\nMake sure your BUILDKITE_TOKEN has `write_builds` scope too!"
}
end

# Make sure your BUILDKITE_TOKEN has `write_builds` scope
def trigger_pipeline(org, pipeline, slug, branch: "main", commit: "HEAD")
slug ||= pipeline
puts "Trigger build for #{pipeline}..."
configuration = File.read("pipelines/#{slug}/initial.yml")
client = Buildkit.new(token: buildkite_token)
resp = client.create_build(org, pipeline, {
branch: branch,
commit: commit,
override_steps_configuration: configuration
})
puts resp.web_url
end

options = {}
OptionParser.new do |opts|
opts.on("--branch=BRANCH")
opts.on("--commit=COMMIT")
end.parse!(into: options)

org = ARGV.shift
pipeline = ARGV.shift
slug = ARGV.shift

trigger_pipeline(org, pipeline, slug, **options)
29 changes: 29 additions & 0 deletions bin/update-initial-pipelines
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
#!/usr/bin/env ruby
# frozen_string_literal: true

require "buildkit"

def buildkite_token
ENV.fetch("BUILDKITE_TOKEN") {
raise "BUILDKITE_TOKEN undefined!\nMake sure your BUILDKITE_TOKEN has `write_pipelines` scope too!"
}
end

# Make sure your BUILDKITE_TOKEN has `write_pipelines` scope
def update_pipeline(pipeline, slug = nil)
slug ||= pipeline
puts "Updating #{pipeline}..."
configuration = File.read("pipelines/#{slug}/initial.yml")
client = Buildkit.new(token: buildkite_token)
resp = client.update_pipeline("rails", pipeline, {
configuration: configuration
})
puts resp.steps
end

update_pipeline("rails", "rails-ci")
update_pipeline("rails-ci", "rails-ci")
update_pipeline("rails-nightly", "rails-ci-nightly")
update_pipeline("rails-ci-nightly", "rails-ci-nightly")
update_pipeline("buildkite-config", "buildkite-config")
update_pipeline("docs-preview", "docs-preview")
13 changes: 9 additions & 4 deletions lib/buildkite/config/diff.rb
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,17 @@ def self.compare
def self.generated_pipeline(repo)
File.symlink(repo, "tmp/rails/.buildkite")

command = ["ruby", ".buildkite/pipeline-generate"]
Dir.chdir("tmp/rails") do
bin = if File.exist?(".buildkite/bin/pipeline-generate")
".buildkite/bin/pipeline-generate"
else
".buildkite/pipeline-generate"
end
command = ["ruby", bin]

pipeline = "rails-ci"
command.push(pipeline)
pipeline = "rails-ci"
command.push(pipeline)

Dir.chdir("tmp/rails") do
io = IO.popen(command)
output = io.read
io.close
Expand Down
13 changes: 12 additions & 1 deletion lib/buildkite/config/docker_build.rb
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,18 @@ def builder(ruby:, &block)
label ":docker: #{build_context.ruby.prefix}#{build_context.ruby.version}"
key "docker-image-#{build_context.ruby.image_key}"
plugin :artifacts, {
download: %w[.dockerignore .buildkite/* .buildkite/**/*]
download: ".dockerignore"
}
plugin :artifacts, {
download: %w[
.buildkite/.empty
.buildkite/docker-compose.yml
.buildkite/Dockerfile
.buildkite/Dockerfile.beanstalkd
.buildkite/mysql-initdb.d
.buildkite/runner
],
compressed: ".buildkite.tgz"
}

plugin :docker_compose, {
Expand Down
13 changes: 12 additions & 1 deletion lib/buildkite/config/rake_command.rb
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,18 @@ def rake(dir = "", task = "test", service: "default", pre_steps: [], &block)
command "rake #{task}"

plugin :artifacts, {
download: %w[.buildkite/* .buildkite/**/*]
download: ".dockerignore"
}
plugin :artifacts, {
download: %w[
.buildkite/.empty
.buildkite/docker-compose.yml
.buildkite/Dockerfile
.buildkite/Dockerfile.beanstalkd
.buildkite/mysql-initdb.d
.buildkite/runner
],
compressed: ".buildkite.tgz"
}

plugin :docker_compose, {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ steps:
-v "$$PWD/tmp":/app/tmp:rw
-e BUNDLE_FROZEN
ruby:latest
./pipeline-annotate
./bin/pipeline-annotate
- command: |
mkdir -p tmp
sh -c "$$ANNOTATE_COMMAND" > tmp/annotate-nightly.diff
Expand All @@ -81,7 +81,7 @@ steps:
-e RAILS_CI_NIGHTLY
-e BUNDLE_FROZEN
ruby:latest
./pipeline-annotate
./bin/pipeline-annotate
- group:
label: ":rails: Rails CI"
Expand All @@ -102,7 +102,6 @@ steps:
env:
CONFIG_REPO: "${BUILDKITE_PULL_REQUEST_REPO}"
CONFIG_BRANCH: "${BUILDKITE_BRANCH}"
BUILDKITE_CONFIG_TRIGGER: true # This variable can be used downstream to avoid things like "if branch==main do Y"
- block: "Rails CI stable"
key: block-rails-ci-stable
prompt: |
Expand All @@ -118,7 +117,6 @@ steps:
env:
CONFIG_REPO: "${BUILDKITE_PULL_REQUEST_REPO}"
CONFIG_BRANCH: "${BUILDKITE_BRANCH}"
BUILDKITE_CONFIG_TRIGGER: true # This variable can be used downstream to avoid things like "if branch==main do Y"
- block: "Rails CI nightly"
key: block-rails-ci-nightly-main
prompt: |
Expand All @@ -134,7 +132,6 @@ steps:
env:
CONFIG_REPO: "${BUILDKITE_PULL_REQUEST_REPO}"
CONFIG_BRANCH: "${BUILDKITE_BRANCH}"
BUILDKITE_CONFIG_TRIGGER: true # This variable can be used downstream to avoid things like "if branch==main do Y"
- block: "Rails CI nightly stable"
key: block-rails-ci-nightly-stable
prompt: |
Expand All @@ -150,4 +147,3 @@ steps:
env:
CONFIG_REPO: "${BUILDKITE_PULL_REQUEST_REPO}"
CONFIG_BRANCH: "${BUILDKITE_BRANCH}"
BUILDKITE_CONFIG_TRIGGER: true # This variable can be used downstream to avoid things like "if branch==main do Y"
Original file line number Diff line number Diff line change
Expand Up @@ -9,30 +9,34 @@ steps:
command: |
PATH=/bin:/usr/bin
set -e
echo "+++ No .buildkite/; using fallback repository"
rm -rf .buildkite
if [ -n "$$CONFIG_REPO" ]; then
GIT_REPO="$$CONFIG_REPO"
elif [ -n "$$CONFIG_FORK" ]; then
GIT_REPO="https://github.com/$$CONFIG_FORK/buildkite-config"
if [ -d ".buildkite" ]; then
echo "+++ :bk-status-failed: .buildkite/ already exists; please remove it and try again"
exit 1
else
GIT_REPO="https://github.com/rails/buildkite-config"
fi
GIT_BRANCH="$${CONFIG_BRANCH-main}"
GIT_BRANCH="$${GIT_BRANCH#*:}"
GIT_REPO="https://github.com/rails/buildkite-config"
GIT_BRANCH="main"
echo "Cloning buildkite-config:"
echo "git clone -b \"$$GIT_BRANCH\" \"$$GIT_REPO\" .buildkite"
git clone -b "$$GIT_BRANCH" "$$GIT_REPO" .buildkite
rm -rf .buildkite/.git
sh -c "$$PIPELINE_COMMAND"
([ -f .buildkite/.dockerignore ] && cp .buildkite/.dockerignore .dockerignore) || true
plugins:
- artifacts#v1.2.0:
upload: [".buildkite/**/*", ".dockerignore"]
- artifacts#v1.9.3:
upload: ".dockerignore"
- artifacts#v1.9.3:
upload:
- .buildkite/.empty
- .buildkite/docker-compose.yml
- .buildkite/Dockerfile
- .buildkite/Dockerfile.beanstalkd
- .buildkite/mysql-initdb.d
- .buildkite/runner
compressed: ".buildkite.tgz"
env:
PIPELINE_COMMAND: >-
docker run --rm
Expand All @@ -50,5 +54,5 @@ steps:
-e RUN_QUEUE
-e QUEUE
ruby:latest
.buildkite/pipeline-generate docs-preview |
.buildkite/bin/pipeline-generate docs-preview |
buildkite-agent pipeline upload
11 changes: 7 additions & 4 deletions pipelines/docs-preview/pipeline.rb
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,7 @@

Buildkite::Builder.pipeline do
plugin :docker, "docker#v5.10.0"
plugin :artifacts, "artifacts#v1.2.0"
# plugin :artifacts, "artifacts#v1.9.2"
plugin :artifacts, "artifacts#v1.9.3"

command do
label "build", emoji: :rails
Expand All @@ -23,7 +22,9 @@
"BUNDLE_WITHOUT=db:job:cable:storage:ujs",
],
}
artifact_paths "preview.tar.gz"
plugin :artifacts, {
upload: "preview.tar.gz"
}
end

command do
Expand All @@ -41,7 +42,9 @@
],
image: "node:latest"
}
plugin :artifacts, { download: "preview.tar.gz" }
plugin :artifacts, {
download: "preview.tar.gz"
}
command "tar -xzf preview.tar.gz"
command "npm install wrangler"
command "npx wrangler pages publish preview --project-name=$CLOUDFLARE_PAGES_PROJECT --branch=\"$BUILDKITE_BRANCH\""
Expand Down
70 changes: 70 additions & 0 deletions pipelines/rails-ci-nightly/initial.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
# This file is never read -- it's just a copy of the pipeline's
# configuration in the Buildkite UI.

steps:
- name: ":pipeline: rails-nightly-initial-pipeline"
command: |
PATH=/bin:/usr/bin
set -e
if [ -d ".buildkite" ]; then
echo "+++ :bk-status-failed: .buildkite/ already exists; please remove it and try again"
exit 1
fi
if [ -n "$$CONFIG_REPO" ]; then
GIT_REPO="$$CONFIG_REPO"
else
GIT_REPO="https://github.com/rails/buildkite-config"
fi
GIT_BRANCH="$${CONFIG_BRANCH-main}"
GIT_BRANCH="$${GIT_BRANCH#*:}"
echo "Cloning buildkite-config:"
echo "git clone -b \"$$GIT_BRANCH\" \"$$GIT_REPO\" .buildkite"
git clone -b "$$GIT_BRANCH" "$$GIT_REPO" .buildkite
rm -rf .buildkite/.git
sh -c "$$PIPELINE_COMMAND"
([ -f .buildkite/.dockerignore ] && cp .buildkite/.dockerignore .dockerignore) || true
plugins:
- artifacts#v1.9.3:
upload: ".dockerignore"
- artifacts#v1.9.3:
upload:
- .buildkite/.empty
- .buildkite/docker-compose.yml
- .buildkite/Dockerfile
- .buildkite/Dockerfile.beanstalkd
- .buildkite/mysql-initdb.d
- .buildkite/runner
compressed: ".buildkite.tgz"
env:
RAILS_CI_NIGHTLY: true
BUNDLE_FROZEN: true
PIPELINE_COMMAND: >-
docker run --rm
-v "$$PWD":/app:ro -w /app
-e RAILS_CI_NIGHTLY
-e CI
-e BUILDKITE
-e BUILDKITE_AGENT_META_DATA_QUEUE
-e BUILDKITE_BRANCH
-e BUILDKITE_BUILD_ID
-e BUILDKITE_PULL_REQUEST
-e BUILDKITE_PULL_REQUEST_BASE_BRANCH
-e BUILDKITE_REBUILT_FROM_BUILD_ID
-e BUILD_QUEUE
-e BUNDLE_FROZEN
-e DOCKER_IMAGE
-e RUN_QUEUE
-e QUEUE
ruby:latest
.buildkite/bin/pipeline-generate rails-ci |
buildkite-agent pipeline upload
timeout_in_minutes: 5
agents:
queue: "${QUEUE-builder}"
Loading

0 comments on commit f9c90fe

Please sign in to comment.