Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add etcd 3.x minor version rollback support to migrate-if-needed.sh #59298

Merged
merged 1 commit into from
Feb 13, 2018
+302 −86
Merged

Add etcd 3.x minor version rollback support to migrate-if-needed.sh #59298

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions cluster/gce/manifests/etcd.manifest
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@
},
{ "name": "DATA_DIRECTORY",
"value": "/var/etcd/data{{ suffix }}"
},
{ "name": "INITIAL_CLUSTER",
"value": "{{ etcd_cluster }}"
}
],
"livenessProbe": {
Expand Down
2 changes: 1 addition & 1 deletion cluster/images/etcd/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,4 @@ FROM BASEIMAGE

EXPOSE 2379 2380 4001 7001
COPY etcd* etcdctl* /usr/local/bin/
COPY migrate-if-needed.sh attachlease rollback /usr/local/bin/
COPY migrate-if-needed.sh start-stop-etcd.sh attachlease rollback /usr/local/bin/
134 changes: 129 additions & 5 deletions cluster/images/etcd/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@
# That binary will be set to the last tag from $(TAGS).
TAGS?=2.2.1 2.3.7 3.0.17 3.1.11 3.2.14
REGISTRY_TAG?=3.2.14
# ROLLBACK_REGISTRY_TAG specified the tag that REGISTRY_TAG may be rolled back to.
ROLLBACK_REGISTRY_TAG?=3.1.11
ARCH?=amd64
REGISTRY?=k8s.gcr.io
# golang version should match the golang version from https://github.com/coreos/etcd/releases for REGISTRY_TAG version of etcd.
Expand Down Expand Up @@ -57,10 +59,10 @@ build:
find ./ -maxdepth 1 -type f | xargs -I {} cp {} $(TEMP_DIR)

# Compile attachlease
docker run -i -v $(shell pwd)/../../../:/go/src/k8s.io/kubernetes -v $(TEMP_DIR):/build -e GOARCH=$(ARCH) golang:$(GOLANG_VERSION) \
docker run --interactive -v $(shell pwd)/../../../:/go/src/k8s.io/kubernetes -v $(TEMP_DIR):/build -e GOARCH=$(ARCH) golang:$(GOLANG_VERSION) \
/bin/bash -c "CGO_ENABLED=0 go build -o /build/attachlease k8s.io/kubernetes/cluster/images/etcd/attachlease"
# Compile rollback
docker run -i -v $(shell pwd)/../../../:/go/src/k8s.io/kubernetes -v $(TEMP_DIR):/build -e GOARCH=$(ARCH) golang:$(GOLANG_VERSION) \
docker run --interactive -v $(shell pwd)/../../../:/go/src/k8s.io/kubernetes -v $(TEMP_DIR):/build -e GOARCH=$(ARCH) golang:$(GOLANG_VERSION) \
/bin/bash -c "CGO_ENABLED=0 go build -o /build/rollback k8s.io/kubernetes/cluster/images/etcd/rollback"


Expand All @@ -81,7 +83,7 @@ else
# For each release create a tmp dir 'etcd_release_tmp_dir' and unpack the release tar there.
for tag in $(TAGS); do \
etcd_release_tmp_dir=$(shell mktemp -d); \
docker run -i -v $$etcd_release_tmp_dir:/etcdbin golang:$(GOLANG_VERSION) /bin/bash -c \
docker run --interactive -v $$etcd_release_tmp_dir:/etcdbin golang:$(GOLANG_VERSION) /bin/bash -c \
"git clone https://github.com/coreos/etcd /go/src/github.com/coreos/etcd \
&& cd /go/src/github.com/coreos/etcd \
&& git checkout v$$tag \
Expand Down Expand Up @@ -114,5 +116,127 @@ ifeq ($(ARCH),amd64)
gcloud docker -- push $(REGISTRY)/etcd:$(REGISTRY_TAG)
endif

all: build
.PHONY: build push
ETCD2_ROLLBACK_NEW_TAG=3.0.17
ETCD2_ROLLBACK_OLD_TAG=2.2.1

# Test a rollback to etcd2 from the earliest etcd3 version.
test-rollback-etcd2:
mkdir -p $(TEMP_DIR)/rollback-etcd2
cd $(TEMP_DIR)/rollback-etcd2

@echo "Starting $(ETCD2_ROLLBACK_NEW_TAG) etcd and writing some sample data."
docker run --tty --interactive -v $(TEMP_DIR)/rollback-etcd2:/var/etcd \
-e "TARGET_STORAGE=etcd3" \
-e "TARGET_VERSION=$(ETCD2_ROLLBACK_NEW_TAG)" \
-e "DATA_DIRECTORY=/var/etcd/data" \
gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \
'INITIAL_CLUSTER=etcd-$$(hostname)=http://localhost:2380 \
/usr/local/bin/migrate-if-needed.sh && \
source /usr/local/bin/start-stop-etcd.sh && \
START_STORAGE=etcd3 START_VERSION=$(ETCD2_ROLLBACK_NEW_TAG) start_etcd && \
ETCDCTL_API=3 /usr/local/bin/etcdctl-$(ETCD2_ROLLBACK_NEW_TAG) --endpoints http://127.0.0.1:$${ETCD_PORT} put /registry/k1 value1 && \
stop_etcd && \
[ $$(cat /var/etcd/data/version.txt) = $(ETCD2_ROLLBACK_NEW_TAG)/etcd3 ]'

@echo "Rolling back to the previous version of etcd and recording keyspace to a flat file."
docker run --tty --interactive -v $(TEMP_DIR)/rollback-etcd2:/var/etcd \
-e "TARGET_STORAGE=etcd2" \
-e "TARGET_VERSION=$(ETCD2_ROLLBACK_OLD_TAG)" \
-e "DATA_DIRECTORY=/var/etcd/data" \
gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \
'INITIAL_CLUSTER=etcd-$$(hostname)=http://localhost:2380 \
/usr/local/bin/migrate-if-needed.sh && \
source /usr/local/bin/start-stop-etcd.sh && \
START_STORAGE=etcd2 START_VERSION=$(ETCD2_ROLLBACK_OLD_TAG) start_etcd && \
/usr/local/bin/etcdctl-$(ETCD2_ROLLBACK_OLD_TAG) --endpoint 127.0.0.1:$${ETCD_PORT} get /registry/k1 > /var/etcd/keyspace.txt && \
stop_etcd'

@echo "Checking if rollback successfully downgraded etcd to $(ETCD2_ROLLBACK_OLD_TAG)"
docker run --tty --interactive -v $(TEMP_DIR)/rollback-etcd2:/var/etcd \
gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \
'[ $$(cat /var/etcd/data/version.txt) = $(ETCD2_ROLLBACK_OLD_TAG)/etcd2 ] && \
grep -q value1 /var/etcd/keyspace.txt'

# Test a rollback from the latest version to the previous version.
test-rollback:
mkdir -p $(TEMP_DIR)/rollback-test
cd $(TEMP_DIR)/rollback-test

@echo "Starting $(REGISTRY_TAG) etcd and writing some sample data."
docker run --tty --interactive -v $(TEMP_DIR)/rollback-test:/var/etcd \
-e "TARGET_STORAGE=etcd3" \
-e "TARGET_VERSION=$(REGISTRY_TAG)" \
-e "DATA_DIRECTORY=/var/etcd/data" \
gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \
'INITIAL_CLUSTER=etcd-$$(hostname)=http://localhost:2380 \
/usr/local/bin/migrate-if-needed.sh && \
source /usr/local/bin/start-stop-etcd.sh && \
START_STORAGE=etcd3 START_VERSION=$(REGISTRY_TAG) start_etcd && \
ETCDCTL_API=3 /usr/local/bin/etcdctl --endpoints http://127.0.0.1:$${ETCD_PORT} put /registry/k1 value1 && \
stop_etcd'

@echo "Rolling back to the previous version of etcd and recording keyspace to a flat file."
docker run --tty --interactive -v $(TEMP_DIR)/rollback-test:/var/etcd \
-e "TARGET_STORAGE=etcd3" \
-e "TARGET_VERSION=$(ROLLBACK_REGISTRY_TAG)" \
-e "DATA_DIRECTORY=/var/etcd/data" \
gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \
'INITIAL_CLUSTER=etcd-$$(hostname)=http://localhost:2380 \
/usr/local/bin/migrate-if-needed.sh && \
source /usr/local/bin/start-stop-etcd.sh && \
START_STORAGE=etcd3 START_VERSION=$(ROLLBACK_REGISTRY_TAG) start_etcd && \
ETCDCTL_API=3 /usr/local/bin/etcdctl --endpoints http://127.0.0.1:$${ETCD_PORT} get --prefix / > /var/etcd/keyspace.txt && \
stop_etcd'

@echo "Checking if rollback successfully downgraded etcd to $(ROLLBACK_REGISTRY_TAG)"
docker run --tty --interactive -v $(TEMP_DIR)/rollback-test:/var/etcd \
gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \
'[ $$(cat /var/etcd/data/version.txt) = $(ROLLBACK_REGISTRY_TAG)/etcd3 ] && \
grep -q value1 /var/etcd/keyspace.txt'
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Those two rollback tests seem pretty similar - would it be possible somehow share the code between those?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I looked into it briefly and the main reason I kept them separate is because we will be deleting the etcd support later this year, and by keeping the tests separate we make the deletion trivial.


# Test migrating from each supported versions to the latest version.
test-migrate:
for tag in $(TAGS); do \
echo "Testing migration from $${tag} to $(REGISTRY_TAG)" && \
mkdir -p $(TEMP_DIR)/migrate-$${tag} && \
cd $(TEMP_DIR)/migrate-$${tag} && \
MAJOR_VERSION=$$(echo $${tag} | cut -c 1) && \
echo "Starting etcd $${tag} and writing sample data to keyspace" && \
docker run --tty --interactive -v $(TEMP_DIR)/migrate-$${tag}:/var/etcd \
-e "TARGET_STORAGE=etcd$${MAJOR_VERSION}" \
-e "TARGET_VERSION=$${tag}" \
-e "DATA_DIRECTORY=/var/etcd/data" \
gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \
"INITIAL_CLUSTER=etcd-\$$(hostname)=http://localhost:2380 \
/usr/local/bin/migrate-if-needed.sh && \
source /usr/local/bin/start-stop-etcd.sh && \
START_STORAGE=etcd$${MAJOR_VERSION} START_VERSION=$${tag} start_etcd && \
if [ $${MAJOR_VERSION} == 2 ]; then \
/usr/local/bin/etcdctl --endpoint http://127.0.0.1:\$${ETCD_PORT} set /registry/k1 value1; \
else \
ETCDCTL_API=3 /usr/local/bin/etcdctl --endpoints http://127.0.0.1:\$${ETCD_PORT} put /registry/k1 value1; \
fi && \
stop_etcd" && \
echo " Migrating from $${tag} to $(REGISTRY_TAG) and capturing keyspace" && \
docker run --tty --interactive -v $(TEMP_DIR)/migrate-$${tag}:/var/etcd \
-e "TARGET_STORAGE=etcd3" \
-e "TARGET_VERSION=$(REGISTRY_TAG)" \
-e "DATA_DIRECTORY=/var/etcd/data" \
gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \
'INITIAL_CLUSTER=etcd-$$(hostname)=http://localhost:2380 \
/usr/local/bin/migrate-if-needed.sh && \
source /usr/local/bin/start-stop-etcd.sh && \
START_STORAGE=etcd3 START_VERSION=$(REGISTRY_TAG) start_etcd && \
ETCDCTL_API=3 /usr/local/bin/etcdctl --endpoints http://127.0.0.1:$${ETCD_PORT} get --prefix / > /var/etcd/keyspace.txt && \
stop_etcd' && \
echo "Checking if migrate from $${tag} successfully upgraded etcd to $(REGISTRY_TAG)" && \
docker run --tty --interactive -v $(TEMP_DIR)/migrate-$${tag}:/var/etcd \
gcr.io/google_containers/etcd-$(ARCH):$(REGISTRY_TAG) /bin/sh -c \
'[ $$(cat /var/etcd/data/version.txt) = $(REGISTRY_TAG)/etcd3 ] && \
grep -q value1 /var/etcd/keyspace.txt'; \
done

test: test-rollback test-rollback-etcd2 test-migrate

all: build test
.PHONY: build push test-rollback test-rollback-etcd2 test-migrate test
8 changes: 8 additions & 0 deletions cluster/images/etcd/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,14 @@ For other architectures, `etcd` is cross-compiled from source. Arch-specific `bu

#### How to release

First, run the migration and rollback tests.

```console
$ make build test
```

Next, build and push the docker images for all supported architectures.

```console
# Build for linux/amd64 (default)
$ make push ARCH=amd64
Expand Down
Loading