From a305269e1830b0ebc00a83af734712ab3be2cad3 Mon Sep 17 00:00:00 2001 From: Zach Loafman Date: Sun, 18 Jan 2015 15:16:52 -0800 Subject: [PATCH] Deferred creation of SkyDNS, monitoring and logging objects This implements phase 1 of the proposal in #3579, moving the creation of the pods, RCs, and services to the master after the apiserver is available. This is such a wide commit because our existing initial config story is special: * Add kube-addons service and associated salt configuration: ** We configure /etc/kubernetes/addons to be a directory of objects that are appropriately configured for the current cluster. ** "/etc/init.d/kube-addons start" slurps up everything in that dir. (Most of the difficult is the business logic in salt around getting that directory built at all.) ** We cheat and overlay cluster/addons into saltbase/salt/kube-addons as config files for the kube-addons meta-service. * Change .yaml.in files to salt templates * Rename {setup,teardown}-{monitoring,logging} to {setup,teardown}-{monitoring,logging}-firewall to properly reflect their real purpose now (the purpose of these functions is now ONLY to bring up the firewall rules, and possibly to relay the IP to the user). * Rework GCE {setup,teardown}-{monitoring,logging}-firewall: Both functions were improperly configuring global rules, yet used lifecycles tied to the cluster. Use $NODE_INSTANCE_PREFIX with the rule. The logging rule needed a $NETWORK specifier. The monitoring rule tried gcloud describe first, but given the instancing, this feels like a waste of time now. * Plumb ENABLE_CLUSTER_MONITORING, ENABLE_CLUSTER_LOGGING, ELASTICSEARCH_LOGGING_REPLICAS and DNS_REPLICAS down to the master, since these are needed there now. (Desperately want just a yaml or json file we can share between providers that has all this crap. Maybe #3525 is an answer?) Huge caveats: I've gone pretty firm testing on GCE, including twiddling the env variables and making sure the objects I expect to come up, come up. I've tested that it doesn't break GKE bringup somehow. But I haven't had a chance to test the other providers. --- build/common.sh | 8 ++ cluster/addons/dns/README.md | 25 ++-- cluster/addons/dns/skydns-rc.yaml.in | 6 +- cluster/addons/dns/skydns-svc.yaml.in | 2 +- .../es-controller.yaml.in | 2 +- cluster/aws/util.sh | 14 +- cluster/azure/util.sh | 8 +- .../templates/create-dynamic-salt-files.sh | 4 + cluster/gce/util.sh | 136 +++++++----------- cluster/gke/util.sh | 12 +- cluster/kube-down.sh | 4 +- cluster/kube-up.sh | 22 +-- cluster/rackspace/util.sh | 8 +- cluster/saltbase/salt/kube-addons/init.sls | 79 ++++++++++ cluster/saltbase/salt/kube-addons/initd | 123 ++++++++++++++++ .../salt/kube-addons/kube-addons.service | 9 ++ cluster/saltbase/salt/top.sls | 1 + cluster/vagrant/provision-master.sh | 4 + cluster/vagrant/util.sh | 10 +- .../templates/create-dynamic-salt-files.sh | 4 + cluster/vsphere/util.sh | 8 +- 21 files changed, 336 insertions(+), 153 deletions(-) create mode 100644 cluster/saltbase/salt/kube-addons/init.sls create mode 100644 cluster/saltbase/salt/kube-addons/initd create mode 100644 cluster/saltbase/salt/kube-addons/kube-addons.service diff --git a/build/common.sh b/build/common.sh index 9aafdbdea5e0e..af87a0d3cc9c4 100644 --- a/build/common.sh +++ b/build/common.sh @@ -551,6 +551,14 @@ function kube::release::package_salt_tarball() { cp -R "${KUBE_ROOT}/cluster/saltbase" "${release_stage}/" + # TODO(#3579): This is a temporary hack. It gathers up the yaml, + # yaml.in files in cluster/addons (minus any demos) and overlays + # them into kube-addons, where we expect them. (This pipeline is a + # fancy copy, stripping anything but the files we don't want.) + local objects + objects=$(cd "${KUBE_ROOT}/cluster/addons" && find . -name \*.yaml -or -name \*.yaml.in | grep -v demo) + tar c -C "${KUBE_ROOT}/cluster/addons" ${objects} | tar x -C "${release_stage}/saltbase/salt/kube-addons" + local package_name="${RELEASE_DIR}/kubernetes-salt.tar.gz" kube::release::create_tarball "${package_name}" "${release_stage}/.." } diff --git a/cluster/addons/dns/README.md b/cluster/addons/dns/README.md index cf2c14630a79e..512bedd757077 100644 --- a/cluster/addons/dns/README.md +++ b/cluster/addons/dns/README.md @@ -1,6 +1,6 @@ # DNS in Kubernetes -This directory holds an example of how to run -[SkyDNS](https://github.com/skynetservices/skydns) in a Kubernetes cluster. +[SkyDNS](https://github.com/skynetservices/skydns) can be configured +to automatically run in a Kubernetes cluster. ## What things get DNS names? The only objects to which we are assigning DNS names are Services. Every @@ -18,23 +18,14 @@ Of course, giving services a name is just half of the problem - DNS names need a domain also. This implementation uses the variable `DNS_DOMAIN` (see below). You can configure your docker daemon with the flag `--dns-search`. -## How do I run it? -The first thing you have to do is substitute the variables into the -configuration. You can then feed the result into `kubectl`. +## How do I configure it? +The following environment variables are used at cluster startup to create the SkyDNS pods and configure the kubelets. If you need to, you can reconfigure your provider as necessary (e.g. `cluster/gce/config-default.sh`): ```shell -DNS_SERVER_IP=10.0.0.10 -DNS_DOMAIN=kubernetes.local -DNS_REPLICAS=2 - -sed -e "s/{DNS_DOMAIN}/$DNS_DOMAIN/g" \ - -e "s/{DNS_REPLICAS}/$DNS_REPLICAS/g" \ - ./cluster/addons/dns/skydns-rc.yaml.in \ - | ./cluster/kubectl.sh create -f - - -sed -e "s/{DNS_SERVER_IP}/$DNS_SERVER_IP/g" \ - ./cluster/addons/dns/skydns-svc.yaml.in \ - | ./cluster/kubectl.sh create -f - +ENABLE_CLUSTER_DNS=true +DNS_SERVER_IP="10.0.0.10" +DNS_DOMAIN="kubernetes.local" +DNS_REPLICAS=1 ``` ## How does it work? diff --git a/cluster/addons/dns/skydns-rc.yaml.in b/cluster/addons/dns/skydns-rc.yaml.in index 1043de7d9975c..15c6d50623dd3 100644 --- a/cluster/addons/dns/skydns-rc.yaml.in +++ b/cluster/addons/dns/skydns-rc.yaml.in @@ -5,7 +5,7 @@ namespace: default labels: k8s-app: skydns desiredState: - replicas: {DNS_REPLICAS} + replicas: {{ pillar['dns_replicas'] }} replicaSelector: k8s-app: skydns podTemplate: @@ -28,7 +28,7 @@ desiredState: image: kubernetes/kube2sky:1.0 command: [ # entrypoint = "/kube2sky", - "-domain={DNS_DOMAIN}", + "-domain={{ pillar['dns_domain'] }}", ] - name: skydns image: kubernetes/skydns:2014-12-23-001 @@ -36,7 +36,7 @@ desiredState: # entrypoint = "/skydns", "-machines=http://localhost:4001", "-addr=0.0.0.0:53", - "-domain={DNS_DOMAIN}.", + "-domain={{ pillar['dns_domain'] }}.", ] ports: - name: dns diff --git a/cluster/addons/dns/skydns-svc.yaml.in b/cluster/addons/dns/skydns-svc.yaml.in index 3898a638bf77d..f94d414d5f34c 100644 --- a/cluster/addons/dns/skydns-svc.yaml.in +++ b/cluster/addons/dns/skydns-svc.yaml.in @@ -4,7 +4,7 @@ id: skydns namespace: default protocol: UDP port: 53 -portalIP: {DNS_SERVER_IP} +portalIP: {{ pillar['dns_server'] }} containerPort: 53 labels: k8s-app: skydns diff --git a/cluster/addons/fluentd-elasticsearch/es-controller.yaml.in b/cluster/addons/fluentd-elasticsearch/es-controller.yaml.in index 87ab48d30fde7..ebad3bab2ffe6 100644 --- a/cluster/addons/fluentd-elasticsearch/es-controller.yaml.in +++ b/cluster/addons/fluentd-elasticsearch/es-controller.yaml.in @@ -2,7 +2,7 @@ apiVersion: v1beta1 kind: ReplicationController id: elasticsearch-logging-controller desiredState: - replicas: {ELASTICSEARCH_LOGGING_REPLICAS} + replicas: {{ pillar['elasticsearch_replicas'] }} replicaSelector: name: elasticsearch-logging podTemplate: diff --git a/cluster/aws/util.sh b/cluster/aws/util.sh index 96e7b4ae1a969..e2b0a6e727e7d 100644 --- a/cluster/aws/util.sh +++ b/cluster/aws/util.sh @@ -106,14 +106,14 @@ function ensure-temp-dir { fi } -function setup-monitoring { +function setup-monitoring-firewall { if [[ "${ENABLE_CLUSTER_MONITORING:-false}" == "true" ]]; then # TODO: Implement this. echo "Monitoring not currently supported on AWS" fi } -function teardown-monitoring { +function teardown-monitoring-firewall { if [[ "${ENABLE_CLUSTER_MONITORING:-false}" == "true" ]]; then # TODO: Implement this. echo "Monitoring not currently supported on AWS" @@ -296,10 +296,14 @@ function kube-up { echo "readonly AWS_ZONE='${ZONE}'" echo "readonly MASTER_HTPASSWD='${htpasswd}'" echo "readonly PORTAL_NET='${PORTAL_NET}'" + echo "readonly ENABLE_CLUSTER_MONITORING='${ENABLE_CLUSTER_MONITORING:-false}'" echo "readonly ENABLE_NODE_MONITORING='${ENABLE_NODE_MONITORING:-false}'" + echo "readonly ENABLE_CLUSTER_LOGGING='${ENABLE_CLUSTER_LOGGING:-false}'" echo "readonly ENABLE_NODE_LOGGING='${ENABLE_NODE_LOGGING:-false}'" echo "readonly LOGGING_DESTINATION='${LOGGING_DESTINATION:-}'" + echo "readonly ELASTICSEARCH_LOGGING_REPLICAS='${ELASTICSEARCH_LOGGING_REPLICAS:-}'" echo "readonly ENABLE_CLUSTER_DNS='${ENABLE_CLUSTER_DNS:-false}'" + echo "readonly DNS_REPLICAS='${DNS_REPLICAS:-}'" echo "readonly DNS_SERVER_IP='${DNS_SERVER_IP:-}'" echo "readonly DNS_DOMAIN='${DNS_DOMAIN:-}'" grep -v "^#" "${KUBE_ROOT}/cluster/aws/templates/create-dynamic-salt-files.sh" @@ -498,10 +502,10 @@ function kube-down { $AWS_CMD delete-vpc --vpc-id $vpc_id > $LOG } -function setup-logging { +function setup-logging-firewall { echo "TODO: setup logging" } -function teardown-logging { +function teardown-logging-firewall { echo "TODO: teardown logging" -} \ No newline at end of file +} diff --git a/cluster/azure/util.sh b/cluster/azure/util.sh index e8c9a2150641a..d8675bbabb449 100644 --- a/cluster/azure/util.sh +++ b/cluster/azure/util.sh @@ -558,18 +558,18 @@ function restart-kube-proxy { } # Setup monitoring using heapster and InfluxDB -function setup-monitoring { +function setup-monitoring-firewall { echo "not implemented" >/dev/null } -function teardown-monitoring { +function teardown-monitoring-firewall { echo "not implemented" >/dev/null } -function setup-logging { +function setup-logging-firewall { echo "TODO: setup logging" } -function teardown-logging { +function teardown-logging-firewall { echo "TODO: teardown logging" } diff --git a/cluster/gce/templates/create-dynamic-salt-files.sh b/cluster/gce/templates/create-dynamic-salt-files.sh index 0f894423d73ba..5440da9433130 100644 --- a/cluster/gce/templates/create-dynamic-salt-files.sh +++ b/cluster/gce/templates/create-dynamic-salt-files.sh @@ -22,10 +22,14 @@ mkdir -p /srv/salt-overlay/pillar cat </srv/salt-overlay/pillar/cluster-params.sls node_instance_prefix: '$(echo "$NODE_INSTANCE_PREFIX" | sed -e "s/'/''/g")' portal_net: '$(echo "$PORTAL_NET" | sed -e "s/'/''/g")' +enable_cluster_monitoring: '$(echo "$ENABLE_CLUSTER_MONITORING" | sed -e "s/'/''/g")' enable_node_monitoring: '$(echo "$ENABLE_NODE_MONITORING" | sed -e "s/'/''/g")' +enable_cluster_logging: '$(echo "$ENABLE_CLUSTER_LOGGING" | sed -e "s/'/''/g")' enable_node_logging: '$(echo "$ENABLE_NODE_LOGGING" | sed -e "s/'/''/g")' logging_destination: '$(echo "$LOGGING_DESTINATION" | sed -e "s/'/''/g")' +elasticsearch_replicas: '$(echo "$ELASTICSEARCH_LOGGING_REPLICAS" | sed -e "s/'/''/g")' enable_cluster_dns: '$(echo "$ENABLE_CLUSTER_DNS" | sed -e "s/'/''/g")' +dns_replicas: '$(echo "$DNS_REPLICAS" | sed -e "s/'/''/g")' dns_server: '$(echo "$DNS_SERVER_IP" | sed -e "s/'/''/g")' dns_domain: '$(echo "$DNS_DOMAIN" | sed -e "s/'/''/g")' EOF diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh index 391e8ca542a96..35fc22e72daf3 100755 --- a/cluster/gce/util.sh +++ b/cluster/gce/util.sh @@ -393,10 +393,14 @@ function kube-up { echo "readonly SALT_TAR_URL='${SALT_TAR_URL}'" echo "readonly MASTER_HTPASSWD='${htpasswd}'" echo "readonly PORTAL_NET='${PORTAL_NET}'" + echo "readonly ENABLE_CLUSTER_MONITORING='${ENABLE_CLUSTER_MONITORING:-false}'" echo "readonly ENABLE_NODE_MONITORING='${ENABLE_NODE_MONITORING:-false}'" + echo "readonly ENABLE_CLUSTER_LOGGING='${ENABLE_CLUSTER_LOGGING:-false}'" echo "readonly ENABLE_NODE_LOGGING='${ENABLE_NODE_LOGGING:-false}'" echo "readonly LOGGING_DESTINATION='${LOGGING_DESTINATION:-}'" + echo "readonly ELASTICSEARCH_LOGGING_REPLICAS='${ELASTICSEARCH_LOGGING_REPLICAS:-}'" echo "readonly ENABLE_CLUSTER_DNS='${ENABLE_CLUSTER_DNS:-false}'" + echo "readonly DNS_REPLICAS='${DNS_REPLICAS:-}'" echo "readonly DNS_SERVER_IP='${DNS_SERVER_IP:-}'" echo "readonly DNS_DOMAIN='${DNS_DOMAIN:-}'" grep -v "^#" "${KUBE_ROOT}/cluster/gce/templates/common.sh" @@ -731,106 +735,70 @@ function restart-kube-proxy { ssh-to-node "$1" "sudo /etc/init.d/kube-proxy restart" } -# Setup monitoring using heapster and InfluxDB -function setup-monitoring { - if [[ "${ENABLE_CLUSTER_MONITORING}" == "true" ]]; then - echo "Setting up cluster monitoring using Heapster." - - detect-project - if ! gcloud compute firewall-rules --project "${PROJECT}" describe monitoring-heapster &> /dev/null; then - if ! gcloud compute firewall-rules create monitoring-heapster \ - --project "${PROJECT}" \ - --target-tags="${MINION_TAG}" \ - --network="${NETWORK}" \ - --allow tcp:80 tcp:8083 tcp:8086; then - echo -e "${color_red}Failed to set up firewall for monitoring ${color_norm}" && false - fi - fi +# Setup monitoring firewalls using heapster and InfluxDB +function setup-monitoring-firewall { + if [[ "${ENABLE_CLUSTER_MONITORING}" != "true" ]]; then + return + fi - local kubectl="${KUBE_ROOT}/cluster/kubectl.sh" - local grafana_host="" - if "${kubectl}" create -f "${KUBE_ROOT}/cluster/addons/cluster-monitoring/" &> /dev/null; then - # wait for pods to be scheduled on a node. - echo "waiting for monitoring pods to be scheduled." - for i in `seq 1 10`; do - grafana_host=$("${kubectl}" get pods -l name=influxGrafana -o template -t {{range.items}}{{.currentState.hostIP}}:{{end}} | sed s/://g) - if [[ $grafana_host != *"<"* ]]; then + echo "Setting up firewalls to Heapster based cluster monitoring." + + detect-project + gcloud compute firewall-rules create "${INSTANCE_PREFIX}-monitoring-heapster" --project "${PROJECT}" \ + --allow tcp:80 tcp:8083 tcp:8086 --target-tags="${MINION_TAG}" --network="${NETWORK}" + + local kubectl="${KUBE_ROOT}/cluster/kubectl.sh" + local grafana_host="" + echo "waiting for monitoring pods to be scheduled." + for i in `seq 1 10`; do + grafana_host=$("${kubectl}" get pods -l name=influxGrafana -o template -t {{range.items}}{{.currentState.hostIP}}:{{end}} | sed s/://g) + if [[ ${grafana_host} != *"<"* ]]; then break - fi - sleep 10 - done - if [[ $grafana_host != *"<"* ]]; then - echo - echo -e "${color_green}Grafana dashboard will be available at ${color_yellow}http://$grafana_host${color_green}. Wait for the monitoring dashboard to be online.${color_norm}" - echo - else - echo -e "${color_red}monitoring pods failed to be scheduled.${color_norm}" - fi - else - echo -e "${color_red}Failed to Setup Monitoring ${color_norm}" - teardown-monitoring fi + sleep 10 + done + if [[ ${grafana_host} != *"<"* ]]; then + echo + echo -e "${color_green}Grafana dashboard will be available at ${color_yellow}http://${grafana_host}${color_green}. Wait for the monitoring dashboard to be online.${color_norm}" + echo + else + echo -e "${color_red}Monitoring pods failed to be scheduled!${color_norm}" fi } -function teardown-monitoring { - if [[ "${ENABLE_CLUSTER_MONITORING}" == "true" ]]; then - detect-project - - local kubectl="${KUBE_ROOT}/cluster/kubectl.sh" - local kubecfg="${KUBE_ROOT}/cluster/kubecfg.sh" - "${kubecfg}" resize monitoring-influxGrafanaController 0 &> /dev/null || true - "${kubecfg}" resize monitoring-heapsterController 0 &> /dev/null || true - "${kubectl}" delete -f "${KUBE_ROOT}/cluster/addons/cluster-monitoring/" &> /dev/null || true - if gcloud compute firewall-rules describe --project "${PROJECT}" monitoring-heapster &> /dev/null; then - gcloud compute firewall-rules delete \ - --project "${PROJECT}" \ - --quiet \ - monitoring-heapster &> /dev/null || true - fi +function teardown-monitoring-firewall { + if [[ "${ENABLE_CLUSTER_MONITORING}" != "true" ]]; then + return fi + + detect-project + gcloud compute firewall-rules delete -q "${INSTANCE_PREFIX}-monitoring-heapster" --project "${PROJECT}" || true } -function setup-logging { +function setup-logging-firewall { # If logging with Fluentd to Elasticsearch is enabled then create pods # and services for Elasticsearch (for ingesting logs) and Kibana (for # viewing logs). - if [[ "${ENABLE_NODE_LOGGING-}" == "true" ]] && \ - [[ "${LOGGING_DESTINATION-}" == "elasticsearch" ]] && \ - [[ "${ENABLE_CLUSTER_LOGGING-}" == "true" ]]; then - local -r kubectl="${KUBE_ROOT}/cluster/kubectl.sh" - if sed -e "s/{ELASTICSEARCH_LOGGING_REPLICAS}/${ELASTICSEARCH_LOGGING_REPLICAS}/g" \ - "${KUBE_ROOT}"/cluster/addons/fluentd-elasticsearch/es-controller.yaml.in | \ - "${kubectl}" create -f - &> /dev/null && \ - "${kubectl}" create -f "${KUBE_ROOT}"/cluster/addons/fluentd-elasticsearch/es-service.yaml &> /dev/null && \ - "${kubectl}" create -f "${KUBE_ROOT}"/cluster/addons/fluentd-elasticsearch/kibana-controller.yaml &> /dev/null && \ - "${kubectl}" create -f "${KUBE_ROOT}"/cluster/addons/fluentd-elasticsearch/kibana-service.yaml &> /dev/null; then - gcloud compute firewall-rules create fluentd-elasticsearch-logging --project "${PROJECT}" \ - --allow tcp:5601 tcp:9200 tcp:9300 --target-tags "${INSTANCE_PREFIX}"-minion || true - local -r region="${ZONE::-2}" - local -r es_ip=$(gcloud compute forwarding-rules --project "${PROJECT}" describe --region "${region}" elasticsearch-logging | grep IPAddress | awk '{print $2}') - local -r kibana_ip=$(gcloud compute forwarding-rules --project "${PROJECT}" describe --region "${region}" kibana-logging | grep IPAddress | awk '{print $2}') - echo - echo -e "${color_green}Cluster logs are ingested into Elasticsearch running at ${color_yellow}http://${es_ip}:9200" - echo -e "${color_green}Kibana logging dashboard will be available at ${color_yellow}http://${kibana_ip}:5601${color_norm}" - echo - else - echo -e "${color_red}Failed to launch Elasticsearch and Kibana pods and services for logging.${color_norm}" - fi + if [[ "${ENABLE_NODE_LOGGING-}" != "true" ]] || \ + [[ "${LOGGING_DESTINATION-}" != "elasticsearch" ]] || \ + [[ "${ENABLE_CLUSTER_LOGGING-}" != "true" ]]; then + return fi + + detect-project + gcloud compute firewall-rules create "${INSTANCE_PREFIX}-fluentd-elasticsearch-logging" --project "${PROJECT}" \ + --allow tcp:5601 tcp:9200 tcp:9300 --target-tags "${MINION_TAG}" --network="${NETWORK}" } -function teardown-logging { - if [[ "${ENABLE_NODE_LOGGING-}" == "true" ]] && \ - [[ "${LOGGING_DESTINATION-}" == "elasticsearch" ]] && \ - [[ "${ENABLE_CLUSTER_LOGGING-}" == "true" ]]; then - local -r kubectl="${KUBE_ROOT}/cluster/kubectl.sh" - "${kubectl}" delete replicationController elasticsearch-logging-controller &> /dev/null || true - "${kubectl}" delete service elasticsearch-logging &> /dev/null || true - "${kubectl}" delete replicationController kibana-logging-controller &> /dev/null || true - "${kubectl}" delete service kibana-logging &> /dev/null || true - gcloud compute firewall-rules delete -q fluentd-elasticsearch-logging --project "${PROJECT}" || true +function teardown-logging-firewall { + if [[ "${ENABLE_NODE_LOGGING-}" != "true" ]] || \ + [[ "${LOGGING_DESTINATION-}" != "elasticsearch" ]] || \ + [[ "${ENABLE_CLUSTER_LOGGING-}" != "true" ]]; then + return fi + + detect-project + gcloud compute firewall-rules delete -q "${INSTANCE_PREFIX}-fluentd-elasticsearch-logging" --project "${PROJECT}" || true } # Perform preparations required to run e2e tests diff --git a/cluster/gke/util.sh b/cluster/gke/util.sh index 2b92efed561aa..aa2388afee4fd 100644 --- a/cluster/gke/util.sh +++ b/cluster/gke/util.sh @@ -115,8 +115,8 @@ function kube-up() { } # Called during cluster/kube-up.sh -function setup-monitoring() { - echo "... in setup-monitoring()" >&2 +function setup-monitoring-firewall() { + echo "... in setup-monitoring-firewall()" >&2 # TODO(mbforbes): This isn't currently supported in GKE. } @@ -239,8 +239,8 @@ function test-teardown() { } # Tears down monitoring. -function teardown-monitoring() { - echo "... in teardown-monitoring()" >&2 +function teardown-monitoring-firewall() { + echo "... in teardown-monitoring-firewall()" >&2 # TODO(mbforbes): This isn't currently supported in GKE. } @@ -257,10 +257,10 @@ function kube-down() { --zone="${ZONE}" "${CLUSTER_NAME}" } -function setup-logging { +function setup-logging-firewall { echo "TODO: setup logging" } -function teardown-logging { +function teardown-logging-firewall { echo "TODO: teardown logging" } diff --git a/cluster/kube-down.sh b/cluster/kube-down.sh index 866a5caaf2e27..cb431f136974b 100755 --- a/cluster/kube-down.sh +++ b/cluster/kube-down.sh @@ -27,8 +27,8 @@ source "${KUBE_ROOT}/cluster/${KUBERNETES_PROVIDER}/util.sh" echo "Bringing down cluster using provider: $KUBERNETES_PROVIDER" verify-prereqs -teardown-monitoring -teardown-logging +teardown-monitoring-firewall +teardown-logging-firewall kube-down diff --git a/cluster/kube-up.sh b/cluster/kube-up.sh index 247287cdeffb8..37d59d9f59db9 100755 --- a/cluster/kube-up.sh +++ b/cluster/kube-up.sh @@ -39,22 +39,10 @@ kube-up echo "... calling validate-cluster" >&2 "${KUBE_ROOT}/cluster/validate-cluster.sh" -echo "... calling setup-monitoring" >&2 -setup-monitoring - -if [[ "${ENABLE_CLUSTER_DNS}" == "true" ]]; then - echo "... setting up cluster DNS" - sed -e "s/{DNS_DOMAIN}/$DNS_DOMAIN/g" \ - -e "s/{DNS_REPLICAS}/$DNS_REPLICAS/g" \ - "${KUBE_ROOT}/cluster/addons/dns/skydns-rc.yaml.in" \ - | "${KUBE_ROOT}/cluster/kubectl.sh" create -f - - - sed -e "s/{DNS_SERVER_IP}/$DNS_SERVER_IP/g" \ - "${KUBE_ROOT}/cluster/addons/dns/skydns-svc.yaml.in" \ - | "${KUBE_ROOT}/cluster/kubectl.sh" create -f - -fi - -echo "... calling setup-logging" >&2 -setup-logging +echo "... calling setup-monitoring-firewall" >&2 +setup-monitoring-firewall + +echo "... calling setup-logging-firewall" >&2 +setup-logging-firewall echo "Done" >&2 diff --git a/cluster/rackspace/util.sh b/cluster/rackspace/util.sh index f8c4552fc0d32..bc1637d619442 100644 --- a/cluster/rackspace/util.sh +++ b/cluster/rackspace/util.sh @@ -342,19 +342,19 @@ kube-up() { echo } -function setup-monitoring { +function setup-monitoring-firewall { echo "TODO" } -function teardown-monitoring { +function teardown-monitoring-firewall { echo "TODO" } -function setup-logging { +function setup-logging-firewall { echo "TODO: setup logging" } -function teardown-logging { +function teardown-logging-firewall { echo "TODO: teardown logging" } diff --git a/cluster/saltbase/salt/kube-addons/init.sls b/cluster/saltbase/salt/kube-addons/init.sls new file mode 100644 index 0000000000000..a073f43afa4dd --- /dev/null +++ b/cluster/saltbase/salt/kube-addons/init.sls @@ -0,0 +1,79 @@ +{% if pillar.get('enable_cluster_monitoring', '').lower() == 'true' %} +/etc/kubernetes/addons/cluster-monitoring: + file.recurse: + - source: salt://kube-addons/cluster-monitoring + - include_pat: E@^.+\.yaml$ + - user: root + - group: root + - dir_mode: 755 + - file_mode: 644 +{% endif %} + +{% if pillar.get('enable_cluster_dns', '').lower() == 'true' %} +/etc/kubernetes/addons/dns/skydns-svc.yaml: + file.managed: + - source: salt://kube-addons/dns/skydns-svc.yaml.in + - template: jinja + - group: root + - dir_mode: 755 + - makedirs: True + +/etc/kubernetes/addons/dns/skydns-rc.yaml: + file.managed: + - source: salt://kube-addons/dns/skydns-rc.yaml.in + - template: jinja + - group: root + - dir_mode: 755 + - makedirs: True +{% endif %} + +{% if pillar.get('enable_node_logging', '').lower() == 'true' + and pillar.get('logging_destination').lower() == 'elasticsearch' + and pillar.get('enable_cluster_logging', '').lower() == 'true' %} +/etc/kubernetes/addons/fluentd-elasticsearch: + file.recurse: + - source: salt://kube-addons/fluentd-elasticsearch + - include_pat: E@^.+\.yaml$ + - user: root + - group: root + - dir_mode: 755 + - file_mode: 644 + +/etc/kubernetes/addons/fluentd-elasticsearch/es-controller.yaml: + file.managed: + - source: salt://kube-addons/fluentd-elasticsearch/es-controller.yaml.in + - template: jinja + - group: root + - dir_mode: 755 + - makedirs: True +{% endif %} + +{% if grains['os_family'] == 'RedHat' %} + +/usr/lib/systemd/system/kube-addons.service: + file.managed: + - source: salt://kube-addons/kube-addons.service + - user: root + - group: root + +/usr/lib/systemd/scripts/kube-addons: + file.managed: + - source: salt://kube-addons/initd + - user: root + - group: root + - mode: 755 + +{% else %} + +/etc/init.d/kube-addons: + file.managed: + - source: salt://kube-addons/initd + - user: root + - group: root + - mode: 755 + +{% endif %} + +kube-addons: + service.running: + - enable: True diff --git a/cluster/saltbase/salt/kube-addons/initd b/cluster/saltbase/salt/kube-addons/initd new file mode 100644 index 0000000000000..8cc1d422b5d8b --- /dev/null +++ b/cluster/saltbase/salt/kube-addons/initd @@ -0,0 +1,123 @@ +#!/bin/bash +# +### BEGIN INIT INFO +# Provides: kube-addons +# Required-Start: $local_fs $network $syslog kube-apiserver +# Required-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Kubernetes Addon Object Manager +# Description: +# Enforces installation of Kubernetes Addon Objects +### END INIT INFO + + +# PATH should only include /usr/* if it runs after the mountnfs.sh script +PATH=/sbin:/usr/sbin:/bin:/usr/bin +DESC="Kubernetes Addon Object Manager" +NAME=kube-addons +DAEMON_LOG_FILE=/var/log/$NAME.log +PIDFILE=/var/run/$NAME.pid +SCRIPTNAME=/etc/init.d/$NAME +KUBECTL=/usr/local/bin/kubectl + +# Define LSB log_* functions. +# Depend on lsb-base (>= 3.2-14) to ensure that this file is present +# and status_of_proc is working. +. /lib/lsb/init-functions + +function addon_manager_async() { + # The business logic for whether a given object should be created + # was already enforced by salt, and /etc/kubernetes/addons is the + # managed result is of that. Start everything below that directory. + echo "== Kubernetes addon manager started at $(date -Is) ==" + for obj in $(find /etc/kubernetes/addons -name \*.yaml); do + ${KUBECTL} create -f ${obj} & + echo "++ addon ${obj} started in pid $! ++" + done + noerrors="true" + for pid in $(jobs -p); do + wait ${pid} || noerrors="false" + echo "++ pid ${pid} complete ++" + done + if [ ${noerrors} == "true" ]; then + echo "== Kubernetes addon manager completed successfully at $(date -Is) ==" + else + echo "== Kubernetes addon manager completed with errors at $(date -Is) ==" + fi + + # We stay around so that status checks by salt make it look like + # the service is good. (We could do this is other ways, but this + # is simple.) + sleep infinity +} + +# +# Function that starts the daemon/service +# +do_start() +{ + addon_manager_async >${DAEMON_LOG_FILE} 2>&1 & + echo $! > ${PIDFILE} + disown +} + +# +# Function that stops the daemon/service +# +do_stop() +{ + kill $(cat ${PIDFILE}) + rm ${PIDFILE} + return +} + +case "$1" in + start) + log_daemon_msg "Starting $DESC" "$NAME" + do_start + case "$?" in + 0|1) log_end_msg 0 || exit 0 ;; + 2) log_end_msg 1 || exit 1 ;; + esac + ;; + stop) + log_daemon_msg "Stopping $DESC" "$NAME" + do_stop + case "$?" in + 0|1) log_end_msg 0 ;; + 2) exit 1 ;; + esac + ;; + status) + if [ ! -e ${PIDFILE} ]; then + exit 1 + fi + pid=$(cat ${PIDFILE}) + # Checks that ${pid} is running AND is us. + ps --no-headers ${pid} | grep ${SCRIPTNAME} > /dev/null || exit $? + ;; + + restart|force-reload) + log_daemon_msg "Restarting $DESC" "$NAME" + do_stop + case "$?" in + 0|1) + do_start + case "$?" in + 0) log_end_msg 0 ;; + 1) log_end_msg 1 ;; # Old process is still running + *) log_end_msg 1 ;; # Failed to start + esac + ;; + *) + # Failed to stop + log_end_msg 1 + ;; + esac + ;; + *) + echo "Usage: $SCRIPTNAME {start|stop|status|restart|force-reload}" >&2 + exit 3 + ;; +esac diff --git a/cluster/saltbase/salt/kube-addons/kube-addons.service b/cluster/saltbase/salt/kube-addons/kube-addons.service new file mode 100644 index 0000000000000..6942e87587b8f --- /dev/null +++ b/cluster/saltbase/salt/kube-addons/kube-addons.service @@ -0,0 +1,9 @@ +[Unit] +Description=Kubernetes Addon Object Manager +Documentation=https://github.com/GoogleCloudPlatform/kubernetes + +[Service] +ExecStart=/usr/lib/systemd/scripts/kube-addons start + +[Install] +WantedBy=multi-user.target diff --git a/cluster/saltbase/salt/top.sls b/cluster/saltbase/salt/top.sls index e4ec1660a700f..f58c6cec1538c 100644 --- a/cluster/saltbase/salt/top.sls +++ b/cluster/saltbase/salt/top.sls @@ -36,6 +36,7 @@ base: - nginx - kube-client-tools - logrotate + - kube-addons {% if grains['cloud'] is defined and grains['cloud'] == 'azure' %} - openvpn {% endif %} diff --git a/cluster/vagrant/provision-master.sh b/cluster/vagrant/provision-master.sh index c7f5a6fab9e5b..22df0ebf07d7c 100755 --- a/cluster/vagrant/provision-master.sh +++ b/cluster/vagrant/provision-master.sh @@ -82,10 +82,14 @@ mkdir -p /srv/salt-overlay/pillar cat </srv/salt-overlay/pillar/cluster-params.sls portal_net: '$(echo "$PORTAL_NET" | sed -e "s/'/''/g")' cert_ip: '$(echo "$MASTER_IP" | sed -e "s/'/''/g")' + enable_cluster_monitoring: '$(echo "$ENABLE_CLUSTER_MONITORING" | sed -e "s/'/''/g")' enable_node_monitoring: '$(echo "$ENABLE_NODE_MONITORING" | sed -e "s/'/''/g")' + enable_cluster_logging: '$(echo "$ENABLE_CLUSTER_LOGGING" | sed -e "s/'/''/g")' enable_node_logging: '$(echo "$ENABLE_NODE_LOGGING" | sed -e "s/'/''/g")' logging_destination: '$(echo "$LOGGING_DESTINATION" | sed -e "s/'/''/g")' + elasticsearch_replicas: '$(echo "$ELASTICSEARCH_LOGGING_REPLICAS" | sed -e "s/'/''/g")' enable_cluster_dns: '$(echo "$ENABLE_CLUSTER_DNS" | sed -e "s/'/''/g")' + dns_replicas: '$(echo "$DNS_REPLICAS" | sed -e "s/'/''/g")' dns_server: '$(echo "$DNS_SERVER_IP" | sed -e "s/'/''/g")' dns_domain: '$(echo "$DNS_DOMAIN" | sed -e "s/'/''/g")' EOF diff --git a/cluster/vagrant/util.sh b/cluster/vagrant/util.sh index ca72fa3b69011..4e8e54829cc93 100644 --- a/cluster/vagrant/util.sh +++ b/cluster/vagrant/util.sh @@ -287,11 +287,11 @@ function restart-kube-proxy { ssh-to-node "$1" "sudo systemctl restart kube-proxy" } -function setup-monitoring { +function setup-monitoring-firewall { echo "TODO" 1>&2 } -function teardown-monitoring { +function teardown-monitoring-firewall { echo "TODO" 1>&2 } @@ -300,10 +300,10 @@ function prepare-e2e() { echo "Vagrant doesn't need special preparations for e2e tests" 1>&2 } -function setup-logging { +function setup-logging-firewall { echo "TODO: setup logging" } -function teardown-logging { +function teardown-logging-firewall { echo "TODO: teardown logging" -} \ No newline at end of file +} diff --git a/cluster/vsphere/templates/create-dynamic-salt-files.sh b/cluster/vsphere/templates/create-dynamic-salt-files.sh index f9a01c41a9b61..cb483ad134061 100755 --- a/cluster/vsphere/templates/create-dynamic-salt-files.sh +++ b/cluster/vsphere/templates/create-dynamic-salt-files.sh @@ -22,10 +22,14 @@ mkdir -p /srv/salt-overlay/pillar cat </srv/salt-overlay/pillar/cluster-params.sls node_instance_prefix: $NODE_INSTANCE_PREFIX portal_net: $PORTAL_NET +enable_cluster_monitoring: $ENABLE_CLUSTER_MONITORING enable_node_monitoring: $ENABLE_NODE_MONITORING +enable_cluster_logging: $ENABLE_CLUSTER_LOGGING enable_node_logging: $ENABLE_NODE_LOGGING logging_destination: $LOGGING_DESTINATION +elasticsearch_replicas: $ELASTICSEARCH_LOGGING_REPLICAS enable_cluster_dns: $ENABLE_CLUSTER_DNS +dns_replicas: $DNS_REPLICAS dns_server: $DNS_SERVER_IP dns_domain: $DNS_DOMAIN EOF diff --git a/cluster/vsphere/util.sh b/cluster/vsphere/util.sh index 03d279fa6b3fd..7469e0ac02d09 100755 --- a/cluster/vsphere/util.sh +++ b/cluster/vsphere/util.sh @@ -478,18 +478,18 @@ function test-teardown { echo "TODO" } -function setup-monitoring { +function setup-monitoring-firewall { echo "TODO" } -function teardown-monitoring { +function teardown-monitoring-firewall { echo "TODO" } -function setup-logging { +function setup-logging-firewall { echo "TODO: setup logging" } -function teardown-logging { +function teardown-logging-firewall { echo "TODO: teardown logging" } \ No newline at end of file