Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Run l7 controller on master #26048

Merged
merged 3 commits into from
May 31, 2016
Merged

Run l7 controller on master #26048

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f4d2334
Add a l7 static pod
bprashanth Apr 5, 2016
e2b980b
Fix e2es
bprashanth May 23, 2016
c01a058
Trusty deployment
bprashanth May 28, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions build/common.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -926,6 +926,7 @@ function kube::release::package_kube_manifests_tarball() {
cp "${salt_dir}/kube-apiserver/abac-authz-policy.jsonl" "${dst_dir}"
cp "${salt_dir}/kube-controller-manager/kube-controller-manager.manifest" "${dst_dir}"
cp "${salt_dir}/kube-addons/kube-addon-manager.yaml" "${dst_dir}"
cp "${salt_dir}/l7-gcp/glbc.manifest" "${dst_dir}"
cp "${KUBE_ROOT}/cluster/gce/trusty/configure-helper.sh" "${dst_dir}/trusty-configure-helper.sh"
cp "${KUBE_ROOT}/cluster/gce/gci/configure-helper.sh" "${dst_dir}/gci-configure-helper.sh"
cp "${KUBE_ROOT}/cluster/gce/gci/health-monitor.sh" "${dst_dir}/health-monitor.sh"
Expand Down
38 changes: 4 additions & 34 deletions ...dons/cluster-loadbalancing/glbc/glbc.yaml → ...alancing/glbc/default-svc-controller.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,33 +1,26 @@
{% set kube_uid = "" -%}
{% if pillar['kube_uid'] is defined -%}
{% set kube_uid = pillar['kube_uid'] %}
{% endif -%}

apiVersion: v1
kind: ReplicationController
metadata:
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If gke keeps re-salting clusters it defeats the purpose of file based persistence, and there are too many edge cases anway, so it's safer to maintain and allocate the uid in the l7 pod.

name: l7-lb-controller-v0.6.2
name: l7-default-backend-v1.0
namespace: kube-system
labels:
k8s-app: glbc
version: v0.6.2
version: v1.0
kubernetes.io/cluster-service: "true"
kubernetes.io/name: "GLBC"
spec:
# There should never be more than 1 controller alive simultaneously.
replicas: 1
selector:
k8s-app: glbc
version: v0.6.2
version: v1.0
template:
metadata:
labels:
k8s-app: glbc
version: v0.6.2
version: v1.0
name: glbc
kubernetes.io/cluster-service: "true"
spec:
terminationGracePeriodSeconds: 600
containers:
- name: default-http-backend
# Any image is permissible as long as:
Expand All @@ -50,26 +43,3 @@ spec:
requests:
cpu: 10m
memory: 20Mi
- image: gcr.io/google_containers/glbc:0.6.2
livenessProbe:
httpGet:
path: /healthz
port: 8081
scheme: HTTP
initialDelaySeconds: 30
# healthz reaches out to GCE
periodSeconds: 30
timeoutSeconds: 5
name: l7-lb-controller
resources:
limits:
cpu: 100m
memory: 100Mi
requests:
cpu: 100m
memory: 50Mi
args:
- --default-backend-service=kube-system/default-http-backend
- --verbose=true
- --sync-period=60s
- --cluster-uid={{kube_uid}}
16 changes: 12 additions & 4 deletions cluster/gce/gci/configure-helper.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -558,7 +558,6 @@ function start-kube-apiserver {
function start-kube-controller-manager {
echo "Start kubernetes controller-manager"
prepare-log-file /var/log/kube-controller-manager.log

# Calculate variables and assemble the command line.
local params="${CONTROLLER_MANAGER_TEST_LOG_LEVEL:-"--v=2"} ${CONTROLLER_MANAGER_TEST_ARGS:-}"
params+=" --cloud-provider=gce"
Expand Down Expand Up @@ -706,9 +705,6 @@ function start-kube-addons {
fi
if [[ "${ENABLE_L7_LOADBALANCING:-}" == "glbc" ]]; then
setup-addon-manifests "addons" "cluster-loadbalancing/glbc"
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This line can be moved into function start-lb-controller

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

but then it won't start-lb-controller without the right env vars, which makes the function name a lie right?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oh, no i guess it can. it splits addon deployment, but consolidates loadbalancer deployment. So ill send a follow up.

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry for late comment, I just came back to office. It is not a functionality problem, just an improvement. So, you don't have to make a PR dedicated for the correction. It is up to you

local -r glbc_yaml="${dst_dir}/cluster-loadbalancing/glbc/glbc.yaml"
remove-salt-config-comments "${glbc_yaml}"
sed -i -e "s@{{ *kube_uid *}}@${KUBE_UID:-}@g" "${glbc_yaml}"
fi
if [[ "${ENABLE_CLUSTER_DNS:-}" == "true" ]]; then
setup-addon-manifests "addons" "dns"
Expand Down Expand Up @@ -761,6 +757,17 @@ function start-fluentd {
fi
}

# Starts a l7 loadbalancing controller for ingress.
function start-lb-controller {
if [[ "${ENABLE_L7_LOADBALANCING:-}" == "glbc" ]]; then
echo "Starting GCE L7 pod"
prepare-log-file /var/log/glbc.log
local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/glbc.manifest"
cp "${src_file}" /etc/kubernetes/manifests/
fi
}


function reset-motd {
# kubelet is installed both on the master and nodes, and the version is easy to parse (unlike kubectl)
local -r version="$(/usr/bin/kubelet --version=true | cut -f2 -d " ")"
Expand Down Expand Up @@ -829,6 +836,7 @@ if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
start-kube-scheduler
start-kube-addons
start-cluster-autoscaler
start-lb-controller
else
start-kube-proxy
# Kube-registry-proxy.
Expand Down
53 changes: 53 additions & 0 deletions cluster/saltbase/salt/l7-gcp/glbc.manifest
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
apiVersion: v1
kind: Pod
metadata:
name: l7-lb-controller-v0.6.3
namespace: kube-system
labels:
k8s-app: glbc
version: v0.6.3
kubernetes.io/cluster-service: "true"
kubernetes.io/name: "GLBC"
spec:
terminationGracePeriodSeconds: 600
hostNetwork: true
containers:
- image: gcr.io/google_containers/glbc:0.6.3
livenessProbe:
httpGet:
path: /healthz
port: 8086
scheme: HTTP
initialDelaySeconds: 30
# healthz reaches out to GCE
periodSeconds: 30
timeoutSeconds: 5
name: l7-lb-controller
volumeMounts:
- mountPath: /etc/gce.conf
name: cloudconfig
readOnly: true
- mountPath: /var/log/glbc.log
name: logfile
readOnly: false
resources:
# Request and limits are set to accomodate this pod alongside the other
# master components on a single core master.
limits:
cpu: 50m
memory: 100Mi
requests:
cpu: 10m
memory: 50Mi
command:
# TODO: split this out into args when we no longer need to pipe stdout to a file #6428
- sh
- -c
- '/glbc --default-backend-service=kube-system/default-http-backend --sync-period=60s --running-in-cluster=false --use-real-cloud=true --config-file-path=/etc/gce.conf --healthz-port=8086 1>>/var/log/glbc.log 2>&1'
volumes:
- hostPath:
path: /etc/gce.conf
name: cloudconfig
- hostPath:
path: /var/log/glbc.log
name: logfile
17 changes: 17 additions & 0 deletions cluster/saltbase/salt/l7-gcp/init.sls
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
/etc/kubernetes/manifests/glbc.manifest:
file.managed:
- source: salt://l7-gcp/glbc.manifest
- template: jinja
- user: root
- group: root
- mode: 644
- makedirs: true
- dir_mode: 755

/var/log/glbc.log:
file.managed:
- user: root
- group: root
- mode: 644


3 changes: 3 additions & 0 deletions cluster/saltbase/salt/top.sls
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ base:
- cni
{% elif pillar.get('network_provider', '').lower() == 'cni' %}
- cni
{% endif %}
{% if pillar.get('enable_l7_loadbalancing', '').lower() == 'glbc' %}
- l7-gcp
{% endif %}
- kube-apiserver
- kube-controller-manager
Expand Down
44 changes: 7 additions & 37 deletions test/e2e/ingress.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ import (
"k8s.io/kubernetes/pkg/api"
"k8s.io/kubernetes/pkg/apis/extensions"
client "k8s.io/kubernetes/pkg/client/unversioned"
"k8s.io/kubernetes/pkg/labels"
utilexec "k8s.io/kubernetes/pkg/util/exec"
"k8s.io/kubernetes/pkg/util/intstr"
"k8s.io/kubernetes/pkg/util/wait"
Expand Down Expand Up @@ -62,6 +61,10 @@ var (
testImage = "gcr.io/google_containers/n-way-http:1.0"
httpContainerPort = 8080

// Name of the config-map and key the ingress controller stores its uid in.
uidConfigMap = "ingress-uid"
uidKey = "uid"

expectedLBCreationTime = 7 * time.Minute
expectedLBHealthCheckTime = 7 * time.Minute

Expand Down Expand Up @@ -232,26 +235,6 @@ func gcloudDelete(resource, name, project string) {
}
}

// kubectlLogLBController logs kubectl debug output for the L7 controller pod.
func kubectlLogLBController(c *client.Client, ns string) {
selector := labels.SelectorFromSet(labels.Set(controllerLabels))
options := api.ListOptions{LabelSelector: selector}
podList, err := c.Pods(api.NamespaceAll).List(options)
if err != nil {
framework.Logf("Cannot log L7 controller output, error listing pods %v", err)
return
}
if len(podList.Items) == 0 {
framework.Logf("Loadbalancer controller pod not found")
return
}
for _, p := range podList.Items {
framework.Logf("\nLast 100 log lines of %v\n", p.Name)
l, _ := framework.RunKubectl("logs", p.Name, fmt.Sprintf("--namespace=%v", ns), "-c", lbContainerName, "--tail=100")
framework.Logf(l)
}
}

type IngressController struct {
ns string
rcPath string
Expand All @@ -263,24 +246,12 @@ type IngressController struct {
}

func (cont *IngressController) getL7AddonUID() (string, error) {
listOpts := api.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set(clusterAddonLBLabels))}
existingRCs, err := cont.c.ReplicationControllers(api.NamespaceSystem).List(listOpts)
cm, err := cont.c.ConfigMaps(api.NamespaceSystem).Get(uidConfigMap)
if err != nil {
return "", err
}
if len(existingRCs.Items) != 1 {
return "", fmt.Errorf("Unexpected number of lb cluster addons %v with label %v in kube-system namespace", len(existingRCs.Items), clusterAddonLBLabels)
}
rc := existingRCs.Items[0]
commandPrefix := "--cluster-uid="
for i, c := range rc.Spec.Template.Spec.Containers {
if c.Name == lbContainerName {
for _, arg := range rc.Spec.Template.Spec.Containers[i].Args {
if strings.HasPrefix(arg, commandPrefix) {
return strings.Replace(arg, commandPrefix, "", -1), nil
}
}
}
if uid, ok := cm.Data[uidKey]; ok {
return uid, nil
}
return "", fmt.Errorf("Could not find cluster UID for L7 addon pod")
}
Expand Down Expand Up @@ -448,7 +419,6 @@ var _ = framework.KubeDescribe("GCE L7 LoadBalancer Controller [Feature:Ingress]
AfterEach(func() {
framework.Logf("Average creation time %+v, health check time %+v", creationTimes, responseTimes)
if CurrentGinkgoTestDescription().Failed {
kubectlLogLBController(client, ns)
framework.Logf("\nOutput of kubectl describe ing:\n")
desc, _ := framework.RunKubectl("describe", "ing", fmt.Sprintf("--namespace=%v", ns))
framework.Logf(desc)
Expand Down