Skip to content

Commit

Permalink
istio-agent: generate proxyless gRPC bootstrap with cert config (#33269)
Browse files Browse the repository at this point in the history 
* generate proxyless gRPC bootstrap in agent

Change-Id: Ia258097cc1da6f41bed8d5e60f676ca73103824d

* cleanup replace fails

Change-Id: I10cc17e3659dd070b2455de3d04507f088ed3797

* echo per-call xds

Change-Id: Ia62a9045b0c9cfb579aa6a8f2c8200324fe0af66

* unit test

Change-Id: Ifb2375977c1bf1edddb52c5f993a2b889f86479d

* create samples directory with templates

Change-Id: I443c7a14cf40e4f9bc5e535e800eb5331467635d

* use agent in proxyless integration test

Change-Id: I3890ac8a99b8fe9a0731aad21191a741df837e81

* debug

Change-Id: Icc5356ef20cec8e9329d50d097472746ef2b02aa

* attempt template fixes

Change-Id: I9960bc0255b977cad921f33966cb8e987d4c2713

* more template fixes

Change-Id: I9adddc49bdf7fed2cfe14388a5cb9804d78ed24e

* block exit

Change-Id: I7d6e0f4662f74058bee9df0c353b23442931c6dd

* fix readiness probe

Change-Id: I0d2c6e1f02435c6dd0eec1fb89dc20310517a5e1

* use statusPort for readiness probe

Change-Id: I9952ffc6e0b4b9e0a5e25692609c8e843470a0e9

* disable agent DNS

Change-Id: I5dc03ad52eb8dbca30d6240d5e9c48976c17ac6c

* echo: give more information about failure to ready

Change-Id: I0b079591187d049b8e18f6037ece924c14495f41

* tf: don't wait for envoy config

Change-Id: I78a54912664dbd72c75c0c2f7d631c4ee9a22106

* cleanup dump

Change-Id: If89133e90f4e95ef92bab4c1bde25272da8490d9

* fix template

Change-Id: I43b0e2c4afbef078a57a92815372e6dd5fd8b694

* match bootstrap to gRPC instead of godoc

Change-Id: Ic34c89d6bc541b09e21ce71cc028dcf88356d7af

* use UDS

Change-Id: Id4e0a629c97be0265b712aeb7a7260cf1c318aa8

* use server features instead of env for v3 support

Change-Id: I11ced33af07458b6d3f163f89ced775c8de4bd73

* set grpc gen

Change-Id: Iafa02f9d693b7adf70fb7043375b2e8ca7183fac

* misc cleanup

Change-Id: Icbeb8d2d539883d32bbb84f947f513ad95f43698

* exclude external svc target

Change-Id: I908f8ab68c4796e62a550880e5185a671128c9d1

* cleanup envoy disable

Change-Id: I1e80e7a78838b58d059321c40c99ad831e9ff5f0

* make unit tests deterministic

Change-Id: Ide55d5322659195597f66d9e69a523d14dcc9751

* grpcgen: basic mTLS support

Change-Id: I2958f0e6557ffb950687287e3a31b587d2c70126

* trim template

Change-Id: Ib611a9c4253b8f34cdf3b9a084035ce65ae1b6a3

* use env instead of flag

Change-Id: Id201f9ae2f2028664f6ee85cf93fd06b50a89d1a

* ready probe

Change-Id: I8bfa491190e8d936863f320c4aadacb626778667

* lint

Change-Id: I5856b2eb4b35224fa60919c94695cf9bebb38202

* mdlint

Change-Id: I6c97236c1b1dbc0d208c7f00980e644e03ed0ff3

* gen

Change-Id: I224aa28bbe23079a0c0c2833ea6a58449c6af4a5

* cleanup bad renames

Change-Id: I5d0725b9b3aec02d7ea4bd0a2ac5e243ec694ab6

* simplify templates

Change-Id: I649ad6e6ddd3aebe5d745366eca1f21d840f2e49

* grpc templates in default install

Change-Id: Idf830c317f2977e0f666502a2b12eef3e2ba8ba7

* reduce grpc env

Change-Id: Iab279af4e4a8bbeca364aa918a4f85d85745dc57

* gen templ

* cleanup

Change-Id: I45f9057867d52b3da946d4f6308491f0ec09c113

* rename istio-proxy to istio-agent to avoid CNI

Change-Id: I4e47bd7692fb8fe6f814cb227dc33d59bbfc52c3

* Revert "rename istio-proxy to istio-agent to avoid CNI"

This reverts commit 76ec858.

* add annotation to exclude pod from CNI

Change-Id: I44fd5c9824bc71cb480b52b1468fea05724905d0

* cni unit test

Change-Id: I8664051111a3effb71741d6050021ddf5e1633ce

* inspect env instead of using annotation

Change-Id: I1ed9c541e0dfb98b8913601bc040a8ca4199dd05
  • Loading branch information
@stevenctl
stevenctl authored Jun 24, 2021
1 parent ca61f98 commit fe710ef
Show file tree
Hide file tree
Showing 29 changed files with 1,149 additions and 110 deletions.
14 changes: 11 additions & 3 deletions cni/cmd/istio-cni/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,10 @@ import (
)

var (
nsSetupBinDir = "/opt/cni/bin"
injectAnnotationKey = annotation.SidecarInject.Name
sidecarStatusKey = annotation.SidecarStatus.Name
nsSetupBinDir = "/opt/cni/bin"
injectAnnotationKey = annotation.SidecarInject.Name
sidecarStatusKey = annotation.SidecarStatus.Name

interceptRuleMgrType = defInterceptRuleMgrType
loggingOptions = log.DefaultOptions()
podRetrievalMaxRetries = 30
Expand Down Expand Up @@ -207,6 +208,13 @@ func cmdAdd(args *skel.CmdArgs) (err error) {
excludePod = true
}

if val, ok := pi.ProxyEnvironments["DISABLE_ENVOY"]; ok {
if val, err := strconv.ParseBool(val); err == nil && val {
log.Infof("Pod excluded due to DISABLE_ENVOY on istio-proxy")
excludePod = true
}
}

log.Infof("Found containers %v", pi.Containers)
if len(pi.Containers) > 1 {
log.WithLabels(
Expand Down
22 changes: 22 additions & 0 deletions cni/cmd/istio-cni/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ var (
testContainers = []string{"mockContainer"}
testLabels = map[string]string{}
testAnnotations = map[string]string{}
testProxyEnv = map[string]string{}
testInitContainers = map[string]struct{}{
"foo-init": {},
}
Expand Down Expand Up @@ -126,6 +127,7 @@ func mockgetK8sPodInfo(client *kubernetes.Clientset, podName, podNamespace strin
pi.Labels = testLabels
pi.Annotations = testAnnotations
pi.InitContainers = testInitContainers
pi.ProxyEnvironments = testProxyEnv

return &pi, nil
}
Expand All @@ -139,6 +141,7 @@ func resetGlobalTestVariables() {
testContainers = []string{"mockContainer"}
testLabels = map[string]string{}
testAnnotations = map[string]string{}
testProxyEnv = map[string]string{}

interceptRuleMgrType = "mock"
testAnnotations[sidecarStatusKey] = "true"
Expand Down Expand Up @@ -368,6 +371,25 @@ func TestCmdAddExcludePodWithIstioInitContainer(t *testing.T) {
}
}

func TestCmdAddExcludePodWithEnvoyDisableEnv(t *testing.T) {
defer resetGlobalTestVariables()

k8Args = "K8S_POD_NAMESPACE=testNS;K8S_POD_NAME=testPodName"
testContainers = []string{"mockContainer", "mockContainer2"}
testInitContainers = map[string]struct{}{
"foo-init": {},
}
testAnnotations[sidecarStatusKey] = "true"
testProxyEnv["DISABLE_ENVOY"] = "true"
getKubePodInfoCalled = true

testCmdAdd(t)

if nsenterFuncCalled {
t.Fatalf("expected nsenterFunc to not get called")
}
}

func TestCmdAddWithKubevirtInterfaces(t *testing.T) {
defer resetGlobalTestVariables()

Expand Down
292 changes: 292 additions & 0 deletions manifests/charts/istio-control/istio-discovery/files/gen-istio.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -885,6 +885,298 @@ data:
securityContext:
fsGroup: 1337
{{- end }}
grpc-simple: |
spec:
initContainers:
- name: grpc-bootstrap-init
image: busybox:1.28
volumeMounts:
- mountPath: /var/lib/grpc/data/
name: grpc-io-proxyless-bootstrap
env:
- name: INSTANCE_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
command:
- sh
- "-c"
- |-
NODE_ID="sidecar~${INSTANCE_IP}~${POD_NAME}.${POD_NAMESPACE}~cluster.local"
echo '
{
"xds_servers": [
{
"server_uri": "dns:///istiod.istio-system.svc:15010",
"channel_creds": [{"type": "insecure"}],
"server_features" : ["xds_v3"]
}
],
"node": {
"id": "'${NODE_ID}'",
"metadata": {
"GENERATOR": "grpc"
}
}
}' > /var/lib/grpc/data/bootstrap.json
containers:
{{- range $index, $container := .Spec.Containers }}
- name: {{ $container.Name }}
env:
- name: GRPC_XDS_BOOTSTRAP
value: /var/lib/grpc/data/bootstrap.json
- name: GRPC_GO_LOG_VERBOSITY_LEVEL
value: "99"
- name: GRPC_GO_LOG_SEVERITY_LEVEL
value: info
volumeMounts:
- mountPath: /var/lib/grpc/data/
name: grpc-io-proxyless-bootstrap
{{- end }}
volumes:
- name: grpc-io-proxyless-bootstrap
emptyDir: {}
grpc-agent: |
{{- $containers := list }}
{{- range $index, $container := .Spec.Containers }}{{ if not (eq $container.Name "istio-proxy") }}{{ $containers = append $containers $container.Name }}{{end}}{{- end}}
metadata:
annotations: {
{{- if eq (len $containers) 1 }}
kubectl.kubernetes.io/default-logs-container: "{{ index $containers 0 }}",
kubectl.kubernetes.io/default-container: "{{ index $containers 0 }}",
{{ end }}
}
spec:
containers:
{{- range $index, $container := .Spec.Containers }}
{{ if not (eq $container.Name "istio-proxy") }}
- name: {{ $container.Name }}
env:
- name: "GRPC_XDS_BOOTSTRAP"
value: "/var/lib/istio/data/grpc-bootstrap.json"
volumeMounts:
- mountPath: /var/lib/istio/data
name: istio-data
# UDS channel between istioagent and gRPC client for XDS/SDS
- mountPath: /etc/istio/proxy
name: istio-xds
{{- end }}
{{- end }}
- name: istio-proxy
{{- if contains "/" (annotation .ObjectMeta `sidecar.istio.io/proxyImage` .Values.global.proxy.image) }}
image: "{{ annotation .ObjectMeta `sidecar.istio.io/proxyImage` .Values.global.proxy.image }}"
{{- else }}
image: "{{ .Values.global.hub }}/{{ .Values.global.proxy.image }}:{{ .Values.global.tag }}"
{{- end }}
args:
- proxy
- sidecar
- --domain
- $(POD_NAMESPACE).svc.{{ .Values.global.proxy.clusterDomain }}
- --log_output_level={{ annotation .ObjectMeta `sidecar.istio.io/agentLogLevel` .Values.global.logging.level }}
{{- if .Values.global.sts.servicePort }}
- --stsPort={{ .Values.global.sts.servicePort }}
{{- end }}
{{- if .Values.global.logAsJson }}
- --log_as_json
{{- end }}
env:
- name: "GRPC_XDS_BOOTSTRAP"
value: "/var/lib/istio/data/grpc-bootstrap.json"
- name: ISTIO_META_GENERATOR
value: grpc
- name: OUTPUT_CERTS
value: /var/lib/istio/data
{{- if eq (env "PILOT_ENABLE_INBOUND_PASSTHROUGH" "true") "false" }}
- name: REWRITE_PROBE_LEGACY_LOCALHOST_DESTINATION
value: "true"
{{- end }}
- name: JWT_POLICY
value: {{ .Values.global.jwtPolicy }}
- name: PILOT_CERT_PROVIDER
value: {{ .Values.global.pilotCertProvider }}
- name: CA_ADDR
{{- if .Values.global.caAddress }}
value: {{ .Values.global.caAddress }}
{{- else }}
value: istiod{{- if not (eq .Values.revision "") }}-{{ .Values.revision }}{{- end }}.{{ .Values.global.istioNamespace }}.svc:15012
{{- end }}
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: INSTANCE_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
- name: SERVICE_ACCOUNT
valueFrom:
fieldRef:
fieldPath: spec.serviceAccountName
- name: HOST_IP
valueFrom:
fieldRef:
fieldPath: status.hostIP
- name: PROXY_CONFIG
value: |
{{ protoToJSON .ProxyConfig }}
- name: ISTIO_META_POD_PORTS
value: |-
[
{{- $first := true }}
{{- range $index1, $c := .Spec.Containers }}
{{- range $index2, $p := $c.Ports }}
{{- if (structToJSON $p) }}
{{if not $first}},{{end}}{{ structToJSON $p }}
{{- $first = false }}
{{- end }}
{{- end}}
{{- end}}
]
- name: ISTIO_META_APP_CONTAINERS
value: "{{ $containers | join "," }}"
- name: ISTIO_META_CLUSTER_ID
value: "{{ valueOrDefault .Values.global.multiCluster.clusterName `Kubernetes` }}"
- name: ISTIO_META_INTERCEPTION_MODE
value: "{{ or (index .ObjectMeta.Annotations `sidecar.istio.io/interceptionMode`) .ProxyConfig.InterceptionMode.String }}"
{{- if .Values.global.network }}
- name: ISTIO_META_NETWORK
value: "{{ .Values.global.network }}"
{{- end }}
{{- if .DeploymentMeta.Name }}
- name: ISTIO_META_WORKLOAD_NAME
value: "{{ .DeploymentMeta.Name }}"
{{ end }}
{{- if and .TypeMeta.APIVersion .DeploymentMeta.Name }}
- name: ISTIO_META_OWNER
value: kubernetes://apis/{{ .TypeMeta.APIVersion }}/namespaces/{{ valueOrDefault .DeploymentMeta.Namespace `default` }}/{{ toLower .TypeMeta.Kind}}s/{{ .DeploymentMeta.Name }}
{{- end}}
{{- if .Values.global.meshID }}
- name: ISTIO_META_MESH_ID
value: "{{ .Values.global.meshID }}"
{{- else if (valueOrDefault .MeshConfig.TrustDomain .Values.global.trustDomain) }}
- name: ISTIO_META_MESH_ID
value: "{{ (valueOrDefault .MeshConfig.TrustDomain .Values.global.trustDomain) }}"
{{- end }}
{{- with (valueOrDefault .MeshConfig.TrustDomain .Values.global.trustDomain) }}
- name: TRUST_DOMAIN
value: "{{ . }}"
{{- end }}
{{- range $key, $value := .ProxyConfig.ProxyMetadata }}
- name: {{ $key }}
value: "{{ $value }}"
{{- end }}
# grpc uses xds:/// to resolve – no need to resolve VIP
- name: ISTIO_META_DNS_CAPTURE
value: "false"
- name: DISABLE_ENVOY
value: "true"
{{with .Values.global.imagePullPolicy }}imagePullPolicy: "{{.}}"{{end}}
{{ if ne (annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort) `0` }}
readinessProbe:
httpGet:
path: /healthz/ready
port: {{ .Values.global.proxy.statusPort }}
initialDelaySeconds: {{ annotation .ObjectMeta `readiness.status.sidecar.istio.io/initialDelaySeconds` .Values.global.proxy.readinessInitialDelaySeconds }}
periodSeconds: {{ annotation .ObjectMeta `readiness.status.sidecar.istio.io/periodSeconds` .Values.global.proxy.readinessPeriodSeconds }}
timeoutSeconds: 3
failureThreshold: {{ annotation .ObjectMeta `readiness.status.sidecar.istio.io/failureThreshold` .Values.global.proxy.readinessFailureThreshold }}
{{ end -}}
resources:
{{- if or (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPU`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemory`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPULimit`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemoryLimit`) }}
{{- if or (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPU`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemory`) }}
requests:
{{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPU`) -}}
cpu: "{{ index .ObjectMeta.Annotations `sidecar.istio.io/proxyCPU` }}"
{{ end }}
{{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemory`) -}}
memory: "{{ index .ObjectMeta.Annotations `sidecar.istio.io/proxyMemory` }}"
{{ end }}
{{- end }}
{{- if or (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPULimit`) (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemoryLimit`) }}
limits:
{{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyCPULimit`) -}}
cpu: "{{ index .ObjectMeta.Annotations `sidecar.istio.io/proxyCPULimit` }}"
{{ end }}
{{ if (isset .ObjectMeta.Annotations `sidecar.istio.io/proxyMemoryLimit`) -}}
memory: "{{ index .ObjectMeta.Annotations `sidecar.istio.io/proxyMemoryLimit` }}"
{{ end }}
{{- end }}
{{- else }}
{{- if .Values.global.proxy.resources }}
{{ toYaml .Values.global.proxy.resources | indent 6 }}
{{- end }}
{{- end }}
volumeMounts:
{{- if eq .Values.global.pilotCertProvider "istiod" }}
- mountPath: /var/run/secrets/istio
name: istiod-ca-cert
{{- end }}
- mountPath: /var/lib/istio/data
name: istio-data
# UDS channel between istioagent and gRPC client for XDS/SDS
- mountPath: /etc/istio/proxy
name: istio-xds
{{- if eq .Values.global.jwtPolicy "third-party-jwt" }}
- mountPath: /var/run/secrets/tokens
name: istio-token
{{- end }}
- name: istio-podinfo
mountPath: /etc/istio/pod
{{- if isset .ObjectMeta.Annotations `sidecar.istio.io/userVolumeMount` }}
{{ range $index, $value := fromJSON (index .ObjectMeta.Annotations `sidecar.istio.io/userVolumeMount`) }}
- name: "{{ $index }}"
{{ toYaml $value | indent 6 }}
{{ end }}
{{- end }}
volumes:
# UDS channel between istioagent and gRPC client for XDS/SDS
- emptyDir:
medium: Memory
name: istio-xds
- name: istio-data
emptyDir: {}
- name: istio-podinfo
downwardAPI:
items:
- path: "labels"
fieldRef:
fieldPath: metadata.labels
- path: "annotations"
fieldRef:
fieldPath: metadata.annotations
{{- if eq .Values.global.jwtPolicy "third-party-jwt" }}
- name: istio-token
projected:
sources:
- serviceAccountToken:
path: istio-token
expirationSeconds: 43200
audience: {{ .Values.global.sds.token.aud }}
{{- end }}
{{- if eq .Values.global.pilotCertProvider "istiod" }}
- name: istiod-ca-cert
configMap:
name: istio-ca-root-cert
{{- end }}
{{- if isset .ObjectMeta.Annotations `sidecar.istio.io/userVolume` }}
{{range $index, $value := fromJSON (index .ObjectMeta.Annotations `sidecar.istio.io/userVolume`) }}
- name: "{{ $index }}"
{{ toYaml $value | indent 4 }}
{{ end }}
{{ end }}
---
# Source: istio-discovery/templates/clusterrole.yaml
apiVersion: rbac.authorization.k8s.io/v1
Expand Down
Loading

0 comments on commit fe710ef

Please sign in to comment.