diff --git a/manifests/charts/istio-control/istio-discovery/files/injection-template.yaml b/manifests/charts/istio-control/istio-discovery/files/injection-template.yaml index 6755cc27bebb..39616139f7f7 100644 --- a/manifests/charts/istio-control/istio-discovery/files/injection-template.yaml +++ b/manifests/charts/istio-control/istio-discovery/files/injection-template.yaml @@ -65,7 +65,9 @@ metadata: {{- end }} } spec: - {{- $holdProxy := or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts }} + {{- $holdProxy := and + (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts) + (not $nativeSidecar) }} initContainers: {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} {{ if .Values.istio_cni.enabled -}} @@ -223,6 +225,17 @@ spec: command: - pilot-agent - wait + {{- else if $nativeSidecar }} + {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}} + lifecycle: + preStop: + exec: + command: + - pilot-agent + - request + - --debug-port={{(annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort)}} + - POST + - drain {{- end }} env: {{- if eq (env "PILOT_ENABLE_INBOUND_PASSTHROUGH" "true") "false" }} diff --git a/manifests/charts/istiod-remote/files/injection-template.yaml b/manifests/charts/istiod-remote/files/injection-template.yaml index 6755cc27bebb..39616139f7f7 100644 --- a/manifests/charts/istiod-remote/files/injection-template.yaml +++ b/manifests/charts/istiod-remote/files/injection-template.yaml @@ -65,7 +65,9 @@ metadata: {{- end }} } spec: - {{- $holdProxy := or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts }} + {{- $holdProxy := and + (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts) + (not $nativeSidecar) }} initContainers: {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} {{ if .Values.istio_cni.enabled -}} @@ -223,6 +225,17 @@ spec: command: - pilot-agent - wait + {{- else if $nativeSidecar }} + {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}} + lifecycle: + preStop: + exec: + command: + - pilot-agent + - request + - --debug-port={{(annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort)}} + - POST + - drain {{- end }} env: {{- if eq (env "PILOT_ENABLE_INBOUND_PASSTHROUGH" "true") "false" }} diff --git a/pilot/cmd/pilot-agent/app/cmd.go b/pilot/cmd/pilot-agent/app/cmd.go index 287daa2d6d2d..020997f30864 100644 --- a/pilot/cmd/pilot-agent/app/cmd.go +++ b/pilot/cmd/pilot-agent/app/cmd.go @@ -163,7 +163,7 @@ func newProxyCommand() *cobra.Command { // If a status port was provided, start handling status probes. if proxyConfig.StatusPort > 0 { if err := initStatusServer(ctx, proxy, proxyConfig, - agentOptions.EnvoyPrometheusPort, proxyArgs.EnableProfiling, agent); err != nil { + agentOptions.EnvoyPrometheusPort, proxyArgs.EnableProfiling, agent, cancel); err != nil { return err } } @@ -214,13 +214,20 @@ func addFlags(proxyCmd *cobra.Command) { "Enable profiling via web interface host:port/debug/pprof/.") } -func initStatusServer(ctx context.Context, proxy *model.Proxy, proxyConfig *meshconfig.ProxyConfig, - envoyPrometheusPort int, enableProfiling bool, agent *istio_agent.Agent, +func initStatusServer( + ctx context.Context, + proxy *model.Proxy, + proxyConfig *meshconfig.ProxyConfig, + envoyPrometheusPort int, + enableProfiling bool, + agent *istio_agent.Agent, + shutdown context.CancelFunc, ) error { o := options.NewStatusServerOptions(proxy, proxyConfig, agent) o.EnvoyPrometheusPort = envoyPrometheusPort o.EnableProfiling = enableProfiling o.Context = ctx + o.Shutdown = shutdown statusServer, err := status.NewServer(*o) if err != nil { return err diff --git a/pilot/cmd/pilot-agent/options/statusserver.go b/pilot/cmd/pilot-agent/options/statusserver.go index c3d917ff1f4d..24ef005a46b7 100644 --- a/pilot/cmd/pilot-agent/options/statusserver.go +++ b/pilot/cmd/pilot-agent/options/statusserver.go @@ -34,5 +34,8 @@ func NewStatusServerOptions(proxy *model.Proxy, proxyConfig *meshconfig.ProxyCon NoEnvoy: agent.EnvoyDisabled(), FetchDNS: agent.GetDNSTable, GRPCBootstrap: agent.GRPCBootstrapPath(), + TriggerDrain: func() { + agent.DrainNow() + }, } } diff --git a/pilot/cmd/pilot-agent/status/server.go b/pilot/cmd/pilot-agent/status/server.go index 8e5a328c91d0..2b2edcd81f1e 100644 --- a/pilot/cmd/pilot-agent/status/server.go +++ b/pilot/cmd/pilot-agent/status/server.go @@ -54,6 +54,7 @@ import ( "istio.io/istio/pkg/kube/apimirror" "istio.io/istio/pkg/log" "istio.io/istio/pkg/monitoring" + "istio.io/istio/pkg/network" "istio.io/istio/pkg/slices" ) @@ -61,7 +62,8 @@ const ( // readyPath is for the pilot agent readiness itself. readyPath = "/healthz/ready" // quitPath is to notify the pilot agent to quit. - quitPath = "/quitquitquit" + quitPath = "/quitquitquit" + drainPath = "/drain" // KubeAppProberEnvName is the name of the command line flag for pilot agent to pass app prober config. // The json encoded string to pass app HTTP probe information from injector(istioctl or webhook). // For example, ISTIO_KUBE_APP_PROBERS='{"/app-health/httpbin/livez":{"httpGet":{"path": "/hello", "port": 8080}}. @@ -128,6 +130,8 @@ type Options struct { EnableProfiling bool // PrometheusRegistry to use. Just for testing. PrometheusRegistry prometheus.Gatherer + Shutdown context.CancelFunc + TriggerDrain func() } // Server provides an endpoint for handling status probes. @@ -147,6 +151,8 @@ type Server struct { http *http.Client enableProfiling bool registry prometheus.Gatherer + shutdown context.CancelFunc + drain func() } func initializeMonitoring() (prometheus.Gatherer, error) { @@ -212,6 +218,10 @@ func NewServer(config Options) (*Server, error) { config: config, enableProfiling: config.EnableProfiling, registry: registry, + shutdown: func() { + config.Shutdown() + }, + drain: config.TriggerDrain, } if LegacyLocalhostProbeDestination.Get() { s.appProbersDestination = "localhost" @@ -358,6 +368,7 @@ func (s *Server) Run(ctx context.Context) { // Keep for backward compat with configs. mux.HandleFunc(`/stats/prometheus`, s.handleStats) mux.HandleFunc(quitPath, s.handleQuit) + mux.HandleFunc(drainPath, s.handleDrain) mux.HandleFunc("/app-health/", s.handleAppProbe) if s.enableProfiling { @@ -387,7 +398,9 @@ func (s *Server) Run(ctx context.Context) { go func() { if err := http.Serve(l, mux); err != nil { - log.Error(err) + if network.IsUnexpectedListenerError(err) { + log.Error(err) + } select { case <-ctx.Done(): // We are shutting down already, don't trigger SIGTERM @@ -671,7 +684,22 @@ func (s *Server) handleQuit(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("OK")) log.Infof("handling %s, notifying pilot-agent to exit", quitPath) - notifyExit() + s.shutdown() +} + +func (s *Server) handleDrain(w http.ResponseWriter, r *http.Request) { + if !isRequestFromLocalhost(r) { + http.Error(w, "Only requests from localhost are allowed", http.StatusForbidden) + return + } + if r.Method != http.MethodPost { + http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("OK")) + log.Infof("handling %s, starting drain", drainPath) + s.drain() } func (s *Server) handleAppProbe(w http.ResponseWriter, req *http.Request) { diff --git a/pilot/cmd/pilot-agent/status/server_test.go b/pilot/cmd/pilot-agent/status/server_test.go index 52283deccef6..d0302c801acd 100644 --- a/pilot/cmd/pilot-agent/status/server_test.go +++ b/pilot/cmd/pilot-agent/status/server_test.go @@ -24,12 +24,9 @@ import ( "net" "net/http" "net/http/httptest" - "os" - "os/signal" "reflect" "strconv" "strings" - "syscall" "testing" "time" @@ -204,26 +201,39 @@ func TestNewServer(t *testing.T) { } } -func TestPprof(t *testing.T) { - pprofPath := "/debug/pprof/cmdline" - // Starts the pilot agent status server. - server, err := NewServer(Options{StatusPort: 0, EnableProfiling: true, PrometheusRegistry: TestingRegistry(t)}) +func NewTestServer(t test.Failer, o Options) *Server { + if o.PrometheusRegistry == nil { + o.PrometheusRegistry = TestingRegistry(t) + } + server, err := NewServer(o) if err != nil { t.Fatalf("failed to create status server %v", err) } ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + t.Cleanup(cancel) go server.Run(ctx) - var statusPort uint16 - for statusPort == 0 { + if err := retry.UntilSuccess(func() error { server.mutex.RLock() - statusPort = server.statusPort + statusPort := server.statusPort server.mutex.RUnlock() + if statusPort == 0 { + return fmt.Errorf("no port allocated") + } + return nil + }, retry.Delay(time.Microsecond)); err != nil { + t.Fatalf("failed to getport: %v", err) } + return server +} + +func TestPprof(t *testing.T) { + pprofPath := "/debug/pprof/cmdline" + // Starts the pilot agent status server. + server := NewTestServer(t, Options{EnableProfiling: true}) client := http.Client{} - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://localhost:%v/%s", statusPort, pprofPath), nil) + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://localhost:%v/%s", server.statusPort, pprofPath), nil) if err != nil { t.Fatalf("[%v] failed to create request", pprofPath) } @@ -871,34 +881,18 @@ func TestAppProbe(t *testing.T) { t.Fatalf("invalid app probers") } config := Options{ - StatusPort: 0, - PrometheusRegistry: TestingRegistry(t), - KubeAppProbers: string(appProber), - PodIP: tc.podIP, - IPv6: tc.ipv6, + KubeAppProbers: string(appProber), + PodIP: tc.podIP, + IPv6: tc.ipv6, } + server := NewTestServer(t, config) // Starts the pilot agent status server. - server, err := NewServer(config) - if err != nil { - t.Fatalf("failed to create status server %v", err) - } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - go server.Run(ctx) - if tc.ipv6 { server.upstreamLocalAddress = &net.TCPAddr{IP: net.ParseIP("::1")} // required because ::6 is NOT a loopback address (IPv6 only has ::1) } - var statusPort uint16 - for statusPort == 0 { - server.mutex.RLock() - statusPort = server.statusPort - server.mutex.RUnlock() - } - client := http.Client{} - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://localhost:%v/%s", statusPort, tc.probePath), nil) + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://localhost:%v/%s", server.statusPort, tc.probePath), nil) if err != nil { t.Fatalf("[%v] failed to create request", tc.probePath) } @@ -979,31 +973,11 @@ func TestHttpsAppProbe(t *testing.T) { appPort := listener.Addr().(*net.TCPAddr).Port // Starts the pilot agent status server. - server, err := NewServer(Options{ - StatusPort: 0, - PrometheusRegistry: TestingRegistry(t), + server := NewTestServer(t, Options{ KubeAppProbers: fmt.Sprintf(`{"/app-health/hello-world/readyz": {"httpGet": {"path": "/hello/sunnyvale", "port": %v, "scheme": "HTTPS"}}, "/app-health/hello-world/livez": {"httpGet": {"port": %v, "scheme": "HTTPS"}}}`, appPort, appPort), }) - if err != nil { - t.Fatalf("failed to create status server %v", err) - } - go server.Run(context.Background()) - - var statusPort uint16 - if err := retry.UntilSuccess(func() error { - server.mutex.RLock() - statusPort = server.statusPort - server.mutex.RUnlock() - if statusPort == 0 { - return fmt.Errorf("no port allocated") - } - return nil - }); err != nil { - t.Fatalf("failed to getport: %v", err) - } - t.Logf("status server starts at port %v, app starts at port %v", statusPort, appPort) - return statusPort, h.lastAlpn.Load + return server.statusPort, h.lastAlpn.Load } testCases := []struct { name string @@ -1105,9 +1079,7 @@ func TestGRPCAppProbe(t *testing.T) { appPort := listener.Addr().(*net.TCPAddr).Port // Starts the pilot agent status server. - server, err := NewServer(Options{ - StatusPort: 0, - PrometheusRegistry: TestingRegistry(t), + server := NewTestServer(t, Options{ KubeAppProbers: fmt.Sprintf(` { "/app-health/foo/livez": { @@ -1140,24 +1112,7 @@ func TestGRPCAppProbe(t *testing.T) { } }`, appPort, appPort, appPort, appPort), }) - if err != nil { - t.Errorf("failed to create status server %v", err) - return - } - go server.Run(context.Background()) - - var statusPort uint16 - if err := retry.UntilSuccess(func() error { - server.mutex.RLock() - statusPort = server.statusPort - server.mutex.RUnlock() - if statusPort == 0 { - return fmt.Errorf("no port allocated") - } - return nil - }); err != nil { - t.Fatalf("failed to getport: %v", err) - } + statusPort := server.statusPort t.Logf("status server starts at port %v, app starts at port %v", statusPort, appPort) testCases := []struct { @@ -1227,11 +1182,9 @@ func TestGRPCAppProbeWithIPV6(t *testing.T) { appPort := listener.Addr().(*net.TCPAddr).Port // Starts the pilot agent status server. - server, err := NewServer(Options{ - StatusPort: 0, - IPv6: true, - PodIP: "::1", - PrometheusRegistry: TestingRegistry(t), + server := NewTestServer(t, Options{ + IPv6: true, + PodIP: "::1", KubeAppProbers: fmt.Sprintf(` { "/app-health/foo/livez": { @@ -1264,27 +1217,8 @@ func TestGRPCAppProbeWithIPV6(t *testing.T) { } }`, appPort, appPort, appPort, appPort), }) - if err != nil { - t.Errorf("failed to create status server %v", err) - return - } server.upstreamLocalAddress = &net.TCPAddr{IP: net.ParseIP("::1")} // required because ::6 is NOT a loopback address (IPv6 only has ::1) - go server.Run(context.Background()) - - var statusPort uint16 - if err := retry.UntilSuccess(func() error { - server.mutex.RLock() - statusPort = server.statusPort - server.mutex.RUnlock() - if statusPort == 0 { - return fmt.Errorf("no port allocated") - } - return nil - }); err != nil { - t.Fatalf("failed to getport: %v", err) - } - t.Logf("status server starts at port %v, app starts at port %v", statusPort, appPort) testCases := []struct { name string @@ -1293,27 +1227,27 @@ func TestGRPCAppProbeWithIPV6(t *testing.T) { }{ { name: "bad-path-should-be-disallowed", - probePath: fmt.Sprintf(":%v/bad-path-should-be-disallowed", statusPort), + probePath: fmt.Sprintf(":%v/bad-path-should-be-disallowed", server.statusPort), statusCode: http.StatusNotFound, }, { name: "foo-livez", - probePath: fmt.Sprintf(":%v/app-health/foo/livez", statusPort), + probePath: fmt.Sprintf(":%v/app-health/foo/livez", server.statusPort), statusCode: http.StatusOK, }, { name: "foo-readyz", - probePath: fmt.Sprintf(":%v/app-health/foo/readyz", statusPort), + probePath: fmt.Sprintf(":%v/app-health/foo/readyz", server.statusPort), statusCode: http.StatusInternalServerError, }, { name: "bar-livez", - probePath: fmt.Sprintf(":%v/app-health/bar/livez", statusPort), + probePath: fmt.Sprintf(":%v/app-health/bar/livez", server.statusPort), statusCode: http.StatusOK, }, { name: "bar-readyz", - probePath: fmt.Sprintf(":%v/app-health/bar/readyz", statusPort), + probePath: fmt.Sprintf(":%v/app-health/bar/readyz", server.statusPort), statusCode: http.StatusInternalServerError, }, } @@ -1450,28 +1384,12 @@ func TestProbeHeader(t *testing.T) { t.Fatalf("invalid app probers") } config := Options{ - StatusPort: 0, - PrometheusRegistry: TestingRegistry(t), - KubeAppProbers: string(appProber), + KubeAppProbers: string(appProber), } // Starts the pilot agent status server. - server, err := NewServer(config) - if err != nil { - t.Fatal("failed to create status server: ", err) - } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - go server.Run(ctx) - - var statusPort uint16 - for statusPort == 0 { - server.mutex.RLock() - statusPort = server.statusPort - server.mutex.RUnlock() - } - + server := NewTestServer(t, config) client := http.Client{} - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://localhost:%v%s", statusPort, probePath), nil) + req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://localhost:%v%s", server.statusPort, probePath), nil) if err != nil { t.Fatal("failed to create request: ", err) } @@ -1489,12 +1407,6 @@ func TestProbeHeader(t *testing.T) { } func TestHandleQuit(t *testing.T) { - statusPort := 15020 - s, err := NewServer(Options{StatusPort: uint16(statusPort), PrometheusRegistry: TestingRegistry(t)}) - if err != nil { - t.Fatal(err) - } - tests := []struct { name string method string @@ -1528,18 +1440,19 @@ func TestHandleQuit(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - // Need to stop SIGTERM from killing the whole test run - termChannel := make(chan os.Signal, 1) - signal.Notify(termChannel, syscall.SIGTERM) - defer signal.Reset(syscall.SIGTERM) - + shutdown := make(chan struct{}) + s := NewTestServer(t, Options{ + Shutdown: func() { + close(shutdown) + }, + }) req, err := http.NewRequest(tt.method, "/quitquitquit", nil) if err != nil { t.Fatal(err) } if tt.remoteAddr != "" { - req.RemoteAddr = tt.remoteAddr + ":15020" + req.RemoteAddr = tt.remoteAddr + ":" + fmt.Sprint(s.statusPort) } resp := httptest.NewRecorder() @@ -1550,12 +1463,16 @@ func TestHandleQuit(t *testing.T) { if tt.expected == http.StatusOK { select { - case <-termChannel: + case <-shutdown: case <-time.After(time.Second): - t.Fatalf("Failed to receive expected SIGTERM") + t.Fatalf("Failed to receive expected shutdown") + } + } else { + select { + case <-shutdown: + t.Fatalf("unexpected shutdown") + default: } - } else if len(termChannel) != 0 { - t.Fatalf("A SIGTERM was sent when it should not have been") } }) } @@ -1589,9 +1506,8 @@ func TestAdditionalProbes(t *testing.T) { defer testServer.Close() for _, tc := range testCases { server, err := NewServer(Options{ - Probes: tc.probes, - PrometheusRegistry: TestingRegistry(t), - AdminPort: uint16(testServer.Listener.Addr().(*net.TCPAddr).Port), + Probes: tc.probes, + AdminPort: uint16(testServer.Listener.Addr().(*net.TCPAddr).Port), }) if err != nil { t.Errorf("failed to construct server") diff --git a/pkg/envoy/admin.go b/pkg/envoy/admin.go index 20213eb4e957..7790f56370a3 100644 --- a/pkg/envoy/admin.go +++ b/pkg/envoy/admin.go @@ -26,13 +26,16 @@ import ( // DrainListeners drains inbound listeners of Envoy so that inflight requests // can gracefully finish and even continue making outbound calls as needed. -func DrainListeners(adminPort uint32, inboundonly bool) error { +func DrainListeners(adminPort uint32, inboundonly bool, skipExit bool) error { var drainURL string if inboundonly { drainURL = "drain_listeners?inboundonly&graceful" } else { drainURL = "drain_listeners?graceful" } + if skipExit { + drainURL += "&skip_exit" + } res, err := doEnvoyPost(drainURL, "", "", adminPort) log.Debugf("Drain listener endpoint response : %s", res.String()) return err diff --git a/pkg/envoy/agent.go b/pkg/envoy/agent.go index ec71bd28a2b7..9d9e77a861f7 100644 --- a/pkg/envoy/agent.go +++ b/pkg/envoy/agent.go @@ -23,6 +23,8 @@ import ( "strings" "time" + "go.uber.org/atomic" + "istio.io/istio/pkg/http" "istio.io/istio/pkg/log" "istio.io/istio/pkg/util/sets" @@ -54,6 +56,7 @@ func NewAgent(proxy Proxy, terminationDrainDuration, minDrainDuration time.Durat adminPort: adminPort, localhost: localhost, knownIstioListeners: knownIstioListeners, + skipDrain: atomic.NewBool(false), } } @@ -63,7 +66,7 @@ type Proxy interface { Run(<-chan error) error // Drains the envoy process. - Drain() error + Drain(skipExit bool) error // Cleanup command for cleans up the proxy. Cleanup() @@ -91,6 +94,8 @@ type Agent struct { knownIstioListeners sets.String exitOnZeroActiveConnections bool + + skipDrain *atomic.Bool } type exitStatus struct { @@ -98,6 +103,15 @@ type exitStatus struct { } // Run starts the envoy and waits until it terminates. +// There are a few exit paths: +// 1. Envoy exits. In this case, we simply log and exit. +// 2. /quitquitquit (on agent, not Envoy) is called. We will set skipDrain and cancel the context, which triggers us to exit immediately. +// 3. SIGTERM. We will drain, wait termination drain duration, then exit. This is the standard pod shutdown; SIGTERM arrives when pod shutdown starts. +// If the pod's terminationGracePeriod is shorter than our drain duration (rare), we may be a SIGKILL. +// 4. /drain + SIGTERM. This is the shutdown when using Kubernetes native sidecars. +// /drain is called when the pod shutdown starts. We start draining, forever. +// Once the app containers shutdown, we get a SIGTERM. We have no use to run anymore, so shutdown immediately. +// If somehow we do not shutdown from the SIGTERM fast enough, we may get a SIGKILL later. func (a *Agent) Run(ctx context.Context) { log.Info("Starting proxy agent") go a.runWait(a.abortCh) @@ -125,11 +139,32 @@ func (a *Agent) Run(ctx context.Context) { } } +func (a *Agent) DisableDraining() { + a.skipDrain.Store(true) +} + +func (a *Agent) DrainNow() { + log.Infof("Agent draining proxy") + err := a.proxy.Drain(true) + if err != nil { + log.Warnf("Error in invoking drain listeners endpoint: %v", err) + } + // If we drained now, skip draining + waiting later + // When we terminate, we will instead exit immediately + a.DisableDraining() +} + +// terminate starts exiting the process. func (a *Agent) terminate() { - log.Infof("Agent draining Proxy") - e := a.proxy.Drain() + log.Infof("Agent draining Proxy for termination") + if a.skipDrain.Load() { + log.Infof("Agent already drained, exiting immediately") + a.abortCh <- errAbort + return + } + e := a.proxy.Drain(false) if e != nil { - log.Warnf("Error in invoking drain listeners endpoint %v", e) + log.Warnf("Error in invoking drain listeners endpoint: %v", e) } // If exitOnZeroActiveConnections is enabled, always sleep minimumDrainDuration then exit // after min(all connections close, terminationGracePeriodSeconds-minimumDrainDuration). diff --git a/pkg/envoy/agent_test.go b/pkg/envoy/agent_test.go index ae88d66079db..b1be06480c60 100644 --- a/pkg/envoy/agent_test.go +++ b/pkg/envoy/agent_test.go @@ -70,7 +70,7 @@ func (tp TestProxy) Run(stop <-chan error) error { return tp.run(stop) } -func (tp TestProxy) Drain() error { +func (tp TestProxy) Drain(bool) error { tp.blockChannel <- "unblock" return nil } diff --git a/pkg/envoy/proxy.go b/pkg/envoy/proxy.go index e838708242a9..1cf9b4a6aad1 100644 --- a/pkg/envoy/proxy.go +++ b/pkg/envoy/proxy.go @@ -103,10 +103,10 @@ func splitComponentLog(level string) (string, []string) { return logLevel, componentLogs } -func (e *envoy) Drain() error { +func (e *envoy) Drain(skipExit bool) error { adminPort := uint32(e.AdminPort) - err := DrainListeners(adminPort, e.Sidecar) + err := DrainListeners(adminPort, e.Sidecar, skipExit) if err != nil { log.Infof("failed draining listeners for Envoy on port %d: %v", adminPort, err) } diff --git a/pkg/istio-agent/agent.go b/pkg/istio-agent/agent.go index f2c8a1601003..d5aa20de267c 100644 --- a/pkg/istio-agent/agent.go +++ b/pkg/istio-agent/agent.go @@ -79,6 +79,13 @@ const ( var _ ready.Prober = &Agent{} +type LifecycleEvent string + +const ( + DrainLifecycleEvent LifecycleEvent = "drain" + ExitLifecycleEvent LifecycleEvent = "exit" +) + // Agent contains the configuration of the agent, based on the injected // environment: // - SDS hostPath if node-agent was used @@ -843,3 +850,7 @@ func (a *Agent) newSecretManager() (*cache.SecretManagerClient, error) { func (a *Agent) GRPCBootstrapPath() string { return a.cfg.GRPCBootstrapPath } + +func (a *Agent) DrainNow() { + a.envoyAgent.DrainNow() +} diff --git a/pkg/kube/inject/testdata/inject/native-sidecar.yaml.injected b/pkg/kube/inject/testdata/inject/native-sidecar.yaml.injected index be37b4e4e38e..f437be303143 100644 --- a/pkg/kube/inject/testdata/inject/native-sidecar.yaml.injected +++ b/pkg/kube/inject/testdata/inject/native-sidecar.yaml.injected @@ -144,6 +144,15 @@ spec: - name: ISTIO_KUBE_APP_PROBERS value: '{"/app-health/other-sidecar/readyz":{"httpGet":{"port":3333}}}' image: gcr.io/istio-testing/proxyv2:latest + lifecycle: + preStop: + exec: + command: + - pilot-agent + - request + - --debug-port=15020 + - POST + - drain name: istio-proxy ports: - containerPort: 15090 diff --git a/pkg/kube/inject/testdata/inject/proxy-override-args-native.yaml.injected b/pkg/kube/inject/testdata/inject/proxy-override-args-native.yaml.injected index 399b57cd31a5..fa27eb29f2e4 100644 --- a/pkg/kube/inject/testdata/inject/proxy-override-args-native.yaml.injected +++ b/pkg/kube/inject/testdata/inject/proxy-override-args-native.yaml.injected @@ -145,6 +145,15 @@ spec: - name: TRUST_DOMAIN value: cluster.local image: gcr.io/istio-testing/proxyv2:latest + lifecycle: + preStop: + exec: + command: + - pilot-agent + - request + - --debug-port=15020 + - POST + - drain name: istio-proxy ports: - containerPort: 15090 diff --git a/pkg/kube/inject/testdata/inputs/custom-template.yaml.37.template.gen.yaml b/pkg/kube/inject/testdata/inputs/custom-template.yaml.37.template.gen.yaml index 665e7113816a..2e7d2b6de9df 100644 --- a/pkg/kube/inject/testdata/inputs/custom-template.yaml.37.template.gen.yaml +++ b/pkg/kube/inject/testdata/inputs/custom-template.yaml.37.template.gen.yaml @@ -76,7 +76,9 @@ templates: {{- end }} } spec: - {{- $holdProxy := or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts }} + {{- $holdProxy := and + (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts) + (not $nativeSidecar) }} initContainers: {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} {{ if .Values.istio_cni.enabled -}} @@ -234,6 +236,17 @@ templates: command: - pilot-agent - wait + {{- else if $nativeSidecar }} + {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}} + lifecycle: + preStop: + exec: + command: + - pilot-agent + - request + - --debug-port={{(annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort)}} + - POST + - drain {{- end }} env: {{- if eq (env "PILOT_ENABLE_INBOUND_PASSTHROUGH" "true") "false" }} diff --git a/pkg/kube/inject/testdata/inputs/default.template.gen.yaml b/pkg/kube/inject/testdata/inputs/default.template.gen.yaml index 334e1630b6e4..81dba4ab1078 100644 --- a/pkg/kube/inject/testdata/inputs/default.template.gen.yaml +++ b/pkg/kube/inject/testdata/inputs/default.template.gen.yaml @@ -76,7 +76,9 @@ templates: {{- end }} } spec: - {{- $holdProxy := or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts }} + {{- $holdProxy := and + (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts) + (not $nativeSidecar) }} initContainers: {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} {{ if .Values.istio_cni.enabled -}} @@ -234,6 +236,17 @@ templates: command: - pilot-agent - wait + {{- else if $nativeSidecar }} + {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}} + lifecycle: + preStop: + exec: + command: + - pilot-agent + - request + - --debug-port={{(annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort)}} + - POST + - drain {{- end }} env: {{- if eq (env "PILOT_ENABLE_INBOUND_PASSTHROUGH" "true") "false" }} diff --git a/pkg/kube/inject/testdata/inputs/enable-core-dump.yaml.5.template.gen.yaml b/pkg/kube/inject/testdata/inputs/enable-core-dump.yaml.5.template.gen.yaml index 334e1630b6e4..81dba4ab1078 100644 --- a/pkg/kube/inject/testdata/inputs/enable-core-dump.yaml.5.template.gen.yaml +++ b/pkg/kube/inject/testdata/inputs/enable-core-dump.yaml.5.template.gen.yaml @@ -76,7 +76,9 @@ templates: {{- end }} } spec: - {{- $holdProxy := or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts }} + {{- $holdProxy := and + (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts) + (not $nativeSidecar) }} initContainers: {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} {{ if .Values.istio_cni.enabled -}} @@ -234,6 +236,17 @@ templates: command: - pilot-agent - wait + {{- else if $nativeSidecar }} + {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}} + lifecycle: + preStop: + exec: + command: + - pilot-agent + - request + - --debug-port={{(annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort)}} + - POST + - drain {{- end }} env: {{- if eq (env "PILOT_ENABLE_INBOUND_PASSTHROUGH" "true") "false" }} diff --git a/pkg/kube/inject/testdata/inputs/hello-existing-cncf-networks-json.yaml.16.template.gen.yaml b/pkg/kube/inject/testdata/inputs/hello-existing-cncf-networks-json.yaml.16.template.gen.yaml index 334e1630b6e4..81dba4ab1078 100644 --- a/pkg/kube/inject/testdata/inputs/hello-existing-cncf-networks-json.yaml.16.template.gen.yaml +++ b/pkg/kube/inject/testdata/inputs/hello-existing-cncf-networks-json.yaml.16.template.gen.yaml @@ -76,7 +76,9 @@ templates: {{- end }} } spec: - {{- $holdProxy := or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts }} + {{- $holdProxy := and + (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts) + (not $nativeSidecar) }} initContainers: {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} {{ if .Values.istio_cni.enabled -}} @@ -234,6 +236,17 @@ templates: command: - pilot-agent - wait + {{- else if $nativeSidecar }} + {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}} + lifecycle: + preStop: + exec: + command: + - pilot-agent + - request + - --debug-port={{(annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort)}} + - POST + - drain {{- end }} env: {{- if eq (env "PILOT_ENABLE_INBOUND_PASSTHROUGH" "true") "false" }} diff --git a/pkg/kube/inject/testdata/inputs/hello-existing-cncf-networks.yaml.15.template.gen.yaml b/pkg/kube/inject/testdata/inputs/hello-existing-cncf-networks.yaml.15.template.gen.yaml index 334e1630b6e4..81dba4ab1078 100644 --- a/pkg/kube/inject/testdata/inputs/hello-existing-cncf-networks.yaml.15.template.gen.yaml +++ b/pkg/kube/inject/testdata/inputs/hello-existing-cncf-networks.yaml.15.template.gen.yaml @@ -76,7 +76,9 @@ templates: {{- end }} } spec: - {{- $holdProxy := or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts }} + {{- $holdProxy := and + (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts) + (not $nativeSidecar) }} initContainers: {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} {{ if .Values.istio_cni.enabled -}} @@ -234,6 +236,17 @@ templates: command: - pilot-agent - wait + {{- else if $nativeSidecar }} + {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}} + lifecycle: + preStop: + exec: + command: + - pilot-agent + - request + - --debug-port={{(annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort)}} + - POST + - drain {{- end }} env: {{- if eq (env "PILOT_ENABLE_INBOUND_PASSTHROUGH" "true") "false" }} diff --git a/pkg/kube/inject/testdata/inputs/hello-image-pull-secret.yaml.11.template.gen.yaml b/pkg/kube/inject/testdata/inputs/hello-image-pull-secret.yaml.11.template.gen.yaml index 334e1630b6e4..81dba4ab1078 100644 --- a/pkg/kube/inject/testdata/inputs/hello-image-pull-secret.yaml.11.template.gen.yaml +++ b/pkg/kube/inject/testdata/inputs/hello-image-pull-secret.yaml.11.template.gen.yaml @@ -76,7 +76,9 @@ templates: {{- end }} } spec: - {{- $holdProxy := or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts }} + {{- $holdProxy := and + (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts) + (not $nativeSidecar) }} initContainers: {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} {{ if .Values.istio_cni.enabled -}} @@ -234,6 +236,17 @@ templates: command: - pilot-agent - wait + {{- else if $nativeSidecar }} + {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}} + lifecycle: + preStop: + exec: + command: + - pilot-agent + - request + - --debug-port={{(annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort)}} + - POST + - drain {{- end }} env: {{- if eq (env "PILOT_ENABLE_INBOUND_PASSTHROUGH" "true") "false" }} diff --git a/pkg/kube/inject/testdata/inputs/hello-probes-noProxyHoldApplication-ProxyConfig.yaml.20.template.gen.yaml b/pkg/kube/inject/testdata/inputs/hello-probes-noProxyHoldApplication-ProxyConfig.yaml.20.template.gen.yaml index 334e1630b6e4..81dba4ab1078 100644 --- a/pkg/kube/inject/testdata/inputs/hello-probes-noProxyHoldApplication-ProxyConfig.yaml.20.template.gen.yaml +++ b/pkg/kube/inject/testdata/inputs/hello-probes-noProxyHoldApplication-ProxyConfig.yaml.20.template.gen.yaml @@ -76,7 +76,9 @@ templates: {{- end }} } spec: - {{- $holdProxy := or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts }} + {{- $holdProxy := and + (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts) + (not $nativeSidecar) }} initContainers: {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} {{ if .Values.istio_cni.enabled -}} @@ -234,6 +236,17 @@ templates: command: - pilot-agent - wait + {{- else if $nativeSidecar }} + {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}} + lifecycle: + preStop: + exec: + command: + - pilot-agent + - request + - --debug-port={{(annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort)}} + - POST + - drain {{- end }} env: {{- if eq (env "PILOT_ENABLE_INBOUND_PASSTHROUGH" "true") "false" }} diff --git a/pkg/kube/inject/testdata/inputs/hello-probes.yaml.18.template.gen.yaml b/pkg/kube/inject/testdata/inputs/hello-probes.yaml.18.template.gen.yaml index 334e1630b6e4..81dba4ab1078 100644 --- a/pkg/kube/inject/testdata/inputs/hello-probes.yaml.18.template.gen.yaml +++ b/pkg/kube/inject/testdata/inputs/hello-probes.yaml.18.template.gen.yaml @@ -76,7 +76,9 @@ templates: {{- end }} } spec: - {{- $holdProxy := or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts }} + {{- $holdProxy := and + (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts) + (not $nativeSidecar) }} initContainers: {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} {{ if .Values.istio_cni.enabled -}} @@ -234,6 +236,17 @@ templates: command: - pilot-agent - wait + {{- else if $nativeSidecar }} + {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}} + lifecycle: + preStop: + exec: + command: + - pilot-agent + - request + - --debug-port={{(annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort)}} + - POST + - drain {{- end }} env: {{- if eq (env "PILOT_ENABLE_INBOUND_PASSTHROUGH" "true") "false" }} diff --git a/pkg/kube/inject/testdata/inputs/hello.yaml.0.template.gen.yaml b/pkg/kube/inject/testdata/inputs/hello.yaml.0.template.gen.yaml index 334e1630b6e4..81dba4ab1078 100644 --- a/pkg/kube/inject/testdata/inputs/hello.yaml.0.template.gen.yaml +++ b/pkg/kube/inject/testdata/inputs/hello.yaml.0.template.gen.yaml @@ -76,7 +76,9 @@ templates: {{- end }} } spec: - {{- $holdProxy := or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts }} + {{- $holdProxy := and + (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts) + (not $nativeSidecar) }} initContainers: {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} {{ if .Values.istio_cni.enabled -}} @@ -234,6 +236,17 @@ templates: command: - pilot-agent - wait + {{- else if $nativeSidecar }} + {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}} + lifecycle: + preStop: + exec: + command: + - pilot-agent + - request + - --debug-port={{(annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort)}} + - POST + - drain {{- end }} env: {{- if eq (env "PILOT_ENABLE_INBOUND_PASSTHROUGH" "true") "false" }} diff --git a/pkg/kube/inject/testdata/inputs/hello.yaml.1.template.gen.yaml b/pkg/kube/inject/testdata/inputs/hello.yaml.1.template.gen.yaml index 334e1630b6e4..81dba4ab1078 100644 --- a/pkg/kube/inject/testdata/inputs/hello.yaml.1.template.gen.yaml +++ b/pkg/kube/inject/testdata/inputs/hello.yaml.1.template.gen.yaml @@ -76,7 +76,9 @@ templates: {{- end }} } spec: - {{- $holdProxy := or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts }} + {{- $holdProxy := and + (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts) + (not $nativeSidecar) }} initContainers: {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} {{ if .Values.istio_cni.enabled -}} @@ -234,6 +236,17 @@ templates: command: - pilot-agent - wait + {{- else if $nativeSidecar }} + {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}} + lifecycle: + preStop: + exec: + command: + - pilot-agent + - request + - --debug-port={{(annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort)}} + - POST + - drain {{- end }} env: {{- if eq (env "PILOT_ENABLE_INBOUND_PASSTHROUGH" "true") "false" }} diff --git a/pkg/kube/inject/testdata/inputs/hello.yaml.10.template.gen.yaml b/pkg/kube/inject/testdata/inputs/hello.yaml.10.template.gen.yaml index 334e1630b6e4..81dba4ab1078 100644 --- a/pkg/kube/inject/testdata/inputs/hello.yaml.10.template.gen.yaml +++ b/pkg/kube/inject/testdata/inputs/hello.yaml.10.template.gen.yaml @@ -76,7 +76,9 @@ templates: {{- end }} } spec: - {{- $holdProxy := or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts }} + {{- $holdProxy := and + (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts) + (not $nativeSidecar) }} initContainers: {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} {{ if .Values.istio_cni.enabled -}} @@ -234,6 +236,17 @@ templates: command: - pilot-agent - wait + {{- else if $nativeSidecar }} + {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}} + lifecycle: + preStop: + exec: + command: + - pilot-agent + - request + - --debug-port={{(annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort)}} + - POST + - drain {{- end }} env: {{- if eq (env "PILOT_ENABLE_INBOUND_PASSTHROUGH" "true") "false" }} diff --git a/pkg/kube/inject/testdata/inputs/hello.yaml.12.template.gen.yaml b/pkg/kube/inject/testdata/inputs/hello.yaml.12.template.gen.yaml index 334e1630b6e4..81dba4ab1078 100644 --- a/pkg/kube/inject/testdata/inputs/hello.yaml.12.template.gen.yaml +++ b/pkg/kube/inject/testdata/inputs/hello.yaml.12.template.gen.yaml @@ -76,7 +76,9 @@ templates: {{- end }} } spec: - {{- $holdProxy := or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts }} + {{- $holdProxy := and + (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts) + (not $nativeSidecar) }} initContainers: {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} {{ if .Values.istio_cni.enabled -}} @@ -234,6 +236,17 @@ templates: command: - pilot-agent - wait + {{- else if $nativeSidecar }} + {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}} + lifecycle: + preStop: + exec: + command: + - pilot-agent + - request + - --debug-port={{(annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort)}} + - POST + - drain {{- end }} env: {{- if eq (env "PILOT_ENABLE_INBOUND_PASSTHROUGH" "true") "false" }} diff --git a/pkg/kube/inject/testdata/inputs/hello.yaml.13.template.gen.yaml b/pkg/kube/inject/testdata/inputs/hello.yaml.13.template.gen.yaml index 334e1630b6e4..81dba4ab1078 100644 --- a/pkg/kube/inject/testdata/inputs/hello.yaml.13.template.gen.yaml +++ b/pkg/kube/inject/testdata/inputs/hello.yaml.13.template.gen.yaml @@ -76,7 +76,9 @@ templates: {{- end }} } spec: - {{- $holdProxy := or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts }} + {{- $holdProxy := and + (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts) + (not $nativeSidecar) }} initContainers: {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} {{ if .Values.istio_cni.enabled -}} @@ -234,6 +236,17 @@ templates: command: - pilot-agent - wait + {{- else if $nativeSidecar }} + {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}} + lifecycle: + preStop: + exec: + command: + - pilot-agent + - request + - --debug-port={{(annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort)}} + - POST + - drain {{- end }} env: {{- if eq (env "PILOT_ENABLE_INBOUND_PASSTHROUGH" "true") "false" }} diff --git a/pkg/kube/inject/testdata/inputs/hello.yaml.14.template.gen.yaml b/pkg/kube/inject/testdata/inputs/hello.yaml.14.template.gen.yaml index 334e1630b6e4..81dba4ab1078 100644 --- a/pkg/kube/inject/testdata/inputs/hello.yaml.14.template.gen.yaml +++ b/pkg/kube/inject/testdata/inputs/hello.yaml.14.template.gen.yaml @@ -76,7 +76,9 @@ templates: {{- end }} } spec: - {{- $holdProxy := or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts }} + {{- $holdProxy := and + (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts) + (not $nativeSidecar) }} initContainers: {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} {{ if .Values.istio_cni.enabled -}} @@ -234,6 +236,17 @@ templates: command: - pilot-agent - wait + {{- else if $nativeSidecar }} + {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}} + lifecycle: + preStop: + exec: + command: + - pilot-agent + - request + - --debug-port={{(annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort)}} + - POST + - drain {{- end }} env: {{- if eq (env "PILOT_ENABLE_INBOUND_PASSTHROUGH" "true") "false" }} diff --git a/pkg/kube/inject/testdata/inputs/hello.yaml.17.template.gen.yaml b/pkg/kube/inject/testdata/inputs/hello.yaml.17.template.gen.yaml index 334e1630b6e4..81dba4ab1078 100644 --- a/pkg/kube/inject/testdata/inputs/hello.yaml.17.template.gen.yaml +++ b/pkg/kube/inject/testdata/inputs/hello.yaml.17.template.gen.yaml @@ -76,7 +76,9 @@ templates: {{- end }} } spec: - {{- $holdProxy := or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts }} + {{- $holdProxy := and + (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts) + (not $nativeSidecar) }} initContainers: {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} {{ if .Values.istio_cni.enabled -}} @@ -234,6 +236,17 @@ templates: command: - pilot-agent - wait + {{- else if $nativeSidecar }} + {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}} + lifecycle: + preStop: + exec: + command: + - pilot-agent + - request + - --debug-port={{(annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort)}} + - POST + - drain {{- end }} env: {{- if eq (env "PILOT_ENABLE_INBOUND_PASSTHROUGH" "true") "false" }} diff --git a/pkg/kube/inject/testdata/inputs/hello.yaml.3.template.gen.yaml b/pkg/kube/inject/testdata/inputs/hello.yaml.3.template.gen.yaml index 334e1630b6e4..81dba4ab1078 100644 --- a/pkg/kube/inject/testdata/inputs/hello.yaml.3.template.gen.yaml +++ b/pkg/kube/inject/testdata/inputs/hello.yaml.3.template.gen.yaml @@ -76,7 +76,9 @@ templates: {{- end }} } spec: - {{- $holdProxy := or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts }} + {{- $holdProxy := and + (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts) + (not $nativeSidecar) }} initContainers: {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} {{ if .Values.istio_cni.enabled -}} @@ -234,6 +236,17 @@ templates: command: - pilot-agent - wait + {{- else if $nativeSidecar }} + {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}} + lifecycle: + preStop: + exec: + command: + - pilot-agent + - request + - --debug-port={{(annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort)}} + - POST + - drain {{- end }} env: {{- if eq (env "PILOT_ENABLE_INBOUND_PASSTHROUGH" "true") "false" }} diff --git a/pkg/kube/inject/testdata/inputs/hello.yaml.4.template.gen.yaml b/pkg/kube/inject/testdata/inputs/hello.yaml.4.template.gen.yaml index 334e1630b6e4..81dba4ab1078 100644 --- a/pkg/kube/inject/testdata/inputs/hello.yaml.4.template.gen.yaml +++ b/pkg/kube/inject/testdata/inputs/hello.yaml.4.template.gen.yaml @@ -76,7 +76,9 @@ templates: {{- end }} } spec: - {{- $holdProxy := or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts }} + {{- $holdProxy := and + (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts) + (not $nativeSidecar) }} initContainers: {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} {{ if .Values.istio_cni.enabled -}} @@ -234,6 +236,17 @@ templates: command: - pilot-agent - wait + {{- else if $nativeSidecar }} + {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}} + lifecycle: + preStop: + exec: + command: + - pilot-agent + - request + - --debug-port={{(annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort)}} + - POST + - drain {{- end }} env: {{- if eq (env "PILOT_ENABLE_INBOUND_PASSTHROUGH" "true") "false" }} diff --git a/pkg/kube/inject/testdata/inputs/kubevirtInterfaces.yaml.9.template.gen.yaml b/pkg/kube/inject/testdata/inputs/kubevirtInterfaces.yaml.9.template.gen.yaml index 334e1630b6e4..81dba4ab1078 100644 --- a/pkg/kube/inject/testdata/inputs/kubevirtInterfaces.yaml.9.template.gen.yaml +++ b/pkg/kube/inject/testdata/inputs/kubevirtInterfaces.yaml.9.template.gen.yaml @@ -76,7 +76,9 @@ templates: {{- end }} } spec: - {{- $holdProxy := or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts }} + {{- $holdProxy := and + (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts) + (not $nativeSidecar) }} initContainers: {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} {{ if .Values.istio_cni.enabled -}} @@ -234,6 +236,17 @@ templates: command: - pilot-agent - wait + {{- else if $nativeSidecar }} + {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}} + lifecycle: + preStop: + exec: + command: + - pilot-agent + - request + - --debug-port={{(annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort)}} + - POST + - drain {{- end }} env: {{- if eq (env "PILOT_ENABLE_INBOUND_PASSTHROUGH" "true") "false" }} diff --git a/pkg/kube/inject/testdata/inputs/merge-probers.yaml.40.template.gen.yaml b/pkg/kube/inject/testdata/inputs/merge-probers.yaml.40.template.gen.yaml index 334e1630b6e4..81dba4ab1078 100644 --- a/pkg/kube/inject/testdata/inputs/merge-probers.yaml.40.template.gen.yaml +++ b/pkg/kube/inject/testdata/inputs/merge-probers.yaml.40.template.gen.yaml @@ -76,7 +76,9 @@ templates: {{- end }} } spec: - {{- $holdProxy := or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts }} + {{- $holdProxy := and + (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts) + (not $nativeSidecar) }} initContainers: {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} {{ if .Values.istio_cni.enabled -}} @@ -234,6 +236,17 @@ templates: command: - pilot-agent - wait + {{- else if $nativeSidecar }} + {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}} + lifecycle: + preStop: + exec: + command: + - pilot-agent + - request + - --debug-port={{(annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort)}} + - POST + - drain {{- end }} env: {{- if eq (env "PILOT_ENABLE_INBOUND_PASSTHROUGH" "true") "false" }} diff --git a/pkg/kube/inject/testdata/inputs/status_params.yaml.8.template.gen.yaml b/pkg/kube/inject/testdata/inputs/status_params.yaml.8.template.gen.yaml index 334e1630b6e4..81dba4ab1078 100644 --- a/pkg/kube/inject/testdata/inputs/status_params.yaml.8.template.gen.yaml +++ b/pkg/kube/inject/testdata/inputs/status_params.yaml.8.template.gen.yaml @@ -76,7 +76,9 @@ templates: {{- end }} } spec: - {{- $holdProxy := or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts }} + {{- $holdProxy := and + (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts) + (not $nativeSidecar) }} initContainers: {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} {{ if .Values.istio_cni.enabled -}} @@ -234,6 +236,17 @@ templates: command: - pilot-agent - wait + {{- else if $nativeSidecar }} + {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}} + lifecycle: + preStop: + exec: + command: + - pilot-agent + - request + - --debug-port={{(annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort)}} + - POST + - drain {{- end }} env: {{- if eq (env "PILOT_ENABLE_INBOUND_PASSTHROUGH" "true") "false" }} diff --git a/pkg/kube/inject/testdata/inputs/traffic-params.yaml.7.template.gen.yaml b/pkg/kube/inject/testdata/inputs/traffic-params.yaml.7.template.gen.yaml index 334e1630b6e4..81dba4ab1078 100644 --- a/pkg/kube/inject/testdata/inputs/traffic-params.yaml.7.template.gen.yaml +++ b/pkg/kube/inject/testdata/inputs/traffic-params.yaml.7.template.gen.yaml @@ -76,7 +76,9 @@ templates: {{- end }} } spec: - {{- $holdProxy := or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts }} + {{- $holdProxy := and + (or .ProxyConfig.HoldApplicationUntilProxyStarts.GetValue .Values.global.proxy.holdApplicationUntilProxyStarts) + (not $nativeSidecar) }} initContainers: {{ if ne (annotation .ObjectMeta `sidecar.istio.io/interceptionMode` .ProxyConfig.InterceptionMode) `NONE` }} {{ if .Values.istio_cni.enabled -}} @@ -234,6 +236,17 @@ templates: command: - pilot-agent - wait + {{- else if $nativeSidecar }} + {{- /* preStop is called when the pod starts shutdown. Initialize drain. We will get SIGTERM once applications are torn down. */}} + lifecycle: + preStop: + exec: + command: + - pilot-agent + - request + - --debug-port={{(annotation .ObjectMeta `status.sidecar.istio.io/port` .Values.global.proxy.statusPort)}} + - POST + - drain {{- end }} env: {{- if eq (env "PILOT_ENABLE_INBOUND_PASSTHROUGH" "true") "false" }}