From 4f6cde0db78c268c06285b400f13cc187992cdee Mon Sep 17 00:00:00 2001 From: Irbe Krumina Date: Sun, 28 Jul 2024 16:48:22 +0300 Subject: [PATCH] cmd/k8s-operator,k8s-operator/apis/v1alpha1: allow Connector to route traffic to a single IP Add a new connector.spec.dnat field that can be used to route traffic to a single IP address reachable from cluster. This can be used to expose to tailnet a cloud service that can be reached from cluster and does not have a DNS name (cloud services that have DNS names can be exposed to tailnet using ExternalName Services, which is a probably preferable way.) Updates tailscale/tailscale#12919 Signed-off-by: Irbe Krumina --- cmd/k8s-operator/connector.go | 38 ++++++++++++++----- cmd/k8s-operator/connector_test.go | 36 ++++++++++++++++++ .../deploy/crds/tailscale.com_connectors.yaml | 26 ++++++++++++- .../deploy/manifests/operator.yaml | 26 ++++++++++++- cmd/k8s-operator/sts.go | 18 ++++----- k8s-operator/api.md | 2 + k8s-operator/apis/v1alpha1/types_connector.go | 18 ++++++++- .../apis/v1alpha1/zz_generated.deepcopy.go | 5 +++ 8 files changed, 144 insertions(+), 25 deletions(-) diff --git a/cmd/k8s-operator/connector.go b/cmd/k8s-operator/connector.go index 4586dfdbfb297..e4a7ce2ed57af 100644 --- a/cmd/k8s-operator/connector.go +++ b/cmd/k8s-operator/connector.go @@ -57,6 +57,7 @@ type ConnectorReconciler struct { subnetRouters set.Slice[types.UID] // for subnet routers gauge exitNodes set.Slice[types.UID] // for exit nodes gauge + dnats set.Slice[types.UID] // for dnat gauge } var ( @@ -66,6 +67,7 @@ var ( gaugeConnectorSubnetRouterResources = clientmetric.NewGauge("k8s_connector_subnetrouter_resources") // gaugeConnectorExitNodeResources tracks the number of Connectors currently managed by this operator instance that are exit nodes. gaugeConnectorExitNodeResources = clientmetric.NewGauge("k8s_connector_exitnode_resources") + gaugeConnectorDNATResources = clientmetric.NewGauge("k8s_connector_dnat_resources") ) func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) { @@ -149,6 +151,9 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque cn.Status.SubnetRoutes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify() return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionTrue, reasonConnectorCreated, reasonConnectorCreated) } + if len(cn.Spec.DNAT) != 0 { + cn.Status.DNAT = cn.Spec.DNAT[0] + } cn.Status.SubnetRoutes = "" return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionTrue, reasonConnectorCreated, reasonConnectorCreated) } @@ -178,33 +183,42 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge Hostname: hostname, ChildResourceLabels: crl, Tags: cn.Spec.Tags.Stringify(), - Connector: &connector{ - isExitNode: cn.Spec.ExitNode, - }, - ProxyClassName: proxyClass, + ProxyClassName: proxyClass, + isExitNode: cn.Spec.ExitNode, } if cn.Spec.SubnetRouter != nil && len(cn.Spec.SubnetRouter.AdvertiseRoutes) > 0 { - sts.Connector.routes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify() + sts.routes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify() + } + + if len(cn.Spec.DNAT) != 0 { + sts.ClusterTargetIP = cn.Spec.DNAT[0] } a.mu.Lock() - if sts.Connector.isExitNode { + if sts.isExitNode { a.exitNodes.Add(cn.UID) } else { a.exitNodes.Remove(cn.UID) } - if sts.Connector.routes != "" { + if sts.routes != "" { a.subnetRouters.Add(cn.GetUID()) } else { a.subnetRouters.Remove(cn.GetUID()) } + if sts.ClusterTargetIP != "" { + a.dnats.Add(cn.GetUID()) + } else { + a.dnats.Remove(cn.GetUID()) + } a.mu.Unlock() gaugeConnectorSubnetRouterResources.Set(int64(a.subnetRouters.Len())) gaugeConnectorExitNodeResources.Set(int64(a.exitNodes.Len())) + gaugeConnectorDNATResources.Set(int64(a.exitNodes.Len())) var connectors set.Slice[types.UID] connectors.AddSlice(a.exitNodes.Slice()) connectors.AddSlice(a.subnetRouters.Slice()) + connectors.AddSlice(a.dnats.Slice()) gaugeConnectorResources.Set(int64(connectors.Len())) _, err := a.ssr.Provision(ctx, logger, sts) @@ -247,12 +261,15 @@ func (a *ConnectorReconciler) maybeCleanupConnector(ctx context.Context, logger a.mu.Lock() a.subnetRouters.Remove(cn.UID) a.exitNodes.Remove(cn.UID) + a.dnats.Remove(cn.UID) a.mu.Unlock() gaugeConnectorExitNodeResources.Set(int64(a.exitNodes.Len())) gaugeConnectorSubnetRouterResources.Set(int64(a.subnetRouters.Len())) + gaugeConnectorDNATResources.Set(int64(a.dnats.Len())) var connectors set.Slice[types.UID] connectors.AddSlice(a.exitNodes.Slice()) connectors.AddSlice(a.subnetRouters.Slice()) + connectors.AddSlice(a.dnats.Slice()) gaugeConnectorResources.Set(int64(connectors.Len())) return true, nil } @@ -261,8 +278,11 @@ func (a *ConnectorReconciler) validate(cn *tsapi.Connector) error { // Connector fields are already validated at apply time with CEL validation // on custom resource fields. The checks here are a backup in case the // CEL validation breaks without us noticing. - if !(cn.Spec.SubnetRouter != nil || cn.Spec.ExitNode) { - return errors.New("invalid spec: a Connector must expose subnet routes or act as an exit node (or both)") + if !(cn.Spec.SubnetRouter != nil || cn.Spec.ExitNode || len(cn.Spec.DNAT) != 0) { + return errors.New("invalid spec: a Connector must expose subnet routes or act as an exit node (or both) or have DNAT set") + } + if (cn.Spec.SubnetRouter != nil || cn.Spec.ExitNode) && len(cn.Spec.DNAT) != 0 { + return errors.New("invalid spec: a Connector must not be both a subnet router and an exit node as well as have a DNAT set") } if cn.Spec.SubnetRouter == nil { return nil diff --git a/cmd/k8s-operator/connector_test.go b/cmd/k8s-operator/connector_test.go index 8a7a5dd535e08..c6481a3162daa 100644 --- a/cmd/k8s-operator/connector_test.go +++ b/cmd/k8s-operator/connector_test.go @@ -191,6 +191,42 @@ func TestConnector(t *testing.T) { expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName) expectMissing[corev1.Secret](t, fc, "operator-ns", fullName) + + // Create a Connector that configures DNAT + cn = &tsapi.Connector{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + UID: types.UID("1234-UID"), + }, + TypeMeta: metav1.TypeMeta{ + Kind: tsapi.ConnectorKind, + APIVersion: "tailscale.io/v1alpha1", + }, + Spec: tsapi.ConnectorSpec{ + DNAT: []string{"10.44.0.1"}, + }, + } + mustCreate(t, fc, cn) + expectReconciled(t, cr, "", "test") + fullName, shortName = findGenName(t, fc, "", "test", "connector") + + opts = configOpts{ + stsName: shortName, + secretName: fullName, + parentType: "connector", + clusterTargetIP: "10.44.0.1", + hostname: "test-connector", + } + expectEqual(t, fc, expectedSecret(t, fc, opts), nil) + expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) + + // Update DNAT value + mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) { + conn.Spec.DNAT = []string{"10.44.0.2"} + }) + opts.clusterTargetIP = "10.44.0.2" + expectReconciled(t, cr, "", "test") + expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation) } func TestConnectorWithProxyClass(t *testing.T) { diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml index 66ff060d40540..ea62b84a8d5ba 100644 --- a/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml @@ -24,6 +24,10 @@ spec: jsonPath: .status.isExitNode name: IsExitNode type: string + - description: DNAT of the Connector if any. + jsonPath: .status.dnat + name: DNAT + type: string - description: Status of the deployed Connector resources. jsonPath: .status.conditions[?(@.type == "ConnectorReady")].reason name: Status @@ -66,6 +70,17 @@ spec: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status type: object properties: + dnat: + description: |- + DNAT is an address routable from within cluster that tailnet + traffic should be routed to. DNAT cannot be set together with + .spec.subnetRouter or .spec.exitNode. + DNAT is currently restricted to a list of a single IP address. + type: array + maxItems: 1 + minItems: 1 + items: + type: string exitNode: description: |- ExitNode defines whether the Connector node should act as a @@ -125,8 +140,10 @@ spec: type: string pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ x-kubernetes-validations: - - rule: has(self.subnetRouter) || self.exitNode == true - message: A Connector needs to be either an exit node or a subnet router, or both. + - rule: (has(self.subnetRouter) || self.exitNode == true) || has(self.dnat) + message: A Connector needs to be either an exit node or a subnet router, or both or have .spec.dnat set. + - rule: (has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true)) != has(self.dnat) + message: A Connector with .spec.dnat set must not be an exit node or subnet router. status: description: |- ConnectorStatus describes the status of the Connector. This is set @@ -194,6 +211,11 @@ spec: x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map + dnat: + description: |- + DNAT is a cluster routable IP address that the tailnet traffic to + this node is routed to. + type: string hostname: description: |- Hostname is the fully qualified domain name of the Connector node. diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index 4633ba3a48a38..9222cf245d65c 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -53,6 +53,10 @@ spec: jsonPath: .status.isExitNode name: IsExitNode type: string + - description: DNAT of the Connector if any. + jsonPath: .status.dnat + name: DNAT + type: string - description: Status of the deployed Connector resources. jsonPath: .status.conditions[?(@.type == "ConnectorReady")].reason name: Status @@ -91,6 +95,17 @@ spec: More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status properties: + dnat: + description: |- + DNAT is an address routable from within cluster that tailnet + traffic should be routed to. DNAT cannot be set together with + .spec.subnetRouter or .spec.exitNode. + DNAT is currently restricted to a list of a single IP address. + items: + type: string + maxItems: 1 + minItems: 1 + type: array exitNode: description: |- ExitNode defines whether the Connector node should act as a @@ -151,8 +166,10 @@ spec: type: array type: object x-kubernetes-validations: - - message: A Connector needs to be either an exit node or a subnet router, or both. - rule: has(self.subnetRouter) || self.exitNode == true + - message: A Connector needs to be either an exit node or a subnet router, or both or have .spec.dnat set. + rule: (has(self.subnetRouter) || self.exitNode == true) || has(self.dnat) + - message: A Connector with .spec.dnat set must not be an exit node or subnet router. + rule: (has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true)) != has(self.dnat) status: description: |- ConnectorStatus describes the status of the Connector. This is set @@ -219,6 +236,11 @@ spec: x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map + dnat: + description: |- + DNAT is a cluster routable IP address that the tailnet traffic to + this node is routed to. + type: string hostname: description: |- Hostname is the fully qualified domain name of the Connector node. diff --git a/cmd/k8s-operator/sts.go b/cmd/k8s-operator/sts.go index 17cc047d000d1..e9204ff6a668e 100644 --- a/cmd/k8s-operator/sts.go +++ b/cmd/k8s-operator/sts.go @@ -119,21 +119,17 @@ type tailscaleSTSConfig struct { Hostname string Tags []string // if empty, use defaultTags - // Connector specifies a configuration of a Connector instance if that's - // what this StatefulSet should be created for. - Connector *connector + // routes is a list of subnet routes that this proxy should expose. + routes string + + // isExitNode defines whether this proxy should act as an exit node. + isExitNode bool ProxyClassName string // name of ProxyClass if one needs to be applied to the proxy ProxyClass *tsapi.ProxyClass // ProxyClass that needs to be applied to the proxy (if there is one) } -type connector struct { - // routes is a list of subnet routes that this Connector should expose. - routes string - // isExitNode defines whether this Connector should act as an exit node. - isExitNode bool -} type tsnetServer interface { CertDomains() []string } @@ -774,8 +770,8 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co if stsC.TailnetTargetFQDN != "" || stsC.TailnetTargetIP != "" { conf.NoStatefulFiltering = "true" } - if stsC.Connector != nil { - routes, err := netutil.CalcAdvertiseRoutes(stsC.Connector.routes, stsC.Connector.isExitNode) + if len(stsC.routes) != 0 || stsC.isExitNode { + routes, err := netutil.CalcAdvertiseRoutes(stsC.routes, stsC.isExitNode) if err != nil { return nil, fmt.Errorf("error calculating routes: %w", err) } diff --git a/k8s-operator/api.md b/k8s-operator/api.md index 1b72df0f2720b..77177e4e0a6c6 100644 --- a/k8s-operator/api.md +++ b/k8s-operator/api.md @@ -84,6 +84,7 @@ _Appears in:_ | `proxyClass` _string_ | ProxyClass is the name of the ProxyClass custom resource that
contains configuration options that should be applied to the
resources created for this Connector. If unset, the operator will
create resources with the default configuration. | | | | `subnetRouter` _[SubnetRouter](#subnetrouter)_ | SubnetRouter defines subnet routes that the Connector node should
expose to tailnet. If unset, none are exposed.
https://tailscale.com/kb/1019/subnets/ | | | | `exitNode` _boolean_ | ExitNode defines whether the Connector node should act as a
Tailscale exit node. Defaults to false.
https://tailscale.com/kb/1103/exit-nodes | | | +| `dnat` _[dnat](#dnat)_ | DNAT is an address routable from within cluster that tailnet
traffic should be routed to. DNAT cannot be set together with
.spec.subnetRouter or .spec.exitNode.
DNAT is currently restricted to a list of a single IP address. | | MaxItems: 1
MinItems: 1
| #### ConnectorStatus @@ -101,6 +102,7 @@ _Appears in:_ | --- | --- | --- | --- | | `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#condition-v1-meta) array_ | List of status conditions to indicate the status of the Connector.
Known condition types are `ConnectorReady`. | | | | `subnetRoutes` _string_ | SubnetRoutes are the routes currently exposed to tailnet via this
Connector instance. | | | +| `dnat` _string_ | DNAT is a cluster routable IP address that the tailnet traffic to
this node is routed to. | | | | `isExitNode` _boolean_ | IsExitNode is set to true if the Connector acts as an exit node. | | | | `tailnetIPs` _string array_ | TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)
assigned to the Connector node. | | | | `hostname` _string_ | Hostname is the fully qualified domain name of the Connector node.
If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
node. | | | diff --git a/k8s-operator/apis/v1alpha1/types_connector.go b/k8s-operator/apis/v1alpha1/types_connector.go index c33ad3c393e70..e36c915c676a4 100644 --- a/k8s-operator/apis/v1alpha1/types_connector.go +++ b/k8s-operator/apis/v1alpha1/types_connector.go @@ -22,6 +22,7 @@ var ConnectorKind = "Connector" // +kubebuilder:resource:scope=Cluster,shortName=cn // +kubebuilder:printcolumn:name="SubnetRoutes",type="string",JSONPath=`.status.subnetRoutes`,description="CIDR ranges exposed to tailnet by a subnet router defined via this Connector instance." // +kubebuilder:printcolumn:name="IsExitNode",type="string",JSONPath=`.status.isExitNode`,description="Whether this Connector instance defines an exit node." +// +kubebuilder:printcolumn:name="DNAT",type="string",JSONPath=`.status.dnat`,description="DNAT of the Connector if any." // +kubebuilder:printcolumn:name="Status",type="string",JSONPath=`.status.conditions[?(@.type == "ConnectorReady")].reason`,description="Status of the deployed Connector resources." // Connector defines a Tailscale node that will be deployed in the cluster. The @@ -55,7 +56,8 @@ type ConnectorList struct { } // ConnectorSpec describes a Tailscale node to be deployed in the cluster. -// +kubebuilder:validation:XValidation:rule="has(self.subnetRouter) || self.exitNode == true",message="A Connector needs to be either an exit node or a subnet router, or both." +// +kubebuilder:validation:XValidation:rule="(has(self.subnetRouter) || self.exitNode == true) || has(self.dnat)",message="A Connector needs to be either an exit node or a subnet router, or both or have .spec.dnat set." +// +kubebuilder:validation:XValidation:rule="(has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true)) != has(self.dnat)",message="A Connector with .spec.dnat set must not be an exit node or subnet router." type ConnectorSpec struct { // Tags that the Tailscale node will be tagged with. // Defaults to [tag:k8s]. @@ -92,8 +94,18 @@ type ConnectorSpec struct { // https://tailscale.com/kb/1103/exit-nodes // +optional ExitNode bool `json:"exitNode"` + // DNAT is an address routable from within cluster that tailnet + // traffic should be routed to. DNAT cannot be set together with + // .spec.subnetRouter or .spec.exitNode. + // DNAT is currently restricted to a list of a single IP address. + // +optional + DNAT dnat `json:"dnat,omitempty"` } +// +kubebuilder:validation:MaxItems=1 +// +kubebuilder:validation:MinItems=1 +type dnat []string + // SubnetRouter defines subnet routes that should be exposed to tailnet via a // Connector node. type SubnetRouter struct { @@ -153,6 +165,10 @@ type ConnectorStatus struct { // Connector instance. // +optional SubnetRoutes string `json:"subnetRoutes"` + // DNAT is a cluster routable IP address that the tailnet traffic to + // this node is routed to. + // +optional + DNAT string `json:"dnat,omitempty"` // IsExitNode is set to true if the Connector acts as an exit node. // +optional IsExitNode bool `json:"isExitNode"` diff --git a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go index 648a6875b19be..f7b50c3783316 100644 --- a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go +++ b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go @@ -85,6 +85,11 @@ func (in *ConnectorSpec) DeepCopyInto(out *ConnectorSpec) { *out = new(SubnetRouter) (*in).DeepCopyInto(*out) } + if in.DNAT != nil { + in, out := &in.DNAT, &out.DNAT + *out = make(dnat, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectorSpec.