Skip to content

Commit

Permalink
cmd/k8s-operator,k8s-operator/apis/v1alpha1: allow Connector to route…
Browse files Browse the repository at this point in the history 
… traffic to a single IP

Add a new connector.spec.dnat field that can be used to route
traffic to a single IP address reachable from cluster.
This can be used to expose to tailnet a cloud service that can be
reached from cluster and does not have a DNS name (cloud services that
have DNS names can be exposed to tailnet using ExternalName Services, which is
a probably preferable way.)

Updates #12919

Signed-off-by: Irbe Krumina <irbe@tailscale.com>
  • Loading branch information
@irbekrm
irbekrm committed Jul 28, 2024
1 parent 1bf82dd commit 29c3a59
Show file tree
Hide file tree
Showing 7 changed files with 142 additions and 25 deletions.
38 changes: 29 additions & 9 deletions cmd/k8s-operator/connector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ type ConnectorReconciler struct {

subnetRouters set.Slice[types.UID] // for subnet routers gauge
exitNodes set.Slice[types.UID] // for exit nodes gauge
dnats set.Slice[types.UID] // for dnat gauge
}

var (
Expand All @@ -66,6 +67,7 @@ var (
gaugeConnectorSubnetRouterResources = clientmetric.NewGauge("k8s_connector_subnetrouter_resources")
// gaugeConnectorExitNodeResources tracks the number of Connectors currently managed by this operator instance that are exit nodes.
gaugeConnectorExitNodeResources = clientmetric.NewGauge("k8s_connector_exitnode_resources")
gaugeConnectorDNATResources = clientmetric.NewGauge("k8s_connector_dnat_resources")
)

func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Request) (res reconcile.Result, err error) {
Expand Down Expand Up @@ -149,6 +151,9 @@ func (a *ConnectorReconciler) Reconcile(ctx context.Context, req reconcile.Reque
cn.Status.SubnetRoutes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify()
return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionTrue, reasonConnectorCreated, reasonConnectorCreated)
}
if len(cn.Spec.DNAT) != 0 {
cn.Status.DNAT = cn.Spec.DNAT[0]
}
cn.Status.SubnetRoutes = ""
return setStatus(cn, tsapi.ConnectorReady, metav1.ConditionTrue, reasonConnectorCreated, reasonConnectorCreated)
}
Expand Down Expand Up @@ -178,33 +183,42 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
Hostname: hostname,
ChildResourceLabels: crl,
Tags: cn.Spec.Tags.Stringify(),
Connector: &connector{
isExitNode: cn.Spec.ExitNode,
},
ProxyClassName: proxyClass,
ProxyClassName: proxyClass,
isExitNode: cn.Spec.ExitNode,
}

if cn.Spec.SubnetRouter != nil && len(cn.Spec.SubnetRouter.AdvertiseRoutes) > 0 {
sts.Connector.routes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify()
sts.routes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify()
}

if len(cn.Spec.DNAT) != 0 {
sts.ClusterTargetIP = cn.Spec.DNAT[0]
}

a.mu.Lock()
if sts.Connector.isExitNode {
if sts.isExitNode {
a.exitNodes.Add(cn.UID)
} else {
a.exitNodes.Remove(cn.UID)
}
if sts.Connector.routes != "" {
if sts.routes != "" {
a.subnetRouters.Add(cn.GetUID())
} else {
a.subnetRouters.Remove(cn.GetUID())
}
if sts.ClusterTargetIP != "" {
a.dnats.Add(cn.GetUID())
} else {
a.dnats.Remove(cn.GetUID())
}
a.mu.Unlock()
gaugeConnectorSubnetRouterResources.Set(int64(a.subnetRouters.Len()))
gaugeConnectorExitNodeResources.Set(int64(a.exitNodes.Len()))
gaugeConnectorDNATResources.Set(int64(a.exitNodes.Len()))
var connectors set.Slice[types.UID]
connectors.AddSlice(a.exitNodes.Slice())
connectors.AddSlice(a.subnetRouters.Slice())
connectors.AddSlice(a.dnats.Slice())
gaugeConnectorResources.Set(int64(connectors.Len()))

_, err := a.ssr.Provision(ctx, logger, sts)
Expand Down Expand Up @@ -247,12 +261,15 @@ func (a *ConnectorReconciler) maybeCleanupConnector(ctx context.Context, logger
a.mu.Lock()
a.subnetRouters.Remove(cn.UID)
a.exitNodes.Remove(cn.UID)
a.dnats.Remove(cn.UID)
a.mu.Unlock()
gaugeConnectorExitNodeResources.Set(int64(a.exitNodes.Len()))
gaugeConnectorSubnetRouterResources.Set(int64(a.subnetRouters.Len()))
gaugeConnectorDNATResources.Set(int64(a.dnats.Len()))
var connectors set.Slice[types.UID]
connectors.AddSlice(a.exitNodes.Slice())
connectors.AddSlice(a.subnetRouters.Slice())
connectors.AddSlice(a.dnats.Slice())
gaugeConnectorResources.Set(int64(connectors.Len()))
return true, nil
}
Expand All @@ -261,8 +278,11 @@ func (a *ConnectorReconciler) validate(cn *tsapi.Connector) error {
// Connector fields are already validated at apply time with CEL validation
// on custom resource fields. The checks here are a backup in case the
// CEL validation breaks without us noticing.
if !(cn.Spec.SubnetRouter != nil || cn.Spec.ExitNode) {
return errors.New("invalid spec: a Connector must expose subnet routes or act as an exit node (or both)")
if !(cn.Spec.SubnetRouter != nil || cn.Spec.ExitNode || len(cn.Spec.DNAT) != 0) {
return errors.New("invalid spec: a Connector must expose subnet routes or act as an exit node (or both) or have DNAT set")
}
if (cn.Spec.SubnetRouter != nil || cn.Spec.ExitNode) && len(cn.Spec.DNAT) != 0 {
return errors.New("invalid spec: a Connector must not be both a subnet router and an exit node as well as have a DNAT set")
}
if cn.Spec.SubnetRouter == nil {
return nil
Expand Down
36 changes: 36 additions & 0 deletions cmd/k8s-operator/connector_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,42 @@ func TestConnector(t *testing.T) {

expectMissing[appsv1.StatefulSet](t, fc, "operator-ns", shortName)
expectMissing[corev1.Secret](t, fc, "operator-ns", fullName)

// Create a Connector that configures DNAT
cn = &tsapi.Connector{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
UID: types.UID("1234-UID"),
},
TypeMeta: metav1.TypeMeta{
Kind: tsapi.ConnectorKind,
APIVersion: "tailscale.io/v1alpha1",
},
Spec: tsapi.ConnectorSpec{
DNAT: []string{"10.44.0.1"},
},
}
mustCreate(t, fc, cn)
expectReconciled(t, cr, "", "test")
fullName, shortName = findGenName(t, fc, "", "test", "connector")

opts = configOpts{
stsName: shortName,
secretName: fullName,
parentType: "connector",
clusterTargetIP: "10.44.0.1",
hostname: "test-connector",
}
expectEqual(t, fc, expectedSecret(t, fc, opts), nil)
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)

// Update DNAT value
mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
conn.Spec.DNAT = []string{"10.44.0.2"}
})
opts.clusterTargetIP = "10.44.0.2"
expectReconciled(t, cr, "", "test")
expectEqual(t, fc, expectedSTS(t, fc, opts), removeHashAnnotation)
}

func TestConnectorWithProxyClass(t *testing.T) {
Expand Down
26 changes: 24 additions & 2 deletions cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ spec:
jsonPath: .status.isExitNode
name: IsExitNode
type: string
- description: DNAT of the Connector if any.
jsonPath: .status.dnat
name: DNAT
type: string
- description: Status of the deployed Connector resources.
jsonPath: .status.conditions[?(@.type == "ConnectorReady")].reason
name: Status
Expand Down Expand Up @@ -66,6 +70,17 @@ spec:
https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
type: object
properties:
dnat:
description: |-
DNAT is an address routable from within cluster that tailnet
traffic should be routed to. DNAT cannot be set together with
.spec.subnetRouter or .spec.exitNode.
DNAT is currently restricted to a list of a single IP address.
type: array
maxItems: 1
minItems: 1
items:
type: string
exitNode:
description: |-
ExitNode defines whether the Connector node should act as a
Expand Down Expand Up @@ -125,8 +140,10 @@ spec:
type: string
pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$
x-kubernetes-validations:
- rule: has(self.subnetRouter) || self.exitNode == true
message: A Connector needs to be either an exit node or a subnet router, or both.
- rule: (has(self.subnetRouter) || self.exitNode == true) || has(self.dnat)
message: A Connector needs to be either an exit node or a subnet router, or both or have .spec.dnat set.
- rule: (has(self.subnetRouter) || self.exitNode == true) != has(self.dnat)
message: A Connector with .spec.dnat set must not be an exit node or subnet router.
status:
description: |-
ConnectorStatus describes the status of the Connector. This is set
Expand Down Expand Up @@ -194,6 +211,11 @@ spec:
x-kubernetes-list-map-keys:
- type
x-kubernetes-list-type: map
dnat:
description: |-
DNAT is a cluster routable IP address that the tailnet traffic to
this node is routed to.
type: string
hostname:
description: |-
Hostname is the fully qualified domain name of the Connector node.
Expand Down
26 changes: 24 additions & 2 deletions cmd/k8s-operator/deploy/manifests/operator.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,10 @@ spec:
jsonPath: .status.isExitNode
name: IsExitNode
type: string
- description: DNAT of the Connector if any.
jsonPath: .status.dnat
name: DNAT
type: string
- description: Status of the deployed Connector resources.
jsonPath: .status.conditions[?(@.type == "ConnectorReady")].reason
name: Status
Expand Down Expand Up @@ -91,6 +95,17 @@ spec:
More info:
https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
properties:
dnat:
description: |-
DNAT is an address routable from within cluster that tailnet
traffic should be routed to. DNAT cannot be set together with
.spec.subnetRouter or .spec.exitNode.
DNAT is currently restricted to a list of a single IP address.
items:
type: string
maxItems: 1
minItems: 1
type: array
exitNode:
description: |-
ExitNode defines whether the Connector node should act as a
Expand Down Expand Up @@ -151,8 +166,10 @@ spec:
type: array
type: object
x-kubernetes-validations:
- message: A Connector needs to be either an exit node or a subnet router, or both.
rule: has(self.subnetRouter) || self.exitNode == true
- message: A Connector needs to be either an exit node or a subnet router, or both or have .spec.dnat set.
rule: (has(self.subnetRouter) || self.exitNode == true) || has(self.dnat)
- message: A Connector with .spec.dnat set must not be an exit node or subnet router.
rule: (has(self.subnetRouter) || self.exitNode == true) != has(self.dnat)
status:
description: |-
ConnectorStatus describes the status of the Connector. This is set
Expand Down Expand Up @@ -219,6 +236,11 @@ spec:
x-kubernetes-list-map-keys:
- type
x-kubernetes-list-type: map
dnat:
description: |-
DNAT is a cluster routable IP address that the tailnet traffic to
this node is routed to.
type: string
hostname:
description: |-
Hostname is the fully qualified domain name of the Connector node.
Expand Down
18 changes: 7 additions & 11 deletions cmd/k8s-operator/sts.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,21 +119,17 @@ type tailscaleSTSConfig struct {
Hostname string
Tags []string // if empty, use defaultTags

// Connector specifies a configuration of a Connector instance if that's
// what this StatefulSet should be created for.
Connector *connector
// routes is a list of subnet routes that this proxy should expose.
routes string

// isExitNode defines whether this proxy should act as an exit node.
isExitNode bool

ProxyClassName string // name of ProxyClass if one needs to be applied to the proxy

ProxyClass *tsapi.ProxyClass // ProxyClass that needs to be applied to the proxy (if there is one)
}

type connector struct {
// routes is a list of subnet routes that this Connector should expose.
routes string
// isExitNode defines whether this Connector should act as an exit node.
isExitNode bool
}
type tsnetServer interface {
CertDomains() []string
}
Expand Down Expand Up @@ -774,8 +770,8 @@ func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *co
if stsC.TailnetTargetFQDN != "" || stsC.TailnetTargetIP != "" {
conf.NoStatefulFiltering = "true"
}
if stsC.Connector != nil {
routes, err := netutil.CalcAdvertiseRoutes(stsC.Connector.routes, stsC.Connector.isExitNode)
if len(stsC.routes) != 0 || stsC.isExitNode {
routes, err := netutil.CalcAdvertiseRoutes(stsC.routes, stsC.isExitNode)
if err != nil {
return nil, fmt.Errorf("error calculating routes: %w", err)
}
Expand Down
18 changes: 17 additions & 1 deletion k8s-operator/apis/v1alpha1/types_connector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ var ConnectorKind = "Connector"
// +kubebuilder:resource:scope=Cluster,shortName=cn
// +kubebuilder:printcolumn:name="SubnetRoutes",type="string",JSONPath=`.status.subnetRoutes`,description="CIDR ranges exposed to tailnet by a subnet router defined via this Connector instance."
// +kubebuilder:printcolumn:name="IsExitNode",type="string",JSONPath=`.status.isExitNode`,description="Whether this Connector instance defines an exit node."
// +kubebuilder:printcolumn:name="DNAT",type="string",JSONPath=`.status.dnat`,description="DNAT of the Connector if any."
// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=`.status.conditions[?(@.type == "ConnectorReady")].reason`,description="Status of the deployed Connector resources."

// Connector defines a Tailscale node that will be deployed in the cluster. The
Expand Down Expand Up @@ -55,7 +56,8 @@ type ConnectorList struct {
}

// ConnectorSpec describes a Tailscale node to be deployed in the cluster.
// +kubebuilder:validation:XValidation:rule="has(self.subnetRouter) || self.exitNode == true",message="A Connector needs to be either an exit node or a subnet router, or both."
// +kubebuilder:validation:XValidation:rule="(has(self.subnetRouter) || self.exitNode == true) || has(self.dnat)",message="A Connector needs to be either an exit node or a subnet router, or both or have .spec.dnat set."
// +kubebuilder:validation:XValidation:rule="(has(self.subnetRouter) || self.exitNode == true) != has(self.dnat)",message="A Connector with .spec.dnat set must not be an exit node or subnet router."
type ConnectorSpec struct {
// Tags that the Tailscale node will be tagged with.
// Defaults to [tag:k8s].
Expand Down Expand Up @@ -92,8 +94,18 @@ type ConnectorSpec struct {
// https://tailscale.com/kb/1103/exit-nodes
// +optional
ExitNode bool `json:"exitNode"`
// DNAT is an address routable from within cluster that tailnet
// traffic should be routed to. DNAT cannot be set together with
// .spec.subnetRouter or .spec.exitNode.
// DNAT is currently restricted to a list of a single IP address.
// +optional
DNAT dnat `json:"dnat,omitempty"`
}

// +kubebuilder:validation:MaxItems=1
// +kubebuilder:validation:MinItems=1
type dnat []string

// SubnetRouter defines subnet routes that should be exposed to tailnet via a
// Connector node.
type SubnetRouter struct {
Expand Down Expand Up @@ -153,6 +165,10 @@ type ConnectorStatus struct {
// Connector instance.
// +optional
SubnetRoutes string `json:"subnetRoutes"`
// DNAT is a cluster routable IP address that the tailnet traffic to
// this node is routed to.
// +optional
DNAT string `json:"dnat,omitempty"`
// IsExitNode is set to true if the Connector acts as an exit node.
// +optional
IsExitNode bool `json:"isExitNode"`
Expand Down
5 changes: 5 additions & 0 deletions k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 29c3a59

Please sign in to comment.