kind/pkg/cluster/internal/kubeadm/config.go

/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubeadm

import (
	"bytes"
	"fmt"
	"sort"
	"strings"

	"github.com/google/safetext/yamltemplate"

	"sigs.k8s.io/kind/pkg/errors"

	"sigs.k8s.io/kind/pkg/internal/apis/config"
	"sigs.k8s.io/kind/pkg/internal/version"
)

// ConfigData is supplied to the kubeadm config template, with values populated
// by the cluster package
type ConfigData struct {
	ClusterName       string
	KubernetesVersion string
	// The ControlPlaneEndpoint, that is the address of the external loadbalancer
	// if defined or the bootstrap node
	ControlPlaneEndpoint string
	// The Local API Server port
	APIBindPort int
	// The API server external listen IP (which we will port forward)
	APIServerAddress string

	// this should really be used for the --provider-id flag
	// ideally cluster config should not depend on the node backend otherwise ...
	NodeProvider string

	// ControlPlane flag specifies the node belongs to the control plane
	ControlPlane bool
	// The IP address or comma separated list IP addresses of of the node
	NodeAddress string
	// The name for the node (not the address)
	NodeName string

	// The Token for TLS bootstrap
	Token string

	// KubeProxyMode defines the kube-proxy mode between iptables, ipvs or nftables
	KubeProxyMode string
	// The subnet used for pods
	PodSubnet string
	// The subnet used for services
	ServiceSubnet string

	// Kubernetes FeatureGates
	FeatureGates map[string]bool

	// Kubernetes API Server RuntimeConfig
	RuntimeConfig map[string]string

	// IPFamily of the cluster, it can be IPv4, IPv6 or DualStack
	IPFamily config.ClusterIPFamily

	// Labels are the labels, in the format "key1=val1,key2=val2", with which the respective node will be labeled
	NodeLabels string

	// RootlessProvider is true if kind is running with rootless mode
	RootlessProvider bool

	// DisableLocalStorageCapacityIsolation is typically set true based on RootlessProvider
	// based on the Kubernetes version, if true kubelet localStorageCapacityIsolation is set false
	DisableLocalStorageCapacityIsolation bool

	// DerivedConfigData contains fields computed from the other fields for use
	// in the config templates and should only be populated by calling Derive()
	DerivedConfigData
}

// DerivedConfigData fields are automatically derived by
// ConfigData.Derive if they are not specified / zero valued
type DerivedConfigData struct {
	// AdvertiseAddress is the first address in NodeAddress
	AdvertiseAddress string
	// DockerStableTag is automatically derived from KubernetesVersion
	DockerStableTag string
	// SortedFeatureGates allows us to iterate FeatureGates deterministically
	SortedFeatureGates []FeatureGate
	// FeatureGatesString is of the form `Foo=true,Baz=false`
	FeatureGatesString string
	// RuntimeConfigString is of the form `Foo=true,Baz=false`
	RuntimeConfigString string
	// KubeadmFeatureGates contains Kubeadm only feature gates
	KubeadmFeatureGates map[string]bool
	// IPv4 values take precedence over IPv6 by default, if true set IPv6 default values
	IPv6 bool
	// kubelet cgroup driver, based on kubernetes version
	CgroupDriver string
	// JoinSkipPhases are the skipPhases values for the JoinConfiguration.
	JoinSkipPhases []string
	// InitSkipPhases are the skipPhases values for the InitConfiguration.
	InitSkipPhases []string
}

type FeatureGate struct {
	Name  string
	Value bool
}

// Derive automatically derives DockerStableTag if not specified
func (c *ConfigData) Derive() {
	// default cgroup driver
	// TODO: refactor and move all deriving logic to this method
	c.CgroupDriver = "systemd"

	// get the first address to use it as the API advertised address
	c.AdvertiseAddress = strings.Split(c.NodeAddress, ",")[0]

	if c.DockerStableTag == "" {
		c.DockerStableTag = strings.Replace(c.KubernetesVersion, "+", "_", -1)
	}

	// get the IP addresses family for defaulting components
	c.IPv6 = c.IPFamily == config.IPv6Family

	// get sorted list of FeatureGate keys
	featureGateKeys := make([]string, 0, len(c.FeatureGates))
	for k := range c.FeatureGates {
		featureGateKeys = append(featureGateKeys, k)
	}
	sort.Strings(featureGateKeys)

	// create a sorted key=value,... string of FeatureGates
	c.SortedFeatureGates = make([]FeatureGate, 0, len(c.FeatureGates))
	featureGates := make([]string, 0, len(c.FeatureGates))
	for _, k := range featureGateKeys {
		v := c.FeatureGates[k]
		featureGates = append(featureGates, fmt.Sprintf("%s=%t", k, v))
		c.SortedFeatureGates = append(c.SortedFeatureGates, FeatureGate{
			Name:  k,
			Value: v,
		})
	}
	c.FeatureGatesString = strings.Join(featureGates, ",")

	// create a sorted key=value,... string of RuntimeConfig
	// first get sorted list of FeatureGate keys
	runtimeConfigKeys := make([]string, 0, len(c.RuntimeConfig))
	for k := range c.RuntimeConfig {
		runtimeConfigKeys = append(runtimeConfigKeys, k)
	}
	sort.Strings(runtimeConfigKeys)
	// stringify
	var runtimeConfig []string
	for _, k := range runtimeConfigKeys {
		v := c.RuntimeConfig[k]
		// TODO: do we need to quote / escape these in the future?
		// Currently runtime config is in practice booleans, no special characters
		runtimeConfig = append(runtimeConfig, fmt.Sprintf("%s=%s", k, v))
	}
	c.RuntimeConfigString = strings.Join(runtimeConfig, ",")

	// skip preflight checks, as these have undesirable side effects
	// and don't tell us much. requires kubeadm 1.22+
	c.JoinSkipPhases = []string{"preflight"}
	c.InitSkipPhases = []string{"preflight"}
	if c.KubeProxyMode == string(config.NoneProxyMode) {
		c.InitSkipPhases = append(c.InitSkipPhases, "addon/kube-proxy")
	}
}

// See docs for these APIs at:
// https://godoc.org/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm#pkg-subdirectories
// EG:
// https://godoc.org/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta1

// ConfigTemplateBetaV2 is the kubeadm config template for API version v1beta2
const ConfigTemplateBetaV2 = `# config generated by kind
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
metadata:
  name: config
kubernetesVersion: {{.KubernetesVersion}}
clusterName: "{{.ClusterName}}"
{{ if .KubeadmFeatureGates}}featureGates:
{{ range $key, $value := .KubeadmFeatureGates }}
  "{{ (StructuralData $key) }}": {{ $value }}
{{end}}{{end}}
controlPlaneEndpoint: "{{ .ControlPlaneEndpoint }}"
# on docker for mac we have to expose the api server via port forward,
# so we need to ensure the cert is valid for localhost so we can talk
# to the cluster after rewriting the kubeconfig to point to localhost
apiServer:
  certSANs: [localhost, "{{.APIServerAddress}}"]
  extraArgs:
    "runtime-config": "{{ .RuntimeConfigString }}"
{{ if .FeatureGates }}
    "feature-gates": "{{ .FeatureGatesString }}"
{{ end}}
controllerManager:
  extraArgs:
{{ if .FeatureGates }}
    "feature-gates": "{{ .FeatureGatesString }}"
{{ end }}
    enable-hostpath-provisioner: "true"
    # configure ipv6 default addresses for IPv6 clusters
    {{ if .IPv6 -}}
    bind-address: "::"
    {{- end }}
scheduler:
  extraArgs:
{{ if .FeatureGates }}
    "feature-gates": "{{ .FeatureGatesString }}"
{{ end }}
    # configure ipv6 default addresses for IPv6 clusters
    {{ if .IPv6 -}}
    bind-address: "::1"
    {{- end }}
networking:
  podSubnet: "{{ .PodSubnet }}"
  serviceSubnet: "{{ .ServiceSubnet }}"
---
apiVersion: kubeadm.k8s.io/v1beta2
kind: InitConfiguration
metadata:
  name: config
# we use a well know token for TLS bootstrap
bootstrapTokens:
- token: "{{ .Token }}"
# we use a well know port for making the API server discoverable inside docker network. 
# from the host machine such port will be accessible via a random local port instead.
localAPIEndpoint:
  advertiseAddress: "{{ .AdvertiseAddress }}"
  bindPort: {{.APIBindPort}}
nodeRegistration:
  criSocket: "unix:///run/containerd/containerd.sock"
  kubeletExtraArgs:
    node-ip: "{{ .NodeAddress }}"
    provider-id: "kind://{{.NodeProvider}}/{{.ClusterName}}/{{.NodeName}}"
    node-labels: "{{ .NodeLabels }}"
---
# no-op entry that exists solely so it can be patched
apiVersion: kubeadm.k8s.io/v1beta2
kind: JoinConfiguration
metadata:
  name: config
{{ if .ControlPlane -}}
controlPlane:
  localAPIEndpoint:
    advertiseAddress: "{{ .AdvertiseAddress }}"
    bindPort: {{.APIBindPort}}
{{- end }}
nodeRegistration:
  criSocket: "unix:///run/containerd/containerd.sock"
  kubeletExtraArgs:
    node-ip: "{{ .NodeAddress }}"
    provider-id: "kind://{{.NodeProvider}}/{{.ClusterName}}/{{.NodeName}}"
    node-labels: "{{ .NodeLabels }}"
discovery:
  bootstrapToken:
    apiServerEndpoint: "{{ .ControlPlaneEndpoint }}"
    token: "{{ .Token }}"
    unsafeSkipCAVerification: true
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
metadata:
  name: config
cgroupDriver: {{ .CgroupDriver }}
cgroupRoot: /kubelet
failSwapOn: false
# configure ipv6 addresses in IPv6 mode
{{ if .IPv6 -}}
address: "::"
healthzBindAddress: "::"
{{- end }}
# disable disk resource management by default
# kubelet will see the host disk that the inner container runtime
# is ultimately backed by and attempt to recover disk space. we don't want that.
imageGCHighThresholdPercent: 100
evictionHard:
  nodefs.available: "0%"
  nodefs.inodesFree: "0%"
  imagefs.available: "0%"
{{if .FeatureGates}}featureGates:
{{ range $index, $gate := .SortedFeatureGates }}
  "{{ (StructuralData $gate.Name) }}": {{ $gate.Value }}
{{end}}{{end}}
{{if ne .KubeProxyMode "none"}}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
metadata:
  name: config
mode: "{{ .KubeProxyMode }}"
{{if .FeatureGates}}featureGates:
{{ range $index, $gate := .SortedFeatureGates }}
  "{{ (StructuralData $gate.Name) }}": {{ $gate.Value }}
{{end}}{{end}}
iptables:
  minSyncPeriod: 1s
conntrack:
# Skip setting sysctl value "net.netfilter.nf_conntrack_max"
# It is a global variable that affects other namespaces
  maxPerCore: 0
# Set sysctl value "net.netfilter.nf_conntrack_tcp_be_liberal"
# for nftables proxy (theoretically for kernels older than 6.1)
# xref: https://github.com/kubernetes/kubernetes/issues/117924
{{if and (eq .KubeProxyMode "nftables") (not .RootlessProvider)}}
  tcpBeLiberal: true
{{end}}
{{if .RootlessProvider}}
# Skip setting "net.netfilter.nf_conntrack_tcp_timeout_established"
  tcpEstablishedTimeout: 0s
# Skip setting "net.netfilter.nf_conntrack_tcp_timeout_close"
  tcpCloseWaitTimeout: 0s
{{end}}{{end}}
`

// ConfigTemplateBetaV3 is the kubeadm config template for API version v1beta3
const ConfigTemplateBetaV3 = `# config generated by kind
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
metadata:
  name: config
kubernetesVersion: {{.KubernetesVersion}}
clusterName: "{{.ClusterName}}"
{{ if .KubeadmFeatureGates}}featureGates:
{{ range $key, $value := .KubeadmFeatureGates }}
  "{{ (StructuralData $key) }}": {{ $value }}
{{end}}{{end}}
controlPlaneEndpoint: "{{ .ControlPlaneEndpoint }}"
# on docker for mac we have to expose the api server via port forward,
# so we need to ensure the cert is valid for localhost so we can talk
# to the cluster after rewriting the kubeconfig to point to localhost
apiServer:
  certSANs: [localhost, "{{.APIServerAddress}}"]
  extraArgs:
    "runtime-config": "{{ .RuntimeConfigString }}"
{{ if .FeatureGates }}
    "feature-gates": "{{ .FeatureGatesString }}"
{{ end}}
controllerManager:
  extraArgs:
{{ if .FeatureGates }}
    "feature-gates": "{{ .FeatureGatesString }}"
{{ end }}
    enable-hostpath-provisioner: "true"
    # configure ipv6 default addresses for IPv6 clusters
    {{ if .IPv6 -}}
    bind-address: "::"
    {{- end }}
scheduler:
  extraArgs:
{{ if .FeatureGates }}
    "feature-gates": "{{ .FeatureGatesString }}"
{{ end }}
    # configure ipv6 default addresses for IPv6 clusters
    {{ if .IPv6 -}}
    bind-address: "::1"
    {{- end }}
networking:
  podSubnet: "{{ .PodSubnet }}"
  serviceSubnet: "{{ .ServiceSubnet }}"
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
metadata:
  name: config
# we use a well know token for TLS bootstrap
bootstrapTokens:
- token: "{{ .Token }}"
# we use a well know port for making the API server discoverable inside docker network. 
# from the host machine such port will be accessible via a random local port instead.
localAPIEndpoint:
  advertiseAddress: "{{ .AdvertiseAddress }}"
  bindPort: {{.APIBindPort}}
nodeRegistration:
  criSocket: "unix:///run/containerd/containerd.sock"
  kubeletExtraArgs:
    node-ip: "{{ .NodeAddress }}"
    provider-id: "kind://{{.NodeProvider}}/{{.ClusterName}}/{{.NodeName}}"
    node-labels: "{{ .NodeLabels }}"
{{ if .InitSkipPhases -}}
skipPhases:
  {{ range $phase := .InitSkipPhases -}}
  - "{{ $phase }}"
  {{- end }}
{{- end }}
---
# no-op entry that exists solely so it can be patched
apiVersion: kubeadm.k8s.io/v1beta3
kind: JoinConfiguration
metadata:
  name: config
{{ if .ControlPlane -}}
controlPlane:
  localAPIEndpoint:
    advertiseAddress: "{{ .AdvertiseAddress }}"
    bindPort: {{.APIBindPort}}
{{- end }}
nodeRegistration:
  criSocket: "unix:///run/containerd/containerd.sock"
  kubeletExtraArgs:
    node-ip: "{{ .NodeAddress }}"
    provider-id: "kind://{{.NodeProvider}}/{{.ClusterName}}/{{.NodeName}}"
    node-labels: "{{ .NodeLabels }}"
discovery:
  bootstrapToken:
    apiServerEndpoint: "{{ .ControlPlaneEndpoint }}"
    token: "{{ .Token }}"
    unsafeSkipCAVerification: true
{{ if .JoinSkipPhases -}}
skipPhases:
  {{ range $phase := .JoinSkipPhases -}}
  - "{{ $phase }}"
  {{- end }}
{{- end }}
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
metadata:
  name: config
cgroupDriver: {{ .CgroupDriver }}
cgroupRoot: /kubelet
failSwapOn: false
# configure ipv6 addresses in IPv6 mode
{{ if .IPv6 -}}
address: "::"
healthzBindAddress: "::"
{{- end }}
# disable disk resource management by default
# kubelet will see the host disk that the inner container runtime
# is ultimately backed by and attempt to recover disk space. we don't want that.
imageGCHighThresholdPercent: 100
evictionHard:
  nodefs.available: "0%"
  nodefs.inodesFree: "0%"
  imagefs.available: "0%"
{{if .FeatureGates}}featureGates:
{{ range $index, $gate := .SortedFeatureGates }}
  "{{ (StructuralData $gate.Name) }}": {{ $gate.Value }}
{{end}}{{end}}
{{if .DisableLocalStorageCapacityIsolation}}localStorageCapacityIsolation: false{{end}}
{{if ne .KubeProxyMode "none"}}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
metadata:
  name: config
mode: "{{ .KubeProxyMode }}"
{{if .FeatureGates}}featureGates:
{{ range $index, $gate := .SortedFeatureGates }}
  "{{ (StructuralData $gate.Name) }}": {{ $gate.Value }}
{{end}}{{end}}
iptables:
  minSyncPeriod: 1s
conntrack:
# Skip setting sysctl value "net.netfilter.nf_conntrack_max"
# It is a global variable that affects other namespaces
  maxPerCore: 0
# Set sysctl value "net.netfilter.nf_conntrack_tcp_be_liberal"
# for nftables proxy (theoretically for kernels older than 6.1)
# xref: https://github.com/kubernetes/kubernetes/issues/117924
{{if and (eq .KubeProxyMode "nftables") (not .RootlessProvider)}}
  tcpBeLiberal: true
{{end}}
{{if .RootlessProvider}}
# Skip setting "net.netfilter.nf_conntrack_tcp_timeout_established"
  tcpEstablishedTimeout: 0s
# Skip setting "net.netfilter.nf_conntrack_tcp_timeout_close"
  tcpCloseWaitTimeout: 0s
{{end}}{{end}}
`

// Config returns a kubeadm config generated from config data, in particular
// the kubernetes version
func Config(data ConfigData) (config string, err error) {
	ver, err := version.ParseGeneric(data.KubernetesVersion)
	if err != nil {
		return "", err
	}

	// ensure featureGates is non-nil, as we may add entries
	if data.FeatureGates == nil {
		data.FeatureGates = make(map[string]bool)
	}

	if data.RootlessProvider {
		if ver.LessThan(version.MustParseSemantic("v1.22.0")) {
			// rootless kind v0.12.x supports Kubernetes v1.22 with KubeletInUserNamespace gate.
			// rootless kind v0.11.x supports older Kubernetes with fake procfs.
			return "", errors.Errorf("version %q is not compatible with rootless provider (hint: kind v0.11.x may work with this version)", ver)
		}
		data.FeatureGates["KubeletInUserNamespace"] = true

		// For avoiding err="failed to get rootfs info: failed to get device for dir \"/var/lib/kubelet\": could not find device with major: 0, minor: 41 in cached partitions map"
		// https://github.com/kubernetes-sigs/kind/issues/2524
		if ver.LessThan(version.MustParseSemantic("v1.25.0-alpha.3.440+0064010cddfa00")) {
			// this feature gate was removed in v1.25 and replaced by an opt-out to disable
			data.FeatureGates["LocalStorageCapacityIsolation"] = false
		} else {
			// added in v1.25 https://github.com/kubernetes/kubernetes/pull/111513
			data.DisableLocalStorageCapacityIsolation = true
		}
	}

	// assume the latest API version, then fallback if the k8s version is too low
	templateSource := ConfigTemplateBetaV3
	if ver.LessThan(version.MustParseSemantic("v1.23.0")) {
		templateSource = ConfigTemplateBetaV2
	}

	t, err := yamltemplate.New("kubeadm-config").Parse(templateSource)
	if err != nil {
		return "", errors.Wrap(err, "failed to parse config template")
	}

	// derive any automatic fields if not supplied
	data.Derive()

	// Kubeadm has its own feature-gate for dual stack
	// we need to enable it for Kubernetes version 1.20 only
	// dual-stack is only supported in 1.20+
	// TODO: remove this when 1.20 is EOL or we no longer support
	// dual-stack for 1.20 in KIND
	if ver.LessThan(version.MustParseSemantic("v1.21.0")) &&
		ver.AtLeast(version.MustParseSemantic("v1.20.0")) {
		data.KubeadmFeatureGates = make(map[string]bool)
		data.KubeadmFeatureGates["IPv6DualStack"] = true
	}

	// before 1.24 kind uses cgroupfs
	// after 1.24 kind uses systemd starting in kind v0.13.0
	// before kind v0.13.0 kubernetes 1.24 wasn't released yet
	if ver.LessThan(version.MustParseSemantic("v1.24.0")) {
		data.CgroupDriver = "cgroupfs"
	}

	// execute the template
	var buff bytes.Buffer
	err = t.Execute(&buff, data)
	if err != nil {
		return "", errors.Wrap(err, "error executing config template")
	}
	return buff.String(), nil
}