add lifecycle hooks

docs/kind-example-config.yaml

@@ -0,0 +1,23 @@
+# this config file contains all config fields with comments
+apiVersion: kind.sigs.k8s.io/v1alpha1
+kind: Config
+# number of nodes in the cluster (currently only 1 is supported)
+numNodes: 1
+# template for kubeadm config, "" -> the default template
+kubeadmConfigTemplate: ""
+# node lifecycle hooks
+nodeLifecycle:
+  # commands run before systemd boots
+  preBoot:
+  - name: "should fail"
+    command: [ "yep-not-a-real-command" ]
+  # commands run before kubeadm init
+  preKubeadm: []
+  # commands run after kubeadm init
+  postKubeadm: []
+  # commands run after all other setup on the node
+  postSetup:
+  - name: "list containers"
+    command: [ "docker", "ps" ]
+    # this command cannot fail, or else kind will fail during create
+    mustSucceed: true

pkg/cluster/cluster.go

@@ -94,16 +94,16 @@ func (c *Context) KubeConfigPath() string {
 }
 
 // Create provisions and starts a kubernetes-in-docker cluster
-func (c *Context) Create(config *config.Config) error {
+func (c *Context) Create(cfg *config.Config) error {
 	// validate config first
-	if err := config.Validate(); err != nil {
+	if err := cfg.Validate(); err != nil {
 		return err
 	}
 
 	// TODO(bentheelder): multiple nodes ...
 	kubeadmConfig, err := c.provisionControlPlane(
 		fmt.Sprintf("kind-%s-control-plane", c.name),
-		config,
+		cfg,
 	)
 
 	// clean up the kubeadm config file
@@ -137,10 +137,10 @@ func (c *Context) Delete() error {
 // and the cluster kubeadm config
 func (c *Context) provisionControlPlane(
 	nodeName string,
-	config *config.Config,
+	cfg *config.Config,
 ) (kubeadmConfigPath string, err error) {
 	// create the "node" container (docker run, but it is paused, see createNode)
-	node, err := createNode(nodeName, config.Image, c.ClusterLabel())
+	node, err := createNode(nodeName, cfg.Image, c.ClusterLabel())
 	if err != nil {
 		return "", err
 	}
@@ -156,6 +156,15 @@ func (c *Context) provisionControlPlane(
 		return "", err
 	}
 
+	// run any pre-boot hooks
+	if cfg.NodeLifecycle != nil {
+		for _, hook := range cfg.NodeLifecycle.PreBoot {
+			if err := node.RunHook(&hook, "preBoot"); err != nil {
+				return "", err
+			}
+		}
+	}
+
 	// signal the node entrypoint to continue booting into systemd
 	if err := node.SignalStart(); err != nil {
 		// TODO(bentheelder): logging here
@@ -186,7 +195,7 @@ func (c *Context) provisionControlPlane(
 
 	// create kubeadm config file
 	kubeadmConfig, err := c.createKubeadmConfig(
-		config.KubeadmConfigTemplate,
+		cfg.KubeadmConfigTemplate,
 		kubeadm.ConfigData{
 			ClusterName:       c.ClusterName(),
 			KubernetesVersion: kubeVersion,
@@ -201,6 +210,15 @@ func (c *Context) provisionControlPlane(
 		return kubeadmConfig, errors.Wrap(err, "failed to copy kubeadm config to node")
 	}
 
+	// run any pre-kubeadm hooks
+	if cfg.NodeLifecycle != nil {
+		for _, hook := range cfg.NodeLifecycle.PreKubeadm {
+			if err := node.RunHook(&hook, "preKubeadm"); err != nil {
+				return kubeadmConfig, err
+			}
+		}
+	}
+
 	// run kubeadm
 	if err := node.Run(
 		// init because this is the control plane node
@@ -217,6 +235,15 @@ func (c *Context) provisionControlPlane(
 		return kubeadmConfig, errors.Wrap(err, "failed to init node with kubeadm")
 	}
 
+	// run any pre-kubeadm hooks
+	if cfg.NodeLifecycle != nil {
+		for _, hook := range cfg.NodeLifecycle.PostKubeadm {
+			if err := node.RunHook(&hook, "postKubeadm"); err != nil {
+				return kubeadmConfig, err
+			}
+		}
+	}
+
 	// set up the $KUBECONFIG
 	kubeConfigPath := c.KubeConfigPath()
 	if err = node.WriteKubeConfig(kubeConfigPath); err != nil {
@@ -236,7 +263,7 @@ func (c *Context) provisionControlPlane(
 
 	// if we are only provisioning one node, remove the master taint
 	// https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#master-isolation
-	if config.NumNodes == 1 {
+	if cfg.NumNodes == 1 {
 		if err = node.Run(
 			"kubectl", "--kubeconfig=/etc/kubernetes/admin.conf",
 			"taint", "nodes", "--all", "node-role.kubernetes.io/master-",
@@ -245,6 +272,15 @@ func (c *Context) provisionControlPlane(
 		}
 	}
 
+	// run any post-overlay hooks
+	if cfg.NodeLifecycle != nil {
+		for _, hook := range cfg.NodeLifecycle.PostSetup {
+			if err := node.RunHook(&hook, "postSetup"); err != nil {
+				return kubeadmConfig, err
+			}
+		}
+	}
+
 	return kubeadmConfig, nil
 }
 

pkg/cluster/config/deepcopy.go

@@ -59,6 +59,12 @@ func (in *NodeLifecycle) DeepCopy() *NodeLifecycle {
 			out.PostKubeadm[i] = *(in.PostKubeadm[i].DeepCopy())
 		}
 	}
+	if in.PostSetup != nil {
+		out.PostSetup = make([]LifecycleHook, len(in.PostSetup))
+		for i := range in.PostSetup {
+			out.PostSetup[i] = *(in.PostSetup[i].DeepCopy())
+		}
+	}
 	return out
 }
 
@@ -69,10 +75,10 @@ func (in *LifecycleHook) DeepCopy() *LifecycleHook {
 	}
 	out := new(LifecycleHook)
 	*out = *in
-	if in.Args != nil {
-		out.Args = make([]string, len(in.Args))
-		for i := range in.Args {
-			out.Args[i] = in.Args[i]
+	if in.Command != nil {
+		out.Command = make([]string, len(in.Command))
+		for i := range in.Command {
+			out.Command[i] = in.Command[i]
 		}
 	}
 	return out

pkg/cluster/config/deepcopy_test.go

@@ -37,22 +37,26 @@ func TestDeepCopy(t *testing.T) {
 				cfg.NodeLifecycle = &NodeLifecycle{
 					PreBoot: []LifecycleHook{
 						{
-							Command: "ps",
-							Args:    []string{"aux"},
+							Command: []string{"ps", "aux"},
 						},
 					},
 					PreKubeadm: []LifecycleHook{
 						{
 							Name:    "docker ps",
-							Command: "docker",
-							Args:    []string{"ps"},
+							Command: []string{"docker", "ps"},
 						},
 					},
 					PostKubeadm: []LifecycleHook{
 						{
 							Name:    "docker ps again",
-							Command: "docker",
-							Args:    []string{"ps", "-a"},
+							Command: []string{"docker", "ps", "-a"},
+						},
+					},
+					PostSetup: []LifecycleHook{
+						{
+							Name:        "docker ps again again",
+							Command:     []string{"docker", "ps", "-a"},
+							MustSucceed: true,
 						},
 					},
 				}

pkg/cluster/config/encoding/encoding_test.go

@@ -127,11 +127,10 @@ apiVersion: kind.sigs.k8s.io/v1alpha1
 nodeLifecycle:
   preKubeadm:
   - name: "pull an image"
-    command: "docker"
-    args: [ "pull", "ubuntu" ]
+    command: [ "docker", "pull", "ubuntu" ]
   - name: "pull another image"
-    command: "docker"
-    args: [ "pull", "debian" ]
+    command: [ "docker", "pull", "debian" ]
+    mustSucceed: true
 `),
 			ExpectedConfig: func() config.Any {
 				cfg := &config.Config{
@@ -139,13 +138,12 @@ nodeLifecycle:
 						PreKubeadm: []config.LifecycleHook{
 							{
 								Name:    "pull an image",
-								Command: "docker",
-								Args:    []string{"pull", "ubuntu"},
+								Command: []string{"docker", "pull", "ubuntu"},
 							},
 							{
-								Name:    "pull another image",
-								Command: "docker",
-								Args:    []string{"pull", "debian"},
+								Name:        "pull another image",
+								Command:     []string{"docker", "pull", "debian"},
+								MustSucceed: true,
 							},
 						},
 					},

pkg/cluster/config/encoding/testdata/valid-with-lifecyclehooks.yaml

@@ -3,8 +3,6 @@ apiVersion: kind.sigs.k8s.io/v1alpha1
 nodeLifecycle:
   preKubeadm:
   - name: "pull an image"
-    command: "docker"
-    args: [ "pull", "ubuntu" ]
+    command: [ "docker", "pull", "ubuntu" ]
   - name: "pull another image"
-    command: "docker"
-    args: [ "pull", "debian" ]
+    command: [ "docker", "pull", "debian" ]

pkg/cluster/config/types.go

@@ -43,10 +43,12 @@ var _ Any = &Config{}
 type NodeLifecycle struct {
 	// PreBoot hooks run before starting systemd
 	PreBoot []LifecycleHook `json:"preBoot,omitempty"`
-	// PreKubeadm hooks run before `kubeadm`
+	// PreKubeadm hooks run immediately before `kubeadm`
 	PreKubeadm []LifecycleHook `json:"preKubeadm,omitempty"`
-	// PostKubeadm hooks run after `kubeadm`
+	// PostKubeadm hooks run immediately after `kubeadm`
 	PostKubeadm []LifecycleHook `json:"postKubeadm,omitempty"`
+	// PostSetup hooks run after any standard `kind` setup on the node
+	PostSetup []LifecycleHook `json:"postSetup,omitempty"`
 }
 
 // LifecycleHook represents a command to run at points in the node lifecycle
@@ -54,7 +56,9 @@ type LifecycleHook struct {
 	// Name is used to improve logging (optional)
 	Name string `json:"name,omitempty"`
 	// Command is the command to run on the node
-	Command string `json:"command"`
-	// Args are the arguments to the command (optional)
-	Args []string `json:"args,omitempty"`
+	Command []string `json:"command"`
+	// MustSucceed - if true then the hook / command failing will cause
+	// cluster creation to fail, otherwise the error will just be logged and
+	// the boot process will continue
+	MustSucceed bool `json:"mustSucceed,omitempty"`
 }

pkg/cluster/config/validate.go

@@ -34,7 +34,7 @@ func (c *Config) Validate() error {
 	}
 	if c.NodeLifecycle != nil {
 		for _, hook := range c.NodeLifecycle.PreBoot {
-			if hook.Command == "" {
+			if len(hook.Command) == 0 {
 				errs = append(errs, fmt.Errorf(
 					"preBoot hooks must set command to a non-empty value",
 				))
@@ -44,7 +44,7 @@ func (c *Config) Validate() error {
 			}
 		}
 		for _, hook := range c.NodeLifecycle.PreKubeadm {
-			if hook.Command == "" {
+			if len(hook.Command) == 0 {
 				errs = append(errs, fmt.Errorf(
 					"preKubeadm hooks must set command to a non-empty value",
 				))
@@ -54,7 +54,7 @@ func (c *Config) Validate() error {
 			}
 		}
 		for _, hook := range c.NodeLifecycle.PostKubeadm {
-			if hook.Command == "" {
+			if len(hook.Command) == 0 {
 				errs = append(errs, fmt.Errorf(
 					"postKubeadm hooks must set command to a non-empty value",
 				))

pkg/cluster/config/validate_test.go

@@ -45,7 +45,7 @@ func TestConfigValidate(t *testing.T) {
 				cfg.NodeLifecycle = &NodeLifecycle{
 					PreBoot: []LifecycleHook{
 						{
-							Command: "",
+							Command: []string{},
 						},
 					},
 				}
@@ -61,7 +61,7 @@ func TestConfigValidate(t *testing.T) {
 					PreKubeadm: []LifecycleHook{
 						{
 							Name:    "pull an image",
-							Command: "",
+							Command: []string{},
 						},
 					},
 				}
@@ -77,7 +77,7 @@ func TestConfigValidate(t *testing.T) {
 					PostKubeadm: []LifecycleHook{
 						{
 							Name:    "pull an image",
-							Command: "",
+							Command: []string{},
 						},
 					},
 				}

pkg/cluster/node.go

@@ -27,9 +27,12 @@ import (
 	"strings"
 	"time"
 
+	"github.com/sirupsen/logrus"
+
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 
+	"k8s.io/test-infra/kind/pkg/cluster/config"
 	"k8s.io/test-infra/kind/pkg/cluster/kubeadm"
 	"k8s.io/test-infra/kind/pkg/exec"
 )
@@ -105,6 +108,28 @@ func (nh *nodeHandle) Run(command string, args ...string) error {
 	return cmd.Run()
 }
 
+// RunHook runs a LifecycleHook on the node
+// It will only return an error if hook.MustSucceed is true
+func (nh *nodeHandle) RunHook(hook *config.LifecycleHook, phase string) error {
+	logger := logrus.WithFields(logrus.Fields{
+		"node":  nh.nameOrID,
+		"phase": phase,
+	})
+	if hook.Name != "" {
+		logger.Infof("Running LifecycleHook \"%s\" ...", hook.Name)
+	} else {
+		logger.Info("Running LifecycleHook ...")
+	}
+	if err := nh.Run(hook.Command[0], hook.Command[1:]...); err != nil {
+		if hook.MustSucceed {
+			logger.WithError(err).Error("LifecycleHook failed")
+			return err
+		}
+		logger.WithError(err).Warn("LifecycleHook failed, continuing ...")
+	}
+	return nil
+}
+
 // CombinedOutputLines execs command, args... on the node, returning the output lines
 func (nh *nodeHandle) CombinedOutputLines(command string, args ...string) ([]string, error) {
 	cmd := exec.Command("docker", "exec")