better cluster boot, more build types

README.md

@@ -1,9 +1,10 @@
 <!--TODO(bentheelder): fill this in much more thoroughly-->
 # `kind` - **K**ubernetes **IN** **D**ocker
 
-## WARNING: `kind` is still a work in progress!
+## WARNING: `kind` is still a work in progress! See [docs/todo.md](./docs/todo.md)
 
 `kind` is a toolset for running local Kubernetes clusters using Docker container "nodes".
+`kind` is designed to be suitable for testing Kubernetes, initally targetting the conformance suite.
 
 It consists of:
  - Go [packages](./pkg) implementing [cluster creation](./pkg/cluster), [image build](./pkg/build), etc.
@@ -17,7 +18,7 @@ For more details see [the design documentation](./docs/design.md).
 
 ## Building
 
-You can build `kind` with `go install ./cmd/kind` or `bazel build //kind/cmd/kind`.
+You can build `kind` with `go install k8s.io/test-infra/kind/cmd/kind` or `bazel build //kind/cmd/kind`.
 
 ## Usage
 
@@ -29,7 +30,9 @@ For more usage, run `kind --help` or `kind [command] --help`.
 
 ## Advanced
 
-`kind build image --source=./kind/images/node` will build the node image
+`kind build base` will build the base image.
+
+`kind build node` will build the node image.
 
 ## Community, discussion, contribution, and support
 

cmd/kind/cmd/build/base/base.go

@@ -28,6 +28,7 @@ type flags struct {
 	Source string
 }
 
+// NewCommand returns a new cobra.Command for building the base image
 func NewCommand() *cobra.Command {
 	flags := &flags{}
 	cmd := &cobra.Command{

cmd/kind/cmd/build/node/node.go

@@ -19,15 +19,18 @@ package node
 import (
 	"os"
 
+	"github.com/golang/glog"
 	"github.com/spf13/cobra"
 
 	"k8s.io/test-infra/kind/pkg/build"
 )
 
 type flags struct {
-	Source string
+	Source    string
+	BuildType string
 }
 
+// NewCommand returns a new cobra.Command for building the node image
 func NewCommand() *cobra.Command {
 	flags := &flags{}
 	cmd := &cobra.Command{
@@ -39,16 +42,20 @@ func NewCommand() *cobra.Command {
 			run(flags, cmd, args)
 		},
 	}
-	cmd.Flags().StringVar(&flags.Source, "source", "", "path to the base image sources")
+	cmd.Flags().StringVar(&flags.BuildType, "type", "docker", "build type, one of [bazel, docker, apt]")
 	return cmd
 }
 
 func run(flags *flags, cmd *cobra.Command, args []string) {
 	// TODO(bentheelder): make this more configurable
-	ctx := build.NewNodeImageBuildContext()
-	ctx.SourceDir = flags.Source
-	err := ctx.Build()
+	ctx, err := build.NewNodeImageBuildContext(flags.BuildType)
 	if err != nil {
+		glog.Errorf("Error creating build context: %v", err)
+		os.Exit(-1)
+	}
+	err = ctx.Build()
+	if err != nil {
+		glog.Errorf("Error building node image: %v", err)
 		os.Exit(-1)
 	}
 }

docs/todo.md

@@ -0,0 +1,37 @@
+# TODO
+
+A non-exhaustive list of tasks (in no-particular order) includes:
+- [x] basic single "node" clusters
+- [x] multiple clusters per host / named clusters
+- [ ] multi-node clusters
+- [x] support for multiple kubernetes builds:
+  - [x] bazel build from source
+  - [x] docker / make build from source
+  - [x] apt (upstream / official release packages)
+    - [ ] support for selecting a non-default package version
+- [ ] kubetest ingregration [WIP]
+  - [ ] point existing ["dind"](https://github.com/kubernetes/test-infra/tree/master/dind) integration here once complete
+- [ ] improved logging and error handling
+- [ ] continuous integration
+- [ ] publish pre-built images to a registry
+- [ ] fake out all internals and unit test [WIP]
+- [ ] pre-load images that are not from the build / possibly build more images
+  - [ ] etcd
+  - [ ] overlay network images?
+- [ ] support multiple overlay networks
+- [ ] support advanced configuration via config file
+  - [ ] kubeadm config template override
+- [ ] more advanced network configuration (not docker0)
+- [ ] support for other CRI within the "node" containers (containerd, cri-o)
+- [ ] switch from `exec.Command("docker", ...)` to the Docker client library
+
+# Wishlist
+
+Longer term / continually appealing items:
+
+- Improved documentation
+- Support for architectures / platforms other than linux / amd64 for the node images
+- Support for client platforms other than docker on linux / docker for mac
+- Less priviliged containers or sharing a CRI via something like [containerd namespaces](https://github.com/containerd/containerd/blob/master/docs/namespaces.md), generally
+ better isolation
+- HA kubeadm / multiple control plane nodes

images/node/Dockerfile

@@ -1,19 +0,0 @@
-# TODO(bentheelder): place these in a registry
-ARG BASE_IMAGE="kind-base"
-FROM ${BASE_IMAGE}
-
-# copy all artifacts provided by the build tooling into the image
-COPY ./files/ /kind/
-
-RUN du -h /kind
-
-# install all the debs if there are any
-RUN [ ! -d /kind/debs/ ] || \
-    dpkg -i /kind/debs/*.deb && \
-    rm -rf \
-        /kind/debs/*.deb \
-        /var/cache/debconf/* \
-        /var/lib/apt/lists/* \
-        /var/log/*
-
-RUN du -h /kind

images/node/README.md

@@ -0,0 +1,10 @@
+## images/node
+
+See: [./../../pkg/build/node_image.go](./../../pkg.build/node_image.go), this
+image is built programmatically with docker run / exec / commit for performance
+reasons with large artifacts.
+
+Roughly this image is [the base image](./../base), with the addition of:
+ - installing the Kubernetes packages / binaries
+ - placing the Kubernetes docker images in /kind/images/*.tar
+ - placing a file in /kind/version containing the Kubernetes semver

pkg/build/doc.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package build implements functionality to build the kind images
+package build

pkg/build/files.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 
 	"k8s.io/test-infra/kind/pkg/exec"
 )
@@ -31,7 +32,9 @@ func TempDir(dir, prefix string) (name string, err error) {
 	if err != nil {
 		return "", err
 	}
-	if runtime.GOOS == "darwin" {
+	// on macOS $TMPDIR is typically /var/..., which is not mountable
+	// /private/var/... is the mountable equivilant
+	if runtime.GOOS == "darwin" && strings.HasPrefix(name, "/var/") {
 		name = filepath.Join("/private", name)
 	}
 	return name, nil

pkg/build/kube/aptbits.go

@@ -0,0 +1,89 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kube
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/golang/glog"
+)
+
+// AptBits implements Bits for the official upstream debian packages
+type AptBits struct {
+}
+
+var _ Bits = &AptBits{}
+
+func init() {
+	RegisterNamedBits("apt", NewAptBits)
+}
+
+// NewAptBits returns a new Bits backed by the upstream debian packages
+func NewAptBits(kubeRoot string) (bits Bits, err error) {
+	return &AptBits{}, nil
+}
+
+// Build implements Bits.Build
+// for AptBits this does nothing
+func (b *AptBits) Build() error {
+	return nil
+}
+
+// Paths implements Bits.Paths
+func (b *AptBits) Paths() map[string]string {
+	return map[string]string{}
+}
+
+// Install implements Bits.Install
+func (b *AptBits) Install(install InstallContext) error {
+	// add apt repo
+	addKey := `curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -`
+	addSources := `cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
+deb http://apt.kubernetes.io/ kubernetes-xenial main
+EOF`
+	if err := install.Run("/bin/sh", "-c", addKey); err != nil {
+		glog.Errorf("Adding Kubernetes apt repository failed! %v", err)
+		return err
+	}
+	if err := install.Run("/bin/sh", "-c", addSources); err != nil {
+		glog.Errorf("Adding Kubernetes apt repository failed! %v", err)
+		return err
+	}
+	// install packages
+	if err := install.Run("/bin/sh", "-c", `clean-install kubelet kubeadm kubectl`); err != nil {
+		glog.Errorf("Installing Kubernetes packages failed! %v", err)
+		return err
+	}
+	// get version to version file
+	lines, err := install.CombinedOutputLines("/bin/sh", "-c", `kubelet --version`)
+	if err != nil {
+		glog.Errorf("Failed to get Kubernetes version! %v", err)
+		return err
+	}
+	// the output should be one line of the form `Kubernetes ${VERSION}`
+	if len(lines) != 1 {
+		glog.Errorf("Failed to parse Kubernetes version with unexpected output: %v", lines)
+		return fmt.Errorf("failed to parse Kubernetes version")
+	}
+	version := strings.SplitN(lines[0], " ", 2)[1]
+	if err := install.Run("/bin/sh", "-c", fmt.Sprintf(`echo "%s" >> /kind/version`, version)); err != nil {
+		glog.Errorf("Failed to get Kubernetes version! %v", err)
+		return err
+	}
+	return nil
+}

pkg/build/kube/bazelbuildbits.go

@@ -0,0 +1,127 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kube
+
+import (
+	"os"
+	"path"
+	"path/filepath"
+
+	"github.com/golang/glog"
+
+	"k8s.io/test-infra/kind/pkg/exec"
+)
+
+// BazelBuildBits implements Bits for a local Bazel build
+type BazelBuildBits struct {
+	kubeRoot string
+	paths    map[string]string
+}
+
+var _ Bits = &BazelBuildBits{}
+
+func init() {
+	RegisterNamedBits("bazel", NewBazelBuildBits)
+}
+
+// NewBazelBuildBits returns a new Bits backed by bazel build,
+// given kubeRoot, the path to the kubernetes source directory
+func NewBazelBuildBits(kubeRoot string) (bits Bits, err error) {
+	// https://docs.bazel.build/versions/master/output_directories.html
+	binDir := filepath.Join(kubeRoot, "bazel-bin")
+	buildDir := filepath.Join(binDir, "build")
+	bits = &BazelBuildBits{
+		kubeRoot: kubeRoot,
+		paths: map[string]string{
+			// debians
+			filepath.Join(buildDir, "debs", "kubeadm.deb"):        "debs/kubeadm.deb",
+			filepath.Join(buildDir, "debs", "kubelet.deb"):        "debs/kubelet.deb",
+			filepath.Join(buildDir, "debs", "kubectl.deb"):        "debs/kubectl.deb",
+			filepath.Join(buildDir, "debs", "kubernetes-cni.deb"): "debs/kubernetes-cni.deb",
+			filepath.Join(buildDir, "debs", "cri-tools.deb"):      "debs/cri-tools.deb",
+			// docker images
+			filepath.Join(buildDir, "kube-apiserver.tar"):          "images/kube-apiserver.tar",
+			filepath.Join(buildDir, "kube-controller-manager.tar"): "images/kube-controller-manager.tar",
+			filepath.Join(buildDir, "kube-scheduler.tar"):          "images/kube-scheduler.tar",
+			filepath.Join(buildDir, "kube-proxy.tar"):              "images/kube-proxy.tar",
+			// version files
+			filepath.Join(kubeRoot, "_output", "git_version"): "version",
+		},
+	}
+	return bits, nil
+}
+
+// Build implements Bits.Build
+func (b *BazelBuildBits) Build() error {
+	// TODO(bentheelder): support other modes of building
+	// cd to k8s source
+	cwd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+	os.Chdir(b.kubeRoot)
+	// make sure we cd back when done
+	defer os.Chdir(cwd)
+
+	// capture version info
+	buildVersionFile(b.kubeRoot)
+	return nil
+
+	// build artifacts
+	cmd := exec.Command("bazel", "build")
+	cmd.Args = append(cmd.Args,
+		// TODO(bentheelder): we assume linux amd64, but we could select
+		// this based on Arch etc. throughout, this flag supports GOOS/GOARCH
+		"--platforms=@io_bazel_rules_go//go/toolchain:linux_amd64",
+		// we want the debian packages
+		"//build/debs:debs",
+		// and the docker images
+		//"//cluster/images/hyperkube:hyperkube.tar",
+		"//build:docker-artifacts",
+	)
+	cmd.Debug = true
+	cmd.InheritOutput = true
+	return cmd.Run()
+}
+
+// Paths implements Bits.Paths
+func (b *BazelBuildBits) Paths() map[string]string {
+	// TODO(bentheelder): maybe copy the map before returning /shrug
+	return b.paths
+}
+
+// Install implements Bits.Install
+func (b *BazelBuildBits) Install(install InstallContext) error {
+	base := install.BasePath()
+
+	debs := path.Join(base, "debs", "*.deb")
+
+	if err := install.Run("/bin/sh", "-c", "dpkg -i "+debs); err != nil {
+		glog.Errorf("Image install failed! %v", err)
+		return err
+	}
+
+	if err := install.Run("/bin/sh", "-c",
+		"rm -rf /kind/bits/debs/*.deb"+
+			" /var/cache/debconf/* /var/lib/apt/lists/* /var/log/*kg",
+	); err != nil {
+		glog.Errorf("Image install failed! %v", err)
+		return err
+	}
+
+	return nil
+}

pkg/build/kube/bits.go

@@ -0,0 +1,93 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kube
+
+import (
+	"fmt"
+	"sync"
+)
+
+// Bits provides the locations of Kubernetes Binaries / Images
+// needed on the cluster nodes
+// Implementations should be registered with RegisterNamedBits
+type Bits interface {
+	// Build returns any errors encountered while building it Kubernetes.
+	// Some implementations (upstream binaries) may use this step to obtain
+	// an existing build isntead
+	Build() error
+	// Paths returns a map of path on host machine to desired path in the image
+	// These paths will be populated in the image relative to some base path,
+	// obtainable by NodeInstall.BasePath()
+	// Note: if Images are populated in iamges/, the cluster provisioning
+	// will load these prior to calling kubeadm
+	Paths() map[string]string
+	// Install should install the built sources on the node, assuming paths
+	// have been populated
+	Install(InstallContext) error
+}
+
+// InstallContext should be implemented by users of Bits
+// to allow installing the bits in a Docker image
+type InstallContext interface {
+	// Returns the base path Paths() were populated relative to
+	BasePath() string
+	// Run execs (cmd, ...args) in the build container and returns error
+	Run(string, ...string) error
+	// CombinedOutputLines is like Run but returns the output lines
+	CombinedOutputLines(string, ...string) ([]string, error)
+}
+
+// NewNamedBits returns a new Bits by named implementation
+// currently this includes:
+// "bazel" -> NewBazelBuildBits(kubeRoot)
+// "docker" or "make" -> NewDockerBuildBits(kubeRoot)
+// "apt" -> NewAptBits(kubeRoot)
+func NewNamedBits(name string, kubeRoot string) (bits Bits, err error) {
+	bitsImpls.Lock()
+	fn, ok := bitsImpls.impls[name]
+	bitsImpls.Unlock()
+	if !ok {
+		return nil, fmt.Errorf("no Bits implementation with name: %s", name)
+	}
+	return fn(kubeRoot)
+}
+
+// RegisterNamedBits registers a new named Bits implementation for use from
+// NewNamedBits
+func RegisterNamedBits(name string, fn func(string) (Bits, error)) {
+	bitsImpls.Lock()
+	bitsImpls.impls[name] = fn
+	bitsImpls.Unlock()
+}
+
+// NamedBitsRegistered returns true if name is in the registry backing
+// NewNamedBits
+func NamedBitsRegistered(name string) bool {
+	var ok bool
+	bitsImpls.Lock()
+	_, ok = bitsImpls.impls[name]
+	bitsImpls.Unlock()
+	return ok
+}
+
+// internal registry of named bits implementations
+var bitsImpls = struct {
+	impls map[string]func(string) (Bits, error)
+	sync.Mutex
+}{
+	impls: map[string]func(string) (Bits, error){},
+}

pkg/build/kube/doc.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package kube implements functionality to build Kubernetes for the purposes
+// of installing into the kind node image
+package kube

pkg/build/kube/dockerbuildbits.go

@@ -0,0 +1,189 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kube
+
+import (
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"k8s.io/test-infra/kind/pkg/exec"
+)
+
+// DockerBuildBits implements Bits for a local docke-ized make / bash build
+type DockerBuildBits struct {
+	kubeRoot string
+	paths    map[string]string
+}
+
+var _ Bits = &DockerBuildBits{}
+
+func init() {
+	RegisterNamedBits("docker", NewDockerBuildBits)
+	RegisterNamedBits("make", NewDockerBuildBits)
+}
+
+// NewDockerBuildBits returns a new Bits backed by the docker-ized build,
+// given kubeRoot, the path to the kubernetes source directory
+func NewDockerBuildBits(kubeRoot string) (bits Bits, err error) {
+	// https://docs.Docker.build/versions/master/output_directories.html
+	binDir := filepath.Join(kubeRoot,
+		"_output", "dockerized", "bin", "linux", "amd64",
+	)
+	imageDir := filepath.Join(kubeRoot,
+		"_output", "release-images", "amd64",
+	)
+	bits = &DockerBuildBits{
+		kubeRoot: kubeRoot,
+		paths: map[string]string{
+			// binaries (hyperkube)
+			filepath.Join(binDir, "kubeadm"): "bin/kubeadm",
+			filepath.Join(binDir, "kubelet"): "bin/kubelet",
+			filepath.Join(binDir, "kubectl"): "bin/kubectl",
+			// docker images
+			filepath.Join(imageDir, "kube-apiserver.tar"):          "images/kube-apiserver.tar",
+			filepath.Join(imageDir, "kube-controller-manager.tar"): "images/kube-controller-manager.tar",
+			filepath.Join(imageDir, "kube-scheduler.tar"):          "images/kube-scheduler.tar",
+			filepath.Join(imageDir, "kube-proxy.tar"):              "images/kube-proxy.tar",
+			// version files
+			filepath.Join(kubeRoot, "_output", "git_version"): "version",
+			// borrow kubelet service files from bazel debians
+			// TODO(bentheelder): probably we should use our own config instead :-)
+			filepath.Join(kubeRoot, "build", "debs", "kubelet.service"): "systemd/kubelet.service",
+			filepath.Join(kubeRoot, "build", "debs", "10-kubeadm.conf"): "systemd/10-kubeadm.conf",
+		},
+	}
+	return bits, nil
+}
+
+// Build implements Bits.Build
+func (b *DockerBuildBits) Build() error {
+	// TODO(bentheelder): support other modes of building
+	// cd to k8s source
+	cwd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+	os.Chdir(b.kubeRoot)
+	// make sure we cd back when done
+	defer os.Chdir(cwd)
+
+	// capture version info
+	err = buildVersionFile(b.kubeRoot)
+	if err != nil {
+		return err
+	}
+
+	// build binaries
+	cmd := exec.Command("build/run.sh", "make", "all")
+	what := []string{
+		"cmd/kubeadm",
+		"cmd/kubectl",
+		"cmd/kubelet",
+		"cmd/cloud-controller-manager",
+		"cmd/kube-apiserver",
+		"cmd/kube-controller-manager",
+		"cmd/kube-scheduler",
+		"cmd/kube-proxy",
+	}
+	cmd.Args = append(cmd.Args,
+		"WHAT="+strings.Join(what, " "), "KUBE_BUILD_PLATFORMS=linux/amd64",
+	)
+	cmd.Env = append(cmd.Env, os.Environ()...)
+	cmd.Env = append(cmd.Env, "KUBE_VERBOSE=0")
+	cmd.Debug = true
+	cmd.InheritOutput = true
+	err = cmd.Run()
+	if err != nil {
+		return errors.Wrap(err, "failed to build binaries")
+	}
+
+	// TODO(bentheelder): this is perhaps a bit overkill
+	// the build will fail if they are already present though
+	// We should find what `make quick-release` does and mimic that
+	err = os.RemoveAll(filepath.Join(
+		".", "_output", "release-images", "amd64",
+	))
+	if err != nil {
+		return errors.Wrap(err, "failed to remove old release-images")
+	}
+
+	// build images
+	// TODO(bentheelder): there has to be a better way to do this, but the
+	// closest seems to be make quick-release, which builds more than we need
+	buildImages := []string{
+		"source build/common.sh;",
+		"source hack/lib/version.sh;",
+		"source build/lib/release.sh;",
+		"kube::version::get_version_vars;",
+		`kube::release::create_docker_images_for_server "${LOCAL_OUTPUT_ROOT}/dockerized/bin/linux/amd64" "amd64"`,
+	}
+	cmd = exec.Command("bash", "-c", strings.Join(buildImages, " "))
+	cmd.Env = append(cmd.Env, os.Environ()...)
+	cmd.Env = append(cmd.Env, "KUBE_BUILD_HYPERKUBE=n")
+	cmd.Debug = true
+	cmd.InheritOutput = true
+	err = cmd.Run()
+	if err != nil {
+		return errors.Wrap(err, "failed to build images")
+	}
+
+	return nil
+}
+
+// Paths implements Bits.Paths
+func (b *DockerBuildBits) Paths() map[string]string {
+	// TODO(bentheelder): maybe copy the map before returning /shrug
+	return b.paths
+}
+
+// Install implements Bits.Install
+func (b *DockerBuildBits) Install(install InstallContext) error {
+	kindBinDir := path.Join(install.BasePath(), "bin")
+
+	// symlink the kubernetes binaries into $PATH
+	binaries := []string{"kubeadm", "kubelet", "kubectl"}
+	for _, binary := range binaries {
+		if err := install.Run("ln", "-s",
+			path.Join(kindBinDir, binary),
+			path.Join("/usr/bin/", binary),
+		); err != nil {
+			return errors.Wrap(err, "failed to symlink binaries")
+		}
+	}
+
+	// enable the kubelet service
+	kubeletService := path.Join(install.BasePath(), "systemd/kubelet.service")
+	if err := install.Run("systemctl", "enable", kubeletService); err != nil {
+		return errors.Wrap(err, "failed to enable kubelet service")
+	}
+
+	// setup the kubelet dropin
+	kubeletDropinSource := path.Join(install.BasePath(), "systemd/10-kubeadm.conf")
+	kubeletDropin := "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf"
+	if err := install.Run("mkdir", "-p", path.Dir(kubeletDropin)); err != nil {
+		return errors.Wrap(err, "failed to configure kubelet service")
+	}
+	if err := install.Run("cp", kubeletDropinSource, kubeletDropin); err != nil {
+		return errors.Wrap(err, "failed to configure kubelet service")
+	}
+
+	return nil
+}

pkg/build/kube/source.go

@@ -14,33 +14,21 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package build
+package kube
 
 import (
 	"fmt"
-	gobuild "go/build"
+	"go/build"
 )
 
-// KubeBits provides the locations of Kubernetes Binaries / Images
-// needed on the cluster nodes
-type KubeBits interface {
-	// Paths returns a map of path on host to desired path in the image
-	Paths() map[string]string
-}
+// ImportPath is the canonical import path for the kubernetes root package
+// this is used by FindSource
+const ImportPath = "k8s.io/kubernetes"
 
-// NodeInstall should be implemented by users of KubeBitsProvider
-// to allow installing the bits
-type NodeInstall interface {
-	// RunOnNode execs (cmd, ...args) on a node and returns error
-	RunOnNode(string, ...string) error
-}
-
-const kubeImportPath = "k8s.io/kubernetes"
-
-// FindKubeSource attempts to locate a kubernetes checkout using go's build package
-func FindKubeSource() (root string, err error) {
+// FindSource attempts to locate a kubernetes checkout using go's build package
+func FindSource() (root string, err error) {
 	// look up the source the way go build would
-	pkg, err := gobuild.Default.Import(kubeImportPath, ".", gobuild.FindOnly)
+	pkg, err := build.Default.Import(ImportPath, ".", build.FindOnly)
 	if err == nil && maybeKubeDir(pkg.Dir) {
 		return pkg.Dir, nil
 	}

pkg/build/kube/version.go

@@ -0,0 +1,64 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kube
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"k8s.io/test-infra/kind/pkg/exec"
+)
+
+// buildVersionFile creates a file for the kubernetes git version in
+// ./_output/version based on hack/print-workspace-status.sh,
+// these are built into the node image and consumed by the cluster tooling
+func buildVersionFile(kubeRoot string) error {
+	cwd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+	os.Chdir(kubeRoot)
+	// make sure we cd back when done
+	defer os.Chdir(cwd)
+
+	// get the version output
+	cmd := exec.Command("hack/print-workspace-status.sh")
+	cmd.Debug = true
+	output, err := cmd.CombinedOutputLines()
+	if err != nil {
+		return err
+	}
+	outputDir := filepath.Join(kubeRoot, "_output")
+	// parse it, and populate it into _output/git_version
+	for _, line := range output {
+		parts := strings.SplitN(line, " ", 2)
+		if len(parts) != 2 {
+			return fmt.Errorf("could not parse kubernetes version")
+		}
+		if parts[0] == "gitVersion" {
+			ioutil.WriteFile(
+				filepath.Join(outputDir, "git_version"),
+				[]byte(parts[1]),
+				0777,
+			)
+		}
+	}
+	return nil
+}

pkg/build/localbuildbits.go

@@ -1,52 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package build
-
-import "path/filepath"
-
-// BazelBuildBits implements KubeBits for a local Bazel build
-type BazelBuildBits struct {
-	paths map[string]string
-}
-
-var _ KubeBits = &BazelBuildBits{}
-
-func (l *BazelBuildBits) Paths() map[string]string {
-	// TODO(bentheelder): maybe copy the map before returning /shrug
-	return l.paths
-}
-
-func NewBazelBuildBits(kubeRoot string) (bits KubeBits, err error) {
-	// https://docs.bazel.build/versions/master/output_directories.html
-	binDir := filepath.Join(kubeRoot, "bazel-bin")
-	bits = &BazelBuildBits{
-		paths: map[string]string{
-			// debians
-			filepath.Join(binDir, "build", "debs", "kubeadm.deb"):        "debs/kubeadm.deb",
-			filepath.Join(binDir, "build", "debs", "kubelet.deb"):        "debs/kubelet.deb",
-			filepath.Join(binDir, "build", "debs", "kubectl.deb"):        "debs/kubectl.deb",
-			filepath.Join(binDir, "build", "debs", "kubernetes-cni.deb"): "debs/kubernetes-cni.deb",
-			filepath.Join(binDir, "build", "debs", "cri-tools.deb"):      "debs/cri-tools.deb",
-			// docker images
-			filepath.Join(binDir, "build", "kube-proxy.tar"):              "images/kube-proxy.tar",
-			filepath.Join(binDir, "build", "kube-controller-manager.tar"): "images/kube-controller-manager.tar",
-			filepath.Join(binDir, "build", "kube-scheduler.tar"):          "images/kube-scheduler.tar",
-			filepath.Join(binDir, "build", "kube-apiserver.tar"):          "images/kube-apiserver.tar",
-		},
-	}
-	return bits, nil
-}

pkg/build/node_image.go

@@ -14,8 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// Package build implements functionality to build the kind images
-// TODO(bentheelder): and k8s
 package build
 
 import (
@@ -28,6 +26,7 @@ import (
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 
+	"k8s.io/test-infra/kind/pkg/build/kube"
 	"k8s.io/test-infra/kind/pkg/build/sources"
 	"k8s.io/test-infra/kind/pkg/exec"
 )
@@ -39,35 +38,46 @@ type NodeImageBuildContext struct {
 	ImageTag  string
 	Arch      string
 	BaseImage string
+	KubeRoot  string
+	Bits      kube.Bits
 }
 
 // NewNodeImageBuildContext creates a new NodeImageBuildContext with
 // default configuration
-func NewNodeImageBuildContext() *NodeImageBuildContext {
+func NewNodeImageBuildContext(mode string) (ctx *NodeImageBuildContext, err error) {
+	kubeRoot := ""
+	// apt should not fail on finding kube root as it does not use it
+	if mode != "apt" {
+		kubeRoot, err = kube.FindSource()
+		if err != nil {
+			return nil, fmt.Errorf("error finding kuberoot: %v", err)
+		}
+	}
+	bits, err := kube.NewNamedBits(mode, kubeRoot)
+	if err != nil {
+		return nil, err
+	}
 	return &NodeImageBuildContext{
 		ImageTag:  "kind-node",
 		Arch:      "amd64",
 		BaseImage: "kind-base",
-	}
+		KubeRoot:  kubeRoot,
+		Bits:      bits,
+	}, nil
 }
 
 // Build builds the cluster node image, the sourcedir must be set on
 // the NodeImageBuildContext
 func (c *NodeImageBuildContext) Build() (err error) {
-	// get k8s source
-	kubeRoot, err := FindKubeSource()
-	if err != nil {
-		return errors.Wrap(err, "could not find kubernetes source")
-	}
-
 	// ensure kubernetes build is up to date first
 	glog.Infof("Starting to build Kubernetes")
-	//c.buildKube(kubeRoot)
+	if err = c.Bits.Build(); err != nil {
+		glog.Errorf("Failed to build Kubernetes: %v", err)
+		return errors.Wrap(err, "failed to build kubernetes")
+	}
 	glog.Infof("Finished building Kubernetes")
-	// TODO(bentheelder): allow other types of bits
-	bits, err := NewBazelBuildBits(kubeRoot)
 
-	// create tempdir to build in
+	// create tempdir to build the image in
 	tmpDir, err := TempDir("", "kind-node-image")
 	if err != nil {
 		return err
@@ -96,48 +106,25 @@ func (c *NodeImageBuildContext) Build() (err error) {
 	glog.Infof("Building node image in: %s", buildDir)
 
 	// populate the kubernetes artifacts first
-	if err := c.populateBits(buildDir, bits); err != nil {
+	if err := c.populateBits(buildDir); err != nil {
 		return err
 	}
 
-	// then the actual docker image
+	// then the perform the actual docker image build
 	return c.buildImage(buildDir)
 }
 
-func (c *NodeImageBuildContext) buildKube(kubeRoot string) error {
-	// TODO(bentheelder): support other modes of building
-	// cd to k8s source
-	cwd, err := os.Getwd()
-	if err != nil {
-		return err
+func (c *NodeImageBuildContext) populateBits(buildDir string) error {
+	// always create bits dir
+	bitsDir := path.Join(buildDir, "bits")
+	if err := os.Mkdir(bitsDir, 0777); err != nil {
+		return errors.Wrap(err, "failed to make bits dir")
 	}
-	os.Chdir(kubeRoot)
-	// make sure we cd back when done
-	defer os.Chdir(cwd)
-
-	// TODO(bentheelder): move this out and next to the KubeBits impl
-	cmd := exec.Command("bazel", "build")
-	cmd.Args = append(cmd.Args,
-		// TODO(bentheelder): we assume linux amd64, but we could select
-		// this based on Arch etc. throughout, this flag supports GOOS/GOARCH
-		"--platforms=@io_bazel_rules_go//go/toolchain:linux_amd64",
-		// we want the debian packages
-		"//build/debs:debs",
-		// and the docker images
-		"//build:docker-artifacts",
-	)
-
-	cmd.Debug = true
-	cmd.InheritOutput = true
-	return cmd.Run()
-}
-
-func (c *NodeImageBuildContext) populateBits(buildDir string, bits KubeBits) error {
 	// copy all bits from their source path to where we will COPY them into
 	// the dockerfile, see images/node/Dockerfile
-	bitPaths := bits.Paths()
+	bitPaths := c.Bits.Paths()
 	for src, dest := range bitPaths {
-		realDest := path.Join(buildDir, "files", dest)
+		realDest := path.Join(bitsDir, dest)
 		if err := copyFile(src, realDest); err != nil {
 			return errors.Wrap(err, "failed to copy build artifact")
 		}
@@ -148,6 +135,33 @@ func (c *NodeImageBuildContext) populateBits(buildDir string, bits KubeBits) err
 // BuildContainerLabelKey is applied to each build container
 const BuildContainerLabelKey = "io.k8s.test-infra.kind-build"
 
+// private kube.InstallContext implementation, local to the image build
+type installContext struct {
+	basePath    string
+	containerID string
+}
+
+var _ kube.InstallContext = &installContext{}
+
+func (ic *installContext) BasePath() string {
+	return ic.basePath
+}
+
+func (ic *installContext) Run(command string, args ...string) error {
+	cmd := exec.Command("docker", "exec", ic.containerID, command)
+	cmd.Args = append(cmd.Args, args...)
+	cmd.Debug = true
+	cmd.InheritOutput = true
+	return cmd.Run()
+}
+
+func (ic *installContext) CombinedOutputLines(command string, args ...string) ([]string, error) {
+	cmd := exec.Command("docker", "exec", ic.containerID, command)
+	cmd.Args = append(cmd.Args, args...)
+	cmd.Debug = true
+	return cmd.CombinedOutputLines()
+}
+
 func (c *NodeImageBuildContext) buildImage(dir string) error {
 	// build the image, tagged as tagImageAs, using the our tempdir as the context
 	glog.Info("Starting image build ...")
@@ -155,6 +169,8 @@ func (c *NodeImageBuildContext) buildImage(dir string) error {
 	// NOTE: we are using docker run + docker commit so we can install
 	// debians without permanently copying them into the image.
 	// if docker gets proper squash support, we can rm them instead
+	// This also allows the KubeBit implementations to perform programmatic
+	// isntall in the image
 	containerID, err := c.createBuildContainer(dir)
 	if err != nil {
 		glog.Errorf("Image build Failed! %v", err)
@@ -176,27 +192,23 @@ func (c *NodeImageBuildContext) buildImage(dir string) error {
 	}
 
 	// make artifacts directory
-	if err = execInBuild("mkdir", "-p", "/kind/bits"); err != nil {
+	if err = execInBuild("mkdir", "/kind/"); err != nil {
 		glog.Errorf("Image build Failed! %v", err)
 		return err
 	}
 
 	// copy artifacts in
-	if err = execInBuild("rsync", "-r", "/build/files/", "/kind/bits/"); err != nil {
+	if err = execInBuild("rsync", "-r", "/build/bits/", "/kind/"); err != nil {
 		glog.Errorf("Image build Failed! %v", err)
 		return err
 	}
 
-	// install debs
-	if err = execInBuild("/bin/sh", "-c", "dpkg -i /kind/bits/debs/*.deb"); err != nil {
-		glog.Errorf("Image build Failed! %v", err)
-		return err
+	// install the kube bits
+	ic := &installContext{
+		basePath:    "/kind/",
+		containerID: containerID,
 	}
-
-	// clean up after debs / remove them, this saves a couple hundred MB
-	if err = execInBuild("/bin/sh", "-c",
-		"rm -rf /kind/bits/debs/*.deb /var/cache/debconf/* /var/lib/apt/lists/* /var/log/*kg",
-	); err != nil {
+	if err = c.Bits.Install(ic); err != nil {
 		glog.Errorf("Image build Failed! %v", err)
 		return err
 	}

pkg/build/sources/generate.go

@@ -15,7 +15,7 @@ limitations under the License.
 */
 
 // Package sources contains the baked in sources kind needs to build.
-// Primarily this includes the node-image dockerfile, which should rarely
+// Primarily this includes the node-image Dockerfile, which should rarely
 // change.
 // These can be overridden with newer files at build-time, see ./../build
 package sources

pkg/cluster/cluster.go

@@ -14,15 +14,19 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// Package cluster implements kind local cluster management
 package cluster
 
 import (
 	"fmt"
+	"io/ioutil"
+	"os"
 	"time"
 
+	"github.com/golang/glog"
+
 	"github.com/pkg/errors"
 
+	"k8s.io/test-infra/kind/pkg/cluster/kubeadm"
 	"k8s.io/test-infra/kind/pkg/exec"
 )
 
@@ -30,7 +34,6 @@ import (
 // kubernetes-in-docker clusters
 type Context struct {
 	config Config
-	// TODO(bentheelder): fill this in
 }
 
 // NewContext returns a new cluster management context with Config config
@@ -46,11 +49,27 @@ func (c *Context) Create() error {
 	if err := c.config.Validate(); err != nil {
 		return err
 	}
-	// create a temp dir to stick kubeconfig in
 
-	// TODO(bentheelder): more advanced provisioning
-	// TODO(bentheelder): multiple nodes
-	return c.provisionNode()
+	// TODO(bentheelder): multiple nodes ...
+	kubeadmConfig, err := c.provisionControlPlane(
+		fmt.Sprintf("kind-%s-control-plane", c.config.Name),
+	)
+
+	// clean up the kubeadm config file
+	// NOTE: in the future we will use this for other nodes first
+	if kubeadmConfig != "" {
+		defer os.Remove(kubeadmConfig)
+	}
+
+	if err != nil {
+		return err
+	}
+
+	println("\nYou can now use the cluster with:\n")
+	println("export KUBECONFIG=\"" + c.config.KubeConfigPath() + "\"")
+	println("kubectl cluster-info\n")
+
+	return nil
 }
 
 // Delete tears down a kubernetes-in-docker cluster
@@ -62,116 +81,138 @@ func (c *Context) Delete() error {
 	return c.deleteNodes(nodes...)
 }
 
-func (c *Context) provisionNode() error {
-	// TODO(bentheelder): multiple nodes...
-	nodeName := "kind-" + c.config.Name + "-control-plane"
+// provisionControlPlane provisions the control plane node
+// and the cluster kubeadm config
+func (c *Context) provisionControlPlane(name string) (kubeadmConfigPath string, err error) {
 	// create the "node" container (docker run, but it is paused, see createNode)
-	if err := c.createNode(nodeName); err != nil {
-		return err
+	node, err := createNode(name, c.config.clusterLabel())
+	if err != nil {
+		return "", err
 	}
 
 	// systemd-in-a-container should have read only /sys
 	// https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/
 	// however, we need other things from `docker run --privileged` ...
 	// and this flag also happens to make /sys rw, amongst other things
-	if err := c.runOnNode(nodeName, []string{
-		"mount", "-o", "remount,ro", "/sys",
-	}); err != nil {
+	if err := node.Run("mount", "-o", "remount,ro", "/sys"); err != nil {
 		// TODO(bentheelder): logging here
 		// TODO(bentheelder): add a flag to retain the broken nodes for debugging
-		c.deleteNodes(nodeName)
-		return err
+		c.deleteNodes(node.nameOrID)
+		return "", err
 	}
-	// TODO(bentheelder): insert other provisioning here
-	// (eg enabling / disabling units, installing kube...)
 
-	// signal the node to boot into systemd
-	if err := c.actuallyStartNode(nodeName); err != nil {
+	// signal the node entrypoint to continue booting into systemd
+	if err := node.SignalStart(); err != nil {
 		// TODO(bentheelder): logging here
-		c.deleteNodes(nodeName)
-		return err
+		// TODO(bentheelder): add a flag to retain the broken nodes for debugging
+		c.deleteNodes(node.nameOrID)
+		return "", err
 	}
 
 	// wait for docker to be ready
-	if !tryUntil(time.Now().Add(time.Second*30), func() bool {
-		out, err := c.outputOnNode(nodeName, []string{"systemctl", "is-active", "docker"})
-		if err != nil {
-			return false
-		}
-		return len(out) == 1 && out[0] == "active"
-	}) {
-		c.deleteNodes(nodeName)
-		return fmt.Errorf("timed out waiting for docker to be ready on node")
+	if !node.WaitForDocker(time.Now().Add(time.Second * 30)) {
+		// TODO(bentheelder): logging here
+		// TODO(bentheelder): add a flag to retain the broken nodes for debugging
+		c.deleteNodes(node.nameOrID)
+		return "", fmt.Errorf("timed out waiting for docker to be ready on node")
 	}
 
-	// run kubeadm init
-	// TODO(bentheelder): configure properly, ensure it uses images we built...
-	if err := c.runOnNode(nodeName, []string{
-		// kubeadm init because this is the control plane node
+	// load the docker image artifacts into the docker daemon
+	node.LoadImages()
+
+	// get installed kubernetes version from the node image
+	kubeVersion, err := node.KubeVersion()
+	if err != nil {
+		// TODO(bentheelder): logging here
+		// TODO(bentheelder): add a flag to retain the broken nodes for debugging
+		c.deleteNodes(node.nameOrID)
+		return "", fmt.Errorf("failed to get kubernetes version from node: %v", err)
+	}
+
+	// create kubeadm config file
+	kubeadmConfig, err := c.createKubeadmConfig("", kubeadm.ConfigData{
+		ClusterName:       c.config.ClusterName(),
+		KubernetesVersion: kubeVersion,
+	})
+
+	// copy the config to the node
+	if err := node.CopyTo(kubeadmConfig, "/kind/kubeadm.conf"); err != nil {
+		// TODO(bentheelder): logging here
+		// TODO(bentheelder): add a flag to retain the broken nodes for debugging
+		c.deleteNodes(node.nameOrID)
+		return kubeadmConfig, errors.Wrap(err, "failed to copy kubeadm config to node")
+	}
+
+	// run kubeadm
+	if err := node.Run(
+		// init because this is the control plane node
 		"kubeadm", "init",
-		// preflight errors are expected, in particular for swap
+		// preflight errors are expected, in particular for swap being enabled
+		// TODO(bentheelder): limit the set of acceptable errors
 		"--ignore-preflight-errors=all",
-		// on docker for mac we have to expose the api server on localhost
-		"--apiserver-cert-extra-sans=localhost",
-	}); err != nil {
-		c.deleteNodes(nodeName)
-		return errors.Wrap(err, "failed to init node with kubeadm")
+		// specify our generated config file
+		"--config=/kind/kubeadm.conf",
+	); err != nil {
+		// TODO(bentheelder): logging here
+		// TODO(bentheelder): add a flag to retain the broken nodes for debugging
+		c.deleteNodes(node.nameOrID)
+		return kubeadmConfig, errors.Wrap(err, "failed to init node with kubeadm")
 	}
 
-	// TODO(bentheelder): apply an overlay network
+	// set up the $KUBECONFIG
+	kubeConfigPath := c.config.KubeConfigPath()
+	if err = node.WriteKubeConfig(kubeConfigPath); err != nil {
+		// TODO(bentheelder): logging here
+		// TODO(bentheelder): add a flag to retain the broken nodes for debugging
+		c.deleteNodes(node.nameOrID)
+		return kubeadmConfig, errors.Wrap(err, "failed to get kubeconfig from node")
+	}
 
-	return nil
-}
+	// TODO(bentheelder): support other overlay networks
+	if err = node.Run(
+		"/bin/sh", "-c",
+		`kubectl apply --kubeconfig=/etc/kubernetes/admin.conf -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')"`,
+	); err != nil {
+		return kubeadmConfig, errors.Wrap(err, "failed to apply overlay network")
+	}
 
-// call `try()`` in a loop until the deadline `until` has passed or `try()`
-// returns true, returns wether try every returned true
-func tryUntil(until time.Time, try func() bool) bool {
-	now := time.Now()
-	for until.After(now) {
-		if try() {
-			return true
+	// if we are only provisioning one node, remove the master taint
+	// https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#master-isolation
+	if c.config.NumNodes == 1 {
+		if err = node.Run(
+			"kubectl", "--kubeconfig=/etc/kubernetes/admin.conf",
+			"taint", "nodes", "--all", "node-role.kubernetes.io/master-",
+		); err != nil {
+			return kubeadmConfig, errors.Wrap(err, "failed to remove master taint")
 		}
 	}
-	return false
+
+	return kubeadmConfig, nil
 }
 
-// createNode `docker run`s the node image, note that due to
-// images/node/entrypoint being the entrypoint, this container will
-// effectively be paused until we call actuallyStartNode(...)
-func (c *Context) createNode(name string) error {
-	// TODO(bentheelder): use config
-	// TODO(bentheelder): logging
-	// TODO(bentheelder): many of these flags should be derived from the config
-	cmd := exec.Command("docker", "run")
-	cmd.Args = append(cmd.Args,
-		"-d", // run the container detached
-		"-t", // we need a pseudo-tty for systemd logs
-		// running containers in a container requires privileged
-		// NOTE: we could try to replicate this with --cap-add, and use less
-		// privileges, but this flag also changes some mounts that are necessary
-		// including some ones docker would otherwise do by default.
-		// for now this is what we want. in the future we may revisit this.
-		"--privileged",
-		"--security-opt", "seccomp=unconfined", // also ignore seccomp
-		"--tmpfs", "/tmp", // various things depend on working /tmp
-		"--tmpfs", "/run", // systemd wants a writable /run
-		// docker in docker needs this, so as not to stack overlays
-		"--tmpfs", "/var/lib/docker:exec",
-		//"-v", "/sys/fs/cgroup:/sys/fs/cgroup:ro",
-		// some k8s things want /lib/modules
-		"-v", "/lib/modules:/lib/modules:ro",
-		"--hostname", name, // make hostname match container name
-		"--name", name, // ... and set the container name
-		// label the node with the cluster ID
-		"--label", c.config.clusterLabel(),
-		// expose API server
-		// TODO(bentheelder): this should probably be configurable
-		"-p", "6443:6443",
-		"kind-node", // use our image, TODO: make this configurable
-	)
-	// TODO(bentheelder): collect output instead of connecting these
-	cmd.InheritOutput = true
-	return cmd.Run()
+// createKubeadmConfig creates the kubeadm config file for the cluster
+// by running data through the template and writing it to a temp file
+// the config file path is returned, this file should be removed later
+func (c *Context) createKubeadmConfig(template string, data kubeadm.ConfigData) (path string, err error) {
+	// create kubeadm config file
+	f, err := ioutil.TempFile("", "")
+	if err != nil {
+		return "", errors.Wrap(err, "failed to create kubeadm config")
+	}
+	path = f.Name()
+	// generate the config contents
+	config, err := kubeadm.Config(template, data)
+	if err != nil {
+		os.Remove(path)
+		return "", err
+	}
+	glog.Infof("Using KubeadmConfig:\n\n%s\n", config)
+	_, err = f.WriteString(config)
+	if err != nil {
+		os.Remove(path)
+		return "", err
+	}
+	return path, nil
 }
 
 func (c *Context) deleteNodes(names ...string) error {
@@ -183,51 +224,6 @@ func (c *Context) deleteNodes(names ...string) error {
 	return cmd.Run()
 }
 
-// runOnNode execs command on the named node
-func (c *Context) runOnNode(nameOrID string, command []string) error {
-	cmd := exec.Command("docker", "exec")
-	cmd.Args = append(cmd.Args,
-		"-t",           // use a tty so we can get output
-		"--privileged", // run with priliges so we can remount etc..
-		nameOrID,       // ... against the "node" container
-	)
-	cmd.Args = append(cmd.Args,
-		command..., // finally, run the command supplied by the user
-	)
-	// TODO(bentheelder): collect output instead of connecting these
-	cmd.InheritOutput = true
-	return cmd.Run()
-}
-
-// outputOnNode execs command on the named node, returning the output lines
-func (c *Context) outputOnNode(nameOrID string, command []string) ([]string, error) {
-	cmd := exec.Command("docker", "exec")
-	cmd.Args = append(cmd.Args,
-		"-t",           // use a tty so we can get output
-		"--privileged", // run with priliges so we can remount etc..
-		nameOrID,       // ... against the "node" container
-	)
-	cmd.Args = append(cmd.Args,
-		command..., // finally, run the command supplied by the user
-	)
-	// TODO(bentheelder): collect output instead of connecting these
-	return cmd.CombinedOutputLines()
-}
-
-// signal our entrypoint (images/node/entrypoint) to boot
-func (c *Context) actuallyStartNode(name string) error {
-	// TODO(bentheelder): use config
-	// TODO(bentheelder): logging
-	cmd := exec.Command("docker", "kill")
-	cmd.Args = append(cmd.Args,
-		"-s", "SIGUSR1",
-		name,
-	)
-	// TODO(bentheelder): collect output instead of connecting these
-	cmd.InheritOutput = true
-	return cmd.Run()
-}
-
 // ListNodes returns the list of container IDs for the "nodes" in the cluster
 func (c *Context) ListNodes(alsoStopped bool) (containerIDs []string, err error) {
 	cmd := exec.Command("docker", "ps")
@@ -237,7 +233,7 @@ func (c *Context) ListNodes(alsoStopped bool) (containerIDs []string, err error)
 		// filter for nodes with the cluster label
 		"--filter", "label="+c.config.clusterLabel(),
 	)
-	// optionally show nodes that are stopped
+	// optionally list nodes that are stopped
 	if alsoStopped {
 		cmd.Args = append(cmd.Args, "-a")
 	}

pkg/cluster/config.go

@@ -19,6 +19,8 @@ package cluster
 import (
 	"bytes"
 	"fmt"
+	"os"
+	"path/filepath"
 	"regexp"
 )
 
@@ -28,19 +30,22 @@ const ClusterLabelKey = "io.k8s.test-infra.kind-cluster"
 // similar to valid docker container names, but since we will prefix
 // and suffix this name, we can relax it a little
 // see Validate() for usage
-var validClusterName = regexp.MustCompile(`^[a-zA-Z0-9_.-]+$`)
+var validNameRE = regexp.MustCompile(`^[a-zA-Z0-9_.-]+$`)
 
 // Config contains cluster options
 type Config struct {
 	// the cluster name
 	Name string
+	// the number of nodes (currently only one is supported)
+	NumNodes int
 	// TODO(bentheelder): fill this in
 }
 
 // NewConfig returns a new cluster config with name
 func NewConfig(name string) Config {
 	return Config{
-		Name: name,
+		Name:     name,
+		NumNodes: 1,
 	}
 }
 
@@ -48,10 +53,17 @@ func NewConfig(name string) Config {
 // with the config, or nil if there are none
 func (c *Config) Validate() error {
 	errs := []error{}
-	if !validClusterName.MatchString(c.Name) {
+	if !validNameRE.MatchString(c.Name) {
 		errs = append(errs, fmt.Errorf(
 			"'%s' is not a valid cluster name, cluster names must match `%s`",
-			c.Name, validClusterName.String(),
+			c.Name, validNameRE.String(),
+		))
+	}
+	// TODO(bentheelder): support multiple nodes
+	if c.NumNodes != 1 {
+		errs = append(errs, fmt.Errorf(
+			"%d nodes requested but only clusters with one node are supported currently",
+			c.NumNodes,
 		))
 	}
 	if len(errs) > 0 {
@@ -87,3 +99,21 @@ func (c ConfigErrors) Errors() []error {
 func (c *Config) clusterLabel() string {
 	return fmt.Sprintf("%s=%s", ClusterLabelKey, c.Name)
 }
+
+// ClusterName returns the Kubernetes cluster name based on the config
+// currently this is .Name prefixed with "kind-"
+func (c *Config) ClusterName() string {
+	return fmt.Sprintf("kind-%s", c.Name)
+}
+
+// KubeConfigPath returns the path to where the Kubeconfig would be placed
+// by kind based on the configuration.
+func (c *Config) KubeConfigPath() string {
+	// TODO(bentheelder): Windows?
+	// configDir matches the standard directory expected by kubectl etc
+	configDir := filepath.Join(os.Getenv("HOME"), ".kube")
+	// note that the file name however does not, we do not want to overwite
+	// the standard config, though in the future we may (?) merge them
+	fileName := fmt.Sprintf("kind-config-%s", c.Name)
+	return filepath.Join(configDir, fileName)
+}

pkg/cluster/doc.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package cluster implements kind kubernetes-in-docker cluster management
+package cluster

pkg/cluster/kubeadm/config.go

@@ -0,0 +1,88 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubeadm
+
+import (
+	"bytes"
+	"strings"
+	"text/template"
+
+	"github.com/pkg/errors"
+)
+
+// ConfigData is supplied to the kubeadm config template, with values populated
+// by the cluster package
+type ConfigData struct {
+	ClusterName       string
+	KubernetesVersion string
+	// UnifiedControlPlaneImage - optional
+	UnifiedControlPlaneImage string
+	// AutoDerivedConfigData is populated by DeriveFields()
+	AutoDerivedConfigData
+}
+
+// AutoDerivedConfigData fields are automatically derived by
+// ConfigData.DeriveFieldsif they are not specified / zero valued
+type AutoDerivedConfigData struct {
+	// DockerStableTag is automatically derived from KubernetesVersion
+	DockerStableTag string
+}
+
+// DeriveFields automatically derives DockerStableTag if not specified
+func (c *ConfigData) DeriveFields() {
+	if c.DockerStableTag == "" {
+		c.DockerStableTag = strings.Replace(c.KubernetesVersion, "+", "_", -1)
+	}
+}
+
+// DefaultConfigTemplate is the default kubeadm config template used by kind
+const DefaultConfigTemplate = `# config generated by kind
+apiVersion: kubeadm.k8s.io/v1alpha2
+kind: MasterConfiguration
+clusterName: {{.ClusterName}}
+# on docker for mac we have to expose the api server via port forward,
+# so we need to ensure the cert is valid for localhost so we can talk
+# to the cluster after rewriting the kubeconfig to point to localhost
+apiServerCertSANs: [localhost]
+kubernetesVersion: {{.KubernetesVersion}}
+{{if ne .UnifiedControlPlaneImage ""}}
+# optionally specify a unified control plane image
+unifiedControlPlaneImage: {{.UnifiedControlPlaneImage}}:{{.DockerStableTag}}
+{{end}}`
+
+// Config returns a kubeadm config from the template and config data,
+// if templateSource == "", DeafultConfigTemplate will be used instead
+// ConfigData will be supplied to the template after conversion to ConfigTemplateData
+func Config(templateSource string, data ConfigData) (config string, err error) {
+	// load the template, using the default if not specified
+	if templateSource == "" {
+		templateSource = DefaultConfigTemplate
+	}
+	t, err := template.New("kubeadm-config").Parse(templateSource)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to parse config template")
+	}
+	// derive any automatic fields if not supplied
+	data.DeriveFields()
+	// execute the template
+	var buff bytes.Buffer
+	err = t.Execute(&buff, data)
+	if err != nil {
+		return "", errors.Wrap(err, "error executing config template")
+	}
+	return buff.String(), nil
+}

pkg/cluster/kubeadm/const.go

@@ -0,0 +1,20 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kubeadm
+
+// APIServerPort is the expected default APIServerPort on the control plane node(s)
+const APIServerPort = 6443

pkg/cluster/kubeadm/doc.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package kubeadm contains kubeadm related constants and configuration
+package kubeadm

pkg/cluster/node.go

@@ -0,0 +1,253 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cluster
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/golang/glog"
+	"github.com/pkg/errors"
+
+	"k8s.io/test-infra/kind/pkg/cluster/kubeadm"
+	"k8s.io/test-infra/kind/pkg/exec"
+)
+
+type nodeHandle struct {
+	// must be one of docker container ID or name
+	nameOrID string
+}
+
+// createNode `docker run`s the node image, note that due to
+// images/node/entrypoint being the entrypoint, this container will
+// effectively be paused until we call actuallyStartNode(...)
+func createNode(name, clusterLabel string) (handle *nodeHandle, err error) {
+	cmd := exec.Command("docker", "run")
+	cmd.Args = append(cmd.Args,
+		"-d", // run the container detached
+		"-t", // we need a pseudo-tty for systemd logs
+		// running containers in a container requires privileged
+		// NOTE: we could try to replicate this with --cap-add, and use less
+		// privileges, but this flag also changes some mounts that are necessary
+		// including some ones docker would otherwise do by default.
+		// for now this is what we want. in the future we may revisit this.
+		"--privileged",
+		"--security-opt", "seccomp=unconfined", // also ignore seccomp
+		"--tmpfs", "/tmp", // various things depend on working /tmp
+		"--tmpfs", "/run", // systemd wants a writable /run
+		// docker in docker needs this, so as not to stack overlays
+		"--tmpfs", "/var/lib/docker:exec",
+		// some k8s things want /lib/modules
+		"-v", "/lib/modules:/lib/modules:ro",
+		"--hostname", name, // make hostname match container name
+		"--name", name, // ... and set the container name
+		// label the node with the cluster ID
+		"--label", clusterLabel,
+		"--expose", "6443", // expose API server port
+		// pick a random ephemeral port to forward to the API server
+		"--publish-all",
+		"kind-node", // use our image, TODO: make this configurable
+	)
+	cmd.Debug = true
+	err = cmd.Run()
+	if err != nil {
+		return nil, err
+	}
+	return &nodeHandle{name}, nil
+}
+
+// SignalStart sends SIGUSR1 to the node, which signals our entrypoint to boot
+// see images/node/entrypoint
+func (nh *nodeHandle) SignalStart() error {
+	cmd := exec.Command("docker", "kill")
+	cmd.Args = append(cmd.Args,
+		"-s", "SIGUSR1",
+		nh.nameOrID,
+	)
+	// TODO(bentheelder): collect output instead of connecting these
+	cmd.InheritOutput = true
+	return cmd.Run()
+}
+
+// Run execs command, args... on the node
+func (nh *nodeHandle) Run(command string, args ...string) error {
+	cmd := exec.Command("docker", "exec")
+	cmd.Args = append(cmd.Args,
+		"-t",           // use a tty so we can get output
+		"--privileged", // run with priliges so we can remount etc..
+		nh.nameOrID,    // ... against the "node" container
+		command,        // with the command specified
+	)
+	cmd.Args = append(cmd.Args,
+		args..., // finally, with the args specified
+	)
+	cmd.InheritOutput = true
+	return cmd.Run()
+}
+
+// CombinedOutputLines execs command, args... on the node, returning the output lines
+func (nh *nodeHandle) CombinedOutputLines(command string, args ...string) ([]string, error) {
+	cmd := exec.Command("docker", "exec")
+	cmd.Args = append(cmd.Args,
+		"-t",           // use a tty so we can get output
+		"--privileged", // run with priliges so we can remount etc..
+		nh.nameOrID,    // ... against the "node" container
+		command,        // with the command specified
+	)
+	cmd.Args = append(cmd.Args,
+		args..., // finally, with the args specified
+	)
+	return cmd.CombinedOutputLines()
+}
+
+// helper to copy source file to dest on the node
+func (nh *nodeHandle) CopyTo(source, dest string) error {
+	cmd := exec.Command("docker", "cp")
+	cmd.Args = append(cmd.Args,
+		source, // from the source file
+		nh.nameOrID+":"+dest, // to the node, at dest
+	)
+	cmd.InheritOutput = true
+	return cmd.Run()
+}
+
+// WaitForDocker waits for Docker to be ready on the node
+// it returns true on success, and false on a timeout
+func (nh *nodeHandle) WaitForDocker(until time.Time) bool {
+	return tryUntil(until, func() bool {
+		out, err := nh.CombinedOutputLines("systemctl", "is-active", "docker")
+		if err != nil {
+			return false
+		}
+		return len(out) == 1 && out[0] == "active"
+	})
+}
+
+// helper that calls `try()`` in a loop until the deadline `until`
+// has passed or `try()`returns true, returns wether try ever returned true
+func tryUntil(until time.Time, try func() bool) bool {
+	now := time.Now()
+	for until.After(now) {
+		if try() {
+			return true
+		}
+	}
+	return false
+}
+
+// LoadImages loads image tarballs stored on the node into docker on the node
+func (nh *nodeHandle) LoadImages() {
+	// load images cached on the node into docker
+	if err := nh.Run(
+		"find",
+		"/kind/images",
+		"-name", "*.tar",
+		"-exec", "docker", "load", "-i", "{}", ";",
+	); err != nil {
+		glog.Warningf("Failed to preload docker images: %v", err)
+		return
+	}
+	// retag images that are missing -amd64 as image:tag -> image-amd64:tag
+	// bazel built images are currently missing these
+	// TODO(bentheelder): this is a bit gross, move this logic out of bash
+	if err := nh.Run(
+		"/bin/bash", "-c",
+		`docker images --format='{{.Repository}}:{{.Tag}}' | grep -v amd64 | xargs -L 1 -I '{}' /bin/bash -c 'docker tag "{}" "$(echo "{}" | sed s/:/-amd64:/)"'`,
+	); err != nil {
+		glog.Warningf("Failed to re-tag docker images: %v", err)
+	}
+
+	nh.Run("docker", "images")
+}
+
+// KubeVersion returns the Kubernetes version installed on the node
+func (nh *nodeHandle) KubeVersion() (version string, err error) {
+	// grab kubernetes version from the node image
+	lines, err := nh.CombinedOutputLines("cat", "/kind/version")
+	if err != nil {
+		return "", errors.Wrap(err, "failed to get file")
+	}
+	if len(lines) != 1 {
+		return "", fmt.Errorf("file should only be one line, got %d lines", len(lines))
+	}
+	return lines[0], nil
+}
+
+// matches kubeconfig server entry like:
+//    server: https://172.17.0.2:6443
+// which we rewrite to:
+//    server: https://localhost:$PORT
+var serverAddressRE = regexp.MustCompile(`^(\s+server:) https://.*:\d+$`)
+
+// WriteKubeConfig writes a fixed KUBECONFIG to dest
+// this should only be called on a control plane node
+func (nh *nodeHandle) WriteKubeConfig(dest string) error {
+	// get the forwarded api server port
+	port, err := nh.GetForwardedPort(kubeadm.APIServerPort)
+	if err != nil {
+		return err
+	}
+
+	lines, err := nh.CombinedOutputLines("cat", "/etc/kubernetes/admin.conf")
+	if err != nil {
+		return errors.Wrap(err, "failed to get kubeconfig from node")
+	}
+
+	// fix the config file, swapping out the server for the forwarded localhost:port
+	var buff bytes.Buffer
+	for _, line := range lines {
+		match := serverAddressRE.FindStringSubmatch(line)
+		if len(match) > 1 {
+			line = fmt.Sprintf("%s https://localhost:%d", match[1], port)
+		}
+		buff.WriteString(line)
+		buff.WriteString("\n")
+	}
+
+	return ioutil.WriteFile(dest, buff.Bytes(), 0600)
+}
+
+// GetForwardedPort takes the port number within the "node" container
+// and returns the port it was forwarded to ouside the container
+func (nh *nodeHandle) GetForwardedPort(port uint16) (uint16, error) {
+	cmd := exec.Command("docker", "port")
+	cmd.Args = append(cmd.Args,
+		nh.nameOrID,             // ports are looked up by container
+		fmt.Sprintf("%d", port), // limit to the port we are looking up
+	)
+	lines, err := cmd.CombinedOutputLines()
+	if err != nil {
+		return 0, err
+	}
+	if len(lines) != 1 {
+		return 0, fmt.Errorf("invalid output: %v", lines)
+	}
+	parts := strings.Split(lines[0], ":")
+	if len(parts) != 2 {
+		return 0, fmt.Errorf("invalid output: %v", lines)
+	}
+	v, err := strconv.ParseUint(parts[1], 10, 16)
+	if err != nil {
+		return 0, err
+	}
+	return uint16(v), nil
+}