better cluster boot, more build types

pkg/build/doc.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package build implements functionality to build the kind images
+package build

pkg/build/files.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 
 	"k8s.io/test-infra/kind/pkg/exec"
 )
@@ -31,7 +32,9 @@ func TempDir(dir, prefix string) (name string, err error) {
 	if err != nil {
 		return "", err
 	}
-	if runtime.GOOS == "darwin" {
+	// on macOS $TMPDIR is typically /var/..., which is not mountable
+	// /private/var/... is the mountable equivilant
+	if runtime.GOOS == "darwin" && strings.HasPrefix(name, "/var/") {
 		name = filepath.Join("/private", name)
 	}
 	return name, nil

pkg/build/kube/aptbits.go

@@ -0,0 +1,89 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kube
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/golang/glog"
+)
+
+// AptBits implements Bits for the official upstream debian packages
+type AptBits struct {
+}
+
+var _ Bits = &AptBits{}
+
+func init() {
+	RegisterNamedBits("apt", NewAptBits)
+}
+
+// NewAptBits returns a new Bits backed by the upstream debian packages
+func NewAptBits(kubeRoot string) (bits Bits, err error) {
+	return &AptBits{}, nil
+}
+
+// Build implements Bits.Build
+// for AptBits this does nothing
+func (b *AptBits) Build() error {
+	return nil
+}
+
+// Paths implements Bits.Paths
+func (b *AptBits) Paths() map[string]string {
+	return map[string]string{}
+}
+
+// Install implements Bits.Install
+func (b *AptBits) Install(install InstallContext) error {
+	// add apt repo
+	addKey := `curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -`
+	addSources := `cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
+deb http://apt.kubernetes.io/ kubernetes-xenial main
+EOF`
+	if err := install.Run("/bin/sh", "-c", addKey); err != nil {
+		glog.Errorf("Adding Kubernetes apt repository failed! %v", err)
+		return err
+	}
+	if err := install.Run("/bin/sh", "-c", addSources); err != nil {
+		glog.Errorf("Adding Kubernetes apt repository failed! %v", err)
+		return err
+	}
+	// install packages
+	if err := install.Run("/bin/sh", "-c", `clean-install kubelet kubeadm kubectl`); err != nil {
+		glog.Errorf("Installing Kubernetes packages failed! %v", err)
+		return err
+	}
+	// get version to version file
+	lines, err := install.CombinedOutputLines("/bin/sh", "-c", `kubelet --version`)
+	if err != nil {
+		glog.Errorf("Failed to get Kubernetes version! %v", err)
+		return err
+	}
+	// the output should be one line of the form `Kubernetes ${VERSION}`
+	if len(lines) != 1 {
+		glog.Errorf("Failed to parse Kubernetes version with unexpected output: %v", lines)
+		return fmt.Errorf("failed to parse Kubernetes version")
+	}
+	version := strings.SplitN(lines[0], " ", 2)[1]
+	if err := install.Run("/bin/sh", "-c", fmt.Sprintf(`echo "%s" >> /kind/version`, version)); err != nil {
+		glog.Errorf("Failed to get Kubernetes version! %v", err)
+		return err
+	}
+	return nil
+}

pkg/build/kube/bazelbuildbits.go

@@ -0,0 +1,127 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kube
+
+import (
+	"os"
+	"path"
+	"path/filepath"
+
+	"github.com/golang/glog"
+
+	"k8s.io/test-infra/kind/pkg/exec"
+)
+
+// BazelBuildBits implements Bits for a local Bazel build
+type BazelBuildBits struct {
+	kubeRoot string
+	paths    map[string]string
+}
+
+var _ Bits = &BazelBuildBits{}
+
+func init() {
+	RegisterNamedBits("bazel", NewBazelBuildBits)
+}
+
+// NewBazelBuildBits returns a new Bits backed by bazel build,
+// given kubeRoot, the path to the kubernetes source directory
+func NewBazelBuildBits(kubeRoot string) (bits Bits, err error) {
+	// https://docs.bazel.build/versions/master/output_directories.html
+	binDir := filepath.Join(kubeRoot, "bazel-bin")
+	buildDir := filepath.Join(binDir, "build")
+	bits = &BazelBuildBits{
+		kubeRoot: kubeRoot,
+		paths: map[string]string{
+			// debians
+			filepath.Join(buildDir, "debs", "kubeadm.deb"):        "debs/kubeadm.deb",
+			filepath.Join(buildDir, "debs", "kubelet.deb"):        "debs/kubelet.deb",
+			filepath.Join(buildDir, "debs", "kubectl.deb"):        "debs/kubectl.deb",
+			filepath.Join(buildDir, "debs", "kubernetes-cni.deb"): "debs/kubernetes-cni.deb",
+			filepath.Join(buildDir, "debs", "cri-tools.deb"):      "debs/cri-tools.deb",
+			// docker images
+			filepath.Join(buildDir, "kube-apiserver.tar"):          "images/kube-apiserver.tar",
+			filepath.Join(buildDir, "kube-controller-manager.tar"): "images/kube-controller-manager.tar",
+			filepath.Join(buildDir, "kube-scheduler.tar"):          "images/kube-scheduler.tar",
+			filepath.Join(buildDir, "kube-proxy.tar"):              "images/kube-proxy.tar",
+			// version files
+			filepath.Join(kubeRoot, "_output", "git_version"): "version",
+		},
+	}
+	return bits, nil
+}
+
+// Build implements Bits.Build
+func (b *BazelBuildBits) Build() error {
+	// TODO(bentheelder): support other modes of building
+	// cd to k8s source
+	cwd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+	os.Chdir(b.kubeRoot)
+	// make sure we cd back when done
+	defer os.Chdir(cwd)
+
+	// capture version info
+	buildVersionFile(b.kubeRoot)
+	return nil
+
+	// build artifacts
+	cmd := exec.Command("bazel", "build")
+	cmd.Args = append(cmd.Args,
+		// TODO(bentheelder): we assume linux amd64, but we could select
+		// this based on Arch etc. throughout, this flag supports GOOS/GOARCH
+		"--platforms=@io_bazel_rules_go//go/toolchain:linux_amd64",
+		// we want the debian packages
+		"//build/debs:debs",
+		// and the docker images
+		//"//cluster/images/hyperkube:hyperkube.tar",
+		"//build:docker-artifacts",
+	)
+	cmd.Debug = true
+	cmd.InheritOutput = true
+	return cmd.Run()
+}
+
+// Paths implements Bits.Paths
+func (b *BazelBuildBits) Paths() map[string]string {
+	// TODO(bentheelder): maybe copy the map before returning /shrug
+	return b.paths
+}
+
+// Install implements Bits.Install
+func (b *BazelBuildBits) Install(install InstallContext) error {
+	base := install.BasePath()
+
+	debs := path.Join(base, "debs", "*.deb")
+
+	if err := install.Run("/bin/sh", "-c", "dpkg -i "+debs); err != nil {
+		glog.Errorf("Image install failed! %v", err)
+		return err
+	}
+
+	if err := install.Run("/bin/sh", "-c",
+		"rm -rf /kind/bits/debs/*.deb"+
+			" /var/cache/debconf/* /var/lib/apt/lists/* /var/log/*kg",
+	); err != nil {
+		glog.Errorf("Image install failed! %v", err)
+		return err
+	}
+
+	return nil
+}

pkg/build/kube/bits.go

@@ -0,0 +1,93 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kube
+
+import (
+	"fmt"
+	"sync"
+)
+
+// Bits provides the locations of Kubernetes Binaries / Images
+// needed on the cluster nodes
+// Implementations should be registered with RegisterNamedBits
+type Bits interface {
+	// Build returns any errors encountered while building it Kubernetes.
+	// Some implementations (upstream binaries) may use this step to obtain
+	// an existing build isntead
+	Build() error
+	// Paths returns a map of path on host machine to desired path in the image
+	// These paths will be populated in the image relative to some base path,
+	// obtainable by NodeInstall.BasePath()
+	// Note: if Images are populated in iamges/, the cluster provisioning
+	// will load these prior to calling kubeadm
+	Paths() map[string]string
+	// Install should install the built sources on the node, assuming paths
+	// have been populated
+	Install(InstallContext) error
+}
+
+// InstallContext should be implemented by users of Bits
+// to allow installing the bits in a Docker image
+type InstallContext interface {
+	// Returns the base path Paths() were populated relative to
+	BasePath() string
+	// Run execs (cmd, ...args) in the build container and returns error
+	Run(string, ...string) error
+	// CombinedOutputLines is like Run but returns the output lines
+	CombinedOutputLines(string, ...string) ([]string, error)
+}
+
+// NewNamedBits returns a new Bits by named implementation
+// currently this includes:
+// "bazel" -> NewBazelBuildBits(kubeRoot)
+// "docker" or "make" -> NewDockerBuildBits(kubeRoot)
+// "apt" -> NewAptBits(kubeRoot)
+func NewNamedBits(name string, kubeRoot string) (bits Bits, err error) {
+	bitsImpls.Lock()
+	fn, ok := bitsImpls.impls[name]
+	bitsImpls.Unlock()
+	if !ok {
+		return nil, fmt.Errorf("no Bits implementation with name: %s", name)
+	}
+	return fn(kubeRoot)
+}
+
+// RegisterNamedBits registers a new named Bits implementation for use from
+// NewNamedBits
+func RegisterNamedBits(name string, fn func(string) (Bits, error)) {
+	bitsImpls.Lock()
+	bitsImpls.impls[name] = fn
+	bitsImpls.Unlock()
+}
+
+// NamedBitsRegistered returns true if name is in the registry backing
+// NewNamedBits
+func NamedBitsRegistered(name string) bool {
+	var ok bool
+	bitsImpls.Lock()
+	_, ok = bitsImpls.impls[name]
+	bitsImpls.Unlock()
+	return ok
+}
+
+// internal registry of named bits implementations
+var bitsImpls = struct {
+	impls map[string]func(string) (Bits, error)
+	sync.Mutex
+}{
+	impls: map[string]func(string) (Bits, error){},
+}

pkg/build/kube/doc.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package kube implements functionality to build Kubernetes for the purposes
+// of installing into the kind node image
+package kube

pkg/build/kube/dockerbuildbits.go

@@ -0,0 +1,189 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kube
+
+import (
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"k8s.io/test-infra/kind/pkg/exec"
+)
+
+// DockerBuildBits implements Bits for a local docke-ized make / bash build
+type DockerBuildBits struct {
+	kubeRoot string
+	paths    map[string]string
+}
+
+var _ Bits = &DockerBuildBits{}
+
+func init() {
+	RegisterNamedBits("docker", NewDockerBuildBits)
+	RegisterNamedBits("make", NewDockerBuildBits)
+}
+
+// NewDockerBuildBits returns a new Bits backed by the docker-ized build,
+// given kubeRoot, the path to the kubernetes source directory
+func NewDockerBuildBits(kubeRoot string) (bits Bits, err error) {
+	// https://docs.Docker.build/versions/master/output_directories.html
+	binDir := filepath.Join(kubeRoot,
+		"_output", "dockerized", "bin", "linux", "amd64",
+	)
+	imageDir := filepath.Join(kubeRoot,
+		"_output", "release-images", "amd64",
+	)
+	bits = &DockerBuildBits{
+		kubeRoot: kubeRoot,
+		paths: map[string]string{
+			// binaries (hyperkube)
+			filepath.Join(binDir, "kubeadm"): "bin/kubeadm",
+			filepath.Join(binDir, "kubelet"): "bin/kubelet",
+			filepath.Join(binDir, "kubectl"): "bin/kubectl",
+			// docker images
+			filepath.Join(imageDir, "kube-apiserver.tar"):          "images/kube-apiserver.tar",
+			filepath.Join(imageDir, "kube-controller-manager.tar"): "images/kube-controller-manager.tar",
+			filepath.Join(imageDir, "kube-scheduler.tar"):          "images/kube-scheduler.tar",
+			filepath.Join(imageDir, "kube-proxy.tar"):              "images/kube-proxy.tar",
+			// version files
+			filepath.Join(kubeRoot, "_output", "git_version"): "version",
+			// borrow kubelet service files from bazel debians
+			// TODO(bentheelder): probably we should use our own config instead :-)
+			filepath.Join(kubeRoot, "build", "debs", "kubelet.service"): "systemd/kubelet.service",
+			filepath.Join(kubeRoot, "build", "debs", "10-kubeadm.conf"): "systemd/10-kubeadm.conf",
+		},
+	}
+	return bits, nil
+}
+
+// Build implements Bits.Build
+func (b *DockerBuildBits) Build() error {
+	// TODO(bentheelder): support other modes of building
+	// cd to k8s source
+	cwd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+	os.Chdir(b.kubeRoot)
+	// make sure we cd back when done
+	defer os.Chdir(cwd)
+
+	// capture version info
+	err = buildVersionFile(b.kubeRoot)
+	if err != nil {
+		return err
+	}
+
+	// build binaries
+	cmd := exec.Command("build/run.sh", "make", "all")
+	what := []string{
+		"cmd/kubeadm",
+		"cmd/kubectl",
+		"cmd/kubelet",
+		"cmd/cloud-controller-manager",
+		"cmd/kube-apiserver",
+		"cmd/kube-controller-manager",
+		"cmd/kube-scheduler",
+		"cmd/kube-proxy",
+	}
+	cmd.Args = append(cmd.Args,
+		"WHAT="+strings.Join(what, " "), "KUBE_BUILD_PLATFORMS=linux/amd64",
+	)
+	cmd.Env = append(cmd.Env, os.Environ()...)
+	cmd.Env = append(cmd.Env, "KUBE_VERBOSE=0")
+	cmd.Debug = true
+	cmd.InheritOutput = true
+	err = cmd.Run()
+	if err != nil {
+		return errors.Wrap(err, "failed to build binaries")
+	}
+
+	// TODO(bentheelder): this is perhaps a bit overkill
+	// the build will fail if they are already present though
+	// We should find what `make quick-release` does and mimic that
+	err = os.RemoveAll(filepath.Join(
+		".", "_output", "release-images", "amd64",
+	))
+	if err != nil {
+		return errors.Wrap(err, "failed to remove old release-images")
+	}
+
+	// build images
+	// TODO(bentheelder): there has to be a better way to do this, but the
+	// closest seems to be make quick-release, which builds more than we need
+	buildImages := []string{
+		"source build/common.sh;",
+		"source hack/lib/version.sh;",
+		"source build/lib/release.sh;",
+		"kube::version::get_version_vars;",
+		`kube::release::create_docker_images_for_server "${LOCAL_OUTPUT_ROOT}/dockerized/bin/linux/amd64" "amd64"`,
+	}
+	cmd = exec.Command("bash", "-c", strings.Join(buildImages, " "))
+	cmd.Env = append(cmd.Env, os.Environ()...)
+	cmd.Env = append(cmd.Env, "KUBE_BUILD_HYPERKUBE=n")
+	cmd.Debug = true
+	cmd.InheritOutput = true
+	err = cmd.Run()
+	if err != nil {
+		return errors.Wrap(err, "failed to build images")
+	}
+
+	return nil
+}
+
+// Paths implements Bits.Paths
+func (b *DockerBuildBits) Paths() map[string]string {
+	// TODO(bentheelder): maybe copy the map before returning /shrug
+	return b.paths
+}
+
+// Install implements Bits.Install
+func (b *DockerBuildBits) Install(install InstallContext) error {
+	kindBinDir := path.Join(install.BasePath(), "bin")
+
+	// symlink the kubernetes binaries into $PATH
+	binaries := []string{"kubeadm", "kubelet", "kubectl"}
+	for _, binary := range binaries {
+		if err := install.Run("ln", "-s",
+			path.Join(kindBinDir, binary),
+			path.Join("/usr/bin/", binary),
+		); err != nil {
+			return errors.Wrap(err, "failed to symlink binaries")
+		}
+	}
+
+	// enable the kubelet service
+	kubeletService := path.Join(install.BasePath(), "systemd/kubelet.service")
+	if err := install.Run("systemctl", "enable", kubeletService); err != nil {
+		return errors.Wrap(err, "failed to enable kubelet service")
+	}
+
+	// setup the kubelet dropin
+	kubeletDropinSource := path.Join(install.BasePath(), "systemd/10-kubeadm.conf")
+	kubeletDropin := "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf"
+	if err := install.Run("mkdir", "-p", path.Dir(kubeletDropin)); err != nil {
+		return errors.Wrap(err, "failed to configure kubelet service")
+	}
+	if err := install.Run("cp", kubeletDropinSource, kubeletDropin); err != nil {
+		return errors.Wrap(err, "failed to configure kubelet service")
+	}
+
+	return nil
+}

pkg/build/kube/source.go

@@ -14,33 +14,21 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package build
+package kube
 
 import (
 	"fmt"
-	gobuild "go/build"
+	"go/build"
 )
 
-// KubeBits provides the locations of Kubernetes Binaries / Images
-// needed on the cluster nodes
-type KubeBits interface {
-	// Paths returns a map of path on host to desired path in the image
-	Paths() map[string]string
-}
+// ImportPath is the canonical import path for the kubernetes root package
+// this is used by FindSource
+const ImportPath = "k8s.io/kubernetes"
 
-// NodeInstall should be implemented by users of KubeBitsProvider
-// to allow installing the bits
-type NodeInstall interface {
-	// RunOnNode execs (cmd, ...args) on a node and returns error
-	RunOnNode(string, ...string) error
-}
-
-const kubeImportPath = "k8s.io/kubernetes"
-
-// FindKubeSource attempts to locate a kubernetes checkout using go's build package
-func FindKubeSource() (root string, err error) {
+// FindSource attempts to locate a kubernetes checkout using go's build package
+func FindSource() (root string, err error) {
 	// look up the source the way go build would
-	pkg, err := gobuild.Default.Import(kubeImportPath, ".", gobuild.FindOnly)
+	pkg, err := build.Default.Import(ImportPath, ".", build.FindOnly)
 	if err == nil && maybeKubeDir(pkg.Dir) {
 		return pkg.Dir, nil
 	}

pkg/build/kube/version.go

@@ -0,0 +1,64 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package kube
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"k8s.io/test-infra/kind/pkg/exec"
+)
+
+// buildVersionFile creates a file for the kubernetes git version in
+// ./_output/version based on hack/print-workspace-status.sh,
+// these are built into the node image and consumed by the cluster tooling
+func buildVersionFile(kubeRoot string) error {
+	cwd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+	os.Chdir(kubeRoot)
+	// make sure we cd back when done
+	defer os.Chdir(cwd)
+
+	// get the version output
+	cmd := exec.Command("hack/print-workspace-status.sh")
+	cmd.Debug = true
+	output, err := cmd.CombinedOutputLines()
+	if err != nil {
+		return err
+	}
+	outputDir := filepath.Join(kubeRoot, "_output")
+	// parse it, and populate it into _output/git_version
+	for _, line := range output {
+		parts := strings.SplitN(line, " ", 2)
+		if len(parts) != 2 {
+			return fmt.Errorf("could not parse kubernetes version")
+		}
+		if parts[0] == "gitVersion" {
+			ioutil.WriteFile(
+				filepath.Join(outputDir, "git_version"),
+				[]byte(parts[1]),
+				0777,
+			)
+		}
+	}
+	return nil
+}

pkg/build/localbuildbits.go

@@ -1,52 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package build
-
-import "path/filepath"
-
-// BazelBuildBits implements KubeBits for a local Bazel build
-type BazelBuildBits struct {
-	paths map[string]string
-}
-
-var _ KubeBits = &BazelBuildBits{}
-
-func (l *BazelBuildBits) Paths() map[string]string {
-	// TODO(bentheelder): maybe copy the map before returning /shrug
-	return l.paths
-}
-
-func NewBazelBuildBits(kubeRoot string) (bits KubeBits, err error) {
-	// https://docs.bazel.build/versions/master/output_directories.html
-	binDir := filepath.Join(kubeRoot, "bazel-bin")
-	bits = &BazelBuildBits{
-		paths: map[string]string{
-			// debians
-			filepath.Join(binDir, "build", "debs", "kubeadm.deb"):        "debs/kubeadm.deb",
-			filepath.Join(binDir, "build", "debs", "kubelet.deb"):        "debs/kubelet.deb",
-			filepath.Join(binDir, "build", "debs", "kubectl.deb"):        "debs/kubectl.deb",
-			filepath.Join(binDir, "build", "debs", "kubernetes-cni.deb"): "debs/kubernetes-cni.deb",
-			filepath.Join(binDir, "build", "debs", "cri-tools.deb"):      "debs/cri-tools.deb",
-			// docker images
-			filepath.Join(binDir, "build", "kube-proxy.tar"):              "images/kube-proxy.tar",
-			filepath.Join(binDir, "build", "kube-controller-manager.tar"): "images/kube-controller-manager.tar",
-			filepath.Join(binDir, "build", "kube-scheduler.tar"):          "images/kube-scheduler.tar",
-			filepath.Join(binDir, "build", "kube-apiserver.tar"):          "images/kube-apiserver.tar",
-		},
-	}
-	return bits, nil
-}

pkg/build/node_image.go

@@ -14,8 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// Package build implements functionality to build the kind images
-// TODO(bentheelder): and k8s
 package build
 
 import (
@@ -28,6 +26,7 @@ import (
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 
+	"k8s.io/test-infra/kind/pkg/build/kube"
 	"k8s.io/test-infra/kind/pkg/build/sources"
 	"k8s.io/test-infra/kind/pkg/exec"
 )
@@ -39,35 +38,46 @@ type NodeImageBuildContext struct {
 	ImageTag  string
 	Arch      string
 	BaseImage string
+	KubeRoot  string
+	Bits      kube.Bits
 }
 
 // NewNodeImageBuildContext creates a new NodeImageBuildContext with
 // default configuration
-func NewNodeImageBuildContext() *NodeImageBuildContext {
+func NewNodeImageBuildContext(mode string) (ctx *NodeImageBuildContext, err error) {
+	kubeRoot := ""
+	// apt should not fail on finding kube root as it does not use it
+	if mode != "apt" {
+		kubeRoot, err = kube.FindSource()
+		if err != nil {
+			return nil, fmt.Errorf("error finding kuberoot: %v", err)
+		}
+	}
+	bits, err := kube.NewNamedBits(mode, kubeRoot)
+	if err != nil {
+		return nil, err
+	}
 	return &NodeImageBuildContext{
 		ImageTag:  "kind-node",
 		Arch:      "amd64",
 		BaseImage: "kind-base",
-	}
+		KubeRoot:  kubeRoot,
+		Bits:      bits,
+	}, nil
 }
 
 // Build builds the cluster node image, the sourcedir must be set on
 // the NodeImageBuildContext
 func (c *NodeImageBuildContext) Build() (err error) {
-	// get k8s source
-	kubeRoot, err := FindKubeSource()
-	if err != nil {
-		return errors.Wrap(err, "could not find kubernetes source")
-	}
-
 	// ensure kubernetes build is up to date first
 	glog.Infof("Starting to build Kubernetes")
-	//c.buildKube(kubeRoot)
+	if err = c.Bits.Build(); err != nil {
+		glog.Errorf("Failed to build Kubernetes: %v", err)
+		return errors.Wrap(err, "failed to build kubernetes")
+	}
 	glog.Infof("Finished building Kubernetes")
-	// TODO(bentheelder): allow other types of bits
-	bits, err := NewBazelBuildBits(kubeRoot)
 
-	// create tempdir to build in
+	// create tempdir to build the image in
 	tmpDir, err := TempDir("", "kind-node-image")
 	if err != nil {
 		return err
@@ -96,48 +106,25 @@ func (c *NodeImageBuildContext) Build() (err error) {
 	glog.Infof("Building node image in: %s", buildDir)
 
 	// populate the kubernetes artifacts first
-	if err := c.populateBits(buildDir, bits); err != nil {
+	if err := c.populateBits(buildDir); err != nil {
 		return err
 	}
 
-	// then the actual docker image
+	// then the perform the actual docker image build
 	return c.buildImage(buildDir)
 }
 
-func (c *NodeImageBuildContext) buildKube(kubeRoot string) error {
-	// TODO(bentheelder): support other modes of building
-	// cd to k8s source
-	cwd, err := os.Getwd()
-	if err != nil {
-		return err
+func (c *NodeImageBuildContext) populateBits(buildDir string) error {
+	// always create bits dir
+	bitsDir := path.Join(buildDir, "bits")
+	if err := os.Mkdir(bitsDir, 0777); err != nil {
+		return errors.Wrap(err, "failed to make bits dir")
 	}
-	os.Chdir(kubeRoot)
-	// make sure we cd back when done
-	defer os.Chdir(cwd)
-
-	// TODO(bentheelder): move this out and next to the KubeBits impl
-	cmd := exec.Command("bazel", "build")
-	cmd.Args = append(cmd.Args,
-		// TODO(bentheelder): we assume linux amd64, but we could select
-		// this based on Arch etc. throughout, this flag supports GOOS/GOARCH
-		"--platforms=@io_bazel_rules_go//go/toolchain:linux_amd64",
-		// we want the debian packages
-		"//build/debs:debs",
-		// and the docker images
-		"//build:docker-artifacts",
-	)
-
-	cmd.Debug = true
-	cmd.InheritOutput = true
-	return cmd.Run()
-}
-
-func (c *NodeImageBuildContext) populateBits(buildDir string, bits KubeBits) error {
 	// copy all bits from their source path to where we will COPY them into
 	// the dockerfile, see images/node/Dockerfile
-	bitPaths := bits.Paths()
+	bitPaths := c.Bits.Paths()
 	for src, dest := range bitPaths {
-		realDest := path.Join(buildDir, "files", dest)
+		realDest := path.Join(bitsDir, dest)
 		if err := copyFile(src, realDest); err != nil {
 			return errors.Wrap(err, "failed to copy build artifact")
 		}
@@ -148,6 +135,33 @@ func (c *NodeImageBuildContext) populateBits(buildDir string, bits KubeBits) err
 // BuildContainerLabelKey is applied to each build container
 const BuildContainerLabelKey = "io.k8s.test-infra.kind-build"
 
+// private kube.InstallContext implementation, local to the image build
+type installContext struct {
+	basePath    string
+	containerID string
+}
+
+var _ kube.InstallContext = &installContext{}
+
+func (ic *installContext) BasePath() string {
+	return ic.basePath
+}
+
+func (ic *installContext) Run(command string, args ...string) error {
+	cmd := exec.Command("docker", "exec", ic.containerID, command)
+	cmd.Args = append(cmd.Args, args...)
+	cmd.Debug = true
+	cmd.InheritOutput = true
+	return cmd.Run()
+}
+
+func (ic *installContext) CombinedOutputLines(command string, args ...string) ([]string, error) {
+	cmd := exec.Command("docker", "exec", ic.containerID, command)
+	cmd.Args = append(cmd.Args, args...)
+	cmd.Debug = true
+	return cmd.CombinedOutputLines()
+}
+
 func (c *NodeImageBuildContext) buildImage(dir string) error {
 	// build the image, tagged as tagImageAs, using the our tempdir as the context
 	glog.Info("Starting image build ...")
@@ -155,6 +169,8 @@ func (c *NodeImageBuildContext) buildImage(dir string) error {
 	// NOTE: we are using docker run + docker commit so we can install
 	// debians without permanently copying them into the image.
 	// if docker gets proper squash support, we can rm them instead
+	// This also allows the KubeBit implementations to perform programmatic
+	// isntall in the image
 	containerID, err := c.createBuildContainer(dir)
 	if err != nil {
 		glog.Errorf("Image build Failed! %v", err)
@@ -176,27 +192,23 @@ func (c *NodeImageBuildContext) buildImage(dir string) error {
 	}
 
 	// make artifacts directory
-	if err = execInBuild("mkdir", "-p", "/kind/bits"); err != nil {
+	if err = execInBuild("mkdir", "/kind/"); err != nil {
 		glog.Errorf("Image build Failed! %v", err)
 		return err
 	}
 
 	// copy artifacts in
-	if err = execInBuild("rsync", "-r", "/build/files/", "/kind/bits/"); err != nil {
+	if err = execInBuild("rsync", "-r", "/build/bits/", "/kind/"); err != nil {
 		glog.Errorf("Image build Failed! %v", err)
 		return err
 	}
 
-	// install debs
-	if err = execInBuild("/bin/sh", "-c", "dpkg -i /kind/bits/debs/*.deb"); err != nil {
-		glog.Errorf("Image build Failed! %v", err)
-		return err
+	// install the kube bits
+	ic := &installContext{
+		basePath:    "/kind/",
+		containerID: containerID,
 	}
-
-	// clean up after debs / remove them, this saves a couple hundred MB
-	if err = execInBuild("/bin/sh", "-c",
-		"rm -rf /kind/bits/debs/*.deb /var/cache/debconf/* /var/lib/apt/lists/* /var/log/*kg",
-	); err != nil {
+	if err = c.Bits.Install(ic); err != nil {
 		glog.Errorf("Image build Failed! %v", err)
 		return err
 	}

pkg/build/sources/generate.go

@@ -15,7 +15,7 @@ limitations under the License.
 */
 
 // Package sources contains the baked in sources kind needs to build.
-// Primarily this includes the node-image dockerfile, which should rarely
+// Primarily this includes the node-image Dockerfile, which should rarely
 // change.
 // These can be overridden with newer files at build-time, see ./../build
 package sources