From 3d579b2b00328df1ff0e5c6de602cb6a747904bd Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Mon, 27 Aug 2018 10:21:43 -0700 Subject: [PATCH] better cluster boot, more build types --- README.md | 9 +- cmd/kind/cmd/build/base/base.go | 1 + cmd/kind/cmd/build/node/node.go | 17 +- docs/todo.md | 37 +++ images/node/Dockerfile | 19 -- images/node/README.md | 10 + pkg/build/doc.go | 18 ++ pkg/build/files.go | 5 +- pkg/build/kube/aptbits.go | 89 ++++++++ pkg/build/kube/bazelbuildbits.go | 127 +++++++++++ pkg/build/kube/bits.go | 93 ++++++++ pkg/build/kube/doc.go | 19 ++ pkg/build/kube/dockerbuildbits.go | 189 +++++++++++++++ pkg/build/{kubebits.go => kube/source.go} | 28 +-- pkg/build/kube/version.go | 64 ++++++ pkg/build/localbuildbits.go | 52 ----- pkg/build/node_image.go | 126 +++++----- pkg/build/sources/generate.go | 2 +- pkg/cluster/cluster.go | 266 +++++++++++----------- pkg/cluster/config.go | 38 +++- pkg/cluster/doc.go | 18 ++ pkg/cluster/kubeadm/config.go | 88 +++++++ pkg/cluster/kubeadm/const.go | 20 ++ pkg/cluster/kubeadm/doc.go | 18 ++ pkg/cluster/node.go | 253 ++++++++++++++++++++ 25 files changed, 1309 insertions(+), 297 deletions(-) create mode 100644 docs/todo.md delete mode 100644 images/node/Dockerfile create mode 100644 images/node/README.md create mode 100644 pkg/build/doc.go create mode 100644 pkg/build/kube/aptbits.go create mode 100644 pkg/build/kube/bazelbuildbits.go create mode 100644 pkg/build/kube/bits.go create mode 100644 pkg/build/kube/doc.go create mode 100644 pkg/build/kube/dockerbuildbits.go rename pkg/build/{kubebits.go => kube/source.go} (56%) create mode 100644 pkg/build/kube/version.go delete mode 100644 pkg/build/localbuildbits.go create mode 100644 pkg/cluster/doc.go create mode 100644 pkg/cluster/kubeadm/config.go create mode 100644 pkg/cluster/kubeadm/const.go create mode 100644 pkg/cluster/kubeadm/doc.go create mode 100644 pkg/cluster/node.go diff --git a/README.md b/README.md index 8c7b6602..3e2b63a4 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,10 @@ # `kind` - **K**ubernetes **IN** **D**ocker -## WARNING: `kind` is still a work in progress! +## WARNING: `kind` is still a work in progress! See [docs/todo.md](./docs/todo.md) `kind` is a toolset for running local Kubernetes clusters using Docker container "nodes". +`kind` is designed to be suitable for testing Kubernetes, initally targetting the conformance suite. It consists of: - Go [packages](./pkg) implementing [cluster creation](./pkg/cluster), [image build](./pkg/build), etc. @@ -17,7 +18,7 @@ For more details see [the design documentation](./docs/design.md). ## Building -You can build `kind` with `go install ./cmd/kind` or `bazel build //kind/cmd/kind`. +You can build `kind` with `go install k8s.io/test-infra/kind/cmd/kind` or `bazel build //kind/cmd/kind`. ## Usage @@ -29,7 +30,9 @@ For more usage, run `kind --help` or `kind [command] --help`. ## Advanced -`kind build image --source=./kind/images/node` will build the node image +`kind build base` will build the base image. + +`kind build node` will build the node image. ## Community, discussion, contribution, and support diff --git a/cmd/kind/cmd/build/base/base.go b/cmd/kind/cmd/build/base/base.go index ce8fae18..e967d1a8 100644 --- a/cmd/kind/cmd/build/base/base.go +++ b/cmd/kind/cmd/build/base/base.go @@ -28,6 +28,7 @@ type flags struct { Source string } +// NewCommand returns a new cobra.Command for building the base image func NewCommand() *cobra.Command { flags := &flags{} cmd := &cobra.Command{ diff --git a/cmd/kind/cmd/build/node/node.go b/cmd/kind/cmd/build/node/node.go index aefa68d3..d87247c5 100644 --- a/cmd/kind/cmd/build/node/node.go +++ b/cmd/kind/cmd/build/node/node.go @@ -19,15 +19,18 @@ package node import ( "os" + "github.com/golang/glog" "github.com/spf13/cobra" "k8s.io/test-infra/kind/pkg/build" ) type flags struct { - Source string + Source string + BuildType string } +// NewCommand returns a new cobra.Command for building the node image func NewCommand() *cobra.Command { flags := &flags{} cmd := &cobra.Command{ @@ -39,16 +42,20 @@ func NewCommand() *cobra.Command { run(flags, cmd, args) }, } - cmd.Flags().StringVar(&flags.Source, "source", "", "path to the base image sources") + cmd.Flags().StringVar(&flags.BuildType, "type", "docker", "build type, one of [bazel, docker, apt]") return cmd } func run(flags *flags, cmd *cobra.Command, args []string) { // TODO(bentheelder): make this more configurable - ctx := build.NewNodeImageBuildContext() - ctx.SourceDir = flags.Source - err := ctx.Build() + ctx, err := build.NewNodeImageBuildContext(flags.BuildType) if err != nil { + glog.Errorf("Error creating build context: %v", err) + os.Exit(-1) + } + err = ctx.Build() + if err != nil { + glog.Errorf("Error building node image: %v", err) os.Exit(-1) } } diff --git a/docs/todo.md b/docs/todo.md new file mode 100644 index 00000000..db68e0b4 --- /dev/null +++ b/docs/todo.md @@ -0,0 +1,37 @@ +# TODO + +A non-exhaustive list of tasks (in no-particular order) includes: +- [x] basic single "node" clusters +- [x] multiple clusters per host / named clusters +- [ ] multi-node clusters +- [x] support for multiple kubernetes builds: + - [x] bazel build from source + - [x] docker / make build from source + - [x] apt (upstream / official release packages) + - [ ] support for selecting a non-default package version +- [ ] kubetest ingregration [WIP] + - [ ] point existing ["dind"](https://github.com/kubernetes/test-infra/tree/master/dind) integration here once complete +- [ ] improved logging and error handling +- [ ] continuous integration +- [ ] publish pre-built images to a registry +- [ ] fake out all internals and unit test [WIP] +- [ ] pre-load images that are not from the build / possibly build more images + - [ ] etcd + - [ ] overlay network images? +- [ ] support multiple overlay networks +- [ ] support advanced configuration via config file + - [ ] kubeadm config template override +- [ ] more advanced network configuration (not docker0) +- [ ] support for other CRI within the "node" containers (containerd, cri-o) +- [ ] switch from `exec.Command("docker", ...)` to the Docker client library + +# Wishlist + +Longer term / continually appealing items: + +- Improved documentation +- Support for architectures / platforms other than linux / amd64 for the node images +- Support for client platforms other than docker on linux / docker for mac +- Less priviliged containers or sharing a CRI via something like [containerd namespaces](https://github.com/containerd/containerd/blob/master/docs/namespaces.md), generally + better isolation +- HA kubeadm / multiple control plane nodes diff --git a/images/node/Dockerfile b/images/node/Dockerfile deleted file mode 100644 index 4292c27e..00000000 --- a/images/node/Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -# TODO(bentheelder): place these in a registry -ARG BASE_IMAGE="kind-base" -FROM ${BASE_IMAGE} - -# copy all artifacts provided by the build tooling into the image -COPY ./files/ /kind/ - -RUN du -h /kind - -# install all the debs if there are any -RUN [ ! -d /kind/debs/ ] || \ - dpkg -i /kind/debs/*.deb && \ - rm -rf \ - /kind/debs/*.deb \ - /var/cache/debconf/* \ - /var/lib/apt/lists/* \ - /var/log/* - -RUN du -h /kind diff --git a/images/node/README.md b/images/node/README.md new file mode 100644 index 00000000..5c28ffa7 --- /dev/null +++ b/images/node/README.md @@ -0,0 +1,10 @@ +## images/node + +See: [./../../pkg/build/node_image.go](./../../pkg.build/node_image.go), this +image is built programmatically with docker run / exec / commit for performance +reasons with large artifacts. + +Roughly this image is [the base image](./../base), with the addition of: + - installing the Kubernetes packages / binaries + - placing the Kubernetes docker images in /kind/images/*.tar + - placing a file in /kind/version containing the Kubernetes semver diff --git a/pkg/build/doc.go b/pkg/build/doc.go new file mode 100644 index 00000000..371b0653 --- /dev/null +++ b/pkg/build/doc.go @@ -0,0 +1,18 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package build implements functionality to build the kind images +package build diff --git a/pkg/build/files.go b/pkg/build/files.go index de1fb100..3dc4d607 100644 --- a/pkg/build/files.go +++ b/pkg/build/files.go @@ -21,6 +21,7 @@ import ( "os" "path/filepath" "runtime" + "strings" "k8s.io/test-infra/kind/pkg/exec" ) @@ -31,7 +32,9 @@ func TempDir(dir, prefix string) (name string, err error) { if err != nil { return "", err } - if runtime.GOOS == "darwin" { + // on macOS $TMPDIR is typically /var/..., which is not mountable + // /private/var/... is the mountable equivilant + if runtime.GOOS == "darwin" && strings.HasPrefix(name, "/var/") { name = filepath.Join("/private", name) } return name, nil diff --git a/pkg/build/kube/aptbits.go b/pkg/build/kube/aptbits.go new file mode 100644 index 00000000..13248127 --- /dev/null +++ b/pkg/build/kube/aptbits.go @@ -0,0 +1,89 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kube + +import ( + "fmt" + "strings" + + "github.com/golang/glog" +) + +// AptBits implements Bits for the official upstream debian packages +type AptBits struct { +} + +var _ Bits = &AptBits{} + +func init() { + RegisterNamedBits("apt", NewAptBits) +} + +// NewAptBits returns a new Bits backed by the upstream debian packages +func NewAptBits(kubeRoot string) (bits Bits, err error) { + return &AptBits{}, nil +} + +// Build implements Bits.Build +// for AptBits this does nothing +func (b *AptBits) Build() error { + return nil +} + +// Paths implements Bits.Paths +func (b *AptBits) Paths() map[string]string { + return map[string]string{} +} + +// Install implements Bits.Install +func (b *AptBits) Install(install InstallContext) error { + // add apt repo + addKey := `curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -` + addSources := `cat </etc/apt/sources.list.d/kubernetes.list +deb http://apt.kubernetes.io/ kubernetes-xenial main +EOF` + if err := install.Run("/bin/sh", "-c", addKey); err != nil { + glog.Errorf("Adding Kubernetes apt repository failed! %v", err) + return err + } + if err := install.Run("/bin/sh", "-c", addSources); err != nil { + glog.Errorf("Adding Kubernetes apt repository failed! %v", err) + return err + } + // install packages + if err := install.Run("/bin/sh", "-c", `clean-install kubelet kubeadm kubectl`); err != nil { + glog.Errorf("Installing Kubernetes packages failed! %v", err) + return err + } + // get version to version file + lines, err := install.CombinedOutputLines("/bin/sh", "-c", `kubelet --version`) + if err != nil { + glog.Errorf("Failed to get Kubernetes version! %v", err) + return err + } + // the output should be one line of the form `Kubernetes ${VERSION}` + if len(lines) != 1 { + glog.Errorf("Failed to parse Kubernetes version with unexpected output: %v", lines) + return fmt.Errorf("failed to parse Kubernetes version") + } + version := strings.SplitN(lines[0], " ", 2)[1] + if err := install.Run("/bin/sh", "-c", fmt.Sprintf(`echo "%s" >> /kind/version`, version)); err != nil { + glog.Errorf("Failed to get Kubernetes version! %v", err) + return err + } + return nil +} diff --git a/pkg/build/kube/bazelbuildbits.go b/pkg/build/kube/bazelbuildbits.go new file mode 100644 index 00000000..1bb775b8 --- /dev/null +++ b/pkg/build/kube/bazelbuildbits.go @@ -0,0 +1,127 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kube + +import ( + "os" + "path" + "path/filepath" + + "github.com/golang/glog" + + "k8s.io/test-infra/kind/pkg/exec" +) + +// BazelBuildBits implements Bits for a local Bazel build +type BazelBuildBits struct { + kubeRoot string + paths map[string]string +} + +var _ Bits = &BazelBuildBits{} + +func init() { + RegisterNamedBits("bazel", NewBazelBuildBits) +} + +// NewBazelBuildBits returns a new Bits backed by bazel build, +// given kubeRoot, the path to the kubernetes source directory +func NewBazelBuildBits(kubeRoot string) (bits Bits, err error) { + // https://docs.bazel.build/versions/master/output_directories.html + binDir := filepath.Join(kubeRoot, "bazel-bin") + buildDir := filepath.Join(binDir, "build") + bits = &BazelBuildBits{ + kubeRoot: kubeRoot, + paths: map[string]string{ + // debians + filepath.Join(buildDir, "debs", "kubeadm.deb"): "debs/kubeadm.deb", + filepath.Join(buildDir, "debs", "kubelet.deb"): "debs/kubelet.deb", + filepath.Join(buildDir, "debs", "kubectl.deb"): "debs/kubectl.deb", + filepath.Join(buildDir, "debs", "kubernetes-cni.deb"): "debs/kubernetes-cni.deb", + filepath.Join(buildDir, "debs", "cri-tools.deb"): "debs/cri-tools.deb", + // docker images + filepath.Join(buildDir, "kube-apiserver.tar"): "images/kube-apiserver.tar", + filepath.Join(buildDir, "kube-controller-manager.tar"): "images/kube-controller-manager.tar", + filepath.Join(buildDir, "kube-scheduler.tar"): "images/kube-scheduler.tar", + filepath.Join(buildDir, "kube-proxy.tar"): "images/kube-proxy.tar", + // version files + filepath.Join(kubeRoot, "_output", "git_version"): "version", + }, + } + return bits, nil +} + +// Build implements Bits.Build +func (b *BazelBuildBits) Build() error { + // TODO(bentheelder): support other modes of building + // cd to k8s source + cwd, err := os.Getwd() + if err != nil { + return err + } + os.Chdir(b.kubeRoot) + // make sure we cd back when done + defer os.Chdir(cwd) + + // capture version info + buildVersionFile(b.kubeRoot) + return nil + + // build artifacts + cmd := exec.Command("bazel", "build") + cmd.Args = append(cmd.Args, + // TODO(bentheelder): we assume linux amd64, but we could select + // this based on Arch etc. throughout, this flag supports GOOS/GOARCH + "--platforms=@io_bazel_rules_go//go/toolchain:linux_amd64", + // we want the debian packages + "//build/debs:debs", + // and the docker images + //"//cluster/images/hyperkube:hyperkube.tar", + "//build:docker-artifacts", + ) + cmd.Debug = true + cmd.InheritOutput = true + return cmd.Run() +} + +// Paths implements Bits.Paths +func (b *BazelBuildBits) Paths() map[string]string { + // TODO(bentheelder): maybe copy the map before returning /shrug + return b.paths +} + +// Install implements Bits.Install +func (b *BazelBuildBits) Install(install InstallContext) error { + base := install.BasePath() + + debs := path.Join(base, "debs", "*.deb") + + if err := install.Run("/bin/sh", "-c", "dpkg -i "+debs); err != nil { + glog.Errorf("Image install failed! %v", err) + return err + } + + if err := install.Run("/bin/sh", "-c", + "rm -rf /kind/bits/debs/*.deb"+ + " /var/cache/debconf/* /var/lib/apt/lists/* /var/log/*kg", + ); err != nil { + glog.Errorf("Image install failed! %v", err) + return err + } + + return nil +} diff --git a/pkg/build/kube/bits.go b/pkg/build/kube/bits.go new file mode 100644 index 00000000..b68329f9 --- /dev/null +++ b/pkg/build/kube/bits.go @@ -0,0 +1,93 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kube + +import ( + "fmt" + "sync" +) + +// Bits provides the locations of Kubernetes Binaries / Images +// needed on the cluster nodes +// Implementations should be registered with RegisterNamedBits +type Bits interface { + // Build returns any errors encountered while building it Kubernetes. + // Some implementations (upstream binaries) may use this step to obtain + // an existing build isntead + Build() error + // Paths returns a map of path on host machine to desired path in the image + // These paths will be populated in the image relative to some base path, + // obtainable by NodeInstall.BasePath() + // Note: if Images are populated in iamges/, the cluster provisioning + // will load these prior to calling kubeadm + Paths() map[string]string + // Install should install the built sources on the node, assuming paths + // have been populated + Install(InstallContext) error +} + +// InstallContext should be implemented by users of Bits +// to allow installing the bits in a Docker image +type InstallContext interface { + // Returns the base path Paths() were populated relative to + BasePath() string + // Run execs (cmd, ...args) in the build container and returns error + Run(string, ...string) error + // CombinedOutputLines is like Run but returns the output lines + CombinedOutputLines(string, ...string) ([]string, error) +} + +// NewNamedBits returns a new Bits by named implementation +// currently this includes: +// "bazel" -> NewBazelBuildBits(kubeRoot) +// "docker" or "make" -> NewDockerBuildBits(kubeRoot) +// "apt" -> NewAptBits(kubeRoot) +func NewNamedBits(name string, kubeRoot string) (bits Bits, err error) { + bitsImpls.Lock() + fn, ok := bitsImpls.impls[name] + bitsImpls.Unlock() + if !ok { + return nil, fmt.Errorf("no Bits implementation with name: %s", name) + } + return fn(kubeRoot) +} + +// RegisterNamedBits registers a new named Bits implementation for use from +// NewNamedBits +func RegisterNamedBits(name string, fn func(string) (Bits, error)) { + bitsImpls.Lock() + bitsImpls.impls[name] = fn + bitsImpls.Unlock() +} + +// NamedBitsRegistered returns true if name is in the registry backing +// NewNamedBits +func NamedBitsRegistered(name string) bool { + var ok bool + bitsImpls.Lock() + _, ok = bitsImpls.impls[name] + bitsImpls.Unlock() + return ok +} + +// internal registry of named bits implementations +var bitsImpls = struct { + impls map[string]func(string) (Bits, error) + sync.Mutex +}{ + impls: map[string]func(string) (Bits, error){}, +} diff --git a/pkg/build/kube/doc.go b/pkg/build/kube/doc.go new file mode 100644 index 00000000..507301ca --- /dev/null +++ b/pkg/build/kube/doc.go @@ -0,0 +1,19 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package kube implements functionality to build Kubernetes for the purposes +// of installing into the kind node image +package kube diff --git a/pkg/build/kube/dockerbuildbits.go b/pkg/build/kube/dockerbuildbits.go new file mode 100644 index 00000000..e5a9aa26 --- /dev/null +++ b/pkg/build/kube/dockerbuildbits.go @@ -0,0 +1,189 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kube + +import ( + "os" + "path" + "path/filepath" + "strings" + + "github.com/pkg/errors" + + "k8s.io/test-infra/kind/pkg/exec" +) + +// DockerBuildBits implements Bits for a local docke-ized make / bash build +type DockerBuildBits struct { + kubeRoot string + paths map[string]string +} + +var _ Bits = &DockerBuildBits{} + +func init() { + RegisterNamedBits("docker", NewDockerBuildBits) + RegisterNamedBits("make", NewDockerBuildBits) +} + +// NewDockerBuildBits returns a new Bits backed by the docker-ized build, +// given kubeRoot, the path to the kubernetes source directory +func NewDockerBuildBits(kubeRoot string) (bits Bits, err error) { + // https://docs.Docker.build/versions/master/output_directories.html + binDir := filepath.Join(kubeRoot, + "_output", "dockerized", "bin", "linux", "amd64", + ) + imageDir := filepath.Join(kubeRoot, + "_output", "release-images", "amd64", + ) + bits = &DockerBuildBits{ + kubeRoot: kubeRoot, + paths: map[string]string{ + // binaries (hyperkube) + filepath.Join(binDir, "kubeadm"): "bin/kubeadm", + filepath.Join(binDir, "kubelet"): "bin/kubelet", + filepath.Join(binDir, "kubectl"): "bin/kubectl", + // docker images + filepath.Join(imageDir, "kube-apiserver.tar"): "images/kube-apiserver.tar", + filepath.Join(imageDir, "kube-controller-manager.tar"): "images/kube-controller-manager.tar", + filepath.Join(imageDir, "kube-scheduler.tar"): "images/kube-scheduler.tar", + filepath.Join(imageDir, "kube-proxy.tar"): "images/kube-proxy.tar", + // version files + filepath.Join(kubeRoot, "_output", "git_version"): "version", + // borrow kubelet service files from bazel debians + // TODO(bentheelder): probably we should use our own config instead :-) + filepath.Join(kubeRoot, "build", "debs", "kubelet.service"): "systemd/kubelet.service", + filepath.Join(kubeRoot, "build", "debs", "10-kubeadm.conf"): "systemd/10-kubeadm.conf", + }, + } + return bits, nil +} + +// Build implements Bits.Build +func (b *DockerBuildBits) Build() error { + // TODO(bentheelder): support other modes of building + // cd to k8s source + cwd, err := os.Getwd() + if err != nil { + return err + } + os.Chdir(b.kubeRoot) + // make sure we cd back when done + defer os.Chdir(cwd) + + // capture version info + err = buildVersionFile(b.kubeRoot) + if err != nil { + return err + } + + // build binaries + cmd := exec.Command("build/run.sh", "make", "all") + what := []string{ + "cmd/kubeadm", + "cmd/kubectl", + "cmd/kubelet", + "cmd/cloud-controller-manager", + "cmd/kube-apiserver", + "cmd/kube-controller-manager", + "cmd/kube-scheduler", + "cmd/kube-proxy", + } + cmd.Args = append(cmd.Args, + "WHAT="+strings.Join(what, " "), "KUBE_BUILD_PLATFORMS=linux/amd64", + ) + cmd.Env = append(cmd.Env, os.Environ()...) + cmd.Env = append(cmd.Env, "KUBE_VERBOSE=0") + cmd.Debug = true + cmd.InheritOutput = true + err = cmd.Run() + if err != nil { + return errors.Wrap(err, "failed to build binaries") + } + + // TODO(bentheelder): this is perhaps a bit overkill + // the build will fail if they are already present though + // We should find what `make quick-release` does and mimic that + err = os.RemoveAll(filepath.Join( + ".", "_output", "release-images", "amd64", + )) + if err != nil { + return errors.Wrap(err, "failed to remove old release-images") + } + + // build images + // TODO(bentheelder): there has to be a better way to do this, but the + // closest seems to be make quick-release, which builds more than we need + buildImages := []string{ + "source build/common.sh;", + "source hack/lib/version.sh;", + "source build/lib/release.sh;", + "kube::version::get_version_vars;", + `kube::release::create_docker_images_for_server "${LOCAL_OUTPUT_ROOT}/dockerized/bin/linux/amd64" "amd64"`, + } + cmd = exec.Command("bash", "-c", strings.Join(buildImages, " ")) + cmd.Env = append(cmd.Env, os.Environ()...) + cmd.Env = append(cmd.Env, "KUBE_BUILD_HYPERKUBE=n") + cmd.Debug = true + cmd.InheritOutput = true + err = cmd.Run() + if err != nil { + return errors.Wrap(err, "failed to build images") + } + + return nil +} + +// Paths implements Bits.Paths +func (b *DockerBuildBits) Paths() map[string]string { + // TODO(bentheelder): maybe copy the map before returning /shrug + return b.paths +} + +// Install implements Bits.Install +func (b *DockerBuildBits) Install(install InstallContext) error { + kindBinDir := path.Join(install.BasePath(), "bin") + + // symlink the kubernetes binaries into $PATH + binaries := []string{"kubeadm", "kubelet", "kubectl"} + for _, binary := range binaries { + if err := install.Run("ln", "-s", + path.Join(kindBinDir, binary), + path.Join("/usr/bin/", binary), + ); err != nil { + return errors.Wrap(err, "failed to symlink binaries") + } + } + + // enable the kubelet service + kubeletService := path.Join(install.BasePath(), "systemd/kubelet.service") + if err := install.Run("systemctl", "enable", kubeletService); err != nil { + return errors.Wrap(err, "failed to enable kubelet service") + } + + // setup the kubelet dropin + kubeletDropinSource := path.Join(install.BasePath(), "systemd/10-kubeadm.conf") + kubeletDropin := "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf" + if err := install.Run("mkdir", "-p", path.Dir(kubeletDropin)); err != nil { + return errors.Wrap(err, "failed to configure kubelet service") + } + if err := install.Run("cp", kubeletDropinSource, kubeletDropin); err != nil { + return errors.Wrap(err, "failed to configure kubelet service") + } + + return nil +} diff --git a/pkg/build/kubebits.go b/pkg/build/kube/source.go similarity index 56% rename from pkg/build/kubebits.go rename to pkg/build/kube/source.go index 168ddd9b..e515c050 100644 --- a/pkg/build/kubebits.go +++ b/pkg/build/kube/source.go @@ -14,33 +14,21 @@ See the License for the specific language governing permissions and limitations under the License. */ -package build +package kube import ( "fmt" - gobuild "go/build" + "go/build" ) -// KubeBits provides the locations of Kubernetes Binaries / Images -// needed on the cluster nodes -type KubeBits interface { - // Paths returns a map of path on host to desired path in the image - Paths() map[string]string -} +// ImportPath is the canonical import path for the kubernetes root package +// this is used by FindSource +const ImportPath = "k8s.io/kubernetes" -// NodeInstall should be implemented by users of KubeBitsProvider -// to allow installing the bits -type NodeInstall interface { - // RunOnNode execs (cmd, ...args) on a node and returns error - RunOnNode(string, ...string) error -} - -const kubeImportPath = "k8s.io/kubernetes" - -// FindKubeSource attempts to locate a kubernetes checkout using go's build package -func FindKubeSource() (root string, err error) { +// FindSource attempts to locate a kubernetes checkout using go's build package +func FindSource() (root string, err error) { // look up the source the way go build would - pkg, err := gobuild.Default.Import(kubeImportPath, ".", gobuild.FindOnly) + pkg, err := build.Default.Import(ImportPath, ".", build.FindOnly) if err == nil && maybeKubeDir(pkg.Dir) { return pkg.Dir, nil } diff --git a/pkg/build/kube/version.go b/pkg/build/kube/version.go new file mode 100644 index 00000000..b4346f9e --- /dev/null +++ b/pkg/build/kube/version.go @@ -0,0 +1,64 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kube + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "k8s.io/test-infra/kind/pkg/exec" +) + +// buildVersionFile creates a file for the kubernetes git version in +// ./_output/version based on hack/print-workspace-status.sh, +// these are built into the node image and consumed by the cluster tooling +func buildVersionFile(kubeRoot string) error { + cwd, err := os.Getwd() + if err != nil { + return err + } + os.Chdir(kubeRoot) + // make sure we cd back when done + defer os.Chdir(cwd) + + // get the version output + cmd := exec.Command("hack/print-workspace-status.sh") + cmd.Debug = true + output, err := cmd.CombinedOutputLines() + if err != nil { + return err + } + outputDir := filepath.Join(kubeRoot, "_output") + // parse it, and populate it into _output/git_version + for _, line := range output { + parts := strings.SplitN(line, " ", 2) + if len(parts) != 2 { + return fmt.Errorf("could not parse kubernetes version") + } + if parts[0] == "gitVersion" { + ioutil.WriteFile( + filepath.Join(outputDir, "git_version"), + []byte(parts[1]), + 0777, + ) + } + } + return nil +} diff --git a/pkg/build/localbuildbits.go b/pkg/build/localbuildbits.go deleted file mode 100644 index 05c65ec1..00000000 --- a/pkg/build/localbuildbits.go +++ /dev/null @@ -1,52 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package build - -import "path/filepath" - -// BazelBuildBits implements KubeBits for a local Bazel build -type BazelBuildBits struct { - paths map[string]string -} - -var _ KubeBits = &BazelBuildBits{} - -func (l *BazelBuildBits) Paths() map[string]string { - // TODO(bentheelder): maybe copy the map before returning /shrug - return l.paths -} - -func NewBazelBuildBits(kubeRoot string) (bits KubeBits, err error) { - // https://docs.bazel.build/versions/master/output_directories.html - binDir := filepath.Join(kubeRoot, "bazel-bin") - bits = &BazelBuildBits{ - paths: map[string]string{ - // debians - filepath.Join(binDir, "build", "debs", "kubeadm.deb"): "debs/kubeadm.deb", - filepath.Join(binDir, "build", "debs", "kubelet.deb"): "debs/kubelet.deb", - filepath.Join(binDir, "build", "debs", "kubectl.deb"): "debs/kubectl.deb", - filepath.Join(binDir, "build", "debs", "kubernetes-cni.deb"): "debs/kubernetes-cni.deb", - filepath.Join(binDir, "build", "debs", "cri-tools.deb"): "debs/cri-tools.deb", - // docker images - filepath.Join(binDir, "build", "kube-proxy.tar"): "images/kube-proxy.tar", - filepath.Join(binDir, "build", "kube-controller-manager.tar"): "images/kube-controller-manager.tar", - filepath.Join(binDir, "build", "kube-scheduler.tar"): "images/kube-scheduler.tar", - filepath.Join(binDir, "build", "kube-apiserver.tar"): "images/kube-apiserver.tar", - }, - } - return bits, nil -} diff --git a/pkg/build/node_image.go b/pkg/build/node_image.go index 0f12734a..37f60841 100644 --- a/pkg/build/node_image.go +++ b/pkg/build/node_image.go @@ -14,8 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package build implements functionality to build the kind images -// TODO(bentheelder): and k8s package build import ( @@ -28,6 +26,7 @@ import ( "github.com/golang/glog" "github.com/pkg/errors" + "k8s.io/test-infra/kind/pkg/build/kube" "k8s.io/test-infra/kind/pkg/build/sources" "k8s.io/test-infra/kind/pkg/exec" ) @@ -39,35 +38,46 @@ type NodeImageBuildContext struct { ImageTag string Arch string BaseImage string + KubeRoot string + Bits kube.Bits } // NewNodeImageBuildContext creates a new NodeImageBuildContext with // default configuration -func NewNodeImageBuildContext() *NodeImageBuildContext { +func NewNodeImageBuildContext(mode string) (ctx *NodeImageBuildContext, err error) { + kubeRoot := "" + // apt should not fail on finding kube root as it does not use it + if mode != "apt" { + kubeRoot, err = kube.FindSource() + if err != nil { + return nil, fmt.Errorf("error finding kuberoot: %v", err) + } + } + bits, err := kube.NewNamedBits(mode, kubeRoot) + if err != nil { + return nil, err + } return &NodeImageBuildContext{ ImageTag: "kind-node", Arch: "amd64", BaseImage: "kind-base", - } + KubeRoot: kubeRoot, + Bits: bits, + }, nil } // Build builds the cluster node image, the sourcedir must be set on // the NodeImageBuildContext func (c *NodeImageBuildContext) Build() (err error) { - // get k8s source - kubeRoot, err := FindKubeSource() - if err != nil { - return errors.Wrap(err, "could not find kubernetes source") - } - // ensure kubernetes build is up to date first glog.Infof("Starting to build Kubernetes") - //c.buildKube(kubeRoot) + if err = c.Bits.Build(); err != nil { + glog.Errorf("Failed to build Kubernetes: %v", err) + return errors.Wrap(err, "failed to build kubernetes") + } glog.Infof("Finished building Kubernetes") - // TODO(bentheelder): allow other types of bits - bits, err := NewBazelBuildBits(kubeRoot) - // create tempdir to build in + // create tempdir to build the image in tmpDir, err := TempDir("", "kind-node-image") if err != nil { return err @@ -96,48 +106,25 @@ func (c *NodeImageBuildContext) Build() (err error) { glog.Infof("Building node image in: %s", buildDir) // populate the kubernetes artifacts first - if err := c.populateBits(buildDir, bits); err != nil { + if err := c.populateBits(buildDir); err != nil { return err } - // then the actual docker image + // then the perform the actual docker image build return c.buildImage(buildDir) } -func (c *NodeImageBuildContext) buildKube(kubeRoot string) error { - // TODO(bentheelder): support other modes of building - // cd to k8s source - cwd, err := os.Getwd() - if err != nil { - return err +func (c *NodeImageBuildContext) populateBits(buildDir string) error { + // always create bits dir + bitsDir := path.Join(buildDir, "bits") + if err := os.Mkdir(bitsDir, 0777); err != nil { + return errors.Wrap(err, "failed to make bits dir") } - os.Chdir(kubeRoot) - // make sure we cd back when done - defer os.Chdir(cwd) - - // TODO(bentheelder): move this out and next to the KubeBits impl - cmd := exec.Command("bazel", "build") - cmd.Args = append(cmd.Args, - // TODO(bentheelder): we assume linux amd64, but we could select - // this based on Arch etc. throughout, this flag supports GOOS/GOARCH - "--platforms=@io_bazel_rules_go//go/toolchain:linux_amd64", - // we want the debian packages - "//build/debs:debs", - // and the docker images - "//build:docker-artifacts", - ) - - cmd.Debug = true - cmd.InheritOutput = true - return cmd.Run() -} - -func (c *NodeImageBuildContext) populateBits(buildDir string, bits KubeBits) error { // copy all bits from their source path to where we will COPY them into // the dockerfile, see images/node/Dockerfile - bitPaths := bits.Paths() + bitPaths := c.Bits.Paths() for src, dest := range bitPaths { - realDest := path.Join(buildDir, "files", dest) + realDest := path.Join(bitsDir, dest) if err := copyFile(src, realDest); err != nil { return errors.Wrap(err, "failed to copy build artifact") } @@ -148,6 +135,33 @@ func (c *NodeImageBuildContext) populateBits(buildDir string, bits KubeBits) err // BuildContainerLabelKey is applied to each build container const BuildContainerLabelKey = "io.k8s.test-infra.kind-build" +// private kube.InstallContext implementation, local to the image build +type installContext struct { + basePath string + containerID string +} + +var _ kube.InstallContext = &installContext{} + +func (ic *installContext) BasePath() string { + return ic.basePath +} + +func (ic *installContext) Run(command string, args ...string) error { + cmd := exec.Command("docker", "exec", ic.containerID, command) + cmd.Args = append(cmd.Args, args...) + cmd.Debug = true + cmd.InheritOutput = true + return cmd.Run() +} + +func (ic *installContext) CombinedOutputLines(command string, args ...string) ([]string, error) { + cmd := exec.Command("docker", "exec", ic.containerID, command) + cmd.Args = append(cmd.Args, args...) + cmd.Debug = true + return cmd.CombinedOutputLines() +} + func (c *NodeImageBuildContext) buildImage(dir string) error { // build the image, tagged as tagImageAs, using the our tempdir as the context glog.Info("Starting image build ...") @@ -155,6 +169,8 @@ func (c *NodeImageBuildContext) buildImage(dir string) error { // NOTE: we are using docker run + docker commit so we can install // debians without permanently copying them into the image. // if docker gets proper squash support, we can rm them instead + // This also allows the KubeBit implementations to perform programmatic + // isntall in the image containerID, err := c.createBuildContainer(dir) if err != nil { glog.Errorf("Image build Failed! %v", err) @@ -176,27 +192,23 @@ func (c *NodeImageBuildContext) buildImage(dir string) error { } // make artifacts directory - if err = execInBuild("mkdir", "-p", "/kind/bits"); err != nil { + if err = execInBuild("mkdir", "/kind/"); err != nil { glog.Errorf("Image build Failed! %v", err) return err } // copy artifacts in - if err = execInBuild("rsync", "-r", "/build/files/", "/kind/bits/"); err != nil { + if err = execInBuild("rsync", "-r", "/build/bits/", "/kind/"); err != nil { glog.Errorf("Image build Failed! %v", err) return err } - // install debs - if err = execInBuild("/bin/sh", "-c", "dpkg -i /kind/bits/debs/*.deb"); err != nil { - glog.Errorf("Image build Failed! %v", err) - return err + // install the kube bits + ic := &installContext{ + basePath: "/kind/", + containerID: containerID, } - - // clean up after debs / remove them, this saves a couple hundred MB - if err = execInBuild("/bin/sh", "-c", - "rm -rf /kind/bits/debs/*.deb /var/cache/debconf/* /var/lib/apt/lists/* /var/log/*kg", - ); err != nil { + if err = c.Bits.Install(ic); err != nil { glog.Errorf("Image build Failed! %v", err) return err } diff --git a/pkg/build/sources/generate.go b/pkg/build/sources/generate.go index 01e6b4df..ebdeea4a 100644 --- a/pkg/build/sources/generate.go +++ b/pkg/build/sources/generate.go @@ -15,7 +15,7 @@ limitations under the License. */ // Package sources contains the baked in sources kind needs to build. -// Primarily this includes the node-image dockerfile, which should rarely +// Primarily this includes the node-image Dockerfile, which should rarely // change. // These can be overridden with newer files at build-time, see ./../build package sources diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 01fc23a9..5ef6af51 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -14,15 +14,19 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package cluster implements kind local cluster management package cluster import ( "fmt" + "io/ioutil" + "os" "time" + "github.com/golang/glog" + "github.com/pkg/errors" + "k8s.io/test-infra/kind/pkg/cluster/kubeadm" "k8s.io/test-infra/kind/pkg/exec" ) @@ -30,7 +34,6 @@ import ( // kubernetes-in-docker clusters type Context struct { config Config - // TODO(bentheelder): fill this in } // NewContext returns a new cluster management context with Config config @@ -46,11 +49,27 @@ func (c *Context) Create() error { if err := c.config.Validate(); err != nil { return err } - // create a temp dir to stick kubeconfig in - // TODO(bentheelder): more advanced provisioning - // TODO(bentheelder): multiple nodes - return c.provisionNode() + // TODO(bentheelder): multiple nodes ... + kubeadmConfig, err := c.provisionControlPlane( + fmt.Sprintf("kind-%s-control-plane", c.config.Name), + ) + + // clean up the kubeadm config file + // NOTE: in the future we will use this for other nodes first + if kubeadmConfig != "" { + defer os.Remove(kubeadmConfig) + } + + if err != nil { + return err + } + + println("\nYou can now use the cluster with:\n") + println("export KUBECONFIG=\"" + c.config.KubeConfigPath() + "\"") + println("kubectl cluster-info\n") + + return nil } // Delete tears down a kubernetes-in-docker cluster @@ -62,116 +81,138 @@ func (c *Context) Delete() error { return c.deleteNodes(nodes...) } -func (c *Context) provisionNode() error { - // TODO(bentheelder): multiple nodes... - nodeName := "kind-" + c.config.Name + "-control-plane" +// provisionControlPlane provisions the control plane node +// and the cluster kubeadm config +func (c *Context) provisionControlPlane(name string) (kubeadmConfigPath string, err error) { // create the "node" container (docker run, but it is paused, see createNode) - if err := c.createNode(nodeName); err != nil { - return err + node, err := createNode(name, c.config.clusterLabel()) + if err != nil { + return "", err } // systemd-in-a-container should have read only /sys // https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/ // however, we need other things from `docker run --privileged` ... // and this flag also happens to make /sys rw, amongst other things - if err := c.runOnNode(nodeName, []string{ - "mount", "-o", "remount,ro", "/sys", - }); err != nil { + if err := node.Run("mount", "-o", "remount,ro", "/sys"); err != nil { // TODO(bentheelder): logging here // TODO(bentheelder): add a flag to retain the broken nodes for debugging - c.deleteNodes(nodeName) - return err + c.deleteNodes(node.nameOrID) + return "", err } - // TODO(bentheelder): insert other provisioning here - // (eg enabling / disabling units, installing kube...) - // signal the node to boot into systemd - if err := c.actuallyStartNode(nodeName); err != nil { + // signal the node entrypoint to continue booting into systemd + if err := node.SignalStart(); err != nil { // TODO(bentheelder): logging here - c.deleteNodes(nodeName) - return err + // TODO(bentheelder): add a flag to retain the broken nodes for debugging + c.deleteNodes(node.nameOrID) + return "", err } // wait for docker to be ready - if !tryUntil(time.Now().Add(time.Second*30), func() bool { - out, err := c.outputOnNode(nodeName, []string{"systemctl", "is-active", "docker"}) - if err != nil { - return false - } - return len(out) == 1 && out[0] == "active" - }) { - c.deleteNodes(nodeName) - return fmt.Errorf("timed out waiting for docker to be ready on node") + if !node.WaitForDocker(time.Now().Add(time.Second * 30)) { + // TODO(bentheelder): logging here + // TODO(bentheelder): add a flag to retain the broken nodes for debugging + c.deleteNodes(node.nameOrID) + return "", fmt.Errorf("timed out waiting for docker to be ready on node") } - // run kubeadm init - // TODO(bentheelder): configure properly, ensure it uses images we built... - if err := c.runOnNode(nodeName, []string{ - // kubeadm init because this is the control plane node + // load the docker image artifacts into the docker daemon + node.LoadImages() + + // get installed kubernetes version from the node image + kubeVersion, err := node.KubeVersion() + if err != nil { + // TODO(bentheelder): logging here + // TODO(bentheelder): add a flag to retain the broken nodes for debugging + c.deleteNodes(node.nameOrID) + return "", fmt.Errorf("failed to get kubernetes version from node: %v", err) + } + + // create kubeadm config file + kubeadmConfig, err := c.createKubeadmConfig("", kubeadm.ConfigData{ + ClusterName: c.config.ClusterName(), + KubernetesVersion: kubeVersion, + }) + + // copy the config to the node + if err := node.CopyTo(kubeadmConfig, "/kind/kubeadm.conf"); err != nil { + // TODO(bentheelder): logging here + // TODO(bentheelder): add a flag to retain the broken nodes for debugging + c.deleteNodes(node.nameOrID) + return kubeadmConfig, errors.Wrap(err, "failed to copy kubeadm config to node") + } + + // run kubeadm + if err := node.Run( + // init because this is the control plane node "kubeadm", "init", - // preflight errors are expected, in particular for swap + // preflight errors are expected, in particular for swap being enabled + // TODO(bentheelder): limit the set of acceptable errors "--ignore-preflight-errors=all", - // on docker for mac we have to expose the api server on localhost - "--apiserver-cert-extra-sans=localhost", - }); err != nil { - c.deleteNodes(nodeName) - return errors.Wrap(err, "failed to init node with kubeadm") + // specify our generated config file + "--config=/kind/kubeadm.conf", + ); err != nil { + // TODO(bentheelder): logging here + // TODO(bentheelder): add a flag to retain the broken nodes for debugging + c.deleteNodes(node.nameOrID) + return kubeadmConfig, errors.Wrap(err, "failed to init node with kubeadm") } - // TODO(bentheelder): apply an overlay network + // set up the $KUBECONFIG + kubeConfigPath := c.config.KubeConfigPath() + if err = node.WriteKubeConfig(kubeConfigPath); err != nil { + // TODO(bentheelder): logging here + // TODO(bentheelder): add a flag to retain the broken nodes for debugging + c.deleteNodes(node.nameOrID) + return kubeadmConfig, errors.Wrap(err, "failed to get kubeconfig from node") + } - return nil -} + // TODO(bentheelder): support other overlay networks + if err = node.Run( + "/bin/sh", "-c", + `kubectl apply --kubeconfig=/etc/kubernetes/admin.conf -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')"`, + ); err != nil { + return kubeadmConfig, errors.Wrap(err, "failed to apply overlay network") + } -// call `try()`` in a loop until the deadline `until` has passed or `try()` -// returns true, returns wether try every returned true -func tryUntil(until time.Time, try func() bool) bool { - now := time.Now() - for until.After(now) { - if try() { - return true + // if we are only provisioning one node, remove the master taint + // https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/#master-isolation + if c.config.NumNodes == 1 { + if err = node.Run( + "kubectl", "--kubeconfig=/etc/kubernetes/admin.conf", + "taint", "nodes", "--all", "node-role.kubernetes.io/master-", + ); err != nil { + return kubeadmConfig, errors.Wrap(err, "failed to remove master taint") } } - return false + + return kubeadmConfig, nil } -// createNode `docker run`s the node image, note that due to -// images/node/entrypoint being the entrypoint, this container will -// effectively be paused until we call actuallyStartNode(...) -func (c *Context) createNode(name string) error { - // TODO(bentheelder): use config - // TODO(bentheelder): logging - // TODO(bentheelder): many of these flags should be derived from the config - cmd := exec.Command("docker", "run") - cmd.Args = append(cmd.Args, - "-d", // run the container detached - "-t", // we need a pseudo-tty for systemd logs - // running containers in a container requires privileged - // NOTE: we could try to replicate this with --cap-add, and use less - // privileges, but this flag also changes some mounts that are necessary - // including some ones docker would otherwise do by default. - // for now this is what we want. in the future we may revisit this. - "--privileged", - "--security-opt", "seccomp=unconfined", // also ignore seccomp - "--tmpfs", "/tmp", // various things depend on working /tmp - "--tmpfs", "/run", // systemd wants a writable /run - // docker in docker needs this, so as not to stack overlays - "--tmpfs", "/var/lib/docker:exec", - //"-v", "/sys/fs/cgroup:/sys/fs/cgroup:ro", - // some k8s things want /lib/modules - "-v", "/lib/modules:/lib/modules:ro", - "--hostname", name, // make hostname match container name - "--name", name, // ... and set the container name - // label the node with the cluster ID - "--label", c.config.clusterLabel(), - // expose API server - // TODO(bentheelder): this should probably be configurable - "-p", "6443:6443", - "kind-node", // use our image, TODO: make this configurable - ) - // TODO(bentheelder): collect output instead of connecting these - cmd.InheritOutput = true - return cmd.Run() +// createKubeadmConfig creates the kubeadm config file for the cluster +// by running data through the template and writing it to a temp file +// the config file path is returned, this file should be removed later +func (c *Context) createKubeadmConfig(template string, data kubeadm.ConfigData) (path string, err error) { + // create kubeadm config file + f, err := ioutil.TempFile("", "") + if err != nil { + return "", errors.Wrap(err, "failed to create kubeadm config") + } + path = f.Name() + // generate the config contents + config, err := kubeadm.Config(template, data) + if err != nil { + os.Remove(path) + return "", err + } + glog.Infof("Using KubeadmConfig:\n\n%s\n", config) + _, err = f.WriteString(config) + if err != nil { + os.Remove(path) + return "", err + } + return path, nil } func (c *Context) deleteNodes(names ...string) error { @@ -183,51 +224,6 @@ func (c *Context) deleteNodes(names ...string) error { return cmd.Run() } -// runOnNode execs command on the named node -func (c *Context) runOnNode(nameOrID string, command []string) error { - cmd := exec.Command("docker", "exec") - cmd.Args = append(cmd.Args, - "-t", // use a tty so we can get output - "--privileged", // run with priliges so we can remount etc.. - nameOrID, // ... against the "node" container - ) - cmd.Args = append(cmd.Args, - command..., // finally, run the command supplied by the user - ) - // TODO(bentheelder): collect output instead of connecting these - cmd.InheritOutput = true - return cmd.Run() -} - -// outputOnNode execs command on the named node, returning the output lines -func (c *Context) outputOnNode(nameOrID string, command []string) ([]string, error) { - cmd := exec.Command("docker", "exec") - cmd.Args = append(cmd.Args, - "-t", // use a tty so we can get output - "--privileged", // run with priliges so we can remount etc.. - nameOrID, // ... against the "node" container - ) - cmd.Args = append(cmd.Args, - command..., // finally, run the command supplied by the user - ) - // TODO(bentheelder): collect output instead of connecting these - return cmd.CombinedOutputLines() -} - -// signal our entrypoint (images/node/entrypoint) to boot -func (c *Context) actuallyStartNode(name string) error { - // TODO(bentheelder): use config - // TODO(bentheelder): logging - cmd := exec.Command("docker", "kill") - cmd.Args = append(cmd.Args, - "-s", "SIGUSR1", - name, - ) - // TODO(bentheelder): collect output instead of connecting these - cmd.InheritOutput = true - return cmd.Run() -} - // ListNodes returns the list of container IDs for the "nodes" in the cluster func (c *Context) ListNodes(alsoStopped bool) (containerIDs []string, err error) { cmd := exec.Command("docker", "ps") @@ -237,7 +233,7 @@ func (c *Context) ListNodes(alsoStopped bool) (containerIDs []string, err error) // filter for nodes with the cluster label "--filter", "label="+c.config.clusterLabel(), ) - // optionally show nodes that are stopped + // optionally list nodes that are stopped if alsoStopped { cmd.Args = append(cmd.Args, "-a") } diff --git a/pkg/cluster/config.go b/pkg/cluster/config.go index 9b862b41..b560415a 100644 --- a/pkg/cluster/config.go +++ b/pkg/cluster/config.go @@ -19,6 +19,8 @@ package cluster import ( "bytes" "fmt" + "os" + "path/filepath" "regexp" ) @@ -28,19 +30,22 @@ const ClusterLabelKey = "io.k8s.test-infra.kind-cluster" // similar to valid docker container names, but since we will prefix // and suffix this name, we can relax it a little // see Validate() for usage -var validClusterName = regexp.MustCompile(`^[a-zA-Z0-9_.-]+$`) +var validNameRE = regexp.MustCompile(`^[a-zA-Z0-9_.-]+$`) // Config contains cluster options type Config struct { // the cluster name Name string + // the number of nodes (currently only one is supported) + NumNodes int // TODO(bentheelder): fill this in } // NewConfig returns a new cluster config with name func NewConfig(name string) Config { return Config{ - Name: name, + Name: name, + NumNodes: 1, } } @@ -48,10 +53,17 @@ func NewConfig(name string) Config { // with the config, or nil if there are none func (c *Config) Validate() error { errs := []error{} - if !validClusterName.MatchString(c.Name) { + if !validNameRE.MatchString(c.Name) { errs = append(errs, fmt.Errorf( "'%s' is not a valid cluster name, cluster names must match `%s`", - c.Name, validClusterName.String(), + c.Name, validNameRE.String(), + )) + } + // TODO(bentheelder): support multiple nodes + if c.NumNodes != 1 { + errs = append(errs, fmt.Errorf( + "%d nodes requested but only clusters with one node are supported currently", + c.NumNodes, )) } if len(errs) > 0 { @@ -87,3 +99,21 @@ func (c ConfigErrors) Errors() []error { func (c *Config) clusterLabel() string { return fmt.Sprintf("%s=%s", ClusterLabelKey, c.Name) } + +// ClusterName returns the Kubernetes cluster name based on the config +// currently this is .Name prefixed with "kind-" +func (c *Config) ClusterName() string { + return fmt.Sprintf("kind-%s", c.Name) +} + +// KubeConfigPath returns the path to where the Kubeconfig would be placed +// by kind based on the configuration. +func (c *Config) KubeConfigPath() string { + // TODO(bentheelder): Windows? + // configDir matches the standard directory expected by kubectl etc + configDir := filepath.Join(os.Getenv("HOME"), ".kube") + // note that the file name however does not, we do not want to overwite + // the standard config, though in the future we may (?) merge them + fileName := fmt.Sprintf("kind-config-%s", c.Name) + return filepath.Join(configDir, fileName) +} diff --git a/pkg/cluster/doc.go b/pkg/cluster/doc.go new file mode 100644 index 00000000..af19392f --- /dev/null +++ b/pkg/cluster/doc.go @@ -0,0 +1,18 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package cluster implements kind kubernetes-in-docker cluster management +package cluster diff --git a/pkg/cluster/kubeadm/config.go b/pkg/cluster/kubeadm/config.go new file mode 100644 index 00000000..a6616e7b --- /dev/null +++ b/pkg/cluster/kubeadm/config.go @@ -0,0 +1,88 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeadm + +import ( + "bytes" + "strings" + "text/template" + + "github.com/pkg/errors" +) + +// ConfigData is supplied to the kubeadm config template, with values populated +// by the cluster package +type ConfigData struct { + ClusterName string + KubernetesVersion string + // UnifiedControlPlaneImage - optional + UnifiedControlPlaneImage string + // AutoDerivedConfigData is populated by DeriveFields() + AutoDerivedConfigData +} + +// AutoDerivedConfigData fields are automatically derived by +// ConfigData.DeriveFieldsif they are not specified / zero valued +type AutoDerivedConfigData struct { + // DockerStableTag is automatically derived from KubernetesVersion + DockerStableTag string +} + +// DeriveFields automatically derives DockerStableTag if not specified +func (c *ConfigData) DeriveFields() { + if c.DockerStableTag == "" { + c.DockerStableTag = strings.Replace(c.KubernetesVersion, "+", "_", -1) + } +} + +// DefaultConfigTemplate is the default kubeadm config template used by kind +const DefaultConfigTemplate = `# config generated by kind +apiVersion: kubeadm.k8s.io/v1alpha2 +kind: MasterConfiguration +clusterName: {{.ClusterName}} +# on docker for mac we have to expose the api server via port forward, +# so we need to ensure the cert is valid for localhost so we can talk +# to the cluster after rewriting the kubeconfig to point to localhost +apiServerCertSANs: [localhost] +kubernetesVersion: {{.KubernetesVersion}} +{{if ne .UnifiedControlPlaneImage ""}} +# optionally specify a unified control plane image +unifiedControlPlaneImage: {{.UnifiedControlPlaneImage}}:{{.DockerStableTag}} +{{end}}` + +// Config returns a kubeadm config from the template and config data, +// if templateSource == "", DeafultConfigTemplate will be used instead +// ConfigData will be supplied to the template after conversion to ConfigTemplateData +func Config(templateSource string, data ConfigData) (config string, err error) { + // load the template, using the default if not specified + if templateSource == "" { + templateSource = DefaultConfigTemplate + } + t, err := template.New("kubeadm-config").Parse(templateSource) + if err != nil { + return "", errors.Wrap(err, "failed to parse config template") + } + // derive any automatic fields if not supplied + data.DeriveFields() + // execute the template + var buff bytes.Buffer + err = t.Execute(&buff, data) + if err != nil { + return "", errors.Wrap(err, "error executing config template") + } + return buff.String(), nil +} diff --git a/pkg/cluster/kubeadm/const.go b/pkg/cluster/kubeadm/const.go new file mode 100644 index 00000000..103b5222 --- /dev/null +++ b/pkg/cluster/kubeadm/const.go @@ -0,0 +1,20 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeadm + +// APIServerPort is the expected default APIServerPort on the control plane node(s) +const APIServerPort = 6443 diff --git a/pkg/cluster/kubeadm/doc.go b/pkg/cluster/kubeadm/doc.go new file mode 100644 index 00000000..f023db7d --- /dev/null +++ b/pkg/cluster/kubeadm/doc.go @@ -0,0 +1,18 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package kubeadm contains kubeadm related constants and configuration +package kubeadm diff --git a/pkg/cluster/node.go b/pkg/cluster/node.go new file mode 100644 index 00000000..fce4138f --- /dev/null +++ b/pkg/cluster/node.go @@ -0,0 +1,253 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cluster + +import ( + "bytes" + "fmt" + "io/ioutil" + "regexp" + "strconv" + "strings" + "time" + + "github.com/golang/glog" + "github.com/pkg/errors" + + "k8s.io/test-infra/kind/pkg/cluster/kubeadm" + "k8s.io/test-infra/kind/pkg/exec" +) + +type nodeHandle struct { + // must be one of docker container ID or name + nameOrID string +} + +// createNode `docker run`s the node image, note that due to +// images/node/entrypoint being the entrypoint, this container will +// effectively be paused until we call actuallyStartNode(...) +func createNode(name, clusterLabel string) (handle *nodeHandle, err error) { + cmd := exec.Command("docker", "run") + cmd.Args = append(cmd.Args, + "-d", // run the container detached + "-t", // we need a pseudo-tty for systemd logs + // running containers in a container requires privileged + // NOTE: we could try to replicate this with --cap-add, and use less + // privileges, but this flag also changes some mounts that are necessary + // including some ones docker would otherwise do by default. + // for now this is what we want. in the future we may revisit this. + "--privileged", + "--security-opt", "seccomp=unconfined", // also ignore seccomp + "--tmpfs", "/tmp", // various things depend on working /tmp + "--tmpfs", "/run", // systemd wants a writable /run + // docker in docker needs this, so as not to stack overlays + "--tmpfs", "/var/lib/docker:exec", + // some k8s things want /lib/modules + "-v", "/lib/modules:/lib/modules:ro", + "--hostname", name, // make hostname match container name + "--name", name, // ... and set the container name + // label the node with the cluster ID + "--label", clusterLabel, + "--expose", "6443", // expose API server port + // pick a random ephemeral port to forward to the API server + "--publish-all", + "kind-node", // use our image, TODO: make this configurable + ) + cmd.Debug = true + err = cmd.Run() + if err != nil { + return nil, err + } + return &nodeHandle{name}, nil +} + +// SignalStart sends SIGUSR1 to the node, which signals our entrypoint to boot +// see images/node/entrypoint +func (nh *nodeHandle) SignalStart() error { + cmd := exec.Command("docker", "kill") + cmd.Args = append(cmd.Args, + "-s", "SIGUSR1", + nh.nameOrID, + ) + // TODO(bentheelder): collect output instead of connecting these + cmd.InheritOutput = true + return cmd.Run() +} + +// Run execs command, args... on the node +func (nh *nodeHandle) Run(command string, args ...string) error { + cmd := exec.Command("docker", "exec") + cmd.Args = append(cmd.Args, + "-t", // use a tty so we can get output + "--privileged", // run with priliges so we can remount etc.. + nh.nameOrID, // ... against the "node" container + command, // with the command specified + ) + cmd.Args = append(cmd.Args, + args..., // finally, with the args specified + ) + cmd.InheritOutput = true + return cmd.Run() +} + +// CombinedOutputLines execs command, args... on the node, returning the output lines +func (nh *nodeHandle) CombinedOutputLines(command string, args ...string) ([]string, error) { + cmd := exec.Command("docker", "exec") + cmd.Args = append(cmd.Args, + "-t", // use a tty so we can get output + "--privileged", // run with priliges so we can remount etc.. + nh.nameOrID, // ... against the "node" container + command, // with the command specified + ) + cmd.Args = append(cmd.Args, + args..., // finally, with the args specified + ) + return cmd.CombinedOutputLines() +} + +// helper to copy source file to dest on the node +func (nh *nodeHandle) CopyTo(source, dest string) error { + cmd := exec.Command("docker", "cp") + cmd.Args = append(cmd.Args, + source, // from the source file + nh.nameOrID+":"+dest, // to the node, at dest + ) + cmd.InheritOutput = true + return cmd.Run() +} + +// WaitForDocker waits for Docker to be ready on the node +// it returns true on success, and false on a timeout +func (nh *nodeHandle) WaitForDocker(until time.Time) bool { + return tryUntil(until, func() bool { + out, err := nh.CombinedOutputLines("systemctl", "is-active", "docker") + if err != nil { + return false + } + return len(out) == 1 && out[0] == "active" + }) +} + +// helper that calls `try()`` in a loop until the deadline `until` +// has passed or `try()`returns true, returns wether try ever returned true +func tryUntil(until time.Time, try func() bool) bool { + now := time.Now() + for until.After(now) { + if try() { + return true + } + } + return false +} + +// LoadImages loads image tarballs stored on the node into docker on the node +func (nh *nodeHandle) LoadImages() { + // load images cached on the node into docker + if err := nh.Run( + "find", + "/kind/images", + "-name", "*.tar", + "-exec", "docker", "load", "-i", "{}", ";", + ); err != nil { + glog.Warningf("Failed to preload docker images: %v", err) + return + } + // retag images that are missing -amd64 as image:tag -> image-amd64:tag + // bazel built images are currently missing these + // TODO(bentheelder): this is a bit gross, move this logic out of bash + if err := nh.Run( + "/bin/bash", "-c", + `docker images --format='{{.Repository}}:{{.Tag}}' | grep -v amd64 | xargs -L 1 -I '{}' /bin/bash -c 'docker tag "{}" "$(echo "{}" | sed s/:/-amd64:/)"'`, + ); err != nil { + glog.Warningf("Failed to re-tag docker images: %v", err) + } + + nh.Run("docker", "images") +} + +// KubeVersion returns the Kubernetes version installed on the node +func (nh *nodeHandle) KubeVersion() (version string, err error) { + // grab kubernetes version from the node image + lines, err := nh.CombinedOutputLines("cat", "/kind/version") + if err != nil { + return "", errors.Wrap(err, "failed to get file") + } + if len(lines) != 1 { + return "", fmt.Errorf("file should only be one line, got %d lines", len(lines)) + } + return lines[0], nil +} + +// matches kubeconfig server entry like: +// server: https://172.17.0.2:6443 +// which we rewrite to: +// server: https://localhost:$PORT +var serverAddressRE = regexp.MustCompile(`^(\s+server:) https://.*:\d+$`) + +// WriteKubeConfig writes a fixed KUBECONFIG to dest +// this should only be called on a control plane node +func (nh *nodeHandle) WriteKubeConfig(dest string) error { + // get the forwarded api server port + port, err := nh.GetForwardedPort(kubeadm.APIServerPort) + if err != nil { + return err + } + + lines, err := nh.CombinedOutputLines("cat", "/etc/kubernetes/admin.conf") + if err != nil { + return errors.Wrap(err, "failed to get kubeconfig from node") + } + + // fix the config file, swapping out the server for the forwarded localhost:port + var buff bytes.Buffer + for _, line := range lines { + match := serverAddressRE.FindStringSubmatch(line) + if len(match) > 1 { + line = fmt.Sprintf("%s https://localhost:%d", match[1], port) + } + buff.WriteString(line) + buff.WriteString("\n") + } + + return ioutil.WriteFile(dest, buff.Bytes(), 0600) +} + +// GetForwardedPort takes the port number within the "node" container +// and returns the port it was forwarded to ouside the container +func (nh *nodeHandle) GetForwardedPort(port uint16) (uint16, error) { + cmd := exec.Command("docker", "port") + cmd.Args = append(cmd.Args, + nh.nameOrID, // ports are looked up by container + fmt.Sprintf("%d", port), // limit to the port we are looking up + ) + lines, err := cmd.CombinedOutputLines() + if err != nil { + return 0, err + } + if len(lines) != 1 { + return 0, fmt.Errorf("invalid output: %v", lines) + } + parts := strings.Split(lines[0], ":") + if len(parts) != 2 { + return 0, fmt.Errorf("invalid output: %v", lines) + } + v, err := strconv.ParseUint(parts[1], 10, 16) + if err != nil { + return 0, err + } + return uint16(v), nil +}