From 809ba3f9a9faa6a49299e94b6a3623a48435437e Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Wed, 23 Oct 2019 10:12:20 -0700 Subject: [PATCH] overhaul base image --- .gitignore | 8 +- images/base/Dockerfile | 125 +++++++----------- .../etc/containerd/config.toml} | 0 images/base/files/etc/crictl.yaml | 1 + .../etc/sysctl.d}/10-network-security.conf | 0 .../etc/systemd/system}/containerd.service | 0 .../{ => files/usr/local/bin}/clean-install | 0 .../base/{ => files/usr/local/bin}/entrypoint | 0 8 files changed, 55 insertions(+), 79 deletions(-) rename images/base/{containerd-config.toml => files/etc/containerd/config.toml} (100%) create mode 100644 images/base/files/etc/crictl.yaml rename images/base/{ => files/etc/sysctl.d}/10-network-security.conf (100%) rename images/base/{ => files/etc/systemd/system}/containerd.service (100%) rename images/base/{ => files/usr/local/bin}/clean-install (100%) rename images/base/{ => files/usr/local/bin}/entrypoint (100%) diff --git a/.gitignore b/.gitignore index a4a9cdc1..93b1594a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,10 @@ # build and test outputs -bin/ -_output/ -_artifacts/ +/bin/ +/_output/ +/_artifacts/ # used for the code generators only -vendor/ +/vendor/ # macOS .DS_Store diff --git a/images/base/Dockerfile b/images/base/Dockerfile index c9e5dce6..42dad635 100644 --- a/images/base/Dockerfile +++ b/images/base/Dockerfile @@ -12,51 +12,53 @@ # See the License for the specific language governing permissions and # limitations under the License. -# kind cluster base image +# kind node base image # # For systemd + docker configuration used below, see the following references: # https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/ -# https://developers.redhat.com/blog/2014/05/05/running-systemd-within-docker-container/ -# https://developers.redhat.com/blog/2016/09/13/running-systemd-in-a-non-privileged-container/ -ARG BASE_IMAGE="ubuntu:19.10" -FROM ${BASE_IMAGE} +# start from ubuntu 19.10, this image is reasonably small as a starting point +# for a kubernetes node image, it doesn't contain much we don't need +FROM ubuntu:19.10 -# setting DEBIAN_FRONTEND=noninteractive stops some apt warnings, this is not -# a real argument, we're (ab)using ARG to get a temporary ENV again. -ARG DEBIAN_FRONTEND=noninteractive +# Configure containerd and runc binaries from kind-ci/containerd-nightlies repository +# The repository contains latest stable releases and nightlies built for multiple architectures +ARG CONTAINERD_VERSION="v1.3.0-7-g0b43a311" +# Configure CNI binaries from upstream +ARG CNI_VERSION="v0.8.2" +# Configure crictl binary from upstream +ARG CRICTL_VERSION="v1.16.1" -COPY clean-install /usr/local/bin/clean-install -RUN chmod +x /usr/local/bin/clean-install +# copy in static files (configs, scripts) +COPY files/ / -# Get dependencies -# The base image already has: ssh, apt, snapd -# This is broken down into (each on a line): +# Install dependencies, first from apt, then from release tarballs. +# NOTE: we use one RUN to minimize layers. +# +# First we must ensure that our util scripts are executable. +# +# The base image already has: ssh, apt, snapd, but we need to install more packages. +# Packages installed are broken down into (each on a line): # - packages needed to run services (systemd) -# - CRI (containerd) # - packages needed for kubernetes components # - misc packages kind uses itself -# Then we cleanup (removing unwanted systemd services) -# Then we disable kmsg in journald (these log entries would be confusing) +# After installing packages we cleanup by: +# - removing unwanted systemd services) +# - disabling kmsg in journald (these log entries would be confusing) # -# ******************************************************************* +# Then we install containerd from our nightly build infrastructure, as this +# build for multiple architectures and allows us to upgrade to patched releases +# more quickly. # -# We then download a ctr binary with a tiny additional feature -# we use that is not yet merged / packaged. +# Next we download and extract crictl and CNI plugin binaries from upstream. # -# See: https://github.com/containerd/containerd/pull/3259 -# -# This binary is built with hack/build/ctr/run.sh -# Sources: https://github.com/BenTheElder/containerd/tree/kind -# Additional commit: -# https://github.com/BenTheElder/containerd/commit/cb7c780af2394ab08d5d8a3932ca7437074ae179 -# -# TODO(bentheelder): remove this once --no-unpack is packaged upstream. -# -# ****************************************************************** -# -# https://developers.redhat.com/blog/2014/05/05/running-systemd-within-docker-container/ -RUN clean-install \ +# Finally we ensure the /etc/kubernetes/manifests directory exists. Normally +# a kubeadm debain / rpm package would ensure that this exists but we install +# freshly built binaries directly when we build the node image. +RUN echo "Ensuring scripts are executable ..." \ + && chmod +x /usr/local/bin/clean-install /usr/local/bin/entrypoint \ + && echo "Installing Packages ..." \ + && DEBIAN_FRONTEND=noninteractive clean-install \ systemd systemd-sysv libsystemd0 \ conntrack iptables iproute2 ethtool socat util-linux mount ebtables udev kmod \ bash ca-certificates curl rsync \ @@ -68,48 +70,28 @@ RUN clean-install \ && rm -f /lib/systemd/system/sockets.target.wants/*initctl* \ && rm -f /lib/systemd/system/basic.target.wants/* \ && echo "ReadKMsg=no" >> /etc/systemd/journald.conf \ - && echo "done installing packages" - -# override the rp_filter settings to enable calico cni to "just work" -COPY 10-network-security.conf /etc/sysctl.d/ - -# Install containerd and runc binaries from kind-ci/containerd-nightlies repository -# The repository contains latest stable releases and nightlies built for multiple architectures -ARG CONTAINERD_VERSION="v1.3.0-7-g0b43a311" -ARG CONTAINERD_BASE_URL="https://github.com/kind-ci/containerd-nightlies/releases/download" -RUN export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm/') \ - && export RELEASE_BASE_URL="${CONTAINERD_BASE_URL}/containerd-${CONTAINERD_VERSION#v}" \ - && curl -sSL --retry 5 --output /tmp/containerd.tgz "${RELEASE_BASE_URL}/containerd-${CONTAINERD_VERSION#v}.linux-${ARCH}.tar.gz" \ + && echo "Installing containerd ..." \ + && export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm/') \ + && export CONTAINERD_BASE_URL="https://github.com/kind-ci/containerd-nightlies/releases/download/containerd-${CONTAINERD_VERSION#v}" \ + && curl -sSL --retry 5 --output /tmp/containerd.tgz "${CONTAINERD_BASE_URL}/containerd-${CONTAINERD_VERSION#v}.linux-${ARCH}.tar.gz" \ && tar -C /usr/local -xzf /tmp/containerd.tgz \ && rm -rf /tmp/containerd.tgz \ - && curl -sSL --retry 5 --output /usr/local/sbin/runc "${RELEASE_BASE_URL}/runc.${ARCH}" \ + && curl -sSL --retry 5 --output /usr/local/sbin/runc "${CONTAINERD_BASE_URL}/runc.${ARCH}" \ && chmod 755 /usr/local/sbin/runc \ - && containerd --version - -# Install containerd systemd unit file -COPY containerd.service /etc/systemd/system -RUN systemctl enable containerd - -# configure containerd with some custom options -COPY containerd-config.toml /etc/containerd/config.toml - -# Install CNI binaries to /opt/cni/bin -# TODO(bentheelder): doc why / what here -ARG CNI_VERSION="v0.8.2" -ARG CNI_BASE_URL="https://github.com/containernetworking/plugins/releases/download/" -RUN export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm/') \ + && containerd --version \ + && systemctl enable containerd \ + && echo "Installing crictl ..." \ + && curl -fSL "https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-${ARCH}.tar.gz" | tar xzC /usr/local/bin \ + && echo "Installing CNI binaries ..." \ + && export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm/') \ && export CNI_TARBALL="${CNI_VERSION}/cni-plugins-linux-${ARCH}-${CNI_VERSION}.tgz" \ - && export CNI_URL="${CNI_BASE_URL}${CNI_TARBALL}" \ + && export CNI_URL="https://github.com/containernetworking/plugins/releases/download/${CNI_TARBALL}" \ && curl -sSL --retry 5 --output /tmp/cni.tgz "${CNI_URL}" \ && mkdir -p /opt/cni/bin \ && tar -C /opt/cni/bin -xzf /tmp/cni.tgz \ - && rm -rf /tmp/cni.tgz - -# Install crictl to /usr/local/bin -ARG CRICTL_VERSION="v1.16.1" -RUN export ARCH=$(dpkg --print-architecture | sed 's/ppc64el/ppc64le/' | sed 's/armhf/arm/') \ - && curl -fSL "https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-${ARCH}.tar.gz" | tar xzC /usr/local/bin \ - && echo 'runtime-endpoint: unix:///var/run/containerd/containerd.sock' > /etc/crictl.yaml + && rm -rf /tmp/cni.tgz \ + && echo "Ensuring /etc/kubernetes/manifests" \ + && mkdir -p /etc/kubernetes/manifests # tell systemd that it is in docker (it will check for the container env) # https://www.freedesktop.org/wiki/Software/systemd/ContainerInterface/ @@ -117,12 +99,5 @@ ENV container docker # systemd exits on SIGRTMIN+3, not SIGTERM (which re-executes it) # https://bugzilla.redhat.com/show_bug.cgi?id=1201657 STOPSIGNAL SIGRTMIN+3 - -# wrap systemd with our special entrypoint, see pkg/build for how this is built -# basically this just lets us set up some things before continuing on to systemd -# while preserving that systemd is PID1 -# for how we leverage this, see pkg/cluster -COPY [ "entrypoint", "/usr/local/bin/" ] -# We need systemd to be PID1 to run the various services (docker, kubelet, etc.) -# NOTE: this is *only* for documentation, the entrypoint is overridden by the node image +# NOTE: this is *only* for documentation, the entrypoint is overridden later ENTRYPOINT [ "/usr/local/bin/entrypoint", "/sbin/init" ] diff --git a/images/base/containerd-config.toml b/images/base/files/etc/containerd/config.toml similarity index 100% rename from images/base/containerd-config.toml rename to images/base/files/etc/containerd/config.toml diff --git a/images/base/files/etc/crictl.yaml b/images/base/files/etc/crictl.yaml new file mode 100644 index 00000000..1a5daba3 --- /dev/null +++ b/images/base/files/etc/crictl.yaml @@ -0,0 +1 @@ +runtime-endpoint: unix:///var/run/containerd/containerd.sock \ No newline at end of file diff --git a/images/base/10-network-security.conf b/images/base/files/etc/sysctl.d/10-network-security.conf similarity index 100% rename from images/base/10-network-security.conf rename to images/base/files/etc/sysctl.d/10-network-security.conf diff --git a/images/base/containerd.service b/images/base/files/etc/systemd/system/containerd.service similarity index 100% rename from images/base/containerd.service rename to images/base/files/etc/systemd/system/containerd.service diff --git a/images/base/clean-install b/images/base/files/usr/local/bin/clean-install similarity index 100% rename from images/base/clean-install rename to images/base/files/usr/local/bin/clean-install diff --git a/images/base/entrypoint b/images/base/files/usr/local/bin/entrypoint similarity index 100% rename from images/base/entrypoint rename to images/base/files/usr/local/bin/entrypoint