Created by: ston1th 2019-11-13 19:47:32
Updated by: ston1th 2021-12-05 21:47:00
Visibility: Public

Kubernetes Installation

Get the basic cluster up and running on Ubuntu 18.04.

Prepare

apt update && apt dist-upgrade -y && apt autoremove -y && apt purge -y nano docker docker-engine docker.io containerd runc && apt install apt-transport-https curl -y

Container Runtime

Docker

Docker has to be installed on all Nodes:

curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
echo "deb [arch=amd64] https://download.docker.com/linux/ubuntu bionic stable" >/etc/apt/sources.list.d/docker.list
apt-get update
apt-get install -y docker-ce docker-ce-cli containerd.io
cat <<EOF> /etc/docker/daemon.json
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2"
}
EOF
systemctl enable docker
systemctl start docker

Add your user to the docker group:

sudo usermod -aG docker $USER
newgrp docker

NOTE: make sure your mountpoint for /var/lib/docker does not include the nosuid mount option. This breaks the usage of linux capabilities(7) inside containers.

CRI-O

OS=xUbuntu_18.04
VER=1.22
curl -sSL https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | apt-key add -
curl -sSL https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VER/$OS/Release.key  | apt-key add -

echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/ /" > /etc/apt/sources.list.d/crio-tools.list
echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VER/$OS/ /" > /etc/apt/sources.list.d/crio.list

apt update
apt install cri-o cri-o-runc cri-tools

cat <<EOF>/etc/modules-load.d/crio.conf
overlay
br_netfilter
EOF
modprobe overlay
modprobe br_netfilter

sed -i 's/,metacopy=on//' /etc/containers/storage.conf

systemctl daemon-reload
systemctl enable crio --now
systemctl restart crio

Kubernetes

Install Kubernetes on all nodes.

curl -sSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" >/etc/apt/sources.list.d/kubernetes.list

v=1.22.4-00
apt update && apt install kubelet=$v kubeadm=$v kubectl=$v kubernetes-cni -y && apt-mark hold kubelet kubeadm kubectl

Apply sysctl on all nodes.

cat <<EOF> /etc/sysctl.conf
kernel.panic=10
kernel.panic_on_oops=1
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
net.ipv6.conf.all.disable_ipv6=1
net.ipv6.conf.default.disable_ipv6=1
vm.overcommit_memory=1
EOF
sysctl -p

Blacklist kernel modules:

cat <<EOF> /etc/modprobe.d/kubernetes-blacklist.conf 
blacklist dccp
blacklist sctp
EOF

iptables

Create startup service:

cat <<EOF> /etc/systemd/system/iptables.service
[Unit]
Description=iptables
ConditionPathExists=/etc/iptables.sh

[Service]
Type=oneshot
ExecStart=/etc/iptables.sh
TimeoutSec=0
RemainAfterExit=yes

[Install]
WantedBy=multi-user.target
EOF
systemctl enable iptables
systemctl daemon-reload

Create iptables.sh:

cat <<'EOF'> /etc/iptables.sh
#!/bin/bash

iface=eth0
ipt=iptables
ipt6=ip6tables

rules=(
"-I INPUT 1 -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT"
"-I INPUT 2 -i ${iface} -p tcp --dport 22 -j ACCEPT"
"-I INPUT 3 -i ${iface} -p tcp --dport 80 -j ACCEPT"
"-I INPUT 4 -i ${iface} -p tcp --dport 443 -j ACCEPT"
"-I INPUT 5 -i ${iface} -p tcp --dport 6443 -j ACCEPT"
"-I INPUT 6 -i ${iface} -j DROP"
)

IFS=$'\n'
for rule in ${rules[@]}; do
  IFS=' '
  r=(${rule})
  $ipt ${r[@]}
  $ipt6 ${r[@]}
  IFS=$'\n'
done
EOF
chmod 700 /etc/iptables.sh

Master

Prepare Config

View the default configs:

kubeadm config print init-defaults --component-configs KubeletConfiguration >kubeadm.conf

For CRI-O:

sed -i 's,/var/run/dockershim.sock,unix:///var/run/crio/crio.sock,' kubeadm.conf

The following parameters need to be adjusted:

kubernetesVersion
advertiseAddress
controlPlaneEndpoint

certSANs

anonymous-auth: "true"
https://github.com/kubernetes/kubernetes/issues/43784

Create the kubeadm config file:

mkdir /etc/kubernetes
cat <<EOF> /etc/kubernetes/kubeadm.yaml
apiVersion: kubeadm.k8s.io/v1beta2
kind: InitConfiguration
---
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
kubernetesVersion: 1.22.4
controlPlaneEndpoint: 192.168.0.2:6443
apiServer:
  certSANs:
  - k8s.example.com
  extraArgs:
    profiling: "false"
    anonymous-auth: "true"
    service-account-lookup: "true"
    service-account-issuer: "kubernetes.default.svc"
    service-account-signing-key-file: "/etc/kubernetes/pki/sa.key"
    disable-admission-plugins: DefaultTolerationSeconds
    enable-admission-plugins: NodeRestriction,AlwaysPullImages,PodPreset,PodSecurityPolicy,LimitRanger,ResourceQuota
    encryption-provider-config: /etc/kubernetes/pki/encryption.conf
    # runtime-config: "apps/v1beta1=true,extensions/v1beta1/deployments=true,settings.k8s.io/v1alpha1=true"
    runtime-config: "settings.k8s.io/v1alpha1=true"
    audit-log-path: /var/log/apiserver/audit.log
    audit-log-maxage: "30"
    audit-log-maxbackup: "10"
    audit-log-maxsize: "100"
    tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
controllerManager:
  extraArgs:
    profiling: "false"
    terminated-pod-gc-threshold: "10"
    feature-gates: "RotateKubeletServerCertificate=true"
    node-monitor-period: "2s"
    node-monitor-grace-period: "16s"
    pod-eviction-timeout: "30s"
scheduler:
  extraArgs:
    profiling: "false"
networking:
  podSubnet: 10.244.0.0/16
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
readOnlyPort: 0
nodeStatusUpdateFrequency: 4s
cgroupDriver: systemd
serverTLSBootstrap: true
featureGates:
  RotateKubeletServerCertificate: true
  AppArmor: true
protectKernelDefaults: true
tlsCipherSuites:
- TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305
- TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
- TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
- TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305
- TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
- TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
EOF

TODO:

apiVersion: kubeadm.k8s.io/v1beta2
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: x.x.x.x

Pod Security Policy

Create basic pod security policies:

cat <<EOF> /etc/kubernetes/privileged-psp.yaml
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
  name: privileged
  annotations:
    seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*'
spec:
  privileged: true
  allowPrivilegeEscalation: true
  allowedCapabilities: ["*"]
  volumes: ["*"]
  hostNetwork: true
  hostPorts:
  - min: 0
    max: 65535
  hostIPC: true
  hostPID: true
  runAsUser:
    rule: 'RunAsAny'
  seLinux:
    rule: 'RunAsAny'
  supplementalGroups:
    rule: 'RunAsAny'
  fsGroup:
    rule: 'RunAsAny'
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: privileged-psp
rules:
- apiGroups: ["policy"]
  resourceNames: ["privileged"]
  resources: ["podsecuritypolicies"]
  verbs: ["use"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: kube-system-psp
  namespace: kube-system
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: privileged-psp
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: Group
  name: system:nodes
- apiGroup: rbac.authorization.k8s.io
  kind: Group
  name: system:serviceaccounts:kube-system
- kind: Group
  name: "petaFuel.Orga.EDV.Standard"
  apiGroup: rbac.authorization.k8s.io
EOF
cat <<EOF> /etc/kubernetes/restricted-psp.yaml
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
  name: zz-restricted
  annotations:
    seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default,runtime/default'
    apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default'
    seccomp.security.alpha.kubernetes.io/defaultProfileName: 'runtime/default'
    apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default'
spec:
  privileged: false
  allowPrivilegeEscalation: false
  requiredDropCapabilities: ["ALL"]
  readOnlyRootFilesystem: false
  volumes:
  - 'configMap'
  - 'downwardAPI'
  - 'emptyDir'
  - 'persistentVolumeClaim'
  - 'projected'
  - 'secret'
  hostNetwork: false
  hostIPC: false
  hostPID: false
  runAsUser:
    rule: 'MustRunAsNonRoot'
  seLinux:
    rule: 'RunAsAny'
  supplementalGroups:
    rule: 'MustRunAs'
    ranges:
    - min: 1
      max: 65535
  fsGroup:
    rule: 'MustRunAs'
    ranges:
    - min: 1
      max: 65535
  hostPorts:
    - min: 1
      max: 65535
---
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
  name: yy-restricted-root
  annotations:
    seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'docker/default,runtime/default'
    apparmor.security.beta.kubernetes.io/allowedProfileNames: 'runtime/default'
    seccomp.security.alpha.kubernetes.io/defaultProfileName: 'runtime/default'
    apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default'
spec:
  privileged: false
  allowPrivilegeEscalation: false
  readOnlyRootFilesystem: false
  volumes:
  - 'configMap'
  - 'downwardAPI'
  - 'emptyDir'
  - 'persistentVolumeClaim'
  - 'projected'
  - 'secret'
  hostNetwork: false
  hostIPC: false
  hostPID: false
  runAsUser:
    rule: 'RunAsAny'
  seLinux:
    rule: 'RunAsAny'
  supplementalGroups:
    rule: 'RunAsAny'
  fsGroup:
    rule: 'RunAsAny'
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: restricted-psp
rules:
- apiGroups: ["policy"]
  resourceNames: ["zz-restricted","yy-restricted-root"]
  resources: ["podsecuritypolicies"]
  verbs: ["use"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: restricted-psp
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: restricted-psp
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: Group
  name: system:authenticated
- apiGroup: rbac.authorization.k8s.io
  kind: Group
  name: system:serviceaccounts
EOF

Secret Encryption

Create secret encryption config on the master node:

mkdir /etc/kubernetes/pki
cat <<EOF> /etc/kubernetes/pki/encryption.conf
apiVersion: apiserver.config.k8s.io/v1
kind: EncryptionConfiguration
resources:
  - resources:
    - secrets
    providers:
    - secretbox:
        keys:
        - name: key1
          secret: $(head -c 32 /dev/urandom | base64)
    - identity: {}
EOF
chmod 600 /etc/kubernetes/pki/encryption.conf

Distribute the encryption.conf file to your other master nodes.

Master Init

Note: save the kubeadm join commands from kubeadm init for later use.

As root:

kubeadm config images pull --config=/etc/kubernetes/kubeadm.yaml
kubeadm init --config=/etc/kubernetes/kubeadm.yaml --upload-certs

KUBECONFIG=/etc/kubernetes/admin.conf kubectl apply -f /etc/kubernetes/privileged-psp.yaml -f /etc/kubernetes/restricted-psp.yaml

Kubectl

Install the kubectl credentials for your root user.

Run as root:

rm -rf ~/.kube
mkdir ~/.kube
cp -i /etc/kubernetes/admin.conf ~/.kube/config

HA Control Plane

You can now join the other control plane nodes to the cluster:

kubeadm join 192.168.0.2:443 --token duoq4i... \
    --discovery-token-ca-cert-hash sha256:abc123... \
    --control-plane --certificate-key abc123...

Container Network

Choose either one: Flannel, Canal or Cilium.

https://cilium.io/blog/2020/02/18/cilium-17/

Flannel

Note: Does not provide NetworkPolicy.

Note: Maybe you need to change the network interface name for flannel: --iface=ens10

# Bare-Metal
curl -s https://raw.githubusercontent.com/coreos/flannel/3f7d3e6c24f641e7ff557ebcea1136fdf4b1b6a1/Documentation/kube-flannel.yml \
| sed 's/vxlan/host-gw/' >/etc/kubernetes/kube-flannel.yaml

# Hetzner Cloud
curl -s https://raw.githubusercontent.com/coreos/flannel/3f7d3e6c24f641e7ff557ebcea1136fdf4b1b6a1/Documentation/kube-flannel.yml \
| sed '/kube-subnet-mgr/ a\        - --iface=ens10' >/etc/kubernetes/kube-flannel.yaml

Install Flannel:

kubectl apply -f /etc/kubernetes/kube-flannel.yaml
kubectl -n kube-system delete \
daemonset.apps/kube-flannel-ds-arm64 \
daemonset.apps/kube-flannel-ds-arm \
daemonset.apps/kube-flannel-ds-ppc64le \
daemonset.apps/kube-flannel-ds-s390x

Uninstall:

kubectl -n kube-system delete -f /etc/kubernetes/kube-flannel.yaml

Canal

Mix of Flannel and Calico.

Note: Does provide NetworkPolicy.

Note: Maybe you need to change the network interface name for flannel: --iface=ens10

# Bare-Metal
curl -s https://docs.projectcalico.org/manifests/canal.yaml \
| sed 's/vxlan/host-gw/' >/etc/kubernetes/canal.yaml

# Hetzner Cloud
curl -s https://docs.projectcalico.org/manifests/canal.yaml \
| sed 's/canal_iface: ""/canal_iface: "ens10"/' >/etc/kubernetes/canal.yaml

Install Canal:

kubectl apply -f /etc/kubernetes/canal.yaml

Uninstall:

kubectl -n kube-system delete -f /etc/kubernetes/canal.yaml

Join Worker Nodes

On the worker nodes run:

kubeadm join 192.168.0.2:6443 --token duoq4i... \
    --discovery-token-ca-cert-hash sha256:abc123...

Get all kubelet certificate signing requests and sign them:

On the master:

for csr in $(kubectl get csr|grep Pending|cut -d" " -f1); do kubectl certificate approve $csr; done

Now your cluster is installed.

Check your nodes and cluster status:

kubectl cluster-info
kubectl get nodes

Upgrading

Master

Upgrade kubeadm:

v=1.16.9-00
apt-mark unhold kubeadm && \
apt update && apt install -y kubeadm=$v && \
apt-mark hold kubeadm

Run the upgrade planner: kubeadm upgrade plan

Upgrade the master: kubeadm upgrade apply 1.16.9

Upgrade additional control-plane nodes: kubeadm upgrade node

Upgrade kubelet:

v=1.16.9-00
apt-mark unhold kubelet kubectl && \
apt update && apt install -y kubelet=$v kubectl=$v && \
apt-mark hold kubelet kubectl && \
systemctl restart kubelet

Worker

Upgrade kubeadm:

v=1.16.9-00
apt-mark unhold kubeadm && \
apt update && apt install -y kubeadm=$v && \
apt-mark hold kubeadm

Drain the node: kubectl drain <nodename> --ignore-daemonsets --delete-local-data

Upgrade the node and kubelet:

v=1.16.9-00
kubeadm upgrade node && \
apt-mark unhold kubelet kubectl && \
apt update && apt install -y kubelet=$v kubectl=$v && \
apt-mark hold kubelet kubectl && \
systemctl restart kubelet

Uncordon the node: kubectl uncordon <nodename>

Full Cluster Reset

yes | kubeadm reset
systemctl stop kubelet
systemctl stop docker
rm -rf /var/lib/cni/
rm -rf /var/lib/rook/
rm -rf /var/lib/kubelet/*
rm -rf /etc/cni/
rm -rf /var/etcd/calico-data/
iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X
ip link set kube-bridge down
ip link delete kube-bridge
ip link delete dummy0
ip link delete flannel.1
ip link delete cni0
systemctl start kubelet
systemctl start docker
docker image list | awk '!/IMAGE/{print $3}' | xargs docker image rm -f