Get the basic cluster up and running on Ubuntu 18.04.
Install:
https://medium.com/@anisinanaj/on-premise-ha-kubernetes-cluster-15e41f18bd12
CSI/Storage:
https://medium.com/@anisinanaj/completing-the-kubernetes-setup-7462383ab3c
apt update && apt dist-upgrade -y && apt autoremove -y && apt purge nano -y
apt install apt-transport-https -y
apt install docker.io -y
cat <<EOF> /etc/docker/daemon.json
{
"exec-opts": ["native.cgroupdriver=systemd"],
"log-driver": "json-file",
"log-opts": {
"max-size": "100m"
},
"storage-driver": "overlay2"
}
EOF
systemctl enable docker
systemctl start docker
Add your user to the docker group:
sudo usermod -aG docker $USER
newgrp docker
Install Kubernetes on all nodes.
apt install curl -y
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" >/etc/apt/sources.list.d/kubernetes.list
apt update && apt install kubelet kubeadm kubectl kubernetes-cni -y
Apply sysctl on all nodes.
cat <<EOF> /etc/sysctl.conf
net.bridge.bridge-nf-call-ip6tables=1
net.bridge.bridge-nf-call-iptables=1
net.ipv4.ip_forward=1
EOF
sysctl net.bridge.bridge-nf-call-ip6tables=1
sysctl net.bridge.bridge-nf-call-iptables=1
sysctl net.ipv4.ip_forward=1
View the default configs:
kubeadm config print init-defaults --component-configs KubeletConfiguration
Create the kubeadm config file:
Change advertiseAddress and kubernetesVersion as needed.
cat <<EOF> kubeadm.yaml
apiVersion: kubeadm.k8s.io/v1beta2
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 192.168.0.2
---
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
kubernetesVersion: v1.16.2
apiServer:
extraArgs:
anonymous-auth: "true"
service-account-lookup: "true"
disable-admission-plugins: DefaultTolerationSeconds
enable-admission-plugins: NodeRestriction,AlwaysPullImages
profiling: "false"
encryption-provider-config: /etc/kubernetes/pki/encryption.conf
controllerManager:
extraArgs:
node-monitor-period: "2s"
node-monitor-grace-period: "16s"
pod-eviction-timeout: "30s"
networking:
podSubnet: 10.244.0.0/16
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
nodeStatusUpdateFrequency: 4s
cgroupDriver: systemd
EOF
Create secret encryption config on the master node:
mkdir -p /etc/kubernetes/pki
cat <<EOF> /etc/kubernetes/pki/encryption.conf
apiVersion: apiserver.config.k8s.io/v1
kind: EncryptionConfiguration
resources:
- resources:
- secrets
providers:
- secretbox:
keys:
- name: key1
secret: $(head -c 32 /dev/urandom | base64)
- identity: {}
EOF
chmod 600 /etc/kubernetes/pki/encryption.conf
Distribute this config file to your other master nodes.
As root (NOTE: save the kubeadm join command, as it is needed later):
kubeadm config images pull --config kubeadm.yaml
kubeadm init --config kubeadm.yaml
# or
kubeadm config images pull --kubernetes-version v1.16.0
kubeadm init --kubernetes-version v1.16.0 --apiserver-advertise-address 192.168.0.2 --pod-network-cidr 10.244.0.0/16
Select one of the following CNI plugins.
Flannel has been testet well.
kubectl edit cm -n kube-system kube-flannel-cfg
# add to cni-conf.json
"cniVersion": "0.2.0",
Note: Maybe you need to change the network interface name for flannel: --iface=ens10
curl -s https://raw.githubusercontent.com/coreos/flannel/3f7d3e6c24f641e7ff557ebcea1136fdf4b1b6a1/Documentation/kube-flannel.yml | sed '/kube-subnet-mgr/ a\ - --iface=ens10' | KUBECONFIG=/etc/kubernetes/admin.conf kubectl apply -f -
KUBECONFIG=/etc/kubernetes/admin.conf kubectl -n kube-system delete daemonset.apps/kube-flannel-ds-arm64 daemonset.apps/kube-flannel-ds-arm daemonset.apps/kube-flannel-ds-ppc64le daemonset.apps/kube-flannel-ds-s390x
# cleanup
kubectl -n kube-system delete -f https://raw.githubusercontent.com/coreos/flannel/3f7d3e6c24f641e7ff557ebcea1136fdf4b1b6a1/Documentation/kube-flannel.yml
KUBECONFIG=/etc/kubernetes/admin.conf kubectl apply -f https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/daemonset/kubeadm-kuberouter-all-features.yaml
# or
KUBECONFIG=/etc/kubernetes/admin.conf kubectl apply -f https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/daemonset/kubeadm-kuberouter-all-features-hostport.yaml
# Add these args:
KUBECONFIG=/etc/kubernetes/admin.conf kubectl -n kube-system edit ds kube-router
- --run-router=true
- --run-firewall=true
- --run-service-proxy=false
- --kubeconfig=/var/lib/kube-router/kubeconfig
- --iptables-sync-period=5s
- --ipvs-sync-period=5s
- --routes-sync-period=5s
- --bgp-graceful-restart
- --hairpin-mode
- -v=3
net.ipv4.vs.expire_nodest_conn=0
KUBECONFIG=/etc/kubernetes/admin.conf kubectl -n kube-system delete ds kube-proxy
docker run --privileged -v /lib/modules:/lib/modules --net=host k8s.gcr.io/kube-proxy-amd64:v1.15.1 kube-proxy --cleanup
# cleanup
kubectl delete -f https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/daemonset/kubeadm-kuberouter-all-features-hostport.yaml
docker run --privileged --net=host cloudnativelabs/kube-router kube-router --cleanup-config
issue: https://github.com/projectcalico/calico/issues/2211
curl -s https://docs.projectcalico.org/v3.4/getting-started/kubernetes/installation/hosted/etcd.yaml | sed 's|192.168.0.0/16|10.244.0.0/16|g' | KUBECONFIG=/etc/kubernetes/admin.conf kubectl apply -f -
curl -s https://docs.projectcalico.org/v3.4/getting-started/kubernetes/installation/hosted/calico.yaml | sed 's|192.168.0.0/16|10.244.0.0/16|g' | KUBECONFIG=/etc/kubernetes/admin.conf kubectl apply -f -
# Delete the 'loop' entry:
kubectl -n kube-system edit cm coredns
# cleanup
kubectl -n kube-system delete -f https://docs.projectcalico.org/v3.4/getting-started/kubernetes/installation/hosted/calico.yaml
kubectl -n kube-system delete -f https://docs.projectcalico.org/v3.4/getting-started/kubernetes/installation/hosted/etcd.yaml
# on every node
rm -rf /var/etcd/calico-data/ /opt/cni/bin{calico,calico-ipam,flannel}
iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X
ifconfig | grep -E "cali|veth" | cut -d" " -f1 | xargs -n1 ip link delete
ip link delete dummy0
modprobe -r ipip
Install the kubectl credentials for you normal non-root user.
Run as normal user:
sudo rm -rf $HOME/.kube
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
As root:
kubeadm join command from the kubeadm init output.
Now your cluster is installed.
Check your nodes and cluster status:
kubectl cluster-info
kubectl get nodes
Upgrade kubeadm:
v=1.16.2-00
apt-mark unhold kubeadm && \
apt update && apt install -y kubeadm=$v && \
apt-mark hold kubeadm
Run the upgrade planner: kubeadm upgrade plan
Upgrade the master: kubeadm upgrade apply v1.16.2
Upgrade additional control-plane nodes: kubeadm upgrade node
Upgrade kubelet:
v=1.16.2-00
apt-mark unhold kubelet kubectl && \
apt update && apt install -y kubelet=$v kubectl=$v && \
apt-mark hold kubelet kubectl && \
systemctl restart kubelet
Upgrade kubeadm:
v=1.16.2-00
apt-mark unhold kubeadm && \
apt update && apt install -y kubeadm=$v && \
apt-mark hold kubeadm
Drain the node: kubectl drain <nodename> --ignore-daemonsets --delete-local-data
Upgrade the node and kubelet:
v=1.16.2-00
kubeadm upgrade node && \
apt-mark unhold kubelet kubectl && \
apt update && apt install -y kubelet=$v kubectl=$v && \
apt-mark hold kubelet kubectl && \
systemctl restart kubelet
Uncordon the node: kubectl uncordon <nodename>
Requires BGP or a Layer2 Network.
kubectl apply -f https://raw.githubusercontent.com/google/metallb/v0.8.1/manifests/metallb.yaml
kubectl apply -f - <<EOF
apiVersion: v1
kind: ConfigMap
metadata:
namespace: metallb-system
name: config
data:
config: |
address-pools:
- name: default
protocol: layer2
addresses:
- 192.168.1.240-192.168.1.250
EOF
You can install some basic services on your cluster.
Install Kubernetes Dashboard:
# TLS
kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta3/aio/deploy/recommended.yaml
# Non TLS
curl -L -s https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0-beta3/aio/deploy/recommended.yaml | sed -e 's/8443/8080/g' -e 's/443/80/g' -e 's/scheme: HTTPS/scheme: HTTP/g' -e 's/--auto-generate-certificates/--insecure-bind-address=0.0.0.0\n - --insecure-port=8080\n - --token-ttl=6000\n - --enable-insecure-login/g' | kubectl apply -f -
kubectl -n kube-system apply -f - <<EOF
apiVersion: v1
kind: ServiceAccount
metadata:
name: admin-user
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: admin-user
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: admin-user
namespace: kube-system
EOF
Proxy connections to the dashboard:
# get your dashboard access token
kubectl -n kube-system get secret $(kubectl -n kube-system get secret | grep admin-user-token | awk '{print $1}') -o jsonpath='{.data.token}' | base64 -d;echo
kubectl -n kubernetes-dashboard port-forward $(kubectl -n kubernetes-dashboard get pod -l k8s-app=kubernetes-dashboard -o jsonpath='{.items[0].metadata.name}') 8001:8443
Deploy a in-cluster container registry:
https://github.com/teamhephy/registry-proxy
kubectl apply -f - <<EOF
apiVersion: apps/v1
kind: Deployment
metadata:
name: kube-registry-v0
namespace: kube-system
labels:
k8s-app: kube-registry
version: v0
kubernetes.io/cluster-service: "true"
spec:
replicas: 1
selector:
matchLabels:
k8s-app: kube-registry
version: v0
template:
metadata:
labels:
k8s-app: kube-registry
version: v0
kubernetes.io/cluster-service: "true"
spec:
containers:
- name: registry
image: registry:2
resources:
limits:
cpu: 100m
memory: 100Mi
env:
- name: REGISTRY_HTTP_ADDR
value: :5000
- name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY
value: /var/lib/registry
volumeMounts:
- name: kube-registry
mountPath: /var/lib/registry
ports:
- containerPort: 5000
name: registry
protocol: TCP
volumes:
- name: kube-registry
emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
name: kube-registry
namespace: kube-system
labels:
k8s-app: kube-registry
spec:
selector:
k8s-app: kube-registry
ports:
- name: registry
port: 5000
protocol: TCP
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kube-registry-proxy
namespace: kube-system
labels:
k8s-app: kube-registry-proxy
tier: node
kubernetes.io/cluster-service: "true"
spec:
selector:
matchLabels:
name: kube-registry-proxy
tier: node
kubernetes.io/cluster-service: "true"
template:
metadata:
labels:
name: kube-registry-proxy
tier: node
kubernetes.io/cluster-service: "true"
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
tolerations:
- key: node-role.kubernetes.io/master
effect: NoSchedule
containers:
- name: kube-registry-proxy
# image: gcr.io/google_containers/kube-registry-proxy:0.4
image: tapjoy/kube-registry-proxy:latest
resources:
limits:
cpu: 100m
memory: 50Mi
env:
- name: REGISTRY_HOST
value: kube-registry.kube-system.svc.cluster.local
- name: REGISTRY_PORT
value: "5000"
ports:
- name: registry
containerPort: 5000
hostPort: 5000
EOF
Push docker images to your registry:
https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
docker build -t 127.0.0.1:5000/db:v1 .
docker push 127.0.0.1:5000/db:v1
Install helm on your master:
# as root
curl -s -L https://get.helm.sh/helm-v2.14.3-linux-amd64.tar.gz >/tmp/helm-v2.14.3-linux-amd64.tar.gz
[[ $(sha256sum /tmp/helm-v2.14.3-linux-amd64.tar.gz|cut -d" " -f1) = "38614a665859c0f01c9c1d84fa9a5027364f936814d1e47839b05327e400bf55" ]] && {
tar xfz /tmp/helm-v2.14.3-linux-amd64.tar.gz --strip-components=1 -C /usr/bin linux-amd64/helm
rm -rf /tmp/helm-v2.14.3-linux-amd64.tar.gz
echo "helm install successful"
} || echo "checksum invalid"
# as user
helm init --client-only
Tiller is optional.
kubectl -n kube-system create serviceaccount tiller
kubectl create clusterrolebinding tiller-cluster-role --clusterrole=cluster-admin --serviceaccount=kube-system:tiller
helm init --service-account tiller
https://www.ibm.com/support/knowledgecenter/en/SSBS6K_3.1.1/manage_cluster/istio.html
Prepare for a Tiller-less installation with helm:
mkdir ~/istio
helm repo add istio.io https://storage.googleapis.com/istio-release/releases/1.2.4/charts/
helm fetch --untar --untardir ~/istio istio.io/istio-init
helm fetch --untar --untardir ~/istio istio.io/istio-cni
helm fetch --untar --untardir ~/istio istio.io/istio
Install istioctl:
curl -s -L https://github.com/istio/istio/releases/download/1.2.3/istio-1.2.3-linux.tar.gz >/tmp/istio-1.2.3-linux.tar.gz
[[ $(sha256sum /tmp/istio-1.2.3-linux.tar.gz|cut -d" " -f1) = "ecf8a759a4c41bd7c0fb3515c61b13f6d1252552a235e147c75212566cf15691" ]] && {
tar xfz /tmp/istio-1.2.3-linux.tar.gz --strip-components=2 -C /usr/bin istio-1.2.3/bin/istioctl
rm -rf /tmp/istio-1.2.3-linux.tar.gz
echo "istioctl install successful"
} || echo "checksum invalid"
kubectl create namespace istio-system
helm template ~/istio/istio-init --name istio-init --namespace istio-system >~/istio/crds.yaml
kubectl apply -f ~/istio/crds.yaml
# check for 23, 28 or 29 CRDs
kubectl get crds | grep -c -E "istio.io|certmanager.k8s.io"
helm template ~/istio/istio-cni --name istio-cni --namespace kube-system >~/istio/cni.yaml
kubectl apply -f ~/istio/cni.yaml
KIALI_USERNAME=$(read -p 'Kiali Username: ' uval && echo -n $uval | base64)
KIALI_PASSPHRASE=$(read -sp 'Kiali Passphrase: ' pval && echo -n $pval | base64)
cat <<EOF> ~/istio/kiali-secret.yaml
apiVersion: v1
kind: Secret
metadata:
name: kiali
namespace: istio-system
labels:
app: kiali
type: Opaque
data:
username: $KIALI_USERNAME
passphrase: $KIALI_PASSPHRASE
EOF
unset KIALI_USERNAME KIALI_PASSPHRASE
kubectl apply -f ~/istio/kiali-secret.yaml
# testing
helm template ~/istio/istio --name istio --namespace istio-system \
--set istio_cni.enabled=true \
--set kiali.enabled=true \
--set gateways.istio-egressgateway.enabled=true \
--set tracing.enabled=true \
--set tracing.ingress.enabled=true \
--set gateways.istio-ingressgateway.sds.enabled=true >~/istio/istio.yaml
# production
helm template ~/istio/istio --name istio --namespace istio-system \
--set gateways.istio-ingressgateway.sds.enabled=true \
--set gateways.istio-egressgateway.enabled=true \
--set security.citadelHealthCheck=true \
--set tracing.enabled=true \
--set tracing.ingress.enabled=true \
--set kiali.enabled=true \
--set istio_cni.enabled=true \
--set global.controlPlaneSecurityEnabled=true \
--set global.mtls.enabled=true \
--set global.arch.s390x=0 \
--set global.arch.ppc64le=0 \
--set global.outboundTrafficPolicy.mode=REGISTRY_ONLY >~/istio/istio.yaml
kubectl apply -f ~/istio/istio.yaml
kubectl label ns <namespace> istio-injection=enabled
kubectl create -f <(istioctl kube-inject -f <your-app-spec>.yaml)
kubectl delete -f ~/istio/istio.yaml
kubectl delete -f ~/istio/cni.yaml
kubectl delete -f ~/istio/crds.yaml
kubectl delete namespace istio-system
kubectl label ns <namespace> istio-injection-
helm repo remove istio.io
rm -rf ~/istio/istio/ ~/istio/istio-cni/ ~/istio/istio-init/
# proceed with help repo add <new version>
# don't forget to update the running envoy sidecars.
This script is based on: https://github.com/truongnh1992/playing-with-istio/blob/c5a78b3c6e62024062154ec4a17ec2af6e005a0b/upgrade-sidecar.sh
cat <<'EOF'> ~/upgrade-sidecar.sh
#!/bin/bash
[[ $# -ne 1 ]] && { echo "usage: $0 <namespace>">&2; exit 1; }
refresh_all_pods() {
local ns=$1
for d in $(kubectl -n "${ns}" get deployment -o jsonpath='{.items[*].metadata.name}'); do
local secs=$(kubectl -n "${ns}" get deployment "${d}" -o jsonpath='{.spec.template.spec.terminationGracePeriodSeconds}')
[[ ${secs} -eq 30 ]] && secs=31 || secs=30
kubectl -n "${ns}" patch deployment "${d}" -p '{"spec":{"template":{"spec":{"terminationGracePeriodSeconds":'${secs}"}}}}"
done
}
refresh_all_pods $1
EOF
chmod 755 upgrade-sidecar.sh
kubectl create ns redirect
kubectl label ns redirect istio-injection=enabled
TODO
https://github.com/istio/istio/issues/9504#issuecomment-466590031
https://github.com/kiali/kiali/tree/master/deploy/kubernetes
curl -L -s https://github.com/linkerd/linkerd2/releases/download/edge-19.8.3/linkerd2-cli-edge-19.8.3-linux >/usr/bin/linkerd
[[ "$(sha256sum /usr/bin/linkerd|cut -d" " -f1)" = "b0024a615843feb3777c09a6f12ab2526f3c89ce306b22cfc26755ce075f191b" ]] && chmod 755 /usr/bin/linkerd || echo "checksum invalid"
mkdir ~/linkerd
linkerd install-cni >~/linkerd/linkerd-cni.yaml
kubectl apply -f ~/linkerd/linkerd-cni.yaml
# testing
linkerd install \
--ignore-cluster \
--linkerd-cni-enabled >~/linkerd/linkerd.yaml
# production
linkerd install \
--controller-replicas 1 \
--ha \
--ignore-cluster \
--linkerd-cni-enabled >~/linkerd/linkerd.yaml
kubectl apply -f ~/linkerd/linkerd.yaml
kubectl annotate ns <namespace> linkerd.io/inject=enabled
kubectl delete -f ~/linkerd/linkerd.yaml
kubectl delete -f ~/linkerd/linkerd-cni.yaml
kubectl annotate ns <namespace> linkerd.io/inject-
https://kubernetes.io/docs/concepts/services-networking/ingress/
TBA
https://github.com/containous/traefik/tree/master/examples/k8s
https://docs.traefik.io/user-guide/kubernetes/
Dashboard: http://127.0.0.1:8001/dashboard/
kubectl apply -f https://raw.githubusercontent.com/containous/traefik/master/examples/k8s/traefik-rbac.yaml
kubectl apply -f https://raw.githubusercontent.com/containous/traefik/master/examples/k8s/traefik-ds.yaml
https://kubernetes.github.io/ingress-nginx/
kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/mandatory.yaml
kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/provider/baremetal/service-nodeport.yaml
kubectl create namespace cert-manager
kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true
curl -L -s https://github.com/jetstack/cert-manager/releases/download/v0.9.0/cert-manager.yaml | sed '/--cluster-resource-namespace/ a\ - --enable-certificate-owner-ref=true' | kubectl apply -f -
kubectl delete -f https://github.com/jetstack/cert-manager/releases/download/v0.9.0/cert-manager.yaml
kubectl delete namespace cert-manager
https://itnext.io/kubernetes-monitoring-with-prometheus-in-15-minutes-8e54d1de2e13
Prepare for a Tiller-less installation with helm:
mkdir ~/monitoring
helm fetch --untar --untardir ~/monitoring stable/metrics-server
helm fetch --untar --untardir ~/monitoring stable/prometheus-operator
Login: admin/prom-operator
kubectl create namespace monitoring
helm template ~/monitoring/prometheus-operator --name pop --namespace monitoring --set kubelet.serviceMonitor.https=true >~/monitoring/prometheus-operator.yaml
kubectl -n monitoring apply -f ~/monitoring/prometheus-operator.yaml
kubectl -n monitoring delete -f ~/monitoring/prometheus-operator.yaml
kubectl delete namespace monitoring
Additional Dashboards:
1. Kubernetes Deployment Statefulset Daemonset metrics: 8588
Kubernetes Deployment metrics: 741
Kubernetes Deployment: 3137
Kubernetes Cluster (Prometheus): 6417
Kubernetes Pods (Prometheus): 5327
Kubernetes pod and cluster monitoring (via Prometheus): 6663
Kubernetes Persistence Volumes: 6739
helm template ~/monitoring/metrics-server --name metrics --namespace kube-system --set args="{--logtostderr,--kubelet-insecure-tls,--kubelet-preferred-address-types=InternalIP}" >~/monitoring/metrics-server.yaml
kubectl -n kube-system apply -f ~/monitoring/metrics-server.yaml
kubectl -n kube-system delete -f ~/monitoring/metrics-server.yaml
helm install stable/prometheus --name prometheus --namespace monitoring --set alertmanager.persistentVolume.enabled=false --set server.persistentVolume.enabled=false
helm del --purge prometheus
helm install stable/grafana --name grafana --namespace monitoring -f grafana/values.yaml
kubectl get secret -n monitoring grafana -o jsonpath="{.data.admin-password}" | base64 -d;echo
helm del --purge grafana
https://www.getambassador.io/user-guide/with-istio
kubectl apply -f https://getambassador.io/yaml/ambassador/ambassador-rbac.yaml
kubectl -n ci annotate svc drone-drone getambassador.io/config="$(cat ambassador-mapping.yaml)"v
https://github.com/kubernetes-incubator/external-dns
https://blog.ptrk.io/using-drone-on-kubernetes-to-deploy-a-basic-application/
helm install --name drone --namespace ci stable/drone
export POD_NAME=$(kubectl get pods -n ci -l "component=server,app=drone,release=drone" -o jsonpath="{.items[0].metadata.name}")
kubectl -n ci port-forward $POD_NAME --address 0.0.0.0 8003:8000
helm del --purge drone
TODO:
TTLAfterFinished https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/
Make Helm Chart for:
https://docs.drone.io/installation/gitea/kubernetes/
https://blog.drone.io/drone-kubernetes-secrets/
https://github.com/drone/drone-kubernetes-secrets
http://plugins.drone.io/vallard/drone-kube/
https://rook.io/docs/rook/v1.1/ceph-quickstart.html
kubectl apply -f https://raw.githubusercontent.com/rook/rook/v1.1.2/cluster/examples/kubernetes/ceph/common.yaml
kubectl apply -f https://raw.githubusercontent.com/rook/rook/v1.1.2/cluster/examples/kubernetes/ceph/operator.yaml
# sleep 10
kubectl apply -f https://raw.githubusercontent.com/rook/rook/v1.1.2/cluster/examples/kubernetes/ceph/cluster-test.yaml
kubectl apply -f https://raw.githubusercontent.com/rook/rook/v1.1.2/cluster/examples/kubernetes/ceph/storageclass-bucket-delete.yaml
# test
kubectl apply -f https://raw.githubusercontent.com/rook/rook/v1.1.2/cluster/examples/kubernetes/mysql.yaml
Access the ceph dashboard:
User: admin
Password: kubectl -n rook-ceph get secret rook-ceph-dashboard-password -o jsonpath='{.data.password}'|base64 -d;echo
Cleanup:
kubectl delete -f https://raw.githubusercontent.com/rook/rook/v1.1.2/cluster/examples/kubernetes/ceph/storageclass-bucket-delete.yaml
kubectl delete -f https://raw.githubusercontent.com/rook/rook/v1.1.2/cluster/examples/kubernetes/ceph/cluster-test.yaml
kubectl delete -f https://raw.githubusercontent.com/rook/rook/v1.1.2/cluster/examples/kubernetes/ceph/operator.yaml
kubectl delete -f https://raw.githubusercontent.com/rook/rook/v1.1.2/cluster/examples/kubernetes/ceph/common.yaml
kubectl delete -f https://raw.githubusercontent.com/rook/rook/v1.1.2/cluster/examples/kubernetes/mysql.yaml
# on all nodes
rm -rf /var/lib/rook
Troubleshooting:
http://cephnotes.ksperis.com/blog/2014/01/21/feature-set-mismatch-error-on-ceph-kernel-client/
# on node
dmesg | grep ceph
# all nodes
sudo systemctl stop docker
sudo iptables -t nat -F
sudo ip link del docker0
sudo systemctl start docker
Reset the cluster:
On all nodes
#!/bin/sh
yes | kubeadm reset
systemctl stop kubelet
systemctl stop docker
rm -rf /var/lib/cni/
rm -rf /var/lib/rook/
rm -rf /var/lib/kubelet/*
rm -rf /etc/cni/
rm -rf /var/etcd/calico-data/
iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X
ip link set kube-bridge down
ip link delete kube-bridge
ip link delete dummy0
ip link delete flannel.1
ip link delete cni0
systemctl start kubelet
systemctl start docker
docker image list | awk '!/IMAGE/{print $3}' | xargs docker image rm -f
Connection Refused:
tcpdump -n -i any host <host> and 'tcp[tcpflags] & (tcp-rst) != 0'
https://medium.com/dowjones/how-did-that-sidecar-get-there-4dcd73f1a0a4
https://github.com/dowjones/k8s-webhook
https://itnext.io/kubernetes-multi-cluster-networking-made-simple-c8f26827813
https://rancher.com/blog/2019/2019-01-17-101-more-kubernetes-security-best-practices/
https://www.cncf.io/blog/2019/01/14/9-kubernetes-security-best-practices-everyone-must-follow/
https://kubernetes.io/docs/concepts/policy/pod-security-policy/
https://github.com/ahmetb/kubernetes-network-policy-recipes
Old: https://github.com/kubernetes/kubernetes/tree/5520386b180d3ddc4fa7b7dfe6f52642cc0c25f3/build/pause
New: https://github.com/kubernetes/kubernetes/tree/c0b73645637c248091358a37c7e99bfcfc983f71/build/pause
https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/
kubectl run busybox --rm -ti --image=busybox /bin/sh
kubectl proxy
http://127.0.0.1:8001/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy/
kubectl port-forward $(kubectl get pod -l app=testapp-srv -o jsonpath='{.items[0].metadata.name}') 8002:8080
kubectl set image deployment testapp-db db=127.0.0.1:5000/testapp/db:v2
kubectl set image deployment testapp-srv srv=127.0.0.1:5000/testapp/srv:v2
kubectl rollout status -w deployment testapp-db
kubectl rollout status -w deployment testapp-srv
while true; do curl -s -H "Host: srv.kube" http://127.0.0.1:8002/; sleep 1; done
kubectl logs -f <container>
kubectl delete pods <pod> --grace-period=0 --force
kubectl patch pod <pod> -p '{"metadata":{"finalizers":null}}'
NS=<namespace to delete>
for r in $(kubectl api-resources --verbs=list --namespaced -o name); do kubectl -n $NS delete $r --all >/dev/null 2>&1; done
kubectl get ns $NS -ojson | jq -M '.metadata.finalizers=[]' >ns.json
awk '/client-certificate/{print $2}' ~/.kube/config|base64 -d >c.pem
awk '/client-key/{print $2}' ~/.kube/config|base64 -d >c.key
curl -k -E c.pem --key c.key -H "Content-Type: application/json" -X PUT --data-binary @ns.json $(awk '/server/{print $2}' ~/.kube/config)/api/v1/namespaces/$NS/finalize
rm -f c.pem c.key ns.json
https://github.com/alexei-led/nsenter
https://www.talos-systems.com/docs/talos/
https://github.com/ahmetb/kubectl-aliases
curl https://raw.githubusercontent.com/ahmetb/kubectl-aliases/master/.kubectl_aliases >~/.kubectl_aliases
echo ". ~/.kubectl_aliases" >>~/.bashrc
https://blog.tekspace.io/setup-kubernetes-cluster-with-ubuntu-16-04/
https://kubedex.com/kubernetes-network-plugins/
https://blog.docker.com/2019/07/intro-guide-to-dockerfile-best-practices/
https://www.padok.fr/en/blog/kubernetes-productivity-tips
https://medium.com/@gregoire.waymel/istio-cert-manager-lets-encrypt-demystified-c1cbed011d67