Create the operator:
mkdir /etc/kubernetes/rook
curl -s https://raw.githubusercontent.com/rook/rook/v1.2.2/cluster/examples/kubernetes/ceph/common.yaml >/etc/kubernetes/rook/common.yaml
curl -s https://raw.githubusercontent.com/rook/rook/v1.2.2/cluster/examples/kubernetes/ceph/operator.yaml >/etc/kubernetes/rook/operator.yaml
kubectl apply -f /etc/kubernetes/rook
Setup: 2 Worker Nodes.
cat <<EOF> /etc/kubernetes/rook/cluster.yaml
apiVersion: ceph.rook.io/v1
kind: CephCluster
metadata:
name: rook-ceph
namespace: rook-ceph
spec:
cephVersion:
image: ceph/ceph:v14.2.6
dataDirHostPath: /var/lib/rook
mon:
count: 2
allowMultiplePerNode: true
dashboard:
enabled: true
storage:
useAllNodes: false
useAllDevices: false
config:
databaseSizeMB: "1024"
journalSizeMB: "1024"
nodes:
- name: "kw1"
directories:
- path: "/rook/storage-dir"
- name: "kw2"
directories:
- path: "/rook/storage-dir"
EOF
kubectl apply -f /etc/kubernetes/rook/cluster.yaml
Setup: 3 Worker Nodes.
cat <<EOF> /etc/kubernetes/rook/cluster.yaml
apiVersion: ceph.rook.io/v1
kind: CephCluster
metadata:
name: rook-ceph
namespace: rook-ceph
spec:
cephVersion:
image: ceph/ceph:v14.2.6
dataDirHostPath: /var/lib/rook
mon:
count: 3
allowMultiplePerNode: true
dashboard:
enabled: true
storage:
useAllNodes: false
useAllDevices: false
config:
databaseSizeMB: "1024"
journalSizeMB: "1024"
nodes:
- name: "node01"
devices:
- name: "sdb"
- name: "node02"
devices:
- name: "sdb"
- name: "node03"
devices:
- name: "sdb"
EOF
kubectl apply -f /etc/kubernetes/rook/cluster.yaml
Setup: 2 Worker Nodes.
cat <<EOF> /etc/kubernetes/rook/filesystem.yaml
apiVersion: ceph.rook.io/v1
kind: CephFilesystem
metadata:
name: myfs
namespace: rook-ceph
spec:
metadataPool:
replicated:
size: 2
dataPools:
- failureDomain: host
replicated:
size: 2
metadataServer:
activeCount: 1
activeStandby: true
EOF
kubectl apply -f /etc/kubernetes/rook/filesystem.yaml
Setup: 3 Worker Nodes.
cat <<EOF> /etc/kubernetes/rook/filesystem.yaml
apiVersion: ceph.rook.io/v1
kind: CephFilesystem
metadata:
name: myfs
namespace: rook-ceph
spec:
metadataPool:
replicated:
size: 3
dataPools:
- failureDomain: host
replicated:
size: 3
metadataServer:
activeCount: 1
activeStandby: true
EOF
kubectl apply -f /etc/kubernetes/rook/filesystem.yaml
Ceph CSI Storage Class:
cat <<EOF> /etc/kubernetes/rook/storageclass.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: csi-cephfs
annotations:
storageclass.kubernetes.io/is-default-class: "true"
provisioner: rook-ceph.cephfs.csi.ceph.com
parameters:
clusterID: rook-ceph
fsName: myfs
pool: myfs-data0
csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
reclaimPolicy: Delete
EOF
kubectl apply -f /etc/kubernetes/rook/storageclass.yaml
Ceph Dashboard Ingress:
User: admin
Password: kubectl -n rook-ceph get secret rook-ceph-dashboard-password -o jsonpath='{.data.password}'|base64 -d;echo
cat <<EOF> /etc/kubernetes/rook/ingress.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: rook-ceph-mgr-dashboard
namespace: rook-ceph
annotations:
kubernetes.io/ingress.class: "nginx"
#nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
spec:
#tls:
#- hosts:
# - ceph.k8s.example.com
rules:
- host: ceph.k8s.example.com
http:
paths:
- path: /
backend:
serviceName: rook-ceph-mgr-dashboard
servicePort: dashboard
EOF
kubectl apply -f /etc/kubernetes/rook/ingress.yaml
mkdir /etc/kubernetes/rook
curl -s https://raw.githubusercontent.com/rook/rook/v1.3.2/cluster/examples/kubernetes/ceph/common.yaml >/etc/kubernetes/rook/common.yaml
curl -s https://raw.githubusercontent.com/rook/rook/v1.3.2/cluster/examples/kubernetes/ceph/operator.yaml >/etc/kubernetes/rook/operator.yaml
curl -s https://raw.githubusercontent.com/rook/rook/v1.3.2/cluster/examples/kubernetes/ceph/cluster-external.yaml >/etc/kubernetes/rook/cluster-external.yaml
kubectl create ns rook-ceph-external
kubectl apply -f /etc/kubernetes/rook
Run this script on one of the ceph nodes:
ceph osd pool application set cephfs_metadata cephfs metadata cephfs
ceph osd pool application set cephfs_data cephfs data cephfs
curl -s -O https://raw.githubusercontent.com/rook/rook/v1.3.2/cluster/examples/kubernetes/ceph/create-external-cluster-resources.sh
echo "export ROOK_EXTERNAL_FSID=$(grep fsid /etc/ceph/ceph.conf | cut -d" " -f4)"
echo "export ROOK_EXTERNAL_CEPH_MON_DATA=\"$(i=97;for m in $(grep -o "mon_host.*" /etc/ceph/ceph.conf|sed -e 's/mon_host = //');do printf "\x$(printf %x $i)=${m}:6789,";i=$((i+1));done)\""
bash create-external-cluster-resources.sh
Now use the export commands from the create-external-cluster-resources.sh script and run the import script on the kubernetes master:
mkdir /etc/kubernetes/rook
curl -s https://raw.githubusercontent.com/rook/rook/v1.3.2/cluster/examples/kubernetes/ceph/common.yaml >/etc/kubernetes/rook/common.yaml
curl -s https://raw.githubusercontent.com/rook/rook/v1.3.2/cluster/examples/kubernetes/ceph/operator.yaml >/etc/kubernetes/rook/operator.yaml
kubectl create ns rook-ceph-external
kubectl apply -f /etc/kubernetes/rook
curl -s -O https://raw.githubusercontent.com/rook/rook/v1.3.2/cluster/examples/kubernetes/ceph/import-external-cluster.sh
# sed patch for import script
sed -i -e '/$CSI_CEPHFS_NODE_SECRET_NAME/a\ --from-literal=adminID=csi-cephfs-provisioner \\\n --from-literal=adminKey="$CSI_CEPHFS_PROVISIONER_SECRET" \\' \
-e '/$CSI_CEPHFS_PROVISIONER_SECRET_NAME/a\ --from-literal=adminID=csi-cephfs-provisioner \\\n --from-literal=adminKey="$CSI_CEPHFS_PROVISIONER_SECRET" \\' \
import-external-cluster.sh
# <hier die export Commands ausführen>
export NAMESPACE=rook-ceph-external
bash import-external-cluster.sh
curl -s https://raw.githubusercontent.com/rook/rook/v1.3.2/cluster/examples/kubernetes/ceph/cluster-external.yaml >/etc/kubernetes/rook/cluster-external.yaml
kubectl apply -f /etc/kubernetes/rook/cluster-external.yaml
cat <<EOF> /etc/kubernetes/rook/cephfs-storageclass.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: cephfs
provisioner: rook-ceph.cephfs.csi.ceph.com
parameters:
clusterID: rook-ceph-external
# cephfs for proxmox clusters
fsName: cephfs
# cephfs_data for proxmox clusters
pool: cephfs_data
csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph-external
csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner
csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph-external
csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph-external
reclaimPolicy: Delete
allowVolumeExpansion: true
mountOptions:
EOF
kubectl apply -f /etc/kubernetes/rook/cephfs-storageclass.yaml
cat <<EOF> /etc/kubernetes/rook/ssd-storageclass.yaml
apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
name: ssd
namespace: rook-ceph-external
spec:
failureDomain: host
replicated:
size: 3
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: ssd
annotations:
storageclass.kubernetes.io/is-default-class: "true"
provisioner: rook-ceph.rbd.csi.ceph.com
parameters:
clusterID: rook-ceph-external
pool: ssd
imageFormat: "2"
imageFeatures: layering
csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph-external
csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner
csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph-external
csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph-external
csi.storage.k8s.io/fstype: ext4
allowVolumeExpansion: true
reclaimPolicy: Delete
EOF
kubectl apply -f /etc/kubernetes/rook/ssd-storageclass.yaml
https://rook.io/docs/rook/v1.3.2/ceph-toolbox.html
cat <<EOF> /etc/kubernetes/rook/toolbox.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: rook-ceph-tools
namespace: rook-ceph
labels:
app: rook-ceph-tools
spec:
replicas: 1
selector:
matchLabels:
app: rook-ceph-tools
template:
metadata:
labels:
app: rook-ceph-tools
spec:
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: rook-ceph-tools
image: rook/ceph:v1.2.2
command: ["/tini"]
args: ["-g", "--", "/usr/local/bin/toolbox.sh"]
imagePullPolicy: IfNotPresent
env:
- name: ROOK_ADMIN_SECRET
valueFrom:
secretKeyRef:
name: rook-ceph-mon
key: admin-secret
securityContext:
privileged: true
volumeMounts:
- mountPath: /dev
name: dev
- mountPath: /sys/bus
name: sysbus
- mountPath: /lib/modules
name: libmodules
- name: mon-endpoint-volume
mountPath: /etc/rook
hostNetwork: true
volumes:
- name: dev
hostPath:
path: /dev
- name: sysbus
hostPath:
path: /sys/bus
- name: libmodules
hostPath:
path: /lib/modules
- name: mon-endpoint-volume
configMap:
name: rook-ceph-mon-endpoints
items:
- key: data
path: mon-endpoints
EOF
kubectl apply -f /etc/kubernetes/rook/toolbox.yaml
Connect to debug container:
kubectl -n rook-ceph exec -it $(kubectl -n rook-ceph get pod -l "app=rook-ceph-tools" -ojsonpath='{.items[0].metadata.name}') bash
Cleanup devices and directories on nodes:
rm -rf /var/lib/rook /rook/storage-dir
for id in $(dmsetup table | grep ceph | cut -d":" -f1); do dmsetup remove $id; done
dd if=/dev/zero of=/dev/sdb bs=1M count=100; sync