GoWiki | k8s | Rook Ceph

# Rook Ceph

## Install

Create the operator:

```
mkdir /etc/kubernetes/rook
curl -s https://raw.githubusercontent.com/rook/rook/v1.2.2/cluster/examples/kubernetes/ceph/common.yaml >/etc/kubernetes/rook/common.yaml
curl -s https://raw.githubusercontent.com/rook/rook/v1.2.2/cluster/examples/kubernetes/ceph/operator.yaml >/etc/kubernetes/rook/operator.yaml
kubectl apply -f /etc/kubernetes/rook
```

### Ceph Cluster

#### Hetzner Cloud

Setup: 2 Worker Nodes.

```
cat <<EOF> /etc/kubernetes/rook/cluster.yaml
apiVersion: ceph.rook.io/v1
kind: CephCluster
metadata:
  name: rook-ceph
  namespace: rook-ceph
spec:
  cephVersion:
    image: ceph/ceph:v14.2.6
  dataDirHostPath: /var/lib/rook
  mon:
    count: 2
    allowMultiplePerNode: true
  dashboard:
    enabled: true
  storage:
    useAllNodes: false
    useAllDevices: false
    config:
      databaseSizeMB: "1024"
      journalSizeMB: "1024"
    nodes:
    - name: "kw1"
      directories:
      - path: "/rook/storage-dir"
    - name: "kw2"
      directories:
      - path: "/rook/storage-dir"
EOF
kubectl apply -f /etc/kubernetes/rook/cluster.yaml
```

#### Bare-Metal

Setup: 3 Worker Nodes.

```
cat <<EOF> /etc/kubernetes/rook/cluster.yaml
apiVersion: ceph.rook.io/v1
kind: CephCluster
metadata:
  name: rook-ceph
  namespace: rook-ceph
spec:
  cephVersion:
    image: ceph/ceph:v14.2.6
  dataDirHostPath: /var/lib/rook
  mon:
    count: 3
    allowMultiplePerNode: true
  dashboard:
    enabled: true
  storage:
    useAllNodes: false
    useAllDevices: false
    config:
      databaseSizeMB: "1024"
      journalSizeMB: "1024"
    nodes:
    - name: "node01"
      devices:
      - name: "sdb"
    - name: "node02"
      devices:
      - name: "sdb"
    - name: "node03"
      devices:
      - name: "sdb"
EOF
kubectl apply -f /etc/kubernetes/rook/cluster.yaml
```

### Ceph Shared Filesystem

#### Hetzner Cloud

Setup: 2 Worker Nodes.

```
cat <<EOF> /etc/kubernetes/rook/filesystem.yaml
apiVersion: ceph.rook.io/v1
kind: CephFilesystem
metadata:
  name: myfs
  namespace: rook-ceph
spec:
  metadataPool:
    replicated:
      size: 2
  dataPools:
  - failureDomain: host
    replicated:
      size: 2
  metadataServer:
    activeCount: 1
    activeStandby: true
EOF
kubectl apply -f /etc/kubernetes/rook/filesystem.yaml
```

#### Bare-Metal

Setup: 3 Worker Nodes.

```
cat <<EOF> /etc/kubernetes/rook/filesystem.yaml
apiVersion: ceph.rook.io/v1
kind: CephFilesystem
metadata:
  name: myfs
  namespace: rook-ceph
spec:
  metadataPool:
    replicated:
      size: 3
  dataPools:
  - failureDomain: host
    replicated:
      size: 3
  metadataServer:
    activeCount: 1
    activeStandby: true
EOF
kubectl apply -f /etc/kubernetes/rook/filesystem.yaml
```

Ceph CSI Storage Class:

```
cat <<EOF> /etc/kubernetes/rook/storageclass.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: csi-cephfs
  annotations:
    storageclass.kubernetes.io/is-default-class: "true"
provisioner: rook-ceph.cephfs.csi.ceph.com
parameters:
  clusterID: rook-ceph
  fsName: myfs
  pool: myfs-data0
  csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
  csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
  csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
  csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
reclaimPolicy: Delete
EOF
kubectl apply -f /etc/kubernetes/rook/storageclass.yaml
```

Ceph Dashboard Ingress:

User: admin

Password: `kubectl -n rook-ceph get secret rook-ceph-dashboard-password -o jsonpath='{.data.password}'|base64 -d;echo`

```
cat <<EOF> /etc/kubernetes/rook/ingress.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: rook-ceph-mgr-dashboard
  namespace: rook-ceph
  annotations:
    kubernetes.io/ingress.class: "nginx"
    #nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
spec:
  #tls:
  #- hosts:
  #  - ceph.k8s.example.com
  rules:
  - host: ceph.k8s.example.com
    http:
      paths:
      - path: /
        backend:
          serviceName: rook-ceph-mgr-dashboard
          servicePort: dashboard
EOF
kubectl apply -f /etc/kubernetes/rook/ingress.yaml
```

## External Ceph Cluster

kubectl create ns rook-ceph-external
kubectl apply -f /etc/kubernetes/rook
```

### Create Ceph Keys

Run this script on one of the ceph nodes:

```
ceph osd pool application set cephfs_metadata cephfs metadata cephfs
ceph osd pool application set cephfs_data cephfs data cephfs
 
curl -s -O https://raw.githubusercontent.com/rook/rook/v1.3.2/cluster/examples/kubernetes/ceph/create-external-cluster-resources.sh
echo "export ROOK_EXTERNAL_FSID=$(grep fsid /etc/ceph/ceph.conf | cut -d" " -f4)"
echo "export ROOK_EXTERNAL_CEPH_MON_DATA=\"$(i=97;for m in $(grep -o "mon_host.*" /etc/ceph/ceph.conf|sed -e 's/mon_host = //');do printf "\x$(printf %x $i)=${m}:6789,";i=$((i+1));done)\""
bash create-external-cluster-resources.sh
```

### Import Cluster

Now use the export commands from the `create-external-cluster-resources.sh` script and run the import script on the kubernetes master:

```
mkdir /etc/kubernetes/rook
curl -s https://raw.githubusercontent.com/rook/rook/v1.3.2/cluster/examples/kubernetes/ceph/common.yaml >/etc/kubernetes/rook/common.yaml
curl -s https://raw.githubusercontent.com/rook/rook/v1.3.2/cluster/examples/kubernetes/ceph/operator.yaml >/etc/kubernetes/rook/operator.yaml
 
kubectl create ns rook-ceph-external
kubectl apply -f /etc/kubernetes/rook
 
 
curl -s -O https://raw.githubusercontent.com/rook/rook/v1.3.2/cluster/examples/kubernetes/ceph/import-external-cluster.sh
# sed patch for import script
sed -i -e '/$CSI_CEPHFS_NODE_SECRET_NAME/a\    --from-literal=adminID=csi-cephfs-provisioner \\\n    --from-literal=adminKey="$CSI_CEPHFS_PROVISIONER_SECRET" \\' \
  -e '/$CSI_CEPHFS_PROVISIONER_SECRET_NAME/a\    --from-literal=adminID=csi-cephfs-provisioner \\\n    --from-literal=adminKey="$CSI_CEPHFS_PROVISIONER_SECRET" \\' \
  import-external-cluster.sh
 
# <hier die export Commands ausführen>
export NAMESPACE=rook-ceph-external
bash import-external-cluster.sh
 
curl -s https://raw.githubusercontent.com/rook/rook/v1.3.2/cluster/examples/kubernetes/ceph/cluster-external.yaml >/etc/kubernetes/rook/cluster-external.yaml
kubectl apply -f /etc/kubernetes/rook/cluster-external.yaml
```

### Create Storage Classes

#### Cephfs

```
cat <<EOF> /etc/kubernetes/rook/cephfs-storageclass.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: cephfs
provisioner: rook-ceph.cephfs.csi.ceph.com
parameters:
  clusterID: rook-ceph-external
  # cephfs for proxmox clusters
  fsName: cephfs
  # cephfs_data for proxmox clusters
  pool: cephfs_data
  csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
  csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph-external
  csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner
  csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph-external
  csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
  csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph-external
reclaimPolicy: Delete
allowVolumeExpansion: true
mountOptions:
EOF
kubectl apply -f /etc/kubernetes/rook/cephfs-storageclass.yaml
```

#### RBD

```
cat <<EOF> /etc/kubernetes/rook/ssd-storageclass.yaml
apiVersion: ceph.rook.io/v1
kind: CephBlockPool
metadata:
  name: ssd
  namespace: rook-ceph-external
spec:
  failureDomain: host
  replicated:
    size: 3
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: ssd
  annotations:
    storageclass.kubernetes.io/is-default-class: "true"
provisioner: rook-ceph.rbd.csi.ceph.com
parameters:
  clusterID: rook-ceph-external
  pool: ssd
  imageFormat: "2"
  imageFeatures: layering
  csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
  csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph-external
  csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner
  csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph-external
  csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
  csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph-external
  csi.storage.k8s.io/fstype: ext4
allowVolumeExpansion: true
reclaimPolicy: Delete
EOF
kubectl apply -f /etc/kubernetes/rook/ssd-storageclass.yaml
```

## Ceph Debug Container

https://rook.io/docs/rook/v1.3.2/ceph-toolbox.html

```
cat <<EOF> /etc/kubernetes/rook/toolbox.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: rook-ceph-tools
  namespace: rook-ceph
  labels:
    app: rook-ceph-tools
spec:
  replicas: 1
  selector:
    matchLabels:
      app: rook-ceph-tools
  template:
    metadata:
      labels:
        app: rook-ceph-tools
    spec:
      dnsPolicy: ClusterFirstWithHostNet
      containers:
      - name: rook-ceph-tools
        image: rook/ceph:v1.2.2
        command: ["/tini"]
        args: ["-g", "--", "/usr/local/bin/toolbox.sh"]
        imagePullPolicy: IfNotPresent
        env:
          - name: ROOK_ADMIN_SECRET
            valueFrom:
              secretKeyRef:
                name: rook-ceph-mon
                key: admin-secret
        securityContext:
          privileged: true
        volumeMounts:
          - mountPath: /dev
            name: dev
          - mountPath: /sys/bus
            name: sysbus
          - mountPath: /lib/modules
            name: libmodules
          - name: mon-endpoint-volume
            mountPath: /etc/rook
      hostNetwork: true
      volumes:
        - name: dev
          hostPath:
            path: /dev
        - name: sysbus
          hostPath:
            path: /sys/bus
        - name: libmodules
          hostPath:
            path: /lib/modules
        - name: mon-endpoint-volume
          configMap:
            name: rook-ceph-mon-endpoints
            items:
            - key: data
              path: mon-endpoints
EOF
kubectl apply -f /etc/kubernetes/rook/toolbox.yaml
```

Connect to debug container:

```
kubectl -n rook-ceph exec -it $(kubectl -n rook-ceph get pod -l "app=rook-ceph-tools" -ojsonpath='{.items[0].metadata.name}') bash
```

# Cleanup

Cleanup devices and directories on nodes:

```
rm -rf /var/lib/rook /rook/storage-dir
for id in $(dmsetup table | grep ceph | cut -d":" -f1); do dmsetup remove $id; done
dd if=/dev/zero of=/dev/sdb bs=1M count=100; sync
```