作者：尹正杰
版权声明：原创作品，谢绝转载！否则将追究法律责任。
一.准备测试环境
1.创建存储池
[root@ceph141 ~]# ceph osd pool create yinzhengjie 2 2
pool 'yinzhengjie' created
[root@ceph141 ~]#
2.对存储池启用rbd功能
[root@ceph141 ~]# ceph osd pool application get yinzhengjie
{}
[root@ceph141 ~]#
[root@ceph141 ~]# ceph osd pool application enable yinzhengjie rbd
enabled application 'rbd' on pool 'yinzhengjie'
[root@ceph141 ~]#
[root@ceph141 ~]# ceph osd pool application get yinzhengjie
{
"rbd": {}
}
[root@ceph141 ~]#
3.对存储池进行环境初始化
[root@ceph141 ~]# rbd pool init yinzhengjie
[root@ceph141 ~]#
[root@ceph141 ~]# rbd pool stats yinzhengjie
Total Images: 0
Total Snapshots: 0
Provisioned Size: 0 B
[root@ceph141 ~]#
4.创建块设备
[root@ceph141 ~]# rbd create wordpress -s 4G -p yinzhengjie
5.创建块设备详细信息
[root@ceph141 ~]# rbd ls -p yinzhengjie
wordpress
[root@ceph141 ~]#
[root@ceph141 ~]# rbd info yinzhengjie/wordpress
rbd image 'wordpress':
size 4 GiB in 1024 objects
order 22 (4 MiB objects)
snapshot_count: 0
id: ad4945cbcd9
block_name_prefix: rbd_data.ad4945cbcd9
format: 2
features: layering, exclusive-lock, object-map, fast-diff, deep-flatten
op_features:
flags:
create_timestamp: Wed Aug 28 00:55:28 2024
access_timestamp: Wed Aug 28 00:55:28 2024
modify_timestamp: Wed Aug 28 00:55:28 2024
[root@ceph141 ~]#
6.映射块设备
[root@ceph141 ~]# rbd map yinzhengjie/wordpress
/dev/rbd0
[root@ceph141 ~]#
7.格式化文件系统
[root@ceph141 ~]# mkfs.xfs /dev/rbd0
meta-data=/dev/rbd0 isize=512 agcount=8, agsize=131072 blks
= sectsz=512 attr=2, projid32bit=1
= crc=1 finobt=1, sparse=1, rmapbt=0
= reflink=1 bigtime=0 inobtcount=0
data = bsize=4096 blocks=1048576, imaxpct=25
= sunit=16 swidth=16 blks
naming =version 2 bsize=4096 ascii-ci=0, ftype=1
log =internal log bsize=4096 blocks=2560, version=2
= sectsz=512 sunit=16 blks, lazy-count=1
realtime =none extsz=4096 blocks=0, rtextents=0
Discarding blocks...Done.
[root@ceph141 ~]#
8.挂载磁盘并拷贝数据
[root@ceph141 ~]# mount /dev/rbd0 /mnt/
[root@ceph141 ~]#
[root@ceph141 ~]# cp /etc/os-release /mnt/
[root@ceph141 ~]#
[root@ceph141 ~]# cp /etc/hosts /mnt/
[root@ceph141 ~]#
[root@ceph141 ~]# ll /mnt/
total 12
drwxr-xr-x 2 root root 37 Aug 28 20:31 ./
drwxr-xr-x 21 root root 4096 Aug 21 20:54 ../
-rw-r--r-- 1 root root 283 Aug 28 20:31 hosts
-rw-r--r-- 1 root root 386 Aug 28 20:31 os-release
[root@ceph141 ~]#
[root@ceph141 ~]# rbd showmapped
id pool namespace image snap device
0 yinzhengjie wordpress - /dev/rbd0
[root@ceph141 ~]#
9.卸载块设备并取消映射
[root@ceph141 ~]# umount /mnt
[root@ceph141 ~]#
[root@ceph141 ~]# rbd unmap /dev/rbd0
[root@ceph141 ~]#
[root@ceph141 ~]# rbd showmapped
[root@ceph141 ~]#
二.服务端定制基础快照并保护模式
1.创建基础快照
[root@ceph141 ~]# rbd snap ls yinzhengjie/wordpress
[root@ceph141 ~]#
[root@ceph141 ~]# rbd snap create yinzhengjie/wordpress@clonewp01
Creating snap: 100% complete...done.
[root@ceph141 ~]#
[root@ceph141 ~]# rbd snap ls yinzhengjie/wordpress
SNAPID NAME SIZE PROTECTED TIMESTAMP
4 clonewp01 4 GiB Wed Aug 28 20:40:06 2024
[root@ceph141 ~]#
2.将快照置于保护模式
[root@ceph141 ~]# rbd snap protect yinzhengjie/wordpress@clonewp01
[root@ceph141 ~]#
[root@ceph141 ~]# rbd snap ls yinzhengjie/wordpress # 很明显，当前镜像的clonewp01快照已经被咱们保护起来了
SNAPID NAME SIZE PROTECTED TIMESTAMP
4 clonewp01 4 GiB yes Wed Aug 28 20:40:06 2024
[root@ceph141 ~]#
3.被"保护"的快照是无法删除的
[root@ceph141 ~]# rbd snap rm yinzhengjie/wordpress@clonewp01
Removing snap: 0% complete...failed.
2024-08-28T22:13:23.313+0800 7fcb7f7fe640 -1 librbd::Operations: snapshot is protected
rbd: snapshot 'clonewp01' is protected from removal.
[root@ceph141 ~]#
三.基于快照进行克隆操作
1.基于基础快照模板克隆新镜像
[root@ceph141 ~]# ceph osd pool ls
.mgr
yinzhengjie-rbd
yinzhengjie
[root@ceph141 ~]#
[root@ceph141 ~]# rbd ls -p yinzhengjie
wordpress
[root@ceph141 ~]#
[root@ceph141 ~]# rbd ls -p yinzhengjie-rbd
k8s
[root@ceph141 ~]#
[root@ceph141 ~]# rbd clone yinzhengjie/wordpress@clonewp01 yinzhengjie/wp01
[root@ceph141 ~]#
[root@ceph141 ~]# rbd clone yinzhengjie/wordpress@clonewp01 yinzhengjie-rbd/wp02 # 克隆时块设备可以不在同一个存储池，但是使用时可能会存在问题。
[root@ceph141 ~]#
[root@ceph141 ~]# rbd clone yinzhengjie/wordpress@clonewp01 yinzhengjie/wp03
[root@ceph141 ~]#
2.查看克隆的信息
[root@ceph141 ~]# rbd ls -p yinzhengjie-rbd -l
NAME SIZE PARENT FMT PROT LOCK
k8s 5 GiB 2
wp02 4 GiB yinzhengjie/wordpress@clonewp01 2
[root@ceph141 ~]#
[root@ceph141 ~]# rbd ls -p yinzhengjie -l
NAME SIZE PARENT FMT PROT LOCK
wordpress 4 GiB 2
wordpress@clonewp01 4 GiB 2 yes
wp01 4 GiB yinzhengjie/wordpress@clonewp01 2
wp03 4 GiB yinzhengjie/wordpress@clonewp01 2
[root@ceph141 ~]#
3.查看基础快照目标是否有子镜像
[root@ceph141 ~]# rbd children yinzhengjie/wordpress@clonewp01
yinzhengjie/wp01
yinzhengjie/wp03
yinzhengjie-rbd/wp02
[root@ceph141 ~]#
四.客户端挂载新镜像进行读写测试
温馨提示:
经过我测试发现如果将多个块设备在同一个节点无法挂载的情况，报错: "wrong fs type, bad option, bad superblock on /dev/rbd1, missing codepage or helper program, or other error."
要解决这个问题，可以将3个块设备分别挂载到不同的节点测试即可。本案例就时将3个块设备分别挂载到不同的节点测试。我测试的版本是Reef 18.2.4。
1.ceph141节点添加映射块设备测试，无需格式化
[root@ceph141 ~]# rbd showmapped
[root@ceph141 ~]#
[root@ceph141 ~]# rbd map yinzhengjie/wp01
/dev/rbd0
[root@ceph141 ~]#
[root@ceph141 ~]# mkdir -pv /yinzhengjie/data/wp01
[root@ceph141 ~]#
[root@ceph141 ~]# rbd showmapped
id pool namespace image snap device
0 yinzhengjie wp01 - /dev/rbd0
[root@ceph141 ~]#
[root@ceph141 ~]# mount /dev/rbd0 /yinzhengjie/data/wp01/
[root@ceph141 ~]#
[root@ceph141 ~]# ll /yinzhengjie/data/wp01/
total 12
drwxr-xr-x 2 root root 37 Aug 28 20:31 ./
drwxr-xr-x 6 root root 4096 Aug 28 21:44 ../
-rw-r--r-- 1 root root 283 Aug 28 20:31 hosts
-rw-r--r-- 1 root root 386 Aug 28 20:31 os-release
[root@ceph141 ~]#
[root@ceph141 ~]# cp /etc/fstab /etc/hostname /yinzhengjie/data/wp01/
[root@ceph141 ~]#
[root@ceph141 ~]# ll /yinzhengjie/data/wp01/
total 20
drwxr-xr-x 2 root root 66 Aug 28 21:45 ./
drwxr-xr-x 6 root root 4096 Aug 28 21:44 ../
-rw-r--r-- 1 root root 657 Aug 28 21:45 fstab
-rw-r--r-- 1 root root 8 Aug 28 21:45 hostname
-rw-r--r-- 1 root root 283 Aug 28 20:31 hosts
-rw-r--r-- 1 root root 386 Aug 28 20:31 os-release
[root@ceph141 ~]#
2.ceph142节点添加映射块设备测试，无需格式化
[root@ceph142 ~]# rbd showmapped
[root@ceph142 ~]#
[root@ceph142 ~]# rbd map yinzhengjie-rbd/wp02
/dev/rbd0
[root@ceph142 ~]#
[root@ceph142 ~]# rbd showmapped
id pool namespace image snap device
0 yinzhengjie-rbd wp02 - /dev/rbd0
[root@ceph142 ~]#
[root@ceph142 ~]# mkdir -pv /yinzhengjie/data/wp02
[root@ceph142 ~]#
[root@ceph142 ~]# mount /dev/rbd0 /yinzhengjie/data/wp02/
[root@ceph142 ~]#
[root@ceph142 ~]# ll /yinzhengjie/data/wp02/
total 12
drwxr-xr-x 2 root root 37 Aug 28 21:48 ./
drwxr-xr-x 3 root root 4096 Aug 28 21:48 ../
-rw-r--r-- 1 root root 283 Aug 28 20:31 hosts
-rw-r--r-- 1 root root 386 Aug 28 20:31 os-release
[root@ceph142 ~]#
[root@ceph142 ~]# rm -f /yinzhengjie/data/wp02/os-release
[root@ceph142 ~]#
[root@ceph142 ~]# ll /yinzhengjie/data/wp02/
total 8
drwxr-xr-x 2 root root 19 Aug 28 21:48 ./
drwxr-xr-x 3 root root 4096 Aug 28 21:48 ../
-rw-r--r-- 1 root root 283 Aug 28 20:31 hosts
[root@ceph142 ~]#
3.ceph143节点添加映射块设备测试，无需格式化
[root@ceph143 ~]# rbd showmapped
[root@ceph143 ~]#
[root@ceph143 ~]# rbd map yinzhengjie/wp03
/dev/rbd0
[root@ceph143 ~]#
[root@ceph143 ~]# rbd showmapped
id pool namespace image snap device
0 yinzhengjie wp03 - /dev/rbd0
[root@ceph143 ~]#
[root@ceph143 ~]# mkdir -pv /yinzhengjie/data/wp03
[root@ceph143 ~]#
[root@ceph143 ~]# mount /dev/rbd0 /yinzhengjie/data/wp03
[root@ceph143 ~]#
[root@ceph143 ~]# ll /yinzhengjie/data/wp03
total 12
drwxr-xr-x 2 root root 37 Aug 28 20:31 ./
drwxr-xr-x 6 root root 4096 Aug 28 21:50 ../
-rw-r--r-- 1 root root 283 Aug 28 20:31 hosts
-rw-r--r-- 1 root root 386 Aug 28 20:31 os-release
[root@ceph143 ~]#
[root@ceph143 ~]# cp /etc/netplan/00-installer-config.yaml /yinzhengjie/data/wp03
[root@ceph143 ~]#
[root@ceph143 ~]# ll /yinzhengjie/data/wp03
total 16
drwxr-xr-x 2 root root 69 Aug 28 21:51 ./
drwxr-xr-x 6 root root 4096 Aug 28 21:50 ../
-rw------- 1 root root 367 Aug 28 21:51 00-installer-config.yaml
-rw-r--r-- 1 root root 283 Aug 28 20:31 hosts
-rw-r--r-- 1 root root 386 Aug 28 20:31 os-release
[root@ceph143 ~]#
五.客户端取消镜像挂载
1.ceph141节点操作
[root@ceph141 ~]# rbd showmapped
id pool namespace image snap device
0 yinzhengjie wp01 - /dev/rbd0
[root@ceph141 ~]#
[root@ceph141 ~]# umount /yinzhengjie/data/wp01
[root@ceph141 ~]#
[root@ceph141 ~]# rbd unmap /dev/rbd0
[root@ceph141 ~]#
[root@ceph141 ~]# rbd showmapped
[root@ceph141 ~]#
2.ceph142节点操作
[root@ceph142 ~]# rbd showmapped
id pool namespace image snap device
0 yinzhengjie-rbd wp02 - /dev/rbd0
[root@ceph142 ~]#
[root@ceph142 ~]# umount /yinzhengjie/data/wp02
[root@ceph142 ~]#
[root@ceph142 ~]# rbd unmap /dev/rbd0
[root@ceph142 ~]#
[root@ceph142 ~]# rbd showmapped
[root@ceph142 ~]#
3.ceph143节点操作
[root@ceph143 ~]# rbd showmapped
id pool namespace image snap device
0 yinzhengjie wp03 - /dev/rbd0
[root@ceph143 ~]#
[root@ceph143 ~]# umount /yinzhengjie/data/wp03
[root@ceph143 ~]#
[root@ceph143 ~]# rbd unmap /dev/rbd0
[root@ceph143 ~]#
[root@ceph143 ~]# rbd showmapped
[root@ceph143 ~]#
六.删除基础镜像的快照
1.子快照需要展平数据
温馨提示:
底层是将基础镜像的快照数据拷贝到子快照中，这个速度取决于镜像的大小
[root@ceph141 ~]# rbd children yinzhengjie/wordpress@clonewp01 # 注意，此处我故意不对"yinzhengjie/wp03"的进行展平。
yinzhengjie/wp01
yinzhengjie/wp03
yinzhengjie-rbd/wp02
[root@ceph141 ~]#
[root@ceph141 ~]# rbd flatten yinzhengjie/wp01
Image flatten: 100% complete...done.
[root@ceph141 ~]#
[root@ceph141 ~]# rbd flatten yinzhengjie-rbd/wp02
Image flatten: 100% complete...done.
[root@ceph141 ~]#
[root@ceph141 ~]#
[root@ceph141 ~]# rbd children yinzhengjie/wordpress@clonewp01 # 很明显，此处基础快照仅有一个我没有展平的子快照啦～
yinzhengjie/wp03
[root@ceph141 ~]#
2.取消保护基础镜像的快照保护
[root@ceph141 ~]# rbd snap unprotect yinzhengjie/wordpress@clonewp01. # 很明显，如果不展平子快照，这里就不允许取消保护
2024-08-28T22:09:04.182+0800 7f880530a640 -1 librbd::SnapshotUnprotectRequest: cannot unprotect: at least 1 child(ren) [d4495e828556] in pool 'yinzhengjie'
2024-08-28T22:09:04.182+0800 7f8805b0b640 -1 librbd::SnapshotUnprotectRequest: encountered error: (16) Device or resource busy
2024-08-28T22:09:04.182+0800 7f8805b0b640 -1 librbd::SnapshotUnprotectRequest: 0x55e0d195abd0 should_complete_error: ret_val=-16
rbd: unprotecting snap failed: 2024-08-28T22:09:04.190+0800 7f880530a640 -1 librbd::SnapshotUnprotectRequest: 0x55e0d195abd0 should_complete_error: ret_val=-16
(16) Device or resource busy
[root@ceph141 ~]#
[root@ceph141 ~]# rbd flatten yinzhengjie/wp03
Image flatten: 100% complete...done.
[root@ceph141 ~]#
[root@ceph141 ~]# rbd children yinzhengjie/wordpress@clonewp01 # Duang，此处看不到有子快照啦。
[root@ceph141 ~]#
[root@ceph141 ~]# rbd snap unprotect yinzhengjie/wordpress@clonewp01 # Duang，可以取消保护啦！
[root@ceph141 ~]#
3.删除基础镜像的快照
[root@ceph141 ~]# rbd snap ls yinzhengjie/wordpress
SNAPID NAME SIZE PROTECTED TIMESTAMP
4 clonewp01 4 GiB Wed Aug 28 20:40:06 2024
[root@ceph141 ~]#
[root@ceph141 ~]# rbd snap rm yinzhengjie/wordpress@clonewp01
Removing snap: 100% complete...done.
[root@ceph141 ~]#
[root@ceph141 ~]# rbd snap ls yinzhengjie/wordpress # Duang，快照删除成功啦！
[root@ceph141 ~]#
七 客户端再次映射测试
温馨提示:
经过测试发现，尽管删除了基础镜像的快照，并不会影响子镜像的使用哟～
1.ceph141节点
[root@ceph141 ~]# rbd showmapped
[root@ceph141 ~]#
[root@ceph141 ~]# rbd map yinzhengjie/wp01
/dev/rbd0
[root@ceph141 ~]#
[root@ceph141 ~]# mount /dev/rbd0 /yinzhengjie/data/wp01/
[root@ceph141 ~]#
[root@ceph141 ~]# ll /yinzhengjie/data/wp01/
total 20
drwxr-xr-x 2 root root 66 Aug 28 21:45 ./
drwxr-xr-x 6 root root 4096 Aug 28 21:44 ../
-rw-r--r-- 1 root root 657 Aug 28 21:45 fstab
-rw-r--r-- 1 root root 8 Aug 28 21:45 hostname
-rw-r--r-- 1 root root 283 Aug 28 20:31 hosts
-rw-r--r-- 1 root root 386 Aug 28 20:31 os-release
[root@ceph141 ~]#
[root@ceph141 ~]# rm -f /yinzhengjie/data/wp01/h*
[root@ceph141 ~]#
[root@ceph141 ~]# ll /yinzhengjie/data/wp01/
total 12
drwxr-xr-x 2 root root 37 Aug 28 22:22 ./
drwxr-xr-x 6 root root 4096 Aug 28 21:44 ../
-rw-r--r-- 1 root root 657 Aug 28 21:45 fstab
-rw-r--r-- 1 root root 386 Aug 28 20:31 os-release
[root@ceph141 ~]#
[root@ceph141 ~]# umount /yinzhengjie/data/wp01
[root@ceph141 ~]#
[root@ceph141 ~]# rbd showmapped
id pool namespace image snap device
0 yinzhengjie wp01 - /dev/rbd0
[root@ceph141 ~]#
[root@ceph141 ~]# rbd unmap /dev/rbd0
[root@ceph141 ~]#
[root@ceph141 ~]# rbd showmapped
[root@ceph141 ~]#
2.ceph142节点
[root@ceph142 ~]# rbd showmapped
[root@ceph142 ~]#
[root@ceph142 ~]# rbd map yinzhengjie-rbd/wp02
/dev/rbd0
[root@ceph142 ~]#
[root@ceph142 ~]# rbd showmapped
id pool namespace image snap device
0 yinzhengjie-rbd wp02 - /dev/rbd0
[root@ceph142 ~]#
[root@ceph142 ~]# mount /dev/rbd0 /yinzhengjie/data/wp02/
[root@ceph142 ~]#
[root@ceph142 ~]# ll /yinzhengjie/data/wp02/
total 8
drwxr-xr-x 2 root root 19 Aug 28 21:48 ./
drwxr-xr-x 3 root root 4096 Aug 28 21:48 ../
-rw-r--r-- 1 root root 283 Aug 28 20:31 hosts
[root@ceph142 ~]#
[root@ceph142 ~]# cp /etc/hostname /yinzhengjie/data/wp02/
[root@ceph142 ~]#
[root@ceph142 ~]# ll /yinzhengjie/data/wp02/
total 12
drwxr-xr-x 2 root root 35 Aug 28 22:24 ./
drwxr-xr-x 3 root root 4096 Aug 28 21:48 ../
-rw-r--r-- 1 root root 8 Aug 28 22:24 hostname
-rw-r--r-- 1 root root 283 Aug 28 20:31 hosts
[root@ceph142 ~]#
[root@ceph142 ~]# umount /yinzhengjie/data/wp02
[root@ceph142 ~]#
[root@ceph142 ~]# rbd unmap /dev/rbd0
[root@ceph142 ~]#
[root@ceph142 ~]# rbd showmapped
[root@ceph142 ~]#
3.ceph143节点
[root@ceph143 ~]# rbd showmapped
[root@ceph143 ~]#
[root@ceph143 ~]# rbd map yinzhengjie/wp03
/dev/rbd0
[root@ceph143 ~]#
[root@ceph143 ~]# rbd showmapped
id pool namespace image snap device
0 yinzhengjie wp03 - /dev/rbd0
[root@ceph143 ~]#
[root@ceph143 ~]# mount /dev/rbd0 /yinzhengjie/data/wp03/
[root@ceph143 ~]#
[root@ceph143 ~]# ll /yinzhengjie/data/wp03/
total 16
drwxr-xr-x 2 root root 69 Aug 28 21:51 ./
drwxr-xr-x 6 root root 4096 Aug 28 21:50 ../
-rw------- 1 root root 367 Aug 28 21:51 00-installer-config.yaml
-rw-r--r-- 1 root root 283 Aug 28 20:31 hosts
-rw-r--r-- 1 root root 386 Aug 28 20:31 os-release
[root@ceph143 ~]#
[root@ceph143 ~]# rm -f /yinzhengjie/data/wp03/*os*
[root@ceph143 ~]#
[root@ceph143 ~]# ll /yinzhengjie/data/wp03/
total 8
drwxr-xr-x 2 root root 38 Aug 28 22:26 ./
drwxr-xr-x 6 root root 4096 Aug 28 21:50 ../
-rw------- 1 root root 367 Aug 28 21:51 00-installer-config.yaml
[root@ceph143 ~]#
[root@ceph143 ~]# umount /yinzhengjie/data/wp03
[root@ceph143 ~]#
[root@ceph143 ~]# rbd unmap /dev/rbd0
[root@ceph143 ~]#
[root@ceph143 ~]# rbd showmapped
[root@ceph143 ~]#
