http://docs.ceph.org.cn/start

192.168.86.11   dbahost1 (admin-node)

192.168.86.16   dbahost6 (mon.node1、osd.2、mds.node1、rgw) 192.168.86.17   dbahost7 (mon.node2、osd.0) 192.168.86.18   dbahost8 (mon.node3、osd.1)

#安装 CEPH 部署工具ceph-deploy

wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add - echo deb http://download.eu.ceph.com/debian-luminous/ $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list sudo apt-get update && sudo apt-get install ceph-deploy

#初始化环境 sudo useradd -d /home/cephadm -m cephadm sudo passwd cephadm ---密码cephadm

echo "cephadm ALL = (root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/cephadm

sudo chmod 0440 /etc/sudoers.d/cephadm

在192.168.86.11上，生成sshkey-gen：

ssh-copy-id cephadm@192.168.86.16  ssh-copy-id cephadm@192.168.86.17  ssh-copy-id cephadm@192.168.86.18

192.168.86.11:

vi

/etc/hosts

192.168.86.11   dbahost1

192.168.86.16   dbahost6 192.168.86.17   dbahost7 192.168.86.18   dbahost8

~/.ssh/config

Host dbahost6

   Hostname dbahost6    User cephadm Host dbahost7    Hostname dbahost7    User cephadm Host dbahost8    Hostname dbahost8    User cephadm     

ssh dbahost1(在控制节点执行ceph-deploy):

部署初始monitor节点： cd my-cluster ceph-deploy new dbahost6

开始部署两个osd数据节点：

ceph-deploy install dbahost1 dbahost6 dbahost7 dbahost8

配置初始监控：

ceph-deploy mon create-initial

配置osd数据节点：

ssh dbahost7 sudo mkdir /var/local/osd0 sudo chmod a+w /var/local/osd0 exit

ssh dbahost8

sudo mkdir /var/local/osd1 sudo chmod a+w /var/local/osd1 exit

ssh dbahost1:

sudo vim ceph.conf

osd_max_object_name_len = 256 osd_max_object_namespace_len = 64

ceph-deploy osd prepare dbahost7:/var/local/osd0 dbahost8:/var/local/osd1

ceph-deploy osd activate dbahost7:/var/local/osd0 dbahost8:/var/local/osd1

ssh dbahost1: ceph-deploy --overwrite-conf admin dbahost1 dbahost6 dbahost7 dbahost8 sudo chmod +r /etc/ceph/ceph.client.admin.keyring

开始部署扩容第3个osd数据节点：

ssh dbahost6 sudo mkdir /var/local/osd2 sudo chmod a+w /var/local/osd2

ssh dbahost1 ceph-deploy --overwrite-conf osd prepare dbahost6:/var/local/osd2 ceph-deploy osd activate dbahost6:/var/local/osd2

部署CephFS的元数据服务器: ceph-deploy mds create dbahost6

部署ceph对象网关服务器radosgw：

ceph-deploy rgw create dbahost6

添加另外2个MONITORS：

ceph-deploy mon add dbahost7 ceph-deploy mon add dbahost8

sudo chmod -R a+r /etc/ceph

ceph quorum_status --format json-pretty

ceph -s ceph -w ceph osd tree ceph osd lspools  ceph fs ls

rbd ls

rbd showmapped rbd map foo rbd unmap foo

#ceph文件系统（就是一个NAS，网络文件系统）： ceph osd pool create cephfs_data 50 ceph osd pool create cephfs_metadata 50 ceph fs new cephfs cephfs_metadata cephfs_data

在16上：

sudo mkdir -p /mnt/mycephfs1 sudo mount -t ceph 192.168.86.17:6789:/ /mnt/mycephfs1 -o name=admin,secret=AQDJGhRZAvkfMRAA2YzxketKJtF90jPh6InDDg==

在18上：

sudo mkdir -p /mnt/mycephfs2 sudo mount -t ceph 192.168.86.16:6789:/ /mnt/mycephfs2 -o name=admin,secret=AQDJGhRZAvkfMRAA2YzxketKJtF90jPh6InDDg==

结果，在16的/mnt/mycephfs1路径下的读写文件，在18上的/mnt/mycephfs2的路径下是能够看到和读写的，也就是二者共享了同一个网络卷。

#ceph块设备：

创建块设备的镜像： rbd create foo --size 4096 -m 192.168.86.16 -k /etc/ceph/ceph.client.admin.keyring rbd info foo rbd feature disable foo exclusive-lock, object-map, fast-diff, deep-flatten

将镜像映射为块设备：

cephadm@dbahost1:/etc/ceph$ sudo rbd map foo --name client.admin -m 192.168.86.16 -k /etc/ceph/ceph.client.admin.keyring /dev/rbd0 cephadm@dbahost1:/etc/ceph$ 

cephadm@dbahost1:/etc/ceph$ ls -l /dev/rbd/rbd/foo

lrwxrwxrwx 1 root root 10 May 12 10:10 /dev/rbd/rbd/foo -> ../../rbd0 cephadm@dbahost1:/etc/ceph$ ls -l /dev/rbd0 brw-rw---- 1 root disk 251, 0 May 12 10:10 /dev/rbd0 cephadm@dbahost1:/etc/ceph$ 

在块设备上创建文件系统（ext4）：

cephadm@dbahost1:/etc/ceph$ sudo mkfs.ext4 -m0 /dev/rbd/rbd/foo mke2fs 1.42.13 (17-May-2015) Discarding device blocks: done                             Creating filesystem with 1048576 4k blocks and 262144 inodes Filesystem UUID: 5f2b8485-4477-4e6e-a049-88dd5655f04c Superblock backups stored on blocks:          32768, 98304, 163840, 229376, 294912, 819200, 884736

Allocating group tables: done                            

Writing inode tables: done                             Creating journal (32768 blocks): done Writing superblocks and filesystem accounting information: done 

cephadm@dbahost1:/etc/ceph$ 

挂载此文件系统：

sudo mkdir /mnt/ceph-block-device sudo mount /dev/rbd/rbd/foo /mnt/ceph-block-device cd /mnt/ceph-block-device

若其它节点再次挂载这个设备， 那么其它节点的卷是overlay的卷，可以看到这个卷原有的文件，新生成的文件互相都看不到。

#ceph对象存储 从管理节点的工作目录，在 client-node 上安装 Ceph 对象网关软件包: ssh dbahost1 cd my-cluster ceph-deploy install --rgw dbahost1 ceph-deploy rgw create dbahost1

可以从此url访问：

http://192.168.86.11:7480

systemctl restart ceph-radosgw.service

为 S3 访问创建 RADOSGW 用户： cephadm@dbahost1:~/my-cluster$  sudo radosgw-admin user create --uid="testuser" --display-name="First User" {     "user_id": "testuser",     "display_name": "First User",     "email": "",     "suspended": 0,     "max_buckets": 1000,     "auid": 0,     "subusers": [],     "keys": [         {             "user": "testuser",             "access_key": "RBT0BVXHZ4OOFYSFKHG1",             "secret_key": "             "         }     ],     "swift_keys": [],     "caps": [],     "op_mask": "read, write, delete",     "default_placement": "",     "placement_tags": [],     "bucket_quota": {         "enabled": false,         "max_size_kb": -1,         "max_objects": -1     },     "user_quota": {         "enabled": false,         "max_size_kb": -1,         "max_objects": -1     },     "temp_url_keys": [] }

新建一个 SWIFT 子用户： cephadm@dbahost1:~/my-cluster$ sudo radosgw-admin subuser create --uid=testuser --subuser=testuser:swift --access=full {     "user_id": "testuser",     "display_name": "First User",     "email": "",     "suspended": 0,     "max_buckets": 1000,     "auid": 0,     "subusers": [         {             "id": "testuser:swift",             "permissions": "full-control"         }     ],     "keys": [         {             "user": "testuser",             "access_key": "RBT0BVXHZ4OOFYSFKHG1",             "secret_key": "XXe5Ex1AzbkErBcRWC9IeaiZXzlDuLOlvpefCNu0"         }     ],     "swift_keys": [         {             "user": "testuser:swift",             "secret_key": "dQ2mMgQkqmwAyAZjjcgquoGN9I27OXN5hkk9bHcd"         }     ],     "caps": [],     "op_mask": "read, write, delete",     "default_placement": "",     "placement_tags": [],     "bucket_quota": {         "enabled": false,         "max_size_kb": -1,         "max_objects": -1     },     "user_quota": {         "enabled": false,         "max_size_kb": -1,         "max_objects": -1     },     "temp_url_keys": [] }

cephadm@dbahost1:~/my-cluster$ 

新建 secret key:

cephadm@dbahost1:~/my-cluster$ sudo radosgw-admin key create --subuser=testuser:swift --key-type=swift --gen-secret {     "user_id": "testuser",     "display_name": "First User",     "email": "",     "suspended": 0,     "max_buckets": 1000,     "auid": 0,     "subusers": [         {             "id": "testuser:swift",             "permissions": "full-control"         }     ],     "keys": [         {             "user": "testuser",             "access_key": "RBT0BVXHZ4OOFYSFKHG1",             "secret_key": "XXe5Ex1AzbkErBcRWC9IeaiZXzlDuLOlvpefCNu0"         }     ],     "swift_keys": [         {             "user": "testuser:swift",             "secret_key": "w2TqR2vWNVCFUBapIOhAstxdoCW1lKkVt9Hyi7GE"         }     ],     "caps": [],     "op_mask": "read, write, delete",     "default_placement": "",     "placement_tags": [],     "bucket_quota": {         "enabled": false,         "max_size_kb": -1,         "max_objects": -1     },     "user_quota": {         "enabled": false,         "max_size_kb": -1,         "max_objects": -1     },     "temp_url_keys": [] }

cephadm@dbahost1:~/my-cluster$ 

#测试s3方式的访问： s3test.py ： import boto import boto.s3.connection

access_key = 'RBT0BVXHZ4OOFYSFKHG1'

secret_key = 'XXe5Ex1AzbkErBcRWC9IeaiZXzlDuLOlvpefCNu0' conn = boto.connect_s3(         aws_access_key_id = access_key,         aws_secret_access_key = secret_key,         host = '192.168.86.11', port = 7480 ,         is_secure=False, calling_format = boto.s3.connection.OrdinaryCallingFormat(),         )

bucket = conn.create_bucket('my-new-bucket')

    for bucket in conn.get_all_buckets():             print "{name}".format(                     name = bucket.name,                     created = bucket.creation_date,  )

@测试 SWIFT 访问

swift -A http://192.168.86.11:7480/auth/1.0 -U testuser:swift -K 'w2TqR2vWNVCFUBapIOhAstxdoCW1lKkVt9Hyi7GE' list

cephadm@dbahost1:~$ swift -A http://192.168.86.11:7480/auth/1.0 -U testuser:swift -K 'w2TqR2vWNVCFUBapIOhAstxdoCW1lKkVt9Hyi7GE' list my-new-bucket cephadm@dbahost1:~$ 

#结合docker的测试：

1、如果使用nfs即CephFS的文件系统，那么在不同host的不同容器内mount的卷， 其内容都是共享的，可以并发读写的。

192.168.86.18:6789:/  86302720  48271360  38031360  56% /mnt/mycephfs11

root@dbahost1:/mnt/mycephfs11# docker run -it --rm -v /mnt/mycephfs11:/mnt/mycephfs11 ubuntu /bin/bash 

root@c0c8aa7425a2:/mnt/mycephfs11# touch container_11

root@c0c8aa7425a2:/mnt/mycephfs11# ls -l  total 1 -rw-r--r-- 1 root root  0 May 12 07:17 container_11 -rw-r--r-- 1 root root  0 May 12 07:16 dbahost1 -rw-r--r-- 1 root root 15 May 11 10:33 dbahost6 -rw-r--r-- 1 root root  0 May 12 07:17 dbahost7 root@c0c8aa7425a2:/mnt/mycephfs11# root@c0c8aa7425a2:/mnt/mycephfs11# ls -l  total 1 -rw-r--r-- 1 root root  0 May 12 07:17 container_11 -rw-r--r-- 1 root root  0 May 12 07:18 container_17 -rw-r--r-- 1 root root  0 May 12 07:16 dbahost1 -rw-r--r-- 1 root root 15 May 11 10:33 dbahost6 -rw-r--r-- 1 root root  0 May 12 07:17 dbahost7 root@c0c8aa7425a2:/mnt/mycephfs11# 

192.168.86.18:6789:/  86302720 48275456  38027264  56% /mnt/mycephfs3

root@dbahost7:/mnt/mycephfs3# docker run -it --rm -v /mnt/mycephfs3:/mnt/mycephfs17 ubuntu /bin/bash   

root@9d60c1af183c:/# df -k 

Filesystem           1K-blocks     Used Available Use% Mounted on none                  28768380 14895136  12388856  55% / tmpfs                   499152        0    499152   0% /dev tmpfs                   499152        0    499152   0% /sys/fs/cgroup 192.168.86.18:6789:/  86302720 48275456  38027264  56% /mnt/mycephfs17 /dev/sda1             28768380 14895136  12388856  55% /etc/hosts shm                      65536        0     65536   0% /dev/shm tmpfs                   499152        0    499152   0% /sys/firmware root@9d60c1af183c:/# cd  /mnt/mycephfs17  root@9d60c1af183c:/mnt/mycephfs17# ls container_11  dbahost1  dbahost6  dbahost7 root@9d60c1af183c:/mnt/mycephfs17# touch container_17 root@9d60c1af183c:/mnt/mycephfs17# ls -l  total 1 -rw-r--r-- 1 root root  0 May 12 07:17 container_11 -rw-r--r-- 1 root root  0 May 12 07:18 container_17 -rw-r--r-- 1 root root  0 May 12 07:16 dbahost1 -rw-r--r-- 1 root root 15 May 11 10:33 dbahost6 -rw-r--r-- 1 root root  0 May 12 07:17 dbahost7 root@9d60c1af183c:/mnt/mycephfs17# 

2、如果使用ceph的块文件，那么在不同host的不同容器内mount的卷， 其内容不是共享的，是overlay的。

但是也可以用在容器共享上面， 因为后面一个容器的mount的卷里面的内容会包含该rbd设备的最新的内容。

http://ceph-users.ceph.narkive.com/LNCcfd0m/ceph-same-rbd-on-multiple-client

Example

create rbd image named foo map foo to /dev/rbd0 on server A, mount /dev/rbd0 to /mnt map foo to /dev/rbd0 on server B, mount /dev/rbd0 to /mnt

root@dbahost7:~# ls -l /dev/rbd/rbd/foo lrwxrwxrwx 1 root root 10 May 12 10:20 /dev/rbd/rbd/foo -> ../../rbd0 root@dbahost7:~#  root@dbahost7:~#  root@dbahost7:~# mount /dev/rbd/rbd/foo /mnt/ceph-block-device-17 root@dbahost7:~#  root@dbahost7:~# docker run -it --rm -v /mnt/ceph-block-device-17:/mnt/block-device17 ubuntu /bin/bash root@e8c518538ff5:/#  root@e8c518538ff5:/# df -k  Filesystem     1K-blocks     Used Available Use% Mounted on none            28768380 14896260  12387732  55% / tmpfs             499152        0    499152   0% /dev tmpfs             499152        0    499152   0% /sys/fs/cgroup /dev/rbd0        3997376     8188   3972804   1% /mnt/block-device17 /dev/sda1       28768380 14896260  12387732  55% /etc/hosts shm                65536        0     65536   0% /dev/shm tmpfs             499152        0    499152   0% /sys/firmware root@e8c518538ff5:/# cd /mnt/block-device17  root@e8c518538ff5:/mnt/block-device17# echo "this is 17" > file17 root@e8c518538ff5:/mnt/block-device17# more file17 this is 17 root@e8c518538ff5:/mnt/block-device17# 

root@dbahost1:/mnt# ls -l /dev/rbd/rbd/foo ls: cannot access '/dev/rbd/rbd/foo': No such file or directory root@dbahost1:/mnt#  root@dbahost1:/mnt# su - cephadm cephadm@dbahost1:~$ sudo rbd map foo --name client.admin -m 192.168.86.16 -k /etc/ceph/ceph.client.admin.keyring /dev/rbd0 cephadm@dbahost1:~$ ls -l /dev/rbd/rbd/foo  lrwxrwxrwx 1 root root 10 May 12 15:58 /dev/rbd/rbd/foo -> ../../rbd0 cephadm@dbahost1:~$ sudo mount /dev/rbd/rbd/foo /mnt/ceph-block-device-11 cephadm@dbahost1:~$ df -k |grep device /dev/rbd0              3997376      8188   3972804   1% /mnt/ceph-block-device-11 cephadm@dbahost1:~$ exit logout root@dbahost1:/mnt# docker run -it --rm -v /mnt/ceph-block-device-11:/mnt/block-device11 ubuntu /bin/bash root@7f1d457f5ebd:/# df -k  Filesystem     1K-blocks     Used Available Use% Mounted on none            28768380 15316948  11967044  57% / tmpfs            1015248        0   1015248   0% /dev tmpfs            1015248        0   1015248   0% /sys/fs/cgroup /dev/rbd0        3997376     8188   3972804   1% /mnt/block-device11 /dev/sda1       28768380 15316948  11967044  57% /etc/hosts shm                65536        0     65536   0% /dev/shm tmpfs            1015248        0   1015248   0% /sys/firmware root@7f1d457f5ebd:/# cd /mnt/block-device11/ root@7f1d457f5ebd:/mnt/block-device11# ls file17  lost+found root@7f1d457f5ebd:/mnt/block-device11# more file17  this is 17 root@7f1d457f5ebd:/mnt/block-device11# echo "this is 11" > file11 root@7f1d457f5ebd:/mnt/block-device11# ls -l total 24 -rw-r--r-- 1 root root    11 May 12 07:59 file11 -rw-r--r-- 1 root root    11 May 12 07:57 file17 drwx------ 2 root root 16384 May 12 02:12 lost+found root@7f1d457f5ebd:/mnt/block-device11# more file11 this is 11 root@7f1d457f5ebd:/mnt/block-device11# 

root@e8c518538ff5:/mnt/block-device17# ls -tlr  total 20 drwx------ 2 root root 16384 May 12 02:12 lost+found -rw-r--r-- 1 root root    11 May 12 07:57 file17 root@e8c518538ff5:/mnt/block-device17#