ceph 13.2.6 部署

[TOC]

架构

ceph版本: mimic 13.2.6

#monitor
168.1.1 ceph-monitor-1  ceph-deploy monitor
168.1.2 ceph-monitor-2  monitor
168.1.3 ceph-monitor-3  monitor mgr

#osd 每台两块盘
168.1.4 ceph-osd-1 osd
168.1.5 ceph-osd-2 osd
168.1.6 ceph-osd-3 osd
168.1.7 ceph-osd-4 osd

前提

ntp服务配置好

创建 Ceph 部署用户

ceph-deploy 工具必须以普通用户登录 Ceph 节点，且此用户拥有无密码使用 sudo 的权限，因为它需要在安装软件及配置文件的过程中，不必输入密码。官方建议所有 Ceph 节点上给 ceph-deploy 创建一个特定的用户，而且不要使用 ceph 这个名字。这里为了方便，我们使用 cephd 这个账户作为特定的用户，而且每个节点上都需要创建该账户，并且拥有 sudo 权限。

# 在 Ceph 集群各节点进行如下操作
# 创建 ceph 特定用户
useradd -d /home/cephd -m cephd
# 添加 sudo 权限
echo "cephd ALL = (root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/cephd
chmod 0440 /etc/sudoers.d/cephd

生成并推送密钥

在ceph-deply admin-node 节点生成密钥，并推送到其他节点

admin节点

sudo su - cephd
ssh-keygen

推送密钥

# 在其他节点上
mkdir -p /home/cephd/.ssh
echo "公钥内容" > /home/cephd/.ssh/authorized_keys
chmod 600 /home/cephd/.ssh/authorized_keys
chmod 700 /home/cephd/.ssh

配置config

增加config (600)

为了方便ssh,在admin节点增加config

Host 192.168.1.1
  Hostname ceph-monitor-1
  User cephd
Host 192.168.1.2
  Hostname ceph-monitor-2
  User cephd
Host 192.168.1.3
  Hostname ceph-monitor-3
  User cephd
Host 192.168.1.4
  Hostname ceph-osd-1
  User cephd
Host 192.168.1.5
  Hostname ceph-osd-2
  User cephd
Host 192.168.1.6
  Hostname ceph-osd-3
  User cephd
Host 192.168.1.7
  Hostname ceph-osd-4
  User cephd

确保sshd_config,开启以下两项

# cat /etc/ssh/sshd_config  以下两项开启
RSAAuthentication yes
PubkeyAuthentication yes

ceph源配置

#epel
yum -y install epel-release      

# ceph 国内源
网易镜像源 http://mirrors.163.com/ceph
阿里镜像源 http://mirrors.aliyun.com/ceph
中科大镜像源 http://mirrors.ustc.edu.cn/ceph
宝德镜像源 http://mirrors.plcloud.com/ceph

# ceph.repo 需要时可以用 
cat > /etc/yum.repos.d/ceph.repo << EOF
[Ceph]
name=Ceph packages for $basearch
baseurl=http://mirrors.163.com/ceph/rpm-mimic/el7/x86_64
enabled=1
gpgcheck=0
type=rpm-md
gpgkey=https://mirrors.163.com/ceph/keys/release.asc
priority=1
[Ceph-noarch]
name=Ceph noarch packages
baseurl=http://mirrors.163.com/ceph/rpm-mimic/el7/noarch
enabled=1
gpgcheck=0
type=rpm-md
gpgkey=https://mirrors.163.com/ceph/keys/release.asc
priority=1
[ceph-source]
name=Ceph source packages
baseurl=http://mirrors.163.com/ceph/rpm-mimic/el7/SRPMS
enabled=1
gpgcheck=0
type=rpm-md
gpgkey=https://mirrors.163.com/ceph/keys/release.asc
priority=1
EOF

#ceph-deploy install 时，export环境变量
CentOS:
export CEPH_DEPLOY_REPO_URL=http://mirrors.ustc.edu.cn/ceph/rpm-mimic/el7
export CEPH_DEPLOY_GPG_URL=http://mirrors.ustc.edu.cn/ceph/keys/release.asc
#或者 
ceph install node-hostname --repo-url http://mirros.aliyun.com/ceph/rpm-mimic/el7/

# 如果遇见超时问题，可以在node节点上
sudo echo "minrate=1" >> /etc/yum.conf
sudo echo "timeout=30000" >> /etc/yum.conf
#ceph-deploy(admin) 节点安装
[cephd@test-ceph101v234_zq ~]$ sudo yum -y install ceph ceph-radosgw ceph-deploy
[cephd@test-ceph101v234_zq ~]$ sudo yum -y install htop sysstat iotop iftop ntp ntpdate

创建集群

#切到cephd用户
sudo su - cephd

# 创建执行目录
$ mkdir ~/my-cluster && cd ~/my-cluster

#初始化monitor
ceph-deploy new ceph-monitor-1 ceph-monitor-2 ceph-monitor-3

#Install Ceph packages,这不会把yum换替换为官方的,ps -ef看到只有yum -y install ceph ceph-radosgw
ceph-deploy install ceph-monitor-1 ceph-monitor-2 ceph-monitor-3

# Deploy the initial monitor(s) and gather the keys
# mon create-initial会根据ceph.conf进行创建mon，判断monitor都创建成功后，会进行keyring的收集，这些keyring在后续创建其他成员的时候要用到
ceph-deploy mon create-initial

#将集群的admin.keyring分发给指定的节点
ceph-deploy admin ceph-monitor-1 ceph-monitor-2 ceph-monitor-3

增加osd节点

osd节点上

# 创建 ceph 特定用户
useradd -d /home/cephd -m cephd
# 添加 sudo 权限
echo "cephd ALL = (root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/cephd
chmod 0440 /etc/sudoers.d/cephd

# 推送admin的密钥，见上面步骤

admin节点

# 创建osd
ceph-deploy osd create --data /dev/sdb ceph-osd-1
ceph-deploy osd create --data /dev/sdc ceph-osd-1

ceph-deploy osd create --data /dev/sdb ceph-osd-2
ceph-deploy osd create --data /dev/sdc ceph-osd-2

ceph-deploy osd create --data /dev/sdb ceph-osd-3
ceph-deploy osd create --data /dev/sdc ceph-osd-3

ceph-deploy osd create --data /dev/sdb ceph-osd-4
ceph-deploy osd create --data /dev/sdc ceph-osd-4

mgr

创建mgr

ceph-deploy mgr create k8s-master-11v28-syq

dashborad

sudo ceph dashboard create-self-signed-cert

mkdir dashboard-ssl
cd dashboard-ssl/

sudo openssl req -new -nodes -x509 \
-subj "/O=IT/CN=ceph-mgr-dashboard" -days 3650 \
-keyout dashboard.key -out dashboard.crt -extensions v3_ca

sudo ceph config-key set mgr mgr/dashboard/crt -i dashboard.crt
sudo ceph config-key set mgr mgr/dashboard/key -i dashboard.key

sudo ceph config set mgr mgr/dashboard/server_addr 192.168.1.2
sudo ceph config set mgr mgr/dashboard/server_port 8080

sudo ceph mgr module disable dashboard
sudo ceph mgr module enable dashboard 
sudo ceph dashboard set-login-credentials admin admin
sudo ceph mgr services  

prometheus

sudo ceph mgr module enable prometheus
sudo ceph config set mgr mgr/prometheus/server_addr 192.168.1.2

遇到的问题

error: GPT headers found

[k8s-ceph-22v28-syq][WARNIN] ceph-volume lvm create: error: GPT headers found, they must be removed on: /dev/sdb
fdisk -l 会看到：
WARNING: fdisk GPT support is currently new, and therefore in an experimental phase. Use at your own discretion.


# 需要手动清理掉硬盘的分区表
dd if=/dev/zero of=/dev/sdx bs=512K count=1

HEALTH_WARN application not enabled on 1 pool(s)

# ceph health detail
HEALTH_WARN application not enabled on 1 pool(s)
POOL_APP_NOT_ENABLED application not enabled on 1 pool(s)
    application not enabled on pool 'kube'
    use 'ceph osd pool application enable <pool-name> <app-name>', where <app-name> is 'cephfs', 'rbd', 'rgw', or freeform for custom applications.
# ceph osd pool application enable kube rbd
enabled application 'rbd' on pool 'kube'
# ceph health
HEALTH_OK

AttributeError: _init_cffi_1_0_external_module

启动dashboard有报错 
It should be python-cffi >= 1.4.

架构

前提