Нотатка для себе. Proxmox Ceph

Ceph destroy

#rm -rf /etc/systemd/system/ceph*
#killall -9 ceph-mon ceph-mgr ceph-mds
#rm -rf /var/lib/ceph/mon/  /var/lib/ceph/mgr/  /var/lib/ceph/mds/
#pveceph purge
#apt purge ceph-mon ceph-osd ceph-mgr ceph-mds
#rm /etc/init.d/ceph 

LVM

#pvdisplay
File descriptor 7 (pipe:[1486785]) leaked on pvdisplay
  invocation. Parent PID 382397: bash
  --- Physical volume ---
 
  PV
  Name              
  /dev/sdc
  VG
  Name              
  ceph-adc898bc-44ea-43a1-8279-964e06acc8d5
  PV
  Size              
  931.51 GiB / not usable 1.71 MiB
 
  Allocatable           yes
  (but full)
  PE
  Size              
  4.00 MiB
  Total
  PE             
  238467
  Free
  PE              
  0
  Allocated
  PE          238467
  PV
  UUID              
  3LBrIs-Eh8t-wljP-WDMv-DxiX-CKAd-vXCyHf

#vgremove ceph-adc898bc-44ea-43a1-8279-964e06acc8d5
File descriptor 7
  (pipe:[1486785]) leaked on vgremove invocation. Parent PID 382397: bash
Do
  you really want to remove volume group
  "ceph-adc898bc-44ea-43a1-8279-964e06acc8d5" containing 1 logical volumes?
  [y/n]: y
Do you really want to remove and DISCARD active logical volume
  ceph-adc898bc-44ea-43a1-8279-964e06acc8d5/osd-block-ca1b00e7-bafc-4c7a-97aa-a83bcd784173?
  [y/n]: y
  Logical volume
  "osd-block-ca1b00e7-bafc-4c7a-97aa-a83bcd784173" successfully removed
 
  Volume group "ceph-adc898bc-44ea-43a1-8279-964e06acc8d5" successfully removed

#pvremove   /dev/sdc
File descriptor 7 (pipe:[1486785]) leaked
  on pvremove invocation. Parent PID 382397: bash
  Labels on physical
  volume "/dev/sdc" successfully wiped.

OSD

#pveceph osd create /dev/sdc
unable to open file
  '/var/lib/ceph/bootstrap-osd/ceph.keyring.tmp.191200' - No such file or
  directory
#mkdir /var/lib/ceph/bootstrap-osd

#pveceph osd create /dev/sdc
create
  OSD on /dev/sdc (bluestore)
wipe disk/partition: /dev/sdc
200+0
  records in
200+0 records out
209715200 bytes (210 MB, 200 MiB)
  copied, 1.54936 s, 135 MB/s
Running command: /bin/ceph-authtool
  --gen-print-key
Running command: /bin/ceph --cluster ceph --name
  client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i -
  osd new 815084bd-ee2c-4b7e-beab-39c76ca8e95f
Running command:
  /sbin/vgcreate --force --yes ceph-4f3d41bc-22ec-42bb-95fa-ca4c9d278493
  /dev/sdc
 stdout: Physical volume "/dev/sdc" successfully
  created.
 stdout: Volume group
  "ceph-4f3d41bc-22ec-42bb-95fa-ca4c9d278493" successfully created
Running
  command: /sbin/lvcreate --yes -l 238467 -n
  osd-block-815084bd-ee2c-4b7e-beab-39c76ca8e95f
  ceph-4f3d41bc-22ec-42bb-95fa-ca4c9d278493
 stdout: Logical volume
  "osd-block-815084bd-ee2c-4b7e-beab-39c76ca8e95f" created.
Running
  command: /bin/ceph-authtool --gen-print-key
Running command: /bin/mount
  -t tmpfs tmpfs /var/lib/ceph/osd/ceph-1
--> Executable selinuxenabled
  not in PATH: /sbin:/bin:/usr/sbin:/usr/bin
Running command: /bin/chown -h
  ceph:ceph
  /dev/ceph-4f3d41bc-22ec-42bb-95fa-ca4c9d278493/osd-block-815084bd-ee2c-4b7e-beab-39c76ca8e95f
Running
  command: /bin/chown -R ceph:ceph /dev/dm-5
Running command: /bin/ln -s
  /dev/ceph-4f3d41bc-22ec-42bb-95fa-ca4c9d278493/osd-block-815084bd-ee2c-4b7e-beab-39c76ca8e95f
  /var/lib/ceph/osd/ceph-1/block
Running command: /bin/ceph --cluster ceph
  --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring
  mon getmap -o /var/lib/ceph/osd/ceph-1/activate.monmap
 stderr:
  2021-05-28T14:36:51.481-0700 7f3cc29d7700 -1 auth: unable to find a keyring on
  /etc/pve/priv/ceph.client.bootstrap-osd.keyring: (2) No such file or
  directory
2021-05-28T14:36:51.481-0700 7f3cc29d7700 -1
  AuthRegistry(0x7f3cbc0596e0) no keyring found at
  /etc/pve/priv/ceph.client.bootstrap-osd.keyring, disabling cephx
 stderr:
  got monmap epoch 2
Running command: /bin/ceph-authtool
  /var/lib/ceph/osd/ceph-1/keyring --create-keyring --name osd.1 --add-key
  AQByYrFgYXVgEhAAPGxq/77NPiOynKFYsVdtqQ==
 stdout: creating
  /var/lib/ceph/osd/ceph-1/keyring
added entity osd.1
  auth(key=AQByYrFgYXVgEhAAPGxq/77NPiOynKFYsVdtqQ==)
Running command:
  /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1/keyring
Running command:
  /bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1/
Running command:
  /bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 1 --monmap
  /var/lib/ceph/osd/ceph-1/activate.monmap --keyfile - --osd-data
  /var/lib/ceph/osd/ceph-1/ --osd-uuid 815084bd-ee2c-4b7e-beab-39c76ca8e95f
  --setuser ceph --setgroup ceph
 stderr: 2021-05-28T14:36:51.874-0700
  7fdeded2de00 -1 bluestore(/var/lib/ceph/osd/ceph-1/) _read_fsid unparsable
  uuid
 stderr: 2021-05-28T14:36:52.158-0700 7fdeded2de00 -1 freelist
  read_size_meta_from_db missing size meta in DB
--> ceph-volume lvm
  prepare successful for: /dev/sdc
Running command: /bin/chown -R ceph:ceph
  /var/lib/ceph/osd/ceph-1
Running command: /bin/ceph-bluestore-tool
  --cluster=ceph prime-osd-dir --dev
  /dev/ceph-4f3d41bc-22ec-42bb-95fa-ca4c9d278493/osd-block-815084bd-ee2c-4b7e-beab-39c76ca8e95f
  --path /var/lib/ceph/osd/ceph-1 --no-mon-config
Running command: /bin/ln
  -snf
  /dev/ceph-4f3d41bc-22ec-42bb-95fa-ca4c9d278493/osd-block-815084bd-ee2c-4b7e-beab-39c76ca8e95f
  /var/lib/ceph/osd/ceph-1/block
Running command: /bin/chown -h ceph:ceph
  /var/lib/ceph/osd/ceph-1/block
Running command: /bin/chown -R ceph:ceph
  /dev/dm-5
Running command: /bin/chown -R ceph:ceph
  /var/lib/ceph/osd/ceph-1
Running command: /bin/systemctl enable
  ceph-volume@lvm-1-815084bd-ee2c-4b7e-beab-39c76ca8e95f
 stderr:
  Created symlink
  /etc/systemd/system/multi-user.target.wants/ceph-volume@lvm-1-815084bd-ee2c-4b7e-beab-39c76ca8e95f.service
  -> /lib/systemd/system/ceph-volume@.service.
Running command:
  /bin/systemctl enable --runtime ceph-osd@1
 stderr: Created symlink
  /run/systemd/system/ceph-osd.target.wants/ceph-osd@1.service ->
  /lib/systemd/system/ceph-osd@.service.
Running command: /bin/systemctl
  start ceph-osd@1
--> ceph-volume lvm activate successful for osd ID:
  1
--> ceph-volume lvm create successful for: /dev/sdc

MDS

Create MDS

#pveceph mds create
creating MDS directory
  '/var/lib/ceph/mds/ceph-ns231'
creating keys for 'mds.ns231'
setting
  ceph as owner for service directory
enabling service
  'ceph-mds@ns231.service'
Created symlink
  /etc/systemd/system/ceph-mds.target.wants/ceph-mds@ns231.service ->
  /lib/systemd/system/ceph-mds@.service.
starting service
  'ceph-mds@ns231.service'

Destroy MDS

#pveceph mds destroy ns231
disabling service
  'ceph-mds@ns231.service'
Removed
  /etc/systemd/system/ceph-mds.target.wants/ceph-mds@ns231.service.
stopping
  service 'ceph-mds@ns231.service'
removing ceph-mds directory
  '/var/lib/ceph/mds/ceph-ns231'
removing ceph auth for 'mds.ns231'

ceph-volume

ceph-volume lvm activate --all

Додавти попердні OSD до монітору.

#ceph auth add osd.0 osd 'allow *' mon 'allow rwx' -i /var/lib/ceph/osd/ceph-0/keyring
added key for osd.1
#ceph-volume lvm activate --all

Export keyring for osd on cluster member. If error is on create new OSD: stderr: [errno 13] RADOS permission denied (error connecting to the cluster) --> RuntimeError: Unable to create a new OSD id

#ceph auth get client.bootstrap-osd > /var/lib/ceph/bootstrap-osd/ceph.keyring
exported keyring for client.bootstrap-osd

....