Highly available Block Storage API

Waiting For Test ,,,

Config Network

  1. [root@localhost ~]# cat /etc/sysconfig/network-scripts/ifcfg-ens33 
  2. TYPE=Ethernet
  3. PROXY_METHOD=none
  4. BROWSER_ONLY=no
  5. BOOTPROTO=static
  6. DEFROUTE=yes
  7. IPV4_FAILURE_FATAL=no
  8. IPV6INIT=yes
  9. IPV6_AUTOCONF=yes
  10. IPV6_DEFROUTE=yes
  11. IPV6_FAILURE_FATAL=no
  12. IPV6_ADDR_GEN_MODE=stable-privacy
  13. NAME=ens33
  14. UUID=4e333024-b8c8-45e4-baee-e46ece81432c
  15. DEVICE=ens33
  16. ONBOOT=yes
  17. IPADDR=192.168.220.101
  18. NETMASK=255.255.255.0
  19. GATEWAY=192.168.220.2
  20. DNS1=114.114.114.114

Fdisk the 2nd Disk

  1. [root@localhost ~]# ls -al /dev/sd*
  2. brw-rw----. 1 root disk 8,  0 Dec 26  2017 /dev/sda
  3. brw-rw----. 1 root disk 8,  1 Dec 26  2017 /dev/sda1
  4. brw-rw----. 1 root disk 8,  2 Dec 26  2017 /dev/sda2
  5. brw-rw----. 1 root disk 8, 16 Dec 26 01:11 /dev/sdb
  6.  
  7. [root@localhost ~]# fdisk /dev/sdb 
  8. Welcome to fdisk (util-linux 2.23.2).
  9.  
  10. Changes will remain in memory only, until you decide to write them.
  11. Be careful before using the write command.
  12.  
  13.  
  14. Command (m for help): n
  15. Partition type:
  16.    p   primary (0 primary, 0 extended, 4 free)
  17.    e   extended
  18. Select (default p): 
  19. Using default response p
  20. Partition number (1-4, default 1): 
  21. First sector (2048-125829119, default 2048): 
  22. Using default value 2048
  23. Last sector, +sectors or +size{K,M,G} (2048-125829119, default 125829119): +30G
  24. Partition 1 of type Linux and of size 30 GiB is set
  25.  
  26. Command (m for help): n
  27. Partition type:
  28.    p   primary (1 primary, 0 extended, 3 free)
  29.    e   extended
  30. Select (default p): 
  31. Using default response p
  32. Partition number (2-4, default 2): 
  33. First sector (62916608-125829119, default 62916608): 
  34. Using default value 62916608
  35. Last sector, +sectors or +size{K,M,G} (62916608-125829119, default 125829119): 
  36. Using default value 125829119
  37. Partition 2 of type Linux and of size 30 GiB is set
  38.  
  39. Command (m for help): p
  40.  
  41. Disk /dev/sdb: 64.4 GB, 64424509440 bytes, 125829120 sectors
  42. Units = sectors of 1 * 512 = 512 bytes
  43. Sector size (logical/physical): 512 bytes / 512 bytes
  44. I/O size (minimum/optimal): 512 bytes / 512 bytes
  45. Disk label type: dos
  46. Disk identifier: 0x285890cb
  47.  
  48.    Device Boot      Start         End      Blocks   Id  System
  49. /dev/sdb1            2048    62916607    31457280   83  Linux
  50. /dev/sdb2        62916608   125829119    31456256   83  Linux
  51.  
  52. Command (m for help): w
  53. The partition table has been altered!
  54.  
  55. Calling ioctl() to re-read partition table.
  56. Syncing disks.
  57.  
  58. [root@localhost ~]# ls -al /dev/sd*
  59. brw-rw----. 1 root disk 8,  0 Dec 26  2017 /dev/sda
  60. brw-rw----. 1 root disk 8,  1 Dec 26  2017 /dev/sda1
  61. brw-rw----. 1 root disk 8,  2 Dec 26  2017 /dev/sda2
  62. brw-rw----. 1 root disk 8, 16 Dec 26 01:11 /dev/sdb
  63. brw-rw----. 1 root disk 8, 17 Dec 26 01:11 /dev/sdb1
  64. brw-rw----. 1 root disk 8, 18 Dec 26 01:11 /dev/sdb2

Format 2nd Disk

  1. [root@localhost ~]# mkfs.ext4 /dev/sdb1
  2. mke2fs 1.42.9 (28-Dec-2013)
  3. Filesystem label=
  4. OS type: Linux
  5. Block size=4096 (log=2)
  6. Fragment size=4096 (log=2)
  7. Stride=0 blocks, Stripe width=0 blocks
  8. 1966080 inodes, 7864320 blocks
  9. 393216 blocks (5.00%) reserved for the super user
  10. First data block=0
  11. Maximum filesystem blocks=2155872256
  12. 240 block groups
  13. 32768 blocks per group, 32768 fragments per group
  14. 8192 inodes per group
  15. Superblock backups stored on blocks: 
  16.         32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632, 2654208, 
  17.         4096000
  18.  
  19. Allocating group tables: done                            
  20. Writing inode tables: done                            
  21. Creating journal (32768 blocks): done
  22. Writing superblocks and filesystem accounting information: done   
  23.  
  24. [root@localhost ~]# mkfs.ext4 /dev/sdb2
  25. mke2fs 1.42.9 (28-Dec-2013)
  26. Filesystem label=
  27. OS type: Linux
  28. Block size=4096 (log=2)
  29. Fragment size=4096 (log=2)
  30. Stride=0 blocks, Stripe width=0 blocks
  31. 1966080 inodes, 7864064 blocks
  32. 393203 blocks (5.00%) reserved for the super user
  33. First data block=0
  34. Maximum filesystem blocks=2155872256
  35. 240 block groups
  36. 32768 blocks per group, 32768 fragments per group
  37. 8192 inodes per group
  38. Superblock backups stored on blocks: 
  39.         32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632, 2654208, 
  40.         4096000
  41.  
  42. Allocating group tables: done                            
  43. Writing inode tables: done                            
  44. Creating journal (32768 blocks): done
  45. Writing superblocks and filesystem accounting information: done

Mount /dev/sdb1

  1. [root@localhost ~]# mkdir -p /date
  2. [root@localhost ~]# mount -t ext4 /dev/sdb1 /date
  3. [root@localhost ~]# df -h|grep /dev/sdb1
  4. /dev/sdb1                 30G   45M   28G   1% /date
  5. [root@localhost ~]# echo "mount -t ext4 /dev/sdb1 /date" >>/etc/rc.d/rc.local
  6. [root@localhost ~]# tail -1 /etc/rc.d/rc.local
  7. mount -t ext4 /dev/sdb1 /date
  8. [root@localhost ~]# chmod +x /etc/rc.d/rc.local

Create Volume on /dev/sdb2

  1. [root@localhost ~]# yum install lvm2 -y
  2. [root@localhost ~]# systemctl enable lvm2-lvmetad.service
  3. Created symlink from /etc/systemd/system/sysinit.target.wants/lvm2-lvmetad.service to /usr/lib/systemd/system/lvm2-lvmetad.service.
  4. [root@localhost ~]# systemctl start lvm2-lvmetad.service
  5.  
  6. [root@localhost ~]# pvcreate /dev/sdb2
  7. WARNING: ext4 signature detected on /dev/sdb2 at offset 1080. Wipe it? [y/n]: y
  8.   Wiping ext4 signature on /dev/sdb2.
  9.   Physical volume "/dev/sdb2" successfully created.
  10. [root@localhost ~]# vgcreate cinder_lvm01 /dev/sdb2
  11.   Volume group "cinder_lvm01" successfully created
  12. [root@localhost ~]# vgdisplay 
  13.   --- Volume group ---
  14.   VG Name               cinder_lvm01
  15.   System ID             
  16.   Format                lvm2
  17.   Metadata Areas        1
  18.   Metadata Sequence No  1
  19.   VG Access             read/write
  20.   VG Status             resizable
  21.   MAX LV                0
  22.   Cur LV                0
  23.   Open LV               0
  24.   Max PV                0
  25.   Cur PV                1
  26.   Act PV                1
  27.   VG Size               <30.00 GiB
  28.   PE Size               4.00 MiB
  29.   Total PE              7679
  30.   Alloc PE / Size       0 / 0   
  31.   Free  PE / Size       7679 / <30.00 GiB
  32.   VG UUID               jyb299-bo5k-E6Z3-Frho-e6Kz-d9Mu-yK0m6c
  33.  
  34.   --- Volume group ---
  35.   VG Name               centos
  36.   System ID             
  37.   Format                lvm2
  38.   Metadata Areas        1
  39.   Metadata Sequence No  4
  40.   VG Access             read/write
  41.   VG Status             resizable
  42.   MAX LV                0
  43.   Cur LV                3
  44.   Open LV               3
  45.   Max PV                0
  46.   Cur PV                1
  47.   Act PV                1
  48.   VG Size               <79.00 GiB
  49.   PE Size               4.00 MiB
  50.   Total PE              20223
  51.   Alloc PE / Size       20222 / 78.99 GiB
  52.   Free  PE / Size       1 / 4.00 MiB
  53.   VG UUID               NHtSF8-nozf-sbB4-vEBt-ogJo-WmuG-F8HYMQ

Install NFS

  1. [root@localhost ~]# yum install nfs-utils rpcbind -y
  2. [root@localhost ~]# mkdir -p /date/{cinder_nfs1,cinder_nfs2}
  3. [root@localhost ~]# chmod -R 777 /date
  4. [root@localhost ~]# echo "/date/cinder_nfs1 *(rw,root_squash,sync,anonuid=165,anongid=165)">/etc/exports
  5. [root@localhost ~]# exportfs -r
  6. [root@localhost ~]# systemctl enable rpcbind nfs-server
  7. Created symlink from /etc/systemd/system/multi-user.target.wants/nfs-server.service to /usr/lib/systemd/system/nfs-server.service.
  8. [root@localhost ~]# systemctl restart rpcbind nfs-server
  9. [root@localhost ~]# showmount -e localhost
  10. Export list for localhost:
  11. /date/cinder_nfs1 *

Install Cinder & Configure

  1. [root@localhost ~]# yum install openstack-cinder targetcli python-keystone lvm2 -y
  2. [root@localhost ~]# cp /etc/cinder/cinder.conf{,.bak}
  3. [root@localhost ~]# cp /etc/lvm/lvm.conf{,.bak
  4. [root@localhost ~]# sed -i '141a filter = [ "a/sdb2/", "r/.*/"]' /etc/lvm/lvm.conf
  5. [root@localhost ~]# echo '192.168.220.101:/date/cinder_nfs1'>/etc/cinder/nfs_shares
  6. [root@localhost ~]# chmod 640 /etc/cinder/nfs_shares
  7. [root@localhost ~]# chown root:cinder /etc/cinder/nfs_shares

Configure cinder.conf

  1. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf DEFAULT auth_strategy keystone
  2. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf DEFAULT log_dir /var/log/cinder
  3. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf DEFAULT state_path /var/lib/cinder
  4. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf DEFAULT glance_api_servers http://controller:9292
  5. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf DEFAULT transport_url rabbit://openstack:openstack@controller
  6. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf DEFAULT enabled_backends lvm,nfs
  7.  
  8. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf database connection mysql+pymysql://cinder:cinder@controller/cinder
  9.  
  10. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf keystone_authtoken auth_uri http://controller:5000
  11. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf keystone_authtoken auth_url http://controller:35357
  12. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf keystone_authtoken memcached_servers controller1:11211,controller2:11211,controller3:11211
  13. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf keystone_authtoken auth_type password
  14. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf keystone_authtoken project_domain_name default
  15. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf keystone_authtoken user_domain_name default
  16. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf keystone_authtoken project_name service
  17. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf keystone_authtoken username cinder
  18. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf keystone_authtoken password cinder
  19.  
  20. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf oslo_concurrency lock_path /var/lib/cinder/tmp
  21.  
  22. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf lvm volume_driver cinder.volume.drivers.lvm.LVMVolumeDriver
  23. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf lvm iscsi_helper lioadm
  24. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf lvm iscsi_protocol iscsi
  25. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf lvm volume_group cinder_lvm01
  26. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf lvm iscsi_ip_address 192.168.220.101
  27. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf lvm volumes_dir $state_path/volumes
  28. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf lvm volume_backend_name lvm01
  29.  
  30. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf nfs volume_driver = cinder.volume.drivers.nfs.NfsDriver
  31. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf nfs nfs_shares_config = /etc/cinder/nfs_shares
  32. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf nfs nfs_mount_point_base = $state_path/mnt
  33. [root@localhost ~]# openstack-config --set /etc/cinder/cinder.conf nfs volume_backend_name = nfs01

Start Cinder Service

  1. [root@localhost ~]# chmod 640 /etc/cinder/cinder.conf
  2. [root@localhost ~]# chgrp cinder /etc/cinder/cinder.conf
  3.  
  4. [root@localhost ~]# systemctl enable openstack-cinder-volume.service target.service
  5. Created symlink from /etc/systemd/system/multi-user.target.wants/openstack-cinder-volume.service to /usr/lib/systemd/system/openstack-cinder-volume.service.
  6. Created symlink from /etc/systemd/system/multi-user.target.wants/target.service to /usr/lib/systemd/system/target.service.
  7. [root@localhost ~]# systemctl start openstack-cinder-volume.service target.service

Add Cinder to Pacemaker

  1. # pcs resource create openstack-cinder-api systemd:openstack-cinder-api --clone interleave=true
  2. # pcs resource create openstack-cinder-scheduler systemd:openstack-cinder-scheduler --clone interleave=true
  3. # pcs resource create openstack-cinder-volume systemd:openstack-cinder-volume
  4.  
  5. # pcs constraint order start openstack-cinder-api-clone then openstack-cinder-scheduler-clone
  6. # pcs constraint colocation add openstack-cinder-scheduler-clone with openstack-cinder-api-clone
  7. # pcs constraint order start openstack-cinder-scheduler-clone then openstack-cinder-volume
  8. # pcs constraint colocation add openstack-cinder-volume with openstack-cinder-scheduler-clone
  9.  
  10. # pcs constraint order start openstack-keystone-clone then openstack-cinder-api-clone

Configure Cinder service

  1. # cp /etc/cinder/cinder.conf /etc/cinder/cinder.conf.bak
  2.  
  3. # openstack-config --set /etc/cinder/cinder.conf DEFAULT host cinder-cluster-1
  4.  
  5. # openstack-config --set /etc/cinder/cinder.conf DEFAULT osapi_volume_listen 10.0.0.11
  6.  
  7. # openstack-config --set /etc/cinder/cinder.conf DEFAULT auth_strategy keystone
  8. # openstack-config --set /etc/cinder/cinder.conf DEFAULT control_exchange cinder
  9.  
  10. # openstack-config --set /etc/cinder/cinder.conf DEFAULT volume_driver cinder.volume.drivers.nfs.NfsDriver
  11. # openstack-config --set /etc/cinder/cinder.conf DEFAULT nfs_shares_config /etc/cinder/nfs_exports
  12. # openstack-config --set /etc/cinder/cinder.conf DEFAULT nfs_sparsed_volumes true
  13. # openstack-config --set /etc/cinder/cinder.conf DEFAULT nfs_mount_options v3
  14.  
  15. # openstack-config --set /etc/cinder/cinder.conf database connection mysql://cinder:password@10.0.0.11/cinder
  16. # openstack-config --set /etc/cinder/cinder.conf database max_retries -1
  17.  
  18. # openstack-config --set /etc/cinder/cinder.conf keystone_authtoken identity_uri http://10.0.0.11:35357/
  19. # openstack-config --set /etc/cinder/cinder.conf keystone_authtoken auth_uri http://10.0.0.11:5000/
  20. # openstack-config --set /etc/cinder/cinder.conf keystone_authtoken admin_tenant_name service
  21. # openstack-config --set /etc/cinder/cinder.conf keystone_authtoken admin_user cinder
  22. # openstack-config --set /etc/cinder/cinder.conf keystone_authtoken admin_password CINDER_PASS
  23.  
  24. # openstack-config --set /etc/cinder/cinder.conf rabbit_hosts 10.0.0.12,10.0.0.13,10.0.0.14
  25. # openstack-config --set /etc/cinder/cinder.conf rabbit_ha_queues True
  26. # openstack-config --set /etc/cinder/cinder.conf heartbeat_timeout_threshold 60
  27. # openstack-config --set /etc/cinder/cinder.conf heartbeat_rate 2

Configure HA Cinder API

  1. $ openstack endpoint create volume --region $KEYSTONE_REGION \
  2.   --publicurl 'http://10.0.0.11:8776/v1/%(tenant_id)s' \
  3.   --adminurl 'http://10.0.0.11:8776/v1/%(tenant_id)s' \
  4.   --internalurl 'http://10.0.0.11:8776/v1/%(tenant_id)s'

Leave a Reply

Your email address will not be published. Required fields are marked *