resource mfs { meta-disk internal; device /dev/drbd1; syncer { verify-alg sha1; } on mfsmaster { disk /dev/vda1; address 172.25.254.10:7789; } on mfsbackup { disk /dev/vda1; address 172.25.254.11:7789; } }安裝mfs master
[root@mfsmaster etc]# cd /etc [root@mfsmaster etc]# mv mfsmaster.cfg.dist mfsmaster.cfg [root@mfsmaster etc]# mv mfsexports.cfg.dist mfsexports.cfg [root@mfsmaster etc]# mv mfsmetalogger.cfg.dist mfsmetalogger.cfg [root@mfsmaster etc]# mv mfstopology.cfg.dist mfstopology.cfgvim mfsmaster.cfg
WORKING_USER = nobody WORKING_GROUP = nobody SYSLOG_IDENT = mfsmaster LOCK_MEMORY = 0 NICE_LEVEL = -19 EXPORTS_FILENAME = /etc/mfsexports.cfg TOPOLOGY_FILENAME = /etc/mfstopology.cfg DATA_PATH = /data1/drbd BACK_LOGS = 50 BACK_META_KEEP_PREVIOUS = 1 REPLICATIONS_DELAY_INIT = 300 REPLICATIONS_DELAY_DISCONNECT = 3600 MATOML_LISTEN_HOST = * MATOML_LISTEN_PORT = 9419 MATOCS_LISTEN_HOST = * MATOCS_LISTEN_PORT = 9420 MATOCL_LISTEN_HOST = * MATOCL_LISTEN_PORT = 9421 CHUNKS_LOOP_CPS = 100000 CHUNKS_LOOP_TIME = 300 CHUNKS_SOFT_DEL_LIMIT = 10 CHUNKS_HARD_DEL_LIMIT = 25 CHUNKS_WRITE_REP_LIMIT = 2 CHUNKS_READ_REP_LIMIT = 10 REJECT_OLD_CLIENTS = 0vim mfsmetalogger.cfg
WORKING_USER = nobody WORKING_GROUP = nobody SYSLOG_IDENT = mfsmetalogger LOCK_MEMORY = 0 NICE_LEVEL = -19 DATA_PATH = /var/lib/mfs BACK_LOGS = 50 BACK_META_KEEP_PREVIOUS = 3 META_DOWNLOAD_FREQ = 24 MASTER_RECONNECTION_DELAY = 5 MASTER_HOST = mfsmaster MASTER_PORT = 9419 MASTER_TIMEOUT = 60ok 為了方便,直接把這幾個文件復制到mfsbackup上就可以
[root@mfsmaster etc]# scp mfs* 172.25.254.11:/etc/ [email protected]'s password: mfsexports.cfg 100% 4060 4.0KB/s 00:00 mfsmaster.cfg 100% 849 0.8KB/s 00:00 mfsmetalogger.cfg 100% 401 0.4KB/s 00:00 mfstopology.cfg 100% 1123 1.1KB/s 00:00master上執行
[root@mfsmaster etc]# drbdsetup /dev/drbd1 primary --force [root@mfsmaster etc]# mkfs.ext4 /dev/drbd1都執行
mkdir -p /data1/drbdchown -R nobody.nobody /data1/drbd/
[root@mfsmaster mfs]# cp /var/lib/mfs/metadata.mfs.empty /data1/drbd/ [root@mfsmaster drbd]# mv metadata.mfs.empty metadata.mfs [root@mfsmaster drbd]# mfsmasterok master 已經可以正常運行了
,為了保證無誤我們在backup上也測試一次。
[root@mfsmaster drbd]# mfsmaster stop [root@mfsmaster drbd]# cd[root@mfsmaster ~] # umount /dev/drbd1 [root@mfsmaster ~]# drbdadm secondary mfs
[root@mfsbackup drbd+mfs-rpm]# drbdadm primary mfs [root@mfsbackup drbd+mfs-rpm]# mount /dev/drbd1 /data1/drbd/ [root@mfsbackup drbd+mfs-rpm]# mfsmaster
這個時候我們的master端已經全線部署完成,接下來我們要實現他的高可用,這個很重要
首先說明,我使用自己的方法很簡單的實現主斷備啟,但是是一次性的,也就是說,只支持一次主的mfsmaster斷了,backup可以接替,並不能來回。需要手工操作。
腳本寫的簡單後續有時間修改,網上有很多成品的,雖然很菜還是寫自己的。。
同樣是keepalived +drbd大神們都是全自動的
我這其實是一次性的但是輕量簡單呗。
安裝keepalived 兩端執行:! Configuration File for keepalived global_defs { notification_email { [email protected] } notification_email_from zabbix@server2.com smtp_server 172.25.254.2 smtp_connect_timeout 30 router_id mfs_master } vrrp_script check_drbd { script "/etc/keepalived/check_drbd.sh" interval 15 } vrrp_instance mfs { state MASTER interface eth0 virtual_router_id 51 priority 150 advert_int 1 authentication { auth_type PASS auth_pass 1111 } virtual_ipaddress { 172.25.254.12 } track_script { check_drbd } notify_master /etc/keepalived/master.sh notify_backup /etc/keepalived/backup.sh }[root@mfsmaster keepalived]# vim check_drbd.sh
/bin/bash: Configuration: command not found # File Name: check_drbd.sh # Author: dty # mail: [email protected] # Created Time: Sun 02 Aug 2015 01:32:43 PM CST ######################################################################### #!/bin/bash A=`ps -C mfsmaster --no-header |wc -l` if [ $A -eq 0 ];then umount /data1/drbd/ drbdadm secondary mfs killall keepalived fi
mfsbackup 上 cd /etc/keepalived 寫兩個腳本 master.sh backup.sh
[root@mfsbackup keepalived]# cat backup.sh master.sh #!/bin/bash mfsmaster stop umount /dev/drbd1 drbdadm secondary mfs ######################################################################### # File Name: master.sh # Author: dty # mail: [email protected] # Created Time: Sun 02 Aug 2015 01:38:12 PM CST ######################################################################### #!/bin/bash drbdadm primary mfs mount /dev/drbd1 /data1/drbd mfsmaster startvim /etc/keepalived.conf
! Configuration File for keepalived global_defs { notification_email { [email protected] } notification_email_from [email protected] smtp_server 172.25.254.2 smtp_connect_timeout 30 router_id mfs_master } vrrp_instance mfs { state BACKUP interface eth0 virtual_router_id 51 priority 80 advert_int 1 authentication { auth_type PASS auth_pass 1111 } virtual_ipaddress { 172.25.254.12 } notify_master /etc/keepalived/master.sh notify_backup /etc/keepalived/backup.sh }
此時我們可以實驗了
開啟mfsmaster,
開啟keepalived
mfsbackup上也啟動 keepalived
[root@mfsbackup keepalived]# /etc/init.d/keepalived start
我們把主上的mfsmaster關掉
可以看到vip已經沒了,轉移到了備機上
服務已經完美轉移了 。
這個時候我們來搞定客戶端和chunkserver的安裝
yum localinstall mfs-chunkserver-1.6.26-1.x86_64.rpm -y
mv mfschunkserver.cfg.dist mfschunkserver
mkdir -p /data1/mfs //設置個目錄存放數據
chown nobody.nobody /data1/mfs
vim /etc/mfschunkserver
WORKING_USER = nobody WORKING_GROUP = nobody SYSLOG_IDENT = mfschunkserver LOCK_MEMORY = 0 NICE_LEVEL = -19 DATA_PATH = /data1/mfs MASTER_RECONNECTION_DELAY = 5 BIND_HOST = * MASTER_HOST = 172.25.254.12 MASTER_PORT = 9420 MASTER_TIMEOUT = 60 CSSERV_LISTEN_HOST = * CSSERV_LISTEN_PORT = 9422 HDD_CONF_FILENAME = /etc/mfshdd.cfg HDD_TEST_FREQ = 10設置chunk存放路徑
[root@server7 ~]# cat /etc/mfshdd.cfg # mount points of HDD drives # #/mnt/hd1 #/mnt/hd2 #etc. /data1/mfs/chunkserver已經部署完成了有多個以此類推就好了.
下來就是客戶端的安裝及使用了
# yum localinstall -y mfs-client-1.6.26-1.x86_64.rpm # cd /etc# cp mfsmount.cfg.dist mfsmount.cfg # vi mfsmount.cfg定義客戶端默認掛載mfsmaster=mfsmaster/mnt/mfs# mfsmount [root@server4 ~]# df -h Filesystem Size Used Avail Use% Mounted on/dev/mapper/VolGroup-lv_root 6.7G 978M 5.4G 16% /tmpfsLinux/1672.html' target='_blank'>inodes with permission denied: 0
[root@server4 mfs]# cp /etc/passwd leaf1 [root@server4 mfs]# cp /etc/passwd leaf2 [root@server4 mfs]# mfsfileinfo leaf1/passwd
leaf1/passwd: chunk 0: 0000000000000008_00000001 / (id:8 ver:1) copy 1: 172.25.254.7:9422 [root@server4 mfs]# mfsfileinfo leaf2/passwd leaf2/passwd: chunk 0: 0000000000000009_00000001 / (id:9 ver:1) copy 1: 172.25.254.8:9422 copy 2: 172.25.254.9:9422 ok 效果就這樣粗來了
其他的mfs的使用這裡不做贅述,主要為了驗證我們的高可用機制是否可行,就到這裡了
ps:當出現問題調轉到backup後,如果主的正常了,我們手動切換必須遵守以下步驟
停掉mfsmaster服務
mfsmaster stop
umount /dev/drbd1
drbdadm secondary mfs
主機:
1.drbdadm primary mfs
2.mount /dev/drbd1 /data1/drbd
3.mfsmaster start
4.keepalived start
還有要提到的就是可能會出現腦裂,這一般時操作不按順序,也可能有其他原因,下面是網上大神寫的,
腦裂的處理方法
drbd1主,drbd2輔
1,正常情況下狀態: [root@drbd1 ~]# cat /proc/drbd version: 8.3.8 (api:88/proto:86-94) : 299AFE04D7AFD98B3CA0AF9 0: cs:Connected ro:Primary/Secondary ds:UpToDate/UpToDate C r---- ns:2144476 nr:0 dw:36468 dr:2115769 al:14 bm:129 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0 [root@drbd2 ~]# cat /proc/drbd version: 8.3.8 (api:88/proto:86-94) srcversion: 299AFE04D7AFD98B3CA0AF9 0: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate C r---- ns:0 nr:2141684 dw:2141684 dr:0 al:0 bm:130 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0 2,drbd1故障後 drbd1狀態: [root@drbd1 ~]# cat /proc/drbd version: 8.3.8 (api:88/proto:86-94) srcversion: 299AFE04D7AFD98B3CA0AF9 0: cs:StandAlone ro:Primary/Unknown ds:UpToDate/DUnknown r---- ns:4 nr:102664 dw:102668 dr:157 al:1 bm:8 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0 drbd2的狀態: [root@drbd2 ~]# cat /proc/drbd version: 8.3.8 (api:88/proto:86-94) srcversion: 299AFE04D7AFD98B3CA0AF9 0: cs:WFConnection ro:Secondary/Unknown ds:UpToDate/DUnknown C r---- ns:0 nr:2141684 dw:2141684 dr:0 al:0 bm:130 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0 3,處理方法: a,將secondary配置成primary角色 [root@drbd2 ~]# drbdsetup /dev/drbd0 primary -o [root@drbd2 ~]# cat /proc/drbd version: 8.3.8 (api:88/proto:86-94) srcversion: 299AFE04D7AFD98B3CA0AF9 0: cs:WFConnection ro:Primary/Unknown ds:UpToDate/Outdated C r---- ns:0 nr:2141684 dw:2141684 dr:0 al:0 bm:130 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0 掛載: [root@drbd2 /]# mount /dev/drbd0 /data1 [root@drbd2 data1]# ll total 10272 -rw-r--r-- 1 root root 10485760 Feb 13 11:26 aa.img drwx------ 2 root root 16384 Feb 13 11:25 lost+found 這個時候drbd2開始提供服務,開始寫數據 drbd1主恢復正常後: [root@drbd1 ~]# cat /proc/drbd version: 8.3.8 (api:88/proto:86-94) srcversion: 299AFE04D7AFD98B3CA0AF9 0: cs:StandAlone ro:Primary/Unknown ds:UpToDate/DUnknown r---- ns:2144476 nr:0 dw:36484 dr:2115769 al:14 bm:129 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:8 drbd1狀態是:StandAlone,此時,drbd1是不會和drbd2互相聯系的 我們來查看下日志: [root@drbd1 ~]# tailf /var/log/messages Feb 13 16:14:27 drbd1 kernel: block drbd0: helper command: /sbin/drbdadm split-brain minor-0 Feb 13 16:14:27 drbd1 kernel: block drbd0: helper command: /sbin/drbdadm split-brain minor-0 exit code 0 (0x0) Feb 13 16:14:27 drbd1 kernel: block drbd0: conn( WFReportParams -> Disconnecting ) Feb 13 16:14:27 drbd1 kernel: block drbd0: error receiving ReportState, l: 4! Feb 13 16:14:27 drbd1 kernel: block drbd0: asender terminated Feb 13 16:14:27 drbd1 kernel: block drbd0: Terminating drbd0_asender Feb 13 16:14:27 drbd1 kernel: block drbd0: Connection closed Feb 13 16:14:27 drbd1 kernel: block drbd0: conn( Disconnecting -> StandAlone ) Feb 13 16:14:27 drbd1 kernel: block drbd0: receiver terminated Feb 13 16:14:27 drbd1 kernel: block drbd0: Terminating drbd0_receiver 腦裂出現! 解決方法: 1>,我們需要將現在的drbd1角色修改為secondary [root@drbd1 ~]# drbdadm secondary r0 [root@drbd1 ~]# drbdadm -- --discard-my-data connect r0 ##該命令告訴drbd,secondary上的數據不正確,以primary上的數據為准。 2>,我們還需要在drbd2上執行下面操作 [root@drbd2 /]# drbdadm connect r0 這樣drbd1就能和drbd2開始連接上了,並且保證數據不會丟失: [root@drbd1 ~]# cat /proc/drbd version: 8.3.8 (api:88/proto:86-94) srcversion: 299AFE04D7AFD98B3CA0AF9 0: cs:Connected ro:Secondary/Primary ds:UpToDate/UpToDate C r---- ns:0 nr:20592 dw:20592 dr:0 al:0 bm:4 lo:0 pe:0 ua:0 ap:0 ep:1 wo:b oos:0