案例说明：
通过iscsi共享存储作为数据库存储文件系统，构建KingbaseES RAC架构。
适用版本：
KingbaseES V008R006C008M030B0010
操作系统版本：
[root@node201 KingbaseHA]# cat /etc/centos-release
CentOS Linux release 7.9.2009 (Core)
集群架构：
如下所示，node1和node2为集群节点：
节点信息：
[root@node201 KingbaseHA]# vi /etc/hosts
192.168.1.201 node201
192.168.1.202 node202
192.168.1.203 node203 iscsi_Srv
一、创建集群部署目录/opt/KingbaseHA
1、编辑脚本（已经安装KingbaseRAC软件）
[root@node201 KingbaseHA]# cd /opt/Kingbase/ES/V8/install/script/
[root@node202 script]# cat rootDeployClusterware.sh
#!/bin/sh# copy KingbaseHA to /opt/KingbaseHA
ROOT_UID=0
#INSTALLDIR=@@INSTALL_DIR@@ #脚本变量修改，路径以绝对路径标识
INSTALLDIR=/opt/Kingbase/ES/V8/KESRealPro/V008R006C008M030B0010
........
2、执行脚本
创建/opt/KingbaseHA目录，并copy文件到此目录：
[root@node202 script]# sh -x rootDeployClusterware.sh
+ ROOT_UID=0
+ INSTALLDIR=/opt/Kingbase/ES/V8/KESRealPro/V008R006C008M030B0010
+ '[' x0 '!=' x0 ']'
+ '[' -d /opt/KingbaseHA ']'
+ mkdir -m 775 /opt/KingbaseHA
+ cp -rf /opt/Kingbase/ES/V8/KESRealPro/V008R006C008M030B0010/KingbaseHA/cluster_manager.conf /opt/Kingbase/ES/V8/KESRealPro/V008R006C008M030B0010/KingbaseHA/cluster_manager.sh /opt/Kingbase/ES/V8/KESRealPro/V008R006C008M030B0010/KingbaseHA/corosync /opt/Kingbase/ES/V8/KESRealPro/V008R006C008M030B0010/KingbaseHA/corosync-qdevice /opt/Kingbase/ES/V8/KESRealPro/V008R006C008M030B0010/KingbaseHA/crmsh /opt/Kingbase/ES/V8/KESRealPro/V008R006C008M030B0010/KingbaseHA/dlm-dlm /opt/Kingbase/ES/V8/KESRealPro/V008R006C008M030B0010/KingbaseHA/fence_agents /opt/Kingbase/ES/V8/KESRealPro/V008R006C008M030B0010/KingbaseHA/gfs2 /opt/Kingbase/ES/V8/KESRealPro/V008R006C008M030B0010/KingbaseHA/gfs2-utils /opt/Kingbase/ES/V8/KESRealPro/V008R006C008M030B0010/KingbaseHA/ipmi_tool /opt/Kingbase/ES/V8/KESRealPro/V008R006C008M030B0010/KingbaseHA/kingbasefs /opt/Kingbase/ES/V8/KESRealPro/V008R006C008M030B0010/KingbaseHA/kronosnet /opt/Kingbase/ES/V8/KESRealPro/V008R006C008M030B0010/KingbaseHA/lib /opt/Kingbase/ES/V8/KESRealPro/V008R006C008M030B0010/KingbaseHA/lib64 /opt/Kingbase/ES/V8/KESRealPro/V008R006C008M030B0010/KingbaseHA/libqb /opt/Kingbase/ES/V8/KESRealPro/V008R006C008M030B0010/KingbaseHA/pacemaker /opt/Kingbase/ES/V8/KESRealPro/V008R006C008M030B0010/KingbaseHA/python2.7 /opt/KingbaseHA/+ exit 0# 查看目录信息
[root@node201 script]# ls -lhd /opt/KingbaseHA/
drwxrwxr-x 17 root root 4.0K Aug 13 14:32 /opt/KingbaseHA/
二、初始化部署环境
1、集群共享存储（通过iscsi共享）
# 数据库共享存储
[root@node201 script]# fdisk -l /dev/sdb
Disk /dev/sdb: 11.5 GB, 11499421696 bytes, 22459808 sectors# qdisk存储
[root@node201 script]# fdisk -l /dev/sdc
Disk /dev/sdc: 536 MB, 536870912 bytes, 1048576 sectors
......
2、编辑集群配置文件
[root@node201 KingbaseHA]# cat cluster_manager.conf |grep -v ^#|grep -v ^$
cluster_name=kcluster
node_name=(node201 node202)
node_ip=(192.168.1.201 192.168.2.202)
enable_qdisk=1
votingdisk=/dev/sdc
sharedata_dir=/sharedata/data_gfs2
sharedata_disk=/dev/sdb
install_dir=/opt/KingbaseHA
env_bash_file=/root/.bashrc
pacemaker_daemon_group=haclient
pacemaker_daemon_user=hacluster
kingbaseowner=kingbase
kingbasegroup=kingbase
kingbase_install_dir=/opt/Kingbase/ES/V8/Server
database="test"
username="system"
password="123456"
initdb_options="-A trust -U $username"
enable_fence=1
enable_qdisk_fence=1
.......
install_rac=1
rac_port=55321
rac_lms_port=53444
rac_lms_count=7
.......
3、仲裁盘qdisk初始化（任意节点）
[root@node201 KingbaseHA]# ./cluster_manager.sh --qdisk_init
qdisk init start
Writing new quorum disk label 'kcluster' to /dev/sdc.
WARNING: About to destroy all data on /dev/sdc; proceed? (Y/N):
y
/dev/block/8:32:
/dev/sdc:Magic: eb7a62c2Label: kclusterCreated: Tue Aug 13 14:41:49 2024Host: node201Kernel Sector Size: 512Recorded Sector Size: 512/dev/block/8:64:
/dev/disk/by-id/scsi-360014050da191d8d53b4d04a277aa8f5:
/dev/disk/by-id/wwn-0x60014050da191d8d53b4d04a277aa8f5:
/dev/disk/by-path/ip-192.168.1.203:3260-iscsi-iqn.2024-08.pip.cc:server-lun-1:
/dev/sde:Magic: eb7a62c2Label: kclusterCreated: Tue Aug 13 14:41:49 2024Host: node201Kernel Sector Size: 512Recorded Sector Size: 512
qdisk init success
4、数据共享存储初始化（任意节点）
[root@node201 KingbaseHA]# ./cluster_manager.sh --cluster_disk_init
rac disk init start
It appears to contain an existing filesystem (gfs2)
This will destroy any data on /dev/sdb
Are you sure you want to proceed? (Y/N): y
Adding journals: Done
Building resource groups: Done
Creating quota file: Done
Writing superblock and syncing: Done
Device: /dev/sdb
Block size: 4096
Device size: 10.71 GB (2807476 blocks)
Filesystem size: 10.71 GB (2807475 blocks)
Journals: 3
Journal size: 32MB
Resource groups: 46
Locking protocol: "lock_dlm"
Lock table: "kcluster:gfs2"
UUID: f8f04582-d09c-4a09-ac11-b35d59d6726b
rac disk init success
5、基础组件初始化（all nodes）
初始化所有集群基础组件，如corosync，pacemaker，corosync-qdevice：
[root@node201 KingbaseHA]# ./cluster_manager.sh --base_configure_init
init kernel soft watchdog start
init kernel soft watchdog success
config host start
config host success
add env varaible in /root/.bashrc
add env variable success
config corosync.conf start
config corosync.conf success
Starting Corosync Cluster Engine (corosync): [WARNING]
add pacemaker daemon user start
useradd: warning: the home directory already exists.
Not copying any file from skel directory into it.
Creating mailbox file: File exists
add pacemaker daemon user success
config pacemaker success
Starting Pacemaker Cluster Manager[ OK ]
config qdevice start
config qdevice success
Starting Qdisk Fenced daemon (qdisk-fenced): [ OK ]
Starting Corosync Qdevice daemon (corosync-qdevice): [ OK ]
Please note the configuration: superuser(system) and port(36321) for database(test) of resource(DB0)
Please note the configuration: superuser(system) and port(36321) for database(test) of resource(DB1)
config kingbase rac start
config kingbase rac success
add_udev_rule start
add_udev_rule success
insmod dlm.ko success
check and mknod for dlm start
check and mknod for dlm success
6、查看集群资源状态
[root@node202 KingbaseHA]# crm status
Cluster Summary:* Stack: corosync* Current DC: node201 (version 2.0.3-4b1f869f0f) - partition with quorum* Last updated: Tue Aug 13 14:44:47 2024* Last change: Tue Aug 13 14:44:39 2024 by hacluster via crmd on node201* 2 nodes configured* 0 resource instances configuredNode List:* Online: [ node201 node202 ]Full List of Resources:* No resources# 资源状态实时监控
[root@node202 KingbaseHA]# crm_mon -1
Cluster Summary:* Stack: corosync* Current DC: node201 (version 2.0.3-4b1f869f0f) - partition with quorum* Last updated: Tue Aug 13 14:45:42 2024* Last change: Tue Aug 13 14:44:39 2024 by hacluster via crmd on node201* 2 nodes configured* 0 resource instances configuredNode List:* Online: [ node201 node202 ]Active Resources:* No active resources
7、初始化共享存储文件系统（gfs2）（all nodes）
[root@node201 KingbaseHA]# ./cluster_manager.sh --init_gfs2
init gfs2 start
current OS kernel version does not support updating gfs2, please confirm whether to continue? (Y/N):
y
init the OS native gfs2 success
8、配置gfs2等资源（任意节点）
配置fence、dlm和gfs2资源：
[root@node201 KingbaseHA]# ./cluster_manager.sh --config_gfs2_resource
config dlm and gfs2 resource start
f8f04582-d09c-4a09-ac11-b35d59d6726bconfig dlm and gfs2 resource success
9、查看集群资源状态
如下所示，集群资源dlm、gfs2、fence_qdisk等资源已被加载：
[root@node201 KingbaseHA]# crm statusCluster Summary:* Stack: corosync* Current DC: node201 (version 2.0.3-4b1f869f0f) - partition with quorum* Last updated: Tue Aug 13 14:48:16 2024* Last change: Tue Aug 13 14:48:13 2024 by root via cibadmin on node201* 2 nodes configured* 6 resource instances configuredNode List:* Online: [ node201 node202 ]Full List of Resources:* fence_qdisk_0 (stonith:fence_qdisk): Started node202* fence_qdisk_1 (stonith:fence_qdisk): Started node201* Clone Set: clone-dlm [dlm]:* Started: [ node201 node202 ]* Clone Set: clone-gfs2 [gfs2]:* gfs2 (ocf::heartbeat:Filesystem): Starting node201* gfs2 (ocf::heartbeat:Filesystem): Starting node20210、查看集群cib配置
[root@node201 KingbaseHA]# crm config show
node 1: node201
node 2: node202
primitive dlm ocf:pacemaker:controld \params daemon="/opt/KingbaseHA/dlm-dlm/sbin/dlm_controld" dlm_tool="/opt/KingbaseHA/dlm-dlm/sbin/dlm_tool" args="-s 0 -f 0" \op start interval=0 \op stop interval=0 \op monitor interval=60 timeout=60
primitive fence_qdisk_0 stonith:fence_qdisk \params qdisk_path="/dev/sdc" qdisk_fence_tool="/opt/KingbaseHA/corosync-qdevice/sbin/qdisk-fence-tool" pcmk_host_list=node201 \op monitor interval=60s \meta failure-timeout=5min
primitive fence_qdisk_1 stonith:fence_qdisk \params qdisk_path="/dev/sdc" qdisk_fence_tool="/opt/KingbaseHA/corosync-qdevice/sbin/qdisk-fence-tool" pcmk_host_list=node202 \op monitor interval=60s \meta failure-timeout=5min
primitive gfs2 Filesystem \params device="-U f8f04582-d09c-4a09-ac11-b35d59d6726b" directory="/sharedata/data_gfs2" fstype=gfs2 \op start interval=0 timeout=60 \op stop interval=0 timeout=60 \op monitor interval=30s timeout=60 OCF_CHECK_LEVEL=20 \meta failure-timeout=5min
clone clone-dlm dlm \meta interleave=true target-role=Started
clone clone-gfs2 gfs2 \meta interleave=true target-role=Started
colocation cluster-colo1 inf: clone-gfs2 clone-dlm
order cluster-order1 clone-dlm clone-gfs2
location fence_qdisk_0-on-node202 fence_qdisk_0 1800: node202
location fence_qdisk_1-on-node201 fence_qdisk_1 1800: node201
property cib-bootstrap-options: \have-watchdog=false \dc-version=2.0.3-4b1f869f0f \cluster-infrastructure=corosync \cluster-name=kcluster
三、创建数据库及实例
1、初始化数据库实例（任意节点）
[root@node201 KingbaseHA]# ./cluster_manager.sh --init_rac
init KingbaseES RAC start
create_rac_share_dir start
create_rac_share_dir success
......
成功。您现在可以用下面的命令开启数据库服务器：./sys_ctl -D /sharedata/data_gfs2/kingbase/data -l 日志文件 start
init KingbaseES RAC success
2、配置数据库资源（任意节点）
如下所示，数据库资源DB已被集群加载：
[root@node201 KingbaseHA]# ./cluster_manager.sh --config_rac_resource
crm configure DB resource start
crm configure DB resource end[root@node201 KingbaseHA]# crm config show
node 1: node201
node 2: node202
primitive DB ocf:kingbase:kingbase \params sys_ctl="/opt/Kingbase/ES/V8/Server/bin/sys_ctl" ksql="/opt/Kingbase/ES/V8/Server/bin/ksql" sys_isready="/opt/Kingbase/ES/V8/Server/bin/sys_isready" kb_data="/sharedata/data_gfs2/kingbase/data" kb_dba=kingbase kb_host=0.0.0.0 kb_user=system kb_port=55321 kb_db=template1 logfile="/opt/Kingbase/ES/V8/Server/log/kingbase1.log" \op start interval=0 timeout=120 \op stop interval=0 timeout=120 \op monitor interval=9s timeout=30 on-fail=stop \meta failure-timeout=5min
primitive dlm ocf:pacemaker:controld \params daemon="/opt/KingbaseHA/dlm-dlm/sbin/dlm_controld" dlm_tool="/opt/KingbaseHA/dlm-dlm/sbin/dlm_tool" args="-s 0 -f 0" \op start interval=0 \op stop interval=0 \op monitor interval=60 timeout=60
primitive fence_qdisk_0 stonith:fence_qdisk \params qdisk_path="/dev/sdc" qdisk_fence_tool="/opt/KingbaseHA/corosync-qdevice/sbin/qdisk-fence-tool" pcmk_host_list=node201 \op monitor interval=60s \meta failure-timeout=5min
primitive fence_qdisk_1 stonith:fence_qdisk \params qdisk_path="/dev/sdc" qdisk_fence_tool="/opt/KingbaseHA/corosync-qdevice/sbin/qdisk-fence-tool" pcmk_host_list=node202 \op monitor interval=60s \meta failure-timeout=5min
primitive gfs2 Filesystem \params device="-U f8f04582-d09c-4a09-ac11-b35d59d6726b" directory="/sharedata/data_gfs2" fstype=gfs2 \op start interval=0 timeout=60 \op stop interval=0 timeout=60 \op monitor interval=30s timeout=60 OCF_CHECK_LEVEL=20 \meta failure-timeout=5min
clone clone-DB DB \meta interleave=true target-role=Started
clone clone-dlm dlm \meta interleave=true target-role=Started
clone clone-gfs2 gfs2 \meta interleave=true target-role=Started
colocation cluster-colo1 inf: clone-gfs2 clone-dlm
order cluster-order1 clone-dlm clone-gfs2
order cluster-order2 clone-dlm clone-gfs2 clone-DB
location fence_qdisk_0-on-node202 fence_qdisk_0 1800: node202
location fence_qdisk_1-on-node201 fence_qdisk_1 1800: node201
property cib-bootstrap-options: \have-watchdog=false \dc-version=2.0.3-4b1f869f0f \cluster-infrastructure=corosync \cluster-name=kcluster \load-threshold="0%"
3、查看资源状态
[root@node201 KingbaseHA]# ./cluster_manager.sh status
corosync (pid 20963) is running...
pacemakerd (pid 21004) is running...
corosync-qdevice (pid 21103) is running...
qdisk-fenced (pid 21077) is running...[root@node201 KingbaseHA]# crm status
Cluster Summary:* Stack: corosync* Current DC: node201 (version 2.0.3-4b1f869f0f) - partition with quorum* Last updated: Tue Aug 13 14:58:01 2024* Last change: Tue Aug 13 14:53:19 2024 by root via cibadmin on node201* 2 nodes configured* 8 resource instances configuredNode List:* Online: [ node201 node202 ]Full List of Resources:* fence_qdisk_0 (stonith:fence_qdisk): Started node202* fence_qdisk_1 (stonith:fence_qdisk): Started node201* Clone Set: clone-dlm [dlm]:* Started: [ node201 node202 ]* Clone Set: clone-gfs2 [gfs2]:* Started: [ node201 node202 ]* Clone Set: clone-DB [DB]: # 数据库资源DB被加载* Started: [ node201 node202 ]
4、启动和停止数据库资源DB
# 数据库资源DB状态
[root@node201 KingbaseHA]# crm resource status clone-DB
resource clone-DB is running on: node201
resource clone-DB is running on: node202# 数据库服务状态
[root@node201 KingbaseHA]# netstat -antlp |grep 55321
tcp 0 0 0.0.0.0:55321 0.0.0.0:* LISTEN 26623/kingbase[root@node201 KingbaseHA]# ps -ef |grep kingbase
......
kingbase 26623 1 0 14:53 ? 00:00:00 /opt/Kingbase/ES/V8/KESRealPro/V008R006C008M030B0010/Server/bin/kingbase -D /sharedata/data_gfs2/kingbase/data -c config_file=/sharedata/data_gfs2/kingbase/data/kingbase.conf -c log_directory=sys_log -h 0.0.0.0
kingbase 26708 26623 0 14:53 ? 00:00:00 kingbase: logger
kingbase 26721 26623 0 14:53 ? 00:00:00 kingbase: lmon
kingbase 26725 26623 0 14:53 ? 00:00:00 kingbase: lms 1
kingbase 26726 26623 0 14:53 ? 00:00:00 kingbase: lms 2
kingbase 26727 26623 0 14:53 ? 00:00:00 kingbase: lms 3
kingbase 26728 26623 0 14:53 ? 00:00:00 kingbase: lms 4
kingbase 26729 26623 0 14:53 ? 00:00:00 kingbase: lms 5
kingbase 26730 26623 0 14:53 ? 00:00:00 kingbase: lms 6
kingbase 26731 26623 0 14:53 ? 00:00:00 kingbase: lms 7
kingbase 26959 26623 0 14:53 ? 00:00:00 kingbase: checkpointer performing checkpoint
kingbase 26960 26623 0 14:53 ? 00:00:00 kingbase: background writer
kingbase 26961 26623 0 14:53 ? 00:00:00 kingbase: global deadlock checker
kingbase 26962 26623 0 14:53 ? 00:00:00 kingbase: transaction syncer
kingbase 26963 26623 0 14:53 ? 00:00:00 kingbase: walwriter
kingbase 26964 26623 0 14:53 ? 00:00:00 kingbase: autovacuum launcher
kingbase 26965 26623 0 14:53 ? 00:00:00 kingbase: archiver last was 000000010000000000000005
kingbase 26968 26623 0 14:53 ? 00:00:00 kingbase: stats collector
kingbase 26969 26623 0 14:53 ? 00:00:00 kingbase: kwr collector
kingbase 26970 26623 0 14:53 ? 00:00:00 kingbase: ksh writer
kingbase 26971 26623 0 14:53 ? 00:00:00 kingbase: ksh collector
kingbase 26972 26623 0 14:53 ? 00:00:00 kingbase: logical replication launcher# 停止数据库资源DB
[root@node202 KingbaseHA]# crm resource stop clone-DB
[root@node202 KingbaseHA]# crm resource status clone-DB
resource clone-DB is NOT running
resource clone-DB is NOT running# 查看集群资源状态
[root@node201 KingbaseHA]# crm status
Cluster Summary:* Stack: corosync* Current DC: node201 (version 2.0.3-4b1f869f0f) - partition with quorum* Last updated: Tue Aug 13 15:06:59 2024* Last change: Tue Aug 13 15:05:58 2024 by root via cibadmin on node201* 2 nodes configured* 8 resource instances configured (2 DISABLED)Node List:* Online: [ node201 node202 ]Full List of Resources:* fence_qdisk_0 (stonith:fence_qdisk): Started node202* fence_qdisk_1 (stonith:fence_qdisk): Started node201* Clone Set: clone-dlm [dlm]:* Started: [ node201 node202 ]* Clone Set: clone-gfs2 [gfs2]:* Started: [ node201 node202 ]* Clone Set: clone-DB [DB]: # 数据库资源被停止* Stopped (disabled): [ node201 node202 ]# 启动数据库资源DB
[root@node201 KingbaseHA]# crm resource start clone-DB
[root@node201 KingbaseHA]# crm resource status clone-DB
resource clone-DB is running on: node201
resource clone-DB is running on: node202
[root@node201 KingbaseHA]# netstat -antlp |grep 55321
tcp 0 0 0.0.0.0:55321 0.0.0.0:* LISTEN 2611/kingbase
四、访问数据库
test=# create database prod;
CREATE DATABASE
test=# \c prod
You are now connected to database "prod" as userName "system".
prod=# create table t1 (id int ,name varchar(10));
CREATE TABLE
prod=# insert into t1 values (generate_series(1,10000),'usr'||generate_series(1,10000));
INSERT 0 10000
prod=# select count(*) from t1;count
-------10000
(1 row)另一节点：
prod=# select count(*) from t1;count
-------10000
(1 row)
五、启停集群
1、停止集群服务
[root@node201 KingbaseHA]# ./cluster_manager.sh stop
Signaling Pacemaker Cluster Manager to terminate[ OK ]
Waiting for cluster services to unload...[ OK ]
Signaling Qdisk Fenced daemon (qdisk-fenced) to terminate: [ OK ]
Waiting for qdisk-fenced services to unload:..[ OK ]
Signaling Corosync Qdevice daemon (corosync-qdevice) to terminate: [ OK ]
Waiting for corosync-qdevice services to unload:.[ OK ]
Signaling Corosync Cluster Engine (corosync) to terminate: [ OK ]
Waiting for corosync services to unload:..[ OK ]
2、启动集群服务
[root@node201 KingbaseHA]# ./cluster_manager.sh start
Waiting for node failover handling:[ OK ]
Starting Corosync Cluster Engine (corosync): [WARNING]
clean qdisk fence flag start
clean qdisk fence flag success
Starting Qdisk Fenced daemon (qdisk-fenced): [ OK ]
Starting Corosync Qdevice daemon (corosync-qdevice): [ OK ]
Waiting for quorate:.............[ OK ]
Starting Pacemaker Cluster Manager[ OK ]
3、查看集群资源状态
如下所示，启动集群后，其资源并未加载：
[root@node201 KingbaseHA]# crm status
Cluster Summary:* Stack: corosync* Current DC: node202 (version 2.0.3-4b1f869f0f) - partition with quorum* Last updated: Tue Aug 13 15:08:52 2024* Last change: Tue Aug 13 15:08:50 2024 by hacluster via crmd on node202* 2 nodes configured* 0 resource instances configuredNode List:* Online: [ node201 node202 ]Full List of Resources:* No resources
4、加载和启动gfs2（dlm）资源
[root@node201 KingbaseHA]# ./cluster_manager.sh --config_gfs2_resource
config dlm and gfs2 resource start
f8f04582-d09c-4a09-ac11-b35d59d6726bconfig dlm and gfs2 resource success
[root@node201 KingbaseHA]# crm status
Cluster Summary:* Stack: corosync* Current DC: node202 (version 2.0.3-4b1f869f0f) - partition with quorum* Last updated: Tue Aug 13 15:16:54 2024* Last change: Tue Aug 13 15:16:51 2024 by root via cibadmin on node201* 2 nodes configured* 6 resource instances configuredNode List:* Online: [ node201 node202 ]Full List of Resources:* fence_qdisk_0 (stonith:fence_qdisk): Started node202* fence_qdisk_1 (stonith:fence_qdisk): Started node201* Clone Set: clone-dlm [dlm]:* Started: [ node201 node202 ]* Clone Set: clone-gfs2 [gfs2]:* gfs2 (ocf::heartbeat:Filesystem): Starting node201* gfs2 (ocf::heartbeat:Filesystem): Starting node202
5、加载和启动数据库资源DB
[root@node201 KingbaseHA]# ./cluster_manager.sh --config_rac_resource
crm configure DB resource start
crm configure DB resource end# 如下所示，数据库资源DB被加载到集群：
[root@node201 KingbaseHA]# crm config show
node 1: node201
node 2: node202
primitive DB ocf:kingbase:kingbase \params sys_ctl="/opt/Kingbase/ES/V8/Server/bin/sys_ctl" ksql="/opt/Kingbase/ES/V8/Server/bin/ksql" sys_isready="/opt/Kingbase/ES/V8/Server/bin/sys_isready" kb_data="/sharedata/data_gfs2/kingbase/data" kb_dba=kingbase kb_host=0.0.0.0 kb_user=system kb_port=55321 kb_db=template1 logfile="/opt/Kingbase/ES/V8/Server/log/kingbase1.log" \op start interval=0 timeout=120 \op stop interval=0 timeout=120 \op monitor interval=9s timeout=30 on-fail=stop \meta failure-timeout=5min
primitive dlm ocf:pacemaker:controld \params daemon="/opt/KingbaseHA/dlm-dlm/sbin/dlm_controld" dlm_tool="/opt/KingbaseHA/dlm-dlm/sbin/dlm_tool" args="-s 0 -f 0" \op start interval=0 \op stop interval=0 \op monitor interval=60 timeout=60
primitive fence_qdisk_0 stonith:fence_qdisk \params qdisk_path="/dev/sdc" qdisk_fence_tool="/opt/KingbaseHA/corosync-qdevice/sbin/qdisk-fence-tool" pcmk_host_list=node201 \op monitor interval=60s \meta failure-timeout=5min
primitive fence_qdisk_1 stonith:fence_qdisk \params qdisk_path="/dev/sdc" qdisk_fence_tool="/opt/KingbaseHA/corosync-qdevice/sbin/qdisk-fence-tool" pcmk_host_list=node202 \op monitor interval=60s \meta failure-timeout=5min
primitive gfs2 Filesystem \params device="-U f8f04582-d09c-4a09-ac11-b35d59d6726b" directory="/sharedata/data_gfs2" fstype=gfs2 \op start interval=0 timeout=60 \op stop interval=0 timeout=60 \op monitor interval=30s timeout=60 OCF_CHECK_LEVEL=20 \meta failure-timeout=5min
clone clone-DB DB \meta interleave=true target-role=Started
clone clone-dlm dlm \meta interleave=true target-role=Started
clone clone-gfs2 gfs2 \meta interleave=true target-role=Started
colocation cluster-colo1 inf: clone-gfs2 clone-dlm
order cluster-order1 clone-dlm clone-gfs2
order cluster-order2 clone-dlm clone-gfs2 clone-DB
location fence_qdisk_0-on-node202 fence_qdisk_0 1800: node202
location fence_qdisk_1-on-node201 fence_qdisk_1 1800: node201
property cib-bootstrap-options: \have-watchdog=false \dc-version=2.0.3-4b1f869f0f \cluster-infrastructure=corosync \cluster-name=kcluster \load-threshold="0%"
6、数据库资源状态
[root@node201 KingbaseHA]# crm resurce status clone-DB
resource clone-DB is running on: node201
resource clone-DB is running on: node202
[root@node201 KingbaseHA]# !net
netstat -antlp |grep 55321
tcp 0 0 0.0.0.0:55321 0.0.0.0:* LISTEN 16835/kingbase
参考文档：
环境准备和具体部署参考：https://help.kingbase.com.cn/v9/highly/RAC/rac/rac-3.html#id17
