Ceph China 小编 2018-06-06

本文转载自雷永鑫博客：http://jiebaby.com/2018/06/05/rgw-ha-lb/#more

0. 说明

keepalived 用于高可用，
haproxy 用于负载均衡

下列中：

192.168.0.203:7480 192.168.0.202:7480 是真实网关服务节点（RGW） 192.168.0.205 192.168.0.204 是负载均衡器（LB） 192.168.0.200 是用于负责均衡器间的高可用的虚拟ip （VIP）

1
2
3

192.168.0.203:7480    192.168.0.202:7480  是真实网关服务节点  （RGW）
         192.168.0.205         192.168.0.204  是负载均衡器     （LB）
        192.168.0.200         是用于负责均衡器间的高可用的虚拟ip   （VIP）

架构示意图如下：

网关高可用和负载均衡架构图

1. 环境检查及安装

yum install keepalived haproxy  -y

1	yum install keepalived haproxy -y

选取两个均衡节点(均衡节点不能和rgw节点重合)，下面在均衡节点（LB）执行。

开启linux ip转发功能

echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.conf sysctl -p

1
2

echo "net.ipv4.ip_forward = 1" >> /etc/sysctl.conf
sysctl -p

允许绑定到非本地ip

echo "net.ipv4.ip_nonlocal_bind = 1" >> /etc/sysctl.conf sysctl -p

1
2

echo "net.ipv4.ip_nonlocal_bind = 1" >> /etc/sysctl.conf
sysctl -p

检查：

/usr/sbin/sysctl net.ipv4.ip_nonlocal_bind /usr/sbin/sysctl net.ipv4.ip_forward cat /proc/sys/net/ipv4/ip_forward

1
2
3

/usr/sbin/sysctl net.ipv4.ip_nonlocal_bind
/usr/sbin/sysctl net.ipv4.ip_forward
cat /proc/sys/net/ipv4/ip_forward

查看是否看起了ip转发功能
如果上述文件中的值为0,说明禁止进行IP转发；如果是1,则说明IP转发功能已经打开。

2.修改keepalived + Haproxy配置文件

下面所有操作均只需在LB（负载均衡器）节点执行，
本例中即在192.168.0.204；192.168.0.205 两个节点，分别修改下面配置文件
keepalived 的配置文件
全部节点采用backup，为了减少VIP来回切换。
` shell
[root@home205 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived

vrrp_script chk_haproxy {
script “killall -0 haproxy” # 检查haproxy进程
interval 2
weight 2
}

vrrp_instance VI_1 {
state BACKPUP
interface eno16780032 # 似情况修改
virtual_router_id 1 # id号局域网唯一
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.0.200 # VIP，配好了，客户端连接该ip即可高可用
}
track_script {
chk_haproxy
}
}

**haproxy 的配置文件**

``` shell
[root@home205 ~]# cat /etc/haproxy/haproxy.cfg 
#---------------------------------------------------------------------
# Example configuration for a possible web application.  See the
# full configuration options online.
#
#   http://haproxy.1wt.eu/download/1.4/doc/configuration.txt
#
#---------------------------------------------------------------------

#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
    # to have these messages end up in /var/log/haproxy.log you will
    # need to:
    #
    # 1) configure syslog to accept network log events.  This is done
    #    by adding the '-r' option to the SYSLOGD_OPTIONS in
    #    /etc/sysconfig/syslog
    #
    # 2) configure local2 events to go to the /var/log/haproxy.log
    #   file. A line like the following can be added to
    #   /etc/sysconfig/syslog
    #
    #    local2.*                       /var/log/haproxy.log
    #
    log         127.0.0.1 local2

    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        root      # 此处要修改成对应的用户，建议用root
    group       root
    daemon

    # turn on stats unix socket
    stats socket /var/lib/haproxy/stats

#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000

#---------------------------------------------------------------------
# main frontend which proxys to the backends
#---------------------------------------------------------------------
frontend rgw *:7480     # 侦听端口
    mode http
    default_backend    rgw

#---------------------------------------------------------------------
# round robin balancing between the various backends
#---------------------------------------------------------------------
backend rgw
    mode http
    balance    roundrobin     # 均衡模式，当前为轮循
    server rgw1 192.168.0.203:7480 check   # 填写真实网关IP和端口
    server rgw2 192.168.0.202:7480 check

**haproxy 的配置文件**

``` shell

[root@home205 ~]# cat /etc/haproxy/haproxy.cfg

#---------------------------------------------------------------------

# Example configuration for a possible web application. See the

# full configuration options online.

# http://haproxy.1wt.eu/download/1.4/doc/configuration.txt

#---------------------------------------------------------------------

# Global settings

#---------------------------------------------------------------------

global

# to have these messages end up in /var/log/haproxy.log you will

# need to:

# 1) configure syslog to accept network log events. This is done

# by adding the '-r' option to the SYSLOGD_OPTIONS in

# /etc/sysconfig/syslog

# 2) configure local2 events to go to the /var/log/haproxy.log

# file. A line like the following can be added to

# /etc/sysconfig/syslog

# local2.* /var/log/haproxy.log

log 127.0.0.1 local2

chroot /var/lib/haproxy

pidfile /var/run/haproxy.pid

maxconn 4000

user root # 此处要修改成对应的用户，建议用root

group root

daemon

# turn on stats unix socket

stats socket /var/lib/haproxy/stats

#---------------------------------------------------------------------

# common defaults that all the 'listen' and 'backend' sections will

# use if not designated in their block

#---------------------------------------------------------------------

defaults

mode http

log global

option httplog

option dontlognull

option http-server-close

option forwardfor except 127.0.0.0/8

option redispatch

retries 3

timeout http-request 10s

timeout queue 1m

timeout connect 10s

timeout client 1m

timeout server 1m

timeout http-keep-alive 10s

timeout check 10s

maxconn 3000

#---------------------------------------------------------------------

# main frontend which proxys to the backends

#---------------------------------------------------------------------

frontend rgw *:7480 # 侦听端口

mode http

default_backend rgw

#---------------------------------------------------------------------

# round robin balancing between the various backends

#---------------------------------------------------------------------

backend rgw

mode http

balance roundrobin # 均衡模式，当前为轮循

server rgw1 192.168.0.203:7480 check # 填写真实网关IP和端口

server rgw2 192.168.0.202:7480 check

3. 启动服务

systemctl start keepalived
 systemctl enable keepalived
 systemctl start haproxy
 systemctl enable haproxy

systemctl start keepalived

systemctl enable keepalived

systemctl start haproxy

systemctl enable haproxy

最后记得防火墙允许相应端口，测试环境可以直接关闭防火墙，关闭selinux。

systemctl stop firewalld

1

systemctl stop firewalld

另：上面演示的是搭建高可用的时候同时搭建负载均衡，如果只搭建网关高可用，则不需要配置haproxy，同时架构也有所变化，直接在两个网关节点（202， 203）安装keepalived（不需要额外节点），如下修改keepalived配置文件就可。（步骤1也转移到网关节点上操作）

[root@home202 ~]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived

vrrp_instance VI_1 {
    state BACKPUP
    interface eno16780032    # 似情况修改
    virtual_router_id 2      # id号局域网唯一
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.0.200        # VIP，配好了，客户端连接该ip即可高可用
    }
}

[root@home202 ~]# cat /etc/keepalived/keepalived.conf

! Configuration File for keepalived

vrrp_instance VI_1 {

state BACKPUP

interface eno16780032 # 似情况修改

virtual_router_id 2 # id号局域网唯一

priority 100

advert_int 1

authentication {

auth_type PASS

auth_pass 1111

}

virtual_ipaddress {

192.168.0.200 # VIP，配好了，客户端连接该ip即可高可用

}

重启服务

systemctl start keepalived
 systemctl enable keepalived

1 2	systemctl start keepalived systemctl enable keepalived

当一个网关节点down机以后，虚拟IP会自动飘到其他网关节点。

NOTE:

相同的真实节点间可以配置多个高可用实例，设置不同的VIP，然后，不同的节点都可以分配一个MASTER，只要不是同一个实例。（只要保证一个实例只有一个MASTER就可以）

在用虚拟机测试的时候发现，切换后的虚拟IP不能跨网段访问，改用物理机后验证没有发现这个问题。

配置Ceph RGW (对象存储网关) 的高可用和负责均衡

0. 说明

1. 环境检查及安装

2.修改keepalived + Haproxy配置文件

3. 启动服务