多级负载均衡搭建

多级负载均衡搭建（四层+七层架构）

框架图：

实验环境
LVS-------eno16777736:10.10.10.11 eno16777736:0 :10.10.10.100
Nginx-----eno16777736:10.10.10.12 lo:0 :10.10.10.100
Nginx-----eno16777736:10.10.10.13 lo:0 :10.10.10.100
apache1-----eno16777736:10.10.10.14
apache2-----eno16777736:10.10.10.15
apache3-----eno16777736:10.10.10.16

两个网站，不同的域名，相同的IP，所以采用以上的方法；Nginx作为反向代理服务器，LVS对nginx采用DR负载均衡

六台机器先关闭防火墙，selinux，NetworkManager等服务

systemctl stop firewalld.service 
setenforce 0
systemctl stop NetworkManager.service 

[root@lvsdr network-scripts]# vim /etc/sysctl.conf
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.default.send_redirects = 0
net.ipv4.conf.eno16777736.send_redirects = 0
[root@lvsdr network-scripts]# sysctl -p
net.ipv4.conf.all.send_redirects = 0
net.ipv4.conf.default.send_redirects = 0
net.ipv4.conf.eno16777736.send_redirects = 0
[root@lvsdr sysconfig]# cd network-scripts/
[root@lvsdr network-scripts]# cp ifcfg-eno16777736 ifcfg-eno16777736:0
[root@lvsdr network-scripts]# more ifcfg-eno16777736:0
TYPE=Ethernet
BOOTPROTO=static
DEFROUTE=yes
PEERDNS=yes
PEERROUTES=yes
IPADDR=10.10.10.100
NETMASK=255.255.255.0
GATEWAY=10.10.10.1
DNS1=8.8.8.8
NAME=eno16777736:0
UUID=a2636a49-9206-46ec-bfee-37f3bf8fc153
DEVICE=eno16777736:0
onBOOT=yes
[root@lvsdr network-scripts]# ifup ifcfg-eno16777736:0
[root@lvsdr network-scripts]# ifconfig
eno16777736: flags=4163  mtu 1500
        inet 10.10.10.11  netmask 255.255.255.0  broadcast 10.10.10.255
        inet6 fe80::20c:29ff:fe53:ae58  prefixlen 64  scopeid 0x20
        ether 00:0c:29:53:ae:58  txqueuelen 1000  (Ethernet)
        RX packets 2709  bytes 252021 (246.1 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 1004  bytes 128927 (125.9 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

eno16777736:0: flags=4163  mtu 1500
        inet 10.10.10.100  netmask 255.255.255.0  broadcast 10.10.10.255
        ether 00:0c:29:53:ae:58  txqueuelen 1000  (Ethernet)

lo: flags=73  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10
        loop  txqueuelen 0  (Local Loopback)
        RX packets 53  bytes 5321 (5.1 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 53  bytes 5321 (5.1 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
 
 另一种设置外部ip和映射ip的命令：
 ifconfig eno16777736:0 10.10.10.100/24

[root@lvsdr network-scripts]# yum -y install ipvsadm
[root@lvsdr network-scripts]# ipvsadm -A -t 10.10.10.100:80 -s rr
[root@lvsdr network-scripts]# ipvsadm -a -t 10.10.10.100:80 -r 10.10.10.12:80 -g
[root@lvsdr network-scripts]# ipvsadm -a -t 10.10.10.100:80 -r 10.10.10.13:80 -g
[root@lvsdr network-scripts]# ipvsadm-save >/etc/sysconfig/ipvsadm
[root@lvsdr network-scripts]# systemctl start ipvsadm.service
[root@lvsdr network-scripts]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.10.10.100:80 rr
  -> 10.10.10.12:80               Route   1      0          0         
  -> 10.10.10.13:80               Route   1      0          0         
[root@lvsdr network-scripts]# ipvsadm -Ln --stats
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port               Conns   InPkts  OutPkts  InBytes OutBytes
  -> RemoteAddress:Port
TCP  10.10.10.100:80                    25      169        0    21636        0
  -> 10.10.10.12:80                     12       90        0    12482        0
  -> 10.10.10.13:80                     13       79        0     9154        0

[root@nginx1 ~]# vim /etc/sysctl.conf
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.all.arp_announce = 2
net.ipv4.conf.default.arp_ignore = 1
net.ipv4.conf.default.arp_announce = 2
net.ipv4.conf.lo.arp_ignore = 1
net.ipv4.conf.lo.arp_announce = 2
[root@nginx1 ~]# sysctl -p
net.ipv4.conf.all.arp_ignore = 1
net.ipv4.conf.all.arp_announce = 2
net.ipv4.conf.default.arp_ignore = 1
net.ipv4.conf.default.arp_announce = 2
net.ipv4.conf.lo.arp_ignore = 1
net.ipv4.conf.lo.arp_announce = 2
[root@nginx1 ~]# ifconfig lo:0 10.10.10.100/32 
[root@nginx1 ~]# ifconfig
eno16777736: flags=4163  mtu 1500
        inet 10.10.10.12  netmask 255.255.255.0  broadcast 10.10.10.255
        inet6 fe80::20c:29ff:fe1a:df3  prefixlen 64  scopeid 0x20
        ether 00:0c:29:1a:0d:f3  txqueuelen 1000  (Ethernet)
        RX packets 55605  bytes 77333193 (73.7 MiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 8437  bytes 865438 (845.1 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

lo: flags=73  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10
        loop  txqueuelen 0  (Local Loopback)
        RX packets 52  bytes 5366 (5.2 KiB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 52  bytes 5366 (5.2 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

lo:0: flags=73  mtu 65536
        inet 10.10.10.100  netmask 0.0.0.0
        loop  txqueuelen 0  (Local Loopback)

[root@nginx1 ~]# route add -host 10.10.10.100 dev lo:0
[root@nginx1 opt]# yum -y install gcc pcre pcre-devel zlib zlib-devel
[root@nginx1 opt]# wget http://nginx.org/download/nginx-1.2.6.tar.gz
[root@nginx1 opt]# useradd -r -s /sbin/nologin -M nginx
[root@nginx1 opt]# tar zxvf nginx-1.2.6.tar.gz 
[root@nginx1 opt]# cd nginx-1.2.6/
[root@nginx1 nginx-1.2.6]# ./configure --prefix=/usr/local/nginx --user=nginx --group=nginx && make && make install
[root@nginx1 nginx-1.2.6]# cd /usr/local/nginx/conf/
[root@nginx1 conf]# vim /usr/local/nginx/conf/nginx.conf
[root@nginx1 conf]# more /usr/local/nginx/conf/nginx.conf

#user  nobody;
worker_processes  1;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;


events {
    worker_connections  1024;
}


http {
    include       mime.types;
    default_type  application/octet-stream;

    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
    #                  '$status $body_bytes_sent "$http_referer" '
    #                  '"$http_user_agent" "$http_x_forwarded_for"';

    #access_log  logs/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    #keepalive_timeout  0;
    keepalive_timeout  65;

    #gzip  on;

upstream chao.com{
        server 10.10.10.14:80;
        server 10.10.10.15:80;
    }
    upstream chao.cn{
        server 10.10.10.16:80;
    }
    server {
        listen       80;
        server_name  www.chao.com;
        location / {
           proxy_pass http://chao.com;
            }
    }
    server {
        listen       80;
        server_name  www.chao.cn;      
        location / {
           proxy_pass http://chao.cn;
            }
    }

}
[root@nginx1 conf]# /usr/local/nginx/sbin/nginx -t       
nginx: the configuration file /usr/local/nginx/conf/nginx.conf syntax is ok
nginx: configuration file /usr/local/nginx/conf/nginx.conf test is successful
[root@nginx1 conf]# /usr/local/nginx/sbin/nginx

yum -y install httpd
echo "www.chaogao.com-apache-1">/var/www/html/index.html
systemctl start httpd

[root@apache1 ~]# more /var/www/html/index.html
www.chaogao.com-apache-1
[root@apache2 ~]# more /var/www/html/index.html
www.chaogao.com-apache-2
[root@apache3 ~]# more /var/www/html/index.html
www.chaogao.com-apache-3

C:WindowsSystem32driversetchosts
10.10.10.100 www.chao.com
10.10.10.100 www.chao.cn