系统centos7 所有节点是yum 安装的kubernetes

admin

#系统centos7 所有节点是yum 安装的kubernetes 版本#前提是集群已经正确运行没有任何问题#原理参照 kubadm 和kargo 三个master做高可用所有node上安装haproxy 负载均衡反代三台kube-apiserver的8080端口 api-server为无状态服务#注意，之前用过nginx配置反代三台apiserver出现创建pod 容器非常慢，3-5分钟，应该有BUG ,建议用haproxy反代,非常顺畅.#controller-manager 和scheduler 为有状态服务，同一时间只有一台当选，会在三台master机之间进行选举，由其中一台担任leader的角色#节点构造如下
cat /etc/hosts#master
5 U, M( S5 ^$ D% |# d# Z192.168.1.61 master1.txg.com  #512M
192.168.1.62 master2.txg.com  #512M
192.168.1.63 master2.txg.com  #512M#master软件包
# rpm -qa|grep kube
kubernetes-client-1.5.2-0.2.gitc55cf2b.el7.x86_64
kubernetes-master-1.5.2-0.2.gitc55cf2b.el7.x86_64
flannel-0.7.0-1.el7.x86_64#etcd-server
6 s* y2 {2 i/ i; W$ ~192.168.1.65 etcd1.txg.com #512M
! F( t- B9 _0 p* @6 c8 q192.168.1.66 etcd2.txg.com #512M
: y9 c4 d4 s# B! y192.168.1.67 etcd3.txg.com #512M#node节点
* k4 T& [- [: v5 G% e5 h192.168.1.68 node1.txg.com #4G
0 o/ ?- p" N) p  ^3 z192.168.1.69 node2.txg.com #4G
192.168.2.68 node3.txg.com #4G
192.168.2.69 node4.txg.com #4G#node节点软件包
[root@node4 ~]# rpm -qa|egrep 'kube|docker'
kubernetes-client-1.5.2-0.5.gita552679.el7.x86_64
docker-common-1.12.6-11.el7.centos.x86_64
docker-1.12.6-11.el7.centos.x86_64
. C; J% P8 L6 C2 [kubernetes-node-1.5.2-0.5.gita552679.el7.x86_64
$ {  A  X+ G* T, v4 `9 @0 C/ Fdocker-client-1.12.6-11.el7.centos.x86_64
2 \) ]) }& I' @0 Sflannel-0.7.0-1.el7.x86_64[root@node4 ~]# uname -a
- O  H) {- s2 e! A# }4 rLinux node4.txg.com 3.10.0-514.6.2.el7.x86_64 #1 SMP Thu Feb 23 03:04:39 UTC 2017 x86_64 x86_64 x86_64 GNU/Linux修改master server上的配置文件，我的配置文件在/etc/kubernetes/下面
0 t3 x  t+ b) Z[root@master1 kubernetes]# pwd
/etc/kubernetes
( I8 A7 Q- n8 b7 S- M[root@master1 kubernetes]# ls
apiserver config controller-manager scheduler ssl sslbk1.修改controller-manager和scheduler配置文件在KUBE_CONTROLLER_MANAGER_ARGS=" " 中间加入 --address=127.0.0.1 --leader-elect=true
KUBE_CONTROLLER_MANAGER_ARGS=" --address=127.0.0.1 --leader-elect=true --service-account-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem --cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem --cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem --root-ca-file=/etc/kubernetes/ssl/ca.pem"
修改scheduler 为 KUBE_SCHEDULER_ARGS=" --address=127.0.0.1 --leader-elect=true"
: ?8 ~1 l/ c/ r8 J9 f让节点有选举master leader 功能，ok master配置完成同步master1上的配置文件到master2 3 节点2.所有node节点安装 haproxy , yum install haproxy#配置haproxy.cfg文件 监听5002端口，所向代理kube-apiserver:8080[root@node4 ~]# cat /etc/haproxy/haproxy.cfg
9 N* ?7 S3 x; }$ Z" d! M( r; d6 B#---------------------------------------------------------------------Example configuration for a possible web application. See thefull configuration options online.http://haproxy.1wt.eu/download/1 ... -------------Global settings#---------------------------------------------------------------------
global
6 }: M" u7 ]: M# to have these messages end up in /var/log/haproxy.log you will
5 l+ g  q) w$ D$ V' W. r2 i- q# need to:
#
! L5 J. I  {6 `' W+ ^" @# 1) configure syslog to accept network log events. This is done
# by adding the '-r' option to the SYSLOGD_OPTIONS in
8 \- j6 M( O3 A. N0 v5 Y" m) h* P# /etc/sysconfig/syslog
) R9 H# Z( h) ?. z$ L0 P#
# 2) configure local2 events to go to the /var/log/haproxy.log
# file. A line like the following can be added to
! Y6 U  w; L) ~8 M# /etc/sysconfig/syslog
$ W3 P& |/ L& R4 T3 w/ n#
#
log 127.0.0.1 local3#      local2.*                 /var/log/haproxy.logchroot      /var/lib/haproxypidfile     /var/run/haproxy.pidmaxconn     4000user        haproxygroup       haproxydaemon# turn on stats unix socketstats socket /var/lib/haproxy/stats#---------------------------------------------------------------------common defaults that all the 'listen' and 'backend' sections willuse if not designated in their block#---------------------------------------------------------------------
defaults
* q4 ?  v8 s0 N$ T7 ~  T. R- Bmode http
% ~, j% r+ ^- H5 X1 B; dlog global
option httplog
option dontlognull
option http-server-close
0 C6 a) y, V6 D. p; Loption forwardfor except 127.0.0.0/8
* J5 O  W$ r0 S7 ^) Yoption redispatch
, U0 c& a6 X4 jretries 3
timeout http-request 10s
timeout queue 1m
timeout connect 10s
timeout client 1m
timeout server 1m
timeout http-keep-alive 10s
timeout check 10s
8 Y( ^) ?* E& B' s# \maxconn 3000#---------------------------------------------------------------------main frontend which proxys to the backends#---------------------------------------------------------------------
/ a( x7 U) x9 b$ afrontend main *:5002
6 h: r" f  D1 d, w2 R$ M; v, fstats uri /haproxyacl url_static path_beg -i /static /images /javascript /stylesheetsacl url_static path_end -i .jpg .gif .png .css .jsuse_backend static if url_staticdefault_backend             app#---------------------------------------------------------------------static backend for serving up images, stylesheets and such#---------------------------------------------------------------------
#backend staticbalance roundrobinserver static 127.0.0.1:4331 check#---------------------------------------------------------------------round robin balancing between the various backends#---------------------------------------------------------------------
, m  @0 ?% M1 j2 q1 D; Tbackend app
mode http
/ N6 W5 Q$ `8 rbalance roundrobin
9 }2 U9 l. j( {) m$ A# d) `server app1 192.168.1.61:8080 check
8 |& S) e( b( C, t% c# C  Wserver app2 192.168.1.62:8080 check
) ^( o5 W) p$ y/ ~! J9 K. c: ]: Dserver app3 192.168.1.63:8080 check#server 部份按照自己apiserver 三台 配置进来即可3.配置rsyslog收集haproxy日志
( K) h7 d) j: J/ H* R  ]# B[root@node4 ~]# echo -e '$ModLoad imudp \n $UDPServerRun 514 \n local3.* /var/log/haproxy.log' >> /etc/rsyslog.conf4.配置node节点配置#配置config 文件 KUBE_MASTER="--master=http://127.0.0.1:5002" 参数指向haproxy的5002端口[root@node4 kubernetes]# pwd
5 y! Z% [: u+ R6 V% k/etc/kubernetes
% H; }. Y7 F" O$ H6 Z) }2 B[root@node4 kubernetes]# ls
1 \7 [$ k, p2 Q) W* r! P0 W# E0 rconfig kubelet proxy[root@node4 kubernetes]# cat configkubernetes system configThe following values are used to configure various aspects of allkubernetes services, includingkube-apiserver.servicekube-controller-manager.servicekube-scheduler.servicekubelet.servicekube-proxy.servicelogging to stderr means we get it in the systemd journalKUBE_LOGTOSTDERR="--logtostderr=true"journal message level, 0 is debugKUBE_LOG_LEVEL="--v=0"Should this cluster be allowed to run privileged docker containersKUBE_ALLOW_PRIV="--allow-privileged=true"How the controller-manager, scheduler, and proxy find the apiserverKUBE_MASTER="--master=http://127.0.0.1:5002"#配置kubelet KUBELET_API_SERVER="--api-servers=http://127.0.0.1:5002"
[root@node4 kubernetes]# cat kubeletkubernetes kubelet (minion) configThe address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces)KUBELET_ADDRESS="--address=0.0.0.0"The port for the info server to serve on#KUBELET_PORT="--port=10250"You may leave this blank to use the actual hostnameKUBELET_HOSTNAME="--hostname-override=192.168.2.69"location of the api-serverKUBELET_API_SERVER="--api-servers=http://127.0.0.1:5002"pod infrastructure container#KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=registry.access.redhat.com/rhel7/pod-infrastructure:latest"Add your own!KUBELET_ARGS="--cluster_dns=172.1.0.2 --cluster_domain=cluster.local"#所有node节点照此配置完成5.#重启所有node节点上的服务,在这里我用ansible来处理，ansible请自行脑补，建议大家用ansible来批量处理会快很多
8 D5 d5 r; n* M#没有安装ansible的，请自行手动重启[root@master1 ~]# ansible -m shell -a ' systemctl restart rsyslog.service ;service haproxy restart ;systemctl restart kubelet.service;systemctl restart kube-proxy.service' 'nodes'
; V5 j8 X3 l0 N' b% R8 j- Tnode3.txg.com | SUCCESS | rc=0 >>
Redirecting to /bin/systemctl restart haproxy.servicenode4.txg.com | SUCCESS | rc=0 >>
  c. y/ w3 e- a: z: aRedirecting to /bin/systemctl restart haproxy.servicenode2.txg.com | SUCCESS | rc=0 >>
7 w& L( }- ]5 T; K1 HRedirecting to /bin/systemctl restart haproxy.servicenode1.txg.com | SUCCESS | rc=0 >>
Redirecting to /bin/systemctl restart haproxy.service#查看所有node上 haproxy 日志 200为正常
$ a3 i0 g$ Z. y3 Q! @; l[root@node3 kubernetes]# tail -f /var/log/haproxy.log
: W* a4 ]& `0 d3 Q: u2017-05-09T11:23:12+08:00 localhost haproxy[18278]: 127.0.0.1:42970 [09/May/2017:11:23:11.992] main app/app1 52/0/0/186/238 200 2507 - - ---- 6/6/5/2/0 0/0 "PUT /api/v1/nodes/192.168.2.69/status HTTP/1.1"
2017-05-09T11:23:22+08:00 localhost haproxy[18278]: 127.0.0.1:42970 [09/May/2017:11:23:12.229] main app/app2 10000/0/1/1/10002 200 2519 - - ---- 6/6/5/1/0 0/0 "GET /api/v1/nodes?fieldSelector=metadata.name%3D192.168.2.69&resourceVersion=0 HTTP/1.1"
2017-05-09T11:23:22+08:00 localhost haproxy[18278]: 127.0.0.1:42970 [09/May/2017:11:23:22.232] main app/app3 60/0/0/123/183 200 2507 - - ---- 6/6/5/2/0 0/0 "PUT /api/v1/nodes/192.168.2.69/status HTTP/1.1"
. J- A, S4 x* s# f* Z0 U8 `3 v, M5 [9 V# Q2017-05-09T11:23:28+08:00 localhost haproxy[18278]: 127.0.0.1:42722 [09/May/2017:11:22:21.385] main app/app1 7384/0/1/0/67387 200 167 - - sD-- 5/5/4/1/0 0/0 "GET /api/v1/watch/pods?fieldSelector=spec.nodeName%3D192.168.2.69&resourceVersion=2348326&timeoutSeconds=424 HTTP/1.1"
2017-05-09T11:23:32+08:00 localhost haproxy[18278]: 127.0.0.1:43096 [09/May/2017:11:23:32.416] main app/app2 0/0/0/1/1 200 2519 - - ---- 6/6/5/1/0 0/0 "GET /api/v1/nodes?fieldSelector=metadata.name%3D192.168.2.69&resourceVersion=0 HTTP/1.1"
- T( c1 }' R1 Y2017-05-09T11:23:32+08:00 localhost haproxy[18278]: 127.0.0.1:43096 [09/May/2017:11:23:32.418] main app/app3 53/0/0/92/145 200 2507 - - ---- 6/6/5/2/0 0/0 "PUT /api/v1/nodes/192.168.2.69/status HTTP/1.1"
2017-05-09T11:23:35+08:00 localhost haproxy[18278]: 127.0.0.1:43096 [09/May/2017:11:23:32.564] main app/app1 2459/0/1/1/2461 200 2507 - - ---- 6/6/5/3/0 0/0 "GET /api/v1/namespaces/kube-system/secrets/default-token-p5l8p HTTP/1.1"
2017-05-09T11:23:42+08:00 localhost haproxy[18278]: 127.0.0.1:38410 [09/May/2017:11:14:38.515] main app/app3 0/0/1/1/544002 200 254800 - - ---- 6/6/4/1/0 0/0 "GET /api/v1/watch/endpoints?resourceVersion=2347840&timeoutSeconds=544 HTTP/1.1"
2017-05-09T11:23:42+08:00 localhost haproxy[18278]: 127.0.0.1:43096 [09/May/2017:11:23:35.024] main app/app3 7540/0/0/1/7541 200 2519 - - ---- 6/6/5/1/0 0/0 "GET /api/v1/nodes?fieldSelector=metadata.name%3D192.168.2.69&resourceVersion=0 HTTP/1.1"
1 H1 o1 z5 k. g0 J+ R: l2017-05-09T11:23:42+08:00 localhost haproxy[18278]: 127.0.0.1:43096 [09/May/2017:11:23:42.566] main app/app1 51/0/1/111/163 200 2507 - - ---- 6/6/5/2/0 0/0 "PUT /api/v1/nodes/192.168.2.69/status HTTP/1.1"#重启所有master节点上的服务ansible -m shell -a 'systemctl restart kube-apiserver.service;systemctl restart kube-controller-manager.service ;systemctl restart kube-scheduler.service ' 'masters'6.查看leader信息位于哪个节点[root@master3 ~]# tail -f /var/log/messages
May 9 11:09:43 master1 kube-scheduler: I0509 11:09:43.354272 4636 leaderelection.go:247] lock is held by master3.txg.com and has not yet expired
" K# H: F1 b* @$ l- PMay 9 11:09:43 master1 kube-controller-manager: I0509 11:09:43.887592 4532 leaderelection.go:247] lock is held by master2.txg.com and has not yet expired#这时， kube-scheduler leader位于master3 和kube-controller-manager 在master2[root@master3 ~]# kubectl -n kube-system get ep kube-controller-manager -o yaml
apiVersion: v1
* c8 h8 v$ I& ckind: Endpoints
metadata:
annotations:
control-plane.alpha.kubernetes.io/leader: '{"holderIdentity":"master2.txg.com","leaseDurationSeconds":15,"acquireTime":"2017-05-08T10:41:07Z","renewTime":"2017-05-09T03:14:02Z","leaderTransitions":0}'
" U# g! U8 ^1 I4 s' E( t' `# H" OcreationTimestamp: 2017-05-08T10:41:07Z
2 j+ j3 I1 }& U. _- W. v$ p7 Aname: kube-controller-manager
. _: Q( {) B; ]- A" c! Lnamespace: kube-system
' e0 S  H4 Y; d$ S' t9 d5 t0 _resourceVersion: "2347791"
selfLink: /api/v1/namespaces/kube-system/endpoints/kube-controller-manager
$ y: ~4 f* {: s; K! ~9 Huid: d7dae24f-33da-11e7-9a51-525400c2bc59
& g+ v6 a' o; [- isubsets: []
[root@master1 ~]# kubectl -n kube-system get ep kube-scheduler -o yaml
apiVersion: v1
" \. M, A3 \$ b9 a, a+ b: [. Y& Qkind: Endpoints
metadata:
* g9 I% ~2 l; c, w" _) mannotations:
control-plane.alpha.kubernetes.io/leader: '{"holderIdentity":"master3.txg.com","leaseDurationSeconds":15,"acquireTime":"2017-05-08T10:41:08Z","renewTime":"2017-05-09T03:14:27Z","leaderTransitions":0}'
$ Y2 S; i3 ^- y1 F2 O2 {creationTimestamp: 2017-05-08T10:41:08Z
& P4 f' S/ |8 M% n  A" t1 rname: kube-scheduler
$ F1 S1 f9 c7 }( `0 ^8 mnamespace: kube-system
6 S  t0 [* u- E7 {& _( TresourceVersion: "2347830"
# X1 Y' Q. z, K7 ^selfLink: /api/v1/namespaces/kube-system/endpoints/kube-scheduler
' `# l8 F# r8 w8 m" B7 ?uid: d87a235a-33da-11e7-9eb5-52540081c06a
subsets: []#至此配置高可用集群配置完成