|
|
#系统centos7 所有节点是yum 安装的kubernetes 版本#前提是集群已经正确运行没有任何问题#原理参照 kubadm 和kargo 三个master做高可用所有node上安装haproxy 负载均衡反代三台kube-apiserver的8080端口 api-server为无状态服务#注意，之前用过nginx配置反代三台apiserver出现创建pod 容器非常慢，3-5分钟，应该有BUG ,建议用haproxy反代,非常顺畅.#controller-manager 和scheduler 为有状态服务，同一时间只有一台当选，会在三台master机之间进行选举，由其中一台担任leader的角色#节点构造如下
* U# _' A8 C! [4 kcat /etc/hosts#master
+ }/ W6 H c) B4 c, [192.168.1.61 master1.txg.com #512M
" w+ J/ I Q, `/ i3 G192.168.1.62 master2.txg.com #512M$ n% _9 W6 Y8 C, V4 [/ {
192.168.1.63 master2.txg.com #512M#master软件包2 r4 [" x# ]$ ^6 B
# rpm -qa|grep kube
`7 p5 | ?# xkubernetes-client-1.5.2-0.2.gitc55cf2b.el7.x86_64
# I9 {7 S2 {" T* `- ?+ skubernetes-master-1.5.2-0.2.gitc55cf2b.el7.x86_64; y" q' F7 j4 [: P, d" J
flannel-0.7.0-1.el7.x86_64#etcd-server1 l) X3 p/ b/ a# t/ R* V: b% K
192.168.1.65 etcd1.txg.com #512M
0 D+ m2 o$ ?$ n( L: s192.168.1.66 etcd2.txg.com #512M
) D; J7 e9 o. m192.168.1.67 etcd3.txg.com #512M#node节点. j8 P! s& A+ f% i2 V$ M7 ^: Z
192.168.1.68 node1.txg.com #4G) T Y$ R4 F6 N0 m
192.168.1.69 node2.txg.com #4G
7 r% _( U5 `" ?! _- f t( t: k192.168.2.68 node3.txg.com #4G
2 D, o H2 K& H8 j2 W4 E) N192.168.2.69 node4.txg.com #4G#node节点软件包8 V E) N# k8 |" k6 k. J
[root@node4 ~]# rpm -qa|egrep 'kube|docker'6 t' N# H$ w6 r$ s' C- \* {
kubernetes-client-1.5.2-0.5.gita552679.el7.x86_64' O( X& t, E* e) K( j1 o
docker-common-1.12.6-11.el7.centos.x86_64
8 q8 m" c8 @2 _/ V$ _docker-1.12.6-11.el7.centos.x86_64$ N& Z$ ~. v0 y) L( u4 U
kubernetes-node-1.5.2-0.5.gita552679.el7.x86_64
& ?" J, b6 i% F9 P( Kdocker-client-1.12.6-11.el7.centos.x86_646 @3 s/ @+ m1 V; N8 Y$ S
flannel-0.7.0-1.el7.x86_64[root@node4 ~]# uname -a' O% f8 e- u2 t2 K: C' p5 d e
Linux node4.txg.com 3.10.0-514.6.2.el7.x86_64 #1 SMP Thu Feb 23 03:04:39 UTC 2017 x86_64 x86_64 x86_64 GNU/Linux修改master server上的配置文件，我的配置文件在/etc/kubernetes/下面
' I$ Y- d9 g. m+ `[root@master1 kubernetes]# pwd* V7 w( T m7 L$ N n5 B+ H
/etc/kubernetes
; k1 w/ H/ Q0 r" o0 T[root@master1 kubernetes]# ls( n+ D. [8 i! h* O6 t% r
apiserver config controller-manager scheduler ssl sslbk1.修改controller-manager和scheduler配置文件在KUBE_CONTROLLER_MANAGER_ARGS=" " 中间加入 --address=127.0.0.1 --leader-elect=true% k1 ~$ [2 O. E
KUBE_CONTROLLER_MANAGER_ARGS=" --address=127.0.0.1 --leader-elect=true --service-account-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem --cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem --cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem --root-ca-file=/etc/kubernetes/ssl/ca.pem"9 S: Y8 k% W( j) X) \' }* I* V, I
修改scheduler 为 KUBE_SCHEDULER_ARGS=" --address=127.0.0.1 --leader-elect=true"
- X5 _5 G/ v5 Y* P5 G: |# ?5 l; v让节点有选举master leader 功能，ok master配置完成同步master1上的配置文件到master2 3 节点2.所有node节点安装 haproxy , yum install haproxy#配置haproxy.cfg文件 监听5002端口，所向代理kube-apiserver:8080[root@node4 ~]# cat /etc/haproxy/haproxy.cfg' z8 }% _' M5 ~5 B+ {; ^
#---------------------------------------------------------------------Example configuration for a possible web application. See thefull configuration options online.http://haproxy.1wt.eu/download/1 ... -------------Global settings#---------------------------------------------------------------------9 F" v' e) u% \$ w: R
global% p% |& y; O4 R+ m; ^" |8 P
# to have these messages end up in /var/log/haproxy.log you will# E% `4 J8 F2 r* w1 ]5 F
# need to:
`- l# U% c" A4 [! S#7 `3 v, U- ~, @7 N; L- D- e: \! w
# 1) configure syslog to accept network log events. This is done
( [. e" O1 G# J# by adding the '-r' option to the SYSLOGD_OPTIONS in P7 O+ l% {( E
# /etc/sysconfig/syslog2 P) T$ ]) K, e# _/ @/ o! ~
#5 n6 A, V$ r- ?5 x: `2 C
# 2) configure local2 events to go to the /var/log/haproxy.log
- P0 t+ k: k. a# file. A line like the following can be added to( S" s; @( G" @1 ]8 O
# /etc/sysconfig/syslog
( r; n y& }$ s#$ B0 g* y9 A. \! Y; }# N* F
#7 n5 k/ R5 \" t; J1 f
log 127.0.0.1 local3# local2.* /var/log/haproxy.logchroot /var/lib/haproxypidfile /var/run/haproxy.pidmaxconn 4000user haproxygroup haproxydaemon# turn on stats unix socketstats socket /var/lib/haproxy/stats#---------------------------------------------------------------------common defaults that all the 'listen' and 'backend' sections willuse if not designated in their block#---------------------------------------------------------------------
+ Q* u' z$ [ t! gdefaults0 S1 c. q2 u p0 T
mode http' o* M# V( E" M8 u- s; V
log global
1 {5 p0 j# C- u% X2 koption httplog& S C& P+ I, @7 J2 d4 p' U& T
option dontlognull
* h. @% o( A6 U3 z5 W+ u% X" Foption http-server-close
' S* m& ]" k! B; e1 P! zoption forwardfor except 127.0.0.0/8' J, w$ p0 \. Q" R A
option redispatch
3 }4 M+ t. |4 Q" O7 vretries 3
/ j2 D& `( X1 T. W' Y. S" @1 gtimeout http-request 10s0 {5 ]; e) q' I1 p% V5 C, x
timeout queue 1m7 _+ ~# |1 y4 Z% s, Z
timeout connect 10s
/ b: n4 M( P; H' r* W$ btimeout client 1m
6 X- I5 J X4 ^& A4 b" s- ttimeout server 1m
/ D- B+ s" A3 e7 V* o: G0 mtimeout http-keep-alive 10s! U1 `6 f" J7 I. w0 q
timeout check 10s
# h( A" R& b7 }+ Xmaxconn 3000#---------------------------------------------------------------------main frontend which proxys to the backends#---------------------------------------------------------------------' z4 b& t' `' |) V3 o
frontend main *:5002
$ l/ E5 k/ q3 Z5 a9 ystats uri /haproxyacl url_static path_beg -i /static /images /javascript /stylesheetsacl url_static path_end -i .jpg .gif .png .css .jsuse_backend static if url_staticdefault_backend app#---------------------------------------------------------------------static backend for serving up images, stylesheets and such#---------------------------------------------------------------------2 v8 R; W) z, s9 D$ v$ s
#backend staticbalance roundrobinserver static 127.0.0.1:4331 check#---------------------------------------------------------------------round robin balancing between the various backends#---------------------------------------------------------------------
) r0 H* ?0 i4 _. Fbackend app
/ N5 `" {8 m+ L/ h1 dmode http. ?0 F! T! i6 L4 ~* z. T- i# {
balance roundrobin" l' b- f4 l, Q3 @2 Q* Y+ p# W
server app1 192.168.1.61:8080 check. z5 s1 ?# Y4 S) E B1 K) o0 ]4 V
server app2 192.168.1.62:8080 check- Z4 `, o3 _: p# G
server app3 192.168.1.63:8080 check#server 部份按照自己apiserver 三台 配置进来即可3.配置rsyslog收集haproxy日志
0 M. y1 W' q$ l# w( R[root@node4 ~]# echo -e '$ModLoad imudp \n $UDPServerRun 514 \n local3.* /var/log/haproxy.log' >> /etc/rsyslog.conf4.配置node节点配置#配置config 文件 KUBE_MASTER="--master=http://127.0.0.1:5002" 参数指向haproxy的5002端口[root@node4 kubernetes]# pwd
]4 S$ d% R' [( K2 b4 o' K/etc/kubernetes
2 ~4 s/ s( j* T6 I" E& H+ N[root@node4 kubernetes]# ls
7 n& c& C, L& G/ I( D ~3 _config kubelet proxy[root@node4 kubernetes]# cat configkubernetes system configThe following values are used to configure various aspects of allkubernetes services, includingkube-apiserver.servicekube-controller-manager.servicekube-scheduler.servicekubelet.servicekube-proxy.servicelogging to stderr means we get it in the systemd journalKUBE_LOGTOSTDERR="--logtostderr=true"journal message level, 0 is debugKUBE_LOG_LEVEL="--v=0"Should this cluster be allowed to run privileged docker containersKUBE_ALLOW_PRIV="--allow-privileged=true"How the controller-manager, scheduler, and proxy find the apiserverKUBE_MASTER="--master=http://127.0.0.1:5002"#配置kubelet KUBELET_API_SERVER="--api-servers=http://127.0.0.1:5002"3 T9 n/ b- Y+ j4 |0 y( C7 }
[root@node4 kubernetes]# cat kubeletkubernetes kubelet (minion) configThe address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces)KUBELET_ADDRESS="--address=0.0.0.0"The port for the info server to serve on#KUBELET_PORT="--port=10250"You may leave this blank to use the actual hostnameKUBELET_HOSTNAME="--hostname-override=192.168.2.69"location of the api-serverKUBELET_API_SERVER="--api-servers=http://127.0.0.1:5002"pod infrastructure container#KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=registry.access.redhat.com/rhel7/pod-infrastructure:latest"Add your own!KUBELET_ARGS="--cluster_dns=172.1.0.2 --cluster_domain=cluster.local"#所有node节点照此配置完成5.#重启所有node节点上的服务,在这里我用ansible来处理，ansible请自行脑补，建议大家用ansible来批量处理会快很多
/ Q% U6 u5 {' m" E, U#没有安装ansible的，请自行手动重启[root@master1 ~]# ansible -m shell -a ' systemctl restart rsyslog.service ;service haproxy restart ;systemctl restart kubelet.service;systemctl restart kube-proxy.service' 'nodes'
3 V* d+ n" ~ ^) gnode3.txg.com | SUCCESS | rc=0 >>' C# W; {# ] [$ A
Redirecting to /bin/systemctl restart haproxy.servicenode4.txg.com | SUCCESS | rc=0 >>1 o7 {) X, C& n1 P+ V
Redirecting to /bin/systemctl restart haproxy.servicenode2.txg.com | SUCCESS | rc=0 >>/ j$ c f3 o1 j* h% l7 r
Redirecting to /bin/systemctl restart haproxy.servicenode1.txg.com | SUCCESS | rc=0 >>) s& t3 _9 M9 Z
Redirecting to /bin/systemctl restart haproxy.service#查看所有node上 haproxy 日志 200为正常7 h$ T0 Q+ k2 Z; G
[root@node3 kubernetes]# tail -f /var/log/haproxy.log
8 ` `9 o+ R1 \3 M9 E- i2017-05-09T11:23:12+08:00 localhost haproxy[18278]: 127.0.0.1:42970 [09/May/2017:11:23:11.992] main app/app1 52/0/0/186/238 200 2507 - - ---- 6/6/5/2/0 0/0 "PUT /api/v1/nodes/192.168.2.69/status HTTP/1.1"
4 p$ E& p7 c: H' F4 y6 }- c' \2017-05-09T11:23:22+08:00 localhost haproxy[18278]: 127.0.0.1:42970 [09/May/2017:11:23:12.229] main app/app2 10000/0/1/1/10002 200 2519 - - ---- 6/6/5/1/0 0/0 "GET /api/v1/nodes?fieldSelector=metadata.name%3D192.168.2.69&resourceVersion=0 HTTP/1.1"' e- |2 l$ H1 L+ b& C9 [7 D, u
2017-05-09T11:23:22+08:00 localhost haproxy[18278]: 127.0.0.1:42970 [09/May/2017:11:23:22.232] main app/app3 60/0/0/123/183 200 2507 - - ---- 6/6/5/2/0 0/0 "PUT /api/v1/nodes/192.168.2.69/status HTTP/1.1"8 _: M; w4 Q- i# X7 E1 u6 L P
2017-05-09T11:23:28+08:00 localhost haproxy[18278]: 127.0.0.1:42722 [09/May/2017:11:22:21.385] main app/app1 7384/0/1/0/67387 200 167 - - sD-- 5/5/4/1/0 0/0 "GET /api/v1/watch/pods?fieldSelector=spec.nodeName%3D192.168.2.69&resourceVersion=2348326&timeoutSeconds=424 HTTP/1.1"
% Y" }3 w9 `3 M3 c" G; B$ I2017-05-09T11:23:32+08:00 localhost haproxy[18278]: 127.0.0.1:43096 [09/May/2017:11:23:32.416] main app/app2 0/0/0/1/1 200 2519 - - ---- 6/6/5/1/0 0/0 "GET /api/v1/nodes?fieldSelector=metadata.name%3D192.168.2.69&resourceVersion=0 HTTP/1.1"! \9 S" b. a0 T$ ~, z
2017-05-09T11:23:32+08:00 localhost haproxy[18278]: 127.0.0.1:43096 [09/May/2017:11:23:32.418] main app/app3 53/0/0/92/145 200 2507 - - ---- 6/6/5/2/0 0/0 "PUT /api/v1/nodes/192.168.2.69/status HTTP/1.1"
9 H, `: m4 x( }& ?0 ?' l2017-05-09T11:23:35+08:00 localhost haproxy[18278]: 127.0.0.1:43096 [09/May/2017:11:23:32.564] main app/app1 2459/0/1/1/2461 200 2507 - - ---- 6/6/5/3/0 0/0 "GET /api/v1/namespaces/kube-system/secrets/default-token-p5l8p HTTP/1.1"
# o$ a7 H( A; g( R; M) U2017-05-09T11:23:42+08:00 localhost haproxy[18278]: 127.0.0.1:38410 [09/May/2017:11:14:38.515] main app/app3 0/0/1/1/544002 200 254800 - - ---- 6/6/4/1/0 0/0 "GET /api/v1/watch/endpoints?resourceVersion=2347840&timeoutSeconds=544 HTTP/1.1"
- N; b3 M. ]* d0 \8 g& p2017-05-09T11:23:42+08:00 localhost haproxy[18278]: 127.0.0.1:43096 [09/May/2017:11:23:35.024] main app/app3 7540/0/0/1/7541 200 2519 - - ---- 6/6/5/1/0 0/0 "GET /api/v1/nodes?fieldSelector=metadata.name%3D192.168.2.69&resourceVersion=0 HTTP/1.1"
' \( N: h/ I/ t6 F2017-05-09T11:23:42+08:00 localhost haproxy[18278]: 127.0.0.1:43096 [09/May/2017:11:23:42.566] main app/app1 51/0/1/111/163 200 2507 - - ---- 6/6/5/2/0 0/0 "PUT /api/v1/nodes/192.168.2.69/status HTTP/1.1"#重启所有master节点上的服务ansible -m shell -a 'systemctl restart kube-apiserver.service;systemctl restart kube-controller-manager.service ;systemctl restart kube-scheduler.service ' 'masters'6.查看leader信息位于哪个节点[root@master3 ~]# tail -f /var/log/messages
; ]* `3 h1 N% p* I# jMay 9 11:09:43 master1 kube-scheduler: I0509 11:09:43.354272 4636 leaderelection.go:247] lock is held by master3.txg.com and has not yet expired
0 M6 A$ k7 b! K6 _# {May 9 11:09:43 master1 kube-controller-manager: I0509 11:09:43.887592 4532 leaderelection.go:247] lock is held by master2.txg.com and has not yet expired#这时， kube-scheduler leader位于master3 和kube-controller-manager 在master2[root@master3 ~]# kubectl -n kube-system get ep kube-controller-manager -o yaml$ J9 Z* R6 n+ h9 o% }" I2 V
apiVersion: v1- ^) w6 n6 M7 n0 S
kind: Endpoints
% l5 A) F! U) P6 a3 z4 Ometadata:8 ~- M7 h( g( m/ B) z% u
annotations:. B7 c$ P9 L' B9 X$ r8 C" M& H
control-plane.alpha.kubernetes.io/leader: '{"holderIdentity":"master2.txg.com","leaseDurationSeconds":15,"acquireTime":"2017-05-08T10:41:07Z","renewTime":"2017-05-09T03:14:02Z","leaderTransitions":0}'- C1 w' u% E( h" w2 \6 h
creationTimestamp: 2017-05-08T10:41:07Z7 J" d+ m7 f. z8 v+ _) N$ W' `# Z- P3 ~
name: kube-controller-manager
0 F' Z1 n T" o; R$ `* G$ xnamespace: kube-system2 [) N O' o* k) |
resourceVersion: "2347791"
4 X/ Q9 |: x2 ^, k$ @6 UselfLink: /api/v1/namespaces/kube-system/endpoints/kube-controller-manager4 m- c' c. ~+ b0 h! m8 ]- N
uid: d7dae24f-33da-11e7-9a51-525400c2bc59" y5 i0 O" H! {/ [
subsets: []
0 h$ G8 y9 G% @1 D5 n6 e s$ V[root@master1 ~]# kubectl -n kube-system get ep kube-scheduler -o yaml
2 Q& `: U( w' O5 g' JapiVersion: v16 `* X+ \8 f6 c' }
kind: Endpoints- [$ F2 J: f/ L/ o/ G$ n, i& G' ]
metadata:1 |( w$ T# Q% P7 E8 S6 i$ R
annotations: M; q+ z4 P9 L3 O0 {0 t A
control-plane.alpha.kubernetes.io/leader: '{"holderIdentity":"master3.txg.com","leaseDurationSeconds":15,"acquireTime":"2017-05-08T10:41:08Z","renewTime":"2017-05-09T03:14:27Z","leaderTransitions":0}'
$ {. }3 t* M" o; q' {. zcreationTimestamp: 2017-05-08T10:41:08Z
; C/ F& k- Y9 ^: z% }/ Wname: kube-scheduler& I( R A `0 d! T
namespace: kube-system
$ A' l3 r3 _6 m0 d( \. f" jresourceVersion: "2347830"0 p X) @! e4 o7 n
selfLink: /api/v1/namespaces/kube-system/endpoints/kube-scheduler( F- B8 e8 _5 I8 K! I
uid: d87a235a-33da-11e7-9eb5-52540081c06a) X, ^: s5 U' L
subsets: []#至此配置高可用集群配置完成 |
|
发表于 2021-7-19 09:39:14