|
|
#系统centos7 所有节点是yum 安装的kubernetes 版本#前提是集群已经正确运行没有任何问题#原理参照 kubadm 和kargo 三个master做高可用所有node上安装haproxy 负载均衡反代三台kube-apiserver的8080端口 api-server为无状态服务#注意，之前用过nginx配置反代三台apiserver出现创建pod 容器非常慢，3-5分钟，应该有BUG ,建议用haproxy反代,非常顺畅.#controller-manager 和scheduler 为有状态服务，同一时间只有一台当选，会在三台master机之间进行选举，由其中一台担任leader的角色#节点构造如下2 W8 J5 u: g+ n+ m; F3 A; k
cat /etc/hosts#master: S' \/ n: B5 k' t \9 {/ B. p8 K
192.168.1.61 master1.txg.com #512M2 J6 y8 D; q( H( N$ J& s
192.168.1.62 master2.txg.com #512M
' ^' L5 ~* b" H- x$ V4 ?- p192.168.1.63 master2.txg.com #512M#master软件包
$ S9 O2 J) R# [1 y# rpm -qa|grep kube, D, l9 J, N* e
kubernetes-client-1.5.2-0.2.gitc55cf2b.el7.x86_64( ^ O6 M$ U) d: a) }) g+ S. a
kubernetes-master-1.5.2-0.2.gitc55cf2b.el7.x86_64
1 B4 e# H# [* Qflannel-0.7.0-1.el7.x86_64#etcd-server+ j; o+ ~0 ^, h0 u8 m$ X
192.168.1.65 etcd1.txg.com #512M
. g2 x! g) D& A2 M1 W V% F( l192.168.1.66 etcd2.txg.com #512M9 ~' |3 x, W3 L# J& w
192.168.1.67 etcd3.txg.com #512M#node节点/ ?" h* }1 o3 @& c
192.168.1.68 node1.txg.com #4G
6 P% @1 h1 e- U5 b$ U192.168.1.69 node2.txg.com #4G( R9 W* i/ r: w- Q) \4 n
192.168.2.68 node3.txg.com #4G
8 f2 |! @: z1 ^4 v; Y: r192.168.2.69 node4.txg.com #4G#node节点软件包' H/ g @3 T- i: r
[root@node4 ~]# rpm -qa|egrep 'kube|docker'
8 k" D, a2 v# o) f+ ~kubernetes-client-1.5.2-0.5.gita552679.el7.x86_647 N0 n4 |) Q2 C5 A
docker-common-1.12.6-11.el7.centos.x86_64 d' M2 g$ Y- B
docker-1.12.6-11.el7.centos.x86_64* k0 J/ J5 q$ }! Y: T$ q/ |0 h) ^
kubernetes-node-1.5.2-0.5.gita552679.el7.x86_64
|% j- S' c" x: X* c5 ]8 x' s; [docker-client-1.12.6-11.el7.centos.x86_64
; N- [/ ?9 y! J0 kflannel-0.7.0-1.el7.x86_64[root@node4 ~]# uname -a' u/ L* S1 \! H9 Y
Linux node4.txg.com 3.10.0-514.6.2.el7.x86_64 #1 SMP Thu Feb 23 03:04:39 UTC 2017 x86_64 x86_64 x86_64 GNU/Linux修改master server上的配置文件，我的配置文件在/etc/kubernetes/下面$ X [: [! u ~1 o1 m
[root@master1 kubernetes]# pwd% z9 u! t7 ?, G7 W) E
/etc/kubernetes
C- u2 [5 F! j7 }. X0 d[root@master1 kubernetes]# ls5 g* e, ?3 S0 M$ s4 f0 e
apiserver config controller-manager scheduler ssl sslbk1.修改controller-manager和scheduler配置文件在KUBE_CONTROLLER_MANAGER_ARGS=" " 中间加入 --address=127.0.0.1 --leader-elect=true
) d9 h" ~! V6 y8 [8 aKUBE_CONTROLLER_MANAGER_ARGS=" --address=127.0.0.1 --leader-elect=true --service-account-private-key-file=/etc/kubernetes/ssl/apiserver-key.pem --cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem --cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem --root-ca-file=/etc/kubernetes/ssl/ca.pem"
# Q9 `7 A4 O5 @3 Y" V' _% ~7 m修改scheduler 为 KUBE_SCHEDULER_ARGS=" --address=127.0.0.1 --leader-elect=true"
6 M1 W- a, Z* \7 g让节点有选举master leader 功能，ok master配置完成同步master1上的配置文件到master2 3 节点2.所有node节点安装 haproxy , yum install haproxy#配置haproxy.cfg文件 监听5002端口，所向代理kube-apiserver:8080[root@node4 ~]# cat /etc/haproxy/haproxy.cfg
! R7 K# a; Q' R4 ]& ^( X" k#---------------------------------------------------------------------Example configuration for a possible web application. See thefull configuration options online.http://haproxy.1wt.eu/download/1 ... -------------Global settings#---------------------------------------------------------------------
9 q% D! r2 D) w0 t2 d1 m& bglobal
$ p, s% F6 x& C3 o* r$ ]# to have these messages end up in /var/log/haproxy.log you will/ @4 T0 [4 N* {
# need to:& J: t- x* Y* f1 c* ^2 r9 n; l% X
#
4 z% V, S8 R6 P# {- W! [# 1) configure syslog to accept network log events. This is done. m) d/ V" l& j" q8 x3 u, d6 X3 g7 w
# by adding the '-r' option to the SYSLOGD_OPTIONS in
P; ^2 D% R j3 _5 p6 f8 z: _6 k* f# /etc/sysconfig/syslog6 s2 J0 E' w! a2 ~' \
## _* g- u+ B- ~3 _6 l* h& N {
# 2) configure local2 events to go to the /var/log/haproxy.log
9 T% L: @, E5 M! E$ D L# file. A line like the following can be added to
3 u: h) N; `9 \* S# /etc/sysconfig/syslog
* S9 i) Y' V; B8 y" z3 P, p `#
; [3 f+ E- C3 E; ?#
4 ~+ \8 P$ }5 G) c: C. @ G; klog 127.0.0.1 local3# local2.* /var/log/haproxy.logchroot /var/lib/haproxypidfile /var/run/haproxy.pidmaxconn 4000user haproxygroup haproxydaemon# turn on stats unix socketstats socket /var/lib/haproxy/stats#---------------------------------------------------------------------common defaults that all the 'listen' and 'backend' sections willuse if not designated in their block#---------------------------------------------------------------------
+ O; i' n/ A d$ \5 Zdefaults/ b6 A$ v; o7 o4 |4 a
mode http
, e D3 b5 |! Klog global
+ [, R% O7 v7 ]6 E, Loption httplog
7 A6 W" b/ S+ ` Woption dontlognull
- K7 C& X( }7 Zoption http-server-close' b/ V: `) c: @, r, y4 C; n
option forwardfor except 127.0.0.0/8
& K& p6 n2 G boption redispatch0 q4 p# @) {0 m- R
retries 37 Z# d( _/ D- O7 l3 m5 k; X: i: Q. M' B
timeout http-request 10s
4 Z- s/ \4 A5 y- v4 j2 K9 O: |timeout queue 1m
2 `0 r' h( r$ d( P4 y; t! W7 ftimeout connect 10s" |1 {5 j: u9 [- z; S3 i+ Z) O
timeout client 1m, O2 r/ n$ G8 [2 _! W
timeout server 1m
3 ]+ z6 l" ?! otimeout http-keep-alive 10s
J r, f& Z+ I# V6 v8 U" j* Vtimeout check 10s8 L2 [! [* Y2 w
maxconn 3000#---------------------------------------------------------------------main frontend which proxys to the backends#---------------------------------------------------------------------
Z z: w" R1 C7 p) J- k6 Wfrontend main *:5002: U2 d) f' ~0 ?' N: Z' \# Q0 E, L
stats uri /haproxyacl url_static path_beg -i /static /images /javascript /stylesheetsacl url_static path_end -i .jpg .gif .png .css .jsuse_backend static if url_staticdefault_backend app#---------------------------------------------------------------------static backend for serving up images, stylesheets and such#---------------------------------------------------------------------. e( G6 ~* ]! C' d1 i6 r+ T8 a
#backend staticbalance roundrobinserver static 127.0.0.1:4331 check#---------------------------------------------------------------------round robin balancing between the various backends#---------------------------------------------------------------------
; t8 Z% X' f) x1 S" {4 S9 mbackend app
+ V! Y& r: ~! m$ M, ~mode http
# B; }1 u3 A) q8 Z( B2 Cbalance roundrobin
# } q; J* c" t6 |8 U& Z3 Zserver app1 192.168.1.61:8080 check
0 [6 j9 Z3 H- q* ^: m! T) K/ e7 l/ sserver app2 192.168.1.62:8080 check
- C% Z1 d+ r' b3 c, f9 bserver app3 192.168.1.63:8080 check#server 部份按照自己apiserver 三台 配置进来即可3.配置rsyslog收集haproxy日志
( P, q. P+ A# q[root@node4 ~]# echo -e '$ModLoad imudp \n $UDPServerRun 514 \n local3.* /var/log/haproxy.log' >> /etc/rsyslog.conf4.配置node节点配置#配置config 文件 KUBE_MASTER="--master=http://127.0.0.1:5002" 参数指向haproxy的5002端口[root@node4 kubernetes]# pwd
& Q9 |/ h8 q. D' o$ r3 u4 C1 m/etc/kubernetes
5 p# p n) A" c& I[root@node4 kubernetes]# ls
{ h+ F9 H5 R0 m: S$ J! ~config kubelet proxy[root@node4 kubernetes]# cat configkubernetes system configThe following values are used to configure various aspects of allkubernetes services, includingkube-apiserver.servicekube-controller-manager.servicekube-scheduler.servicekubelet.servicekube-proxy.servicelogging to stderr means we get it in the systemd journalKUBE_LOGTOSTDERR="--logtostderr=true"journal message level, 0 is debugKUBE_LOG_LEVEL="--v=0"Should this cluster be allowed to run privileged docker containersKUBE_ALLOW_PRIV="--allow-privileged=true"How the controller-manager, scheduler, and proxy find the apiserverKUBE_MASTER="--master=http://127.0.0.1:5002"#配置kubelet KUBELET_API_SERVER="--api-servers=http://127.0.0.1:5002"
) Y) g. |5 N6 y, V[root@node4 kubernetes]# cat kubeletkubernetes kubelet (minion) configThe address for the info server to serve on (set to 0.0.0.0 or "" for all interfaces)KUBELET_ADDRESS="--address=0.0.0.0"The port for the info server to serve on#KUBELET_PORT="--port=10250"You may leave this blank to use the actual hostnameKUBELET_HOSTNAME="--hostname-override=192.168.2.69"location of the api-serverKUBELET_API_SERVER="--api-servers=http://127.0.0.1:5002"pod infrastructure container#KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=registry.access.redhat.com/rhel7/pod-infrastructure:latest"Add your own!KUBELET_ARGS="--cluster_dns=172.1.0.2 --cluster_domain=cluster.local"#所有node节点照此配置完成5.#重启所有node节点上的服务,在这里我用ansible来处理，ansible请自行脑补，建议大家用ansible来批量处理会快很多' H8 w4 ?" t+ t; \
#没有安装ansible的，请自行手动重启[root@master1 ~]# ansible -m shell -a ' systemctl restart rsyslog.service ;service haproxy restart ;systemctl restart kubelet.service;systemctl restart kube-proxy.service' 'nodes'4 |, x6 E$ p3 ?, `. q) i9 {
node3.txg.com | SUCCESS | rc=0 >>
0 y$ _! h. c6 @; v) }% g0 ^8 @2 lRedirecting to /bin/systemctl restart haproxy.servicenode4.txg.com | SUCCESS | rc=0 >>4 Y( D* N' W I( Q; ?1 g
Redirecting to /bin/systemctl restart haproxy.servicenode2.txg.com | SUCCESS | rc=0 >>
) A& j8 ?, n8 kRedirecting to /bin/systemctl restart haproxy.servicenode1.txg.com | SUCCESS | rc=0 >>2 f, l6 `" H- c, Q2 W
Redirecting to /bin/systemctl restart haproxy.service#查看所有node上 haproxy 日志 200为正常6 K& G0 V% g# x
[root@node3 kubernetes]# tail -f /var/log/haproxy.log
2 C5 }8 O: A; _2017-05-09T11:23:12+08:00 localhost haproxy[18278]: 127.0.0.1:42970 [09/May/2017:11:23:11.992] main app/app1 52/0/0/186/238 200 2507 - - ---- 6/6/5/2/0 0/0 "PUT /api/v1/nodes/192.168.2.69/status HTTP/1.1"+ e, {3 R0 I$ K d9 z% ~) `
2017-05-09T11:23:22+08:00 localhost haproxy[18278]: 127.0.0.1:42970 [09/May/2017:11:23:12.229] main app/app2 10000/0/1/1/10002 200 2519 - - ---- 6/6/5/1/0 0/0 "GET /api/v1/nodes?fieldSelector=metadata.name%3D192.168.2.69&resourceVersion=0 HTTP/1.1"7 T+ a5 y! ]1 H- `& R2 Q9 b0 q
2017-05-09T11:23:22+08:00 localhost haproxy[18278]: 127.0.0.1:42970 [09/May/2017:11:23:22.232] main app/app3 60/0/0/123/183 200 2507 - - ---- 6/6/5/2/0 0/0 "PUT /api/v1/nodes/192.168.2.69/status HTTP/1.1"6 D B! H- l4 G% H2 i9 J
2017-05-09T11:23:28+08:00 localhost haproxy[18278]: 127.0.0.1:42722 [09/May/2017:11:22:21.385] main app/app1 7384/0/1/0/67387 200 167 - - sD-- 5/5/4/1/0 0/0 "GET /api/v1/watch/pods?fieldSelector=spec.nodeName%3D192.168.2.69&resourceVersion=2348326&timeoutSeconds=424 HTTP/1.1"$ _+ j4 i2 {- I" y& Y8 N
2017-05-09T11:23:32+08:00 localhost haproxy[18278]: 127.0.0.1:43096 [09/May/2017:11:23:32.416] main app/app2 0/0/0/1/1 200 2519 - - ---- 6/6/5/1/0 0/0 "GET /api/v1/nodes?fieldSelector=metadata.name%3D192.168.2.69&resourceVersion=0 HTTP/1.1"
% g" o) d0 M& k) h4 r' W5 i" v2017-05-09T11:23:32+08:00 localhost haproxy[18278]: 127.0.0.1:43096 [09/May/2017:11:23:32.418] main app/app3 53/0/0/92/145 200 2507 - - ---- 6/6/5/2/0 0/0 "PUT /api/v1/nodes/192.168.2.69/status HTTP/1.1"
- P( D" e* @( E$ A( q7 N$ [, ^, C2017-05-09T11:23:35+08:00 localhost haproxy[18278]: 127.0.0.1:43096 [09/May/2017:11:23:32.564] main app/app1 2459/0/1/1/2461 200 2507 - - ---- 6/6/5/3/0 0/0 "GET /api/v1/namespaces/kube-system/secrets/default-token-p5l8p HTTP/1.1"1 `* u( C+ m4 q9 H$ k4 Q: \) k: I! b$ ^
2017-05-09T11:23:42+08:00 localhost haproxy[18278]: 127.0.0.1:38410 [09/May/2017:11:14:38.515] main app/app3 0/0/1/1/544002 200 254800 - - ---- 6/6/4/1/0 0/0 "GET /api/v1/watch/endpoints?resourceVersion=2347840&timeoutSeconds=544 HTTP/1.1"
z1 ^9 ]$ ~$ h( k. H8 r2017-05-09T11:23:42+08:00 localhost haproxy[18278]: 127.0.0.1:43096 [09/May/2017:11:23:35.024] main app/app3 7540/0/0/1/7541 200 2519 - - ---- 6/6/5/1/0 0/0 "GET /api/v1/nodes?fieldSelector=metadata.name%3D192.168.2.69&resourceVersion=0 HTTP/1.1"7 q- J# J" W: Y/ }& e& ?
2017-05-09T11:23:42+08:00 localhost haproxy[18278]: 127.0.0.1:43096 [09/May/2017:11:23:42.566] main app/app1 51/0/1/111/163 200 2507 - - ---- 6/6/5/2/0 0/0 "PUT /api/v1/nodes/192.168.2.69/status HTTP/1.1"#重启所有master节点上的服务ansible -m shell -a 'systemctl restart kube-apiserver.service;systemctl restart kube-controller-manager.service ;systemctl restart kube-scheduler.service ' 'masters'6.查看leader信息位于哪个节点[root@master3 ~]# tail -f /var/log/messages K$ X. `5 `+ g( i5 Y6 {$ W# F
May 9 11:09:43 master1 kube-scheduler: I0509 11:09:43.354272 4636 leaderelection.go:247] lock is held by master3.txg.com and has not yet expired
, r m4 [+ q* U5 p, C" W7 KMay 9 11:09:43 master1 kube-controller-manager: I0509 11:09:43.887592 4532 leaderelection.go:247] lock is held by master2.txg.com and has not yet expired#这时， kube-scheduler leader位于master3 和kube-controller-manager 在master2[root@master3 ~]# kubectl -n kube-system get ep kube-controller-manager -o yaml
- B+ T2 x% \2 }+ oapiVersion: v1/ l1 f4 u x$ J
kind: Endpoints
, T2 |; t0 f2 d+ z* p) pmetadata:
5 y3 e8 H5 E0 ?. [annotations:1 C' l3 o, H) B+ b
control-plane.alpha.kubernetes.io/leader: '{"holderIdentity":"master2.txg.com","leaseDurationSeconds":15,"acquireTime":"2017-05-08T10:41:07Z","renewTime":"2017-05-09T03:14:02Z","leaderTransitions":0}'3 j0 ~. w) t. |7 ]! t
creationTimestamp: 2017-05-08T10:41:07Z
/ A A3 |$ c! g0 P$ i0 z( Cname: kube-controller-manager
4 u- d/ M7 ?# Vnamespace: kube-system& N0 Y9 w0 o6 V O7 T, Y: _
resourceVersion: "2347791"0 `: i6 u, Q( w. U; Z4 p- w
selfLink: /api/v1/namespaces/kube-system/endpoints/kube-controller-manager
* e" V. a: E2 n5 huid: d7dae24f-33da-11e7-9a51-525400c2bc59, z9 T3 ?/ m/ Y8 M) W
subsets: []
2 T1 W4 B2 D, _, F[root@master1 ~]# kubectl -n kube-system get ep kube-scheduler -o yaml
( d2 z: i8 ]4 c* |/ NapiVersion: v1
5 y) C- I( }* v: B; F0 a, qkind: Endpoints
* G" u! G3 L# S g2 m- bmetadata:
" l" L8 a* S" j r. T3 u8 Q: Oannotations:
' M' C3 }1 E, y: w$ {1 acontrol-plane.alpha.kubernetes.io/leader: '{"holderIdentity":"master3.txg.com","leaseDurationSeconds":15,"acquireTime":"2017-05-08T10:41:08Z","renewTime":"2017-05-09T03:14:27Z","leaderTransitions":0}'
8 O$ E! b0 D& L U8 S: ^; v2 kcreationTimestamp: 2017-05-08T10:41:08Z; I/ m0 Z3 ?: Y* [0 K( v& l
name: kube-scheduler
' D5 s, r+ ~; ?* M+ I1 {% K7 `namespace: kube-system
% P u1 j: {$ |resourceVersion: "2347830"
( ?4 h' \* A$ |+ iselfLink: /api/v1/namespaces/kube-system/endpoints/kube-scheduler' P( p# k) l% u0 u$ y
uid: d87a235a-33da-11e7-9eb5-52540081c06a
) u* I% N: c, }; [. j. ^subsets: []#至此配置高可用集群配置完成 |
|
发表于 2021-7-19 09:39:14