|
|
楼主 |
发表于 2025-1-1 19:51:59
|
显示全部楼层
创建目录
/ A0 k: |+ g" O: x2 C$ j根据自身实际情况创建指定路径，此路径用来存放k8s二进制文件以及用到的镜像文件+ I, f- @" M9 g& J
e* z' e: @3 z5 R& i* Mmkdir -p /approot1/k8s/{bin,images,pkg,tmp/{ssl,service}}
( u/ B. W/ E+ A! h$ T& N# @1 `关闭防火墙
( n$ w0 V! k( \for i in 192.168.91.19 192.168.91.20;do \+ k; }+ m1 U6 f' _/ r8 w
ssh $i "systemctl disable firewalld"; \% G3 H3 l6 ^9 B T4 N2 M; u
ssh $i "systemctl stop firewalld"; \
5 [! n' B6 c6 B1 ]done4 [) t) Y4 `- I+ F! F' ~7 ~9 I
关闭selinux
% s. S6 T: ]' X0 l临时关闭* U! I' H; \+ j+ d
( E( Q+ ^, l& s
for i in 192.168.91.19 192.168.91.20;do \- C) U4 U. P! n& D% \* L4 x9 w
ssh $i "setenforce 0"; \6 s; t9 R9 g$ Q* B( ~9 G* L
done
* y; a1 o) w: g/ w% o8 h* X# `+ F永久关闭3 j2 J0 k0 m. r M- T: t
: h, n( w1 N9 M+ i, S h
for i in 192.168.91.19 192.168.91.20;do \! r+ |1 \- \+ G) d
ssh $i "sed -i '/SELINUX/s/enforcing/disabled/g' /etc/selinux/config"; \' Q" Z6 t- I! I% [+ |0 q, c0 P
done
0 Y% P- q5 f1 g$ C+ U5 h. L关闭swap
% l# o4 k8 U$ ]1 U) v. S临时关闭- l3 r2 ]) ~7 m7 e, b6 T
& G' C7 @2 |, _! j
for i in 192.168.91.19 192.168.91.20;do \3 n+ g# e9 `6 V. l8 {+ x
ssh $i "swapoff -a"; \: \4 H8 n1 u; t1 w! n; t
done
* }( |2 w& [: B8 Q* \5 v永久关闭
5 S" l# c% P9 I# ~' I, S( a
' r7 l2 a* a) O; {" g6 p* t \4 n: Gfor i in 192.168.91.19 192.168.91.20;do \
0 {# {7 u8 ]+ S! W+ q `ssh $i "sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab"; \2 p+ V5 O6 t# p2 `! d
done
! _, d& v+ l" c& `: \. \开启内核模块/ N; t* {5 O- z& n6 ?& L
临时开启
" G0 K" Z. [. v- B8 B% T( s, ?
0 E% t$ B5 |5 [! s1 V6 T/ {. ?for i in 192.168.91.19 192.168.91.20;do \
. S+ s: _$ a1 m) cssh $i "modprobe ip_vs"; \
+ m& X1 G- |; Z! yssh $i "modprobe ip_vs_rr"; \
+ Z7 M( ~$ P1 o3 b; Hssh $i "modprobe ip_vs_wrr"; \
4 o& K9 w% P& A' ussh $i "modprobe ip_vs_sh"; \
# D' `! q$ B1 G$ j% n4 Kssh $i "modprobe nf_conntrack"; \
) k/ j% \, {4 ^& E9 g9 b* g. ^+ ^ssh $i "modprobe nf_conntrack_ipv4"; \
% l4 x0 m' z2 Z/ A9 hssh $i "modprobe br_netfilter"; \
) R: o; ~; y/ e$ cssh $i "modprobe overlay"; \
5 c) E; _6 @6 @; Y$ {done
7 x: c/ R- k6 x) O永久开启$ {5 P/ @% S; A) E m( t$ k
: @0 ?( Y1 P" L. X/ p' {; |. A9 ?
vim /approot1/k8s/tmp/service/k8s-modules.conf
4 w1 G0 K( _$ r; |! S: rip_vs; l$ m, B2 S0 \; J$ w+ T
ip_vs_rr6 c, D$ O8 [+ L- E" ]8 g& C
ip_vs_wrr
( {+ [, p$ Y$ z5 a2 {0 m" cip_vs_sh F) ?3 J$ Q, p& |5 L* T) R
nf_conntrack
: F3 t' E2 h! [; `1 |nf_conntrack_ipv4
! ] M) b% o: `% D; V3 Qbr_netfilter
( l) D* S X2 k. boverlay
+ z# Y/ @& w2 f- U( ^! ~分发到所有节点9 G$ \/ O; b; A- X
for i in 192.168.91.19 192.168.91.20;do \
) \* @5 l- Q& h h/ xscp /approot1/k8s/tmp/service/k8s-modules.conf $i:/etc/modules-load.d/; \
) x' P3 {6 [( X- G/ hdone6 c+ F: T, n C) _' m H8 [
启用systemd自动加载模块服务
1 Q: q; W3 I: ]) M. z3 y( B, L4 \for i in 192.168.91.19 192.168.91.20;do \% X$ D- a% k& @ X d* @
ssh $i "systemctl enable systemd-modules-load"; \6 g- g1 H% V6 ~; i' ~
ssh $i "systemctl restart systemd-modules-load"; \7 ^3 f7 q: \' I. T5 {. x
ssh $i "systemctl is-active systemd-modules-load"; \, N( a( @: l- k. X* H; S
done
! I ]4 y {: J返回active表示 自动加载模块服务 启动成功8 U2 _' W! j7 ?' Y
3 P0 i; Z7 e9 Q6 e L/ P
配置系统参数
7 t0 m) J2 D/ U2 a5 e以下的参数适用于3.x和4.x系列的内核
4 O! v% @8 l2 ~; J1 \) F5 g7 E, s
7 @5 k2 p+ F# v; p( p) |vim /approot1/k8s/tmp/service/kubernetes.conf
4 m$ B( P( t6 |( s建议编辑之前，在 vim 里面先执行 :set paste ，避免复制进去的内容和文档的不一致，比如多了注释，或者语法对齐异常
7 X" i4 A1 a% F x H% t3 l) Z8 U0 y9 Y4 _' [
# 开启数据包转发功能（实现vxlan）. D2 l& c, ^* ?. ?
net.ipv4.ip_forward=1! r' i8 ~* q" A1 c* c& w2 S# j
# iptables对bridge的数据进行处理
) w: j( ?! l) V6 x3 Jnet.bridge.bridge-nf-call-iptables=12 O. O! w5 p8 j0 E
net.bridge.bridge-nf-call-ip6tables=15 }& G+ i, M' o" P; w" k" e2 _/ K
net.bridge.bridge-nf-call-arptables=1& a4 `/ f& Q& k) }$ A2 o
# 关闭tcp_tw_recycle，否则和NAT冲突，会导致服务不通0 @& `" D6 F- X% L( k
net.ipv4.tcp_tw_recycle=0+ d$ _7 ^7 l- h2 s; ?
# 不允许将TIME-WAIT sockets重新用于新的TCP连接$ \7 B; z& H' x' e
net.ipv4.tcp_tw_reuse=05 P$ R3 `+ G% G( n- D5 `/ g
# socket监听(listen)的backlog上限
* K g/ I6 n T( `/ N; pnet.core.somaxconn=32768
! a$ Y4 Q" L0 |6 c( j9 |# 最大跟踪连接数，默认 nf_conntrack_buckets * 4
6 y' M& m9 U* q* unet.netfilter.nf_conntrack_max=1000000' Y) q$ l4 x- s7 V5 a7 K. U
# 禁止使用 swap 空间，只有当系统 OOM 时才允许使用它5 c: E6 F! j" N! b
vm.swappiness=0
h% [5 `6 ], u: L, w h# 计算当前的内存映射文件数。
1 m \& X1 A% o" j# `6 ~) {: L; ]% q6 \vm.max_map_count=655360+ a0 h# M! _5 y7 m; N
# 内核可分配的最大文件数
* t6 s8 D1 G" ^3 u' Z& Q' Gfs.file-max=6553600. W) {3 G' ?/ z/ v" r' b
# 持久连接
7 j x2 I- d* @2 [* xnet.ipv4.tcp_keepalive_time=6005 A2 b# I6 W$ F z- w
net.ipv4.tcp_keepalive_intvl=300 U0 e4 Y4 Y. l& t2 i# m! ?" U$ G( J
net.ipv4.tcp_keepalive_probes=10% v" y2 I" p' Z5 H; g, V, r$ }
分发到所有节点
a# r( X o7 ^5 ^8 B( t( k: mfor i in 192.168.91.19 192.168.91.20;do \& Z: |3 j$ t; W
scp /approot1/k8s/tmp/service/kubernetes.conf $i:/etc/sysctl.d/; \
* y, J$ J/ F3 Q2 e# j3 o4 Tdone
3 ~/ J* ^& M3 |' K& G加载系统参数3 V1 z! o1 {3 H j* Q) V
for i in 192.168.91.19 192.168.91.20;do \
& z. x8 C# [1 A$ {( O( rssh $i "sysctl -p /etc/sysctl.d/kubernetes.conf"; \+ d1 {- B1 m5 a7 A9 ?0 {
done
9 S2 \7 k' E4 x; q! I3 p清空iptables规则. j7 j; F# W: k; K; i9 a' _5 ?
for i in 192.168.91.19 192.168.91.20;do \) }& d3 U, P1 B) Z. K6 L6 _
ssh $i "iptables -F && iptables -X && iptables -F -t nat && iptables -X -t nat"; \
& b- y/ l+ l( @& N( f: G% mssh $i "iptables -P FORWARD ACCEPT"; \# U8 B! D2 Y4 U3 ^4 V W; j" U
done
$ ^3 l9 j- b2 k% m2 Z4 k M配置 PATH 变量8 ~2 A) J! }, H# K2 i( a0 b
for i in 192.168.91.19 192.168.91.20;do \& m# |, \4 t& r" B6 N
ssh $i "echo 'PATH=$PATH:/approot1/k8s/bin' >> $HOME/.bashrc"; \: _5 B" A# R" d
done
6 f( y0 a# |, ?$ j2 m: }& |* d: u9 zsource $HOME/.bashrc3 c# D' y9 \1 r; o b+ r$ M
下载二进制文件
' c- |6 U, a: `3 j( |其中一台节点操作即可" w! I& B- g7 i8 ]" T( a$ s, I
1 ^7 Q1 v" p# K8 r
github下载会比较慢，可以从本地上传到 /approot1/k8s/pkg/ 目录下
& [2 }/ T( @6 S! n( z, }7 H: e
, ]5 Y" A. y2 ^) ]5 iwget -O /approot1/k8s/pkg/kubernetes.tar.gz \
/ ^; }9 Q ]6 ?, e* Ehttps://dl.k8s.io/v1.23.3/kubernetes-server-linux-amd64.tar.gz; C3 b* K* u( G1 y& Y7 z. \
) Y. E0 I. ^/ H" X* t8 H5 \
wget -O /approot1/k8s/pkg/etcd.tar.gz \6 ]) x1 {8 i d2 Q$ h( ~2 p
https://github.com/etcd-io/etcd/ ... -linux-amd64.tar.gz6 S& g1 U4 O* o' d, d& @, D
解压并删除不必要的文件; e# q p# m/ B' _5 j
5 }6 y- F$ f7 t5 w
cd /approot1/k8s/pkg/
/ L2 o* m6 \! ^& q; P3 o5 V7 S0 Z9 Ofor i in $(ls *.tar.gz);do tar xvf $i && rm -f $i;done# j5 \3 b. w) Z+ v
mv kubernetes/server/bin/ kubernetes/2 v3 w3 f R9 c6 @( G9 g
rm -rf kubernetes/{addons,kubernetes-src.tar.gz,LICENSES,server}& x6 b0 [$ a" E& [' m3 C8 ~6 @; _
rm -f kubernetes/bin/*_tag kubernetes/bin/*.tar
" D% r( {+ N4 c; F. S+ f% Z% Irm -rf etcd-v3.5.1-linux-amd64/Documentation etcd-v3.5.1-linux-amd64/*.md' p6 X$ n; V# O/ o0 w! [" G
部署 master 节点' P2 j+ _$ |8 i) d- l, Z+ A$ ]7 ~
创建 ca 根证书, l7 w* v, h2 G& u* s
wget -O /approot1/k8s/bin/cfssl https://github.com/cloudflare/cf ... l_1.6.1_linux_amd64
' f; j+ b! p# ^wget -O /approot1/k8s/bin/cfssljson https://github.com/cloudflare/cf ... n_1.6.1_linux_amd644 r! L8 h% `. ?2 x ]" c
chmod +x /approot1/k8s/bin/*# ?- u5 ^ E+ y. z
vim /approot1/k8s/tmp/ssl/ca-config.json% Z9 e' O. {1 X& I) F* c7 |
{
( D6 h6 X" y. }: x% E: ? "signing": {
# Z) d; G$ y' y0 N: j0 h' z" ]: u, W$ F "default": {. [" D# K- T7 t# L- N3 e
"expiry": "87600h"& \# ^3 [) e/ `1 P
},
. ~+ a; w, n+ }' O1 s "profiles": {
6 L [! |& k/ H; Q "kubernetes": {
, f8 I% h. A) f0 C* z; E "usages": [
7 R3 h; L' y* L/ V& A "signing",
- M4 e: P' {+ c+ l) g7 i "key encipherment",
8 B3 n8 f1 c- a "server auth",; P. V ]+ \2 B6 F) b$ S+ R
"client auth"6 p+ _) s" H' Q4 F7 c: i* t8 L6 I
],; V: U& a8 f+ T, o0 y! A
"expiry": "876000h"
9 M' N) E" ?& S4 T7 [; z }
+ b. L( U& [; J) I }$ j0 X$ A5 Q9 A# a1 O D, z
} S5 B$ K7 z: X% W
}
* \' T+ f x x$ [vim /approot1/k8s/tmp/ssl/ca-csr.json
5 D+ C( s% B+ C4 E{
9 e& `+ ]# a4 ?2 f* x "CN": "kubernetes",2 Q% D% k( [$ j* C! a E& A
"key": {
- _6 Q+ N! A+ ^9 x3 Y0 O! W "algo": "rsa",
0 K. n8 p- m. H% @' ` "size": 2048! a( Z O% U7 \# d
},
$ d/ a- c$ g% l$ t8 T! A1 P "names": [
( v% `* V& G0 q6 Z {1 G/ {0 a0 _; P. y
"C": "CN",
; r& o- n' h, o" }+ B "ST": "ShangHai",
0 @! @7 M" Z, Q+ K" V- @, k3 R: ^ "L": "ShangHai",
/ n- Q+ x" C& T+ w( k* g "O": "k8s",5 x2 I1 d5 K! ]2 z& g( z4 E
"OU": "System"
1 K0 N: k. g' L3 _" ~ }# ?1 Q, @/ L; h- |2 @
],5 F* d- w$ h4 U) i) S( f) f* L1 }
"ca": {* H$ S# O9 [# O+ {" A7 B
"expiry": "876000h"
% b( [+ O* N W) T8 h8 v- [ }
+ P% l A% l- }0 m3 O} N; T. U* j! ?! ?+ n2 [+ r0 k; z
cd /approot1/k8s/tmp/ssl/7 O8 w7 o' }% P+ g
cfssl gencert -initca ca-csr.json | cfssljson -bare ca- N( ~" r1 l5 T6 T0 y8 h/ d1 P; K
部署 etcd 组件: M; h7 _* q/ B% ]# P6 Y$ `
创建 etcd 证书$ I; p' c0 ^6 d$ F0 [3 i
vim /approot1/k8s/tmp/ssl/etcd-csr.json/ z% N" H9 Q) v
这里的192.168.91.19需要改成自己的ip，不要一股脑的复制黏贴
5 ~/ |; i& J$ m- f. b p3 ^. \+ e' l4 C5 S8 Q/ ^
注意json的格式
3 b2 @9 u& s& J5 G' n! h0 \7 A! R
# z+ M2 O! ]/ J! E8 o) o7 _/ j{# v2 ]2 [7 |3 V4 ]' f4 s
"CN": "etcd",7 ?& q) ]% }& x! Q/ ~: v# b
"hosts": [: F P; K5 L- b; |* Q7 v$ H
"127.0.0.1",
* S6 z y! C7 ^ "192.168.91.19"
* P* S3 b% H1 l/ j3 ]/ E ],
: T# i0 z6 e$ \! r; [$ m "key": {
6 X1 X. C B7 F5 P. T G: R "algo": "rsa",6 k1 j$ w s% K* {/ s$ d y. t
"size": 2048
# k ^$ G4 Z+ w5 w+ u7 y, [" N- _ },# V' f& g6 @1 ?
"names": [
( e1 c. R- m. U- l {8 F$ L% w; n6 w: M k( o3 a
"C": "CN",
$ y5 u# y W0 O& C "ST": "ShangHai",! ^4 n Y4 U. p" ]. h9 e
"L": "ShangHai",
0 a9 E; d( I1 ^7 p3 r; _) @2 g$ R( w: N "O": "k8s",# C/ M5 Z! |- c
"OU": "System"
% }- N Q9 k, C7 \# X }
9 M( m1 y+ M7 \3 k* {/ N9 j ]3 o& e6 O$ p3 q6 B E# F
}6 t& n; O% [$ K3 ^+ L( w, |! D- q
cd /approot1/k8s/tmp/ssl/4 T8 Y2 B, E& a
cfssl gencert -ca=ca.pem \9 G% _6 z6 r X a
-ca-key=ca-key.pem \2 f6 I! Y8 e" ]' b
-config=ca-config.json \& f8 b$ h7 K/ b1 X
-profile=kubernetes etcd-csr.json | cfssljson -bare etcd* W6 w5 H# o- Q
配置 etcd 为 systemctl 管理4 ? [6 t* o% b' J. ` G
vim /approot1/k8s/tmp/service/kube-etcd.service.192.168.91.19
) k& K! u* ]+ G- @. }" m& P5 D这里的192.168.91.19需要改成自己的ip，不要一股脑的复制黏贴4 H0 q* p. O& j8 k9 K6 }
8 _* V/ T, A% w4 K; f
etcd 参数
' R# T* }1 Q5 X- [( u- e1 s. Q& r% q, _2 H
[Unit]" e7 X D4 a. u6 C; o2 L
Description=Etcd Server4 y; a# F$ {1 ?% a
After=network.target; E; q' d' J( [/ g: }
After=network-online.target' I7 c2 r3 \( ?! y
Wants=network-online.target- Q9 A& e' x$ I9 T! G
Documentation=https://github.com/coreos0 ^% d" g0 V$ o, v
4 n7 x1 r# t# W# p8 r) H7 {
[Service]
7 ?$ ]$ y/ o) lType=notify
/ u' ^ w7 z6 }0 }, P+ p9 t2 BWorkingDirectory=/approot1/k8s/data/etcd
9 U) \8 t0 v. ]4 N! b: ]ExecStart=/approot1/k8s/bin/etcd \
) f! r6 [- g: z- H% t9 @; q --name=etcd-192.168.91.19 \
- t& {/ f) G2 }; w --cert-file=/etc/kubernetes/ssl/etcd.pem \- \4 z. S% j3 B: w9 s5 l
--key-file=/etc/kubernetes/ssl/etcd-key.pem \
6 R: L1 }5 I( l* \ --peer-cert-file=/etc/kubernetes/ssl/etcd.pem \
' E6 p. g# X1 n* [4 c( ] --peer-key-file=/etc/kubernetes/ssl/etcd-key.pem \
; S0 ?6 ?. ?5 b+ h% m- o; P3 ~2 i --trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
D% q( e, f1 a2 P/ K: r1 R --peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \
# n: A' O1 d6 a9 o1 S) }; t2 a% g --initial-advertise-peer-urls=https://192.168.91.19:2380 \
, u& k7 W, ]* n' z* x --listen-peer-urls=https://192.168.91.19:2380 \3 I. ~1 w, S/ _" N' L! R+ \
--listen-client-urls=https://192.168.91.19:2379,http://127.0.0.1:2379 \
! k2 s' [# U0 X% i5 ? --advertise-client-urls=https://192.168.91.19:2379 \' n9 G7 B5 _, H
--initial-cluster-token=etcd-cluster-0 \
$ ?/ { ]- @. _6 ] e7 q: } --initial-cluster=etcd-192.168.91.19=https://192.168.91.19:2380 \
2 x3 j1 W' b3 h2 P- X --initial-cluster-state=new \/ J |" q, V) c& u
--data-dir=/approot1/k8s/data/etcd \* _& p- p; _, e% I
--wal-dir= \
) Z$ S9 I+ V; U( g% t+ ~2 G --snapshot-count=50000 \0 m8 [# O" @( v1 Z, c( n' X
--auto-compaction-retention=1 \/ k" m2 i5 A* `4 z! @/ c4 |/ Z5 h7 ^
--auto-compaction-mode=periodic \
- d5 d: \3 @. j8 L --max-request-bytes=10485760 \
; l: E4 ]+ U( [/ l3 ^ --quota-backend-bytes=8589934592
8 G# N( A" F3 H7 m7 R/ S" ^Restart=always
0 K/ _/ x# K; m! N0 H& CRestartSec=15' K0 C9 J. A7 _6 X1 A
LimitNOFILE=65536
7 A u3 h$ x/ J, K$ oOOMScoreAdjust=-9995 k6 L/ |3 J! E6 j7 V
3 t/ j( w3 ?6 R& R9 `
[Install]
0 w% R' Q2 G, K; E A: FWantedBy=multi-user.target
9 H& i# @' e/ B5 Q0 }% C/ S' D分发证书以及创建相关路径
, H+ Z4 C) v( j9 Z% y1 T" b如果是多节点，只需要在192.168.91.19后面加上对应的ip即可，以空格为分隔，注意将192.168.91.19修改为自己的ip，切莫一股脑复制
' w" S; D% o. [, N" [+ M# S6 x: | z; ?; a5 N1 {$ j( @
对应的目录也要确保和自己规划的一致，如果和我的有不同，注意修改，否则服务会启动失败
# c6 _8 X& f5 T0 }, Q+ t
' ^8 h ~5 f0 M" C2 vfor i in 192.168.91.19;do \6 e! U, ?' O3 i( |2 ^
ssh $i "mkdir -p /etc/kubernetes/ssl"; \
1 ~* q' ]- e, s4 K3 tssh $i "mkdir -m 700 -p /approot1/k8s/data/etcd"; \
. l* ?, {! L# s2 Y5 f( z3 R( G, jssh $i "mkdir -p /approot1/k8s/bin"; \% r1 v% n' k; [ n J2 B
scp /approot1/k8s/tmp/ssl/{ca*.pem,etcd*.pem} $i:/etc/kubernetes/ssl/; \
* Y5 k( n# c) A |scp /approot1/k8s/tmp/service/kube-etcd.service.$i $i:/etc/systemd/system/kube-etcd.service; \
( Z; T5 J, C/ T9 pscp /approot1/k8s/pkg/etcd-v3.5.1-linux-amd64/etcd* $i:/approot1/k8s/bin/; \
% {9 {3 w4 \( wdone8 {* n6 I6 j8 s5 ]& ^
启动 etcd 服务- x& K( O- H, q: X$ f @
如果是多节点，只需要在192.168.91.19后面加上对应的ip即可，以空格为分隔，注意将192.168.91.19修改为自己的ip，切莫一股脑复制2 U' o5 F. Q' _/ y
1 R* m: I- J: S& h; r& Yfor i in 192.168.91.19;do \
$ i) z" p- H3 l# P0 d$ Cssh $i "systemctl daemon-reload"; \
3 T3 L m. }: K) t8 H8 W% n1 Jssh $i "systemctl enable kube-etcd"; \* I$ h5 M0 o, k, K( N) w% x
ssh $i "systemctl restart kube-etcd --no-block"; \
0 f4 R# Q+ }- N! Hssh $i "systemctl is-active kube-etcd"; \
, U, O* z9 h1 { _6 @done* Y+ T! t: E g9 S% [* @' U8 a: a6 F
返回 activating 表示 etcd 还在启动中，可以稍等一会，然后再执行 for i in 192.168.91.19;do ssh $i "systemctl is-active kube-etcd";done9 y& P) n( Z& h5 ^3 q. n3 ?4 k
8 k. K1 A3 R4 h4 T2 u1 n% W+ ^返回active表示 etcd 启动成功，如果是多节点 etcd ，其中一个没有返回active属于正常的，可以使用下面的方式来验证集群
8 J% d. d( X' r4 h! Z" Z) A% U0 g' L; d" w4 x
如果是多节点，只需要在192.168.91.19后面加上对应的ip即可，以空格为分隔，注意将192.168.91.19修改为自己的ip，切莫一股脑复制( _0 L% [% V2 [9 P! c6 I2 M' G
$ F6 S% [9 i% w/ @8 _" t0 rfor i in 192.168.91.19;do \! t3 M' Q0 d7 |+ \
ssh $i "ETCDCTL_API=3 /approot1/k8s/bin/etcdctl \
( h- S& d: Q9 m& x* m --endpoints=https://${i}:2379 \+ E& G; `& w$ n, v9 t3 c/ T
--cacert=/etc/kubernetes/ssl/ca.pem \) G. }# J r. J; D5 W- Y; p% r6 a
--cert=/etc/kubernetes/ssl/etcd.pem \
+ l! D1 w1 W" N# ]4 C0 ^' ^1 | --key=/etc/kubernetes/ssl/etcd-key.pem \2 w: m6 O( Q5 h0 X
endpoint health"; \
" Q4 d4 @/ D6 t7 t- v j" idone+ F! S/ I, }5 {, H; z) F% M" m
https://192.168.91.19:2379 is healthy: successfully committed proposal: took = 7.135668ms
) G- j# @, D0 V9 P n% y5 h! S* J4 y2 T- C4 i7 z, H
返回以上信息，并显示 successfully 表示节点是健康的" P( ?: O7 k6 ^' n$ k- B! d
: C9 D8 }* ?) g4 m* N- e# ]部署 apiserver 组件
# i: c$ ]+ _* W q- t4 K- X. i: W创建 apiserver 证书
5 ~; f% u+ x, Z5 dvim /approot1/k8s/tmp/ssl/kubernetes-csr.json
# ]/ K0 O- q7 S$ i& B$ ~2 |1 X+ M这里的192.168.91.19需要改成自己的ip，不要一股脑的复制黏贴+ |- u6 F2 G9 K* I3 r# i ~- o
* I1 ^: L B0 T, L- k
注意json的格式6 {2 H0 E& `" J! V/ Q
3 }3 ^' c+ s5 _- O$ l$ a9 a
10.88.0.1 是 k8s 的服务 ip，千万不要和现有的网络一致，避免出现冲突
. P6 u# _* b7 Q" B# R* V, i+ c9 O
{
& }" K: ]. L( w8 r8 I# F: D5 U "CN": "kubernetes",% L8 z" l3 I% U4 P; H
"hosts": [1 S4 E. T2 \# T0 i4 U; G3 M
"127.0.0.1",
7 Y! y3 z) U5 x7 [- z/ L6 x "192.168.91.19",
& r% u6 F2 f5 G7 g: c- \ "10.88.0.1",
& W. u2 a; l( C, a "kubernetes",
) y6 ?0 e% W/ H H- S "kubernetes.default",# p2 [; k# ]( a( Q" [* y& o5 U) T
"kubernetes.default.svc",( X4 T. D+ f' [: ?/ g8 U
"kubernetes.default.svc.cluster",0 h7 [3 q" F2 w& m
"kubernetes.default.svc.cluster.local"$ \, c- }6 N, e" g$ P
],
' ]% Y3 J" z6 T! R( @$ a "key": {
4 D" }7 g0 ~1 ` "algo": "rsa",
! P" @) i/ B# l+ C1 y |, X4 q "size": 2048
* `' J7 ^' B4 { },
5 w: v, ^0 x* {7 l& l0 N "names": [8 Q. i8 ?# ]3 W& S
{
4 C3 ~' F7 L8 t9 `7 \3 ]6 p' ]! |7 S; t "C": "CN",+ B$ ?7 F# Z1 m; Z$ {+ G- Y4 H. F' V1 K
"ST": "ShangHai",5 D w. Q- Q% {2 w* t
"L": "ShangHai",# E1 G7 }# s2 b/ F' c- u
"O": "k8s",
4 D# c- [$ ~8 y8 o: _" b "OU": "System"
1 z# ?6 i' J# A' ~+ H, _# v }! H, r4 j, J+ z# x3 S
]
2 Z, E- f/ t* D7 l3 K}. u7 \: A) x E6 K* Q) Z0 W
cd /approot1/k8s/tmp/ssl/! F8 l7 u/ I1 b" ?& n9 T% M
cfssl gencert -ca=ca.pem \
/ [3 [2 T' u: P" X& v; J-ca-key=ca-key.pem \3 P" O7 Z: s* t5 ?" y* }
-config=ca-config.json \7 V! V/ }6 F) b
-profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes
. v7 S5 d1 W n5 n! d3 s创建 metrics-server 证书
% h8 F; [& a8 m: Z- x9 Xvim /approot1/k8s/tmp/ssl/metrics-server-csr.json
, u6 D: M j: u. u; ~* F, C{3 Y& |% H5 J$ z+ w; n
"CN": "aggregator",& d) S+ k* R% o
"hosts": [& X8 A2 x. f z9 Q
],
7 i/ x* _ I+ f* }" j "key": {
/ e; R/ c. F* r0 A( P1 N "algo": "rsa",4 W' E. n" O2 C# g0 ?8 R1 q
"size": 2048# ^9 F7 E5 E0 M4 [ Y. J: z" z
},
" q& P) c$ F$ S* S' U$ f "names": [
K& |9 M2 w- I: }) Z {
& N+ H2 Y' b. a- Y3 \ "C": "CN",
- d- `8 k$ o0 S6 i7 {- W+ b6 x "ST": "ShangHai",
+ _. c' m X, L8 e/ f "L": "ShangHai", S7 Y# T3 v; [3 S: k6 w
"O": "k8s",
# b6 f7 O( r0 a7 K4 c1 | "OU": "System"1 c+ l( }. C; d) y
}
% i' f5 O" j1 S5 K- G( \6 G5 @' t ]* \6 K4 T/ g/ U1 T5 ]$ B
}* B7 Y3 _9 E) H3 |
cd /approot1/k8s/tmp/ssl/3 g5 u7 i5 A8 @ V$ E, [
cfssl gencert -ca=ca.pem \$ l4 k1 E1 [ `
-ca-key=ca-key.pem \
* Y3 v( F3 E% [3 E$ n1 L% U-config=ca-config.json \1 ^6 m9 l1 w, Q; `7 B8 M
-profile=kubernetes metrics-server-csr.json | cfssljson -bare metrics-server
, U* E, G4 q8 s1 E配置 apiserver 为 systemctl 管理
+ i. @' q0 d$ Mvim /approot1/k8s/tmp/service/kube-apiserver.service.192.168.91.19) a/ j, R D5 g) t/ N( L9 F
这里的192.168.91.19需要改成自己的ip，不要一股脑的复制黏贴# {" e' P: q; u9 }- a5 X9 v
+ J7 Z w5 H1 L' B--service-cluster-ip-range 参数的 ip 网段要和 kubernetes-csr.json 里面的 10.88.0.1 是一个网段的: y2 x3 a) j- Q" S- K* k
8 S, ]1 J+ f9 @7 k0 E. q5 L: _: m
--etcd-servers 如果 etcd 是多节点的，这里要写上所有的 etcd 节点 y. g5 N( T( ~7 v. T) |
2 r- I) T. |! G3 U) g$ ]
apiserver 参数1 q: L. t2 P( p: q4 n" ]6 b7 {
n2 Z' v6 h/ L$ V- P3 ~! P) U
[Unit]/ Q' x T& v! Y! y
Description=Kubernetes API Server
m) z7 R; V8 J3 H- ~Documentation=https://github.com/GoogleCloudPlatform/kubernetes' `( c6 y( f2 \: E: ^+ B
After=network.target3 O! W+ ~( c/ M8 _" E6 Q; j. V# j
/ R: x# J! ~* ?' c% X+ i[Service]1 w% @" E5 U3 X$ s% |' I
ExecStart=/approot1/k8s/bin/kube-apiserver \6 H/ C' X* `+ |( @) T- m# K0 m
--allow-privileged=true \+ [( {2 ^/ c ?9 F4 {. H! f* o
--anonymous-auth=false \
; N9 ^9 u! \* r$ z y: P/ S --api-audiences=api,istio-ca \
6 ^; ?" q. d7 Q9 R2 v# v" T2 ^ --authorization-mode=Node,RBAC \. p* W. L! N6 h0 i' C1 A
--bind-address=192.168.91.19 \+ e% Y G) G" u; Q- [/ L
--client-ca-file=/etc/kubernetes/ssl/ca.pem \
7 `$ [& l, B5 H, `- D --endpoint-reconciler-type=lease \. L/ [' E& r. {% m
--etcd-cafile=/etc/kubernetes/ssl/ca.pem \
+ o, N4 e6 f. q6 x6 R' Q+ d --etcd-certfile=/etc/kubernetes/ssl/kubernetes.pem \0 s- {7 r6 v3 D# i! Y5 g
--etcd-keyfile=/etc/kubernetes/ssl/kubernetes-key.pem \
" |# K! B3 S. [5 G6 A --etcd-servers=https://192.168.91.19:2379 \
1 L7 I" D0 u0 g; d --kubelet-certificate-authority=/etc/kubernetes/ssl/ca.pem \
: W9 d) E1 N! ]( f) K --kubelet-client-certificate=/etc/kubernetes/ssl/kubernetes.pem \# |0 F4 e& \8 H( r
--kubelet-client-key=/etc/kubernetes/ssl/kubernetes-key.pem \2 [ u/ k3 E. l0 p) ~
--secure-port=6443 \
% k% G: ` e& J) W" Z0 h9 } --service-account-issuer=https://kubernetes.default.svc \
) O( v; z& D3 u$ Q& x- g G --service-account-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \
& Y) O; c/ f* z, ^ --service-account-key-file=/etc/kubernetes/ssl/ca.pem \* p: n0 T8 O) B3 F1 \
--service-cluster-ip-range=10.88.0.0/16 \- Q6 k4 ?# _- o7 p y0 R. t
--service-node-port-range=30000-32767 \
9 o5 }6 u8 T' _5 C --tls-cert-file=/etc/kubernetes/ssl/kubernetes.pem \( W: O$ O8 o" z9 t; _; l
--tls-private-key-file=/etc/kubernetes/ssl/kubernetes-key.pem \( x2 e5 O6 S8 y* l: \
--requestheader-client-ca-file=/etc/kubernetes/ssl/ca.pem \
8 ?% L& |2 N4 N6 m --requestheader-allowed-names= \
! P, v1 I( t& Q --requestheader-extra-headers-prefix=X-Remote-Extra- \: V% J$ f% ], b. }1 [. F
--requestheader-group-headers=X-Remote-Group \
9 ~/ Q8 Q; K+ }0 K/ \0 X; C --requestheader-username-headers=X-Remote-User \1 H P8 N+ j; X+ E2 `/ ]& A
--proxy-client-cert-file=/etc/kubernetes/ssl/metrics-server.pem \
1 @. h* J/ x! `: ]5 ^ --proxy-client-key-file=/etc/kubernetes/ssl/metrics-server-key.pem \
; D2 i; L3 e7 ~ e* y --enable-aggregator-routing=true \
' a3 a6 } X0 F) l; k6 ] --v=2
7 H$ R y. C" F3 C% `. n& }# MRestart=always1 B ?3 B- [) T* Y# Q6 s$ X
RestartSec=56 u5 P# m( |! C& s) z
Type=notify
2 U! K( b c, FLimitNOFILE=655368 a! B, c9 e" y8 d- c
: Q3 j9 j; u& N, y+ N; @3 V1 P
[Install]$ r2 T; {9 X; P6 Q/ i H8 o8 W& R
WantedBy=multi-user.target
( l% |8 N6 }" ]/ h8 J6 w$ p分发证书以及创建相关路径. x" K) O6 N+ N, W0 }) k5 _& i
如果是多节点，只需要在192.168.91.19后面加上对应的ip即可，以空格为分隔，注意将192.168.91.19修改为自己的ip，切莫一股脑复制) s5 r- F. F0 v! W+ I
: W. |8 M0 ?' ~, f
对应的目录也要确保和自己规划的一致，如果和我的有不同，注意修改，否则服务会启动失败0 V% g7 i1 R# |% T* ?
' f, k: I4 i1 v3 v7 Ifor i in 192.168.91.19;do \. s% F9 K8 ]" s1 K5 P
ssh $i "mkdir -p /etc/kubernetes/ssl"; \; D1 K/ a: Q; K! i& E* Z$ O
ssh $i "mkdir -p /approot1/k8s/bin"; \$ @- R/ ?: z* u
scp /approot1/k8s/tmp/ssl/{ca*.pem,kubernetes*.pem,metrics-server*.pem} $i:/etc/kubernetes/ssl/; \, G( c/ R8 T! D; e+ f p3 F$ b, [; Q0 R
scp /approot1/k8s/tmp/service/kube-apiserver.service.$i $i:/etc/systemd/system/kube-apiserver.service; \6 D8 A! M# A: A' b8 c
scp /approot1/k8s/pkg/kubernetes/bin/kube-apiserver $i:/approot1/k8s/bin/; \
% S% X7 {, L! Q1 r3 y( udone
' _8 ~7 l8 w, F$ C7 L启动 apiserver 服务6 r/ @0 N5 r+ x( h
如果是多节点，只需要在192.168.91.19后面加上对应的ip即可，以空格为分隔，注意将192.168.91.19修改为自己的ip，切莫一股脑复制1 y4 d2 }( x: N" j. L
: m2 ^- @! }; o4 S+ B4 Vfor i in 192.168.91.19;do \' ~7 C z5 i, C) g# I7 D
ssh $i "systemctl daemon-reload"; \8 r s0 X) v! B& ^8 r2 }0 q7 Z
ssh $i "systemctl enable kube-apiserver"; \
4 I1 G. E. d0 A# M2 I$ P/ y, X6 b1 Zssh $i "systemctl restart kube-apiserver --no-block"; \" I U- W1 D2 _3 W9 V
ssh $i "systemctl is-active kube-apiserver"; \
# v# M' i& B( m9 Jdone% a3 E' I5 |: B2 X0 Z
返回 activating 表示 apiserver 还在启动中，可以稍等一会，然后再执行 for i in 192.168.91.19;do ssh $i "systemctl is-active kube-apiserver";done
4 Y0 b4 U( }% X3 C. I/ W9 x; Q& t3 \* k
返回active表示 apiserver 启动成功- \3 Q# v, C- O) q
9 @* ^4 f' X4 Q% V; G0 x6 R5 t# _
curl -k --cacert /etc/kubernetes/ssl/ca.pem \
+ P8 h/ C: s! s1 G' ?, a--cert /etc/kubernetes/ssl/kubernetes.pem \
4 m" V5 J* @; u- F6 u--key /etc/kubernetes/ssl/kubernetes-key.pem \
: y5 @/ `+ n* H; dhttps://192.168.91.19:6443/api+ {9 p6 A$ a; S. _6 F# | Z
正常返回如下信息，说明 apiserver 服务运行正常
8 L \1 ~) @. L2 \) \" {+ v0 R8 x# i
. ?1 q9 w) [3 y% ?" ]; @{
3 y* e* C$ ]2 O6 ?, `2 ?$ j- r "kind": "APIVersions",& D* C. T. F8 s$ x0 F3 z8 v- H. e* u
"versions": [! u" }9 a2 S; t3 l! q( A M
"v1"& O/ w d* z1 O" X% e2 G: e
],) I. V+ K! e. N$ f) g# S0 F
"serverAddressByClientCIDRs": [
, \; I* n& h9 T- ~7 J {
% g( E E% R9 ~$ @ "clientCIDR": "0.0.0.0/0",
K; H( G8 u4 v J$ X3 f "serverAddress": "192.168.91.19:6443"
5 c' K9 U& J& v. U* J' M }8 E" R7 `. m* J; ^6 j" W, \ u9 B
]
& U) m* [- {, D}
; d, D: W+ `* B) R查看 k8s 的所有 kind （对象类别）
! x- t2 Y- N( s/ b I
Y8 \" X2 X qcurl -s -k --cacert /etc/kubernetes/ssl/ca.pem \6 T( W- B6 r# o' F
--cert /etc/kubernetes/ssl/kubernetes.pem \, E7 j! f. o2 U) `' X+ u9 [! q! }
--key /etc/kubernetes/ssl/kubernetes-key.pem \' \+ L3 B# K) ]$ p$ U2 E, I% N
https://192.168.91.19:6443/api/v1/ | grep kind | sort -u7 ~7 u9 c) J) h& d
"kind": "APIResourceList", I) \, \" ~1 _, P
"kind": "Binding",
6 w2 y$ N6 m6 S5 V" Q "kind": "ComponentStatus",
+ d# S+ ~& e8 v O$ ` "kind": "ConfigMap",
: v; I' B* t+ r! ] "kind": "Endpoints",3 ?* G9 r0 Z: n& I
"kind": "Event",! S+ E: B$ i3 b9 f% u; L, Z0 M
"kind": "Eviction",* ^% }; n2 o0 f: Y' W$ c* x0 w
"kind": "LimitRange",
8 S, L& H1 I% x$ D- M9 H "kind": "Namespace",
) P U, x) l+ ^7 B "kind": "Node",6 r6 ~5 u4 o. `% i% w8 k
"kind": "NodeProxyOptions",3 K/ V: M0 H4 {4 K* ^2 }7 Z- j1 |
"kind": "PersistentVolume",& M/ a# m a: I" U0 U
"kind": "PersistentVolumeClaim",- \# w9 G4 T: D1 E
"kind": "Pod",
8 S( [ k- M8 h5 k R& |6 R8 I "kind": "PodAttachOptions",
" g8 y6 ^3 X2 m& `0 E "kind": "PodExecOptions",
7 |" }! J% f% n6 r. T "kind": "PodPortForwardOptions",/ b( {2 `9 E/ _! U; o5 u, N) C
"kind": "PodProxyOptions",9 H4 d9 s. {5 @; s
"kind": "PodTemplate",
7 ~, O; i4 {' ]2 y2 B. p( f "kind": "ReplicationController",+ A W+ X* r3 B5 h4 J
"kind": "ResourceQuota",, I1 h' J. q) s# _6 p1 k ~
"kind": "Scale",
M) [. F# v0 v* S4 y- U1 U "kind": "Secret",
8 G8 q" z6 z* l: i9 T& [; J" t "kind": "Service",- q3 p1 P% o' _( l$ ?6 B
"kind": "ServiceAccount",
* a, C+ v Z( X* m "kind": "ServiceProxyOptions",( U: _" }+ L$ ~5 K, Z: v
"kind": "TokenRequest",9 q+ k1 S4 U2 i" O+ e! K* {+ x$ K% H
配置 kubectl 管理
) m9 y- k* W# j. z创建 admin 证书4 G2 P2 y% e0 _4 Z& c0 y4 p
vim /approot1/k8s/tmp/ssl/admin-csr.json' v2 ]" P D* a' N) m
{
7 Z& i& k8 J ?0 G$ s3 S7 k+ o "CN": "admin",
" w. j8 [7 X/ v! G "hosts": [: Z* I- u8 V( O. u& r
],
g9 b9 @3 j9 }) ~+ h5 Q$ G! f "key": {( X R* Z" k; v4 Q
"algo": "rsa",3 m2 G6 }/ w) D9 y
"size": 20480 t2 Y7 ^6 A' p" P; a( w: R
},7 c1 P$ L' m8 m' [$ I. E
"names": [
|3 Z. _6 V7 W$ B' v" Q {
7 ]+ a5 `' h* a. u7 Q4 k( K: t "C": "CN",
% y( v' e6 V/ a/ [, H4 v1 A" U "ST": "ShangHai",6 c; U* q- l6 o8 |
"L": "ShangHai",& R; {# t, p! h* I. }4 E
"O": "system:masters",
' u4 i7 [* G: a/ J& |, J& x3 { "OU": "System"
2 o w: m5 @, O" i }
6 r; h0 D8 K% X* I9 H ]& {. N1 R: K; h5 D2 r5 r
}
- _- p7 i6 f- r% | icd /approot1/k8s/tmp/ssl/
, j# L3 J, h7 c0 s% _cfssl gencert -ca=ca.pem \
& ~+ ]5 [1 ^: b$ p( @ V-ca-key=ca-key.pem \
* Z( y& T$ |: w% Q-config=ca-config.json \# @( m! [! b6 T
-profile=kubernetes admin-csr.json | cfssljson -bare admin/ c* v3 M# ?" L2 c7 l, E
创建 kubeconfig 证书) k# I0 { t; T* u! @
设置集群参数0 A9 R0 n6 |( D8 f% W+ k* c
/ \6 b! Q) @" y/ `4 j--server 为 apiserver 的访问地址，修改成自己的 ip 地址和 service 文件里面指定的 --secure-port 参数的端口，切记，一定要带上https:// 协议，否则生成的证书，kubectl 命令访问不到 apiserver e" T; E5 N) t8 M! D" u9 u) M
[3 ~9 n; a- X$ X# L
cd /approot1/k8s/tmp/ssl/$ L; ~5 R$ M( ~7 | f
/approot1/k8s/pkg/kubernetes/bin/kubectl config set-cluster kubernetes \" H% B1 i$ a$ B3 a
--certificate-authority=ca.pem \
7 C X: ~. j4 j3 P--embed-certs=true \
' x/ m5 L( A/ M- U! w; u--server=https://192.168.91.19:6443 \) o! \+ i6 m+ S" \7 }% P
--kubeconfig=kubectl.kubeconfig: I6 H$ |6 o1 k; x/ q
设置客户端认证参数
8 e V" o B: ~/ {3 ~
; x4 S# r5 R+ E% ncd /approot1/k8s/tmp/ssl/
" H9 ^4 t; J) X+ B( i/approot1/k8s/pkg/kubernetes/bin/kubectl config set-credentials admin \/ |7 B6 i) h2 Y
--client-certificate=admin.pem \; N$ w% W$ t! p+ B
--client-key=admin-key.pem \, i. B5 z- M0 ]1 @# Y4 E: L% N5 R
--embed-certs=true \
1 y& N5 k* M1 p( c6 ~* V ]/ Q3 r* `--kubeconfig=kubectl.kubeconfig+ c& n1 e4 ~) y! s0 D
设置上下文参数
2 d; q8 S t" U# B |! _0 p% @/ J3 p* h- |
cd /approot1/k8s/tmp/ssl/
5 m' E( ] A$ w/approot1/k8s/pkg/kubernetes/bin/kubectl config set-context kubernetes \
; Y3 i- I% z# W2 p# u--cluster=kubernetes \$ K, E- Z+ }; X, f, A# x) w. R
--user=admin \ J. X* P. v/ I
--kubeconfig=kubectl.kubeconfig
) q5 _; v$ G- C9 i0 N设置默认上下文: h9 }3 J* U F* d' @( ]
* k( D/ i" j2 _; n) |cd /approot1/k8s/tmp/ssl/8 ?' k8 O/ w6 _7 k
/approot1/k8s/pkg/kubernetes/bin/kubectl config use-context kubernetes --kubeconfig=kubectl.kubeconfig
9 z& q; G# l# b5 @$ e" L) \. L分发 kubeconfig 证书到所有 master 节点$ p% G. f% x$ N7 ~% f- n/ U
如果是多节点，只需要在192.168.91.19后面加上对应的ip即可，以空格为分隔，注意将192.168.91.19修改为自己的ip，切莫一股脑复制# I0 s- j% V4 ~9 ~) Z A1 g) r
1 }% C3 o, Y1 I6 O. Z
for i in 192.168.91.19;do \, A" E& _0 r+ B" [" _
ssh $i "mkdir -p /etc/kubernetes/ssl"; \
& v# A8 R. R; R1 J3 kssh $i "mkdir -p /approot1/k8s/bin"; \
4 K7 T9 U" g& Mssh $i "mkdir -p $HOME/.kube"; \
6 W1 ` b% e$ i5 P8 Wscp /approot1/k8s/pkg/kubernetes/bin/kubectl $i:/approot1/k8s/bin/; \
5 i" f H- e1 w- E' R# t& Jssh $i "echo 'source <(kubectl completion bash)' >> $HOME/.bashrc"
# q3 O6 e" k0 t* G/ g2 J: D! c" o; h6 M: Oscp /approot1/k8s/tmp/ssl/kubectl.kubeconfig $i:$HOME/.kube/config; \
$ R; J; K6 Q M2 v8 S' R$ rdone
. m. v, S! k/ r) G* z# ?. W部署 controller-manager 组件0 i0 p0 c0 |2 u! P7 h5 G# }: U
创建 controller-manager 证书
9 \0 D/ u/ {$ J9 \/ Svim /approot1/k8s/tmp/ssl/kube-controller-manager-csr.json: r( T' j9 d% Y w
这里的192.168.91.19需要改成自己的ip，不要一股脑的复制黏贴) M/ T6 Y' r8 p
+ C" m) G$ a) f4 N" ]
注意json的格式" z+ P% Y, R% p2 k9 _' T7 u- L
, k7 M( v, T1 [/ F3 g9 X$ f1 {% X1 ]{6 |. `, _; ^7 b
"CN": "system:kube-controller-manager",
) J. C0 A+ }5 a* j% A8 s "key": {3 j' @" `6 F; x2 r. j/ F
"algo": "rsa",5 _0 t: M: c3 k% `+ ~3 q
"size": 2048" _ g# _: W, \: F
},% `' Z& ?# C3 b/ F
"hosts": [' {4 c5 q8 L0 a: q5 o- H* `, T& G, I! q
"127.0.0.1",5 C; P# B0 S3 ?' E" j+ F. J
"192.168.91.19"
( y% J! z& ]; S% j7 o ],
5 b. d$ g; d- ^( b$ h "names": [
. ~9 c* q9 H/ x9 B7 i {
: a! c N7 j( n; s* }) a c' o7 M "C": "CN",
: T$ y# ~" @: [! O# c9 W "ST": "ShangHai",. @* H) ~- j, J/ s3 G
"L": "ShangHai",& w& M; f }$ S1 O1 |2 y
"O": "system:kube-controller-manager",; m% T: L# l6 b8 m
"OU": "System"
. _& i0 W" H4 z5 g( Y/ } }' q, f0 V9 r8 d/ |) ~, v
]0 e5 v+ D" ~2 p' u) |( t
}2 B5 o. N/ \8 U1 d: f
cd /approot1/k8s/tmp/ssl/: e R* |5 D$ S. a' c) ^
cfssl gencert -ca=ca.pem \: R7 S( a! j1 q
-ca-key=ca-key.pem \5 M2 q* Y+ v( m% r% m! n: |
-config=ca-config.json \. e' C5 {' f. f8 \$ `
-profile=kubernetes kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager
! C* x4 O, W' b1 h# q) x2 n创建 kubeconfig 证书* v5 u& z7 h' y, t
设置集群参数* M* I7 Y+ M* F4 [
& ], a* Q' C2 z--server 为 apiserver 的访问地址，修改成自己的 ip 地址和 service 文件里面指定的 --secure-port 参数的端口，切记，一定要带上https:// 协议，否则生成的证书，kubectl 命令访问不到 apiserver, H; a' i a3 n, h3 g
7 y0 g* C7 L4 t! U# z3 B
cd /approot1/k8s/tmp/ssl/# f/ d( _9 n0 x- \
/approot1/k8s/pkg/kubernetes/bin/kubectl config set-cluster kubernetes \
' l0 t5 R! }, f4 a6 ~9 f, e9 M--certificate-authority=ca.pem \
|) r$ Z, Q. E' g( T8 R. K--embed-certs=true \
, i5 V) L- e7 r* p--server=https://192.168.91.19:6443 \) ?% {2 G9 y+ @# h) h! t% @. z6 Q
--kubeconfig=kube-controller-manager.kubeconfig
4 E0 Q% u- P7 ~" j# |# b+ F: L1 u设置客户端认证参数% ~1 U3 D/ _6 w* P5 @2 M0 D
$ e. G2 r2 M- q- z; J+ Ocd /approot1/k8s/tmp/ssl/' I- ~; i! ^# N) e9 N# Z- s
/approot1/k8s/pkg/kubernetes/bin/kubectl config set-credentials system:kube-controller-manager \ N& R2 O% u9 h
--client-certificate=kube-controller-manager.pem \. }; _1 w3 D- k
--client-key=kube-controller-manager-key.pem \4 Z9 r- b7 S1 Y. ?- }# w% N
--embed-certs=true \3 ?8 O c; R, d6 {: r( w
--kubeconfig=kube-controller-manager.kubeconfig$ p6 M+ ]9 f6 h8 o- l! E1 B$ H
设置上下文参数' p0 F, _, e# O! H$ F5 V
- z. v; N; N2 e6 F& u1 wcd /approot1/k8s/tmp/ssl/
, a$ ~" b) }" j/approot1/k8s/pkg/kubernetes/bin/kubectl config set-context system:kube-controller-manager \6 I0 ?8 k, ]6 t3 Q3 ]4 o( p+ H& l
--cluster=kubernetes \) ]0 J l4 W+ u V3 l
--user=system:kube-controller-manager \
3 b/ |3 |8 T# ~1 D--kubeconfig=kube-controller-manager.kubeconfig- J8 H9 Y& l+ ], K7 a
设置默认上下文
- M& D+ r3 w; G7 x+ q- ?0 l
! V8 P. W0 _# _0 x8 x7 jcd /approot1/k8s/tmp/ssl/1 l( Z( ]2 t K+ \+ g* o$ ~
/approot1/k8s/pkg/kubernetes/bin/kubectl config \& K5 b# z0 t8 E- t( l5 l/ v
use-context system:kube-controller-manager \; `! x' {7 l9 l! Z* ^
--kubeconfig=kube-controller-manager.kubeconfig" W+ Y- `, y! |$ w
配置 controller-manager 为 systemctl 管理$ b5 e6 Q' e/ a9 k' S$ W# ~: D
vim /approot1/k8s/tmp/service/kube-controller-manager.service
. |" J9 M& E8 j" f+ F这里的192.168.91.19需要改成自己的ip，不要一股脑的复制黏贴, Z) G: g7 e) n9 Q. ^: ^
7 Q5 n* Y" [/ M: J. r/ }6 j" b--service-cluster-ip-range 参数的 ip 网段要和 kubernetes-csr.json 里面的 10.88.0.1 是一个网段的
0 m/ q5 R) w/ k5 @/ }! v4 \& v8 x: r; L+ V1 }( D: o
--cluster-cidr 为 pod 运行的网段，要和 --service-cluster-ip-range 参数的网段以及现有的网络不一致，避免出现冲突
" k, s" b' B j8 C' K( N% R
$ C& `- e1 B5 u* i0 N2 Ncontroller-manager 参数
: R4 `) r7 V# z1 l* U$ m/ y5 P7 ~3 c
( E" s* F/ z0 w( ~% e- Q4 J[Unit]
8 f Y6 h/ B$ s4 K2 n7 zDescription=Kubernetes Controller Manager0 E& L: c8 d" J3 H
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
1 B7 K# {: t0 n* F9 I7 P, f1 V; L7 ~
[Service]) c- @7 A) ~0 _. I! s- ?. @
ExecStart=/approot1/k8s/bin/kube-controller-manager \
7 h$ N1 M+ L' G K) ]9 { --bind-address=0.0.0.0 \
6 ^: ]6 k8 F1 ]8 ]1 f --allocate-node-cidrs=true \
: T" o7 v$ u5 l( ^; @- y' w* e% L --cluster-cidr=172.20.0.0/16 \
- d0 a" t$ z. m0 i0 z6 c- g5 v --cluster-name=kubernetes \5 c1 r* H; c9 y/ A3 ~$ N! V; n
--cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \ k* T$ e2 @. s( H
--cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \
$ P; k- `. w" H' D2 j+ \ --kubeconfig=/etc/kubernetes/kube-controller-manager.kubeconfig \
2 O$ p( z9 G6 @1 ^# B --leader-elect=true \' A! ~% o( `2 Z4 w
--node-cidr-mask-size=24 \& V$ ]9 s6 q$ I, ?" M( p
--root-ca-file=/etc/kubernetes/ssl/ca.pem \% l; o8 G6 d z1 x
--service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem \
! \, k" y# d' p1 l" d/ e" F! e --service-cluster-ip-range=10.88.0.0/16 \ j' ?2 C( u6 p, A2 {0 ?5 V
--use-service-account-credentials=true \
7 K) a4 c4 _& g. ^6 W6 J --v=2
5 n" ]" M: F. y9 s9 RRestart=always3 q6 x V$ }% n2 D& g
RestartSec=55 {$ U) C* \. S) T* F8 B, C1 L
9 L B$ }7 A- _$ M; i2 o[Install]
! J( S* f- q/ p# d, x" Y' {% ~WantedBy=multi-user.target
$ o, ]% I3 p! h5 b5 ^分发证书以及创建相关路径- D: c* ^# ]% a# K+ K1 P. @
如果是多节点，只需要在192.168.91.19后面加上对应的ip即可，以空格为分隔，注意将192.168.91.19修改为自己的ip，切莫一股脑复制" A0 e9 `3 @' e% t1 E
6 t% Q% a, x: \5 R6 f
对应的目录也要确保和自己规划的一致，如果和我的有不同，注意修改，否则服务会启动失败
& i0 W1 d2 Z' k6 g" T6 m1 {
: T! }% R; Q% X" L, U v. Vfor i in 192.168.91.19;do \0 A* j9 S- y2 |
ssh $i "mkdir -p /etc/kubernetes/ssl"; \1 J. y1 [# E9 K* L# q( |
ssh $i "mkdir -p /approot1/k8s/bin"; \
. J* o6 m' {, c, p+ M m% Jscp /approot1/k8s/tmp/ssl/kube-controller-manager.kubeconfig $i:/etc/kubernetes/; \
# ~* d% o) N7 M! q6 L6 j* a6 Nscp /approot1/k8s/tmp/ssl/ca*.pem $i:/etc/kubernetes/ssl/; \5 B+ x/ @' C' ?2 @4 j- q
scp /approot1/k8s/tmp/service/kube-controller-manager.service $i:/etc/systemd/system/; \
2 T" n8 u: V3 ^5 [scp /approot1/k8s/pkg/kubernetes/bin/kube-controller-manager $i:/approot1/k8s/bin/; \) h4 J4 g7 c, _; U5 D" z# @
done# E: h! J) `0 _
启动 controller-manager 服务# ]) S. {; M" q% Y4 F( h
如果是多节点，只需要在192.168.91.19后面加上对应的ip即可，以空格为分隔，注意将192.168.91.19修改为自己的ip，切莫一股脑复制
3 Q2 e9 R1 W% n }2 g
$ q' W7 |, X5 j; A9 V: ffor i in 192.168.91.19;do \
9 O$ b, V( ]* r* A4 Dssh $i "systemctl daemon-reload"; \
" x: S, E; L4 Q) W9 ~6 wssh $i "systemctl enable kube-controller-manager"; \' Z9 A& I+ T$ m% S
ssh $i "systemctl restart kube-controller-manager --no-block"; \
9 s$ N8 Q; L. ?% \- kssh $i "systemctl is-active kube-controller-manager"; \
: l/ s" W2 c5 q# b9 }; n! _' _done
' A1 q# O+ o. @& B' f返回 activating 表示 controller-manager 还在启动中，可以稍等一会，然后再执行 for i in 192.168.91.19;do ssh $i "systemctl is-active kube-controller-manager";done
" E8 v4 h( P4 O! e" [: B/ D7 y1 o, T' i
返回active表示 controller-manager 启动成功* ?) s8 d/ |/ d. `
# Y5 F& P' P% z6 V部署 scheduler 组件; r% i9 m- X8 w; d+ D
创建 scheduler 证书
) D& f2 [- ~& {vim /approot1/k8s/tmp/ssl/kube-scheduler-csr.json$ X- W$ d# z; G2 L8 [) e) Q
这里的192.168.91.19需要改成自己的ip，不要一股脑的复制黏贴
; ^; R# y! G( A. G8 p' ^9 ^" B) ]" {0 o! m5 W1 H
注意json的格式
A* S Y5 e, }* N+ ~
$ X" v+ B% ?1 \; C{
( y9 h2 M1 r5 b8 \, [. H9 B" Y "CN": "system:kube-scheduler",% ?/ m9 a" W7 Q: Q
"key": {+ J6 y! g* J, l$ A" R$ x, P0 ~
"algo": "rsa",
z2 h6 i6 H7 L% g* ~ "size": 2048( X @( U I2 z2 V7 j+ p4 G7 z$ _# `* T6 C
},
- M' m% f: P' h5 t# }. L "hosts": [& e# Q6 K* A, R5 [
"127.0.0.1",+ N9 z8 @* i* M' _
"192.168.91.19"
3 M0 L6 O, N- G/ A0 Y ],2 O9 W+ t& r, R
"names": [
0 G" `) j5 A2 K# N {& h" F' J" U, I; k5 a
"C": "CN",, y4 y8 o% O9 _* u- n
"ST": "ShangHai",# ?0 O1 \. I8 M8 l9 U8 P, \
"L": "ShangHai",
* p& B N5 Z. J: a& X6 [ "O": "system:kube-scheduler",( o; B; X' ]0 u; s" u
"OU": "System"
* l1 F- B" \* r* b; b }
: c- h2 e9 v9 u ]" F \& y, G/ X# Q! d: z
}4 b9 f: i, U9 j
cd /approot1/k8s/tmp/ssl/
9 c& K8 {$ v! j: Y/ c! @" f% ^, w7 Pcfssl gencert -ca=ca.pem \7 L X) x8 |# B" I. @1 r
-ca-key=ca-key.pem \
; \* K( p5 f; Z# F$ v-config=ca-config.json \
! \2 o6 {2 o8 Q" _-profile=kubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler* H* P+ i2 s* N
创建 kubeconfig 证书, C: X' K3 P% _; E8 z! T5 c
设置集群参数
8 Y+ I5 _7 r, y _/ Z7 `
- f4 A* ] Y% @' d1 b% t( d8 L--server 为 apiserver 的访问地址，修改成自己的 ip 地址和 service 文件里面指定的 --secure-port 参数的端口，切记，一定要带上https:// 协议，否则生成的证书，kubectl 命令访问不到 apiserver# N# D, t' S- ] \2 ]+ w9 e
) f2 c i* q4 tcd /approot1/k8s/tmp/ssl/" B8 e, `' z( f/ t" J
/approot1/k8s/pkg/kubernetes/bin/kubectl config set-cluster kubernetes \
. ~, Z" [7 F0 b8 s--certificate-authority=ca.pem \
1 H' S1 o9 ^5 J7 ]) K, [--embed-certs=true \% q. U$ K! {0 R: M* d6 P
--server=https://192.168.91.19:6443 \
9 X+ C1 \6 z* S$ b8 |--kubeconfig=kube-scheduler.kubeconfig& V# p" c& ~ p2 M( Q- r; }4 U
设置客户端认证参数
# A4 \, j- Y9 t1 S4 r! R1 H( l+ ^' t7 ?, Z4 P+ b
cd /approot1/k8s/tmp/ssl/
! ^% b3 [7 B* r7 @ F/approot1/k8s/pkg/kubernetes/bin/kubectl config set-credentials system:kube-scheduler \
, }+ N+ T# I$ \$ V) M! ?--client-certificate=kube-scheduler.pem \
; K- P9 ?# W3 [. j/ f--client-key=kube-scheduler-key.pem \
4 _$ g3 V8 Y! C5 Y: O- E--embed-certs=true \5 N; }' x0 a; N+ z
--kubeconfig=kube-scheduler.kubeconfig
. e& f$ n- G$ i设置上下文参数
( a$ [3 Q7 L& P' h9 ^! f3 g- V! h3 O" s' a" R8 X) W# Q
cd /approot1/k8s/tmp/ssl/
$ T8 m8 s$ J) ~4 C! F/approot1/k8s/pkg/kubernetes/bin/kubectl config set-context system:kube-scheduler \
+ F4 @) r d- |--cluster=kubernetes \" Y6 U* E% I0 d& ^$ g5 Y
--user=system:kube-scheduler \
: b0 c6 x$ o, O0 h7 B7 t/ D J--kubeconfig=kube-scheduler.kubeconfig
3 e5 E' H: k3 S* U# E1 N0 l3 a设置默认上下文
/ a# K0 @$ A9 ^; q! k6 i4 o
& p- V) ~/ C! H3 zcd /approot1/k8s/tmp/ssl/1 x9 e# \2 h' L. s$ ?
/approot1/k8s/pkg/kubernetes/bin/kubectl config \
) j. N a( G8 c% I9 s' wuse-context system:kube-scheduler \
! t! a- _$ E1 w6 i5 I9 h& C% a--kubeconfig=kube-scheduler.kubeconfig! h* ~, F+ l. ]1 J
配置 scheduler 为 systemctl 管理
6 M V1 J+ f! H" N" g) cvim /approot1/k8s/tmp/service/kube-scheduler.service
+ |; g _3 W( l" lscheduler 参数( [, ^$ ^2 D- Z( i7 w
, }4 m, j, S, C
[Unit]
0 N, O) ~. V' f" s8 D. F' S0 zDescription=Kubernetes Scheduler
4 \5 y, L9 Y5 D+ PDocumentation=https://github.com/GoogleCloudPlatform/kubernetes
9 J2 `$ [( U7 L3 Y4 J: q0 F2 f9 H2 B; H/ ?
[Service]
# e" [& f9 Y) g" e9 ?( ZExecStart=/approot1/k8s/bin/kube-scheduler \9 z2 ]2 C) s- H0 ?2 O
--authentication-kubeconfig=/etc/kubernetes/kube-scheduler.kubeconfig \1 W! E% U. [) o/ R
--authorization-kubeconfig=/etc/kubernetes/kube-scheduler.kubeconfig \
1 w7 |* z. e: h- W! [3 H$ K+ w --bind-address=0.0.0.0 \/ j4 O5 }7 n, C: s3 w6 r6 o
--kubeconfig=/etc/kubernetes/kube-scheduler.kubeconfig \8 R: |& I' |5 P9 ?$ w5 [ W) `
--leader-elect=true \
0 z3 {! V/ A- J --v=2
3 R) [+ C, X3 b# Y3 c- GRestart=always
6 v) U( ]# f) L+ u4 jRestartSec=5
6 j% Q) ]8 W9 S! Q8 @$ l l' p) L2 b, [2 O
[Install]
1 ]7 `* M1 ^/ I2 R( `$ RWantedBy=multi-user.target
7 u% O6 b" j: V: z9 ^分发证书以及创建相关路径7 T" X" y: E; V, D
如果是多节点，只需要在192.168.91.19后面加上对应的ip即可，以空格为分隔，注意将192.168.91.19修改为自己的ip，切莫一股脑复制$ K \/ I+ U( z! u+ e, \
3 r( p4 O" f" @5 Q; \" [# m \
对应的目录也要确保和自己规划的一致，如果和我的有不同，注意修改，否则服务会启动失败: b- j R% _/ }% `9 b9 F
/ s$ \9 ] v) a& `" ]0 B2 mfor i in 192.168.91.19;do \$ i; T" b3 M) X% [# R
ssh $i "mkdir -p /etc/kubernetes/ssl"; \' n0 V" B0 F0 P) C( P" \% [- {& Q
ssh $i "mkdir -p /approot1/k8s/bin"; \$ ^" U' Y. ~ ?, w
scp /approot1/k8s/tmp/ssl/{ca*.pem,kube-scheduler.kubeconfig} $i:/etc/kubernetes/; \# v6 J# J2 s- Z1 C3 l
scp /approot1/k8s/tmp/service/kube-scheduler.service $i:/etc/systemd/system/; \$ h( _/ D( y, }6 h5 I& G9 |# \
scp /approot1/k8s/pkg/kubernetes/bin/kube-scheduler $i:/approot1/k8s/bin/; \8 \+ D* g# T7 b
done
( _! X& I6 F( E) E; O启动 scheduler 服务
1 w( ~* y5 [, _8 W& X: H9 Z/ P6 o如果是多节点，只需要在192.168.91.19后面加上对应的ip即可，以空格为分隔，注意将192.168.91.19修改为自己的ip，切莫一股脑复制/ e9 Q* X9 v, R4 f2 G. _ I3 r, d
9 M( m/ x3 _+ J6 t, Ofor i in 192.168.91.19;do \' _" u* P5 E( ]/ {) k+ _: p# E
ssh $i "systemctl daemon-reload"; \+ [6 O' A3 a$ U4 c
ssh $i "systemctl enable kube-scheduler"; \6 O+ [1 y& Q7 H1 L F1 g3 n; y6 D
ssh $i "systemctl restart kube-scheduler --no-block"; \0 U! J* g9 Y/ P7 d
ssh $i "systemctl is-active kube-scheduler"; \2 I0 h3 a5 L0 p9 w
done! r+ K- x/ j3 r1 q+ n
返回 activating 表示 scheduler 还在启动中，可以稍等一会，然后再执行 for i in 192.168.91.19;do ssh $i "systemctl is-active kube-scheduler";done
) S- q9 l: b$ |' X4 p
0 q1 Y5 o* U7 D' N$ ?6 r返回active表示 scheduler 启动成功4 L% P9 f5 d9 ^/ }" V- U
; z9 h# ?* _/ h
部署 work 节点
. B! ?+ ?8 F9 _" W部署 containerd 组件
/ U. Z I- j( w( E: D& Z+ v' o下载二进制文件+ L! \) g& o; x" F; p
github 下载 containerd 的时候，记得选择cri-containerd-cni 开头的文件，这个包里面包含了 containerd 以及 crictl 管理工具和 cni 网络插件，包括 systemd service 文件、config.toml 、 crictl.yaml 以及 cni 配置文件都是配置好的，简单修改一下就可以使用了$ S) o9 F+ h( w
# T1 Y" Z6 x) D6 P7 S
虽然 cri-containerd-cni 也有 runc ，但是缺少依赖，所以还是要去 runc github 重新下载一个
! j3 @1 L5 L% Y# X: f, ]: f. i: G3 ^( @8 w6 ], O' R& V
wget -O /approot1/k8s/pkg/containerd.tar.gz \
) M6 ?5 s4 Y \% h' x( C" fhttps://github.com/containerd/co ... -linux-amd64.tar.gz
& Z1 v) W; S3 H% Y+ u: u% B i dwget -O /approot1/k8s/pkg/runc https://github.com/opencontainer ... d/v1.0.3/runc.amd649 z$ H5 ?7 A6 E d) b
mkdir /approot1/k8s/pkg/containerd+ S/ p9 q1 |* O( ]: ]6 u
cd /approot1/k8s/pkg/ t n! F( @& h) y
for i in $(ls *containerd*.tar.gz);do tar xvf $i -C /approot1/k8s/pkg/containerd && rm -f $i;done9 Q4 @4 `! B7 s7 H3 l
chmod +x /approot1/k8s/pkg/runc
& Z9 A" N/ @2 Z3 }" emv /approot1/k8s/pkg/containerd/usr/local/bin/{containerd,containerd-shim*,crictl,ctr} /approot1/k8s/pkg/containerd/' Q; D. \6 Q+ ~) d. d0 j
mv /approot1/k8s/pkg/containerd/opt/cni/bin/{bridge,flannel,host-local,loopback,portmap} /approot1/k8s/pkg/containerd/, X, e. B! T9 U2 t; C% t% U
rm -rf /approot1/k8s/pkg/containerd/{etc,opt,usr}9 I$ m! m$ L, L
配置 containerd 为 systemctl 管理9 h; @1 j" ?! x" v+ l
vim /approot1/k8s/tmp/service/containerd.service# n0 f" C6 v6 a8 k8 _
注意二进制文件存放路径
& r4 n l. j+ z4 z' @3 G0 Z$ t3 {5 u) h; w, C) P! \
如果 runc 二进制文件不在 /usr/bin/ 目录下，需要有 Environment 参数，指定 runc 二进制文件的路径给 PATH ，否则当 k8s 启动 pod 的时候会报错 exec: "runc": executable file not found in $PATH: unknown$ o! ^5 @' a+ k& Q- m' d
0 Y0 q' d$ Q' M[Unit]
& Q L4 w0 O' L. UDescription=containerd container runtime5 @ i% p: | N/ n
Documentation=https://containerd.io
8 A/ r7 i* P9 Q! b: KAfter=network.target* ^1 t' `0 ?/ e& x
( z& p) ~0 W m3 t4 S8 ?/ B0 l
[Service]
! a$ r$ h& y1 F+ a; ~' gEnvironment="PATH=$PATH:/approot1/k8s/bin"( ~5 |/ ~7 q: Q" h3 U5 a3 h# d9 u
ExecStartPre=-/sbin/modprobe overlay
4 |0 U# v+ B' o+ ^9 D3 |) t4 q qExecStart=/approot1/k8s/bin/containerd
: U" ?0 j$ \: P7 _" W0 U; M. jRestart=always% ~% \5 P" e( d! V* L
RestartSec=5: T8 C1 u* z/ O
Delegate=yes7 k$ i$ {$ }" N& v9 R" o; o @
KillMode=process. e4 d/ s8 m6 e# j
OOMScoreAdjust=-999
! ?1 c9 J0 |& l$ |7 ?LimitNOFILE=1048576& S5 I+ M3 j2 Y
# Having non-zero Limit*s causes performance problems due to accounting overhead
5 z* \3 f+ `+ L* i0 g2 y2 Q# in the kernel. We recommend using cgroups to do container-local accounting.
. z i/ P% `: N9 P1 Y6 [/ V% rLimitNPROC=infinity
7 o4 [. H6 }" Q' X/ qLimitCORE=infinity
. h+ C" N! p3 X" m: e* r9 `
/ u4 l/ m! |3 l& W[Install]
8 H" G* x! V) m nWantedBy=multi-user.target
6 m. F& Z' d, r$ ?2 y3 c配置 containerd 配置文件% ]- C- T5 P" y2 a( b* O
vim /approot1/k8s/tmp/service/config.toml5 v6 e* i" t; L0 u" H" w) a
root 容器存储路径，修改成磁盘空间充足的路径* j9 r7 k8 R$ }) r
* `4 H- a, Z: Y8 i
bin_dir containerd 服务以及 cni 插件存储路径
' |- k$ D) z% j) ]4 z E3 q$ U1 p2 J0 M1 l6 Z
sandbox_image pause 镜像名称以及镜像tag
* Q: S7 Q3 V2 Y& ^6 t5 B1 A/ j8 n8 O5 I
disabled_plugins = []
6 @ J: V ~# G( F/ p ?% Fimports = []3 k; k, H3 g' ^
oom_score = 0
' Y( _1 F: Z+ J p9 {9 jplugin_dir = "". _0 A7 k! M3 ?1 X" u
required_plugins = []6 E1 U. R3 {7 Z: e; M2 F
root = "/approot1/data/containerd"
; E7 M% s$ x0 a5 dstate = "/run/containerd"
L. T2 C/ V3 v3 T( Y# E5 c) ^version = 2
$ ]9 ?; J) Z- i. L, u
, Y9 n# }7 w3 i+ B! H[cgroup]
. n* A" o& f" o+ ?; S' v, G4 c path = ""- b( a. M; F8 F4 ?$ J* L
. a5 `# \* P- `% k+ Z" |
[debug]
. G' y! \8 f4 @% u address = ""
) U7 j L. v Q3 C format = ""
: {/ s3 \9 R7 s7 Z# L gid = 0
( v, V- z, X0 n1 i/ W; f/ { level = ""
$ M% i9 K. I" n# g4 H1 g7 ` uid = 0/ j8 m1 q" a& C, j' b4 L. J' W' k
) ~! w. n; g9 h9 i$ Y$ N8 _
[grpc]% t( W5 M. Q) A+ S6 s, u2 o
address = "/run/containerd/containerd.sock"
. b4 r8 Z6 b" X gid = 0
3 ]. b% x2 L' ~- x max_recv_message_size = 16777216& y/ }/ E- r Q) V
max_send_message_size = 16777216. g# m) D2 \% x% V+ G6 P. s$ R
tcp_address = "": r# b* }; `% i8 E0 N5 W
tcp_tls_cert = "") S; Z) c9 ^& E0 P
tcp_tls_key = ""
1 Z. a8 t+ \1 `3 d3 D uid = 03 D7 y- X7 G+ D, }
* M8 d0 d' |. s' T5 ~[metrics]8 Q: P3 {% U$ N% t' Y) y' C3 X4 C
address = "": R X) x2 c2 K7 Y! Q
grpc_histogram = false
8 K! J2 c) r) a! C0 o* I6 y0 Q' T
" E8 z8 s1 B" J/ _# Z6 S; `[plugins]
9 s% G d5 G2 N& B% H. U _, k9 t5 j# T8 z
[plugins."io.containerd.gc.v1.scheduler"]
% S2 N/ z: ?$ ]/ V R" w deletion_threshold = 0
& {7 Q. Q' c! T0 c8 v: W mutation_threshold = 100
$ Y" o) j4 {4 Z6 L9 `9 o pause_threshold = 0.02
$ I4 t) l) o/ w) v& m/ `# j, ~# O schedule_delay = "0s"; l9 k, f3 A# a, ~/ S2 L
startup_delay = "100ms": Z+ U& {) X& a6 ]
; W* }; K2 Q, Y* n3 [ [plugins."io.containerd.grpc.v1.cri"]2 i# j' K0 b. E& z* y8 g A3 E1 ?- {
disable_apparmor = false
) n1 K% ~3 Q; n6 f+ P* b, v disable_cgroup = false# r; y+ g! D5 z- _- o- b2 |) j
disable_hugetlb_controller = true
f2 v5 A: b7 ~, _* W disable_proc_mount = false0 E5 W8 D1 x5 A7 a' f8 _3 J
disable_tcp_service = true1 ?& P# ~4 m, f" G
enable_selinux = false6 K% I( V6 \: j* a
enable_tls_streaming = false
; V5 e5 k+ G6 E ignore_image_defined_volumes = false
( {. U+ p% I) o! f! E* g3 D max_concurrent_downloads = 3
8 |+ K& k. G( x max_container_log_line_size = 16384
) T6 |# s+ e; U netns_mounts_under_state_dir = false
9 ^. a9 F1 V8 g' q5 J/ }* F4 A restrict_oom_score_adj = false1 J3 b4 U3 l& K$ m7 w4 @
sandbox_image = "k8s.gcr.io/pause:3.6"2 c/ {& c0 V! {2 ~
selinux_category_range = 1024
' w8 O% \& W0 ]4 b c stats_collect_period = 10 M3 ]% R2 t0 P7 |
stream_idle_timeout = "4h0m0s"
" a& U) Y% W; l. ?7 ^ u8 }2 i stream_server_address = "127.0.0.1"
+ F+ h9 s( d) v& e3 j stream_server_port = "0"
9 f6 B+ `4 x7 r. Q5 n systemd_cgroup = false4 ]: Y# v9 C \; z- ?9 [
tolerate_missing_hugetlb_controller = true/ _( h% {- d( M' {" B
unset_seccomp_profile = ""
) p4 X \# w9 B5 q2 ~- K: x8 ]: [7 H& \
[plugins."io.containerd.grpc.v1.cri".cni]
: \ i! \, t- Z. a" a1 ?& E6 s6 t G bin_dir = "/approot1/k8s/bin". ?" N! j* I! _5 b( Y" `
conf_dir = "/etc/cni/net.d", F1 f7 M& M9 M, \6 D
conf_template = "/etc/cni/net.d/cni-default.conf"
( S( J# F1 Q8 i8 Z max_conf_num = 1
6 z! H8 B/ C" G! E0 J
7 A6 U3 A q& _/ l- Q: i5 `8 L [plugins."io.containerd.grpc.v1.cri".containerd]
" ?4 R# ~4 i0 V4 p$ v' \ default_runtime_name = "runc"
, I6 S5 e5 P$ B3 l- O* A disable_snapshot_annotations = true- ?+ Q& Y* x" R3 A% L/ b5 }6 z
discard_unpacked_layers = false
0 t" e6 b4 E% C* G8 R/ x no_pivot = false& v$ c9 m7 p& Q
snapshotter = "overlayfs"" H. I/ J6 c+ v
1 G6 ~' B% F' \" w4 w o [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
9 k5 p3 v3 z3 H base_runtime_spec = ""
, t. U V2 y7 Z4 V/ M' K container_annotations = []
$ M, v% ^' S4 s pod_annotations = []7 {7 }6 U0 Q1 q
privileged_without_host_devices = false$ {0 ^) n8 M0 F; f$ Q& U; q& u
runtime_engine = ""5 ?/ c8 g- k+ @# {* }4 Z
runtime_root = ""
" i+ [0 R0 x3 E2 j4 o) L runtime_type = ""2 I3 o9 P. ~# D* |
5 T0 Q6 K( H2 C& T% u% v- D
[plugins."io.containerd.grpc.v1.cri".containerd.default_runtime.options]
. D+ s2 c% X9 T, O5 r7 o) j5 \ j! c2 B& _& t. c' o/ C
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]$ t! g1 V# X `. {) j
. D# d0 O" F& |" `5 v6 b; o
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
( s: A! @6 N* g4 F/ H$ t( X; q base_runtime_spec = ""
* j2 ^, Y% d5 d1 J- M container_annotations = []
) X3 \$ E% w/ i0 ^ pod_annotations = []8 ?2 D1 h3 Z# _) m4 v
privileged_without_host_devices = false
7 Z3 T! ]/ ^# H3 ?/ ^" e T runtime_engine = ""/ L+ |( q! G1 w; N9 C
runtime_root = ""; \2 A, `. b ^
runtime_type = "io.containerd.runc.v2"
; F* A- x! V# b" q$ x& [% h9 g( D7 u8 C1 |
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]9 H' L* F1 Q V5 P
BinaryName = "": T! o7 @; x, g! f$ x7 p
CriuImagePath = ""4 N+ o1 {3 `2 U0 W" H6 q
CriuPath = ""
7 I: v7 {# g) {- u |0 Q CriuWorkPath = ""7 b4 j" ^+ n( w
IoGid = 0! `) d8 b; ^% f0 o& ]- b
IoUid = 0; U. m; h$ F/ @/ T; l+ @
NoNewKeyring = false
# W+ \2 v9 Z4 M1 R" D+ b NoPivotRoot = false# N" \# I1 R( x) j) T
Root = ""
8 R" {) }8 X+ P/ Z4 L5 t ShimCgroup = "". Q2 O$ |, }* r; S" P4 A. m
SystemdCgroup = true" K0 X8 N& ?+ O! B! g! Y
: ~% X/ U* p/ _% E4 V
[plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime] O! f1 v# M3 ?
base_runtime_spec = ""+ S: i/ J j% l; x/ x9 e; N" f
container_annotations = []
$ X) q8 E! @3 ^+ S6 `7 Y pod_annotations = []) T3 c2 j" M: }( m% D
privileged_without_host_devices = false
2 ^# |7 G2 o4 a q5 S9 h5 `3 Q runtime_engine = ""9 k' Q3 t0 b4 [' j( G
runtime_root = ""% \. V; _, N8 i# M1 J
runtime_type = ""
1 d) {" Q, D6 a# b* d" i J5 u, E+ p8 ~& P- T
[plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime.options]$ S! \! S6 k* ?$ p4 i* D
# e( B. \2 d( u- ]% R! I [plugins."io.containerd.grpc.v1.cri".image_decryption]% \* Q! g" d$ h' b5 B" `
key_model = "node"
9 V" q8 w( {/ f' T# {- T3 W1 ]. t4 \" _- ~% |" a. C* i
[plugins."io.containerd.grpc.v1.cri".registry]$ v9 n, f0 @4 X/ T; ]) l
config_path = ""# a9 b5 f8 t1 m$ ^; j* p
5 E* J A7 E2 ]* S
[plugins."io.containerd.grpc.v1.cri".registry.auths]) N- G' {8 n- \) x
6 U: K& _1 g5 ~. @: h+ _$ @
[plugins."io.containerd.grpc.v1.cri".registry.configs]
+ \+ s4 `/ H" n5 y8 e
- B7 t; t5 u2 { [plugins."io.containerd.grpc.v1.cri".registry.headers]
( U8 f: Y( O4 x" B- U: \7 {* U3 s2 B6 | X) M- z$ s; @$ m/ U
[plugins."io.containerd.grpc.v1.cri".registry.mirrors]* G) M8 o* ^7 g/ l" L$ ?+ W2 b
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
5 G2 U/ S2 @6 @7 d endpoint = ["https://docker.mirrors.ustc.edu.cn", "http://hub-mirror.c.163.com"]% t5 ?' |; g m+ e4 V% {# G! ?
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."gcr.io"]) R' t& | a6 h1 z. c. D2 C9 t. a
endpoint = ["https://gcr.mirrors.ustc.edu.cn"]. G" y+ x% n; t/ ?
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."k8s.gcr.io"]
, {. D2 X, ^* Z9 x8 n% h w. R7 y endpoint = ["https://gcr.mirrors.ustc.edu.cn/google-containers/"]
; ]3 R5 H! X- {8 ^' `2 e* ` [plugins."io.containerd.grpc.v1.cri".registry.mirrors."quay.io"]
) k0 g4 ]4 X' S4 N. }" I, r9 q) A endpoint = ["https://quay.mirrors.ustc.edu.cn"]
+ Y) l0 {) e4 P/ q) E: D7 U9 h v+ ~, @5 B/ C4 n5 R8 z& _
[plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]0 e+ z" l @* Y1 k1 g, d
tls_cert_file = ""' |& f) `# q. e9 K! \% z* H; X9 o5 o
tls_key_file = ""
/ e6 Y0 H" _5 \5 d, j5 [! X0 i) l# D( S/ O
[plugins."io.containerd.internal.v1.opt"]
- \ l# W: p9 R R( ` path = "/opt/containerd"! s1 _$ x! J6 w% \5 f
% b- U/ W% W) H, J' V7 F# }4 T6 j: {5 q
[plugins."io.containerd.internal.v1.restart"]
1 Q8 f; a! b6 i& o5 k9 U0 X interval = "10s"
* ?5 y. A* |( T/ O
9 \1 w' A3 X! t3 s6 O+ V( c$ r. S [plugins."io.containerd.metadata.v1.bolt"]0 y. W" _# |! W
content_sharing_policy = "shared"$ }# L1 E* q+ P1 L8 N0 Y
# ?7 |5 Q2 ]& a% ]8 P' g
[plugins."io.containerd.monitor.v1.cgroups"]9 g1 \; Q. {4 y9 a
no_prometheus = false# F3 }2 D5 C( x
7 ^% u% w% K" H& e' J4 U# e [plugins."io.containerd.runtime.v1.linux"] R+ ]) E$ o1 K/ Y6 }5 Y
no_shim = false
7 F7 D, L. p! b, f runtime = "runc"
J# I1 x8 Q0 n; i; ?# {0 z" k runtime_root = ""
A2 \5 f* p: A* {2 J; M shim = "containerd-shim"" b' o. s+ G% y6 S1 X
shim_debug = false
1 C' i0 r, V5 b/ ~% u8 m
& B) d, P. L! S# [& T, P7 X! g [plugins."io.containerd.runtime.v2.task"]3 w9 j7 M# c# a* x4 C
platforms = ["linux/amd64"]9 N4 k1 Z' G2 k5 ]' w' B) v1 B( R
# g) ]1 c( k: C5 e' o [plugins."io.containerd.service.v1.diff-service"]
4 ]3 O& u# U3 W+ s$ {8 `4 ~# h) O default = ["walking"]; ~3 B+ K. {( L
4 V8 ^. p- Z [& X& G/ }2 V
[plugins."io.containerd.snapshotter.v1.aufs"]) I/ \/ Z3 h/ c# b
root_path = ""
7 x% `! r. d2 \1 V- `) n0 Y! F/ q; m6 j7 O0 `
[plugins."io.containerd.snapshotter.v1.btrfs"]
^: Y, @$ G5 i8 Z [: j root_path = ""
7 y* }6 P0 r) M/ E7 F W1 L
& Q6 D9 U9 |% L( \ [plugins."io.containerd.snapshotter.v1.devmapper"], o9 b' |. _3 v; `, i
async_remove = false
" r8 t" a; }" w, k/ Y base_image_size = ""7 @# `6 `3 A+ c+ A
pool_name = ""
3 x2 d3 p; B- M! W* ]9 T root_path = ""/ v+ B7 D2 a! x5 G5 z
: M* k9 V7 p, A8 n2 W# y$ @" Y
[plugins."io.containerd.snapshotter.v1.native"]+ v E6 C/ P% L: E6 `6 ^" m7 s
root_path = ""
+ c9 j! J" Z% u; v1 O
. R- J" @7 E" y3 X, o7 e [plugins."io.containerd.snapshotter.v1.overlayfs"]
& Y% r$ ~* ~8 v' E% ^4 @ root_path = ""7 h* T8 ~3 h# J) Y( n
! ^6 U1 k9 I/ Y9 n) j8 o! v- C1 s
[plugins."io.containerd.snapshotter.v1.zfs"], B; Y8 I. L+ e4 h6 _3 [
root_path = ""( Q! f. ?& I" w: ]; B
) t2 ^3 O; u' y3 T+ y$ x% N
[proxy_plugins]8 h' F! P, C6 ^! L; E+ y8 J/ S, U( Q- ] w
$ s, B! C, V6 g
[stream_processors]6 t7 a/ N0 B( m
9 E% f, X c# _8 p [stream_processors."io.containerd.ocicrypt.decoder.v1.tar"]
+ O; z& O! W. f) W accepts = ["application/vnd.oci.image.layer.v1.tar+encrypted"]: B/ S5 x' l/ ^# u0 `
args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"]2 ^% P' e( E# y! g; D
env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"]
: @: k2 g& D( k* G T, D% O path = "ctd-decoder"1 H% O+ u* r9 }3 r( f: h' K5 A
returns = "application/vnd.oci.image.layer.v1.tar"2 T; p. ~2 E: J$ B
; V& X. y: @. ^ [stream_processors."io.containerd.ocicrypt.decoder.v1.tar.gzip"]( }/ n# Z! W0 G6 C% I" D: q3 t+ j7 u
accepts = ["application/vnd.oci.image.layer.v1.tar+gzip+encrypted"]
- O7 j+ a2 u' N- V6 ] args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"]
6 w/ l- h( Y0 e2 ~$ | env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"]
, x9 R9 i [& k0 W. | path = "ctd-decoder"
5 n# x' J; T# Z( T- _- ? returns = "application/vnd.oci.image.layer.v1.tar+gzip"' _) `, W& B) G+ k2 g
& y; Z2 G6 w1 r0 c P( U2 x) j
[timeouts]
# G: [& O, b( @& j6 B8 r6 Z* M7 J "io.containerd.timeout.shim.cleanup" = "5s"
- q- B& g( G5 m7 } "io.containerd.timeout.shim.load" = "5s"
7 L& k6 c9 Y: D: W3 f7 @ "io.containerd.timeout.shim.shutdown" = "3s"0 J7 a, U8 l/ G
"io.containerd.timeout.task.state" = "2s"
2 f# W2 d- m' z1 Q% l6 R
1 `- J& a9 \6 C# k8 G; G[ttrpc]' C) \; D6 V4 @3 Z) G) C
address = ""
) c1 i, u0 U, C% n. \. l gid = 00 R) `( @( n, I) T: E
uid = 0
' i/ |2 H- s1 j配置 crictl 管理工具
/ w' i4 v% F2 }! b I/ Hvim /approot1/k8s/tmp/service/crictl.yaml% V" { t$ g2 F2 T' h
runtime-endpoint: unix:///run/containerd/containerd.sock+ ^( x7 |% ~4 s! g# L
配置 cni 网络插件
0 a7 s9 u5 ?. Gvim /approot1/k8s/tmp/service/cni-default.conf
# Z6 A ?6 z! [- c& H( ssubnet 参数要和 controller-manager 的 --cluster-cidr 参数一致* M/ _4 |! s& x( O" g
4 O- a6 F1 [* ]1 @" D2 U+ |7 g
{
5 @; @' T7 Q3 ]% `0 |. r( {, l; B" ]( h "name": "mynet",/ h0 U- L. q# U- S5 Z
"cniVersion": "0.3.1",
# M7 u3 o$ c# W" v9 \, w "type": "bridge"," x1 z z" K6 |2 s9 E* D3 f. u" }
"bridge": "mynet0",
3 G# r" S* T: Z0 o( p/ z1 y "isDefaultGateway": true,, _) Z ~; Q4 c7 T+ R$ ]
"ipMasq": true,
- s& B# ^* s& r" o7 d "hairpinMode": true,
. S5 g0 t% x0 Z" ~2 B% z0 ]* v4 y "ipam": {) W j! G% }$ L9 i) J* B4 Q
"type": "host-local",
+ [( ^, ~+ B' L" X "subnet": "172.20.0.0/16"( F# J' `7 ]5 i* }
}
+ w" i- w" k E6 z: m}# m. |7 [. i# z2 a
分发配置文件以及创建相关路径
4 \7 o$ ]9 i- ~7 X" `( M/ Y* Bfor i in 192.168.91.19 192.168.91.20;do \, U% d U. v: c* G9 z, ?1 _1 k2 E
ssh $i "mkdir -p /etc/containerd"; \
3 A) N7 z/ L K8 q* n; Ossh $i "mkdir -p /approot1/k8s/bin"; \5 t0 {% T9 I x" Z' z
ssh $i "mkdir -p /etc/cni/net.d"; \5 ?. J' ^: L0 o) o
scp /approot1/k8s/tmp/service/containerd.service $i:/etc/systemd/system/; \- y; n' b3 k" S, ? n
scp /approot1/k8s/tmp/service/config.toml $i:/etc/containerd/; \- X( \7 X+ q7 s* L' h7 H2 n4 e2 ^
scp /approot1/k8s/tmp/service/cni-default.conf $i:/etc/cni/net.d/; \, y+ |5 E D$ k
scp /approot1/k8s/tmp/service/crictl.yaml $i:/etc/; \
/ L! q# B0 z) L6 f! m; L' Oscp /approot1/k8s/pkg/containerd/* $i:/approot1/k8s/bin/; \" y2 @0 |5 F$ w2 a; A. R
scp /approot1/k8s/pkg/runc $i:/approot1/k8s/bin/; \
- D- W& ^2 M3 U3 T( h9 X* Adone
( a+ X# { D& T; L启动 containerd 服务4 E2 I$ z1 [% V& Y6 J! \! A
for i in 192.168.91.19 192.168.91.20;do \
4 U9 o8 W4 L" m1 J% X* Cssh $i "systemctl daemon-reload"; \
+ u( i) ^: x. @ssh $i "systemctl enable containerd"; \
! b% W* b8 |/ v, }ssh $i "systemctl restart containerd --no-block"; \6 v6 d ]( E" ?, s) [
ssh $i "systemctl is-active containerd"; \
& z& t S" `# a$ U* a9 b' ]done
! |+ X% {: z& d( f, M6 O8 X返回 activating 表示 containerd 还在启动中，可以稍等一会，然后再执行 for i in 192.168.91.19 192.168.91.20;do ssh $i "systemctl is-active containerd";done. S5 j ]/ G1 N' j- l
/ ?# h/ r; q# _+ i7 H; u返回active表示 containerd 启动成功
8 r9 e, {( e' v! S" \2 L* u! R+ ^+ d. F) r; G2 u y+ q
导入 pause 镜像. r0 \% G, r) i& Y$ K, T1 Y
ctr 导入镜像有一个特殊的地方，如果导入的镜像想要 k8s 可以使用，需要加上 -n k8s.io 参数，而且必须是ctr -n k8s.io image import <xxx.tar> 这样的格式，如果是 ctr image import <xxx.tar> -n k8s.io 就会报错 ctr: flag provided but not defined: -n 这个操作确实有点骚气，不太适应
- ]- D- I! p- c2 a4 a
' N9 g" p' P( B. l9 ?# g1 C$ o如果镜像导入的时候没有加上 -n k8s.io ，启动 pod 的时候 kubelet 会重新去拉取 pause 容器，如果配置的镜像仓库没有这个 tag 的镜像就会报错) L ] a, H' i6 N; [$ ]
) H2 u' \5 l! A- S# C
for i in 192.168.91.19 192.168.91.20;do \
0 x& i; R* y6 w. R- Tscp /approot1/k8s/images/pause-v3.6.tar $i:/tmp/ j' r8 _1 H* a0 O
ssh $i "ctr -n=k8s.io image import /tmp/pause-v3.6.tar && rm -f /tmp/pause-v3.6.tar"; \1 ]: _* e) O# B# b: z: M$ B. `% l$ L
done
3 _: |8 P$ X( `! \1 o查看镜像
4 ~0 u' Y% d9 R! |$ y1 H& d0 T5 F' o
for i in 192.168.91.19 192.168.91.20;do \, \" n) ^) W R! z8 G3 e
ssh $i "ctr -n=k8s.io image list | grep pause"; \5 P( k) I; O; `; \0 y. j
done6 V) ]8 Z# I8 ~
部署 kubelet 组件4 i3 O( r. P% |, P' I+ W& F3 M
创建 kubelet 证书 J# j% N& @: q' s7 U% L2 G& R E
vim /approot1/k8s/tmp/ssl/kubelet-csr.json.192.168.91.19
$ T5 b; E3 z" r9 u& }& T0 p这里的192.168.91.19需要改成自己的ip，不要一股脑的复制黏贴，有多少个node节点就创建多少个json文件，json文件内的 ip 也要修改为 work 节点的 ip，别重复了5 \" ], b$ H& e3 |6 G @
1 L' L. E3 S- h. a7 e+ X* w e. y{
, S2 Q# m( v1 u* z "CN": "system:node:192.168.91.19",7 M1 V: K, g6 V. o. r, e
"key": {
/ y7 C4 l, b2 _4 i, \ "algo": "rsa",
( {5 ~% g3 p5 C5 ^ ]7 @ "size": 2048
; n5 D4 T. c" P$ \ },
/ S& c; i- L7 ]& J" t "hosts": [
" i2 [& C, A0 j, Q8 B4 a" p "127.0.0.1",- n) d8 R+ m$ T4 o1 C
"192.168.91.19"
; Y; S* S& V6 z ],
7 e) y! Z4 ?( M "names": [
1 E! J$ @% |$ s ~; a6 `8 ~4 d! y {# m. s, T+ }* u
"C": "CN",1 {1 |- w1 d. j6 w" I; A' M! E
"ST": "ShangHai",
' n- P+ Y4 x: t. [0 n0 y "L": "ShangHai",
& c, U( N ?$ J3 z2 ]# I8 d2 v "O": "system:nodes",
% v2 r+ y' Y. O7 E; l% b" N9 i- \ "OU": "System"
; d9 g- @. i3 B, T) h3 k$ _ }& s! T% Q' y5 s
]
1 ]! v2 a, X$ ?% }7 D: f}
! \. p; S6 Y3 c- G z& \" Xfor i in 192.168.91.19 192.168.91.20;do \ R- b. Y5 a ?) f
cd /approot1/k8s/tmp/ssl/; \$ l) ?% G- i# M; w M
cfssl gencert -ca=ca.pem \. _6 y( h8 k5 Q
-ca-key=ca-key.pem \
! x; A8 p; Z; h2 r* e-config=ca-config.json \
) `9 c! j' ^: ^* [9 p) M- i-profile=kubernetes kubelet-csr.json.$i | cfssljson -bare kubelet.$i; \# U! T9 [+ P0 h: a
done
b) Z* N- F' j/ l- ~; \0 Q3 d创建 kubeconfig 证书
, h! I4 }4 x! X+ a4 P7 R设置集群参数2 \8 u( X. ?' H- r9 p
' `1 \; T; k8 Y/ ^7 y/ ?+ Q
--server 为 apiserver 的访问地址，修改成自己的 ip 地址和 service 文件里面指定的 --secure-port 参数的端口，切记，一定要带上https:// 协议，否则生成的证书，kubectl 命令访问不到 apiserver2 S5 C# `, R; z
& X4 l' q- B7 ]; O% efor i in 192.168.91.19 192.168.91.20;do \5 l s) A7 ~/ u) z4 O
cd /approot1/k8s/tmp/ssl/; \# m/ o, [5 ~+ l4 b* O; e5 y) b* d
/approot1/k8s/pkg/kubernetes/bin/kubectl config set-cluster kubernetes \
i2 t. {' i- W+ u2 i--certificate-authority=ca.pem \
; Z* W$ W9 |# M: T- I--embed-certs=true \4 n, }1 k, C: _ H
--server=https://192.168.91.19:6443 \
( l. `# u. `8 S" ^" I! O) \--kubeconfig=kubelet.kubeconfig.$i; \
" q* l# i$ P0 k' K7 @done; J- s' J. F' u: g2 t; s
设置客户端认证参数/ {7 D( K3 B% d& H g
2 a) Q4 r& `" F" Mfor i in 192.168.91.19 192.168.91.20;do \8 J. u. ]- T" ~
cd /approot1/k8s/tmp/ssl/; \$ B9 |: a( }8 J7 A5 [$ q
/approot1/k8s/pkg/kubernetes/bin/kubectl config set-credentials system:node:$i \
. _* Y0 {) n+ d$ G+ V--client-certificate=kubelet.$i.pem \
2 Q1 A9 n/ L# p7 [: e--client-key=kubelet.$i-key.pem \$ q- g1 N# [& ?4 Q2 C0 f
--embed-certs=true \/ S% s1 p2 _$ t4 Q$ ]# X3 N
--kubeconfig=kubelet.kubeconfig.$i; \# k+ u0 @' f' x) x3 |; _& E
done
' X: {, a3 R" w; M9 ]; J% x2 ?设置上下文参数# @9 S# l3 \2 q h b$ C/ A
. {) \* w1 O6 U/ Z# S8 C8 Hfor i in 192.168.91.19 192.168.91.20;do \
4 ^1 g1 [, e4 L$ F4 I8 acd /approot1/k8s/tmp/ssl/; \
* d9 b: I4 J9 M) e/approot1/k8s/pkg/kubernetes/bin/kubectl config set-context default \ w" N+ j# E% K- H7 J0 W
--cluster=kubernetes \& E+ Y1 s( t2 Q- l- W8 f% N
--user=system:node:$i \3 h. m/ z) {$ l2 B# o) L) W
--kubeconfig=kubelet.kubeconfig.$i; \) j( R. }; l' x$ Y; r \7 i
done2 X2 }2 ]6 h W3 P, n( h: D5 C$ l9 j
设置默认上下文
- Y r1 a2 @' M$ V0 ?
`# G" _7 \6 \/ g8 Yfor i in 192.168.91.19 192.168.91.20;do \
! G8 z; _* G3 Q2 V1 Ncd /approot1/k8s/tmp/ssl/; \+ v# x) F- y, S& E) {: r
/approot1/k8s/pkg/kubernetes/bin/kubectl config \ S* u" y7 \# j% Y& r4 f( D# G
use-context default \
/ Q* N. c3 ^ R' P, x3 `, I5 m--kubeconfig=kubelet.kubeconfig.$i; \
' Q7 A+ I; _+ m8 L) I- cdone& Z8 z" B4 v7 Y
配置 kubelet 配置文件$ {" M9 @8 k c* T! Q! U% K
vim /approot1/k8s/tmp/service/config.yaml$ i7 U* S9 ?5 T3 D/ T) v
clusterDNS 参数的 ip 注意修改，和 apiserver 的 --service-cluster-ip-range 参数一个网段，和 k8s 服务 ip 要不一样，一般 k8s 服务的 ip 取网段第一个ip， clusterdns 选网段的第二个ip( }( v7 c! d& P% P) e+ _. B
) w ^) m2 s5 `1 U* ?0 @" B
kind: KubeletConfiguration) D5 k: Y4 \2 |7 N6 m, k' _# T8 G" T
apiVersion: kubelet.config.k8s.io/v1beta16 N6 ^0 ~* X ~
address: 0.0.0.0/ h8 r7 H, s0 W2 R# J8 c' W
authentication:
5 v# n4 \2 ?5 ^* k anonymous:- q. K( l% z5 h# W( S: T* @: l' y
enabled: false0 G! h+ T9 p; X( Y' _6 F
webhook:
; r% H' ?8 f! f$ [9 Z$ ? cacheTTL: 2m0s/ a' O9 w4 ?4 a2 i$ s) B3 Q" O* t1 s2 a2 `
enabled: true; G. Z6 C2 C! {5 V! V5 @! |
x509:
- e! U' u- d9 n6 v3 o% z1 I+ F clientCAFile: /etc/kubernetes/ssl/ca.pem
5 N- b2 N8 o/ u# ]authorization:% ?! h2 o" F0 B) {. D- y5 X
mode: Webhook/ y }* o* R, K3 v4 g. i' m
webhook:7 {3 F/ I. l, U" L
cacheAuthorizedTTL: 5m0s
& g# _- _, [% Y2 S% Q cacheUnauthorizedTTL: 30s
% Q9 e2 _* h' D# n0 hcgroupDriver: systemd
7 [2 Q( |! i2 Z% u7 o. bcgroupsPerQOS: true
) q. { C" ?3 k, G4 j$ c3 t8 EclusterDNS:1 D: y4 q% L3 D7 R6 ^2 c
- 10.88.0.2
, y" {: \" ?8 d8 AclusterDomain: cluster.local
& w' `3 H( X K% T! v5 X1 c( IconfigMapAndSecretChangeDetectionStrategy: Watch
3 M! s4 V7 m4 s- f1 X) u* Y1 PcontainerLogMaxFiles: 3! L2 q& j/ H8 F* o8 \
containerLogMaxSize: 10Mi% Q% G& A$ i% W( L* j2 ~9 Z
enforceNodeAllocatable:
9 o- r( e1 n5 E3 K, ^' N7 q- pods
6 i) A" _- x, A! R3 t# S( veventBurst: 10. K9 W! G9 e8 Q: j9 W6 S2 B. W; O; A
eventRecordQPS: 5
; ~( Z9 j8 Q' `- Z- F4 }evictionHard:
Q: o8 ^! H. W9 f) @- |1 y imagefs.available: 15%
& h' [/ F: ]) l memory.available: 300Mi3 R1 k" A: i% a% G: ~0 n
nodefs.available: 10%
X E- B' A# R0 e nodefs.inodesFree: 5%
% t8 _4 w0 ]1 H( F" h% ?# z6 ~% L+ BevictionPressureTransitionPeriod: 5m0s1 W* n, V4 J' u6 k& x: Q
failSwapOn: true' \2 P% M" u9 _4 C; ^. `
fileCheckFrequency: 40s0 y$ N# |# ^# r6 E* G
hairpinMode: hairpin-veth
6 x6 M* s# a& R* ^& A5 DhealthzBindAddress: 0.0.0.0$ U% \2 E: r) x D. L
healthzPort: 10248
9 q1 H- ^1 b2 ]9 Z+ OhttpCheckFrequency: 40s
" e1 K9 z( B6 ?) G2 w1 R3 m/ r" bimageGCHighThresholdPercent: 851 u$ p, c! `9 j" j( `/ D* v
imageGCLowThresholdPercent: 80
" J" R5 u7 ?* I$ u! QimageMinimumGCAge: 2m0s- ]" w7 }; s: {! g& F! ~
kubeAPIBurst: 100' f8 f6 U- Z3 P
kubeAPIQPS: 50
4 T P4 ^, E8 ^8 C8 S. tmakeIPTablesUtilChains: true5 u4 C) ] p3 m3 U8 G, H
maxOpenFiles: 10000009 ]( s$ `+ B V) ]+ f- z
maxPods: 110
- L6 o) N+ @7 f+ |; k0 znodeLeaseDurationSeconds: 40
2 _2 V* w8 I l- c6 ZnodeStatusReportFrequency: 1m0s. R. N: v8 Q4 y. `! n/ j4 E2 i
nodeStatusUpdateFrequency: 10s
5 f! ~% Y9 u7 P+ @3 \, [7 ^oomScoreAdj: -999
; S( E J# @/ T& S" h. @podPidsLimit: -1 f" C0 i/ Z" B5 u: L* |
port: 10250
& Z$ B2 [5 _$ s# disable readOnlyPort0 T5 ]" K; M! a9 ~4 e. X
readOnlyPort: 0
: q, `3 k! Q% k5 t& V7 M% aresolvConf: /etc/resolv.conf
: M& ?# L/ g% rruntimeRequestTimeout: 2m0s3 F1 @7 O2 x- \1 r! e7 D
serializeImagePulls: true
+ ?6 s- @$ N6 ?+ MstreamingConnectionIdleTimeout: 4h0m0s) n1 [' H$ m3 P' ]/ D+ @% K$ X
syncFrequency: 1m0s
7 |- ?% K5 T; A( b: e) X$ l$ o) DtlsCertFile: /etc/kubernetes/ssl/kubelet.pem
/ Q" q7 w6 T0 X, ~5 s7 \4 [- ptlsPrivateKeyFile: /etc/kubernetes/ssl/kubelet-key.pem
+ ~; r6 I' ?( [& n0 q' }7 n- G T5 a配置 kubelet 为 systemctl 管理; `& F7 f9 L; A X% M2 n
vim /approot1/k8s/tmp/service/kubelet.service.192.168.91.195 }& W$ ^+ {4 p
这里的192.168.91.19需要改成自己的ip，不要一股脑的复制黏贴，有多少个node节点就创建多少个service文件，service 文件内的 ip 也要修改为 work 节点的 ip，别重复了% C$ b# @1 y9 }2 H
# c, f) K# U) `% o--container-runtime 参数默认是 docker ，如果使用 docker 以外的，需要配置为 remote ，并且要配置 --container-runtime-endpoint 参数来指定 sock 文件的路径( |% h( P, U$ f3 P; G* {1 m1 Q3 g# g
" q- x) Q: F/ v& J2 c
kubelet 参数; b7 A! u$ L& d
5 F) A( I. \8 j- @1 U[Unit]2 ~: u, S$ |- U( [ [% r
Description=Kubernetes Kubelet
+ i; U/ Z8 {' g1 a, N, l9 HDocumentation=https://github.com/GoogleCloudPlatform/kubernetes4 N h- ^; r5 p% g+ C
. A6 n p9 T' h d- y[Service]
/ z" h. W1 u% ]WorkingDirectory=/approot1/k8s/data/kubelet
) o8 B3 X0 Y# X- t; V1 LExecStart=/approot1/k8s/bin/kubelet \8 O; e1 d+ x; D# l' f; I r; `9 }
--config=/approot1/k8s/data/kubelet/config.yaml \
) E5 K% X2 T- ?4 P7 c --cni-bin-dir=/approot1/k8s/bin \
4 q4 F# Z/ p i9 v9 k4 P --cni-conf-dir=/etc/cni/net.d \
0 l* A! M8 f: w& g8 l) |/ q9 l8 r --container-runtime=remote \9 k9 l# Q+ N6 R9 Q/ |( E! P" |
--container-runtime-endpoint=unix:///run/containerd/containerd.sock \4 v z' C5 q; J: ]* q& x1 J
--hostname-override=192.168.91.19 \1 B. \3 b4 x `6 A
--image-pull-progress-deadline=5m \: j: o- _+ s3 G6 ~2 \( Y* Y9 F
--kubeconfig=/etc/kubernetes/kubelet.kubeconfig \- J" |8 o: k; L5 G ?/ n! w
--network-plugin=cni \
8 y" R z6 a. l) |, D8 k --pod-infra-container-image=k8s.gcr.io/pause:3.6 \6 D8 v& l; I0 ^* P0 s8 T
--root-dir=/approot1/k8s/data/kubelet \
4 ]3 K6 l' a" v --v=2# t7 S5 E6 l% c
Restart=always: s7 U: |# p* `- W l- C
RestartSec=5+ D+ c' y, k8 Y/ w
, l' }3 v# d* m0 C- f[Install]
% k& d K7 [( p- yWantedBy=multi-user.target3 L! i3 f* Y' L/ w/ |) F" K
分发证书以及创建相关路径$ s* D8 k- w& E! @
如果是多节点，只需要在192.168.91.19后面加上对应的ip即可，以空格为分隔，注意将192.168.91.19修改为自己的ip，切莫一股脑复制
4 _& y. `0 ~! B$ U! B! t; T; R9 k. ?6 T% N6 y
对应的目录也要确保和自己规划的一致，如果和我的有不同，注意修改，否则服务会启动失败
t w' I) ^% ]) o2 D1 t& j% z9 k" @) l! y8 L" [
for i in 192.168.91.19 192.168.91.20;do \
" F1 A u) U: n+ b. s- a: ?ssh $i "mkdir -p /approot1/k8s/data/kubelet"; \* v- K$ j2 J o* d9 k
ssh $i "mkdir -p /approot1/k8s/bin"; \
' p" `, l- O' G0 y: A! m. [ssh $i "mkdir -p /etc/kubernetes/ssl"; \! _ ]2 r! z, H' E# h; q
scp /approot1/k8s/tmp/ssl/ca*.pem $i:/etc/kubernetes/ssl/; \% u5 q% H6 Y! i! j0 q; z
scp /approot1/k8s/tmp/ssl/kubelet.$i.pem $i:/etc/kubernetes/ssl/kubelet.pem; \* H6 n2 x1 g! P
scp /approot1/k8s/tmp/ssl/kubelet.$i-key.pem $i:/etc/kubernetes/ssl/kubelet-key.pem; \5 E6 g4 \+ v7 d7 f! A1 y/ I9 K X6 L
scp /approot1/k8s/tmp/ssl/kubelet.kubeconfig.$i $i:/etc/kubernetes/kubelet.kubeconfig; \
! x7 M/ m4 U" P) ^6 i7 @7 _; B& Escp /approot1/k8s/tmp/service/kubelet.service.$i $i:/etc/systemd/system/kubelet.service; \+ H X _$ J- _+ T4 q& X
scp /approot1/k8s/tmp/service/config.yaml $i:/approot1/k8s/data/kubelet/; \6 s) v4 d# G! p6 n% V$ l+ ?- s" K
scp /approot1/k8s/pkg/kubernetes/bin/kubelet $i:/approot1/k8s/bin/; \
7 M! h9 Q/ z* S# s' |5 ~3 V/ @done
% m: a: w% o. G; m3 d# m2 b启动 kubelet 服务
. {- ^4 z9 a) b V4 }, |for i in 192.168.91.19 192.168.91.20;do \
* J# P9 \2 ]) n) {2 C! Pssh $i "systemctl daemon-reload"; \
% O0 x/ |8 p: h6 x7 T( {! Sssh $i "systemctl enable kubelet"; \
5 f$ B$ v" x( G( {7 P% F: n( Fssh $i "systemctl restart kubelet --no-block"; \
1 b2 S. y6 Q* g# Z* f! k' _! |ssh $i "systemctl is-active kubelet"; \
7 {% N4 u' ^' n& k0 vdone' R; ?; Z- j8 ~ v5 ~- A9 [4 g
返回 activating 表示 kubelet 还在启动中，可以稍等一会，然后再执行 for i in 192.168.91.19 192.168.91.20;do ssh $i "systemctl is-active kubelet";done
* P1 a$ [$ O# z$ S L" u7 Y5 p
" p" m8 i! ^! A( E- @2 E' `返回active表示 kubelet 启动成功: ^ i6 H7 W1 ?1 a' Q. U% B3 y! C
5 A- b* F7 [" R# X! \' Y
查看节点是否 Ready
5 S) I& f, B* b+ [kubectl get node
, [6 P- G0 V: e6 L! |预期出现类似如下输出，STATUS 字段为 Ready 表示节点正常
- S" f" E" B1 ^1 G% X, v1 e; n& b3 }! f
NAME STATUS ROLES AGE VERSION# }2 L7 P! v# U& f
192.168.91.19 Ready <none> 20m v1.23.3
' ^9 m+ V [9 T8 R/ h. k192.168.91.20 Ready <none> 20m v1.23.39 H) K6 ^ N/ N ]7 \
部署 proxy 组件* g; Y% w+ [: `
创建 proxy 证书
; j' a3 I p/ c; o! D7 a7 }9 }+ qvim /approot1/k8s/tmp/ssl/kube-proxy-csr.json
# N: Q8 Y( c4 V" k* s% A8 t2 L* E{
, \! s8 m2 O3 x "CN": "system:kube-proxy",
; C. f' E- `' D0 |+ e: _ "key": {
- N1 z G2 v! J: k "algo": "rsa",9 {7 G w3 }# w& V
"size": 2048
8 z7 `2 r" ?2 ?" s. n" W },
3 M5 \ O% L, J: `3 M0 e "hosts": [],- B2 @: o( N+ @* e% x9 b1 M
"names": [
3 X9 j- T" K$ P! t9 N" X {
2 j( O- H6 `) D9 y "C": "CN",
# G* P$ o5 p& ]' V7 V, D6 {- O8 C "ST": "ShangHai",
% f3 D* D7 ^# O* J "L": "ShangHai",
" M' |$ M" G+ O& T7 o9 d "O": "system:kube-proxy",- p6 |1 k' f1 v t% W$ P8 }8 r! D8 g
"OU": "System"2 n. K+ ~/ o4 j. P0 I5 E
}
, F+ \; @: G' O3 H/ X& s ] ]
$ y# z4 R3 {. D2 n9 q}) \9 Z/ j" h+ G3 B- T
cd /approot1/k8s/tmp/ssl/; \" E1 z8 M; t4 O2 `) u* c% j) J1 \
cfssl gencert -ca=ca.pem \
4 e1 [ k! V- I' ?, O-ca-key=ca-key.pem \9 ]. |% m/ V4 R: M
-config=ca-config.json \
V0 l" `8 L! I: c-profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy5 N, o6 [4 t7 w6 } x
创建 kubeconfig 证书 C- R8 l- B7 Z8 b' S! ]8 U- f$ l
设置集群参数% Z9 U. J% Q3 E/ L! E! e: } t
" z6 _6 f$ k+ ^5 p, e9 V9 r5 n
--server 为 apiserver 的访问地址，修改成自己的 ip 地址和 service 文件里面指定的 --secure-port 参数的端口，切记，一定要带上https:// 协议，否则生成的证书，kubectl 命令访问不到 apiserver5 N- r* D8 O" g( q" v% ]/ }& Z; y
- I" d) p$ F8 l. e9 w
cd /approot1/k8s/tmp/ssl// }3 L# A+ X8 b% |5 s- ?* }9 y( M
/approot1/k8s/pkg/kubernetes/bin/kubectl config set-cluster kubernetes \
: b- U) G5 Q' R+ t$ c--certificate-authority=ca.pem \5 }1 [5 I7 X: [! M7 |7 v& }1 Z
--embed-certs=true \" X- G" Z6 \$ i
--server=https://192.168.91.19:6443 \
% G4 S* M6 n3 \% }/ P& d# j. Q/ l, Z--kubeconfig=kube-proxy.kubeconfig
. r% g! y1 n/ M# f' b& n: ], I设置客户端认证参数! S6 a& \7 X- p2 ~1 E
% W1 ?. p& L9 J* Gcd /approot1/k8s/tmp/ssl/
) y% _9 r6 u$ o9 r1 y/approot1/k8s/pkg/kubernetes/bin/kubectl config set-credentials kube-proxy \8 y. R- A& }. h* Q
--client-certificate=kube-proxy.pem \
2 r3 B/ k+ |- K5 ?: l--client-key=kube-proxy-key.pem \+ i) O) Y8 Z2 e; b
--embed-certs=true \
4 j, h# `2 l4 |& w3 z! G--kubeconfig=kube-proxy.kubeconfig* u: a* p: N+ j$ ~
设置上下文参数2 K+ @& t6 ~5 ?- d$ |" M4 D
% E3 r0 |% Y2 @* i' n+ @+ e2 zcd /approot1/k8s/tmp/ssl/
+ X" A! y7 }# c' U) R/approot1/k8s/pkg/kubernetes/bin/kubectl config set-context default \
' F+ C) P' I; u; ?--cluster=kubernetes \- b, a5 ~, H2 f0 {: S* K7 e
--user=kube-proxy \* k I! P% p t O0 q# j
--kubeconfig=kube-proxy.kubeconfig
7 s, d+ n" R, c1 f* P- E设置默认上下文
, G/ f( g" R3 X) b7 S# {* t: C& b% y, d9 N4 k# c* _
cd /approot1/k8s/tmp/ssl/
- q: V3 R$ ?" P. T; a1 w' \/approot1/k8s/pkg/kubernetes/bin/kubectl config \
7 [6 n+ O# m, q6 [7 Buse-context default \3 `+ @$ S: t! ~2 ?" G! r. Y
--kubeconfig=kube-proxy.kubeconfig( i" E4 `$ ~0 U5 b
配置 kube-proxy 配置文件
! d# j8 U8 j' L1 E; vvim /approot1/k8s/tmp/service/kube-proxy-config.yaml.192.168.91.19) }% O' C: l$ s$ T
这里的192.168.91.19需要改成自己的ip，不要一股脑的复制黏贴，有多少个node节点就创建多少个service文件，service 文件内的 ip 也要修改为 work 节点的 ip，别重复了
0 @1 k( c( `8 h) V' g" o* v" D1 g2 C, B ~0 B6 E: {* X
clusterCIDR 参数要和 controller-manager 的 --cluster-cidr 参数一致
3 |- i+ V% C4 ?7 f' o* ]3 b2 I7 k) B' [1 A. ~3 J( \0 P! ~0 d( h' q
hostnameOverride 要和 kubelet 的 --hostname-override 参数一致，否则会出现 node not found 的报错
I' \/ }7 W% i
# p; g# n8 S0 D; o6 r7 S4 F. S& Okind: KubeProxyConfiguration+ V2 f. U$ O: @% }* H
apiVersion: kubeproxy.config.k8s.io/v1alpha1
1 x2 ~* e' v3 Z+ n) g$ c, e1 rbindAddress: 0.0.0.0
0 P$ W V. P8 EclientConnection:
5 O+ a- C9 b* g kubeconfig: "/etc/kubernetes/kube-proxy.kubeconfig"
* q: u1 g0 S1 q: A$ tclusterCIDR: "172.20.0.0/16"$ {4 c, D# l$ u( n, T: n
conntrack:+ H7 r" t A' N1 s5 _ ?8 x
maxPerCore: 32768
|% z* ]4 y. |( V' p+ r min: 131072
: N2 D4 z. e3 T9 I1 O! j# m tcpCloseWaitTimeout: 1h0m0s
! r5 T7 o, F8 d. l; j h& v tcpEstablishedTimeout: 24h0m0s
+ j' A) I% G# i5 v. R6 B9 T9 ghealthzBindAddress: 0.0.0.0:10256
# V! W' |* c/ ?1 j3 R2 t5 N3 thostnameOverride: "192.168.91.19"# g( V3 S9 f" g8 T1 g; N; u
metricsBindAddress: 0.0.0.0:10249. Y3 ^5 J7 l. Z! u
mode: "ipvs"
- v% Z, R) X0 h配置 proxy 为 systemctl 管理
! ?; H' x( ~3 C( [5 M. L( R1 @vim /approot1/k8s/tmp/service/kube-proxy.service" C( o, L. @, `2 m( j, d9 \1 b( e u1 ^
[Unit]2 j; [4 Y* \" o: ]& y$ U* v
Description=Kubernetes Kube-Proxy Server
8 ^; N* S, U7 P0 \* J6 eDocumentation=https://github.com/GoogleCloudPlatform/kubernetes
. c. |+ f' \' _0 E; b `3 S: KAfter=network.target: @) n5 J) I N5 v( \" V4 ]0 {$ C
0 ]" N, {* L/ m. _1 V
[Service]
! S* ~9 A1 y1 p x1 d) v# kube-proxy 根据 --cluster-cidr 判断集群内部和外部流量
{4 r$ _ V6 \) X## 指定 --cluster-cidr 或 --masquerade-all 选项后" L( d5 `1 {: v' P
## kube-proxy 会对访问 Service IP 的请求做 SNAT
" ]. O8 `( O3 Z vWorkingDirectory=/approot1/k8s/data/kube-proxy( y" v, `: V& R) |1 L1 G0 r
ExecStart=/approot1/k8s/bin/kube-proxy \1 Y1 \$ m5 C) V- {0 ? a" O
--config=/approot1/k8s/data/kube-proxy/kube-proxy-config.yaml
( O Y5 ~) N% e! I! {3 ]7 ORestart=always1 Q/ S& j: C- F/ B
RestartSec=5
+ \" x$ M$ b9 s. m( MLimitNOFILE=65536
! v( ~/ X# y8 j5 b) h: `8 B. L. w- p6 K4 L/ X+ [) ^- L
[Install]
- C9 t( n- K! H" }: Q9 |WantedBy=multi-user.target* p' N% Z7 e" W) r9 u2 F
分发证书以及创建相关路径! `6 Y+ q7 I, U) m
如果是多节点，只需要在192.168.91.19后面加上对应的ip即可，以空格为分隔，注意将192.168.91.19修改为自己的ip，切莫一股脑复制
! K5 t5 _% m- \* z, b5 e! ]
: D9 H" e* m9 _% R) d, h( l) x对应的目录也要确保和自己规划的一致，如果和我的有不同，注意修改，否则服务会启动失败
0 J/ |0 B. u8 G" @
, p/ ~* D; c9 K9 \1 Y1 kfor i in 192.168.91.19 192.168.91.20;do \7 P) Z( O+ i( v& {. t2 z2 c( j
ssh $i "mkdir -p /approot1/k8s/data//kube-proxy"; \4 Q5 N) E/ q; S
ssh $i "mkdir -p /approot1/k8s/bin"; \* J7 r4 F6 X E; I, X& n
ssh $i "mkdir -p /etc/kubernetes/ssl"; \( e, i5 P0 ]1 U8 ~" x' ~% d. ^5 j
scp /approot1/k8s/tmp/ssl/kube-proxy.kubeconfig $i:/etc/kubernetes/; \
* X: Z2 e3 L! i8 zscp /approot1/k8s/tmp/service/kube-proxy.service $i:/etc/systemd/system/; \; B5 \3 u. R, ]* i9 `2 V
scp /approot1/k8s/tmp/service/kube-proxy-config.yaml.$i $i:/approot1/k8s/data/kube-proxy/kube-proxy-config.yaml; \
& h6 P" ~8 M2 Tscp /approot1/k8s/pkg/kubernetes/bin/kube-proxy $i:/approot1/k8s/bin/; \
$ c3 c. t m" o% B7 P ^done
' V7 ~5 F: c! q$ U启动 kube-proxy 服务, F# `) `; K" u( X3 W" Z
for i in 192.168.91.19 192.168.91.20;do \
/ Y) P7 E8 |( U/ Z7 U, Mssh $i "systemctl daemon-reload"; \% ~ A6 n c( g/ A
ssh $i "systemctl enable kube-proxy"; \
! y, Q1 N. A' W# Y* ossh $i "systemctl restart kube-proxy --no-block"; \
3 X9 t& h- k4 N+ V/ t. Cssh $i "systemctl is-active kube-proxy"; \* l( f, C' r! f& S/ s; ?8 t
done
; W6 Y0 X" n" a5 b6 k返回 activating 表示 kubelet 还在启动中，可以稍等一会，然后再执行 for i in 192.168.91.19 192.168.91.20;do ssh $i "systemctl is-active kubelet";done
3 E0 q2 w: H1 W3 _6 l* |- `
; m& \, k6 D* k返回active表示 kubelet 启动成功( J( ?, U s# F1 X0 E( E8 T
0 k$ B; ^" u: L: h; v
部署 flannel 组件) O: r9 q8 v( P8 e9 p" p3 x# v
flannel github
' Q: f" H! q R. y6 d* U8 T. K5 w B: R1 _" O6 ?% y* _" @
配置 flannel yaml 文件; n6 N7 B/ t6 l
vim /approot1/k8s/tmp/service/flannel.yaml. N r( |7 u* r. P; T
net-conf.json 内的 Network 参数需要和 controller-manager 的 --cluster-cidr 参数一致
; `, w$ s/ K; N; h. c, B9 W
# E; x' ]$ u4 b---8 d( [! @8 Y3 b( ?) j
apiVersion: policy/v1beta1
! b# P& c. ^- _0 j( O+ ukind: PodSecurityPolicy* E. |$ q* G% ]$ v3 D A, b
metadata:% r& U% o$ H8 {& k$ o* h. D
name: psp.flannel.unprivileged
1 A b; g# |! N annotations: b c6 J7 t% u+ P* v
seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default) j8 e) o4 r6 q- l/ {5 S
seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default. B& q Q, | D/ }6 T
apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default
) n" A7 N4 B" j$ H apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default) w4 W8 O$ _% f5 n& W0 ~
spec:
# m* t/ K7 Z- S' N: Z8 } privileged: false; T I" T. U9 f2 @, n4 I1 R5 X
volumes:" x. M+ s& i0 f; @4 ^$ y
- configMap" f, g+ A9 g9 B/ \$ x N6 u& @
- secret/ O9 G1 J3 F4 H" P6 P# r2 j/ A
- emptyDir2 v. m( z3 t5 f6 p# ?: Q. \/ O; a Z
- hostPath
9 Q5 c N7 \9 x1 a allowedHostPaths:8 w( J& b* V% s( p A; U
- pathPrefix: "/etc/cni/net.d"
) l$ D. k; S$ x1 ]( y, T - pathPrefix: "/etc/kube-flannel"
1 B2 [; A2 N' i `1 W; o - pathPrefix: "/run/flannel"8 o+ i; }' `3 u( V; K. _7 ~
readOnlyRootFilesystem: false
$ ~# j% b- I! K1 Z! [ # Users and groups
1 D; X9 B! R9 F runAsUser:& O7 t) w& r4 q( K* g ]% b8 l
rule: RunAsAny
* I# a- ]5 v1 _. S7 b supplementalGroups:8 R6 D# C4 P5 C: `
rule: RunAsAny, p6 k+ L! y8 X" n: j: }+ J
fsGroup:# a s2 R1 i4 } J
rule: RunAsAny
L& ]2 G% s- [$ r G. _ # Privilege Escalation1 j6 K- t. M# n y0 [6 A. }
allowPrivilegeEscalation: false4 @6 F% ?& w5 U. \, F' B& P) b
defaultAllowPrivilegeEscalation: false
/ {9 S9 k! ^; `- m+ k) C9 {+ t) L # Capabilities
$ S. G8 ?8 e; z& \3 l: N allowedCapabilities: ['NET_ADMIN', 'NET_RAW']
: ?3 q$ Y4 m" z5 m- E4 W defaultAddCapabilities: []& b; D3 V1 [. E) `
requiredDropCapabilities: []
% B# c1 f# L+ S& G7 v7 { # Host namespaces) v5 {& C9 u6 A9 }+ g
hostPID: false
0 c4 h& g: R3 y2 ? hostIPC: false
* Q3 F3 ~; `$ e* M3 i0 F% l hostNetwork: true2 R% w9 N* z3 U! P, n
hostPorts:. b3 e- r( B% j5 C L- m
- min: 0
; s- ~& l2 o6 J8 F3 l. a* T K max: 65535
: U6 ~% @$ W1 _' u # SELinux; _5 l& h& I; [* g
seLinux:
! X! N5 Y9 X" P9 E" q # SELinux is unused in CaaSP
( R/ U& v, _1 \- r rule: 'RunAsAny', b' P4 \2 m9 ~; Y
---0 c& ]/ W; G- d% L
kind: ClusterRole
6 v' I) N& \6 s) m' e+ Z9 X/ i0 YapiVersion: rbac.authorization.k8s.io/v11 g2 N" ~) {: x( D
metadata:& V0 m1 g- T& ?+ }& L
name: flannel1 Q T( \) t, s( E! u; @0 T
rules:
8 @, q% m4 q- W9 }1 @- apiGroups: ['policy']
. z; l$ O& I/ |9 H- Y: G7 p( ~ resources: ['podsecuritypolicies']
. b/ q" [. o- z( A0 y verbs: ['use']2 j9 w' j4 m" b+ l, `8 \
resourceNames: ['psp.flannel.unprivileged']- [5 r1 G2 H5 a& ]8 }) w$ N: k
- apiGroups:; E- s% V1 t( B9 s
- ""
, J+ H: ]9 c% R* @0 L( ` resources:
! U0 w; `. R. L3 r. |& O0 t, Q - pods
# h/ [# U$ m4 k$ A* q* | verbs:4 H( x+ j9 m# j" `; B* ]) _. y: K0 ]
- get1 B) G* [0 V6 P3 B9 W5 N
- apiGroups:* F0 \. d- A- U9 t0 [8 Q2 s9 t
- "") {5 w7 s8 q( {* w
resources:/ l( F/ a1 R7 D. a) [, C% U9 @& J
- nodes
; n/ i. x0 @4 U0 k$ d$ O verbs:. E0 s# N' u) `* U4 U
- list6 d) M% p4 I, C
- watch \/ n2 S* k- M2 b/ O% I
- apiGroups:( K8 i8 O2 L/ q. P9 W
- ""
2 D* H7 q# j# T! M* m# Z resources:- {6 S# m6 g) O! ]& p+ P u# Z
- nodes/status
7 m8 R+ O4 v: [! p( E% n verbs:
2 R( ?% _4 T: U8 X4 ]: u - patch
7 O, _ q' y2 _3 w7 l' ]---+ S1 t/ k2 f- i! x) f; I- r* B
kind: ClusterRoleBinding$ G8 ?1 K' @% C% c; p, D c; j
apiVersion: rbac.authorization.k8s.io/v1+ j8 E* v7 c1 U" w( [3 U
metadata:
/ w- n1 M5 j: \+ E. E. u name: flannel
: r+ e% r0 o8 l- d( R( f* groleRef:
, s" f3 d; Q0 z7 T! i1 Q apiGroup: rbac.authorization.k8s.io
: e h% x' s( w+ V3 p, X: j kind: ClusterRole* g) D. H; ?0 G/ m2 Y: d
name: flannel( v/ [4 D* P9 R$ h2 Q8 E
subjects:( O+ R2 Y: K* w1 O: H
- kind: ServiceAccount
0 b; Q" n( E( M5 F, u name: flannel6 \4 d& x6 s+ K, y( b
namespace: kube-system- V( }: \% X) g. J9 x4 v( Y7 \
---
2 W7 Z3 n+ ?9 @1 W* eapiVersion: v1* L% [4 q P0 }, ~! F
kind: ServiceAccount
/ J, x% H+ {2 h1 [! ~7 Dmetadata:& E& l1 s | y) E3 a: @" S
name: flannel
+ F+ q7 v& }, [' {( {8 L namespace: kube-system0 R* |5 r* P% k1 J% l
---$ F6 E* X5 S1 j( S2 b
kind: ConfigMap
! y0 A" Z- l% R3 F; OapiVersion: v1& E# i0 C* D- C- g* ~
metadata:& v. s% G2 c) ], R2 Z7 H. R
name: kube-flannel-cfg6 J3 s: A+ C5 v8 {! y' H
namespace: kube-system
& {. n3 d6 e W( i5 N& i labels:
5 o' q/ U, M8 W: ^6 V/ d tier: node
! d3 _- X0 Y8 I# n0 V- Z app: flannel3 Z& Q0 M, Q: y& R* c4 t2 m
data:, W, {% Q0 D2 ]$ d
cni-conf.json: | o( a# p8 D0 n( v) ~
{
8 ~; h' ^; o0 Z/ Y4 A "name": "cbr0",; j, {9 u4 y; h. Z
"cniVersion": "0.3.1",
1 m7 G9 t2 _$ u5 z "plugins": [/ {, g5 r5 `7 l- R* V" D3 \
{( {6 M6 m# g5 H7 s: X% }
"type": "flannel",
; @. c: @' a6 p4 B9 x3 N "delegate": {1 E8 F% s1 B5 z* L
"hairpinMode": true,
! r$ T3 x" b0 f "isDefaultGateway": true2 M: X) m1 c% J
}! t) M, Z- `# N( g) b4 ^
},
$ ]/ Q' O5 R4 E& n% R {! `; n/ r Z6 l! }# q* p
"type": "portmap",: e' W2 {! v. V. U8 v9 A" J0 _. W
"capabilities": {+ [; ]) e9 U% r/ ^9 i3 u
"portMappings": true
. ]5 d4 {- H6 v u- J: Y }/ T2 M9 Y7 x* W& k
}
$ g8 y8 l) f7 S1 s' b/ Y ]' r, j0 m. x* C" S7 o! p- e
}
3 T7 Q9 Z7 h- L7 u) W net-conf.json: |3 h1 f; \- b4 g8 i1 H
{: {% S: e7 D/ @; Z8 n
"Network": "172.20.0.0/16",
; ^$ x f! _3 x; c "Backend": {7 s8 G# g; S/ G J# ?
"Type": "vxlan"
1 I+ ~* c# R. }9 |, r/ ]. W4 ` }
8 R6 T! T! K4 q3 H) s }( b0 U9 u7 H5 r! G
---2 n7 ^3 u' A( d6 p
apiVersion: apps/v1( f9 F9 Q; B! x$ O: ~
kind: DaemonSet
8 S/ k4 _9 u+ g2 c5 Nmetadata:
u7 u1 L$ T8 j% `2 ]3 V( ^ name: kube-flannel-ds$ w$ s0 Q8 M0 y8 ^* T- r7 E3 T, B8 {$ |
namespace: kube-system+ Z: F' |* N: R+ h. m& b4 I& [2 M
labels:* a. U. m: n: e9 q C/ A; G
tier: node8 v9 n( N' u. ~4 N
app: flannel
7 M% v: M) Q# ^3 f4 G7 v' Pspec:
/ F# O, W( Y: y1 d selector:) c0 i% J0 h. z9 X
matchLabels:
5 O5 A6 @& n' h( H/ ?! l* v4 O app: flannel7 x4 i) b/ c+ Y+ [# l# E
template:3 |' W# Z& u& d$ c5 d
metadata:
/ A+ a3 w3 i- V4 W- o; k O% X labels:
: k4 t% U3 C! d- t) A tier: node
u! T+ G) r: w app: flannel
2 L% S# }( u) {7 h6 ] spec:) ]# Y) f( N/ x4 Q" g! j
affinity:- G1 K6 d% D8 F6 L
nodeAffinity:
1 v% \$ h `" `: h requiredDuringSchedulingIgnoredDuringExecution:
* Y9 }6 v- L. C1 h nodeSelectorTerms:1 c' C2 o) z4 B$ F$ ~4 o: ^' V
- matchExpressions:
" H& W* g& \: J( |8 U. s4 N - key: kubernetes.io/os
# s- D `# _" }8 ~2 ^ operator: In
; ~; n2 C# ]& h6 x7 { values:
) [+ f- {, V/ m; K3 {; ] - linux0 o5 n% @+ ~4 d7 s! h4 Y
hostNetwork: true6 \% p- K2 W' o+ v
priorityClassName: system-node-critical
4 o! W! }5 m- [/ B. y- D& t4 v tolerations:
. U+ H; t; C* W6 W2 V - operator: Exists5 L/ c4 N# v/ p% s3 V5 C1 b- A" U+ G
effect: NoSchedule g6 a! D# v* e5 H
serviceAccountName: flannel
1 k* W2 e' Z8 Y& l7 B1 g, S8 y6 A initContainers:
# h8 f- Z" q0 W1 r+ V - name: install-cni
+ j4 O0 i/ @; F7 j9 A: p image: quay.io/coreos/flannel:v0.15.1
9 ]$ M" A8 F. Y" C _ command:
# m: q2 y2 S; l9 }/ ] - cp! ^& S6 m5 c" |1 ~+ M7 H! z
args:
( t+ W* q, m( G8 T* E: V - -f0 L5 H! X! J$ x
- /etc/kube-flannel/cni-conf.json. w$ d- {6 x( F2 I. p
- /etc/cni/net.d/10-flannel.conflist/ z7 f$ |/ [% h$ d% g' Q
volumeMounts:4 ~* X# ]5 C v
- name: cni
7 g( z; Z/ n }8 A0 a mountPath: /etc/cni/net.d
/ L$ D6 N8 r" j+ R - name: flannel-cfg7 X, |. q4 p( w9 d7 _2 o7 U1 M1 U
mountPath: /etc/kube-flannel/
8 S- G+ e* c5 ^+ [" X! u* B containers:8 f: h$ p5 J+ u3 f+ j
- name: kube-flannel6 Z! f* u5 U( i! }
image: quay.io/coreos/flannel:v0.15.1. N. m2 ~8 _7 A# H4 d
command:, A% ], J* E$ \8 G) c# s
- /opt/bin/flanneld
, F/ S; E0 \& r args:5 ~$ D" @+ E% s# a) _6 y# r( \
- --ip-masq
5 b' u4 U: R b+ z - --kube-subnet-mgr8 ]7 O( y7 f! y: k( q5 ~! a. _+ j
resources:' B# l E! H+ M1 l O8 x
requests:
E4 n6 L4 w* k9 ^! c; ^' b cpu: "100m"
8 H& {) Y5 p2 [, r" J v9 I) p memory: "50Mi"
6 j+ o( ]/ {: |& C limits:7 u S) q- Y! | F% P* @
cpu: "100m"( a; F2 ]4 ]" b1 o
memory: "50Mi"1 {* m" R9 t- F! t3 A, h
securityContext:
/ e4 ]4 l+ x/ h/ I privileged: false
, y+ M' b- e1 B2 H, L* m Q1 r& P capabilities:( `6 X) E* l- ]
add: ["NET_ADMIN", "NET_RAW"]
6 K; C8 G% N/ y3 F% h env:& o9 x; m4 ]! u. b' Y
- name: POD_NAME
0 X! M5 T9 R' `& g valueFrom:
) D* C; J/ {0 m" g8 k& F) ]" P* ~ fieldRef:
1 u- S: T: A# t fieldPath: metadata.name0 p. n4 d0 R7 h) [. [- C. P
- name: POD_NAMESPACE
9 u! l, B. B5 E( z valueFrom:7 Z' u/ [( H% \$ T+ ^
fieldRef:- V& y; Y: S) \: q* Z, Z
fieldPath: metadata.namespace
1 S0 G- l2 D' Z6 T% J volumeMounts:
4 ~/ j- G" ?4 h0 I - name: run
1 f( B O9 ^+ U8 m mountPath: /run/flannel" K( F" N% V5 k1 B- P; f
- name: flannel-cfg* I1 A% {8 B- u8 Q
mountPath: /etc/kube-flannel/$ r1 s$ {1 c+ `6 w
volumes:
* j: z% G/ r$ T( w - name: run4 h1 O$ c8 [8 f3 O% M% _
hostPath:
' p6 l" \9 G" D9 | path: /run/flannel) {5 e2 ~- W+ R5 W' X9 @( {# ~4 @
- name: cni+ h4 n9 V. M* u* u M; |! l. q
hostPath:9 i) T! ~. W( m8 t& Y2 Z9 ?/ c
path: /etc/cni/net.d
0 @/ @2 V' v& i+ J3 ?# L5 z - name: flannel-cfg
# ^8 I- V# y7 n6 v J% B$ h' F configMap:6 H) H( w7 P/ ?! L3 e( F5 m0 y N% c
name: kube-flannel-cfg8 N, R% v7 \5 t" l- Z
配置 flannel cni 网卡配置文件
1 y" f; f1 H: `- o" Hvim /approot1/k8s/tmp/service/10-flannel.conflist& b/ `7 g. P, ]4 p
{* [, e7 e; I8 ^7 b1 D# Y7 l k. c d+ f' ]
"name": "cbr0"," C0 t' b4 P- E" C8 P& G3 N1 z
"cniVersion": "0.3.1"," J3 j6 z' f# L+ I+ m4 g
"plugins": [
/ X& V% X' ^# e9 a {" i0 i [" d7 U5 N7 a
"type": "flannel",6 P R8 u- z; i& x
"delegate": {7 r# h- T, L( x) S* B
"hairpinMode": true,
; `# K0 u: ^/ y, r6 m "isDefaultGateway": true! ^" P$ y9 l6 j8 C0 K
}' Y3 r8 p/ w2 u7 T+ j7 `; y
},/ ]7 k: l9 k& e+ J& g( I4 U
{6 R# y& A5 f2 r* i* x% z6 b
"type": "portmap",# Y0 Q" u* D9 l8 X; x1 Y7 M
"capabilities": {4 t! o# a3 K! a( A
"portMappings": true
, n3 ~9 [7 Y8 y0 g }9 v. W% q" \! E0 ^: t' f
}9 y8 _( Y d; N% S
]
) B1 P, ?# a8 L3 A, q0 W) \}
2 y" r" V! Y: J- c0 P. I导入 flannel 镜像& Z W8 |0 P1 r0 q
for i in 192.168.91.19 192.168.91.20;do \
( ?7 z( A) C/ ?6 H" t; p7 Yscp /approot1/k8s/images/flannel-v0.15.1.tar $i:/tmp/" b) a% t# u0 N% G, h
ssh $i "ctr -n=k8s.io image import /tmp/flannel-v0.15.1.tar && rm -f /tmp/flannel-v0.15.1.tar"; \
! Y' Q7 O! b- B+ O j+ w: L. T" M& Vdone7 b: D6 d; |1 O6 @
查看镜像+ T* E! o7 ^( H/ M9 h/ a
8 r, i) F- N; Y d a* s
for i in 192.168.91.19 192.168.91.20;do \
: @: N0 c- V0 M$ @* {- ossh $i "ctr -n=k8s.io image list | grep flannel"; \
9 A5 |( u: F8 l. kdone" A- _. B7 V3 I& W1 l- G! F
分发 flannel cni 网卡配置文件$ @5 ]9 b, S) J2 W J5 k
for i in 192.168.91.19 192.168.91.20;do \) y5 `; G+ b. v; ?3 N
ssh $i "rm -f /etc/cni/net.d/10-default.conf"; \
- w0 r+ ~0 y M' M. Escp /approot1/k8s/tmp/service/10-flannel.conflist $i:/etc/cni/net.d/; \
3 D* T& a$ s; ^, {$ bdone# P) \& ~ W+ ?$ V' U0 \$ Y
分发完 flannel cni 网卡配置文件后，节点会出现暂时的 NotReady 状态，需要等到节点都变回 Ready 状态后，再运行 flannel 组件
" \, [9 y: m5 k+ x& p* }( i* j0 q# X" Y$ c* ]# p' }2 Z
在 k8s 中运行 flannel 组件- ?: [2 M' x% J9 B8 _
kubectl apply -f /approot1/k8s/tmp/service/flannel.yaml4 K2 ?+ t) h% t0 l5 t0 H
检查 flannel pod 是否运行成功
8 B. f+ H+ }( j6 J0 _! @kubectl get pod -n kube-system | grep flannel$ A/ j" I& g0 B6 f1 J* {' ]8 G
预期输出类似如下结果
3 [) T9 w$ E, r% J, o. s6 Z/ C. m& K% x6 {% f
flannel 属于 DaemonSet ，属于和节点共存亡类型的 pod ，k8s 有多少 node ，flannel 就有多少 pod ，当 node 被删除的时候， flannel pod 也会随之删除
6 }! Q0 s& [7 H
4 a7 T$ z Q2 Y) M6 W# ukube-flannel-ds-86rrv 1/1 Running 0 8m54s
7 } @5 P$ w9 D8 \1 r& n* ]kube-flannel-ds-bkgzx 1/1 Running 0 8m53s7 n5 Z( \( l- C9 l! a! g- @4 p
suse 12 发行版会出现 Init:CreateContainerError 的情况，此时需要 kubectl describe pod -n kube-system <flannel_pod_name> 查看报错原因，Error: failed to create containerd container: get apparmor_parser version: exec: "apparmor_parser": executable file not found in $PATH 出现这个报错，只需要使用 which apparmor_parser 找到 apparmor_parser 所在路径，然后做一个软连接到 kubelet 命令所在目录即可，然后重启 pod ，注意，所有 flannel 所在节点都需要执行这个软连接操作) `& F/ Z/ C) y- W- n1 F, ?
1 M. w$ @# P# ^/ p- m
部署 coredns 组件5 ?+ A6 K8 N% y$ L @/ I0 ?
配置 coredns yaml 文件) e& [( T, Z+ L3 J1 S) Y" }7 @7 A
vim /approot1/k8s/tmp/service/coredns.yaml) k, E+ ~, l- n9 W# z; A
clusterIP 参数要和 kubelet 配置文件的 clusterDNS 参数一致3 ?# o3 r1 L1 [% A( w$ k
$ Q( g9 |6 _$ _' R8 J4 j0 j" j
apiVersion: v1
! o9 V" @. `$ @5 `7 Y& `0 m$ akind: ServiceAccount
, s" m4 A$ x4 y7 Kmetadata:
5 X2 N# i0 u, {) x- H5 E name: coredns6 s3 W4 g4 v8 X; V4 F$ S
namespace: kube-system- S" ~" Q7 b1 J# n8 g2 t
labels:0 W8 J* E8 ~. {! t* P: ^
kubernetes.io/cluster-service: "true"
+ `6 u0 t0 s+ e5 X. M addonmanager.kubernetes.io/mode: Reconcile
9 f% e5 T' E& H1 `4 D9 {. |---
" b7 T! W2 @' Y9 W# HapiVersion: rbac.authorization.k8s.io/v1) l! g: u9 M) w, l
kind: ClusterRole
, J' I+ K- W0 ]6 [) Bmetadata:9 N5 P/ ]% ] s
labels:) B0 r$ Q, w% y% G% E& i
kubernetes.io/bootstrapping: rbac-defaults$ H7 Y# T) J! }: O
addonmanager.kubernetes.io/mode: Reconcile
9 Z/ }6 h4 ^& Q! r4 O/ \3 A name: system:coredns' u' J! a, v4 t- S
rules:
: f; }# |9 A+ s* J( R: V$ S P- apiGroups:
6 c r; w/ t! w: Z+ y# L$ W - ""+ M+ t/ F( V4 B6 B
resources:4 L8 ^- p, H! f( R$ o
- endpoints
) _! }. X, W4 `1 B* Q/ } - services
- d) Q) ]6 A1 J2 y - pods
2 H R' b% M- \ - namespaces
3 s( J# t: B! z0 g2 B0 \" i* S1 q verbs:
8 [/ R8 `, K+ A) ~ - list7 Y3 y9 t0 Z8 c6 R! I; J
- watch; H; ?: z& A1 H$ @% `& j: `
- apiGroups:6 X- g4 ], K# K, P
- ""9 w& K! X2 G; ~8 w2 U4 F
resources:
8 k! m- A5 {! M; B- F" } - nodes( I8 C8 b- j- w p0 S" z) c, H
verbs:
8 G' I/ R( H& Q - get/ s+ [6 U! B q8 X k' n( h5 E7 s
- apiGroups:8 k: t! e* J" \. v- u( Q* B. l$ N& }
- discovery.k8s.io$ ?0 T. Z1 T, b& \
resources:: e0 }# J& P, a; v+ Z
- endpointslices
( |0 U4 _! v- q3 l/ g: g! Q/ p verbs:
9 R$ b) [! q3 |' ^! M) \) B" u - list/ R. z- y: [' V+ F' P
- watch
+ j" }! l) c8 R' R0 a! a3 H---
+ k1 H0 f0 `* y1 RapiVersion: rbac.authorization.k8s.io/v1, l5 q b l9 T" ?
kind: ClusterRoleBinding
: L" M4 n8 e% Cmetadata:8 {6 x9 B/ l/ i+ C/ U6 v# \; K
annotations:
4 V( e9 j/ |" q8 \: z) q) Q rbac.authorization.kubernetes.io/autoupdate: "true"
/ ~9 x9 j# U; u5 D k0 ^. L& v labels:
8 Z9 U' l% Z) B. s kubernetes.io/bootstrapping: rbac-defaults
0 C3 o. C+ S: w: R addonmanager.kubernetes.io/mode: EnsureExists
7 ~1 S8 q& R& f' r3 m6 W3 y9 } name: system:coredns
: `' u v8 _: R* i" D+ m& v+ BroleRef:
' W U4 V& r: C$ s5 e; ^ apiGroup: rbac.authorization.k8s.io* c, q0 c, @$ @, j( X
kind: ClusterRole
$ U0 V8 w& D$ x3 i$ ] name: system:coredns
, B8 |1 u3 s! S' U& Fsubjects:
/ C7 o8 ~: f" i# x* h- D6 y- kind: ServiceAccount
' E' w% d; K& _: W9 t6 }. ? name: coredns
) n. O" ?* f7 m, d! \$ m namespace: kube-system2 n' a0 j) D5 R% I" T0 A
---
( {3 p( @, C, y4 J# f/ X3 @apiVersion: v19 v7 s* ~0 t8 c; W7 N. W
kind: ConfigMap. q: o! l3 | T; }- F
metadata:
/ h7 N6 j" ^* _9 B! S, | name: coredns
, H! H5 g5 b0 d3 _4 l namespace: kube-system
$ y; j# G7 G3 M: A labels:
4 {0 l' i3 U3 X$ q" x) W& I addonmanager.kubernetes.io/mode: EnsureExists" |0 R8 b' h- H3 j c. M E+ K
data:4 Q! k# f3 E! F8 V6 r7 Q6 x
Corefile: |
/ W& j9 ~2 c4 C( \8 c; H& N .:53 {
4 W4 A2 C, B8 }$ D errors
, D$ V2 S" o" x* r health {: I3 T9 e2 }: @' _
lameduck 5s
( c% A, K/ f& z$ `, B+ `9 W }5 S4 d5 h7 J$ Z1 x
ready
( m: W* B4 y% V: M5 B' k: O kubernetes cluster.local in-addr.arpa ip6.arpa {' `; G9 a* z _& ~& W
pods insecure& ^8 }: q& t0 O9 g/ O3 L
fallthrough in-addr.arpa ip6.arpa7 W, L, R# ~6 L/ m7 q
ttl 30
' _* A5 \+ Z5 g z% q. q! @ }. j. d- d4 _: b1 d& n0 `
prometheus :9153
$ R) o2 [, \4 S. A1 m5 A2 l# F5 S2 y forward . /etc/resolv.conf {
9 M# G# ]( K0 p# m! D5 v0 F% ?9 r max_concurrent 1000
/ n$ J' Q; G' `0 V }
7 ^3 d( ?+ }) p* y& Z u: m) k cache 30# v3 ?6 l& F# H9 b. N" N; O
reload
4 C( w* u5 {8 _) g7 | L loadbalance
0 ^& v# X4 L/ G2 R }/ s7 R% c) Z/ R! r( b: u
---, ^* r, Y- r* O
apiVersion: apps/v1; L2 l! C9 x3 l
kind: Deployment
2 Q8 ?. G+ r1 `metadata:4 j5 ]- y" s' j4 }) ?. b# [' n
name: coredns. g/ Q1 J) w! H$ @/ G g) ?
namespace: kube-system7 W7 o P$ W2 f% b
labels:1 o2 Q* _' l( M$ y/ j5 B5 t
k8s-app: kube-dns
6 u. J+ s& e9 w kubernetes.io/cluster-service: "true"6 l$ D0 ~3 z5 _
addonmanager.kubernetes.io/mode: Reconcile
6 R+ l0 c% l/ [! s0 v# K& t6 S* h kubernetes.io/name: "CoreDNS"
5 U3 g* X/ r- aspec:* G1 ?# [" {: r/ I
replicas: 12 ], [+ r+ T- V8 t6 M ?
strategy:
' b1 T2 ]% x* g9 O( l q* V$ } type: RollingUpdate& Z( H: a E0 C2 K- c$ e4 l8 h
rollingUpdate:) t( p" F6 v6 n6 _% Y
maxUnavailable: 1
. R2 N$ L, R# K# Z0 t6 P$ e1 ^ selector:
; P+ K; R. o& B9 |1 J' K6 J matchLabels:# b% V6 r" w: p) S% f" P3 t
k8s-app: kube-dns
l! i! g) ]5 t H, s4 j* q template:
; [4 o, W( H+ f1 w8 G metadata:
/ w" U. W4 l, j3 f) k labels:2 v; h/ H; f" h7 x1 R
k8s-app: kube-dns/ S" Q' L) ^5 S# s h! H" F3 i
spec:
" h9 f" d, s! S3 w6 I securityContext:& x7 {7 {4 l- n3 D7 A$ y
seccompProfile:
' W% M* [; Z1 v! N type: RuntimeDefault
& z0 |8 d* p$ r1 R' C priorityClassName: system-cluster-critical; i0 I: r8 G& u8 V d3 x B
serviceAccountName: coredns$ E/ z' c6 }, I
affinity:
3 A2 A. D7 A7 f8 S podAntiAffinity:; J1 M1 h: ?' F8 x B5 @& M: H
preferredDuringSchedulingIgnoredDuringExecution:
9 W1 P( D. @2 m9 c9 l# e; g8 v - weight: 100
- p' E5 s5 P8 l% U G# h1 F0 _4 a) g podAffinityTerm:
& K! T1 V2 E" O* i) R8 E) n2 X: i labelSelector:
0 p) p. k+ N: I( v matchExpressions:0 s* B9 C3 V/ t: ]% f5 R: Y6 J7 C1 K
- key: k8s-app
2 a, T; R, G9 Y# u, i operator: In
6 n( v" \3 e/ e( F values: ["kube-dns"]
|- {* U) J/ I+ A) M7 e topologyKey: kubernetes.io/hostname+ A; v# L d9 P4 |( u+ ]' S
tolerations:
8 {0 E! w" E1 L5 g! M8 J - key: "CriticalAddonsOnly"
. u7 `6 H! o2 f4 q6 v8 w! F8 h4 j operator: "Exists"
/ q* \% H9 m& ?/ b7 c x& _+ B+ Z nodeSelector:# V/ x7 ^* b" W9 i& v. D4 ?
kubernetes.io/os: linux2 z m3 U1 `, n. g' T; O, s* j
containers:% B \) }' v- v& L
- name: coredns
% {1 E. e, r5 ~, R- o) E& {7 i image: docker.io/coredns/coredns:1.8.6
% j" h5 N# F; f, ~, g% O imagePullPolicy: IfNotPresent% O% |- R; A3 \+ v
resources:4 k% ?9 g+ A' ?5 S- U
limits:
! \$ h2 ^' G$ v6 U! x. s memory: 300Mi
" N* F1 }) h0 k/ P6 y6 e3 b3 F requests:# T7 X+ l5 J' J- ?; E6 a
cpu: 100m
. @6 g- s& T5 N memory: 70Mi
7 W1 X! Y! }4 ~% p T/ q3 ?6 [ args: [ "-conf", "/etc/coredns/Corefile" ]: v7 D9 I2 k1 W4 \: J
volumeMounts:# |' |$ P1 N. U
- name: config-volume
: J$ O* i5 t% b# M+ s) m mountPath: /etc/coredns$ Y/ J. e3 s# m
readOnly: true
9 {; A, R( j1 ^& u ports:
6 w3 e3 X3 V4 R% A/ M, F+ Z - containerPort: 53) m3 V* k$ D4 D" V
name: dns5 D, e" f' H' c/ G' H* p$ L
protocol: UDP: x. ]" L2 ^$ E
- containerPort: 53
, s( [2 z7 y; _! P0 v% o, H; G4 ~7 } name: dns-tcp
0 x* m7 j2 Y6 r& r; c protocol: TCP
' L0 }$ g" V/ E8 L - containerPort: 9153
2 X5 S# Q2 K& r+ t6 `7 w name: metrics/ x6 I2 C! G5 s, j0 c' A; ?
protocol: TCP8 K) U6 V) t2 U4 M" [1 r! P: m
livenessProbe:+ B# m! o; r) r/ `, v* ]4 o
httpGet:7 ]- J$ D& y( ?
path: /health
4 d+ Z, ~4 r! v' A$ s port: 8080
! u( }3 K5 V3 v, N: L scheme: HTTP8 A0 v) C3 j3 k/ t
initialDelaySeconds: 60
' d3 J# P# s% A* T; r0 a O timeoutSeconds: 5
8 c4 V+ x; G9 d8 D5 x successThreshold: 1
. _* ~. w* z2 b% j* E# A/ W failureThreshold: 5
7 O- V: @5 L8 [ readinessProbe:
4 m5 `; f, f! e, Q httpGet:7 T+ m1 ?" x& x; M
path: /ready3 M3 i; v) k4 G; a' w4 i" C
port: 8181
% D0 x. P6 [% U8 q! L scheme: HTTP e. A, k& S7 w2 e: p9 ?/ a* o
securityContext:
7 G5 M- p0 m0 Y6 V- ?1 h+ p% M allowPrivilegeEscalation: false F& g- v/ B, _+ k
capabilities:
. H$ a# D: q9 a' N2 U0 ~6 i add:
0 w$ h {. J' \4 g. a8 H - NET_BIND_SERVICE
$ J- C" g; ~/ r; T; H drop:9 m5 }) P; o9 y5 k, j
- all( W8 k$ g; T! W
readOnlyRootFilesystem: true9 Z) a/ s, R1 U; E# M# f% e+ M
dnsPolicy: Default9 l3 e4 i1 y, g f! i6 V9 ^
volumes:6 L/ w: b' R; [( f
- name: config-volume
; m3 {& T1 S1 e3 P4 {# z configMap:
4 q2 x# i3 d& h2 d: f( ~ name: coredns
/ a, C5 j1 h- a5 T. Z L5 J& c' O items:
5 t9 w% }1 A. @. J; u" Q+ x: R - key: Corefile+ s. U1 |$ N3 O5 {) [: |
path: Corefile- L" G" [6 A# B$ \1 Q. B5 h
---3 g' V1 j1 U. d) g s) Z
apiVersion: v11 a7 N2 a9 y: {8 P, P
kind: Service
: C& s4 G; i" G3 |& |metadata:8 @" i1 `" ]& Q! |; U1 |
name: kube-dns" ^' z3 p' S, O$ _9 X* \+ N
namespace: kube-system; I* d% j P. y5 M
annotations:$ |& L: i+ u) h2 V7 ^- T. d
prometheus.io/port: "9153"2 Y- @+ h+ b1 X% h0 w3 a* l
prometheus.io/scrape: "true", o* [) c8 r5 _& b7 ?
labels:3 K7 l5 e" I& @& E5 E: V
k8s-app: kube-dns
9 m3 R) F* U" j2 x kubernetes.io/cluster-service: "true"# U! o. j" ?' r# ?* \% C
addonmanager.kubernetes.io/mode: Reconcile
$ A: u i3 I/ \ c1 u* C+ W kubernetes.io/name: "CoreDNS"! K- j# {6 h2 ~5 Z- K
spec:
, r7 Z$ S& C# |4 d4 N9 F( x selector:' R* ^7 }0 g3 @! `5 Q
k8s-app: kube-dns
. O4 ?, } y- t) k! v$ X clusterIP: 10.88.0.27 q9 j# q; }; L
ports:1 N" z; |$ h# c9 ^$ G. ? A
- name: dns0 s; E) c& B0 }1 B( t
port: 537 V5 S- w% G1 F: a$ g
protocol: UDP
! _6 i- W( J9 P3 v - name: dns-tcp
. H! B8 k& b2 G port: 53
. g7 ?. H l% O7 t/ K6 Z protocol: TCP2 H1 s7 [8 j1 g) s4 k
- name: metrics& a3 X# `, T8 d$ m& l, v; W
port: 9153
1 c) r5 a' V7 t& n, C protocol: TCP
, c2 X# u) B, B( t" [导入 coredns 镜像
, i! T0 \! ~, c" g Zfor i in 192.168.91.19 192.168.91.20;do \5 Z7 f( @" P2 K
scp /approot1/k8s/images/coredns-v1.8.6.tar $i:/tmp/
- M6 [0 |5 P0 a' assh $i "ctr -n=k8s.io image import /tmp/coredns-v1.8.6.tar && rm -f /tmp/coredns-v1.8.6.tar"; \: N7 _0 ]% u) z9 l% K
done q$ d! g: @. d7 o2 }' U! @
查看镜像+ w5 w. H+ V8 [. n o U; F
- U7 v5 v& s1 m8 L0 O7 V- f; Y
for i in 192.168.91.19 192.168.91.20;do \
& G% {4 i2 {8 S1 r$ i0 h( Issh $i "ctr -n=k8s.io image list | grep coredns"; \
6 W7 K3 l# M& I, a* y# N* edone
- w) g* _* n/ S4 a9 l在 k8s 中运行 coredns 组件2 }% W9 ?3 t+ B, j
kubectl apply -f /approot1/k8s/tmp/service/coredns.yaml
; ^" c3 M& t/ h检查 coredns pod 是否运行成功+ t* S3 x5 ^1 T' o! R, I1 N0 B
kubectl get pod -n kube-system | grep coredns F+ L( i+ |, A) B2 h. D
预期输出类似如下结果
9 M4 \" { b, C1 e1 ]$ o* I
: h3 g9 o$ ^- f4 p g因为 coredns yaml 文件内的 replicas 参数是 1 ，因此这里只有一个 pod ，如果改成 2 ，就会出现两个 pod% }' l% Q: N: B# t
5 x0 v/ G9 p( @
coredns-5fd74ff788-cddqf 1/1 Running 0 10s
. M3 S' j7 k8 ^: J' H# ^: p部署 metrics-server 组件
Q7 D4 M6 u7 x1 f配置 metrics-server yaml 文件- G$ X! l0 q- o, z& m: j
vim /approot1/k8s/tmp/service/metrics-server.yaml& y, @9 o# A; d
apiVersion: v1
, {& b# ]; t0 W7 zkind: ServiceAccount4 e5 v2 s7 d+ K2 ~* Y" {" V
metadata:6 `) g, v. w/ }& j/ ]) G. f* S, w
labels:: S* z: T3 R! s$ L% L7 U
k8s-app: metrics-server
- j# L E, E! J% I! L name: metrics-server: @1 c1 t8 a4 w* E$ c
namespace: kube-system
* ^ p5 y6 o+ ]+ |- x9 n---7 N/ f6 f( L, S7 a" T9 A
apiVersion: rbac.authorization.k8s.io/v10 m) j v" S: w
kind: ClusterRole
3 N# y0 m- y8 G& j2 [" O& e( n# N6 Dmetadata:6 V; W' }6 Z8 @& [. c
labels:
9 H u+ v$ Q, z+ C" \2 C5 M k8s-app: metrics-server
0 b+ J$ e3 U5 }4 Y$ D8 x rbac.authorization.k8s.io/aggregate-to-admin: "true") A, Z2 J7 X6 Z& T
rbac.authorization.k8s.io/aggregate-to-edit: "true"8 }, h& H8 J: O9 Q) Z0 B
rbac.authorization.k8s.io/aggregate-to-view: "true"! J0 {( p' t9 A+ g5 Y+ S
name: system:aggregated-metrics-reader
- J' D8 w' T7 B: g0 E% I# trules:
; x4 p2 s8 P9 t9 q( X+ `0 p' N- apiGroups:
5 ]2 C6 P4 P. v- h3 ? - metrics.k8s.io
2 }; x1 x/ F2 Q3 L resources:
2 `0 Q V6 F3 A5 g7 W - pods
) Y- f: L5 i9 R% v# L - nodes. I2 N M8 A, ~" x3 \% a4 P
verbs:* H2 _/ @' P) S, n" g. W( x1 v
- get6 H- H5 e& U% {9 k$ C6 J
- list
+ @1 Q5 S; i: b6 n ~3 A0 a5 k - watch1 \/ b" n, N' d1 h, n
---
1 T; @4 Y/ ?! j( GapiVersion: rbac.authorization.k8s.io/v1- N0 R/ g! ^' `3 X9 ~% |
kind: ClusterRole% R8 N4 H; I7 Z4 B
metadata:& L+ z) t$ Q. J. ^$ |: R6 D
labels:
) w9 g6 {4 q, P k8s-app: metrics-server
; X& x U7 e6 G3 A6 I7 R8 P name: system:metrics-server" |! m/ k# c1 T. _; k% i
rules:
0 X( Q1 p, M. A) {0 J% J: A- apiGroups:
+ V t, \5 Y# _$ f5 Z4 B2 u - ""
$ I9 Q0 [- D& F2 R" r+ J. q# U resources:
4 j" j. Y, Y3 h/ [" z - pods
' q& b( r9 G3 [9 f$ V - nodes
' ?5 d! _8 u# G! g - nodes/stats
9 H Q5 m8 l& ], r' m# N - namespaces! t- ^. K0 d, f5 B
- configmaps
! K$ X" y7 T4 z verbs:
$ r; _. `' d5 f6 `, M - get
7 S/ \) U& U, w$ G - list' k: g$ _6 B4 m8 Q R
- watch
7 Q9 d' d5 H# w" ] @6 Q: ~---. D! s8 I, I0 h; F& V" p( ^3 n9 ~0 e
apiVersion: rbac.authorization.k8s.io/v1
7 O+ W5 u$ ~3 P- wkind: RoleBinding
" w; a, q. { v. `8 C xmetadata:
- Y9 T, o6 @* N labels:
8 ^1 r0 B2 u. ]$ C w k8s-app: metrics-server
) T7 l# q1 g( b. X7 s8 F name: metrics-server-auth-reader. n, O" O3 g/ Q5 n+ m4 f
namespace: kube-system, x {4 A! F5 F9 v. z, j: s
roleRef:
" C c/ @0 |) J c ^8 e7 y& t3 q apiGroup: rbac.authorization.k8s.io6 B/ s6 D ^1 ^ j2 J+ M
kind: Role0 n) ~5 o, o& U3 s9 c
name: extension-apiserver-authentication-reader: P8 @* f1 Z- Q4 n
subjects:( X2 m# U$ ^/ V/ t
- kind: ServiceAccount* {: y- _" {" W" D* t
name: metrics-server
5 U) k- Y; c, k$ B namespace: kube-system R0 C/ v+ `6 n9 B
---
! [3 k; e* y5 w9 Y4 r. f+ xapiVersion: rbac.authorization.k8s.io/v1
% `' |$ O: R& {- [% b: x" Vkind: ClusterRoleBinding
$ b X* w! Z6 }5 A7 W+ |4 kmetadata:
4 K, _0 k6 V: z7 h5 _" L% t labels:; t+ W) c% c: W' g4 Z
k8s-app: metrics-server
4 m" }) D: E& w; M5 \( P' Z name: metrics-server:system:auth-delegator! z. D; o7 T+ j8 [% ]' M
roleRef:
- ?+ R0 c) g$ z, O8 L% N( c" D) @ apiGroup: rbac.authorization.k8s.io
2 ?5 \% M: Y# Y: i2 `8 ^) h kind: ClusterRole' T! [5 g2 Q5 P7 f! X- i& B3 L
name: system:auth-delegator6 b- |0 b& }7 o- T: ^$ l2 F3 q# b1 Z
subjects:
_' a7 N9 g9 G1 v- kind: ServiceAccount( Y# w$ g+ C5 C- H! S1 o
name: metrics-server
1 b5 Y) s$ l" Y R namespace: kube-system, @ C, Q" ^$ X- F. ^' ~. ^5 |: s
---0 X8 Y$ G# z: y( C' ^0 ]
apiVersion: rbac.authorization.k8s.io/v18 c" P' Q8 G/ l: E7 a! z. W
kind: ClusterRoleBinding N3 b; S5 u. W5 \
metadata:6 j& |+ Z/ e, k @) b
labels:# ^( S0 T0 ]: y3 ^8 }8 L- Y: I
k8s-app: metrics-server
& r0 {& V. b4 F9 d* m name: system:metrics-server
: F; P% O7 w2 N/ proleRef:/ k$ s2 v/ p! H& n: X+ D: ?3 d
apiGroup: rbac.authorization.k8s.io
7 r! i# f* I8 D8 n8 Q9 {+ q- P kind: ClusterRole
5 a% o* U, t$ w8 T name: system:metrics-server
% B" j" F R* K! R# J6 ?: ~subjects:: A% G. ~/ p8 t0 ], |2 X
- kind: ServiceAccount, A& ~5 u! O! E( M" N. r
name: metrics-server% s+ N, v8 \- p! M& w# k: v" Y
namespace: kube-system
8 c v$ o6 |3 J3 r( d) a f3 i' x# A---! C+ O8 ~7 I2 S# N2 p" ]; b
apiVersion: v1
$ j! G d( u1 d- y+ J! M/ [kind: Service- G. }! g# d5 ]' ^
metadata:" F: [, t' R$ q3 x! u
labels:
+ M! T6 N: O5 U k8s-app: metrics-server4 {& w' R- ^! K/ y* a
name: metrics-server9 x/ A4 D1 u4 D0 y
namespace: kube-system
* f5 n6 y4 [( k3 G8 @spec:
3 T2 g/ {% N& I" L" j* ^3 t9 k. g, L ports:
# q9 }1 k0 b( @, h: ~ - name: https9 X! R1 B g1 g+ a4 B1 U; d
port: 4437 h, J. ?) M# P4 f/ c
protocol: TCP
5 n/ Q# ^; } w: r. T4 n0 G- F targetPort: https* k- i( p0 \: E+ _/ D
selector:6 I; D- `$ _* `* ]8 l
k8s-app: metrics-server" y1 G S" c& ^, s. X$ U4 D* l3 r
---
; c+ j8 c1 U9 g; TapiVersion: apps/v1: G/ P- v6 V$ T0 |$ }+ A
kind: Deployment* t7 u( }+ c) w
metadata:
V2 F3 ~3 n& h' Q6 s labels:- i, |6 J- h, U1 R- I0 }: g& {; p4 P
k8s-app: metrics-server
3 c5 K1 [& L0 d! A: Y" o) B name: metrics-server
# N0 L7 E" C( l- Y* n8 M# Q namespace: kube-system& m) z1 a; [1 ^, y( F1 J( @( _5 m
spec:
+ A) v$ c6 V F/ N" e selector:2 d: m0 v' j1 C6 X" y3 E
matchLabels:
( p; W* ~* _ f2 Q" r k8s-app: metrics-server0 T: ~8 t; ^, {9 x' o5 F
strategy:( ?' q7 w) n6 F
rollingUpdate:
0 V6 f0 a' N/ s maxUnavailable: 0
; @: ]. v0 Z% }* F' X) E5 X template:, z( }9 K l; i6 r3 W4 n$ M6 r2 @' i
metadata: O. }. r% q$ B. u& b7 R. z; q
labels:
7 l1 Z7 ~# _0 y( t/ l; n+ o k8s-app: metrics-server# \7 `2 `% {" q9 \
spec:
/ R# f; u' y( r3 j containers:3 r# N! Y& N A) o& v: }1 N- W
- args:
k) H9 M, t; @/ \ - --cert-dir=/tmp) q t5 o: ]+ d* i
- --secure-port=44434 b+ O1 @, @0 y& M
- --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
4 n% `/ P; q6 b9 g- a - --kubelet-insecure-tls
& r) x& U1 I7 ^- ^2 W - --kubelet-use-node-status-port
/ D4 B6 U; x& H% \4 i! n - --metric-resolution=15s8 w# j- }! ] T+ y) M* N7 v
image: k8s.gcr.io/metrics-server/metrics-server:v0.5.2
3 }% `1 B8 j7 c+ I' ~" _9 d6 u& R imagePullPolicy: IfNotPresent" [# f% R7 r$ T
livenessProbe:6 b( T+ }6 B1 g4 R A- m
failureThreshold: 3
0 v& V6 ?5 Q4 b l httpGet:' R5 u3 m o; ?9 L
path: /livez
; E# Z- c* w* g+ u, v* h" ?8 K1 j port: https. ^& B, V4 l0 ]& S1 W
scheme: HTTPS
* Q- R# v! [! E4 U, V periodSeconds: 10% J/ u1 v% f& S/ H( ~0 ?7 G! \
name: metrics-server1 V, G ?0 \0 h- a" j" W- @: M
ports:
" J6 G) K( R' |! Y, U3 i - containerPort: 4443% d* \* a! t1 d, d7 ?' {+ c, \& G, m
name: https
6 C4 r% h5 |' M# ]3 ^2 P protocol: TCP# }' m. z1 m7 a4 c& @/ k
readinessProbe:
. ?% O) \9 x. k( f5 T3 f failureThreshold: 3- R* |9 {/ J6 A) F
httpGet:
' r- D5 `5 J9 K @4 c1 f path: /readyz
# G; |- q$ J }! ^: c port: https/ D- i, \+ g" |* f8 E1 r* D. Y
scheme: HTTPS; s& ^+ V- I2 y( ]* x6 U
initialDelaySeconds: 20. [# E( h( B. i3 p
periodSeconds: 109 L2 r1 [7 V$ o0 y% f
resources:
4 ~5 Y0 I4 m3 U9 f$ b, L M requests:
) S3 C2 Z* n$ g# D( q cpu: 100m! D8 \0 u' J$ h
memory: 200Mi! Y7 e7 ^' J/ d2 Y, I m- i% p
securityContext:
% W- b2 G) i9 A) ~. s readOnlyRootFilesystem: true
2 z9 ]" p# B6 e0 B7 R runAsNonRoot: true/ f7 J+ W2 A( F0 _& W1 i, W
runAsUser: 1000
- M# {: E# F Z; ` volumeMounts:% X$ c$ g! I+ V$ c. f: b. D/ Z! i' a
- mountPath: /tmp
7 O" I" ?% t, w" ?7 Q3 |9 ~- { name: tmp-dir
; d8 L# N- j1 E3 M nodeSelector:
( |8 Z: L( L h6 I) b6 x0 P- L o7 A kubernetes.io/os: linux# U+ x' f: ?5 m1 F5 Q* X! y
priorityClassName: system-cluster-critical
i0 c; P9 }9 `7 I8 _: _) {( N serviceAccountName: metrics-server
" h) Y3 I3 Z) y volumes:
" s$ J @9 a7 Z1 Q' _ - emptyDir: {}
6 H7 F$ A: k2 a3 H name: tmp-dir K7 T& h+ P0 G8 _
---5 ? g1 p& o) _$ u
apiVersion: apiregistration.k8s.io/v1+ _6 s2 C7 }; [6 \8 n
kind: APIService
6 M/ G" j" X8 w( u; a1 z4 Nmetadata:
" Z% k7 O9 n; G# d labels:
7 f0 W* D% w. s$ `1 F. e) z k8s-app: metrics-server
, C8 |% Q R5 z4 L. @+ }& x name: v1beta1.metrics.k8s.io
( ~7 h. }9 L- D( A8 u5 r! @+ ^- v4 g9 zspec:
; b, n: a" i3 P1 @8 m5 p# { group: metrics.k8s.io
" Z+ d1 |; ^, Q% E- l groupPriorityMinimum: 100
1 p; {% g4 l, ~$ g. ~$ a insecureSkipTLSVerify: true( J3 o- t& s. n+ n" K Y
service:7 f2 L7 g+ X* A# o; v
name: metrics-server
/ v" A2 F: _( E+ U namespace: kube-system8 ]6 ^3 ~$ l, f! N
version: v1beta1) X# D2 w* m# N: p6 i9 z
versionPriority: 100
& z- o) e0 |9 K5 W2 N: T导入 metrics-server 镜像- ~- t3 ^) O1 O; Z$ a0 b
for i in 192.168.91.19 192.168.91.20;do \$ V' b3 q: N4 A! X7 i
scp /approot1/k8s/images/metrics-server-v0.5.2.tar $i:/tmp/
2 |0 O$ U( e& \- C6 E0 bssh $i "ctr -n=k8s.io image import /tmp/metrics-server-v0.5.2.tar && rm -f /tmp/metrics-server-v0.5.2.tar"; \
+ s. m# O. F9 R! `8 N" Fdone t+ _4 I7 k! ~0 N" H) n
查看镜像5 T1 |) {% \! T5 Z3 L, N; o
) h; p8 M5 E7 W/ w- _
for i in 192.168.91.19 192.168.91.20;do \
9 Z1 E3 O$ a) B3 E$ ussh $i "ctr -n=k8s.io image list | grep metrics-server"; \) R b Z9 b' V
done
" I& ?5 G n5 ^) [在 k8s 中运行 metrics-server 组件
( S0 x2 b! z% E9 z! L4 qkubectl apply -f /approot1/k8s/tmp/service/metrics-server.yaml
+ I0 H- }4 m4 K) {0 X检查 metrics-server pod 是否运行成功" P- J9 S9 N9 v$ I; L% c
kubectl get pod -n kube-system | grep metrics-server
* X3 w0 P! m' R, Q' B7 j- F预期输出类似如下结果& \2 Z" {- b( J0 H% c9 R
( u& w% F) Z6 V$ D( Q7 j( E" m( G
metrics-server-6c95598969-qnc76 1/1 Running 0 71s( x6 g" D+ W) K4 J4 G7 s* |( `9 w
验证 metrics-server 功能
8 z" h' I6 Y$ A0 P7 H2 S2 q7 c0 R3 n* ?8 U; O. {9 r
查看节点资源使用情况
/ f* u4 f9 q' G: Q# e0 {& y, A1 k+ Y+ P# q/ J
kubectl top node
- L, x8 d% c+ v6 k% }预期输出类似如下结果
* ~1 G/ p! ?/ q8 s" r- m
. ^8 _4 ~* a8 m! P. [" c, Tmetrics-server 启动会偏慢，速度取决于机器配置，如果输出 is not yet 或者 is not ready 就等一会再执行一次 kubectl top node
5 l$ H& c+ I5 H0 `5 s, N |6 r6 y1 \
0 d2 o' C; F' ONAME CPU(cores) CPU% MEMORY(bytes) MEMORY%% X# D3 O) K+ Y5 N4 n* @
192.168.91.19 285m 4% 2513Mi 32%& E+ ~* X0 E+ S2 Z
192.168.91.20 71m 3% 792Mi 21%
! ~6 l( n& n( s! p) {查看指定 namespace 的 pod 资源使用情况 l2 e' H; p. F6 }
; r. ?9 k. P3 F7 q6 P
kubectl top pod -n kube-system" R; C9 H8 W- y( F! T% |
预期输出类似如下结果7 O0 M; \' d4 J
) M/ F3 \! l( ~9 ^# L- WNAME CPU(cores) MEMORY(bytes)
, D- J8 r1 o6 Z( j' dcoredns-5fd74ff788-cddqf 11m 18Mi
, z2 g7 E) Z9 \- X, Tkube-flannel-ds-86rrv 4m 18Mi+ j' d& c1 p+ r N; Z' t
kube-flannel-ds-bkgzx 6m 22Mi \8 {+ R( U; ]8 [% p6 T
kube-flannel-ds-v25xc 6m 22Mi
6 Z4 k( V. ]& q) p p; z/ i# Fmetrics-server-6c95598969-qnc76 6m 22Mi: [, \, F1 w. {6 m1 t
部署 dashboard 组件9 }( ` W6 N$ n6 W+ o$ m
配置 dashboard yaml 文件
; b4 R: {* o1 G& e5 P! pvim /approot1/k8s/tmp/service/dashboard.yaml
/ ?% t# u9 m9 c& T( L3 X---' {+ \: I2 A+ } s8 U/ O- X
apiVersion: v1
$ s# _0 x4 Q: n- v1 H, D9 S; Tkind: ServiceAccount7 l+ L* l- V. o7 h6 H
metadata:
( E- G: P' n; {9 Z4 M C) Y, t2 _8 ~' E name: admin-user
4 J3 ]$ U. N% a0 q namespace: kube-system
8 ?' Q2 J* C0 c- o* P4 w# y! Z Q
---
( {9 n2 V1 \0 s6 V9 ZapiVersion: rbac.authorization.k8s.io/v1
& H# m! S2 D7 a- wkind: ClusterRoleBinding
. }. {, r$ w$ n9 M" t! ametadata:& ^8 b* H, [7 v: R, I0 h1 y, o
name: admin-user8 X! I! M6 H' ]1 d8 ]" ]* r/ K
roleRef:
( z& D X# J: Z apiGroup: rbac.authorization.k8s.io N$ l) Z5 S( [7 h e* i6 S+ x
kind: ClusterRole- ^ E/ \# @; A/ N" |
name: cluster-admin
9 k8 _) P( q% e1 P* t8 w# Lsubjects:+ Y9 M7 N5 f$ m
- kind: ServiceAccount
& f# e. Q( T- ]& U2 n name: admin-user
4 Z$ {8 H7 w4 x$ K) A1 E7 I namespace: kube-system3 {& J! Q9 g9 I* |
# _2 t- R- o6 r e# n# r5 h2 M---
4 v* R, Z. u7 B7 ^' }* WapiVersion: v12 {# V( R# O0 y3 m
kind: ServiceAccount, I6 ?( K) r, A7 r
metadata:
# u8 H) ^& R5 U7 M* h name: dashboard-read-user( @5 E2 |/ W, M$ g
namespace: kube-system
( s+ q; B0 z1 q, N7 g2 y
% Q; r1 B' Q, B0 o' |1 g% I" D' a---. H8 G1 [) p" }0 U
apiVersion: rbac.authorization.k8s.io/v1
{2 B, a2 ]) n$ hkind: ClusterRoleBinding
I* V0 z( V! Imetadata:4 H8 v* _* H; [1 ]! V
name: dashboard-read-binding
/ i, K- o8 v5 y% rroleRef: W' @' s" \1 Y' C" r" y
apiGroup: rbac.authorization.k8s.io
: o) ~' \- u* t+ F9 G3 E% E kind: ClusterRole/ a$ B4 C- N" @$ I
name: dashboard-read-clusterrole! [3 _% K4 w- B o# a6 w" V6 {
subjects:: y8 u; a+ G7 c7 @! I3 d& x
- kind: ServiceAccount' E6 N" p. M5 u
name: dashboard-read-user
3 e U% a# O$ D4 f* ^( j& X2 Z: P6 f namespace: kube-system
5 S/ v4 h9 p' H- U; C9 x. F+ A' O$ T+ x& s
---, Z2 p1 n3 ?* H
apiVersion: rbac.authorization.k8s.io/v1
8 U* \* h* E, D' Z" _kind: ClusterRole
) }2 x5 C; M' L6 e+ K8 @4 `, Y5 kmetadata:( H* r0 y5 A4 ]/ Y- L& O% p
name: dashboard-read-clusterrole
4 |' q) R- m$ trules:/ Q* ]8 G+ r7 J
- apiGroups:
8 R) h: k' F) H- K5 f - ""3 G2 @5 I: q" ?4 s
resources:" w& @8 F% s3 X1 s0 ]( T# w$ k
- configmaps
7 @$ Q& \& r7 d& G& E S$ C2 F! r - endpoints: F2 L5 p7 u6 r( K) N& f& U
- nodes
% ^! L7 J2 {# G! T - persistentvolumes. J' E8 E- W8 Q* H: r/ J4 z
- persistentvolumeclaims
! w5 @1 ?% q h( ?2 X1 Z - persistentvolumeclaims/status( h/ p( z6 m8 s+ v
- pods* x5 o0 e9 q, D
- replicationcontrollers
0 N& T v6 B7 y7 z9 B - replicationcontrollers/scale
@* I9 K# {) n: s' J/ P+ p - serviceaccounts
" H( L6 X# x6 B! G% o, u* U - services
+ J4 d2 v1 w1 T+ l. k - services/status5 H* Y+ O% B& e' N/ R' E
verbs:6 x2 S7 u8 H- v+ ~, Q& \# \) X
- get: H6 T& t! s" @- K, R1 w
- list1 a& D5 G. _0 K& x% ?9 B
- watch
" w! {% G f7 J% t& P# V- apiGroups:
) M+ }! H* y7 P+ r4 x - ""5 X4 w8 d# c, l( D- g: F
resources:% E D/ P& i! {4 c
- bindings1 P" z2 N+ `0 N) F5 o% k$ P8 K
- events
1 c/ o, n" p, C4 {" X( o9 n - limitranges
: @+ m, s. O7 O' w& _: [ d - namespaces/status0 A2 c) Q% r* J2 D j- V4 Y9 t
- pods/log
% h3 O4 E! ~) t. x" W9 E- N - pods/status
& o3 X! h" M/ ~ - replicationcontrollers/status
' X: t/ D- j$ w) m. }3 a0 k6 |! p - resourcequotas* K z8 w1 ?, U1 r! r* y: K
- resourcequotas/status+ {2 G3 A# M3 I; S6 M4 E9 v& v
verbs:9 Q. f2 o# a: F! s4 n9 n5 C, ~
- get- a: ~7 s! c! t% g
- list
$ P Y: \; V2 C9 ^ - watch) Z8 n& g6 S* E7 k: A3 c
- apiGroups:
& u" L1 G$ |( d9 } - ""
6 l F) C8 a1 U1 R: O$ H resources:: w, N( s+ w/ z
- namespaces
, N+ A) s& y/ J4 a/ ] \, @ verbs:; a! q' r5 |; }, g I9 e) j0 b
- get# T, h% F8 \2 a# C; O
- list
7 V4 N& S* x r w3 l - watch
! M0 o% w1 F8 |% Z3 F2 y- apiGroups:
5 d$ s! t/ Y9 L+ b0 n - apps
( k* T9 v0 k% r. D# C resources:' q# v3 q2 o/ e$ l$ p7 m! B) Y) v
- controllerrevisions9 }7 Z. [- I# w! P$ b
- daemonsets
1 R+ B- L4 l& ~+ i) @ - daemonsets/status9 G- |/ w* l7 B1 F# E' {
- deployments: X' i; c% X6 U
- deployments/scale
( x; E8 D/ {( D4 P; s% p/ c - deployments/status3 ], t2 A1 g; _ M6 _: g1 E
- replicasets
, b2 E" M! v# |8 j. r- I - replicasets/scale
; i' p* a4 h3 Z- X: t - replicasets/status8 Y1 J; m; ]8 v/ {
- statefulsets! r" \$ X5 y3 C8 W; b# F8 N
- statefulsets/scale
7 b& [* H& P Q - statefulsets/status
: W5 ?5 t* _ C verbs:
$ r' s) f" k) G4 t - get
: F6 X0 w( w" ~! H) p - list" L# Q# l( F1 ]
- watch, ~8 \2 P+ o# o8 n; j( p- o
- apiGroups:0 T- V1 R, o% A8 e
- autoscaling. c* D; N( O. j1 _. X' n2 P
resources:4 k4 K, M5 `, G: f: _9 l
- horizontalpodautoscalers- w. i. o! h7 v! e' \
- horizontalpodautoscalers/status
& c2 b6 D) b2 ~+ B, b verbs:% t2 q3 c# b! \5 {
- get
: H, G( G \7 S6 _4 e4 A - list2 L" I9 B& ^+ }
- watch
: K# J" v! D7 P- apiGroups:8 a" `. Y- ]+ `/ U
- batch
* a0 m$ ?& @& j resources:2 J. o* \3 @4 T x6 W7 |; ~
- cronjobs) n* W8 m! x6 Z- P9 o. A
- cronjobs/status( c8 X) P$ V! A' }5 |
- jobs
" @7 P5 K' X% Z- E3 @ - jobs/status8 y: V! t8 J! ~7 d% z/ y$ _0 {% i+ G
verbs:' `& o; g3 R$ V7 L- X3 J
- get! K9 _0 A* p2 b$ Y
- list, ]( F L4 Q: x) v1 k! T. n
- watch' c o0 g- e% J) q
- apiGroups:+ h, T4 ^# A4 \% r! U
- extensions
9 v* k9 j- E* [( \2 L( Z$ [ resources:
) n! t1 J* [9 [1 z - daemonsets0 \4 }6 y' b4 U; ]3 N. `
- daemonsets/status
2 ]- {7 W- u6 y# r+ e - deployments
+ k+ z) F* u2 m6 u - deployments/scale
4 p: ^8 x y( H, W2 |$ b; U( D% G - deployments/status$ n( g3 _ _( h
- ingresses0 ~. T- O4 A' v4 m1 Z
- ingresses/status' ^6 U$ O4 A8 C X2 y5 p
- replicasets
; m1 l" _3 F; a0 i - replicasets/scale# Z) `- j( y' @% C4 f
- replicasets/status: M* u5 e* t! _% b3 P: C- x
- replicationcontrollers/scale- ]# u4 R0 @6 F3 P+ V, e
verbs:
8 d" R1 J& R5 I: H/ y - get! z4 t& t- b" G$ C1 Z; x
- list
/ S8 u4 \5 I: T1 x; k# Z - watch
( L1 L' k1 \5 N1 {- apiGroups:: E5 t' H7 B( q+ P
- policy H7 B$ m9 k" c3 E/ j
resources:
1 R* }# E# W/ [ - poddisruptionbudgets
3 Q+ N: t" s4 G/ J! ]8 {9 T- K - poddisruptionbudgets/status* x! |' v8 k% L+ j: e) ?& B
verbs:' a4 G0 D9 N/ {4 X3 N7 T
- get
% x/ n' B4 N q7 P - list
1 S) \$ k+ h( W3 R$ G l - watch
4 g( P$ ~8 _1 l8 K8 n o) i- N3 N- apiGroups:9 S; V: K6 K" t( |3 S. Q6 d
- networking.k8s.io
) }# Z$ |# _9 F! T$ R resources:; K" Z# J3 w+ s* ~! ^' x% }
- ingresses0 [5 R H' I) ^8 _4 m+ i
- ingresses/status
- b, f9 W2 J8 c( d4 T# b0 P - networkpolicies0 y( O# U1 n$ N. X/ l# a* H& o
verbs:
* ~# ]! C: P5 v( {- h - get
6 O5 B- H7 t/ j) ^* x - list! A4 E( y" X s2 ~$ ~( _3 Z" ~
- watch% W: {& C# k7 M/ o. v, o
- apiGroups:* g+ E% b4 _) K6 z% A# [; t( {6 }
- storage.k8s.io
) A6 |) R/ k8 D) g2 O resources:
% X- S# ?0 T1 ]/ s3 ?6 a5 _ - storageclasses1 h x, y, H3 \( o4 u
- volumeattachments
i ]3 B8 p1 b f! m verbs:
6 S& h4 U/ h0 d6 O" B - get5 E2 ]! g8 ~/ H7 U
- list6 `! m9 q7 I8 v# i" |
- watch2 r. K0 P( j) p; n( o6 L1 b
- apiGroups:; u# n, {+ w+ @
- rbac.authorization.k8s.io4 y) l) E8 V# A
resources:2 R) W6 z! j, Q# p% U8 n( q$ }
- clusterrolebindings1 m: h0 ^: [ D9 d" C- W9 \
- clusterroles
- U Y4 z, A% o. U; B - roles0 R& h. A4 x0 c! [8 g+ ^ c$ {
- rolebindings
9 ^- g3 J- W! s& x4 @ verbs:* G2 Y" e" p! j% [5 P6 J. a6 @
- get
: p$ T: _! z k - list
2 f; w( \9 k) _- P) F - watch
2 q# G7 g9 g p
5 d; C9 Y) S% n$ E) k' X---8 G: q6 {, J. e! w) y
apiVersion: v17 r0 y) T7 R" n
kind: ServiceAccount
, b4 U3 T: ^0 J# D6 q' s% Nmetadata:# F! C8 l% D7 v" X
labels:
" _' F9 ~0 g4 Q k8s-app: kubernetes-dashboard
5 \: {. g- H' t% l name: kubernetes-dashboard5 t o: `. a- L, Q: [2 R! u; n; _
namespace: kube-system
8 s1 u8 K. V4 _& c( h
5 D ], I8 h F X/ k---$ F/ R; c$ c. B: W% l* k" {" ]* r
kind: Service% i) Q. `9 D9 v4 _
apiVersion: v1* J: r4 |& G" {
metadata:
, E& D" J1 S1 ?+ D: `; `' ? labels:
, f2 w' w6 _1 k7 L3 r( D2 m5 i k8s-app: kubernetes-dashboard
8 G2 [5 v+ s* \4 G1 X3 t' | kubernetes.io/cluster-service: "true" Y, x7 J2 A8 g0 r+ n- f
name: kubernetes-dashboard
* d3 P# v6 e: x namespace: kube-system4 W! V2 V9 F1 y% T' i1 W
spec:( A2 y! W) } z5 T
ports:
% A3 A0 r# ], {5 J - port: 443
: C6 _0 y+ F$ u. |( E5 @- s targetPort: 84431 z, x) a" C# y, w) E
selector:/ v# o( z4 v% `3 p8 A
k8s-app: kubernetes-dashboard {; j( [; P: `* D
type: NodePort
; \- V' V1 Z4 e: a; [2 T5 r2 ^ k/ d6 y# U) E
---( N/ j0 B% s2 c8 c0 l
apiVersion: v1
8 V, a$ i/ Q+ k( R% q. [kind: Secret
: c' T* o6 P9 A. i- ^5 Pmetadata:+ ]$ j' C$ F A- r7 f
labels:
" ~9 |0 d$ M) p k8s-app: kubernetes-dashboard$ A# Q6 [0 y/ v; [/ y. m O) l) \
name: kubernetes-dashboard-certs
& I5 {* c7 S: Y9 s3 l! P( d namespace: kube-system
3 ^, _* ^' ?8 g$ n/ X* y) Atype: Opaque h' V( Y8 U) s2 `
( ~' Q% f7 b4 Y% e---
( @9 W9 o; u7 E6 @. @apiVersion: v1
: d8 p. o: m' [% \( ekind: Secret3 ?# P5 L$ ^8 K! d+ d6 A/ T: x
metadata:
2 p4 U$ A) N+ f7 @. Y labels:
+ v; W2 j p s7 O T k8s-app: kubernetes-dashboard5 i+ L4 S: F1 Z7 l0 ^% n2 F
name: kubernetes-dashboard-csrf1 }5 C, _' d0 s
namespace: kube-system
. Q3 c7 n# y+ {: Qtype: Opaque& {( Y9 j: h) l2 b& D
data:8 |* C! C, T( v% j. w- S. F
csrf: ""
+ ?+ j) b! M# t" P3 L& W: I7 k1 P5 d
+ Q9 g0 M, b3 b; z5 o---
* ]3 `, Z5 l# _7 m4 Y( m d9 xapiVersion: v1
4 H& A/ W! g* Hkind: Secret
6 L7 A# m1 w E/ s" Pmetadata:9 {* d3 @' f- | `* e- m% d8 ^. J, E* b# v
labels:
3 W4 u9 g g& A7 ?1 X k8s-app: kubernetes-dashboard* e! P4 h4 o9 T7 r
name: kubernetes-dashboard-key-holder
4 r- R$ x& I5 ]. g1 ^. a( Z namespace: kube-system3 C* b, `1 I, d+ w: r
type: Opaque
, O% d4 c$ b: {+ W! o0 m$ M' z5 c! u; S9 R; Y; T% A/ ]2 [- j; Y- o5 J
---# w, @/ R( u5 s; ^" X) c
kind: ConfigMap
& R! \( u9 ]9 ~% Y3 CapiVersion: v1" Q; e3 [/ t8 E! _- Q
metadata:" F+ s( u) }% j3 l8 A
labels:: r9 c; ~; R8 c; N6 V6 g9 U2 ]
k8s-app: kubernetes-dashboard% `6 n- g" K( ]! o/ }+ X
name: kubernetes-dashboard-settings
! r* R4 K$ F2 S O( K namespace: kube-system" e( J# |2 |, d) Z2 `2 L+ d
% F( y: ^% l3 L/ f# J---9 n+ H+ j4 L+ G; f2 q
kind: Role1 k* r5 k9 u3 G* D7 N3 A. b$ D& {
apiVersion: rbac.authorization.k8s.io/v1( l6 S- i t- s) A0 U, \
metadata:
0 D. ?2 A+ W2 i labels:7 ~! Y* Z; B! u! u
k8s-app: kubernetes-dashboard0 I. [2 p4 v$ L2 y/ R9 x }- k
name: kubernetes-dashboard
& d/ K$ K) X: m- s( Z# y- | namespace: kube-system
& b& V2 \8 \9 X$ Z/ D5 f+ @rules:
; ~3 Q% j1 Y k- F: }* i # Allow Dashboard to get, update and delete Dashboard exclusive secrets.1 Q- U4 d* c% v8 W' h0 ^, P
- apiGroups: [""]" v m& p# W5 Q5 y2 `
resources: ["secrets"]
) H9 S/ B5 B8 V8 _2 ? resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"]0 [7 H0 G& Q; F8 D1 F$ o, T
verbs: ["get", "update", "delete"]
! \+ H, g7 I* D6 ]% ] # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
7 H! P0 r8 ~" P - apiGroups: [""]
& X9 @# `# h% a resources: ["configmaps"]
, N6 a4 A: B) q7 q1 y- k( P resourceNames: ["kubernetes-dashboard-settings"]) `4 ]( f+ k4 l8 n4 w3 h A9 y
verbs: ["get", "update"]
1 H. a6 n( S' s% ?1 ] # Allow Dashboard to get metrics.( g! M6 m) U# X2 q
- apiGroups: [""]3 M- ^5 n1 N9 p j
resources: ["services"]
* W# w1 N# |- D% b6 }8 z: o resourceNames: ["heapster", "dashboard-metrics-scraper"]
2 o7 r3 i/ W# I" E verbs: ["proxy"]
6 K9 S: z6 t( E) w/ q. M3 l - apiGroups: [""]. _4 }, N4 ]: N# @' ^9 g
resources: ["services/proxy"]0 P, X& h7 O' \* R' V' n$ F
resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"]7 W& `6 s7 h7 R9 d" J
verbs: ["get"], @4 ]: _" C$ I. Y& g" L. _" L/ k
, M0 ]& \( H( l6 i
---
1 `& x Y/ ?/ b0 tkind: ClusterRole) `2 K: D; w' y& N5 {" e. c9 m
apiVersion: rbac.authorization.k8s.io/v15 m4 `9 g3 k2 j; Z; t4 L" B5 W8 \- G
metadata:
( K, @0 c! r! v8 ^9 n& g) b g3 Z6 a# d( j labels:* B5 o8 e1 o5 C2 U5 t
k8s-app: kubernetes-dashboard
/ d N" k# K. v7 ^/ t/ P name: kubernetes-dashboard% \1 F* c; l6 \6 K( E( k
rules:
; P9 @2 d! x0 \0 b- L5 d # Allow Metrics Scraper to get metrics from the Metrics server
1 F) g/ F Q% Z9 ^- q4 n8 n - apiGroups: ["metrics.k8s.io"] Z/ b7 v$ L7 S
resources: ["pods", "nodes"]
/ x6 }2 {) o% U( d% n) t verbs: ["get", "list", "watch"]- U( @' \& N; e
/ n- T8 r; Q* D; s; y5 w5 h
---
" R& q! ^! o$ XapiVersion: rbac.authorization.k8s.io/v1
* B. S2 y8 y/ Zkind: RoleBinding: i. `, y/ E' S
metadata:) v/ U2 I8 j5 ]! H$ U$ h& N
labels:; R0 n. x& u" G2 ?8 e- d
k8s-app: kubernetes-dashboard9 o% L+ z' T" ~' L
name: kubernetes-dashboard- v0 p3 R" b( G5 i
namespace: kube-system, c& o1 I3 \9 B% o: K) a8 a
roleRef:, u9 W |" l [9 v7 Z
apiGroup: rbac.authorization.k8s.io
5 ]! Z) {7 D4 Q. I T kind: Role
; H/ M+ L- Q& ]" b, y2 D7 p name: kubernetes-dashboard
' K1 g. D/ l1 e. E% K1 S8 Psubjects:
% z) ]! P8 q7 w% A3 t - kind: ServiceAccount* r2 K' g- D7 H: r0 R
name: kubernetes-dashboard n% _, T' y1 y3 g0 e$ l6 v
namespace: kube-system) x$ @$ b5 i! H
! C3 f* ^# S% q$ g" }---- R: z5 q0 T/ H
apiVersion: rbac.authorization.k8s.io/v11 C, P1 q# W6 S O6 n* W
kind: ClusterRoleBinding* n! {, I& c" Q: F
metadata:
5 A; n. ^& r8 t2 B name: kubernetes-dashboard
" t- N# l! \, W8 N* ~6 EroleRef:7 v3 x" N' J8 m, ~9 f" g
apiGroup: rbac.authorization.k8s.io
O h& V/ ]$ [' ]; k/ E kind: ClusterRole
$ G4 c, Q( Z. Z. V( p0 s4 h name: kubernetes-dashboard# Z3 l/ L/ s4 E- k
subjects:8 e; n2 ~7 ]) t: L5 L
- kind: ServiceAccount
' c7 G0 D9 y1 {) q name: kubernetes-dashboard8 \' P4 b6 f8 K1 _$ `" ~9 n
namespace: kube-system- }) ?& q* ?* E9 F4 `
9 F1 B6 p% F4 T4 ?4 o--- i: ?% c3 V" A3 Y9 k: C' y0 H
kind: Deployment6 }7 W+ h5 v# d! h: R( g1 Y8 P- h
apiVersion: apps/v1
( d1 b+ N! N) _1 vmetadata:! i) c8 c( {$ P- {
labels:" y9 P- ~5 ~# T1 p( G8 _+ W2 @
k8s-app: kubernetes-dashboard
. \! k$ b2 ?" F/ k) U5 @# t name: kubernetes-dashboard* s8 F2 ^' l. s& U' S1 p
namespace: kube-system# y* P) }2 D. E, P
spec:* _# U; {8 E' @$ ~, x
replicas: 1. s" R! N N+ T v3 @. ^: H
revisionHistoryLimit: 10
9 i( [9 Y9 c& B" z selector:
5 G+ B) X5 t$ J1 M# K: K4 N matchLabels:
4 j+ u/ g+ R) ] t7 l, U" o! S6 h k8s-app: kubernetes-dashboard
: t" S: J; `7 H& L0 V template:
+ r; g" n. M; x2 [% @ metadata:1 y: _; o k8 ?, i2 [
labels:- x, q I2 g$ m- M! M& h/ o& T: t
k8s-app: kubernetes-dashboard8 `1 D/ _6 `1 |+ I
spec:$ C, ?0 d& P2 Z; B
containers:. M$ t# c& l- L9 a! }; A
- name: kubernetes-dashboard
7 y1 e- k1 l# k% v# t( ` image: kubernetesui/dashboard:v2.4.0
, w6 J6 l1 t7 Q* Y imagePullPolicy: IfNotPresent- l1 V, W+ x: L
ports:! x+ d/ V$ \0 A$ ] V. J$ n
- containerPort: 8443" X* K2 A! r7 _" Z
protocol: TCP
/ J' [; [6 s3 L' n3 q: I* h args:( ~% ?7 B! L: |0 I
- --auto-generate-certificates, ?+ I" v3 V$ T, r( t/ I+ M
- --namespace=kube-system! Z; X* ]4 h" i6 G% I6 s) v
- --token-ttl=1800 p% q0 }! U5 V: B! b4 x
- --sidecar-host=http://dashboard-metrics-scraper:8000: O4 u8 P% S% P8 J0 K
# Uncomment the following line to manually specify Kubernetes API server Host4 O9 J9 F& }: M. Y& k4 H
# If not specified, Dashboard will attempt to auto discover the API server and connect4 u& ]$ r! q' f$ X6 z
# to it. Uncomment only if the default does not work.5 k% r1 @( K' n9 t7 m# D! E; f; O/ x
# - --apiserver-host=http://my-address:port( d q$ U, e' e$ t
volumeMounts:
- s+ J6 S, G5 }, k% B - name: kubernetes-dashboard-certs8 J+ k6 S( i8 ~
mountPath: /certs
4 M5 o$ a1 d+ Q # Create on-disk volume to store exec logs+ z% d# y+ H0 T; H. k7 o2 ?
- mountPath: /tmp' N1 L |' H" Q) _! j5 S/ G/ X
name: tmp-volume; f# P& u; Z- Y$ V; o( S+ ^0 C. |, Q
livenessProbe:
; i" F& e6 H: c) u2 d4 d httpGet:
1 `6 @/ m' T5 C& p$ w, f% w: m( T scheme: HTTPS
; H5 r/ V3 S: E+ e2 | path: /
9 Y/ j5 f5 p" N0 q& l port: 84432 T( ]! ~9 _' j9 W
initialDelaySeconds: 30
+ D0 _7 W; l- }8 a g) u! s+ y! m* a* n timeoutSeconds: 30$ x) ?, h: k1 J3 L$ b. l
securityContext:2 Z& w$ v# n( s) |" t% T7 N" M
allowPrivilegeEscalation: false
" H: Q7 l4 B4 w: ^ readOnlyRootFilesystem: true
/ q% s, k$ I! U; Q1 t2 X runAsUser: 1001
' d$ Z+ A9 _6 Z! e. Y) ^7 z9 M runAsGroup: 2001# b9 P; d7 o& v* N; C
volumes:
0 G/ S: y7 f! w - name: kubernetes-dashboard-certs7 t) I' N/ I! m' d7 M
secret:
: {! \8 o) m, Q- k5 T6 u" [! c secretName: kubernetes-dashboard-certs
& g8 j. N7 n! p+ b9 f! k+ } - name: tmp-volume* H0 E0 s) x7 Y N$ M
emptyDir: {}$ U7 k6 d, r$ M) O* p/ Z8 _2 `
serviceAccountName: kubernetes-dashboard) F, g* {( t1 o2 A0 ]
nodeSelector:, l& {' w9 x% u j" O& J5 g
"kubernetes.io/os": linux5 a% |; L- b# b! C5 Q+ o6 K
# Comment the following tolerations if Dashboard must not be deployed on master; T& g9 U5 w8 h
tolerations:
! }: I1 k% _' a( f) `9 w - key: node-role.kubernetes.io/master
. L6 [9 Y. o$ @/ k effect: NoSchedule3 M' h0 Q; q; g- b+ {' A4 S. C
9 f% S# B7 y: d5 k) B( c3 {9 H I
---7 R# Z5 V" O) t4 E3 i4 k
kind: Service+ W, g+ p/ X8 b# ^
apiVersion: v1* O% e* C3 ?5 ^& @1 P! {/ N
metadata:! t m7 ]) G. l9 |0 K
labels:
" p7 Q& V# `8 w k8s-app: dashboard-metrics-scraper/ H5 U3 |+ s2 V b: ?$ Z
name: dashboard-metrics-scraper/ Z" ?5 k! N" x+ x
namespace: kube-system* _- f/ k0 v% z+ O# S7 g9 |' P3 M
spec:
; i5 ~9 i6 V7 k+ E/ S ports: Z1 U6 R) U2 n! u# B' N% ?- S3 R
- port: 8000/ D" z6 `/ ~+ W' G/ ^7 m
targetPort: 8000
, ?5 O+ T5 z: G+ T: Q; l selector:
, y0 F4 q' e8 g6 a1 U: f k8s-app: dashboard-metrics-scraper: B; N6 U3 X4 Z0 [, y
# D, s. p5 e: d, ]5 R0 C
---. e) F. q; C9 Q1 w( T
kind: Deployment* z; g4 B$ d( X9 P9 z
apiVersion: apps/v1
5 A: @4 X% `4 p* B8 ]& h" `) A7 Ometadata:
' S e" z! z4 s2 A4 l labels:
1 E$ f# ?: F5 V, l k8s-app: dashboard-metrics-scraper. k& F C2 v% h1 P# X3 D- Y
name: dashboard-metrics-scraper
/ W/ v7 i) f9 x L namespace: kube-system
# X% R* J& ^( [5 wspec:
2 F$ \; c' R6 @. m replicas: 1
/ v0 T% x: V; e revisionHistoryLimit: 10' |8 i6 {" g3 ]. b. a/ ~
selector:- E1 }6 ]8 ]- j# K f& F* _
matchLabels:
7 }- d! c+ y: d+ g k8s-app: dashboard-metrics-scraper
* o8 z2 `% @7 k' W( h/ E" ~ template:
- K4 u1 n- d$ b metadata:
3 d! u. t( T/ I) m% w labels:
, n+ l0 Q, L( k, N- n. P3 E k8s-app: dashboard-metrics-scraper p5 f- L3 h0 x$ O0 j: [, A0 T
spec:
: ^$ i. w# n2 \ securityContext:
, m; y3 v5 o4 p seccompProfile:. _/ R& m3 X A9 W1 j2 ?( I$ B
type: RuntimeDefault
3 {' a. }: i; N6 Q- ? containers:
- B$ P8 l6 L/ L5 |. _ N, M - name: dashboard-metrics-scraper
" Z8 O" h, ~( d( d& G: c image: kubernetesui/metrics-scraper:v1.0.7
: _4 l% v1 `' ? imagePullPolicy: IfNotPresent
( o6 d5 Y1 c- \' v. N. d# A ports:- ?, e) J( `1 e3 n. a* s4 }+ Q
- containerPort: 8000
\$ W; T, h0 F protocol: TCP2 ]# W2 d6 x1 C! D' u2 n
livenessProbe:
& x* J7 E% q0 c0 L1 i1 }' [7 S& R0 X httpGet:
5 h8 h: b) _& o: A/ p* l scheme: HTTP
( V7 i( F* T: Z path: /; h8 I$ y$ M9 n( h
port: 8000
0 E: V1 t& M- i& t4 p initialDelaySeconds: 30- y- V8 n1 w( q0 A5 ~5 ]
timeoutSeconds: 30
2 ]7 t6 t7 l4 S1 }* h5 B+ D9 | volumeMounts:
2 x! Q( F2 O" S; S, t - mountPath: /tmp0 u4 `0 Y9 v, Z& `; @" `, t& L! h
name: tmp-volume
8 X, M0 L7 I0 Z securityContext:
# y: u( C) q4 M: j allowPrivilegeEscalation: false
/ N$ d. b2 b0 D6 ^2 j' I# @5 x2 H0 [ readOnlyRootFilesystem: true4 m) g% s! k# ?' ]; V
runAsUser: 10014 ?2 `% w" @+ K+ G
runAsGroup: 2001
2 b: X) ^7 D( e( K! K serviceAccountName: kubernetes-dashboard
& Z9 v1 G" P* I7 G nodeSelector:
1 W1 b0 y/ k; N" A+ p* ~ "kubernetes.io/os": linux
. v- E6 ~( I4 o+ F( | # Comment the following tolerations if Dashboard must not be deployed on master
' m, K1 Q! }4 |5 H( Q0 {! L tolerations:: A6 s- d. z% J2 s$ [5 f6 z1 S
- key: node-role.kubernetes.io/master# }" }( s0 }4 v8 n2 }
effect: NoSchedule
% p9 ]! z* i- w# v B, j1 Y5 G5 v volumes:, y/ l/ G4 f* ^ T; N( j
- name: tmp-volume) t3 L0 F2 p' U4 u5 U0 m# f& U
emptyDir: {}& w2 J4 m! X( \* F9 s- g
导入 dashboard 镜像5 W0 s0 u/ s2 ]
for i in 192.168.91.19 192.168.91.20;do \
) U( H9 u/ M) F2 y. v( Rscp /approot1/k8s/images/dashboard-*.tar $i:/tmp/+ g& z* P# @6 s% t! t
ssh $i "ctr -n=k8s.io image import /tmp/dashboard-v2.4.0.tar && rm -f /tmp/dashboard-v2.4.0.tar"; \
0 H$ r9 L# h @. ?ssh $i "ctr -n=k8s.io image import /tmp/dashboard-metrics-scraper-v1.0.7.tar && rm -f /tmp/dashboard-metrics-scraper-v1.0.7.tar"; \
# A# D- q4 @5 n, F( l1 xdone
; ?& q3 p5 X6 L% H; B: a* M查看镜像4 ?# u/ L( g8 U8 E& {6 o
, B% |* o& Y9 R. \ M% [) U. Tfor i in 192.168.91.19 192.168.91.20;do \
, d* n& r7 w. `* x" sssh $i "ctr -n=k8s.io image list | egrep 'dashboard|metrics-scraper'"; \9 C! D! [- } u$ @. m b4 X
done
$ X9 n1 q0 W, O# s: E在 k8s 中运行 dashboard 组件
& P6 ?, O; B/ V8 `kubectl apply -f /approot1/k8s/tmp/service/dashboard.yaml3 V5 W& ?3 P2 P& y
检查 dashboard pod 是否运行成功( X% y# l6 n' R6 V e
kubectl get pod -n kube-system | grep dashboard
- x/ t, X; i5 n预期输出类似如下结果: c; g4 x, f3 E
( L6 Q2 ~, r" }0 t- a7 X* C7 adashboard-metrics-scraper-799d786dbf-v28pm 1/1 Running 0 2m55s
g5 f1 ~4 l. s2 m5 n5 y( fkubernetes-dashboard-9f8c8b989-rhb7z 1/1 Running 0 2m55s A- u1 ?* `7 w
查看 dashboard 访问端口
3 y" i! h2 _: P0 d. [* O! c在 service 当中没有指定 dashboard 的访问端口，所以需要自己获取，也可以修改 yaml 文件指定访问端口
0 Y9 v, Y& G( W7 ?/ a$ _9 x
: i. d3 Y0 D; ?* W* K- c预期输出类似如下结果
0 q( d9 D2 `8 h, l! a( _
9 E7 A) t. o, T' A我这边是将 30210 端口映射给 pod 的 443 端口
Q3 M, E8 u, M# D4 ?
! Y, P5 i! m( m8 G" Ekubernetes-dashboard NodePort 10.88.127.68 <none> 443:30210/TCP 5m30s( o0 {# ~/ r" L9 p
根据得到的端口访问 dashboard 页面，例如： https://192.168.91.19:302109 o- K7 `1 P) ]3 W+ E
/ R c) l& P- \6 W0 [8 j
查看 dashboard 登录 token
9 Q/ W3 u' |! {. x获取 token 文件名称: L0 H- q% {% o; m/ d
5 S8 l, T7 _/ Nkubectl get secrets -n kube-system | grep admin
$ q4 @- C) m! U, k预期输出类似如下结果7 o$ f8 Y+ |1 s
6 X7 L- t$ V& l8 o& X% H2 F8 S
admin-user-token-zvrst kubernetes.io/service-account-token 3 9m2s
. o: j) t9 n" y: [" g9 w* h获取 token 内容
4 z4 O3 F# e; b. | _6 ?& U8 u n2 d5 L0 K6 @2 j+ v* @9 i5 ?
kubectl get secrets -n kube-system admin-user-token-zvrst -o jsonpath={.data.token}|base64 -d, ^7 O7 d- U# Z$ e5 k/ t; R
预期输出类似如下结果0 r( h; M5 R' e1 p6 l e, P
) ^% W7 p) f! VeyJhbGciOiJSUzI1NiIsImtpZCI6InA4M1lhZVgwNkJtekhUd3Vqdm9vTE1ma1JYQ1ZuZ3c3ZE1WZmJhUXR4bUUifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJhZG1pbi11c2VyLXRva2VuLXp2cnN0Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImFkbWluLXVzZXIiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiJhYTE3NTg1ZC1hM2JiLTQ0YWYtOWNhZS0yNjQ5YzA0YThmZWYiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZS1zeXN0ZW06YWRtaW4tdXNlciJ9.K2o9p5St9tvIbXk7mCQCwsZQV11zICwN-JXhRv1hAnc9KFcAcDOiO4NxIeicvC2H9tHQBIJsREowVwY3yGWHj_MQa57EdBNWMrN1hJ5u-XzpzJ6JbQxns8ZBrCpIR8Fxt468rpTyMyqsO2UBo-oXQ0_ZXKss6X6jjxtGLCQFkz1ZfFTQW3n49L4ENzW40sSj4dnaX-PsmosVOpsKRHa8TPndusAT-58aujcqt31Z77C4M13X_vAdjyDLK9r5ZXwV2ryOdONwJye_VtXXrExBt9FWYtLGCQjKn41pwXqEfidT8cY6xbA7XgUVTr9miAmZ-jf1UeEw-nm8FOw9Bb5v6A
) P3 C% W4 a- a5 B9 t: t6 {2 g( I3 B- v; W2 ?7 M& ?
到此，基于 containerd 二进制部署 k8s v1.23.3 就结束了. f8 x' v q& o2 K* }
. \" c' Y' v0 ]* q6 Z8 _, o |
|
发表于 2024-9-17 10:35:55