|
|
楼主 |
发表于 2025-1-1 19:51:59
|
显示全部楼层
创建目录
' ?5 N/ Q( e r A7 N根据自身实际情况创建指定路径，此路径用来存放k8s二进制文件以及用到的镜像文件
# s2 b C6 j$ L/ e) W ]* f' `0 M, u A, Z7 z$ Y
mkdir -p /approot1/k8s/{bin,images,pkg,tmp/{ssl,service}}$ ]& f* y1 U2 @% J
关闭防火墙
' L2 S" J8 N+ v" _% y; _7 U* Hfor i in 192.168.91.19 192.168.91.20;do \; j* e7 b! z% }+ `' q; F7 v
ssh $i "systemctl disable firewalld"; \/ t: t+ e7 P! @3 U* N4 D1 r
ssh $i "systemctl stop firewalld"; \
- o& C+ _7 h* m0 R$ z5 U1 F. Ydone
; ~' L0 ~. y+ S关闭selinux6 @' e# |* y4 i' x
临时关闭
1 s' x: O% T. w* O. x0 _* k
7 n: Q- q7 i1 Q2 B4 ifor i in 192.168.91.19 192.168.91.20;do \
" [# H8 p; }, e" s4 u% t1 tssh $i "setenforce 0"; \6 S$ P# }2 }: y
done
" ~" \- F! ?) [2 u7 i7 h永久关闭
. G1 S% ~. X8 u* j! v* C( W; }9 ^- T* _9 R( n
for i in 192.168.91.19 192.168.91.20;do \2 p- m M# R' ~7 k* ~6 }' h* k
ssh $i "sed -i '/SELINUX/s/enforcing/disabled/g' /etc/selinux/config"; \, o; d! Z1 T2 Z, N/ r) k+ Y
done7 `; l1 i& e5 y$ h- V
关闭swap* \& Z& o. h' P u, \
临时关闭
* L5 N& ]+ D; L! d8 |. i3 F1 I5 e: U# [; ^" o- l7 t/ D
for i in 192.168.91.19 192.168.91.20;do \6 E, F4 W& H5 f* x, c0 O) W* c
ssh $i "swapoff -a"; \
( R2 b3 I; r- ~- ~done4 f9 Q. j; G& @: z! m% O
永久关闭8 r% {+ G# u3 S. P7 C4 \5 s" B
3 U/ U# @# [" |6 h# Afor i in 192.168.91.19 192.168.91.20;do \' B: E2 t8 X S2 ^* s
ssh $i "sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab"; \! h. R9 E- N$ Y/ W$ y1 m
done* K1 r) X9 a1 s7 M1 p5 D0 U
开启内核模块
0 k" ]5 A/ s7 K! Y* }- r' Z3 U, l临时开启
- \1 {: Y$ t$ V1 W7 @+ r# s" }% i0 U8 T( t* F( o' `! o
for i in 192.168.91.19 192.168.91.20;do \
, ]) v5 m( |: | b3 Y2 Ussh $i "modprobe ip_vs"; \
6 m4 r4 R' }0 [* q% }' X; _8 @ssh $i "modprobe ip_vs_rr"; \" q+ Z1 h# E# N" X
ssh $i "modprobe ip_vs_wrr"; \
' y; ]1 |2 |& w1 e4 Cssh $i "modprobe ip_vs_sh"; \
7 u/ k7 f5 j3 b6 q# N# @" u% A7 Ussh $i "modprobe nf_conntrack"; \
+ Y& V; j* y& I t. Zssh $i "modprobe nf_conntrack_ipv4"; \
" f; p& x# `2 v6 [# tssh $i "modprobe br_netfilter"; \$ S! }+ M$ g6 A& O, _, c! c: N: R) a
ssh $i "modprobe overlay"; \
" Z" I2 j) g* U" Q; b+ ]done
& H$ Z/ ~( W! I' j0 T% l) \永久开启
- L* H- D m6 Z$ u' h! V* J/ x" e5 e1 ^5 p0 |0 Y0 q( L: w
vim /approot1/k8s/tmp/service/k8s-modules.conf" A5 b Q8 h9 s% c- Y9 Z) X
ip_vs8 O* j7 S! G2 l. n3 w+ b
ip_vs_rr
6 j8 d; r# y& I8 v; `! xip_vs_wrr
' \( [. }3 g* T/ F+ pip_vs_sh
4 F( Z2 w! Z( G5 Cnf_conntrack
$ o+ _2 O( E- w/ Wnf_conntrack_ipv4
a1 h5 m l; H, z. }br_netfilter; n* f" @ N6 R/ i
overlay; K4 i" A) x2 e9 X
分发到所有节点
/ v- ]/ w! Z1 M9 F- V1 \for i in 192.168.91.19 192.168.91.20;do \5 M: S' C5 y# a# n) ]; W- r, ]) a8 B
scp /approot1/k8s/tmp/service/k8s-modules.conf $i:/etc/modules-load.d/; \; W& t8 [) \0 V( I9 |- ]
done c! @5 M- h; P D5 |5 T
启用systemd自动加载模块服务
' m( a+ J$ F3 L$ U9 C. P4 G) D) ~) |for i in 192.168.91.19 192.168.91.20;do \
# p; C1 K) @" p" Rssh $i "systemctl enable systemd-modules-load"; \
( {* Q+ |6 E! h( d/ |9 Essh $i "systemctl restart systemd-modules-load"; \' g" K$ S; a: p6 `% z9 ?
ssh $i "systemctl is-active systemd-modules-load"; \/ u; X0 ~5 A8 ]2 ~, F& P
done w: h+ A2 n" p( T" b
返回active表示 自动加载模块服务 启动成功
( x W* M7 {. ]; z; G$ p4 { v& ]4 p! E) ]1 F# m/ m& }2 U! \& J
配置系统参数; @8 [ n+ e4 c" N& h. m
以下的参数适用于3.x和4.x系列的内核& C6 `2 m5 U6 n1 }( ]3 p
; a3 h. s( e' W0 e# Nvim /approot1/k8s/tmp/service/kubernetes.conf2 G4 n k8 K0 U% q X
建议编辑之前，在 vim 里面先执行 :set paste ，避免复制进去的内容和文档的不一致，比如多了注释，或者语法对齐异常
9 i/ T7 B5 A* Q7 ]" {. h5 M% U
, H- @$ P# p$ x3 P# 开启数据包转发功能（实现vxlan）
$ B% u9 b. j6 Ynet.ipv4.ip_forward=18 b/ P2 W6 [6 m! w p5 N( j
# iptables对bridge的数据进行处理
, l% s7 _! x' X0 S% Onet.bridge.bridge-nf-call-iptables=1
( S p, d Y: C& @' anet.bridge.bridge-nf-call-ip6tables=1" w5 |; N7 y9 v- ]: ^* \& T
net.bridge.bridge-nf-call-arptables=1) A/ O4 x O% J9 c, y3 ^
# 关闭tcp_tw_recycle，否则和NAT冲突，会导致服务不通& l3 D% y2 E# P5 x& L
net.ipv4.tcp_tw_recycle=0) j+ t2 v. g& P
# 不允许将TIME-WAIT sockets重新用于新的TCP连接) ^" X- D+ e/ i6 Y C
net.ipv4.tcp_tw_reuse=0
8 R( @/ `. N6 Y, w4 j0 [' m# socket监听(listen)的backlog上限+ w! ?4 X9 I3 Z5 t. P
net.core.somaxconn=32768 \" V4 s, S' y' J2 i2 U1 Y4 j
# 最大跟踪连接数，默认 nf_conntrack_buckets * 4
7 W* y# p } C/ Fnet.netfilter.nf_conntrack_max=10000003 v( n$ J) ?7 B" L- n* U
# 禁止使用 swap 空间，只有当系统 OOM 时才允许使用它
+ X1 v1 X0 h8 `2 F5 P: ?: Svm.swappiness=0! D& V- M/ ?' N
# 计算当前的内存映射文件数。
4 ^9 G3 ?* v/ s/ N+ ivm.max_map_count=655360
: M7 D8 W5 W+ @3 V# 内核可分配的最大文件数) W/ N/ o! }3 n4 {) C: J
fs.file-max=6553600' E4 P6 N# L2 K6 h
# 持久连接
: ?/ i$ ~$ e# V' l8 Tnet.ipv4.tcp_keepalive_time=600- G, t9 j! E( `/ b
net.ipv4.tcp_keepalive_intvl=305 Z3 K3 ^4 w% ]( a
net.ipv4.tcp_keepalive_probes=10
v l0 E' d% Y8 V分发到所有节点
@6 D; }- V1 ^) R/ }1 I: hfor i in 192.168.91.19 192.168.91.20;do \
& E. T! H5 h: U7 k, dscp /approot1/k8s/tmp/service/kubernetes.conf $i:/etc/sysctl.d/; \( \" _: A) v2 N/ b
done0 U! e! w% q' h2 G8 G+ H
加载系统参数- i" U0 T9 K& r6 v
for i in 192.168.91.19 192.168.91.20;do \5 U' U3 G9 v8 P$ {! e5 p- \% P% V
ssh $i "sysctl -p /etc/sysctl.d/kubernetes.conf"; \
! u) z, |# l" C3 Adone
' Z! p* V$ L2 U4 j, G- k' T' m清空iptables规则3 R( T4 [3 l4 |" y9 F( q
for i in 192.168.91.19 192.168.91.20;do \
. I8 G- {3 C8 Y% _& ^2 nssh $i "iptables -F && iptables -X && iptables -F -t nat && iptables -X -t nat"; \
# _8 l5 c8 y1 l- Q2 \ cssh $i "iptables -P FORWARD ACCEPT"; \
( f& x0 g! _. q Rdone
6 @% |, l1 X& {* u( C: r配置 PATH 变量9 i0 C% y0 h h8 K; |, Q
for i in 192.168.91.19 192.168.91.20;do \
4 g W' L* o7 c6 n$ I' |+ z' d+ Nssh $i "echo 'PATH=$PATH:/approot1/k8s/bin' >> $HOME/.bashrc"; \% F8 F0 `0 }9 ~1 j: v+ @! j+ u
done
1 M+ h% r$ A+ q" l) @source $HOME/.bashrc# B2 P' K& V1 j7 l3 A
下载二进制文件2 U8 ~0 g) i! k3 R2 y6 R
其中一台节点操作即可
4 h* x0 u3 m. W( r# E3 t$ O/ C: s
W1 n) l2 q. V/ Q$ Ggithub下载会比较慢，可以从本地上传到 /approot1/k8s/pkg/ 目录下
% m4 e' L, f6 F9 w) E T9 l. e6 M" C8 {1 Y$ G1 z
wget -O /approot1/k8s/pkg/kubernetes.tar.gz \: U7 g7 \- x/ W' E- q9 J
https://dl.k8s.io/v1.23.3/kubernetes-server-linux-amd64.tar.gz0 T7 D7 j6 J- b- ~! O }( X
" L2 X3 h' F1 }) U
wget -O /approot1/k8s/pkg/etcd.tar.gz \& P+ i' X8 T) i/ g" V& [& M
https://github.com/etcd-io/etcd/ ... -linux-amd64.tar.gz+ }5 {8 D( O' g R& Y
解压并删除不必要的文件, { X% R+ Z% }; G7 z
( |' f; h% X7 x, L
cd /approot1/k8s/pkg/6 y6 G+ f6 c: O. |7 `
for i in $(ls *.tar.gz);do tar xvf $i && rm -f $i;done$ Z0 W$ S& n1 o o+ s
mv kubernetes/server/bin/ kubernetes/
' ~( s/ [$ }8 ^. u- s/ H. S9 irm -rf kubernetes/{addons,kubernetes-src.tar.gz,LICENSES,server}! g( B7 A, Z4 z! |* n
rm -f kubernetes/bin/*_tag kubernetes/bin/*.tar
6 Z1 q$ h( u) o* Q8 Erm -rf etcd-v3.5.1-linux-amd64/Documentation etcd-v3.5.1-linux-amd64/*.md
) z5 P6 T. ?, t) O# U- j6 g部署 master 节点
% ~0 B$ Q3 ?6 t5 x. F% K9 d创建 ca 根证书+ y+ G* N0 Y" q9 k, t& _
wget -O /approot1/k8s/bin/cfssl https://github.com/cloudflare/cf ... l_1.6.1_linux_amd64" F0 L7 w/ P- z) y! F2 ~% n
wget -O /approot1/k8s/bin/cfssljson https://github.com/cloudflare/cf ... n_1.6.1_linux_amd64
: T" ^8 i2 i+ p6 x$ gchmod +x /approot1/k8s/bin/*+ L+ N: h4 W& ^8 m: `$ c: g N
vim /approot1/k8s/tmp/ssl/ca-config.json8 c+ I) _% O+ o+ S/ C/ w
{
# e+ U3 c" X1 h1 | "signing": {
- Z" f K2 d" W/ }& M5 {0 T& V; o "default": {5 [* Q/ j U2 }3 c+ D y) s, B: i
"expiry": "87600h"& j. ?& B1 P% N) ]" b( [
},# s8 T/ b. R W0 b. _) e$ b
"profiles": {
: m/ J+ Q! o, o L8 L% C4 J( e "kubernetes": {* g' H7 m0 b% O0 |6 {
"usages": [: N% @2 V- Z! E% z+ T# e
"signing",. L j3 N6 W9 a2 f/ T' q! x
"key encipherment",$ ^" E7 _! a0 Z9 q
"server auth",& O8 z8 e( s# C% K2 y3 ?
"client auth"
5 t3 m$ [6 f! B4 b9 ~8 }" V ],* R0 G B* E# v2 _+ k' l
"expiry": "876000h"+ y$ D" v6 L* T2 Q
}
$ `# ]0 G, v* B4 R/ `- } }
+ X" C: T* h$ ?, Z" C8 \+ n8 i1 K; k1 D }" E. O5 B) b8 b* \2 E% p9 Y6 G! u
}/ j9 T5 N' @& J
vim /approot1/k8s/tmp/ssl/ca-csr.json
1 h3 B! p; y2 R* S4 ?+ [. b{
0 v7 ]& Q1 ]0 U "CN": "kubernetes",
) S2 Y. d) Q9 V "key": {5 W* t% }4 ^+ K+ n2 `4 J: T2 x
"algo": "rsa",
% F+ ~8 l. c& ?, Y+ l; _ "size": 2048
) r4 o. r3 a/ _- y$ x- k$ ? },
0 F: L- f5 P: Q1 o9 h, {/ q "names": [
7 n/ q/ I9 f: K5 V {
' ?9 N0 a2 M; P4 O "C": "CN"," V2 _( [ s2 E2 p
"ST": "ShangHai",
% b( U7 N0 [* n" g7 w% c# G "L": "ShangHai",8 y3 \& _; I' ]5 I! l D% o# `, ^0 r
"O": "k8s", D/ I3 F3 w2 K' B# M" N9 @. c, U2 S. w
"OU": "System"0 \6 Q6 Q- j4 Y% d& U% J/ p
}; K6 A$ D5 h! c$ E2 {
],0 e/ _8 R: A) C! r' l
"ca": {- d+ V) u {( ^
"expiry": "876000h"/ M. L; \3 K: w) Z
}
1 \/ H) O; H+ D: I/ I# ]; T}
# l) o) Z, o* j' O! |cd /approot1/k8s/tmp/ssl/
7 n2 Y2 X+ l2 v( x: G% gcfssl gencert -initca ca-csr.json | cfssljson -bare ca
! f: f# B( m% ?+ {: o部署 etcd 组件6 R9 a0 Z2 o" B/ a
创建 etcd 证书- W, G& L0 o$ k$ D: F
vim /approot1/k8s/tmp/ssl/etcd-csr.json9 \5 n1 d- t7 v% {0 b9 D8 f2 z
这里的192.168.91.19需要改成自己的ip，不要一股脑的复制黏贴; {8 i8 Y! o0 W2 H% R
4 ~( ~. u# u8 k5 c- P4 F. r
注意json的格式
7 g, A4 ^& k/ d7 i! R/ s8 a( j+ Y) d
{
( i. v. m/ v+ ]0 ^3 L7 I. i: ?$ R "CN": "etcd",. E$ J2 Z1 _3 s: |: c5 N
"hosts": [8 K" B# C. v8 N$ v! D2 _: e" v' R
"127.0.0.1",7 u! y$ `6 q" z# k) [: c8 _
"192.168.91.19"
) Q) Z# E6 n) F) Z. V' N v# j ],: Z' L! B% s$ A8 B
"key": {
g5 a8 k- q6 T" c* a+ c8 R( _ "algo": "rsa",
% G4 G: X; e3 J0 ~ "size": 2048
% ]7 w7 N5 U6 c0 b y },6 G, K* F4 g7 C+ L7 O
"names": [
$ {" ~7 C: o% }+ ?# b6 @ {
5 J1 q. w1 C" W# w3 |* ] "C": "CN",0 Y; Z3 ]) W6 U$ c$ r5 b6 R
"ST": "ShangHai",& G- h# A$ g0 t3 S
"L": "ShangHai",' [, O8 P1 X5 m1 _* I* y
"O": "k8s",4 c' p, _9 a+ P+ `
"OU": "System"! Q% b: v" Y8 A
}+ C( [. _1 V' C/ C1 W- p
]- f& B. v& z: x7 a
}
# i: s0 g5 y8 }$ J$ ucd /approot1/k8s/tmp/ssl/0 H) v E& }4 s
cfssl gencert -ca=ca.pem \
% } E: a" ]+ _( m9 _, S-ca-key=ca-key.pem \
1 k5 p. o# d6 b- B4 ~-config=ca-config.json \% o9 |& F/ z1 _
-profile=kubernetes etcd-csr.json | cfssljson -bare etcd0 W& t. q& `% m8 _. F
配置 etcd 为 systemctl 管理% e& b# o* r" b( Y" W* o; f
vim /approot1/k8s/tmp/service/kube-etcd.service.192.168.91.19
8 ~. w; V. }; z& Q8 W) \这里的192.168.91.19需要改成自己的ip，不要一股脑的复制黏贴
/ A) i/ U$ |: ^
0 {& {3 e6 H) }( z( z7 |etcd 参数* S- k. p# N2 E/ s/ g1 B% x) e
: I- \2 B1 k) Y) C2 o# y3 h8 h[Unit]
9 g3 C; z/ R+ E, i2 t) T5 HDescription=Etcd Server
' X/ i; I8 x$ ^5 p7 l/ fAfter=network.target% @+ A$ L8 l# U2 N9 r% |# I
After=network-online.target8 f" A5 Q: s3 f
Wants=network-online.target9 V/ e& z1 ]- F
Documentation=https://github.com/coreos
! M+ ]# r0 Q# w3 V
: G4 p* k7 l' t& @) e' ]7 K[Service]% M) A* z+ A" B: g
Type=notify
3 M% ~" y! q n5 HWorkingDirectory=/approot1/k8s/data/etcd
1 Y9 h- m1 j, I& D# }$ z& _ExecStart=/approot1/k8s/bin/etcd \
8 a3 |4 N3 G6 {' f, D" a --name=etcd-192.168.91.19 \
, x% Z3 Y5 L& f9 U' A% c" a --cert-file=/etc/kubernetes/ssl/etcd.pem \
7 V7 m, t+ g& R F" U --key-file=/etc/kubernetes/ssl/etcd-key.pem \5 B5 z7 {4 U4 s1 H
--peer-cert-file=/etc/kubernetes/ssl/etcd.pem \! N7 t; o* V) j' U2 G0 H
--peer-key-file=/etc/kubernetes/ssl/etcd-key.pem \
9 |4 j7 z- D- U W9 f, K --trusted-ca-file=/etc/kubernetes/ssl/ca.pem \7 y. p! r3 l6 u- m" J0 l* m3 ]
--peer-trusted-ca-file=/etc/kubernetes/ssl/ca.pem \ o5 ?& G, R2 _; K+ L- j
--initial-advertise-peer-urls=https://192.168.91.19:2380 \! n$ f' s* R+ [* e7 B1 G! j
--listen-peer-urls=https://192.168.91.19:2380 \
) V& d E, E7 I( g2 V --listen-client-urls=https://192.168.91.19:2379,http://127.0.0.1:2379 \) ]5 n" f8 y+ f; m8 P. ]. D
--advertise-client-urls=https://192.168.91.19:2379 \6 ^. o: a6 { S* C& ?+ Y
--initial-cluster-token=etcd-cluster-0 \+ I8 y1 e7 D" F4 E5 F5 \
--initial-cluster=etcd-192.168.91.19=https://192.168.91.19:2380 \
; H+ ~* S* O+ b9 o --initial-cluster-state=new \+ R% j. u2 V1 e" W6 U! B0 R
--data-dir=/approot1/k8s/data/etcd \
$ G+ t3 X8 W1 U --wal-dir= \
' T1 b; Z# o/ F5 Z6 y --snapshot-count=50000 \
; S [. n) s5 ?& K5 w/ G --auto-compaction-retention=1 \
$ D% I( }! @" S9 n( y9 F2 H! ]& ~$ t! |- m --auto-compaction-mode=periodic \
G( A. f' T; Z% N --max-request-bytes=10485760 \5 M: J, |+ N/ D9 V) G. w& i
--quota-backend-bytes=8589934592
# R: j9 I6 w+ k c9 f6 `" \Restart=always
/ T( @5 k8 ]4 i8 [! Z9 hRestartSec=15
9 \; U9 P' q7 \2 p1 FLimitNOFILE=65536
& A) p# `! o7 HOOMScoreAdjust=-9996 |4 E3 q4 x3 n. E) I3 w4 J
. k. R. @* z5 S, ?1 U3 i7 `
[Install], u1 q! u, E4 ~
WantedBy=multi-user.target
5 G9 [1 _2 W8 e6 [& T7 e S9 d' D分发证书以及创建相关路径! [* |7 @ O; Q% d9 F
如果是多节点，只需要在192.168.91.19后面加上对应的ip即可，以空格为分隔，注意将192.168.91.19修改为自己的ip，切莫一股脑复制
+ ~4 `% N0 Y- v8 V" ~9 f; m' d% m% B1 |% l; s' d8 m
对应的目录也要确保和自己规划的一致，如果和我的有不同，注意修改，否则服务会启动失败* h# @# F6 D9 S* O& Z
# B9 f* E" J9 c4 l! M7 m, r
for i in 192.168.91.19;do \
" R. Q" c6 p% S/ Y. j, c0 xssh $i "mkdir -p /etc/kubernetes/ssl"; \
k) O( W% C O6 F+ ussh $i "mkdir -m 700 -p /approot1/k8s/data/etcd"; \
, [! i D5 b; r. Lssh $i "mkdir -p /approot1/k8s/bin"; \! l- N8 q4 ^: T* D9 E
scp /approot1/k8s/tmp/ssl/{ca*.pem,etcd*.pem} $i:/etc/kubernetes/ssl/; \1 Z( P2 v) Q% ]: ]5 a+ L
scp /approot1/k8s/tmp/service/kube-etcd.service.$i $i:/etc/systemd/system/kube-etcd.service; \4 @5 s" R+ ]+ ?) E
scp /approot1/k8s/pkg/etcd-v3.5.1-linux-amd64/etcd* $i:/approot1/k8s/bin/; \) N5 Z/ U- x2 }- o, Y4 ^
done3 B, m4 q1 A y% a
启动 etcd 服务! g% G/ C4 X, G9 q8 k4 m
如果是多节点，只需要在192.168.91.19后面加上对应的ip即可，以空格为分隔，注意将192.168.91.19修改为自己的ip，切莫一股脑复制
( ?& h1 l6 k" ~4 D! t7 v+ B/ W7 {+ ?: w$ O J4 i4 [3 K
for i in 192.168.91.19;do \ K" \( f: n# |+ c0 k
ssh $i "systemctl daemon-reload"; \
6 \' P2 m+ m- xssh $i "systemctl enable kube-etcd"; \
: w5 B. k' B: F' O7 g& A" Yssh $i "systemctl restart kube-etcd --no-block"; \
# K5 ], {: g# [+ k# k8 W) Z# g6 B6 Y" Mssh $i "systemctl is-active kube-etcd"; \. H9 h' x6 E( r4 {% J
done
* F m& w8 D! i2 t- i. [. t返回 activating 表示 etcd 还在启动中，可以稍等一会，然后再执行 for i in 192.168.91.19;do ssh $i "systemctl is-active kube-etcd";done
+ W8 w5 V: h2 P7 T9 N# u( p( _0 Z6 k3 E6 M6 G2 p
返回active表示 etcd 启动成功，如果是多节点 etcd ，其中一个没有返回active属于正常的，可以使用下面的方式来验证集群
' d% ~* C3 S9 M4 g Y* f$ j- N$ m, w& j$ k% ? b" @
如果是多节点，只需要在192.168.91.19后面加上对应的ip即可，以空格为分隔，注意将192.168.91.19修改为自己的ip，切莫一股脑复制: H! Z7 C% x: w2 r0 W4 s5 ^, S
" i7 {; w3 F1 C- H) p$ Ffor i in 192.168.91.19;do \
# `# @2 C" j* z, C- I Z5 Jssh $i "ETCDCTL_API=3 /approot1/k8s/bin/etcdctl \% N5 t' t- R1 q4 D" K: H
--endpoints=https://${i}:2379 \* ?3 L& F2 F% O
--cacert=/etc/kubernetes/ssl/ca.pem \4 Z: n" d- h% Y1 H* V; s5 h
--cert=/etc/kubernetes/ssl/etcd.pem \
& q9 T3 L# \& _- u* M3 X ^ --key=/etc/kubernetes/ssl/etcd-key.pem \; \$ j1 j! B E7 w7 B% V3 T* {" l
endpoint health"; \& Y L# h# q; \% C: l: q
done# l; Q. A% e9 C7 D# \# E; |
https://192.168.91.19:2379 is healthy: successfully committed proposal: took = 7.135668ms
" T2 x' O5 m* d/ V" r
$ g* K1 B3 b2 Q: {+ J- v返回以上信息，并显示 successfully 表示节点是健康的
* G( ]5 O5 F" D5 V- v( G+ `- `/ t$ y9 P. u, \7 R
部署 apiserver 组件
: E) N+ Y' B4 G: X" A9 m创建 apiserver 证书- a2 ]' `# z# n, q
vim /approot1/k8s/tmp/ssl/kubernetes-csr.json
3 \! V2 R2 T* z: \这里的192.168.91.19需要改成自己的ip，不要一股脑的复制黏贴
s3 V& Z+ S" y/ _9 W; @4 v k, S2 C+ t3 S1 V& i9 C. j
注意json的格式
% A( ?7 F# K; j# H
$ B2 Z( ?9 u$ Z# t- B9 v1 x10.88.0.1 是 k8s 的服务 ip，千万不要和现有的网络一致，避免出现冲突. z$ z1 F1 S% w4 _+ r
+ Z, S6 {5 ^2 M5 M# R$ t- J{6 Z' m4 _* n9 T3 U
"CN": "kubernetes",: f+ Y9 m& V" v3 u/ r9 ?( V
"hosts": [' k( w6 M. |/ g- Q" ?) c; I' S
"127.0.0.1",4 f0 Q8 K1 B; z7 s9 P
"192.168.91.19",0 U S. W+ [0 V- z. c' x4 \& F
"10.88.0.1",
* N; V, u% U# C, P6 A* c "kubernetes",
, `2 D; Q% A- p: U+ e7 f2 f "kubernetes.default",
; R# Z! p ?2 v- n; H4 s "kubernetes.default.svc",
' L3 g2 |7 {$ a N% d* C% q "kubernetes.default.svc.cluster"," A! n' K! U1 _% f. f% ]
"kubernetes.default.svc.cluster.local"
% U$ ]& B2 C( [3 l: D ],
/ U' E( e o/ l" k ]& |, V5 c "key": {
7 u8 I$ R% G) }1 Q; ^$ D "algo": "rsa",. b1 L- P- K6 T) l, A
"size": 20485 z$ n( B( I; M; y8 v3 \- x% W/ A
},
: J* r. K2 t' H! o4 \9 H! |; \ "names": [! A) p& g) B" y3 ]; E
{
) c- X' j; H% K# T "C": "CN",7 N' W% E# M* e9 R. u+ F8 j) Q
"ST": "ShangHai",
1 O% _, |9 S9 s& x7 Z$ G+ U "L": "ShangHai",: ]* e6 y$ |7 h+ H/ l/ b
"O": "k8s",. S J" { h. ^' r
"OU": "System"2 l4 r* W3 B3 H
}
' M% v; f2 H( P; C6 z9 K! c9 O" ^ ]
8 N" _' @, u! W) W} s- d5 d: ^; Z# F4 K
cd /approot1/k8s/tmp/ssl/7 g0 ?6 h) f) `, K, P
cfssl gencert -ca=ca.pem \
0 N% a, M+ E; C, v& D, L& G& t-ca-key=ca-key.pem \
@ d/ a: L, j" e4 e-config=ca-config.json \
6 ]! q1 M2 n, c# V( t T. g-profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes
( ^$ b R8 S# n创建 metrics-server 证书& @0 O/ I$ X, k2 p* m
vim /approot1/k8s/tmp/ssl/metrics-server-csr.json
- W, `3 d, D+ t1 _& Z- s8 x{# X ] C: x0 ?! ]7 k9 h* O
"CN": "aggregator",
8 E3 L9 z( B/ [: a "hosts": [
1 f; f8 T7 \3 |4 [! H+ U! S, M ],
; {7 c( V& m" |6 j, @! u4 Q3 M "key": {" n" c0 X, m0 K% ~ @8 o
"algo": "rsa",
9 E$ S9 {5 w, U' ?% r J g "size": 20484 {' `5 ?3 }$ p: b2 Z3 a3 m
},1 U, E9 k, N( t/ E( ]
"names": [
! a- |+ _; Y1 S6 C9 N4 Z8 \' b {
5 l" p( _+ n( o/ f6 W6 R "C": "CN",
. L; s6 L+ @3 l! d3 u; C "ST": "ShangHai",; F/ R2 u: f# R n+ E
"L": "ShangHai",8 S2 D e! t5 j7 ~9 S# I+ |
"O": "k8s",7 P- ]1 v, b1 ]! X' `; t
"OU": "System"
/ @+ O5 S( L7 Z1 x5 e; B }" b' u& e$ v* n1 e3 v
]$ I/ B% c5 K- ]/ ?, f. d
}
4 S+ c9 q: _" M5 o& tcd /approot1/k8s/tmp/ssl/$ q* g2 t8 m0 j( B3 m4 ?
cfssl gencert -ca=ca.pem \
% z3 R' x9 Y7 ^! f1 [-ca-key=ca-key.pem \
b: s- V+ C/ a0 E/ t7 O-config=ca-config.json \
+ m8 P* W, P+ b) P0 X: {& Y8 \3 V-profile=kubernetes metrics-server-csr.json | cfssljson -bare metrics-server
& X; S; [9 J0 }0 g8 h m1 w配置 apiserver 为 systemctl 管理) Z' X6 M! w7 ~
vim /approot1/k8s/tmp/service/kube-apiserver.service.192.168.91.19
: r8 T; `, K [4 K, o' c这里的192.168.91.19需要改成自己的ip，不要一股脑的复制黏贴- d9 T; W) h1 W% V8 u! f: U6 {
; t) t7 d0 f7 d
--service-cluster-ip-range 参数的 ip 网段要和 kubernetes-csr.json 里面的 10.88.0.1 是一个网段的
: `& x1 V% a) f7 W- q) g7 e
7 P3 ~# _7 }& \7 I# W--etcd-servers 如果 etcd 是多节点的，这里要写上所有的 etcd 节点
- ^5 q$ w. r3 g5 @6 g6 G9 g. X" Z, o) ~- Y; P4 |- X
apiserver 参数% |0 h% ?* k0 n. I
: C6 q2 a! x1 G5 G
[Unit]
6 f( [3 C; S% Z0 a5 P* M3 a9 B) g5 p2 DDescription=Kubernetes API Server1 s t* X+ L2 _- o, g3 R7 j
Documentation=https://github.com/GoogleCloudPlatform/kubernetes1 v; W9 w" g0 L7 I; k) n, ^
After=network.target
( w% H! V' P4 n% f k: A
5 q1 }1 ?, Y: v: {( e; |4 X[Service]
# c8 e( k p( H; V& PExecStart=/approot1/k8s/bin/kube-apiserver \
W0 i/ G2 [6 ?' q" Q0 {9 I; U --allow-privileged=true \
, @, e- c; B* f1 y0 q --anonymous-auth=false \. B* z5 j, a& J0 u& {3 [
--api-audiences=api,istio-ca \
5 d0 v) Z1 N8 {! H& K --authorization-mode=Node,RBAC \
5 o% ~. _4 a0 v) } --bind-address=192.168.91.19 \
5 g- m8 x, V5 z, i3 J5 k3 e --client-ca-file=/etc/kubernetes/ssl/ca.pem \
1 [. ?5 \3 v; G2 Q- ~% R --endpoint-reconciler-type=lease \: X9 x5 \1 g" A, G& t
--etcd-cafile=/etc/kubernetes/ssl/ca.pem \& G+ |* N# l |- ~
--etcd-certfile=/etc/kubernetes/ssl/kubernetes.pem \) D% s) O" U3 r: e
--etcd-keyfile=/etc/kubernetes/ssl/kubernetes-key.pem \
" S2 f( N7 X2 m' T- g M- M --etcd-servers=https://192.168.91.19:2379 \, Y2 r# [6 z& J7 s% R
--kubelet-certificate-authority=/etc/kubernetes/ssl/ca.pem \' n1 p' ]0 m/ N6 k. O0 z& M
--kubelet-client-certificate=/etc/kubernetes/ssl/kubernetes.pem \
* r5 B7 D4 L+ b' ? --kubelet-client-key=/etc/kubernetes/ssl/kubernetes-key.pem \
' B, w% U0 J; [: |% R$ ]) O --secure-port=6443 \+ b8 l8 v3 ^3 n8 c k! C
--service-account-issuer=https://kubernetes.default.svc \/ k' i% C6 } P# S0 j+ I5 [
--service-account-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \# f; d5 D, N% n" c) W) G
--service-account-key-file=/etc/kubernetes/ssl/ca.pem \) Q8 b% e' d' F! U; }) z
--service-cluster-ip-range=10.88.0.0/16 \. h8 D4 g9 f0 m1 |& M* }" o
--service-node-port-range=30000-32767 \% ~1 y% B) V& q; E0 V7 b8 B
--tls-cert-file=/etc/kubernetes/ssl/kubernetes.pem \
0 l* S- G9 v1 i q/ A& L, x2 E0 r --tls-private-key-file=/etc/kubernetes/ssl/kubernetes-key.pem \0 P* `" _/ h# p g* f* \, v
--requestheader-client-ca-file=/etc/kubernetes/ssl/ca.pem \7 d$ L% U+ w( X2 k( u
--requestheader-allowed-names= \
4 o ~8 n; Z; S0 ?# m --requestheader-extra-headers-prefix=X-Remote-Extra- \
9 G3 d! d$ u' z% ?, Z --requestheader-group-headers=X-Remote-Group \
3 L7 P1 [& J" Y: b/ [3 T% t --requestheader-username-headers=X-Remote-User \. E2 z ]7 Y) o
--proxy-client-cert-file=/etc/kubernetes/ssl/metrics-server.pem \
. t2 a( J6 x* {/ N8 q --proxy-client-key-file=/etc/kubernetes/ssl/metrics-server-key.pem \; s8 \, ^1 ^7 r; j: t" P" V
--enable-aggregator-routing=true \
1 p$ c8 }" m: u8 o: f --v=2; x1 j4 C; v) a, t9 `
Restart=always
( [/ d8 \' A% q- R6 N6 kRestartSec=5$ ]' e+ ]; ^* C2 Q4 [8 [
Type=notify
/ J) X7 X& m5 r$ n9 c4 Q4 h/ L' BLimitNOFILE=655364 u% u- M1 L: M( h1 S9 r0 R$ x
m' Q9 Y( i+ y; J
[Install]$ N# T; k7 k. _, R U k1 d
WantedBy=multi-user.target
1 R4 t7 T5 ^/ H) P2 s3 Y! R分发证书以及创建相关路径! D' e% U* n) f
如果是多节点，只需要在192.168.91.19后面加上对应的ip即可，以空格为分隔，注意将192.168.91.19修改为自己的ip，切莫一股脑复制" v( n8 W4 H# t8 U8 Y
: Q8 p; Q; v- {( K- P# P; _对应的目录也要确保和自己规划的一致，如果和我的有不同，注意修改，否则服务会启动失败
/ U) U7 C& {0 v/ T
3 x, _7 q7 _- d$ [; r' H k; @for i in 192.168.91.19;do \
5 K$ p/ R1 O! W/ K! q, }/ }* c bssh $i "mkdir -p /etc/kubernetes/ssl"; \
( K7 `' R- z8 @ssh $i "mkdir -p /approot1/k8s/bin"; \
' o/ s0 I3 D0 j3 [* jscp /approot1/k8s/tmp/ssl/{ca*.pem,kubernetes*.pem,metrics-server*.pem} $i:/etc/kubernetes/ssl/; \* z5 ~& t+ ?; m! b
scp /approot1/k8s/tmp/service/kube-apiserver.service.$i $i:/etc/systemd/system/kube-apiserver.service; \
6 F2 r" s% I; D) ^9 m" k( F) Fscp /approot1/k8s/pkg/kubernetes/bin/kube-apiserver $i:/approot1/k8s/bin/; \& o( W9 n Z: ?+ A
done* q+ ~+ W0 b+ M! ^ D) \ k8 E: o% u6 W. g
启动 apiserver 服务
* O! S/ i- z* J$ ]6 j1 U如果是多节点，只需要在192.168.91.19后面加上对应的ip即可，以空格为分隔，注意将192.168.91.19修改为自己的ip，切莫一股脑复制
+ g7 `* l7 k3 a. I% }6 g7 d, Q1 w7 ]8 E) l/ W( p* C
for i in 192.168.91.19;do \1 a# I& u+ o0 N( |& n
ssh $i "systemctl daemon-reload"; \
, S( {! i2 ^5 A' P6 j; |ssh $i "systemctl enable kube-apiserver"; \
, m! g7 K$ q, ~! w0 j% Essh $i "systemctl restart kube-apiserver --no-block"; \
+ B' B1 R- t' g) i. N" Gssh $i "systemctl is-active kube-apiserver"; \# {3 q/ U3 q C
done0 u# e# z0 }) q! T. _1 B$ e
返回 activating 表示 apiserver 还在启动中，可以稍等一会，然后再执行 for i in 192.168.91.19;do ssh $i "systemctl is-active kube-apiserver";done/ N. ]6 u" `4 S) v4 g- v/ X
5 |6 q/ m7 t8 |! Y5 `3 |0 Z返回active表示 apiserver 启动成功
1 g- y0 i9 K$ r k: _, o1 _3 R4 P8 g0 c& n
curl -k --cacert /etc/kubernetes/ssl/ca.pem \
9 f* }; F% \& r--cert /etc/kubernetes/ssl/kubernetes.pem \
( ~; Q$ |* H) B9 a: P--key /etc/kubernetes/ssl/kubernetes-key.pem \/ L3 ~5 i6 _+ x, Z% ~
https://192.168.91.19:6443/api' y7 ~" s/ w, b; f. ~
正常返回如下信息，说明 apiserver 服务运行正常- ^$ g/ L* f0 ?3 _* I7 M- k
& S6 i* x) o+ W* S R/ ]
{, w8 j8 w( T9 g- B
"kind": "APIVersions"," k; l! k- L$ X3 b ~3 H4 h
"versions": [
; E7 V. \ F& l. x- n "v1"
: ]: V3 L7 c) X+ h/ x% ^# V2 u ],% B2 U- T' O2 }/ s+ f$ z% O& P+ h
"serverAddressByClientCIDRs": [' F" F. [. L! s
{
( l2 G: n4 Z2 V7 E3 O3 L2 I& ] "clientCIDR": "0.0.0.0/0",% K! {( M6 L& K% u3 Q* X
"serverAddress": "192.168.91.19:6443" [! [8 u- Y7 F
}
4 E3 }0 E2 O$ }- r( s* i! q ]
8 l/ X7 k9 A: Z) o; L} O$ E! _" s( }1 K( A! C J
查看 k8s 的所有 kind （对象类别）5 S9 _# H/ b% M+ s$ r) P7 d3 A. W
. c4 @2 m: b' b* W9 E: q
curl -s -k --cacert /etc/kubernetes/ssl/ca.pem \
: v9 o3 K! @! U3 r' n7 J6 t) c--cert /etc/kubernetes/ssl/kubernetes.pem \7 n3 x$ b7 x% M% i* w2 H
--key /etc/kubernetes/ssl/kubernetes-key.pem \5 O1 t$ V z1 k* F) I$ u
https://192.168.91.19:6443/api/v1/ | grep kind | sort -u, H5 V) g( o4 \0 x
"kind": "APIResourceList",+ K7 G( r& ^0 l5 c v4 o
"kind": "Binding",. o8 e9 Y! @0 V; h/ l: x
"kind": "ComponentStatus",' z, K7 v( N" b% S$ a
"kind": "ConfigMap",# h. s: L& J; B7 g
"kind": "Endpoints",6 f, Y& F. B3 g( t; u) j. @4 c
"kind": "Event",
6 q# C& J: X1 N9 P J4 F' k "kind": "Eviction",0 O6 h) u, a& E' l
"kind": "LimitRange",
x6 d! h- K% Q J8 O! Q0 l: _ "kind": "Namespace",* e) m7 |3 Z; _* g/ @, I) k
"kind": "Node",
1 H. g) j" V# A. }7 h "kind": "NodeProxyOptions",: ^) R% ^$ J; C; h* m+ ^$ L3 z
"kind": "PersistentVolume",
8 H8 B: L8 X( r$ S2 ~ "kind": "PersistentVolumeClaim",
; e: K g" U8 J2 b- ~9 t "kind": "Pod",- H: E7 s/ ^# v) ~) x8 I
"kind": "PodAttachOptions",1 T# f8 p" r9 ^( {
"kind": "PodExecOptions",3 r& O( g V: g& H2 z2 Q! b
"kind": "PodPortForwardOptions",' ]- e1 g' S, v; b3 r- m
"kind": "PodProxyOptions",
6 }9 T4 J4 G' g# h( r "kind": "PodTemplate",3 T" m B8 g- n" ?9 W/ z
"kind": "ReplicationController",3 G2 b9 B" _+ }" ?
"kind": "ResourceQuota",
' I2 T0 z: V8 u* ~* q0 `' Y9 ] "kind": "Scale",
' }- {" b' C" R2 F8 N6 f "kind": "Secret",$ ?1 P* S$ l. |9 i) O2 {0 v# T3 _2 ~
"kind": "Service", W, ]% [; n1 i p# l& Z3 W
"kind": "ServiceAccount",
1 ?: x9 Q% d5 c! P' r4 z0 n9 q/ m "kind": "ServiceProxyOptions",) g7 p' [( U" t; U
"kind": "TokenRequest",1 c( O! `1 ^; i
配置 kubectl 管理+ N; W) E# ]3 ]3 j% C$ E
创建 admin 证书2 [1 |/ \9 m% W# g' H3 ^, b
vim /approot1/k8s/tmp/ssl/admin-csr.json) T" B' I) F- i& y* A$ O
{; s7 S$ Q( J3 G9 \, Y
"CN": "admin",
! `- U8 r. }: |. ] "hosts": [4 S1 X9 {% z( i$ B( R
],
0 z$ h# c2 |- q; N! o "key": {! T* p+ K4 @5 I6 Y/ J" N
"algo": "rsa",
8 L6 { T1 h. f3 ^' S- h. n, h "size": 2048
0 C3 [. p/ y2 ]8 i/ R! Q+ v4 C },
7 [0 w, C/ G' f& E# m0 B "names": [; I# u! A1 N- w! t/ z" c
{' U, J. H! H6 @$ c( J$ F
"C": "CN"," J8 s4 ~; E& G2 d- d4 O. F
"ST": "ShangHai",; T# D: X! X. M% e* t5 I: d
"L": "ShangHai",/ {1 v/ S4 N6 V8 ^ Q
"O": "system:masters",6 F0 M- K4 _$ l
"OU": "System"
2 W, S0 s, W$ u( u2 y) T0 ~1 Z& ] }5 x) C# B6 e+ U3 C5 N/ D
]
3 B4 u; g' V+ X7 v( z' W& k}1 d. c1 i* C, B9 R0 D
cd /approot1/k8s/tmp/ssl/
. ]! X! F6 _7 I4 l- ~cfssl gencert -ca=ca.pem \
) G5 ]. l) w, a6 W e1 y- q3 H# k* w# u: D-ca-key=ca-key.pem \- S1 J7 U) N) o8 E" |5 ?6 V
-config=ca-config.json \
& |$ r$ }9 M$ ~8 P @-profile=kubernetes admin-csr.json | cfssljson -bare admin
4 }! S# f( t5 d6 C7 S* i' u创建 kubeconfig 证书
( H+ V1 E, X3 e. t7 t% ^设置集群参数
" N: g) Y! K- A( P! |1 X' d& d, m [0 a9 v3 N. c
--server 为 apiserver 的访问地址，修改成自己的 ip 地址和 service 文件里面指定的 --secure-port 参数的端口，切记，一定要带上https:// 协议，否则生成的证书，kubectl 命令访问不到 apiserver
/ m0 o1 Y9 }7 d$ n" S
" F. g& X! N1 d7 X* L5 d, }! C1 `cd /approot1/k8s/tmp/ssl/
Q; e* c7 U9 @, l' n8 _- j/approot1/k8s/pkg/kubernetes/bin/kubectl config set-cluster kubernetes \
- A. w9 L0 [* p! p, e5 U--certificate-authority=ca.pem \
! F5 h7 {; \! Z8 h--embed-certs=true \/ b1 O2 U3 _+ _9 e, U" {; a
--server=https://192.168.91.19:6443 \
7 X& n2 I2 d! O6 u4 W--kubeconfig=kubectl.kubeconfig
: d; {0 T9 B, V5 ~/ H; R设置客户端认证参数
; ?- s! R \( }4 G- ]4 o% Y5 M) |5 i) o6 u# z. ^9 t
cd /approot1/k8s/tmp/ssl/8 M8 M0 W7 \/ Z# O! l
/approot1/k8s/pkg/kubernetes/bin/kubectl config set-credentials admin \
( ^) h# L3 h/ M% C0 `, j- X--client-certificate=admin.pem \
8 s' ]$ B* b, o/ P) \--client-key=admin-key.pem \
' L! B9 G$ l, I. r1 H--embed-certs=true \. ^5 P$ B' v5 t6 L" N; I
--kubeconfig=kubectl.kubeconfig0 ?2 P$ }: K! z6 ~# g& Q
设置上下文参数4 u v' \2 k1 N" G$ B
% U* z. ~3 v2 @/ k2 z' R
cd /approot1/k8s/tmp/ssl/7 b% K* X9 M# A y
/approot1/k8s/pkg/kubernetes/bin/kubectl config set-context kubernetes \
) X; A p2 j8 N5 P--cluster=kubernetes \
5 ]' x1 e0 v( b$ g" P% K--user=admin \
+ L* B8 u" Z+ m0 V% t6 Q9 t--kubeconfig=kubectl.kubeconfig
/ W! E& l2 p0 |- t设置默认上下文
' [ U' ~, i* r, K, v7 m! H( D# n; p% f y" i
cd /approot1/k8s/tmp/ssl/, T+ G) V5 v4 ^% ^0 K! Y
/approot1/k8s/pkg/kubernetes/bin/kubectl config use-context kubernetes --kubeconfig=kubectl.kubeconfig
% n2 V4 o9 ^) b: Y% \6 K9 ?分发 kubeconfig 证书到所有 master 节点
2 _9 v9 @) ?& D E如果是多节点，只需要在192.168.91.19后面加上对应的ip即可，以空格为分隔，注意将192.168.91.19修改为自己的ip，切莫一股脑复制, z5 ?6 |$ {1 f' F: n" b
) j5 ?* s& A- ?7 ofor i in 192.168.91.19;do \) h/ H1 z" {. Z' m8 ]
ssh $i "mkdir -p /etc/kubernetes/ssl"; \
( S5 m( N4 r/ pssh $i "mkdir -p /approot1/k8s/bin"; \# ?1 ^. K$ O. ]: O" l
ssh $i "mkdir -p $HOME/.kube"; \4 y. Y" |( p9 ?( A/ S6 G
scp /approot1/k8s/pkg/kubernetes/bin/kubectl $i:/approot1/k8s/bin/; \
% t$ w, S3 R& S3 `$ w0 ?, O& U. t1 vssh $i "echo 'source <(kubectl completion bash)' >> $HOME/.bashrc"
9 i- ]# ~3 {: q; Nscp /approot1/k8s/tmp/ssl/kubectl.kubeconfig $i:$HOME/.kube/config; \
6 q! J1 Y' ]+ g. i$ gdone
2 b/ r& P7 Q% {) c5 Z部署 controller-manager 组件; y4 T3 C( \; ?- Q- d$ ^
创建 controller-manager 证书
, V: u2 d. C& R }9 B. W6 lvim /approot1/k8s/tmp/ssl/kube-controller-manager-csr.json
$ D9 N6 l3 v3 J7 r+ W1 g) M这里的192.168.91.19需要改成自己的ip，不要一股脑的复制黏贴
0 p& n( X. ?* [# \2 i4 s1 Q' L' f! ~
注意json的格式
0 U3 u) P8 u8 ?$ M1 |
" E' s5 C" y y7 ^{
. W4 h4 W' S! c. G! F x. l1 P "CN": "system:kube-controller-manager",
& \' H4 W- {. r5 J "key": {
3 x1 n8 |1 L- y: U "algo": "rsa",
( w8 l4 p: k9 y "size": 20482 G- H. H0 V0 m" I
},
# U* [4 b0 C$ ~3 Y; i "hosts": [* b. K& J' V, @% O# d" `
"127.0.0.1",4 N4 r; V6 u- v: q( t0 I% M& P
"192.168.91.19"4 c" ]& X H/ r
],: L( q3 K; a3 ` ]
"names": [ |6 q- ~# x1 P' Z
{7 \* F0 U/ }/ ?' J9 f. n# w
"C": "CN",1 [! ?. k1 |. y* O1 ]8 l
"ST": "ShangHai", s8 X1 c/ |7 Y* L9 _. u
"L": "ShangHai",: }( g" a! ?* p. z
"O": "system:kube-controller-manager",& x5 k; Z: d8 Q v
"OU": "System"
Y0 U0 ?& ]% m% ^ }& o) a* m2 V; d% N
]
- E. c# f+ Y0 y1 p+ P% E}
' ]1 h* H5 ^, Z0 Z7 Lcd /approot1/k8s/tmp/ssl/
, H& t% Z- ]$ o' J1 ?1 Kcfssl gencert -ca=ca.pem \
% R* T7 i1 C. S, M# n-ca-key=ca-key.pem \
h4 i' M8 a4 A-config=ca-config.json \
: r1 c5 }/ J0 W8 ?! B-profile=kubernetes kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager+ v i; ]" }" C% L
创建 kubeconfig 证书 G# v9 K( G2 J% ?% ~
设置集群参数- T7 t! i, M, E( ]2 j
) w. b2 z/ g) Y/ v% k* I, d
--server 为 apiserver 的访问地址，修改成自己的 ip 地址和 service 文件里面指定的 --secure-port 参数的端口，切记，一定要带上https:// 协议，否则生成的证书，kubectl 命令访问不到 apiserver
5 t/ t, w8 I; K/ \0 i% `4 k7 R, z' [* N7 Q4 H7 I4 s
cd /approot1/k8s/tmp/ssl/
, ]5 `! |3 E) h+ A2 `& y: q/approot1/k8s/pkg/kubernetes/bin/kubectl config set-cluster kubernetes \5 o1 \5 I& V- }. O
--certificate-authority=ca.pem \7 `& u/ A1 f0 r) U, i* `
--embed-certs=true \8 i/ A+ |5 q7 ~/ V0 X
--server=https://192.168.91.19:6443 \
: b/ g% H( B# R: Y6 X3 X+ W--kubeconfig=kube-controller-manager.kubeconfig
; K5 ^" Y# z$ A设置客户端认证参数0 b4 p. C- `- N7 d* d3 C0 j
9 F P) C6 B& D3 a2 u+ B
cd /approot1/k8s/tmp/ssl/
* `1 [. Z. k8 v0 p& U2 P, c/approot1/k8s/pkg/kubernetes/bin/kubectl config set-credentials system:kube-controller-manager \+ Y- `" G& P: U2 m. r' d, F+ s6 t
--client-certificate=kube-controller-manager.pem \, \3 I& V: Q6 U
--client-key=kube-controller-manager-key.pem \- y' B+ e9 Q& ?% L
--embed-certs=true \+ U2 j+ l/ [' H# w8 `9 o
--kubeconfig=kube-controller-manager.kubeconfig
& {, P8 x% T* z8 ^: W' m设置上下文参数' M5 C9 g8 `! Q) F( a% F1 A) y8 M
& n! o4 f% C$ E, ?9 g
cd /approot1/k8s/tmp/ssl/+ X! U* w2 Z# s
/approot1/k8s/pkg/kubernetes/bin/kubectl config set-context system:kube-controller-manager \
8 c( w5 s% y3 u- r$ ?3 A8 S# t--cluster=kubernetes \
0 K5 t) K/ j! X. V5 ]0 U--user=system:kube-controller-manager \2 o. B8 J1 P+ [* ?0 K: J
--kubeconfig=kube-controller-manager.kubeconfig' J p0 b! C% l8 Y* ^( J
设置默认上下文
e8 m' ^( u: }' h
8 g" t9 a% y5 k+ q m7 m h( vcd /approot1/k8s/tmp/ssl/4 ]- Q' z% h. d2 e+ j T
/approot1/k8s/pkg/kubernetes/bin/kubectl config \, T, n0 f8 l6 V1 R
use-context system:kube-controller-manager \
/ T3 z9 d" A3 s0 N/ D--kubeconfig=kube-controller-manager.kubeconfig; s& B( F# q4 z1 J
配置 controller-manager 为 systemctl 管理
. M8 j9 A' A* N: i# d( ]vim /approot1/k8s/tmp/service/kube-controller-manager.service" P. k2 K& X# ?5 R; v& W8 T0 T9 d
这里的192.168.91.19需要改成自己的ip，不要一股脑的复制黏贴1 V0 x5 e) t& O$ ^
) a9 ], X# t9 N* ?1 N--service-cluster-ip-range 参数的 ip 网段要和 kubernetes-csr.json 里面的 10.88.0.1 是一个网段的
6 e$ K5 W" o/ W p" e# h/ h- U) B( Y/ u5 G1 L
--cluster-cidr 为 pod 运行的网段，要和 --service-cluster-ip-range 参数的网段以及现有的网络不一致，避免出现冲突
2 U# M' T* I G c8 i, @' m. d: e" Y9 J. P
controller-manager 参数% N5 n4 ]0 l7 X4 k
: h1 r* \1 O$ v+ |- j
[Unit]
5 P9 B% W; V6 RDescription=Kubernetes Controller Manager
' K* w* t; L0 C1 c$ Z& oDocumentation=https://github.com/GoogleCloudPlatform/kubernetes9 r7 j; S8 P [( ^6 z6 K+ F+ w# {, H
5 T3 d! u1 X- v, q/ ]# I, f3 t: o[Service]) k _: U' G0 z f8 V$ D
ExecStart=/approot1/k8s/bin/kube-controller-manager \
( _6 M ~. g, x7 C' r% s --bind-address=0.0.0.0 \" Z \8 c, r7 I4 G" N$ I. n
--allocate-node-cidrs=true \8 l8 g$ f; n# _/ A% B. x& J5 C" a. B
--cluster-cidr=172.20.0.0/16 \& `3 j8 D4 L7 [* V% ?# \
--cluster-name=kubernetes \2 e! Q, Q) f; N/ q, K
--cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \; L8 \/ x% u& c# h% J9 y
--cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \0 e& d0 N8 M6 _& [0 Y; T
--kubeconfig=/etc/kubernetes/kube-controller-manager.kubeconfig \; k7 O* ?. V5 S
--leader-elect=true \
& R2 G0 a! j; b2 U2 K --node-cidr-mask-size=24 \3 P, }0 t+ k8 w
--root-ca-file=/etc/kubernetes/ssl/ca.pem \- D: Q) F' {" z- o- ]5 _2 Q
--service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem \8 ?0 N3 W2 B) Z& j) g. @& U( W: W: \( }
--service-cluster-ip-range=10.88.0.0/16 \* }( X1 r( D! A2 B F
--use-service-account-credentials=true \/ m, c# G# K/ b: @. \2 ]" p |
--v=2
$ E% O* g9 o8 e# f. XRestart=always
" c: N; `$ w1 t' K8 r1 mRestartSec=5) x7 n1 [+ B0 ], M; o) p, ]
: \; n, |! _8 Q6 |! @. |4 r[Install]7 y$ r9 M% j/ U
WantedBy=multi-user.target
# D8 A* x9 s; ^, }分发证书以及创建相关路径
& ~& _* y7 ]2 c, y, E5 j1 w如果是多节点，只需要在192.168.91.19后面加上对应的ip即可，以空格为分隔，注意将192.168.91.19修改为自己的ip，切莫一股脑复制
$ c% g7 m3 `0 w) h- m5 [
4 w3 v5 t! q$ U0 r* N对应的目录也要确保和自己规划的一致，如果和我的有不同，注意修改，否则服务会启动失败
+ l- H' e; H: N, |6 R' L& L
* s- w% ` m3 o& l; R$ U" f$ ofor i in 192.168.91.19;do \& k8 |3 e. W/ h; e2 ]% a
ssh $i "mkdir -p /etc/kubernetes/ssl"; \5 K+ r+ a+ x% G3 ~' Y
ssh $i "mkdir -p /approot1/k8s/bin"; \9 M/ ]& M" v! j* z) Y _+ E
scp /approot1/k8s/tmp/ssl/kube-controller-manager.kubeconfig $i:/etc/kubernetes/; \% ?4 F& ^5 v& u2 u& b0 S
scp /approot1/k8s/tmp/ssl/ca*.pem $i:/etc/kubernetes/ssl/; \# u" ?( M5 _6 S8 }
scp /approot1/k8s/tmp/service/kube-controller-manager.service $i:/etc/systemd/system/; \
$ D4 [8 l) I! hscp /approot1/k8s/pkg/kubernetes/bin/kube-controller-manager $i:/approot1/k8s/bin/; \
; [7 s9 V7 T( `& S, sdone/ Z9 I# v# G2 L& D A
启动 controller-manager 服务7 f g* P9 O, s* y" y/ \
如果是多节点，只需要在192.168.91.19后面加上对应的ip即可，以空格为分隔，注意将192.168.91.19修改为自己的ip，切莫一股脑复制
) s) M4 `$ y5 Y, J6 Q
3 o6 M8 y; h+ k& X' n( x, [& T; mfor i in 192.168.91.19;do \, u+ ?+ R' }: C1 n9 z. p2 l$ C) ^% w
ssh $i "systemctl daemon-reload"; \7 ]* |+ V' O+ d/ u4 V) _3 j
ssh $i "systemctl enable kube-controller-manager"; \
$ p# ?# d7 A+ K% y$ N* yssh $i "systemctl restart kube-controller-manager --no-block"; \
. u2 a7 b( x. Y2 ? hssh $i "systemctl is-active kube-controller-manager"; \
; n+ D& T6 K1 e$ D- ~4 s/ wdone& Y2 s7 }4 B* c
返回 activating 表示 controller-manager 还在启动中，可以稍等一会，然后再执行 for i in 192.168.91.19;do ssh $i "systemctl is-active kube-controller-manager";done
& \, G. Q3 V/ x" P3 @, H" E# c) N. ^. k( U# K6 @
返回active表示 controller-manager 启动成功9 t% G: w1 M5 h) F# l5 |
- P/ Y- b c T; L: z$ g2 k
部署 scheduler 组件
$ A% N% F, A# O' Y; O, A创建 scheduler 证书- A3 }2 ]6 i% F# E: o6 m
vim /approot1/k8s/tmp/ssl/kube-scheduler-csr.json( I, `+ N- b# [: |# ?; l+ W& |/ G
这里的192.168.91.19需要改成自己的ip，不要一股脑的复制黏贴
# M5 e( n# g! z: p t$ H z: F/ N! E: e- _/ P( [
注意json的格式
/ o( x+ j$ ~1 J, I
( @9 S& J+ B }+ y{$ l& X! Z4 L; o/ ^, a
"CN": "system:kube-scheduler",& [2 }6 K6 |) p8 Y# O7 `/ _# v
"key": {
0 P/ t& r" q D/ T/ Z( w- s0 s& Y "algo": "rsa",
# o4 k; ^. i# |# h% e* Y "size": 2048& B, l8 S+ i s2 o
},
w# `' S# _" _ i2 J6 F" A3 h& s "hosts": [
9 ^1 k) E. u' a& m+ i% ~ "127.0.0.1",. O$ U# j* y, a2 l
"192.168.91.19"! N1 ~ O0 O# f' W G8 k
],- V: f2 E" s: a6 U$ f; ?9 h# L
"names": [
5 S, P' O- w* y$ F5 T: x {" o. d, A3 a' X# p
"C": "CN",, f3 r6 I5 w0 {% L: O5 V
"ST": "ShangHai",
! }+ p: Y$ w/ n* M* c "L": "ShangHai", w. s/ J7 c2 w- j$ v3 z
"O": "system:kube-scheduler",
! F7 H* O) }, V X6 | "OU": "System"
* V- k/ e. P, J( _( m }
: B w/ ?8 v2 k* ^- b$ ` ]
6 ~+ I( h) G; X) b9 U! c; `}' W6 `/ k0 X1 Y( M& r5 ]$ x; T
cd /approot1/k8s/tmp/ssl/0 d1 P# A. x) ~0 E1 ]) z0 w5 S. ~
cfssl gencert -ca=ca.pem \
) g- g' W5 w) }# |8 ~# u1 ~-ca-key=ca-key.pem \$ X$ g6 s G6 `+ H5 e
-config=ca-config.json \
. Q% v) C; b; K: w3 K-profile=kubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler9 F7 `. c# I; }
创建 kubeconfig 证书
Z% y* i0 p. W* n3 ]设置集群参数3 a" h4 Q, i# V" S! p2 G
, |$ _* X* k+ j B--server 为 apiserver 的访问地址，修改成自己的 ip 地址和 service 文件里面指定的 --secure-port 参数的端口，切记，一定要带上https:// 协议，否则生成的证书，kubectl 命令访问不到 apiserver
: V; K6 d$ |# e( K5 ^! v3 \$ N
- {3 z1 {( W- l" O& r) m5 ~/ Dcd /approot1/k8s/tmp/ssl/$ C$ O9 ]4 K; l9 |
/approot1/k8s/pkg/kubernetes/bin/kubectl config set-cluster kubernetes \
4 `" O% Z* G, T7 ]% v3 g--certificate-authority=ca.pem \
/ L: T1 o0 ^3 k1 Y) I: I--embed-certs=true \0 K6 _; L* I' k1 V+ S) b4 Y
--server=https://192.168.91.19:6443 \
. E7 v* a) }2 M5 n5 H--kubeconfig=kube-scheduler.kubeconfig( B& F+ m! I1 y, S
设置客户端认证参数
- l' v/ a+ _9 t9 `5 S( B, j
: r& C) d# i1 H0 wcd /approot1/k8s/tmp/ssl/
4 K& J' s( Y( ~6 R1 o, z; I/approot1/k8s/pkg/kubernetes/bin/kubectl config set-credentials system:kube-scheduler \2 N$ D) Z6 l& b) [
--client-certificate=kube-scheduler.pem \
" L: v7 J0 }6 b--client-key=kube-scheduler-key.pem \
W3 h g- O& D) |- M--embed-certs=true \
) o7 i: L! M4 A5 p--kubeconfig=kube-scheduler.kubeconfig% c9 [. P8 Z/ D3 D2 _
设置上下文参数. O. Z! b& l( z/ g$ I
+ Z% e3 y$ I" a0 L" V5 N: x, I/ Zcd /approot1/k8s/tmp/ssl/2 b' c' l1 {) t w1 o: ]: D: u
/approot1/k8s/pkg/kubernetes/bin/kubectl config set-context system:kube-scheduler \
- C1 Q) w1 J# U+ q--cluster=kubernetes \
2 n0 f6 I5 `$ b--user=system:kube-scheduler \5 W; z, y$ b0 }0 V8 u2 n6 L
--kubeconfig=kube-scheduler.kubeconfig
: M2 [( e! C( l设置默认上下文) a/ ~' \6 W& G: o2 `: L
% L" U% t# L9 I4 k
cd /approot1/k8s/tmp/ssl/7 v8 V# L2 y( s3 u* b7 i: E
/approot1/k8s/pkg/kubernetes/bin/kubectl config \
4 X/ M% O: R5 Cuse-context system:kube-scheduler \
N, \! N- b3 j+ p j--kubeconfig=kube-scheduler.kubeconfig
$ p. w! e% E4 V `6 W# \配置 scheduler 为 systemctl 管理: ]0 T& ?6 N) Y: _' w) l- P7 X( O
vim /approot1/k8s/tmp/service/kube-scheduler.service5 K2 v$ b: G; L/ W
scheduler 参数 q( P% a! x+ N$ e
, f$ f; D; ?0 f; ]! S# D
[Unit]
% z* f7 _7 e, I9 jDescription=Kubernetes Scheduler0 ?( g; @$ j$ G5 l' l! m, |6 @
Documentation=https://github.com/GoogleCloudPlatform/kubernetes0 L7 v2 U3 V& y+ g8 L
U: H: v# H. O$ E
[Service]
' Q3 c5 Z: d- S. n% ~1 RExecStart=/approot1/k8s/bin/kube-scheduler \2 F; l, l8 T" G* @7 C3 z3 s# a
--authentication-kubeconfig=/etc/kubernetes/kube-scheduler.kubeconfig \
; Z/ E! I( A# |! x: I; i+ U8 ~ --authorization-kubeconfig=/etc/kubernetes/kube-scheduler.kubeconfig \1 @0 C4 k) ~. F( M" _- s; e0 i D
--bind-address=0.0.0.0 \* [) A# T8 b0 L2 f# E
--kubeconfig=/etc/kubernetes/kube-scheduler.kubeconfig \! _& c+ c) j0 s# \6 u
--leader-elect=true \
; R, k0 B) L D0 B --v=2
- b+ r' q1 g0 o! U2 Z+ K! v* HRestart=always; B {9 K; Z9 A- d7 k4 z
RestartSec=5
8 E( `5 S' j; u3 W# [0 o& j- E6 _, X2 T# v; M1 y1 d: G
[Install]' R6 G; o7 y# z4 \" g ^; ^
WantedBy=multi-user.target$ x t+ g0 R& V9 Y ]/ W
分发证书以及创建相关路径! j& Z5 l% ]) c" \) }' s# p
如果是多节点，只需要在192.168.91.19后面加上对应的ip即可，以空格为分隔，注意将192.168.91.19修改为自己的ip，切莫一股脑复制9 e! R3 A8 o9 m' U* w
: L }0 z/ d; y; G6 H' Q
对应的目录也要确保和自己规划的一致，如果和我的有不同，注意修改，否则服务会启动失败
) m+ q% e5 x" j i- e0 Y( s2 }& ~. s
% B9 u- @* L* @2 }1 d0 Yfor i in 192.168.91.19;do \; n) E$ R# w4 s2 L; ~ \
ssh $i "mkdir -p /etc/kubernetes/ssl"; \
- ~' J# {, t6 |6 F3 Q* e7 vssh $i "mkdir -p /approot1/k8s/bin"; \
# U4 O! ?4 c! W! a Uscp /approot1/k8s/tmp/ssl/{ca*.pem,kube-scheduler.kubeconfig} $i:/etc/kubernetes/; \3 @" s: F: B( G3 h% i0 j7 C$ v% w
scp /approot1/k8s/tmp/service/kube-scheduler.service $i:/etc/systemd/system/; \- i$ I! N: \4 T! g3 F
scp /approot1/k8s/pkg/kubernetes/bin/kube-scheduler $i:/approot1/k8s/bin/; \. k( G% }% [! A1 P2 `% O; W
done" B, L+ U: i2 I# a. I# g0 q
启动 scheduler 服务
. d2 t) d% c& B0 J如果是多节点，只需要在192.168.91.19后面加上对应的ip即可，以空格为分隔，注意将192.168.91.19修改为自己的ip，切莫一股脑复制
2 B' [7 d' C$ W$ C" O2 N4 c" l5 l. ^2 q3 h
for i in 192.168.91.19;do \3 H6 t3 L( R# z+ Z/ a6 M. X% W& t
ssh $i "systemctl daemon-reload"; \# O# l# Z. n/ x( @" x) ~
ssh $i "systemctl enable kube-scheduler"; \1 x, i1 z/ x. K/ l7 o# ^% {
ssh $i "systemctl restart kube-scheduler --no-block"; \
4 Y& e6 L1 P# M Cssh $i "systemctl is-active kube-scheduler"; \5 ^) i p# h% \( _
done
6 k' W( L* q s0 a返回 activating 表示 scheduler 还在启动中，可以稍等一会，然后再执行 for i in 192.168.91.19;do ssh $i "systemctl is-active kube-scheduler";done
) F' W' e! Y: @, g, t4 e' P0 I5 {! K# J& X+ p% d4 b
返回active表示 scheduler 启动成功$ g. x: ~$ ]" V3 a8 Z' R
6 b+ d4 F* k, i5 Z
部署 work 节点* f0 v% s' z/ Q1 m) [; c& ?# c
部署 containerd 组件8 d/ g$ B* U. n) }3 D$ P
下载二进制文件
0 z) T# r! t6 t! v0 C- Z; Pgithub 下载 containerd 的时候，记得选择cri-containerd-cni 开头的文件，这个包里面包含了 containerd 以及 crictl 管理工具和 cni 网络插件，包括 systemd service 文件、config.toml 、 crictl.yaml 以及 cni 配置文件都是配置好的，简单修改一下就可以使用了8 D4 M& P' p# n' l
, I* f$ s' P4 \0 D& W/ u& e, Z, J0 E4 U
虽然 cri-containerd-cni 也有 runc ，但是缺少依赖，所以还是要去 runc github 重新下载一个
/ l% W3 N q. Z6 _9 N/ P) R! l, n) H
wget -O /approot1/k8s/pkg/containerd.tar.gz \
# j" K* L9 X Ohttps://github.com/containerd/co ... -linux-amd64.tar.gz. j- Q1 w6 D$ ]- R1 y2 V
wget -O /approot1/k8s/pkg/runc https://github.com/opencontainer ... d/v1.0.3/runc.amd642 J8 Z2 S# @9 A. A8 N8 K2 n
mkdir /approot1/k8s/pkg/containerd
0 {$ F' w( s7 }9 C5 q+ gcd /approot1/k8s/pkg/
/ G4 q# D f5 ~6 Mfor i in $(ls *containerd*.tar.gz);do tar xvf $i -C /approot1/k8s/pkg/containerd && rm -f $i;done
5 p% h& B- V* J$ N1 C% r9 Achmod +x /approot1/k8s/pkg/runc3 c+ s. n+ U T1 m1 q% O0 [
mv /approot1/k8s/pkg/containerd/usr/local/bin/{containerd,containerd-shim*,crictl,ctr} /approot1/k8s/pkg/containerd/" O, q* o& S4 x$ D% y) D
mv /approot1/k8s/pkg/containerd/opt/cni/bin/{bridge,flannel,host-local,loopback,portmap} /approot1/k8s/pkg/containerd/
- P$ ?+ `$ D# J- @. krm -rf /approot1/k8s/pkg/containerd/{etc,opt,usr}/ p8 H4 V0 L- c# ]1 K/ n2 M% ?
配置 containerd 为 systemctl 管理
8 t( k: ^! T/ N$ v3 S% c7 ?vim /approot1/k8s/tmp/service/containerd.service
: V0 V* l) L. |6 e9 V M) H注意二进制文件存放路径
% B1 p/ i6 A+ c/ R9 m: b) ^9 H! Z# [9 Z+ }( P {+ B+ g, K8 N2 U1 b" C
如果 runc 二进制文件不在 /usr/bin/ 目录下，需要有 Environment 参数，指定 runc 二进制文件的路径给 PATH ，否则当 k8s 启动 pod 的时候会报错 exec: "runc": executable file not found in $PATH: unknown
$ ]7 V/ W7 y$ ^/ \9 K( z
+ @$ R/ {& W- P; K- b7 G" E& n[Unit]
+ R; r7 S1 s3 f/ P. ~9 `( I8 Y1 UDescription=containerd container runtime
; |; c" x9 A6 Q3 MDocumentation=https://containerd.io
% r! K* N3 T- N. vAfter=network.target
3 C; B6 W9 R+ ]* [3 Y7 |" e6 n! l* m: l" I7 {3 Y8 c1 Z% w6 C
[Service]3 x6 F/ z S6 K3 k
Environment="PATH=$PATH:/approot1/k8s/bin"
2 K/ m& B Z! k' Q* O8 p/ cExecStartPre=-/sbin/modprobe overlay
5 p9 H% l* _4 ~2 E# N) [3 WExecStart=/approot1/k8s/bin/containerd% U( ~# F" I' M) ]+ r ~- G. R
Restart=always
- l8 o( K2 s7 |) B `4 GRestartSec=5
7 B; f9 R7 d9 Y8 c6 p( k; ADelegate=yes+ G; k/ z, z& q- X$ G% y* T8 j3 D
KillMode=process
3 W; y4 p7 R0 Y5 S, |& ` W4 ROOMScoreAdjust=-999
{& @2 c9 A1 E! JLimitNOFILE=10485769 Z3 f. h* E8 T& d5 `7 {/ w
# Having non-zero Limit*s causes performance problems due to accounting overhead4 S4 ?. _+ f; T; C2 r* `
# in the kernel. We recommend using cgroups to do container-local accounting.
5 T9 J- s V; z$ jLimitNPROC=infinity
7 H: W& b3 n6 D! mLimitCORE=infinity! T b$ c% ~7 N7 T9 a. I V
. E7 Y: L5 t% v+ v* n[Install]
- Q# L3 O1 q. F2 h, U9 e+ MWantedBy=multi-user.target/ g- m: u; S6 i, W! B
配置 containerd 配置文件
8 V" O! M' K9 p3 K2 fvim /approot1/k8s/tmp/service/config.toml
0 E+ j& ]+ F5 A6 ^- @8 groot 容器存储路径，修改成磁盘空间充足的路径
2 s' ]" _8 _2 I' G2 j* I2 _8 S& N
) D+ ]5 ^: T1 P- Q% Rbin_dir containerd 服务以及 cni 插件存储路径
- \7 n* G2 g- ?* G" R1 X( p1 A- o3 j- @
sandbox_image pause 镜像名称以及镜像tag
* ~& w( A7 |/ ^0 b4 e" v. m4 a& g- O
( V l, \' P: x+ B, _# Zdisabled_plugins = []
( i) K, O6 N) Q, C+ bimports = []
! g0 l* T4 k+ _' R4 boom_score = 0
\; }9 v% r% }2 K) \/ O' }plugin_dir = ""
- w4 M/ e$ e& F/ _required_plugins = []
( n( s, W2 C6 ]) Proot = "/approot1/data/containerd"
/ C; Y; R4 @% D$ U% {) Vstate = "/run/containerd"7 z1 a# z; x8 M6 d3 |- a
version = 2
" J O* W1 o8 s, A+ L+ I" o+ r4 @5 v: r2 H
[cgroup]
6 ]9 g/ b2 s; z' S8 n. J# U t1 M$ p path = ""- s8 w5 ^" i! k& R
6 x! o2 W# c8 S8 d9 X& O[debug]
0 P4 ^/ |+ \3 m$ P" r address = ""0 n+ f' R4 S& ]9 S
format = ""
/ a4 u' {' Y& f& e7 D gid = 0$ `2 V! W" M: M# [9 n7 E
level = ""
9 c* O" h. X& z( K- X- c uid = 0, |+ e6 `5 M) `1 E0 @! O4 w% A D
# E& y+ x$ t6 t& S9 z
[grpc]
) G! M* |& E c. g6 ]3 F( c address = "/run/containerd/containerd.sock"
& O$ J2 t2 |; f' \5 ~0 t gid = 0/ v- V7 j9 [4 H. I' v% k( j
max_recv_message_size = 16777216
3 n1 h9 ^8 N4 u. c1 e max_send_message_size = 16777216$ p3 R E( Q4 x
tcp_address = ""
$ s9 P, B9 E. H" E2 S1 s& C3 N tcp_tls_cert = ""1 i( T3 ^* R0 d9 d3 _' u8 |( t
tcp_tls_key = ""( U0 F, K1 Q: T1 T
uid = 0
0 b4 p. q" D% _6 n$ l3 t" Q3 O0 g: k. m9 g8 Y
[metrics]
. j& B. V3 U5 R address = ""8 m$ H! ~+ y4 E: ]7 b' U
grpc_histogram = false
4 ^( z% \7 r i# V
4 j, C7 I" T4 ?! }[plugins]6 P8 W! ]- |0 C2 Z, i
1 |# f$ [3 n- x/ Z; ^$ M [plugins."io.containerd.gc.v1.scheduler"]
: A1 @* D4 v, W" W9 s deletion_threshold = 0
* H% {, ?% d) ]0 @ mutation_threshold = 100
, W1 M z$ Y- z pause_threshold = 0.020 B8 }; d2 B3 @- l4 e' P; p$ c
schedule_delay = "0s") R9 v5 R8 a" S0 O& o9 f' w+ {" `
startup_delay = "100ms"- U; \& C+ F4 A2 ~4 d a3 U
5 e L8 d2 ?$ z( c+ l
[plugins."io.containerd.grpc.v1.cri"]3 v" t2 Q) V0 v; _. B! A+ C
disable_apparmor = false
1 z1 z5 m4 M" S+ C; O disable_cgroup = false
* q* f K: o7 q9 m6 d disable_hugetlb_controller = true0 L G1 E$ @2 n# B$ d
disable_proc_mount = false0 G9 k4 ?" ]$ P. w/ a
disable_tcp_service = true1 S8 l4 p% L/ J$ m
enable_selinux = false6 \8 p% h) |& @% {, M
enable_tls_streaming = false
- J' O+ A0 |# K' R- ^7 d ignore_image_defined_volumes = false% _' W7 L' Q. Y7 n/ Y9 O
max_concurrent_downloads = 3! |) H7 i& ~2 l4 C
max_container_log_line_size = 163849 I; w' _2 f1 S
netns_mounts_under_state_dir = false
4 r1 P' J5 g% j: E3 }1 M; m$ G4 u restrict_oom_score_adj = false6 r" F5 u! @: B1 t8 Z J" T
sandbox_image = "k8s.gcr.io/pause:3.6"
/ e. G+ p8 S/ `; R* K selinux_category_range = 1024
/ X4 F- Z+ S4 W% A8 q stats_collect_period = 10$ q! c4 Q( Z- s
stream_idle_timeout = "4h0m0s"
/ t3 a. h1 n' m( U; @- k stream_server_address = "127.0.0.1"
0 k# K P+ o" a+ r4 }; v stream_server_port = "0"
& T( q" b a# J- O6 u systemd_cgroup = false0 A3 S0 X$ x% {' H$ @$ J9 C4 \
tolerate_missing_hugetlb_controller = true; B6 v! b1 _/ O) I0 Y1 H
unset_seccomp_profile = "", m+ ]) F; e" N, m M
% \- c$ x& Y+ i. y" B$ Z [plugins."io.containerd.grpc.v1.cri".cni]$ W- v' Y+ W. P3 S
bin_dir = "/approot1/k8s/bin" _4 \% v& i+ ~$ x1 d4 C
conf_dir = "/etc/cni/net.d"
) K& Y: ?5 j$ y, o; l, X# [ S conf_template = "/etc/cni/net.d/cni-default.conf"
* h. Q; Q, v! J- B- H; I max_conf_num = 1' k6 ~0 S+ h3 l) H$ A0 |! V
; ?/ Q& `6 N2 H0 d [plugins."io.containerd.grpc.v1.cri".containerd]5 I% c: F% _) G* w+ G; I3 A% F
default_runtime_name = "runc"' J# F" B. @0 s( |! a# ?. T% z- r
disable_snapshot_annotations = true( o$ O M1 T7 w
discard_unpacked_layers = false
0 R) ]- E" v; h! {4 ` no_pivot = false4 D& N/ Q' [ I2 a
snapshotter = "overlayfs"
+ Z1 B. m2 Q# u& \
: I+ v% _& g2 E& | [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
+ C2 `( Z0 a8 ?& c2 w3 p6 s5 r base_runtime_spec = ""6 ?9 Y5 t- x" [
container_annotations = []
" B$ {& H2 ^' O% m; z' _ pod_annotations = []) O: O4 w* h$ Z% @$ [8 T* q
privileged_without_host_devices = false2 a8 t8 P8 E& h2 i
runtime_engine = ""
9 Z6 n" o Z/ I5 v: r9 D- H runtime_root = ""- ^5 I5 V0 e5 g
runtime_type = ""2 _% p, {9 g" v
. K: a# [( b& q3 j3 G [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime.options]6 O |. \. A/ j7 g. m
; q: F+ a h8 F: [% o1 c7 U [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
( X8 Z* \& @8 l. W& W) d
3 y% _: u: a8 h7 C7 l [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
, q5 x2 {% v. Y; m j+ S base_runtime_spec = ""% ]& C4 n! {! [+ C1 D5 T
container_annotations = []
/ ~/ h+ y: a7 a+ |0 ^. e8 j pod_annotations = []
( U+ K$ t! v. C# I" G privileged_without_host_devices = false# Y+ x- Y# g$ k0 Q8 z8 ~/ h
runtime_engine = ""
0 T, N$ A- y" a# ~ runtime_root = ""5 O# }8 s% l1 V' T0 H
runtime_type = "io.containerd.runc.v2"' G, q2 c9 Z- d' O
" D; L/ w. f/ ?( |4 i, Q
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
* v) y R3 \8 ?; h7 S BinaryName = ""( z0 d3 u2 O3 P& ~! F
CriuImagePath = ""0 M3 L6 J6 |, v3 F
CriuPath = ""
. A0 P1 q; M, x. a CriuWorkPath = ""/ \" x, `: H* W1 r4 {# \% J
IoGid = 0
5 r5 m& L. y2 C3 D5 D. F IoUid = 0
; l, q$ n* v: N: Z NoNewKeyring = false0 i- r+ d* ~+ h8 J
NoPivotRoot = false
$ `' M' b# `) z* k2 I0 S8 B' x: z Root = "". a2 K9 j9 V8 g3 t7 I' _! I# r9 C) s/ g' c3 M
ShimCgroup = "". j+ ~: L" w1 ~+ S
SystemdCgroup = true
4 e" x: e. [ U' Y) K4 L: n# E9 i! D7 e- L
[plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]# ]) _. s9 }! d' x9 L+ O
base_runtime_spec = ""
/ }0 g) o6 `; ^2 d container_annotations = []
/ O9 K4 m# A" P7 W% r- f& G& ~1 l& n pod_annotations = []
8 X% Y/ R; h; s& w privileged_without_host_devices = false
A6 N: r. T$ M# m: ? runtime_engine = ""
3 D2 K0 V( Z3 h7 u/ v7 | runtime_root = "", O- ]& k5 ~, [
runtime_type = ""$ L) K# W, i; j" R) g: B5 j& C
# b. [1 H: ~0 P% i [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime.options]
5 C7 j6 k, Y) }' b
0 q3 I0 ^4 t0 e+ d5 F; d' ? [plugins."io.containerd.grpc.v1.cri".image_decryption]! `' O- a/ O/ ~) @: E. b! q
key_model = "node" t3 ~- B( Q, k4 g! B$ j
1 N: n+ |5 U% j( o [plugins."io.containerd.grpc.v1.cri".registry]
P7 l# s4 e) y- ^9 | config_path = ""$ H0 _ F/ C. V* N, y
8 S4 i& i2 j" y [plugins."io.containerd.grpc.v1.cri".registry.auths]
3 C0 U, k# Z$ h+ [2 ]3 X& |2 p$ I( [3 I) a+ \
[plugins."io.containerd.grpc.v1.cri".registry.configs]
1 V" K5 |, [. d; N: t+ T0 H, l; ^0 x1 B3 X& n' ?
[plugins."io.containerd.grpc.v1.cri".registry.headers]
^% ]& g1 Y0 u8 u7 }, A2 n1 E, A3 }% t( s' @
[plugins."io.containerd.grpc.v1.cri".registry.mirrors]" D+ d/ V4 v: u) y6 c
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]5 W2 q. f0 G7 R' `
endpoint = ["https://docker.mirrors.ustc.edu.cn", "http://hub-mirror.c.163.com"]: R( k" p+ R6 l
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."gcr.io"]
+ u# J6 h4 j& H8 y endpoint = ["https://gcr.mirrors.ustc.edu.cn"]' Q0 D& k; ?8 g* d/ V5 w
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."k8s.gcr.io"]2 {2 H" A$ [* S6 w
endpoint = ["https://gcr.mirrors.ustc.edu.cn/google-containers/"]! W7 j" p- u% w2 l5 q
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."quay.io"]' c, S9 Q( Q1 U9 q
endpoint = ["https://quay.mirrors.ustc.edu.cn"]9 d5 B2 c6 B( p' J
* k' t8 R2 A Q2 N, B# X! d/ | [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
+ ]3 g3 J5 C( o1 G7 A# s tls_cert_file = ""; k' n+ g9 ~( g6 h2 U) S
tls_key_file = ""
! v+ X/ @) ?, j6 w: G' }
2 i/ _; g7 G. j3 o [plugins."io.containerd.internal.v1.opt"]
6 d5 Z2 }7 t) c6 e2 F path = "/opt/containerd", a5 n8 u( W# p
1 M( L% b. O+ ] [plugins."io.containerd.internal.v1.restart"]* E g3 `/ v; M( {/ F( L3 U0 i( i
interval = "10s"
1 ]+ M o4 K9 ], V5 s. E
9 _7 P8 ~6 c! ~' g0 W2 j# W9 L [plugins."io.containerd.metadata.v1.bolt"]
, J$ r w! l2 @) |: w content_sharing_policy = "shared"
" T8 R! ^" q0 [4 t4 _+ e2 B; \& E, F. h
[plugins."io.containerd.monitor.v1.cgroups"]6 E& s0 P0 W2 n2 t
no_prometheus = false
' ?3 e9 O8 t! |' Y+ N' ?
# y+ i3 D! M, S0 a9 }1 o [plugins."io.containerd.runtime.v1.linux"]
" q+ k, `& r- P1 S, }( _ A5 | no_shim = false% q' D, `! {; ~) ?, \" ~
runtime = "runc"
( s' h g: ]% t" j runtime_root = "": u T5 J5 n* R2 f- K, x
shim = "containerd-shim"3 q5 a" _% W. O) r9 I$ ~; F
shim_debug = false
! d+ [& G/ I$ M
- a; ?( M3 I5 N( K& R, V* Q [plugins."io.containerd.runtime.v2.task"]9 k. S2 r/ O* s; }
platforms = ["linux/amd64"]
, C& y) Y0 s. q
$ A3 t- t; @ W ] [plugins."io.containerd.service.v1.diff-service"]" i1 c! k, m: w2 A# e# N
default = ["walking"]
5 P+ ]3 S: Z1 r+ y! U9 A1 y) S: n6 R0 @' l8 w
' a7 C3 s2 X6 n+ z8 Q [plugins."io.containerd.snapshotter.v1.aufs"]
" a! C& O) o1 ~ root_path = ""
( L( b! Q0 o# S3 l8 h8 A. T! ?
; z' B: [. N* X* [8 b; F [plugins."io.containerd.snapshotter.v1.btrfs"]# I5 d5 p! h* f$ a% q
root_path = "" {+ l4 s3 @. {& K$ K
+ e( Y. I; t0 y' Y- g/ B* t8 \
[plugins."io.containerd.snapshotter.v1.devmapper"]
1 m0 ^. Q5 _- F8 Z& b% {' l async_remove = false( n8 ]$ d& M1 c' @; ~! W
base_image_size = ""9 b/ U F4 t0 r$ u, b6 u& ^4 `' g
pool_name = ""9 A8 D0 E/ E4 {' ~3 V9 B$ `
root_path = ""; k$ W, b: ^8 N. C0 T! f
5 j/ e3 m1 a* a5 m4 u [plugins."io.containerd.snapshotter.v1.native"]) t4 a/ D' R9 X+ U. b. K$ ?! z
root_path = ""+ X4 }, p1 j1 ?" v
/ t+ M# Y# f8 Q% Y
[plugins."io.containerd.snapshotter.v1.overlayfs"]; h. Q! B( I& ]. C
root_path = ""
% ?; w; E7 }! Q+ K0 W3 h: E& n8 N; C8 l% w8 r* A( h, H6 `8 R9 U) b* ]$ q
[plugins."io.containerd.snapshotter.v1.zfs"]- P5 a8 m* g0 s) _; [% |7 V4 i; S
root_path = ""* R. a6 k+ r e. s% U& r# U/ v! K
0 Q( ^; p# f* I; c) ?[proxy_plugins]
6 M- ~' F+ f) [6 G- X+ _+ A; q6 x& ?( \; r0 d3 `
[stream_processors]' b' Z% e3 O F+ C( H. Q8 ?
9 X" A; M# ] i& s& P$ Y9 s6 V
[stream_processors."io.containerd.ocicrypt.decoder.v1.tar"]( s, r N1 h: p, S. g5 ^
accepts = ["application/vnd.oci.image.layer.v1.tar+encrypted"]
9 T' \& k& K" R+ g0 |$ M& a3 m args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"]! d2 v |1 ?; I; f
env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"]" J: C; [6 q/ h1 e
path = "ctd-decoder"1 S) R9 X3 y7 p7 u
returns = "application/vnd.oci.image.layer.v1.tar"
; m2 E8 A9 ^ O% {% y; @! l7 y( R4 N0 }$ V- c- }0 x) w, I7 `
[stream_processors."io.containerd.ocicrypt.decoder.v1.tar.gzip"]$ e2 y# K. _ ?1 L
accepts = ["application/vnd.oci.image.layer.v1.tar+gzip+encrypted"] @% r6 k8 q5 J8 O. |# H
args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"]: c( {& C$ @& q8 W, I. G! S) I1 Z" Q
env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"]
2 ?1 I/ Q: ]1 m) h( t S! x1 l path = "ctd-decoder"
1 O0 p2 |- V2 R b. ]6 s' i returns = "application/vnd.oci.image.layer.v1.tar+gzip": h6 c% N; q. t0 d f5 X
0 R/ u# ]- V% @: S[timeouts]
+ z; C) |, K6 w$ m. r "io.containerd.timeout.shim.cleanup" = "5s"2 n* Z6 ^3 \: d# I! O y
"io.containerd.timeout.shim.load" = "5s"
$ z$ [7 _) @0 W' [, H+ K7 ~ "io.containerd.timeout.shim.shutdown" = "3s"# t# x5 F, L3 Y+ j. L) r
"io.containerd.timeout.task.state" = "2s". }( K# r5 @- o6 h
- Y" E5 ^& M- v% `5 l% B# d) W4 t[ttrpc]
+ s, q' U+ o. Y0 _7 R" U+ z address = ""
8 L6 y D! e" O% j2 p0 Y, q gid = 0* N) q7 }! r# M0 s' x7 K
uid = 0
% B" l* e1 g, D# |配置 crictl 管理工具/ ]* Z2 r+ t0 z' p3 R# @
vim /approot1/k8s/tmp/service/crictl.yaml. z2 p3 v6 v- [! K& q
runtime-endpoint: unix:///run/containerd/containerd.sock/ y- r6 F; M1 q" J& x/ \
配置 cni 网络插件% D2 _5 B7 K% I
vim /approot1/k8s/tmp/service/cni-default.conf5 u6 H$ a2 Y% M1 q3 Y
subnet 参数要和 controller-manager 的 --cluster-cidr 参数一致
/ |9 L1 K% E& o& z2 N8 u
: f; K6 F! u/ G{2 }) I4 h4 g; F% L+ k
"name": "mynet",
+ O: D% A! H) {4 t0 S) R "cniVersion": "0.3.1",
$ S# S: j; r, { w$ t "type": "bridge",
) ?8 X9 X0 ]* t( o6 R2 u "bridge": "mynet0",
( I5 ]2 c, ?. g9 }$ P6 L& x "isDefaultGateway": true," F. x2 R* [6 S& N/ e5 l$ h
"ipMasq": true,
) l7 j# d5 g, q3 k. `% {4 ] "hairpinMode": true,2 k* v" V8 g" M {
"ipam": {
1 D8 q( ]/ k& p( R$ v* P6 d, \ "type": "host-local",
$ K1 M1 X+ ^3 T4 X1 h) A& d" k. z "subnet": "172.20.0.0/16"1 O3 d# h+ g, J: C
}- q. ?9 {0 o; E6 e( b$ P
}; ~9 G# M4 ~. g t: B6 _7 O6 C! j
分发配置文件以及创建相关路径
5 o* S" W, x4 M* ]$ `1 rfor i in 192.168.91.19 192.168.91.20;do \# P9 L7 s" R' e X0 `! C
ssh $i "mkdir -p /etc/containerd"; \ W0 \4 d0 @, \4 u, C4 v1 H
ssh $i "mkdir -p /approot1/k8s/bin"; \/ b" h2 J! w- I
ssh $i "mkdir -p /etc/cni/net.d"; \. }$ N' a/ U4 M1 k3 D' K
scp /approot1/k8s/tmp/service/containerd.service $i:/etc/systemd/system/; \+ ~' R6 K C7 u
scp /approot1/k8s/tmp/service/config.toml $i:/etc/containerd/; \
9 s) q- p" E- |% g5 |scp /approot1/k8s/tmp/service/cni-default.conf $i:/etc/cni/net.d/; \
( ]6 j8 b: z; _+ P( N" N( Wscp /approot1/k8s/tmp/service/crictl.yaml $i:/etc/; \
- ] e% r1 ~* s" E jscp /approot1/k8s/pkg/containerd/* $i:/approot1/k8s/bin/; \
4 {4 I: d% |# V' F* D+ I( qscp /approot1/k8s/pkg/runc $i:/approot1/k8s/bin/; \
" w9 Y; g/ n4 E1 h6 d, n* K) |8 h# h& Adone$ I& v- r# ]% g& I
启动 containerd 服务3 d* c. R* S0 G% G$ o# s
for i in 192.168.91.19 192.168.91.20;do \; o$ X- w$ w6 s- q8 z
ssh $i "systemctl daemon-reload"; \, ^- p$ F( G& v' p4 ?/ }
ssh $i "systemctl enable containerd"; \
5 H' B6 [5 k- B: g" ^/ E/ [ssh $i "systemctl restart containerd --no-block"; \: v* M3 m" u) I5 _6 c
ssh $i "systemctl is-active containerd"; \8 y, T* Z N: N( w2 j/ e
done
, N! J" N* C5 G2 q$ @6 F4 ~. k2 q返回 activating 表示 containerd 还在启动中，可以稍等一会，然后再执行 for i in 192.168.91.19 192.168.91.20;do ssh $i "systemctl is-active containerd";done+ z$ I q, j( j- S" x
7 q+ [9 A* C& K B
返回active表示 containerd 启动成功
- A( a+ p( }" [$ J" Y. _
. M5 }/ f1 X! e) r; z导入 pause 镜像
2 c( X: w2 \& Fctr 导入镜像有一个特殊的地方，如果导入的镜像想要 k8s 可以使用，需要加上 -n k8s.io 参数，而且必须是ctr -n k8s.io image import <xxx.tar> 这样的格式，如果是 ctr image import <xxx.tar> -n k8s.io 就会报错 ctr: flag provided but not defined: -n 这个操作确实有点骚气，不太适应8 T0 m# n* b( F
8 O. k; v- A8 _8 ]7 E
如果镜像导入的时候没有加上 -n k8s.io ，启动 pod 的时候 kubelet 会重新去拉取 pause 容器，如果配置的镜像仓库没有这个 tag 的镜像就会报错
_+ y# V, v: a- G2 y% Z, ~+ w: }% R5 ^& ?! L/ H* K
for i in 192.168.91.19 192.168.91.20;do \) z. a% Z5 d- c. ^ T
scp /approot1/k8s/images/pause-v3.6.tar $i:/tmp/
" _$ w( \ ]) P8 q% kssh $i "ctr -n=k8s.io image import /tmp/pause-v3.6.tar && rm -f /tmp/pause-v3.6.tar"; \# ^/ P4 x; G* Z$ q# T/ z) @* V
done
3 Z, _7 H8 {$ _6 o- L& c查看镜像
: g+ _7 J3 I. i N" G' Y6 ?! v; u
4 A4 h/ n8 Y( Y' `8 }1 u% N8 @6 v4 cfor i in 192.168.91.19 192.168.91.20;do \* L6 t3 f5 m/ z* W0 w; z( ~$ i
ssh $i "ctr -n=k8s.io image list | grep pause"; \, r" U4 O6 u) P# O: n1 n
done( ^; q9 {" |- w4 S6 ^
部署 kubelet 组件
# K2 \4 \* B. f. N: k* r) \创建 kubelet 证书) {4 T4 g. L$ S7 D" O
vim /approot1/k8s/tmp/ssl/kubelet-csr.json.192.168.91.19
7 N6 I& k8 B% y" c' o这里的192.168.91.19需要改成自己的ip，不要一股脑的复制黏贴，有多少个node节点就创建多少个json文件，json文件内的 ip 也要修改为 work 节点的 ip，别重复了, t) ]: ]% @" G2 u; A$ Q% Q
# \5 @; ~' J7 \4 e% ?{
( [9 m8 O& Z2 n" N "CN": "system:node:192.168.91.19",
6 D/ {1 u2 g$ F "key": {
3 Y, B# C2 L4 j5 F/ p4 [8 ` "algo": "rsa",
) b% D4 I$ V% U) j/ Z* j1 B4 i: d; g "size": 2048
( H: a+ A( c7 E/ o: Z6 m5 H },: F* H( }5 H6 O# H: d
"hosts": [
! j: e6 o7 Z' {5 Q' O "127.0.0.1",
* \3 r) C1 G9 W0 Q/ A, y "192.168.91.19"$ m4 P6 E: L: _1 v
],8 H9 _: ?' w* o" _9 Y1 u2 w" n) q* H# a
"names": [
0 N8 S0 ]/ h& m# i8 g8 G3 t# A( Z {
5 I9 i' u2 ?' S "C": "CN",. }' i8 m! @3 J4 Y' K1 p p6 s
"ST": "ShangHai",
+ `. W: O% I- v3 E k6 r3 u5 u "L": "ShangHai",
( ^2 e' V2 u5 G- Z# H2 T9 G "O": "system:nodes"," e% _& S' h6 D8 c
"OU": "System"
% G9 ?8 L4 F' b; S! ]4 @ }
% R, T8 {+ Q( a' {8 w6 E- ? ]
2 U, a9 s* }5 ^: _* y- G$ J}
& p; Q. w# _/ z5 }8 K" d+ Ifor i in 192.168.91.19 192.168.91.20;do \+ g: E2 L# Q+ L
cd /approot1/k8s/tmp/ssl/; \
% y5 [1 J1 _7 |5 o' |. Mcfssl gencert -ca=ca.pem \3 C# S7 G1 r" h/ ?
-ca-key=ca-key.pem \
- J3 x( w% G9 a$ i0 S-config=ca-config.json \
+ V: a" H1 s: t1 ?# r. E* f9 R* u-profile=kubernetes kubelet-csr.json.$i | cfssljson -bare kubelet.$i; \ Y* H/ ^8 z/ a( Q) Q& T
done8 U( N" h1 o! \
创建 kubeconfig 证书
% T6 ^3 X* G$ r! J( C0 u设置集群参数
* O7 e: S( W: ~9 D6 Y. q
" e" }1 O) o" ^. o: x) [' a--server 为 apiserver 的访问地址，修改成自己的 ip 地址和 service 文件里面指定的 --secure-port 参数的端口，切记，一定要带上https:// 协议，否则生成的证书，kubectl 命令访问不到 apiserver
1 e- O. J+ `4 H3 [0 w8 Z2 v$ g% t6 k, B( `& V K5 H5 s
for i in 192.168.91.19 192.168.91.20;do \
. Y& {* W8 B# m8 v0 \: ?+ Ncd /approot1/k8s/tmp/ssl/; \
+ A* }% H. Z Q7 j/approot1/k8s/pkg/kubernetes/bin/kubectl config set-cluster kubernetes \9 Z5 p# y z2 z
--certificate-authority=ca.pem \
5 b2 g, |" T" x0 t) u--embed-certs=true \
( F0 B9 Y5 |( o' B' W" P--server=https://192.168.91.19:6443 \9 f) t- Y- H {
--kubeconfig=kubelet.kubeconfig.$i; \( d! Q1 S5 P: {* G. h7 U0 |
done
- l `1 M# x% f' ?7 @1 M9 _4 L' ]设置客户端认证参数, P1 B7 I; E9 E
4 P: X8 u7 X7 F8 ]* [8 v2 r8 [for i in 192.168.91.19 192.168.91.20;do \! s: d4 f- N# T; q/ [/ F
cd /approot1/k8s/tmp/ssl/; \8 X9 j/ l- M6 y4 i5 N; ~
/approot1/k8s/pkg/kubernetes/bin/kubectl config set-credentials system:node:$i \
% i7 B4 T: [( ^9 |2 h6 |--client-certificate=kubelet.$i.pem \! b5 S3 [% S( `: Z( i
--client-key=kubelet.$i-key.pem \
/ Z0 ?, }2 m Y--embed-certs=true \
% E% F0 _' t9 ~0 m0 N--kubeconfig=kubelet.kubeconfig.$i; \
. y- N! A) r( r5 {- tdone
2 e5 G# e# y" I0 W7 v9 S1 g9 o设置上下文参数
, d- I; E: @7 \$ A
0 S% V: ?1 G' O! E- ^for i in 192.168.91.19 192.168.91.20;do \0 b; {9 i2 c* I7 a
cd /approot1/k8s/tmp/ssl/; \
9 C1 j0 l( X4 W$ e0 R/approot1/k8s/pkg/kubernetes/bin/kubectl config set-context default \* ^) D- {$ e6 @' P x; P
--cluster=kubernetes \! d, t6 f% U0 m8 m0 s2 t% U
--user=system:node:$i \4 H# {) d7 r* u) x$ g( ^
--kubeconfig=kubelet.kubeconfig.$i; \1 ~7 i! i0 H- `. o) }
done
7 ?9 M3 E5 m8 Q2 c% \设置默认上下文2 n, d2 {' x& h7 A ]$ o
: [6 B2 S3 [* t# }6 `! T
for i in 192.168.91.19 192.168.91.20;do \9 `- `+ R: C2 \/ N
cd /approot1/k8s/tmp/ssl/; \5 ^4 V0 j H' T/ Z/ T' E& F
/approot1/k8s/pkg/kubernetes/bin/kubectl config \
9 u3 T( {+ ]7 V9 ~ l& N0 D2 `9 l, Puse-context default \* s; v8 J/ i( o6 w9 S4 f, `1 P* G
--kubeconfig=kubelet.kubeconfig.$i; \! {$ I. [; m6 L4 m( A
done8 l% j6 Z2 Y* k n* j
配置 kubelet 配置文件
" e2 {1 k* M6 n3 H Pvim /approot1/k8s/tmp/service/config.yaml
. J! F' [6 I" |4 p3 ZclusterDNS 参数的 ip 注意修改，和 apiserver 的 --service-cluster-ip-range 参数一个网段，和 k8s 服务 ip 要不一样，一般 k8s 服务的 ip 取网段第一个ip， clusterdns 选网段的第二个ip
m9 O4 t' F2 {9 J' ?8 G+ \( C3 N
& B* j: l6 U0 I0 r0 P* a5 }0 r0 zkind: KubeletConfiguration0 ?; R1 w/ w5 |8 C
apiVersion: kubelet.config.k8s.io/v1beta1
: g6 P3 I0 G9 a1 J1 Eaddress: 0.0.0.0
$ ?$ j, m! e* c2 O+ Vauthentication:
; b$ v% ~$ b% X0 s1 C/ b anonymous:
# S6 t" W; z# A- N1 L4 t enabled: false% L5 ?, n$ P3 t% p# r* i5 W
webhook:
5 N" ?$ F$ M1 l ^# B, l- R3 a+ v5 [ I cacheTTL: 2m0s
$ B' d C" L. T3 b enabled: true
! ? V: G. {- G+ w. w1 y$ Y x509:
: R* t1 W4 t) m' k, R" ^( ` clientCAFile: /etc/kubernetes/ssl/ca.pem
6 }5 _( J ^4 V8 o7 `' l5 a" A; ~authorization:
# b' X" Z0 O6 c% T mode: Webhook
6 P3 M" M0 O% t* ^ webhook:. t( v9 q' w( N$ M3 z |4 O9 w
cacheAuthorizedTTL: 5m0s
" [- m% q( M( W& h- F) U6 d cacheUnauthorizedTTL: 30s6 q* F6 U9 `) `4 N1 k
cgroupDriver: systemd5 g: `& k# L9 y4 R% w" x4 o- s
cgroupsPerQOS: true
+ ]" J$ \" ~: FclusterDNS:5 @2 F$ i, \: O* Y f7 a
- 10.88.0.2
2 a6 z) y2 }$ y4 TclusterDomain: cluster.local
; z: N- ] ?8 [5 U& jconfigMapAndSecretChangeDetectionStrategy: Watch3 M D P& k/ m7 {5 O+ o9 D
containerLogMaxFiles: 3* {$ d0 i ?; O% Y
containerLogMaxSize: 10Mi
9 p' g- i" ]) J' F8 X, P# J5 L) w! T$ XenforceNodeAllocatable:( f/ K% F8 F; E+ a9 T
- pods
$ K3 Y% E1 Q7 n) X8 ueventBurst: 10 |% P0 x f0 f8 p9 O. {: _
eventRecordQPS: 55 S1 l: r/ o6 x4 F4 N. a8 c
evictionHard:* Z- |# O1 k6 E. c' l+ |9 R0 C
imagefs.available: 15%9 {, R9 j2 f& P w
memory.available: 300Mi
6 u* M4 j8 L2 n) | nodefs.available: 10%
/ P/ s+ x- ^+ C. a; }- c nodefs.inodesFree: 5%
; x l# Y3 @5 w5 c/ U. n7 V PevictionPressureTransitionPeriod: 5m0s
/ M- s2 [5 Q' M' g3 v" j2 ffailSwapOn: true
$ T+ t" v" k4 \- L( G: qfileCheckFrequency: 40s
, V! J6 i9 z: ?/ f9 C: PhairpinMode: hairpin-veth
& U+ x+ e; M' R# H3 YhealthzBindAddress: 0.0.0.0% C4 e% Y8 u; A; S7 Z9 H% A7 k/ K
healthzPort: 10248
2 B7 s7 Y1 \; y3 ahttpCheckFrequency: 40s
) `2 w( f, M/ d4 C5 GimageGCHighThresholdPercent: 85# y# m6 N9 c/ R& M1 G
imageGCLowThresholdPercent: 801 N' e5 e& S* d5 y
imageMinimumGCAge: 2m0s
" S+ ^1 C3 i1 S" N, C: dkubeAPIBurst: 100
* @& Y, F8 l" ~) PkubeAPIQPS: 50
6 q( C! p; f' D. b7 O0 \makeIPTablesUtilChains: true
! F/ A; R: Y+ O$ f" KmaxOpenFiles: 1000000- ?; p7 D+ @$ D
maxPods: 110$ m5 [$ k& Z8 h
nodeLeaseDurationSeconds: 40. _) | P' x# j5 m, s) M0 _' r
nodeStatusReportFrequency: 1m0s
- Q' {, m" L% B. {# T4 p4 `nodeStatusUpdateFrequency: 10s' s3 n% @. Y# s
oomScoreAdj: -9998 G7 T3 a1 \$ H7 f
podPidsLimit: -1
9 ^/ `$ ~: E3 Mport: 10250* p# a2 ]" w1 D, `6 R& d0 m
# disable readOnlyPort
3 O! U% }/ {# c) D- x0 vreadOnlyPort: 0
7 a' W2 ~1 w/ w, oresolvConf: /etc/resolv.conf& J) z5 T3 j% a
runtimeRequestTimeout: 2m0s
1 m4 R6 m9 M' ` y5 u1 h( A# G% IserializeImagePulls: true
& x) p/ ~+ j- LstreamingConnectionIdleTimeout: 4h0m0s
8 w* N& ?: `, y% G, t1 HsyncFrequency: 1m0s
1 U3 u2 v( R5 | [( |tlsCertFile: /etc/kubernetes/ssl/kubelet.pem1 V% e; I2 ?1 R5 {
tlsPrivateKeyFile: /etc/kubernetes/ssl/kubelet-key.pem
- \% A2 O/ l! w配置 kubelet 为 systemctl 管理
9 D1 ]5 N( R" |, |% O, |vim /approot1/k8s/tmp/service/kubelet.service.192.168.91.19
) _. S2 j% Z, Q7 n: X1 Q这里的192.168.91.19需要改成自己的ip，不要一股脑的复制黏贴，有多少个node节点就创建多少个service文件，service 文件内的 ip 也要修改为 work 节点的 ip，别重复了& Y3 v" G. j0 P8 n; X
; U/ A/ `* v( |8 ~--container-runtime 参数默认是 docker ，如果使用 docker 以外的，需要配置为 remote ，并且要配置 --container-runtime-endpoint 参数来指定 sock 文件的路径; P+ n9 j, K. J' v/ _( h- f
0 D! ?9 B( ?' T2 h
kubelet 参数
% O. ~6 _8 F$ f2 S7 T" R0 w8 h" M9 M8 o! S! e
[Unit]& T5 i- M! S0 j j. `+ b3 D) I
Description=Kubernetes Kubelet( A) \* j/ } N, k7 M" x+ m
Documentation=https://github.com/GoogleCloudPlatform/kubernetes4 N2 B7 |6 |5 k8 C
% @* E( g# H: S- Z
[Service]
* d7 R: v6 A& h1 ~1 s- G. QWorkingDirectory=/approot1/k8s/data/kubelet' q1 Y' J, @& V; o1 b3 P" r
ExecStart=/approot1/k8s/bin/kubelet \
6 P; Q( t# {2 ~2 J --config=/approot1/k8s/data/kubelet/config.yaml \
1 w( N9 r' |# K --cni-bin-dir=/approot1/k8s/bin \
9 r J' P$ i' D0 j' V1 I/ v --cni-conf-dir=/etc/cni/net.d \; ~5 m+ l6 f% w" C7 u
--container-runtime=remote \! \! W& v9 S) F# B' G0 X1 h
--container-runtime-endpoint=unix:///run/containerd/containerd.sock \
! L; W8 Q$ N( F( O# P --hostname-override=192.168.91.19 \; s9 N9 a& O: F1 ^! t
--image-pull-progress-deadline=5m \# E" H8 |: k+ O5 i+ s
--kubeconfig=/etc/kubernetes/kubelet.kubeconfig \! u8 E2 {" `: ~9 _& |: Z
--network-plugin=cni \
' [7 }; G7 D. C; W3 ~ --pod-infra-container-image=k8s.gcr.io/pause:3.6 \
% u8 a8 j* K1 n --root-dir=/approot1/k8s/data/kubelet \7 k: v$ _2 S# |4 t5 h# v' R( c( D
--v=2
n, c2 x7 T2 w6 HRestart=always
' M) x9 e$ r; q. `" B) k' sRestartSec=5
" L4 ] i, u+ t: p% s$ D' G
s- r- W4 c( y+ X# L- h[Install]% O! L) K' }- B! d/ K
WantedBy=multi-user.target
; |5 U& v4 {4 |9 P0 x; J$ F4 I分发证书以及创建相关路径
7 A& D$ i0 u8 n( T/ G# m如果是多节点，只需要在192.168.91.19后面加上对应的ip即可，以空格为分隔，注意将192.168.91.19修改为自己的ip，切莫一股脑复制' u4 S* E8 z9 j5 d8 a9 J
8 q3 O- Z: t( ~7 H6 A, u
对应的目录也要确保和自己规划的一致，如果和我的有不同，注意修改，否则服务会启动失败
" S# [; ~) [9 k* e" P! J
6 p7 V8 O) J+ ?& Ofor i in 192.168.91.19 192.168.91.20;do \! ]. [9 K: {/ D1 z3 z4 R, v
ssh $i "mkdir -p /approot1/k8s/data/kubelet"; \
$ d) g9 m6 W- `( R3 sssh $i "mkdir -p /approot1/k8s/bin"; \; Z( S) t w' P. N$ r
ssh $i "mkdir -p /etc/kubernetes/ssl"; \
3 o7 ~5 c$ o% f. q; R( Dscp /approot1/k8s/tmp/ssl/ca*.pem $i:/etc/kubernetes/ssl/; \( L$ O) h5 z/ y' Z* U: o5 {1 z
scp /approot1/k8s/tmp/ssl/kubelet.$i.pem $i:/etc/kubernetes/ssl/kubelet.pem; \
0 A5 S5 n7 E" b4 n0 h1 }3 P: ~- pscp /approot1/k8s/tmp/ssl/kubelet.$i-key.pem $i:/etc/kubernetes/ssl/kubelet-key.pem; \* u% B$ ]7 b6 o$ U
scp /approot1/k8s/tmp/ssl/kubelet.kubeconfig.$i $i:/etc/kubernetes/kubelet.kubeconfig; \$ Y4 ^7 s3 l0 ?/ J4 N0 C
scp /approot1/k8s/tmp/service/kubelet.service.$i $i:/etc/systemd/system/kubelet.service; \
9 k! U9 A+ V% p" M, v! W% Lscp /approot1/k8s/tmp/service/config.yaml $i:/approot1/k8s/data/kubelet/; \% B( E( x3 k' e4 a5 w
scp /approot1/k8s/pkg/kubernetes/bin/kubelet $i:/approot1/k8s/bin/; \( g0 G: V$ y) K+ Z: `) Z
done
8 X7 U2 X% D, M* k; ~% f启动 kubelet 服务
$ I' F# ^5 k2 E1 y$ B+ Qfor i in 192.168.91.19 192.168.91.20;do \ Y) ^* p) Z9 J/ R: b% B
ssh $i "systemctl daemon-reload"; \; A; ^! {1 }( E
ssh $i "systemctl enable kubelet"; \9 k2 Y1 ` p/ c/ s
ssh $i "systemctl restart kubelet --no-block"; \0 R, E5 f% k/ U( [5 l7 L
ssh $i "systemctl is-active kubelet"; \
) |' ~* P0 P* F# H# Z7 {: H5 Udone
$ K3 b! _" e0 h5 n8 g& l5 s9 N返回 activating 表示 kubelet 还在启动中，可以稍等一会，然后再执行 for i in 192.168.91.19 192.168.91.20;do ssh $i "systemctl is-active kubelet";done
; u) @. c9 E) ^- D" [
J' |, @# g% M8 q返回active表示 kubelet 启动成功) S8 s8 q& v( F/ q
. d5 R; a' t' ?/ a: ^
查看节点是否 Ready
' Q+ Z& @# Q2 o8 _kubectl get node
* h7 j2 Z. M4 `8 e预期出现类似如下输出，STATUS 字段为 Ready 表示节点正常! K! f7 A0 t8 G& K
# F0 }- k. V. e1 e
NAME STATUS ROLES AGE VERSION8 I, D9 t" Y6 c$ q) p X) p
192.168.91.19 Ready <none> 20m v1.23.3
3 @0 \5 J9 Y C% D192.168.91.20 Ready <none> 20m v1.23.31 j1 ]/ w; M! t" k
部署 proxy 组件
* D* X8 F, q. _$ W/ o' _. H创建 proxy 证书1 k2 r9 A! W( G/ r8 @
vim /approot1/k8s/tmp/ssl/kube-proxy-csr.json
$ h7 k" X4 U3 [{
; h# V! G7 N/ m3 f; Q "CN": "system:kube-proxy",5 O& ~+ x4 Z7 [0 a! v
"key": {- o \4 b( M) }( G; P
"algo": "rsa",
* @0 n% \! t! Y! y' ?* t! ^% c9 o& l. M "size": 2048
" Z# R( K0 P! ^7 n' b) Y },
3 ?+ S! N1 X1 R; ^ k! d, z+ r "hosts": [],
' Z7 \, h7 K0 R! ?+ v1 x* G2 u' e: U "names": [2 Y V5 h' _3 A: f& x+ k- U3 k( Q
{
1 ]) N! i( {0 `/ i "C": "CN",# W1 B" C+ P x7 E+ m& Y3 t
"ST": "ShangHai",. I/ _* f- ?* F2 y- ~
"L": "ShangHai",: z, w! k. P+ Z
"O": "system:kube-proxy",
1 o. ]# t* x0 p' b# k' U "OU": "System"
( g9 ]: ~) z8 b4 v( C" z }' P4 J( L5 z8 z+ j
]
0 F1 U. ]/ J4 f s+ \( V}
9 n, L6 m& H4 i0 y: |cd /approot1/k8s/tmp/ssl/; \" r' Q' D# T0 R- y. `
cfssl gencert -ca=ca.pem \; ]) ?- U# u$ x
-ca-key=ca-key.pem \
: i+ I' @2 S9 u; B8 d9 w-config=ca-config.json \# W* `' T4 O$ q8 w
-profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy" L( z% t9 f) C
创建 kubeconfig 证书" ]* i. U0 K ^2 n3 ?4 S
设置集群参数: x; N9 _$ v6 P" y
+ n- h3 p) a/ ?0 y9 x% _
--server 为 apiserver 的访问地址，修改成自己的 ip 地址和 service 文件里面指定的 --secure-port 参数的端口，切记，一定要带上https:// 协议，否则生成的证书，kubectl 命令访问不到 apiserver1 b& C# E: ^* ~* o
+ l, ^( j4 X3 {/ W( Pcd /approot1/k8s/tmp/ssl/
+ `7 H" r- L, l9 f2 P# ^4 T/approot1/k8s/pkg/kubernetes/bin/kubectl config set-cluster kubernetes \
% l8 U) h. `$ c8 I$ j0 ^--certificate-authority=ca.pem \" b/ \% L! R: c! p
--embed-certs=true \* m/ Z% g& D$ B+ _
--server=https://192.168.91.19:6443 \3 T2 S+ Z! e% }9 s6 u% D
--kubeconfig=kube-proxy.kubeconfig/ Q% J3 f) `4 A0 ]( F' n$ Y G
设置客户端认证参数. k9 \ L# S J- S3 V% w! p
5 v' H. E+ `; F; A: @2 ]8 x
cd /approot1/k8s/tmp/ssl/
/ d, l# y, D! @7 X6 H p) @. m# i$ w/approot1/k8s/pkg/kubernetes/bin/kubectl config set-credentials kube-proxy \1 X/ f4 U6 {! K7 k. ]! F% f/ T7 X" N
--client-certificate=kube-proxy.pem \4 O5 o# [$ Y. D8 k8 c, C: H- B6 v/ J
--client-key=kube-proxy-key.pem \, Y: F$ g& f$ G6 ]1 k
--embed-certs=true \: n# f# Z7 l8 e, L, _
--kubeconfig=kube-proxy.kubeconfig
* p. [. y I! W7 ?& ~# z! s设置上下文参数/ v" {3 {" k2 i* @
" f( _- ^. w, q" M
cd /approot1/k8s/tmp/ssl/
* F; K- B7 [. d5 Q/approot1/k8s/pkg/kubernetes/bin/kubectl config set-context default \1 J- l$ G2 H2 q1 O0 ~
--cluster=kubernetes \
" X/ V* |' x, X6 x+ b- @. l' c* D& ^--user=kube-proxy \; l u( k3 N: W" q, i( P
--kubeconfig=kube-proxy.kubeconfig. @! N( K& c) E8 Q3 `% C' {2 q2 i
设置默认上下文! K. P7 s; B: r7 n7 ^' t$ k" w
5 H) }; j% R% `$ H6 U) Z
cd /approot1/k8s/tmp/ssl/% r% }0 `; V; d" x b7 c
/approot1/k8s/pkg/kubernetes/bin/kubectl config \3 R9 D L# m+ C( h. w% s$ L
use-context default \
w& F6 z) B6 ~: w--kubeconfig=kube-proxy.kubeconfig
$ t0 k, Y* t3 m/ d& ^% Y& r配置 kube-proxy 配置文件
) |$ N0 D9 R. [; N* L, P& j) Z1 Avim /approot1/k8s/tmp/service/kube-proxy-config.yaml.192.168.91.19
9 Q) w8 h- g- u8 v5 h这里的192.168.91.19需要改成自己的ip，不要一股脑的复制黏贴，有多少个node节点就创建多少个service文件，service 文件内的 ip 也要修改为 work 节点的 ip，别重复了
5 @- {* m1 G3 T$ K* J4 U2 {% ]* `( W3 b
clusterCIDR 参数要和 controller-manager 的 --cluster-cidr 参数一致" L7 ]* F6 v r% c$ _/ U7 e* T
/ B4 ?9 A6 S" V* C1 c, _, xhostnameOverride 要和 kubelet 的 --hostname-override 参数一致，否则会出现 node not found 的报错
5 U5 x3 Z7 w! }/ J3 {
* Y6 S! d( C5 ^" p$ l3 M+ t: ~kind: KubeProxyConfiguration& V2 i8 G( @0 g, W$ M1 P3 Z
apiVersion: kubeproxy.config.k8s.io/v1alpha1
. v+ @3 o, w4 Q. QbindAddress: 0.0.0.02 }5 ?9 _0 `' O2 f
clientConnection:
9 c1 z/ x# G# v7 x4 w: H kubeconfig: "/etc/kubernetes/kube-proxy.kubeconfig"9 ~; K G c- p7 n' r% m M' k
clusterCIDR: "172.20.0.0/16"
2 P1 P- E! ^* h- Pconntrack:' ~6 [ R+ P/ k' p. V& V2 i- U
maxPerCore: 32768- X* l5 _2 L! a! }& x
min: 131072# J3 c* f* b# Y/ q
tcpCloseWaitTimeout: 1h0m0s' O% r6 T* a$ N3 G' `
tcpEstablishedTimeout: 24h0m0s
6 M+ W+ m8 p$ @, QhealthzBindAddress: 0.0.0.0:102563 F" \4 `1 k L! A2 V
hostnameOverride: "192.168.91.19"
1 |' E' R ?5 [1 M; e# o8 PmetricsBindAddress: 0.0.0.0:102499 Z) ?/ d7 T" X% d! E2 L
mode: "ipvs"
4 i8 o9 q' H% _6 L3 v, U配置 proxy 为 systemctl 管理
# i! w) I* P, y3 R. X. p7 `- p7 v0 M# xvim /approot1/k8s/tmp/service/kube-proxy.service7 V6 O8 H; n T! h7 ?% V
[Unit]
1 s8 p9 a. x( c3 pDescription=Kubernetes Kube-Proxy Server
6 T+ l4 P# I! O4 m' aDocumentation=https://github.com/GoogleCloudPlatform/kubernetes
0 p5 l. C0 F3 w2 kAfter=network.target
3 H' f6 C; }5 a- Q
/ U- t0 o4 Z- m9 E: t, T5 q* k[Service]2 Q/ N7 u, [" {6 l; y, [( A k( Q
# kube-proxy 根据 --cluster-cidr 判断集群内部和外部流量
7 C3 h$ `4 ?9 f## 指定 --cluster-cidr 或 --masquerade-all 选项后
9 @7 R/ L: {9 M/ b1 O' w## kube-proxy 会对访问 Service IP 的请求做 SNAT: r' K( S4 _' m
WorkingDirectory=/approot1/k8s/data/kube-proxy
- B/ {. W3 ^; Y. h4 Y" R4 o" h g! G2 gExecStart=/approot1/k8s/bin/kube-proxy \
3 K+ O# n8 }* g7 E --config=/approot1/k8s/data/kube-proxy/kube-proxy-config.yaml
, a( ?! w1 j8 N: {Restart=always
/ f I, p3 a* p- t s9 k# W/ uRestartSec=5
|# |9 z2 N& u' Q; KLimitNOFILE=65536/ ]% Z& z! j: U0 _' B* O
; w% k' Q+ H- {4 j+ w
[Install]" S# I+ q1 N1 ~* Y! x C' e% \
WantedBy=multi-user.target
9 ?, U: q- c! D! H' ~分发证书以及创建相关路径
* F3 J" a, f$ t3 P# X0 }2 w如果是多节点，只需要在192.168.91.19后面加上对应的ip即可，以空格为分隔，注意将192.168.91.19修改为自己的ip，切莫一股脑复制* j9 L( ]% W4 W
7 k; ?8 P1 ~7 `2 M {6 P
对应的目录也要确保和自己规划的一致，如果和我的有不同，注意修改，否则服务会启动失败
0 ~' c: H' Q) v- C
* u8 V; m( }- \9 m1 C, bfor i in 192.168.91.19 192.168.91.20;do \
! Q' l/ g4 |& zssh $i "mkdir -p /approot1/k8s/data//kube-proxy"; \5 T+ J+ t S% W. o2 N
ssh $i "mkdir -p /approot1/k8s/bin"; \
/ U, r9 T) f; Tssh $i "mkdir -p /etc/kubernetes/ssl"; \6 b0 j' k& k* J& F+ A- R
scp /approot1/k8s/tmp/ssl/kube-proxy.kubeconfig $i:/etc/kubernetes/; \
5 a8 l0 [/ u# N1 }% ?! e. {' Qscp /approot1/k8s/tmp/service/kube-proxy.service $i:/etc/systemd/system/; \
0 P, A! t7 p4 s8 Z) o- s' d: X, `' Tscp /approot1/k8s/tmp/service/kube-proxy-config.yaml.$i $i:/approot1/k8s/data/kube-proxy/kube-proxy-config.yaml; \
" m6 V! ~! I. q, F8 J5 fscp /approot1/k8s/pkg/kubernetes/bin/kube-proxy $i:/approot1/k8s/bin/; \
- l! ^) L/ R; D& ^. n5 U" F' w* ^% Xdone% N9 j* l& f! A2 x
启动 kube-proxy 服务/ Z8 u: R' T, ?
for i in 192.168.91.19 192.168.91.20;do \% P' U. f5 Q- R- z6 n
ssh $i "systemctl daemon-reload"; \9 A, q/ W8 @9 h* b5 y
ssh $i "systemctl enable kube-proxy"; \
( c6 }2 H% g/ essh $i "systemctl restart kube-proxy --no-block"; \) s( E6 H% k1 A' t# n% u; ]
ssh $i "systemctl is-active kube-proxy"; \+ o9 G$ Q6 C3 r+ H& [6 x+ d
done* O$ O, ~, `2 A+ |) b+ z
返回 activating 表示 kubelet 还在启动中，可以稍等一会，然后再执行 for i in 192.168.91.19 192.168.91.20;do ssh $i "systemctl is-active kubelet";done
- ~' A: D; t( D, l9 x q
) J. v# \7 Q- P6 @1 e1 ~' b/ m返回active表示 kubelet 启动成功
5 y* j7 i- t) J) v8 V. I# E! l* J0 @5 V7 _, ^. h1 A
部署 flannel 组件9 J" m# i c# q4 A( X
flannel github" N: G& {8 K9 d* O0 z% s" N
! {+ W' ^* }- Z. D! Z6 Q& f/ S配置 flannel yaml 文件- [6 _- i5 o: P# b
vim /approot1/k8s/tmp/service/flannel.yaml
- y) S! ~( q. Lnet-conf.json 内的 Network 参数需要和 controller-manager 的 --cluster-cidr 参数一致5 q! Z8 o& B+ m/ ~) t4 g0 V
f) G o# X) } m: a---
" q- L8 j2 ^7 ?' E& K# hapiVersion: policy/v1beta1
+ u: A2 |3 {! V: z, f. i# B6 Bkind: PodSecurityPolicy/ F3 E9 N, l# p4 I
metadata:! [' _5 U( V7 c
name: psp.flannel.unprivileged* K0 J9 a! C. J: T( m5 v6 V
annotations:
8 a+ Q {( ~' K- C+ }1 u4 z! U T2 S seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default% z( y2 W( D& e, L& N
seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default& n) m9 p7 [4 v2 o8 T# n# {/ D
apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default* i; i @& {# ?
apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default8 D+ z( m% C7 S$ X* h+ Y
spec:- a6 Q& `1 u8 O( h" F+ O
privileged: false' P% Y5 R& z" b4 M- @
volumes: k4 l, m! i0 b
- configMap
# Y+ Z1 o2 i4 c- J' v' w$ K - secret
, G: Z a. W1 T - emptyDir
2 N% e* |- s- {0 J - hostPath
( D# P. T$ _9 _, I4 u+ \( O allowedHostPaths:
/ @9 g2 f s9 N2 k4 B. C - pathPrefix: "/etc/cni/net.d"
6 U% Y& i. L" A/ s - pathPrefix: "/etc/kube-flannel"
" m4 T$ j: u* }8 D7 [- k g- H - pathPrefix: "/run/flannel"
. A1 D$ h+ P6 O readOnlyRootFilesystem: false0 u/ b& R+ M- }$ ~1 X1 l7 Q$ S0 ]. z
# Users and groups! X& t# W& }7 D i: q4 Y* I
runAsUser:
: I* s6 {, L0 W, f# a rule: RunAsAny8 U; W, m6 p6 K9 s& `/ }
supplementalGroups:. }+ ^5 p; n" J
rule: RunAsAny
+ l$ Q! A7 Q( `- i1 \ fsGroup:& ~ G9 @$ G: C5 ^" b+ G" `
rule: RunAsAny
3 U; Q/ @* Z- W, q) o # Privilege Escalation
) V. i: i" [$ O2 ]+ O. V allowPrivilegeEscalation: false
# C @2 |; V3 M# U defaultAllowPrivilegeEscalation: false' z; W. `! [2 l$ @% i% J) H
# Capabilities: H! d# |. t: q4 i7 r8 G) E" l
allowedCapabilities: ['NET_ADMIN', 'NET_RAW']( n: g* ?7 t6 o8 s, i+ _/ d3 z
defaultAddCapabilities: []# s+ [" I: n: H
requiredDropCapabilities: []
" V6 p: W( z) B0 [0 d9 B: S- ~ # Host namespaces4 g N+ D! Z1 _$ W1 h# C- O
hostPID: false
, d0 ]0 B8 {. t hostIPC: false
% D& w* z l/ h1 ?7 C* t7 ^, ?. n# h hostNetwork: true
5 M# d* L' Y) F7 `* L hostPorts:6 J% J' a9 V& N$ D
- min: 0# f' n4 g f! t. N
max: 655350 t) u j6 w4 x* Q
# SELinux
, D! k7 I& ^7 ~/ A( i n. Z( k seLinux:) Q8 t1 k( q( O: E" X
# SELinux is unused in CaaSP* `: S3 p* Z8 R& j7 O2 s
rule: 'RunAsAny'
* [& F0 L8 M+ W---) W! i( Q% a* b9 |3 L+ I% ~' h
kind: ClusterRole$ ]7 W) x9 p& ?2 @; {! |5 ]; {
apiVersion: rbac.authorization.k8s.io/v1
9 d" h$ Y. n7 _, o+ Q) s, }+ ` Y1 Emetadata:
9 @. s" M2 v; x1 V name: flannel
3 B a" v! M" V0 V F6 @# N' brules:
& o: J6 E3 ]7 d- apiGroups: ['policy']2 W1 p4 \0 F' q7 N1 b9 @% [
resources: ['podsecuritypolicies']9 P+ N% j4 g5 M$ p+ o: J
verbs: ['use']
2 g7 S: e8 [7 d) l; `0 M2 b% G resourceNames: ['psp.flannel.unprivileged']
1 I' h4 |$ y; M' N1 w' ?2 B$ V- apiGroups:6 w2 T( \% D% |& {
- ""9 X* S1 e8 I, w5 C% Q2 N8 `, ]
resources:$ a9 V X. ^ d- E2 P( Z; t
- pods% v8 C: ?& c6 r1 ?8 E4 C( R; ?
verbs:! f/ A1 e% p$ u f
- get9 _) Z0 D! s3 i4 w4 p' U
- apiGroups:9 S, O$ {6 E6 m! H. F0 Z1 _
- "". f' T6 }- s3 c T+ p. L! f
resources:; O' a1 B" z2 j2 a" f
- nodes$ o( i: I0 `4 O) R( X3 E- K
verbs:
- L2 b/ C: a& q- p5 U9 Y! R - list
k% w2 y7 f8 v! h+ k - watch5 R; ^8 p; m1 @7 U, a$ o; m
- apiGroups:
7 T- ~& i3 Z% E) L - ""
8 y- g. X* [7 l7 U `1 k resources:& b b$ o. r9 Y7 R. P0 K% \4 L4 @
- nodes/status; ?; m9 l' A$ W
verbs:* [0 D9 l2 d3 R7 N3 S# u
- patch
3 a( L( b/ {8 \; V4 o; b---
+ k1 Z, ]# F" f8 Xkind: ClusterRoleBinding" P/ b" |! P7 L3 ?/ o
apiVersion: rbac.authorization.k8s.io/v1# e0 Q$ }! \ y' O
metadata:) w! f. r$ z- C8 @2 o
name: flannel/ j" E: e2 B) ]) k" ^
roleRef:6 j7 \% f! i" O! u3 @9 C
apiGroup: rbac.authorization.k8s.io4 R/ A/ y8 ^4 |* U6 F$ D8 u$ n2 [
kind: ClusterRole
1 x* ]" x9 D* M9 J1 \ name: flannel
% @8 F8 C" e/ W+ L7 i, W$ ]subjects:
0 y% i" z: P9 G, Y1 z- kind: ServiceAccount
& b# X6 I3 A5 H$ u1 |6 @ name: flannel
+ e/ ~& |/ U2 _0 W6 y/ s* [ namespace: kube-system
( |6 q- l1 w- R+ g---' w: D; v/ l: ] D2 k
apiVersion: v1
# b N( r* U0 l) Z0 I: I4 V8 L( Mkind: ServiceAccount
" T& o/ i$ S8 ^( Kmetadata:- S" D9 X& K2 W# t; {' `; x
name: flannel
7 |; Z7 ~5 X" K& e( t( n7 j- U, L9 r namespace: kube-system
, N( B9 H4 L' b---4 D/ ]& N7 ~1 H0 b9 L
kind: ConfigMap
3 U2 \: \. ~- o( QapiVersion: v1
+ Z. ^. k. K6 ~. h$ kmetadata:
' K( ~$ m h! M7 a* s3 u j name: kube-flannel-cfg2 ?1 \7 ~' I4 V7 a) g, b
namespace: kube-system
& x5 k1 M. Q& x" y& `) \. c labels:
& }+ b0 o0 X# T! Q tier: node
- a- v) M9 a; z5 f" u4 b, ` app: flannel
+ h/ ~( U) m* C" X: l3 ddata:
0 J: o. @' ~1 e' H cni-conf.json: |
W& I) b; ]) d9 ~& f& f [ {
# J r6 `7 m3 ^3 a6 n0 c "name": "cbr0",2 n1 a7 Q6 k! Y. ~
"cniVersion": "0.3.1",
~) v0 o8 S8 V. [ "plugins": [
) L1 X" v, ~2 x$ P' L1 F2 y0 M3 \ {; u( k; u6 H& x) W% ]+ z4 E
"type": "flannel",
* l/ d6 D2 @4 U* G+ F. { "delegate": {
, b5 _+ L3 H5 T" p0 d$ s "hairpinMode": true,* H9 A: m% U3 _
"isDefaultGateway": true; \' g' \2 e5 Z% U: t% [
}1 j. k4 u0 y7 J) T& H
},
' X* }% k4 o y& F {
: F5 c Y) u- \5 w6 Q' L- W0 |$ O, x "type": "portmap",. D. O! u8 L8 [. m
"capabilities": {
$ {- N# @- f& F3 Q7 h1 U, J "portMappings": true
1 I" g& g. k- N# } }
- n/ F0 ]3 A$ {% R }' d, C/ H: j5 v2 a* p$ `5 Z, V
]5 C0 b% ?) W9 g9 s# n# g8 f* C; \" k
}8 Q: B' `( ~8 O9 B0 ~* Q& _3 H
net-conf.json: |
* h) d; k! a, X7 x0 Z$ Z {" L2 P' |0 d6 a5 S# u* n" A
"Network": "172.20.0.0/16",
' G5 A( ]3 f6 }' p6 X "Backend": {" ?2 E9 ?( f. O, V" z( u
"Type": "vxlan"% p( L6 q4 O0 `0 H4 q! s3 b# \3 Z
}
$ X$ {2 x% c" u6 P }
9 I$ Z. d6 r0 e8 T---
: \: J) b- f4 C: g1 q% _$ I- t4 JapiVersion: apps/v1( l4 l) E. ^. I6 Z( Y, K
kind: DaemonSet9 D: j8 m! d8 G/ _* c l
metadata:
) ]# H! g) j) V9 @ name: kube-flannel-ds
; X w; @2 e4 t. s; I namespace: kube-system
7 }1 e4 b8 n- d# l labels:7 ?5 q- r/ }& f, _7 K" u
tier: node; y: V$ |# { M3 i/ j2 F
app: flannel
7 c3 W# w- C. |$ U! Uspec:
% X, s0 W ^ \3 e2 Q* I" k selector:' R9 M" q' t5 d/ Z& a+ L: [
matchLabels:& J, Q5 ^& X0 @( Z. y6 @6 a/ Y* d0 f
app: flannel, B* ?* M+ e$ ]3 V( _
template:
5 T0 ~4 W& F' a2 A( T1 o metadata:
7 H) k3 m! e8 C% t labels:
* H. B$ A9 H& Q* A: K tier: node
/ M0 s7 R. J9 @6 T app: flannel
, O: C0 G W; r1 ]* g N) g6 Y spec:' S# \9 Y9 ?. ]! W- Q v% Z: H
affinity:
/ l& z* ?; R+ G$ A! H" V nodeAffinity:% }8 X% _- I4 Y- L4 ]- g
requiredDuringSchedulingIgnoredDuringExecution:3 d- R$ K0 ` B9 l8 G# v. `
nodeSelectorTerms:9 I. f1 n- a( Q! {( e
- matchExpressions: l. w5 L' |# f! f
- key: kubernetes.io/os
0 d- l& Q7 F5 H% u operator: In
$ |# W/ Y- Y/ {' b values:
; R% r* l6 A% y5 d1 c! a - linux
4 k! A6 M; }4 {! e( F7 y hostNetwork: true
0 M$ Z# k. k, }( l priorityClassName: system-node-critical
, N Z6 i* I! G1 [- X tolerations:" ~% ~' B5 h7 P' K
- operator: Exists4 x; j/ Y+ @# o, [1 h
effect: NoSchedule5 L3 J7 F: a1 C7 H
serviceAccountName: flannel
8 R1 l8 p1 i% R" M4 j' V6 O initContainers:
+ k& n; G( a0 G - name: install-cni/ [6 O; v/ C5 D
image: quay.io/coreos/flannel:v0.15.19 f; w# d! F8 @4 I
command:+ l; g: _" Z A; Q
- cp
! ]8 Q3 f9 P( } u/ s4 D args:
: w- x. D; h0 f o" t2 O- Y5 g9 v - -f
0 Y/ h3 i# V- o% A - /etc/kube-flannel/cni-conf.json
; A8 w0 P5 x/ o3 A& j - /etc/cni/net.d/10-flannel.conflist
# o5 {6 U: W! q6 h# G# Y$ K volumeMounts:
, A9 B8 o: L1 X3 U - name: cni& Y: O' P9 ~6 |1 T# m
mountPath: /etc/cni/net.d' o; m5 e3 S4 m5 ^( ]
- name: flannel-cfg ?; A2 e) Z. D. x/ {* ^5 r
mountPath: /etc/kube-flannel/
" O7 ?2 {- n5 z& O% Z) t8 `0 y containers:
' I" U; {- q3 [" `9 {) H - name: kube-flannel
! e; Y( }9 q* Z/ t( {& w image: quay.io/coreos/flannel:v0.15.1
/ p; W& T$ p7 [1 \% i8 Z command:
4 ~% w; M0 e9 Z- p' u1 t - /opt/bin/flanneld z8 A- D- y$ p, g* s Q# v1 q
args:
/ l& q3 T4 A$ o9 O. G3 |! f. @/ _ - --ip-masq* x+ j8 n& ^4 J8 [! a- C$ p3 x
- --kube-subnet-mgr# {+ }5 D- H1 v2 h8 l- t# n
resources:' [# p2 }: k9 [& a
requests:+ L4 p3 ?9 A/ U) n+ ~3 T$ ^( I1 w! d
cpu: "100m"/ }1 U4 S9 G) X" F( e' f( e: s
memory: "50Mi"
) A2 f0 a5 t1 q+ N' l2 O6 N7 O limits:
$ Z! L9 H% w. J2 ^ cpu: "100m"
+ _& H; y# o: }* F( ? o' e8 T memory: "50Mi"' @) K5 L3 U* F" m$ C( T, x6 K$ D' i" N. E
securityContext:
8 G! {% r' Z2 N+ v- }/ t privileged: false
. V: F. x* C% D% S7 ?4 b capabilities:
8 K; {3 Y8 u6 \1 F4 ]$ H add: ["NET_ADMIN", "NET_RAW"]
* ]/ h) u, ?" w, B W env:9 w( \1 [6 ]3 E& ?6 b7 W
- name: POD_NAME, i$ |' v h' g; ?
valueFrom:* s2 H7 w* @7 P- I
fieldRef:* @3 D9 J' p/ f! s" @/ t( `" L
fieldPath: metadata.name0 P, x2 O2 l, S- t# J9 a
- name: POD_NAMESPACE
2 O- f* ^3 u1 Y) o1 W. p& D4 R valueFrom:7 ?( D) y' B; N1 Z' `
fieldRef:
& ~( r( ^( A1 ~6 @: S fieldPath: metadata.namespace9 [- L" W: r5 A9 W, S x
volumeMounts:
* A; @9 R" ?* k - name: run6 x2 k, B$ O1 X; r
mountPath: /run/flannel
* Z% q+ k/ V1 u' w1 f2 p - name: flannel-cfg4 [1 E" u) {# Z7 y! [0 v
mountPath: /etc/kube-flannel/# W e" _. Q: e& Z
volumes: U/ O! v) u H
- name: run
+ a' W& I- ^+ J1 x" x6 b- e hostPath:
# C+ b! m& d, S( b. ^) ]5 ? path: /run/flannel
3 Y- u0 M5 n7 n - name: cni
. S& a8 N% z5 f- b# Y; ?- n5 d hostPath:
6 s: G4 j7 k3 L, s' r path: /etc/cni/net.d
$ h% A; W5 P* Q, l; R" [/ }6 i7 g1 v - name: flannel-cfg0 c1 }* S: F8 S( w- i7 n
configMap:
& {; E/ a3 ]* ` name: kube-flannel-cfg
5 x- ]6 z7 k2 \, |8 Z6 v. F配置 flannel cni 网卡配置文件2 n& Y R0 V$ Q! [9 A
vim /approot1/k8s/tmp/service/10-flannel.conflist: Z6 D9 q+ d7 z6 r4 x
{
8 a' K7 q3 ?0 \7 t8 M "name": "cbr0",* Y! W% j$ F' l) I' l
"cniVersion": "0.3.1",
; g0 X' J, e( t9 ^2 U$ r "plugins": [
( n) X3 U3 _3 Y) x- B7 m! ] {) ` B5 C) l1 N1 P+ w
"type": "flannel",- G3 P3 r. a0 ?
"delegate": {( i3 o+ Y7 I( `4 n! V1 A- l4 k
"hairpinMode": true,
/ i% b9 ?2 O R* {6 ^ \ "isDefaultGateway": true
8 u- J9 Y& ~* \) w1 O* z }
. e6 d% D3 [/ F) @2 W+ q },% P6 `2 x/ v& r! D6 ^
{9 Y# B |) U9 n- K4 \/ ]
"type": "portmap",* W9 o! J" K3 B7 K7 ~$ o. E
"capabilities": {
" [0 G% u( n5 K: U* Z- n "portMappings": true
; B) H, o! H$ K+ k! D }
' G" Y0 t1 J* t/ H; e A }! Y+ @. Z0 b; [2 j0 X' t
]
) \6 d/ |: }' d8 }3 k. j5 Q}4 n; s* [ H' I( H) p. o# \
导入 flannel 镜像) @2 i' K. x; y$ f* U1 z
for i in 192.168.91.19 192.168.91.20;do \
# E* b0 A& e9 t' p9 m7 Fscp /approot1/k8s/images/flannel-v0.15.1.tar $i:/tmp/
% j) A: Q4 ]* J! Q- f% A3 k; ossh $i "ctr -n=k8s.io image import /tmp/flannel-v0.15.1.tar && rm -f /tmp/flannel-v0.15.1.tar"; \% Z- }* V6 n9 r* O8 S
done
E8 |1 }9 c" {查看镜像; V! I( c& j9 Z# |
( F- r' \6 Y9 m6 N6 h7 l
for i in 192.168.91.19 192.168.91.20;do \
' F2 k: M. D* ~& i/ w1 fssh $i "ctr -n=k8s.io image list | grep flannel"; \7 ^& z1 d/ o' |) ~" j' W
done+ P4 o. M% ~+ N) e$ v
分发 flannel cni 网卡配置文件
0 d4 h8 B7 N& x+ R7 |% `3 efor i in 192.168.91.19 192.168.91.20;do \
9 {, |5 u: C5 W. [ssh $i "rm -f /etc/cni/net.d/10-default.conf"; \
4 W; _ z! M4 O7 f$ S4 qscp /approot1/k8s/tmp/service/10-flannel.conflist $i:/etc/cni/net.d/; \ r+ B) ?2 L6 P: m+ `* J
done
/ T7 ^+ Y5 l" g/ s- ^4 b3 U分发完 flannel cni 网卡配置文件后，节点会出现暂时的 NotReady 状态，需要等到节点都变回 Ready 状态后，再运行 flannel 组件. R; W2 ^9 R7 a+ ~
! X6 G! z# i- G, C6 s, v
在 k8s 中运行 flannel 组件+ ~; c& h' p$ q* _; K
kubectl apply -f /approot1/k8s/tmp/service/flannel.yaml1 O/ u( a0 V( h H
检查 flannel pod 是否运行成功8 u4 m0 v0 V- j% [1 ]" W$ N/ z
kubectl get pod -n kube-system | grep flannel
7 v2 S- A4 s {# r9 H/ K预期输出类似如下结果9 k- B2 a5 L, o- h
o: T+ ^2 x( `7 gflannel 属于 DaemonSet ，属于和节点共存亡类型的 pod ，k8s 有多少 node ，flannel 就有多少 pod ，当 node 被删除的时候， flannel pod 也会随之删除: l& N9 |# M5 |
1 k, n( T" k5 W. A
kube-flannel-ds-86rrv 1/1 Running 0 8m54s9 p/ x( J+ X2 ~( X" @
kube-flannel-ds-bkgzx 1/1 Running 0 8m53s6 {5 ?7 M9 Z' ?2 R) p
suse 12 发行版会出现 Init:CreateContainerError 的情况，此时需要 kubectl describe pod -n kube-system <flannel_pod_name> 查看报错原因，Error: failed to create containerd container: get apparmor_parser version: exec: "apparmor_parser": executable file not found in $PATH 出现这个报错，只需要使用 which apparmor_parser 找到 apparmor_parser 所在路径，然后做一个软连接到 kubelet 命令所在目录即可，然后重启 pod ，注意，所有 flannel 所在节点都需要执行这个软连接操作' E" z: z& w. i! d( q; p T5 x
( O# Z0 b' v; O! F% d
部署 coredns 组件: ]- v1 o0 }) P; F& \6 U
配置 coredns yaml 文件
6 q! \5 j0 }; P- Y& y# d& w) |vim /approot1/k8s/tmp/service/coredns.yaml5 Q* j8 |8 h2 \+ n* O
clusterIP 参数要和 kubelet 配置文件的 clusterDNS 参数一致7 Y6 W, C; e( \; J3 N L8 s+ {
& H( {& l2 @8 ? V# s1 b+ F- [& WapiVersion: v1
" Z( r. t6 r2 f& A( kkind: ServiceAccount
4 [. I+ `! p4 `- U" Y8 M5 x# Dmetadata:
% E5 W9 L5 e/ D2 J- b& b name: coredns
) C# h/ m" V- z: m! P. ^8 y) U- t namespace: kube-system$ q- y, Q3 Z0 }0 H
labels:
( q# H/ A9 _% |1 N$ H$ L kubernetes.io/cluster-service: "true"
$ Q; s. _* d1 S$ `% ~( u+ h addonmanager.kubernetes.io/mode: Reconcile4 R: n R4 P6 w3 }- b5 d
---
# G& D' j R, BapiVersion: rbac.authorization.k8s.io/v1/ F( ^; H1 N& l% Q9 x: x3 Q
kind: ClusterRole7 `6 t: p/ h7 H4 g9 m" w
metadata:
a% r) D& _; U+ {/ j$ ?) ^ labels:
0 Y _9 c3 V2 \, w! [' a kubernetes.io/bootstrapping: rbac-defaults6 O5 {* I# f& X- L5 h
addonmanager.kubernetes.io/mode: Reconcile7 V5 X# ~2 [8 _7 t* _+ R- |
name: system:coredns j/ q) a8 O0 n' X5 I# Z" n" H
rules:+ j! w: A, x6 `' h0 C5 S' ?. q
- apiGroups:0 ~+ R' R' T- o9 U6 k. f
- ""4 L4 Q% f9 N1 a/ v/ O' }
resources:
8 s4 D) q( g; R# e: J) d9 d6 u! e - endpoints b% U3 I/ t- P+ c* ]3 }; s
- services2 u8 i5 Q( C# `. M+ ?. A5 Z/ r
- pods
0 V3 U" X8 O, w+ t - namespaces( i/ ]8 }7 P0 Y E2 {$ u
verbs:
2 \5 a- Q, x) R0 ?8 | - list
" `& p' q7 L- F5 ]- n - watch
, P' T4 r u; k4 W$ X. `! \- apiGroups:
" [2 z; R- |# L - ""
+ g" [0 v( d5 ^8 z+ z' s" Q3 k resources:
- i2 f2 U* w' E, {1 F j - nodes' ^8 y2 E- I- j2 Z8 h0 m* O* F
verbs:' g+ L8 F; t- [2 g: _
- get3 c$ G: G* p7 M) i; |# c
- apiGroups:$ v* z8 ^; ^- e4 e
- discovery.k8s.io
! _ z2 ~, ]4 j* @ M) H+ p" p resources:
* ^/ D% L! e" D4 G$ A - endpointslices( R L% ~) G6 F9 c
verbs:
% T, ]1 t3 Y9 B - list
8 N" D1 O+ |1 m - watch
0 T6 J/ i6 e, ]( Z3 ]6 f- M! x---! u$ {" c% Z! ~* [
apiVersion: rbac.authorization.k8s.io/v1
1 L9 [: _; Z" d5 q- M- Jkind: ClusterRoleBinding
7 o$ @: O2 g: f2 |5 I+ _( z nmetadata:
2 h2 b5 E. o$ O N% \ annotations:
. _! w, S c! C& ?" P* P rbac.authorization.kubernetes.io/autoupdate: "true"8 H; R5 c0 G4 ?: E0 G
labels:* j. u" ]' u' v: ~* s
kubernetes.io/bootstrapping: rbac-defaults6 O1 j% ^+ Y( B" a/ F
addonmanager.kubernetes.io/mode: EnsureExists
& S/ ^7 k5 }; [9 b1 f R name: system:coredns+ k# f! |% K, ^ B* z# K2 F/ l0 f
roleRef:
% V( M( L4 R! A( X: g apiGroup: rbac.authorization.k8s.io3 J+ `! B ^% q; u1 P
kind: ClusterRole# N+ d$ J& N. q0 G
name: system:coredns
/ x7 i; X+ I. o- T6 D$ K- ~1 rsubjects:8 X8 Q* M1 q3 f# e+ u
- kind: ServiceAccount- p* B Q0 l' v# Z8 D# w7 {
name: coredns9 D) n8 W# {! d1 J
namespace: kube-system
- w7 p1 `: y C0 j7 \* m+ u- Z---( N! {/ Z% H- Q3 W9 f
apiVersion: v1
% J1 \! H' P w: W3 q! W. gkind: ConfigMap- l6 B% \( J# {7 `! t
metadata:
) d; i- l' {+ D2 I8 s" i1 u name: coredns' ` z$ _" z0 d( i0 ^
namespace: kube-system
! }. Z5 f* F; I! v labels:1 o* z+ G- k4 t8 V% u0 `4 r- s
addonmanager.kubernetes.io/mode: EnsureExists
. Z9 w/ P' F6 Z/ W ?data:$ Y6 t2 e& b. b/ _ m) u7 D! k
Corefile: |
+ I% ]- j" y- ]0 d4 V5 H6 o4 A .:53 {( ~9 S$ ?- V! V0 B; o/ E, w
errors+ }/ K7 C0 x& V4 i& i( k
health {: M& J2 ?7 z3 L( H
lameduck 5s
$ F: \9 t1 Z: p3 y; A }
5 ^$ ~3 m- z9 }4 | ready
. q' f3 t: Y2 V% } _7 S kubernetes cluster.local in-addr.arpa ip6.arpa {
' j9 V9 D5 @& r8 o$ A pods insecure$ H$ h2 P' U3 s, i" ~6 w( o* u1 \
fallthrough in-addr.arpa ip6.arpa
# |& P6 z1 l" t& b% `. E ttl 30: o( f( o/ n8 C+ H' i
}
: S+ n' V* h G3 ^9 G7 _ prometheus :9153. p v o6 a) t6 h
forward . /etc/resolv.conf {
2 ?$ k* u/ f( p& d max_concurrent 1000
' E; |. r/ Q; J# \0 p( c }
8 E# [8 R }. [; V1 |, Q5 i4 ^ cache 30- Z; e1 l' P% g9 C: z+ C6 n
reload- G& b% m6 a `4 [5 i% o( g
loadbalance
* u0 Q$ k1 Z1 q9 j7 y. q% m }
& `2 J* t% w* d( s$ b3 D- ~---
" e6 e9 @1 C# X2 a' C/ vapiVersion: apps/v1& n$ D" `9 ^# c# ^& @
kind: Deployment7 ]4 c+ b% \9 A- M% e
metadata:
$ S/ O0 H2 V' K. B8 N1 M% g8 C name: coredns
' x- |* Z8 s; U namespace: kube-system
: ^; f" M- y% q( g( j labels:5 y- \8 T" _/ {% g( P
k8s-app: kube-dns
: J# t7 N6 G6 u. C" R4 C kubernetes.io/cluster-service: "true"6 u6 U Q0 g9 T- V% r) S& m/ k
addonmanager.kubernetes.io/mode: Reconcile8 z8 m7 R3 e0 L: u1 O6 u* \
kubernetes.io/name: "CoreDNS"
+ Y' h5 N. ?7 ~8 jspec:! z- k( w" w7 t) w+ Y
replicas: 1
: q, Q2 ?) x! a& H: r' W strategy: O% g& M; W G' B
type: RollingUpdate/ e0 J$ b& i; ]. L
rollingUpdate:4 J) G. b! A- Q
maxUnavailable: 1
4 C! ?: {0 B+ { selector:% E; s+ V$ n. w) h- C. C" Y
matchLabels:" n% I2 r% d9 R* U
k8s-app: kube-dns8 S5 P( L% n! W4 u+ @
template:
# k- j0 }: d$ n' [ metadata:
5 N7 X9 \. ^( ?) y# u labels:0 d# b" m2 s# V4 B7 Y6 p
k8s-app: kube-dns
! S, Q% }; I3 k- {' T' W+ X3 g spec:4 Z" c% R6 q8 ?2 k
securityContext:
8 ?3 E: _" b. A0 o; w6 w seccompProfile:1 o& n! i/ [( f" X8 v
type: RuntimeDefault, s2 U/ ]1 K5 ?( O) M
priorityClassName: system-cluster-critical; B9 n) g8 ^! p" _
serviceAccountName: coredns
7 ^, v, }, G, V3 V& c, H: h affinity:
7 m8 s& A# K4 n: Q+ A podAntiAffinity:
0 i8 d6 t# T- L9 H7 y: v2 \ preferredDuringSchedulingIgnoredDuringExecution:
( I$ O& H: V" o - weight: 100
5 h; i% I; F; ?% `/ Q3 \8 @ podAffinityTerm:
}$ p5 n$ }4 W2 C) h labelSelector:" ^2 J% `% @" r* y7 @
matchExpressions:
& [1 W% J' Y" g8 {" F" m6 l - key: k8s-app; Z Q+ m$ n; D, {5 c
operator: In/ \; @8 |' D! Y! b: z
values: ["kube-dns"]& D# U4 G6 S3 o
topologyKey: kubernetes.io/hostname# ]4 B6 V/ _2 U7 m% ^
tolerations:6 a+ V( x+ n; ^" Q% `
- key: "CriticalAddonsOnly"1 y$ W/ V1 D$ I1 u r
operator: "Exists"
O, ]. u6 M7 q) W) q! z nodeSelector:: K- |& X+ L+ j. b' l/ R
kubernetes.io/os: linux
6 y# R- Y5 ~* P L; } containers:/ c% q f# `1 X( ~( v6 }( C# N
- name: coredns1 e6 v9 x6 M+ |6 d4 v. r7 Y# I% a
image: docker.io/coredns/coredns:1.8.6
2 k! k) Y! c( K$ P+ `! F6 N# t imagePullPolicy: IfNotPresent
w4 `6 p3 O" @% r2 D8 I resources:
% z/ O F O0 r6 x) u, _5 p3 F limits:: x# M# u8 S5 K
memory: 300Mi/ F( X) y: X I
requests:9 `9 a. [+ J/ V% I+ A
cpu: 100m
: P! p+ N' ?- ]+ C) Y' L* R) c memory: 70Mi
$ U0 @2 w! X* B8 o' U6 ` args: [ "-conf", "/etc/coredns/Corefile" ]
+ Q+ Q e/ O) z volumeMounts:! E4 h* ^9 [# K9 g) n
- name: config-volume
5 X9 Q8 ?5 r7 m7 r mountPath: /etc/coredns% m& S' v* }" u6 R% V3 h' a7 U
readOnly: true; N9 c" j; ` e$ Y+ y
ports:
6 I9 s* T) H: ~- t* D. t; q4 x! C - containerPort: 53
+ x( Q; J& j4 G1 {, W$ }4 z name: dns+ x4 V+ q5 ^0 @
protocol: UDP8 V" C8 _! N: M
- containerPort: 53/ Y& p1 H( [ P/ w- A
name: dns-tcp
8 G3 K# k, T0 @4 g/ Z* |6 {4 |/ l protocol: TCP( j3 C$ r+ S3 j& ?' b% ^
- containerPort: 9153
6 `5 W9 Q; g, B( L6 \- T, f7 f name: metrics6 j4 m2 G" Q1 B- l
protocol: TCP+ ^! n3 @& A' P0 W7 B
livenessProbe:9 j& O! E9 s( B3 U. {4 n1 `& ]
httpGet:/ W) i/ ^' E" \( K+ C0 N* A* ^1 |
path: /health
; b+ d9 V0 X0 J' `% @( Y n2 |9 ` port: 80805 T; C9 Z% z+ R) A
scheme: HTTP* T1 a/ w' ]* c' |0 M! O8 c' L6 B% d
initialDelaySeconds: 60
+ W8 x, p5 j- ?, F* W timeoutSeconds: 56 i7 s& D' V2 v3 t! n
successThreshold: 11 n( I7 R) n$ r- H
failureThreshold: 5' G5 l0 Y- {3 r2 j- `; ?
readinessProbe:8 A: j4 z8 P6 a, y# w5 g8 [) j, j
httpGet:: R3 O' ^7 W/ m" R
path: /ready1 ^$ `7 ~) j7 H) ]5 ^9 M0 k
port: 8181
# h, |6 e2 x$ X2 p. Y U. b. J scheme: HTTP
; `) I# C- `' y6 w& z5 E securityContext:
% `* M0 B- P! V! y% R) u2 r. H& E$ k allowPrivilegeEscalation: false
" q/ i# Z% P8 V/ z1 M capabilities:% |. F. D- z$ r4 Q
add:
k) G( ?/ }, m- ^. ~ - NET_BIND_SERVICE
; U e% x: C( L- n; r drop:( N. i* z% M9 s' t* ^
- all& I1 X! H# h' R4 b4 z# r9 g
readOnlyRootFilesystem: true
/ A V: y8 H6 }4 U dnsPolicy: Default2 N/ n- E& h1 k- e
volumes:* G1 P7 ^. C, R8 A
- name: config-volume2 Q2 L" k: U" j9 P
configMap:' W: G, z8 S/ M: @1 D
name: coredns( @- Z* t( V& w1 h0 c
items:2 a) g( z. e m( M2 A
- key: Corefile0 @3 K8 N9 c- q% Z @( H
path: Corefile6 H2 u% [" s' ^/ t! S# T) X0 y
---
2 s& B) }# e. J2 ~9 O7 IapiVersion: v17 M8 e6 Y4 e* S
kind: Service" D$ K8 {4 e0 c
metadata:
1 R+ } F/ h2 M' ?+ v5 D r8 `( l name: kube-dns8 g4 J5 \; a, A
namespace: kube-system" k1 y2 |2 L! K2 o1 ~
annotations:
. \4 R9 W. D$ A r" ? prometheus.io/port: "9153" L9 @0 T; u4 h' W# L8 \
prometheus.io/scrape: "true"
# s9 Y4 [2 D1 S( { labels:! g4 |: o- v7 G7 o1 H6 @; U; V- [
k8s-app: kube-dns
5 x! g$ w& m: O+ z kubernetes.io/cluster-service: "true") \5 ~6 o, w `+ z
addonmanager.kubernetes.io/mode: Reconcile7 R2 R0 N4 c8 o& H5 j
kubernetes.io/name: "CoreDNS"
3 M p/ U/ `; [* Z. k* ~6 mspec:2 D, H; A8 B3 \9 D1 H1 y4 [
selector:
0 P' G7 e- V. S& Z" a1 x; o k8s-app: kube-dns
+ a2 `* ^- T ~' Y* [ clusterIP: 10.88.0.2, N" H4 g1 Y8 A. _, K& g
ports:3 E* r# X$ W$ t& w$ `# A- }% I
- name: dns8 D d+ p4 A8 V8 T( E, p
port: 53$ M. r& e5 w( U# U. K5 Q
protocol: UDP+ Y8 c* L8 j, }, }! O T+ l
- name: dns-tcp
7 g2 j6 L" A8 { port: 53
; |( [1 q) a9 ~0 d: W- ~* m protocol: TCP
5 q# g1 A. x* G1 |' z. g% t/ G - name: metrics h1 N1 {8 F6 l5 x: G( W0 n, u
port: 91538 Q8 {* t* \$ e6 p1 |
protocol: TCP
0 L% [- b3 |% f0 A% z导入 coredns 镜像
4 y! u4 }5 C/ R" bfor i in 192.168.91.19 192.168.91.20;do \6 C }! g$ r H1 f* i
scp /approot1/k8s/images/coredns-v1.8.6.tar $i:/tmp/$ q& w0 }: a! M7 ]. a8 S+ D
ssh $i "ctr -n=k8s.io image import /tmp/coredns-v1.8.6.tar && rm -f /tmp/coredns-v1.8.6.tar"; \) {5 o( P& _1 |' }; E6 Y
done
6 _0 K4 |3 Q- E5 y( g# q查看镜像
& t. O* x5 G- j3 o( g# J- {, M
; C& ~9 j ~5 N: z# V1 Nfor i in 192.168.91.19 192.168.91.20;do \# e9 }$ W: c" v/ ]+ a5 m9 Y
ssh $i "ctr -n=k8s.io image list | grep coredns"; \
0 u( K. A( ^7 p4 d* z" l* C" pdone
' g0 I _9 _5 w/ Q' k在 k8s 中运行 coredns 组件
f9 ?( E. t h" R& m9 O' d1 r7 \kubectl apply -f /approot1/k8s/tmp/service/coredns.yaml8 u1 E' @6 M9 o i9 o5 q1 z
检查 coredns pod 是否运行成功* d$ {. `+ m! L
kubectl get pod -n kube-system | grep coredns5 g+ z/ Q- G3 R; i, {; X
预期输出类似如下结果
* P2 W9 T8 \( |* g0 f) H1 [4 r$ E2 T
x! y* W, j- [5 O; Z因为 coredns yaml 文件内的 replicas 参数是 1 ，因此这里只有一个 pod ，如果改成 2 ，就会出现两个 pod
" w5 O: _ l" M2 w: Z8 x4 C1 V3 _, ]8 c4 Y. @/ T' `, ^
coredns-5fd74ff788-cddqf 1/1 Running 0 10s, }8 A" r" G, q* p, M
部署 metrics-server 组件6 Y8 H) z8 f% C
配置 metrics-server yaml 文件7 ]0 |! o2 z8 t
vim /approot1/k8s/tmp/service/metrics-server.yaml
( r2 d6 K8 S9 _0 _8 G. Z* zapiVersion: v1+ k- [: A5 \ T
kind: ServiceAccount
* ?; d% w( a( u4 ~metadata:
% V1 f6 m8 z' _2 h% t- U8 \5 @ labels:
+ F- B& }9 P. m" e' n7 ? k8s-app: metrics-server
0 D' M: q! O, R0 g* Q( K name: metrics-server' g+ y, X2 r- M b1 ]9 b9 ~8 ^1 T
namespace: kube-system' }: L1 q/ R0 B* ~- i8 y0 I! S
---
: _/ a4 n3 } n f, LapiVersion: rbac.authorization.k8s.io/v1% v4 K' }4 N8 S$ G' n9 |* s8 d
kind: ClusterRole; v) j/ G+ e! k; _4 ^ v& h
metadata:) Z' K. ]. i) J7 s9 ~( W
labels:" T! q7 l9 U4 T5 D' |" _' i
k8s-app: metrics-server) V% [; D" Z; }/ f' E6 `
rbac.authorization.k8s.io/aggregate-to-admin: "true" r0 T/ [# b- U9 V6 _" h
rbac.authorization.k8s.io/aggregate-to-edit: "true"0 |' Y. C& `0 c
rbac.authorization.k8s.io/aggregate-to-view: "true"
/ `( q8 Y. l* ] name: system:aggregated-metrics-reader
& D; R" C+ N* g" Y+ v' _ ]4 H! o, frules:
1 ]5 y" K+ l1 W n- apiGroups:' Y3 @* M9 |8 L: f
- metrics.k8s.io
( g6 k: o2 S4 {" f. G) n; Z' ?0 M resources:
4 k% D$ J( f& c5 e, L - pods
( }. {% _9 Q$ L0 N' \2 e: ] - nodes, ]. Y# D. E, k1 g: Z, G
verbs:% J* D+ _, S& G9 q* H! F8 O, c
- get
2 E, p9 o- J. K5 m+ Q! B - list
. q$ H* w6 z1 X* h3 ^ - watch! I4 C0 c# p% E/ F8 X/ ]. k% K
---& V: }' f2 [- V4 v! u9 |2 P. U# I
apiVersion: rbac.authorization.k8s.io/v10 |8 }. U) N$ h2 K
kind: ClusterRole1 c j( I4 d6 s& ~7 N( A
metadata:2 d7 c& S, p* y! N" u1 W2 R
labels:
) A& ~& X- h9 p. }9 J k8s-app: metrics-server
# P: ]) y2 v$ | D) `- H* r name: system:metrics-server
; d- Q; W1 g4 D1 R% grules:
& B: |9 N! U$ l6 ~- apiGroups:
" Z$ `6 c# Q1 v3 x" i4 U - ""
$ L' f9 t, W/ {9 R resources:* \% x5 {. h" y1 m; ~$ }. `' I
- pods* B l/ Q7 T4 g
- nodes1 `7 P) P: g( E: ?! f9 B
- nodes/stats
1 ^6 ?6 a5 q4 X6 l - namespaces9 o4 a4 S. e K) Y" i: n, `$ B7 n
- configmaps& r0 @+ t( S) M. x
verbs:# P" o, M$ }: v2 ?) F5 J
- get% X& G+ z+ D; a t- @( d2 ~
- list
. f+ ]/ X3 R" V5 f* K) n - watch, q6 @8 [# a6 p/ s; {4 f& z
---0 V0 W# a% C O D
apiVersion: rbac.authorization.k8s.io/v1
0 F& d3 h0 w: W2 z: o+ Z; mkind: RoleBinding
1 j7 z! `; \, k; C0 m5 i. _1 Cmetadata:$ O, s2 x6 e( [( n3 j$ C. ~6 F4 Q
labels:, c9 N+ |& L' y \: r, \
k8s-app: metrics-server
2 a: [2 ~1 d, ^- J: M- `' m. E6 T name: metrics-server-auth-reader+ p" C; O! ~) k. q& r1 \* \
namespace: kube-system a' s% E. C5 k
roleRef:( A; _! U% b1 n! B# S @8 E _
apiGroup: rbac.authorization.k8s.io7 L3 l( z. r9 c3 [+ d
kind: Role# E' m0 j3 @5 W3 K1 }- z% `4 z- E
name: extension-apiserver-authentication-reader& i3 v3 H4 b% c# y
subjects:
' D3 q( W/ i6 S- kind: ServiceAccount. |* j# Z, y$ ?4 V1 B
name: metrics-server
1 z0 Z, q% A3 U4 s7 b( j namespace: kube-system5 F% @6 ]5 C3 ~5 C& `
---
- I9 w" [# m. N, c( S& S' A1 GapiVersion: rbac.authorization.k8s.io/v1
% j! |' \5 I- y/ i' _5 dkind: ClusterRoleBinding
% B; L/ q7 s2 p! o/ h P# z& Tmetadata:
. w/ d; V$ j$ L labels:5 @7 y( s% _: f6 I
k8s-app: metrics-server
9 w3 K$ \0 _; G name: metrics-server:system:auth-delegator2 N8 b9 z! _! t# ?1 V& I
roleRef:
/ j% p! a/ G7 p) M+ l* | apiGroup: rbac.authorization.k8s.io
2 o3 K! d# D& J kind: ClusterRole& c- I* B3 i8 p* y( q3 `: M
name: system:auth-delegator
! h; R6 V! m5 tsubjects:' b6 ^: e8 n( m" B' z/ i
- kind: ServiceAccount' `2 X% C: W; g+ v7 o! v+ {
name: metrics-server8 ^+ q$ V- R- Q' d4 N
namespace: kube-system" Z3 S: s2 o& F7 N5 a/ L7 m
---0 U- z! i% w; A3 J3 Q
apiVersion: rbac.authorization.k8s.io/v1+ H, Q& ]- ?4 J4 g3 T- r) j0 s5 a
kind: ClusterRoleBinding8 A1 r n$ S7 i
metadata:* S& J# w1 W1 s( H9 o7 o
labels:1 k/ Z; r8 {6 f
k8s-app: metrics-server
. ]9 i1 p' J- A: i1 d. P name: system:metrics-server0 A& S2 R! v6 s% E" }0 a
roleRef:$ t$ s( ~. G2 s! F) T/ K
apiGroup: rbac.authorization.k8s.io( w& X1 k1 ^0 ?/ x5 n" b1 J" J
kind: ClusterRole* N3 D1 B6 Q- a" @
name: system:metrics-server
3 S; T. c$ [4 t9 Esubjects:% `* a v; \2 `0 U# R* E, K
- kind: ServiceAccount2 h# s7 Y3 D3 A4 u
name: metrics-server) r. _# l4 d$ I) A0 a+ @+ A+ D
namespace: kube-system9 m* J$ k/ Q" R
---' p% L5 p+ d a# u" a1 _% [- C/ i1 G
apiVersion: v1& \; M( t4 _ x: L% H& g/ D
kind: Service
+ c# Z: d6 s T2 z U5 lmetadata:' I/ D% e- c5 m# y3 B
labels:
$ n) W( Z1 r- b4 N) A3 z k8s-app: metrics-server
, `. d: _6 [4 r4 S) K& I1 `0 w% w7 R2 X name: metrics-server0 ^) h% |" b+ C: i0 y
namespace: kube-system
9 Q s9 s3 U4 @: L+ K( O8 Kspec:
5 q; W/ Z( {. r9 K ports:' c8 i) b$ V5 B0 b
- name: https
, v8 k/ M- v7 N& O port: 443) q! A; W4 M- S0 I5 Q8 x
protocol: TCP
]# O( X: B6 l) e" W: w( J targetPort: https3 r0 {$ |8 a# H0 M
selector:0 @* E' Y0 v! z2 S+ u1 _
k8s-app: metrics-server
/ X! v2 u' d7 H1 n. {---
% e8 O8 _8 c0 dapiVersion: apps/v13 O' D6 y+ d* n6 P9 ]; a
kind: Deployment
F6 S. _ @) {: J" smetadata:
! q9 y0 e& ?7 Z3 ? labels:
' Y! Z2 ?" @( I* z k8s-app: metrics-server* E% ^' }3 Q( y: |
name: metrics-server
( _5 f% v/ b0 J. G namespace: kube-system! l& t5 Z+ {( B3 M- l
spec:, |0 }( J2 p( y; Q
selector:
6 \: R; R- A" j) X matchLabels:
* ?7 {1 G, L- g; ?4 F" @ k8s-app: metrics-server
# x' b. v* `7 A2 g/ y1 f$ E strategy:+ q" `+ c- A1 _+ Y9 r
rollingUpdate:# G1 k1 g# s/ n0 R
maxUnavailable: 0 A" V) n8 H6 S- y; Y
template:" _& t4 ?! m" w6 G: T! S
metadata:
1 k+ C; o9 a- d4 n* Z* C labels:
. `- T. j; E$ @! a# {! E k8s-app: metrics-server
% W9 \% M& Z1 }+ x spec:
Q$ I' Q6 H2 g0 } containers:
5 B+ R( B$ `/ G; \8 |" I6 H- N - args:' Y! m' S, `& W% m' ^
- --cert-dir=/tmp8 V0 d9 ]8 k, S1 [& V
- --secure-port=4443
3 y! x2 q$ g% g7 c! | - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname0 @, Q! _/ R& g: p ^
- --kubelet-insecure-tls
5 x9 O* t1 w- r5 v8 i - --kubelet-use-node-status-port a8 J' O3 b7 g/ K; P% R- d
- --metric-resolution=15s
( _: E* y2 U& p& i image: k8s.gcr.io/metrics-server/metrics-server:v0.5.2- X& O! V1 E6 M4 }6 m) ?
imagePullPolicy: IfNotPresent2 s& R: R* x$ r1 q+ T8 {. r
livenessProbe:1 X& O1 [. P7 O# K. o3 ~
failureThreshold: 3 P4 X, a0 b3 |& A9 E4 `$ y' s5 l% E
httpGet:
; P) b' K7 S6 M1 z7 g( m path: /livez k8 n2 H5 n8 C( U* m
port: https- j' ]/ E3 R, b, m L/ N
scheme: HTTPS
2 r/ h3 ?% P6 d- h periodSeconds: 105 K) D/ p: |2 `. o- R# J
name: metrics-server3 w, h' }! N+ ~" C, i z
ports:6 H' Q, s' X3 Q' {
- containerPort: 4443
5 B$ p5 ^9 L% e m9 Q A5 S/ F name: https
! r8 e. a* o5 x7 v& y0 ^ protocol: TCP
; e& J5 X5 r6 M x" `9 U# |" p/ j readinessProbe:# m. B3 X2 }1 ^: B1 N
failureThreshold: 3
/ W5 L* ~% g( P S$ X8 A httpGet:
+ C2 [1 |8 Z- N: T1 \5 { ^ path: /readyz$ T/ U6 Z& b" v! O3 ~* c2 G
port: https1 y2 f6 {& r2 l' t+ F
scheme: HTTPS: ^( m" j! M; g5 U B
initialDelaySeconds: 20
3 P( {# W) b1 _/ z0 G T periodSeconds: 10) G0 D# \+ S' c8 N/ I* V( b5 O
resources:
/ M5 s3 N5 J$ t" O$ I% X! ] requests:
9 }: b/ T6 ]: q4 }% `2 K) n cpu: 100m
9 X n! |$ i5 g' G: v" @- n memory: 200Mi0 z# k; L8 e5 \& M! k
securityContext:
9 ^1 b6 W- Z U( T D readOnlyRootFilesystem: true, F# _3 W, j2 o. r$ j
runAsNonRoot: true
" Q" |' V7 l2 Z: F runAsUser: 10000 ^/ c, I% @- B3 u" w: h2 E! y
volumeMounts:
5 v9 ]0 r, t n4 w# L, J! {4 | - mountPath: /tmp5 h( y) s1 C' {1 X" f- K5 X
name: tmp-dir: l( x- s" J3 Z7 U( W5 b
nodeSelector:" q6 H: C9 y' w4 {+ _. c! `
kubernetes.io/os: linux
+ I- Y. Q$ d8 x2 _ priorityClassName: system-cluster-critical5 X# v2 v- M/ h
serviceAccountName: metrics-server$ N& {& a" X2 _$ Z. j! \- \
volumes:* l" e, m! G6 ` @" b
- emptyDir: {}$ g" H4 n/ V5 l2 k+ ^
name: tmp-dir
+ m. G# ?; K' }! T/ r+ |: n% p---
# r$ ^; f, m, \" o8 ^" GapiVersion: apiregistration.k8s.io/v1' P) j' Y1 r7 x1 t3 H5 C! L3 x
kind: APIService
; |. H i& ?3 ~# }7 H- Smetadata:
/ Z* Q9 z( }4 v/ x4 ~ labels:
7 c. H! x% Z: c' K0 \# A+ e+ u2 _3 c1 M k8s-app: metrics-server
# o% ^1 T+ U' f2 A! X; `2 a' x name: v1beta1.metrics.k8s.io( L( X1 Z' w* |0 P, n* `
spec:
6 o+ h# T' X( t7 ^" E) \, J6 z9 C group: metrics.k8s.io- k1 ]1 w% h2 h8 \/ T7 i3 \' F
groupPriorityMinimum: 100* y, k& X6 i# B
insecureSkipTLSVerify: true* H h$ g& l( D# _
service:
, ^: b0 B- h7 |4 \: y name: metrics-server; P7 Q4 w8 K# }1 Q! `6 {
namespace: kube-system: i6 Q5 Y' w0 L% k b2 |; e) _- E
version: v1beta1
) ]3 m) q Q/ A versionPriority: 100
K, Y/ [9 P( c; t; Q导入 metrics-server 镜像* U: o2 G8 M. m. S3 S
for i in 192.168.91.19 192.168.91.20;do \7 m, K: i; o9 C) g# {# p
scp /approot1/k8s/images/metrics-server-v0.5.2.tar $i:/tmp/9 y& P2 j* B/ E
ssh $i "ctr -n=k8s.io image import /tmp/metrics-server-v0.5.2.tar && rm -f /tmp/metrics-server-v0.5.2.tar"; \
! f: m8 ^8 z* a7 ~, K8 f1 Ddone1 i. D5 M f0 \* N8 {
查看镜像
) T' w" j4 o( ~6 h, p
' C" f: i0 W5 K9 R+ H( L$ Bfor i in 192.168.91.19 192.168.91.20;do \9 W4 c3 O0 ~6 w2 S5 {7 U2 s
ssh $i "ctr -n=k8s.io image list | grep metrics-server"; \
/ j A- C' z5 R5 k$ Edone% k/ X% `' ?$ W5 q8 ]. G
在 k8s 中运行 metrics-server 组件. P+ s8 U: N( ^% C7 @
kubectl apply -f /approot1/k8s/tmp/service/metrics-server.yaml" t+ K+ r1 H. @& k, m: e7 }
检查 metrics-server pod 是否运行成功, P8 a+ Y8 u( D, Q: V3 Z5 s
kubectl get pod -n kube-system | grep metrics-server
) R' j6 L2 T. I& z( ~预期输出类似如下结果
z: `* K# O1 r: c2 _( ^
0 A- K- a6 @1 x+ m& J; [! p Umetrics-server-6c95598969-qnc76 1/1 Running 0 71s
( S' T) I, \! A3 E验证 metrics-server 功能
+ J* Z' L8 L. ~! T' k: W, M' \: {" D$ u. R
查看节点资源使用情况8 W7 l% _3 D! S1 x% @$ l
' [. L/ {5 B, S# Q4 [$ ikubectl top node
( r8 I( n5 Q" e预期输出类似如下结果
& K; J/ H" v# w# [- {( n- n) C
5 s) ^, ]. d4 P1 T# `0 @0 Q% \2 fmetrics-server 启动会偏慢，速度取决于机器配置，如果输出 is not yet 或者 is not ready 就等一会再执行一次 kubectl top node
; `1 A; n7 [/ f) U! s
9 S) |1 _2 F. xNAME CPU(cores) CPU% MEMORY(bytes) MEMORY%3 ^; `* x- j; ?' r3 i4 [2 m
192.168.91.19 285m 4% 2513Mi 32%
. m) l$ o5 R) W1 q& m5 b192.168.91.20 71m 3% 792Mi 21%
# t/ }, v# o8 Q" a' b查看指定 namespace 的 pod 资源使用情况
! m) f/ {6 M( g0 T. z9 z
9 y! M" [! P4 X2 }* {kubectl top pod -n kube-system- d; o* k0 m) N& y. |! ?8 _1 G
预期输出类似如下结果
+ |1 L7 A( ?/ [# e+ B- j p6 u! e- E7 o% Z
NAME CPU(cores) MEMORY(bytes)
0 s* k5 ]1 r5 M: t! Zcoredns-5fd74ff788-cddqf 11m 18Mi6 K) [7 m2 T u! ?) T" ~- F
kube-flannel-ds-86rrv 4m 18Mi: I8 V* x' Z3 \
kube-flannel-ds-bkgzx 6m 22Mi
! R3 {2 Y6 O1 y( F& skube-flannel-ds-v25xc 6m 22Mi
a& r+ e9 M- H3 V/ {! z, @metrics-server-6c95598969-qnc76 6m 22Mi
% ?: s* Z$ w3 @' c: G部署 dashboard 组件3 I3 K. }0 u& R* m6 @
配置 dashboard yaml 文件" X1 I8 [/ _: V. k4 Y0 I# K6 | D3 Y. h0 ?
vim /approot1/k8s/tmp/service/dashboard.yaml" y9 o3 H: S2 K! n3 w
---
0 M4 s4 d5 D* V; r1 N" i$ \apiVersion: v1
0 s& r2 w8 T* `* X3 I) e" |kind: ServiceAccount
) [; M0 ]# {3 \. |metadata:0 [& q, D* E9 R( Z" i- F8 q+ M
name: admin-user* H n2 D/ X& k. k. b3 W& t
namespace: kube-system
k5 G0 ]! ]6 {! A) D. a& X4 q- j/ w) z2 a
---* \3 `+ ~, [( t9 M/ w j2 U) M
apiVersion: rbac.authorization.k8s.io/v1
, C3 K/ |/ R- s/ Gkind: ClusterRoleBinding
2 y$ |7 x- d2 i5 n" B+ d5 Emetadata:
( n' B9 o& A5 X c name: admin-user
2 i1 T# c. W4 D1 croleRef:
3 b6 R' v. E2 a& s; Y E# g* ~ apiGroup: rbac.authorization.k8s.io
6 M# m8 y% A: D Q kind: ClusterRole
7 O- T: p4 M9 v! i8 B name: cluster-admin
7 r) J* r- m; P r( |9 lsubjects:
; N9 j1 U9 o3 L ~- t/ e- kind: ServiceAccount
' [9 @% R5 [! r, ?0 j" O" b name: admin-user6 L9 }7 n& n, S" I3 ?. F- `
namespace: kube-system
4 K: c2 i: P2 i2 `6 K) n5 j9 @7 t0 b0 G$ \
---* ] l4 l3 [# ^. M7 w
apiVersion: v1
1 c! X' }8 I3 p' x) E1 skind: ServiceAccount" |) e0 s, n. B2 C+ Y' e1 I
metadata:
( _6 m6 k. r* g4 [$ }+ F name: dashboard-read-user
" }/ D' [$ x- H namespace: kube-system: r" I% Z0 G5 S$ Z ]
9 t& r1 b. l: H0 I! N
---& F5 n" M9 n" h1 N
apiVersion: rbac.authorization.k8s.io/v10 ]$ w* `: |! s" r/ e: n
kind: ClusterRoleBinding
7 k: t* s/ E, l9 |, U+ R. dmetadata:
0 g4 f; j+ C$ {0 N name: dashboard-read-binding1 R7 F- d( t; j) U$ E0 @# d) _
roleRef:
1 ?5 n# I# _$ m# z' j apiGroup: rbac.authorization.k8s.io
; {2 p3 g, s5 H* t" b kind: ClusterRole- {+ s+ u+ P6 n' n" j# p
name: dashboard-read-clusterrole
, ?/ N2 V6 E0 F+ ~subjects:3 S! d5 p7 q2 O. r
- kind: ServiceAccount$ ?; N5 W# m/ O+ W/ q
name: dashboard-read-user
! c- ^, i( a$ |( K3 \) n( C namespace: kube-system% o# E7 n* c% ^/ Y3 Q! l1 {
- p* p$ t' n3 S4 z/ s) a, x7 R$ Y/ X---
0 H1 n( Y' S$ E7 DapiVersion: rbac.authorization.k8s.io/v1
3 ]7 h, y5 B* a$ b. F- X) ckind: ClusterRole
8 D& R! [: l; u" n% @" Xmetadata:" u; j; K! o( G7 K( r
name: dashboard-read-clusterrole
/ ?7 h/ e& t9 Y4 Xrules:5 W, A* ^% I# {" U# Y7 I
- apiGroups:( G7 m s5 s; G
- ""
: v9 j% ]9 A/ `, x* b resources:8 C. E% ]0 H: O, l. W$ C
- configmaps" b _. T3 n9 [8 {
- endpoints
! l6 J, A5 s9 o8 W9 n7 ~ - nodes
5 c- X. D4 Z9 y/ e. b2 N - persistentvolumes( S2 {1 [( a! G( S1 X! V0 E
- persistentvolumeclaims9 T& n2 r! x; \, O) a* u. R/ F
- persistentvolumeclaims/status8 C: @2 [, i; d1 _4 }0 q
- pods/ T' Y7 D5 A- f* d
- replicationcontrollers2 `# m1 u3 b# E
- replicationcontrollers/scale
7 m9 q9 [/ @# q/ |# M - serviceaccounts/ a+ L5 R. ~+ o; n& L3 S- r5 B2 ]+ ]
- services9 E7 s7 F- u; k3 K
- services/status- W0 K# a/ T! T& F$ u
verbs:
1 H4 |) G6 }% k+ D - get1 O. m+ n- \. V( d/ h$ B
- list9 l6 \. [5 Y, m }" l
- watch# z+ E& ^6 K/ O" j/ r9 u
- apiGroups:
6 F* k0 B( i: O- v - """ ?, z9 P; a: j9 b2 i
resources:
* J; v& I5 Y6 Z/ V; y& {+ R - bindings
- p' |0 w- }8 Q. A: q# v - events
/ `7 Z: @$ l! ^( q - limitranges3 q+ O" m7 H% I* m9 m8 h" ^ l# E
- namespaces/status6 ^+ F/ f0 o+ P0 M" d
- pods/log
5 Q1 V8 _6 E, a - pods/status
4 d5 ?+ |) ^: [ H7 Y4 t - replicationcontrollers/status
% j* V2 ~) i+ M - resourcequotas. ~1 u% ~! D9 k; J- m* W- M
- resourcequotas/status
' ]9 B2 I! y1 c) j verbs:5 v S* l( R' _
- get( ], P: ]+ G- p2 T8 n ~( N
- list
+ M9 b6 J* f0 S$ L+ n- o - watch$ u8 d# D8 q: w" L: T
- apiGroups:/ B( f7 z7 h7 Q
- ""
! X7 b$ Q; c& G5 x. T" u resources:: E, o! Z0 w5 a- I
- namespaces W* C, R1 _+ w/ _7 x3 d; o4 G; Y
verbs:0 M& w! ^) A- E3 f, l% v1 F7 q3 X
- get5 v/ }, t0 r) I; _# C$ D( Y
- list
6 [7 t6 E. U" I4 y" a2 v5 C - watch
) ^1 n/ U& b4 b0 j- apiGroups:( c* N& F- _5 I* j
- apps
+ T8 x$ N( W1 c+ `7 |* H$ b, g resources:( U9 \/ D6 H, }5 [
- controllerrevisions
! {6 p/ h9 `+ d3 P% S+ o# I - daemonsets6 Y% r0 r5 D. J0 H' m8 E! Y
- daemonsets/status& o# I, u$ K9 L9 C9 Q& M
- deployments9 x O+ O9 N# Y& L, i# y: d
- deployments/scale
) z* C9 e3 z1 e; Q( { - deployments/status
+ l `9 d. c; }1 S - replicasets
; M/ E! d3 ^& W2 d* z - replicasets/scale" ^: H9 ^* v% N, g* n% j
- replicasets/status8 n; r# ~4 j0 r5 F ~* W9 l
- statefulsets/ T# {, v, l1 I& }4 D! S- r2 q) `9 l' Y3 }. x
- statefulsets/scale6 D5 j* Y6 f6 w* @' K- [, b$ P/ H
- statefulsets/status+ L3 n, m. D) q& k! e2 O- a
verbs:0 A1 ]0 [/ t) X) Q
- get" H. M; i1 j% m( \6 a4 X; k$ A0 r
- list
3 a$ d- v' z0 W @8 h - watch* F# ?# Y" d3 B, g9 s
- apiGroups:
9 n4 {- l4 z! `" q/ v - autoscaling
?, ]8 P( H; |% S, m resources:! ?3 I, H( L/ w2 ?
- horizontalpodautoscalers
[( L) x+ R# I" ~6 v# {" d S - horizontalpodautoscalers/status
1 o% r7 X7 Z+ [ verbs:( L" H8 j9 H' t+ r
- get
! x8 E: I# T2 W - list
: W! j3 S) f. I% G& l4 j4 X9 Y - watch& G& s l0 _/ d3 Q
- apiGroups:7 \7 b; k* l+ J: h
- batch) `4 }7 X: l2 G+ z
resources:7 M) E, I2 F7 L* k
- cronjobs
: j; S7 e9 U3 B* S9 d, ^" C - cronjobs/status. v6 a. x% D' Y9 P/ I
- jobs8 y+ }: q# a" I4 h9 J
- jobs/status
3 S- V6 `0 h+ j& ^1 u! } verbs:
5 t w M2 }, L' x. v. [4 v, ` - get1 h; T) N8 R" v8 ]# g& I8 v2 g
- list9 w; N9 s( y5 A _$ m
- watch& {3 r: _( _# @( j* ?
- apiGroups:
# W1 l( S- K" G& G, ?) C2 @ - extensions: r0 \2 C! P2 a; _1 X. X: r/ Y
resources:
( M& A' Y) n, D7 }) F/ v - daemonsets) H e5 A1 x$ g( H( K8 T6 A" H# X
- daemonsets/status' T" \0 A4 v3 i; h- N1 _" [
- deployments9 k* P$ m- Z w! v k
- deployments/scale9 s5 O+ L) k+ {9 [
- deployments/status+ W e( u3 B( P4 s- D
- ingresses
" W# E0 }' i L! N+ O: a - ingresses/status
1 w% b! n4 [. g$ w2 l% \( ]4 a) Y - replicasets: p Z" H; a, o, Q R
- replicasets/scale
* j0 h6 y. [2 g. V/ k2 q( j0 V7 } - replicasets/status
$ W6 m$ P! T4 _( b; l - replicationcontrollers/scale8 a+ i9 w" j7 y9 Y8 A! r: \
verbs:
) w @7 E1 c! X - get
) [: [. |! ~" ]1 U D7 }1 `- a - list
$ J5 R" b7 j1 X; i5 u/ _5 a - watch
9 e5 l: G! Q! k1 N& r8 J- apiGroups:
& I( \. P* o. r6 F1 M - policy
6 L; ?" R p. [' o8 _/ u) n; M resources:
7 F; a% @' T& U! H. B - poddisruptionbudgets7 v& X& ?* p: ], }6 x& E" q, I: t
- poddisruptionbudgets/status" u( A7 k6 @* L4 m; u! d
verbs:
7 n# F: D. B! i" m* w# ] - get1 f( W& w5 }4 d6 B) i$ a
- list
3 c- w' w6 a# s$ d7 R0 { - watch
$ V) n+ e, @$ s R- apiGroups:9 Y3 e' t0 d7 ]% b# [
- networking.k8s.io* k& Q o6 I1 z% }2 d
resources:9 x" _' D9 @+ b+ ? y$ I
- ingresses( h( r$ r7 S# w' M& `: ^- P' p, j
- ingresses/status- Z' R/ ^/ m4 M4 u, {" E% i
- networkpolicies
' e8 y! ]/ D& E2 s$ O O5 n" M verbs:
8 }+ S, f3 _' o+ d. S - get0 q. ^* `. O$ h% D" }% C
- list ]& o% u' J8 p5 F( ^! S1 q5 R
- watch6 ^ Q3 T# d: L. ~9 K0 N6 G
- apiGroups:
) ?# Q" A2 o* _" `+ @ - storage.k8s.io+ W- b8 g) f3 l6 i: {! W
resources:
2 R/ j2 A6 ?# d) a- ` - storageclasses
, C( ^' J7 {# l* X6 J' J0 n - volumeattachments# y, i) Z" i4 x* {( o
verbs:; c) D1 g3 N- y0 b! p6 A( \
- get
+ j5 W8 H$ [/ W# L. j' b7 @. x - list
/ Q; s+ L5 {% z" L# E+ F - watch
; G& ]2 |1 v3 Y7 T8 V- y3 G: _- apiGroups: l! o( _$ g6 c1 m9 B
- rbac.authorization.k8s.io
# X" u' Q1 H. J resources:
/ C$ f( d4 y- }: } - clusterrolebindings9 }/ n, k6 n% h5 g7 z; s7 B& W
- clusterroles
. T% e; i7 E: [+ [* A4 e+ [ - roles
% u C9 x) Z3 c - rolebindings! |- _6 ?9 x0 Y; e5 T/ M
verbs:
9 U3 ~4 S1 T9 ^+ G' f - get- g( k3 ] u/ o- C
- list
' [& G8 o1 c* v. e - watch' P4 x) s8 B0 y/ N+ F
5 h& d8 F$ ?& j
---4 M; l, d: A; S8 [1 s, r
apiVersion: v19 O; o. V& v+ x' _5 L$ ?
kind: ServiceAccount5 M( N/ w8 U( |' _
metadata:5 o6 f" s* Q" }# h0 s
labels:# L2 b% X5 \9 }5 v$ V. G
k8s-app: kubernetes-dashboard
3 A7 g& T# z! }# g K. Y8 o% m name: kubernetes-dashboard
( f% O$ s0 ]* }1 N namespace: kube-system; t# Q8 n4 C9 \& P1 \0 S. y
, h$ ?) i( T! g9 J. }$ M
---
) u1 W8 x# q2 F2 S$ K7 rkind: Service
. n2 U; G+ c2 Q* C, f' M0 q/ g2 OapiVersion: v1
- V- ~4 ~2 ]# x2 S' w K* _metadata:
% |1 E3 m* g( N+ Z: \! s8 T labels:4 y8 W* g+ g# I" {" ~- t+ N; d% {
k8s-app: kubernetes-dashboard
9 I7 R! T9 h0 W4 i7 N7 s4 n kubernetes.io/cluster-service: "true"
% P% T& }) {& a, y4 ~0 @( R1 ? name: kubernetes-dashboard
; a6 {% l) O7 M+ Q4 m) L, Q ]% l namespace: kube-system
$ K( P& d. K0 }& x, \4 ~7 W$ W9 jspec:3 g7 h$ T$ U, {* n
ports:+ h8 T5 c! O- v c5 _% b. r0 P
- port: 443 C$ X% f; A2 `8 Q
targetPort: 84438 P0 L1 B) I g' a
selector:
4 W' n, {( |: b k8s-app: kubernetes-dashboard" y6 | ^" @2 Q+ U8 [( S5 a
type: NodePort
% G: y: k3 A$ N" b5 F, M+ @. W1 W( U) Z
---8 S# t d" a D/ ~9 c
apiVersion: v12 o" R0 Y' c! E' |. S- J
kind: Secret
$ z# H$ Q& q9 X& y8 h4 P2 g6 imetadata:4 m% X4 Y! P0 O& c
labels:
2 o5 L; {* l' j* c4 Y+ F) X k8s-app: kubernetes-dashboard
, G0 ]+ e, u3 z& y( G! f$ L( W name: kubernetes-dashboard-certs
0 @- x/ T0 R. S2 h& v4 d+ w& H namespace: kube-system, C( [5 E" I5 @4 y' ^4 e5 s3 x# L
type: Opaque
2 {; [; }" v- J* j3 G, l9 I! ]7 A/ b" V7 Y
---& c2 s5 H$ l# S6 k: S. g; \8 E
apiVersion: v1
0 {# Y! \2 j9 o, N wkind: Secret
* ?9 w- ?0 |! z- s% z3 P6 Gmetadata:( Z h. `# j4 D! X* o
labels:) A% @: ], `" f+ W& q/ Y0 V6 l" B
k8s-app: kubernetes-dashboard: e' @6 b+ H& c
name: kubernetes-dashboard-csrf
2 a# @! N, n& j9 K6 e namespace: kube-system0 E9 C! T# w# d) W& {% I
type: Opaque
7 |- T0 V5 i1 V& xdata:4 n4 l2 M& D- p% s+ A
csrf: ""' O. C2 m9 |/ x2 ]. ]; c+ L# _
( L' E3 s2 P) C6 `; e---$ x; T4 q4 [( T
apiVersion: v1
" F9 \5 X$ c2 r; l1 \* p/ `kind: Secret0 t8 a& x6 Z4 Q, n1 N. u
metadata:: h6 N( [' R* X+ N, z" N {
labels:/ }, a+ t1 J# {& v9 P8 O
k8s-app: kubernetes-dashboard
$ w7 Q- E/ J$ g8 D7 t N: K) M name: kubernetes-dashboard-key-holder& D, a. E, |# E' i' e j' {* V- y
namespace: kube-system& i# _8 v: V% m
type: Opaque
* H4 D: L! N4 Q, J3 n3 c& R3 }0 Z1 K8 u# q$ f! s) \$ Z
---
" p+ I4 R* P$ X% H* f( Bkind: ConfigMap
5 j, Q+ u1 Z# PapiVersion: v1
& A7 _* w a& y/ r2 l, N9 zmetadata:
8 f8 K% }5 _; v4 A! _8 z |/ d labels:
. o' y( p, @( v/ ]6 x k8s-app: kubernetes-dashboard
5 b6 m' I* D& p, N name: kubernetes-dashboard-settings/ r* q2 {4 v' I- O; U* d5 _
namespace: kube-system* F: T! F( W& J u4 E9 ^9 B
1 Q- m& l& [6 N. I3 F! G; {# v
---
3 v9 F; t# |3 {8 b. ] Mkind: Role% t8 P; r9 ]# I& @, S ^. v- W
apiVersion: rbac.authorization.k8s.io/v1
! K& f$ `. j. X$ _6 J' Dmetadata:
: j7 b2 U+ \$ b0 y labels:
# L! ~: g; } O6 e& \ k8s-app: kubernetes-dashboard0 e$ z( H; s& k! a
name: kubernetes-dashboard
: g7 ]0 o. r8 G& O namespace: kube-system
/ v8 v) w8 M+ x( l( N0 F U0 rrules:
& ?+ x- g- _2 b# ` # Allow Dashboard to get, update and delete Dashboard exclusive secrets.# l3 B) B+ _9 a2 E7 Z
- apiGroups: [""]
4 K* L+ y4 ?( S: {9 T resources: ["secrets"]
( M. t2 f) N5 x, A% S resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"]4 E5 N2 T; u4 ?& o. [- R; ^
verbs: ["get", "update", "delete"]
3 B$ M l. A' \2 q$ [8 G" W # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
+ n! m" b( M# l' T. w" k - apiGroups: [""]1 H6 S/ E3 C) c& d
resources: ["configmaps"]# Y* N% g4 y2 f1 ^: n
resourceNames: ["kubernetes-dashboard-settings"]
9 x6 q5 i# [( @' q verbs: ["get", "update"]
5 n( e- d' L" V4 t5 o # Allow Dashboard to get metrics." f; k1 q) C x: k
- apiGroups: [""] g: ?* T5 C& i6 o
resources: ["services"]: {( T) Z9 B/ g9 ]- j w0 w8 d
resourceNames: ["heapster", "dashboard-metrics-scraper"]& A4 K o& p5 v4 w' F
verbs: ["proxy"]
! M: |8 W0 I6 U! x - apiGroups: [""]* ~% f4 d' V% w
resources: ["services/proxy"]
, u( \) W1 D$ U/ n* [ resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"]* h$ U* w- L% L' L7 a* r- o
verbs: ["get"]
4 p# S) X7 _% A6 h. g/ l9 D8 o) J9 E& V5 b* L' ]/ g
--- z1 P5 l* ^: D& q6 ~* ~
kind: ClusterRole
o1 h, T( r# j: capiVersion: rbac.authorization.k8s.io/v1' I7 T5 h/ H* @4 U7 B7 i' B& Z
metadata:# ^! a" @) N' ]! v6 j: Q
labels:
0 v* q0 h# \' u$ S! N k8s-app: kubernetes-dashboard
; T/ s# a- U: s4 R name: kubernetes-dashboard; G9 F8 W4 I$ z y, V! M/ H/ ^& S
rules:
/ I! F6 N$ Z3 f- ~2 X9 R, R1 l0 I # Allow Metrics Scraper to get metrics from the Metrics server1 N1 l% y: Z8 t
- apiGroups: ["metrics.k8s.io"]
6 @/ y" c A% y- A, a resources: ["pods", "nodes"]
" S8 k, X6 I9 \1 R. u verbs: ["get", "list", "watch"]
/ L% W: x d% c5 T L# ^( k& I! `6 R! V
---
9 c6 [% t7 y7 dapiVersion: rbac.authorization.k8s.io/v1
) |: [4 W9 {: `' `8 @kind: RoleBinding
9 a! J8 y7 {$ [4 Nmetadata:0 |6 A2 Q' J* k4 r& s6 t& z
labels:
# a7 n- G4 b+ F, V; y) f) W3 H k8s-app: kubernetes-dashboard2 J) G' x7 s. [3 T
name: kubernetes-dashboard
2 e1 e" r9 Y- t8 `9 {9 G namespace: kube-system
- `4 b* t* Y# jroleRef:( ^; [) }: h, C. Y
apiGroup: rbac.authorization.k8s.io- y! J& q* G' l" R5 z
kind: Role
. d6 c& I" H# E2 }5 D name: kubernetes-dashboard( E1 w# ~: @' `# Y0 t' `
subjects:
5 M9 Z( D, ^+ }0 a0 O- s i - kind: ServiceAccount
" f8 B7 Z# V x" G3 Y3 ] name: kubernetes-dashboard( H2 s3 p- }! ^* p) ^3 z, N
namespace: kube-system$ _4 X4 Y. [6 c- ?' W# Q }. G2 ~
0 ^) u' Z8 i$ [; C% S7 m
---/ a# ^" l$ h C9 q* `
apiVersion: rbac.authorization.k8s.io/v1
! m" ^4 e- b3 c% W) w$ M9 `kind: ClusterRoleBinding
0 j+ b9 e ^4 p& a: |% A6 p! Xmetadata:
9 q; V/ Z) g+ E8 t6 _# } name: kubernetes-dashboard+ b8 Q) a. g4 p- Z# S+ h u
roleRef:) M8 X3 ^+ P# \, l- A
apiGroup: rbac.authorization.k8s.io. w3 }2 B, i A
kind: ClusterRole
& b# L% r& }7 r name: kubernetes-dashboard
$ e2 q7 {1 }3 Esubjects:
: w" ?( b; x* b, G5 ` - kind: ServiceAccount
' @6 @& v3 k( N- E" C: y! a name: kubernetes-dashboard$ G+ Q8 i& O2 F+ ?6 R
namespace: kube-system) o \* f" }% [1 A6 S$ ?
# S& [+ Q- R* m( N---+ _* {2 [6 \1 D& ~- O2 i- j. M
kind: Deployment4 O- I/ |% W3 l& f8 [: v- D, c
apiVersion: apps/v1, D. |+ f3 A y
metadata:
7 `8 ` p% v4 R$ k; F; f$ O# o6 v labels:
3 |& E- G/ i+ q0 A# R" e8 q; Q k8s-app: kubernetes-dashboard
3 I' I, i U3 ? name: kubernetes-dashboard
/ S+ Z" C- g7 d; e$ v" D' w' ^0 A5 T namespace: kube-system
' o! h8 e3 R' @4 n- O; Mspec:
- W1 I9 ^9 W. j) h; F& i. v' n$ e replicas: 10 u8 P1 t9 r. K
revisionHistoryLimit: 10
1 K2 o( p( |' S/ ~+ p0 S selector:
1 s. R! V4 {! f8 u$ c( L$ H* E matchLabels:
& R6 [8 ^$ z9 I5 a& d$ f k8s-app: kubernetes-dashboard
% m' v2 Z) L$ M) y( N template:0 G0 t: l$ x% n$ b
metadata:
% a7 r% c0 g/ z( G. q& F labels:- d+ o) }7 L! X3 h2 c/ o2 t' S) v
k8s-app: kubernetes-dashboard! S/ F" x' c1 e# `% j z& Q( T
spec: ?* {7 D7 x T
containers:
+ ^# ^0 J7 Z9 j" M5 d e9 s - name: kubernetes-dashboard
1 f2 X( m1 r. t2 {+ |4 i8 u% i image: kubernetesui/dashboard:v2.4.0
7 C ^' n4 {7 ^; \ imagePullPolicy: IfNotPresent
3 Y& W2 p7 s/ w& _1 Z- @: g ports:
# p$ ?, r6 Q. O$ i* X! d - containerPort: 84430 W+ u& y3 E/ J M# d( o* B
protocol: TCP4 }7 P7 s- n2 U2 a
args:
: Z% ^; ^7 _ a; @ - --auto-generate-certificates; j2 k9 Z# R; ~4 V
- --namespace=kube-system
8 n9 t" h: ]9 X0 Q: ^ - --token-ttl=1800
9 [4 f0 F6 o" O! D - --sidecar-host=http://dashboard-metrics-scraper:8000
" | ?" S) p" `% z2 D7 O # Uncomment the following line to manually specify Kubernetes API server Host
* k2 H8 q2 N4 H! W! k # If not specified, Dashboard will attempt to auto discover the API server and connect# E4 P* _8 z3 v, p5 K# p8 a6 J8 E2 @
# to it. Uncomment only if the default does not work.; Q0 y* L1 j" _! ]7 M( I+ a: N1 \, C( T
# - --apiserver-host=http://my-address:port2 ~* j# g! U e/ Y
volumeMounts:4 v3 l h# B( q$ h0 i
- name: kubernetes-dashboard-certs _9 X. A* n" k
mountPath: /certs& ^. x) I5 W! ?7 J/ w) Q
# Create on-disk volume to store exec logs
$ I w0 W: d# G& m - mountPath: /tmp' e- a- E3 ~5 f" B1 j
name: tmp-volume) n, V8 g' o3 W0 \7 y: }& q R. t
livenessProbe:* j! [+ F9 u' O7 [
httpGet:
: u d* `2 ]' \- w3 s* z4 \, ~) x scheme: HTTPS: @8 ~3 e. a! ~" H. o% J. _
path: /7 w, |, T" |' u* Q% M2 y# |
port: 8443
0 y( f' { S; U8 v7 q initialDelaySeconds: 305 `/ a# \# j. \2 |
timeoutSeconds: 30
4 T Y( p; \7 r, t* S' F securityContext:
* V; S7 @* b% E7 J) I2 b' r. k allowPrivilegeEscalation: false
1 `) A" _4 }6 [# r) M) H( o readOnlyRootFilesystem: true, R8 A7 P, p, ~! ~8 q# [
runAsUser: 1001; M; j( i6 o3 d1 K( I- s
runAsGroup: 20017 y- ^6 r' s8 y. T$ |
volumes:
3 Y2 P: u, X$ Z4 V% W - name: kubernetes-dashboard-certs
& b4 n2 M5 C2 F! e& w' A" @ secret:" l; _3 V3 u0 @$ ?
secretName: kubernetes-dashboard-certs
* e5 O% u, f& j7 c% S9 O+ H( E0 i" }: N7 k - name: tmp-volume, o) G6 E3 }4 r' F
emptyDir: {}4 B5 O6 K; L# L) v" L. N# M
serviceAccountName: kubernetes-dashboard
* T8 {2 M% w* h! u3 Y, Z nodeSelector:
) ^; K+ ]3 N. Z8 u# t/ Q "kubernetes.io/os": linux T3 \+ b$ T0 i" n2 u/ }/ r
# Comment the following tolerations if Dashboard must not be deployed on master9 R* T8 {1 `* s; ~9 A
tolerations:/ s$ [% V5 [& W
- key: node-role.kubernetes.io/master" Q& M& c& h/ v" G
effect: NoSchedule! }# y/ p; k% k; o i
% J7 p( t# J$ G---
$ n- ~8 q, U, Vkind: Service
. S$ t9 \1 a6 D- MapiVersion: v1, R* C$ B: q/ J# h8 V) e1 K
metadata:+ z B% j! C, T
labels:
1 e: {/ [3 u/ R6 E% l* i' I k8s-app: dashboard-metrics-scraper1 w7 P! {9 a% b; t+ Z F
name: dashboard-metrics-scraper$ y% c8 }9 \# N, C# b0 c6 w
namespace: kube-system# J6 z5 v, F; R9 `& l8 p
spec:
* E9 E& d4 i7 j ]9 g3 o ports:) E: _- O. _# g$ I0 }6 c
- port: 80006 s4 t( G0 B) K6 K {8 G
targetPort: 8000
' L) h. L5 i' m" _( Q' r selector:4 V0 _9 J' B+ z8 a! k2 a9 X0 M
k8s-app: dashboard-metrics-scraper u/ r5 ]$ t) c
, [" R" P* C$ X( F' Y. O2 ~. K
---
1 F. X( j9 u* n9 h# z; Fkind: Deployment
( ]. G, ?0 O; B$ u: |0 F. TapiVersion: apps/v1
: y: A1 s, }8 b" k/ b1 X: o2 umetadata:3 r) x L; V! t8 w; k9 [3 U
labels:
0 M3 ^8 _+ H, T( T7 s k8s-app: dashboard-metrics-scraper
! @$ ]$ a4 P; o$ h4 j name: dashboard-metrics-scraper* {; D) `5 \" w: X: i- s m
namespace: kube-system
7 ~! D( o: r" j, t; |. Y7 Dspec:
# d" [: c% A6 ` replicas: 1
) o' P. ? O5 V5 D revisionHistoryLimit: 10
" \1 q7 P8 d: `1 I# g' A selector:
$ `% }* H F( y. G% O: O2 } matchLabels:
r8 q3 O6 o) D: S7 ?; l3 \' u k8s-app: dashboard-metrics-scraper" A1 k9 H+ N! ~+ ]
template:
! U; d5 |9 G3 |, {# i metadata:1 V9 K% l5 W# c8 c! u
labels:% }" X" p# C8 ]+ f- o
k8s-app: dashboard-metrics-scraper
# S& x( h7 R+ _/ q spec:
! q' b9 A! J' c1 n* h securityContext:
' r n6 g& x: t; \/ S% `" W/ ^) p' ~ seccompProfile:
# i8 f0 }6 y3 F$ b; [5 I type: RuntimeDefault
) q6 m6 L' s7 t containers:
0 B1 Z) \% }8 w5 R0 d - name: dashboard-metrics-scraper7 c, S5 U! Y6 r2 w6 J# _; u2 S
image: kubernetesui/metrics-scraper:v1.0.7
7 f0 V% r- T i8 n imagePullPolicy: IfNotPresent1 `. w( I# B) \: ?. t( y8 L
ports:7 }2 y- x" u# K0 n) `2 U; T
- containerPort: 8000
d6 r7 p0 @8 T. |5 J6 ^1 m protocol: TCP
: y0 d3 E1 |; j; Y8 n+ v livenessProbe:
8 |" c. U& i3 _0 ? httpGet:
y+ R2 v: X! X6 b' }2 b- a' t3 J" B# x scheme: HTTP$ N; J# y7 M' |
path: /
r/ g2 E0 y7 Q- M& ? port: 8000
I% T+ g) Z$ i. J initialDelaySeconds: 30; H9 y0 z4 u/ d `2 T
timeoutSeconds: 30
4 C. r* a6 [: t* S# X) @ volumeMounts:6 \, b- S9 ]( K$ \1 |" V" O
- mountPath: /tmp
3 J5 M0 u0 _! k N name: tmp-volume
( x: }2 j# }& `& h4 n0 f# `( M9 t securityContext:
- E4 P$ r( X* X0 c- u, u$ X allowPrivilegeEscalation: false
) @" x. {( @2 E2 A readOnlyRootFilesystem: true
+ q+ K2 g$ D( w9 n/ B# h- X' ^ runAsUser: 1001
+ B$ Y8 C p: M; x9 N; a( C# w runAsGroup: 2001
1 J$ R9 }2 i: w4 u! j: x+ m* ~+ O serviceAccountName: kubernetes-dashboard
8 m, A0 u$ p+ h1 T+ D: i nodeSelector:
`& U9 W6 T* I' U c; X "kubernetes.io/os": linux
5 U4 U6 M/ b. h4 c+ v& M0 S) K # Comment the following tolerations if Dashboard must not be deployed on master
' w0 F" w+ @4 D% x% A5 D tolerations:
& c4 A1 P" D' p - key: node-role.kubernetes.io/master
# t4 H0 r. k* q) f effect: NoSchedule
& n& L/ `( ~' c m! ^9 B volumes:
1 ^& k3 A" P7 X: u - name: tmp-volume0 i7 N7 l, V: r1 r! c# k2 L
emptyDir: {}
/ B; x" p1 E3 j* }& X" O8 m导入 dashboard 镜像# v. c& x# W1 ]% q
for i in 192.168.91.19 192.168.91.20;do \8 }+ [! {' Y* o+ S1 J+ F# k
scp /approot1/k8s/images/dashboard-*.tar $i:/tmp/: D/ a$ G' ]+ P* T4 y
ssh $i "ctr -n=k8s.io image import /tmp/dashboard-v2.4.0.tar && rm -f /tmp/dashboard-v2.4.0.tar"; \/ y8 N2 Q8 o+ a, @# F8 d+ X
ssh $i "ctr -n=k8s.io image import /tmp/dashboard-metrics-scraper-v1.0.7.tar && rm -f /tmp/dashboard-metrics-scraper-v1.0.7.tar"; \: ?( g; N Z5 |5 Y
done
4 a" {1 {+ F: X) C2 U L( D查看镜像5 `/ {5 l+ ?& D( L" `- n% ]; P! v
2 ]* f% Q( i0 i o$ }- d! \: p
for i in 192.168.91.19 192.168.91.20;do \& ^- B' t6 r! @" c$ s
ssh $i "ctr -n=k8s.io image list | egrep 'dashboard|metrics-scraper'"; \
$ |% u* G" p9 }6 I/ m& [) F2 ~! Ndone- ^7 `; H0 C. L6 O2 ~
在 k8s 中运行 dashboard 组件
5 ]6 P h$ G+ v5 d0 ]6 M; Jkubectl apply -f /approot1/k8s/tmp/service/dashboard.yaml
8 f2 e. I/ }1 V" h6 o5 r" h检查 dashboard pod 是否运行成功
) }8 A3 n" a1 C6 d2 Tkubectl get pod -n kube-system | grep dashboard
$ F. ]3 s( ^8 ]+ L1 B r预期输出类似如下结果" P" R5 {6 n4 R- }, F
\5 B7 V9 ]* H% r
dashboard-metrics-scraper-799d786dbf-v28pm 1/1 Running 0 2m55s
( ~# g6 u, t- |+ v! ~. R `kubernetes-dashboard-9f8c8b989-rhb7z 1/1 Running 0 2m55s7 i& `& S# M% A1 g5 T
查看 dashboard 访问端口6 [3 A: b. Z- [ ^) C; m. X
在 service 当中没有指定 dashboard 的访问端口，所以需要自己获取，也可以修改 yaml 文件指定访问端口
# E* X; _ ?' U' d. H! d
: z: @+ { f( C5 m6 u4 X$ p4 b8 g预期输出类似如下结果; T( k" P- K( p2 U! y5 y
' v/ |( X; x2 X# S4 w7 r* b# ~
我这边是将 30210 端口映射给 pod 的 443 端口; y$ z% f: f0 @# X x: G
, m4 Z+ Q& K" u) z, P! s& Hkubernetes-dashboard NodePort 10.88.127.68 <none> 443:30210/TCP 5m30s I7 J$ U9 M1 F. T
根据得到的端口访问 dashboard 页面，例如： https://192.168.91.19:30210
4 m& b9 I) U/ k" I& x
3 d" y. k; w& f, R% S; N查看 dashboard 登录 token! m, ?) j( ~9 i% Q& A4 I
获取 token 文件名称3 [( B0 Q% `. h) \
* r- G) R$ H7 K+ D& L1 }
kubectl get secrets -n kube-system | grep admin
9 e/ v7 e* \7 \* A+ L& t9 a& S& g预期输出类似如下结果
) f, c5 i) E: e! o- `! O3 M. A* S3 P4 d- _3 ^
admin-user-token-zvrst kubernetes.io/service-account-token 3 9m2s7 @7 c: a m3 @* j
获取 token 内容0 h& c& O, _& N0 Y" k; ^
/ s& p7 u4 t/ _3 @kubectl get secrets -n kube-system admin-user-token-zvrst -o jsonpath={.data.token}|base64 -d7 x! T# l4 H8 s) v; I- _
预期输出类似如下结果
" N0 z9 e$ @8 N- [! M6 g9 k0 d& `+ r( ]3 [) ^
eyJhbGciOiJSUzI1NiIsImtpZCI6InA4M1lhZVgwNkJtekhUd3Vqdm9vTE1ma1JYQ1ZuZ3c3ZE1WZmJhUXR4bUUifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJhZG1pbi11c2VyLXRva2VuLXp2cnN0Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImFkbWluLXVzZXIiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiJhYTE3NTg1ZC1hM2JiLTQ0YWYtOWNhZS0yNjQ5YzA0YThmZWYiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZS1zeXN0ZW06YWRtaW4tdXNlciJ9.K2o9p5St9tvIbXk7mCQCwsZQV11zICwN-JXhRv1hAnc9KFcAcDOiO4NxIeicvC2H9tHQBIJsREowVwY3yGWHj_MQa57EdBNWMrN1hJ5u-XzpzJ6JbQxns8ZBrCpIR8Fxt468rpTyMyqsO2UBo-oXQ0_ZXKss6X6jjxtGLCQFkz1ZfFTQW3n49L4ENzW40sSj4dnaX-PsmosVOpsKRHa8TPndusAT-58aujcqt31Z77C4M13X_vAdjyDLK9r5ZXwV2ryOdONwJye_VtXXrExBt9FWYtLGCQjKn41pwXqEfidT8cY6xbA7XgUVTr9miAmZ-jf1UeEw-nm8FOw9Bb5v6A
+ R. R/ P3 A# F! l" K6 b" |' Y0 v6 p Q3 Q( m
到此，基于 containerd 二进制部署 k8s v1.23.3 就结束了6 j: L8 q8 w( F( Y
# ~" x: n* D, f |
|
发表于 2024-9-17 10:35:55