云计算运维一步步编译安装Kubernetes之master计算节点安装

部署kube-apiserver集群

主机名角色IP
k8s-node01.boysec.cnkube-apiserver10.1.1.100
k8s-node02.boysec.cnkube-apiserver10.1.1.110
k8s-master.boysec.cn4层负载均衡10.1.1.120
k8s-slave.boysec.cn4层负载均衡10.1.1.130

注意:这里10.1.1.120和10.1.1.130使用nginx做4层负载均衡器,用keepalived跑一个vip:10.1.1.50,代理两个kube-apiserver,实现高可用这里部署文档以k8s-node01.boysec.cn主机为例,另外一台运算节点安装部署方法类似

下载安装kube-apiserver

Kubernetes下载地址

1
2
3
4
5
cd /server/tools/ 
tar xf kubernetes-server-linux-amd64.tar.gz -C /opt
mv /opt/kubernetes /opt/kubernetes-v1.17.9
ln -s /opt/kubernetes-v1.17.9 /opt/kubernetes
mkdir /opt/kubernetes/server/bin/{cert,conf} -p

签发证书

在k8s-dns.boysec.cn上

vim /opt/certs/client-csr.json

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
{
"CN": "k8s-node",
"hosts": [
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "beijing",
"L": "beijing",
"O": "od",
"OU": "ops"
}
]
}

生成client证书和私钥

1
2
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client-csr.json | cfssljson -bare client
ls client*

签发kube-apiserver证书

vim /opt/certs/apiserver-csr.json

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
{
"CN": "apiserver",
"hosts": [
"127.0.0.1",
"192.168.0.1",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local",
"10.1.1.50",
"10.1.1.100",
"10.1.1.110",
"10.1.1.120",
"10.1.1.130"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "beijing",
"L": "beijing",
"O": "od",
"OU": "ops"
}
]
}

生成apiserver证书和私钥

1
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server apiserver-csr.json |cfssljson -bare apiserver

拷贝证书至各运算节点,并创建配置

k8s-node01.boysec.cn上:

1
mkdir /opt/kubernetes/server/bin/{cert,conf} -p  

拷贝证书、私钥,注意私钥文件属性600

1
[root@k8s-dns certs]# scp apiserver-key.pem apiserver.pem ca-key.pem ca.pem client-key.pem client.pem k8s-node01:/opt/kubernetes/server/bin/cert

检查

1
2
3
4
5
6
7
8
[root@k8s-node01 ]# ls -l /opt/kubernetes/server/bin/cert 
total 16
-rw------- 1 root root 1679 Aug 4 22:09 apiserver-key.pem
-rw-r--r-- 1 root root 1606 Aug 4 22:09 apiserver.pem
-rw------- 1 root root 1679 Aug 4 22:11 ca-key.pem
-rw-r--r-- 1 root root 1354 Aug 4 22:11 ca.pem
-rw------- 1 root root 1675 Aug 4 22:09 client-key.pem
-rw-r--r-- 1 root root 1371 Aug 4 22:09 client.pem

创建配置

vim /opt/kubernetes/server/bin/conf/audit.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
apiVersion: audit.k8s.io/v1beta1 # This is required.
kind: Policy
# Don't generate audit events for all requests in RequestReceived stage.
omitStages:
- "RequestReceived"
rules:
# Log pod changes at RequestResponse level
- level: RequestResponse
resources:
- group: ""
# Resource "pods" doesn't match requests to any subresource of pods,
# which is consistent with the RBAC policy.
resources: ["pods"]
# Log "pods/log", "pods/status" at Metadata level
- level: Metadata
resources:
- group: ""
resources: ["pods/log", "pods/status"]

# Don't log requests to a configmap called "controller-leader"
- level: None
resources:
- group: ""
resources: ["configmaps"]
resourceNames: ["controller-leader"]

# Don't log watch requests by the "system:kube-proxy" on endpoints or services
- level: None
users: ["system:kube-proxy"]
verbs: ["watch"]
resources:
- group: "" # core API group
resources: ["endpoints", "services"]

# Don't log authenticated requests to certain non-resource URL paths.
- level: None
userGroups: ["system:authenticated"]
nonResourceURLs:
- "/api*" # Wildcard matching.
- "/version"

# Log the request body of configmap changes in kube-system.
- level: Request
resources:
- group: "" # core API group
resources: ["configmaps"]
# This rule only applies to resources in the "kube-system" namespace.
# The empty string "" can be used to select non-namespaced resources.
namespaces: ["kube-system"]

# Log configmap and secret changes in all other namespaces at the Metadata level.
- level: Metadata
resources:
- group: "" # core API group
resources: ["secrets", "configmaps"]

# Log all other resources in core and extensions at the Request level.
- level: Request
resources:
- group: "" # core API group
- group: "extensions" # Version of group should NOT be included.

# A catch-all rule to log all other requests at the Metadata level.
- level: Metadata
# Long-running requests like watches that fall under this rule will not
# generate an audit event in RequestReceived.
omitStages:
- "RequestReceived"

创建启动脚本

k8s-node01.boysec.cn上:

vim /opt/kubernetes/server/bin/kube-apiserver.sh

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
#!/bin/bash
./kube-apiserver \
--apiserver-count 2 \
--audit-log-path /data/logs/kubernetes/kube-apiserver/audit-log \
--audit-policy-file ./conf/audit.yaml \
--authorization-mode RBAC \
--client-ca-file ./cert/ca.pem \
--requestheader-client-ca-file ./cert/ca.pem \
--enable-admission-plugins NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota \
--etcd-cafile ./cert/ca.pem \
--etcd-certfile ./cert/client.pem \
--etcd-keyfile ./cert/client-key.pem \
--etcd-servers https://10.1.1.100:2379,https://10.1.1.110:2379,https://10.1.1.130:2379 \
--service-account-key-file ./cert/ca-key.pem \
--service-cluster-ip-range 192.168.0.0/16 \
--service-node-port-range 3000-29999 \
--target-ram-mb=1024 \
--kubelet-client-certificate ./cert/client.pem \
--kubelet-client-key ./cert/client-key.pem \
--log-dir /data/logs/kubernetes/kube-apiserver \
--tls-cert-file ./cert/apiserver.pem \
--tls-private-key-file ./cert/apiserver-key.pem \
--v 2

调整权限和目录

1
2
chmod +x /opt/kubernetes/server/bin/kube-apiserver.sh
mkdir -p /data/logs/kubernetes/kube-apiserver

创建supervisor配置

vim /etc/supervisord.d/kube-apiserver.ini

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
[program:kube-apiserver-100]
command=/opt/kubernetes/server/bin/kube-apiserver.sh ; the program (relative uses PATH, can take args)
numprocs=1 ; number of processes copies to start (def 1)
directory=/opt/kubernetes/server/bin ; directory to cwd to before exec (def no cwd)
autostart=true ; start at supervisord start (default: true)
autorestart=true ; retstart at unexpected quit (default: true)
startsecs=22 ; number of secs prog must stay running (def. 1)
startretries=3 ; max # of serial start failures (default 3)
exitcodes=0,2 ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT ; signal used to kill process (default TERM)
stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10)
user=root ; setuid to this UNIX account to run the program
redirect_stderr=false ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/kubernetes/kube-apiserver/apiserver.stdout.log ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4 ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false ; emit events on stdout writes (default false)
stderr_logfile=/data/logs/kubernetes/kube-apiserver/apiserver.stderr.log ; stderr log path, NONE for none; default AUTO
stderr_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stderr_logfile_backups=4 ; # of stderr logfile backups (default 10)
stderr_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stderr_events_enabled=false ; emit events on stderr writes (default false)

启动服务并检查

1
2
3
supervisorctl update
supervisorctl status
netstat -lnpt|grep 6443

配4层反向代理

k8s-master.boysec.cn,k8s-slave.boysec.cn上:

vim /etc/nginx/nginx.conf

1
2
3
4
5
6
7
8
9
10
11
12
stream {
upstream kube-apiserver {
server 10.1.1.100:6443 max_fails=3 fail_timeout=30s;
server 10.1.1.110:6443 max_fails=3 fail_timeout=30s;
}
server {
listen 7443;
proxy_connect_timeout 2s;
proxy_timeout 900s;
proxy_pass kube-apiserver;
}
}

2021-08-13报错

1
2
3
4
5
[root@k8s-master ~]# nginx -t
nginx: [emerg] unknown directive "stream" in /etc/nginx/nginx.conf:77
nginx: configuration file /etc/nginx/nginx.conf test failed
## 解决
yum -y install nginx-mod-stream

keepalived配置

vim /etc/keepalived/check_port.sh

1
2
3
4
5
6
7
8
9
10
11
12
13
#!/bin/bash
nginxpid=$(ps -C nginx --no-header|wc -l)
#1.判断 Nginx 是否存活,如果不存活则尝试启动 Nginx
if [ $nginxpid -eq 0 ];then
systemctl start nginx
sleep 3
#2.等待 3 秒后再次获取一次 Nginx 状态
nginxpid=$(ps -C nginx --no-header|wc -l)
#3.再次进行判断, 如 Nginx 还不存活则停止 Keepalived,让地址进行漂移,并退出脚本
if [ $nginxpid -eq 0 ];then
systemctl stop keepalived
fi
fi

vim /etc/keepalived/keepalived.conf

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
! Configuration File for keepalived

global_defs {
router_id 10.1.1.120

}

vrrp_script chk_nginx {
script "/etc/keepalived/check_port.sh"
interval 2
weight -20
}

vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 251
priority 100
advert_int 1
mcast_src_ip 10.1.1.120
nopreempt # 非抢占式

authentication {
auth_type PASS
auth_pass 11111111
}
track_script {
chk_nginx
}
virtual_ipaddress {
10.1.1.50
}
}

vim /etc/keepalived/keepalived.conf

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
! Configuration File for keepalived
global_defs {
router_id 10.1.1.130
}
vrrp_script chk_nginx {
script "/etc/keepalived/check_port.sh"
interval 2
weight -20
}
vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 251
mcast_src_ip 10.1.1.130
nopreempt
priority 90
advert_int 1
authentication {
auth_type PASS
auth_pass 11111111
}
track_script {
chk_nginx
}
virtual_ipaddress {
10.1.1.50
}
}
nopreempt注解

VRRP will normally preempt a lower priority machine when a higher priority machine comes online. “nopreempt” allows the lower priority machine to maintain the master role, even when a higher priority machine comes back online.

NOTE: For this to work, the initial state of thisentry must be BACKUP.

当优先级较高的机器联机时,VRRP通常会抢占优先级较低的机器。“noprempt”允许低优先级机器保持主角色,即使高优先级机器重新联机。

注意:要使其工作,此条目的初始状态必须为BACKUP。

启动代理并检查

1
2
3
4
5
6
7
systemctl start keepalived
systemctl enable keepalived
systemctl start nginx
systemctl enable nginx

netstat -lnpt|grep 7443
ip addr|grep 10.1.1.50

部署controller-manager

集群规划

主机名角色ip
k8s-node01.boysec.cncontroller-manager10.1.1.100
k8s-node02.boysec.cncontroller-manager10.1.1.110

注意:这里部署文档以k8s-node01.boysec.cn主机为例,另外一台运算节点安装部署方法类似

创建启动脚本

vim /opt/kubernetes/server/bin/kube-controller-manager.sh

1
2
3
4
5
6
7
8
9
10
#!/bin/sh
./kube-controller-manager \
--cluster-cidr 172.7.0.0/16 \
--leader-elect true \
--log-dir /data/logs/kubernetes/kube-controller-manager \
--master http://127.0.0.1:8080 \
--service-account-private-key-file ./cert/ca-key.pem \
--service-cluster-ip-range 192.168.0.0/16 \
--root-ca-file ./cert/ca.pem \
--v 2

调整文件权限,创建目录

1
2
chmod +x /opt/kubernetes/server/bin/kube-controller-manager.sh
mkdir -p /data/logs/kubernetes/kube-controller-manager

创建supervisor配置

vim /etc/supervisord.d/kube-conntroller-manager.ini

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
[program:kube-controller-manager-100]
command=/opt/kubernetes/server/bin/kube-controller-manager.sh ; the program (relative uses PATH, can take args)
numprocs=1 ; number of processes copies to start (def 1)
directory=/opt/kubernetes/server/bin ; directory to cwd to before exec (def no cwd)
autostart=true ; start at supervisord start (default: true)
autorestart=true ; retstart at unexpected quit (default: true)
startsecs=22 ; number of secs prog must stay running (def. 1)
startretries=3 ; max # of serial start failures (default 3)
exitcodes=0,2 ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT ; signal used to kill process (default TERM)
stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10)
user=root ; setuid to this UNIX account to run the program
redirect_stderr=false ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/kubernetes/kube-controller-manager/controll.stdout.log ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4 ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false ; emit events on stdout writes (default false)
stderr_logfile=/data/logs/kubernetes/kube-controller-manager/controll.stderr.log ; stderr log path, NONE for none; default AUTO
stderr_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stderr_logfile_backups=4 ; # of stderr logfile backups (default 10)
stderr_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stderr_events_enabled=false ; emit events on stderr writes (default false)

启动服务并检查

1
2
supervisorctl update
supervisorctl status

部署kube-scheduler

集群规划

主机名角色ip
k8s-node01.boysec.cnkube-scheduler10.1.1.100
k8s-node02.boysec.cnkube-scheduler10.1.1.110

注意:这里部署文档以k8s-node01.boysec.cn主机为例,另外一台运算节点安装部署方法类似

创建启动脚本

vim /opt/kubernetes/server/bin/kube-scheduler.sh

1
2
3
4
5
6
#!/bin/sh
./kube-scheduler \
--leader-elect \
--log-dir /data/logs/kubernetes/kube-scheduler \
--master http://127.0.0.1:8080 \
--v 2

调整文件权限,创建目录

1
2
chmod +x /opt/kubernetes/server/bin/kube-scheduler.sh
mkdir -p /data/logs/kubernetes/kube-scheduler

创建supervisor配置

vim /etc/supervisord.d/kube-scheduler.ini

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
[program:kube-scheduler-100]
command=/opt/kubernetes/server/bin/kube-scheduler.sh ; the program (relative uses PATH, can take args)
numprocs=1 ; number of processes copies to start (def 1)
directory=/opt/kubernetes/server/bin ; directory to cwd to before exec (def no cwd)
autostart=true ; start at supervisord start (default: true)
autorestart=true ; retstart at unexpected quit (default: true)
startsecs=22 ; number of secs prog must stay running (def. 1)
startretries=3 ; max # of serial start failures (default 3)
exitcodes=0,2 ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT ; signal used to kill process (default TERM)
stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10)
user=root ; setuid to this UNIX account to run the program
redirect_stderr=false ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/kubernetes/kube-scheduler/scheduler.stdout.log ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4 ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false ; emit events on stdout writes (default false)
stderr_logfile=/data/logs/kubernetes/kube-scheduler/scheduler.stderr.log ; stderr log path, NONE for none; default AUTO
stderr_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stderr_logfile_backups=4 ; # of stderr logfile backups (default 10)
stderr_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stderr_events_enabled=false ; emit events on stderr writes (default false)

启动服务并检查

1
2
3
4
5
supervisorctl update
supervisorctl status

ln -s /opt/kubernetes/server/bin/kubectl /usr/bin/kubectl
kubectl get cs