SoulChild随笔记
SoulChild随笔记

kubernetes 1.20.7 二进制安装-apiserver+nginx+keepalived(五)

SoulChild SoulChild 收录于 系统服务  kubernetes
   约 1861 字  预计阅读 9 分钟   - 次阅读  
警告
本文最后更新于 2021-06-30 17:32，文中内容可能已过时。

一、安装apiserver

1.下载server包

1
2
3
4
5

cd /server/packages/
wget https://dl.k8s.io/v1.20.7/kubernetes-server-linux-amd64.tar.gz

# 分发到其他机器
for i in {202..203};do scp kubernetes-server-linux-amd64.tar.gz 172.17.20.$i:`pwd` ;done

2.安装apiserver

1
2
3

# 三台机器执行
tar xf kubernetes-server-linux-amd64.tar.gz
cp kubernetes/server/bin/kube-apiserver /usr/local/bin

3.创建审计日志策略配置文件

None - 符合这条规则的日志将不会记录

Metadata - 记录请求的元数据（请求的用户、时间戳、资源、动词等等）， 但是不记录请求或者响应的消息体。

Request - 记录事件的元数据和请求的消息体，但是不记录响应的消息体。 这不适用于非资源类型的请求。

RequestResponse - 记录事件的元数据，请求和响应的消息体。这不适用于非资源类型的请求。

审计配置的api文档:https://kubernetes.io/zh/docs/reference/config-api/apiserver-audit.v1/#resource-types

下面配置来源kubernetes code

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158

cat > /etc/kubernetes/audit-policy.yaml <<EOF
apiVersion: audit.k8s.io/v1
kind: Policy
rules:
  # 不记录一些高容量和低风险的日志.
  - level: None
    users: ["system:kube-proxy"]
    verbs: ["watch"]
    resources:
      - group: "" # core
        resources: ["endpoints", "services", "services/status"]
  - level: None
    # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port.
    # TODO(#46983): Change this to the ingress controller service account.
    users: ["system:unsecured"]
    namespaces: ["kube-system"]
    verbs: ["get"]
    resources:
      - group: "" # core
        resources: ["configmaps"]
  - level: None
    users: ["kubelet"] # legacy kubelet identity
    verbs: ["get"]
    resources:
      - group: "" # core
        resources: ["nodes", "nodes/status"]
  - level: None
    userGroups: ["system:nodes"]
    verbs: ["get"]
    resources:
      - group: "" # core
        resources: ["nodes", "nodes/status"]
  - level: None
    users:
      - system:kube-controller-manager
      - system:kube-scheduler
      - system:serviceaccount:kube-system:endpoint-controller
    verbs: ["get", "update"]
    namespaces: ["kube-system"]
    resources:
      - group: "" # core
        resources: ["endpoints"]
  - level: None
    users: ["system:apiserver"]
    verbs: ["get"]
    resources:
      - group: "" # core
        resources: ["namespaces", "namespaces/status", "namespaces/finalize"]
  - level: None
    users: ["cluster-autoscaler"]
    verbs: ["get", "update"]
    namespaces: ["kube-system"]
    resources:
      - group: "" # core
        resources: ["configmaps", "endpoints"]
  # Don't log HPA fetching metrics.
  - level: None
    users:
      - system:kube-controller-manager
    verbs: ["get", "list"]
    resources:
      - group: "metrics.k8s.io"
  # Don't log these read-only URLs.
  - level: None
    nonResourceURLs:
      - /healthz*
      - /version
      - /swagger*
  # Don't log events requests because of performance impact.
  - level: None
    resources:
      - group: "" # core
        resources: ["events"]
  # node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes
  - level: Request
    users: ["kubelet", "system:node-problem-detector", "system:serviceaccount:kube-system:node-problem-detector"]
    verbs: ["update","patch"]
    resources:
      - group: "" # core
        resources: ["nodes/status", "pods/status"]
    omitStages:
      - "RequestReceived"
  - level: Request
    userGroups: ["system:nodes"]
    verbs: ["update","patch"]
    resources:
      - group: "" # core
        resources: ["nodes/status", "pods/status"]
    omitStages:
      - "RequestReceived"
  # deletecollection calls can be large, don't log responses for expected namespace deletions
  - level: Request
    users: ["system:serviceaccount:kube-system:namespace-controller"]
    verbs: ["deletecollection"]
    omitStages:
      - "RequestReceived"
  # Secrets, ConfigMaps, TokenRequest and TokenReviews can contain sensitive & binary data,
  # so only log at the Metadata level.
  - level: Metadata
    resources:
      - group: "" # core
        resources: ["secrets", "configmaps", "serviceaccounts/token"]
      - group: authentication.k8s.io
        resources: ["tokenreviews"]
    omitStages:
      - "RequestReceived"
  # Get responses can be large; skip them.
  - level: Request
    verbs: ["get", "list", "watch"]
    resources:
      - group: "" # core
      - group: "admissionregistration.k8s.io"
      - group: "apiextensions.k8s.io"
      - group: "apiregistration.k8s.io"
      - group: "apps"
      - group: "authentication.k8s.io"
      - group: "authorization.k8s.io"
      - group: "autoscaling"
      - group: "batch"
      - group: "certificates.k8s.io"
      - group: "extensions"
      - group: "metrics.k8s.io"
      - group: "networking.k8s.io"
      - group: "node.k8s.io"
      - group: "policy"
      - group: "rbac.authorization.k8s.io"
      - group: "scheduling.k8s.io"
      - group: "storage.k8s.io"
    omitStages:
      - "RequestReceived"
  # Default level for known APIs
  - level: RequestResponse
    resources:
      - group: "" # core
      - group: "admissionregistration.k8s.io"
      - group: "apiextensions.k8s.io"
      - group: "apiregistration.k8s.io"
      - group: "apps"
      - group: "authentication.k8s.io"
      - group: "authorization.k8s.io"
      - group: "autoscaling"
      - group: "batch"
      - group: "certificates.k8s.io"
      - group: "extensions"
      - group: "metrics.k8s.io"
      - group: "networking.k8s.io"
      - group: "node.k8s.io"
      - group: "policy"
      - group: "rbac.authorization.k8s.io"
      - group: "scheduling.k8s.io"
      - group: "storage.k8s.io"
    omitStages:
      - "RequestReceived"
  # Default level for all other requests.
  - level: Metadata
    omitStages:
      - "RequestReceived"
EOF

分发审计策略配置到其他机器

1

for i in {202..203};do scp /etc/kubernetes/audit-policy.yaml 172.17.20.$i:/etc/kubernetes/ ;done

4.配置静态加密Secret数据

官方文档: https://kubernetes.io/zh/docs/tasks/administer-cluster/encrypt-data/

默认情况下secret的数据存储在etcd上是没有加密的,为了相对安全一些我们可以配置静态secret数据加密,将存在etcd中的secret数据进行加密。使用静态加密需要给apiserver添加--encryption-provider-config参数,并指定一个配置文件。

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

cat > /etc/kubernetes/static-secret-encryption.yaml <<EOF
apiVersion: apiserver.config.k8s.io/v1
kind: EncryptionConfiguration
resources:
  - resources:
    - secrets
    providers:
    - aescbc:
        keys:
        - name: key1
          secret: {{Secret}}
    - identity: {}
EOF

# 生成随机secret
sed -i "s#{{Secret}}#`head -c 32 /dev/urandom | base64`#" /etc/kubernetes/static-secret-encryption.yaml

# 分发配置到其他机器
for i in {202..203};do scp /etc/kubernetes/static-secret-encryption.yaml 172.17.20.$i:/etc/kubernetes/ ;done

5.创建令牌认证文件

kubelet在启动时会向apiserver发送给csr请求，这里我们创建一个权限有限的token,用于kubelet向apiserver申请证书。

1
2

# token,user,uid,"group1,group2,group3"
echo "$(head -c 16 /dev/urandom | od -An -t x | tr -d ' '),kubelet-bootstrap,10001,"system:bootstrappers"" > /etc/kubernetes/token.csv

6.创建systemd启动脚本

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

cat > /etc/systemd/system/kube-apiserver.service <<EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-apiserver \\
  --advertise-address=172.17.20.201 \\
  --allow-privileged=true \\
  --apiserver-count=3 \\
  --audit-log-maxage=15 \\
  --audit-log-maxsize=100 \\
  --audit-log-maxbackup=5 \\
  --audit-log-path=/var/log/kube-apiserver-audit.log \\
  --audit-policy-file=/etc/kubernetes/audit-policy.yaml \\
  --authorization-mode=Node,RBAC \\
  --bind-address=172.17.20.201 \\
  --client-ca-file=/etc/kubernetes/pki/ca/ca.pem \\
  --delete-collection-workers=2 \\
  --enable-admission-plugins=NodeRestriction \\
  --enable-bootstrap-token-auth \\
  --token-auth-file=/etc/kubernetes/token.csv \\
  --encryption-provider-config=/etc/kubernetes/static-secret-encryption.yaml \\
  --etcd-cafile=/etc/kubernetes/pki/ca/ca.pem \\
  --etcd-certfile=/etc/kubernetes/pki/etcd/etcd.pem \\
  --etcd-keyfile=/etc/kubernetes/pki/etcd/etcd-key.pem \\
  --etcd-servers=https://172.17.20.201:2379,https://172.17.20.202:2379,https://172.17.20.203:2379 \\
  --event-ttl=72h \\
  --feature-gates=EphemeralContainers=true \\
  --kubelet-certificate-authority=/etc/kubernetes/pki/ca/ca.pem \\
  --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem \\
  --kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem \\
  --logtostderr=false \\
  --log-file=/var/log/kube-apiserver.log \\
  --log-file-max-size=1 \\
  --proxy-client-cert-file=/etc/kubernetes/pki/apiserver.pem \\
  --proxy-client-key-file=/etc/kubernetes/pki/apiserver-key.pem \\
  --requestheader-client-ca-file=/etc/kubernetes/pki/ca/ca.pem \\
  --requestheader-allowed-names="aggregator" \\
  --requestheader-username-headers="X-Remote-User" \\
  --requestheader-group-headers="X-Remote-Group" \\
  --requestheader-extra-headers-prefix="X-Remote-Extra-" \\
  --runtime-config=api/all=true \\
  --secure-port=6443 \\
  --service-account-issuer=https://kubernetes.default.svc.cluster.local \\
  --service-account-key-file=/etc/kubernetes/pki/apiserver.pem  \\
  --service-account-signing-key-file=/etc/kubernetes/pki/apiserver-key.pem \\
  --service-cluster-ip-range=10.1.0.0/16 \\
  --service-node-port-range=30000-32767 \\
  --tls-cert-file=/etc/kubernetes/pki/apiserver.pem \\
  --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem

Restart=on-failure
RestartSec=10
Type=notify
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

参数说明:

--advertise-address: 向集群成员通知apiserver的IP地址

--allow-privileged: 设置为true，允许特权容器

--apiserver-count: 集群中运行的apiserver数量

--audit-log-maxage: 保留旧审计日志文件的最大天数

--audit-log-maxbackup: 保留的旧审计日志文件的最大数量

--audit-log-maxsize: 审计日志文件轮转大小(单个日志文件的最大大小)单位MB

--audit-log-path: 指定用来写入审计事件的日志文件路径。不指定此标志会禁用日志- 表示stdout

--audit-policy-file: 审计策略的配置文件路径

--authorization-mode: 在安全端口上进行鉴权的插件的顺序列表。逗号分隔的列表：AlwaysAllow,AlwaysDeny,ABAC,Webhook,Node,RBAC

--bind-address: 安全端口的监听IP地址。

--client-ca-file: 指定签发客户端证书的根CA机构证书,apiserver会使用此CA证书校验证书合法性，并且对客户端证书中的CommonName对身份进行校验

--delete-collection-workers: 调用DeleteCollection的worker数量。 这可以用于加速namespace清理。

--enable-admission-plugins: 指定要启用除了默认插件以外的哪些插件。

  • 默认已启用的插件:NamespaceLifecycle、LimitRanger、ServiceAccount、TaintNodesByCondition、Priority、DefaultTolerationSeconds、DefaultStorageClass、StorageObjectInUseProtection、PersistentVolumeClaimResize、RuntimeClass、CertificateApproval、CertificateSigning、CertificateSubjectRestriction、DefaultIngressClass、MutatingAdmissionWebhook、ValidatingAdmissionWebhook、ResourceQuota

  • 所有可选插件: AlwaysAdmit、AlwaysDeny、AlwaysPullImages、CertificateApproval、CertificateSigning、CertificateSubjectRestriction、DefaultIngressClass、DefaultStorageClass、DefaultTolerationSeconds、DenyEscalatingExec、DenyExecOnPrivileged、EventRateLimit、ExtendedResourceToleration、ImagePolicyWebhook、LimitPodHardAntiAffinityTopology、LimitRanger、MutatingAdmissionWebhook、NamespaceAutoProvision、NamespaceExists、NamespaceLifecycle、NodeRestriction、OwnerReferencesPermissionEnforcement、PersistentVolumeClaimResize、PersistentVolumeLabel、PodNodeSelector、PodSecurityPolicy、PodTolerationRestriction、Priority、ResourceQuota、RuntimeClass、SecurityContextDeny、ServiceAccount、StorageObjectInUseProtection、TaintNodesByCondition、ValidatingAdmissionWebhook

--enable-bootstrap-token-auth: 设置是否允许将kube-system命名空间中，类型为bootstrap.kubernetes.io/token的Secret用于TLS引导身份验证。

--encryption-provider-config: 静态加密secret的配置文件

--etcd-cafile: 用于和etcd通信的证书的颁发机构的证书(即颁发etcd客户端证书的CA机构证书)

--etcd-certfile: 用于和etcd通信的证书

--etcd-healthcheck-timeout: 检查etcd健康状况时的超时时长。默认2s

--etcd-keyfile: 用户和etcd通信的证书的配套私钥

--etcd-servers: etcd服务器的列表,格式为scheme://ip:port,多个地址使用逗号分隔

--etcd-servers-overrides: 将不同的资源存入不同的etcd集群。格式为group/resource#servers。例如将event事件单独存储: /events#http://etcd4:2379,http://etcd5:2379,http://etcd6:2379

--event-ttl: 事件的保留时长默认1h0m0s

--feature-gates: 控制试验性功能。多个条目用逗号分隔，可选项: https://v1-20.docs.kubernetes.io/zh/docs/reference/command-line-tools-reference/kube-apiserver/

--kubelet-certificate-authority: 校验kubelet服务端的证书(因为apiserver也会请求kubelet，比如logs,exec等操作)，这里指定kubelet服务端证书的ca机构证书

--kubelet-client-certificate: apiserver访问kubelet使用的客户端证书

--kubelet-client-key: apiserver访问kubelet使用的客户端证书的私钥

--logtostderr: 默认true，输出日志到stderr，而不是输出到文件

--log-file: 日志文件路径

--log-file-max-size: 日志文件最大大小,默认1800M

--profiling: 启用web性能分析

--contention-profiling: 当web性能分析启用时, 设置是否启用锁竞争分析功能

--proxy-client-cert-file: apiserver访问其他服务使用的客户端证书，用于访问扩展apiserver服务器使用。

--proxy-client-key-file: apiserver的客户端证书私钥

--requestheader-client-ca-file: 签发apiserver客户端证书的CA证书(kube-apiserver访问扩展apiserver的客户端证书的CA证书)

--requestheader-allowed-names: 这个选项用来指定一个CN(Common Name)列表。客户端证书的CN是我们指定的CN列表之一才可以建立连接。如果没有指定CN列表,只要是--requestheader-client-ca-file签发的客户端证书都可以通过apiserver认证。

--requestheader-username-headers: apiserver将客户端请求的用户名发给扩展apiserver是通过请求头发送的。这里是设置请求头的名称。推荐使用(X-Remote-User)，前提是和扩展apiserver对应。

--requestheader-group-headers: 同上，这个是用户组，推荐使用(X-Remote-Group)

--requestheader-extra-headers-prefix: 扩展信息前缀。推荐使用(X-Remote-Extra-)

--runtime-config: 用于启用或禁用内置api。可选项如下

  • v1=true|false 启用或禁用核心API组
  • /=true|false 启用或禁用特定API组和版本 (例如 apps/v1=true)
  • api/all=true|false 启用或禁用所有API版本
  • api/ga=true|false 启用或禁用v[0-9]+形式的所有API版本
  • api/beta=true|false 启用或禁用v[0-9]+beta[0-9]+形式的所有API版本
  • api/alpha=true|false 启用或禁用v[0-9]+alpha[0-9]+形式的所有API版本

--secure-port: 监听的安全端口(https)默认6443

--service-account-issuer: 当使用sa token 卷投射(Service Account Token Volume Projection)时，这里指的是jwt token颁发者的标识符(通过在线jwt解析的网站可以看到iss字段就是我们设置的值).取值必须是 HTTPS URL。可以设置成(https://kubernetes.default.svc.cluster.local)。

--service-account-key-file: 指定的是私钥对应的公钥，用于在身份认证过程中确保ServiceAccount token没有被篡改

--service-account-signing-key-file: 当使用sa token 卷投射(Service Account Token Volume Projection)时,这里指定的是对token签名的私钥

关于sa token 卷投射: https://kubernetes.io/zh/docs/tasks/configure-pod-container/configure-service-account/#service-account-token-volume-projection

--service-cluster-ip-range: cluster ip范围。

--service-node-port-range: nodeport的端口范围

--tls-cert-file: apiserver的服务端证书

--tls-private-key-file: apiserver的服务端证书私钥

其他两台的配置 apiserver02

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

cat > /etc/systemd/system/kube-apiserver.service <<EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-apiserver \\
  --advertise-address=172.17.20.202 \\
  --allow-privileged=true \\
  --apiserver-count=3 \\
  --audit-log-maxage=15 \\
  --audit-log-maxsize=100 \\
  --audit-log-maxbackup=5 \\
  --audit-log-path=/var/log/kube-apiserver-audit.log \\
  --audit-policy-file=/etc/kubernetes/audit-policy.yaml \\
  --authorization-mode=Node,RBAC \\
  --bind-address=172.17.20.202 \\
  --client-ca-file=/etc/kubernetes/pki/ca/ca.pem \\
  --delete-collection-workers=2 \\
  --enable-admission-plugins=NodeRestriction \\
  --enable-bootstrap-token-auth \\
  --encryption-provider-config=/etc/kubernetes/static-secret-encryption.yaml \\
  --etcd-cafile=/etc/kubernetes/pki/ca/ca.pem \\
  --etcd-certfile=/etc/kubernetes/pki/etcd/etcd.pem \\
  --etcd-keyfile=/etc/kubernetes/pki/etcd/etcd-key.pem \\
  --etcd-servers=https://172.17.20.201:2379,https://172.17.20.202:2379,https://172.17.20.203:2379 \\
  --event-ttl=72h \\
  --feature-gates=EphemeralContainers=true \\
  --kubelet-certificate-authority=/etc/kubernetes/pki/ca/ca.pem \\
  --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem \\
  --kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem \\
  --logtostderr=false \\
  --log-file=/var/log/kube-apiserver.log \\
  --log-file-max-size=1 \\
  --proxy-client-cert-file=/etc/kubernetes/pki/apiserver.pem \\
  --proxy-client-key-file=/etc/kubernetes/pki/apiserver-key.pem \\
  --requestheader-client-ca-file=/etc/kubernetes/pki/ca/ca.pem \\
  --requestheader-allowed-names="aggregator" \\
  --requestheader-username-headers="X-Remote-User" \\
  --requestheader-group-headers="X-Remote-Group" \\
  --requestheader-extra-headers-prefix="X-Remote-Extra-" \\
  --runtime-config=api/all=true \\
  --secure-port=6443 \\
  --service-account-issuer=https://kubernetes.default.svc.cluster.local \\
  --service-account-key-file=/etc/kubernetes/pki/apiserver.pem  \\
  --service-account-signing-key-file=/etc/kubernetes/pki/apiserver-key.pem \\
  --service-cluster-ip-range=10.1.0.0/16 \\
  --service-node-port-range=30000-32767 \\
  --tls-cert-file=/etc/kubernetes/pki/apiserver.pem \\
  --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem

Restart=on-failure
RestartSec=10
Type=notify
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

apiserver03

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

cat > /etc/systemd/system/kube-apiserver.service <<EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-apiserver \\
  --advertise-address=172.17.20.203 \\
  --allow-privileged=true \\
  --apiserver-count=3 \\
  --audit-log-maxage=15 \\
  --audit-log-maxsize=100 \\
  --audit-log-maxbackup=5 \\
  --audit-log-path=/var/log/kube-apiserver-audit.log \\
  --audit-policy-file=/etc/kubernetes/audit-policy.yaml \\
  --authorization-mode=Node,RBAC \\
  --bind-address=172.17.20.203 \\
  --client-ca-file=/etc/kubernetes/pki/ca/ca.pem \\
  --delete-collection-workers=2 \\
  --enable-admission-plugins=NodeRestriction \\
  --enable-bootstrap-token-auth \\
  --encryption-provider-config=/etc/kubernetes/static-secret-encryption.yaml \\
  --etcd-cafile=/etc/kubernetes/pki/ca/ca.pem \\
  --etcd-certfile=/etc/kubernetes/pki/etcd/etcd.pem \\
  --etcd-keyfile=/etc/kubernetes/pki/etcd/etcd-key.pem \\
  --etcd-servers=https://172.17.20.201:2379,https://172.17.20.202:2379,https://172.17.20.203:2379 \\
  --event-ttl=72h \\
  --feature-gates=EphemeralContainers=true \\
  --kubelet-certificate-authority=/etc/kubernetes/pki/ca/ca.pem \\
  --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem \\
  --kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem \\
  --logtostderr=false \\
  --log-file=/var/log/kube-apiserver.log \\
  --log-file-max-size=1 \\
  --proxy-client-cert-file=/etc/kubernetes/pki/apiserver.pem \\
  --proxy-client-key-file=/etc/kubernetes/pki/apiserver-key.pem \\
  --requestheader-client-ca-file=/etc/kubernetes/pki/ca/ca.pem \\
  --requestheader-allowed-names="aggregator" \\
  --requestheader-username-headers="X-Remote-User" \\
  --requestheader-group-headers="X-Remote-Group" \\
  --requestheader-extra-headers-prefix="X-Remote-Extra-" \\
  --runtime-config=api/all=true \\
  --secure-port=6443 \\
  --service-account-issuer=https://kubernetes.default.svc.cluster.local \\
  --service-account-key-file=/etc/kubernetes/pki/apiserver.pem  \\
  --service-account-signing-key-file=/etc/kubernetes/pki/apiserver-key.pem \\
  --service-cluster-ip-range=10.1.0.0/16 \\
  --service-node-port-range=30000-32767 \\
  --tls-cert-file=/etc/kubernetes/pki/apiserver.pem \\
  --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem

Restart=on-failure
RestartSec=10
Type=notify
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

7.启动服务

1
2

systemctl start kube-apiserver
systemctl enable kube-apiserver

二、安装配置nginx

1. 安装

1
2
3
4
5
6

mkdir /server/packages/ -p
cd /server/packages/
wget http://nginx.org/download/nginx-1.16.1.tar.gz
tar xf nginx-1.16.1.tar.gz
cd nginx-1.16.1/
./configure --prefix=/usr/local/nginx --with-stream --without-http --without-http_uwsgi_module

编译参数说明:

--with-stream: 开启tcp/udp代理

--without-http: 禁用http服务器

--without-http_uwsgi_module: 禁用uwsgi传输功能

2.配置

两台nginx相同

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

cat > /usr/local/nginx/conf/nginx.conf <<EOF
worker_processes  auto;

error_log  logs/error.log;
pid        logs/nginx.pid;

events {
    worker_connections  1024;
}

stream {
    log_format  main  '$remote_addr [$time_local] '
                      '$protocol $status $bytes_sent $bytes_received '
                      '$session_time "$upstream_addr" '
                      '"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"';

    access_log  logs/access.log  main;

    upstream kube-apiserver {
        hash $remote_addr consistent;
        server 172.17.20.201:6443 max_fails=3 fail_timeout=10s;
        server 172.17.20.202:6443 max_fails=3 fail_timeout=10s;
        server 172.17.20.203:6443 max_fails=3 fail_timeout=10s;
    }
    server {
        listen      6443;
        proxy_connect_timeout 1s;
        proxy_pass kube-apiserver;

    }
}
EOF

3.启动服务

1

/usr/local/nginx/sbin/nginx

三、安装keepalived

1. 安装

1

yum install -y keepalived

2.配置

nginx01的keepalived

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

cat > /etc/keepalived/keepalived.conf <<EOF
global_defs {
   router_id nginx01
}

# 定义一个状态检查,script中也可以写一个脚本，但脚本需有返回值
vrrp_script check_nginx {
    # 每2秒检查一次nginx进程状态，根据命令执行的状态码去判断服务是否正常
    script "/usr/bin/killall -0 nginx"
    interval 2
    # 2次状态吗为非0才为失败状态
    fall 2
    # 2次状态码为0才为正常状态
    rise 2
}

vrrp_instance nginx {
    # 设置为master
    state MASTER
    # 指定vip绑定网卡
    interface eth0
    # vrrp标识1-255(需要和备节点一致)
    virtual_router_id 51
    # 指定优先级，优先级高的会优先成为master
    priority 100
    # 组播包间隔时间
    advert_int 1
    # 认证
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    # 配置vip
    virtual_ipaddress {
        172.17.20.200
    }

    # 引用上面定义的状态检查
    track_script {
        check_nginx
    }
}
EOF

nginx02的keepalived

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

cat > /etc/keepalived/keepalived.conf <<EOF
global_defs {
   router_id nginx02
}

# 定义一个状态检查,script中也可以写一个脚本，但脚本需有返回值
vrrp_script check_nginx {
    # 每2秒检查一次nginx进程状态，根据命令执行的状态码去判断服务是否正常
    script "/usr/bin/killall -0 nginx"
    interval 2
    # 2次状态吗为非0才为失败状态
    fall 2
    # 2次状态码为0才为正常状态
    rise 2
}

vrrp_instance nginx {
    # 设置为master
    state BACKUP
    # 指定vip绑定网卡
    interface eth0
    # vrrp标识1-255(需要和备节点一致)
    virtual_router_id 51
    # 指定优先级，优先级高的会优先成为master
    priority 99
    # 组播包间隔时间
    advert_int 1
    # 认证
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    # 配置vip
    virtual_ipaddress {
        172.17.20.200
    }

    # 引用上面定义的状态检查
    track_script {
        check_nginx
    }
}
EOF

3.启动服务

1
2

systemctl start keepalived
systemctl enable keepalived

4.验证vip漂移

停止nginx01的keepalived,观察vip是否会漂移到nginx02上 启动nginx01的keepalived,观察vip是否会漂移到nginx01上(当前配置是抢占模式，nginx01恢复后会抢占VIP)

更新于 2021-06-30 17:32 
本站使用「署名 4.0 国际」创作共享协议，可自由转载、引用，但需署名作者且注明文章出处
nginxkeepalievdk8s
返回 | 主页
请我喝杯水
SoulChild 微信号 微信号
SoulChild 微信打赏 微信打赏
0%