K8S 集群部署 ¶
kubernetes 所有 master 节点配置 ¶
在所有k8s master节点上。
1. 创建相关目录
mkdir -p /etc/kubernetes/
mkdir -p /etc/kubernetes/ssl
mkdir -p /var/log/kubernetes
kube apiserver 部署 ¶
apiserver 证书请求文件 ¶
mkdir kube-apiserver
1. 创建 文件
cat > kube-apiserver/kube-apiserver-csr.json << "EOF"
{
"CN": "kubernetes",
"hosts": [
"127.0.0.1",
"192.168.3.40",
"192.168.3.41",
"192.168.3.42",
"192.168.3.43",
"192.168.3.44",
"192.168.3.45",
"192.168.3.46",
"192.168.3.47",
"192.168.3.48",
"192.168.3.49",
"192.168.3.50",
"192.168.3.51",
"192.168.3.52",
"192.168.3.53",
"192.168.3.54",
"192.168.3.55",
"192.168.3.56",
"192.168.3.57",
"192.168.3.58",
"192.168.3.59",
"192.168.3.60",
"192.168.3.61",
"192.168.3.62",
"192.168.3.63",
"192.168.3.64",
"192.168.3.65",
"192.168.3.66",
"192.168.3.67",
"192.168.3.68",
"192.168.3.69",
"192.168.3.70",
"192.168.3.200",
"10.96.0.1",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "kubemsb",
"OU": "CN"
}
]
}
EOF
2. 生成 apiserver 证书
cfssl gencert -ca=ca/ca.pem -ca-key=ca/ca-key.pem -config=ca/ca-config.json -profile=kubernetes kube-apiserver/kube-apiserver-csr.json | cfssljson -bare kube-apiserver/kube-apiserver
创建 TLS 机制所需 TOKEN ¶
TLS Bootstraping：Master apiserver启用TLS认证后，Node节点kubelet和kube-proxy与kube-apiserver进行通信，必须使用CA签发的有效证书才可以，当Node节点很多时，这种客户端证书颁发需要大量工作，同样也会增加集群扩展复杂度。为了简化流程，Kubernetes引入了TLS bootstraping机制来自动颁发客户端证书，kubelet会以一个低权限用户自动向apiserver申请证书，kubelet的证书由apiserver动态签署。所以强烈建议在Node上使用这种方式，目前主要用于kubelet，kube-proxy还是由我们统一颁发一个证书。
1. 创建 token 文件
cat > kube-apiserver/token.csv << EOF
$(head -c 16 /dev/urandom | od -An -t x | tr -d ' '),kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF
创建 apiserver 服务配置文件 ¶
cat > kube-apiserver/k8s-master01-kube-apiserver.conf << "EOF"
KUBE_APISERVER_OPTS="--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \
--anonymous-auth=false \
--bind-address=192.168.3.50 \
--secure-port=6443 \
--advertise-address=192.168.3.50 \
--authorization-mode=Node,RBAC \
--runtime-config=api/all=true \
--enable-bootstrap-token-auth \
--service-cluster-ip-range=10.96.0.0/16 \
--token-auth-file=/etc/kubernetes/token.csv \
--service-node-port-range=30000-32767 \
--tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem \
--tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem \
--client-ca-file=/etc/kubernetes/ssl/ca.pem \
--kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem \
--kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem \
--service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \
--service-account-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \
--service-account-issuer=api \
--etcd-cafile=/etc/etcd/ssl/ca.pem \
--etcd-certfile=/etc/etcd/ssl/etcd.pem \
--etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \
--etcd-servers=https://192.168.3.40:2379,https://192.168.3.41:2379,https://192.168.3.42:2379 \
--allow-privileged=true \
--apiserver-count=3 \
--audit-log-maxage=30 \
--audit-log-maxbackup=3 \
--audit-log-maxsize=100 \
--audit-log-path=/var/log/kube-apiserver-audit.log \
--event-ttl=1h \
--v=4"
EOF
cat > kube-apiserver/k8s-master02-kube-apiserver.conf << "EOF"
KUBE_APISERVER_OPTS="--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \
--anonymous-auth=false \
--bind-address=192.168.3.51 \
--secure-port=6443 \
--advertise-address=192.168.3.51 \
--authorization-mode=Node,RBAC \
--runtime-config=api/all=true \
--enable-bootstrap-token-auth \
--service-cluster-ip-range=10.96.0.0/16 \
--token-auth-file=/etc/kubernetes/token.csv \
--service-node-port-range=30000-32767 \
--tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem \
--tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem \
--client-ca-file=/etc/kubernetes/ssl/ca.pem \
--kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem \
--kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem \
--service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \
--service-account-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \
--service-account-issuer=api \
--etcd-cafile=/etc/etcd/ssl/ca.pem \
--etcd-certfile=/etc/etcd/ssl/etcd.pem \
--etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \
--etcd-servers=https://192.168.3.40:2379,https://192.168.3.41:2379,https://192.168.3.42:2379 \
--allow-privileged=true \
--apiserver-count=3 \
--audit-log-maxage=30 \
--audit-log-maxbackup=3 \
--audit-log-maxsize=100 \
--audit-log-path=/var/log/kube-apiserver-audit.log \
--event-ttl=1h \
--v=4"
EOF
cat > kube-apiserver/k8s-master03-kube-apiserver.conf << "EOF"
KUBE_APISERVER_OPTS="--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \
--anonymous-auth=false \
--bind-address=192.168.3.52 \
--secure-port=6443 \
--advertise-address=192.168.3.52 \
--authorization-mode=Node,RBAC \
--runtime-config=api/all=true \
--enable-bootstrap-token-auth \
--service-cluster-ip-range=10.96.0.0/16 \
--token-auth-file=/etc/kubernetes/token.csv \
--service-node-port-range=30000-32767 \
--tls-cert-file=/etc/kubernetes/ssl/kube-apiserver.pem \
--tls-private-key-file=/etc/kubernetes/ssl/kube-apiserver-key.pem \
--client-ca-file=/etc/kubernetes/ssl/ca.pem \
--kubelet-client-certificate=/etc/kubernetes/ssl/kube-apiserver.pem \
--kubelet-client-key=/etc/kubernetes/ssl/kube-apiserver-key.pem \
--service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \
--service-account-signing-key-file=/etc/kubernetes/ssl/ca-key.pem \
--service-account-issuer=api \
--etcd-cafile=/etc/etcd/ssl/ca.pem \
--etcd-certfile=/etc/etcd/ssl/etcd.pem \
--etcd-keyfile=/etc/etcd/ssl/etcd-key.pem \
--etcd-servers=https://192.168.3.40:2379,https://192.168.3.41:2379,https://192.168.3.42:2379 \
--allow-privileged=true \
--apiserver-count=3 \
--audit-log-maxage=30 \
--audit-log-maxbackup=3 \
--audit-log-maxsize=100 \
--audit-log-path=/var/log/kube-apiserver-audit.log \
--event-ttl=1h \
--v=4"
EOF
创建 apiserver 服务管理配置文件 ¶
三个 master 节点均执行
cat > kube-apiserver/kube-apiserver.service << "EOF"
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=etcd.service
Wants=etcd.service
[Service]
EnvironmentFile=-/etc/kubernetes/kube-apiserver.conf
ExecStart=/usr/local/bin/kube-apiserver $KUBE_APISERVER_OPTS
Restart=on-failure
RestartSec=5
Type=notify
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
分发证书及 token 文件到各 master 节点 ¶
1. 从 ca 发送至各 master 节点
#!/bin/bash
TARGET_HOSTS=("k8s-master01" "k8s-master02" "k8s-master03")
DIRECTORY=kube-apiserver/
CA_DIRECTORY=ca/
# 通过循环执行 scp 命令
for host in "${TARGET_HOSTS[@]}"; do
echo "正在复制文件到 $host..."
# 复制证书和密钥文件到 /etc/kubernetes/ssl/ 目录
scp ${DIRECTORY}kube-apiserver*.pem $host:/etc/kubernetes/ssl/
# 复制 ca 到 /etc/kubernetes/ssl 目录
scp ${CA_DIRECTORY}ca*.pem $host:/etc/kubernetes/ssl/
# 复制 systemd 服务文件到 /usr/lib/systemd/system/ 目录
scp ${DIRECTORY}kube-apiserver.service $host:/usr/lib/systemd/system/
# 复制 token.csv
scp ${DIRECTORY}token.csv $host:/etc/kubernetes/
# 复制对应主机的配置文件到 /etc/kubernetes/ 目录
case $host in
"k8s-master01")
scp ${DIRECTORY}k8s-master01-kube-apiserver.conf $host:/etc/kubernetes/kube-apiserver.conf
;;
"k8s-master02")
scp ${DIRECTORY}k8s-master02-kube-apiserver.conf $host:/etc/kubernetes/kube-apiserver.conf
;;
"k8s-master03")
scp ${DIRECTORY}k8s-master03-kube-apiserver.conf $host:/etc/kubernetes/kube-apiserver.conf
;;
*)
echo "未识别的主机名: $host"
;;
esac
done
# 在目标主机上执行 systemctl 命令
for host in "${TARGET_HOSTS[@]}"; do
echo "在 $host 上执行 systemctl 命令..."
ssh -t $host 'sudo systemctl daemon-reload'
ssh -t $host 'sudo systemctl enable --now kube-apiserver'
ssh -t $host 'sudo systemctl status kube-apiserver'
done
启动 apiserver 服务 ¶
所有节点均执行
1. 启动 api-server
systemctl daemon-reload
systemctl enable --now kube-apiserver
systemctl status kube-apiserver
2. 验证 apiserver 访问
curl --insecure https://192.168.3.50:6443/
curl --insecure https://192.168.3.51:6443/
curl --insecure https://192.168.3.52:6443/