## 创建APIServer
```
docker run -it --name kube-apiserver -d --restart=always \
--network=host \
-v /opt/ssl/etcd/etcd-ca.pem:/opt/ssl/etcd/etcd-ca.pem \
-v /opt/ssl/etcd/etcd_client.pem:/opt/ssl/etcd/etcd_client.pem \
-v /opt/ssl/etcd/etcd_client-key.pem:/opt/ssl/etcd/etcd_client-key.pem \
-v /opt/ssl/k8s/k8s-ca.pem:/opt/ssl/k8s/k8s-ca.pem \
-v /opt/ssl/k8s/k8s_server.pem:/opt/ssl/k8s/k8s_server.pem \
-v /opt/ssl/k8s/k8s_server-key.pem:/opt/ssl/k8s/k8s_server-key.pem \
-v /opt/ssl/k8s/k8s-front-proxy-ca.pem:/opt/ssl/k8s/k8s-front-proxy-ca.pem \
-v /opt/ssl/k8s/k8s_front_proxy_client.pem:/opt/ssl/k8s/k8s_front_proxy_client.pem \
-v /opt/ssl/k8s/k8s_front_proxy_client-key.pem:/opt/ssl/k8s/k8s_front_proxy_client-key.pem \
-v /opt/ssl/k8s/sa.pub:/opt/ssl/k8s/sa.pub \
-v /var/lib/lxcfs/proc/cpuinfo:/proc/cpuinfo:rw \
-v /var/lib/lxcfs/proc/diskstats:/proc/diskstats:rw \
-v /var/lib/lxcfs/proc/meminfo:/proc/meminfo:rw \
-v /var/lib/lxcfs/proc/stat:/proc/stat:rw \
-v /var/lib/lxcfs/proc/swaps:/proc/swaps:rw \
-v /var/lib/lxcfs/proc/uptime:/proc/uptime:rw \
-m 2048m \
-v /var/log/kubernetes/:/var/log/kubernetes/ \
--entrypoint="/usr/local/bin/kube-apiserver" \
gcr.io/google_containers/kube-apiserver:v1.12.1 \
--bind-address="0.0.0.0" \
--advertise-address="10.10.10.230" \
--secure-port=5443 \
--insecure-port=0 \
--service-cluster-ip-range="10.253.0.0/16" \
--service-node-port-range="60000-65000" \
--etcd-cafile="/opt/ssl/etcd/etcd-ca.pem" \
--etcd-certfile="/opt/ssl/etcd/etcd_client.pem" \
--etcd-keyfile="/opt/ssl/etcd/etcd_client-key.pem" \
--etcd-prefix="/registry" \
--etcd-servers="https://10.10.10.231:2379,https://10.10.10.232:2379,https://10.10.10.233:2379" \
--client-ca-file="/opt/ssl/k8s/k8s-ca.pem" \
--tls-cert-file="/opt/ssl/k8s/k8s_server.pem" \
--tls-private-key-file="/opt/ssl/k8s/k8s_server-key.pem" \
--kubelet-client-certificate="/opt/ssl/k8s/k8s_server.pem" \
--kubelet-client-key="/opt/ssl/k8s/k8s_server-key.pem" \
--service-account-key-file="/opt/ssl/k8s/sa.pub" \
--requestheader-client-ca-file=/opt/ssl/k8s/k8s-front-proxy-ca.pem \
--proxy-client-cert-file=/opt/ssl/k8s/k8s_front_proxy_client.pem \
--proxy-client-key-file=/opt/ssl/k8s/k8s_front_proxy_client-key.pem \
--requestheader-allowed-names=front-proxy-client \
--requestheader-group-headers=X-Remote-Group \
--requestheader-extra-headers-prefix=X-Remote-Extra- \
--requestheader-username-headers=X-Remote-User \
--cors-allowed-origins=".*" \
--enable-swagger-ui \
--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \
--authorization-mode="Node,RBAC" \
--runtime-config="settings.k8s.io/v1alpha1=true" \
--enable-admission-plugins="AlwaysPullImages,DefaultStorageClass,DefaultTolerationSeconds,Initializers,LimitRanger,NamespaceExists,NamespaceLifecycle,NodeRestriction,OwnerReferencesPermissionEnforcement,PodNodeSelector,PersistentVolumeClaimResize,PodPreset,PodTolerationRestriction,ResourceQuota,SecurityContextDeny,ServiceAccount,StorageObjectInUseProtection" \
--disable-admission-plugins="DenyEscalatingExec,ExtendedResourceToleration,ImagePolicyWebhook,LimitPodHardAntiAffinityTopology,MutatingAdmissionWebhook,NamespaceAutoProvision,Priority,ValidatingAdmissionWebhook,EventRateLimit,PodSecurityPolicy" \
--allow-privileged \
--kubelet-https \
--enable-bootstrap-token-auth=true \
--audit-log-path="/var/log/kubernetes/api-server-audit.log" \
--alsologtostderr \
--logtostderr \
--log-dir="/var/log/kubernetes/"
```
## 创建 HAProxy
```
# 编辑配置文件
mkdir -p /opt/k8s/haproxy
vi /opt/k8s/haproxy/haproxy.cfg
global
log 127.0.0.1 local0
log 127.0.0.1 local1 notice
tune.ssl.default-dh-param 2048
defaults
log global
mode http
option dontlognull
timeout connect 5000ms
timeout client 1800000ms
timeout server 1800000ms
listen stats
bind :9090
mode http
balance
stats uri /haproxy_stats
stats auth admin:admin123
stats admin if TRUE
frontend kube-apiserver-https
mode tcp
bind :6443
default_backend kube-apiserver-backend
backend kube-apiserver-backend
mode tcp
server 10.10.10.231-api 10.10.10.231:5443 check
server 10.10.10.232-api 10.10.10.232:5443 check
server 10.10.10.233-api 10.10.10.233:5443 check
# 运行 Haproxy
docker run -it --name ha-proxy -d --restart=always \
--network=host \
-v /opt/k8s/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg \
-v /var/lib/lxcfs/proc/cpuinfo:/proc/cpuinfo:rw \
-v /var/lib/lxcfs/proc/diskstats:/proc/diskstats:rw \
-v /var/lib/lxcfs/proc/meminfo:/proc/meminfo:rw \
-v /var/lib/lxcfs/proc/stat:/proc/stat:rw \
-v /var/lib/lxcfs/proc/swaps:/proc/swaps:rw \
-v /var/lib/lxcfs/proc/uptime:/proc/uptime:rw \
-m 2048m \
docker.io/haproxy:1.7-alpine
# 运行 Keepalived
docker run -it --name keepalived -d --restart=always \
--privileged --network=host \
--cap-add=NET_ADMIN \
-e KEEPALIVED_VIRTUAL_IPS=10.10.10.230 \
-e KEEPALIVED_INTERFACE=ens192 \
-e KEEPALIVED_UNICAST_PEERS="#PYTHON2BASH:['10.10.10.231', '10.10.10.232', '10.10.10.233']" \
-e KEEPALIVED_PASSWORD=d0cker \
-e KEEPALIVED_PRIORITY=150 \
-e KEEPALIVED_ROUTER_ID=51 \
docker.io/osixia/keepalived:1.4.5
```
