云计算运维kubernetes集群里集成Apollo配置中心

Apollo介绍

Apollo（阿波罗）是携程框架部门研发的分布式配置中心，能够集中化管理应用不同环境、不同集群的配置，配置修改后能够实时推送到应用端，并且具备规范的权限、流程治理等特性，适用于微服务配置管理场景。

Apollo安装部署

Apollo官方地址

官方release包

基础架构

简化模型

交付apollo-configservice

准备软件包

在k8s-dns.boysec.cn上：

下载官方release包

1
2
cd /tools/
mkdir /data/dockerfile/apollo-configservice && unzip -o apollo-configservice-1.7.1-github.zip -d /data/dockerfile/apollo-configservice

执行数据库脚本

注意：MySQL版本应为5.6或以上！

  • 更新yum源
1
2
3
4
5
6
vi /etc/yum.repos.d/MariaDB.repo
[mariadb]
name = MariaDB
baseurl = https://mirrors.ustc.edu.cn/mariadb/yum/10.2/centos7-amd64/
gpgkey=https://mirrors.ustc.edu.cn/mariadb/yum/RPM-GPG-KEY-MariaDB
gpgcheck=1
  • 导入GPG-KEY
1
rpm --import https://mirrors.ustc.edu.cn/mariadb/yum/RPM-GPG-KEY-MariaDB
  • 更新数据库版本
1
yum install MariaDB-server -y
  • 配置my.cnf
1
2
3
4
5
6
7
8
9
vim /etc/my.cnf.d/mysql-clients.cnf
[mysql]
default-character-set = utf8mb4

vim /etc/my.cnf.d/server.cnf
[mysqld]
character_set_server = utf8mb4
collation_server = utf8mb4_general_ci
init_connect = "SET NAMES 'utf8mb4'"

数据库脚本地址

1
2
mysql -uroot -p
mysql> source ./apolloconfigdb.sql

数据库用户授权

1
MariaDB [(none)]> grant INSERT,DELETE,UPDATE,SELECT on TestBetaApolloConfigDB.* to "apolloconfig"@"10.1.1.%" identified by "123456";

修改初始数据

1
update TestBetaApolloConfigDB.ServerConfig set ServerConfig.Value="http://config-test.od.com/eureka" where ServerConfig.Key="eureka.service.url";

制作Docker镜像

配置数据库连接串

1
2
3
4
5
[root@k8s-dns apollo-configservice]# cat config/application-github.properties 
# DataSource
spring.datasource.url = jdbc:mysql://mysql.od.com:3306/TestBetaApolloConfigDB?characterEncoding=utf8
spring.datasource.username = apolloconfig
spring.datasource.password = 123456

更新startup.sh

官方脚本

vim scripts/startup.sh

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
#!/bin/bash
SERVICE_NAME=apollo-configservice
## Adjust log dir if necessary
LOG_DIR=/opt/logs/apollo-config-server
## Adjust server port if necessary
SERVER_PORT=8080
APOLLO_CONFIG_SERVICE_NAME=$(hostname -i)
SERVER_URL="http://${APOLLO_CONFIG_SERVICE_NAME}:${SERVER_PORT}"

## Adjust memory settings if necessary
#export JAVA_OPTS="-Xms6144m -Xmx6144m -Xss256k -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=384m -XX:NewSize=4096m -XX:MaxNewSize=4096m -XX:SurvivorRatio=8"

## Only uncomment the following when you are using server jvm
#export JAVA_OPTS="$JAVA_OPTS -server -XX:-ReduceInitialCardMarks"

########### The following is the same for configservice, adminservice, portal ###########
export JAVA_OPTS="$JAVA_OPTS -XX:ParallelGCThreads=4 -XX:MaxTenuringThreshold=9 -XX:+DisableExplicitGC -XX:+ScavengeBeforeFullGC -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+ExplicitGCInvokesConcurrent -XX:+HeapDumpOnOutOfMemoryError -XX:-OmitStackTraceInFastThrow -Duser.timezone=Asia/Shanghai -Dclient.encoding.override=UTF-8 -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom"
export JAVA_OPTS="$JAVA_OPTS -Dserver.port=$SERVER_PORT -Dlogging.file=$LOG_DIR/$SERVICE_NAME.log -XX:HeapDumpPath=$LOG_DIR/HeapDumpOnOutOfMemoryError/"

# Find Java
if [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]]; then
    javaexe="$JAVA_HOME/bin/java"
elif type -p java > /dev/null 2>&1; then
    javaexe=$(type -p java)
elif [[ -x "/usr/bin/java" ]];  then
    javaexe="/usr/bin/java"
else
    echo "Unable to find Java"
    exit 1
fi

if [[ "$javaexe" ]]; then
    version=$("$javaexe" -version 2>&1 | awk -F '"' '/version/ {print $2}')
    version=$(echo "$version" | awk -F. '{printf("%03d%03d",$1,$2);}')
    # now version is of format 009003 (9.3.x)
    if [ $version -ge 011000 ]; then
        JAVA_OPTS="$JAVA_OPTS -Xlog:gc*:$LOG_DIR/gc.log:time,level,tags -Xlog:safepoint -Xlog:gc+heap=trace"
    elif [ $version -ge 010000 ]; then
        JAVA_OPTS="$JAVA_OPTS -Xlog:gc*:$LOG_DIR/gc.log:time,level,tags -Xlog:safepoint -Xlog:gc+heap=trace"
    elif [ $version -ge 009000 ]; then
        JAVA_OPTS="$JAVA_OPTS -Xlog:gc*:$LOG_DIR/gc.log:time,level,tags -Xlog:safepoint -Xlog:gc+heap=trace"
    else
        JAVA_OPTS="$JAVA_OPTS -XX:+UseParNewGC"
        JAVA_OPTS="$JAVA_OPTS -Xloggc:$LOG_DIR/gc.log -XX:+PrintGCDetails"
        JAVA_OPTS="$JAVA_OPTS -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=60 -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:CMSFullGCsBeforeCompaction=9 -XX:+CMSClassUnloadingEnabled  -XX:+PrintGCDateStamps -XX:+PrintGCApplicationConcurrentTime -XX:+PrintHeapAtGC -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=5M"
    fi
fi

printf "$(date) ==== Starting ==== \n"

cd `dirname $0`/..
chmod 755 $SERVICE_NAME".jar"
./$SERVICE_NAME".jar" start

rc=$?;

if [[ $rc != 0 ]];
then
    echo "$(date) Failed to start $SERVICE_NAME.jar, return code: $rc"
    exit $rc;
fi

tail -f /dev/null

编写Dockerfile

1
2
3
4
5
6
7
8
9
10
11
12
FROM wangxiansen/jre8:8u112

ENV VERSION 1.7.1

RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime &&\
    echo "Asia/Shanghai" > /etc/timezone

ADD apollo-configservice-${VERSION}.jar /apollo-configservice/apollo-configservice.jar
ADD config/ /apollo-configservice/config
ADD scripts/ /apollo-configservice/scripts

CMD ["/apollo-configservice/scripts/startup.sh"]

制作镜像并推送

1
2
docker build . -t harbor.od.com/infra/apollo-configservice:v1.7.1
docker push  harbor.od.com/infra/apollo-configservice:v1.7.1

解析域名

1
2
3
vi /var/named/chroot/etc/od.com.zone  
mysql   A    10.1.1.250
config-test A 10.1.1.50

准备资源配置清单

1
mkdir /var/k8s-yaml/apollo-configservice && cd /var/k8s-yaml/apollo-configservice

vim /var/k8s-yaml/apollo-configservice/deployment.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
kind: Deployment
apiVersion: apps/v1
metadata:
  name: apollo-configservice
  namespace: test
  labels: 
    name: apollo-configservice
spec:
  replicas: 1
  selector:
    matchLabels: 
      name: apollo-configservice
  template:
    metadata:
      labels: 
        app: apollo-configservice 
        name: apollo-configservice
    spec:
      volumes:
      - name: configmap-volume
        configMap:
          name: apollo-configservice-cm
      containers:
      - name: apollo-configservice
        image: harbor.od.com/infra/apollo-configservice:v1.7.1
        ports:
        - containerPort: 8080
          protocol: TCP
        volumeMounts:
        - name: configmap-volume
          mountPath: /apollo-configservice/config
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        imagePullPolicy: IfNotPresent
      imagePullSecrets:
      - name: harbor
      restartPolicy: Always
      terminationGracePeriodSeconds: 30
      securityContext: 
        runAsUser: 0
      schedulerName: default-scheduler
  strategy:
    type: RollingUpdate
    rollingUpdate: 
      maxUnavailable: 1
      maxSurge: 1
  revisionHistoryLimit: 7
  progressDeadlineSeconds: 600

vim /var/k8s-yaml/apollo-configservice/svc.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
kind: Service
apiVersion: v1
metadata: 
  name: apollo-configservice
  namespace: test
spec:
  ports:
  - protocol: TCP
    port: 8080
    targetPort: 8080
  selector: 
    app: apollo-configservice
  clusterIP: None
  type: ClusterIP
  sessionAffinity: None

vim /var/k8s-yaml/apollo-configservice/ingress.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
kind: Ingress
apiVersion: extensions/v1beta1
metadata: 
  name: apollo-configservice
  namespace: test
spec:
  rules:
  - host: config-test.od.com
    http:
      paths:
      - path: /
        backend: 
          serviceName: apollo-configservice
          servicePort: 8080

vim /var/k8s-yaml/apollo-configservice/configmap.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
apiVersion: v1
kind: ConfigMap
metadata:
  name: apollo-configservice-cm
  namespace: test
data:
  application-github.properties: |
    # DataSource
    spring.datasource.url = jdbc:mysql://mysql.od.com:3306/TestBetaApolloConfigDB?characterEncoding=utf8
    spring.datasource.username = apolloconfig
    spring.datasource.password = 123456
    eureka.service.url = http://config-test.od.com/eureka
  app.properties: |
    appId=100003171

应用资源配置清单

在任意一台k8s运算节点执行：

1
2
3
4
5
6
kubectl create ns test
kubectl create  secret docker-registry harbor --docker-server=harbor.od.com --docker-username=admin --docker-password=Harbor12345 -n test
kubectl apply -f http://k8s-yaml.od.com/apollo-configservice/deployment.yaml
kubectl apply -f http://k8s-yaml.od.com/apollo-configservice/svc.yaml
kubectl apply -f http://k8s-yaml.od.com/apollo-configservice/ingress.yaml
kubectl apply -f http://k8s-yaml.od.com/apollo-configservice/configmap.yaml

浏览器访问

http://config-test.od.com

交付apollo-adminservice

准备软件包

在k8s-dns.boysec.cn上：

下载官方release包

1
[root@k8s-dns tools]# mkdir /data/dockerfile/apollo-adminservice && unzip -o apollo-adminservice-1.7.1-github.zip -d /data/dockerfile/apollo-adminservice

制作Docker镜像

配置数据库连接串

1
2
3
4
5
[root@k8s-dns apollo-adminservice]# cat config/application-github.properties 
# DataSource
spring.datasource.url = jdbc:mysql://mysql.od.com:3306/TestBetaApolloConfigDB?characterEncoding=utf8
spring.datasource.username = apolloconfig
spring.datasource.password = 123456

更新starup.sh

vi scripts/startup.sh

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
#!/bin/bash
SERVICE_NAME=apollo-adminservice
## Adjust log dir if necessary
LOG_DIR=/opt/logs/apollo-adminservice
## Adjust server port if necessary
SERVER_PORT=8080
APOLLO_ADMIN_SERVICE_NAME=$(hostname -i)
# SERVER_URL="http://localhost:${SERVER_PORT}"
SERVER_URL="http://${APOLLO_ADMIN_SERVICE_NAME}:${SERVER_PORT}"

## Adjust memory settings if necessary
#export JAVA_OPTS="-Xms2560m -Xmx2560m -Xss256k -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=384m -XX:NewSize=1536m -XX:MaxNewSize=1536m -XX:SurvivorRatio=8"

## Only uncomment the following when you are using server jvm
#export JAVA_OPTS="$JAVA_OPTS -server -XX:-ReduceInitialCardMarks"

########### The following is the same for configservice, adminservice, portal ###########
export JAVA_OPTS="$JAVA_OPTS -XX:ParallelGCThreads=4 -XX:MaxTenuringThreshold=9 -XX:+DisableExplicitGC -XX:+ScavengeBeforeFullGC -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+ExplicitGCInvokesConcurrent -XX:+PrintGCDetails -XX:+HeapDumpOnOutOfMemoryError -XX:-OmitStackTraceInFastThrow -Duser.timezone=Asia/Shanghai -Dclient.encoding.override=UTF-8 -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom"
export JAVA_OPTS="$JAVA_OPTS -Dserver.port=$SERVER_PORT -Dlogging.file=$LOG_DIR/$SERVICE_NAME.log -XX:HeapDumpPath=$LOG_DIR/HeapDumpOnOutOfMemoryError/"

# Find Java
if [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]]; then
    javaexe="$JAVA_HOME/bin/java"
elif type -p java > /dev/null 2>&1; then
    javaexe=$(type -p java)
elif [[ -x "/usr/bin/java" ]];  then
    javaexe="/usr/bin/java"
else
    echo "Unable to find Java"
    exit 1
fi

if [[ "$javaexe" ]]; then
    version=$("$javaexe" -version 2>&1 | awk -F '"' '/version/ {print $2}')
    version=$(echo "$version" | awk -F. '{printf("%03d%03d",$1,$2);}')
    # now version is of format 009003 (9.3.x)
    if [ $version -ge 011000 ]; then
        JAVA_OPTS="$JAVA_OPTS -Xlog:gc*:$LOG_DIR/gc.log:time,level,tags -Xlog:safepoint -Xlog:gc+heap=trace"
    elif [ $version -ge 010000 ]; then
        JAVA_OPTS="$JAVA_OPTS -Xlog:gc*:$LOG_DIR/gc.log:time,level,tags -Xlog:safepoint -Xlog:gc+heap=trace"
    elif [ $version -ge 009000 ]; then
        JAVA_OPTS="$JAVA_OPTS -Xlog:gc*:$LOG_DIR/gc.log:time,level,tags -Xlog:safepoint -Xlog:gc+heap=trace"
    else
        JAVA_OPTS="$JAVA_OPTS -XX:+UseParNewGC"
        JAVA_OPTS="$JAVA_OPTS -Xloggc:$LOG_DIR/gc.log -XX:+PrintGCDetails"
        JAVA_OPTS="$JAVA_OPTS -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=60 -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:CMSFullGCsBeforeCompaction=9 -XX:+CMSClassUnloadingEnabled  -XX:+PrintGCDateStamps -XX:+PrintGCApplicationConcurrentTime -XX:+PrintHeapAtGC -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=5M"
    fi
fi

printf "$(date) ==== Starting ==== \n"

cd `dirname $0`/..
chmod 755 $SERVICE_NAME".jar"
./$SERVICE_NAME".jar" start

rc=$?;

if [[ $rc != 0 ]];
then
    echo "$(date) Failed to start $SERVICE_NAME.jar, return code: $rc"
    exit $rc;
fi

tail -f /dev/null

编写Dockerfile

vi /data/dockerfile/apollo-adminservice/Dockerfile

1
2
3
4
5
6
7
8
9
10
11
12
FROM wangxiansen/jre8:8u112

ENV VERSION 1.7.1

RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime &&\
    echo "Asia/Shanghai" > /etc/timezone

ADD apollo-adminservice-${VERSION}.jar /apollo-adminservice/apollo-adminservice.jar
ADD config/ /apollo-adminservice/config
ADD scripts/ /apollo-adminservice/scripts

CMD ["/apollo-adminservice/scripts/startup.sh"]

制作镜像并推送

1
2
3
[root@k8s-dns apollo-adminservice]# docker build . -t harbor.od.com/infra/apollo-adminservice:v1.7.1

[root@k8s-dns apollo-adminservice]# docker push harbor.od.com/infra/apollo-adminservice:v1.7.1

准备资源配置清单

1
mkdir /var/k8s-yaml/apollo-adminservice && cd /var/k8s-yaml/apollo-adminservice

vim /var/k8s-yaml/apollo-adminservice/deployment.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
kind: Deployment
apiVersion: apps/v1
metadata:
  name: apollo-adminservice
  namespace: test
  labels: 
    name: apollo-adminservice
spec:
  replicas: 1
  selector:
    matchLabels: 
      name: apollo-adminservice
  template:
    metadata:
      labels: 
        app: apollo-adminservice 
        name: apollo-adminservice
    spec:
      volumes:
      - name: configmap-volume
        configMap:
          name: apollo-adminservice-cm
      containers:
      - name: apollo-adminservice
        image: harbor.od.com/infra/apollo-adminservice:v1.7.1
        ports:
        - containerPort: 8080
          protocol: TCP
        volumeMounts:
        - name: configmap-volume
          mountPath: /apollo-adminservice/config
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        imagePullPolicy: IfNotPresent
      imagePullSecrets:
      - name: harbor
      restartPolicy: Always
      terminationGracePeriodSeconds: 30
      securityContext: 
        runAsUser: 0
      schedulerName: default-scheduler
  strategy:
    type: RollingUpdate
    rollingUpdate: 
      maxUnavailable: 1
      maxSurge: 1
  revisionHistoryLimit: 7
  progressDeadlineSeconds: 600

vim /var/k8s-yaml/apollo-adminservice/configmap.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
apiVersion: v1
kind: ConfigMap
metadata:
  name: apollo-adminservice-cm
  namespace: test
data:
  application-github.properties: |
    # DataSource
    spring.datasource.url = jdbc:mysql://mysql.od.com:3306/TestBetaApolloConfigDB?characterEncoding=utf8
    spring.datasource.username = apolloconfig
    spring.datasource.password = 123456
    eureka.service.url = http://config-test.od.com/eureka
  app.properties: |
    appId=100003172

应用资源配置清单

在任意一台k8s运算节点执行：

1
2
kubectl apply -f http://k8s-yaml.od.com/apollo-adminservice/configmap.yaml
kubectl apply -f http://k8s-yaml.od.com/apollo-adminservice/deployment.yaml

浏览器访问

http://config-test.od.com

交付apollo-portal

准备软件包

在运维主机HDSS7-200.host.com上：
下载官方release包

1
2
[root@k8s-dns tools]# mkdir /data/dockerfile/apollo-portal && unzip -o apollo-portal-1.7.1-github.zip -d /data/dockerfile/apollo-portal
[root@k8s-dns tools]# cd /data/dockerfile/apollo-portal

执行数据库脚本

在数据库主机HDSS7-11.host.com上：
数据库脚本地址

1
2
3
4
5
[root@k8s-dns apollo-portal]# mysql -uroot -p123456 < apolloportaldb.sql 
## 授权
grant INSERT,DELETE,UPDATE,SELECT on ApolloPortalDB.* to "apolloportal"@"10.1.1.%" identified by "123456";

update ServerConfig set value='[{"orgId":"boy1","orgName":"Linux男孩"},{"orgId":"boy","orgName":"LinuxBoy"}]' where id=2;

制作Docker镜像

配置数据库连接串

1
2
3
4
5
[root@k8s-dns apollo-portal]# cat config/application-github.properties 
# DataSource
spring.datasource.url = jdbc:mysql://mysql.od.com:3306/ApolloPortalDB?characterEncoding=utf8
spring.datasource.username = apolloportal
spring.datasource.password = 123456

配置Portal的meta service

1
2
[root@k8s-dns apollo-portal]# cat config/apollo-env.properties 
dev.meta=http://config.od.com

更新starup.sh

vim scripts/startup.sh

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
#!/bin/bash
SERVICE_NAME=apollo-portal
## Adjust log dir if necessary
LOG_DIR=/opt/logs/apollo-portal-server
## Adjust server port if necessary
SERVER_PORT=8080
APOLLO_PORTAL_SERVICE_NAME=$(hostname -i)
# SERVER_URL="http://localhost:$SERVER_PORT"
SERVER_URL="http://${APOLLO_PORTAL_SERVICE_NAME}:${SERVER_PORT}"

## Adjust memory settings if necessary
#export JAVA_OPTS="-Xms2560m -Xmx2560m -Xss256k -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=384m -XX:NewSize=1536m -XX:MaxNewSize=1536m -XX:SurvivorRatio=8"

## Only uncomment the following when you are using server jvm
#export JAVA_OPTS="$JAVA_OPTS -server -XX:-ReduceInitialCardMarks"

########### The following is the same for configservice, adminservice, portal ###########
export JAVA_OPTS="$JAVA_OPTS -XX:ParallelGCThreads=4 -XX:MaxTenuringThreshold=9 -XX:+DisableExplicitGC -XX:+ScavengeBeforeFullGC -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+ExplicitGCInvokesConcurrent -XX:+PrintGCDetails -XX:+HeapDumpOnOutOfMemoryError -XX:-OmitStackTraceInFastThrow -Duser.timezone=Asia/Shanghai -Dclient.encoding.override=UTF-8 -Dfile.encoding=UTF-8 -Djava.security.egd=file:/dev/./urandom"
export JAVA_OPTS="$JAVA_OPTS -Dserver.port=$SERVER_PORT -Dlogging.file=$LOG_DIR/$SERVICE_NAME.log -XX:HeapDumpPath=$LOG_DIR/HeapDumpOnOutOfMemoryError/"

# Find Java
if [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]]; then
    javaexe="$JAVA_HOME/bin/java"
elif type -p java > /dev/null 2>&1; then
    javaexe=$(type -p java)
elif [[ -x "/usr/bin/java" ]];  then
    javaexe="/usr/bin/java"
else
    echo "Unable to find Java"
    exit 1
fi

if [[ "$javaexe" ]]; then
    version=$("$javaexe" -version 2>&1 | awk -F '"' '/version/ {print $2}')
    version=$(echo "$version" | awk -F. '{printf("%03d%03d",$1,$2);}')
    # now version is of format 009003 (9.3.x)
    if [ $version -ge 011000 ]; then
        JAVA_OPTS="$JAVA_OPTS -Xlog:gc*:$LOG_DIR/gc.log:time,level,tags -Xlog:safepoint -Xlog:gc+heap=trace"
    elif [ $version -ge 010000 ]; then
        JAVA_OPTS="$JAVA_OPTS -Xlog:gc*:$LOG_DIR/gc.log:time,level,tags -Xlog:safepoint -Xlog:gc+heap=trace"
    elif [ $version -ge 009000 ]; then
        JAVA_OPTS="$JAVA_OPTS -Xlog:gc*:$LOG_DIR/gc.log:time,level,tags -Xlog:safepoint -Xlog:gc+heap=trace"
    else
        JAVA_OPTS="$JAVA_OPTS -XX:+UseParNewGC"
        JAVA_OPTS="$JAVA_OPTS -Xloggc:$LOG_DIR/gc.log -XX:+PrintGCDetails"
        JAVA_OPTS="$JAVA_OPTS -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=60 -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:CMSFullGCsBeforeCompaction=9 -XX:+CMSClassUnloadingEnabled  -XX:+PrintGCDateStamps -XX:+PrintGCApplicationConcurrentTime -XX:+PrintHeapAtGC -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=5M"
    fi
fi

printf "$(date) ==== Starting ==== \n"

cd `dirname $0`/..
chmod 755 $SERVICE_NAME".jar"
./$SERVICE_NAME".jar" start

rc=$?;

if [[ $rc != 0 ]];
then
    echo "$(date) Failed to start $SERVICE_NAME.jar, return code: $rc"
    exit $rc;
fi

tail -f /dev/null

编写Dockerfile

1
2
3
4
5
6
7
8
9
10
11
12
FROM wangxiansen/jre8:8u112

ENV VERSION 1.7.1

RUN ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime &&\
    echo "Asia/Shanghai" > /etc/timezone

ADD apollo-portal-${VERSION}.jar /apollo-portal/apollo-portal.jar
ADD config/ /apollo-portal/config
ADD scripts/ /apollo-portal/scripts

CMD ["/apollo-portal/scripts/startup.sh"]

制作镜像并推送

1
2
3
[root@k8s-dns apollo-portal]# docker build . -t harbor.od.com/infra/apollo-portal:v1.7.1

[root@k8s-dns apollo-portal]# docker push harbor.od.com/infra/apollo-portal:v1.7.1

解析域名

1
2
3
[root@k8s-dns apollo-portal]# cat /var/named/chroot/etc/od.com.zone   
...
portal   A 10.1.1.50

准备资源配置清单

1
2
mkdir /var/k8s-yaml/apollo-portal
cd /var/k8s-yaml/apollo-portal

vim /var/k8s-yaml/apollo-portal/deployment.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
kind: Deployment
apiVersion: apps/v1
metadata:
  name: apollo-portal
  namespace: test
  labels: 
    name: apollo-portal
spec:
  replicas: 1
  selector:
    matchLabels: 
      name: apollo-portal
  template:
    metadata:
      labels: 
        app: apollo-portal 
        name: apollo-portal
    spec:
      volumes:
      - name: configmap-volume
        configMap:
          name: apollo-portal-cm
      containers:
      - name: apollo-portal
        image: harbor.od.com/infra/apollo-portal:v1.7.1
        ports:
        - containerPort: 8080
          protocol: TCP
        volumeMounts:
        - name: configmap-volume
          mountPath: /apollo-portal/config
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        imagePullPolicy: IfNotPresent
      imagePullSecrets:
      - name: harbor
      restartPolicy: Always
      terminationGracePeriodSeconds: 30
      securityContext: 
        runAsUser: 0
      schedulerName: default-scheduler
  strategy:
    type: RollingUpdate
    rollingUpdate: 
      maxUnavailable: 1
      maxSurge: 1
  revisionHistoryLimit: 7
  progressDeadlineSeconds: 600

vim /var/k8s-yaml/apollo-portal/svc.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
kind: Service
apiVersion: v1
metadata: 
  name: apollo-portal
  namespace: test
spec:
  ports:
  - protocol: TCP
    port: 8080
    targetPort: 8080
  selector: 
    app: apollo-portal
  clusterIP: None
  type: ClusterIP
  sessionAffinity: None

vim /var/k8s-yaml/apollo-portal/ingress.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
kind: Ingress
apiVersion: extensions/v1beta1
metadata: 
  name: apollo-portal
  namespace: test
spec:
  rules:
  - host: portal.od.com
    http:
      paths:
      - path: /
        backend: 
          serviceName: apollo-portal
          servicePort: 8080

vim /var/k8s-yaml/apollo-portal/configmap.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
apiVersion: v1
kind: ConfigMap
metadata:
  name: apollo-portal-cm
  namespace: test
data:
  application-github.properties: |
    # DataSource
    spring.datasource.url = jdbc:mysql://mysql.od.com:3306/ApolloPortalDB?characterEncoding=utf8
    spring.datasource.username = apolloportal
    spring.datasource.password = 123456
  app.properties: |
    appId=100003173
  apollo-env.properties: |
    dev.meta=http://config-test.od.com

应用资源配置清单

在任意一台k8s运算节点执行：

1
2
3
4
5
6
7
8
9

[root@k8s-node01 ~]# kubectl apply -f http://k8s-yaml.od.com/apollo-portal/configmap.yaml
configmap/apollo-portal-cm created
[root@k8s-node01 ~]# kubectl apply -f http://k8s-yaml.od.com/apollo-portal/deployment.yaml
deployment.apps/apollo-portal created
[root@k8s-node01 ~]# kubectl apply -f http://k8s-yaml.od.com/apollo-portal/svc.yaml
service/apollo-portal created
[root@k8s-node01 ~]# kubectl apply -f http://k8s-yaml.od.com/apollo-portal/ingress.yaml
ingress.extensions/apollo-portal created

浏览器访问

http://portal.od.com

  • 用户名：apollo
  • 密码： admin

实战dubbo微服务接入Apollo配置中心

apollo项目地址

这里将k8s-slave.boysec.cn作为zookeeper独立服务来使用

配置apollo-portal

创建项目

注意：此处AppID要与gitee上项目id相同

进入配置页面

dubbo.registry：zookeeper://zk4.od.com:2181

dubbo.port：20880

发布配置

点击发布，配置生效

使用jenkins进行CI

略（注意记录镜像的tag）

上线新构建的项目

准备资源配置清单

vim /var/k8s-yaml/dubbo-demo-service/apollo-deployment.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
kind: Deployment
apiVersion: apps/v1
metadata:
  name: dubbo-demo-service
  namespace: test
  labels: 
    name: dubbo-demo-service
spec:
  replicas: 1
  selector:
    matchLabels: 
      name: dubbo-demo-service
  template:
    metadata:
      labels: 
        app: dubbo-demo-service
        name: dubbo-demo-service
    spec:
      containers:
      - name: dubbo-demo-service
        image: harbor.od.com/app/dubbo-demo-service:apollo_1230
        ports:
        - containerPort: 20880
          protocol: TCP
        env:
        - name: JAR_BALL
          value: dubbo-server.jar
        - name: C_OPTS
          value: -Denv=dev -Dapollo.meta=http://config-test.od.com
        imagePullPolicy: IfNotPresent
      imagePullSecrets:
      - name: harbor
      restartPolicy: Always
      terminationGracePeriodSeconds: 30
      securityContext: 
        runAsUser: 0
      schedulerName: default-scheduler
  strategy:
    type: RollingUpdate
    rollingUpdate: 
      maxUnavailable: 1
      maxSurge: 1
  revisionHistoryLimit: 7
  progressDeadlineSeconds: 600

注意：增加了env段配置
注意：docker镜像新版的tag

应用资源配置清单

在任意一台k8s运算节点上执行：

1
kubectl apply -f http://k8s-yaml.od.com/dubbo-demo-service/apollo-deployment.yaml

观察项目运行情况

http://dubbo-monitor.od.com/

Apollo实例列表：

创建dubbo消费者服务

配置apollo-portal

dubbo.registry：zookeeper://zk4.od.com:2181

准备资源配置清单

vim /var/k8s-yaml/dubbo-demo-consumer/apollo-deployment.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
kind: Deployment
apiVersion: apps/v1
metadata:
  name: dubbo-demo-consumer
  namespace: test
  labels: 
    name: dubbo-demo-consumer
spec:
  replicas: 1
  selector:
    matchLabels: 
      name: dubbo-demo-consumer
  template:
    metadata:
      labels: 
        app: dubbo-demo-consumer
        name: dubbo-demo-consumer
    spec:
      containers:
      - name: dubbo-demo-consumer
        image: harbor.od.com/app/dubbo-demo-consumer:apollo_20210824_1654
        ports:
        - containerPort: 20880
          protocol: TCP
        - containerPort: 8080
          protocol: TCP
        env:
        - name: C_OPTS
          value: -Denv=dev -Dapollo.meta=http://config-test.od.com
        - name: JAR_BALL
          value: dubbo-client.jar
        imagePullPolicy: IfNotPresent
      imagePullSecrets:
      - name: harbor
      restartPolicy: Always
      terminationGracePeriodSeconds: 30
      securityContext: 
        runAsUser: 0
      schedulerName: default-scheduler
  strategy:
    type: RollingUpdate
    rollingUpdate: 
      maxUnavailable: 1
      maxSurge: 1
  revisionHistoryLimit: 7
  progressDeadlineSeconds: 600

注意：增加了env段配置
注意：docker镜像新版的tag

应用资源配置清单

在任意一台k8s运算节点上执行：

1
kubectl apply -f http://k8s-yaml.od.com/dubbo-demo-consumer/apollo-deployment.yaml

实战维护多套dubbo微服务环境

生产实践

  1. 迭代新需求/修复BUG（编码->提GIT）
  2. 测试环境发版，测试（应用通过编译打包发布至TEST命名空间）
  3. 测试通过，上线（应用镜像直接发布至PROD命名空间）

系统架构

  • 物理架构
主机名角色ip
k8s-master.boysec.cnzk-test(测试环境Test)10.1.1.120
k8s-slave.boysec.cnzk-prod(生产环境Prod)10.1.1.130
k8s-node01.boysec.cnkubernetes运算节点10.1.1.100
k8s-node02.boysec.cnkubernetes运算节点10.1.1.110
k8s-dns.boysec.cn运维主机，harbor仓库10.1.1.250
  • K8S内系统架构
环境命名空间应用
测试环境（TEST）testapollo-config，apollo-admin
测试环境（TEST）testdubbo-demo-service，dubbo-demo-web
生产环境（PROD）prodapollo-config，apollo-admin
生产环境（PROD）proddubbo-demo-service，dubbo-demo-web
ops环境（infra）infrajenkins，dubbo-monitor，apollo-portal

修改/添加域名解析

DNS主机k8s-dns.boysec.cn上：

1
2
3
4
5
6
vim /var/named/chroot/etc/od.com.zone
...
zk-test 60 IN A 10.1.1.120
zk-prod 60 IN A 10.1.1.130
config-test A 10.1.1.50
config-prod A 10.1.1.50

Apollo的k8s应用配置

  • 删除app命名空间内应用，创建test命名空间，创建prod命名空间
  • 删除infra命名空间内apollo-configservice，apollo-adminservice应用
  • 数据库内删除ApolloConfigDB，创建ApolloConfigTestDB，创建ApolloConfigProdDB
1
2
3
4
5
6
7
8
9
10
11
12
13
mysql> drop database ApolloConfigDB;

mysql> create database ApolloConfigTestDB;
mysql> use ApolloConfigTestDB;
mysql> source ./apolloconfig.sql
mysql> update ApolloConfigTestDB.ServerConfig set ServerConfig.Value="http://config-test.od.com/eureka" where ServerConfig.Key="eureka.service.url";
mysql> grant INSERT,DELETE,UPDATE,SELECT on ApolloConfigTestDB.* to "apolloconfig"@"10.1.1.%" identified by "123456";

mysql> create database ApolloConfigProdDB;
mysql> use ApolloConfigProdDB;
mysql> source ./apolloconfig.sql
mysql> update ApolloConfigProdDB.ServerConfig set ServerConfig.Value="http://config-prod.od.com/eureka" where ServerConfig.Key="eureka.service.url";
mysql> grant INSERT,DELETE,UPDATE,SELECT on ApolloConfigProdDB.* to "apolloconfig"@"10.1.1.%" identified by "123456";
  • 准备apollo-config，apollo-admin的资源配置清单（各2套）

注：apollo-config/apollo-admin的configmap配置要点

  • Test环境
1
2
3
4
5
6
application-github.properties: |
    # DataSource
    spring.datasource.url = jdbc:mysql://mysql.od.com:3306/ApolloConfigTestDB?characterEncoding=utf8
    spring.datasource.username = apolloconfig
    spring.datasource.password = 123456
    eureka.service.url = http://config-test.od.com/eureka
  • Prod环境
1
2
3
4
5
6
application-github.properties: |
    # DataSource
    spring.datasource.url = jdbc:mysql://mysql.od.com:3306/ApolloConfigProdDB?characterEncoding=utf8
    spring.datasource.username = apolloconfig
    spring.datasource.password = 123456
    eureka.service.url = http://config-prod.od.com/eureka
  • 依次应用，分别发布在test和prod命名空间
  • 修改apollo-portal的configmap并重启portal
1
2
3
apollo-env.properties: |
    TEST.meta=http://config-test.od.com
    PROD.meta=http://config-prod.od.com

Apollo的portal配置

管理员工具

删除应用、集群、AppNamespace，将已配置应用删除

系统参数

  • Key

    apollo.portal.envs

  • Value

    TEST,PROD

查询

  • Value

    TEST,PROD

保存

新建dubbo-demo-service和dubbo-demo-web项目

在TEST/PROD环境分别增加配置项并发布

发布dubbo微服务

  • 准备dubbo-demo-service和dubbo-demo-web的资源配置清单（各2套）
  • 依次应用，分别发布至test和prod命名空间
  • 使用dubbo-monitor查验