kubernetes集群之微服务tomcat服务

制作tomcat容器的底包镜像

Tomcat是Apache 软件基金会（Apache Software Foundation）的Jakarta 项目中的一个核心项目，由Apache、Sun 和其他一些公司及个人共同开发而成。由于有了Sun 的参与和支持，最新的Servlet 和JSP 规范总是能在Tomcat 中得到体现，Tomcat 5支持最新的Servlet 2.4 和JSP 2.0 规范。因为Tomcat 技术先进、性能稳定，而且免费，因而深受Java 爱好者的喜爱并得到了部分软件开发商的认可，成为目前比较流行的Web 应用服务器。

1
2
3
wget https://mirrors.tuna.tsinghua.edu.cn/apache/tomcat/tomcat-8/v8.5.70/bin/apache-tomcat-8.5.70.tar.gz
mkdir /data/dockerfile/tomcat
tar xf apache-tomcat-8.5.69.tar.gz -C /data/dockerfile/tomcat/

配置日志

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
[root@k8s-dns tomcat]# vim apache-tomcat-8.5.70/conf/logging.properties
handlers = 1catalina.org.apache.juli.AsyncFileHandler, 2localhost.org.apache.juli.AsyncFileHandler, , java.util.logging.ConsoleHandler

...

1catalina.org.apache.juli.AsyncFileHandler.level = INFO
1catalina.org.apache.juli.AsyncFileHandler.directory = ${catalina.base}/logs
1catalina.org.apache.juli.AsyncFileHandler.prefix = catalina.
1catalina.org.apache.juli.AsyncFileHandler.encoding = UTF-8

2localhost.org.apache.juli.AsyncFileHandler.level = INFO
2localhost.org.apache.juli.AsyncFileHandler.directory = ${catalina.base}/logs
2localhost.org.apache.juli.AsyncFileHandler.prefix = localhost.
2localhost.org.apache.juli.AsyncFileHandler.encoding = UTF-8

#3manager.org.apache.juli.AsyncFileHandler.level = FINE
#3manager.org.apache.juli.AsyncFileHandler.directory = ${catalina.base}/logs
#3manager.org.apache.juli.AsyncFileHandler.prefix = manager.
#3manager.org.apache.juli.AsyncFileHandler.encoding = UTF-8
#
#4host-manager.org.apache.juli.AsyncFileHandler.level = FINE
#4host-manager.org.apache.juli.AsyncFileHandler.directory = ${catalina.base}/logs
#4host-manager.org.apache.juli.AsyncFileHandler.prefix = host-manager.
#4host-manager.org.apache.juli.AsyncFileHandler.encoding = UTF-8

java.util.logging.ConsoleHandler.level = INFO
java.util.logging.ConsoleHandler.formatter = org.apache.juli.OneLineFormatter
java.util.logging.ConsoleHandler.encoding = UTF-8
...

制作Docker镜像

1
2
3
4
5
6
7
8
9
10
11
12
13
[root@k8s-dns ~]# cd /data/dockerfile/tomcat/
[root@k8s-dns tomcat]# cat Dockerfile 
From harbor.od.com/base/jre8:8u112
RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime &&\ 
    echo 'Asia/Shanghai' >/etc/timezone
ENV CATALINA_HOME /opt/tomcat
ENV LANG zh_CN.UTF-8
ADD apache-tomcat-8.5.70/ /opt/tomcat
ADD config.yml /opt/prom/config.yml
ADD jmx_javaagent-0.13.0.jar /opt/prom/jmx_javaagent-0.13.0.jar
WORKDIR /opt/tomcat
ADD entrypoint.sh /entrypoint.sh
CMD ["/entrypoint.sh"]

vim /data/dockerfile/tomcat/config.yml

1
2
3
---
rules:
  - pattern: '.*'
1
wget https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.13.0/jmx_prometheus_javaagent-0.13.0.jar -O jmx_javaagent-0.13.0.jar

vim /data/dockerfile/tomcat/entrypoint.sh

1
2
3
4
5
6
7
8
9
10
11
#!/bin/bash
M_OPTS="-Duser.timezone=Asia/Shanghai -javaagent:/opt/prom/jmx_javaagent-0.13.0.jar=$(hostname -i):${M_PORT:-"12346"}:/opt/prom/config.yml"
C_OPTS=${C_OPTS}
MIN_HEAP=${MIN_HEAP:-"128m"}
MAX_HEAP=${MAX_HEAP:-"128m"}
JAVA_OPTS=${JAVA_OPTS:-"-Xmn384m -Xss256k -Duser.timezone=GMT+08  -XX:+DisableExplicitGC -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSParallelRemarkEnabled -XX:+UseCMSCompactAtFullCollection -XX:CMSFullGCsBeforeCompaction=0 -XX:+CMSClassUnloadingEnabled -XX:LargePageSizeInBytes=128m -XX:+UseFastAccessorMethods -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -XX:+PrintClassHistogram  -Dfile.encoding=UTF8 -Dsun.jnu.encoding=UTF8"}
CATALINA_OPTS="${CATALINA_OPTS}"
JAVA_OPTS="${M_OPTS} ${C_OPTS} -Xms${MIN_HEAP} -Xmx${MAX_HEAP} ${JAVA_OPTS}"
sed -i -e "1a\JAVA_OPTS=\"$JAVA_OPTS\"" -e "1a\CATALINA_OPTS=\"$CATALINA_OPTS\"" /opt/tomcat/bin/catalina.sh

cd /opt/tomcat && /opt/tomcat/bin/catalina.sh run 2>&1 >> /opt/tomcat/logs/stdout.log

赋予执行权限：

chmod +x /data/dockerfile/tomcat/entrypoint.sh

构建镜像

1
2
[root@k8s-dns tomcat]# docker build . -t harbor.od.com/base/tomcat:v8.5.70
[root@k8s-dns tomcat]# docker push harbor.od.com/base/tomcat:v8.5.70

jenkins构建

  • Enter an item name

    tomcat-dubbo

  • Pipeline -> OK

  • Discard old builds

    Days to keep builds : 3
    Max # of builds to keep : 30

  • This project is parameterized

  1. Add Parameter -> String Parameter

    Name : app_name
    Default Value :
    Description : 项目名称，例: dubbo-demo-tomcat

  2. Add Parameter -> String Parameter

    Name : image_name
    Default Value :
    Description : 项目所需镜像，例: app/dubbo-demo-tomcat

  3. Add Parameter -> String Parameter

    Name : git_repo
    Default Value :
    Description : 项目所需git中央仓库地址，例: https://gitee.com/dabou/dubbo-demo-web.git

  4. Add Parameter -> String Parameter

    Name : git_ver
    Default Value :
    Description : 项目在git中央仓库种对应的版本号或分支

  5. Add Parameter -> String Parameter

    Name : add_tag
    Default Value :
    Description : 创建项目docker镜像标签，例: 20210815_1232

  6. Add Parameter -> String Parameter

    Name : mvn_dir
    Default Value : ./
    Description : 编译项目的目录，默认为”根”。

  7. Add Parameter -> String Parameter

    Name : target_dir
    Default Value : ./dubbo-client/target
    Description : 编译完成后产生的jar or war包所在目录 例: ./dubbo-client/target

  8. Add Parameter -> String Parameter

    Name : mvn_cmd
    Default Value : mvn clean package -Dmaven.test.skip=true
    Description : 编译命令 例: mvn clean package -e -q -Dmaven.test.skip=true

  9. Add Parameter -> Choice Parameter

    Name : base_image
    Default Value :

    • base/tomcat:v8.5.70
    • base/tomcat:v9.5.63
      Description : 项目所用的tomcat版本镜像。

  10. Add Parameter -> Choice Parameter

    Name : maven
    Default Value :

    • 3.6.3-8u242
    • 3.2.5-6u025
    • 2.2.1-6u025
      Description : 编译时maven版本。

  11. Add Parameter -> String Parameter

Name : root_url

Default Value : ROOT

Description : WEB服务目录。

Pipeline-Script

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
pipeline {
  agent any 
    stages {
    stage('pull') { //get project code from repo 
      steps {
        sh "git clone ${params.git_repo} ${params.app_name}/${env.BUILD_NUMBER} && cd ${params.app_name}/${env.BUILD_NUMBER} && git checkout ${params.git_ver}"
        }
    }
    stage('build') { //exec mvn cmd
      steps {
        sh "cd ${params.app_name}/${env.BUILD_NUMBER}  && /var/jenkins_home/maven-${params.maven}/bin/${params.mvn_cmd}"
      }
    }
    stage('unzip') { //unzip  target/*.war -c target/project_dir
      steps {
        sh "cd ${params.app_name}/${env.BUILD_NUMBER} && cd ${params.target_dir} && mkdir project_dir && unzip *.war -d ./project_dir"
      }
    }
    stage('image') { //build image and push to registry
      steps {
        writeFile file: "${params.app_name}/${env.BUILD_NUMBER}/Dockerfile", text: """FROM harbor.od.com/${params.base_image}
ADD ${params.target_dir}/project_dir /opt/tomcat/webapps/${params.root_url}"""
        sh "cd  ${params.app_name}/${env.BUILD_NUMBER} && docker build -t harbor.od.com/${params.image_name}:${params.git_ver}_${params.add_tag} . && docker push harbor.od.com/${params.image_name}:${params.git_ver}_${params.add_tag}"
      }
    }
  }
}

准备资源配置清单

vim /var/k8s-yaml/dubbo-tomcat/deployment.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
kind: Deployment
apiVersion: apps/v1
metadata:
  name: dubbo-demo-consumer
  namespace: test
  labels: 
    name: dubbo-demo-consumer
spec:
  replicas: 1
  selector:
    matchLabels: 
      name: dubbo-demo-consumer
  template:
    metadata:
      labels: 
        app: dubbo-demo-consumer
        name: dubbo-demo-consumer
    spec:
      containers:
      - name: dubbo-demo-consumer
        image: harbor.od.com/app/dubbo-demo-consumer:tomcat:20210301_1800
        ports:
        - containerPort: 8080
          protocol: TCP
        - containerPort: 20880
          protocol: TCP
        env:
        - name: JAR_BALL
          value: dubbo-client.jar
        - name: C_OPTS
          value: -Denv=dev -Dapollo.meta=http://config-test.od.com
        imagePullPolicy: IfNotPresent
      imagePullSecrets:
      - name: harbor
      restartPolicy: Always
      terminationGracePeriodSeconds: 30
      securityContext: 
        runAsUser: 0
      schedulerName: default-scheduler
  strategy:
    type: RollingUpdate
    rollingUpdate: 
      maxUnavailable: 1
      maxSurge: 1
  revisionHistoryLimit: 7
  progressDeadlineSeconds: 600

应用资源配置清单

在任意一台k8s运算节点执行：

1
kubectl apply -f http://k8s-yaml.od.com/dubbo-tomcat/deployment.yaml

测试

注意

tomcat配合Apollo时需要注意修改dubbo.registry=zookeeper://zk-test.od.com:2181