0%

K8s环境下Nginx容器自动热更新

说明

Nginx自身支持热更新操作。如我们在宿主机上通过yum/apt/编译安装/二进制安装nginx,在我们修改配置文件之后,执行nginx -s reload命令可以不停服务重新加载配置。

对于使用docker部署的nginx,我们也可以使用docker exec -it nginx-container service nginx reload 来完成修改后的配置文件重新加载,但无论对于docker/k8s这样过多的人工干预还是很痛苦的。

本文将使用sidecar(边车)方案来完成nginx的自动热更新。

构建nginx热更新镜像

Dockerfile

Sidecar容器,nginx-reloader镜像的Dockerfile如下:

1
2
3
4
5
6
7
8
9
10
11
12
13
# cat Dockerfile

FROM golang:1.12.0 as build
RUN go get github.com/fsnotify/fsnotify
RUN go get github.com/shirou/gopsutil/process
RUN mkdir -p /go/src/app
ADD main.go /go/src/app/
WORKDIR /go/src/app
RUN CGO_ENABLED=0 GOOS=linux go build -a -o nginx-reloader .
# main image
FROM nginx:1.14.2-alpine
COPY --from=build /go/src/app/nginx-reloader /
CMD ["/nginx-reloader"]

nginx热更新功能是用一个叫main.go的go脚本实现的:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
# cat main.go

package main

import (
"log"
"os"
"path/filepath"
"syscall"

"github.com/fsnotify/fsnotify"
proc "github.com/shirou/gopsutil/process"
)

const (
nginxProcessName = "nginx"
defaultNginxConfPath = "/etc/nginx"
watchPathEnvVarName = "WATCH_NGINX_CONF_PATH"
)

var stderrLogger = log.New(os.Stderr, "error: ", log.Lshortfile)
var stdoutLogger = log.New(os.Stdout, "", log.Lshortfile)

func getMasterNginxPid() (int, error) {
processes, processesErr := proc.Processes()
if processesErr != nil {
return 0, processesErr
}

nginxProcesses := map[int32]int32{}

for _, process := range processes {
processName, processNameErr := process.Name()
if processNameErr != nil {
return 0, processNameErr
}

if processName == nginxProcessName {
ppid, ppidErr := process.Ppid()

if ppidErr != nil {
return 0, ppidErr
}

nginxProcesses[process.Pid] = ppid
}
}

var masterNginxPid int32

for pid, ppid := range nginxProcesses {
if ppid == 0 {
masterNginxPid = pid

break
}
}

stdoutLogger.Println("found master nginx pid:", masterNginxPid)

return int(masterNginxPid), nil
}

func signalNginxReload(pid int) error {
stdoutLogger.Printf("signaling master nginx process (pid: %d) -> SIGHUP\n", pid)
nginxProcess, nginxProcessErr := os.FindProcess(pid)

if nginxProcessErr != nil {
return nginxProcessErr
}

return nginxProcess.Signal(syscall.SIGHUP)
}

func main() {
watcher, watcherErr := fsnotify.NewWatcher()
if watcherErr != nil {
stderrLogger.Fatal(watcherErr)
}
defer watcher.Close()

done := make(chan bool)
go func() {
for {
select {
case event, ok := <-watcher.Events:
if !ok {
return
}

if event.Op&fsnotify.Create == fsnotify.Create {
if filepath.Base(event.Name) == "..data" {
stdoutLogger.Println("config map updated")

nginxPid, nginxPidErr := getMasterNginxPid()
if nginxPidErr != nil {
stderrLogger.Printf("getting master nginx pid failed: %s", nginxPidErr.Error())

continue
}

if err := signalNginxReload(nginxPid); err != nil {
stderrLogger.Printf("signaling master nginx process failed: %s", err)
}
}
}
case err, ok := <-watcher.Errors:
if !ok {
return
}
stderrLogger.Printf("received watcher.Error: %s", err)
}
}
}()

pathToWatch, ok := os.LookupEnv(watchPathEnvVarName)
if !ok {
pathToWatch = defaultNginxConfPath
}

stdoutLogger.Printf("adding path: `%s` to watch\n", pathToWatch)

if err := watcher.Add(pathToWatch); err != nil {
stderrLogger.Fatal(err)
}
<-done
}

部署nginx

创建命名空间

1
2
3
4
5
6
# cat ns-nginx.yml

apiVersion: v1
kind: Namespace
metadata:
name: nginx

创建nginx configmap

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
# cat nginx-configmap.yml

apiVersion: v1
kind: ConfigMap
metadata:
name: nginx-config
namespace: kube-mon
data:
nginx.conf: |
user root;
worker_processes 2;
error_log /var/log/nginx/error.log error;
pid /var/run/nginx.pid;
events {
worker_connections 10240;
}
http {
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
access_log /var/log/nginx/access.log main;
sendfile on;
#tcp_nopush on;
keepalive_timeout 65;
#gzip on;
include /etc/nginx/conf.d/*.conf;
}

创建vhost

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
# cat ai-download.yml 

apiVersion: v1
kind: ConfigMap
metadata:
name: download-config
namespace: kube-mon
data:
ai-download.conf: |
server {
listen 80; #端口
#server_name localhost; #服务名
charset utf-8; #避免中文乱码
root /tmp; #显示的根索引目录,注意这里要改成你自己的,目录要存在

location / {
autoindex on; #开启索引功能
autoindex_exact_size off; #关闭计算文件确切大小(单位bytes),只显示大概大小(单位kb、mb、gb)
autoindex_localtime on; #显示本机时间而非 GMT 时间
}
}

创建deployment

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
# cat nginx-all-reloader.yml

---
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx
namespace: kube-mon
spec:
replicas: 1
selector:
matchLabels:
app: nginx
template:
metadata:
name: nginx
labels:
app: nginx
spec:
# 需打开共享进程命名空间特性
shareProcessNamespace: true
volumes:
- name: nginx-config
configMap:
name: nginx-config
- name: ai-config
configMap:
name: download-config
- name: mongodb-efs-data
hostPath:
# 宿主上目录位置
path: /mnt/data-s3-fs/mongodb-efs
#type: Directory
containers:
- name: nginx
image: nginx
ports:
- name: http
containerPort: 80
- name: https
containerPort: 443
volumeMounts:
- name: nginx-config
mountPath: /etc/nginx/nginx.conf
subPath: nginx.conf
readOnly: true
- name: ai-config
mountPath: /etc/nginx/conf.d
readOnly: true
- name: mongodb-efs-data
mountPath: /tmp
readOnly: true

- name: nginx-reloader
image: registry.cn-shanghai.aliyuncs.com/tengfeiwu/nginx-reloader:20210521
env:
- name: WATCH_NGINX_CONF_PATH
value: /etc/nginx/conf.d
volumeMounts:
- name: ai-config
mountPath: /etc/nginx/conf.d
readOnly: true
---
apiVersion: v1
kind: Service
metadata:
name: nginx
namespace: kube-mon
spec:
selector:
app: nginx
type: NodePort
ports:
- name: http
port: 80
nodePort: 32080

部署应用

1
2
3
4
5
6
7
8
# 创建namespace
kubectl apply -f ns-nginx.yml
# 创建nginx configmap
kubectl apply -f nginx-configmap.yml
# 创建download配置文件,支持热更新
kubectl apply -f ai-download.yml
# 部署应用
kubectl apply -f nginx-all-reloader.yml

实现效果

手动修改ai-download.yml 后再apply,reloader监测到configmap变化,会主动向nginx主进程发起HUP信号,实现配置热更新。