日志系统

架构图

clickhouse

setup.sh

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
#!/bin/bash
echo "Asia/Shanghai" > /etc/timezone

docker stop clickhouse
docker rm clickhouse
docker run -d --name clickhouse \
--restart=always \
--network host \
-m 4g \
--add-host localhost:10.0.18.2 \
-v /etc/localtime:/etc/localtime:ro \
-v /etc/timezone:/etc/timezone:ro \
-e TZ='Asia/Shanghai' \
--ulimit nofile=262144:262144 \
-v $(pwd)/data:/var/lib/clickhouse \
-v $(pwd)/config:/etc/clickhouse-server \
-v $(pwd)/initdb:/docker-entrypoint-initdb.d \
-v $(pwd)/log:/var/log/clickhouse-server \
clickhouse/clickhouse-server:23.12-alpine

# --volume=$(pwd)/scripts/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh \
# docker logs -f clickhouse-server

# 证书需要自己生成
# openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout $(pwd)/config/server.key -out $(pwd)/config/server.crt

设置密码

在users.xml中的password中输入明文

nginx建表

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
SET allow_experimental_object_type=1;
CREATE TABLE log.ngxlog
(
    `create_time` DateTime('Asia/Shanghai'),
    `kafka_offset` UInt64,
    `query_params_imsi` Nullable(String),
    `query_params_imei` Nullable(String),
    `query_params_hsman` Nullable(String),
    `query_params_hstype` Nullable(String),
    `query_params_exttype` Nullable(String),
    `nginx_time_local` String,
    `nginx_upstream_addr` String,
    `nginx_uri` String,
    `nginx_status` UInt16,
    `nginx_host` String,
    `mcc` Nullable(String),
    `hostname` String,
    `message` Nullable(String),
    `nginx_query_string` Nullable(String),
    `nginx_upstream_response_time` Float32,
    `nginx_request_time` Float32
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(create_time)
ORDER BY create_time
TTL create_time + toIntervalDay(180)
SETTINGS index_granularity = 1024

服务建表

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
CREATE TABLE log.applog_test
(
    `app_name` Nullable(String),
    `app_type` Nullable(String),
    `hostname` Nullable(String),
    `logver` Nullable(String),
    `create_time` DateTime('Asia/Shanghai'),
    `container_name` Nullable(String),
    `kafka_offset` UInt64,
    `log_level` Nullable(String),
    `message` Nullable(String),
    `object` Nullable(String)
)
ENGINE = MergeTree
PARTITION BY toYYYYMMDD(create_time)
ORDER BY create_time
TTL create_time + toIntervalDay(7)
SETTINGS index_granularity = 1024

clickvisual

setup.sh

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
#!/bin/bash

clickvisual_version=1.0.0-rc3

docker kill clickvisual
docker rm clickvisual
docker run -d --net host \
--restart=always \
--name clickvisual \
-v /etc/timezone:/etc/timezone:ro \
-v /etc/localtime:/etc/localtime:ro \
-e EGO_CONFIG_PATH=/clickvisual/config/docker.toml \
-e EGO_LOG_WRITER=stderr \
-v $(pwd)/config:/clickvisual/config/ \
clickvisual/clickvisual:${clickvisual_version}


# -p 19001:19001 \
# -p 19006:19006 \

连接数据库

在docker.toml中修改mysql的配置

kafka

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
#!/bin/bash

port=9092
container_name="kafka-${port}"

docker stop ${container_name}
docker rm ${container_name}
docker run -d \
--hostname $(hostname) \
--add-host $(hostname):10.0.18.2 \
--name ${container_name} \
--restart=always \
-v /etc/timezone:/etc/timezone:ro \
-v /etc/localtime:/etc/localtime:ro \
-v $(pwd)/config-${port}/run.sh:/run.sh \
-v $(pwd)/data-standalone:/data \
--net host \
-e NODE_ID="1001" \
-e LISTENERS="PRIVATE://:9092,CONTROLLER://:9093,PUBLIC://:9094" \
-e ADVERTISED_LISTENERS="PRIVATE://10.0.18.2:9092" \
registry.cn-hangzhou.aliyuncs.com/buyfakett/kafka-standalone:2.13-2.8.0

# 配置通过 run.sh 重写

kafka-map

1
2
3
4
5
6
7
8
docker run -d \
    -p 8087:8080 \
    -v ./data:/usr/local/kafka-map/data \
    -e DEFAULT_USERNAME=buyfakett \
    -e DEFAULT_PASSWORD= \
    --name kafka-map \
    --restart always \
    dushixiang/kafka-map:v1.3.3

redpandadata-console

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
#!/bin/bash

##################
# 用于kafka查看
# 部分功能收费如登录功能
# https://docs.redpanda.com/current/get-started/
####################################################

port=9096

docker rm -f redpandadata-console-${port}
docker run --network=host -d \
--name redpandadata-console-${port} \
-e KAFKA_BROKERS=10.0.18.2:9092 \
-e SERVER_LISTENPORT=${port} \
docker.redpanda.com/redpandadata/console:v2.3.3

filebeat

setup.sh

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
#!/bin/bash

echo 'Asia/Shanghai' > /etc/timezone

docker stop filebeat
docker rm filebeat
docker run -d \
--net host \
-m 1024M \
--restart=always \
--name=filebeat \
--hostname=$(hostname) \
--user=root \
--volume="$(pwd)/config/filebeat.docker.yml:/usr/share/filebeat/filebeat.yml:ro" \
-v /etc/timezone:/etc/timezone:ro \
-v /etc/localtime:/etc/localtime:ro \
-v /data/logs/:/data/logs/ \
-v ./filebeat/registry/:/usr/share/filebeat/data/registry/ \
docker.elastic.co/beats/filebeat:7.0.0 filebeat -e -strict.perms=false \
#setup -E setup.kibana.host=kibana:5601 \
#-E output.elasticsearch.hosts=["elasticsearch:9200"]

#docker exec -it filebeat /bin/bash
#docker logs --tail=200 -f filebeat

config/filebeat.docker.yml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
filebeat.config:
  modules:
    path: ${path.config}/modules.d/*.yml
    reload.enabled: false

processors:
- add_cloud_metadata: ~

filebeat.inputs:
- type: log
  paths:
    - /data/logs/nginx/json_*.log
  fields:
      type: "test_nginxlog"
      saltid: hdy-nmg-server-001
      appType: nginx
      env: test
      object: "未分类"
  fields_under_root: true


output.kafka:
  # initial brokers for reading cluster metadata
  hosts: ["10.0.18.2:9092"]
  # message topic selection + partitioning
  topics:
    - topic:  "test_nginxlog"
      when.contains:
        type: "test_nginxlog"
  partition.round_robin:
    reachable_only: false

  required_acks: 1
  compression: gzip
  max_message_bytes: 1000000

vector

setup.sh

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
#!/bin/bash

docker kill vector
docker rm vector

docker run -d --name vector \
--restart=always \
--net host \
-v /etc/timezone:/etc/timezone:ro \
-v /etc/localtime:/etc/localtime:ro \
-v $(pwd)/config/:/etc/vector/conf.d/ \
-v $(pwd)/data/:/var/lib/vector \
timberio/vector:0.33.0-alpine --config-dir /etc/vector/conf.d/

# reload
# docker kill --signal=HUP vector

config/vector.toml

1
2
3
4
5
6
data_dir = "/var/lib/vector"
timezone = "Asia/Shanghai"

[api]
enabled=true
address="0.0.0.0:8686"

config/ngxlog.toml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
[sources.ngxlog_source]
type = "kafka"
bootstrap_servers = "10.0.18.2:9092"
group_id = "vector-clickhouse-hdy-nmg-server-001"
topics = [ "test_nginxlog" ]
offset_key = "kafka_offset"
decoding.codec = "json"
auto_offset_reset = "earliest"

[transforms.ngxlog_trans]
type="remap"
inputs=["ngxlog_source"]
source='''
# 解析 nginx 日志并添加 "nginx_" 前缀
.message_parsed = object(parse_json(.message) ?? {}) ?? {}
.message_parsed = map_keys(.message_parsed, recursive: false) -> |key| { "nginx_" + key }
. = merge(., .message_parsed)

# 解析 nginx_query_string 字段并添加 "query_params_" 前缀
.query_params = parse_query_string(.nginx_query_string) ?? {}
. = merge(., map_keys(.query_params) -> |key| { "query_params_" + key })

# 解析 nginx_request_body 字段并添加 "nginx_request_body_" 前缀
.nginx_request_body = object(parse_json(.nginx_request_body) ?? {}) ?? {}
. = merge(., map_keys(.nginx_request_body) -> |key| { "nginx_request_body_" + key })

# 解析 nginx_resp_body 字段并添加 "nginx_resp_body_" 前缀
.nginx_resp_body = object(parse_json(.nginx_resp_body) ?? {}) ?? {}
. = merge(., map_keys(.nginx_resp_body) -> |key| { "nginx_resp_body_" + key })

# 解析 nginx_resp_body 的 data 字段并添加 "nginx_resp_body_data_" 前缀
.nginx_resp_body_data = object(.nginx_resp_body.data) ?? {}
. = merge(., map_keys(.nginx_resp_body_data) -> |key| { "nginx_resp_body_data_" + key })

# 解析 nginx_http_Authorization 字段并解密添加 "auth_message_" 前缀
.auth_message_encode = split(.nginx_http_Authorization, ".")[1] ?? ""
.auth_message_encode_length = length(.auth_message_encode) ?? 0
.auth_message_mod = mod(.auth_message_encode_length, 4)
if (.auth_message_mod == 1) {
        .auth_message_encode = .auth_message_encode + "==="
} else if (.auth_message_mod == 2) {
        .auth_message_encode = .auth_message_encode + "=="
} else if (.auth_message_mod == 3) {
        .auth_message_encode = .auth_message_encode + "="
} else if (.auth_message_mod == 0) {
        .auth_message_encode = .auth_message_encode
}
.auth_message_decode,.err.auth_message_decode = decode_base64(.auth_message_encode)
.auth_message = object(parse_json(.auth_message_decode) ?? {}) ?? {}
. = merge(., map_keys(.auth_message) -> |key| { "auth_message_" + key })

# 时间字段解析
.create_time = parse_timestamp(.nginx_time_local,format: "%d/%b/%Y:%H:%M:%S %z") ?? now()

# 主机名
.hostname = .saltid

# mcc 解析
.mcc = chunks(.query_params_imsi, 3)[0] ?? null
'''

[sinks.ngxlog_clickhouse_sink]
type = "clickhouse"
inputs = [ "ngxlog_trans" ]
# 数据库配置
endpoint = "http://10.0.18.2:8123"
database = "log"
table = "ngxlog_test"
healthcheck.enabled = true
auth.strategy = "basic"
auth.user = "default"
auth.password = "kOob87lU"
# 批量入库
batch.max_bytes = 10000000
batch.max_events = 1000
batch.timeout_secs = 3
# 缓存
# buffer.type = "disk"
# buffer.max_size = 1024000000
# 内存缓存
buffer.type = "memory"
buffer.max_events = 1000
# 缓存满了之后，block/drop_newest
buffer.when_full = "block"
# 压缩格式
compression = "gzip"
# 默认false，时间格式自动解析 RFC3339/ISO 8601
date_time_best_effort = true
# 自动丢弃多余字段
skip_unknown_fields = true

config/applog.toml

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
[sources.source_applog_test]
type = "kafka"
bootstrap_servers = "10.0.18.2:9092"
group_id = "vector-loki-hdy-nmg-server-001"
topics = [ "log" ]
offset_key = "kafka_offset"
decoding.codec = "json"
auto_offset_reset = "earliest"

[sinks.sink_loki_applog]
type = "loki"
inputs = [ "source_applog_test" ]
endpoint = "http://10.0.18.2:3100"
healthcheck.enabled = true
encoding.codec = "raw_message"
batch.max_bytes = 1000000
batch.max_events = 100000
batch.timeout_secs = 1
buffer.type = "memory"
buffer.max_events = 1000
buffer.when_full = "block"
compression = "gzip"
labels."object" = "{{ .object }}"
labels."appName" = "{{ .appName }}"
labels."containerName" = "{{ .containerName }}"
labels."hostname" = "{{ .hostname }}"
labels."logver" = "{{ .logver }}"

loki

setup.sh

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
version: "3"

networks:

services:
  loki:
    container_name: loki
    image: grafana/loki:2.4.1
    volumes:
      - /etc/timezone:/etc/timezone
      - /etc/localtime:/etc/localtime
      - ./config/loki-config.yaml:/etc/config/loki-config.yaml
      - ./loki-data:/loki/data
    ports:
      - "3100:3100"
    command: -config.file=/etc/config/loki-config.yaml -target=all,table-manager
    restart: always
    logging:
      driver: "json-file"
      options:
        max-size: "10m"
        max-file: "3"
    deploy:
      resources:
        limits:
          cpus: '3'
          memory: 4G