diff --git a/filebast/s1-lessie-server01/filebeat.yml b/filebast/s1-lessie-server01/filebeat.yml new file mode 100644 index 0000000..60a59c7 --- /dev/null +++ b/filebast/s1-lessie-server01/filebeat.yml @@ -0,0 +1,92 @@ +# 配置索引模板名称和模式 +setup.template.name: "lessie-sit" +setup.template.pattern: "lessie-sit*" +setup.template.enabled: true +setup.ilm.enabled: true + +#主配置文件加载子配置文件 +filebeat.config.inputs: + enabled: true + path: /etc/filebeat/inputs.d/*.yml + reload.enabled: true + reload.period: 10s + + +# 处理器 +processors: +# lessie ------------------------- + + - dissect: + when: + equals: + log_type: lessie_search.log + tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{message}' + field: "message" + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true + + # 针对带有 [level: | event: | msg: | context:] 的日志,再做一次 dissect + - dissect: + when: + regexp: + mylog.message: '^\[level:.*\]' + tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{context}]' + field: "mylog.message" + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true + + # 把 context 再拆成独立字段 + - script: + lang: javascript + id: parse_context + source: > + function process(event) { + var ctx = event.Get("mylog.context"); + if (ctx) { + var parts = ctx.split(","); + parts.forEach(function(p) { + var kv = p.split(":"); + if (kv.length == 2) { + event.Put("mylog." + kv[0].trim(), kv[1].trim()); + } + }); + } + } +# lessie ------------------------ + + - decode_json_fields: + when: + equals: + log_type: go.log + fields: ["message"] + target: "" + overwrite_keys: true + add_error_key: true + + +#输出 +output.elasticsearch: + hosts: ["http://106.53.194.199:9200"] + username: "admin" + password: "123456" + index: "%{[environment]}-%{[application]}-%{+yyyy.MM.dd}" # 按天分割索引 + bulk_max_size: 50 # 单批次传输最大文档数 + worker: 1 # 并行工作线程数 + timeout: 15s + +# 日志记录 +logging.level: info +logging.to_files: true +logging.files: + path: /var/log/filebeat + name: filebeat.log + keepfiles: 7 + permissions: 0644 + +# 设置队列和内存使用 +queue.mem: + events: 1024 + flush.min_events: 512 + flush.timeout: 10s \ No newline at end of file diff --git a/filebast/s1-lessie-server01/go_lessie_sourcing_api.yml b/filebast/s1-lessie-server01/go_lessie_sourcing_api.yml new file mode 100644 index 0000000..7a06541 --- /dev/null +++ b/filebast/s1-lessie-server01/go_lessie_sourcing_api.yml @@ -0,0 +1,25 @@ +- type: log + id: s1_go_lessie_sourcing_api + enabled: true + paths: + - /data/webapps/go_lessie_sourcing_api/logs/*.log + follow_symlinks: true + harvester_limit: 1 + fields: + application: go-lessie-sourcing_api # 自定义字段,标识应用名称 + log_type: go.log # 自定义字段,标识日志类型 + environment: s1 # 自定义字段,标识机器环境名称 + instance: weblessie-server # 自定义字段,标识机器名称 + fields_under_root: true + # multiline.pattern: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}' + # multiline.negate: true + # multiline.match: after + ignore_older: 24h # 忽略旧日志文件(避免处理已归档的日志) + scan_frequency: 10s # 定期扫描新文件的频率 + clean_inactive: 25h # 清除超过一天未更新的文件 + close_inactive: 5m # 文件超过5分钟无更新则关闭 + close_renamed: true # 处理被重命名的文件 + start_position: beginning # 从文件的开头读取 + + + diff --git a/filebast/s1-lessie-server01/s1_lessie_search.yml b/filebast/s1-lessie-server01/s1_lessie_search.yml new file mode 100644 index 0000000..04092a0 --- /dev/null +++ b/filebast/s1-lessie-server01/s1_lessie_search.yml @@ -0,0 +1,25 @@ +- type: log + id: s1_lessie_search + enabled: true + paths: + - /data/webapps/lessie_sourcing_agents/logs/lessie_sourcing_agents_*.log + include_lines: ['^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}'] # 只包含匹配该正则表达式的行 + fields: + application: lessie_search + log_type: lessie_search.log + environment: s1 + instance: weblessie-server + ip: 43.130.56.138 + fields_under_root: true + # multiline.pattern: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}' + # multiline.negate: true + # multiline.match: after + ignore_older: 24h # 忽略旧日志文件(避免处理已归档的日志) + scan_frequency: 10s # 定期扫描新文件的频率 + clean_inactive: 25h # 清除超过一天未更新的文件 + close_inactive: 5m # 文件超过5分钟无更新则关闭 + close_renamed: true # 处理被重命名的文件 + start_position: beginning # 从文件的开头读取 + + + diff --git a/filebast/s2s3-lessie-server01/filebeat.yml b/filebast/s2s3-lessie-server01/filebeat.yml new file mode 100644 index 0000000..3614cc0 --- /dev/null +++ b/filebast/s2s3-lessie-server01/filebeat.yml @@ -0,0 +1,104 @@ +# 配置索引模板名称和模式 +setup.template.name: "lessie-sit" +setup.template.pattern: "lessie-sit*" +setup.template.enabled: true +setup.ilm.enabled: true + +#主配置文件加载子配置文件 +filebeat.config.inputs: + enabled: true + path: /etc/filebeat/inputs.d/*.yml + reload.enabled: true + reload.period: 10s + + +# 处理器 +processors: + - dissect: + when: + equals: + log_type: email-log + tokenizer: '%{timestamp} [%{thread}] %{level} %{class} - [%{method_line}] - %{message}' + field: "message" + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true + +# lessie ------------------------- + + - dissect: + when: + equals: + log_type: lessie_search.log + tokenizer: '%{timestamp} - %{level} - %{module} - %{function} - %{message}' + field: "message" + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true + + # 针对带有 [level: | event: | msg: | context:] 的日志,再做一次 dissect + - dissect: + when: + regexp: + mylog.message: '^\[level:.*\]' + tokenizer: '[level: %{event_level} | event: %{event} | msg: %{msg} | context: %{context}]' + field: "mylog.message" + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true + + # 把 context 再拆成独立字段 + - script: + lang: javascript + id: parse_context + source: > + function process(event) { + var ctx = event.Get("mylog.context"); + if (ctx) { + var parts = ctx.split(","); + parts.forEach(function(p) { + var kv = p.split(":"); + if (kv.length == 2) { + event.Put("mylog." + kv[0].trim(), kv[1].trim()); + } + }); + } + } +# lessie ------------------------ + + - decode_json_fields: + when: + equals: + log_type: go.log + fields: ["message"] + target: "" + overwrite_keys: true + add_error_key: true + + +#输出 +output.elasticsearch: + hosts: ["http://106.53.194.199:9200"] + username: "admin" + password: "123456" + index: "%{[environment]}-%{[application]}-%{+yyyy.MM.dd}" # 按天分割索引 + bulk_max_size: 50 # 单批次传输最大文档数 + worker: 1 # 并行工作线程数 + timeout: 15s + +# 日志记录 +logging.level: info +logging.to_files: true +logging.files: + path: /var/log/filebeat + name: filebeat.log + keepfiles: 7 + permissions: 0644 + +# 设置队列和内存使用 +queue.mem: + events: 1024 + flush.min_events: 512 + flush.timeout: 10s + + diff --git a/filebast/s2s3-lessie-server01/s2_go_lessie_sourcing_api.yml b/filebast/s2s3-lessie-server01/s2_go_lessie_sourcing_api.yml new file mode 100644 index 0000000..f03ec68 --- /dev/null +++ b/filebast/s2s3-lessie-server01/s2_go_lessie_sourcing_api.yml @@ -0,0 +1,25 @@ +- type: log + id: s2_go_lessie_sourcing_api + enabled: true + paths: + - /data/webapps/go_lessie_sourcing_api/logs/*.log + follow_symlinks: true + harvester_limit: 1 + fields: + application: go-lessie-sourcing_api # 自定义字段,标识应用名称 + log_type: go.log # 自定义字段,标识日志类型 + environment: s2 # 自定义字段,标识机器环境名称 + instance: webdrive-server # 自定义字段,标识机器名称 + fields_under_root: true + # multiline.pattern: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}' + # multiline.negate: true + # multiline.match: after + ignore_older: 24h # 忽略旧日志文件(避免处理已归档的日志) + scan_frequency: 10s # 定期扫描新文件的频率 + clean_inactive: 25h # 清除超过一天未更新的文件 + close_inactive: 5m # 文件超过5分钟无更新则关闭 + close_renamed: true # 处理被重命名的文件 + start_position: beginning # 从文件的开头读取 + + + diff --git a/filebast/s2s3-lessie-server01/s2_lessie_search.yml b/filebast/s2s3-lessie-server01/s2_lessie_search.yml new file mode 100644 index 0000000..1688e3c --- /dev/null +++ b/filebast/s2s3-lessie-server01/s2_lessie_search.yml @@ -0,0 +1,24 @@ +- type: log + id: s2_lessie_search + enabled: true + paths: + - /data/webapps/lessie_sourcing_agents/logs/lessie_sourcing_agents_*.log + include_lines: ['^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}'] # 只包含匹配该正则表达式的行 + fields: + application: lessie_search + log_type: lessie_search.log + environment: s2 + instance: webdrive-server + ip: 43.159.145.241 + fields_under_root: true + # multiline.pattern: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}' + # multiline.negate: true + # multiline.match: after + ignore_older: 24h # 忽略旧日志文件(避免处理已归档的日志) + scan_frequency: 10s # 定期扫描新文件的频率 + clean_inactive: 25h # 清除超过一天未更新的文件 + close_inactive: 5m # 文件超过5分钟无更新则关闭 + close_renamed: true # 处理被重命名的文件 + start_position: beginning # 从文件的开头读取 + + diff --git a/filebast/s2s3-lessie-server01/s3_go_lessie_sourcing_api.yml b/filebast/s2s3-lessie-server01/s3_go_lessie_sourcing_api.yml new file mode 100644 index 0000000..21bda35 --- /dev/null +++ b/filebast/s2s3-lessie-server01/s3_go_lessie_sourcing_api.yml @@ -0,0 +1,25 @@ +- type: log + id: s3_go_lessie_sourcing_api + enabled: true + paths: + - /data/webapps/s3_go_lessie_sourcing_api/logs/*.log + follow_symlinks: true + harvester_limit: 1 + fields: + application: go-lessie-sourcing_api # 自定义字段,标识应用名称 + log_type: go.log # 自定义字段,标识日志类型 + environment: s3 # 自定义字段,标识机器环境名称 + instance: webdrive-server # 自定义字段,标识机器名称 + fields_under_root: true + # multiline.pattern: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}' + # multiline.negate: true + # multiline.match: after + ignore_older: 24h # 忽略旧日志文件(避免处理已归档的日志) + scan_frequency: 10s # 定期扫描新文件的频率 + clean_inactive: 25h # 清除超过一天未更新的文件 + close_inactive: 5m # 文件超过5分钟无更新则关闭 + close_renamed: true # 处理被重命名的文件 + start_position: beginning # 从文件的开头读取 + + + diff --git a/filebast/s2s3-lessie-server01/s3_lessie_search.yml b/filebast/s2s3-lessie-server01/s3_lessie_search.yml new file mode 100644 index 0000000..c277869 --- /dev/null +++ b/filebast/s2s3-lessie-server01/s3_lessie_search.yml @@ -0,0 +1,25 @@ +- type: log + id: s3_lessie_search + enabled: true + paths: + - /data/webapps/qmm_sourcing_agents/logs/lessie_sourcing_agents_*.log + include_lines: ['^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}'] # 只包含匹配该正则表达式的行 + fields: + application: lessie_search + log_type: lessie_search.log + environment: s3 + instance: webdrive-server + ip: 43.159.145.241 + fields_under_root: true + # multiline.pattern: '^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}' + # multiline.negate: true + # multiline.match: after + ignore_older: 24h # 忽略旧日志文件(避免处理已归档的日志) + scan_frequency: 10s # 定期扫描新文件的频率 + clean_inactive: 25h # 清除超过一天未更新的文件 + close_inactive: 5m # 文件超过5分钟无更新则关闭 + close_renamed: true # 处理被重命名的文件 + start_position: beginning # 从文件的开头读取 + + + diff --git a/filebast/sit/filebeat.yml b/filebast/sit/filebeat.yml index cd096ec..a1a251e 100644 --- a/filebast/sit/filebeat.yml +++ b/filebast/sit/filebeat.yml @@ -1,7 +1,7 @@ +setup.template.name: "lessie-sit" +setup.template.pattern: "lessie-sit*" setup.template.enabled: true setup.ilm.enabled: true -setup.template.name: "sit-flymoonlog" -setup.template.pattern: "sit-flymoonlog*" @@ -19,33 +19,42 @@ processors: - dissect: when: equals: - log_type: sys-info - tokenizer: '%{timestamp} [%{thread}] %{log_level} %{class_name} - [%{method},%{line}] - %{message}' + log_type: admin.log + tokenizer: '%{timestamp} [%{thread}] %{log_level} %{log_message}' field: "message" target_prefix: "parsed_sys_info" - # - include_fields: - # fields: ["@timestamp", "log_type", "message", "application", "host.ip", "host.name", "log.file.path", "parsed_sys_info.timestamp", "parsed_sys_info.log_level", "parsed_sys_info.message", "parsed_sys_info.method", "parsed_sys_info.thread", "_id", "_index" ] + ignore_missing: true + overwrite_keys: false - dissect: when: equals: - log_type: sys-error - tokenizer: '%{timestamp} [%{thread}] %{log_level} %{logger} - [%{method},%{line}] - %{message}' + log_type: email.log + tokenizer: '%{timestamp} [%{thread}] %{level} %{class} - [%{method_line}] - %{message}' field: "message" - target_prefix: "parsed_sys_error" - # - include_fields: - # fields: ["@timestamp", "log_type", "message", "application", "host.ip", "host.name", "log.file.path", "parsed_sys_info.timestamp", "parsed_sys_info.log_level", "parsed_sys_info.message", "parsed_sys_info.method", "parsed_sys_info.thread", "_id", "_index", "parsed_sys_info.logger" ] + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true - dissect: when: equals: - log_type: sys-user - tokenizer: '%{timestamp} [%{thread}] %{log_level} %{module} - [%{method},%{line}] - %{message}' + log_type: agent.log + tokenizer: '%{timestamp} %{level} - [%{method},%{line}] - %{message}' field: "message" - target_prefix: "parsed_sys_user" - # - include_fields: - # fields: ["@timestamp", "log_type", "message", "application", "host.ip", "host.name", "log.file.path", "parsed_sys_info.timestamp", "parsed_sys_info.log_level", "parsed_sys_info.message", "parsed_sys_info.method", "parsed_sys_info.thread", "_id", "_index", "parsed_sys_info.module" ] + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true + - dissect: + when: + equals: + log_type: payment.log + tokenizer: '%{timestamp} [%{thread}] %{level} %{class} - [%{method},%{line}] - %{message}' + field: "message" + target_prefix: "mylog" + ignore_missing: true + overwrite_keys: true #输出 @@ -53,7 +62,7 @@ output.elasticsearch: hosts: ["http://192.168.60.21:9200"] username: "admin" password: "123456" - index: "sit-flymoonlog-%{[environment]}-%{[application]}-%{+yyyy.MM}" # 按月分割索引 + index: "%{[environment]}-%{[application]}-%{+yyyy.MM.dd}" # 按天分割索引 bulk_max_size: 50 # 单批次传输最大文档数 worker: 1 # 并行工作线程数 timeout: 15s diff --git a/filebast/sit/fly-moon-payment.yml b/filebast/sit/fly-moon-payment.yml index bb5606f..e3e20f4 100644 --- a/filebast/sit/fly-moon-payment.yml +++ b/filebast/sit/fly-moon-payment.yml @@ -1,14 +1,15 @@ - type: log - id: pord01_fly-moon-agent + id: sit_flymoon-payment enabled: true paths: - - /root/logs/flymoon-agent/sys-info.log + - /root/logs/flymoon-payment/sys-info.log fields: - application: flymoon-agent # 自定义字段,标识应用名称 - log_type: sys-info # 自定义字段,标识日志类型 - environment: pord01 # 自定义字段,标识机器环境名称 + application: flymoon-payment + log_type: payment.log + environment: sit + instance: sit-server fields_under_root: true - multiline.pattern: '^\d{2}:\d{2}:\d{2}\.\d{3}' # 针对email的sys-info.log的日志格式多行 + multiline.pattern: '^\d{2}:\d{2}:\d{2}\.\d{3}' multiline.negate: true multiline.match: after ignore_older: 24h # 忽略旧日志文件(避免处理已归档的日志) @@ -17,6 +18,4 @@ close_inactive: 5m # 文件超过5分钟无更新则关闭 close_renamed: true # 处理被重命名的文件 start_position: beginning # 从文件的开头读取 - - \ No newline at end of file diff --git a/filebast/sit/flymoon-admin.yml b/filebast/sit/flymoon-admin.yml index 40409da..0b73670 100644 --- a/filebast/sit/flymoon-admin.yml +++ b/filebast/sit/flymoon-admin.yml @@ -1,12 +1,13 @@ -- type: filestream - id: input_sit_flymoon-admin_sys-info +- type: log + id: sit_flymoon-admin enabled: true paths: - /root/logs/flymoon-admin/sys-info.log fields: application: flymoon-admin # 自定义字段,标识应用名称 - log_type: sys-info # 自定义字段,标识日志类型 + log_type: admin.log # 自定义字段,标识日志类型 environment: sit # 自定义字段,标识机器环境名称 + instance: sit-server # 自定义字段,标识机器名称 fields_under_root: true multiline.pattern: '^\d{2}:\d{2}:\d{2}\.\d{3}' # 针对info的日志格式 multiline.negate: true @@ -16,57 +17,4 @@ clean_inactive: 25h # 清除超过一天未更新的文件 close_inactive: 5m # 文件超过5分钟无更新则关闭 close_renamed: true # 处理被重命名的文件 - start_position: beginning # 从文件的开头读取 - -# - type: filestream -# id: input_sit_flymoon-admin_sys-error -# enabled: true -# paths: -# - /root/logs/flymoon-admin/sys-error.log -# fields: -# application: flymoon-admin # 自定义字段,标识应用名称 -# log_type: sys-error # 自定义字段,标识日志类型 -# environment: sit # 自定义字段,标识机器环境名称 -# fields_under_root: true -# multiline.pattern: '^\d{2}:\d{2}:\d{2}\.\d{3}' # 针对error的日志格式 -# multiline.negate: true -# multiline.match: after -# ignore_older: 24h # 忽略旧日志文件(避免处理已归档的日志) -# scan_frequency: 10s # 定期扫描新文件的频率 -# clean_inactive: 25h # 清除超过一天未更新的文件 -# close_inactive: 5m # 文件超过5分钟无更新则关闭 -# close_renamed: true # 处理被重命名的文件 -# start_position: beginning # 从文件的开头读取 - -# - type: filestream -# id: input_sit_flymoon-admin_sys-user -# enabled: true -# paths: -# - /root/logs/flymoon-admin/sys-user.log -# fields: -# application: flymoon-admin # 自定义字段,标识应用名称 -# log_type: sys-user # 自定义字段,标识日志类型 -# environment: sit # 自定义字段,标识机器环境名称 -# fields_under_root: true -# multiline.pattern: '^\d{2}:\d{2}:\d{2}\.\d{3}' # 针对user的日志格式 -# multiline.negate: true -# multiline.match: after -# ignore_older: 24h # 忽略旧日志文件(避免处理已归档的日志) -# scan_frequency: 10s # 定期扫描新文件的频率 -# clean_inactive: 25h # 清除超过一天未更新的文件 -# close_inactive: 5m # 文件超过5分钟无更新则关闭 -# close_renamed: true # 处理被重命名的文件 -# start_position: beginning # 从文件的开头读取 - - - - -#设置索引模版 -# setup.template.name: "sit-flymoon-admin" -# setup.template.pattern: "sit-flymoon-admin*" - -# output.elasticsearch: -# hosts: ["http://192.168.60.21:9200"] -# username: "admin" -# password: "123456" -# index: "sit-flymoon-admin-%{+yyyy.MM}" # 按月分割索引 + start_position: beginning # 从文件的开头读取 \ No newline at end of file diff --git a/filebast/sit/flymoon-agent.yml b/filebast/sit/flymoon-agent.yml new file mode 100644 index 0000000..dd5ca44 --- /dev/null +++ b/filebast/sit/flymoon-agent.yml @@ -0,0 +1,23 @@ +- type: log + id: sit_flymoon-agent + enabled: true + paths: + - /root/logs/flymoon-agent/sys-info.log + fields: + application: flymoon-agent # 自定义字段,标识应用名称 + log_type: agent.log # 自定义字段,标识日志类型 + environment: sit # 自定义字段,标识机器环境名称 + instance: sit-server # 自定义字段,标识机器名称 + fields_under_root: true + multiline.pattern: '^\d{2}:\d{2}:\d{2}\.\d{3}' # 针对email的sys-info.log的日志格式多行 + multiline.negate: true + multiline.match: after + ignore_older: 24h # 忽略旧日志文件(避免处理已归档的日志) + scan_frequency: 10s # 定期扫描新文件的频率 + clean_inactive: 25h # 清除超过一天未更新的文件 + close_inactive: 5m # 文件超过5分钟无更新则关闭 + close_renamed: true # 处理被重命名的文件 + start_position: beginning # 从文件的开头读取 + + + \ No newline at end of file diff --git a/filebast/sit/flymoon-partner.yml b/filebast/sit/flymoon-partner.yml deleted file mode 100644 index 48bad90..0000000 --- a/filebast/sit/flymoon-partner.yml +++ /dev/null @@ -1,72 +0,0 @@ -- type: filestream - id: input_sit_flymoon-partner_sys-info - enabled: true - paths: - - /root/logs/flymoon-partner/sys-info.log - fields: - application: flymoon-partner # 自定义字段,标识应用名称 - log_type: sys-info # 自定义字段,标识日志类型 - environment: sit # 自定义字段,标识机器环境名称 - fields_under_root: true - multiline.pattern: '^\d{2}:\d{2}:\d{2}\.\d{3}' # 针对info的日志格式多行 - multiline.negate: true - multiline.match: after - ignore_older: 24h # 忽略旧日志文件(避免处理已归档的日志) - scan_frequency: 10s # 定期扫描新文件的频率 - clean_inactive: 25h # 清除超过一天未更新的文件 - close_inactive: 5m # 文件超过5分钟无更新则关闭 - close_renamed: true # 处理被重命名的文件 - start_position: beginning # 从文件的开头读取 - -# - type: filestream -# id: input_sit_flymoon-admin_sys-error -# enabled: true -# paths: -# - /root/logs/flymoon-admin/sys-error.log -# fields: -# application: flymoon-admin # 自定义字段,标识应用名称 -# log_type: sys-error # 自定义字段,标识日志类型 -# environment: sit # 自定义字段,标识机器环境名称 -# fields_under_root: true -# multiline.pattern: '^\d{2}:\d{2}:\d{2}\.\d{3}' # 针对error的日志格式 -# multiline.negate: true -# multiline.match: after -# ignore_older: 24h # 忽略旧日志文件(避免处理已归档的日志) -# scan_frequency: 10s # 定期扫描新文件的频率 -# clean_inactive: 25h # 清除超过一天未更新的文件 -# close_inactive: 5m # 文件超过5分钟无更新则关闭 -# close_renamed: true # 处理被重命名的文件 -# start_position: beginning # 从文件的开头读取 - -# - type: filestream -# id: input_sit_flymoon-admin_sys-user -# enabled: true -# paths: -# - /root/logs/flymoon-admin/sys-user.log -# fields: -# application: flymoon-admin # 自定义字段,标识应用名称 -# log_type: sys-user # 自定义字段,标识日志类型 -# environment: sit # 自定义字段,标识机器环境名称 -# fields_under_root: true -# multiline.pattern: '^\d{2}:\d{2}:\d{2}\.\d{3}' # 针对user的日志格式 -# multiline.negate: true -# multiline.match: after -# ignore_older: 24h # 忽略旧日志文件(避免处理已归档的日志) -# scan_frequency: 10s # 定期扫描新文件的频率 -# clean_inactive: 25h # 清除超过一天未更新的文件 -# close_inactive: 5m # 文件超过5分钟无更新则关闭 -# close_renamed: true # 处理被重命名的文件 -# start_position: beginning # 从文件的开头读取 - - - - -#设置索引模版 -# setup.template.name: "sit-flymoon-admin" -# setup.template.pattern: "sit-flymoon-admin*" - -# output.elasticsearch: -# hosts: ["http://192.168.60.21:9200"] -# username: "admin" -# password: "123456" -# index: "sit-flymoon-admin-%{+yyyy.MM}" # 按月分割索引 diff --git a/filebast/sit/flymoon-task.yml b/filebast/sit/flymoon-task.yml deleted file mode 100644 index 39ac148..0000000 --- a/filebast/sit/flymoon-task.yml +++ /dev/null @@ -1,72 +0,0 @@ -- type: filestream - id: input_sit_flymoon-task_sys-info - enabled: true - paths: - - /root/logs/flymoon-task/sys-info.log - fields: - application: flymoon-task # 自定义字段,标识应用名称 - log_type: sys-info # 自定义字段,标识日志类型 - environment: sit # 自定义字段,标识机器环境名称 - fields_under_root: true - multiline.pattern: '^\d{2}:\d{2}:\d{2}\.\d{3}' # 针对info的日志格式多行 - multiline.negate: true - multiline.match: after - ignore_older: 24h # 忽略旧日志文件(避免处理已归档的日志) - scan_frequency: 10s # 定期扫描新文件的频率 - clean_inactive: 25h # 清除超过一天未更新的文件 - close_inactive: 5m # 文件超过5分钟无更新则关闭 - close_renamed: true # 处理被重命名的文件 - start_position: beginning # 从文件的开头读取 - -# - type: filestream -# id: input_sit_flymoon-admin_sys-error -# enabled: true -# paths: -# - /root/logs/flymoon-admin/sys-error.log -# fields: -# application: flymoon-admin # 自定义字段,标识应用名称 -# log_type: sys-error # 自定义字段,标识日志类型 -# environment: sit # 自定义字段,标识机器环境名称 -# fields_under_root: true -# multiline.pattern: '^\d{2}:\d{2}:\d{2}\.\d{3}' # 针对error的日志格式 -# multiline.negate: true -# multiline.match: after -# ignore_older: 24h # 忽略旧日志文件(避免处理已归档的日志) -# scan_frequency: 10s # 定期扫描新文件的频率 -# clean_inactive: 25h # 清除超过一天未更新的文件 -# close_inactive: 5m # 文件超过5分钟无更新则关闭 -# close_renamed: true # 处理被重命名的文件 -# start_position: beginning # 从文件的开头读取 - -# - type: filestream -# id: input_sit_flymoon-admin_sys-user -# enabled: true -# paths: -# - /root/logs/flymoon-admin/sys-user.log -# fields: -# application: flymoon-admin # 自定义字段,标识应用名称 -# log_type: sys-user # 自定义字段,标识日志类型 -# environment: sit # 自定义字段,标识机器环境名称 -# fields_under_root: true -# multiline.pattern: '^\d{2}:\d{2}:\d{2}\.\d{3}' # 针对user的日志格式 -# multiline.negate: true -# multiline.match: after -# ignore_older: 24h # 忽略旧日志文件(避免处理已归档的日志) -# scan_frequency: 10s # 定期扫描新文件的频率 -# clean_inactive: 25h # 清除超过一天未更新的文件 -# close_inactive: 5m # 文件超过5分钟无更新则关闭 -# close_renamed: true # 处理被重命名的文件 -# start_position: beginning # 从文件的开头读取 - - - - -#设置索引模版 -# setup.template.name: "sit-flymoon-admin" -# setup.template.pattern: "sit-flymoon-admin*" - -# output.elasticsearch: -# hosts: ["http://192.168.60.21:9200"] -# username: "admin" -# password: "123456" -# index: "sit-flymoon-admin-%{+yyyy.MM}" # 按月分割索引 diff --git a/filebast/us-prod-01/filebeat.yml b/filebast/us-prod-01/filebeat.yml index d4728eb..4f735d9 100644 --- a/filebast/us-prod-01/filebeat.yml +++ b/filebast/us-prod-01/filebeat.yml @@ -32,7 +32,7 @@ output.elasticsearch: hosts: ["http://106.53.194.199:9200"] username: "admin" password: "123456" - index: "%{[environment]}-%{[application]}-%{+yyyy.MM}" # 按月分割索引 + index: "%{[environment]}-%{[application]}-%{+yyyy.MM.dd}" # 按天分割索引 bulk_max_size: 50 # 单批次传输最大文档数 worker: 1 # 并行工作线程数 timeout: 15s diff --git a/filebast/us-prod-02/filebeat.yml b/filebast/us-prod-02/filebeat.yml index cff784f..3e0ea95 100644 --- a/filebast/us-prod-02/filebeat.yml +++ b/filebast/us-prod-02/filebeat.yml @@ -50,7 +50,7 @@ output.elasticsearch: hosts: ["http://106.53.194.199:9200"] username: "admin" password: "123456" - index: "%{[environment]}-%{[application]}-%{+yyyy.MM}" # 按月分割索引 + index: "%{[environment]}-%{[application]}-%{+yyyy.MM.dd}" # 按天分割索引 bulk_max_size: 50 # 单批次传输最大文档数 worker: 1 # 并行工作线程数 timeout: 15s diff --git a/filebast/us-prod-03/filebeat.yml b/filebast/us-prod-03/filebeat.yml index 0fe8c45..68aaf77 100644 --- a/filebast/us-prod-03/filebeat.yml +++ b/filebast/us-prod-03/filebeat.yml @@ -49,7 +49,7 @@ output.elasticsearch: hosts: ["http://106.53.194.199:9200"] username: "admin" password: "123456" - index: "%{[environment]}-%{[application]}-%{+yyyy.MM}" # 按月分割索引 + index: "%{[environment]}-%{[application]}-%{+yyyy.MM.dd}" # 按天分割索引 bulk_max_size: 50 # 单批次传输最大文档数 worker: 1 # 并行工作线程数 timeout: 15s diff --git a/filebast/相关命令 b/filebast/相关命令 index d515052..4d93173 100644 --- a/filebast/相关命令 +++ b/filebast/相关命令 @@ -1,4 +1,8 @@ -1、查看启动输出:journalctl -u filebeat -f +1、查看启动输出: +journalctl -u filebeat -f -2、测试 -filebeat test config、filebeat test output \ No newline at end of file +2、查看后100行日志: + + +3、测试 +filebeat test config && filebeat test output \ No newline at end of file