Upload files to "Grafana"
This commit is contained in:
1 parent
36d7d6b65f
commit
092f736331
5 files changed
+351
No files matched your search
@@ -0,0 +1,94 @@
|
|||||||
|
networks:
|
||||||
|
logs-network:
|
||||||
|
driver: bridge
|
||||||
|
name: logs-network
|
||||||
|
|
||||||
|
services:
|
||||||
|
# ========== ПРИЕМНИК ЛОГОВ ==========
|
||||||
|
syslog-ng:
|
||||||
|
image: lscr.io/linuxserver/syslog-ng:latest
|
||||||
|
container_name: syslog-ng
|
||||||
|
restart: unless-stopped
|
||||||
|
networks:
|
||||||
|
- logs-network
|
||||||
|
ports:
|
||||||
|
- "1514:1514/udp" # MikroTik
|
||||||
|
- "1515:1515/udp" # Synology
|
||||||
|
- "1516:1516/udp" # Proxmox
|
||||||
|
- "1517:1517/udp" # TerraMaster
|
||||||
|
volumes:
|
||||||
|
- /opt/docker/grafana/syslog-ng/config:/config
|
||||||
|
- /opt/docker/grafana/syslog-ng/logs:/var/log/syslog-ng
|
||||||
|
- /opt/docker/grafana/syslog-ng/config/syslog-ng.conf:/etc/syslog-ng/syslog-ng.conf:ro
|
||||||
|
environment:
|
||||||
|
- TZ="Europe/Samara"
|
||||||
|
- PUID=1000
|
||||||
|
- PGID=1000
|
||||||
|
|
||||||
|
# ========== АГЕНТ СБОРА ЛОГОВ ==========
|
||||||
|
promtail:
|
||||||
|
image: grafana/promtail:latest
|
||||||
|
container_name: promtail
|
||||||
|
restart: unless-stopped
|
||||||
|
networks:
|
||||||
|
- logs-network
|
||||||
|
depends_on:
|
||||||
|
- loki
|
||||||
|
- syslog-ng
|
||||||
|
volumes:
|
||||||
|
- /opt/docker/grafana/promtail-config.yaml:/etc/promtail/config.yaml:ro
|
||||||
|
- /opt/docker/grafana/syslog-ng/logs:/var/log/syslog-ng:ro
|
||||||
|
command: -config.file=/etc/promtail/config.yaml
|
||||||
|
|
||||||
|
# ========== ХРАНИЛИЩЕ ЛОГОВ ==========
|
||||||
|
loki:
|
||||||
|
image: grafana/loki:latest
|
||||||
|
container_name: loki
|
||||||
|
restart: unless-stopped
|
||||||
|
networks:
|
||||||
|
- logs-network
|
||||||
|
ports:
|
||||||
|
- "3100:3100"
|
||||||
|
volumes:
|
||||||
|
- /opt/docker/grafana/loki-config.yaml:/loki-config.yaml:ro
|
||||||
|
- /opt/docker/grafana/data/loki:/loki:rw
|
||||||
|
user: "0:0"
|
||||||
|
command: -config.file=/loki-config.yaml
|
||||||
|
|
||||||
|
# ========== ВЕБ-ИНТЕРФЕЙС ==========
|
||||||
|
grafana:
|
||||||
|
image: grafana/grafana:latest
|
||||||
|
container_name: grafana
|
||||||
|
restart: unless-stopped
|
||||||
|
networks:
|
||||||
|
- logs-network
|
||||||
|
ports:
|
||||||
|
- "3005:3000"
|
||||||
|
depends_on:
|
||||||
|
- loki
|
||||||
|
volumes:
|
||||||
|
- /opt/docker/grafana/data/grafana:/var/lib/grafana
|
||||||
|
environment:
|
||||||
|
- GF_SECURITY_ADMIN_USER=admin
|
||||||
|
- GF_SECURITY_ADMIN_PASSWORD=admin
|
||||||
|
- GF_USERS_ALLOW_SIGN_UP=false
|
||||||
|
|
||||||
|
# ========== АВТООЧИСТКА ФАЙЛОВ ==========
|
||||||
|
log-cleaner:
|
||||||
|
image: alpine:latest
|
||||||
|
container_name: log-cleaner
|
||||||
|
restart: unless-stopped
|
||||||
|
networks:
|
||||||
|
- logs-network
|
||||||
|
depends_on:
|
||||||
|
- syslog-ng
|
||||||
|
volumes:
|
||||||
|
- /opt/docker/grafana/syslog-ng/logs:/logs
|
||||||
|
command: >
|
||||||
|
sh -c "
|
||||||
|
while true; do
|
||||||
|
find /logs -name '*.log' -type f -mtime +7 -delete;
|
||||||
|
echo 'Cleanup completed at '$(date);
|
||||||
|
sleep 86400;
|
||||||
|
done
|
||||||
|
"
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
auth_enabled: false
|
||||||
|
|
||||||
|
server:
|
||||||
|
http_listen_port: 3100
|
||||||
|
|
||||||
|
common:
|
||||||
|
path_prefix: /loki
|
||||||
|
storage:
|
||||||
|
filesystem:
|
||||||
|
chunks_directory: /loki/chunks
|
||||||
|
rules_directory: /loki/rules
|
||||||
|
replication_factor: 1
|
||||||
|
ring:
|
||||||
|
kvstore:
|
||||||
|
store: inmemory
|
||||||
|
|
||||||
|
schema_config:
|
||||||
|
configs:
|
||||||
|
- from: 2020-10-24
|
||||||
|
store: tsdb
|
||||||
|
object_store: filesystem
|
||||||
|
schema: v13
|
||||||
|
index:
|
||||||
|
prefix: index_
|
||||||
|
period: 24h
|
||||||
|
|
||||||
|
limits_config:
|
||||||
|
retention_period: 168h # 7 дней
|
||||||
|
reject_old_samples: true
|
||||||
|
reject_old_samples_max_age: 168h
|
||||||
|
max_query_series: 5000
|
||||||
|
|
||||||
|
compactor:
|
||||||
|
working_directory: /loki/compactor
|
||||||
|
compaction_interval: 10m
|
||||||
|
retention_enabled: true
|
||||||
|
retention_delete_delay: 2h
|
||||||
|
retention_delete_worker_count: 150
|
||||||
|
delete_request_store: filesystem
|
||||||
|
|
||||||
|
table_manager:
|
||||||
|
retention_deletes_enabled: true
|
||||||
|
retention_period: 168h
|
||||||
@@ -0,0 +1,129 @@
|
|||||||
|
server:
|
||||||
|
http_listen_port: 9080
|
||||||
|
grpc_listen_port: 0
|
||||||
|
|
||||||
|
positions:
|
||||||
|
filename: /tmp/positions.yaml
|
||||||
|
|
||||||
|
clients:
|
||||||
|
- url: http://loki:3100/loki/api/v1/push
|
||||||
|
|
||||||
|
scrape_configs:
|
||||||
|
- job_name: mikrotik
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- localhost
|
||||||
|
labels:
|
||||||
|
job: syslog
|
||||||
|
device: mikrotik
|
||||||
|
__path__: /var/log/syslog-ng/mikrotik.log
|
||||||
|
pipeline_stages:
|
||||||
|
- regex:
|
||||||
|
expression: '^(?P<timestamp>\S+)\s+(?P<router_ip>\d+\.\d+\.\d+\.\d+)\s+(?P<message>.*)$'
|
||||||
|
- labels:
|
||||||
|
router_ip:
|
||||||
|
- template:
|
||||||
|
source: router_name
|
||||||
|
template: '{{ if eq .router_ip "10.112.100.254" }}GateWay_01{{ else if eq .router_ip "192.168.114.1" }}Home{{ else if eq .router_ip "172.19.200.2" }}Home-loopback{{ else }}{{ .router_ip }}{{ end }}'
|
||||||
|
- labels:
|
||||||
|
router_name:
|
||||||
|
- template:
|
||||||
|
source: topic
|
||||||
|
template: '{{ if contains "logged in" .message }}account{{ else if contains "logged out" .message }}account{{ else if contains "test from" .message }}script{{ else if contains "log rule changed" .message }}system{{ else if contains "error" .message }}error{{ else if contains "warning" .message }}warning{{ else }}info{{ end }}'
|
||||||
|
- labels:
|
||||||
|
topic:
|
||||||
|
- template:
|
||||||
|
source: level
|
||||||
|
template: '{{ if contains "error" .message }}error{{ else if contains "warning" .message }}warn{{ else if contains "critical" .message }}critical{{ else }}info{{ end }}'
|
||||||
|
- labels:
|
||||||
|
level:
|
||||||
|
|
||||||
|
- job_name: synology
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- localhost
|
||||||
|
labels:
|
||||||
|
job: syslog
|
||||||
|
device: synology
|
||||||
|
__path__: /var/log/syslog-ng/synology.log
|
||||||
|
pipeline_stages:
|
||||||
|
- drop:
|
||||||
|
expression: '.*(XFERDB is not exist|SLIBLogSetByArray|Fail to send log to socket|Error connecting to target server).*'
|
||||||
|
drop_counter_reason: 'synology_logcenter_noise'
|
||||||
|
- drop:
|
||||||
|
expression: '.*(SLIBCFileGetKeyValue|file_check_key_value|Webapi do not descript|command_client\.cpp|syncobackupd\.socket|disk_status_summary_calculate|setresuid|setresgid).*'
|
||||||
|
drop_counter_reason: 'synology_system_noise'
|
||||||
|
- regex:
|
||||||
|
expression: '^(?P<timestamp>\S+)\s+(?P<facility>\w+)\s+(?P<message>.*)$'
|
||||||
|
- labels:
|
||||||
|
facility:
|
||||||
|
- template:
|
||||||
|
source: category
|
||||||
|
template: '{{ if contains "logged in" .message }}auth{{ else if contains "logged out" .message }}auth{{ else if contains "SSH" .message }}auth{{ else if contains "SMB" .message }}smb{{ else if contains "File Station" .message }}file{{ else if contains "Backup" .message }}backup{{ else if contains "rsync" .message }}backup{{ else }}system{{ end }}'
|
||||||
|
- labels:
|
||||||
|
category:
|
||||||
|
- template:
|
||||||
|
source: level
|
||||||
|
template: '{{ if contains "fail" .message }}error{{ else if contains "Failed" .message }}error{{ else if contains "error" .message }}error{{ else if contains "WARNING" .message }}warn{{ else if contains "refused" .message }}warn{{ else }}info{{ end }}'
|
||||||
|
- labels:
|
||||||
|
level:
|
||||||
|
|
||||||
|
- job_name: proxmox
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- localhost
|
||||||
|
labels:
|
||||||
|
job: syslog
|
||||||
|
device: proxmox
|
||||||
|
__path__: /var/log/syslog-ng/proxmox.log
|
||||||
|
pipeline_stages:
|
||||||
|
- regex:
|
||||||
|
expression: '^(?P<timestamp>\S+)\s+(?P<source_ip>\d+\.\d+\.\d+\.\d+)\s+(?P<program>\w+)(?:\[(?P<pid>\d+)\])?:\s+(?P<message>.*)$'
|
||||||
|
- labels:
|
||||||
|
program:
|
||||||
|
source_ip:
|
||||||
|
- template:
|
||||||
|
source: level
|
||||||
|
template: '{{ if contains "error" .message }}error{{ else if contains "warning" .message }}warn{{ else if contains "critical" .message }}critical{{ else }}info{{ end }}'
|
||||||
|
- labels:
|
||||||
|
level:
|
||||||
|
|
||||||
|
- job_name: terramaster
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- localhost
|
||||||
|
labels:
|
||||||
|
job: syslog
|
||||||
|
device: terramaster
|
||||||
|
__path__: /var/log/syslog-ng/terramaster.log
|
||||||
|
pipeline_stages:
|
||||||
|
- drop:
|
||||||
|
expression: '.*(will insert msg|INSERT INTO|SLOW SQL|Sending message to all clients|SendMail now|msgcenter-go|msgcente>
|
||||||
|
drop_counter_reason: 'terramaster_debug'
|
||||||
|
- drop:
|
||||||
|
expression: '.*(pam_winbind|PAM_CONV|PAM_RHOST|ITEM\(PAM).*'
|
||||||
|
drop_counter_reason: 'terramaster_pam'
|
||||||
|
- drop:
|
||||||
|
expression: '.*(#033\[|\x1b\[|\[0m|\[1m|\[31;1m|\[32m|\[33m|\[35m).*'
|
||||||
|
drop_counter_reason: 'terramaster_ansi'
|
||||||
|
- drop:
|
||||||
|
expression: '.*(Listen normally on|Listening on routing socket|Deactivated successfully|Started Network Time Service|n>
|
||||||
|
drop_counter_reason: 'terramaster_ntp'
|
||||||
|
- drop:
|
||||||
|
expression: '.*(veth[0-9a-f]+|br-[0-9a-f]+|bridge2|docker0).*'
|
||||||
|
drop_counter_reason: 'terramaster_docker'
|
||||||
|
- drop:
|
||||||
|
expression: '.*(TIME_ERROR: 0x2041).*'
|
||||||
|
drop_counter_reason: 'terramaster_time_error'
|
||||||
|
- regex:
|
||||||
|
expression: '^(?P<timestamp>\S+)\s+(?P<source_ip>\d+\.\d+\.\d+\.\d+)\s+(?P<message>.*)$'
|
||||||
|
- labels:
|
||||||
|
source_ip:
|
||||||
|
- regex:
|
||||||
|
source: message
|
||||||
|
expression: '\[(?P<level_raw>INFO|WARN|WARNING|ERROR|CRITICAL|DEBUG|DEBU)\]'
|
||||||
|
- template:
|
||||||
|
source: level
|
||||||
|
template: '{{ if eq .level_raw "WARNING" }}warn{{ else if eq .level_raw "CRITICAL" }}critical{{ else if eq .level_raw >
|
||||||
|
- labels:
|
||||||
|
level:
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
server:
|
||||||
|
http_listen_port: 9080
|
||||||
|
grpc_listen_port: 0
|
||||||
|
|
||||||
|
positions:
|
||||||
|
filename: /tmp/positions.yaml
|
||||||
|
|
||||||
|
clients:
|
||||||
|
- url: http://loki:3100/loki/api/v1/push
|
||||||
|
|
||||||
|
scrape_configs:
|
||||||
|
- job_name: mikrotik
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- localhost
|
||||||
|
labels:
|
||||||
|
job: syslog
|
||||||
|
device: mikrotik
|
||||||
|
__path__: /var/log/syslog-ng/mikrotik.log
|
||||||
|
|
||||||
|
- job_name: synology
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- localhost
|
||||||
|
labels:
|
||||||
|
job: syslog
|
||||||
|
device: synology
|
||||||
|
__path__: /var/log/syslog-ng/synology.log
|
||||||
|
|
||||||
|
- job_name: proxmox
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- localhost
|
||||||
|
labels:
|
||||||
|
job: syslog
|
||||||
|
device: proxmox
|
||||||
|
__path__: /var/log/syslog-ng/proxmox.log
|
||||||
|
|
||||||
|
- job_name: terramaster
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- localhost
|
||||||
|
labels:
|
||||||
|
job: syslog
|
||||||
|
device: terramaster
|
||||||
|
__path__: /var/log/syslog-ng/terramaster.log
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
@version: 4.2
|
||||||
|
@include "scl.conf"
|
||||||
|
|
||||||
|
source s_mikrotik {
|
||||||
|
network(ip(0.0.0.0) port(1514) transport("udp"));
|
||||||
|
};
|
||||||
|
|
||||||
|
source s_synology {
|
||||||
|
network(ip(0.0.0.0) port(1515) transport("udp"));
|
||||||
|
};
|
||||||
|
|
||||||
|
source s_proxmox {
|
||||||
|
network(ip(0.0.0.0) port(1516) transport("udp"));
|
||||||
|
};
|
||||||
|
|
||||||
|
source s_terramaster {
|
||||||
|
network(ip(0.0.0.0) port(1517) transport("udp"));
|
||||||
|
};
|
||||||
|
|
||||||
|
destination d_mikrotik {
|
||||||
|
file("/var/log/syslog-ng/mikrotik.log" template("${ISODATE} ${HOST} ${MESSAGE}\n"));
|
||||||
|
};
|
||||||
|
|
||||||
|
destination d_synology {
|
||||||
|
file("/var/log/syslog-ng/synology.log" template("${ISODATE} ${HOST} ${MESSAGE}\n"));
|
||||||
|
};
|
||||||
|
|
||||||
|
destination d_proxmox {
|
||||||
|
file("/var/log/syslog-ng/proxmox.log" template("${ISODATE} ${HOST} ${MESSAGE}\n"));
|
||||||
|
};
|
||||||
|
|
||||||
|
destination d_terramaster {
|
||||||
|
file("/var/log/syslog-ng/terramaster.log" template("${ISODATE} ${HOST} ${MESSAGE}\n"));
|
||||||
|
};
|
||||||
|
|
||||||
|
log { source(s_mikrotik); destination(d_mikrotik); };
|
||||||
|
log { source(s_synology); destination(d_synology); };
|
||||||
|
log { source(s_proxmox); destination(d_proxmox); };
|
||||||
|
log { source(s_terramaster); destination(d_terramaster); };
|
||||||
Reference in new issue
Block a user