Upload files to "Grafana"
This commit is contained in:
1 parent
36d7d6b65f
commit
092f736331
5 files changed
+351
No files matched your search
@@ -0,0 +1,94 @@
|
||||
networks:
|
||||
logs-network:
|
||||
driver: bridge
|
||||
name: logs-network
|
||||
|
||||
services:
|
||||
# ========== ПРИЕМНИК ЛОГОВ ==========
|
||||
syslog-ng:
|
||||
image: lscr.io/linuxserver/syslog-ng:latest
|
||||
container_name: syslog-ng
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- logs-network
|
||||
ports:
|
||||
- "1514:1514/udp" # MikroTik
|
||||
- "1515:1515/udp" # Synology
|
||||
- "1516:1516/udp" # Proxmox
|
||||
- "1517:1517/udp" # TerraMaster
|
||||
volumes:
|
||||
- /opt/docker/grafana/syslog-ng/config:/config
|
||||
- /opt/docker/grafana/syslog-ng/logs:/var/log/syslog-ng
|
||||
- /opt/docker/grafana/syslog-ng/config/syslog-ng.conf:/etc/syslog-ng/syslog-ng.conf:ro
|
||||
environment:
|
||||
- TZ="Europe/Samara"
|
||||
- PUID=1000
|
||||
- PGID=1000
|
||||
|
||||
# ========== АГЕНТ СБОРА ЛОГОВ ==========
|
||||
promtail:
|
||||
image: grafana/promtail:latest
|
||||
container_name: promtail
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- logs-network
|
||||
depends_on:
|
||||
- loki
|
||||
- syslog-ng
|
||||
volumes:
|
||||
- /opt/docker/grafana/promtail-config.yaml:/etc/promtail/config.yaml:ro
|
||||
- /opt/docker/grafana/syslog-ng/logs:/var/log/syslog-ng:ro
|
||||
command: -config.file=/etc/promtail/config.yaml
|
||||
|
||||
# ========== ХРАНИЛИЩЕ ЛОГОВ ==========
|
||||
loki:
|
||||
image: grafana/loki:latest
|
||||
container_name: loki
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- logs-network
|
||||
ports:
|
||||
- "3100:3100"
|
||||
volumes:
|
||||
- /opt/docker/grafana/loki-config.yaml:/loki-config.yaml:ro
|
||||
- /opt/docker/grafana/data/loki:/loki:rw
|
||||
user: "0:0"
|
||||
command: -config.file=/loki-config.yaml
|
||||
|
||||
# ========== ВЕБ-ИНТЕРФЕЙС ==========
|
||||
grafana:
|
||||
image: grafana/grafana:latest
|
||||
container_name: grafana
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- logs-network
|
||||
ports:
|
||||
- "3005:3000"
|
||||
depends_on:
|
||||
- loki
|
||||
volumes:
|
||||
- /opt/docker/grafana/data/grafana:/var/lib/grafana
|
||||
environment:
|
||||
- GF_SECURITY_ADMIN_USER=admin
|
||||
- GF_SECURITY_ADMIN_PASSWORD=admin
|
||||
- GF_USERS_ALLOW_SIGN_UP=false
|
||||
|
||||
# ========== АВТООЧИСТКА ФАЙЛОВ ==========
|
||||
log-cleaner:
|
||||
image: alpine:latest
|
||||
container_name: log-cleaner
|
||||
restart: unless-stopped
|
||||
networks:
|
||||
- logs-network
|
||||
depends_on:
|
||||
- syslog-ng
|
||||
volumes:
|
||||
- /opt/docker/grafana/syslog-ng/logs:/logs
|
||||
command: >
|
||||
sh -c "
|
||||
while true; do
|
||||
find /logs -name '*.log' -type f -mtime +7 -delete;
|
||||
echo 'Cleanup completed at '$(date);
|
||||
sleep 86400;
|
||||
done
|
||||
"
|
||||
@@ -0,0 +1,43 @@
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_port: 3100
|
||||
|
||||
common:
|
||||
path_prefix: /loki
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
replication_factor: 1
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2020-10-24
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
limits_config:
|
||||
retention_period: 168h # 7 дней
|
||||
reject_old_samples: true
|
||||
reject_old_samples_max_age: 168h
|
||||
max_query_series: 5000
|
||||
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
compaction_interval: 10m
|
||||
retention_enabled: true
|
||||
retention_delete_delay: 2h
|
||||
retention_delete_worker_count: 150
|
||||
delete_request_store: filesystem
|
||||
|
||||
table_manager:
|
||||
retention_deletes_enabled: true
|
||||
retention_period: 168h
|
||||
@@ -0,0 +1,129 @@
|
||||
server:
|
||||
http_listen_port: 9080
|
||||
grpc_listen_port: 0
|
||||
|
||||
positions:
|
||||
filename: /tmp/positions.yaml
|
||||
|
||||
clients:
|
||||
- url: http://loki:3100/loki/api/v1/push
|
||||
|
||||
scrape_configs:
|
||||
- job_name: mikrotik
|
||||
static_configs:
|
||||
- targets:
|
||||
- localhost
|
||||
labels:
|
||||
job: syslog
|
||||
device: mikrotik
|
||||
__path__: /var/log/syslog-ng/mikrotik.log
|
||||
pipeline_stages:
|
||||
- regex:
|
||||
expression: '^(?P<timestamp>\S+)\s+(?P<router_ip>\d+\.\d+\.\d+\.\d+)\s+(?P<message>.*)$'
|
||||
- labels:
|
||||
router_ip:
|
||||
- template:
|
||||
source: router_name
|
||||
template: '{{ if eq .router_ip "10.112.100.254" }}GateWay_01{{ else if eq .router_ip "192.168.114.1" }}Home{{ else if eq .router_ip "172.19.200.2" }}Home-loopback{{ else }}{{ .router_ip }}{{ end }}'
|
||||
- labels:
|
||||
router_name:
|
||||
- template:
|
||||
source: topic
|
||||
template: '{{ if contains "logged in" .message }}account{{ else if contains "logged out" .message }}account{{ else if contains "test from" .message }}script{{ else if contains "log rule changed" .message }}system{{ else if contains "error" .message }}error{{ else if contains "warning" .message }}warning{{ else }}info{{ end }}'
|
||||
- labels:
|
||||
topic:
|
||||
- template:
|
||||
source: level
|
||||
template: '{{ if contains "error" .message }}error{{ else if contains "warning" .message }}warn{{ else if contains "critical" .message }}critical{{ else }}info{{ end }}'
|
||||
- labels:
|
||||
level:
|
||||
|
||||
- job_name: synology
|
||||
static_configs:
|
||||
- targets:
|
||||
- localhost
|
||||
labels:
|
||||
job: syslog
|
||||
device: synology
|
||||
__path__: /var/log/syslog-ng/synology.log
|
||||
pipeline_stages:
|
||||
- drop:
|
||||
expression: '.*(XFERDB is not exist|SLIBLogSetByArray|Fail to send log to socket|Error connecting to target server).*'
|
||||
drop_counter_reason: 'synology_logcenter_noise'
|
||||
- drop:
|
||||
expression: '.*(SLIBCFileGetKeyValue|file_check_key_value|Webapi do not descript|command_client\.cpp|syncobackupd\.socket|disk_status_summary_calculate|setresuid|setresgid).*'
|
||||
drop_counter_reason: 'synology_system_noise'
|
||||
- regex:
|
||||
expression: '^(?P<timestamp>\S+)\s+(?P<facility>\w+)\s+(?P<message>.*)$'
|
||||
- labels:
|
||||
facility:
|
||||
- template:
|
||||
source: category
|
||||
template: '{{ if contains "logged in" .message }}auth{{ else if contains "logged out" .message }}auth{{ else if contains "SSH" .message }}auth{{ else if contains "SMB" .message }}smb{{ else if contains "File Station" .message }}file{{ else if contains "Backup" .message }}backup{{ else if contains "rsync" .message }}backup{{ else }}system{{ end }}'
|
||||
- labels:
|
||||
category:
|
||||
- template:
|
||||
source: level
|
||||
template: '{{ if contains "fail" .message }}error{{ else if contains "Failed" .message }}error{{ else if contains "error" .message }}error{{ else if contains "WARNING" .message }}warn{{ else if contains "refused" .message }}warn{{ else }}info{{ end }}'
|
||||
- labels:
|
||||
level:
|
||||
|
||||
- job_name: proxmox
|
||||
static_configs:
|
||||
- targets:
|
||||
- localhost
|
||||
labels:
|
||||
job: syslog
|
||||
device: proxmox
|
||||
__path__: /var/log/syslog-ng/proxmox.log
|
||||
pipeline_stages:
|
||||
- regex:
|
||||
expression: '^(?P<timestamp>\S+)\s+(?P<source_ip>\d+\.\d+\.\d+\.\d+)\s+(?P<program>\w+)(?:\[(?P<pid>\d+)\])?:\s+(?P<message>.*)$'
|
||||
- labels:
|
||||
program:
|
||||
source_ip:
|
||||
- template:
|
||||
source: level
|
||||
template: '{{ if contains "error" .message }}error{{ else if contains "warning" .message }}warn{{ else if contains "critical" .message }}critical{{ else }}info{{ end }}'
|
||||
- labels:
|
||||
level:
|
||||
|
||||
- job_name: terramaster
|
||||
static_configs:
|
||||
- targets:
|
||||
- localhost
|
||||
labels:
|
||||
job: syslog
|
||||
device: terramaster
|
||||
__path__: /var/log/syslog-ng/terramaster.log
|
||||
pipeline_stages:
|
||||
- drop:
|
||||
expression: '.*(will insert msg|INSERT INTO|SLOW SQL|Sending message to all clients|SendMail now|msgcenter-go|msgcente>
|
||||
drop_counter_reason: 'terramaster_debug'
|
||||
- drop:
|
||||
expression: '.*(pam_winbind|PAM_CONV|PAM_RHOST|ITEM\(PAM).*'
|
||||
drop_counter_reason: 'terramaster_pam'
|
||||
- drop:
|
||||
expression: '.*(#033\[|\x1b\[|\[0m|\[1m|\[31;1m|\[32m|\[33m|\[35m).*'
|
||||
drop_counter_reason: 'terramaster_ansi'
|
||||
- drop:
|
||||
expression: '.*(Listen normally on|Listening on routing socket|Deactivated successfully|Started Network Time Service|n>
|
||||
drop_counter_reason: 'terramaster_ntp'
|
||||
- drop:
|
||||
expression: '.*(veth[0-9a-f]+|br-[0-9a-f]+|bridge2|docker0).*'
|
||||
drop_counter_reason: 'terramaster_docker'
|
||||
- drop:
|
||||
expression: '.*(TIME_ERROR: 0x2041).*'
|
||||
drop_counter_reason: 'terramaster_time_error'
|
||||
- regex:
|
||||
expression: '^(?P<timestamp>\S+)\s+(?P<source_ip>\d+\.\d+\.\d+\.\d+)\s+(?P<message>.*)$'
|
||||
- labels:
|
||||
source_ip:
|
||||
- regex:
|
||||
source: message
|
||||
expression: '\[(?P<level_raw>INFO|WARN|WARNING|ERROR|CRITICAL|DEBUG|DEBU)\]'
|
||||
- template:
|
||||
source: level
|
||||
template: '{{ if eq .level_raw "WARNING" }}warn{{ else if eq .level_raw "CRITICAL" }}critical{{ else if eq .level_raw >
|
||||
- labels:
|
||||
level:
|
||||
@@ -0,0 +1,46 @@
|
||||
server:
|
||||
http_listen_port: 9080
|
||||
grpc_listen_port: 0
|
||||
|
||||
positions:
|
||||
filename: /tmp/positions.yaml
|
||||
|
||||
clients:
|
||||
- url: http://loki:3100/loki/api/v1/push
|
||||
|
||||
scrape_configs:
|
||||
- job_name: mikrotik
|
||||
static_configs:
|
||||
- targets:
|
||||
- localhost
|
||||
labels:
|
||||
job: syslog
|
||||
device: mikrotik
|
||||
__path__: /var/log/syslog-ng/mikrotik.log
|
||||
|
||||
- job_name: synology
|
||||
static_configs:
|
||||
- targets:
|
||||
- localhost
|
||||
labels:
|
||||
job: syslog
|
||||
device: synology
|
||||
__path__: /var/log/syslog-ng/synology.log
|
||||
|
||||
- job_name: proxmox
|
||||
static_configs:
|
||||
- targets:
|
||||
- localhost
|
||||
labels:
|
||||
job: syslog
|
||||
device: proxmox
|
||||
__path__: /var/log/syslog-ng/proxmox.log
|
||||
|
||||
- job_name: terramaster
|
||||
static_configs:
|
||||
- targets:
|
||||
- localhost
|
||||
labels:
|
||||
job: syslog
|
||||
device: terramaster
|
||||
__path__: /var/log/syslog-ng/terramaster.log
|
||||
@@ -0,0 +1,39 @@
|
||||
@version: 4.2
|
||||
@include "scl.conf"
|
||||
|
||||
source s_mikrotik {
|
||||
network(ip(0.0.0.0) port(1514) transport("udp"));
|
||||
};
|
||||
|
||||
source s_synology {
|
||||
network(ip(0.0.0.0) port(1515) transport("udp"));
|
||||
};
|
||||
|
||||
source s_proxmox {
|
||||
network(ip(0.0.0.0) port(1516) transport("udp"));
|
||||
};
|
||||
|
||||
source s_terramaster {
|
||||
network(ip(0.0.0.0) port(1517) transport("udp"));
|
||||
};
|
||||
|
||||
destination d_mikrotik {
|
||||
file("/var/log/syslog-ng/mikrotik.log" template("${ISODATE} ${HOST} ${MESSAGE}\n"));
|
||||
};
|
||||
|
||||
destination d_synology {
|
||||
file("/var/log/syslog-ng/synology.log" template("${ISODATE} ${HOST} ${MESSAGE}\n"));
|
||||
};
|
||||
|
||||
destination d_proxmox {
|
||||
file("/var/log/syslog-ng/proxmox.log" template("${ISODATE} ${HOST} ${MESSAGE}\n"));
|
||||
};
|
||||
|
||||
destination d_terramaster {
|
||||
file("/var/log/syslog-ng/terramaster.log" template("${ISODATE} ${HOST} ${MESSAGE}\n"));
|
||||
};
|
||||
|
||||
log { source(s_mikrotik); destination(d_mikrotik); };
|
||||
log { source(s_synology); destination(d_synology); };
|
||||
log { source(s_proxmox); destination(d_proxmox); };
|
||||
log { source(s_terramaster); destination(d_terramaster); };
|
||||
Reference in new issue
Block a user