diff --git a/README.md b/README.md index 0c54fb8..963336e 100644 --- a/README.md +++ b/README.md @@ -11,13 +11,16 @@ One-file [Grafana Alloy](https://grafana.com/docs/alloy/latest/) setup that ship | Source | Component | Notes | |---|---|---| -| Host metrics | `prometheus.exporter.unix` | CPU, memory, load, disk I/O, filesystem, network, uname, boot time | +| Host metrics | `prometheus.exporter.unix` | CPU, memory, load, disk I/O, filesystem, network, uname, boot time, systemd, vmstat, sockstat — full default-collector set minus `ipvs/btrfs/infiniband/xfs/zfs` | | Container metrics | `prometheus.exporter.cadvisor` | CPU, memory, fs usage/limit, network, `last_seen` | | Container logs | `loki.source.docker` | all running containers, labeled with `container`, `stream`, `instance` | -| System logs | `loki.source.journal` (via `journal_module`) | systemd journal with `unit`, `boot_id`, `transport`, `level` labels | +| System logs (journal) | `loki.source.journal` (via `journal_module`) | systemd journal with `unit`, `boot_id`, `transport`, `level` labels | +| System logs (files) | `loki.source.file` | `/var/log/syslog`, `/var/log/messages`, `/var/log/*.log` | | Remote config | `remotecfg` | polls Grafana Fleet Management every 60s | -`keep`-filter on metric names trims the firehose down to the standard Grafana Cloud integration dashboards (node-exporter + docker). +Filtering follows the upstream Grafana Cloud integration configs: node metrics drop only `node_scrape_collector_*` meta-metrics (everything else ships); cadvisor uses the documented allowlist; logs are unfiltered. + +> **Log duplication caveat.** On systems where rsyslog mirrors journald to `/var/log/syslog` (e.g. Debian/Ubuntu defaults), enabling both pipelines double-ships the same lines. If that's the case for your hosts, drop one source — typically the file-based one is redundant on systemd-only stacks. ## Quick start @@ -57,7 +60,7 @@ Runs `privileged: true` + `network_mode: host`, matching the upstream Grafana Cl | `/dev/disk/:/dev/disk:ro` | node-exporter diskstats device labels | | `/var/run/docker.sock` | `discovery.docker` + `loki.source.docker` | | `/var/lib/docker:ro` | cadvisor container metadata | -| `/var/log/journal:ro` | `loki.source.journal` | +| `/var/log:/var/log:ro` | `loki.source.journal` (`/var/log/journal`) + `loki.source.file` (syslog/messages/*.log) | | `/etc/machine-id:ro` | stable host id for the journal reader | | `alloy-data` (named volume) | WAL + remotecfg cache | diff --git a/docker-compose.yml b/docker-compose.yml index 19fac33..411b175 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,7 +28,7 @@ services: - /:/rootfs:ro - /dev/disk/:/dev/disk:ro - /var/lib/docker/:/var/lib/docker:ro - - /var/log/journal:/var/log/journal:ro + - /var/log:/var/log:ro # journal + syslog/messages/*.log for the Linux-Node integration - /etc/machine-id:/etc/machine-id:ro # stable host id for systemd journal reader configs: - { source: alloy_config, target: /etc/alloy/config.alloy } @@ -114,13 +114,15 @@ configs: forward_to = [prometheus.relabel.integrations_node_exporter.receiver] } + // Linux-Node integration ships everything node_exporter emits and only drops + // the per-collector scrape meta-metrics (matches the upstream cloud-config). prometheus.relabel "integrations_node_exporter" { forward_to = [prometheus.remote_write.metrics_service.receiver] rule { source_labels = ["__name__"] - regex = "up|node_arp_entries|node_boot_time_seconds|node_context_switches_total|node_cpu_seconds_total|node_disk_io_time_seconds_total|node_disk_io_time_weighted_seconds_total|node_disk_read_bytes_total|node_disk_read_time_seconds_total|node_disk_reads_completed_total|node_disk_write_time_seconds_total|node_disk_writes_completed_total|node_disk_written_bytes_total|node_filefd_allocated|node_filefd_maximum|node_filesystem_avail_bytes|node_filesystem_device_error|node_filesystem_files|node_filesystem_files_free|node_filesystem_readonly|node_filesystem_size_bytes|node_intr_total|node_load1|node_load15|node_load5|node_md_disks|node_md_disks_required|node_memory_Active_anon_bytes|node_memory_Active_bytes|node_memory_Active_file_bytes|node_memory_AnonHugePages_bytes|node_memory_AnonPages_bytes|node_memory_Bounce_bytes|node_memory_Buffers_bytes|node_memory_Cached_bytes|node_memory_CommitLimit_bytes|node_memory_Committed_AS_bytes|node_memory_DirectMap1G_bytes|node_memory_DirectMap2M_bytes|node_memory_DirectMap4k_bytes|node_memory_Dirty_bytes|node_memory_HugePages_Free|node_memory_HugePages_Rsvd|node_memory_HugePages_Surp|node_memory_HugePages_Total|node_memory_Hugepagesize_bytes|node_memory_Inactive_anon_bytes|node_memory_Inactive_bytes|node_memory_Inactive_file_bytes|node_memory_Mapped_bytes|node_memory_MemAvailable_bytes|node_memory_MemFree_bytes|node_memory_MemTotal_bytes|node_memory_SReclaimable_bytes|node_memory_SUnreclaim_bytes|node_memory_ShmemHugePages_bytes|node_memory_ShmemPmdMapped_bytes|node_memory_Shmem_bytes|node_memory_Slab_bytes|node_memory_SwapTotal_bytes|node_memory_VmallocChunk_bytes|node_memory_VmallocTotal_bytes|node_memory_VmallocUsed_bytes|node_memory_WritebackTmp_bytes|node_memory_Writeback_bytes|node_netstat_Icmp6_InErrors|node_netstat_Icmp6_InMsgs|node_netstat_Icmp6_OutMsgs|node_netstat_Icmp_InErrors|node_netstat_Icmp_InMsgs|node_netstat_Icmp_OutMsgs|node_netstat_IpExt_InOctets|node_netstat_IpExt_OutOctets|node_netstat_TcpExt_ListenDrops|node_netstat_TcpExt_ListenOverflows|node_netstat_TcpExt_TCPSynRetrans|node_netstat_Tcp_InErrs|node_netstat_Tcp_InSegs|node_netstat_Tcp_OutRsts|node_netstat_Tcp_OutSegs|node_netstat_Tcp_RetransSegs|node_netstat_Udp6_InDatagrams|node_netstat_Udp6_InErrors|node_netstat_Udp6_NoPorts|node_netstat_Udp6_OutDatagrams|node_netstat_Udp6_RcvbufErrors|node_netstat_Udp6_SndbufErrors|node_netstat_UdpLite_InErrors|node_netstat_Udp_InDatagrams|node_netstat_Udp_InErrors|node_netstat_Udp_NoPorts|node_netstat_Udp_OutDatagrams|node_netstat_Udp_RcvbufErrors|node_netstat_Udp_SndbufErrors|node_network_carrier|node_network_info|node_network_mtu_bytes|node_network_receive_bytes_total|node_network_receive_compressed_total|node_network_receive_drop_total|node_network_receive_errs_total|node_network_receive_fifo_total|node_network_receive_multicast_total|node_network_receive_packets_total|node_network_speed_bytes|node_network_transmit_bytes_total|node_network_transmit_compressed_total|node_network_transmit_drop_total|node_network_transmit_errs_total|node_network_transmit_fifo_total|node_network_transmit_multicast_total|node_network_transmit_packets_total|node_network_transmit_queue_length|node_network_up|node_nf_conntrack_entries|node_nf_conntrack_entries_limit|node_os_info|node_sockstat_FRAG6_inuse|node_sockstat_FRAG_inuse|node_sockstat_RAW6_inuse|node_sockstat_RAW_inuse|node_sockstat_TCP6_inuse|node_sockstat_TCP_alloc|node_sockstat_TCP_inuse|node_sockstat_TCP_mem|node_sockstat_TCP_mem_bytes|node_sockstat_TCP_orphan|node_sockstat_TCP_tw|node_sockstat_UDP6_inuse|node_sockstat_UDPLITE6_inuse|node_sockstat_UDPLITE_inuse|node_sockstat_UDP_inuse|node_sockstat_UDP_mem|node_sockstat_UDP_mem_bytes|node_sockstat_sockets_used|node_softnet_dropped_total|node_softnet_processed_total|node_softnet_times_squeezed_total|node_systemd_unit_state|node_textfile_scrape_error|node_time_zone_offset_seconds|node_timex_estimated_error_seconds|node_timex_maxerror_seconds|node_timex_offset_seconds|node_timex_sync_status|node_uname_info|node_vmstat_oom_kill|node_vmstat_pgfault|node_vmstat_pgmajfault|node_vmstat_pgpgin|node_vmstat_pgpgout|node_vmstat_pswpin|node_vmstat_pswpout|process_max_fds|process_open_fds" - action = "keep" + regex = "node_scrape_collector_.+" + action = "drop" } } @@ -140,6 +142,19 @@ configs: forward_to = [loki.relabel.integrations_node_exporter.receiver] } + // File-based logs from the Linux-Node integration: syslog/messages/*.log. + // On systemd hosts, rsyslog often mirrors journald — see README "Caveats". + local.file_match "integrations_node_exporter_files" { + path_targets = [{ + __address__ = "localhost", + __path__ = "/var/log/{syslog,messages,*.log}", + }] + } + loki.source.file "integrations_node_exporter_files" { + targets = local.file_match.integrations_node_exporter_files.targets + forward_to = [loki.relabel.integrations_node_exporter.receiver] + } + //JOURNAL declare "journal_module" { argument "forward_to" { @@ -199,7 +214,7 @@ configs: rule { source_labels = ["__name__"] - regex = "container_cpu_usage_seconds_total|container_fs_reads_total|container_fs_usage_bytes|container_fs_writes_total|container_last_seen|container_memory_usage_bytes|container_memory_working_set_bytes|container_network_receive_bytes_total|container_network_receive_errors_total|container_network_receive_packets_dropped_total|container_network_transmit_bytes_total|container_network_transmit_errors_total|container_network_transmit_packets_dropped_total|container_spec_memory_reservation_limit_bytes|machine_memory_bytes|machine_scrape_error|up" + regex = "container_cpu_usage_seconds_total|container_fs_reads_total|container_fs_usage_bytes|container_fs_writes_total|container_last_seen|container_memory_usage_bytes|container_network_receive_bytes_total|container_network_receive_errors_total|container_network_receive_packets_dropped_total|container_network_transmit_bytes_total|container_network_transmit_errors_total|container_network_transmit_packets_dropped_total|container_spec_memory_reservation_limit_bytes|machine_memory_bytes|machine_scrape_error|up" action = "keep" } }