From a4aa225a7048cd7cdf25268f8e8ceedaa215b988 Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Fri, 24 Apr 2026 14:20:50 +0700 Subject: [PATCH] fix: drop docker logs older than 6d before sending to loki MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit grafana cloud loki rejects entries older than 7 days with HTTP 400. loki.source.docker has no "tail since" option, so on first start it replays logs from each container's start time — long-running containers (coolify-proxy, traffmonetizer) produced weeks of backlog that loki refused to ingest. insert a loki.process drop stage (older_than=144h) between the docker source and loki.write.gc so stale entries are filtered before egress. journal source already bounded by max_age=12h — no filter needed there. --- docker-compose.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 62ea797..9ed1236 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -114,10 +114,21 @@ configs: } } + // Grafana Cloud Loki rejects entries older than 7 days (HTTP 400). + // Long-running containers replay weeks of backlog on first start — drop + // anything older than 6 days to stay safely under the cutoff. + loki.process "drop_old" { + forward_to = [loki.write.gc.receiver] + stage.drop { + older_than = "144h" + drop_counter_reason = "entry_too_old" + } + } + loki.source.docker "containers" { host = "unix:///var/run/docker.sock" targets = discovery.relabel.container_logs.output - forward_to = [loki.write.gc.receiver] + forward_to = [loki.process.drop_old.receiver] } loki.relabel "journal" {