diff --git a/.github/workflows/docker-publish.yaml b/.github/workflows/docker-publish.yaml index 03b016aa..8a66fd65 100644 --- a/.github/workflows/docker-publish.yaml +++ b/.github/workflows/docker-publish.yaml @@ -22,18 +22,27 @@ jobs: - variant: latest enable_otel: "false" enable_tsnet: "false" + enable_redis: "false" suffix: "" - variant: otel enable_otel: "true" enable_tsnet: "false" + enable_redis: "false" suffix: "-otel" - variant: tsnet enable_otel: "false" enable_tsnet: "true" + enable_redis: "false" suffix: "-tsnet" + - variant: redis + enable_otel: "false" + enable_tsnet: "false" + enable_redis: "true" + suffix: "-redis" - variant: full enable_otel: "true" enable_tsnet: "true" + enable_redis: "true" suffix: "-full" steps: @@ -75,6 +84,7 @@ jobs: build-args: | ENABLE_OTEL=${{ matrix.enable_otel }} ENABLE_TSNET=${{ matrix.enable_tsnet }} + ENABLE_REDIS=${{ matrix.enable_redis }} VERSION=${{ github.ref_name }} cache-from: type=gha,scope=${{ matrix.variant }} cache-to: type=gha,mode=max,scope=${{ matrix.variant }} diff --git a/Dockerfile b/Dockerfile index c2d6a5f1..e2654091 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ COPY . . # Build args ARG ENABLE_OTEL=false ARG ENABLE_TSNET=false +ARG ENABLE_REDIS=false ARG VERSION=dev # Build static binary (CGO disabled for scratch/alpine compatibility) @@ -24,6 +25,9 @@ RUN set -eux; \ if [ "$ENABLE_TSNET" = "true" ]; then \ if [ -n "$TAGS" ]; then TAGS="$TAGS,tsnet"; else TAGS="tsnet"; fi; \ fi; \ + if [ "$ENABLE_REDIS" = "true" ]; then \ + if [ -n "$TAGS" ]; then TAGS="$TAGS,redis"; else TAGS="redis"; fi; \ + fi; \ if [ -n "$TAGS" ]; then TAGS="-tags $TAGS"; fi; \ CGO_ENABLED=0 GOOS=linux \ go build -ldflags="-s -w -X github.com/nextlevelbuilder/goclaw/cmd.Version=${VERSION}" \ diff --git a/cmd/gateway.go b/cmd/gateway.go index dd2f1b2b..f72d996c 100644 --- a/cmd/gateway.go +++ b/cmd/gateway.go @@ -346,6 +346,10 @@ func runGateway() { initOTelExporter(context.Background(), cfg, traceCollector) } + // Redis cache: compiled via build tags. Build with 'go build -tags redis' to enable. + redisClient := initRedisClient(cfg) + defer shutdownRedis(redisClient) + // Wire cron retry config from config.json cronRetryCfg := cfg.Cron.ToRetryConfig() // Apply retry config via type assertion on the concrete cron store. @@ -575,7 +579,7 @@ func runGateway() { } var mcpPool *mcpbridge.Pool - contextFileInterceptor, delegateMgr, mcpPool = wireExtras(pgStores, agentRouter, providerRegistry, msgBus, pgStores.Sessions, toolsReg, toolPE, skillsLoader, hasMemory, traceCollector, workspace, cfg.Gateway.InjectionAction, cfg, sandboxMgr, dynamicLoader) + contextFileInterceptor, delegateMgr, mcpPool = wireExtras(pgStores, agentRouter, providerRegistry, msgBus, pgStores.Sessions, toolsReg, toolPE, skillsLoader, hasMemory, traceCollector, workspace, cfg.Gateway.InjectionAction, cfg, sandboxMgr, dynamicLoader, redisClient) if mcpPool != nil { defer mcpPool.Stop() } diff --git a/cmd/gateway_managed.go b/cmd/gateway_managed.go index c1f7344d..fc582ef9 100644 --- a/cmd/gateway_managed.go +++ b/cmd/gateway_managed.go @@ -44,18 +44,22 @@ func wireExtras( appCfg *config.Config, sandboxMgr sandbox.Manager, dynamicLoader *tools.DynamicToolLoader, + redisClient any, // nil when built without -tags redis or when Redis is unconfigured ) (*tools.ContextFileInterceptor, *tools.DelegateManager, *mcpbridge.Pool) { - // 1. Context file interceptor (created before resolver so callbacks can reference it) + // 1. Build cache instances (in-memory or Redis depending on build tags) + agentCtxCache, userCtxCache, gwCache := makeCaches(redisClient) + + // 1a. Context file interceptor (created before resolver so callbacks can reference it) var contextFileInterceptor *tools.ContextFileInterceptor var delegateMgr *tools.DelegateManager if stores.Agents != nil { - contextFileInterceptor = tools.NewContextFileInterceptor(stores.Agents, workspace) + contextFileInterceptor = tools.NewContextFileInterceptor(stores.Agents, workspace, agentCtxCache, userCtxCache) } // 1b. Group writer cache (wraps ListGroupFileWriters with TTL cache) var groupWriterCache *store.GroupWriterCache if stores.Agents != nil { - groupWriterCache = store.NewGroupWriterCache(stores.Agents) + groupWriterCache = store.NewGroupWriterCache(stores.Agents, gwCache) } // 2. User seeding callback: seeds per-user context files on first chat diff --git a/cmd/gateway_redis.go b/cmd/gateway_redis.go new file mode 100644 index 00000000..b230b5c6 --- /dev/null +++ b/cmd/gateway_redis.go @@ -0,0 +1,56 @@ +//go:build redis + +package cmd + +import ( + "log/slog" + + "github.com/redis/go-redis/v9" + + "github.com/nextlevelbuilder/goclaw/internal/cache" + "github.com/nextlevelbuilder/goclaw/internal/config" + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +// initRedisClient creates a Redis client when built with -tags redis. +// Returns nil (typed as any) if GOCLAW_REDIS_DSN is empty or connection fails. +func initRedisClient(cfg *config.Config) any { + dsn := cfg.Database.RedisDSN + if dsn == "" { + slog.Debug("Redis available but not configured (set GOCLAW_REDIS_DSN)") + return nil + } + client, err := cache.NewRedisClient(dsn) + if err != nil { + slog.Warn("Redis connection failed, falling back to in-memory cache", "error", err) + return nil + } + slog.Info("Redis cache connected") + return client +} + +// makeCaches creates typed cache instances backed by Redis (or in-memory if client is nil). +func makeCaches(raw any) ( + agentCtxCache cache.Cache[[]store.AgentContextFileData], + userCtxCache cache.Cache[[]store.AgentContextFileData], + groupWriterCache cache.Cache[[]store.GroupFileWriterData], +) { + client, _ := raw.(*redis.Client) + if client == nil { + slog.Info("cache backend: in-memory (Redis not connected)") + return cache.NewInMemoryCache[[]store.AgentContextFileData](), + cache.NewInMemoryCache[[]store.AgentContextFileData](), + cache.NewInMemoryCache[[]store.GroupFileWriterData]() + } + slog.Info("cache backend: redis") + return cache.NewRedisCache[[]store.AgentContextFileData](client, "ctx:agent"), + cache.NewRedisCache[[]store.AgentContextFileData](client, "ctx:user"), + cache.NewRedisCache[[]store.GroupFileWriterData](client, "grp:writers") +} + +// shutdownRedis closes the Redis client connection. +func shutdownRedis(raw any) { + if client, ok := raw.(*redis.Client); ok && client != nil { + client.Close() + } +} diff --git a/cmd/gateway_redis_noop.go b/cmd/gateway_redis_noop.go new file mode 100644 index 00000000..9cd472e1 --- /dev/null +++ b/cmd/gateway_redis_noop.go @@ -0,0 +1,30 @@ +//go:build !redis + +package cmd + +import ( + "log/slog" + + "github.com/nextlevelbuilder/goclaw/internal/cache" + "github.com/nextlevelbuilder/goclaw/internal/config" + "github.com/nextlevelbuilder/goclaw/internal/store" +) + +// initRedisClient is a no-op when built without the "redis" tag. +// Build with `go build -tags redis` to enable Redis cache. +func initRedisClient(_ *config.Config) any { return nil } + +// makeCaches returns in-memory cache instances when Redis is not compiled in. +func makeCaches(_ any) ( + agentCtxCache cache.Cache[[]store.AgentContextFileData], + userCtxCache cache.Cache[[]store.AgentContextFileData], + groupWriterCache cache.Cache[[]store.GroupFileWriterData], +) { + slog.Info("cache backend: in-memory") + return cache.NewInMemoryCache[[]store.AgentContextFileData](), + cache.NewInMemoryCache[[]store.AgentContextFileData](), + cache.NewInMemoryCache[[]store.GroupFileWriterData]() +} + +// shutdownRedis is a no-op when built without the "redis" tag. +func shutdownRedis(_ any) {} diff --git a/docker-compose.redis.yml b/docker-compose.redis.yml new file mode 100644 index 00000000..3c0660c3 --- /dev/null +++ b/docker-compose.redis.yml @@ -0,0 +1,28 @@ +# Redis overlay — rebuilds with Redis cache support + adds Redis server. +# +# Usage: +# docker compose -f docker-compose.yml -f docker-compose.postgres.yml -f docker-compose.redis.yml up +# + +services: + redis: + image: redis:7-alpine + ports: + - "6379:6379" + volumes: + - redis-data:/data + command: redis-server --appendonly yes + restart: unless-stopped + + goclaw: + build: + args: + ENABLE_REDIS: "true" + environment: + - GOCLAW_REDIS_DSN=redis://redis:6379/0 + depends_on: + redis: + condition: service_started + +volumes: + redis-data: diff --git a/go.mod b/go.mod index ef9a3873..7f66c123 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/jackc/pgx/v5 v5.6.0 github.com/mattn/go-runewidth v0.0.16 github.com/mymmrac/telego v1.6.0 + github.com/redis/go-redis/v9 v9.18.0 github.com/spf13/cobra v1.10.2 github.com/titanous/json5 v1.0.0 go.opentelemetry.io/otel v1.40.0 @@ -23,7 +24,6 @@ require ( go.opentelemetry.io/otel/sdk v1.40.0 go.opentelemetry.io/otel/trace v1.40.0 golang.org/x/time v0.14.0 - modernc.org/sqlite v1.45.0 tailscale.com v1.94.2 ) @@ -60,6 +60,7 @@ require ( github.com/coder/websocket v1.8.12 // indirect github.com/creachadair/msync v0.7.1 // indirect github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/gaissmai/bart v0.18.0 // indirect @@ -96,6 +97,7 @@ require ( github.com/x448/float16 v0.8.4 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect + go.uber.org/atomic v1.11.0 // indirect go4.org/mem v0.0.0-20240501181205-ae6ca9944745 // indirect go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect golang.org/x/image v0.27.0 // indirect @@ -129,8 +131,6 @@ require ( github.com/lib/pq v1.10.9 github.com/mark3labs/mcp-go v0.44.0 github.com/mattn/go-isatty v0.0.20 // indirect - github.com/ncruces/go-strftime v1.0.0 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/spf13/pflag v1.0.9 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect @@ -149,7 +149,7 @@ require ( golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect golang.org/x/crypto v0.47.0 // indirect golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect - golang.org/x/net v0.49.0 // indirect + golang.org/x/net v0.49.0 golang.org/x/sync v0.19.0 golang.org/x/sys v0.40.0 // indirect golang.org/x/text v0.33.0 // indirect @@ -157,7 +157,4 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect google.golang.org/grpc v1.78.0 // indirect google.golang.org/protobuf v1.36.11 // indirect - modernc.org/libc v1.67.6 // indirect - modernc.org/mathutil v1.7.1 // indirect - modernc.org/memory v1.11.0 // indirect ) diff --git a/go.sum b/go.sum index 21719c22..05a4ae90 100644 --- a/go.sum +++ b/go.sum @@ -60,6 +60,10 @@ github.com/aymanbagabas/go-udiff v0.3.1 h1:LV+qyBQ2pqe0u42ZsUEtPiCaUoqgA9gYRDs3v github.com/aymanbagabas/go-udiff v0.3.1/go.mod h1:G0fsKmG+P6ylD0r6N/KgQD/nWzgfnl8ZBcNLgcbrw8E= github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/bwmarrin/discordgo v0.29.0 h1:FmWeXFaKUwrcL3Cx65c20bTRW+vOb6k8AnaP+EgjDno= @@ -131,6 +135,8 @@ github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa h1:h8TfIT1xc8FWbww github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa/go.mod h1:Nx87SkVqTKd8UtT+xu7sM/l+LgXs6c0aHrlKusR+2EQ= github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc h1:8WFBn63wegobsYAX0YjD+8suexZDga5CctH4CCTx2+8= github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4= github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU= github.com/digitalocean/go-smbios v0.0.0-20180907143718-390a4f403a8e h1:vUmf0yezR0y7jJ5pceLHthLaYf4bA5T14B6q39S4q2Q= @@ -194,8 +200,6 @@ github.com/google/go-tpm v0.9.4 h1:awZRf9FwOeTunQmHoDYSHJps3ie6f1UlhS1fOdPEt1I= github.com/google/go-tpm v0.9.4/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/nftables v0.2.1-0.20240414091927-5e242ec57806 h1:wG8RYIyctLhdFk6Vl1yPGtSRtwGpVkWyZww1OCil2MI= github.com/google/nftables v0.2.1-0.20240414091927-5e242ec57806/go.mod h1:Beg6V6zZ3oEn0JuiUQ4wqwuyqqzasOltcoXPtgLbFp4= -github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= -github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -205,9 +209,6 @@ github.com/grbit/go-json v0.11.0 h1:bAbyMdYrYl/OjYsSqLH99N2DyQ291mHy726Mx+sYrnc= github.com/grbit/go-json v0.11.0/go.mod h1:IYpHsdybQ386+6g3VE6AXQ3uTGa5mquBme5/ZWmtzek= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7 h1:X+2YciYSxvMQK0UZ7sg45ZVabVZBeBuvMkmuI2V3Fak= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.7/go.mod h1:lW34nIZuQ8UDPdkon5fmfp2l3+ZkQ2me/+oecHYLOII= -github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4= -github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= -github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hdevalence/ed25519consensus v0.2.0 h1:37ICyZqdyj0lAZ8P4D1d1id3HqbbG1N3iBb1Tb4rdcU= github.com/hdevalence/ed25519consensus v0.2.0/go.mod h1:w3BHWjwJbFU29IRHL1Iqkw3sus+7FctEyM4RqDxYNzo= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= @@ -293,8 +294,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mymmrac/telego v1.6.0 h1:Zc8rgyHozvd/7ZgyrigyHdAF9koHYMfilYfyB6wlFC0= github.com/mymmrac/telego v1.6.0/go.mod h1:xt6ZWA8zi8KmuzryE1ImEdl9JSwjHNpM4yhC7D8hU4Y= -github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= -github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -318,8 +317,8 @@ github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNw github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE= github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs= +github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -410,6 +409,8 @@ github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE= github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg= github.com/ysmood/leakless v0.9.0 h1:qxCG5VirSBvmi3uynXFkcnLMzkphdh3xx5FtrORwDCU= github.com/ysmood/leakless v0.9.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.64.0 h1:ssfIgGNANqpVFCndZvcuyKbl0g+UAVcbBcqGkG28H0Y= @@ -432,6 +433,8 @@ go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZY go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= @@ -509,34 +512,6 @@ honnef.co/go/tools v0.7.0-0.dev.0.20251022135355-8273271481d0 h1:5SXjd4ET5dYijLa honnef.co/go/tools v0.7.0-0.dev.0.20251022135355-8273271481d0/go.mod h1:EPDDhEZqVHhWuPI5zPAsjU0U7v9xNIWjoOVyZ5ZcniQ= howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM= howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= -modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis= -modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= -modernc.org/ccgo/v4 v4.30.1 h1:4r4U1J6Fhj98NKfSjnPUN7Ze2c6MnAdL0hWw6+LrJpc= -modernc.org/ccgo/v4 v4.30.1/go.mod h1:bIOeI1JL54Utlxn+LwrFyjCx2n2RDiYEaJVSrgdrRfM= -modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA= -modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= -modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= -modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/gc/v3 v3.1.1 h1:k8T3gkXWY9sEiytKhcgyiZ2L0DTyCQ/nvX+LoCljoRE= -modernc.org/gc/v3 v3.1.1/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= -modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= -modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.67.6 h1:eVOQvpModVLKOdT+LvBPjdQqfrZq+pC39BygcT+E7OI= -modernc.org/libc v1.67.6/go.mod h1:JAhxUVlolfYDErnwiqaLvUqc8nfb2r6S6slAgZOnaiE= -modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= -modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= -modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= -modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= -modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= -modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= -modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= -modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.45.0 h1:r51cSGzKpbptxnby+EIIz5fop4VuE4qFoVEjNvWoObs= -modernc.org/sqlite v1.45.0/go.mod h1:CzbrU2lSB1DKUusvwGz7rqEKIq+NUd8GWuBBZDs9/nA= -modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= -modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= -modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= software.sslmate.com/src/go-pkcs12 v0.4.0 h1:H2g08FrTvSFKUj+D309j1DPfk5APnIdAQAB8aEykJ5k= software.sslmate.com/src/go-pkcs12 v0.4.0/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI= tailscale.com v1.94.2 h1:H+0NYSG81K1RBXnh6FfWee9G1KEeX9pvYspPrVdIfII= diff --git a/internal/cache/redis.go b/internal/cache/redis.go new file mode 100644 index 00000000..daee2128 --- /dev/null +++ b/internal/cache/redis.go @@ -0,0 +1,94 @@ +//go:build redis + +package cache + +import ( + "context" + "encoding/json" + "log/slog" + "time" + + "github.com/redis/go-redis/v9" +) + +// RedisCache is a Cache implementation backed by Redis. +// All Redis errors are treated as cache misses (fail-open) to avoid breaking callers. +type RedisCache[V any] struct { + client *redis.Client + prefix string // key namespace, e.g. "ctx:agent" +} + +// NewRedisCache creates a Redis-backed cache with the given key prefix. +// Keys are stored as "goclaw:{prefix}:{key}". +func NewRedisCache[V any](client *redis.Client, prefix string) *RedisCache[V] { + return &RedisCache[V]{client: client, prefix: prefix} +} + +func (c *RedisCache[V]) fullKey(key string) string { + return "goclaw:" + c.prefix + ":" + key +} + +func (c *RedisCache[V]) keyPattern() string { + return "goclaw:" + c.prefix + ":*" +} + +func (c *RedisCache[V]) Get(ctx context.Context, key string) (V, bool) { + var zero V + data, err := c.client.Get(ctx, c.fullKey(key)).Bytes() + if err != nil { + return zero, false + } + var val V + if err := json.Unmarshal(data, &val); err != nil { + slog.Warn("redis cache: unmarshal error", "key", c.fullKey(key), "error", err) + return zero, false + } + return val, true +} + +func (c *RedisCache[V]) Set(ctx context.Context, key string, value V, ttl time.Duration) { + data, err := json.Marshal(value) + if err != nil { + slog.Warn("redis cache: marshal error", "key", c.fullKey(key), "error", err) + return + } + if err := c.client.Set(ctx, c.fullKey(key), data, ttl).Err(); err != nil { + slog.Warn("redis cache: set error", "key", c.fullKey(key), "error", err) + } +} + +func (c *RedisCache[V]) Delete(ctx context.Context, key string) { + if err := c.client.Del(ctx, c.fullKey(key)).Err(); err != nil { + slog.Warn("redis cache: delete error", "key", c.fullKey(key), "error", err) + } +} + +func (c *RedisCache[V]) DeleteByPrefix(ctx context.Context, prefix string) { + pattern := "goclaw:" + c.prefix + ":" + prefix + "*" + c.deleteByPattern(ctx, pattern) +} + +func (c *RedisCache[V]) Clear(ctx context.Context) { + c.deleteByPattern(ctx, c.keyPattern()) +} + +// deleteByPattern scans for keys matching pattern and deletes them in batches. +func (c *RedisCache[V]) deleteByPattern(ctx context.Context, pattern string) { + var cursor uint64 + for { + keys, next, err := c.client.Scan(ctx, cursor, pattern, 100).Result() + if err != nil { + slog.Warn("redis cache: scan error", "pattern", pattern, "error", err) + return + } + if len(keys) > 0 { + if err := c.client.Del(ctx, keys...).Err(); err != nil { + slog.Warn("redis cache: batch delete error", "count", len(keys), "error", err) + } + } + cursor = next + if cursor == 0 { + break + } + } +} diff --git a/internal/cache/redis_client.go b/internal/cache/redis_client.go new file mode 100644 index 00000000..d00594ae --- /dev/null +++ b/internal/cache/redis_client.go @@ -0,0 +1,35 @@ +//go:build redis + +package cache + +import ( + "context" + "fmt" + "time" + + "github.com/redis/go-redis/v9" +) + +// NewRedisClient creates a Redis client from a DSN (e.g. "redis://localhost:6379/0") +// and verifies connectivity with a PING. +func NewRedisClient(dsn string) (*redis.Client, error) { + opts, err := redis.ParseURL(dsn) + if err != nil { + return nil, fmt.Errorf("redis: invalid DSN: %w", err) + } + opts.PoolSize = 10 + opts.MinIdleConns = 2 + opts.DialTimeout = 5 * time.Second + opts.ReadTimeout = 3 * time.Second + opts.WriteTimeout = 3 * time.Second + + client := redis.NewClient(opts) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := client.Ping(ctx).Err(); err != nil { + client.Close() + return nil, fmt.Errorf("redis: ping failed: %w", err) + } + return client, nil +} diff --git a/internal/cache/redis_test.go b/internal/cache/redis_test.go new file mode 100644 index 00000000..8e33fb3e --- /dev/null +++ b/internal/cache/redis_test.go @@ -0,0 +1,142 @@ +//go:build redis + +package cache + +import ( + "context" + "os" + "testing" + "time" + + "github.com/redis/go-redis/v9" +) + +func testRedisClient(t *testing.T) *redis.Client { + t.Helper() + dsn := os.Getenv("REDIS_TEST_DSN") + if dsn == "" { + dsn = "redis://localhost:6379/15" // use DB 15 for tests + } + client, err := NewRedisClient(dsn) + if err != nil { + t.Skipf("Redis not available: %v", err) + } + t.Cleanup(func() { + client.FlushDB(context.Background()) + client.Close() + }) + return client +} + +func TestRedisCache_GetSet(t *testing.T) { + client := testRedisClient(t) + c := NewRedisCache[string](client, "test:getset") + ctx := context.Background() + + // miss + _, ok := c.Get(ctx, "k1") + if ok { + t.Fatal("expected miss") + } + + // hit + c.Set(ctx, "k1", "hello", time.Minute) + v, ok := c.Get(ctx, "k1") + if !ok || v != "hello" { + t.Fatalf("expected hello, got %q ok=%v", v, ok) + } +} + +func TestRedisCache_TTLExpiry(t *testing.T) { + client := testRedisClient(t) + c := NewRedisCache[int](client, "test:ttl") + ctx := context.Background() + + c.Set(ctx, "k1", 42, 100*time.Millisecond) + v, ok := c.Get(ctx, "k1") + if !ok || v != 42 { + t.Fatalf("expected 42, got %d ok=%v", v, ok) + } + + time.Sleep(150 * time.Millisecond) + _, ok = c.Get(ctx, "k1") + if ok { + t.Fatal("expected miss after TTL") + } +} + +func TestRedisCache_Delete(t *testing.T) { + client := testRedisClient(t) + c := NewRedisCache[string](client, "test:del") + ctx := context.Background() + + c.Set(ctx, "k1", "v1", time.Minute) + c.Delete(ctx, "k1") + + _, ok := c.Get(ctx, "k1") + if ok { + t.Fatal("expected miss after delete") + } +} + +func TestRedisCache_DeleteByPrefix(t *testing.T) { + client := testRedisClient(t) + c := NewRedisCache[string](client, "test:prefix") + ctx := context.Background() + + c.Set(ctx, "agent:1:a", "v1", time.Minute) + c.Set(ctx, "agent:1:b", "v2", time.Minute) + c.Set(ctx, "agent:2:a", "v3", time.Minute) + + c.DeleteByPrefix(ctx, "agent:1:") + + if _, ok := c.Get(ctx, "agent:1:a"); ok { + t.Fatal("agent:1:a should be deleted") + } + if _, ok := c.Get(ctx, "agent:1:b"); ok { + t.Fatal("agent:1:b should be deleted") + } + if _, ok := c.Get(ctx, "agent:2:a"); !ok { + t.Fatal("agent:2:a should still exist") + } +} + +func TestRedisCache_Clear(t *testing.T) { + client := testRedisClient(t) + c := NewRedisCache[int](client, "test:clear") + ctx := context.Background() + + c.Set(ctx, "a", 1, time.Minute) + c.Set(ctx, "b", 2, time.Minute) + c.Clear(ctx) + + if _, ok := c.Get(ctx, "a"); ok { + t.Fatal("expected miss after clear") + } + if _, ok := c.Get(ctx, "b"); ok { + t.Fatal("expected miss after clear") + } +} + +// TestRedisCache_StructRoundtrip verifies JSON serialization of complex types. +type testStruct struct { + Name string `json:"name"` + Age int `json:"age"` +} + +func TestRedisCache_StructRoundtrip(t *testing.T) { + client := testRedisClient(t) + c := NewRedisCache[[]testStruct](client, "test:struct") + ctx := context.Background() + + data := []testStruct{{Name: "Alice", Age: 30}, {Name: "Bob", Age: 25}} + c.Set(ctx, "users", data, time.Minute) + + got, ok := c.Get(ctx, "users") + if !ok { + t.Fatal("expected hit") + } + if len(got) != 2 || got[0].Name != "Alice" || got[1].Age != 25 { + t.Fatalf("roundtrip mismatch: %+v", got) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 747c8716..f27a0be1 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -65,10 +65,11 @@ type TailscaleConfig struct { EnableTLS bool `json:"enable_tls,omitempty"` // use ListenTLS for auto HTTPS certs } -// DatabaseConfig configures the PostgreSQL connection. -// PostgresDSN is NEVER read from config.json (secret) — only from env GOCLAW_POSTGRES_DSN. +// DatabaseConfig configures the PostgreSQL connection and optional Redis cache. +// DSN fields are NEVER read from config.json (secrets) — only from env vars. type DatabaseConfig struct { PostgresDSN string `json:"-"` // from env GOCLAW_POSTGRES_DSN only + RedisDSN string `json:"-"` // from env GOCLAW_REDIS_DSN only (optional, requires -tags redis) } // SkillsConfig configures the skills storage system. diff --git a/internal/config/config_load.go b/internal/config/config_load.go index dccbc1e2..233e8d4d 100644 --- a/internal/config/config_load.go +++ b/internal/config/config_load.go @@ -166,6 +166,7 @@ func (c *Config) applyEnvOverrides() { // Database envStr("GOCLAW_POSTGRES_DSN", &c.Database.PostgresDSN) + envStr("GOCLAW_REDIS_DSN", &c.Database.RedisDSN) // Deprecation warning for GOCLAW_MODE (removed — PostgreSQL is always active) if v := os.Getenv("GOCLAW_MODE"); v != "" { diff --git a/internal/store/group_writer_cache.go b/internal/store/group_writer_cache.go index 0c9da3c8..b4c2c83f 100644 --- a/internal/store/group_writer_cache.go +++ b/internal/store/group_writer_cache.go @@ -17,14 +17,15 @@ const groupWriterCacheTTL = 5 * time.Minute // Used by tools and agent loop to check group write permissions without repeated DB queries. type GroupWriterCache struct { agentStore AgentStore - cache *cache.InMemoryCache[[]GroupFileWriterData] + cache cache.Cache[[]GroupFileWriterData] } // NewGroupWriterCache creates a new cache backed by the given agent store. -func NewGroupWriterCache(as AgentStore) *GroupWriterCache { +// The cache implementation is injected (in-memory or Redis) so callers control the backend. +func NewGroupWriterCache(as AgentStore, c cache.Cache[[]GroupFileWriterData]) *GroupWriterCache { return &GroupWriterCache{ agentStore: as, - cache: cache.NewInMemoryCache[[]GroupFileWriterData](), + cache: c, } } diff --git a/internal/tools/context_file_interceptor.go b/internal/tools/context_file_interceptor.go index 17ff5137..0cdc4a59 100644 --- a/internal/tools/context_file_interceptor.go +++ b/internal/tools/context_file_interceptor.go @@ -81,12 +81,17 @@ type ContextFileInterceptor struct { } // NewContextFileInterceptor creates an interceptor backed by the given agent store. -func NewContextFileInterceptor(as store.AgentStore, workspace string) *ContextFileInterceptor { +// Cache implementations are injected (in-memory or Redis) so callers control the backend. +func NewContextFileInterceptor( + as store.AgentStore, + workspace string, + agentCache, userCache cache.Cache[[]store.AgentContextFileData], +) *ContextFileInterceptor { return &ContextFileInterceptor{ agentStore: as, workspace: workspace, - agentCache: cache.NewInMemoryCache[[]store.AgentContextFileData](), - userCache: cache.NewInMemoryCache[[]store.AgentContextFileData](), + agentCache: agentCache, + userCache: userCache, ttl: defaultContextCacheTTL, } } diff --git a/internal/tools/context_file_interceptor_test.go b/internal/tools/context_file_interceptor_test.go index 314a6339..e467c268 100644 --- a/internal/tools/context_file_interceptor_test.go +++ b/internal/tools/context_file_interceptor_test.go @@ -8,6 +8,7 @@ import ( "github.com/google/uuid" + "github.com/nextlevelbuilder/goclaw/internal/cache" "github.com/nextlevelbuilder/goclaw/internal/store" ) @@ -94,7 +95,10 @@ func TestInterceptor_CacheHit(t *testing.T) { {AgentID: agentID, FileName: "SOUL.md", Content: "you are helpful"}, }, } - intc := NewContextFileInterceptor(as, "") + intc := NewContextFileInterceptor(as, "", + cache.NewInMemoryCache[[]store.AgentContextFileData](), + cache.NewInMemoryCache[[]store.AgentContextFileData](), + ) ctx := store.WithAgentID(context.Background(), agentID) @@ -126,7 +130,10 @@ func TestInterceptor_InvalidateAgent_ClearsCache(t *testing.T) { {AgentID: agentID, FileName: "SOUL.md", Content: "old content"}, }, } - intc := NewContextFileInterceptor(as, "") + intc := NewContextFileInterceptor(as, "", + cache.NewInMemoryCache[[]store.AgentContextFileData](), + cache.NewInMemoryCache[[]store.AgentContextFileData](), + ) ctx := store.WithAgentID(context.Background(), agentID) // Warm up cache with old content @@ -175,7 +182,10 @@ func TestInterceptor_InvalidateAgent_ClearsUserCache(t *testing.T) { {AgentID: agentID, UserID: userID, FileName: "USER.md", Content: "old user content"}, }, } - intc := NewContextFileInterceptor(as, "") + intc := NewContextFileInterceptor(as, "", + cache.NewInMemoryCache[[]store.AgentContextFileData](), + cache.NewInMemoryCache[[]store.AgentContextFileData](), + ) // Warm user cache intc.readUserFile(context.Background(), agentID, userID, "USER.md") @@ -215,7 +225,10 @@ func TestInterceptor_InvalidateAgent_DoesNotAffectOtherAgents(t *testing.T) { {AgentID: agentA, FileName: "SOUL.md", Content: "agent A soul"}, } - intc := NewContextFileInterceptor(as, "") + intc := NewContextFileInterceptor(as, "", + cache.NewInMemoryCache[[]store.AgentContextFileData](), + cache.NewInMemoryCache[[]store.AgentContextFileData](), + ) ctx := context.Background() // Warm agent A cache @@ -240,7 +253,10 @@ func TestInterceptor_TTLExpiry(t *testing.T) { {AgentID: agentID, FileName: "SOUL.md", Content: "soul content"}, }, } - intc := NewContextFileInterceptor(as, "") + intc := NewContextFileInterceptor(as, "", + cache.NewInMemoryCache[[]store.AgentContextFileData](), + cache.NewInMemoryCache[[]store.AgentContextFileData](), + ) intc.ttl = 10 * time.Millisecond // very short TTL for testing ctx := context.Background()