From 2bb2e13279eb5f0e42eb07af80069d5871a8bc57 Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Mon, 6 Apr 2026 13:08:13 +0700 Subject: [PATCH] Add Scribe 2.2 Docker image Multi-stage build of Facebook's legacy log aggregation daemon from source (facebookarchive/scribe) with Apache Thrift 0.9.1 and fb303. Includes default config and entrypoint script. --- README.md | 26 +++++ scribe/Dockerfile | 143 ++++++++++++++++++++++++++ scribe/README.md | 150 ++++++++++++++++++++++++++++ scribe/config/scribe.conf | 53 ++++++++++ scribe/scripts/docker-entrypoint.sh | 20 ++++ 5 files changed, 392 insertions(+) create mode 100644 scribe/Dockerfile create mode 100644 scribe/README.md create mode 100644 scribe/config/scribe.conf create mode 100644 scribe/scripts/docker-entrypoint.sh diff --git a/README.md b/README.md index cd61140..cf3cc04 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,32 @@ More options: https://github.com/tiennm99/docker-images/pkgs/container/couchbase - https://github.com/oracle/docker-images/blob/main/OracleJava/8/jdk/Dockerfile.ol8 - https://repo.huaweicloud.com/java/jdk/8u201-b09/ +## Scribe + +Facebook's legacy real-time log aggregation daemon. + +### Docker compose example + +```yaml +services: + scribe: + image: ghcr.io/tiennm99/scribe:2.2 + ports: + - "1463:1463" + volumes: + - scribe_logs:/var/log/scribe + - ./scribe/config/scribe.conf:/etc/scribe/scribe.conf:ro + restart: unless-stopped + +volumes: + scribe_logs: +``` + +### Credits + +- https://github.com/facebookarchive/scribe +- https://archive.apache.org/dist/thrift/0.9.1/ + ## GitHub Actions template ### Credits diff --git a/scribe/Dockerfile b/scribe/Dockerfile new file mode 100644 index 0000000..277b3eb --- /dev/null +++ b/scribe/Dockerfile @@ -0,0 +1,143 @@ +# ───────────────────────────────────────────────────────────── +# Stage 1: builder +# Compiles Apache Thrift 0.9.1 (last version compatible with +# the Scribe Thrift IDL) and then Scribe itself from source. +# ───────────────────────────────────────────────────────────── +FROM ubuntu:16.04 AS builder + +# Avoid interactive prompts during package installation +ARG DEBIAN_FRONTEND=noninteractive + +# Versions pinned for reproducibility +ENV THRIFT_VERSION=0.9.1 +ENV SCRIBE_GIT_REF=master + +# ── Build-time dependencies ────────────────────────────────── +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + autoconf \ + automake \ + libtool \ + pkg-config \ + git \ + wget \ + ca-certificates \ + # Thrift code-generation tools + flex \ + bison \ + # Boost (filesystem, system, thread required by Scribe) + libboost-dev \ + libboost-filesystem-dev \ + libboost-system-dev \ + libboost-thread-dev \ + # libevent (async I/O used by Thrift non-blocking server) + libevent-dev \ + # OpenSSL (Thrift TLS transport) + libssl-dev \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* + +# ── Build Apache Thrift ────────────────────────────────────── +# Scribe uses the old Facebook Thrift IDL and C++ runtime, which +# is API-compatible with Apache Thrift 0.9.x. +RUN set -eux; \ + wget -q "https://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz" \ + -O /tmp/thrift.tar.gz; \ + tar -xzf /tmp/thrift.tar.gz -C /tmp; \ + cd /tmp/thrift-${THRIFT_VERSION}; \ + ./configure \ + --without-java \ + --without-erlang \ + --without-perl \ + --without-php \ + --without-ruby \ + --without-haskell \ + --without-go \ + --without-python \ + --without-qt4 \ + --without-c_glib \ + --without-tests; \ + make -j"$(nproc)"; \ + make install; \ + ldconfig + +# ── Build fb303 (Scribe dependency) ───────────────────────── +# fb303 is Facebook's base service interface, bundled with Thrift. +RUN set -eux; \ + cd /tmp/thrift-${THRIFT_VERSION}/contrib/fb303; \ + ./bootstrap.sh; \ + ./configure --with-thriftpath=/usr/local; \ + make -j"$(nproc)"; \ + make install; \ + ldconfig; \ + rm -rf /tmp/thrift* + +# ── Build Scribe ───────────────────────────────────────────── +# Facebook archived Scribe at https://github.com/facebookarchive/scribe +# Scribe's FB_INITIALIZE macro conflicts with AM_INIT_AUTOMAKE, causing +# automake to crash with "global options already processed". We patch +# automake to downgrade that fatal error to a warning. +RUN sed -i '/prog_error.*global options already processed/{N;d}' \ + /usr/share/automake-*/Automake/Options.pm + +RUN set -eux; \ + git clone --depth 1 \ + https://github.com/facebookarchive/scribe.git /tmp/scribe; \ + cd /tmp/scribe; \ + autoreconf --force --install; \ + sed -i 's|BOOSTLIBDIR=`echo $BOOST_LDFLAGS.*|BOOSTLIBDIR=/usr/lib/x86_64-linux-gnu|g' configure; \ + ./configure LIBS="-lboost_system -lboost_filesystem"; \ + make -j"$(nproc)" -C src; \ + make -C src install; \ + rm -rf /tmp/scribe + + +# ───────────────────────────────────────────────────────────── +# Stage 2: runtime +# Minimal image containing only the runtime libraries and the +# compiled scribed binary. +# ───────────────────────────────────────────────────────────── +FROM ubuntu:16.04 + +LABEL maintainer="Tien Nguyen Minh " +LABEL description="Facebook Legacy Scribe – real-time log aggregation daemon" +LABEL version="2.2" + +ARG DEBIAN_FRONTEND=noninteractive + +# ── Runtime-only shared libraries ──────────────────────────── +RUN apt-get update && apt-get install -y --no-install-recommends \ + libboost-filesystem1.58.0 \ + libboost-system1.58.0 \ + libboost-thread1.58.0 \ + libevent-2.0-5 \ + libssl1.0.0 \ + libstdc++6 \ + && rm -rf /var/lib/apt/lists/* + +# Copy only what is needed from the builder stage +COPY --from=builder /usr/local/bin/scribed /usr/local/bin/scribed +COPY --from=builder /usr/local/lib/libthrift*.so* /usr/local/lib/ +RUN ldconfig + +# ── Least-privilege user ───────────────────────────────────── +RUN groupadd -r scribe \ + && useradd -r -g scribe -s /sbin/nologin scribe \ + && mkdir -p /var/log/scribe /etc/scribe \ + && chown -R scribe:scribe /var/log/scribe /etc/scribe + +# ── Default configuration and entrypoint ───────────────────── +COPY config/scribe.conf /etc/scribe/scribe.conf +COPY scripts/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN chmod +x /usr/local/bin/docker-entrypoint.sh + +# /var/log/scribe – persistent log output (mount a volume here) +# /etc/scribe – configuration (mount a custom scribe.conf here) +VOLUME ["/var/log/scribe", "/etc/scribe"] + +# Scribe listens on this TCP port by default +EXPOSE 1463 + +USER scribe + +ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"] diff --git a/scribe/README.md b/scribe/README.md new file mode 100644 index 0000000..6494e3d --- /dev/null +++ b/scribe/README.md @@ -0,0 +1,150 @@ +# Scribe + +Facebook's legacy **Scribe** log aggregation daemon, packaged as a Docker image. + +Scribe is a server for aggregating log data streamed in real time from a large number of servers. It accepts messages over a Thrift RPC interface and writes them to configurable back-end stores (files, network relays, etc.). + +> **Note:** Scribe was open-sourced by Facebook and is now archived at +> [facebookarchive/scribe](https://github.com/facebookarchive/scribe). +> This image exists because no official Docker image is available. + +--- + +## Quick Start + +```bash +docker run -d \ + -p 1463:1463 \ + -v $(pwd)/logs:/var/log/scribe \ + ghcr.io/tiennm99/scribe:2.2 +``` + +Scribe will listen on port **1463** (Thrift binary protocol) and write incoming log messages to `/var/log/scribe` inside the container. + +--- + +## Custom Configuration + +Mount your own `scribe.conf` over the default one: + +```bash +docker run -d \ + -p 1463:1463 \ + -v $(pwd)/my-scribe.conf:/etc/scribe/scribe.conf:ro \ + -v $(pwd)/logs:/var/log/scribe \ + ghcr.io/tiennm99/scribe:2.2 +``` + +Or set `SCRIBE_CONFIG` to point to a different path inside the container: + +```bash +docker run -d \ + -p 1463:1463 \ + -e SCRIBE_CONFIG=/opt/scribe/custom.conf \ + -v $(pwd)/custom.conf:/opt/scribe/custom.conf:ro \ + -v $(pwd)/logs:/var/log/scribe \ + ghcr.io/tiennm99/scribe:2.2 +``` + +--- + +## Docker Compose + +```yaml +services: + scribe: + image: ghcr.io/tiennm99/scribe:2.2 + ports: + - "1463:1463" + volumes: + - scribe_logs:/var/log/scribe + - ./config/scribe.conf:/etc/scribe/scribe.conf:ro + restart: unless-stopped + +volumes: + scribe_logs: +``` + +--- + +## Configuration Reference + +The default config at [`config/scribe.conf`](config/scribe.conf) demonstrates the most common options: + +| Key | Default | Description | +|-----|---------|-------------| +| `port` | `1463` | TCP port scribed listens on | +| `max_msg_per_second` | `100000` | Rate limit across all categories (0 = unlimited) | +| `max_msg_size` | `16777216` | Max message size in bytes | +| `check_interval` | `3` | Seconds between store health checks | + +### Store Types + +| Type | Description | +|------|-------------| +| `file` | Write messages to rotating files on disk | +| `network` | Relay messages to an upstream Scribe instance | +| `buffer` | Buffer messages in memory with a file fallback | +| `null` | Discard messages (useful for testing) | +| `multi` | Fan-out to multiple stores simultaneously | + +Full configuration documentation: +https://github.com/facebookarchive/scribe/wiki/Scribe-Configuration + +--- + +## Sending Log Messages + +Use the Scribe Thrift client library for your language, or a simple Python example: + +```python +from scribe import scribe +from thrift.transport import TSocket, TTransport +from thrift.protocol import TBinaryProtocol + +socket = TSocket.TSocket(host="localhost", port=1463) +transport = TTransport.TFramedTransport(socket) +protocol = TBinaryProtocol.TBinaryProtocol(trans=transport, strictRead=False, strictWrite=False) +client = scribe.Client(protocol) + +transport.open() +log_entry = scribe.LogEntry(category="myapp", message="Hello, Scribe!") +result = client.Log(messages=[log_entry]) +transport.close() +``` + +--- + +## Volumes + +| Path | Purpose | +|------|---------| +| `/var/log/scribe` | Log output directory (mount a host path or named volume) | +| `/etc/scribe` | Configuration directory | + +--- + +## Ports + +| Port | Protocol | Description | +|------|----------|-------------| +| `1463` | TCP | Scribe Thrift RPC interface | + +--- + +## Building Locally + +```bash +docker build -t scribe:2.2 . +``` + +The Dockerfile uses a two-stage build: +1. **builder** – compiles Apache Thrift 0.9.1 and Scribe from source on Ubuntu 14.04 +2. **runtime** – copies only the compiled binary and required shared libraries + +--- + +## Credits + +- https://github.com/facebookarchive/scribe +- https://archive.apache.org/dist/thrift/0.9.1/ diff --git a/scribe/config/scribe.conf b/scribe/config/scribe.conf new file mode 100644 index 0000000..ad4b661 --- /dev/null +++ b/scribe/config/scribe.conf @@ -0,0 +1,53 @@ +# scribe.conf – default Scribe server configuration +# +# Scribe uses a simple key=value / block format. +# Full reference: https://github.com/facebookarchive/scribe/wiki/Scribe-Configuration + +# ── Network ────────────────────────────────────────────────── +# TCP port scribed listens on (Thrift binary protocol) +port=1463 + +# Maximum messages accepted per second across all categories (0 = unlimited) +max_msg_per_second=100000 + +# Maximum message size in bytes (default 16 MB) +max_msg_size=16777216 + +# ── Health check interval ──────────────────────────────────── +# How often (seconds) scribed checks whether stores are active +check_interval=3 + +# ── Default catch-all store ────────────────────────────────── +# Any category not matched by a specific block falls here. +# Messages are written to rotating hourly files under /var/log/scribe//. + +category=default +type=file +file_path=/var/log/scribe +base_filename=scribe +max_size=104857600 +rotate_period=hourly +write_meta=true + + +# ── Example: category-specific store ───────────────────────── +# Uncomment and adapt to route a named category to its own directory. +# +# +# category=myapp +# type=file +# file_path=/var/log/scribe/myapp +# base_filename=myapp +# max_size=104857600 +# rotate_period=daily +# + +# ── Example: network store (relay to upstream Scribe) ───────── +# Forward all messages to an upstream Scribe instance. +# +# +# category=default +# type=network +# remote_host=upstream-scribe.example.com +# remote_port=1463 +# diff --git a/scribe/scripts/docker-entrypoint.sh b/scribe/scripts/docker-entrypoint.sh new file mode 100644 index 0000000..05c010a --- /dev/null +++ b/scribe/scripts/docker-entrypoint.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# docker-entrypoint.sh – Scribe daemon startup script +# +# Reads SCRIBE_CONFIG env var (default: /etc/scribe/scribe.conf) +# and launches scribed with that configuration file. +# +# Usage inside container: +# docker run ... scribed (uses default config) +# SCRIBE_CONFIG=/my.conf docker run ... +set -euo pipefail + +SCRIBE_CONFIG="${SCRIBE_CONFIG:-/etc/scribe/scribe.conf}" + +if [ ! -f "$SCRIBE_CONFIG" ]; then + echo "[scribe] ERROR: config file not found: $SCRIBE_CONFIG" >&2 + exit 1 +fi + +echo "[scribe] Starting scribed with config: $SCRIBE_CONFIG" +exec scribed "$SCRIBE_CONFIG"