mirror of
https://github.com/tiennm99/monkeyd-crawler.git
synced 2026-09-03 00:18:06 +00:00
fix: embed font data instead of a path, and bundle a fallback font
PDF rendering failed on Linux with "stat usr/share/fonts/...: no such file or directory": fpdf joins the font path onto its own font directory, which it defaults to ".", so path.Join turns an absolute path into a working-directory-relative one. It only resolved when the process happened to run from the filesystem root, which is why Windows was unaffected. Font data is now read by pdfout and handed over as bytes. LoadFont also falls back to a bundled DejaVu Sans, so a host with no fonts installed still renders. An explicitly requested font remains a hard error when unreadable rather than being silently substituted. Tests verify the bundled font parses and covers Vietnamese, which is the reason it exists.
This commit is contained in:
@@ -40,7 +40,7 @@ The PDF is named after the novel unless you pass `-out`.
|
||||
| `-font-size` | `12` | Body font size in points |
|
||||
| `-line-spacing` | `1.55` | Line height as a multiple of font size |
|
||||
| `-margin` | `6` | Page margin in mm |
|
||||
| `-font` | auto | Path to a `.ttf`; defaults to a system font with Vietnamese coverage |
|
||||
| `-font` | auto | Path to a `.ttf`; defaults to a system font, else the bundled one |
|
||||
| `-workers` | `4` | Concurrent chapter fetches |
|
||||
| `-delay` | `400ms` | Minimum delay between requests |
|
||||
| `-retries` | `3` | Retries per request |
|
||||
@@ -106,14 +106,14 @@ chapter ordering, and the numbering gap. No network access required.
|
||||
cmd/monkeyd-crawler/ CLI
|
||||
export/ URL -> PDF in one call; shared by the CLI and importers
|
||||
monkeyd/ fetching, HTML/CSS parsing, crawl orchestration
|
||||
pdfout/ PDF rendering and font discovery
|
||||
pdfout/ PDF rendering, font resolution, bundled fallback font
|
||||
```
|
||||
|
||||
## Use as a library
|
||||
|
||||
The packages are importable, so another Go program can produce the same PDF
|
||||
without shelling out to the binary. `export.Export` is the whole pipeline —
|
||||
chapter list, fetch, font discovery, render:
|
||||
chapter list, fetch, font resolution, render:
|
||||
|
||||
```go
|
||||
result, err := export.Export(ctx, export.Request{
|
||||
@@ -128,9 +128,27 @@ cache or *no* request delay with the `NoCache` and `NoDelay` fields rather than
|
||||
by zeroing `CacheDir` or `Delay`. Pass a `Log` function to receive the progress
|
||||
messages the CLI prints to stderr.
|
||||
|
||||
Callers running in a container should note that `pdfout.FindFont` searches
|
||||
system font paths: a minimal image with no fonts installed needs either a font
|
||||
present or an explicit `FontFile`.
|
||||
## Fonts
|
||||
|
||||
The PDF embeds a TrueType font, and Vietnamese needs one that covers the Latin
|
||||
Extended Additional block — a basic-Latin font silently drops the diacritics.
|
||||
The font is resolved in this order:
|
||||
|
||||
1. the path given to `-font` / `Request.FontFile`, which is an error if it
|
||||
cannot be read — a named font is not silently substituted;
|
||||
2. a system font known to cover Vietnamese (see `pdfout.FindFont`);
|
||||
3. the bundled DejaVu Sans, compiled into the binary.
|
||||
|
||||
Step 3 means rendering never depends on the host having fonts installed, which
|
||||
is what a minimal container usually looks like. See
|
||||
[`pdfout/fonts/NOTICE.md`](pdfout/fonts/NOTICE.md) for the bundled font's
|
||||
provenance and licensing.
|
||||
|
||||
Font data is handed to the PDF writer as bytes, not as a path. `fpdf`'s
|
||||
path-taking `AddUTF8Font` joins the name onto its own font directory, which it
|
||||
defaults to `"."`; an absolute path is thereby rewritten into a
|
||||
working-directory-relative one and fails wherever the process does not run from
|
||||
the filesystem root.
|
||||
|
||||
## Scope
|
||||
|
||||
|
||||
@@ -44,7 +44,7 @@ func run() error {
|
||||
|
||||
fmt.Fprintf(os.Stderr, "\n%s\n", result.Summary())
|
||||
fmt.Fprintf(os.Stderr, "font: %s at %.0fpt on %s page (%.0f x %.0f mm)\n",
|
||||
filepath.Base(result.FontFile), req.FontSize, result.Page.Name, result.Page.W, result.Page.H)
|
||||
filepath.Base(result.FontName), req.FontSize, result.Page.Name, result.Page.W, result.Page.H)
|
||||
fmt.Println(result.Path)
|
||||
return nil
|
||||
}
|
||||
|
||||
+8
-8
@@ -75,7 +75,7 @@ type Result struct {
|
||||
SourceURL string
|
||||
Chapters int
|
||||
Words int
|
||||
FontFile string
|
||||
FontName string // font path, or pdfout.BundledFontName
|
||||
Page pdfout.PageSize
|
||||
}
|
||||
|
||||
@@ -115,11 +115,11 @@ func Export(ctx context.Context, req Request) (*Result, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fontFile := req.FontFile
|
||||
if fontFile == "" {
|
||||
if fontFile, err = pdfout.FindFont(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Falls back to the bundled font, so a host with no fonts installed still
|
||||
// renders; only an explicitly requested font can fail here.
|
||||
font, err := pdfout.LoadFont(req.FontFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
outPath := req.OutPath
|
||||
@@ -131,7 +131,7 @@ func Export(ctx context.Context, req Request) (*Result, error) {
|
||||
opts := pdfout.Options{
|
||||
Page: page,
|
||||
Margin: req.Margin,
|
||||
FontFile: fontFile,
|
||||
Font: font,
|
||||
FontSize: req.FontSize,
|
||||
LineSpacing: req.LineSpacing,
|
||||
Title: novel.Title,
|
||||
@@ -147,7 +147,7 @@ func Export(ctx context.Context, req Request) (*Result, error) {
|
||||
SourceURL: novel.URL,
|
||||
Chapters: len(chapters),
|
||||
Words: monkeyd.TotalWords(chapters),
|
||||
FontFile: fontFile,
|
||||
FontName: font.Name,
|
||||
Page: page,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -4,6 +4,9 @@ go 1.26.4
|
||||
|
||||
require (
|
||||
github.com/go-pdf/fpdf v0.9.0
|
||||
golang.org/x/image v0.44.0
|
||||
golang.org/x/net v0.57.0
|
||||
golang.org/x/sync v0.22.0
|
||||
)
|
||||
|
||||
require golang.org/x/text v0.40.0 // indirect
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
github.com/go-pdf/fpdf v0.9.0 h1:PPvSaUuo1iMi9KkaAn90NuKi+P4gwMedWPHhj8YlJQw=
|
||||
github.com/go-pdf/fpdf v0.9.0/go.mod h1:oO8N111TkmKb9D7VvWGLvLJlaZUQVPM+6V42pp3iV4Y=
|
||||
golang.org/x/image v0.44.0 h1:+tDekMZED9+LrtB3G5xzRggpVh9CARjZqROla3R3R+I=
|
||||
golang.org/x/image v0.44.0/go.mod h1:V8K3KE9KKKE+pLpQDOeN18w9oacNSvy1tDOirTu4xtY=
|
||||
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
|
||||
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
|
||||
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
|
||||
|
||||
+56
-2
@@ -1,12 +1,37 @@
|
||||
package pdfout
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
)
|
||||
|
||||
// bundledTTF is the font used when nothing else is available, so rendering never
|
||||
// depends on the host having fonts installed — a minimal container typically has
|
||||
// none. See fonts/NOTICE.md for provenance and licensing.
|
||||
//
|
||||
//go:embed fonts/DejaVuSans.ttf
|
||||
var bundledTTF []byte
|
||||
|
||||
// BundledFontName labels the embedded font in diagnostics. It is not a path;
|
||||
// the font is compiled into the binary.
|
||||
const BundledFontName = "DejaVu Sans (bundled)"
|
||||
|
||||
// Font is font data ready to embed in a PDF.
|
||||
//
|
||||
// The data is carried as bytes rather than as a path because fpdf joins a font
|
||||
// path onto its own font directory, which it defaults to "." — turning an
|
||||
// absolute path into a working-directory-relative one that resolves only when
|
||||
// the process happens to run from the filesystem root.
|
||||
type Font struct {
|
||||
// Name identifies the font for diagnostics: the file path it was read
|
||||
// from, or BundledFontName.
|
||||
Name string
|
||||
Data []byte
|
||||
}
|
||||
|
||||
// Vietnamese text needs the Latin Extended Additional block (ư, ạ, ế, ộ …).
|
||||
// Every font listed here ships with its platform and covers it; fonts with only
|
||||
// basic Latin would silently drop the diacritics.
|
||||
@@ -42,6 +67,8 @@ func fontCandidates() []string {
|
||||
}
|
||||
|
||||
// FindFont returns the first available system font suitable for Vietnamese.
|
||||
// Callers that just need something that works should use LoadFont, which falls
|
||||
// back to the bundled font instead of failing.
|
||||
func FindFont() (string, error) {
|
||||
candidates := fontCandidates()
|
||||
for _, path := range candidates {
|
||||
@@ -49,6 +76,33 @@ func FindFont() (string, error) {
|
||||
return path, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("no Vietnamese-capable system font found (looked for %v); "+
|
||||
"pass -font with a path to a .ttf file", candidates)
|
||||
return "", fmt.Errorf("no Vietnamese-capable system font found (looked for %v)", candidates)
|
||||
}
|
||||
|
||||
// LoadFont resolves the font to embed.
|
||||
//
|
||||
// An explicit path wins, and is a hard error when it cannot be read: a caller
|
||||
// that named a font wants that font, not a substitute. Otherwise a system font
|
||||
// is used, and when none can be read the bundled font is returned — so with an
|
||||
// empty path LoadFont always succeeds.
|
||||
func LoadFont(path string) (Font, error) {
|
||||
if path != "" {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return Font{}, fmt.Errorf("read font: %w", err)
|
||||
}
|
||||
return Font{Name: path, Data: data}, nil
|
||||
}
|
||||
if found, err := FindFont(); err == nil {
|
||||
if data, err := os.ReadFile(found); err == nil {
|
||||
return Font{Name: found, Data: data}, nil
|
||||
}
|
||||
// A listed font that cannot be read is no better than a missing one.
|
||||
}
|
||||
return BundledFont(), nil
|
||||
}
|
||||
|
||||
// BundledFont returns the font compiled into the binary.
|
||||
func BundledFont() Font {
|
||||
return Font{Name: BundledFontName, Data: bundledTTF}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
package pdfout
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"golang.org/x/image/font/sfnt"
|
||||
)
|
||||
|
||||
func TestLoadFontUsesExplicitPath(t *testing.T) {
|
||||
// Any readable file is enough: LoadFont does not parse, it only reads.
|
||||
path := filepath.Join(t.TempDir(), "custom.ttf")
|
||||
want := []byte("not really a font")
|
||||
if err := os.WriteFile(path, want, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
font, err := LoadFont(path)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadFont: %v", err)
|
||||
}
|
||||
if font.Name != path {
|
||||
t.Errorf("Name = %q, want %q", font.Name, path)
|
||||
}
|
||||
if string(font.Data) != string(want) {
|
||||
t.Errorf("Data = %q, want %q", font.Data, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A caller that names a font wants that font; silently substituting another
|
||||
// would produce a PDF that does not match the request.
|
||||
func TestLoadFontFailsOnUnreadableExplicitPath(t *testing.T) {
|
||||
font, err := LoadFont(filepath.Join(t.TempDir(), "missing.ttf"))
|
||||
if err == nil {
|
||||
t.Fatalf("LoadFont = %+v, want an error", font)
|
||||
}
|
||||
}
|
||||
|
||||
// With no path given, LoadFont must always produce something usable — that is
|
||||
// the whole point of the bundled font.
|
||||
func TestLoadFontFallsBackWithoutExplicitPath(t *testing.T) {
|
||||
font, err := LoadFont("")
|
||||
if err != nil {
|
||||
t.Fatalf("LoadFont(\"\") = %v, want no error", err)
|
||||
}
|
||||
if len(font.Data) == 0 {
|
||||
t.Error("resolved font carries no data")
|
||||
}
|
||||
if font.Name == "" {
|
||||
t.Error("resolved font has no name")
|
||||
}
|
||||
// On a host with fonts installed this is a system path; on one without, it
|
||||
// is the bundled font. Either is fine, but it must be one of them.
|
||||
if system, err := FindFont(); err == nil {
|
||||
if font.Name != system && font.Name != BundledFontName {
|
||||
t.Errorf("Name = %q, want %q or %q", font.Name, system, BundledFontName)
|
||||
}
|
||||
} else if font.Name != BundledFontName {
|
||||
t.Errorf("Name = %q, want %q when no system font exists", font.Name, BundledFontName)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundledFontIsParseable(t *testing.T) {
|
||||
font := BundledFont()
|
||||
if font.Name != BundledFontName {
|
||||
t.Errorf("Name = %q, want %q", font.Name, BundledFontName)
|
||||
}
|
||||
if _, err := sfnt.Parse(font.Data); err != nil {
|
||||
t.Fatalf("bundled font does not parse as a TrueType font: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The bundled font exists to render Vietnamese. A replacement that lacks these
|
||||
// glyphs would silently emit blanks, so check before trusting it.
|
||||
func TestBundledFontCoversVietnamese(t *testing.T) {
|
||||
parsed, err := sfnt.Parse(BundledFont().Data)
|
||||
if err != nil {
|
||||
t.Fatalf("parse bundled font: %v", err)
|
||||
}
|
||||
var buf sfnt.Buffer
|
||||
for _, r := range []rune{'ư', 'ơ', 'đ', 'ạ', 'ế', 'ộ', 'ữ', 'ằ', 'ỷ', 'ỹ', 'Ọ', 'Ế', 'Ư', 'Đ'} {
|
||||
index, err := parsed.GlyphIndex(&buf, r)
|
||||
if err != nil {
|
||||
t.Errorf("GlyphIndex(%q): %v", r, err)
|
||||
continue
|
||||
}
|
||||
// Glyph 0 is .notdef — the character is absent from the font.
|
||||
if index == 0 {
|
||||
t.Errorf("bundled font has no glyph for %q (U+%04X)", r, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,24 @@
|
||||
# Bundled font
|
||||
|
||||
`DejaVuSans.ttf` is embedded into the binary and used when no font is supplied
|
||||
and no suitable system font is found. It covers the Latin Extended Additional
|
||||
block, which is what Vietnamese diacritics need — a font with only basic Latin
|
||||
coverage silently drops them.
|
||||
|
||||
The following is recorded in the font file's own name table:
|
||||
|
||||
- Version: `Version 2.37`
|
||||
- Copyright: `Copyright (c) 2003 by Bitstream, Inc. All Rights Reserved.`
|
||||
`Copyright (c) 2006 by Tavmjong Bah. All Rights Reserved.`
|
||||
`DejaVu changes are in public domain`
|
||||
- License information: <http://dejavu.sourceforge.net/wiki/index.php/License>
|
||||
|
||||
The DejaVu fonts are free and redistributable, which is why they ship with most
|
||||
Linux distributions. This copy came from Alpine's `font-dejavu` package, which
|
||||
does not include the license text as a separate file. For a vendored copy of the
|
||||
full license text, take `LICENSE` from the upstream DejaVu release and add it to
|
||||
this directory.
|
||||
|
||||
To swap the bundled font, replace `DejaVuSans.ttf` and update this file. Verify
|
||||
the replacement covers Vietnamese first — `TestBundledFontCoversVietnamese` in
|
||||
`../font_test.go` checks a representative set of characters.
|
||||
+7
-4
@@ -53,7 +53,7 @@ type Chapter struct {
|
||||
type Options struct {
|
||||
Page PageSize
|
||||
Margin float64 // mm
|
||||
FontFile string
|
||||
Font Font // resolve with LoadFont
|
||||
FontSize float64 // pt
|
||||
LineSpacing float64 // multiple of font size
|
||||
Title string
|
||||
@@ -73,9 +73,12 @@ func Write(path string, opts Options, chapters []Chapter) error {
|
||||
pdf.SetMargins(opts.Margin, opts.Margin, opts.Margin)
|
||||
pdf.SetAutoPageBreak(true, opts.Margin+footerReserve)
|
||||
|
||||
// AddUTF8Font embeds a subset of the TrueType file, which is what makes the
|
||||
// Vietnamese diacritics render instead of falling back to "?".
|
||||
pdf.AddUTF8Font(bodyFont, "", opts.FontFile)
|
||||
// Embeds a subset of the TrueType data, which is what makes the Vietnamese
|
||||
// diacritics render instead of falling back to "?". The bytes are passed
|
||||
// directly rather than by path: the path-taking variant joins the name onto
|
||||
// fpdf's own font directory (default "."), which mangles an absolute path
|
||||
// into a working-directory-relative one.
|
||||
pdf.AddUTF8FontFromBytes(bodyFont, "", opts.Font.Data)
|
||||
pdf.SetFont(bodyFont, "", opts.FontSize)
|
||||
pdf.SetTitle(opts.Title, true)
|
||||
|
||||
|
||||
+5
-3
@@ -8,14 +8,16 @@ import (
|
||||
|
||||
func testOptions(t *testing.T) Options {
|
||||
t.Helper()
|
||||
font, err := FindFont()
|
||||
// LoadFont("") cannot fail — it ends at the bundled font — so unlike the
|
||||
// old FindFont call this never skips for want of a system font.
|
||||
font, err := LoadFont("")
|
||||
if err != nil {
|
||||
t.Skipf("no system font available: %v", err)
|
||||
t.Fatalf("LoadFont: %v", err)
|
||||
}
|
||||
return Options{
|
||||
Page: Presets["phone"],
|
||||
Margin: 6,
|
||||
FontFile: font,
|
||||
Font: font,
|
||||
FontSize: 12,
|
||||
LineSpacing: 1.55,
|
||||
Title: "TRỞ LẠI NĂM THÁNG CŨ",
|
||||
|
||||
Reference in New Issue
Block a user