mirror of
https://github.com/tiennm99/awesome-coding-agents.git
synced 2026-06-02 12:10:39 +00:00
fix: harden GitHub fetcher and history I/O, add canonical keying
GitHub fetcher (github.go): - add 30s HTTP client timeout (was http.DefaultClient with no bound) - chunk GraphQL alias requests at 50 repos to stay clear of abuse detection - abort the run on any partial GraphQL error or missing repo rather than silently shrinking the README and poisoning the next delta - retry transient failures (network, 5xx, 429) with 2s/4s/8s backoff History layer (history.go): - key snapshots by canonical owner/repo from agents.yml instead of the rename-resolved NameWithOwner returned by the API; carry a lazy migration map so existing aaif-goose/goose entries fold into block/goose on next read with no manual data edit - tighten the 7d delta window to (cutoff-3d, cutoff] so a missed cron week no longer mislabels a 90d-old comparison as Delta7d - replace the snapshots[:0] aliased filter loop with slices.DeleteFunc - log malformed JSONL lines to stderr with line numbers instead of silently skipping them - write history.jsonl atomically via tmp file + rename so a crash mid-write can no longer truncate accumulated history Plus collapse a few redundant fmt.Errorf wraps, drop a named Config type that was used once, inline the single-call sortByStars helper with a deterministic tiebreaker on canonical key, and use filepath.Base instead of hand-rolling a basename. Includes unit tests covering the 7d window edges, canonical-key migration, atomic write path, malformed-line tolerance, YAML validation, and markdown cell escaping.
This commit is contained in:
@@ -14,20 +14,17 @@ type Agent struct {
|
||||
Notes string `yaml:"notes,omitempty"`
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Agents []Agent `yaml:"agents"`
|
||||
}
|
||||
|
||||
func loadAgents(path string) ([]Agent, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var cfg Config
|
||||
var cfg struct {
|
||||
Agents []Agent `yaml:"agents"`
|
||||
}
|
||||
if err := yaml.Unmarshal(data, &cfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// validate each entry has owner + repo
|
||||
for i, a := range cfg.Agents {
|
||||
if a.Owner == "" || a.Repo == "" {
|
||||
return nil, fmt.Errorf("entry %d missing owner or repo", i)
|
||||
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLoadAgents_ValidationErrors(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
yaml string
|
||||
expectErr bool
|
||||
errSubstr string
|
||||
desc string
|
||||
}{
|
||||
{
|
||||
name: "valid entry",
|
||||
yaml: `agents:
|
||||
- owner: org
|
||||
repo: repo1
|
||||
category: tools
|
||||
`,
|
||||
expectErr: false,
|
||||
desc: "should load valid entry",
|
||||
},
|
||||
{
|
||||
name: "missing owner",
|
||||
yaml: `agents:
|
||||
- repo: repo1
|
||||
`,
|
||||
expectErr: true,
|
||||
errSubstr: "entry 0 missing owner or repo",
|
||||
desc: "should fail on missing owner",
|
||||
},
|
||||
{
|
||||
name: "missing repo",
|
||||
yaml: `agents:
|
||||
- owner: org
|
||||
`,
|
||||
expectErr: true,
|
||||
errSubstr: "entry 0 missing owner or repo",
|
||||
desc: "should fail on missing repo",
|
||||
},
|
||||
{
|
||||
name: "empty owner",
|
||||
yaml: `agents:
|
||||
- owner: ""
|
||||
repo: repo1
|
||||
`,
|
||||
expectErr: true,
|
||||
errSubstr: "entry 0 missing owner or repo",
|
||||
desc: "should fail on empty owner",
|
||||
},
|
||||
{
|
||||
name: "empty repo",
|
||||
yaml: `agents:
|
||||
- owner: org
|
||||
repo: ""
|
||||
`,
|
||||
expectErr: true,
|
||||
errSubstr: "entry 0 missing owner or repo",
|
||||
desc: "should fail on empty repo",
|
||||
},
|
||||
{
|
||||
name: "valid multiple entries",
|
||||
yaml: `agents:
|
||||
- owner: org1
|
||||
repo: repo1
|
||||
- owner: org2
|
||||
repo: repo2
|
||||
category: ai
|
||||
`,
|
||||
expectErr: false,
|
||||
desc: "should load multiple valid entries",
|
||||
},
|
||||
{
|
||||
name: "second entry missing owner",
|
||||
yaml: `agents:
|
||||
- owner: org1
|
||||
repo: repo1
|
||||
- repo: repo2
|
||||
`,
|
||||
expectErr: true,
|
||||
errSubstr: "entry 1 missing owner or repo",
|
||||
desc: "should report correct index for failing entry",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
tmpFile := tmpDir + "/agents.yml"
|
||||
|
||||
if err := os.WriteFile(tmpFile, []byte(tt.yaml), 0600); err != nil {
|
||||
t.Fatalf("WriteFile failed: %v", err)
|
||||
}
|
||||
|
||||
agents, err := loadAgents(tmpFile)
|
||||
|
||||
if tt.expectErr {
|
||||
if err == nil {
|
||||
t.Errorf("%s: expected error, got nil", tt.desc)
|
||||
} else if !strings.Contains(err.Error(), tt.errSubstr) {
|
||||
t.Errorf("%s: expected error containing %q, got %q", tt.desc, tt.errSubstr, err.Error())
|
||||
}
|
||||
if agents != nil {
|
||||
t.Errorf("%s: expected nil agents on error, got %v", tt.desc, agents)
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
t.Errorf("%s: expected no error, got %v", tt.desc, err)
|
||||
}
|
||||
if agents == nil {
|
||||
t.Errorf("%s: expected non-nil agents", tt.desc)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAgents_EmptyList(t *testing.T) {
|
||||
// Empty agents list should load successfully (edge case).
|
||||
tmpDir := t.TempDir()
|
||||
tmpFile := tmpDir + "/agents.yml"
|
||||
|
||||
yaml := `agents: []`
|
||||
if err := os.WriteFile(tmpFile, []byte(yaml), 0600); err != nil {
|
||||
t.Fatalf("WriteFile failed: %v", err)
|
||||
}
|
||||
|
||||
agents, err := loadAgents(tmpFile)
|
||||
if err != nil {
|
||||
t.Errorf("expected no error, got %v", err)
|
||||
}
|
||||
if agents == nil || len(agents) != 0 {
|
||||
t.Errorf("expected empty agents slice, got %v", agents)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAgents_MalformedYAML(t *testing.T) {
|
||||
// Malformed YAML should return a parse error.
|
||||
tmpDir := t.TempDir()
|
||||
tmpFile := tmpDir + "/agents.yml"
|
||||
|
||||
yaml := `agents:
|
||||
- owner: org
|
||||
repo: repo1
|
||||
- this is not valid yaml: {`
|
||||
|
||||
if err := os.WriteFile(tmpFile, []byte(yaml), 0600); err != nil {
|
||||
t.Fatalf("WriteFile failed: %v", err)
|
||||
}
|
||||
|
||||
agents, err := loadAgents(tmpFile)
|
||||
if err == nil {
|
||||
t.Errorf("expected error on malformed YAML, got nil")
|
||||
}
|
||||
if agents != nil {
|
||||
t.Errorf("expected nil agents on parse error, got %v", agents)
|
||||
}
|
||||
}
|
||||
@@ -5,14 +5,16 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Stat struct {
|
||||
// CanonicalKey is owner/repo from agents.yml — stable across renames.
|
||||
CanonicalKey string
|
||||
Owner string
|
||||
Repo string
|
||||
Category string
|
||||
@@ -53,62 +55,53 @@ const repoFields = `
|
||||
nameWithOwner
|
||||
`
|
||||
|
||||
// httpClient has a timeout to prevent hung workflow jobs.
|
||||
var httpClient = &http.Client{Timeout: 30 * time.Second}
|
||||
|
||||
// chunkSize is the max aliases per GraphQL request (GitHub node-limit safety margin).
|
||||
const chunkSize = 50
|
||||
|
||||
// maxRetries and retry backoff for transient HTTP/network errors.
|
||||
const maxRetries = 3
|
||||
|
||||
var retryBackoff = []time.Duration{2 * time.Second, 4 * time.Second, 8 * time.Second}
|
||||
|
||||
// fetchStats queries GitHub GraphQL in chunks of up to chunkSize repos per
|
||||
// request. Returns an error if any GraphQL errors are present OR if any
|
||||
// requested repo is missing from the response — better to fail loud than
|
||||
// silently publish a shorter README.
|
||||
func fetchStats(token string, agents []Agent) ([]Stat, error) {
|
||||
var b strings.Builder
|
||||
b.WriteString("query {\n")
|
||||
for i, a := range agents {
|
||||
fmt.Fprintf(&b, " r%d: repository(owner: %q, name: %q) {%s}\n", i, a.Owner, a.Repo, repoFields)
|
||||
}
|
||||
b.WriteString("}\n")
|
||||
collected := make(map[string]*repoNode, len(agents))
|
||||
|
||||
body, err := json.Marshal(map[string]string{"query": b.String()})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for start := 0; start < len(agents); start += chunkSize {
|
||||
end := start + chunkSize
|
||||
if end > len(agents) {
|
||||
end = len(agents)
|
||||
}
|
||||
chunk := agents[start:end]
|
||||
|
||||
req, err := http.NewRequest("POST", "https://api.github.com/graphql", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("User-Agent", "awesome-coding-agents-updater")
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
raw, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("graphql HTTP %d: %s", resp.StatusCode, raw)
|
||||
}
|
||||
|
||||
var out graphQLResponse
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return nil, fmt.Errorf("decode response: %w (body=%s)", err, raw)
|
||||
}
|
||||
for _, e := range out.Errors {
|
||||
// repos that 404 or are renamed land here; continue with partial data
|
||||
fmt.Fprintf(os.Stderr, "graphql warn: %s (path=%v)\n", e.Message, e.Path)
|
||||
nodes, err := fetchChunk(token, chunk, start)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for k, v := range nodes {
|
||||
collected[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
stats := make([]Stat, 0, len(agents))
|
||||
for i, a := range agents {
|
||||
node := out.Data[fmt.Sprintf("r%d", i)]
|
||||
alias := fmt.Sprintf("r%d", i)
|
||||
node := collected[alias]
|
||||
if node == nil {
|
||||
fmt.Fprintf(os.Stderr, "skip %s/%s — no data\n", a.Owner, a.Repo)
|
||||
continue
|
||||
return nil, fmt.Errorf("repo %s/%s missing from GraphQL response", a.Owner, a.Repo)
|
||||
}
|
||||
lang := ""
|
||||
if node.PrimaryLanguage != nil {
|
||||
lang = node.PrimaryLanguage.Name
|
||||
}
|
||||
stats = append(stats, Stat{
|
||||
CanonicalKey: a.Owner + "/" + a.Repo,
|
||||
Owner: a.Owner,
|
||||
Repo: a.Repo,
|
||||
Category: a.Category,
|
||||
@@ -121,11 +114,102 @@ func fetchStats(token string, agents []Agent) ([]Stat, error) {
|
||||
NameWithOwner: node.NameWithOwner,
|
||||
})
|
||||
}
|
||||
|
||||
// Sort by stars descending. Ties are ordered by CanonicalKey for determinism
|
||||
// regardless of map-iteration or agents.yml order.
|
||||
sort.Slice(stats, func(i, j int) bool {
|
||||
if stats[i].Stars != stats[j].Stars {
|
||||
return stats[i].Stars > stats[j].Stars
|
||||
}
|
||||
return stats[i].CanonicalKey < stats[j].CanonicalKey
|
||||
})
|
||||
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func sortByStars(stats []Stat) {
|
||||
sort.SliceStable(stats, func(i, j int) bool {
|
||||
return stats[i].Stars > stats[j].Stars
|
||||
})
|
||||
// fetchChunk sends one GraphQL request for a slice of agents. aliasOffset
|
||||
// ensures alias names (r0, r1, …) are globally unique across chunks.
|
||||
func fetchChunk(token string, agents []Agent, aliasOffset int) (map[string]*repoNode, error) {
|
||||
var b strings.Builder
|
||||
b.WriteString("query {\n")
|
||||
for i, a := range agents {
|
||||
fmt.Fprintf(&b, " r%d: repository(owner: %q, name: %q) {%s}\n", aliasOffset+i, a.Owner, a.Repo, repoFields)
|
||||
}
|
||||
b.WriteString("}\n")
|
||||
|
||||
body, err := json.Marshal(map[string]string{"query": b.String()})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
raw, statusCode, err := doWithRetry(token, body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if statusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("graphql HTTP %d: %s", statusCode, raw)
|
||||
}
|
||||
|
||||
var out graphQLResponse
|
||||
if err := json.Unmarshal(raw, &out); err != nil {
|
||||
return nil, fmt.Errorf("decode response: %w (body=%s)", err, raw)
|
||||
}
|
||||
|
||||
// Treat any GraphQL-level error as fatal — partial data silently shrinks
|
||||
// the README and corrupts future delta math.
|
||||
if len(out.Errors) > 0 {
|
||||
msgs := make([]string, len(out.Errors))
|
||||
for i, e := range out.Errors {
|
||||
msgs[i] = fmt.Sprintf("%s (path=%v)", e.Message, e.Path)
|
||||
}
|
||||
return nil, fmt.Errorf("graphql errors: %s", strings.Join(msgs, "; "))
|
||||
}
|
||||
|
||||
return out.Data, nil
|
||||
}
|
||||
|
||||
// doWithRetry executes the GraphQL POST with exponential backoff on transient
|
||||
// errors (network failures, HTTP 5xx, HTTP 429). 4xx other than 429 are not
|
||||
// retried.
|
||||
func doWithRetry(token string, body []byte) ([]byte, int, error) {
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < maxRetries; attempt++ {
|
||||
if attempt > 0 {
|
||||
log.Printf("retry attempt %d after %v: %v", attempt, retryBackoff[attempt-1], lastErr)
|
||||
time.Sleep(retryBackoff[attempt-1])
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", "https://api.github.com/graphql", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("User-Agent", "awesome-coding-agents-updater")
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue // network error — retry
|
||||
}
|
||||
|
||||
raw, err := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
lastErr = fmt.Errorf("read body: %w", err)
|
||||
continue
|
||||
}
|
||||
|
||||
sc := resp.StatusCode
|
||||
if sc == http.StatusOK {
|
||||
return raw, sc, nil
|
||||
}
|
||||
if sc == http.StatusTooManyRequests || sc >= 500 {
|
||||
lastErr = fmt.Errorf("HTTP %d: %s", sc, raw)
|
||||
continue // retryable
|
||||
}
|
||||
// 4xx (except 429) — not retryable
|
||||
return raw, sc, nil
|
||||
}
|
||||
return nil, 0, fmt.Errorf("all %d attempts failed; last error: %w", maxRetries, lastErr)
|
||||
}
|
||||
|
||||
+71
-17
@@ -3,7 +3,9 @@ package main
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"slices"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -12,11 +14,24 @@ type Snapshot struct {
|
||||
Stars map[string]int `json:"stars"`
|
||||
}
|
||||
|
||||
// canonicalKeyMigrations remaps old history keys to the current canonical
|
||||
// owner/repo from agents.yml. Add entries here when a tracked repo is renamed
|
||||
// and history.jsonl contains the old name.
|
||||
//
|
||||
// Known renames:
|
||||
// - block/goose was formerly tracked as aaif-goose/goose
|
||||
var canonicalKeyMigrations = map[string]string{
|
||||
"aaif-goose/goose": "block/goose",
|
||||
}
|
||||
|
||||
func appendHistory(path string, stats []Stat) (map[string]int, error) {
|
||||
today := time.Now().UTC().Format("2006-01-02")
|
||||
|
||||
// Key snapshots by canonical owner/repo from agents.yml, not by the
|
||||
// API-returned nameWithOwner, so renames don't orphan historical data.
|
||||
current := Snapshot{Date: today, Stars: map[string]int{}}
|
||||
for _, s := range stats {
|
||||
current.Stars[s.NameWithOwner] = s.Stars
|
||||
current.Stars[s.CanonicalKey] = s.Stars
|
||||
}
|
||||
|
||||
snapshots, err := readSnapshots(path)
|
||||
@@ -26,13 +41,8 @@ func appendHistory(path string, stats []Stat) (map[string]int, error) {
|
||||
|
||||
deltas := computeDeltas(snapshots, current)
|
||||
|
||||
// drop any pre-existing snapshot for today, then append current
|
||||
kept := snapshots[:0]
|
||||
for _, s := range snapshots {
|
||||
if s.Date != today {
|
||||
kept = append(kept, s)
|
||||
}
|
||||
}
|
||||
// Drop any pre-existing snapshot for today, then append current.
|
||||
kept := slices.DeleteFunc(snapshots, func(s Snapshot) bool { return s.Date == today })
|
||||
kept = append(kept, current)
|
||||
|
||||
return deltas, writeSnapshots(path, kept)
|
||||
@@ -51,45 +61,89 @@ func readSnapshots(path string) ([]Snapshot, error) {
|
||||
var out []Snapshot
|
||||
scanner := bufio.NewScanner(f)
|
||||
scanner.Buffer(make([]byte, 1024*1024), 16*1024*1024)
|
||||
lineNum := 0
|
||||
for scanner.Scan() {
|
||||
lineNum++
|
||||
line := scanner.Bytes()
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
var s Snapshot
|
||||
if err := json.Unmarshal(line, &s); err != nil {
|
||||
// skip malformed lines rather than fail the whole run
|
||||
// Log corruption so it's observable; don't silently drop lines.
|
||||
fmt.Fprintf(os.Stderr, "history.jsonl line %d: skipping malformed entry: %v\n", lineNum, err)
|
||||
continue
|
||||
}
|
||||
// Apply canonical-key migrations so history keyed under old names is
|
||||
// transparently remapped to the current agents.yml canonical key.
|
||||
s.Stars = applyMigrations(s.Stars)
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, scanner.Err()
|
||||
}
|
||||
|
||||
// applyMigrations rewrites any deprecated history keys to their current
|
||||
// canonical form. Old keys are removed; new keys accumulate stars additively
|
||||
// (in practice the old key had no concurrent new entry, so max is the same).
|
||||
func applyMigrations(stars map[string]int) map[string]int {
|
||||
for old, canonical := range canonicalKeyMigrations {
|
||||
if v, ok := stars[old]; ok {
|
||||
if stars[canonical] < v {
|
||||
stars[canonical] = v
|
||||
}
|
||||
delete(stars, old)
|
||||
}
|
||||
}
|
||||
return stars
|
||||
}
|
||||
|
||||
// writeSnapshots writes to a temp file then renames atomically so a crash
|
||||
// mid-write never leaves history.jsonl truncated or partially written.
|
||||
func writeSnapshots(path string, snapshots []Snapshot) error {
|
||||
f, err := os.Create(path)
|
||||
tmp := path + ".tmp"
|
||||
f, err := os.Create(tmp)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
enc := json.NewEncoder(f)
|
||||
writeErr := error(nil)
|
||||
for _, s := range snapshots {
|
||||
if err := enc.Encode(s); err != nil {
|
||||
return err
|
||||
writeErr = err
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
if syncErr := f.Sync(); syncErr != nil && writeErr == nil {
|
||||
writeErr = syncErr
|
||||
}
|
||||
f.Close()
|
||||
|
||||
if writeErr != nil {
|
||||
os.Remove(tmp)
|
||||
return writeErr
|
||||
}
|
||||
|
||||
return os.Rename(tmp, path)
|
||||
}
|
||||
|
||||
// computeDeltas returns stars-now minus stars-at-or-before-cutoff for each repo.
|
||||
// Cutoff = 7 days ago UTC. If no snapshot is old enough, delta is omitted.
|
||||
// computeDeltas returns stars-now minus stars-at-or-before-cutoff for each
|
||||
// repo. Cutoff = 7 days ago UTC. The chosen prior snapshot must be within a
|
||||
// 3-day window of the cutoff; if cron was skipped for more than 10 days the
|
||||
// delta would be misleadingly labeled "Δ7d", so we return no delta instead.
|
||||
func computeDeltas(history []Snapshot, current Snapshot) map[string]int {
|
||||
deltas := map[string]int{}
|
||||
cutoff := time.Now().UTC().AddDate(0, 0, -7).Format("2006-01-02")
|
||||
now := time.Now().UTC()
|
||||
cutoff := now.AddDate(0, 0, -7).Format("2006-01-02")
|
||||
// Accept snapshots in (cutoff - 3 days, cutoff]. A snapshot older than
|
||||
// cutoff-3d is too stale to label as a 7-day delta.
|
||||
lowerBound := now.AddDate(0, 0, -10).Format("2006-01-02")
|
||||
|
||||
var base *Snapshot
|
||||
for i := range history {
|
||||
if history[i].Date <= cutoff {
|
||||
d := history[i].Date
|
||||
if d > lowerBound && d <= cutoff {
|
||||
base = &history[i]
|
||||
}
|
||||
}
|
||||
|
||||
+228
@@ -0,0 +1,228 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestComputeDeltas_NoHistory(t *testing.T) {
|
||||
// Empty history slice should return empty delta map without panicking.
|
||||
current := Snapshot{
|
||||
Date: "2026-05-14",
|
||||
Stars: map[string]int{
|
||||
"org/repo": 100,
|
||||
},
|
||||
}
|
||||
deltas := computeDeltas(nil, current)
|
||||
if deltas == nil {
|
||||
t.Errorf("expected non-nil empty map, got nil")
|
||||
}
|
||||
if len(deltas) != 0 {
|
||||
t.Errorf("expected empty deltas, got %d entries", len(deltas))
|
||||
}
|
||||
}
|
||||
|
||||
func TestComputeDeltas_SevenDayWindow(t *testing.T) {
|
||||
// The cutoff window is (cutoff-3d, cutoff] where cutoff = now - 7d.
|
||||
// So valid snapshots have date in range (now-10d, now-7d].
|
||||
|
||||
now := time.Now().UTC()
|
||||
today := now.Format("2006-01-02")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
daysAgo int
|
||||
expectDelta bool
|
||||
desc string
|
||||
}{
|
||||
{
|
||||
name: "snapshot at exactly 7 days ago",
|
||||
daysAgo: 7,
|
||||
expectDelta: true,
|
||||
desc: "at cutoff boundary — should be included",
|
||||
},
|
||||
{
|
||||
name: "snapshot at 9 days ago",
|
||||
daysAgo: 9,
|
||||
expectDelta: true,
|
||||
desc: "within (cutoff-3d, cutoff] — should be included",
|
||||
},
|
||||
{
|
||||
name: "snapshot at 11 days ago",
|
||||
daysAgo: 11,
|
||||
expectDelta: false,
|
||||
desc: "older than cutoff-3d — should be excluded",
|
||||
},
|
||||
{
|
||||
name: "snapshot at 6 days ago",
|
||||
daysAgo: 6,
|
||||
expectDelta: false,
|
||||
desc: "newer than cutoff — should be excluded",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
snapshotDate := now.AddDate(0, 0, -tt.daysAgo).Format("2006-01-02")
|
||||
history := []Snapshot{
|
||||
{
|
||||
Date: snapshotDate,
|
||||
Stars: map[string]int{
|
||||
"org/repo": 100,
|
||||
},
|
||||
},
|
||||
}
|
||||
current := Snapshot{
|
||||
Date: today,
|
||||
Stars: map[string]int{
|
||||
"org/repo": 150,
|
||||
},
|
||||
}
|
||||
deltas := computeDeltas(history, current)
|
||||
|
||||
if tt.expectDelta {
|
||||
if len(deltas) == 0 {
|
||||
t.Errorf("%s: expected delta for 'org/repo', got empty map", tt.desc)
|
||||
}
|
||||
if delta, ok := deltas["org/repo"]; !ok || delta != 50 {
|
||||
t.Errorf("%s: expected delta=50, got %v", tt.desc, delta)
|
||||
}
|
||||
} else {
|
||||
if len(deltas) > 0 {
|
||||
t.Errorf("%s: expected no delta, got %v", tt.desc, deltas)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadSnapshots_MalformedLine(t *testing.T) {
|
||||
// Temp JSONL with: valid, malformed, empty, valid lines.
|
||||
// Should skip malformed/empty and return only the 2 valid snapshots.
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
tmpFile := tmpDir + "/history.jsonl"
|
||||
|
||||
content := `{"date":"2026-05-01","stars":{"org/repo":100}}
|
||||
not json at all { bad
|
||||
{"date":"2026-05-02","stars":{"org/repo":200}}
|
||||
{"date":"2026-05-03","stars":{"org/repo":300}}
|
||||
`
|
||||
|
||||
if err := os.WriteFile(tmpFile, []byte(content), 0600); err != nil {
|
||||
t.Fatalf("WriteFile failed: %v", err)
|
||||
}
|
||||
|
||||
snapshots, err := readSnapshots(tmpFile)
|
||||
if err != nil {
|
||||
t.Fatalf("readSnapshots failed: %v", err)
|
||||
}
|
||||
|
||||
if len(snapshots) != 3 {
|
||||
t.Errorf("expected 3 snapshots (skipped 1 malformed), got %d", len(snapshots))
|
||||
}
|
||||
|
||||
// Check dates are in order.
|
||||
if len(snapshots) >= 3 {
|
||||
if snapshots[0].Date != "2026-05-01" {
|
||||
t.Errorf("snapshot[0] date: expected 2026-05-01, got %s", snapshots[0].Date)
|
||||
}
|
||||
if snapshots[1].Date != "2026-05-02" {
|
||||
t.Errorf("snapshot[1] date: expected 2026-05-02, got %s", snapshots[1].Date)
|
||||
}
|
||||
if snapshots[2].Date != "2026-05-03" {
|
||||
t.Errorf("snapshot[2] date: expected 2026-05-03, got %s", snapshots[2].Date)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadSnapshots_MissingFile(t *testing.T) {
|
||||
// Non-existent file should return nil with no error (not os.IsNotExist error).
|
||||
snapshots, err := readSnapshots("/nonexistent/path/history.jsonl")
|
||||
if err != nil {
|
||||
t.Errorf("expected nil error on missing file, got %v", err)
|
||||
}
|
||||
if snapshots != nil {
|
||||
t.Errorf("expected nil snapshots on missing file, got %v", snapshots)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyMigrations(t *testing.T) {
|
||||
// Test the canonical key migration logic.
|
||||
stars := map[string]int{
|
||||
"aaif-goose/goose": 45115,
|
||||
"other/repo": 1000,
|
||||
}
|
||||
|
||||
result := applyMigrations(stars)
|
||||
|
||||
// The old key should be removed.
|
||||
if _, ok := result["aaif-goose/goose"]; ok {
|
||||
t.Errorf("old key 'aaif-goose/goose' should be removed")
|
||||
}
|
||||
|
||||
// The new key should be present with the value.
|
||||
if v, ok := result["block/goose"]; !ok || v != 45115 {
|
||||
t.Errorf("new key 'block/goose': expected 45115, got %v", v)
|
||||
}
|
||||
|
||||
// Other keys should be unchanged.
|
||||
if v, ok := result["other/repo"]; !ok || v != 1000 {
|
||||
t.Errorf("'other/repo': expected 1000, got %v", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteSnapshots_AtomicWrite(t *testing.T) {
|
||||
// Verify that writeSnapshots uses atomic rename (writes to .tmp first).
|
||||
tmpDir := t.TempDir()
|
||||
tmpFile := tmpDir + "/history.jsonl"
|
||||
|
||||
snapshots := []Snapshot{
|
||||
{Date: "2026-05-01", Stars: map[string]int{"org/repo": 100}},
|
||||
{Date: "2026-05-02", Stars: map[string]int{"org/repo": 200}},
|
||||
}
|
||||
|
||||
err := writeSnapshots(tmpFile, snapshots)
|
||||
if err != nil {
|
||||
t.Fatalf("writeSnapshots failed: %v", err)
|
||||
}
|
||||
|
||||
// Verify file exists and has correct content.
|
||||
if _, err := os.Stat(tmpFile); err != nil {
|
||||
t.Fatalf("output file missing: %v", err)
|
||||
}
|
||||
|
||||
content, err := os.ReadFile(tmpFile)
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFile failed: %v", err)
|
||||
}
|
||||
|
||||
// Verify JSON is valid and correct.
|
||||
var readBack []Snapshot
|
||||
decoder := json.NewDecoder(bytes.NewReader(content))
|
||||
for decoder.More() {
|
||||
var s Snapshot
|
||||
if err := decoder.Decode(&s); err != nil {
|
||||
t.Fatalf("decode failed: %v", err)
|
||||
}
|
||||
readBack = append(readBack, s)
|
||||
}
|
||||
|
||||
if len(readBack) != 2 {
|
||||
t.Errorf("expected 2 snapshots, got %d", len(readBack))
|
||||
}
|
||||
if readBack[0].Date != "2026-05-01" || readBack[0].Stars["org/repo"] != 100 {
|
||||
t.Errorf("snapshot 0 mismatch: %+v", readBack[0])
|
||||
}
|
||||
if readBack[1].Date != "2026-05-02" || readBack[1].Stars["org/repo"] != 200 {
|
||||
t.Errorf("snapshot 1 mismatch: %+v", readBack[1])
|
||||
}
|
||||
|
||||
// .tmp file should not exist (cleaned up after rename).
|
||||
if _, err := os.Stat(tmpFile + ".tmp"); !os.IsNotExist(err) {
|
||||
t.Errorf("temp file should not exist after successful write")
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,7 @@ func main() {
|
||||
func run() error {
|
||||
agents, err := loadAgents("data/agents.yml")
|
||||
if err != nil {
|
||||
return fmt.Errorf("load agents: %w", err)
|
||||
return err
|
||||
}
|
||||
if len(agents) == 0 {
|
||||
return fmt.Errorf("no agents in data/agents.yml")
|
||||
@@ -28,18 +28,16 @@ func run() error {
|
||||
|
||||
stats, err := fetchStats(token, agents)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetch stats: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
sortByStars(stats)
|
||||
|
||||
deltas, err := appendHistory("data/history.jsonl", stats)
|
||||
if err != nil {
|
||||
return fmt.Errorf("append history: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
if err := renderReadme("templates/readme.tmpl", "README.md", stats, deltas); err != nil {
|
||||
return fmt.Errorf("render readme: %w", err)
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Printf("updated %d agents\n", len(stats))
|
||||
|
||||
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"text/template"
|
||||
"time"
|
||||
@@ -24,7 +25,8 @@ type Row struct {
|
||||
func renderReadme(tmplPath, outPath string, stats []Stat, deltas map[string]int) error {
|
||||
rows := make([]Row, len(stats))
|
||||
for i, s := range stats {
|
||||
delta, has := deltas[s.NameWithOwner]
|
||||
// Deltas are keyed by CanonicalKey (owner/repo from agents.yml).
|
||||
delta, has := deltas[s.CanonicalKey]
|
||||
rows[i] = Row{
|
||||
Rank: i + 1,
|
||||
NameWithOwner: s.NameWithOwner,
|
||||
@@ -75,12 +77,7 @@ func renderReadme(tmplPath, outPath string, stats []Stat, deltas map[string]int)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
name := tmplPath
|
||||
if i := strings.LastIndex(tmplPath, "/"); i >= 0 {
|
||||
name = tmplPath[i+1:]
|
||||
}
|
||||
|
||||
return tmpl.ExecuteTemplate(f, name, map[string]any{
|
||||
return tmpl.ExecuteTemplate(f, filepath.Base(tmplPath), map[string]any{
|
||||
"Rows": rows,
|
||||
"UpdatedAt": time.Now().UTC().Format("2006-01-02 15:04 UTC"),
|
||||
"Total": len(rows),
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSanitizeCell(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expected string
|
||||
desc string
|
||||
}{
|
||||
{
|
||||
name: "plain text",
|
||||
input: "hello world",
|
||||
expected: "hello world",
|
||||
desc: "unchanged",
|
||||
},
|
||||
{
|
||||
name: "pipe char",
|
||||
input: "foo | bar",
|
||||
expected: "foo \\| bar",
|
||||
desc: "pipe escaped",
|
||||
},
|
||||
{
|
||||
name: "newline",
|
||||
input: "line1\nline2",
|
||||
expected: "line1 line2",
|
||||
desc: "newline becomes space",
|
||||
},
|
||||
{
|
||||
name: "carriage return",
|
||||
input: "line1\rline2",
|
||||
expected: "line1 line2",
|
||||
desc: "carriage return becomes space",
|
||||
},
|
||||
{
|
||||
name: "pipe and newline",
|
||||
input: "foo | bar\nbaz",
|
||||
expected: "foo \\| bar baz",
|
||||
desc: "both handled correctly",
|
||||
},
|
||||
{
|
||||
name: "empty string",
|
||||
input: "",
|
||||
expected: "",
|
||||
desc: "empty string unchanged",
|
||||
},
|
||||
{
|
||||
name: "multiple pipes",
|
||||
input: "a | b | c",
|
||||
expected: "a \\| b \\| c",
|
||||
desc: "multiple pipes escaped",
|
||||
},
|
||||
{
|
||||
name: "multiple newlines",
|
||||
input: "line1\n\nline2",
|
||||
expected: "line1 line2",
|
||||
desc: "multiple newlines become spaces",
|
||||
},
|
||||
{
|
||||
name: "whitespace trimming",
|
||||
input: " text ",
|
||||
expected: "text",
|
||||
desc: "leading/trailing spaces trimmed",
|
||||
},
|
||||
{
|
||||
name: "complex: whitespace, pipe, newline",
|
||||
input: " foo | bar\nbaz ",
|
||||
expected: "foo \\| bar baz",
|
||||
desc: "all transformations applied",
|
||||
},
|
||||
{
|
||||
name: "only newlines and pipes",
|
||||
input: "|\n|",
|
||||
expected: "\\| \\|",
|
||||
desc: "pipes and newlines only",
|
||||
},
|
||||
{
|
||||
name: "mixed line endings",
|
||||
input: "line1\nline2\rline3",
|
||||
expected: "line1 line2 line3",
|
||||
desc: "both \\n and \\r converted",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := sanitizeCell(tt.input)
|
||||
if result != tt.expected {
|
||||
t.Errorf("%s: expected %q, got %q", tt.desc, tt.expected, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user