mirror of
https://github.com/tiennm99/miti99bot.git
synced 2026-09-06 04:16:48 +00:00
refactor(loldle): source all champion data from loldle.net
loldle.net's JS bundle ships the complete set of classic-mode axes in plaintext, so ddragon merging is no longer needed. Scraper now produces the final schema directly. Schema changes: drop title, skinCount, image, and genre (ddragon-only). Replace genre (class tags like Fighter/Mage) with species (Human/Darkin/ Vastayan) — the axis loldle.net actually uses. Promote region to a multi-value field so multi-region champions compare correctly. Handlers no longer show "Name — Title" on win/giveup.
This commit is contained in:
@@ -1,11 +1,12 @@
|
||||
name: scrape-loldle-data
|
||||
|
||||
# Rebuilds src/modules/loldle/champions.json every Monday 06:00 UTC by
|
||||
# scraping loldle.net's JS bundle + merging ddragon championFull.
|
||||
# Opens a PR if the output changed. Manually triggerable from Actions tab.
|
||||
# scraping loldle.net's JS bundle (sole source of truth for classic-mode
|
||||
# fields). Opens a PR if the output changed. Manually triggerable from the
|
||||
# Actions tab.
|
||||
#
|
||||
# Note: the bundled data is shipped with the Worker — the change only takes
|
||||
# effect after `npm run deploy` is run on the updated main branch.
|
||||
# Note: the bundled data ships with the Worker — the change only takes effect
|
||||
# after `npm run deploy` is run on the updated main branch.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
@@ -41,11 +42,9 @@ jobs:
|
||||
title: "data: weekly loldle.net champion refresh"
|
||||
body: |
|
||||
Automated weekly refresh of `src/modules/loldle/champions.json`
|
||||
from loldle.net + ddragon championFull.
|
||||
|
||||
Fields sourced from loldle.net: `gender`, `attackType`, `lane`,
|
||||
`region`, `releaseDate`. Display fields (`title`, `resource`,
|
||||
`genre`, `skinCount`, `image`) come from ddragon.
|
||||
from loldle.net's JS bundle — the canonical source for all
|
||||
classic-mode fields (`gender`, `species`, `resource`,
|
||||
`attackType`, `region`, `lane`, `releaseDate`).
|
||||
|
||||
Review the diff, merge, then run `npm run deploy` to ship.
|
||||
add-paths: |
|
||||
|
||||
@@ -1,16 +1,15 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* @file scrape-loldle-data — rebuilds src/modules/loldle/champions.json by
|
||||
* scraping loldle.net for canonical game fields (gender, positions,
|
||||
* range_type, regions, release_date) and merging with ddragon championFull
|
||||
* for display fields (title, resource, genre tags, skin count, sprite image).
|
||||
* @file scrape-loldle-data — rebuilds src/modules/loldle/champions.json from
|
||||
* loldle.net's JS bundle, the canonical source for the classic-mode axes:
|
||||
* gender, species, resource, attackType, region, lane, releaseDate.
|
||||
*
|
||||
* loldle.net embeds the full champion array in plaintext inside its JS bundle
|
||||
* at `<script src="js/index.<hash>.js">`, one record per champion with the
|
||||
* exact shape the bot needs. No CryptoJS decoding, no ddragon merge.
|
||||
*
|
||||
* Writes both champions.json (authoring format) and champions-data.js (ESM
|
||||
* wrapper consumed by the bot). Replaces the hand-run build-loldle-data step.
|
||||
*
|
||||
* Source of truth — loldle.net embeds the full champion array in plaintext
|
||||
* inside its JS bundle at `<script src="js/index.<hash>.js">`, one record per
|
||||
* champion with the exact shape the bot needs. No CryptoJS decoding needed.
|
||||
* wrapper consumed by the bot).
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/scrape-loldle-data.js
|
||||
@@ -22,9 +21,6 @@ import { writeFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
const LOLDLE_CLASSIC = "https://loldle.net/classic";
|
||||
const DDRAGON_VERSIONS = "https://ddragon.leagueoflegends.com/api/versions.json";
|
||||
const ddragonChampUrl = (v) =>
|
||||
`https://ddragon.leagueoflegends.com/cdn/${v}/data/en_US/championFull.json`;
|
||||
|
||||
const LANE_MAP = {
|
||||
top: "top",
|
||||
@@ -37,7 +33,7 @@ const LANE_MAP = {
|
||||
const GENDER_MAP = { male: "male", female: "female", other: "divers" };
|
||||
|
||||
const CHAMPION_RECORD_RX =
|
||||
/\{_id:"[a-f0-9]+",championId:"[^"]+",championName:"([^"]+)",gender:"([^"]+)",positions:\[([^\]]+)\],species:\[[^\]]+\],resource:"[^"]+",range_type:\[([^\]]+)\],regions:\[([^\]]+)\],release_date:"(\d{4})-\d{2}-\d{2}"\}/g;
|
||||
/\{_id:"[a-f0-9]+",championId:"[^"]+",championName:"([^"]+)",gender:"([^"]+)",positions:\[([^\]]+)\],species:\[([^\]]+)\],resource:"([^"]+)",range_type:\[([^\]]+)\],regions:\[([^\]]+)\],release_date:"(\d{4})-\d{2}-\d{2}"\}/g;
|
||||
|
||||
async function fetchText(url) {
|
||||
const res = await fetch(url);
|
||||
@@ -45,12 +41,6 @@ async function fetchText(url) {
|
||||
return res.text();
|
||||
}
|
||||
|
||||
async function fetchJson(url) {
|
||||
const res = await fetch(url);
|
||||
if (!res.ok) throw new Error(`fetch ${url}: ${res.status} ${res.statusText}`);
|
||||
return res.json();
|
||||
}
|
||||
|
||||
function parseJsArrayStrings(inner) {
|
||||
return [...inner.matchAll(/"([^"]+)"/g)].map((m) => m[1]);
|
||||
}
|
||||
@@ -69,7 +59,7 @@ async function scrapeLoldle() {
|
||||
const seen = new Set();
|
||||
const records = [];
|
||||
for (const m of bundle.matchAll(CHAMPION_RECORD_RX)) {
|
||||
const [, name, gender, positionsRaw, rangeTypeRaw, regionsRaw, year] = m;
|
||||
const [, name, gender, positionsRaw, speciesRaw, resource, rangeTypeRaw, regionsRaw, year] = m;
|
||||
if (seen.has(name)) continue;
|
||||
seen.add(name);
|
||||
|
||||
@@ -77,14 +67,18 @@ async function scrapeLoldle() {
|
||||
.map((p) => LANE_MAP[p.toLowerCase()])
|
||||
.filter(Boolean);
|
||||
const regions = parseJsArrayStrings(regionsRaw).map(normalizeRegion);
|
||||
const species = parseJsArrayStrings(speciesRaw).map((s) => s.toLowerCase());
|
||||
const rangeType = parseJsArrayStrings(rangeTypeRaw)[0]?.toLowerCase();
|
||||
|
||||
records.push({
|
||||
id: name,
|
||||
name,
|
||||
gender: GENDER_MAP[gender.toLowerCase()] ?? "divers",
|
||||
species: species.join(","),
|
||||
resource,
|
||||
attackType: rangeType === "melee" ? "close" : "range",
|
||||
lane: lanes.join(","),
|
||||
region: regions.join(","),
|
||||
lane: lanes.join(","),
|
||||
releaseDate: Number(year),
|
||||
});
|
||||
}
|
||||
@@ -94,69 +88,19 @@ async function scrapeLoldle() {
|
||||
"loldle.net: zero champion records parsed — bundle format changed, update CHAMPION_RECORD_RX",
|
||||
);
|
||||
}
|
||||
records.sort((a, b) => a.name.localeCompare(b.name));
|
||||
return records;
|
||||
}
|
||||
|
||||
async function fetchDdragon() {
|
||||
const versions = await fetchJson(DDRAGON_VERSIONS);
|
||||
const version = versions[0];
|
||||
const full = await fetchJson(ddragonChampUrl(version));
|
||||
return { version, champions: full.data };
|
||||
}
|
||||
|
||||
function mergeRecords(loldleRecords, ddragonChampions) {
|
||||
const byName = new Map(loldleRecords.map((r) => [r.name, r]));
|
||||
const merged = [];
|
||||
const missing = [];
|
||||
|
||||
for (const champ of Object.values(ddragonChampions)) {
|
||||
const lol = byName.get(champ.name);
|
||||
if (!lol) {
|
||||
missing.push(champ.name);
|
||||
continue;
|
||||
}
|
||||
merged.push({
|
||||
id: champ.id,
|
||||
name: champ.name,
|
||||
title: champ.title,
|
||||
resource: champ.partype,
|
||||
genre: champ.tags.join(","),
|
||||
skinCount: champ.skins.length,
|
||||
image: champ.image,
|
||||
gender: lol.gender,
|
||||
attackType: lol.attackType,
|
||||
releaseDate: lol.releaseDate,
|
||||
region: lol.region,
|
||||
lane: lol.lane,
|
||||
});
|
||||
}
|
||||
|
||||
if (missing.length > 0) {
|
||||
console.warn(
|
||||
`warn: ${missing.length} ddragon champions absent from loldle.net (likely just-released): ${missing.join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
merged.sort((a, b) => a.name.localeCompare(b.name));
|
||||
return merged;
|
||||
}
|
||||
|
||||
const root = resolve(import.meta.dirname, "..");
|
||||
const jsonPath = resolve(root, "src/modules/loldle/champions.json");
|
||||
const esmPath = resolve(root, "src/modules/loldle/champions-data.js");
|
||||
|
||||
console.log("scraping loldle.net…");
|
||||
const loldleRecords = await scrapeLoldle();
|
||||
console.log(` parsed ${loldleRecords.length} champions from loldle.net`);
|
||||
const records = await scrapeLoldle();
|
||||
console.log(` parsed ${records.length} champions`);
|
||||
|
||||
console.log("fetching ddragon championFull…");
|
||||
const { version, champions } = await fetchDdragon();
|
||||
console.log(` ddragon ${version}: ${Object.keys(champions).length} champions`);
|
||||
|
||||
const merged = mergeRecords(loldleRecords, champions);
|
||||
console.log(`merged ${merged.length} champions`);
|
||||
|
||||
const json = JSON.stringify(merged, null, 4);
|
||||
const json = JSON.stringify(records, null, 4);
|
||||
writeFileSync(jsonPath, `${json}\n`);
|
||||
console.log(`wrote ${jsonPath}`);
|
||||
|
||||
|
||||
+695
-2554
File diff suppressed because it is too large
Load Diff
+695
-2554
File diff suppressed because it is too large
Load Diff
@@ -5,10 +5,10 @@
|
||||
|
||||
export const CLASSIC_ATTRIBUTES = [
|
||||
{ key: "gender", label: "Gender", type: "exact" },
|
||||
{ key: "genre", label: "Genre", type: "multi" },
|
||||
{ key: "species", label: "Species", type: "multi" },
|
||||
{ key: "attackType", label: "Range", type: "exact" },
|
||||
{ key: "resource", label: "Resource", type: "exact" },
|
||||
{ key: "region", label: "Region", type: "exact" },
|
||||
{ key: "region", label: "Region", type: "multi" },
|
||||
{ key: "lane", label: "Lane", type: "multi" },
|
||||
{ key: "releaseDate", label: "Year", type: "year" },
|
||||
];
|
||||
@@ -102,8 +102,11 @@ function formatValue(key, value) {
|
||||
case "attackType":
|
||||
return str === "close" ? "Melee" : "Ranged";
|
||||
case "region":
|
||||
return str.split("-").map(capitalize).join(" ");
|
||||
case "genre":
|
||||
return str
|
||||
.split(",")
|
||||
.map((s) => s.split("-").map(capitalize).join(" "))
|
||||
.join(", ");
|
||||
case "species":
|
||||
case "lane":
|
||||
return str
|
||||
.split(",")
|
||||
|
||||
@@ -141,7 +141,7 @@ export async function handleLoldle(ctx, db) {
|
||||
|
||||
const reply = renderGuess(guess.name, results);
|
||||
const elapsed = formatDuration(Date.now() - (game.startedAt ?? Date.now()));
|
||||
const champ = `${escapeHtml(target.name)} — ${escapeHtml(target.title)}`;
|
||||
const champ = escapeHtml(target.name);
|
||||
|
||||
if (won) {
|
||||
const s = await recordResult(db, subject, true);
|
||||
@@ -181,9 +181,7 @@ export async function handleGiveup(ctx, db) {
|
||||
const target = champions.find((c) => c.id === game.target);
|
||||
await startFreshGame(db, subject);
|
||||
await trySendSticker(ctx, GIVEUP_STICKERS);
|
||||
const answer = target
|
||||
? `${escapeHtml(target.name)} — ${escapeHtml(target.title)}`
|
||||
: escapeHtml(game.target);
|
||||
const answer = target ? escapeHtml(target.name) : escapeHtml(game.target);
|
||||
return ctx.reply(`🏳️ Answer was ${answer}.\n${NEW_ROUND_HINT}`, { parse_mode: "HTML" });
|
||||
}
|
||||
|
||||
|
||||
@@ -4,10 +4,10 @@ import { CLASSIC_ATTRIBUTES, compareChampions } from "../../../src/modules/loldl
|
||||
const aatrox = {
|
||||
id: "Aatrox",
|
||||
gender: "male",
|
||||
genre: "Fighter",
|
||||
species: "darkin",
|
||||
attackType: "close",
|
||||
resource: "Blood Well",
|
||||
region: "runeterra",
|
||||
resource: "Manaless",
|
||||
region: "runeterra,shurima",
|
||||
lane: "top",
|
||||
releaseDate: 2013,
|
||||
};
|
||||
@@ -15,7 +15,7 @@ const aatrox = {
|
||||
const ahri = {
|
||||
id: "Ahri",
|
||||
gender: "female",
|
||||
genre: "Mage,Assassin",
|
||||
species: "vastayan",
|
||||
attackType: "range",
|
||||
resource: "Mana",
|
||||
region: "ionia",
|
||||
@@ -26,7 +26,7 @@ const ahri = {
|
||||
const akali = {
|
||||
id: "Akali",
|
||||
gender: "female",
|
||||
genre: "Assassin",
|
||||
species: "human",
|
||||
attackType: "close",
|
||||
resource: "Energy",
|
||||
region: "ionia",
|
||||
@@ -49,18 +49,24 @@ describe("compareChampions", () => {
|
||||
expect(byKey(r, "gender").result).toBe("wrong");
|
||||
expect(byKey(r, "attackType").result).toBe("wrong");
|
||||
expect(byKey(r, "resource").result).toBe("wrong");
|
||||
expect(byKey(r, "region").result).toBe("wrong");
|
||||
});
|
||||
|
||||
it("multi-value partial overlap is partial", () => {
|
||||
const r = compareChampions(akali, ahri);
|
||||
expect(byKey(r, "genre").result).toBe("partial");
|
||||
const r = compareChampions({ ...akali, lane: "mid,top" }, { ...ahri, lane: "mid" });
|
||||
expect(byKey(r, "lane").result).toBe("partial");
|
||||
const r2 = compareChampions(
|
||||
{ ...aatrox, region: "runeterra,shurima" },
|
||||
{ ...ahri, region: "runeterra,ionia" },
|
||||
);
|
||||
expect(byKey(r2, "region").result).toBe("partial");
|
||||
});
|
||||
|
||||
it("multi-value identical sets are correct even if order/case differ", () => {
|
||||
const r = compareChampions({ ...akali, genre: "assassin" }, { ...akali, genre: "Assassin" });
|
||||
expect(byKey(r, "genre").result).toBe("correct");
|
||||
const r = compareChampions(
|
||||
{ ...akali, species: "human,ninja" },
|
||||
{ ...akali, species: "Ninja,Human" },
|
||||
);
|
||||
expect(byKey(r, "species").result).toBe("correct");
|
||||
});
|
||||
|
||||
it("year direction hints up when guess < target", () => {
|
||||
|
||||
@@ -4,7 +4,7 @@ import { renderBoard, renderGuess } from "../../../src/modules/loldle/render.js"
|
||||
/** Mirror the shape compareChampions returns — only the fields render.js reads. */
|
||||
const sampleResults = [
|
||||
{ key: "gender", label: "Gender", result: "correct", guessValue: "Male" },
|
||||
{ key: "genre", label: "Genre", result: "correct", guessValue: "Mage, Support" },
|
||||
{ key: "species", label: "Species", result: "correct", guessValue: "Human, Darkin" },
|
||||
{ key: "attackType", label: "Range", result: "correct", guessValue: "Ranged" },
|
||||
{ key: "resource", label: "Resource", result: "correct", guessValue: "Mana" },
|
||||
{ key: "region", label: "Region", result: "correct", guessValue: "Runeterra" },
|
||||
|
||||
Reference in New Issue
Block a user