feat: initial JavaScript port of store-scraper-bot

Node.js 20+ ESM port mirroring Java/Go implementations.

- 13 Telegram commands matching Java identifiers
- MongoDB schema parity (common, group, apple_app, google_app collections)
- Apple/Google scrapers calling store-scraper.vercel.app with 10-min cache
- Daily 7am Vietnam-time cron with weekend-silent mode
- HTML table renderer matching Java/Go output
- Docker + Compose (prod and dev)

Untested end-to-end against live Telegram or upstream API.
This commit is contained in:
tiennm99 committed 2026-04-26 20:16:00 +07:00
commit a656a18ce8
41 files changed
+1852

No files matched your search

+57
View File
@@ -0,0 +1,57 @@
import { getCachedAppleApp, saveAppleApp } from '../repository/apple-app-repository.js';
import { newAppleApp } from '../models/apple-app.js';
// Mirrors Java AppStoreScraper (api/apple/AppStoreScraper.java).
const BASE_URL = 'https://store-scraper.vercel.app/apple';
export function buildAppleRequestByTrackId(id, country) {
return { id, country, ratings: true };
}
export function buildAppleRequestByBundleId(appId, country) {
return { appId, country, ratings: true };
}
export function createAppleScraper(config) {
const { logger, appCacheSeconds } = config;
async function rawApp(req) {
const res = await fetch(`${BASE_URL}/app`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(req),
});
if (!res.ok) throw new Error(`apple HTTP status ${res.status}`);
return await res.text();
}
async function app(req) {
const text = await rawApp(req);
return JSON.parse(text);
}
async function cache(resp) {
if (!resp || !resp.appId) return;
try {
await saveAppleApp(newAppleApp(resp.appId, resp, Date.now()));
} catch (err) {
logger.warn({ appId: resp.appId, err: err.message }, 'failed to cache apple app');
}
}
async function getApp(appId, country) {
const cached = await getCachedAppleApp(appId, appCacheSeconds);
if (cached) return cached.app;
const resp = await app(buildAppleRequestByBundleId(appId, country));
await cache(resp);
return resp;
}
async function fetchAndCache(req) {
const resp = await app(req);
await cache(resp);
return resp;
}
return { rawApp, app, getApp, fetchAndCache };
}
+55
View File
@@ -0,0 +1,55 @@
import { getCachedGoogleApp, saveGoogleApp } from '../repository/google-app-repository.js';
import { newGoogleApp } from '../models/google-app.js';
// Mirrors Java GooglePlayScraper (api/google/GooglePlayScraper.java).
const BASE_URL = 'https://store-scraper.vercel.app/google';
export function buildGoogleRequest(appId, country) {
return { appId, country: country || 'vn' };
}
export function createGoogleScraper(config) {
const { logger, appCacheSeconds } = config;
async function rawApp(req) {
const res = await fetch(`${BASE_URL}/app`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(req),
});
if (!res.ok) throw new Error(`google HTTP status ${res.status}`);
return await res.text();
}
async function app(req) {
const text = await rawApp(req);
return JSON.parse(text);
}
async function cache(resp, fallbackId) {
if (!resp) return;
const id = resp.appId || fallbackId;
if (!id) return;
try {
await saveGoogleApp(newGoogleApp(id, resp, Date.now()));
} catch (err) {
logger.warn({ appId: id, err: err.message }, 'failed to cache google app');
}
}
async function getApp(appId, country) {
const cached = await getCachedGoogleApp(appId, appCacheSeconds);
if (cached) return cached.app;
const resp = await app(buildGoogleRequest(appId, country));
await cache(resp, appId);
return resp;
}
async function fetchAndCache(req) {
const resp = await app(req);
await cache(resp, req.appId);
return resp;
}
return { rawApp, app, getApp, fetchAndCache };
}
+118
View File
@@ -0,0 +1,118 @@
import TelegramBot from 'node-telegram-bot-api';
import { createInfoCommand } from './commands/info.js';
import { createAddGroupCommand } from './commands/add-group.js';
import { createDeleteGroupCommand } from './commands/delete-group.js';
import { createListGroupCommand } from './commands/list-group.js';
import { createAddAppleAppCommand } from './commands/add-apple-app.js';
import { createDeleteAppleAppCommand } from './commands/delete-apple-app.js';
import { createAddGoogleAppCommand } from './commands/add-google-app.js';
import { createDeleteGoogleAppCommand } from './commands/delete-google-app.js';
import { createListAppCommand } from './commands/list-app.js';
import { createCheckAppCommand } from './commands/check-app.js';
import { createCheckAppScoresCommand } from './commands/check-app-scores.js';
import { createRawAppleAppCommand } from './commands/raw-apple-app.js';
import { createRawGoogleAppCommand } from './commands/raw-google-app.js';
// HTML parse mode for all messages (Java parity).
const PARSE_MODE = 'HTML';
export function createBot(config, appleScraper, googleScraper) {
const tg = new TelegramBot(config.telegramBotToken, { polling: true });
const logger = config.logger;
const sender = {
async sendMessage(chatId, html) {
try {
await tg.sendMessage(chatId, html, {
parse_mode: PARSE_MODE,
disable_web_page_preview: true,
});
} catch (err) {
logger.warn({ chatId, err: err.message }, 'send message failed');
}
},
async sendMessageSilent(chatId, html) {
try {
await tg.sendMessage(chatId, html, {
parse_mode: PARSE_MODE,
disable_web_page_preview: true,
disable_notification: true,
});
} catch (err) {
logger.warn({ chatId, err: err.message }, 'send silent message failed');
}
},
async sendDocument(chatId, filename, body) {
try {
await tg.sendDocument(
chatId,
Buffer.from(body, 'utf8'),
{},
{ filename, contentType: 'application/json' },
);
} catch (err) {
logger.warn({ chatId, err: err.message }, 'send document failed');
}
},
};
// Java command identifiers — keep names matching exactly.
const commands = {
info: createInfoCommand(),
addgroup: createAddGroupCommand(config),
delgroup: createDeleteGroupCommand(config),
listgroup: createListGroupCommand(config),
addapple: createAddAppleAppCommand(appleScraper),
delapple: createDeleteAppleAppCommand(),
addgoogle: createAddGoogleAppCommand(googleScraper),
delgoogle: createDeleteGoogleAppCommand(),
listapp: createListAppCommand(),
checkapp: createCheckAppCommand(config, appleScraper, googleScraper),
checkappscore: createCheckAppScoresCommand(appleScraper, googleScraper),
rawappleapp: createRawAppleAppCommand(appleScraper),
rawgoogleapp: createRawGoogleAppCommand(googleScraper),
};
tg.on('message', async (msg) => {
const name = parseCommandName(msg.text, config.telegramBotUsername);
if (!name) return;
const handler = commands[name];
if (!handler) {
logger.debug({ command: name }, 'Unknown command');
return;
}
logger.info(
{ command: name, userId: msg.from?.id, chatId: msg.chat.id },
'Executing command',
);
try {
await handler(msg, sender);
} catch (err) {
logger.error({ err: err.message, command: name }, 'panic in command');
await sender.sendMessage(msg.chat.id, 'Internal server error');
}
});
tg.on('polling_error', (err) => {
logger.warn({ err: err.message }, 'polling error');
});
tg.getMe()
.then((me) => logger.info({ username: me.username }, 'Authorized on account'))
.catch((err) => logger.error({ err: err.message }, 'getMe failed'));
return { sender, telegram: tg };
}
// Extracts "info" from "/info", "/info arg", "/info@bot", "/info@bot arg".
function parseCommandName(text, botUsername) {
if (!text || text[0] !== '/') return null;
const space = text.indexOf(' ');
const head = space < 0 ? text.slice(1) : text.slice(1, space);
const at = head.indexOf('@');
if (at < 0) return head;
const cmd = head.slice(0, at);
const target = head.slice(at + 1);
if (botUsername && target && target.toLowerCase() !== botUsername.toLowerCase()) return null;
return cmd;
}
+48
View File
@@ -0,0 +1,48 @@
import { buildAppleRequestByBundleId, buildAppleRequestByTrackId } from '../../api/apple-scraper.js';
import * as groupRepo from '../../repository/group-repository.js';
import { authorizeGroup, getCommandArguments, splitArgs } from './command-utils.js';
// /addapple <id|appId> [country=vn] — Java AddAppleAppCommand.
export function createAddAppleAppCommand(appleScraper) {
return async (msg, sender) => {
if (!(await authorizeGroup(msg.chat.id, sender))) return;
const args = splitArgs(getCommandArguments(msg.text));
if (args.length < 1 || args.length > 2) {
await sender.sendMessage(msg.chat.id, 'Invalid arguments');
return;
}
const country = args.length === 2 ? args[1] : 'vn';
// Java: try parsing arg[0] as Long (trackId); else treat as bundleId.
const trackId = Number.parseInt(args[0], 10);
const req =
Number.isFinite(trackId) && String(trackId) === args[0]
? buildAppleRequestByTrackId(trackId, country)
: buildAppleRequestByBundleId(args[0], country);
let resp;
try {
resp = await appleScraper.fetchAndCache(req);
} catch {
resp = null;
}
if (!resp || !resp.appId) {
await sender.sendMessage(msg.chat.id, 'Error when request app info');
return;
}
try {
const added = await groupRepo.addAppleApp(msg.chat.id, resp.appId, country);
if (!added) {
await sender.sendMessage(msg.chat.id, `Apple app <code>${resp.appId}</code> is already added`);
return;
}
await sender.sendMessage(
msg.chat.id,
`Apple app <code>${resp.appId}</code>, country <b>${country}</b> added successfully`,
);
} catch {
await sender.sendMessage(msg.chat.id, 'Internal server error');
}
};
}
+42
View File
@@ -0,0 +1,42 @@
import { buildGoogleRequest } from '../../api/google-scraper.js';
import * as groupRepo from '../../repository/group-repository.js';
import { authorizeGroup, getCommandArguments, splitArgs } from './command-utils.js';
// /addgoogle <appId> [country=vn] — Java AddGoogleAppCommand.
export function createAddGoogleAppCommand(googleScraper) {
return async (msg, sender) => {
if (!(await authorizeGroup(msg.chat.id, sender))) return;
const args = splitArgs(getCommandArguments(msg.text));
if (args.length < 1 || args.length > 2) {
await sender.sendMessage(msg.chat.id, 'Invalid arguments');
return;
}
const appId = args[0];
const country = args.length === 2 ? args[1] : 'vn';
let resp;
try {
resp = await googleScraper.fetchAndCache(buildGoogleRequest(appId, country));
} catch {
resp = null;
}
if (!resp) {
await sender.sendMessage(msg.chat.id, 'Error when request app info');
return;
}
try {
const added = await groupRepo.addGoogleApp(msg.chat.id, appId, country);
if (!added) {
await sender.sendMessage(msg.chat.id, `Google app <code>${appId}</code> is already added`);
return;
}
await sender.sendMessage(
msg.chat.id,
`Google app <code>${appId}</code>, country <b>${country}</b> added successfully`,
);
} catch {
await sender.sendMessage(msg.chat.id, 'Internal server error');
}
};
}
+35
View File
@@ -0,0 +1,35 @@
import * as adminRepo from '../../repository/admin-repository.js';
import * as groupRepo from '../../repository/group-repository.js';
import { getCommandArguments, requireAdminUser, splitArgs } from './command-utils.js';
// /addgroup [groupId] — Java AddGroupCommand. Admin-only.
export function createAddGroupCommand(config) {
return async (msg, sender) => {
if (!(await requireAdminUser(msg.from.id, msg.chat.id, config, sender))) return;
const args = splitArgs(getCommandArguments(msg.text));
if (args.length > 1) {
await sender.sendMessage(msg.chat.id, 'Invalid arguments');
return;
}
let groupId = msg.chat.id;
if (args.length === 1) {
const parsed = Number.parseInt(args[0], 10);
if (!Number.isFinite(parsed)) {
await sender.sendMessage(msg.chat.id, 'Invalid arguments');
return;
}
groupId = parsed;
}
try {
const added = await adminRepo.addGroup(groupId);
if (!added) {
await sender.sendMessage(msg.chat.id, 'Group is already added');
return;
}
await groupRepo.initGroup(groupId);
await sender.sendMessage(msg.chat.id, 'Group added successfully');
} catch {
await sender.sendMessage(msg.chat.id, 'Internal server error');
}
};
}
+58
View File
@@ -0,0 +1,58 @@
import * as groupRepo from '../../repository/group-repository.js';
import { buildTable } from '../../util/table.js';
import { authorizeGroup, getCommandArguments, splitArgs } from './command-utils.js';
// /checkappscore — Java CheckAppScoreCommand. Reports score + ratings.
// Score rounded to 1 decimal (Java Precision.round(score, 1) parity).
export function createCheckAppScoresCommand(appleScraper, googleScraper) {
return async (msg, sender) => {
if (!(await authorizeGroup(msg.chat.id, sender))) return;
if (splitArgs(getCommandArguments(msg.text)).length !== 0) {
await sender.sendMessage(msg.chat.id, 'Invalid arguments');
return;
}
try {
const group = await groupRepo.getGroup(msg.chat.id);
const headers = ['AppId', 'Score', 'Ratings'];
const appleRows = await scoreRowsFor(group.appleApps, appleScraper);
const googleRows = await scoreRowsFor(group.googleApps, googleScraper);
const out =
'<b>Apple Apps</b>\n' +
renderTable(appleRows, headers) +
'\n<b>Google Apps</b>\n' +
renderTable(googleRows, headers);
await sender.sendMessage(msg.chat.id, out);
} catch {
await sender.sendMessage(msg.chat.id, 'Internal server error');
}
};
}
async function scoreRowsFor(apps, scraper) {
const rows = [];
for (const a of apps) {
try {
const resp = await scraper.getApp(a.appId, a.country);
if (!resp) {
rows.push([a.appId, '?', '?']);
continue;
}
rows.push([a.appId, formatScore(resp.score), String(resp.ratings ?? 0)]);
} catch {
rows.push([a.appId, '?', '?']);
}
}
return rows;
}
function renderTable(rows, headers) {
if (rows.length === 0) return '<i>(none)</i>\n';
return `<pre>${buildTable(headers, rows)}</pre>\n`;
}
function formatScore(score) {
const v = Number(score);
if (!Number.isFinite(v)) return '?';
return (Math.round(v * 10) / 10).toFixed(1);
}
+84
View File
@@ -0,0 +1,84 @@
import * as groupRepo from '../../repository/group-repository.js';
import { buildTable } from '../../util/table.js';
import { daysBetween, formatDateInTz } from '../../util/time.js';
import { authorizeGroup, getCommandArguments, splitArgs } from './command-utils.js';
// /checkapp — Java CheckAppCommand. Reports update status per app, per store.
export function createCheckAppCommand(config, appleScraper, googleScraper) {
return async (msg, sender) => {
if (!(await authorizeGroup(msg.chat.id, sender))) return;
if (splitArgs(getCommandArguments(msg.text)).length !== 0) {
await sender.sendMessage(msg.chat.id, 'Invalid arguments');
return;
}
try {
const group = await groupRepo.getGroup(msg.chat.id);
const nowMs = Date.now();
const threshold = config.numDaysWarningNotUpdated;
const headers = ['AppId', 'Updated', 'Days', 'OK'];
const appleRows = await appleRowsFor(group.appleApps, appleScraper, nowMs, threshold, config.timezone);
const googleRows = await googleRowsFor(group.googleApps, googleScraper, nowMs, threshold, config.timezone);
const out =
'<b>Apple Apps</b>\n' +
renderTable(appleRows, headers) +
'\n<b>Google Apps</b>\n' +
renderTable(googleRows, headers);
await sender.sendMessage(msg.chat.id, out);
} catch {
await sender.sendMessage(msg.chat.id, 'Internal server error');
}
};
}
async function appleRowsFor(apps, scraper, nowMs, threshold, timezone) {
const rows = [];
for (const a of apps) {
try {
const resp = await scraper.getApp(a.appId, a.country);
if (!resp) {
rows.push([a.appId, '?', '?', mark(false)]);
continue;
}
const updatedMs = Date.parse(resp.updated);
if (Number.isNaN(updatedMs)) {
rows.push([a.appId, resp.updated || '?', '?', mark(false)]);
continue;
}
const days = daysBetween(updatedMs, nowMs);
rows.push([a.appId, formatDateInTz(new Date(updatedMs), timezone), String(days), mark(days <= threshold)]);
} catch {
rows.push([a.appId, '?', '?', mark(false)]);
}
}
return rows;
}
async function googleRowsFor(apps, scraper, nowMs, threshold, timezone) {
const rows = [];
for (const a of apps) {
try {
const resp = await scraper.getApp(a.appId, a.country);
if (!resp) {
rows.push([a.appId, '?', '?', mark(false)]);
continue;
}
const updatedMs = Number(resp.updated);
const days = daysBetween(updatedMs, nowMs);
rows.push([a.appId, formatDateInTz(new Date(updatedMs), timezone), String(days), mark(days <= threshold)]);
} catch {
rows.push([a.appId, '?', '?', mark(false)]);
}
}
return rows;
}
function renderTable(rows, headers) {
if (rows.length === 0) return '<i>(none)</i>\n';
return `<pre>${buildTable(headers, rows)}</pre>\n`;
}
function mark(ok) {
return ok ? '✅' : '❌';
}
+37
View File
@@ -0,0 +1,37 @@
import * as adminRepo from '../../repository/admin-repository.js';
export function splitArgs(text) {
if (!text) return [];
return text.trim().split(/\s+/).filter((s) => s.length > 0);
}
// Strips the "/<cmd>" or "/<cmd>@botname" prefix from message.text.
export function getCommandArguments(text) {
if (!text) return '';
const trimmed = text.trim();
const space = trimmed.indexOf(' ');
if (space < 0) return '';
return trimmed.slice(space + 1).trim();
}
export async function authorizeGroup(chatId, sender) {
try {
const ok = await adminRepo.hasGroup(chatId);
if (!ok) {
await sender.sendMessage(chatId, 'Group is not allowed to use bot');
return false;
}
return true;
} catch {
await sender.sendMessage(chatId, 'Group is not allowed to use bot');
return false;
}
}
export async function requireAdminUser(userId, chatId, config, sender) {
if (!config.isAdmin(userId)) {
await sender.sendMessage(chatId, 'You are not authorized to use this command');
return false;
}
return true;
}
+24
View File
@@ -0,0 +1,24 @@
import * as groupRepo from '../../repository/group-repository.js';
import { authorizeGroup, getCommandArguments, splitArgs } from './command-utils.js';
// /delapple <appId> — Java DeleteAppleAppCommand.
export function createDeleteAppleAppCommand() {
return async (msg, sender) => {
if (!(await authorizeGroup(msg.chat.id, sender))) return;
const args = splitArgs(getCommandArguments(msg.text));
if (args.length !== 1) {
await sender.sendMessage(msg.chat.id, 'Invalid arguments');
return;
}
try {
const removed = await groupRepo.removeAppleApp(msg.chat.id, args[0]);
if (!removed) {
await sender.sendMessage(msg.chat.id, 'Apple app is not added');
return;
}
await sender.sendMessage(msg.chat.id, 'Apple app deleted successfully');
} catch {
await sender.sendMessage(msg.chat.id, 'Internal server error');
}
};
}
+24
View File
@@ -0,0 +1,24 @@
import * as groupRepo from '../../repository/group-repository.js';
import { authorizeGroup, getCommandArguments, splitArgs } from './command-utils.js';
// /delgoogle <appId> — Java DeleteGoogleAppCommand.
export function createDeleteGoogleAppCommand() {
return async (msg, sender) => {
if (!(await authorizeGroup(msg.chat.id, sender))) return;
const args = splitArgs(getCommandArguments(msg.text));
if (args.length !== 1) {
await sender.sendMessage(msg.chat.id, 'Invalid arguments');
return;
}
try {
const removed = await groupRepo.removeGoogleApp(msg.chat.id, args[0]);
if (!removed) {
await sender.sendMessage(msg.chat.id, 'Google app is not added');
return;
}
await sender.sendMessage(msg.chat.id, 'Google app deleted successfully');
} catch {
await sender.sendMessage(msg.chat.id, 'Internal server error');
}
};
}
+33
View File
@@ -0,0 +1,33 @@
import * as adminRepo from '../../repository/admin-repository.js';
import { getCommandArguments, requireAdminUser, splitArgs } from './command-utils.js';
// /delgroup [groupId] — Java DeleteGroupCommand. Admin-only.
export function createDeleteGroupCommand(config) {
return async (msg, sender) => {
if (!(await requireAdminUser(msg.from.id, msg.chat.id, config, sender))) return;
const args = splitArgs(getCommandArguments(msg.text));
if (args.length > 1) {
await sender.sendMessage(msg.chat.id, 'Invalid arguments');
return;
}
let groupId = msg.chat.id;
if (args.length === 1) {
const parsed = Number.parseInt(args[0], 10);
if (!Number.isFinite(parsed)) {
await sender.sendMessage(msg.chat.id, 'Invalid arguments');
return;
}
groupId = parsed;
}
try {
const removed = await adminRepo.removeGroup(groupId);
if (!removed) {
await sender.sendMessage(msg.chat.id, 'Group is not added');
return;
}
await sender.sendMessage(msg.chat.id, 'Group deleted successfully');
} catch {
await sender.sendMessage(msg.chat.id, 'Internal server error');
}
};
}
+12
View File
@@ -0,0 +1,12 @@
import { getCommandArguments, splitArgs } from './command-utils.js';
// /info — Java InfoCommand. Reports the chat (group) ID.
export function createInfoCommand() {
return async (msg, sender) => {
if (splitArgs(getCommandArguments(msg.text)).length !== 0) {
await sender.sendMessage(msg.chat.id, 'Invalid arguments');
return;
}
await sender.sendMessage(msg.chat.id, `Id của nhóm là <code>${msg.chat.id}</code>\n`);
};
}
+31
View File
@@ -0,0 +1,31 @@
import * as groupRepo from '../../repository/group-repository.js';
import { buildTable } from '../../util/table.js';
import { authorizeGroup, getCommandArguments, splitArgs } from './command-utils.js';
// /listapp — Java ListAppCommand. Two tables (Apple / Google) of tracked apps.
export function createListAppCommand() {
return async (msg, sender) => {
if (!(await authorizeGroup(msg.chat.id, sender))) return;
if (splitArgs(getCommandArguments(msg.text)).length !== 0) {
await sender.sendMessage(msg.chat.id, 'Invalid arguments');
return;
}
try {
const group = await groupRepo.getGroup(msg.chat.id);
const out =
'<b>Apple Apps</b>\n' +
formatAppTable(group.appleApps) +
'\n<b>Google Apps</b>\n' +
formatAppTable(group.googleApps);
await sender.sendMessage(msg.chat.id, out);
} catch {
await sender.sendMessage(msg.chat.id, 'Internal server error');
}
};
}
function formatAppTable(apps) {
if (apps.length === 0) return '<i>(none)</i>\n';
const rows = apps.map((a, i) => [String(i + 1), a.appId, a.country]);
return `<pre>${buildTable(['#', 'AppId', 'Country'], rows)}</pre>\n`;
}
+25
View File
@@ -0,0 +1,25 @@
import * as adminRepo from '../../repository/admin-repository.js';
import { getCommandArguments, requireAdminUser, splitArgs } from './command-utils.js';
// /listgroup — Java ListGroupCommand. Admin-only.
export function createListGroupCommand(config) {
return async (msg, sender) => {
if (!(await requireAdminUser(msg.from.id, msg.chat.id, config, sender))) return;
if (splitArgs(getCommandArguments(msg.text)).length !== 0) {
await sender.sendMessage(msg.chat.id, 'Invalid arguments');
return;
}
try {
const groups = await adminRepo.getAllGroups();
if (groups.length === 0) {
await sender.sendMessage(msg.chat.id, 'No groups found');
return;
}
const lines = [`<b>Authorized groups (${groups.length}):</b>`];
groups.forEach((gid, i) => lines.push(`${i + 1}. <code>${gid}</code>`));
await sender.sendMessage(msg.chat.id, lines.join('\n') + '\n');
} catch {
await sender.sendMessage(msg.chat.id, 'Internal server error');
}
};
}
+32
View File
@@ -0,0 +1,32 @@
import { buildAppleRequestByBundleId, buildAppleRequestByTrackId } from '../../api/apple-scraper.js';
import { getCommandArguments, splitArgs } from './command-utils.js';
// /rawappleapp <id|appId> [country=vn] — Java RawAppleAppCommand.
// Sends raw upstream JSON as a Telegram document attachment.
export function createRawAppleAppCommand(appleScraper) {
return async (msg, sender) => {
const args = splitArgs(getCommandArguments(msg.text));
if (args.length < 1 || args.length > 2) {
await sender.sendMessage(msg.chat.id, 'Invalid arguments');
return;
}
const country = args.length === 2 ? args[1] : 'vn';
const trackId = Number.parseInt(args[0], 10);
const req =
Number.isFinite(trackId) && String(trackId) === args[0]
? buildAppleRequestByTrackId(trackId, country)
: buildAppleRequestByBundleId(args[0], country);
let raw;
try {
raw = await appleScraper.rawApp(req);
} catch {
raw = '';
}
if (!raw) {
await sender.sendMessage(msg.chat.id, 'Error when request app info');
return;
}
await sender.sendDocument(msg.chat.id, `${args[0]}.json`, raw);
};
}
+27
View File
@@ -0,0 +1,27 @@
import { buildGoogleRequest } from '../../api/google-scraper.js';
import { getCommandArguments, splitArgs } from './command-utils.js';
// /rawgoogleapp <appId> [country=vn] — Java RawGoogleAppCommand.
export function createRawGoogleAppCommand(googleScraper) {
return async (msg, sender) => {
const args = splitArgs(getCommandArguments(msg.text));
if (args.length < 1 || args.length > 2) {
await sender.sendMessage(msg.chat.id, 'Invalid arguments');
return;
}
const appId = args[0];
const country = args.length === 2 ? args[1] : 'vn';
let raw;
try {
raw = await googleScraper.rawApp(buildGoogleRequest(appId, country));
} catch {
raw = '';
}
if (!raw) {
await sender.sendMessage(msg.chat.id, 'Error when request app info');
return;
}
await sender.sendDocument(msg.chat.id, `${appId}.json`, raw);
};
}
+87
View File
@@ -0,0 +1,87 @@
import 'dotenv/config';
import { createLogger } from './logger.js';
const DEFAULT_DATABASE_NAME = 'store-scraper-bot';
function getEnv(key, fallback = '') {
const v = process.env[key];
return v && v.length > 0 ? v : fallback;
}
function getEnvInt(key, fallback) {
const v = process.env[key];
if (!v) return fallback;
const n = Number.parseInt(v, 10);
return Number.isFinite(n) ? n : fallback;
}
function parseAdminIds(raw) {
return raw
.split(',')
.map((s) => s.trim())
.filter((s) => s.length > 0)
.map((s) => {
try {
return BigInt(s);
} catch {
return null;
}
})
.filter((v) => v !== null);
}
// Mirrors Java/Go databaseFromURI: extract db name from connection string.
function databaseFromUri(uri) {
const idx = uri.indexOf('://');
let rest = idx >= 0 ? uri.slice(idx + 3) : uri;
const slash = rest.indexOf('/');
if (slash < 0) return DEFAULT_DATABASE_NAME;
let tail = rest.slice(slash + 1);
const q = tail.indexOf('?');
if (q >= 0) tail = tail.slice(0, q);
tail = tail.trim();
return tail.length > 0 ? tail : DEFAULT_DATABASE_NAME;
}
export function loadConfig() {
const telegramBotToken = getEnv('TELEGRAM_BOT_TOKEN');
if (!telegramBotToken) throw new Error('TELEGRAM_BOT_TOKEN is required');
const telegramBotUsername = getEnv('TELEGRAM_BOT_USERNAME');
if (!telegramBotUsername) throw new Error('TELEGRAM_BOT_USERNAME is required');
// Java parity: prefer MONGODB_CONNECTION_STRING, fall back to MONGO_URI.
const mongoUri = getEnv('MONGODB_CONNECTION_STRING', getEnv('MONGO_URI', 'mongodb://localhost:27017'));
const mongoDatabase = getEnv('MONGO_DATABASE') || databaseFromUri(mongoUri);
const mongoTimeoutMs = getEnvInt('MONGO_TIMEOUT_SECONDS', 10) * 1000;
const env = getEnv('ENV', 'DEVELOPMENT') === 'PRODUCTION' ? 'PRODUCTION' : 'DEVELOPMENT';
const adminIdsRaw = getEnv('ADMIN_IDS');
if (!adminIdsRaw) throw new Error('ADMIN_IDS is required');
const adminIds = parseAdminIds(adminIdsRaw);
if (adminIds.length === 0) throw new Error('at least one admin ID is required');
const config = {
telegramBotToken,
telegramBotUsername,
mongoUri,
mongoDatabase,
mongoTimeoutMs,
env,
adminIds,
creatorId: adminIds[0],
sourceCommit: getEnv('SOURCE_COMMIT', 'unknown'),
appCacheSeconds: getEnvInt('APP_CACHE_SECONDS', 600),
numDaysWarningNotUpdated: getEnvInt('NUM_DAYS_WARNING_NOT_UPDATED', 30),
scheduleCheckAppTime: getEnv('SCHEDULE_CHECK_APP_TIME', '0 7 * * *'),
timezone: 'Asia/Ho_Chi_Minh',
logger: createLogger(env),
};
config.isAdmin = (userId) => {
const id = typeof userId === 'bigint' ? userId : BigInt(userId);
return config.adminIds.some((a) => a === id);
};
return config;
}
+43
View File
@@ -0,0 +1,43 @@
import { loadConfig } from './config.js';
import { closeMongoDB, initMongoDB } from './repository/mongodb.js';
import { initAdmin } from './repository/admin-repository.js';
import { createAppleScraper } from './api/apple-scraper.js';
import { createGoogleScraper } from './api/google-scraper.js';
import { createBot } from './bot/bot.js';
import { createScheduler } from './scheduler/scheduler.js';
async function main() {
const config = loadConfig();
const logger = config.logger;
logger.info({ env: config.env, commit: config.sourceCommit }, 'Starting Store Scraper Bot');
await initMongoDB(config);
await initAdmin(); // Java parity: ensure singleton "common/admin" doc exists.
const appleScraper = createAppleScraper(config);
const googleScraper = createGoogleScraper(config);
const bot = createBot(config, appleScraper, googleScraper);
const scheduler = createScheduler(config, bot.sender, appleScraper, googleScraper);
scheduler.start();
logger.info('Starting Telegram bot polling');
const shutdown = async (signal) => {
logger.info({ signal }, 'Received shutdown signal, stopping bot...');
try {
scheduler.stop();
await bot.telegram.stopPolling();
await closeMongoDB();
} finally {
process.exit(0);
}
};
process.on('SIGINT', () => shutdown('SIGINT'));
process.on('SIGTERM', () => shutdown('SIGTERM'));
}
main().catch((err) => {
console.error('Fatal:', err);
process.exit(1);
});
+16
View File
@@ -0,0 +1,16 @@
import { pino } from 'pino';
export function createLogger(env) {
const isDev = env !== 'PRODUCTION';
return pino(
isDev
? {
level: 'debug',
transport: {
target: 'pino-pretty',
options: { colorize: true, translateTime: 'SYS:HH:MM:ss', ignore: 'pid,hostname' },
},
}
: { level: 'info' },
);
}
+23
View File
@@ -0,0 +1,23 @@
// Admin singleton document — Java parity (_id="admin", class="Admin").
export const ADMIN_ID = 'admin';
export function newAdmin() {
return { _id: ADMIN_ID, class: 'Admin', groups: [] };
}
export function adminAddGroup(admin, groupId) {
if (admin.groups.includes(groupId)) return false;
admin.groups.push(groupId);
return true;
}
export function adminRemoveGroup(admin, groupId) {
const i = admin.groups.indexOf(groupId);
if (i < 0) return false;
admin.groups.splice(i, 1);
return true;
}
export function adminHasGroup(admin, groupId) {
return admin.groups.includes(groupId);
}
+8
View File
@@ -0,0 +1,8 @@
// AppleApp cache entry — Java parity (_id=appId, class="AppleApp").
export function newAppleApp(appId, response, millis) {
return { _id: appId, class: 'AppleApp', app: response, millis };
}
export function isAppleAppExpired(entry, nowMillis, cacheMillis) {
return nowMillis - entry.millis > cacheMillis;
}
+8
View File
@@ -0,0 +1,8 @@
// GoogleApp cache entry — Java parity (_id=appId, class="GoogleApp").
export function newGoogleApp(appId, response, millis) {
return { _id: appId, class: 'GoogleApp', app: response, millis };
}
export function isGoogleAppExpired(entry, nowMillis, cacheMillis) {
return nowMillis - entry.millis > cacheMillis;
}
+46
View File
@@ -0,0 +1,46 @@
// Group document — Java parity (_id is string form of Telegram chat ID).
export function groupIdToKey(groupId) {
return String(groupId);
}
export function groupKeyToId(key) {
return Number(key);
}
export function newGroup(groupId) {
return {
_id: groupIdToKey(groupId),
class: 'Group',
appleApps: [],
googleApps: [],
};
}
function addApp(list, appId, country) {
if (list.some((a) => a.appId === appId)) return false;
list.push({ appId, country });
return true;
}
function removeApp(list, appId) {
const i = list.findIndex((a) => a.appId === appId);
if (i < 0) return false;
list.splice(i, 1);
return true;
}
export function groupAddAppleApp(group, appId, country) {
return addApp(group.appleApps, appId, country);
}
export function groupRemoveAppleApp(group, appId) {
return removeApp(group.appleApps, appId);
}
export function groupAddGoogleApp(group, appId, country) {
return addApp(group.googleApps, appId, country);
}
export function groupRemoveGoogleApp(group, appId) {
return removeApp(group.googleApps, appId);
}
+47
View File
@@ -0,0 +1,47 @@
import { getCollection } from './mongodb.js';
import { ADMIN_ID, adminAddGroup, adminHasGroup, adminRemoveGroup, newAdmin } from '../models/admin.js';
// Stored in "common" collection at _id="admin" (Java parity).
function collection() {
return getCollection('common');
}
export async function initAdmin() {
const c = collection();
const count = await c.countDocuments({ _id: ADMIN_ID });
if (count > 0) return;
await save(newAdmin());
}
export async function getAdmin() {
const doc = await collection().findOne({ _id: ADMIN_ID });
return doc ?? newAdmin();
}
export async function save(admin) {
await collection().replaceOne({ _id: ADMIN_ID }, admin, { upsert: true });
}
export async function addGroup(groupId) {
const admin = await getAdmin();
if (!adminAddGroup(admin, groupId)) return false;
await save(admin);
return true;
}
export async function removeGroup(groupId) {
const admin = await getAdmin();
if (!adminRemoveGroup(admin, groupId)) return false;
await save(admin);
return true;
}
export async function hasGroup(groupId) {
const admin = await getAdmin();
return adminHasGroup(admin, groupId);
}
export async function getAllGroups() {
const admin = await getAdmin();
return admin.groups;
}
+22
View File
@@ -0,0 +1,22 @@
import { getCollection } from './mongodb.js';
import { isAppleAppExpired } from '../models/apple-app.js';
function collection() {
return getCollection('apple_app');
}
export async function getAppleApp(appId) {
return collection().findOne({ _id: appId });
}
export async function saveAppleApp(entry) {
await collection().replaceOne({ _id: entry._id }, entry, { upsert: true });
}
export async function getCachedAppleApp(appId, appCacheSeconds) {
const entry = await getAppleApp(appId);
if (!entry) return null;
const cacheMillis = appCacheSeconds * 1000;
if (isAppleAppExpired(entry, Date.now(), cacheMillis)) return null;
return entry;
}
+22
View File
@@ -0,0 +1,22 @@
import { getCollection } from './mongodb.js';
import { isGoogleAppExpired } from '../models/google-app.js';
function collection() {
return getCollection('google_app');
}
export async function getGoogleApp(appId) {
return collection().findOne({ _id: appId });
}
export async function saveGoogleApp(entry) {
await collection().replaceOne({ _id: entry._id }, entry, { upsert: true });
}
export async function getCachedGoogleApp(appId, appCacheSeconds) {
const entry = await getGoogleApp(appId);
if (!entry) return null;
const cacheMillis = appCacheSeconds * 1000;
if (isGoogleAppExpired(entry, Date.now(), cacheMillis)) return null;
return entry;
}
+59
View File
@@ -0,0 +1,59 @@
import { getCollection } from './mongodb.js';
import {
groupAddAppleApp,
groupAddGoogleApp,
groupIdToKey,
groupRemoveAppleApp,
groupRemoveGoogleApp,
newGroup,
} from '../models/group.js';
function collection() {
return getCollection('group');
}
export async function exists(groupId) {
const count = await collection().countDocuments({ _id: groupIdToKey(groupId) });
return count > 0;
}
export async function getGroup(groupId) {
const doc = await collection().findOne({ _id: groupIdToKey(groupId) });
return doc ?? newGroup(groupId);
}
export async function saveGroup(group) {
await collection().replaceOne({ _id: group._id }, group, { upsert: true });
}
export async function initGroup(groupId) {
if (await exists(groupId)) return;
await saveGroup(newGroup(groupId));
}
export async function deleteGroup(groupId) {
await collection().deleteOne({ _id: groupIdToKey(groupId) });
}
async function mutateAndSave(groupId, mutator) {
const group = await getGroup(groupId);
if (!mutator(group)) return false;
await saveGroup(group);
return true;
}
export function addAppleApp(groupId, appId, country) {
return mutateAndSave(groupId, (g) => groupAddAppleApp(g, appId, country));
}
export function removeAppleApp(groupId, appId) {
return mutateAndSave(groupId, (g) => groupRemoveAppleApp(g, appId));
}
export function addGoogleApp(groupId, appId, country) {
return mutateAndSave(groupId, (g) => groupAddGoogleApp(g, appId, country));
}
export function removeGoogleApp(groupId, appId) {
return mutateAndSave(groupId, (g) => groupRemoveGoogleApp(g, appId));
}
+30
View File
@@ -0,0 +1,30 @@
import { MongoClient } from 'mongodb';
let client;
let database;
export async function initMongoDB(config) {
client = new MongoClient(config.mongoUri, {
serverSelectionTimeoutMS: config.mongoTimeoutMs,
});
await client.connect();
await client.db(config.mongoDatabase).command({ ping: 1 });
database = client.db(config.mongoDatabase);
config.logger.info(
{ database: config.mongoDatabase, uri: config.mongoUri },
'Connected to MongoDB',
);
}
export async function closeMongoDB() {
if (client) await client.close();
}
export function getDatabase() {
if (!database) throw new Error('MongoDB not initialized');
return database;
}
export function getCollection(name) {
return getDatabase().collection(name);
}
+136
View File
@@ -0,0 +1,136 @@
import cron from 'node-cron';
import * as adminRepo from '../repository/admin-repository.js';
import * as groupRepo from '../repository/group-repository.js';
import { buildTable, formatNumber, truncateString } from '../util/table.js';
import { daysBetween, formatDateInTz, formatDateTimeInTz, weekdayInTz } from '../util/time.js';
export function createScheduler(config, sender, appleScraper, googleScraper) {
const logger = config.logger;
let task;
function start() {
task = cron.schedule(config.scheduleCheckAppTime, runDailyCheck, {
scheduled: true,
timezone: config.timezone,
});
logger.info(
{ schedule: config.scheduleCheckAppTime, timezone: config.timezone },
'Scheduler started',
);
}
function stop() {
if (task) task.stop();
logger.info('Scheduler stopped');
}
async function runDailyCheck() {
const now = new Date();
const dow = weekdayInTz(now, config.timezone);
const silent = dow === 0 || dow === 6;
logger.info({ silent }, 'Running daily check job');
let groups;
try {
groups = await adminRepo.getAllGroups();
} catch (err) {
logger.error({ err: err.message }, 'Failed to get groups');
return;
}
for (const gid of groups) {
try {
await checkGroup(gid, silent, now);
} catch (err) {
logger.error({ err: err.message, groupId: gid }, 'check group failed');
}
}
logger.info({ groupsChecked: groups.length }, 'Daily check job completed');
}
async function checkGroup(groupId, silent, now) {
const group = await groupRepo.getGroup(groupId);
if (group.appleApps.length === 0 && group.googleApps.length === 0) {
logger.info({ groupId }, 'Group has no apps, skipping');
return;
}
const threshold = config.numDaysWarningNotUpdated;
const stale = [];
for (const info of group.appleApps) {
try {
const app = await appleScraper.getApp(info.appId, info.country);
if (!app) continue;
const updatedMs = Date.parse(app.updated);
if (Number.isNaN(updatedMs)) continue;
const days = daysBetween(updatedMs, now.getTime());
if (days > threshold) {
stale.push({
appId: info.appId,
title: app.title,
days,
updated: formatDateInTz(new Date(updatedMs), config.timezone),
score: app.score,
reviews: Number(app.reviews ?? 0),
ratings: Number(app.ratings ?? 0),
isApple: true,
});
}
} catch (err) {
logger.error({ err: err.message, appId: info.appId }, 'Apple fetch failed');
}
}
for (const info of group.googleApps) {
try {
const app = await googleScraper.getApp(info.appId, info.country);
if (!app) continue;
const updatedMs = Number(app.updated);
const days = daysBetween(updatedMs, now.getTime());
if (days > threshold) {
stale.push({
appId: info.appId,
title: app.title,
days,
updated: formatDateInTz(new Date(updatedMs), config.timezone),
score: app.score,
reviews: Number(app.reviews ?? 0),
ratings: Number(app.ratings ?? 0),
isApple: false,
});
}
} catch (err) {
logger.error({ err: err.message, appId: info.appId }, 'Google fetch failed');
}
}
if (stale.length === 0) {
logger.info({ groupId }, 'All apps up-to-date');
return;
}
const message = buildReport(groupId, stale, now);
if (silent) await sender.sendMessageSilent(groupId, message);
else await sender.sendMessage(groupId, message);
}
function buildReport(groupId, apps, now) {
const headers = ['App', 'Store', 'Days', 'Updated', 'Score', 'Reviews', 'Ratings'];
const rows = apps.map((a) => [
truncateString(a.title || '', 30),
a.isApple ? 'Apple' : 'Google',
String(a.days),
a.updated,
Number(a.score ?? 0).toFixed(1),
String(a.reviews),
formatNumber(a.ratings),
]);
return (
`<b>Daily App Check Report</b>\n` +
`Date: ${formatDateTimeInTz(now, config.timezone)}\n` +
`Group: <code>${groupId}</code>\n` +
`Apps not updated in &gt;${config.numDaysWarningNotUpdated} days: <b>${apps.length}</b>\n\n` +
`<pre>${buildTable(headers, rows)}</pre>`
);
}
return { start, stop, runDailyCheck };
}
+66
View File
@@ -0,0 +1,66 @@
// Mirrors Java bot/table/Table.java:
// - left-aligned columns padded to max(header, cell) width
// - "│" column separator
// - row separator inserted every 5 rows using "─" cells joined by "─┼─"
// Output is intended to be wrapped in <pre> for Telegram HTML rendering.
export function buildTable(headers, rows) {
const widths = computeWidths(headers, rows);
const parts = [];
parts.push(writeRow(headers, widths));
parts.push('\n');
parts.push(writeSeparator(widths));
for (let i = 0; i < rows.length; i++) {
parts.push('\n');
if (i > 0 && i % 5 === 0) {
parts.push(writeSeparator(widths));
parts.push('\n');
}
parts.push(writeRow(rows[i], widths));
}
return parts.join('');
}
function computeWidths(headers, rows) {
const widths = headers.map((h) => h.length);
for (const row of rows) {
for (let i = 0; i < row.length && i < widths.length; i++) {
if (row[i].length > widths[i]) widths[i] = row[i].length;
}
}
return widths;
}
function writeRow(cells, widths) {
const out = [];
for (let i = 0; i < widths.length; i++) {
out.push(padRight(cells[i] ?? '', widths[i]));
if (i < widths.length - 1) out.push(' │ ');
}
return out.join('');
}
function writeSeparator(widths) {
const out = [];
for (let i = 0; i < widths.length; i++) {
out.push('─'.repeat(widths[i]));
if (i < widths.length - 1) out.push('─┼─');
}
return out.join('');
}
function padRight(s, len) {
return s.length >= len ? s : s + ' '.repeat(len - s.length);
}
export function truncateString(s, maxLen) {
if (s.length <= maxLen) return s;
if (maxLen <= 3) return s.slice(0, maxLen);
return s.slice(0, maxLen - 3) + '...';
}
export function formatNumber(n) {
const v = Number(n);
if (v >= 1_000_000) return `${(v / 1_000_000).toFixed(1)}M`;
if (v >= 1_000) return `${(v / 1_000).toFixed(1)}K`;
return String(v);
}
+36
View File
@@ -0,0 +1,36 @@
// Format date as YYYY-MM-DD in given IANA timezone.
export function formatDateInTz(date, timezone) {
const fmt = new Intl.DateTimeFormat('en-CA', {
timeZone: timezone,
year: 'numeric',
month: '2-digit',
day: '2-digit',
});
return fmt.format(date);
}
// Format datetime as "YYYY-MM-DD HH:MM" in given IANA timezone.
export function formatDateTimeInTz(date, timezone) {
const fmt = new Intl.DateTimeFormat('en-CA', {
timeZone: timezone,
year: 'numeric',
month: '2-digit',
day: '2-digit',
hour: '2-digit',
minute: '2-digit',
hour12: false,
});
// en-CA produces "YYYY-MM-DD, HH:MM" — strip the comma.
return fmt.format(date).replace(', ', ' ');
}
// Day-of-week (0=Sun..6=Sat) for `date` in given timezone.
export function weekdayInTz(date, timezone) {
const fmt = new Intl.DateTimeFormat('en-US', { timeZone: timezone, weekday: 'short' });
const map = { Sun: 0, Mon: 1, Tue: 2, Wed: 3, Thu: 4, Fri: 5, Sat: 6 };
return map[fmt.format(date)];
}
export function daysBetween(fromMs, toMs) {
return Math.floor((toMs - fromMs) / (24 * 60 * 60 * 1000));
}