From a4646c91cadabdd763a62542a291f9afcd1a6096 Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Sun, 9 Nov 2025 10:12:19 +0700 Subject: [PATCH] Add raw app response commands for Apple and Google Introduces RawAppleAppCommand and RawGoogleAppCommand to allow admins to fetch and download raw JSON responses from the Apple App Store and Google Play Store scrapers. Refactors AppStoreScraper and GooglePlayScraper to expose rawApp methods and improves method naming for clarity. Updates StoreScrapeBot to register the new commands and adjusts .env.example for better configuration clarity. --- .env.example | 10 +-- .../api/apple/AppStoreScraper.java | 23 ++++-- .../api/google/GooglePlayScraper.java | 20 +++-- .../storescraperbot/bot/StoreScrapeBot.java | 5 ++ .../bot/command/RawAppleAppCommand.java | 81 +++++++++++++++++++ .../bot/command/RawGoogleAppCommand.java | 70 ++++++++++++++++ 6 files changed, 189 insertions(+), 20 deletions(-) create mode 100644 src/main/java/com/miti99/storescraperbot/bot/command/RawAppleAppCommand.java create mode 100644 src/main/java/com/miti99/storescraperbot/bot/command/RawGoogleAppCommand.java diff --git a/.env.example b/.env.example index ca95628..0908926 100644 --- a/.env.example +++ b/.env.example @@ -3,13 +3,13 @@ # Couchbase configuration COUCHBASE_CONNECTION_STRING=couchbase://localhost -COUCHBASE_USERNAME=admin -COUCHBASE_PASSWORD=your_password_here -COUCHBASE_BUCKET_NAME=store_scraper +COUCHBASE_USERNAME=your_couchbase_username +COUCHBASE_PASSWORD=your_couchbase_password +COUCHBASE_BUCKET_NAME=store-scraper-bot # Telegram Bot configuration -TELEGRAM_BOT_TOKEN=your_telegram_bot_token_here -TELEGRAM_BOT_USERNAME=your_bot_username +TELEGRAM_BOT_TOKEN=your_telegram_bot_token +TELEGRAM_BOT_USERNAME=your_telegram_bot_username # Java configuration JAVA_OPTS=-Xmx512m diff --git a/src/main/java/com/miti99/storescraperbot/api/apple/AppStoreScraper.java b/src/main/java/com/miti99/storescraperbot/api/apple/AppStoreScraper.java index 818b80e..47b6386 100644 --- a/src/main/java/com/miti99/storescraperbot/api/apple/AppStoreScraper.java +++ b/src/main/java/com/miti99/storescraperbot/api/apple/AppStoreScraper.java @@ -24,7 +24,7 @@ public class AppStoreScraper { public static final String BASE_URL = "https://miti-app-store-scraper.vercel.app/"; @SneakyThrows - public static AppleAppResponse app(AppleAppRequest request) { + public static String rawApp(AppleAppRequest request) { var httpRequest = HttpRequest.newBuilder() .uri(URI.create(BASE_URL + "/app")) @@ -38,12 +38,19 @@ public class AppStoreScraper { .followRedirects(Redirect.NORMAL) // .connectTimeout(Duration.ofMillis(TIMEOUT)) .build()) { - var body = httpClient.send(httpRequest, BodyHandlers.ofString()).body(); - return GsonUtil.fromJson(body, AppleAppResponse.class); + return httpClient.send(httpRequest, BodyHandlers.ofString()).body(); + } catch (Exception e) { + log.error("rawAppResponse error - request: '{}'", GsonUtil.toJson(request), e); + return null; } } - public static AppleAppResponse getResponse(String appId, String country) { + @SneakyThrows + public static AppleAppResponse app(AppleAppRequest request) { + return GsonUtil.fromJson(rawApp(request), AppleAppResponse.class); + } + + public static AppleAppResponse getAppResponse(String appId, String country) { boolean isInCache = AppleAppRepository.INSTANCE.exist(appId); if (isInCache) { var app = AppleAppRepository.INSTANCE.load(appId); @@ -59,7 +66,7 @@ public class AppStoreScraper { } public static LocalDate getAppUpdated(String appId, String country) { - var response = getResponse(appId, country); + var response = getAppResponse(appId, country); if (response == null) { log.error("response is null"); return LocalDate.ofInstant(Instant.ofEpochMilli(0), ZoneId.systemDefault()); @@ -68,7 +75,7 @@ public class AppStoreScraper { } public static double getAppScore(String appId, String country) { - var response = getResponse(appId, country); + var response = getAppResponse(appId, country); if (response == null) { log.error("response is null"); return 0.0; @@ -77,7 +84,7 @@ public class AppStoreScraper { } public static long getAppReviews(String appId, String country) { - var response = getResponse(appId, country); + var response = getAppResponse(appId, country); if (response == null) { log.error("response is null"); return 0L; @@ -86,7 +93,7 @@ public class AppStoreScraper { } public static long getAppRatings(String appId, String country) { - var response = getResponse(appId, country); + var response = getAppResponse(appId, country); if (response == null) { log.error("response is null"); return 0L; diff --git a/src/main/java/com/miti99/storescraperbot/api/google/GooglePlayScraper.java b/src/main/java/com/miti99/storescraperbot/api/google/GooglePlayScraper.java index f0f5bd7..917ba7e 100644 --- a/src/main/java/com/miti99/storescraperbot/api/google/GooglePlayScraper.java +++ b/src/main/java/com/miti99/storescraperbot/api/google/GooglePlayScraper.java @@ -24,7 +24,7 @@ public class GooglePlayScraper { public static final String BASE_URL = "https://miti-google-play-scraper.vercel.app/"; @SneakyThrows - public static GoogleAppResponse app(GoogleAppRequest request) { + public static String rawApp(GoogleAppRequest request) { var httpRequest = HttpRequest.newBuilder() .uri(URI.create(BASE_URL + "/app")) @@ -38,12 +38,18 @@ public class GooglePlayScraper { // .connectTimeout(Duration.ofMillis(TIMEOUT)) .followRedirects(Redirect.NORMAL) .build()) { - var body = httpClient.send(httpRequest, BodyHandlers.ofString()).body(); - return GsonUtil.fromJson(body, GoogleAppResponse.class); + return httpClient.send(httpRequest, BodyHandlers.ofString()).body(); + } catch (Exception e) { + log.error("rawAppResponse error - request: '{}'", GsonUtil.toJson(request), e); + return null; } } - private static GoogleAppResponse getResponse(String appId, String country) { + public static GoogleAppResponse app(GoogleAppRequest request) { + return GsonUtil.fromJson(rawApp(request), GoogleAppResponse.class); + } + + private static GoogleAppResponse getAppResponse(String appId, String country) { boolean isInCache = GoogleAppRepository.INSTANCE.exist(appId); if (isInCache) { var app = GoogleAppRepository.INSTANCE.load(appId); @@ -59,7 +65,7 @@ public class GooglePlayScraper { } public static LocalDate getLastUpdateOfApp(String appId, String country) { - var response = getResponse(appId, country); + var response = getAppResponse(appId, country); long lastUpdateMillis = 0; if (response != null) { lastUpdateMillis = response.updated(); @@ -70,7 +76,7 @@ public class GooglePlayScraper { } public static double getAppScore(String appId, String country) { - var response = getResponse(appId, country); + var response = getAppResponse(appId, country); if (response == null) { log.error("response is null"); return 0.0; @@ -79,7 +85,7 @@ public class GooglePlayScraper { } public static long getAppRatings(String appId, String country) { - var response = getResponse(appId, country); + var response = getAppResponse(appId, country); if (response == null) { log.error("response is null"); return 0L; diff --git a/src/main/java/com/miti99/storescraperbot/bot/StoreScrapeBot.java b/src/main/java/com/miti99/storescraperbot/bot/StoreScrapeBot.java index 513a7d9..34d5dfa 100644 --- a/src/main/java/com/miti99/storescraperbot/bot/StoreScrapeBot.java +++ b/src/main/java/com/miti99/storescraperbot/bot/StoreScrapeBot.java @@ -15,6 +15,8 @@ import com.miti99.storescraperbot.bot.command.DeleteGroupCommand; import com.miti99.storescraperbot.bot.command.InfoCommand; import com.miti99.storescraperbot.bot.command.ListAppCommand; import com.miti99.storescraperbot.bot.command.ListGroupCommand; +import com.miti99.storescraperbot.bot.command.RawAppleAppCommand; +import com.miti99.storescraperbot.bot.command.RawGoogleAppCommand; import com.miti99.storescraperbot.bot.entity.NonUpdatedApp; import com.miti99.storescraperbot.bot.table.Table; import com.miti99.storescraperbot.constant.Constant; @@ -40,6 +42,9 @@ public class StoreScrapeBot extends CommandLongPollingTelegramBot { super(StoreScrapeBotTelegramClient.INSTANCE, true, StoreScrapeBotUsernameSupplier.INSTANCE); register(InfoCommand.INSTANCE); + register(RawAppleAppCommand.INSTANCE); + register(RawGoogleAppCommand.INSTANCE); + register(AddGroupCommand.INSTANCE); register(DeleteGroupCommand.INSTANCE); register(ListGroupCommand.INSTANCE); diff --git a/src/main/java/com/miti99/storescraperbot/bot/command/RawAppleAppCommand.java b/src/main/java/com/miti99/storescraperbot/bot/command/RawAppleAppCommand.java new file mode 100644 index 0000000..b6c46ff --- /dev/null +++ b/src/main/java/com/miti99/storescraperbot/bot/command/RawAppleAppCommand.java @@ -0,0 +1,81 @@ +package com.miti99.storescraperbot.bot.command; + +import com.miti99.storescraperbot.api.apple.AppStoreScraper; +import com.miti99.storescraperbot.api.apple.request.AppleAppRequest; +import com.miti99.storescraperbot.bot.StoreScrapeBotTelegramClient; +import com.miti99.storescraperbot.repository.AdminRepository; +import com.miti99.storescraperbot.util.GsonUtil; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import lombok.SneakyThrows; +import lombok.extern.log4j.Log4j2; +import org.telegram.telegrambots.meta.api.methods.send.SendDocument; +import org.telegram.telegrambots.meta.api.objects.InputFile; +import org.telegram.telegrambots.meta.api.objects.User; +import org.telegram.telegrambots.meta.api.objects.chat.Chat; +import org.telegram.telegrambots.meta.generics.TelegramClient; + +@Log4j2 +public class RawAppleAppCommand extends BaseStoreScraperBotCommand { + public static final RawAppleAppCommand INSTANCE = new RawAppleAppCommand(); + + RawAppleAppCommand() { + super( + "rawappleapp", + " [country]. Lấy raw response khi request lên service. id: iTunes 'trackId', appId: iTunes 'bundleId'. Một số app cần country để hoạt động đúng, country mặc định là 'vn'"); + } + + @Override + @SneakyThrows + protected void executeCommand( + TelegramClient telegramClient, User user, Chat chat, String[] arguments) { + var admin = AdminRepository.INSTANCE.load(); + if (!admin.getGroups().contains(chat.getId())) { + StoreScrapeBotTelegramClient.INSTANCE.sendMessage( + chat.getId(), "Group is not allowed to use bot"); + return; + } + + if (arguments.length < 1 || arguments.length > 2) { + StoreScrapeBotTelegramClient.INSTANCE.sendMessage(chat.getId(), "Invalid arguments"); + return; + } + + var appId = arguments[0]; + long id = -1; + var country = arguments.length == 2 ? arguments[1] : "vn"; + String response = ""; + try { + try { + id = Long.parseLong(appId); + } catch (Exception e) { + // Input không phải id, bỏ qua + } + if (id != -1) { + response = AppStoreScraper.rawApp(new AppleAppRequest(id, country)); + } else { + response = AppStoreScraper.rawApp(new AppleAppRequest(appId, country)); + } + } catch (Exception e) { + log.error("request app error for appId: '{}', id: '{}'", appId, id, e); + StoreScrapeBotTelegramClient.INSTANCE.sendMessage( + chat.getId(), "Error when request app info"); + return; + } + + if (response == null) response = ""; + + var inputStream = new ByteArrayInputStream(response.getBytes(StandardCharsets.UTF_8)); + + var file = new InputFile(inputStream, "%s.json".formatted(appId)); + + var sendDocument = + SendDocument.builder() + .chatId(chat.getId()) + .document(file) + // .caption("raw") + .build(); + + StoreScrapeBotTelegramClient.INSTANCE.execute(sendDocument); + } +} diff --git a/src/main/java/com/miti99/storescraperbot/bot/command/RawGoogleAppCommand.java b/src/main/java/com/miti99/storescraperbot/bot/command/RawGoogleAppCommand.java new file mode 100644 index 0000000..59809b9 --- /dev/null +++ b/src/main/java/com/miti99/storescraperbot/bot/command/RawGoogleAppCommand.java @@ -0,0 +1,70 @@ +package com.miti99.storescraperbot.bot.command; + +import com.miti99.storescraperbot.api.google.GooglePlayScraper; +import com.miti99.storescraperbot.api.google.request.GoogleAppRequest; +import com.miti99.storescraperbot.bot.StoreScrapeBotTelegramClient; +import com.miti99.storescraperbot.repository.AdminRepository; +import com.miti99.storescraperbot.util.GsonUtil; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import lombok.SneakyThrows; +import lombok.extern.log4j.Log4j2; +import org.telegram.telegrambots.meta.api.methods.send.SendDocument; +import org.telegram.telegrambots.meta.api.objects.InputFile; +import org.telegram.telegrambots.meta.api.objects.User; +import org.telegram.telegrambots.meta.api.objects.chat.Chat; +import org.telegram.telegrambots.meta.generics.TelegramClient; + +@Log4j2 +public class RawGoogleAppCommand extends BaseStoreScraperBotCommand { + public static final RawGoogleAppCommand INSTANCE = new RawGoogleAppCommand(); + + RawGoogleAppCommand() { + super( + "rawgoogleapp", + " [country]. Lấy raw response khi request lên service. Một số app cần country để hoạt động đúng, country mặc định là 'vn'"); + } + + @Override + @SneakyThrows + protected void executeCommand( + TelegramClient telegramClient, User user, Chat chat, String[] arguments) { + var admin = AdminRepository.INSTANCE.load(); + if (!admin.getGroups().contains(chat.getId())) { + StoreScrapeBotTelegramClient.INSTANCE.sendMessage( + chat.getId(), "Group is not allowed to use bot"); + return; + } + + if (arguments.length < 1 || arguments.length > 2) { + StoreScrapeBotTelegramClient.INSTANCE.sendMessage(chat.getId(), "Invalid arguments"); + return; + } + + var appId = arguments[0]; + var country = arguments.length == 2 ? arguments[1] : "vn"; + String response = ""; + try { + response = GooglePlayScraper.rawApp(new GoogleAppRequest(appId, country)); + } catch (Exception e) { + log.error("request app error for appId: '{}'", appId, e); + StoreScrapeBotTelegramClient.INSTANCE.sendMessage( + chat.getId(), "Error when request app info"); + return; + } + if (response == null) response = ""; + + var inputStream = new ByteArrayInputStream(response.getBytes(StandardCharsets.UTF_8)); + + var file = new InputFile(inputStream, "%s.json".formatted(appId)); + + var sendDocument = + SendDocument.builder() + .chatId(chat.getId()) + .document(file) + // .caption("raw") + .build(); + + StoreScrapeBotTelegramClient.INSTANCE.execute(sendDocument); + } +}