Add raw app response commands for Apple and Google

Introduces RawAppleAppCommand and RawGoogleAppCommand to allow admins to fetch and download raw JSON responses from the Apple App Store and Google Play Store scrapers. Refactors AppStoreScraper and GooglePlayScraper to expose rawApp methods and improves method naming for clarity. Updates StoreScrapeBot to register the new commands and adjusts .env.example for better configuration clarity.

.env.example

@@ -3,13 +3,13 @@
 
 # Couchbase configuration
 COUCHBASE_CONNECTION_STRING=couchbase://localhost
-COUCHBASE_USERNAME=admin
-COUCHBASE_PASSWORD=your_password_here
-COUCHBASE_BUCKET_NAME=store_scraper
+COUCHBASE_USERNAME=your_couchbase_username
+COUCHBASE_PASSWORD=your_couchbase_password
+COUCHBASE_BUCKET_NAME=store-scraper-bot
 
 # Telegram Bot configuration
-TELEGRAM_BOT_TOKEN=your_telegram_bot_token_here
-TELEGRAM_BOT_USERNAME=your_bot_username
+TELEGRAM_BOT_TOKEN=your_telegram_bot_token
+TELEGRAM_BOT_USERNAME=your_telegram_bot_username
 
 # Java configuration
 JAVA_OPTS=-Xmx512m

src/main/java/com/miti99/storescraperbot/api/apple/AppStoreScraper.java

@@ -24,7 +24,7 @@ public class AppStoreScraper {
   public static final String BASE_URL = "https://miti-app-store-scraper.vercel.app/";
 
   @SneakyThrows
-  public static AppleAppResponse app(AppleAppRequest request) {
+  public static String rawApp(AppleAppRequest request) {
     var httpRequest =
         HttpRequest.newBuilder()
             .uri(URI.create(BASE_URL + "/app"))
@@ -38,12 +38,19 @@ public class AppStoreScraper {
             .followRedirects(Redirect.NORMAL)
             // .connectTimeout(Duration.ofMillis(TIMEOUT))
             .build()) {
-      var body = httpClient.send(httpRequest, BodyHandlers.ofString()).body();
-      return GsonUtil.fromJson(body, AppleAppResponse.class);
+      return httpClient.send(httpRequest, BodyHandlers.ofString()).body();
+    } catch (Exception e) {
+      log.error("rawAppResponse error - request: '{}'", GsonUtil.toJson(request), e);
+      return null;
     }
   }
 
-  public static AppleAppResponse getResponse(String appId, String country) {
+  @SneakyThrows
+  public static AppleAppResponse app(AppleAppRequest request) {
+    return GsonUtil.fromJson(rawApp(request), AppleAppResponse.class);
+  }
+
+  public static AppleAppResponse getAppResponse(String appId, String country) {
     boolean isInCache = AppleAppRepository.INSTANCE.exist(appId);
     if (isInCache) {
       var app = AppleAppRepository.INSTANCE.load(appId);
@@ -59,7 +66,7 @@ public class AppStoreScraper {
   }
 
   public static LocalDate getAppUpdated(String appId, String country) {
-    var response = getResponse(appId, country);
+    var response = getAppResponse(appId, country);
     if (response == null) {
       log.error("response is null");
       return LocalDate.ofInstant(Instant.ofEpochMilli(0), ZoneId.systemDefault());
@@ -68,7 +75,7 @@ public class AppStoreScraper {
   }
 
   public static double getAppScore(String appId, String country) {
-    var response = getResponse(appId, country);
+    var response = getAppResponse(appId, country);
     if (response == null) {
       log.error("response is null");
       return 0.0;
@@ -77,7 +84,7 @@ public class AppStoreScraper {
   }
 
   public static long getAppReviews(String appId, String country) {
-    var response = getResponse(appId, country);
+    var response = getAppResponse(appId, country);
     if (response == null) {
       log.error("response is null");
       return 0L;
@@ -86,7 +93,7 @@ public class AppStoreScraper {
   }
 
   public static long getAppRatings(String appId, String country) {
-    var response = getResponse(appId, country);
+    var response = getAppResponse(appId, country);
     if (response == null) {
       log.error("response is null");
       return 0L;

src/main/java/com/miti99/storescraperbot/api/google/GooglePlayScraper.java

@@ -24,7 +24,7 @@ public class GooglePlayScraper {
   public static final String BASE_URL = "https://miti-google-play-scraper.vercel.app/";
 
   @SneakyThrows
-  public static GoogleAppResponse app(GoogleAppRequest request) {
+  public static String rawApp(GoogleAppRequest request) {
     var httpRequest =
         HttpRequest.newBuilder()
             .uri(URI.create(BASE_URL + "/app"))
@@ -38,12 +38,18 @@ public class GooglePlayScraper {
             // .connectTimeout(Duration.ofMillis(TIMEOUT))
             .followRedirects(Redirect.NORMAL)
             .build()) {
-      var body = httpClient.send(httpRequest, BodyHandlers.ofString()).body();
-      return GsonUtil.fromJson(body, GoogleAppResponse.class);
+      return httpClient.send(httpRequest, BodyHandlers.ofString()).body();
+    } catch (Exception e) {
+      log.error("rawAppResponse error - request: '{}'", GsonUtil.toJson(request), e);
+      return null;
     }
   }
 
-  private static GoogleAppResponse getResponse(String appId, String country) {
+  public static GoogleAppResponse app(GoogleAppRequest request) {
+    return GsonUtil.fromJson(rawApp(request), GoogleAppResponse.class);
+  }
+
+  private static GoogleAppResponse getAppResponse(String appId, String country) {
     boolean isInCache = GoogleAppRepository.INSTANCE.exist(appId);
     if (isInCache) {
       var app = GoogleAppRepository.INSTANCE.load(appId);
@@ -59,7 +65,7 @@ public class GooglePlayScraper {
   }
 
   public static LocalDate getLastUpdateOfApp(String appId, String country) {
-    var response = getResponse(appId, country);
+    var response = getAppResponse(appId, country);
     long lastUpdateMillis = 0;
     if (response != null) {
       lastUpdateMillis = response.updated();
@@ -70,7 +76,7 @@ public class GooglePlayScraper {
   }
 
   public static double getAppScore(String appId, String country) {
-    var response = getResponse(appId, country);
+    var response = getAppResponse(appId, country);
     if (response == null) {
       log.error("response is null");
       return 0.0;
@@ -79,7 +85,7 @@ public class GooglePlayScraper {
   }
 
   public static long getAppRatings(String appId, String country) {
-    var response = getResponse(appId, country);
+    var response = getAppResponse(appId, country);
     if (response == null) {
       log.error("response is null");
       return 0L;

src/main/java/com/miti99/storescraperbot/bot/StoreScrapeBot.java

@@ -15,6 +15,8 @@ import com.miti99.storescraperbot.bot.command.DeleteGroupCommand;
 import com.miti99.storescraperbot.bot.command.InfoCommand;
 import com.miti99.storescraperbot.bot.command.ListAppCommand;
 import com.miti99.storescraperbot.bot.command.ListGroupCommand;
+import com.miti99.storescraperbot.bot.command.RawAppleAppCommand;
+import com.miti99.storescraperbot.bot.command.RawGoogleAppCommand;
 import com.miti99.storescraperbot.bot.entity.NonUpdatedApp;
 import com.miti99.storescraperbot.bot.table.Table;
 import com.miti99.storescraperbot.constant.Constant;
@@ -40,6 +42,9 @@ public class StoreScrapeBot extends CommandLongPollingTelegramBot {
     super(StoreScrapeBotTelegramClient.INSTANCE, true, StoreScrapeBotUsernameSupplier.INSTANCE);
     register(InfoCommand.INSTANCE);
 
+    register(RawAppleAppCommand.INSTANCE);
+    register(RawGoogleAppCommand.INSTANCE);
+
     register(AddGroupCommand.INSTANCE);
     register(DeleteGroupCommand.INSTANCE);
     register(ListGroupCommand.INSTANCE);

src/main/java/com/miti99/storescraperbot/bot/command/RawAppleAppCommand.java

@@ -0,0 +1,81 @@
+package com.miti99.storescraperbot.bot.command;
+
+import com.miti99.storescraperbot.api.apple.AppStoreScraper;
+import com.miti99.storescraperbot.api.apple.request.AppleAppRequest;
+import com.miti99.storescraperbot.bot.StoreScrapeBotTelegramClient;
+import com.miti99.storescraperbot.repository.AdminRepository;
+import com.miti99.storescraperbot.util.GsonUtil;
+import java.io.ByteArrayInputStream;
+import java.nio.charset.StandardCharsets;
+import lombok.SneakyThrows;
+import lombok.extern.log4j.Log4j2;
+import org.telegram.telegrambots.meta.api.methods.send.SendDocument;
+import org.telegram.telegrambots.meta.api.objects.InputFile;
+import org.telegram.telegrambots.meta.api.objects.User;
+import org.telegram.telegrambots.meta.api.objects.chat.Chat;
+import org.telegram.telegrambots.meta.generics.TelegramClient;
+
+@Log4j2
+public class RawAppleAppCommand extends BaseStoreScraperBotCommand {
+  public static final RawAppleAppCommand INSTANCE = new RawAppleAppCommand();
+
+  RawAppleAppCommand() {
+    super(
+        "rawappleapp",
+        "<id/appId> [country]. Lấy raw response khi request lên service. id: <i>iTunes 'trackId'</i>, appId: <i>iTunes 'bundleId'</i>. Một số app cần country để hoạt động đúng, country mặc định là 'vn'");
+  }
+
+  @Override
+  @SneakyThrows
+  protected void executeCommand(
+      TelegramClient telegramClient, User user, Chat chat, String[] arguments) {
+    var admin = AdminRepository.INSTANCE.load();
+    if (!admin.getGroups().contains(chat.getId())) {
+      StoreScrapeBotTelegramClient.INSTANCE.sendMessage(
+          chat.getId(), "Group is not allowed to use bot");
+      return;
+    }
+
+    if (arguments.length < 1 || arguments.length > 2) {
+      StoreScrapeBotTelegramClient.INSTANCE.sendMessage(chat.getId(), "Invalid arguments");
+      return;
+    }
+
+    var appId = arguments[0];
+    long id = -1;
+    var country = arguments.length == 2 ? arguments[1] : "vn";
+    String response = "";
+    try {
+      try {
+        id = Long.parseLong(appId);
+      } catch (Exception e) {
+        // Input không phải id, bỏ qua
+      }
+      if (id != -1) {
+        response = AppStoreScraper.rawApp(new AppleAppRequest(id, country));
+      } else {
+        response = AppStoreScraper.rawApp(new AppleAppRequest(appId, country));
+      }
+    } catch (Exception e) {
+      log.error("request app error for appId: '{}', id: '{}'", appId, id, e);
+      StoreScrapeBotTelegramClient.INSTANCE.sendMessage(
+          chat.getId(), "Error when request app info");
+      return;
+    }
+
+    if (response == null) response = "";
+
+    var inputStream = new ByteArrayInputStream(response.getBytes(StandardCharsets.UTF_8));
+
+    var file = new InputFile(inputStream, "%s.json".formatted(appId));
+
+    var sendDocument =
+        SendDocument.builder()
+            .chatId(chat.getId())
+            .document(file)
+            // .caption("raw")
+            .build();
+
+    StoreScrapeBotTelegramClient.INSTANCE.execute(sendDocument);
+  }
+}

src/main/java/com/miti99/storescraperbot/bot/command/RawGoogleAppCommand.java

@@ -0,0 +1,70 @@
+package com.miti99.storescraperbot.bot.command;
+
+import com.miti99.storescraperbot.api.google.GooglePlayScraper;
+import com.miti99.storescraperbot.api.google.request.GoogleAppRequest;
+import com.miti99.storescraperbot.bot.StoreScrapeBotTelegramClient;
+import com.miti99.storescraperbot.repository.AdminRepository;
+import com.miti99.storescraperbot.util.GsonUtil;
+import java.io.ByteArrayInputStream;
+import java.nio.charset.StandardCharsets;
+import lombok.SneakyThrows;
+import lombok.extern.log4j.Log4j2;
+import org.telegram.telegrambots.meta.api.methods.send.SendDocument;
+import org.telegram.telegrambots.meta.api.objects.InputFile;
+import org.telegram.telegrambots.meta.api.objects.User;
+import org.telegram.telegrambots.meta.api.objects.chat.Chat;
+import org.telegram.telegrambots.meta.generics.TelegramClient;
+
+@Log4j2
+public class RawGoogleAppCommand extends BaseStoreScraperBotCommand {
+  public static final RawGoogleAppCommand INSTANCE = new RawGoogleAppCommand();
+
+  RawGoogleAppCommand() {
+    super(
+        "rawgoogleapp",
+        "<appId> [country]. Lấy raw response khi request lên service. Một số app cần country để hoạt động đúng, country mặc định là 'vn'");
+  }
+
+  @Override
+  @SneakyThrows
+  protected void executeCommand(
+      TelegramClient telegramClient, User user, Chat chat, String[] arguments) {
+    var admin = AdminRepository.INSTANCE.load();
+    if (!admin.getGroups().contains(chat.getId())) {
+      StoreScrapeBotTelegramClient.INSTANCE.sendMessage(
+          chat.getId(), "Group is not allowed to use bot");
+      return;
+    }
+
+    if (arguments.length < 1 || arguments.length > 2) {
+      StoreScrapeBotTelegramClient.INSTANCE.sendMessage(chat.getId(), "Invalid arguments");
+      return;
+    }
+
+    var appId = arguments[0];
+    var country = arguments.length == 2 ? arguments[1] : "vn";
+    String response = "";
+    try {
+      response = GooglePlayScraper.rawApp(new GoogleAppRequest(appId, country));
+    } catch (Exception e) {
+      log.error("request app error for appId: '{}'", appId, e);
+      StoreScrapeBotTelegramClient.INSTANCE.sendMessage(
+          chat.getId(), "Error when request app info");
+      return;
+    }
+    if (response == null) response = "";
+
+    var inputStream = new ByteArrayInputStream(response.getBytes(StandardCharsets.UTF_8));
+
+    var file = new InputFile(inputStream, "%s.json".formatted(appId));
+
+    var sendDocument =
+        SendDocument.builder()
+            .chatId(chat.getId())
+            .document(file)
+            // .caption("raw")
+            .build();
+
+    StoreScrapeBotTelegramClient.INSTANCE.execute(sendDocument);
+  }
+}