feat(self scraper): try to crawl without country

But fail
This commit is contained in:
2025-11-12 23:10:00 +07:00
parent 1418c20366
commit 64938a0eb7
25 changed files with 254 additions and 36 deletions
+8
View File
@@ -1,3 +1,11 @@
# store-scraper-bot
Telegram bot that support scrape infos of an app on stores
## Credits
Many thanks to [facundoolano](https://github.com/facundoolano)'
s [google-play-scraper](https://github.com/facundoolano/google-play-scraper)
and [app-store-scraper](https://github.com/facundoolano/app-store-scraper), myy api code are based
on
their logics.
+1
View File
@@ -1,3 +1,4 @@
lombok.accessors.fluent = true
lombok.addLombokGeneratedAnnotation = true
lombok.experimental.flagUsage = error
lombok.log.apacheCommons.flagUsage = error
@@ -0,0 +1,95 @@
package com.miti99.storescraperbot.api.apple;
import com.miti99.storescraperbot.api.apple.entity.AppleAppDetail;
import com.miti99.storescraperbot.api.apple.request.ITunesLookupRequest;
import com.miti99.storescraperbot.api.apple.response.ITunesLookupResponse;
import com.miti99.storescraperbot.repository.AppleAppRepository;
import com.miti99.storescraperbot.util.GsonUtil;
import com.miti99.storescraperbot.util.RequestUtil;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpClient.Redirect;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse.BodyHandlers;
import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneId;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.log4j.Log4j2;
@Log4j2
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class AppStoreScraper {
private static final String LOOKUP_URL = "https://itunes.apple.com/lookup?";
@SneakyThrows
public static String rawLookup(ITunesLookupRequest request) {
var httpRequest =
HttpRequest.newBuilder()
.uri(URI.create(LOOKUP_URL + RequestUtil.makeGetParams(request)))
// .timeout(Duration.ofMillis(TIMEOUT))
.header("Content-Type", "application/json")
.GET()
.build();
try (var httpClient =
HttpClient.newBuilder()
.followRedirects(Redirect.NORMAL)
// .connectTimeout(Duration.ofMillis(TIMEOUT))
.build()) {
return httpClient.send(httpRequest, BodyHandlers.ofString()).body();
} catch (Exception e) {
log.error("rawLookup error - request: '{}'", GsonUtil.toJson(request), e);
return null;
}
}
@SneakyThrows
public static AppleAppDetail app(ITunesLookupRequest request) {
return GsonUtil.fromJson(rawLookup(request), ITunesLookupResponse.class).getAppDetail();
}
public static AppleAppDetail app(String appId) {
return app(new ITunesLookupRequest(appId));
}
public static AppleAppDetail app(long id) {
return app(new ITunesLookupRequest(id));
}
public static AppleAppDetail getAppResponse(String appId) {
boolean isInCache = AppleAppRepository.INSTANCE.exist(appId);
if (isInCache) {
var app = AppleAppRepository.INSTANCE.load(appId);
return app.detail();
} else {
var response = app(appId);
AppleAppRepository.INSTANCE.init(appId);
var app = AppleAppRepository.INSTANCE.load(appId);
app.detail(response);
AppleAppRepository.INSTANCE.save(appId, app);
return response;
}
}
public static LocalDate getAppUpdated(String appId) {
var detail = getAppResponse(appId);
if (detail == null) {
log.error("detail is null");
return LocalDate.ofInstant(Instant.ofEpochMilli(0), ZoneId.systemDefault());
}
return LocalDate.ofInstant(
Instant.parse(detail.currentVersionReleaseDate()), ZoneId.systemDefault());
}
public static double getAppScore(String appId) {
var response = getAppResponse(appId);
if (response == null) {
log.error("response is null");
return 0.0;
}
return response.averageUserRating();
}
}
@@ -0,0 +1,13 @@
package com.miti99.storescraperbot.api.apple.entity;
import lombok.Value;
/**
* Chỉ define một số field cần thiết nên không đầy đủ. Khi cần thì check raw response để define thêm
*/
@Value
public class AppleAppDetail {
String bundleId;
String currentVersionReleaseDate;
double averageUserRating;
}
@@ -0,0 +1,20 @@
package com.miti99.storescraperbot.api.apple.request;
import lombok.RequiredArgsConstructor;
import lombok.Value;
@RequiredArgsConstructor
@Value
public class ITunesLookupRequest {
Long id;
String bundleId;
String entity = "software";
public ITunesLookupRequest(Long id) {
this(id, null);
}
public ITunesLookupRequest(String bundleId) {
this(null, bundleId);
}
}
@@ -0,0 +1,23 @@
package com.miti99.storescraperbot.api.apple.response;
import com.miti99.storescraperbot.api.apple.entity.AppleAppDetail;
import com.miti99.storescraperbot.util.GsonUtil;
import java.util.List;
import lombok.Value;
import lombok.extern.log4j.Log4j2;
@Log4j2
@Value
public class ITunesLookupResponse {
int resultCount;
List<AppleAppDetail> results;
public AppleAppDetail getAppDetail() {
if (resultCount != 1) {
log.warn("resultCount('{}') != 1", resultCount);
log.warn("results: {}", GsonUtil.toJson(results));
}
return results.getFirst();
}
}
@@ -54,12 +54,12 @@ public class AppStoreScraper {
boolean isInCache = AppleAppRepository.INSTANCE.exist(appId);
if (isInCache) {
var app = AppleAppRepository.INSTANCE.load(appId);
return app.getApp();
return app.app();
} else {
var response = app(new AppleAppRequest(appId, country));
AppleAppRepository.INSTANCE.init(appId);
var app = AppleAppRepository.INSTANCE.load(appId);
app.setApp(response);
app.app(response);
AppleAppRepository.INSTANCE.save(appId, app);
return response;
}
@@ -53,12 +53,12 @@ public class GooglePlayScraper {
boolean isInCache = GoogleAppRepository.INSTANCE.exist(appId);
if (isInCache) {
var app = GoogleAppRepository.INSTANCE.load(appId);
return app.getApp();
return app.app();
} else {
var response = app(new GoogleAppRequest(appId, country));
GoogleAppRepository.INSTANCE.init(appId);
var app = GoogleAppRepository.INSTANCE.load(appId);
app.setApp(response);
app.app(response);
GoogleAppRepository.INSTANCE.save(appId, app);
return response;
}
@@ -79,7 +79,7 @@ public class StoreScrapeBot extends CommandLongPollingTelegramBot {
public void runCheckApp() {
var admin = AdminRepository.INSTANCE.load();
for (var groupId : admin.getGroups()) {
for (var groupId : admin.groups()) {
checkAppForGroup(groupId);
}
}
@@ -89,7 +89,7 @@ public class StoreScrapeBot extends CommandLongPollingTelegramBot {
var now = LocalDate.now();
var nonUpdatedAppleApps = new ArrayList<NonUpdatedApp>();
for (var app : group.getAppleApps()) {
for (var app : group.appleApps()) {
var appId = app.appId();
var updated = AppStoreScraper.getAppUpdated(appId, app.country());
long days = ChronoUnit.DAYS.between(updated, now);
@@ -98,7 +98,7 @@ public class StoreScrapeBot extends CommandLongPollingTelegramBot {
}
}
var nonUpdatedGoogleApps = new ArrayList<NonUpdatedApp>();
for (var app : group.getGoogleApps()) {
for (var app : group.googleApps()) {
var appId = app.appId();
var updated = GooglePlayScraper.getLastUpdateOfApp(appId, app.country());
long days = ChronoUnit.DAYS.between(updated, now);
@@ -26,7 +26,7 @@ public class AddAppleAppCommand extends BaseStoreScraperBotCommand {
protected void executeCommand(
TelegramClient telegramClient, User user, Chat chat, String[] arguments) {
var admin = AdminRepository.INSTANCE.load();
if (!admin.getGroups().contains(chat.getId())) {
if (!admin.groups().contains(chat.getId())) {
StoreScrapeBotTelegramClient.INSTANCE.sendMessage(
chat.getId(), "Group is not allowed to use bot");
return;
@@ -64,13 +64,13 @@ public class AddAppleAppCommand extends BaseStoreScraperBotCommand {
var group = GroupRepository.INSTANCE.load(groupId);
var finalAppId = appId;
if (group.getAppleApps().stream().anyMatch(app -> finalAppId.equals(app.appId()))) {
if (group.appleApps().stream().anyMatch(app -> finalAppId.equals(app.appId()))) {
StoreScrapeBotTelegramClient.INSTANCE.sendMessage(
chat.getId(), "Apple app <code>%s</code> is already added".formatted(appId));
return;
}
group.getAppleApps().add(new AppleAppInfo(appId, country));
group.appleApps().add(new AppleAppInfo(appId, country));
GroupRepository.INSTANCE.save(groupId, group);
StoreScrapeBotTelegramClient.INSTANCE.sendMessage(
chat.getId(),
@@ -26,7 +26,7 @@ public class AddGoogleAppCommand extends BaseStoreScraperBotCommand {
protected void executeCommand(
TelegramClient telegramClient, User user, Chat chat, String[] arguments) {
var admin = AdminRepository.INSTANCE.load();
if (!admin.getGroups().contains(chat.getId())) {
if (!admin.groups().contains(chat.getId())) {
StoreScrapeBotTelegramClient.INSTANCE.sendMessage(
chat.getId(), "Group is not allowed to use bot");
return;
@@ -52,13 +52,13 @@ public class AddGoogleAppCommand extends BaseStoreScraperBotCommand {
long groupId = chat.getId();
var group = GroupRepository.INSTANCE.load(groupId);
if (group.getGoogleApps().stream().anyMatch(app -> appId.equals(app.appId()))) {
if (group.googleApps().stream().anyMatch(app -> appId.equals(app.appId()))) {
StoreScrapeBotTelegramClient.INSTANCE.sendMessage(
chat.getId(), "Google app <code>%s</code> is already added".formatted(appId));
return;
}
group.getGoogleApps().add(new GoogleAppInfo(appId, country));
group.googleApps().add(new GoogleAppInfo(appId, country));
GroupRepository.INSTANCE.save(groupId, group);
StoreScrapeBotTelegramClient.INSTANCE.sendMessage(
chat.getId(),
@@ -41,13 +41,13 @@ public class AddGroupCommand extends BaseStoreScraperBotCommand {
}
var admin = AdminRepository.INSTANCE.load();
if (admin.getGroups().contains(groupId)) {
if (admin.groups().contains(groupId)) {
StoreScrapeBotTelegramClient.INSTANCE.sendMessage(chat.getId(), "Group is already added");
return;
}
GroupRepository.INSTANCE.init(groupId);
admin.getGroups().add(groupId);
admin.groups().add(groupId);
AdminRepository.INSTANCE.save(admin);
StoreScrapeBotTelegramClient.INSTANCE.sendMessage(chat.getId(), "Group added successfully");
}
@@ -24,7 +24,7 @@ public class CheckAppCommand extends BaseStoreScraperBotCommand {
protected void executeCommand(
TelegramClient telegramClient, User user, Chat chat, String[] arguments) {
var admin = AdminRepository.INSTANCE.load();
if (!admin.getGroups().contains(chat.getId())) {
if (!admin.groups().contains(chat.getId())) {
StoreScrapeBotTelegramClient.INSTANCE.sendMessage(
chat.getId(), "Group is not allowed to use bot");
return;
@@ -43,7 +43,7 @@ public class CheckAppCommand extends BaseStoreScraperBotCommand {
sb.append("<b>Apple Apps:</b>\n");
sb.append("<code>\n");
var appleTable = new Table("AppId", "Updated", "Days", "OK");
for (var app : group.getAppleApps()) {
for (var app : group.appleApps()) {
var appId = app.appId();
var updated = AppStoreScraper.getAppUpdated(appId, app.country());
long days = ChronoUnit.DAYS.between(updated, now);
@@ -57,7 +57,7 @@ public class CheckAppCommand extends BaseStoreScraperBotCommand {
sb.append("<b>Google Apps:</b>\n");
sb.append("<code>\n");
var googleTable = new Table("AppId", "Updated", "Days", "OK");
for (var app : group.getGoogleApps()) {
for (var app : group.googleApps()) {
var appId = app.appId();
var updated = GooglePlayScraper.getLastUpdateOfApp(appId, app.country());
long days = ChronoUnit.DAYS.between(updated, now);
@@ -23,7 +23,7 @@ public class CheckAppScoreCommand extends BaseStoreScraperBotCommand {
protected void executeCommand(
TelegramClient telegramClient, User user, Chat chat, String[] arguments) {
var admin = AdminRepository.INSTANCE.load();
if (!admin.getGroups().contains(chat.getId())) {
if (!admin.groups().contains(chat.getId())) {
StoreScrapeBotTelegramClient.INSTANCE.sendMessage(
chat.getId(), "Group is not allowed to use bot");
return;
@@ -41,7 +41,7 @@ public class CheckAppScoreCommand extends BaseStoreScraperBotCommand {
sb.append("<b>Apple Apps:</b>\n");
sb.append("<code>\n");
var appleTable = new Table("AppId", "Score", "Ratings");
for (var app : group.getAppleApps()) {
for (var app : group.appleApps()) {
var appId = app.appId();
var country = app.country();
double score =
@@ -55,7 +55,7 @@ public class CheckAppScoreCommand extends BaseStoreScraperBotCommand {
sb.append("<b>Google Apps:</b>\n");
sb.append("<code>\n");
var googleTable = new Table("AppId", "Score", "Ratings");
for (var app : group.getGoogleApps()) {
for (var app : group.googleApps()) {
var appId = app.appId();
var country = app.country();
double score =
@@ -18,7 +18,7 @@ public class DeleteAppleAppCommand extends BaseStoreScraperBotCommand {
protected void executeCommand(
TelegramClient telegramClient, User user, Chat chat, String[] arguments) {
var admin = AdminRepository.INSTANCE.load();
if (!admin.getGroups().contains(chat.getId())) {
if (!admin.groups().contains(chat.getId())) {
StoreScrapeBotTelegramClient.INSTANCE.sendMessage(
chat.getId(), "Group is not allowed to use bot");
return;
@@ -33,12 +33,12 @@ public class DeleteAppleAppCommand extends BaseStoreScraperBotCommand {
long groupId = chat.getId();
var group = GroupRepository.INSTANCE.load(groupId);
if (group.getAppleApps().stream().noneMatch(app -> appId.equals(app.appId()))) {
if (group.appleApps().stream().noneMatch(app -> appId.equals(app.appId()))) {
StoreScrapeBotTelegramClient.INSTANCE.sendMessage(chat.getId(), "Apple app is not added");
return;
}
group.getAppleApps().removeIf(app -> appId.equals(app.appId()));
group.appleApps().removeIf(app -> appId.equals(app.appId()));
GroupRepository.INSTANCE.save(groupId, group);
StoreScrapeBotTelegramClient.INSTANCE.sendMessage(
chat.getId(), "Apple app deleted successfully");
@@ -18,7 +18,7 @@ public class DeleteGoogleAppCommand extends BaseStoreScraperBotCommand {
protected void executeCommand(
TelegramClient telegramClient, User user, Chat chat, String[] arguments) {
var admin = AdminRepository.INSTANCE.load();
if (!admin.getGroups().contains(chat.getId())) {
if (!admin.groups().contains(chat.getId())) {
StoreScrapeBotTelegramClient.INSTANCE.sendMessage(
chat.getId(), "Group is not allowed to use bot");
return;
@@ -33,12 +33,12 @@ public class DeleteGoogleAppCommand extends BaseStoreScraperBotCommand {
long groupId = chat.getId();
var group = GroupRepository.INSTANCE.load(groupId);
if (group.getGoogleApps().stream().noneMatch(app -> appId.equals(app.appId()))) {
if (group.googleApps().stream().noneMatch(app -> appId.equals(app.appId()))) {
StoreScrapeBotTelegramClient.INSTANCE.sendMessage(chat.getId(), "Google app is not added");
return;
}
group.getGoogleApps().removeIf(app -> appId.equals(app.appId()));
group.googleApps().removeIf(app -> appId.equals(app.appId()));
GroupRepository.INSTANCE.save(groupId, group);
StoreScrapeBotTelegramClient.INSTANCE.sendMessage(
chat.getId(), "Google app deleted successfully");
@@ -40,12 +40,12 @@ public class DeleteGroupCommand extends BaseStoreScraperBotCommand {
}
var admin = AdminRepository.INSTANCE.load();
if (!admin.getGroups().contains(groupId)) {
if (!admin.groups().contains(groupId)) {
StoreScrapeBotTelegramClient.INSTANCE.sendMessage(chat.getId(), "Group is not added");
return;
}
admin.getGroups().remove(groupId);
admin.groups().remove(groupId);
AdminRepository.INSTANCE.save(admin);
StoreScrapeBotTelegramClient.INSTANCE.sendMessage(chat.getId(), "Group deleted successfully");
}
@@ -19,7 +19,7 @@ public class ListAppCommand extends BaseStoreScraperBotCommand {
protected void executeCommand(
TelegramClient telegramClient, User user, Chat chat, String[] arguments) {
var admin = AdminRepository.INSTANCE.load();
if (!admin.getGroups().contains(chat.getId())) {
if (!admin.groups().contains(chat.getId())) {
StoreScrapeBotTelegramClient.INSTANCE.sendMessage(
chat.getId(), "Group is not allowed to use bot");
return;
@@ -38,7 +38,7 @@ public class ListAppCommand extends BaseStoreScraperBotCommand {
sb.append("<code>\n");
var appleTable = new Table("#", "AppId", "Country");
int i = 0;
for (var app : group.getAppleApps()) {
for (var app : group.appleApps()) {
i++;
appleTable.addRow(i, app.appId(), app.country());
}
@@ -49,7 +49,7 @@ public class ListAppCommand extends BaseStoreScraperBotCommand {
sb.append("<code>\n");
var googleTable = new Table("#", "AppId", "Country");
i = 0;
for (var app : group.getGoogleApps()) {
for (var app : group.googleApps()) {
i++;
googleTable.addRow(i, app.appId(), app.country());
}
@@ -28,7 +28,7 @@ public class ListGroupCommand extends BaseStoreScraperBotCommand {
}
var admin = AdminRepository.INSTANCE.load();
var groups = admin.getGroups();
var groups = admin.groups();
var sb = new StringBuilder();
sb.append("<b>Groups:</b>\n");
for (var groupId : groups) {
@@ -29,7 +29,7 @@ public class RawAppleAppCommand extends BaseStoreScraperBotCommand {
protected void executeCommand(
TelegramClient telegramClient, User user, Chat chat, String[] arguments) {
var admin = AdminRepository.INSTANCE.load();
if (!admin.getGroups().contains(chat.getId())) {
if (!admin.groups().contains(chat.getId())) {
StoreScrapeBotTelegramClient.INSTANCE.sendMessage(
chat.getId(), "Group is not allowed to use bot");
return;
@@ -29,7 +29,7 @@ public class RawGoogleAppCommand extends BaseStoreScraperBotCommand {
protected void executeCommand(
TelegramClient telegramClient, User user, Chat chat, String[] arguments) {
var admin = AdminRepository.INSTANCE.load();
if (!admin.getGroups().contains(chat.getId())) {
if (!admin.groups().contains(chat.getId())) {
StoreScrapeBotTelegramClient.INSTANCE.sendMessage(
chat.getId(), "Group is not allowed to use bot");
return;
@@ -1,5 +1,6 @@
package com.miti99.storescraperbot.model;
import com.miti99.storescraperbot.api.apple.entity.AppleAppDetail;
import com.miti99.storescraperbot.api.old.apple.response.AppleAppResponse;
import lombok.Getter;
import lombok.Setter;
@@ -8,4 +9,5 @@ import lombok.Setter;
@Setter
public class AppleApp extends AbstractModel<String> {
AppleAppResponse app;
AppleAppDetail detail;
}
@@ -71,7 +71,7 @@ public abstract class AbstractRepository<K, V extends AbstractModel<K>> {
return;
}
V data = classV.getDeclaredConstructor().newInstance();
data.setKey(key);
data.key(key);
save(key, data);
} catch (Exception e) {
log.error("Error while initializing data", e);
@@ -0,0 +1,29 @@
package com.miti99.storescraperbot.util;
import com.google.gson.reflect.TypeToken;
import java.lang.reflect.Type;
import java.util.Map;
import java.util.stream.Collectors;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class RequestUtil {
private static final Type MAP_STRING_STRING_TYPE =
new TypeToken<Map<String, String>>() {}.getType();
private static Map<String, String> objToMapStringString(Object obj) {
return GsonUtil.GSON.fromJson(GsonUtil.GSON.toJson(obj), MAP_STRING_STRING_TYPE);
}
public static String makeGetParams(Map<String, String> params) {
return params.entrySet().stream()
.filter(entry -> entry.getValue() != null)
.map(entry -> entry.getKey() + "=" + entry.getValue())
.collect(Collectors.joining("&"));
}
public static String makeGetParams(Object obj) {
return makeGetParams(objToMapStringString(obj));
}
}
@@ -0,0 +1,27 @@
package com.miti99.storescraperbot.api.apple;
import static org.junit.jupiter.api.Assertions.*;
import com.miti99.storescraperbot.util.GsonUtil;
import org.junit.jupiter.api.Test;
class AppStoreScraperTest {
@Test
void testComMptKvtm() {
var response = AppStoreScraper.app("com.mpt.kvtm");
System.out.println(GsonUtil.toJson(response));
}
@Test
void testComMptBuraco() {
// var response = AppStoreScraper.app("com.mpt.buraco");
var response = AppStoreScraper.app(1638264682);
System.out.println(GsonUtil.toJson(response));
}
@Test
void testComMptDoudizhu() {
var response = AppStoreScraper.app("com.mpt.doudizhu");
System.out.println(GsonUtil.toJson(response));
}
}