diff --git a/README.md b/README.md index eba39d8..b19d497 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,3 @@ # store-scraper-bot + Telegram bot that support scrape infos of an app on stores diff --git a/build.gradle.kts b/build.gradle.kts index f81696e..d2fac22 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -9,12 +9,29 @@ repositories { mavenCentral() } +configurations { + compileOnly { + extendsFrom(configurations.annotationProcessor.get()) + } + testCompileOnly { + extendsFrom(configurations.testAnnotationProcessor.get()) + } +} + dependencies { - testImplementation(platform("org.junit:junit-bom:5.10.0")) + annotationProcessor("org.projectlombok:lombok:1.18.36") + implementation("com.couchbase.client:java-client:3.4.11") + implementation("org.apache.logging.log4j:log4j-1.2-api:2.24.3") + implementation("org.apache.logging.log4j:log4j-core:2.24.3") + implementation("org.apache.logging.log4j:log4j-slf4j2-impl:2.24.3") + implementation("org.telegram:telegrambots-abilities:8.0.0") + + testAnnotationProcessor("org.projectlombok:lombok:1.18.36") + testImplementation(platform("org.junit:junit-bom:5.11.4")) testImplementation("org.junit.jupiter:junit-jupiter") testRuntimeOnly("org.junit.platform:junit-platform-launcher") } tasks.test { useJUnitPlatform() -} \ No newline at end of file +} diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..534ffa6 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,16 @@ +services: + couchbase: + image: couchbase:community-7.6.2 + ports: + - "8091-8097:8091-8097" + - "9123:9123" + - "11207:11207" + - "11210:11210" + - "11280:11280" + - "18091-18097:18091-18097" + volumes: + - couchbase_data:/opt/couchbase/var + restart: unless-stopped + +volumes: + couchbase_data: diff --git a/src/main/java/com/miti99/Main.java b/src/main/java/com/miti99/storescraperbot/Main.java similarity index 95% rename from src/main/java/com/miti99/Main.java rename to src/main/java/com/miti99/storescraperbot/Main.java index 15ce5a2..7139c2b 100644 --- a/src/main/java/com/miti99/Main.java +++ b/src/main/java/com/miti99/storescraperbot/Main.java @@ -1,4 +1,4 @@ -package com.miti99; +package com.miti99.storescraperbot; // TIP To Run code, press or // click the icon in the gutter. diff --git a/src/main/java/com/miti99/storescraperbot/api/apple/AppStoreScraper.java b/src/main/java/com/miti99/storescraperbot/api/apple/AppStoreScraper.java new file mode 100644 index 0000000..a023f43 --- /dev/null +++ b/src/main/java/com/miti99/storescraperbot/api/apple/AppStoreScraper.java @@ -0,0 +1,40 @@ +package com.miti99.storescraperbot.api.apple; + +import com.couchbase.client.core.deps.io.netty.handler.codec.http.HttpHeaderNames; +import com.couchbase.client.core.deps.io.netty.handler.codec.http.HttpHeaderValues; +import com.miti99.storescraperbot.api.apple.request.AppleAppRequest; +import com.miti99.storescraperbot.api.apple.response.AppleAppResponse; +import com.miti99.storescraperbot.util.JacksonUtil; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpClient.Redirect; +import java.net.http.HttpRequest; +import java.net.http.HttpRequest.BodyPublishers; +import java.net.http.HttpResponse.BodyHandlers; +import lombok.SneakyThrows; + +public class AppStoreScraper { + public static final String BASE_URL = "https://miti-app-store-scraper.vercel.app/"; + + @SneakyThrows + public static AppleAppResponse app(AppleAppRequest request) { + var httpRequest = + HttpRequest.newBuilder() + .uri(URI.create(BASE_URL + "/app")) + // .timeout(Duration.ofMillis(TIMEOUT)) + .header( + HttpHeaderNames.CONTENT_TYPE.toString(), + HttpHeaderValues.APPLICATION_JSON.toString()) + .POST(BodyPublishers.ofString(JacksonUtil.writeValueAsString(request))) + .build(); + + var body = + HttpClient.newBuilder() + .followRedirects(Redirect.NORMAL) + // .connectTimeout(Duration.ofMillis(TIMEOUT)) + .build() + .send(httpRequest, BodyHandlers.ofString()) + .body(); + return JacksonUtil.readValue(body, AppleAppResponse.class); + } +} diff --git a/src/main/java/com/miti99/storescraperbot/api/apple/request/AppleAppRequest.java b/src/main/java/com/miti99/storescraperbot/api/apple/request/AppleAppRequest.java new file mode 100644 index 0000000..526c97b --- /dev/null +++ b/src/main/java/com/miti99/storescraperbot/api/apple/request/AppleAppRequest.java @@ -0,0 +1,5 @@ +package com.miti99.storescraperbot.api.apple.request; + +public record AppleAppRequest(String appId) { + +} diff --git a/src/main/java/com/miti99/storescraperbot/api/apple/response/AppleAppResponse.java b/src/main/java/com/miti99/storescraperbot/api/apple/response/AppleAppResponse.java new file mode 100644 index 0000000..16759cb --- /dev/null +++ b/src/main/java/com/miti99/storescraperbot/api/apple/response/AppleAppResponse.java @@ -0,0 +1,67 @@ +package com.miti99.storescraperbot.api.apple.response; + +import java.util.List; +import java.util.Map; + +public record AppleAppResponse( + String title, + String description, + String descriptionHTML, + String summary, + String installs, + long minInstalls, + long maxInstalls, + double score, + String scoreText, + long ratings, + long reviews, + Map histogram, + double price, + boolean free, + String currency, + String priceText, + boolean offersIAP, + String IAPRange, + String androidVersion, + String androidVersionText, + String androidMaxVersion, + String developer, + String developerId, + String developerEmail, + String developerWebsite, + String developerAddress, + String developerLegalName, + String developerLegalEmail, + String developerLegalAddress, + String developerLegalPhoneNumber, + String privacyPolicy, + String developerInternalID, + String genre, + String genreId, + List categories, + String icon, + String headerImage, + List screenshots, + String video, + String videoImage, + String previewVideo, + String contentRating, + String contentRatingDescription, + boolean adSupported, + String released, + String updated, + String version, + String recentChanges, + List comments, + boolean preregister, + boolean earlyAccessEnabled, + boolean isAvailableInPlayPass, + boolean editorsChoice, + List features, + String appId, + String url) { + + public record Category(String name, String id) {} + + public record Feature(String title, String description) {} +} diff --git a/src/main/java/com/miti99/storescraperbot/api/google/GooglePlayScraper.java b/src/main/java/com/miti99/storescraperbot/api/google/GooglePlayScraper.java new file mode 100644 index 0000000..4b12fc4 --- /dev/null +++ b/src/main/java/com/miti99/storescraperbot/api/google/GooglePlayScraper.java @@ -0,0 +1,40 @@ +package com.miti99.storescraperbot.api.google; + +import com.couchbase.client.core.deps.io.netty.handler.codec.http.HttpHeaderNames; +import com.couchbase.client.core.deps.io.netty.handler.codec.http.HttpHeaderValues; +import com.miti99.storescraperbot.api.google.request.GoogleAppRequest; +import com.miti99.storescraperbot.api.google.response.GoogleAppResponse; +import com.miti99.storescraperbot.util.JacksonUtil; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpClient.Redirect; +import java.net.http.HttpRequest; +import java.net.http.HttpRequest.BodyPublishers; +import java.net.http.HttpResponse.BodyHandlers; +import lombok.SneakyThrows; + +public class GooglePlayScraper { + public static final String BASE_URL = "https://miti-google-play-scraper.vercel.app/"; + + @SneakyThrows + public static GoogleAppResponse app(GoogleAppRequest request) { + var httpRequest = + HttpRequest.newBuilder() + .uri(URI.create(BASE_URL + "/app")) + // .timeout(Duration.ofMillis(TIMEOUT)) + .header( + HttpHeaderNames.CONTENT_TYPE.toString(), + HttpHeaderValues.APPLICATION_JSON.toString()) + .POST(BodyPublishers.ofString(JacksonUtil.writeValueAsString(request))) + .build(); + + var body = + HttpClient.newBuilder() + // .connectTimeout(Duration.ofMillis(TIMEOUT)) + .followRedirects(Redirect.NORMAL) + .build() + .send(httpRequest, BodyHandlers.ofString()) + .body(); + return JacksonUtil.readValue(body, GoogleAppResponse.class); + } +} diff --git a/src/main/java/com/miti99/storescraperbot/api/google/request/GoogleAppRequest.java b/src/main/java/com/miti99/storescraperbot/api/google/request/GoogleAppRequest.java new file mode 100644 index 0000000..f4eb905 --- /dev/null +++ b/src/main/java/com/miti99/storescraperbot/api/google/request/GoogleAppRequest.java @@ -0,0 +1,5 @@ +package com.miti99.storescraperbot.api.google.request; + +public record GoogleAppRequest(String appId) { + +} diff --git a/src/main/java/com/miti99/storescraperbot/api/google/response/GoogleAppResponse.java b/src/main/java/com/miti99/storescraperbot/api/google/response/GoogleAppResponse.java new file mode 100644 index 0000000..3384d6a --- /dev/null +++ b/src/main/java/com/miti99/storescraperbot/api/google/response/GoogleAppResponse.java @@ -0,0 +1,67 @@ +package com.miti99.storescraperbot.api.google.response; + +import java.util.List; +import java.util.Map; + +public record GoogleAppResponse( + String title, + String description, + String descriptionHTML, + String summary, + String installs, + long minInstalls, + long maxInstalls, + double score, + String scoreText, + long ratings, + long reviews, + Map histogram, + double price, + boolean free, + String currency, + String priceText, + boolean offersIAP, + String IAPRange, + String androidVersion, + String androidVersionText, + String androidMaxVersion, + String developer, + String developerId, + String developerEmail, + String developerWebsite, + String developerAddress, + String developerLegalName, + String developerLegalEmail, + String developerLegalAddress, + String developerLegalPhoneNumber, + String privacyPolicy, + String developerInternalID, + String genre, + String genreId, + List categories, + String icon, + String headerImage, + List screenshots, + String video, + String videoImage, + String previewVideo, + String contentRating, + String contentRatingDescription, + boolean adSupported, + Long released, + long updated, + String version, + String recentChanges, + List comments, + boolean preregister, + boolean earlyAccessEnabled, + boolean isAvailableInPlayPass, + boolean editorsChoice, + List features, + String appId, + String url) { + + public record Category(String name, String id) {} + + public record Feature(String title, String description) {} +} diff --git a/src/main/java/com/miti99/storescraperbot/config/Config.java b/src/main/java/com/miti99/storescraperbot/config/Config.java new file mode 100644 index 0000000..dd387c5 --- /dev/null +++ b/src/main/java/com/miti99/storescraperbot/config/Config.java @@ -0,0 +1,5 @@ +package com.miti99.storescraperbot.config; + +public class Config { + +} diff --git a/src/main/java/com/miti99/storescraperbot/util/JacksonUtil.java b/src/main/java/com/miti99/storescraperbot/util/JacksonUtil.java new file mode 100644 index 0000000..de46f13 --- /dev/null +++ b/src/main/java/com/miti99/storescraperbot/util/JacksonUtil.java @@ -0,0 +1,47 @@ +package com.miti99.storescraperbot.util; + +import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.core.JsonParser.Feature; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import lombok.SneakyThrows; + +public class JacksonUtil { + public static ObjectMapper MAPPER = objectMapper(); + + private static ObjectMapper objectMapper() { + var objectMapper = new ObjectMapper(); + + objectMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + objectMapper.disable( + SerializationFeature.FAIL_ON_EMPTY_BEANS, + SerializationFeature.INDENT_OUTPUT, + SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + + objectMapper.enable( + DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT, + DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, + DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL); + objectMapper.enable( + Feature.ALLOW_SINGLE_QUOTES, Feature.ALLOW_UNQUOTED_FIELD_NAMES, Feature.IGNORE_UNDEFINED); + objectMapper.enable(SerializationFeature.WRITE_ENUMS_USING_TO_STRING); + + objectMapper.setSerializationInclusion(Include.NON_NULL); + objectMapper.setVisibility(PropertyAccessor.FIELD, Visibility.ANY); + + return objectMapper; + } + + @SneakyThrows + public static T readValue(String input, Class valueType) { + return MAPPER.readValue(input, valueType); + } + + @SneakyThrows + public static String writeValueAsString(Object input) { + return MAPPER.writeValueAsString(input); + } +} diff --git a/src/test/java/com/miti99/storescraperbot/api/apple/AppStoreScraperTest.java b/src/test/java/com/miti99/storescraperbot/api/apple/AppStoreScraperTest.java new file mode 100644 index 0000000..9a32b53 --- /dev/null +++ b/src/test/java/com/miti99/storescraperbot/api/apple/AppStoreScraperTest.java @@ -0,0 +1,16 @@ +package com.miti99.storescraperbot.api.apple; + +import static org.junit.jupiter.api.Assertions.*; + +import com.miti99.storescraperbot.api.apple.request.AppleAppRequest; +import com.miti99.storescraperbot.util.JacksonUtil; +import org.junit.jupiter.api.Test; + +class AppStoreScraperTest { + @Test + void testApp() { + var request = new AppleAppRequest("com.mpt.kvtm"); + var response = AppStoreScraper.app(request); + System.out.println(JacksonUtil.writeValueAsString(response)); + } +} diff --git a/src/test/java/com/miti99/storescraperbot/api/google/GooglePlayScraperTest.java b/src/test/java/com/miti99/storescraperbot/api/google/GooglePlayScraperTest.java new file mode 100644 index 0000000..e9da330 --- /dev/null +++ b/src/test/java/com/miti99/storescraperbot/api/google/GooglePlayScraperTest.java @@ -0,0 +1,16 @@ +package com.miti99.storescraperbot.api.google; + +import static org.junit.jupiter.api.Assertions.*; + +import com.miti99.storescraperbot.api.google.request.GoogleAppRequest; +import com.miti99.storescraperbot.util.JacksonUtil; +import org.junit.jupiter.api.Test; + +class GooglePlayScraperTest { + @Test + void testApp() { + var request = new GoogleAppRequest("vn.kvtm.js"); + var response = GooglePlayScraper.app(request); + System.out.println(JacksonUtil.writeValueAsString(response)); + } +}