tiennm99/store-scraper-bot-java
1
0
Fork 0
mirror of https://github.com/tiennm99/store-scraper-bot-java.git synced 2026-05-14 07:52:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(mongo): cleanup

Browse Source
This commit is contained in:
tiennm99
2025-12-15 22:46:21 +07:00
parent c351535f7c
commit eff6049623
11 changed files with 38 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+1 -4
View File
@@ -2,10 +2,7 @@
# cp .env.example .env
# MongoDB configuration
MONGODB_CONNECTION_STRING=mongodb://localhost:27017
MONGODB_USERNAME=your_mongodb_username
MONGODB_PASSWORD=your_mongodb_password
MONGODB_DATABASE_NAME=store-scraper-bot
MONGODB_CONNECTION_STRING=mongodb://localhost:27017/store-scraper-bot
# Telegram Bot configuration
TELEGRAM_BOT_TOKEN=your_telegram_bot_token
build.gradle.kts
+1 -1
View File
@@ -22,13 +22,13 @@ configurations {
dependencies {
annotationProcessor("org.projectlombok:lombok:1.18.36")
implementation("org.mongodb:mongodb-driver-sync:5.2.1")
implementation("com.fasterxml.jackson.core:jackson-databind:2.18.2")
implementation("com.google.code.gson:gson:2.11.0")
implementation("com.google.guava:guava:33.4.0-jre")
implementation("org.apache.commons:commons-math3:3.6.1")
implementation("org.apache.logging.log4j:log4j-core:2.24.3")
implementation("org.apache.logging.log4j:log4j-slf4j2-impl:2.24.3")
implementation("org.mongodb:mongodb-driver-sync:5.2.1")
implementation("org.telegram:telegrambots-client:8.0.0")
implementation("org.telegram:telegrambots-extensions:8.0.0")
implementation("org.telegram:telegrambots-longpolling:8.0.0")
compose.dev.couchbase.yml
-18
View File
@@ -1,18 +0,0 @@
services:
couchbase:
image: couchbase:community-7.6.2
env_file:
- .env
ports:
- "8091-8097:8091-8097"
- "9123:9123"
- "11207:11207"
- "11210:11210"
- "11280:11280"
- "18091-18097:18091-18097"
volumes:
- couchbase_data:/opt/couchbase/var
restart: unless-stopped
volumes:
couchbase_data:
compose.dev.mongodb.yml → compose.dev.yml
View File
src/main/java/com/miti99/storescraperbot/constant/Constant.java
+2
View File
@@ -15,4 +15,6 @@ public class Constant {
public static final ZoneId VIETNAM_ZONE_ID = ZoneId.of(VIETNAM_ZONE_ID_STRING);
public static final long SECONDS_PER_DAY = ChronoUnit.DAYS.getDuration().getSeconds();
public static final Set<DayOfWeek> WEEKENDS = Set.of(DayOfWeek.SATURDAY, DayOfWeek.SUNDAY);
public static final String DEFAULT_DATABASE_NAME = "store-scraper-bot";
}
src/main/java/com/miti99/storescraperbot/env/Environment.java
Vendored
+1 -5
View File
@@ -9,11 +9,7 @@ import java.util.Optional;
import java.util.stream.Collectors;
public class Environment {
public static final String MONGODB_CONNECTION_STRING =
System.getenv("MONGODB_CONNECTION_STRING");
public static final String MONGODB_USERNAME = System.getenv("MONGODB_USERNAME");
public static final String MONGODB_PASSWORD = System.getenv("MONGODB_PASSWORD");
public static final String MONGODB_DATABASE_NAME = System.getenv("MONGODB_DATABASE_NAME");
public static final String MONGODB_CONNECTION_STRING = System.getenv("MONGODB_CONNECTION_STRING");
public static final String TELEGRAM_BOT_TOKEN = System.getenv("TELEGRAM_BOT_TOKEN");
public static final String TELEGRAM_BOT_USERNAME = System.getenv("TELEGRAM_BOT_USERNAME");
src/main/java/com/miti99/storescraperbot/repository/AbstractRepository.java
+5 -18
View File
@@ -3,13 +3,11 @@ package com.miti99.storescraperbot.repository;
import static com.miti99.storescraperbot.repository.AbstractSingletonRepository.COMMON_COLLECTION_NAME;
import com.google.common.base.CaseFormat;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.Filters;
import com.mongodb.client.model.ReplaceOptions;
import com.miti99.storescraperbot.env.Environment;
import com.miti99.storescraperbot.model.AbstractModel;
import com.miti99.storescraperbot.util.MongoDBUtil;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.model.Filters;
import com.mongodb.client.model.ReplaceOptions;
import java.lang.reflect.ParameterizedType;
import lombok.extern.log4j.Log4j2;
@@ -58,13 +56,6 @@ public abstract class AbstractRepository<K, V extends AbstractModel<K>> {
return MongoDBUtil.DATABASE.getCollection(collectionName, classV);
}
/**
* @return expire seconds. 0 mean never expire.
*/
protected long getExpireSeconds() {
return 0;
}
protected void init(K key) {
try {
if (exist(key)) {
@@ -85,12 +76,8 @@ public abstract class AbstractRepository<K, V extends AbstractModel<K>> {
protected void save(K key, V data) {
var databaseKey = getDatabaseKey(key);
try {
var replaceOptions = new ReplaceOptions();
if (getExpireSeconds() > 0) {
// MongoDB TTL indexes need to be created at the collection level
// For now, we'll just save without TTL and handle TTL through indexes
}
collection().replaceOne(Filters.eq("_id", databaseKey), data, replaceOptions);
collection()
.replaceOne(Filters.eq("_id", databaseKey), data, new ReplaceOptions().upsert(true));
} catch (Exception e) {
log.error("save error - key {}, databaseKey {}", key, databaseKey, e);
}
src/main/java/com/miti99/storescraperbot/repository/AbstractSingletonRepository.java
+1 -2
View File
@@ -5,8 +5,7 @@ import lombok.extern.log4j.Log4j2;
/**
* Repository chỉ chứa 1 key duy nhất, public các method liên quan nhưng không cho truyền params
* vào. Các repository loại này được lưu trong 1 collection duy nhất là "common" (do MongoDB không
* giới hạn số lượng collection nên gom nhóm các repository loại này lại để quản lý tập trung)
* vào. Các repository loại này được lưu trong 1 collection duy nhất là "common"
*/
@Log4j2
public abstract class AbstractSingletonRepository<K, V extends AbstractModel<K>>
src/main/java/com/miti99/storescraperbot/repository/AppleAppRepository.java
-12
View File
@@ -1,19 +1,7 @@
package com.miti99.storescraperbot.repository;
import com.miti99.storescraperbot.constant.Constant;
import com.miti99.storescraperbot.model.AppleApp;
import com.miti99.storescraperbot.util.MongoDBUtil;
public class AppleAppRepository extends AbstractCollectionRepository<String, AppleApp> {
public static final AppleAppRepository INSTANCE = new AppleAppRepository();
static {
// Create TTL index for cached data
MongoDBUtil.createTTLIndexIfNotExists("appleapp", "clazz", Constant.APP_CACHE_SECONDS);
}
@Override
protected long getExpireSeconds() {
return Constant.APP_CACHE_SECONDS;
}
}
src/main/java/com/miti99/storescraperbot/repository/GoogleAppRepository.java
-12
View File
@@ -1,19 +1,7 @@
package com.miti99.storescraperbot.repository;
import com.miti99.storescraperbot.constant.Constant;
import com.miti99.storescraperbot.model.GoogleApp;
import com.miti99.storescraperbot.util.MongoDBUtil;
public class GoogleAppRepository extends AbstractCollectionRepository<String, GoogleApp> {
public static final GoogleAppRepository INSTANCE = new GoogleAppRepository();
static {
// Create TTL index for cached data
MongoDBUtil.createTTLIndexIfNotExists("googleapp", "clazz", Constant.APP_CACHE_SECONDS);
}
@Override
protected long getExpireSeconds() {
return Constant.APP_CACHE_SECONDS;
}
}
src/main/java/com/miti99/storescraperbot/util/MongoDBUtil.java
+27 -49
View File
@@ -1,19 +1,20 @@
package com.miti99.storescraperbot.util;
import static com.miti99.storescraperbot.env.Environment.MONGODB_DATABASE_NAME;
import static com.miti99.storescraperbot.env.Environment.MONGODB_CONNECTION_STRING;
import static com.miti99.storescraperbot.env.Environment.MONGODB_USERNAME;
import static com.miti99.storescraperbot.env.Environment.MONGODB_PASSWORD;
import com.miti99.storescraperbot.constant.Constant;
import com.mongodb.ConnectionString;
import com.mongodb.MongoClientSettings;
import com.mongodb.MongoException;
import com.mongodb.ServerApi;
import com.mongodb.ServerApiVersion;
import com.mongodb.client.MongoClient;
import com.mongodb.client.MongoClients;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.Indexes;
import com.mongodb.client.model.CreateIndexOptions;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.bson.Document;
@Log4j2
@NoArgsConstructor(access = AccessLevel.PRIVATE)
@@ -22,26 +23,31 @@ public class MongoDBUtil {
public static final MongoDatabase DATABASE;
static {
String connectionString = MONGODB_CONNECTION_STRING;
String username = MONGODB_USERNAME;
String password = MONGODB_PASSWORD;
String mongoUri;
if (username != null && !username.isEmpty() && password != null && !password.isEmpty()) {
mongoUri = String.format("mongodb://%s:%s@%s", username, password, connectionString);
} else {
mongoUri = connectionString;
var serverApi = ServerApi.builder().version(ServerApiVersion.V1).build();
var connectionString = new ConnectionString(MONGODB_CONNECTION_STRING);
var settings =
MongoClientSettings.builder()
.applyConnectionString(connectionString)
.serverApi(serverApi)
.build();
MONGO_CLIENT = MongoClients.create(settings);
var databaseName =
connectionString.getDatabase() != null
? connectionString.getDatabase()
: Constant.DEFAULT_DATABASE_NAME;
DATABASE = MONGO_CLIENT.getDatabase(databaseName);
try {
DATABASE.runCommand(new Document("ping", 1));
log.info("Pinged your deployment. You successfully connected to MongoDB!");
} catch (MongoException e) {
log.error(e);
}
MONGO_CLIENT = MongoClients.create(mongoUri);
DATABASE = MONGO_CLIENT.getDatabase(MONGODB_DATABASE_NAME);
log.info("MongoDB connection established to database: {}", MONGODB_DATABASE_NAME);
}
public static void createCollectionIfNotExists(String collectionName) {
try {
boolean collectionExists = false;
for (String name : DATABASE.listCollectionNames()) {
for (var name : DATABASE.listCollectionNames()) {
if (name.equals(collectionName)) {
collectionExists = true;
break;
@@ -58,32 +64,4 @@ public class MongoDBUtil {
log.error("createCollectionIfNotExists error - collectionName: '{}'", collectionName, e);
}
}
public static void createTTLIndexIfNotExists(String collectionName, String fieldName, long expireAfterSeconds) {
try {
MongoCollection<?> collection = DATABASE.getCollection(collectionName);
// Check if TTL index already exists
boolean indexExists = false;
for (var index : collection.listIndexes()) {
String indexOptions = index.toJson();
if (indexOptions.contains("\"expireAfterSeconds\": " + expireAfterSeconds)) {
indexExists = true;
break;
}
}
if (!indexExists) {
CreateIndexOptions options = new CreateIndexOptions().expireAfter(expireAfterSeconds, java.util.concurrent.TimeUnit.SECONDS);
collection.createIndex(Indexes.descending(fieldName), options);
log.info("TTL index created on {} in collection {} with expire time: {} seconds",
fieldName, collectionName, expireAfterSeconds);
} else {
log.info("TTL index already existed on {} in collection {}", fieldName, collectionName);
}
} catch (Exception e) {
log.error("createTTLIndexIfNotExists error - collectionName: '{}', fieldName: '{}'",
collectionName, fieldName, e);
}
}
}
}