diff --git a/.env.example b/.env.example index 7656eb7..f04f2ed 100644 --- a/.env.example +++ b/.env.example @@ -1,11 +1,11 @@ # Copy this file to .env and customize the values for your environment # cp .env.example .env -# Couchbase configuration -COUCHBASE_CONNECTION_STRING=couchbase://localhost -COUCHBASE_USERNAME=your_couchbase_username -COUCHBASE_PASSWORD=your_couchbase_password -COUCHBASE_BUCKET_NAME=store-scraper-bot +# MongoDB configuration +MONGODB_CONNECTION_STRING=mongodb://localhost:27017 +MONGODB_USERNAME=your_mongodb_username +MONGODB_PASSWORD=your_mongodb_password +MONGODB_DATABASE_NAME=store-scraper-bot # Telegram Bot configuration TELEGRAM_BOT_TOKEN=your_telegram_bot_token diff --git a/build.gradle.kts b/build.gradle.kts index 92c29bc..f8658f6 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -22,7 +22,7 @@ configurations { dependencies { annotationProcessor("org.projectlombok:lombok:1.18.36") - implementation("com.couchbase.client:java-client:3.7.6") + implementation("org.mongodb:mongodb-driver-sync:5.2.1") implementation("com.fasterxml.jackson.core:jackson-databind:2.18.2") implementation("com.google.code.gson:gson:2.11.0") implementation("com.google.guava:guava:33.4.0-jre") diff --git a/compose.dev.yml b/compose.dev.couchbase.yml similarity index 100% rename from compose.dev.yml rename to compose.dev.couchbase.yml diff --git a/compose.dev.mongodb.yml b/compose.dev.mongodb.yml new file mode 100644 index 0000000..45a9979 --- /dev/null +++ b/compose.dev.mongodb.yml @@ -0,0 +1,13 @@ +services: + mongodb: + image: mongo:7.0 + env_file: + - .env + ports: + - "27017:27017" + volumes: + - mongodb_data:/data/db + restart: unless-stopped + +volumes: + mongodb_data: \ No newline at end of file diff --git a/src/main/java/com/miti99/storescraperbot/env/Environment.java b/src/main/java/com/miti99/storescraperbot/env/Environment.java index ee82cf3..1fbe2dc 100644 --- a/src/main/java/com/miti99/storescraperbot/env/Environment.java +++ b/src/main/java/com/miti99/storescraperbot/env/Environment.java @@ -9,11 +9,11 @@ import java.util.Optional; import java.util.stream.Collectors; public class Environment { - public static final String COUCHBASE_CONNECTION_STRING = - System.getenv("COUCHBASE_CONNECTION_STRING"); - public static final String COUCHBASE_USERNAME = System.getenv("COUCHBASE_USERNAME"); - public static final String COUCHBASE_PASSWORD = System.getenv("COUCHBASE_PASSWORD"); - public static final String COUCHBASE_BUCKET_NAME = System.getenv("COUCHBASE_BUCKET_NAME"); + public static final String MONGODB_CONNECTION_STRING = + System.getenv("MONGODB_CONNECTION_STRING"); + public static final String MONGODB_USERNAME = System.getenv("MONGODB_USERNAME"); + public static final String MONGODB_PASSWORD = System.getenv("MONGODB_PASSWORD"); + public static final String MONGODB_DATABASE_NAME = System.getenv("MONGODB_DATABASE_NAME"); public static final String TELEGRAM_BOT_TOKEN = System.getenv("TELEGRAM_BOT_TOKEN"); public static final String TELEGRAM_BOT_USERNAME = System.getenv("TELEGRAM_BOT_USERNAME"); diff --git a/src/main/java/com/miti99/storescraperbot/model/AbstractModel.java b/src/main/java/com/miti99/storescraperbot/model/AbstractModel.java index 7e56d62..e99fd54 100644 --- a/src/main/java/com/miti99/storescraperbot/model/AbstractModel.java +++ b/src/main/java/com/miti99/storescraperbot/model/AbstractModel.java @@ -1,6 +1,8 @@ package com.miti99.storescraperbot.model; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSetter; +import com.fasterxml.jackson.annotation.Nulls; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -9,6 +11,7 @@ import lombok.Setter; @NoArgsConstructor @Setter public abstract class AbstractModel { + @JsonSetter(nulls = Nulls.SKIP) protected K key; @JsonProperty("class") diff --git a/src/main/java/com/miti99/storescraperbot/repository/AbstractCollectionRepository.java b/src/main/java/com/miti99/storescraperbot/repository/AbstractCollectionRepository.java index 780079e..9887367 100644 --- a/src/main/java/com/miti99/storescraperbot/repository/AbstractCollectionRepository.java +++ b/src/main/java/com/miti99/storescraperbot/repository/AbstractCollectionRepository.java @@ -4,7 +4,7 @@ import com.miti99.storescraperbot.model.AbstractModel; import lombok.extern.log4j.Log4j2; /** - * Các repository tương ứng với 1 Couchbase collection, public các method protected ở class + * Các repository tương ứng với 1 MongoDB collection, public các method protected ở class * AbstractRepository để các nơi khác gọi */ @Log4j2 diff --git a/src/main/java/com/miti99/storescraperbot/repository/AbstractRepository.java b/src/main/java/com/miti99/storescraperbot/repository/AbstractRepository.java index 330c3e1..dc286f4 100644 --- a/src/main/java/com/miti99/storescraperbot/repository/AbstractRepository.java +++ b/src/main/java/com/miti99/storescraperbot/repository/AbstractRepository.java @@ -2,14 +2,15 @@ package com.miti99.storescraperbot.repository; import static com.miti99.storescraperbot.repository.AbstractSingletonRepository.COMMON_COLLECTION_NAME; -import com.couchbase.client.java.Collection; -import com.couchbase.client.java.kv.UpsertOptions; import com.google.common.base.CaseFormat; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.ReplaceOptions; import com.miti99.storescraperbot.env.Environment; import com.miti99.storescraperbot.model.AbstractModel; -import com.miti99.storescraperbot.util.CouchbaseUtil; +import com.miti99.storescraperbot.util.MongoDBUtil; import java.lang.reflect.ParameterizedType; -import java.time.Duration; import lombok.extern.log4j.Log4j2; /** @@ -21,12 +22,11 @@ public abstract class AbstractRepository> { protected static final String SEPARATOR = "_"; // protected final Class classK = getKeyClass(); protected final Class classV = getDataClass(); - protected final String scopeName = Environment.ENV.name().toLowerCase(); protected final String collectionName; protected AbstractRepository(String collectionName) { this.collectionName = collectionName; - CouchbaseUtil.createCollection(scopeName, collectionName); + MongoDBUtil.createCollectionIfNotExists(collectionName); } protected AbstractRepository() { @@ -35,7 +35,7 @@ public abstract class AbstractRepository> { throw new RuntimeException( "Collection named '%s' is reserved".formatted(COMMON_COLLECTION_NAME)); } - CouchbaseUtil.createCollection(scopeName, collectionName); + MongoDBUtil.createCollectionIfNotExists(collectionName); } // protected Class getKeyClass() { @@ -54,8 +54,8 @@ public abstract class AbstractRepository> { ((ParameterizedType) getClass().getGenericSuperclass()).getActualTypeArguments()[1]; } - protected Collection collection() { - return CouchbaseUtil.BUCKET.scope(scopeName).collection(collectionName); + protected MongoCollection collection() { + return MongoDBUtil.DATABASE.getCollection(collectionName, classV); } /** @@ -85,13 +85,12 @@ public abstract class AbstractRepository> { protected void save(K key, V data) { var databaseKey = getDatabaseKey(key); try { - if (getExpireSeconds() == 0) { - collection().upsert(databaseKey, data); - } else { - var upsertOptions = - UpsertOptions.upsertOptions().expiry(Duration.ofSeconds(getExpireSeconds())); - collection().upsert(databaseKey, data, upsertOptions); + var replaceOptions = new ReplaceOptions(); + if (getExpireSeconds() > 0) { + // MongoDB TTL indexes need to be created at the collection level + // For now, we'll just save without TTL and handle TTL through indexes } + collection().replaceOne(Filters.eq("_id", databaseKey), data, replaceOptions); } catch (Exception e) { log.error("save error - key {}, databaseKey {}", key, databaseKey, e); } @@ -100,7 +99,7 @@ public abstract class AbstractRepository> { protected boolean exist(K key) { var databaseKey = getDatabaseKey(key); try { - return collection().exists(databaseKey).exists(); + return collection().countDocuments(Filters.eq("_id", databaseKey)) > 0; } catch (Exception e) { log.error("exist error - key {}, databaseKey {}", key, databaseKey, e); return false; @@ -110,11 +109,7 @@ public abstract class AbstractRepository> { protected V load(K key) { var databaseKey = getDatabaseKey(key); try { - var getResult = collection().get(databaseKey); - if (getResult == null) { - return null; - } - return getResult.contentAs(classV); + return collection().find(Filters.eq("_id", databaseKey)).first(); } catch (Exception e) { log.error("load error - key {}, databaseKey {}", key, databaseKey, e); return null; @@ -124,7 +119,7 @@ public abstract class AbstractRepository> { protected void delete(K key) { var databaseKey = getDatabaseKey(key); try { - collection().remove(databaseKey); + collection().deleteOne(Filters.eq("_id", databaseKey)); } catch (Exception e) { log.error("delete error", e); } diff --git a/src/main/java/com/miti99/storescraperbot/repository/AbstractSingletonRepository.java b/src/main/java/com/miti99/storescraperbot/repository/AbstractSingletonRepository.java index a728a75..94877b6 100644 --- a/src/main/java/com/miti99/storescraperbot/repository/AbstractSingletonRepository.java +++ b/src/main/java/com/miti99/storescraperbot/repository/AbstractSingletonRepository.java @@ -5,8 +5,8 @@ import lombok.extern.log4j.Log4j2; /** * Repository chỉ chứa 1 key duy nhất, public các method liên quan nhưng không cho truyền params - * vào. Các repository loại này được lưu trong 1 collection duy nhất là "common" (do Couchbase có - * giới hạn số lượng collection mỗi cluster nên gom nhóm các repository loại này lại) + * vào. Các repository loại này được lưu trong 1 collection duy nhất là "common" (do MongoDB không + * giới hạn số lượng collection nên gom nhóm các repository loại này lại để quản lý tập trung) */ @Log4j2 public abstract class AbstractSingletonRepository> diff --git a/src/main/java/com/miti99/storescraperbot/repository/AppleAppRepository.java b/src/main/java/com/miti99/storescraperbot/repository/AppleAppRepository.java index 6309e88..4a9c897 100644 --- a/src/main/java/com/miti99/storescraperbot/repository/AppleAppRepository.java +++ b/src/main/java/com/miti99/storescraperbot/repository/AppleAppRepository.java @@ -2,10 +2,16 @@ package com.miti99.storescraperbot.repository; import com.miti99.storescraperbot.constant.Constant; import com.miti99.storescraperbot.model.AppleApp; +import com.miti99.storescraperbot.util.MongoDBUtil; public class AppleAppRepository extends AbstractCollectionRepository { public static final AppleAppRepository INSTANCE = new AppleAppRepository(); + static { + // Create TTL index for cached data + MongoDBUtil.createTTLIndexIfNotExists("appleapp", "clazz", Constant.APP_CACHE_SECONDS); + } + @Override protected long getExpireSeconds() { return Constant.APP_CACHE_SECONDS; diff --git a/src/main/java/com/miti99/storescraperbot/repository/GoogleAppRepository.java b/src/main/java/com/miti99/storescraperbot/repository/GoogleAppRepository.java index 707bf04..5d906a9 100644 --- a/src/main/java/com/miti99/storescraperbot/repository/GoogleAppRepository.java +++ b/src/main/java/com/miti99/storescraperbot/repository/GoogleAppRepository.java @@ -2,10 +2,16 @@ package com.miti99.storescraperbot.repository; import com.miti99.storescraperbot.constant.Constant; import com.miti99.storescraperbot.model.GoogleApp; +import com.miti99.storescraperbot.util.MongoDBUtil; public class GoogleAppRepository extends AbstractCollectionRepository { public static final GoogleAppRepository INSTANCE = new GoogleAppRepository(); + static { + // Create TTL index for cached data + MongoDBUtil.createTTLIndexIfNotExists("googleapp", "clazz", Constant.APP_CACHE_SECONDS); + } + @Override protected long getExpireSeconds() { return Constant.APP_CACHE_SECONDS; diff --git a/src/main/java/com/miti99/storescraperbot/util/CouchbaseUtil.java b/src/main/java/com/miti99/storescraperbot/util/CouchbaseUtil.java deleted file mode 100644 index 61c93d7..0000000 --- a/src/main/java/com/miti99/storescraperbot/util/CouchbaseUtil.java +++ /dev/null @@ -1,81 +0,0 @@ -package com.miti99.storescraperbot.util; - -import static com.miti99.storescraperbot.env.Environment.COUCHBASE_BUCKET_NAME; -import static com.miti99.storescraperbot.env.Environment.COUCHBASE_CONNECTION_STRING; -import static com.miti99.storescraperbot.env.Environment.COUCHBASE_PASSWORD; -import static com.miti99.storescraperbot.env.Environment.COUCHBASE_USERNAME; - -import com.couchbase.client.java.Bucket; -import com.couchbase.client.java.Cluster; -import com.couchbase.client.java.ClusterOptions; -import java.time.Duration; -import lombok.AccessLevel; -import lombok.NoArgsConstructor; -import lombok.extern.log4j.Log4j2; - -@Log4j2 -@NoArgsConstructor(access = AccessLevel.PRIVATE) -public class CouchbaseUtil { - public static final Cluster CLUSTER; - public static final Bucket BUCKET; - - static { - CLUSTER = - Cluster.connect( - COUCHBASE_CONNECTION_STRING, - ClusterOptions.clusterOptions(COUCHBASE_USERNAME, COUCHBASE_PASSWORD) - .environment(env -> {})); - - BUCKET = CLUSTER.bucket(COUCHBASE_BUCKET_NAME); - BUCKET.waitUntilReady(Duration.ofSeconds(10)); - } - - public static void createScope(String scopeName) { - var collectionManager = BUCKET.collections(); - try { - boolean scopeExists = - collectionManager.getAllScopes().stream().anyMatch(s -> s.name().equals(scopeName)); - - if (!scopeExists) { - collectionManager.createScope(scopeName); - log.info("Scope created: {}", scopeName); - } else { - log.info("Scope existed: {}", scopeName); - } - } catch (Exception e) { - log.error("createScope error - scopeName: '{}'", scopeName, e); - } - } - - public static void createCollection(String scopeName, String collectionName) { - var collectionManager = BUCKET.collections(); - try { - var scopeSpecOpt = - collectionManager.getAllScopes().stream() - .filter(s -> s.name().equals(scopeName)) - .findFirst(); - if (scopeSpecOpt.isEmpty()) { - createScope(scopeName); - createCollection(scopeName, collectionName); - return; - } - - var scopeSpec = scopeSpecOpt.get(); - boolean collectionExists = - scopeSpec.collections().stream().anyMatch(c -> c.name().equals(collectionName)); - - if (!collectionExists) { - collectionManager.createCollection(scopeName, collectionName); - log.info("Collection created: {} in {}", collectionName, scopeName); - } else { - log.info("Collection existed: {} in {}", collectionName, scopeName); - } - } catch (Exception e) { - log.error( - "createCollection error - collectionName: '{}', scopeName: '{}'", - collectionName, - scopeName, - e); - } - } -} diff --git a/src/main/java/com/miti99/storescraperbot/util/MongoDBUtil.java b/src/main/java/com/miti99/storescraperbot/util/MongoDBUtil.java new file mode 100644 index 0000000..4fa0fb5 --- /dev/null +++ b/src/main/java/com/miti99/storescraperbot/util/MongoDBUtil.java @@ -0,0 +1,89 @@ +package com.miti99.storescraperbot.util; + +import static com.miti99.storescraperbot.env.Environment.MONGODB_DATABASE_NAME; +import static com.miti99.storescraperbot.env.Environment.MONGODB_CONNECTION_STRING; +import static com.miti99.storescraperbot.env.Environment.MONGODB_USERNAME; +import static com.miti99.storescraperbot.env.Environment.MONGODB_PASSWORD; + +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.mongodb.client.MongoCollection; +import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.Indexes; +import com.mongodb.client.model.CreateIndexOptions; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.extern.log4j.Log4j2; + +@Log4j2 +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class MongoDBUtil { + public static final MongoClient MONGO_CLIENT; + public static final MongoDatabase DATABASE; + + static { + String connectionString = MONGODB_CONNECTION_STRING; + String username = MONGODB_USERNAME; + String password = MONGODB_PASSWORD; + + String mongoUri; + if (username != null && !username.isEmpty() && password != null && !password.isEmpty()) { + mongoUri = String.format("mongodb://%s:%s@%s", username, password, connectionString); + } else { + mongoUri = connectionString; + } + + MONGO_CLIENT = MongoClients.create(mongoUri); + DATABASE = MONGO_CLIENT.getDatabase(MONGODB_DATABASE_NAME); + log.info("MongoDB connection established to database: {}", MONGODB_DATABASE_NAME); + } + + public static void createCollectionIfNotExists(String collectionName) { + try { + boolean collectionExists = false; + for (String name : DATABASE.listCollectionNames()) { + if (name.equals(collectionName)) { + collectionExists = true; + break; + } + } + + if (!collectionExists) { + DATABASE.createCollection(collectionName); + log.info("Collection created: {}", collectionName); + } else { + log.info("Collection existed: {}", collectionName); + } + } catch (Exception e) { + log.error("createCollectionIfNotExists error - collectionName: '{}'", collectionName, e); + } + } + + public static void createTTLIndexIfNotExists(String collectionName, String fieldName, long expireAfterSeconds) { + try { + MongoCollection collection = DATABASE.getCollection(collectionName); + + // Check if TTL index already exists + boolean indexExists = false; + for (var index : collection.listIndexes()) { + String indexOptions = index.toJson(); + if (indexOptions.contains("\"expireAfterSeconds\": " + expireAfterSeconds)) { + indexExists = true; + break; + } + } + + if (!indexExists) { + CreateIndexOptions options = new CreateIndexOptions().expireAfter(expireAfterSeconds, java.util.concurrent.TimeUnit.SECONDS); + collection.createIndex(Indexes.descending(fieldName), options); + log.info("TTL index created on {} in collection {} with expire time: {} seconds", + fieldName, collectionName, expireAfterSeconds); + } else { + log.info("TTL index already existed on {} in collection {}", fieldName, collectionName); + } + } catch (Exception e) { + log.error("createTTLIndexIfNotExists error - collectionName: '{}', fieldName: '{}'", + collectionName, fieldName, e); + } + } +} \ No newline at end of file