feat(db): use mongodb instead of couchbase (WIP)

.env.example

@@ -1,11 +1,11 @@
 # Copy this file to .env and customize the values for your environment
 # cp .env.example .env
 
-# Couchbase configuration
-COUCHBASE_CONNECTION_STRING=couchbase://localhost
-COUCHBASE_USERNAME=your_couchbase_username
-COUCHBASE_PASSWORD=your_couchbase_password
-COUCHBASE_BUCKET_NAME=store-scraper-bot
+# MongoDB configuration
+MONGODB_CONNECTION_STRING=mongodb://localhost:27017
+MONGODB_USERNAME=your_mongodb_username
+MONGODB_PASSWORD=your_mongodb_password
+MONGODB_DATABASE_NAME=store-scraper-bot
 
 # Telegram Bot configuration
 TELEGRAM_BOT_TOKEN=your_telegram_bot_token

build.gradle.kts

@@ -22,7 +22,7 @@ configurations {
 
 dependencies {
     annotationProcessor("org.projectlombok:lombok:1.18.36")
-    implementation("com.couchbase.client:java-client:3.7.6")
+    implementation("org.mongodb:mongodb-driver-sync:5.2.1")
     implementation("com.fasterxml.jackson.core:jackson-databind:2.18.2")
     implementation("com.google.code.gson:gson:2.11.0")
     implementation("com.google.guava:guava:33.4.0-jre")

compose.dev.mongodb.yml

@@ -0,0 +1,13 @@
+services:
+  mongodb:
+    image: mongo:7.0
+    env_file:
+      - .env
+    ports:
+      - "27017:27017"
+    volumes:
+      - mongodb_data:/data/db
+    restart: unless-stopped
+
+volumes:
+  mongodb_data:

src/main/java/com/miti99/storescraperbot/env/Environment.java

@@ -9,11 +9,11 @@ import java.util.Optional;
 import java.util.stream.Collectors;
 
 public class Environment {
-  public static final String COUCHBASE_CONNECTION_STRING =
-      System.getenv("COUCHBASE_CONNECTION_STRING");
-  public static final String COUCHBASE_USERNAME = System.getenv("COUCHBASE_USERNAME");
-  public static final String COUCHBASE_PASSWORD = System.getenv("COUCHBASE_PASSWORD");
-  public static final String COUCHBASE_BUCKET_NAME = System.getenv("COUCHBASE_BUCKET_NAME");
+  public static final String MONGODB_CONNECTION_STRING =
+      System.getenv("MONGODB_CONNECTION_STRING");
+  public static final String MONGODB_USERNAME = System.getenv("MONGODB_USERNAME");
+  public static final String MONGODB_PASSWORD = System.getenv("MONGODB_PASSWORD");
+  public static final String MONGODB_DATABASE_NAME = System.getenv("MONGODB_DATABASE_NAME");
 
   public static final String TELEGRAM_BOT_TOKEN = System.getenv("TELEGRAM_BOT_TOKEN");
   public static final String TELEGRAM_BOT_USERNAME = System.getenv("TELEGRAM_BOT_USERNAME");

src/main/java/com/miti99/storescraperbot/model/AbstractModel.java

@@ -1,6 +1,8 @@
 package com.miti99.storescraperbot.model;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonSetter;
+import com.fasterxml.jackson.annotation.Nulls;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
@@ -9,6 +11,7 @@ import lombok.Setter;
 @NoArgsConstructor
 @Setter
 public abstract class AbstractModel<K> {
+  @JsonSetter(nulls = Nulls.SKIP)
   protected K key;
 
   @JsonProperty("class")

src/main/java/com/miti99/storescraperbot/repository/AbstractCollectionRepository.java

@@ -4,7 +4,7 @@ import com.miti99.storescraperbot.model.AbstractModel;
 import lombok.extern.log4j.Log4j2;
 
 /**
- * Các repository tương ứng với 1 Couchbase collection, public các method protected ở class
+ * Các repository tương ứng với 1 MongoDB collection, public các method protected ở class
  * AbstractRepository để các nơi khác gọi
  */
 @Log4j2

src/main/java/com/miti99/storescraperbot/repository/AbstractRepository.java

@@ -2,14 +2,15 @@ package com.miti99.storescraperbot.repository;
 
 import static com.miti99.storescraperbot.repository.AbstractSingletonRepository.COMMON_COLLECTION_NAME;
 
-import com.couchbase.client.java.Collection;
-import com.couchbase.client.java.kv.UpsertOptions;
 import com.google.common.base.CaseFormat;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.model.Filters;
+import com.mongodb.client.model.ReplaceOptions;
 import com.miti99.storescraperbot.env.Environment;
 import com.miti99.storescraperbot.model.AbstractModel;
-import com.miti99.storescraperbot.util.CouchbaseUtil;
+import com.miti99.storescraperbot.util.MongoDBUtil;
 import java.lang.reflect.ParameterizedType;
-import java.time.Duration;
 import lombok.extern.log4j.Log4j2;
 
 /**
@@ -21,12 +22,11 @@ public abstract class AbstractRepository<K, V extends AbstractModel<K>> {
   protected static final String SEPARATOR = "_";
   // protected final Class<K> classK = getKeyClass();
   protected final Class<V> classV = getDataClass();
-  protected final String scopeName = Environment.ENV.name().toLowerCase();
   protected final String collectionName;
 
   protected AbstractRepository(String collectionName) {
     this.collectionName = collectionName;
-    CouchbaseUtil.createCollection(scopeName, collectionName);
+    MongoDBUtil.createCollectionIfNotExists(collectionName);
   }
 
   protected AbstractRepository() {
@@ -35,7 +35,7 @@ public abstract class AbstractRepository<K, V extends AbstractModel<K>> {
       throw new RuntimeException(
           "Collection named '%s' is reserved".formatted(COMMON_COLLECTION_NAME));
     }
-    CouchbaseUtil.createCollection(scopeName, collectionName);
+    MongoDBUtil.createCollectionIfNotExists(collectionName);
   }
 
   // protected Class<K> getKeyClass() {
@@ -54,8 +54,8 @@ public abstract class AbstractRepository<K, V extends AbstractModel<K>> {
         ((ParameterizedType) getClass().getGenericSuperclass()).getActualTypeArguments()[1];
   }
 
-  protected Collection collection() {
-    return CouchbaseUtil.BUCKET.scope(scopeName).collection(collectionName);
+  protected MongoCollection<V> collection() {
+    return MongoDBUtil.DATABASE.getCollection(collectionName, classV);
   }
 
   /**
@@ -85,13 +85,12 @@ public abstract class AbstractRepository<K, V extends AbstractModel<K>> {
   protected void save(K key, V data) {
     var databaseKey = getDatabaseKey(key);
     try {
-      if (getExpireSeconds() == 0) {
-        collection().upsert(databaseKey, data);
-      } else {
-        var upsertOptions =
-            UpsertOptions.upsertOptions().expiry(Duration.ofSeconds(getExpireSeconds()));
-        collection().upsert(databaseKey, data, upsertOptions);
+      var replaceOptions = new ReplaceOptions();
+      if (getExpireSeconds() > 0) {
+        // MongoDB TTL indexes need to be created at the collection level
+        // For now, we'll just save without TTL and handle TTL through indexes
       }
+      collection().replaceOne(Filters.eq("_id", databaseKey), data, replaceOptions);
     } catch (Exception e) {
       log.error("save error - key {}, databaseKey {}", key, databaseKey, e);
     }
@@ -100,7 +99,7 @@ public abstract class AbstractRepository<K, V extends AbstractModel<K>> {
   protected boolean exist(K key) {
     var databaseKey = getDatabaseKey(key);
     try {
-      return collection().exists(databaseKey).exists();
+      return collection().countDocuments(Filters.eq("_id", databaseKey)) > 0;
     } catch (Exception e) {
       log.error("exist error - key {}, databaseKey {}", key, databaseKey, e);
       return false;
@@ -110,11 +109,7 @@ public abstract class AbstractRepository<K, V extends AbstractModel<K>> {
   protected V load(K key) {
     var databaseKey = getDatabaseKey(key);
     try {
-      var getResult = collection().get(databaseKey);
-      if (getResult == null) {
-        return null;
-      }
-      return getResult.contentAs(classV);
+      return collection().find(Filters.eq("_id", databaseKey)).first();
     } catch (Exception e) {
       log.error("load error - key {}, databaseKey {}", key, databaseKey, e);
       return null;
@@ -124,7 +119,7 @@ public abstract class AbstractRepository<K, V extends AbstractModel<K>> {
   protected void delete(K key) {
     var databaseKey = getDatabaseKey(key);
     try {
-      collection().remove(databaseKey);
+      collection().deleteOne(Filters.eq("_id", databaseKey));
     } catch (Exception e) {
       log.error("delete error", e);
     }

src/main/java/com/miti99/storescraperbot/repository/AbstractSingletonRepository.java

@@ -5,8 +5,8 @@ import lombok.extern.log4j.Log4j2;
 
 /**
  * Repository chỉ chứa 1 key duy nhất, public các method liên quan nhưng không cho truyền params
- * vào. Các repository loại này được lưu trong 1 collection duy nhất là "common" (do Couchbase có
- * giới hạn số lượng collection mỗi cluster nên gom nhóm các repository loại này lại)
+ * vào. Các repository loại này được lưu trong 1 collection duy nhất là "common" (do MongoDB không
+ * giới hạn số lượng collection nên gom nhóm các repository loại này lại để quản lý tập trung)
  */
 @Log4j2
 public abstract class AbstractSingletonRepository<K, V extends AbstractModel<K>>

src/main/java/com/miti99/storescraperbot/repository/AppleAppRepository.java

@@ -2,10 +2,16 @@ package com.miti99.storescraperbot.repository;
 
 import com.miti99.storescraperbot.constant.Constant;
 import com.miti99.storescraperbot.model.AppleApp;
+import com.miti99.storescraperbot.util.MongoDBUtil;
 
 public class AppleAppRepository extends AbstractCollectionRepository<String, AppleApp> {
   public static final AppleAppRepository INSTANCE = new AppleAppRepository();
 
+  static {
+    // Create TTL index for cached data
+    MongoDBUtil.createTTLIndexIfNotExists("appleapp", "clazz", Constant.APP_CACHE_SECONDS);
+  }
+
   @Override
   protected long getExpireSeconds() {
     return Constant.APP_CACHE_SECONDS;

src/main/java/com/miti99/storescraperbot/repository/GoogleAppRepository.java

@@ -2,10 +2,16 @@ package com.miti99.storescraperbot.repository;
 
 import com.miti99.storescraperbot.constant.Constant;
 import com.miti99.storescraperbot.model.GoogleApp;
+import com.miti99.storescraperbot.util.MongoDBUtil;
 
 public class GoogleAppRepository extends AbstractCollectionRepository<String, GoogleApp> {
   public static final GoogleAppRepository INSTANCE = new GoogleAppRepository();
 
+  static {
+    // Create TTL index for cached data
+    MongoDBUtil.createTTLIndexIfNotExists("googleapp", "clazz", Constant.APP_CACHE_SECONDS);
+  }
+
   @Override
   protected long getExpireSeconds() {
     return Constant.APP_CACHE_SECONDS;

src/main/java/com/miti99/storescraperbot/util/CouchbaseUtil.java

@@ -1,81 +0,0 @@
-package com.miti99.storescraperbot.util;
-
-import static com.miti99.storescraperbot.env.Environment.COUCHBASE_BUCKET_NAME;
-import static com.miti99.storescraperbot.env.Environment.COUCHBASE_CONNECTION_STRING;
-import static com.miti99.storescraperbot.env.Environment.COUCHBASE_PASSWORD;
-import static com.miti99.storescraperbot.env.Environment.COUCHBASE_USERNAME;
-
-import com.couchbase.client.java.Bucket;
-import com.couchbase.client.java.Cluster;
-import com.couchbase.client.java.ClusterOptions;
-import java.time.Duration;
-import lombok.AccessLevel;
-import lombok.NoArgsConstructor;
-import lombok.extern.log4j.Log4j2;
-
-@Log4j2
-@NoArgsConstructor(access = AccessLevel.PRIVATE)
-public class CouchbaseUtil {
-  public static final Cluster CLUSTER;
-  public static final Bucket BUCKET;
-
-  static {
-    CLUSTER =
-        Cluster.connect(
-            COUCHBASE_CONNECTION_STRING,
-            ClusterOptions.clusterOptions(COUCHBASE_USERNAME, COUCHBASE_PASSWORD)
-                .environment(env -> {}));
-
-    BUCKET = CLUSTER.bucket(COUCHBASE_BUCKET_NAME);
-    BUCKET.waitUntilReady(Duration.ofSeconds(10));
-  }
-
-  public static void createScope(String scopeName) {
-    var collectionManager = BUCKET.collections();
-    try {
-      boolean scopeExists =
-          collectionManager.getAllScopes().stream().anyMatch(s -> s.name().equals(scopeName));
-
-      if (!scopeExists) {
-        collectionManager.createScope(scopeName);
-        log.info("Scope created: {}", scopeName);
-      } else {
-        log.info("Scope existed: {}", scopeName);
-      }
-    } catch (Exception e) {
-      log.error("createScope error - scopeName: '{}'", scopeName, e);
-    }
-  }
-
-  public static void createCollection(String scopeName, String collectionName) {
-    var collectionManager = BUCKET.collections();
-    try {
-      var scopeSpecOpt =
-          collectionManager.getAllScopes().stream()
-              .filter(s -> s.name().equals(scopeName))
-              .findFirst();
-      if (scopeSpecOpt.isEmpty()) {
-        createScope(scopeName);
-        createCollection(scopeName, collectionName);
-        return;
-      }
-
-      var scopeSpec = scopeSpecOpt.get();
-      boolean collectionExists =
-          scopeSpec.collections().stream().anyMatch(c -> c.name().equals(collectionName));
-
-      if (!collectionExists) {
-        collectionManager.createCollection(scopeName, collectionName);
-        log.info("Collection created: {} in {}", collectionName, scopeName);
-      } else {
-        log.info("Collection existed: {} in {}", collectionName, scopeName);
-      }
-    } catch (Exception e) {
-      log.error(
-          "createCollection error - collectionName: '{}', scopeName: '{}'",
-          collectionName,
-          scopeName,
-          e);
-    }
-  }
-}

src/main/java/com/miti99/storescraperbot/util/MongoDBUtil.java

@@ -0,0 +1,89 @@
+package com.miti99.storescraperbot.util;
+
+import static com.miti99.storescraperbot.env.Environment.MONGODB_DATABASE_NAME;
+import static com.miti99.storescraperbot.env.Environment.MONGODB_CONNECTION_STRING;
+import static com.miti99.storescraperbot.env.Environment.MONGODB_USERNAME;
+import static com.miti99.storescraperbot.env.Environment.MONGODB_PASSWORD;
+
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.model.Indexes;
+import com.mongodb.client.model.CreateIndexOptions;
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+
+@Log4j2
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class MongoDBUtil {
+  public static final MongoClient MONGO_CLIENT;
+  public static final MongoDatabase DATABASE;
+
+  static {
+    String connectionString = MONGODB_CONNECTION_STRING;
+    String username = MONGODB_USERNAME;
+    String password = MONGODB_PASSWORD;
+
+    String mongoUri;
+    if (username != null && !username.isEmpty() && password != null && !password.isEmpty()) {
+      mongoUri = String.format("mongodb://%s:%s@%s", username, password, connectionString);
+    } else {
+      mongoUri = connectionString;
+    }
+
+    MONGO_CLIENT = MongoClients.create(mongoUri);
+    DATABASE = MONGO_CLIENT.getDatabase(MONGODB_DATABASE_NAME);
+    log.info("MongoDB connection established to database: {}", MONGODB_DATABASE_NAME);
+  }
+
+  public static void createCollectionIfNotExists(String collectionName) {
+    try {
+      boolean collectionExists = false;
+      for (String name : DATABASE.listCollectionNames()) {
+        if (name.equals(collectionName)) {
+          collectionExists = true;
+          break;
+        }
+      }
+
+      if (!collectionExists) {
+        DATABASE.createCollection(collectionName);
+        log.info("Collection created: {}", collectionName);
+      } else {
+        log.info("Collection existed: {}", collectionName);
+      }
+    } catch (Exception e) {
+      log.error("createCollectionIfNotExists error - collectionName: '{}'", collectionName, e);
+    }
+  }
+
+  public static void createTTLIndexIfNotExists(String collectionName, String fieldName, long expireAfterSeconds) {
+    try {
+      MongoCollection<?> collection = DATABASE.getCollection(collectionName);
+
+      // Check if TTL index already exists
+      boolean indexExists = false;
+      for (var index : collection.listIndexes()) {
+        String indexOptions = index.toJson();
+        if (indexOptions.contains("\"expireAfterSeconds\": " + expireAfterSeconds)) {
+          indexExists = true;
+          break;
+        }
+      }
+
+      if (!indexExists) {
+        CreateIndexOptions options = new CreateIndexOptions().expireAfter(expireAfterSeconds, java.util.concurrent.TimeUnit.SECONDS);
+        collection.createIndex(Indexes.descending(fieldName), options);
+        log.info("TTL index created on {} in collection {} with expire time: {} seconds",
+                fieldName, collectionName, expireAfterSeconds);
+      } else {
+        log.info("TTL index already existed on {} in collection {}", fieldName, collectionName);
+      }
+    } catch (Exception e) {
+      log.error("createTTLIndexIfNotExists error - collectionName: '{}', fieldName: '{}'",
+               collectionName, fieldName, e);
+    }
+  }
+}