From dac6a82b7107f8433dec227e619ebb9c25bd7fb2 Mon Sep 17 00:00:00 2001 From: tiennm99 Date: Thu, 6 Nov 2025 20:30:16 +0700 Subject: [PATCH] Migrate from Redis to Couchbase for data storage Replaces Redis with Couchbase as the primary data store. Updates environment variables, dependencies, and Docker Compose configuration to use Couchbase. Refactors repository and model classes to use Couchbase APIs, removes Redis utility and related code, and adds Couchbase utility for collection management. Updates AdminRepository to use 'admin' as the key instead of an empty string. --- .env.example | 7 +- build.gradle.kts | 3 +- compose.dev.yml | 18 ++-- compose.yml | 2 +- .../miti99/storescraperbot/config/Config.java | 6 +- .../storescraperbot/model/AbstractModel.java | 4 +- .../repository/AbstractRepository.java | 61 +++++++------- .../repository/AdminRepository.java | 12 +-- .../storescraperbot/util/CouchbaseUtil.java | 83 +++++++++++++++++++ .../storescraperbot/util/RedisUtil.java | 17 ---- 10 files changed, 146 insertions(+), 67 deletions(-) create mode 100644 src/main/java/com/miti99/storescraperbot/util/CouchbaseUtil.java delete mode 100644 src/main/java/com/miti99/storescraperbot/util/RedisUtil.java diff --git a/.env.example b/.env.example index bf790bb..ca95628 100644 --- a/.env.example +++ b/.env.example @@ -1,8 +1,11 @@ # Copy this file to .env and customize the values for your environment # cp .env.example .env -# Redis configuration -REDIS_URL=localhost:6379 +# Couchbase configuration +COUCHBASE_CONNECTION_STRING=couchbase://localhost +COUCHBASE_USERNAME=admin +COUCHBASE_PASSWORD=your_password_here +COUCHBASE_BUCKET_NAME=store_scraper # Telegram Bot configuration TELEGRAM_BOT_TOKEN=your_telegram_bot_token_here diff --git a/build.gradle.kts b/build.gradle.kts index 9bb2710..e3b7896 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -22,15 +22,14 @@ configurations { dependencies { annotationProcessor("org.projectlombok:lombok:1.18.36") + implementation("com.couchbase.client:java-client:3.4.11") implementation("com.google.code.gson:gson:2.11.0") implementation("com.google.guava:guava:33.4.0-jre") - implementation("org.apache.commons:commons-text:1.13.0") implementation("org.apache.logging.log4j:log4j-core:2.24.3") implementation("org.apache.logging.log4j:log4j-slf4j2-impl:2.24.3") implementation("org.telegram:telegrambots-client:8.0.0") implementation("org.telegram:telegrambots-extensions:8.0.0") implementation("org.telegram:telegrambots-longpolling:8.0.0") - implementation("redis.clients:jedis:5.2.0") testAnnotationProcessor("org.projectlombok:lombok:1.18.36") testImplementation(platform("org.junit:junit-bom:5.11.4")) diff --git a/compose.dev.yml b/compose.dev.yml index 3f4d356..4a2e505 100644 --- a/compose.dev.yml +++ b/compose.dev.yml @@ -1,12 +1,18 @@ services: - redis: - image: redis:7.2-alpine + couchbase: + image: couchbase:community-7.6.2 + env_file: + - .env ports: - - "6379:6379" + - "8091-8097:8091-8097" + - "9123:9123" + - "11207:11207" + - "11210:11210" + - "11280:11280" + - "18091-18097:18091-18097" volumes: - - redis_data:/data + - couchbase_data:/opt/couchbase/var restart: unless-stopped - command: redis-server --appendonly yes volumes: - redis_data: \ No newline at end of file + couchbase_data: diff --git a/compose.yml b/compose.yml index 8b86a6e..c6504ba 100644 --- a/compose.yml +++ b/compose.yml @@ -3,5 +3,5 @@ services: build: context: . -# If you need redis, add redis service +# If you need database, add database service # Check compose.dev.yml for example diff --git a/src/main/java/com/miti99/storescraperbot/config/Config.java b/src/main/java/com/miti99/storescraperbot/config/Config.java index e89db87..2ecc0d6 100644 --- a/src/main/java/com/miti99/storescraperbot/config/Config.java +++ b/src/main/java/com/miti99/storescraperbot/config/Config.java @@ -8,7 +8,11 @@ import java.util.Set; import java.util.stream.Collectors; public class Config { - public static final String REDIS_URL = System.getenv("REDIS_URL"); + public static final String COUCHBASE_CONNECTION_STRING = + System.getenv("COUCHBASE_CONNECTION_STRING"); + public static final String COUCHBASE_USERNAME = System.getenv("COUCHBASE_USERNAME"); + public static final String COUCHBASE_PASSWORD = System.getenv("COUCHBASE_PASSWORD"); + public static final String COUCHBASE_BUCKET_NAME = System.getenv("COUCHBASE_BUCKET_NAME"); public static final String TELEGRAM_BOT_TOKEN = System.getenv("TELEGRAM_BOT_TOKEN"); public static final String TELEGRAM_BOT_USERNAME = System.getenv("TELEGRAM_BOT_USERNAME"); diff --git a/src/main/java/com/miti99/storescraperbot/model/AbstractModel.java b/src/main/java/com/miti99/storescraperbot/model/AbstractModel.java index 3d430c8..7e56d62 100644 --- a/src/main/java/com/miti99/storescraperbot/model/AbstractModel.java +++ b/src/main/java/com/miti99/storescraperbot/model/AbstractModel.java @@ -1,6 +1,6 @@ package com.miti99.storescraperbot.model; -import com.google.gson.annotations.SerializedName; +import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -11,6 +11,6 @@ import lombok.Setter; public abstract class AbstractModel { protected K key; - @SerializedName("class") + @JsonProperty("class") protected String clazz = getClass().getSimpleName(); } diff --git a/src/main/java/com/miti99/storescraperbot/repository/AbstractRepository.java b/src/main/java/com/miti99/storescraperbot/repository/AbstractRepository.java index 161eddf..d2865b8 100644 --- a/src/main/java/com/miti99/storescraperbot/repository/AbstractRepository.java +++ b/src/main/java/com/miti99/storescraperbot/repository/AbstractRepository.java @@ -1,29 +1,26 @@ package com.miti99.storescraperbot.repository; +import com.couchbase.client.java.Collection; +import com.couchbase.client.java.kv.UpsertOptions; import com.google.common.base.CaseFormat; import com.miti99.storescraperbot.config.Config; -import com.miti99.storescraperbot.constant.Constant; import com.miti99.storescraperbot.model.AbstractModel; -import com.miti99.storescraperbot.util.GsonUtil; -import com.miti99.storescraperbot.util.RedisUtil; +import com.miti99.storescraperbot.util.CouchbaseUtil; import java.lang.reflect.ParameterizedType; -import lombok.AccessLevel; -import lombok.NoArgsConstructor; +import java.time.Duration; import lombok.extern.log4j.Log4j2; -import redis.clients.jedis.params.SetParams; @Log4j2 -@NoArgsConstructor(access = AccessLevel.PROTECTED) public abstract class AbstractRepository> { - public static final String SEPARATOR = ":"; + public static final String SEPARATOR = "_"; // protected final Class classK = getKeyClass(); protected final Class classV = getDataClass(); - protected final String prefix = - String.join( - SEPARATOR, - Constant.APP_NAME, - Config.ENV.name().toLowerCase(), - CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_UNDERSCORE, classV.getSimpleName())); + protected final String scopeName = Config.ENV.name().toLowerCase(); + protected final String collectionName = CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_UNDERSCORE, classV.getSimpleName()); + + protected AbstractRepository() { + CouchbaseUtil.createCollection(scopeName, collectionName); + } // protected Class getKeyClass() { // return (Class) @@ -42,8 +39,12 @@ public abstract class AbstractRepository> { ((ParameterizedType) getClass().getGenericSuperclass()).getActualTypeArguments()[1]; } + protected Collection collection() { + return CouchbaseUtil.BUCKET.scope(scopeName).collection(collectionName); + } + /** - * @return expire seconds. <= 0 mean never expire. + * @return expire seconds. 0 mean never expire. */ protected long getExpireSeconds() { return 0; @@ -63,17 +64,18 @@ public abstract class AbstractRepository> { } protected String getDatabaseKey(K key) { - return String.join(SEPARATOR, prefix, String.valueOf(key)); + return String.valueOf(key); } public void save(K key, V data) { var databaseKey = getDatabaseKey(key); - try (var jedis = RedisUtil.getJedis()) { - var json = GsonUtil.toJson(data); - if (getExpireSeconds() <= 0) { - jedis.set(databaseKey, json); + try { + if (getExpireSeconds() == 0) { + collection().upsert(databaseKey, data); } else { - jedis.set(databaseKey, json, SetParams.setParams().ex(getExpireSeconds())); + var upsertOptions = + UpsertOptions.upsertOptions().expiry(Duration.ofSeconds(getExpireSeconds())); + collection().upsert(databaseKey, data, upsertOptions); } } catch (Exception e) { log.error("save error - key {}, databaseKey {}", key, databaseKey, e); @@ -82,8 +84,8 @@ public abstract class AbstractRepository> { public boolean exist(K key) { var databaseKey = getDatabaseKey(key); - try (var jedis = RedisUtil.getJedis()) { - return jedis.exists(databaseKey); + try { + return collection().exists(databaseKey).exists(); } catch (Exception e) { log.error("exist error - key {}, databaseKey {}", key, databaseKey, e); return false; @@ -92,9 +94,12 @@ public abstract class AbstractRepository> { public V load(K key) { var databaseKey = getDatabaseKey(key); - try (var jedis = RedisUtil.getJedis()) { - var json = jedis.get(databaseKey); - return GsonUtil.fromJson(json, classV); + try { + var getResult = collection().get(databaseKey); + if (getResult == null) { + return null; + } + return getResult.contentAs(classV); } catch (Exception e) { log.error("load error - key {}, databaseKey {}", key, databaseKey, e); return null; @@ -103,8 +108,8 @@ public abstract class AbstractRepository> { public void delete(K key) { var databaseKey = getDatabaseKey(key); - try (var jedis = RedisUtil.getJedis()) { - jedis.del(databaseKey); + try { + collection().remove(databaseKey); } catch (Exception e) { log.error("delete error", e); } diff --git a/src/main/java/com/miti99/storescraperbot/repository/AdminRepository.java b/src/main/java/com/miti99/storescraperbot/repository/AdminRepository.java index 4f29514..5911acf 100644 --- a/src/main/java/com/miti99/storescraperbot/repository/AdminRepository.java +++ b/src/main/java/com/miti99/storescraperbot/repository/AdminRepository.java @@ -2,23 +2,19 @@ package com.miti99.storescraperbot.repository; import com.miti99.storescraperbot.model.Admin; -/** Đây là repository chỉ chứa 1 key duy nhất, key là "" (rỗng) */ +/** Đây là repository chỉ chứa 1 key duy nhất, key là "admin" */ public class AdminRepository extends AbstractRepository { public static final AdminRepository INSTANCE = new AdminRepository(); - protected AdminRepository() { - super(); - } - public void init() { - init(""); + init("admin"); } public Admin load() { - return load(""); + return load("admin"); } public void save(Admin data) { - save("", data); + save("admin", data); } } diff --git a/src/main/java/com/miti99/storescraperbot/util/CouchbaseUtil.java b/src/main/java/com/miti99/storescraperbot/util/CouchbaseUtil.java new file mode 100644 index 0000000..78334a8 --- /dev/null +++ b/src/main/java/com/miti99/storescraperbot/util/CouchbaseUtil.java @@ -0,0 +1,83 @@ +package com.miti99.storescraperbot.util; + +import static com.miti99.storescraperbot.config.Config.COUCHBASE_BUCKET_NAME; +import static com.miti99.storescraperbot.config.Config.COUCHBASE_CONNECTION_STRING; +import static com.miti99.storescraperbot.config.Config.COUCHBASE_PASSWORD; +import static com.miti99.storescraperbot.config.Config.COUCHBASE_USERNAME; + +import com.couchbase.client.java.Bucket; +import com.couchbase.client.java.Cluster; +import com.couchbase.client.java.ClusterOptions; +import com.couchbase.client.java.manager.collection.CollectionSpec; +import java.time.Duration; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import lombok.extern.log4j.Log4j2; + +@Log4j2 +@NoArgsConstructor(access = AccessLevel.PRIVATE) +public class CouchbaseUtil { + public static final Cluster CLUSTER; + public static final Bucket BUCKET; + + static { + CLUSTER = + Cluster.connect( + COUCHBASE_CONNECTION_STRING, + ClusterOptions.clusterOptions(COUCHBASE_USERNAME, COUCHBASE_PASSWORD) + .environment(env -> {})); + + BUCKET = CLUSTER.bucket(COUCHBASE_BUCKET_NAME); + BUCKET.waitUntilReady(Duration.ofSeconds(10)); + } + + public static void createScope(String scopeName) { + var collectionManager = BUCKET.collections(); + try { + boolean scopeExists = + collectionManager.getAllScopes().stream().anyMatch(s -> s.name().equals(scopeName)); + + if (!scopeExists) { + collectionManager.createScope(scopeName); + log.info("Scope created: {}", scopeName); + } else { + log.info("Scope existed: {}", scopeName); + } + } catch (Exception e) { + log.error("createScope error - scopeName: '{}'", scopeName, e); + } + } + + public static void createCollection(String scopeName, String collectionName) { + var collectionManager = BUCKET.collections(); + try { + var scopeSpecOpt = + collectionManager.getAllScopes().stream() + .filter(s -> s.name().equals(scopeName)) + .findFirst(); + if (scopeSpecOpt.isEmpty()) { + createScope(scopeName); + createCollection(scopeName, collectionName); + return; + } + + var scopeSpec = scopeSpecOpt.get(); + boolean collectionExists = + scopeSpec.collections().stream().anyMatch(c -> c.name().equals(collectionName)); + + if (!collectionExists) { + var spec = CollectionSpec.create(collectionName, scopeName); + collectionManager.createCollection(spec); + log.info("Collection created: {} in {}", collectionName, scopeName); + } else { + log.info("Collection existed: {} in {}", collectionName, scopeName); + } + } catch (Exception e) { + log.error( + "createCollection error - collectionName: '{}', scopeName: '{}'", + collectionName, + scopeName, + e); + } + } +} diff --git a/src/main/java/com/miti99/storescraperbot/util/RedisUtil.java b/src/main/java/com/miti99/storescraperbot/util/RedisUtil.java deleted file mode 100644 index 78143cc..0000000 --- a/src/main/java/com/miti99/storescraperbot/util/RedisUtil.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.miti99.storescraperbot.util; - -import static com.miti99.storescraperbot.config.Config.REDIS_URL; - -import lombok.AccessLevel; -import lombok.NoArgsConstructor; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisPool; - -@NoArgsConstructor(access = AccessLevel.PRIVATE) -public class RedisUtil { - private static final JedisPool REDIS_POOL = new JedisPool(REDIS_URL); - - public static Jedis getJedis() { - return REDIS_POOL.getResource(); - } -}