Migrate data storage from Couchbase to Redis

Replaces Couchbase with Redis for data persistence, updating environment variables, Docker Compose files, and repository logic. Removes Couchbase and Jackson dependencies, introduces Gson for JSON serialization, and refactors models and repositories to support Redis. Adds utility classes for Redis and Gson, and updates tests and scrapers to use Gson.
This commit is contained in:
2025-11-05 21:47:58 +07:00
parent a532c34d99
commit def45c7402
24 changed files with 120 additions and 257 deletions
+2 -5
View File
@@ -1,11 +1,8 @@
# Copy this file to .env and customize the values for your environment
# cp .env.example .env
# Couchbase configuration
COUCHBASE_CONNECTION_STRING=couchbase://localhost
COUCHBASE_USERNAME=admin
COUCHBASE_PASSWORD=your_password_here
COUCHBASE_BUCKET_NAME=store_scraper
# Redis configuration
REDIS_URL=localhost:6379
# Telegram Bot configuration
TELEGRAM_BOT_TOKEN=your_telegram_bot_token_here
+4 -2
View File
@@ -22,13 +22,15 @@ configurations {
dependencies {
annotationProcessor("org.projectlombok:lombok:1.18.36")
implementation("com.couchbase.client:java-client:3.4.11")
implementation("com.google.code.gson:gson:2.11.0")
implementation("com.google.guava:guava:33.4.0-jre")
implementation("org.apache.commons:commons-text:1.13.0")
implementation("org.apache.logging.log4j:log4j-core:2.24.3")
implementation("org.apache.logging.log4j:log4j-slf4j2-impl:2.24.3")
// implementation("org.telegram:telegrambots-abilities:8.0.0")
implementation("org.telegram:telegrambots-client:8.0.0")
implementation("org.telegram:telegrambots-extensions:8.0.0")
implementation("org.telegram:telegrambots-longpolling:8.0.0")
implementation("redis.clients:jedis:5.2.0")
testAnnotationProcessor("org.projectlombok:lombok:1.18.36")
testImplementation(platform("org.junit:junit-bom:5.11.4"))
+6 -12
View File
@@ -1,18 +1,12 @@
services:
couchbase:
image: couchbase:community-7.6.2
env_file:
- .env
redis:
image: redis:7.2-alpine
ports:
- "8091-8097:8091-8097"
- "9123:9123"
- "11207:11207"
- "11210:11210"
- "11280:11280"
- "18091-18097:18091-18097"
- "6379:6379"
volumes:
- couchbase_data:/opt/couchbase/var
- redis_data:/data
restart: unless-stopped
command: redis-server --appendonly yes
volumes:
couchbase_data:
redis_data:
+2 -32
View File
@@ -2,36 +2,6 @@ services:
server:
build:
context: .
env_file:
- .env
depends_on:
- couchbase
hostname: couchbase
networks:
- internal
command: >
bash -c "couchbase-server -- -advertised_hostname couchbase"
couchbase:
image: couchbase:community-7.6.2
# env_file:
# - .env
# ports: # Enable these ports if you need
# - "8091-8097:8091-8097"
# - "9123:9123"
# - "11207:11207"
# - "11210:11210"
# - "11280:11280"
# - "18091-18097:18091-18097"
volumes:
- couchbase_data:/opt/couchbase/var
restart: unless-stopped
networks:
- internal
volumes:
couchbase_data:
networks:
internal:
driver: bridge
# If you need redis, add redis service
# Check compose.dev.yml for example
@@ -1,10 +1,8 @@
package com.miti99.storescraperbot.api.apple;
import com.couchbase.client.core.deps.io.netty.handler.codec.http.HttpHeaderNames;
import com.couchbase.client.core.deps.io.netty.handler.codec.http.HttpHeaderValues;
import com.miti99.storescraperbot.api.apple.request.AppleAppRequest;
import com.miti99.storescraperbot.api.apple.response.AppleAppResponse;
import com.miti99.storescraperbot.util.JacksonUtil;
import com.miti99.storescraperbot.util.GsonUtil;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpClient.Redirect;
@@ -22,10 +20,8 @@ public class AppStoreScraper {
HttpRequest.newBuilder()
.uri(URI.create(BASE_URL + "/app"))
// .timeout(Duration.ofMillis(TIMEOUT))
.header(
HttpHeaderNames.CONTENT_TYPE.toString(),
HttpHeaderValues.APPLICATION_JSON.toString())
.POST(BodyPublishers.ofString(JacksonUtil.writeValueAsString(request)))
.header("Content-Type", "application/json")
.POST(BodyPublishers.ofString(GsonUtil.toJson(request)))
.build();
var body =
@@ -35,6 +31,6 @@ public class AppStoreScraper {
.build()
.send(httpRequest, BodyHandlers.ofString())
.body();
return JacksonUtil.readValue(body, AppleAppResponse.class);
return GsonUtil.fromJson(body, AppleAppResponse.class);
}
}
@@ -1,10 +1,8 @@
package com.miti99.storescraperbot.api.google;
import com.couchbase.client.core.deps.io.netty.handler.codec.http.HttpHeaderNames;
import com.couchbase.client.core.deps.io.netty.handler.codec.http.HttpHeaderValues;
import com.miti99.storescraperbot.api.google.request.GoogleAppRequest;
import com.miti99.storescraperbot.api.google.response.GoogleAppResponse;
import com.miti99.storescraperbot.util.JacksonUtil;
import com.miti99.storescraperbot.util.GsonUtil;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpClient.Redirect;
@@ -22,10 +20,8 @@ public class GooglePlayScraper {
HttpRequest.newBuilder()
.uri(URI.create(BASE_URL + "/app"))
// .timeout(Duration.ofMillis(TIMEOUT))
.header(
HttpHeaderNames.CONTENT_TYPE.toString(),
HttpHeaderValues.APPLICATION_JSON.toString())
.POST(BodyPublishers.ofString(JacksonUtil.writeValueAsString(request)))
.header("Content-Type", "application/json")
.POST(BodyPublishers.ofString(GsonUtil.toJson(request)))
.build();
var body =
@@ -35,6 +31,6 @@ public class GooglePlayScraper {
.build()
.send(httpRequest, BodyHandlers.ofString())
.body();
return JacksonUtil.readValue(body, GoogleAppResponse.class);
return GsonUtil.fromJson(body, GoogleAppResponse.class);
}
}
@@ -8,11 +8,7 @@ import java.util.Set;
import java.util.stream.Collectors;
public class Config {
public static final String COUCHBASE_CONNECTION_STRING =
System.getenv("COUCHBASE_CONNECTION_STRING");
public static final String COUCHBASE_USERNAME = System.getenv("COUCHBASE_USERNAME");
public static final String COUCHBASE_PASSWORD = System.getenv("COUCHBASE_PASSWORD");
public static final String COUCHBASE_BUCKET_NAME = System.getenv("COUCHBASE_BUCKET_NAME");
public static final String REDIS_URL = System.getenv("REDIS_URL");
public static final String TELEGRAM_BOT_TOKEN = System.getenv("TELEGRAM_BOT_TOKEN");
public static final String TELEGRAM_BOT_USERNAME = System.getenv("TELEGRAM_BOT_USERNAME");
@@ -0,0 +1,5 @@
package com.miti99.storescraperbot.constant;
public class Constant {
public static final String APP_NAME = "store_scraper_bot";
}
@@ -1,11 +1,14 @@
package com.miti99.storescraperbot.model;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.gson.annotations.SerializedName;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
@Getter
@RequiredArgsConstructor
public abstract class AbstractModel<K> {
final K key;
@JsonProperty("class")
@SerializedName("class")
protected String clazz = getClass().getSimpleName();
}
@@ -10,4 +10,8 @@ import lombok.Setter;
@Setter
public class Admin extends AbstractModel<String> {
List<Long> groups = new ArrayList<>();
public Admin(String key) {
super(key);
}
}
@@ -9,4 +9,8 @@ import lombok.Setter;
public class AppleApp extends AbstractModel<String> {
long cacheTime;
AppleAppResponse rawResponse;
public AppleApp(String key) {
super(key);
}
}
@@ -9,4 +9,8 @@ import lombok.Setter;
public class GoogleApp extends AbstractModel<String> {
long cacheTime;
GoogleAppResponse rawResponse;
public GoogleApp(String key) {
super(key);
}
}
@@ -11,6 +11,10 @@ import lombok.Setter;
public class Group extends AbstractModel<Long> {
List<App> apps;
public Group(Long key) {
super(key);
}
public static class App {
String appId;
AppType type;
@@ -1,29 +1,32 @@
package com.miti99.storescraperbot.repository;
import com.couchbase.client.java.Collection;
import com.google.common.base.CaseFormat;
import com.miti99.storescraperbot.config.Config;
import com.miti99.storescraperbot.constant.Constant;
import com.miti99.storescraperbot.model.AbstractModel;
import com.miti99.storescraperbot.util.CouchbaseUtil;
import com.miti99.storescraperbot.util.GsonUtil;
import com.miti99.storescraperbot.util.RedisUtil;
import java.lang.reflect.ParameterizedType;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import lombok.extern.log4j.Log4j2;
/** 1 repository = 1 collection */
@Log4j2
@NoArgsConstructor(access = AccessLevel.PROTECTED)
public abstract class AbstractRepository<K, V extends AbstractModel<K>> {
public static final String SEPARATOR = "_";
// protected static ObjectMapper objectMapper = new ObjectMapper();
public static final String SEPARATOR = ":";
protected final Class<K> classK = getKeyClass();
protected final Class<V> classV = getDataClass();
// protected final JavaType type = objectMapper.getTypeFactory().constructType(classV);
protected final String scopeName = Config.ENV.name().toLowerCase();
protected final String collectionName;
protected final String prefix =
String.join(
SEPARATOR,
Constant.APP_NAME,
Config.ENV.name().toLowerCase(),
CaseFormat.UPPER_CAMEL.to(CaseFormat.LOWER_UNDERSCORE, classV.getSimpleName()));
protected AbstractRepository(String collectionName) {
this.collectionName = collectionName.toLowerCase();
CouchbaseUtil.createCollection(scopeName, collectionName);
}
public Collection collection() {
return CouchbaseUtil.BUCKET.scope(scopeName).collection(collectionName);
protected Class<K> getKeyClass() {
return (Class<K>)
((ParameterizedType) getClass().getGenericSuperclass()).getActualTypeArguments()[0];
}
/**
@@ -43,7 +46,7 @@ public abstract class AbstractRepository<K, V extends AbstractModel<K>> {
if (exist(key)) {
return;
}
V data = classV.getDeclaredConstructor().newInstance();
V data = classV.getDeclaredConstructor(classK).newInstance(key);
save(key, data);
} catch (Exception e) {
log.error("Error while initializing data", e);
@@ -51,13 +54,14 @@ public abstract class AbstractRepository<K, V extends AbstractModel<K>> {
}
protected String getDatabaseKey(K key) {
return String.join(SEPARATOR, classV.getSimpleName(), String.valueOf(key));
return String.join(SEPARATOR, prefix, String.valueOf(key));
}
public void save(K key, V data) {
var databaseKey = getDatabaseKey(key);
try {
collection().upsert(databaseKey, data);
try (var jedis = RedisUtil.getJedis()) {
var json = GsonUtil.toJson(data);
jedis.set(databaseKey, json);
} catch (Exception e) {
log.error("save error - key {}, databaseKey {}", key, databaseKey, e);
}
@@ -65,8 +69,8 @@ public abstract class AbstractRepository<K, V extends AbstractModel<K>> {
public boolean exist(K key) {
var databaseKey = getDatabaseKey(key);
try {
return collection().exists(databaseKey).exists();
try (var jedis = RedisUtil.getJedis()) {
return jedis.exists(databaseKey);
} catch (Exception e) {
log.error("exist error - key {}, databaseKey {}", key, databaseKey, e);
return false;
@@ -75,12 +79,9 @@ public abstract class AbstractRepository<K, V extends AbstractModel<K>> {
public V load(K key) {
var databaseKey = getDatabaseKey(key);
try {
var getResult = collection().get(databaseKey);
if (getResult == null) {
return null;
}
return getResult.contentAs(classV);
try (var jedis = RedisUtil.getJedis()) {
var json = jedis.get(databaseKey);
return GsonUtil.fromJson(json, classV);
} catch (Exception e) {
log.error("load error - key {}, databaseKey {}", key, databaseKey, e);
return null;
@@ -89,8 +90,8 @@ public abstract class AbstractRepository<K, V extends AbstractModel<K>> {
public void delete(K key) {
var databaseKey = getDatabaseKey(key);
try {
collection().remove(databaseKey);
try (var jedis = RedisUtil.getJedis()) {
jedis.del(databaseKey);
} catch (Exception e) {
log.error("delete error", e);
}
@@ -2,14 +2,12 @@ package com.miti99.storescraperbot.repository;
import com.miti99.storescraperbot.model.Admin;
/**
* Đây là repository chỉ chứa 1 key duy nhất, key là "" (rỗng)
*/
/** Đây là repository chỉ chứa 1 key duy nhất, key là "" (rỗng) */
public class AdminRepository extends AbstractRepository<String, Admin> {
public static final AdminRepository INSTANCE = new AdminRepository();
protected AdminRepository() {
super("admin");
super();
}
public void init() {
@@ -1,12 +1,7 @@
package com.miti99.storescraperbot.repository;
import com.miti99.storescraperbot.model.AppleApp;
import com.miti99.storescraperbot.model.Group;
public class AppleAppRepository extends AbstractRepository<String, AppleApp> {
public static final AppleAppRepository INSTANCE = new AppleAppRepository();
protected AppleAppRepository() {
super("apple");
}
}
@@ -1,12 +1,7 @@
package com.miti99.storescraperbot.repository;
import com.miti99.storescraperbot.model.GoogleApp;
import com.miti99.storescraperbot.model.Group;
public class GoogleAppRepository extends AbstractRepository<String, GoogleApp> {
private static final GoogleAppRepository INSTANCE = new GoogleAppRepository();
protected GoogleAppRepository() {
super("google");
}
}
@@ -4,8 +4,4 @@ import com.miti99.storescraperbot.model.Group;
public class GroupRepository extends AbstractRepository<Long, Group> {
public static final GroupRepository INSTANCE = new GroupRepository();
protected GroupRepository() {
super("group");
}
}
@@ -1,81 +0,0 @@
package com.miti99.storescraperbot.util;
import static com.miti99.storescraperbot.config.Config.COUCHBASE_BUCKET_NAME;
import static com.miti99.storescraperbot.config.Config.COUCHBASE_CONNECTION_STRING;
import static com.miti99.storescraperbot.config.Config.COUCHBASE_PASSWORD;
import static com.miti99.storescraperbot.config.Config.COUCHBASE_USERNAME;
import com.couchbase.client.java.Bucket;
import com.couchbase.client.java.Cluster;
import com.couchbase.client.java.ClusterOptions;
import com.couchbase.client.java.manager.collection.CollectionSpec;
import com.miti99.storescraperbot.config.Config;
import java.time.Duration;
import lombok.extern.log4j.Log4j2;
@Log4j2
public class CouchbaseUtil {
public static final Cluster CLUSTER;
public static final Bucket BUCKET;
static {
CLUSTER =
Cluster.connect(
COUCHBASE_CONNECTION_STRING,
ClusterOptions.clusterOptions(COUCHBASE_USERNAME, COUCHBASE_PASSWORD)
.environment(env -> {}));
BUCKET = CLUSTER.bucket(COUCHBASE_BUCKET_NAME);
BUCKET.waitUntilReady(Duration.ofSeconds(10));
}
public static void createScope(String scopeName) {
var collectionManager = BUCKET.collections();
try {
boolean scopeExists =
collectionManager.getAllScopes().stream().anyMatch(s -> s.name().equals(scopeName));
if (!scopeExists) {
collectionManager.createScope(scopeName);
log.info("Scope created: {}", scopeName);
} else {
log.info("Scope existed: {}", scopeName);
}
} catch (Exception e) {
log.error("createScope error - scopeName: '{}'", scopeName, e);
}
}
public static void createCollection(String scopeName, String collectionName) {
var collectionManager = BUCKET.collections();
try {
var scopeSpecOpt =
collectionManager.getAllScopes().stream()
.filter(s -> s.name().equals(scopeName))
.findFirst();
if (scopeSpecOpt.isEmpty()) {
createScope(scopeName);
createCollection(scopeName, collectionName);
return;
}
var scopeSpec = scopeSpecOpt.get();
boolean collectionExists =
scopeSpec.collections().stream().anyMatch(c -> c.name().equals(collectionName));
if (!collectionExists) {
var spec = CollectionSpec.create(collectionName, scopeName);
collectionManager.createCollection(spec);
log.info("Collection created: {} in {}", collectionName, scopeName);
} else {
log.info("Collection existed: {} in {}", collectionName, scopeName);
}
} catch (Exception e) {
log.error(
"createCollection error - collectionName: '{}', scopeName: '{}'",
collectionName,
scopeName,
e);
}
}
}
@@ -0,0 +1,18 @@
package com.miti99.storescraperbot.util;
import com.google.gson.Gson;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class GsonUtil {
public static final Gson GSON = new Gson();
public static <T> T fromJson(String input, Class<T> valueType) {
return GSON.fromJson(input, valueType);
}
public static String toJson(Object input) {
return GSON.toJson(input);
}
}
@@ -1,47 +0,0 @@
package com.miti99.storescraperbot.util;
import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.PropertyAccessor;
import com.fasterxml.jackson.core.JsonParser.Feature;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import lombok.SneakyThrows;
public class JacksonUtil {
public static ObjectMapper MAPPER = objectMapper();
private static ObjectMapper objectMapper() {
var objectMapper = new ObjectMapper();
objectMapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
objectMapper.disable(
SerializationFeature.FAIL_ON_EMPTY_BEANS,
SerializationFeature.INDENT_OUTPUT,
SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
objectMapper.enable(
DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT,
DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY,
DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL);
objectMapper.enable(
Feature.ALLOW_SINGLE_QUOTES, Feature.ALLOW_UNQUOTED_FIELD_NAMES, Feature.IGNORE_UNDEFINED);
objectMapper.enable(SerializationFeature.WRITE_ENUMS_USING_TO_STRING);
objectMapper.setSerializationInclusion(Include.NON_NULL);
objectMapper.setVisibility(PropertyAccessor.FIELD, Visibility.ANY);
return objectMapper;
}
@SneakyThrows
public static <T> T readValue(String input, Class<T> valueType) {
return MAPPER.readValue(input, valueType);
}
@SneakyThrows
public static String writeValueAsString(Object input) {
return MAPPER.writeValueAsString(input);
}
}
@@ -0,0 +1,17 @@
package com.miti99.storescraperbot.util;
import static com.miti99.storescraperbot.config.Config.REDIS_URL;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class RedisUtil {
private static final JedisPool REDIS_POOL = new JedisPool(REDIS_URL);
public static Jedis getJedis() {
return REDIS_POOL.getResource();
}
}
@@ -1,9 +1,7 @@
package com.miti99.storescraperbot.api.apple;
import static org.junit.jupiter.api.Assertions.*;
import com.miti99.storescraperbot.api.apple.request.AppleAppRequest;
import com.miti99.storescraperbot.util.JacksonUtil;
import com.miti99.storescraperbot.util.GsonUtil;
import org.junit.jupiter.api.Test;
class AppStoreScraperTest {
@@ -11,6 +9,6 @@ class AppStoreScraperTest {
void testApp() {
var request = new AppleAppRequest("com.mpt.kvtm");
var response = AppStoreScraper.app(request);
System.out.println(JacksonUtil.writeValueAsString(response));
System.out.println(GsonUtil.toJson(response));
}
}
@@ -1,9 +1,7 @@
package com.miti99.storescraperbot.api.google;
import static org.junit.jupiter.api.Assertions.*;
import com.miti99.storescraperbot.api.google.request.GoogleAppRequest;
import com.miti99.storescraperbot.util.JacksonUtil;
import com.miti99.storescraperbot.util.GsonUtil;
import org.junit.jupiter.api.Test;
class GooglePlayScraperTest {
@@ -11,6 +9,6 @@ class GooglePlayScraperTest {
void testApp() {
var request = new GoogleAppRequest("vn.kvtm.js");
var response = GooglePlayScraper.app(request);
System.out.println(JacksonUtil.writeValueAsString(response));
System.out.println(GsonUtil.toJson(response));
}
}