From d8d3f12d1c49ca0c763807f4b7b4f9e5a0fbce86 Mon Sep 17 00:00:00 2001 From: Alemiz Date: Sat, 25 Mar 2023 13:19:46 +0100 Subject: [PATCH] String improvements & use SQLite for storage --- .gitignore | 3 +- .../common/string/JoinLocalString.java | 72 ++++++++ .../common/string/LocalString.java | 20 +++ .../common/string/StaticLocalString.java | 3 + .../common/string/StringWrapper.java | 68 ++++++++ .../translationlib/common/TranslateTest.java | 10 ++ gradle/libs.versions.toml | 2 +- service/build.gradle.kts | 2 +- .../service/TranslationLibService.java | 22 --- .../service/manager/TermsManager.java | 2 +- .../service/repository/TermsRepository.java | 161 +++++++++++++----- .../service/utils/Configuration.java | 10 +- service/src/main/resources/service.properties | 6 +- .../main/resources/simplelogger.properties | 5 +- 14 files changed, 304 insertions(+), 82 deletions(-) create mode 100644 common/src/main/java/eu/mizerak/alemiz/translationlib/common/string/JoinLocalString.java create mode 100644 common/src/main/java/eu/mizerak/alemiz/translationlib/common/string/StringWrapper.java diff --git a/.gitignore b/.gitignore index 520f33b..e956371 100644 --- a/.gitignore +++ b/.gitignore @@ -74,4 +74,5 @@ bin/ service.properties common/src/main/generated/ -service/src/main/generated/ \ No newline at end of file +service/src/main/generated/ +*.db \ No newline at end of file diff --git a/common/src/main/java/eu/mizerak/alemiz/translationlib/common/string/JoinLocalString.java b/common/src/main/java/eu/mizerak/alemiz/translationlib/common/string/JoinLocalString.java new file mode 100644 index 0000000..af4641b --- /dev/null +++ b/common/src/main/java/eu/mizerak/alemiz/translationlib/common/string/JoinLocalString.java @@ -0,0 +1,72 @@ +package eu.mizerak.alemiz.translationlib.common.string; + +import java.util.Locale; +import java.util.function.Function; + +public class JoinLocalString implements LocalString { + + private final LocalString left; + private final LocalString right; + + private final String delimiter; + + public JoinLocalString(LocalString left, LocalString right) { + this(left, right, ""); + } + + public JoinLocalString(LocalString left, LocalString right, String delimiter) { + this.left = left; + this.right = right; + this.delimiter = delimiter; + } + + @Override + public String getKey() { + return this.left.getKey() + "_" + this.right.getKey(); + } + + @Override + public LocalString reload() { + this.left.reload(); + this.right.reload(); + return this; + } + + @Override + public LocalString enableReload(boolean enable) { + this.left.enableReload(enable); + this.right.enableReload(enable); + return this; + } + + @Override + public LocalString withArgument(String name, Function, String> mapper) { + throw new UnsupportedOperationException("Joined string does not support arguments"); + } + + @Override + public LocalString clearArguments() { + throw new UnsupportedOperationException("Joined string does not support arguments"); + } + + @Override + public String getFormatted() { + return this.left.getFormatted() + this.delimiter + this.right.getFormatted(); + } + + @Override + public String getFormatted(Locale locale) { + return this.left.getFormatted(locale) + this.delimiter + this.right.getFormatted(locale); + } + + @Override + public String getText(T object) { + return this.left.getText(object) + this.delimiter + this.right.getText(object); + } + + @Override + public void uploadFallbackMessage() { + this.left.uploadFallbackMessage(); + this.right.uploadFallbackMessage(); + } +} diff --git a/common/src/main/java/eu/mizerak/alemiz/translationlib/common/string/LocalString.java b/common/src/main/java/eu/mizerak/alemiz/translationlib/common/string/LocalString.java index 1cb9531..a49912f 100644 --- a/common/src/main/java/eu/mizerak/alemiz/translationlib/common/string/LocalString.java +++ b/common/src/main/java/eu/mizerak/alemiz/translationlib/common/string/LocalString.java @@ -33,6 +33,14 @@ static LocalString immutable(String text) { return new StaticLocalString<>(term); } + static LocalString wrapper(String text) { + return new StringWrapper<>(text); + } + + static LocalString empty() { + return (LocalString) StringWrapper.EMPTY; + } + String getKey(); LocalString reload(); @@ -54,4 +62,16 @@ default LocalString withArgument(String name, Object argument) { String getText(T object); void uploadFallbackMessage(); + + default LocalString append(String string) { + return new JoinLocalString<>(this, LocalString.wrapper(string)); + } + + default LocalString append(LocalString string) { + return new JoinLocalString<>(this, string); + } + + default LocalString append(LocalString string, String delimiter) { + return new JoinLocalString<>(this, string, delimiter); + } } diff --git a/common/src/main/java/eu/mizerak/alemiz/translationlib/common/string/StaticLocalString.java b/common/src/main/java/eu/mizerak/alemiz/translationlib/common/string/StaticLocalString.java index 4fb3ff6..8b45321 100644 --- a/common/src/main/java/eu/mizerak/alemiz/translationlib/common/string/StaticLocalString.java +++ b/common/src/main/java/eu/mizerak/alemiz/translationlib/common/string/StaticLocalString.java @@ -4,6 +4,9 @@ import java.util.Locale; +/** + * A implementation of a LocalString, which does support formatting, but does not support translations. + */ public class StaticLocalString extends LocalStringBase { public static final Locale DEFAULT_LOCALE = Locale.ENGLISH; diff --git a/common/src/main/java/eu/mizerak/alemiz/translationlib/common/string/StringWrapper.java b/common/src/main/java/eu/mizerak/alemiz/translationlib/common/string/StringWrapper.java new file mode 100644 index 0000000..dcf6c75 --- /dev/null +++ b/common/src/main/java/eu/mizerak/alemiz/translationlib/common/string/StringWrapper.java @@ -0,0 +1,68 @@ +package eu.mizerak.alemiz.translationlib.common.string; + +import java.util.Locale; +import java.util.function.Function; + +/** + * A very simple implementation of a LocalString, which does not do any translations or formatting + * nor does support language translations. + */ +public class StringWrapper implements LocalString { + public static StringWrapper EMPTY = new StringWrapper<>(""); + + private final String text; + + protected StringWrapper(String text) { + this.text = text; + } + + @Override + public String getKey() { + return "wrapper"; + } + + @Override + public LocalString reload() { + throw new UnsupportedOperationException("Static string does not support reload"); + } + + @Override + public LocalString enableReload(boolean b) { + throw new UnsupportedOperationException("Static string does not support reload"); + } + + @Override + public LocalString withArgument(String s, Function, String> function) { + return this; + } + + @Override + public LocalString clearArguments() { + return this; + } + + @Override + public String getFormatted() { + return this.text; + } + + @Override + public String getFormatted(Locale locale) { + return this.text; + } + + @Override + public String getText(T t) { + return this.text; + } + + @Override + public void uploadFallbackMessage() { + throw new UnsupportedOperationException("Static can not be uploaded"); + } + + @Override + public String toString() { + return this.text; + } +} diff --git a/common/src/test/java/eu/mizerak/alemiz/translationlib/common/TranslateTest.java b/common/src/test/java/eu/mizerak/alemiz/translationlib/common/TranslateTest.java index d296a54..18ed5e5 100644 --- a/common/src/test/java/eu/mizerak/alemiz/translationlib/common/TranslateTest.java +++ b/common/src/test/java/eu/mizerak/alemiz/translationlib/common/TranslateTest.java @@ -53,4 +53,14 @@ public void testStatic() { assertEquals("Hi " + User.ENGLISH.getName(), string1.getText(User.ENGLISH)); } + + @Test + public void testJoinedStrings() { + LocalString string = LocalString.from("test_string4", "Hello {user}") + .withArgument("user", ctx -> ctx.getObject().getName()); + LocalString string2 = LocalString.immutable(" this is test"); + + String text = string.append(string2).getText(User.ENGLISH); + assertEquals("Hello " + User.ENGLISH.getName() + " this is test", text); + } } diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 2cc94fe..2dd09c7 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -24,4 +24,4 @@ avaje-inject-generator = { module = "io.avaje:avaje-inject-generator", version.r # Other dependencies lombok = { module = "org.projectlombok:lombok", version.ref = "lombok" } gson = { module = "com.google.code.gson:gson", version = "2.10.1" } -mongo-driver = { module = "org.mongodb:mongodb-driver-sync", version = "4.7.0" } \ No newline at end of file +sqlite-driver = { module = "org.xerial:sqlite-jdbc", version = "3.41.2.0" } \ No newline at end of file diff --git a/service/build.gradle.kts b/service/build.gradle.kts index 4766d4d..832a17c 100644 --- a/service/build.gradle.kts +++ b/service/build.gradle.kts @@ -9,7 +9,7 @@ group = "eu.mizerak.alemiz.translationlib.service" dependencies { implementation(project(":common")) implementation(libs.slf4j.simple) - implementation(libs.mongo.driver) + implementation(libs.sqlite.driver) implementation(libs.javalin) implementation(libs.avaje.inject) diff --git a/service/src/main/java/eu/mizerak/alemiz/translationlib/service/TranslationLibService.java b/service/src/main/java/eu/mizerak/alemiz/translationlib/service/TranslationLibService.java index 1948908..ac3e34d 100644 --- a/service/src/main/java/eu/mizerak/alemiz/translationlib/service/TranslationLibService.java +++ b/service/src/main/java/eu/mizerak/alemiz/translationlib/service/TranslationLibService.java @@ -3,11 +3,6 @@ import com.google.gson.FieldNamingPolicy; import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import com.mongodb.ConnectionString; -import com.mongodb.MongoClientSettings; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import com.mongodb.client.MongoDatabase; import eu.mizerak.alemiz.translationlib.common.gson.LocaleSerializer; import eu.mizerak.alemiz.translationlib.common.structure.RestStatus; import eu.mizerak.alemiz.translationlib.service.access.AccessRole; @@ -79,27 +74,12 @@ public static Configuration loadConfiguration() throws IOException { private final Javalin server; private final Gson gson; - private final MongoClient mongoClient; - private final MongoDatabase mongoDatabase; - private final TranslationDataScrapper scrapper; private final TermsManager termsManager; public TranslationLibService(Configuration configuration) { this.configuration = configuration; - MongoClientSettings settings = MongoClientSettings.builder() - .applyConnectionString(new ConnectionString(configuration.getMongoUrl())) - .applyToConnectionPoolSettings(builder -> builder - .minSize(2) - .maxSize(configuration.getMaxMongoPoolSize())) - .applyToSocketSettings(builder -> builder - .connectTimeout(10, TimeUnit.SECONDS) - .readTimeout(10, TimeUnit.SECONDS)) - .build(); - this.mongoClient = MongoClients.create(settings); - this.mongoDatabase = mongoClient.getDatabase(configuration.getMongoDatabase()); - this.gson = new GsonBuilder() .registerTypeAdapter(Locale.class, new LocaleSerializer()) .registerTypeAdapterFactory(new DataCollectionTypeAdapterFactory()) @@ -110,8 +90,6 @@ public TranslationLibService(Configuration configuration) { BeanScope scope = BeanScope.builder() .bean(TranslationLibService.class, this) .bean(Configuration.class, configuration) - .bean(MongoClient.class, this.mongoClient) - .bean(MongoDatabase.class, this.mongoDatabase) .bean(Gson.class, this.gson) .build(); diff --git a/service/src/main/java/eu/mizerak/alemiz/translationlib/service/manager/TermsManager.java b/service/src/main/java/eu/mizerak/alemiz/translationlib/service/manager/TermsManager.java index 57fdb10..e377e6c 100644 --- a/service/src/main/java/eu/mizerak/alemiz/translationlib/service/manager/TermsManager.java +++ b/service/src/main/java/eu/mizerak/alemiz/translationlib/service/manager/TermsManager.java @@ -27,7 +27,7 @@ public class TermsManager { private final Map groups = new ConcurrentHashMap<>(); public void onInit() { - Collection terms = this.repository.getAllTerms(null); + Collection terms = this.repository.getAllTerms(); for (TranslationTerm term : terms) { this.terms.put(term.getKey(), term); for (String tag : term.getTags()) { diff --git a/service/src/main/java/eu/mizerak/alemiz/translationlib/service/repository/TermsRepository.java b/service/src/main/java/eu/mizerak/alemiz/translationlib/service/repository/TermsRepository.java index af3d585..8440e29 100644 --- a/service/src/main/java/eu/mizerak/alemiz/translationlib/service/repository/TermsRepository.java +++ b/service/src/main/java/eu/mizerak/alemiz/translationlib/service/repository/TermsRepository.java @@ -1,24 +1,19 @@ package eu.mizerak.alemiz.translationlib.service.repository; import com.google.gson.Gson; -import com.mongodb.client.FindIterable; -import com.mongodb.client.MongoCursor; -import com.mongodb.client.MongoDatabase; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.ReplaceOptions; -import com.mongodb.client.model.UpdateOptions; +import com.google.gson.reflect.TypeToken; import eu.mizerak.alemiz.translationlib.common.structure.TranslationTerm; +import eu.mizerak.alemiz.translationlib.service.utils.Configuration; +import io.avaje.inject.PostConstruct; +import io.avaje.inject.PreDestroy; import jakarta.inject.Inject; import jakarta.inject.Singleton; import lombok.extern.slf4j.Slf4j; -import org.bson.Document; -import org.bson.conversions.Bson; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; +import java.lang.reflect.Type; +import java.sql.*; +import java.util.*; @Slf4j @Singleton @@ -26,55 +21,139 @@ public class TermsRepository { public static final String COLLECTION = "terms"; @Inject - MongoDatabase database; + Configuration config; @Inject Gson gson; + Connection connection; + + @PostConstruct + void onStartup() { + try { + Class.forName("org.sqlite.JDBC"); + this.connection = DriverManager.getConnection("jdbc:sqlite:" + this.config.getTermsDatabase()); + } catch (Exception e) { + throw new IllegalStateException("Unable to open connection to SQLite database", e); + } + + try (Statement statement = this.connection.createStatement()) { + statement.execute("CREATE TABLE IF NOT EXISTS `terms` (`key` varchar(200) NOT NULL PRIMARY KEY, `internal_id` varchar(200) DEFAULT NULL, " + + "`tags` MEDIUMTEXT DEFAULT NULL, `translations` LONGTEXT NOT NULL);"); + } catch (SQLException e) { + throw new IllegalStateException("Unable to init terms database"); + } + } + + @PreDestroy + void onShutdown() throws SQLException { + if (this.connection != null) { + this.connection.close(); + } + } + public void addTerm(@NotNull TranslationTerm term) { - Bson filter = Filters.eq("key", term.getKey()); - this.database.getCollection(COLLECTION).replaceOne(filter, createDocument(term), - new ReplaceOptions().upsert(true)); + try { + if (this.isTermCreated(term.getKey())) { + this.updateTerm(term); + } else { + this.insertTerm(term); + } + } catch (SQLException e) { + throw new IllegalStateException("Unable to update translation term " + term.getKey(), e); + } } - public void addTerms(Collection terms) { - List documents = new ArrayList<>(); - for (TranslationTerm term : terms) { - documents.add(createDocument(term)); + private void updateTerm(@NotNull TranslationTerm term) throws SQLException { + try (PreparedStatement statement = this.connection.prepareStatement("UPDATE `terms` SET `internal_id` = ?, `tags` = ?, `translations` = ? WHERE `key` = ?")) { + if (term.getInternalId() == null || term.getInternalId().trim().isEmpty()) { + statement.setNull(1, Types.NULL); + } else { + statement.setString(1, term.getInternalId()); + } + + if (term.getTags() == null || term.getTags().isEmpty()) { + statement.setNull(2, Types.NULL); + } else { + statement.setString(2, String.join(",", term.getTags())); + } + statement.setString(3, gson.toJson(term.getTranslations())); + statement.setString(4, term.getKey()); + + statement.execute(); } - this.database.getCollection(COLLECTION).insertMany(documents); } - public void removeTerm(@NotNull TranslationTerm term) { - this.database.getCollection(COLLECTION).deleteOne(Filters.eq("key", term.getKey())); + private void insertTerm(@NotNull TranslationTerm term) throws SQLException { + try (PreparedStatement statement = this.connection.prepareStatement("INSERT INTO `terms` (`key`, `internal_id`, `tags`, `translations`) VALUES (?, ?, ?, ?);")) { + statement.setString(1, term.getKey()); + if (term.getInternalId() == null || term.getInternalId().trim().isEmpty()) { + statement.setNull(2, Types.NULL); + } else { + statement.setString(2, term.getInternalId()); + } + + if (term.getTags() == null || term.getTags().isEmpty()) { + statement.setNull(3, Types.NULL); + } else { + statement.setString(3, String.join(",", term.getTags())); + } + statement.setString(4, gson.toJson(term.getTranslations())); + + statement.execute(); + } } - public boolean isTermCreated(@NotNull System key) { - return this.database.getCollection(COLLECTION).countDocuments(Filters.eq("key", key)) > 1; + public void addTerms(Collection terms) { + for (TranslationTerm term : terms) { + this.addTerm(term); + } } - public Collection getAllTerms(@Nullable Bson filter) { - FindIterable documents; - if (filter == null) { - documents = this.database.getCollection(COLLECTION).find(); - } else { - documents = this.database.getCollection(COLLECTION).find(filter); + public void removeTerm(@NotNull TranslationTerm term) { + try (PreparedStatement statement = this.connection.prepareStatement("DELETE FROM `terms` WHERE `key` = ?;")) { + statement.setString(1, term.getKey()); + statement.execute(); + } catch (SQLException e) { + throw new IllegalStateException("Unable to delete term " + term.getKey(), e); } + } - List terms = new ArrayList<>(); - try (MongoCursor cursor = documents.cursor()) { - while (cursor.hasNext()) { - terms.add(createTerm(cursor.next())); + public boolean isTermCreated(@NotNull String key) { + try (PreparedStatement queryStatement = this.connection.prepareStatement("SELECT `key` FROM `terms` WHERE `key` = ?")) { + queryStatement.setString(1, key); + try (ResultSet query = queryStatement.executeQuery()) { + return query.getRow() > 0; } + } catch (SQLException e) { + throw new IllegalStateException("Unable check if term exists " + key, e); } - return terms; } - private Document createDocument(TranslationTerm term) { - return Document.parse(gson.toJson(term)); - } + public Collection getAllTerms() { + try (Statement statement = this.connection.createStatement()) { + try (ResultSet resultSet = statement.executeQuery("SELECT * FROM `terms`;")) { + List terms = new ArrayList<>(); + while (resultSet.next()) { + TranslationTerm term = new TranslationTerm(); + term.setKey(resultSet.getString("key")); + term.setInternalId(resultSet.getString("internal_id")); - private TranslationTerm createTerm(Document document) { - return gson.fromJson(document.toJson(), TranslationTerm.class); + String tags = resultSet.getString("tags"); + if (tags != null && !tags.trim().isEmpty()) { + term.getTags().addAll(Arrays.asList(tags.split(","))); + } + + String translations = resultSet.getString("translations"); + Type type = new TypeToken>(){}.getType(); + term.getTranslations().putAll(gson.fromJson(translations, type)); + + terms.add(term); + } + return terms; + } + } catch (SQLException e) { + throw new IllegalStateException("Unable to load terms from database", e); + } } } diff --git a/service/src/main/java/eu/mizerak/alemiz/translationlib/service/utils/Configuration.java b/service/src/main/java/eu/mizerak/alemiz/translationlib/service/utils/Configuration.java index 5763ff1..a37752b 100644 --- a/service/src/main/java/eu/mizerak/alemiz/translationlib/service/utils/Configuration.java +++ b/service/src/main/java/eu/mizerak/alemiz/translationlib/service/utils/Configuration.java @@ -13,9 +13,7 @@ public class Configuration { private String httpHost; private int httpPort; - private String mongoUrl; - private String mongoDatabase; - private int maxMongoPoolSize; + private String termsDatabase; private List accessTokens; @@ -25,11 +23,9 @@ public static Configuration parse(Properties properties) { return Configuration.builder() .properties(properties) .httpHost(parseValue("http.host", properties)) + .termsDatabase(parseValue("termsdb-file", properties)) .httpPort(Integer.parseInt(parseValue("http.port", properties))) - .mongoUrl(parseValue("mongodb.connection-string", properties)) - .mongoDatabase(parseValue("mongodb.database", properties)) - .maxMongoPoolSize(Integer.parseInt(parseValue("mongodb.max-pool-size", properties))) - .accessTokens(Collections.unmodifiableList(Arrays.asList(parseValue("service.access-tokens", properties).split(",")))) + .accessTokens(List.of(parseValue("service.access-tokens", properties).split(","))) .scrapperName(parseValue("scrapper.name", properties)) .build(); } diff --git a/service/src/main/resources/service.properties b/service/src/main/resources/service.properties index dbc2c9a..7f8a306 100644 --- a/service/src/main/resources/service.properties +++ b/service/src/main/resources/service.properties @@ -5,10 +5,8 @@ http.host=0.0.0.0 http.port=8080 -# MongoDB settings -mongodb.connection-string=${MONGODB_URL} -mongodb.database=${MONGODB_DATABASE:translate_lib} -mongodb.max-pool-size=8 +# Translations db path +translations-file=${TRANSLATIONS_FILE:translations.db} # Access tokens divided by semicon service.access-tokens=my_secret_token,my_secret_token2 diff --git a/service/src/main/resources/simplelogger.properties b/service/src/main/resources/simplelogger.properties index 348b21a..0d9ea37 100644 --- a/service/src/main/resources/simplelogger.properties +++ b/service/src/main/resources/simplelogger.properties @@ -6,7 +6,4 @@ org.slf4j.simpleLogger.showShortLogName=true org.slf4j.simpleLogger.logFile=System.out # Adjust the logger to show time in readable format org.slf4j.simpleLogger.showDateTime=true -org.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss -# Adjust MongoDB logger levels -org.slf4j.simpleLogger.log.org.mongodb.driver.cluster=warn -org.slf4j.simpleLogger.log.org.mongodb.driver.connection=warn \ No newline at end of file +org.slf4j.simpleLogger.dateTimeFormat=HH:mm:ss \ No newline at end of file