Merge pull request #19 from oramasearch/feat/adds-fst

feat: adds FST-based full-text search

nlp/src/lib.rs

@@ -34,6 +34,7 @@ impl Clone for TextParser {
 impl TextParser {
     pub fn from_language(locale: Locale) -> Self {
         let (tokenizer, stemmer) = match locale {
+            Locale::IT => (Tokenizer::italian(), Stemmer::create(Algorithm::Italian)),
             Locale::EN => (Tokenizer::english(), Stemmer::create(Algorithm::English)),
             // @todo: manage other locales
             _ => (Tokenizer::english(), Stemmer::create(Algorithm::English)),

nlp/src/tokenizer.rs

@@ -17,6 +17,14 @@ impl Tokenizer {
             stop_words,
         }
     }
+
+    pub fn italian() -> Self {
+        let stop_words: HashSet<&str> = Locale::IT.stop_words().unwrap();
+        Tokenizer {
+            split_regex: Locale::IT.split_regex().unwrap(),
+            stop_words
+        }
+    }
 
     pub fn tokenize<'a, 'b>(&'a self, input: &'b str) -> impl Iterator<Item = String> + 'b
     where

rustorama/.gitignore

@@ -0,0 +1 @@
+reviews.json

rustorama/README.md

@@ -0,0 +1,3 @@
+# Rustorama
+
+Download the dataset from https://www.kaggle.com/datasets/abdallahwagih/amazon-reviews and save it as `reviews.json`. Then run `deno run -A insert.js` to upload all the reviews to a local rustorama instance.

rustorama/config.json

@@ -0,0 +1,8 @@
+{
+  "data_dir": "/tmp/rustorama",
+  "http": {
+    "host": "127.0.0.1",
+    "port": 8080,
+    "allow_cors": true
+  }
+}