From 51d2d6b256318624db7caf230efaf7f07cb7ad32 Mon Sep 17 00:00:00 2001 From: Chen Xu Date: Thu, 11 Jan 2024 01:39:49 +0800 Subject: [PATCH 1/3] Dedup --- .vscode/settings.json | 1 - Cargo.lock | 176 +++++++++++++++++++++++++++ Cargo.toml | 4 +- client-interface/src/lib.rs | 2 + websocket-client/src/lib.rs | 38 +++--- websocket-server/Cargo.toml | 3 + websocket-server/src/global_state.rs | 120 +++++++++++++++--- websocket-server/src/lib.rs | 16 ++- websocket-server/src/search.rs | 153 +++++++++++++++++++---- 9 files changed, 454 insertions(+), 59 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 26770e7..5f209f2 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,4 @@ { - "rust-analyzer.check.allTargets": false, "editor.formatOnSave": true, "[toml]": { "editor.formatOnSave": false diff --git a/Cargo.lock b/Cargo.lock index 3c4751b..2b683b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -147,6 +147,15 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" +[[package]] +name = "async-lock" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b" +dependencies = [ + "event-listener", +] + [[package]] name = "async-trait" version = "0.1.75" @@ -253,6 +262,12 @@ version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +[[package]] +name = "bytecount" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e5f035d16fc623ae5f74981db80a439803888314e3a555fd6f04acd51a3205" + [[package]] name = "bytemuck" version = "1.14.0" @@ -271,6 +286,37 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +[[package]] +name = "camino" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-platform" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ceed8ef69d8518a5dda55c07425450b58a4e1946f4951eab6d7191ee86c2443d" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", +] + [[package]] name = "cc" version = "1.0.83" @@ -824,6 +870,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "error-chain" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" +dependencies = [ + "version_check", +] + [[package]] name = "error-code" version = "2.3.1" @@ -834,6 +889,12 @@ dependencies = [ "str-buf", ] +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + [[package]] name = "exr" version = "1.71.0" @@ -1127,6 +1188,12 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "h2" version = "0.3.22" @@ -1229,6 +1296,12 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "home" version = "0.5.9" @@ -1749,6 +1822,30 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "moka" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cebde309854872ea4fcaf4d7c870ad8d5873091c6bfb7ce91fd08ea648f20b0" +dependencies = [ + "async-lock", + "async-trait", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "futures-util", + "once_cell", + "parking_lot", + "quanta", + "rustc_version", + "skeptic", + "smallvec", + "tagptr", + "thiserror", + "triomphe", + "uuid", +] + [[package]] name = "mqtt-client" version = "0.6.0" @@ -2177,6 +2274,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "pulldown-cmark" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a1a2f1f0a7ecff9c31abbe177637be0e97a0aef46cf8738ece09327985d998" +dependencies = [ + "bitflags 1.3.2", + "memchr", + "unicase", +] + [[package]] name = "qoi" version = "0.4.1" @@ -2186,6 +2294,21 @@ dependencies = [ "bytemuck", ] +[[package]] +name = "quanta" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca0b7bac0b97248c40bb77288fc52029cf1459c0461ea1b05ee32ccf011de2c" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + [[package]] name = "quick-xml" version = "0.22.0" @@ -2243,6 +2366,15 @@ dependencies = [ "fastrand 1.9.0", ] +[[package]] +name = "raw-cpuid" +version = "11.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d86a7c4638d42c44551f4791a20e687dbb4c3de1f33c43dd71e355cd429def1" +dependencies = [ + "bitflags 2.4.1", +] + [[package]] name = "raw-window-handle" version = "0.5.2" @@ -2634,6 +2766,9 @@ name = "semver" version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" +dependencies = [ + "serde", +] [[package]] name = "serde" @@ -2698,6 +2833,17 @@ dependencies = [ "digest", ] +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -2713,6 +2859,21 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" +[[package]] +name = "skeptic" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16d23b015676c90a0f01c197bfdc786c20342c73a0afdda9025adb0bc42940a8" +dependencies = [ + "bytecount", + "cargo_metadata", + "error-chain", + "glob", + "pulldown-cmark", + "tempfile", + "walkdir", +] + [[package]] name = "sketches-ddsketch" version = "0.2.1" @@ -2827,6 +2988,12 @@ dependencies = [ "libc", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "tantivy" version = "0.21.1" @@ -3300,6 +3467,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "triomphe" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859eb650cfee7434994602c3a68b25d77ad9e68c8a6cd491616ef86661382eb3" + [[package]] name = "try-lock" version = "0.2.5" @@ -3614,10 +3787,13 @@ dependencies = [ "clap 4.4.12", "client-interface", "futures-util", + "hex", "log", + "moka", "poem", "serde", "serde_json", + "sha2", "tantivy", "tokio", ] diff --git a/Cargo.toml b/Cargo.toml index 9cd65cd..3768e07 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,9 +24,11 @@ platform-dirs = "0.3" futures-util = "0.3" poem = { version = "2" } tantivy = { version = "0.21" } -tantivy-jieba = "0.10.0" chrono = "0.4" png = { version = "0.17" } +sha2 = { version = "0.10" } +hex = { version = "0.4" } +moka = { version = "0.12", features = ["future"] } reqwest = { version = "0.11" } rumqttc = { version = "0.23" } arboard = { version = "3" } diff --git a/client-interface/src/lib.rs b/client-interface/src/lib.rs index 2e8b89b..1cd6a8f 100644 --- a/client-interface/src/lib.rs +++ b/client-interface/src/lib.rs @@ -115,6 +115,8 @@ mod ws { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct ServerClipboardRecord { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, pub source: String, #[serde(flatten)] pub content: ServerClipboardContent, diff --git a/websocket-client/src/lib.rs b/websocket-client/src/lib.rs index a7751e4..3b56158 100644 --- a/websocket-client/src/lib.rs +++ b/websocket-client/src/lib.rs @@ -229,23 +229,29 @@ impl WebSocketSink { impl ClipboardSink for WebSocketSink { async fn publish(&mut self, data: Option) -> anyhow::Result<()> { let raw_string = match data { - Some(data) => match data.content { - ClipboardContent::Text(text) => { - let data = ServerClipboardRecord { - source: data.source, - content: ServerClipboardContent::Text(text), - }; - Some(serde_json::to_string(&data)?) + Some(data) => { + match data.content { + ClipboardContent::Text(text) => { + let data = ServerClipboardRecord { + id: None, + source: data.source, + content: ServerClipboardContent::Text(text), + }; + Some(serde_json::to_string(&data)?) + } + ClipboardContent::Image(img) => { + // Convert data to ServerClipboardData + let data = ServerClipboardRecord { + id: None, + source: data.source, + content: ServerClipboardContent::ImageUrl( + self.upload_image(&img).await?, + ), + }; + Some(serde_json::to_string(&data)?) + } } - ClipboardContent::Image(img) => { - // Convert data to ServerClipboardData - let data = ServerClipboardRecord { - source: data.source, - content: ServerClipboardContent::ImageUrl(self.upload_image(&img).await?), - }; - Some(serde_json::to_string(&data)?) - } - }, + } None => None, }; self.publish_raw_string(raw_string).await?; diff --git a/websocket-server/Cargo.toml b/websocket-server/Cargo.toml index 554c44d..54042a4 100644 --- a/websocket-server/Cargo.toml +++ b/websocket-server/Cargo.toml @@ -16,5 +16,8 @@ poem = { workspace = true, features = ["websocket", "rustls", "static-files", "m tantivy = { workspace = true } chrono = { workspace = true } futures-util = { workspace = true } +moka = { workspace = true } +sha2 = { workspace = true } +hex = { workspace = true } client-interface = { workspace = true, features = ["websocket"] } \ No newline at end of file diff --git a/websocket-server/src/global_state.rs b/websocket-server/src/global_state.rs index dd453cb..508bd55 100644 --- a/websocket-server/src/global_state.rs +++ b/websocket-server/src/global_state.rs @@ -2,7 +2,10 @@ use std::{collections::HashSet, path::PathBuf}; use client_interface::ServerClipboardContent; use log::{debug, info, warn}; +use moka::future::Cache; +use sha2::Digest; use tokio::{ + io::AsyncReadExt, runtime::{Builder, Handle}, sync::broadcast::Sender, }; @@ -17,6 +20,7 @@ pub struct GlobalState { _rt: tokio::runtime::Runtime, thread_pool: Handle, image_path: PathBuf, + cache: Cache, } impl GlobalState { @@ -40,6 +44,7 @@ impl GlobalState { _rt: rt, thread_pool: handle, image_path: args.image_path.clone().unwrap(), + cache: Cache::new(10_000), } } @@ -72,38 +77,53 @@ impl GlobalState { self.online_device_list.iter().cloned().collect() } - pub async fn add_entry(&self, msg: ClipboardMessage, store: bool) -> anyhow::Result<()> { + pub async fn add_entry(&self, mut msg: ClipboardMessage, store: bool) -> anyhow::Result<()> { debug!("Publishing message: {:?}", msg); self.sender.send(msg.clone())?; if self.validate_message_content(&msg).await.is_err() { warn!("Ignored invalid clipboard entry."); return Ok(()); } - if matches!(msg.entry.content, ServerClipboardContent::Text(_)) { - let search = self.search.clone(); - self.thread_pool - .spawn_blocking(move || -> anyhow::Result<()> { - if store { - debug!("Store clipboard entry {:?}", msg); - match search.add_entry(&msg) { - Ok(_) => {} - Err(e) => { - warn!("Failed to store clipboard entry: {}", e); - } + match &msg.entry.content { + ServerClipboardContent::ImageUrl(url) => { + let digest = self.image_digest(url).await?; + msg.entry.id = Some(digest); + } + ServerClipboardContent::Text(text) => { + let mut hasher = ::new(); + hasher.update(text.as_bytes()); + let digest = hex::encode(std::convert::Into::<[u8; 64]>::into(hasher.finalize())); + msg.entry.id = Some(digest); + } + } + let search = self.search.clone(); + self.thread_pool + .spawn_blocking(move || -> anyhow::Result<()> { + if store { + debug!("Store clipboard entry {:?}", msg); + match search.add_entry(&msg) { + Ok(_) => {} + Err(e) => { + warn!("Failed to store clipboard entry: {}", e); } } - Ok(()) - }) - .await??; - } + } + Ok(()) + }) + .await??; Ok(()) } pub async fn query(&self, param: QueryParam) -> anyhow::Result { let search = self.search.clone(); - self.thread_pool + let result = self + .thread_pool .spawn_blocking(move || -> anyhow::Result { search.query(param) }) - .await? + .await??; + for msg in result.data.iter() { + self.update_image_digest_cache(msg).await; + } + Ok(result) } async fn validate_message_content(&self, msg: &ClipboardMessage) -> anyhow::Result<()> { @@ -123,4 +143,68 @@ impl GlobalState { } Ok(()) } + + pub async fn get_entry_by_id(&self, id: &str) -> anyhow::Result> { + let search = self.search.clone(); + let id = id.to_string(); + match self + .thread_pool + .spawn_blocking(move || search.get_entry_by_id(&id)) + .await? + { + Ok(Some(msg)) => { + self.update_image_digest_cache(&msg).await; + Ok(Some(msg)) + } + Ok(None) => Ok(None), + Err(e) => Err(e), + } + } + + async fn update_image_digest_cache(&self, msg: &ClipboardMessage) { + // Whenever we do query, we update the image digest cache if possible. + if let ServerClipboardContent::ImageUrl(url) = &msg.entry.content { + self.cache + .insert( + url.to_string(), + msg.entry + .id + .to_owned() + .expect("Internal error, index corrupted."), + ) + .await; + } + } + + async fn image_digest(&self, url: &str) -> anyhow::Result { + let path = self.image_path.join(url); + let path = path.to_str().unwrap(); + let digest = self + .cache + .get_with(path.to_string(), async move { + let Ok(mut file) = tokio::fs::File::open(path).await else { + return Default::default(); + }; + let mut buf = Vec::with_capacity(4096); + let mut read_bytes = 0; + let mut hasher = ::new(); + while let Ok(n) = file.read_buf(&mut buf).await { + if n == 0 { + break; + } + read_bytes += n; + hasher.update(&buf[0..n]); + buf.clear(); + } + if read_bytes == 0 { + return Default::default(); + } + hex::encode(std::convert::Into::<[u8; 64]>::into(hasher.finalize())) + }) + .await; + if digest.is_empty() { + anyhow::bail!("Image not found."); + } + Ok(digest) + } } diff --git a/websocket-server/src/lib.rs b/websocket-server/src/lib.rs index e494b07..09676ab 100644 --- a/websocket-server/src/lib.rs +++ b/websocket-server/src/lib.rs @@ -1,7 +1,7 @@ use std::{path::PathBuf, sync::Arc, time::Duration}; use chrono::Utc; -use client_interface::ClipboardMessage; +use client_interface::{ClipboardMessage, ServerClipboardContent}; use futures_util::{SinkExt, StreamExt}; use log::{debug, info, trace, warn}; use poem::{ @@ -17,6 +17,7 @@ use poem::{ }, EndpointExt, IntoResponse, Request, Route, Server, }; +use sha2::Digest; use tokio::sync::{broadcast::channel, RwLock}; use crate::global_state::GlobalState; @@ -181,7 +182,7 @@ async fn upload_image( } break; } - let filename = format!("{}-{}.png", timestamp, suffix); + let filename: String = format!("{}-{}.png", timestamp, suffix); let filepath: PathBuf = dir.join(&filename); let part_name = field.name().map(ToString::to_string); let file_name = field.file_name().map(ToString::to_string); @@ -193,6 +194,16 @@ async fn upload_image( bytes.len(), filepath, ); + let mut hasher = ::new(); + hasher.update(&bytes); + let digest = hex::encode(Into::<[u8; 64]>::into(hasher.finalize())); + let existing_entry = data.0.read().await.get_entry_by_id(&digest).await.unwrap(); + if let Some(existing_entry) = existing_entry { + if let ServerClipboardContent::ImageUrl(url) = &existing_entry.entry.content { + debug!("Image already exists: {:?}", existing_entry); + return Ok(url.clone()); + } + } tokio::fs::write(&filepath, bytes).await.map_err(|e| { warn!("Failed to write file: {}", e); poem::Error::from_status(StatusCode::INTERNAL_SERVER_ERROR) @@ -296,6 +307,7 @@ mod tests { fn test_serde() { use client_interface::{ServerClipboardContent, ServerClipboardRecord}; let data = ServerClipboardRecord { + id: None, source: "test".to_string(), content: ServerClipboardContent::Text("test".to_string()), }; diff --git a/websocket-server/src/search.rs b/websocket-server/src/search.rs index a595e6e..8ce4552 100644 --- a/websocket-server/src/search.rs +++ b/websocket-server/src/search.rs @@ -7,7 +7,7 @@ use tantivy::{ directory::MmapDirectory, doc, merge_policy::LogMergePolicy, - query::{AllQuery, BooleanQuery, Query, QueryParser, RangeQuery, TermSetQuery}, + query::{AllQuery, BooleanQuery, Query, QueryParser, RangeQuery, TermQuery, TermSetQuery}, query_grammar::Occur, schema::{Field, IndexRecordOption, Schema, TextFieldIndexing, TextOptions, FAST, STORED}, tokenizer::{LowerCaser, NgramTokenizer, TextAnalyzer}, @@ -22,8 +22,10 @@ const TOKENIZER_NAME: &str = "ngram_m_n"; pub struct Search { index: Index, reader: IndexReader, + id: Field, source: Field, content: Field, + url: Field, timestamp: Field, query_parser: QueryParser, } @@ -50,8 +52,10 @@ impl Search { .set_index_option(IndexRecordOption::WithFreqsAndPositions), ) .set_stored(); - let source = schema_builder.add_text_field("source", token_options); + let id = schema_builder.add_text_field("id", token_options.clone()); + let source = schema_builder.add_text_field("source", token_options.clone()); let content = schema_builder.add_text_field("content", text_options); + let url = schema_builder.add_text_field("url", token_options); let timestamp = schema_builder.add_i64_field("timestamp", FAST | STORED); let schema = schema_builder.build(); let index = match index_path { @@ -62,7 +66,6 @@ impl Search { None => Index::create_in_ram(schema.clone()), }; index.tokenizers().register(TOKENIZER_NAME, tokenizer); - // .register("jieba", tantivy_jieba::JiebaTokenizer {}); let reader = index .reader_builder() .reload_policy(ReloadPolicy::OnCommit) @@ -74,28 +77,114 @@ impl Search { Self { index, reader, + id, source, content, + url, timestamp, query_parser, } } + pub fn get_entry_by_id(&self, id: &str) -> anyhow::Result> { + let q = TermQuery::new(Term::from_field_text(self.id, id), IndexRecordOption::Basic); + let collector = TopDocs::with_limit(1); + let searcher = self.reader.searcher(); + let result: Vec<(f64, DocAddress)> = searcher + .search(&q, &collector)? + .into_iter() + .map(|(ts, d)| (ts as f64, d)) + .collect(); + if result.is_empty() { + return Ok(None); + } + let (_, doc_address) = result[0]; + let doc = searcher.doc(doc_address)?; + let data = doc + .get_first(self.content) + .and_then(|v| v.as_text()) + .map(|v| v.to_string()) + .unwrap_or_default(); + let id = doc + .get_first(self.id) + .and_then(|v| v.as_text()) + .map(|v| v.to_string()) + .unwrap_or_default(); + let source = doc + .get_first(self.source) + .and_then(|v| v.as_text()) + .map(|v| v.to_string()) + .unwrap_or_default(); + let url = doc + .get_first(self.url) + .and_then(|v| v.as_text()) + .map(|v| v.to_string()) + .unwrap_or_default(); + let timestamp = doc + .get_first(self.timestamp) + .and_then(|v| v.as_i64()) + .unwrap_or_default(); + if url.is_empty() { + Ok(Some(ClipboardMessage { + entry: ServerClipboardRecord { + id: Some(id), + source: source.to_string(), + content: ServerClipboardContent::Text(data.to_string()), + }, + timestamp, + })) + } else { + Ok(Some(ClipboardMessage { + entry: ServerClipboardRecord { + id: Some(id), + source: source.to_string(), + content: ServerClipboardContent::ImageUrl(url.to_string()), + }, + timestamp, + })) + } + } + pub fn add_entry(&self, entry: &ClipboardMessage) -> anyhow::Result<()> { debug!("Adding entry: from {}", entry.entry.source); - if let ServerClipboardContent::Text(text) = &entry.entry.content { - let mut index_writer = self.index.writer(50_000_000)?; - index_writer.set_merge_policy(Box::::default()); - index_writer.add_document(doc!( - self.source => entry.entry.source.clone(), - self.content => text.clone(), - self.timestamp => entry.timestamp - ))?; - index_writer.commit()?; - } else { - // TODO: Save image to somewhere - debug!("Not text, skipping"); + assert!(entry.entry.id.is_some()); + let id = entry.entry.id.as_ref().unwrap().clone(); + let q = TermQuery::new( + Term::from_field_text(self.id, &id), + IndexRecordOption::Basic, + ); + let collector = TopDocs::with_limit(1); + let searcher = self.reader.searcher(); + let result: Vec<(f64, DocAddress)> = searcher + .search(&q, &collector)? + .into_iter() + .map(|(ts, d)| (ts as f64, d)) + .collect(); + if !result.is_empty() { + debug!("Entry already exists, skipping"); + return Ok(()); } + let mut index_writer = self.index.writer(50_000_000)?; + index_writer.set_merge_policy(Box::::default()); + index_writer.add_document(match &entry.entry.content { + ServerClipboardContent::Text(text) => { + doc!( + self.id => id, + self.source => entry.entry.source.clone(), + self.content => text.clone(), + self.timestamp => entry.timestamp + ) + } + ServerClipboardContent::ImageUrl(url) => { + doc!( + self.id => id, + self.source => entry.entry.source.clone(), + self.url => url.clone(), + self.timestamp => entry.timestamp + ) + } + })?; + index_writer.commit()?; Ok(()) } @@ -211,21 +300,43 @@ impl Search { .and_then(|v| v.as_text()) .map(|v| v.to_string()) .unwrap_or_default(); + let id = d + .get_first(self.id) + .and_then(|v| v.as_text()) + .map(|v| v.to_string()) + .unwrap_or_default(); let source = d .get_first(self.source) .and_then(|v| v.as_text()) .map(|v| v.to_string()) .unwrap_or_default(); + let url = d + .get_first(self.url) + .and_then(|v| v.as_text()) + .map(|v| v.to_string()) + .unwrap_or_default(); let timestamp = d .get_first(self.timestamp) .and_then(|v| v.as_i64()) .unwrap_or_default(); - ClipboardMessage { - entry: ServerClipboardRecord { - source: source.to_string(), - content: ServerClipboardContent::Text(data.to_string()), - }, - timestamp, + if url.is_empty() { + ClipboardMessage { + entry: ServerClipboardRecord { + id: Some(id), + source: source.to_string(), + content: ServerClipboardContent::Text(data.to_string()), + }, + timestamp, + } + } else { + ClipboardMessage { + entry: ServerClipboardRecord { + id: Some(id), + source: source.to_string(), + content: ServerClipboardContent::ImageUrl(url.to_string()), + }, + timestamp, + } } }) }) From 379518f4dbd37d1fb1ef12f31f705f8bfb2bd38f Mon Sep 17 00:00:00 2001 From: Chen Xu Date: Thu, 11 Jan 2024 02:26:09 +0800 Subject: [PATCH 2/3] 0.7.0 --- Cargo.lock | 16 ++++++++-------- Cargo.toml | 2 +- client-interface/Cargo.toml | 2 +- clip-sync-cli/Cargo.toml | 2 +- clip-sync-config/Cargo.toml | 2 +- clip-sync-server/Cargo.toml | 2 +- clip-sync-ui/package.json | 2 +- mqtt-client/Cargo.toml | 2 +- websocket-client/Cargo.toml | 2 +- websocket-server/Cargo.toml | 2 +- 10 files changed, 17 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2b683b0..ec47324 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -426,7 +426,7 @@ checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" [[package]] name = "client-interface" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "chrono", @@ -438,7 +438,7 @@ dependencies = [ [[package]] name = "clip-sync" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "arboard", @@ -464,7 +464,7 @@ dependencies = [ [[package]] name = "clip-sync-cli" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "chrono", @@ -492,7 +492,7 @@ dependencies = [ [[package]] name = "clip-sync-config" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "clap 4.4.12", @@ -510,7 +510,7 @@ dependencies = [ [[package]] name = "clip-sync-server" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "clap 4.4.12", @@ -1848,7 +1848,7 @@ dependencies = [ [[package]] name = "mqtt-client" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "bincode", @@ -3760,7 +3760,7 @@ dependencies = [ [[package]] name = "websocket-client" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "clap 4.4.12", @@ -3780,7 +3780,7 @@ dependencies = [ [[package]] name = "websocket-server" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 3768e07..ae5393e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,7 +50,7 @@ websocket-server = { path = "websocket-server" } [package] name = "clip-sync" authors = ["Chen Xu "] -version = "0.6.0" +version = "0.7.0" edition = "2021" description = "A clipboard sync tool" rust-version = "1.75.0" diff --git a/client-interface/Cargo.toml b/client-interface/Cargo.toml index d865d4a..36b621f 100644 --- a/client-interface/Cargo.toml +++ b/client-interface/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "client-interface" -version = "0.6.0" +version = "0.7.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/clip-sync-cli/Cargo.toml b/clip-sync-cli/Cargo.toml index 4fab4a8..fee462d 100644 --- a/clip-sync-cli/Cargo.toml +++ b/clip-sync-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "clip-sync-cli" -version = "0.6.0" +version = "0.7.0" edition = "2021" [dependencies] diff --git a/clip-sync-config/Cargo.toml b/clip-sync-config/Cargo.toml index 6a27caf..9fa7d20 100644 --- a/clip-sync-config/Cargo.toml +++ b/clip-sync-config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "clip-sync-config" -version = "0.6.0" +version = "0.7.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/clip-sync-server/Cargo.toml b/clip-sync-server/Cargo.toml index 0c4dd04..7b28e0a 100644 --- a/clip-sync-server/Cargo.toml +++ b/clip-sync-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "clip-sync-server" -version = "0.6.0" +version = "0.7.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/clip-sync-ui/package.json b/clip-sync-ui/package.json index a617b25..f7e2e80 100644 --- a/clip-sync-ui/package.json +++ b/clip-sync-ui/package.json @@ -1,7 +1,7 @@ { "name": "clip-sync-ui", "private": true, - "version": "0.5.0", + "version": "0.7.0", "type": "module", "scripts": { "dev": "vite", diff --git a/mqtt-client/Cargo.toml b/mqtt-client/Cargo.toml index 3063f36..9b34a01 100644 --- a/mqtt-client/Cargo.toml +++ b/mqtt-client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mqtt-client" -version = "0.6.0" +version = "0.7.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/websocket-client/Cargo.toml b/websocket-client/Cargo.toml index 59b2e5c..800ef78 100644 --- a/websocket-client/Cargo.toml +++ b/websocket-client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "websocket-client" -version = "0.6.0" +version = "0.7.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/websocket-server/Cargo.toml b/websocket-server/Cargo.toml index 54042a4..65c79c6 100644 --- a/websocket-server/Cargo.toml +++ b/websocket-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "websocket-server" -version = "0.6.0" +version = "0.7.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 9475e4c93af430e6092dcd869c962856310f16df Mon Sep 17 00:00:00 2001 From: Chen Xu Date: Thu, 11 Jan 2024 02:31:23 +0800 Subject: [PATCH 3/3] Server URL --- clip-sync-config/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clip-sync-config/src/lib.rs b/clip-sync-config/src/lib.rs index 00d84df..cb218a2 100644 --- a/clip-sync-config/src/lib.rs +++ b/clip-sync-config/src/lib.rs @@ -24,9 +24,9 @@ impl Args { pub fn get_server_url(&self) -> Option { if self.roles.contains(&"websocket-client".to_string()) { if let Ok(mut url) = url::Url::parse(&self.websocket_client.server_url) { - let scheme = if url.scheme() == "wss" { + let scheme = if (url.scheme() == "wss") || url.scheme() == "https" { "https" - } else if url.scheme() == "ws" { + } else if (url.scheme() == "ws") || url.scheme() == "http" { "http" } else { return None;