From c6aad2c9e8dc6460aa84997263cc29c68cc226bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=B3=A2?= Date: Tue, 30 Jan 2024 17:29:25 +0800 Subject: [PATCH 01/10] =?UTF-8?q?support=20fnv1a=5F64=20=E4=BB=A5=E5=8F=8A?= =?UTF-8?q?=20twemproxy=E6=94=AF=E6=8C=81=E7=9A=84=E5=8E=9F=E5=A7=8B?= =?UTF-8?q?=E7=89=88=E6=9C=AC=E7=9A=84ketama?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- endpoint/src/redisservice/config.rs | 29 +++++++- sharding/src/distribution/consistent.rs | 14 +++- sharding/src/distribution/mod.rs | 1 + sharding/src/hash/fnv1.rs | 18 +++++ sharding/src/hash/mod.rs | 5 +- tests/src/shard_test.rs | 95 +++++++++++++++++++++++++ 6 files changed, 156 insertions(+), 6 deletions(-) create mode 100644 sharding/src/hash/fnv1.rs diff --git a/endpoint/src/redisservice/config.rs b/endpoint/src/redisservice/config.rs index 9b01c4d3b..591e00c2d 100644 --- a/endpoint/src/redisservice/config.rs +++ b/endpoint/src/redisservice/config.rs @@ -14,6 +14,8 @@ const NO_CHECK_SUFFIX: &str = "-nocheck"; pub struct RedisNamespace { pub(crate) basic: Basic, pub(crate) backends: Vec, + // 对于一致性hash，为了确保ip变化后，分片不变，一般会为每组分片取一个name，来确定分片的hash始终固定 + pub(crate) backend_names: Vec, } #[derive(Debug, Clone, Default, Deserialize, Serialize)] @@ -51,8 +53,26 @@ impl RedisNamespace { return None; } + // check backends，分离出names + let mut backends = Vec::with_capacity(ns.backends.len()); + for b in &mut ns.backends { + let domain_name: Vec<&str> = b.split(" ").collect(); + // 后端地址格式： 域名,域名 name, name不能是rm、rs开头 + if domain_name.len() == 2 + && !domain_name[1].starts_with("rm") + && !domain_name[1].starts_with("rs") + { + backends.push(domain_name[0].to_string()); + ns.backend_names.push(domain_name[1].to_string()); + } + } + if backends.len() > 0 { + ns.backends = backends; + log::info!("+++ found redis backends with name: {}", cfg); + } + if !ns.validate_and_correct() { - log::error!("shards {} is not power of two: {}", ns.backends.len(), cfg); + log::error!("malformed names or shards {}: {}", ns.backends.len(), cfg); return None; } @@ -86,8 +106,8 @@ impl RedisNamespace { #[inline(always)] fn validate_and_correct(&mut self) -> bool { let dist = &self.basic.distribution; - // 需要检测dist时（默认场景），对于range/modrange类型的dist需要限制后端数量为2^n + // 需要检测dist时（默认场景），对于range/modrange类型的dist需要限制后端数量为2^n if dist.starts_with(sharding::distribution::DIST_RANGE) || dist.starts_with(sharding::distribution::DIST_MOD_RANGE) { @@ -103,6 +123,11 @@ impl RedisNamespace { } } + // 如果backend有name，则所有的后端都必须有name + if self.backend_names.len() > 0 && self.backend_names.len() != self.backends.len() { + return false; + } + true } } diff --git a/sharding/src/distribution/consistent.rs b/sharding/src/distribution/consistent.rs index 28d34eb7d..920616ddc 100644 --- a/sharding/src/distribution/consistent.rs +++ b/sharding/src/distribution/consistent.rs @@ -25,6 +25,10 @@ impl Consistent { } pub fn from>(shards: &[T]) -> Self { + Consistent::new(shards, false) + } + + pub fn new>(shards: &[T], origin_alg: bool) -> Self { let mut map = BTreeMap::default(); for idx in 0..shards.len() { let factor = 40; @@ -32,20 +36,24 @@ impl Consistent { let data: String = shards[idx].to_string() + "-" + &i.to_string(); let out_bytes = md5::compute(data.as_str()); for j in 0..4 { - let hash = (((out_bytes[3 + j * 4] & 0xFF) as i64) << 24) + let mut hash = (((out_bytes[3 + j * 4] & 0xFF) as i64) << 24) | (((out_bytes[2 + j * 4] & 0xFF) as i64) << 16) | (((out_bytes[1 + j * 4] & 0xFF) as i64) << 8) | ((out_bytes[0 + j * 4] & 0xFF) as i64); - let mut hash = hash.wrapping_rem(i32::MAX as i64); + // twemproxy 为代表的原始版ketama算法，不需要此计算, 但业务sdk的修正版需要此计算 + if !origin_alg { + hash = hash.wrapping_rem(i32::MAX as i64); + } + if hash < 0 { hash = hash.wrapping_mul(-1); } - map.insert(hash, idx); } } } + Self { buckets: map } } } diff --git a/sharding/src/distribution/mod.rs b/sharding/src/distribution/mod.rs index a9bee7457..7e26ffa35 100644 --- a/sharding/src/distribution/mod.rs +++ b/sharding/src/distribution/mod.rs @@ -68,6 +68,7 @@ impl Distribute { "modula" => Self::Modula(Modula::from(names.len(), false)), "absmodula" => Self::Modula(Modula::from(names.len(), true)), "ketama" => Self::Consistent(Consistent::from(names)), + "ketama-origin" => Self::Consistent(Consistent::new(names, true)), "range" => Self::Range(Range::from(num, names.len())), "modrange" => Self::ModRange(ModRange::from(num, names.len())), "splitmod" => Self::SplitMod(SplitMod::from(num, names.len())), diff --git a/sharding/src/hash/fnv1.rs b/sharding/src/hash/fnv1.rs new file mode 100644 index 000000000..c48057e42 --- /dev/null +++ b/sharding/src/hash/fnv1.rs @@ -0,0 +1,18 @@ +/// 按需支持fnv1系列所有相关的hash算法，目前支持fnv1a_64 + +#[derive(Debug, Default, Clone)] +pub struct Fnv1a64; + +const FNV_64_INIT: u64 = 0xcbf29ce484222325; +const FNV_64_PRIME: u64 = 0x100000001b3; + +impl super::Hash for Fnv1a64 { + fn hash(&self, key: &S) -> i64 { + let mut hash = FNV_64_INIT as u32; + for i in 0..key.len() { + hash ^= key.at(i) as u32; + hash = hash.wrapping_mul(FNV_64_PRIME as u32); + } + hash as i64 + } +} diff --git a/sharding/src/hash/mod.rs b/sharding/src/hash/mod.rs index a39665b99..92300d794 100644 --- a/sharding/src/hash/mod.rs +++ b/sharding/src/hash/mod.rs @@ -4,6 +4,7 @@ pub mod bkdrsub; pub mod crc32; pub mod crc32local; pub mod crc64; +pub mod fnv1; pub mod lbcrc32local; pub mod padding; pub mod random; @@ -26,7 +27,7 @@ pub mod crc; use enum_dispatch::enum_dispatch; -use self::{bkdrsub::Bkdrsub, crc64::Crc64}; +use self::{bkdrsub::Bkdrsub, crc64::Crc64, fnv1::Fnv1a64}; // 占位hash，主要用于兼容服务框架，供mq等业务使用 pub const HASH_PADDING: &str = "padding"; @@ -84,6 +85,7 @@ pub enum Hasher { Rawcrc32local(Rawcrc32local), // raw or crc32local Crc32Abs(Crc32Abs), // crc32abs: 基于i32转换，然后直接取abs；其他走i64提升为正数 Crc64(Crc64), // Crc64 算法，对整个key做crc64计算 + Fnv1a64(Fnv1a64), Random(RandomHash), // random hash RawSuffix(RawSuffix), } @@ -130,6 +132,7 @@ impl Hasher { "crc32abs" => Self::Crc32Abs(Default::default()), "crc64" => Self::Crc64(Default::default()), "random" => Self::Random(Default::default()), + "fnv1a64" => Self::Fnv1a64(Default::default()), _ => { // 默认采用mc的crc32-s hash log::error!("found unknown hash:{}, use crc32-short instead", alg); diff --git a/tests/src/shard_test.rs b/tests/src/shard_test.rs index 1c026ff35..02ccfae0f 100644 --- a/tests/src/shard_test.rs +++ b/tests/src/shard_test.rs @@ -574,3 +574,98 @@ fn crc64_file_check(shards_count: usize, file_name: &str) { println!("file:{}, succeed count:{}", file_name, success_count); assert!(success_count > 0); } + +#[test] +fn fnv1a_64_ketama_check() { + let root_dir = "sharding_datas/fnv1"; + let file_name = "fnv1a_64-ketama-data"; + let shard_file = format!("{}/{}", root_dir, file_name); + + let file = File::open(shard_file).unwrap(); + let mut reader = BufReader::new(file); + let mut success_count = 0; + + // 业务为了避免ip变更而导致节点变化，会增加nickname,like: node1,node2... + let shard_count = 256; + let mut servers = Vec::with_capacity(shard_count); + for i in 1..(shard_count + 1) { + servers.push(format!("node{}", i).to_string()); + } + let hasher = Hasher::from("fnv1a64"); + let dist = Distribute::from("ketama", &servers); + + const PORT_BASE: u16 = 58064; + let mut idx_dest = 0; + loop { + let mut line = String::with_capacity(64); + match reader.read_line(&mut line) { + Ok(len) => { + if len == 0 { + // println!("process file/{} completed!", file_name); + break; + } + line = line.trim().to_string(); + + if line.trim().len() == 0 || line.starts_with("#") { + // println!("ignore comment: {}", line); + continue; + } + + // port=58064 指示接下来的key所在的分片 + if line.starts_with("port=") { + let port_str = line.split("=").nth(1).unwrap(); + let port = u16::from_str_radix(port_str, 10).unwrap(); + idx_dest = (port - PORT_BASE) as usize; + println!("+++ will proc idx/port: {}/{}", idx_dest, port); + continue; + } + + let key = line; + let hash = hasher.hash(&key.as_bytes()); + let idx = dist.index(hash); + // assert_eq!( + // idx_dest, idx, + // "line/key: {} - {}:{}/{}", + // key, hash, idx_dest, idx + // ); + if idx_dest != idx { + println!( + "+++ wrong key: {}/{}, hash/idx:{}:{}, idx in redis:{}", + key, + key.len(), + hash, + idx, + idx_dest + ); + continue; + } + success_count += 1; + + println!("proc succeed line:{}", key); + } + Err(e) => { + println!("found err: {:?}", e); + break; + } + } + } + println!("file:{}, succeed count:{}", file_name, success_count); + assert!(success_count > 0); +} + +#[test] +fn fnv1_tmp_test() { + // 业务为了避免ip变更而导致节点变化，会增加nickname,like: node1,node2... + let shard_count = 256; + let mut servers = Vec::with_capacity(shard_count); + for i in 1..(shard_count + 1) { + servers.push(format!("node{}", i).to_string()); + } + let hasher = Hasher::from("fnv1a64"); + let dist = Distribute::from("ketama", &servers); + + let key = "throttle_android_6244642063"; + let hash = hasher.hash(&key.as_bytes()); + let idx = dist.index(hash); + println!("+++ key:{}, hash:{}, idx:{}", key, hash, idx); +} From a3439a2a7975c88837d578f8f8146914660373e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=B3=A2?= Date: Tue, 30 Jan 2024 18:50:33 +0800 Subject: [PATCH 02/10] update ketama origin name --- endpoint/src/redisservice/config.rs | 1 + endpoint/src/redisservice/topo.rs | 7 ++++++- sharding/src/distribution/consistent.rs | 1 + sharding/src/distribution/mod.rs | 2 +- sharding/src/hash/fnv1.rs | 1 + tests/src/shard_test.rs | 16 ++++++++++++++++ 6 files changed, 26 insertions(+), 2 deletions(-) diff --git a/endpoint/src/redisservice/config.rs b/endpoint/src/redisservice/config.rs index 591e00c2d..fe3bb5f05 100644 --- a/endpoint/src/redisservice/config.rs +++ b/endpoint/src/redisservice/config.rs @@ -15,6 +15,7 @@ pub struct RedisNamespace { pub(crate) basic: Basic, pub(crate) backends: Vec, // 对于一致性hash，为了确保ip变化后，分片不变，一般会为每组分片取一个name，来确定分片的hash始终固定 + #[serde(default)] pub(crate) backend_names: Vec, } diff --git a/endpoint/src/redisservice/topo.rs b/endpoint/src/redisservice/topo.rs index a37f19520..4dff8d163 100644 --- a/endpoint/src/redisservice/topo.rs +++ b/endpoint/src/redisservice/topo.rs @@ -126,7 +126,12 @@ where fn update(&mut self, namespace: &str, cfg: &str) { if let Some(ns) = RedisNamespace::try_from(cfg) { self.hasher = Hasher::from(&ns.basic.hash); - self.distribute = Distribute::from(ns.basic.distribution.as_str(), &ns.backends); + let backends = match ns.backend_names.len() { + 0 => &ns.backends, + _ => &ns.backend_names, + }; + log::debug!("+++ dist with backends:{:?}", backends); + self.distribute = Distribute::from(ns.basic.distribution.as_str(), backends); self.cfg.update(namespace, ns); } } diff --git a/sharding/src/distribution/consistent.rs b/sharding/src/distribution/consistent.rs index 920616ddc..117304a3c 100644 --- a/sharding/src/distribution/consistent.rs +++ b/sharding/src/distribution/consistent.rs @@ -29,6 +29,7 @@ impl Consistent { } pub fn new>(shards: &[T], origin_alg: bool) -> Self { + log::debug!("+++ use ketama with origin:{}", origin_alg); let mut map = BTreeMap::default(); for idx in 0..shards.len() { let factor = 40; diff --git a/sharding/src/distribution/mod.rs b/sharding/src/distribution/mod.rs index 7e26ffa35..a3ffb65f7 100644 --- a/sharding/src/distribution/mod.rs +++ b/sharding/src/distribution/mod.rs @@ -68,7 +68,7 @@ impl Distribute { "modula" => Self::Modula(Modula::from(names.len(), false)), "absmodula" => Self::Modula(Modula::from(names.len(), true)), "ketama" => Self::Consistent(Consistent::from(names)), - "ketama-origin" => Self::Consistent(Consistent::new(names, true)), + "ketama_origin" => Self::Consistent(Consistent::new(names, true)), "range" => Self::Range(Range::from(num, names.len())), "modrange" => Self::ModRange(ModRange::from(num, names.len())), "splitmod" => Self::SplitMod(SplitMod::from(num, names.len())), diff --git a/sharding/src/hash/fnv1.rs b/sharding/src/hash/fnv1.rs index c48057e42..8e910c721 100644 --- a/sharding/src/hash/fnv1.rs +++ b/sharding/src/hash/fnv1.rs @@ -13,6 +13,7 @@ impl super::Hash for Fnv1a64 { hash ^= key.at(i) as u32; hash = hash.wrapping_mul(FNV_64_PRIME as u32); } + log::debug!("+++ use Fnv1a64"); hash as i64 } } diff --git a/tests/src/shard_test.rs b/tests/src/shard_test.rs index 02ccfae0f..ff178d7e0 100644 --- a/tests/src/shard_test.rs +++ b/tests/src/shard_test.rs @@ -669,3 +669,19 @@ fn fnv1_tmp_test() { let idx = dist.index(hash); println!("+++ key:{}, hash:{}, idx:{}", key, hash, idx); } + +#[test] +fn print_fnv1_backends_for_vintage() { + for i in 0..256 { + let port = 58064 + i; + let master = format!( + " - rm{}.eos.grid.sina.com.cn:{},rs{}.hebe.grid.sina.com.cn:{} node{}", + port, + port, + port, + port, + i + 1, + ); + println!("{}", master); + } +} From 0b840d1c8f05d6d364c3d55dd219375677e45719 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=B3=A2?= Date: Wed, 31 Jan 2024 10:57:57 +0800 Subject: [PATCH 03/10] update tests --- .../sharding_datas/fnv1/fnv1a_64-ketama-data | 332 ++++++++++++++++++ tests/src/shard_test.rs | 32 +- 2 files changed, 346 insertions(+), 18 deletions(-) create mode 100644 tests/sharding_datas/fnv1/fnv1a_64-ketama-data diff --git a/tests/sharding_datas/fnv1/fnv1a_64-ketama-data b/tests/sharding_datas/fnv1/fnv1a_64-ketama-data new file mode 100644 index 000000000..149f497d1 --- /dev/null +++ b/tests/sharding_datas/fnv1/fnv1a_64-ketama-data @@ -0,0 +1,332 @@ +port=58064 +user_today_2007394873320 +local_isp_free01A2Ko2AwNbzsZzsC8YOPSumAbeOzO_2b_3xeCDdU6TpRKf4Q. +user_today_2012525307775 +local_isp_free01A1X0Dh9WrXxDGwdMp0aP_4h27DaKpBJqiP_UYpKFaouOLOg. +user_today_2589091355 + +port=58065 +user_today_2011671903998 +local_abtest_6450443858 +user_today_7768864026 +G_ttt_6618895639_ftf_7339743650 +user_today_7883331242 +local_isp_free01AwNqmPxHpoXV1tZJKLmEzIGDQ5u7Dj2BwQmAwJ6XTiwkIBU. +throttle_android_5728436605 + +port=58066 +local_abtest_7801978228 +throttle_android_24098a3024863680295d3d1d79731149 +user_today_6230999339 +local_isp_free01AjlxESZol3Vw5vEV-W2TeBcLD7Ew52pxRGdci6-jxbN5JJg. +_TTT_USER_CONFIG_7846824287 +user_today_6200320258 +oasis_oasis_3149831343 + +port=58067 +local_abtest_userinfo_6180164992 +user_today_7853764690 +user_today_7745908360 +local_isp_free01A-g6utWfF7gukziRmFD2JDBhPpHseG5r_Q46g96fZtUE5Hg. +user_today_1026334870553 +throttle_iphone_7813656303 +throttle__2409895a324b2c0278b266fffe842be7 +G_ttt_5676045247_ftf_5822706681 +throttle__2239419857 +G_ttt_7264142535_ftf_1022:1008089d9e9d6cb495f6cfa155189f7e821cc86872502526 +user_today_7609496639 + + +port=58068 +local_abtest_5990906370 +local_abtest_userinfo_7647088080 +G_ttt_6589413257_ftf_5403650170 +local_abtest_userinfo_5975742191 +local_abtest_userinfo_7872273045 +local_abtest_userinfo_2923744897 +#throttle_android_7365346961 +oasis_oasis_7490358734 +local_abtest_userinfo_6452722698 +throttle_android_2408822a7204e8b07c903ca7bcb6420e + +port=58069 +user_today_1880983255 +user_today_2002654010822 +throttle__11324219255 +user_today_2009626985318 +local_abtest_7861820263 +local_abtest_userinfo_3196941517 +#throttle_android_5867820131 +G_ttt_7742402748_ftf_ +throttle__24098a20b849ad40cc11dc7e74931057 + +port=58070 +local_isp_free5118046801 +user_ltv_1279633232 +user_today_2011504340783 +local_isp_free01AyR9KSztv2VlxlhkKVwXCLk69anthXzc9-vMtb3vAQ8zYXQ. +throttle__2408824ec104ced01198d7b798cd3888 +local_isp_free6855684332 +local_abtest_7593896549 +#throttle_android_1680973297 +oasis_oasis_7447446162 +user_today_3879312983 +user_today_7059207789 +local_isp_free01A78I-sc2NA7XXYwGH4GANl6FvI3NBYB-yyPs_Er_aV2cfD4. +local_isp_free7782012434 +local_isp_free7001466421 + + +port=58085 +user_today_5541090410 +throttle_iphone_14117225245 +local_abtest_1892600681 +user_today_6284111104 +#throttle_iphone_7829398810 +local_abtest_7309709649 +user_today_5615810434 +user_today_7760817508 +local_isp_free01A_4y-W7b6eG0XlFpjARC8OIlB2tCkL6ihGqVzgzectHxrGA. +user_today_5507145080 +stone3864950202 + +port=58109 +local_abtest_userinfo_1699351254 +local_abtest_userinfo_7373559045 +local_abtest_2009342770198 +oasis_oasis_6265298691 +local_abtest_userinfo_6383698186 +user_today_2011812533296 +local_isp_free01AxKZ04W-nRuvqXZQ_THmN0VZao1_5t47UVQQOL6uKu-feXw. +user_ltv_6107585286 +user_today_5210655623 +local_abtest_7558264136 +local_isp_free6460101972 +local_abtest_userinfo_5744148801 +oasis_oasis_7027315162 +throttle_iphone_3810112297 +throttle_android_6364802638 + +port=58131 +local_abtest_userinfo_3177195290 +G_ttt_6996331467_ftf_1022:100808daf50ee62946b3b988d32081598fe75a6025941641 +local_abtest_7398633683 +throttle_iphone_6449865524 +local_abtest_2011635660501 +throttle_android_36411283 +local_abtest_5670729934 +local_isp_free01A8jJEa4-Wat5DC8f-Lj2XMDNUtaQr0FHXzEceTgL71JX2GY. +#throttle_iphone_3165529661 +user_today_1021207891374 +local_top_mblog_expire_5639969109 +G_ttt_5868540282_ftf_5776930791 +throttle_iphone_1014960773 +local_abtest_userinfo_2011909458881 +oasis_oasis_6086988668 + + +port=58163 +local_abtest_userinfo_6548486174 +user_today_1015261615268 +user_ltv_6620759565 +user_today_6988731159 +G_ttt_7855644994_ftf_7255461169 +local_isp_free01A52Ohw1nP3uubmqK4hW5wJwMPhYZ6eS1Jz9XhV_vSoCimhA. +user_today_1021220148070 +local_abtest_3749891184 +user_ltv_5071448642 +G_ttt_6934257267_ftf_1022:100808eace67c79c66847f6741cdcb5dbcdff95330052953 +user_today_1883624127 + + +port=58182 +local_abtest_userinfo_6181621220 +throttle_iphone_5258957790 +local_abtest_7438044538 +local_isp_free7189393655 +user_today_1025434190271 +user_today_7535313756 +throttle__117281807 +local_TC_lbs_position_6338956500 +user_today_2384602257 +stone5760342037 +user_today_1009375851 +oasis_oasis_5537842411 +local_abtest_userinfo_7485236643 + +port=58184 +local_isp_free01AySeb3dhq6CgOVwEpfyRlbcgoq5njGu0hUkfjWmbWHHdhg8. +throttle_android_7148617270 +local_abtest_5449802296 +local_isp_free01A-vgisxqQwic7RW78tEQSwZ566RX3XOGD2g9rQPKO6WZD1Y. +throttle_android_24098a0c142093d0b85e37b7ebb8310b +local_SESS2_7874371850_WAP_LOGIN_INFO_LOCK +local_isp_free7829213155 +oasis_oasis_1677479157 +local_abtest_userinfo_2813760500 +oasis_oasis_2314138817 +throttle__2408842c6410279b1792b388f22f3cbc +local_isp_free5951344573 +throttle_iphone_6527539683 + + +port=58205 +G_ttt_7826330391_ftf_1642591402 +user_ltv_5668511357 +user_today_7872190278 +user_today_5404269693 +local_isp_free01ApNCng9rNtZOWiBPSdLTPWhYGqrQwkBQBmKE7O0irjf9HC4. +local_abtest_userinfo_7635296248 +oasis_oasis_1629915322 +local_abtest_5203057104 +user_today_1958091815 +throttle_iphone_429317369 +local_isp_free01A6acWvLbxjfcFVi_wOZkSaFHR007NfaIsQnQzfETRaJinP4. +user_today_2005443602477 +user_today_2002951794389 +G_ttt_5269063612_ftf_ +local_abtest_1676430873 +user_today_1021142892389 + +port=58215 +user_today_7500796876 +G_ttt_5894851348_ftf_1195908387 +throttle__11424078100 +user_today_3174889172 +user_today_5529714038 +G_ttt_6543603129_ftf_6181575379 +local_isp_free7190624362 +user_today_1021169988768 +user_today_1060843071122 +local_isp_free01A4PlTrUCaw-HpQWSO1xWi2tmBGyWP6_OMQzDhF3urAFHM6k. +throttle_android_1468885213 +local_abtest_userinfo_5619788809 +user_today_7894439543 + +port=58222 +local_abtest_userinfo_2810900151 +throttle_variety_5494801587 +user_today_7417928281 +throttle_android_240989008df0367eb8d97dfffe08b44 +local_abtest_userinfo_3939534225 +throttle_android_7811937429 +#throttle_android_7835288009 +throttle_android_7855943079 +user_today_1021142278046 +G_ttt_1779547520_ftf_1258256457 + +port=58239 +local_isp_free01A3sd5KogCU83eQUoagRLQi_L9S67etCli9n3hUwhjJbw1kE. +local_isp_free5377579356 +oasis_oasis_6876071028 +oasis_oasis_5833523555 +user_today_7333377032 +#throttle_iphone_5351430659 +user_today_6470978556 +local_abtest_userinfo_1772488763 +local_isp_free01A6CnZNMbojbCOCMtnxzbHYbvQuY4ZgapzaeNcw8V93v4NRo. +local_isp_free01A00suM1EvIdvYAeMiuKB54KFrrXujQysaqQcx4AdwZH4giM. +SESS2_6739566320_WAP_LOGIN_INFO_LOCK +throttle_iphone_2122245883 + +port=58258 +G_ttt_7782837700_ftf_2299825394 +local_isp_free2008067998764 +throttle_android_7822445000 +user_today_7609153659 +G_ttt_1799191017_ftf_ +local_abtest_userinfo_5747501244 +user_today_7812739373 +throttle_weixinminiprogram_7827072473 +local_abtest_5676998478 +user_today_1024249930683 +local_TC_publisher_list_6566899061 +local_abtest_userinfo_1807048002 +user_today_7466602526 + + +port=58263 +user_today_6314205726 +#throttle_iphone_2833639710 +#throttle_iphone_7292521885 +#throttle_android_5323604660 +#throttle_android_6270811159 +local_abtest_5790420500 +oasis_oasis_5754902226 +local_abtest_userinfo_6608187508 +s_follow_6357228809 +oasis_oasis_5725430306 +user_today_2012085418423 +throttle__1257017687 +local_abtest_userinfo_3265517303 +pic_activity_pic_activity_source_id_4992763323745141 + +port=58289 +throttle__12022525328 +user_today_7795724800 +G_ttt_3277931691_ftf_2175199144 +throttle__22280185184 +local_abtest_userinfo_3061225241 +throttle__240884109c30310c833716b3c88176a3 +throttle_android_586216676 +user_today_7597836142 +local_abtest_userinfo_2011505876651 +throttle_android_7898118587 +local_isp_free01A_9vyPNEYAtMgeny_lKrdORGaMAl0UolWo0RyVvVxNChj24. +user_today_7617790169 +throttle_weibofastios_7847081160 +G_ttt_5760274716_ftf_5455747538 +throttle_android_1065853163 + +port=58306 +user_today_1050210959614 +local_abtest_7865764302 +user_today_6819651756 +local_isp_free6088295991 +user_today_2012396416958 +user_today_2118718715 +local_abtest_userinfo_7891716189 +throttle__240989492a2392017ad3cb2c294afac +local_isp_free5846431000 +local_abtest_5041055370 +local_isp_free01A30LvgR9bU9pluWR78CG8WZ24u2CXjZSXsozky6pLzO_qb4. +user_today_7092540434 +#throttle_iphone_5037370901 + + +port=58318 +throttle_android_7890031318 +local_abtest_7247209433 +local_abtest_7782228567 +user_today_3027604257 +local_abtest_5660873470 +oasis_oasis_6979207421 +user_today_5435797704 +local_isp_free01A1t9oW3sQj1ucTUp59YuNaJJ47lh19UylsN-r8sMX_ca4OI. +local_isp_free01A9otDypLvzSAoaylh5CC9riPLECwzS5XGiEqd71ylLk75ss. +throttle_iphone_1621705340 +local_abtest_6662435150 +local_isp_free01AwsRWtCFhc-2Q24pPnpCx6x_GbiMS35fjRxWo9750zr8yhU. +local_abtest_userinfo_7436334312 +throttle__2409895c431c37c2d8c6bcda7ff2235 + +port=58319 +local_abtest_5147924726 +local_isp_free5692915026 +local_abtest_userinfo_2011551557426 +oasis_oasis_3430228262 +local_abtest_1107488180 +local_abtest_userinfo_7181479701 +user_today_2854998792 +local_abtest_1876810337 +local_isp_free5900015504 +throttle_android_6350026753 +local_abtest_5714567945 +local_abtest_5238929672 +#throttle_android_2129160480 +throttle_huawei_11496167104 +local_abtest_5889915296 +local_isp_free01A5cNqtWThet7F7Xv7lVNaFwAFBIaL-j3IHztTl6uYxLk2gQ. +user_today_6582511774 + + diff --git a/tests/src/shard_test.rs b/tests/src/shard_test.rs index ff178d7e0..71664ccd9 100644 --- a/tests/src/shard_test.rs +++ b/tests/src/shard_test.rs @@ -592,7 +592,7 @@ fn fnv1a_64_ketama_check() { servers.push(format!("node{}", i).to_string()); } let hasher = Hasher::from("fnv1a64"); - let dist = Distribute::from("ketama", &servers); + let dist = Distribute::from("ketama_origin", &servers); const PORT_BASE: u16 = 58064; let mut idx_dest = 0; @@ -623,25 +623,21 @@ fn fnv1a_64_ketama_check() { let key = line; let hash = hasher.hash(&key.as_bytes()); let idx = dist.index(hash); - // assert_eq!( - // idx_dest, idx, - // "line/key: {} - {}:{}/{}", - // key, hash, idx_dest, idx + assert_eq!( + idx_dest, idx, + "line/key: {} - {}:{}/{}", + key, hash, idx_dest, idx + ); + // 原始业务数据存在漂移，导致无法准确匹配 + // if idx_dest != idx { + // println!( + // "+++ wrong key: {}/{}, hash/idx:{}:{}, idx in redis:{}", key, key.len(), hash, idx,idx_dest // ); - if idx_dest != idx { - println!( - "+++ wrong key: {}/{}, hash/idx:{}:{}, idx in redis:{}", - key, - key.len(), - hash, - idx, - idx_dest - ); - continue; - } + // continue; + // } success_count += 1; - println!("proc succeed line:{}", key); + // println!("proc succeed line:{}/{}", key, key.len()); } Err(e) => { println!("found err: {:?}", e); @@ -662,7 +658,7 @@ fn fnv1_tmp_test() { servers.push(format!("node{}", i).to_string()); } let hasher = Hasher::from("fnv1a64"); - let dist = Distribute::from("ketama", &servers); + let dist = Distribute::from("ketama_origin", &servers); let key = "throttle_android_6244642063"; let hash = hasher.hash(&key.as_bytes()); From 440ce78841a2ee37f2301ebe952cea7c6d288dd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=B3=A2?= Date: Wed, 13 Mar 2024 11:03:42 +0800 Subject: [PATCH 04/10] =?UTF-8?q?update=20fnv1a64=20to=20fnv1a=5F64?= =?UTF-8?q?=EF=BC=8C=E4=BF=9D=E6=8C=81=E9=85=8D=E7=BD=AE=E5=90=8D=E7=A7=B0?= =?UTF-8?q?=E4=B8=80=E8=87=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sharding/src/hash/fnv1.rs | 6 +++--- sharding/src/hash/mod.rs | 6 +++--- tests/src/shard_test.rs | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sharding/src/hash/fnv1.rs b/sharding/src/hash/fnv1.rs index 8e910c721..19fc80bbf 100644 --- a/sharding/src/hash/fnv1.rs +++ b/sharding/src/hash/fnv1.rs @@ -1,19 +1,19 @@ /// 按需支持fnv1系列所有相关的hash算法，目前支持fnv1a_64 #[derive(Debug, Default, Clone)] -pub struct Fnv1a64; +pub struct Fnv1a_64; const FNV_64_INIT: u64 = 0xcbf29ce484222325; const FNV_64_PRIME: u64 = 0x100000001b3; -impl super::Hash for Fnv1a64 { +impl super::Hash for Fnv1a_64 { fn hash(&self, key: &S) -> i64 { let mut hash = FNV_64_INIT as u32; for i in 0..key.len() { hash ^= key.at(i) as u32; hash = hash.wrapping_mul(FNV_64_PRIME as u32); } - log::debug!("+++ use Fnv1a64"); + hash as i64 } } diff --git a/sharding/src/hash/mod.rs b/sharding/src/hash/mod.rs index 92300d794..84e5a2445 100644 --- a/sharding/src/hash/mod.rs +++ b/sharding/src/hash/mod.rs @@ -27,7 +27,7 @@ pub mod crc; use enum_dispatch::enum_dispatch; -use self::{bkdrsub::Bkdrsub, crc64::Crc64, fnv1::Fnv1a64}; +use self::{bkdrsub::Bkdrsub, crc64::Crc64, fnv1::Fnv1a_64}; // 占位hash，主要用于兼容服务框架，供mq等业务使用 pub const HASH_PADDING: &str = "padding"; @@ -85,7 +85,7 @@ pub enum Hasher { Rawcrc32local(Rawcrc32local), // raw or crc32local Crc32Abs(Crc32Abs), // crc32abs: 基于i32转换，然后直接取abs；其他走i64提升为正数 Crc64(Crc64), // Crc64 算法，对整个key做crc64计算 - Fnv1a64(Fnv1a64), + Fnv1a_64(Fnv1a_64), Random(RandomHash), // random hash RawSuffix(RawSuffix), } @@ -132,7 +132,7 @@ impl Hasher { "crc32abs" => Self::Crc32Abs(Default::default()), "crc64" => Self::Crc64(Default::default()), "random" => Self::Random(Default::default()), - "fnv1a64" => Self::Fnv1a64(Default::default()), + "fnv1a_64" => Self::Fnv1a_64(Default::default()), _ => { // 默认采用mc的crc32-s hash log::error!("found unknown hash:{}, use crc32-short instead", alg); diff --git a/tests/src/shard_test.rs b/tests/src/shard_test.rs index 71664ccd9..cfb794778 100644 --- a/tests/src/shard_test.rs +++ b/tests/src/shard_test.rs @@ -591,7 +591,7 @@ fn fnv1a_64_ketama_check() { for i in 1..(shard_count + 1) { servers.push(format!("node{}", i).to_string()); } - let hasher = Hasher::from("fnv1a64"); + let hasher = Hasher::from("fnv1a_64"); let dist = Distribute::from("ketama_origin", &servers); const PORT_BASE: u16 = 58064; From c581e6028cbb7b3126d61ccba951f91cc7a50113 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=B3=A2?= Date: Wed, 13 Mar 2024 12:05:47 +0800 Subject: [PATCH 05/10] =?UTF-8?q?hash=E7=AE=97=E6=B3=95=E4=B8=AD=E7=9A=84?= =?UTF-8?q?=E4=B8=8B=E5=88=92=E7=BA=BF=E4=B8=8D=E8=83=BD=E7=94=A8=E5=9C=A8?= =?UTF-8?q?=E7=BB=93=E6=9E=84=E5=90=8D=EF=BC=8C=E7=BB=9F=E4=B8=80=E6=94=B9?= =?UTF-8?q?=E4=B8=BAF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sharding/src/hash/fnv1.rs | 4 ++-- sharding/src/hash/mod.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sharding/src/hash/fnv1.rs b/sharding/src/hash/fnv1.rs index 19fc80bbf..f2272459e 100644 --- a/sharding/src/hash/fnv1.rs +++ b/sharding/src/hash/fnv1.rs @@ -1,12 +1,12 @@ /// 按需支持fnv1系列所有相关的hash算法，目前支持fnv1a_64 #[derive(Debug, Default, Clone)] -pub struct Fnv1a_64; +pub struct Fnv1aF64; const FNV_64_INIT: u64 = 0xcbf29ce484222325; const FNV_64_PRIME: u64 = 0x100000001b3; -impl super::Hash for Fnv1a_64 { +impl super::Hash for Fnv1aF64 { fn hash(&self, key: &S) -> i64 { let mut hash = FNV_64_INIT as u32; for i in 0..key.len() { diff --git a/sharding/src/hash/mod.rs b/sharding/src/hash/mod.rs index 84e5a2445..58b388fbf 100644 --- a/sharding/src/hash/mod.rs +++ b/sharding/src/hash/mod.rs @@ -27,7 +27,7 @@ pub mod crc; use enum_dispatch::enum_dispatch; -use self::{bkdrsub::Bkdrsub, crc64::Crc64, fnv1::Fnv1a_64}; +use self::{bkdrsub::Bkdrsub, crc64::Crc64, fnv1::Fnv1aF64}; // 占位hash，主要用于兼容服务框架，供mq等业务使用 pub const HASH_PADDING: &str = "padding"; @@ -85,7 +85,7 @@ pub enum Hasher { Rawcrc32local(Rawcrc32local), // raw or crc32local Crc32Abs(Crc32Abs), // crc32abs: 基于i32转换，然后直接取abs；其他走i64提升为正数 Crc64(Crc64), // Crc64 算法，对整个key做crc64计算 - Fnv1a_64(Fnv1a_64), + Fnv1aF64(Fnv1aF64), Random(RandomHash), // random hash RawSuffix(RawSuffix), } @@ -132,7 +132,7 @@ impl Hasher { "crc32abs" => Self::Crc32Abs(Default::default()), "crc64" => Self::Crc64(Default::default()), "random" => Self::Random(Default::default()), - "fnv1a_64" => Self::Fnv1a_64(Default::default()), + "fnv1a_64" => Self::Fnv1aF64(Default::default()), _ => { // 默认采用mc的crc32-s hash log::error!("found unknown hash:{}, use crc32-short instead", alg); From 8cfb5b27b33181cc40e877e8f4ed7eae4749b352 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=B3=A2?= Date: Wed, 13 Mar 2024 17:05:50 +0800 Subject: [PATCH 06/10] update test --- tests/src/hash_test.rs | 4 ++++ tests/src/shard_test.rs | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/src/hash_test.rs b/tests/src/hash_test.rs index 0d3141aed..ddf711591 100644 --- a/tests/src/hash_test.rs +++ b/tests/src/hash_test.rs @@ -309,4 +309,8 @@ mod hash_test { let crc32_lblocal_hasher = crc32_lblocal_hasher.hash(key); assert_eq!(crc32_lblocal, crc32_lblocal_hasher, "key:{key:?}"); } + + fn tmp_print_ports() { + for i in 58064..58319 {} + } } diff --git a/tests/src/shard_test.rs b/tests/src/shard_test.rs index cfb794778..d6a8dc336 100644 --- a/tests/src/shard_test.rs +++ b/tests/src/shard_test.rs @@ -657,7 +657,7 @@ fn fnv1_tmp_test() { for i in 1..(shard_count + 1) { servers.push(format!("node{}", i).to_string()); } - let hasher = Hasher::from("fnv1a64"); + let hasher = Hasher::from("fnv1a_64"); let dist = Distribute::from("ketama_origin", &servers); let key = "throttle_android_6244642063"; From 8c67c72e2c3c79c7b40d9bb9dfaca8741254e854 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=B3=A2?= Date: Wed, 13 Mar 2024 17:06:45 +0800 Subject: [PATCH 07/10] update tests --- tests/src/hash_test.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/src/hash_test.rs b/tests/src/hash_test.rs index ddf711591..0d3141aed 100644 --- a/tests/src/hash_test.rs +++ b/tests/src/hash_test.rs @@ -309,8 +309,4 @@ mod hash_test { let crc32_lblocal_hasher = crc32_lblocal_hasher.hash(key); assert_eq!(crc32_lblocal, crc32_lblocal_hasher, "key:{key:?}"); } - - fn tmp_print_ports() { - for i in 58064..58319 {} - } } From 379ef7263972eb8c18bf6d3c57c0c9a498e7fe1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=B3=A2?= Date: Wed, 13 Mar 2024 17:59:39 +0800 Subject: [PATCH 08/10] =?UTF-8?q?=E5=AF=B9redis=20backends=E7=9A=84name?= =?UTF-8?q?=E5=81=9A=E9=87=8D=E5=A4=8D=E6=80=A7=E6=A0=A1=E9=AA=8C=EF=BC=8C?= =?UTF-8?q?=E6=AF=8F=E7=BB=84=E5=9F=9F=E5=90=8D=E7=9A=84name=E5=BF=85?= =?UTF-8?q?=E9=A1=BB=E4=B8=8D=E5=90=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- endpoint/src/redisservice/config.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/endpoint/src/redisservice/config.rs b/endpoint/src/redisservice/config.rs index fe3bb5f05..ddfa04d9d 100644 --- a/endpoint/src/redisservice/config.rs +++ b/endpoint/src/redisservice/config.rs @@ -1,6 +1,6 @@ //use ds::time::Duration; -use std::fmt::Debug; +use std::{collections::HashSet, fmt::Debug}; use serde::{Deserialize, Serialize}; //use sharding::distribution::{DIST_ABS_MODULA, DIST_MODULA}; @@ -124,9 +124,16 @@ impl RedisNamespace { } } - // 如果backend有name，则所有的后端都必须有name - if self.backend_names.len() > 0 && self.backend_names.len() != self.backends.len() { - return false; + // 如果backend有name，则所有的后端都必须有name，且name不能重复 + if self.backend_names.len() > 0 { + if self.backend_names.len() != self.backends.len() { + return false; + } + let mut names_unique = HashSet::with_capacity(self.backend_names.len()); + names_unique.extend(self.backend_names.clone()); + if names_unique.len() != self.backend_names.len() { + return false; + } } true From e9e2d8f90c9237d357388c3593cec054bde010e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=B3=A2?= Date: Thu, 14 Mar 2024 11:06:46 +0800 Subject: [PATCH 09/10] =?UTF-8?q?=E6=94=AF=E6=8C=81=E6=8C=89=E9=9C=80?= =?UTF-8?q?=E6=9E=84=E5=BB=BAcfg=EF=BC=8C=E6=96=B9=E4=BE=BF256=E7=AB=AF?= =?UTF-8?q?=E5=8F=A3=E4=B8=9A=E5=8A=A1=E7=9A=84=E9=85=8D=E7=BD=AE=E6=8E=A8?= =?UTF-8?q?=E9=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/src/all.rs | 1 + tests/src/cfg_build.rs | 68 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 tests/src/cfg_build.rs diff --git a/tests/src/all.rs b/tests/src/all.rs index 2331664f1..24db6ab1b 100644 --- a/tests/src/all.rs +++ b/tests/src/all.rs @@ -17,6 +17,7 @@ mod asserts; mod layout; // mod mysql; mod bkdrsub; +mod cfg_build; mod cow; mod decrypt; mod discovery; diff --git a/tests/src/cfg_build.rs b/tests/src/cfg_build.rs new file mode 100644 index 000000000..3fd85faad --- /dev/null +++ b/tests/src/cfg_build.rs @@ -0,0 +1,68 @@ +use std::{ + fs::File, + io::{BufWriter, Write}, +}; + +#[test] +fn build_redis_cfg() { + let start_port = 58064; + let end_port = 58319 + 1; + let hash = "fnv1a_64"; + let dist = "ketama_origin"; + let namespace = "mapi"; + + let mut ports = String::with_capacity(1024); + for p in start_port..end_port { + ports += &format!("{},", p); + } + let _ = ports.split_off(ports.len() - 1); + + let mut shards = String::with_capacity(4096); + for p in start_port..end_port { + shards += &format!( + " - rm{}.eos.grid.sina.com.cn:{},rs{}.hebe.grid.sina.com.cn:{} node{}\n", + p, + p, + p, + p, + (p - start_port + 1) + ); + } + + let mut cfg_str = String::with_capacity(8192); + cfg_str += "basic:\n"; + cfg_str += " access_mod: rw\n"; + cfg_str += &format!(" hash: {}\n", hash); + cfg_str += &format!(" distribution: {}\n", dist); + cfg_str += &format!(" listen: {}\n", ports); + cfg_str += " resource_type: eredis\n"; + cfg_str += " timeout_ms_master: 500\n timeout_ms_slave: 500\n"; + + cfg_str += "backends:\n"; + cfg_str += &shards; + + let cfg_file = format!("../static.{}.cfg", namespace); + let file = File::create(cfg_file).unwrap(); + let mut writer = BufWriter::new(file); + writer.write_all(cfg_str.as_bytes()).unwrap(); + writer.flush().unwrap(); + + let mut shards_4_table = String::with_capacity(8192); + for p in start_port..end_port { + shards_4_table += &format!( + "rm{}.eos.grid.sina.com.cn:{},rs{}.hebe.grid.sina.com.cn:{} node{}\n", + p, + p, + p, + p, + (p - start_port + 1) + ); + } + let cfg_file = format!("../static.{}.cfg.table", namespace); + let file = File::create(cfg_file).unwrap(); + let mut writer = BufWriter::new(file); + writer + .write_all(format!("shards:\n{}", shards_4_table).as_bytes()) + .unwrap(); + writer.flush().unwrap(); +} From 096a6d96128cb8acbd8c247ab3a79ddd935512a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E6=B3=A2?= Date: Thu, 14 Mar 2024 11:18:40 +0800 Subject: [PATCH 10/10] =?UTF-8?q?=E5=AF=B9redis=20name=E6=A3=80=E6=B5=8B?= =?UTF-8?q?=E6=97=B6=EF=BC=8C=E6=8E=92=E9=99=A4','=EF=BC=8C=E9=81=BF?= =?UTF-8?q?=E5=85=8D=E9=94=99=E8=AF=AF=E6=A0=BC=E5=BC=8F=E7=9A=84slave?= =?UTF-8?q?=E8=A2=AB=E5=BD=93=E4=BD=9Credis=20name?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- endpoint/src/redisservice/config.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/endpoint/src/redisservice/config.rs b/endpoint/src/redisservice/config.rs index ddfa04d9d..9f1433bb7 100644 --- a/endpoint/src/redisservice/config.rs +++ b/endpoint/src/redisservice/config.rs @@ -58,10 +58,11 @@ impl RedisNamespace { let mut backends = Vec::with_capacity(ns.backends.len()); for b in &mut ns.backends { let domain_name: Vec<&str> = b.split(" ").collect(); - // 后端地址格式： 域名,域名 name, name不能是rm、rs开头 + // 后端地址格式： 域名,域名 name, name不能是rm、rs、','开头，避免把异常格式的slave当作name if domain_name.len() == 2 && !domain_name[1].starts_with("rm") && !domain_name[1].starts_with("rs") + && !domain_name[1].starts_with(",") { backends.push(domain_name[0].to_string()); ns.backend_names.push(domain_name[1].to_string());