diff --git a/Cargo.lock b/Cargo.lock index b4bab21..7d66248 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,21 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "0.6.14" @@ -57,12 +72,44 @@ version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +[[package]] +name = "autocfg" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "cc" +version = "1.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26a5c3fd7bfa1ce3897a3a3501d362b2d87b7f2583ebcb4a949ec25911025cbc" + [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets", +] + [[package]] name = "clap" version = "4.5.9" @@ -122,6 +169,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + [[package]] name = "crc" version = "3.2.1" @@ -177,6 +230,29 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "indexmap" version = "2.2.6" @@ -205,6 +281,7 @@ name = "innodb_recovery" version = "0.1.0" dependencies = [ "anyhow", + "chrono", "clap", "crc", "indicatif", @@ -232,6 +309,15 @@ version = "1.70.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" +[[package]] +name = "js-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -275,6 +361,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "num_enum" version = "0.7.2" @@ -607,6 +702,60 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "wasm-bindgen" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.72", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" + [[package]] name = "winapi" version = "0.3.9" @@ -629,6 +778,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.52.0" diff --git a/Cargo.toml b/Cargo.toml index 10b888e..008c9fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ default-run="page_explorer" [dependencies] anyhow = "1.0.86" +chrono = "0.4.38" clap = { version = "4.5.9", features = ["derive"] } crc = "3.2.1" indicatif = "0.17.8" diff --git a/src/bin/page_explorer.rs b/src/bin/page_explorer.rs index 3cd2bdb..f4d9572 100644 --- a/src/bin/page_explorer.rs +++ b/src/bin/page_explorer.rs @@ -9,7 +9,7 @@ use std::{ use clap::Parser; use innodb::innodb::{ buffer_manager::{ - lru::LRUBufferManager, simple::SimpleBufferManager, BufferManager, DummyBufferMangaer, + lru::LRUBufferManager, BufferManager, DummyBufferMangaer, }, page::{ index::{record::RecordType, IndexPage}, @@ -125,7 +125,8 @@ impl PageExplorer { let values = row.parse_values(self.buffer_mgr.as_mut()); assert_eq!(values.len(), table.field_count()); debug!("{:?}", values); - self.write_row(row.record.header.info_flags.deleted, &values).expect("Failed to write row"); + self.write_row(row.record.header.info_flags.deleted, &values) + .expect("Failed to write row"); } } RecordType::NodePointer => { diff --git a/src/bin/page_extractor.rs b/src/bin/page_extractor.rs index 030379a..8282c36 100644 --- a/src/bin/page_extractor.rs +++ b/src/bin/page_extractor.rs @@ -75,7 +75,7 @@ fn validate_page(page: &[u8]) -> PageValidationResult { }; trace!("Bad page: {:#?}", page); - return PageValidationResult::NotAPage; + PageValidationResult::NotAPage } fn main() { diff --git a/src/bin/tablespace_sort.rs b/src/bin/tablespace_sort.rs index 10fb673..4fc72eb 100644 --- a/src/bin/tablespace_sort.rs +++ b/src/bin/tablespace_sort.rs @@ -18,7 +18,7 @@ struct Page { impl Page { fn offset(&self) -> u32 { let num: [u8; 4] = self.data[4..8].try_into().unwrap(); - return u32::from_be_bytes(num); + u32::from_be_bytes(num) } } diff --git a/src/innodb/buffer_manager/lru.rs b/src/innodb/buffer_manager/lru.rs index a34d260..bbc869f 100644 --- a/src/innodb/buffer_manager/lru.rs +++ b/src/innodb/buffer_manager/lru.rs @@ -1,14 +1,11 @@ use std::{ - borrow::BorrowMut, cell::RefCell, - collections::{HashMap, LinkedList}, + collections::HashMap, fs::File, io::{Read, Seek, SeekFrom}, path::{Path, PathBuf}, slice, - sync::atomic::{AtomicU32, AtomicU64, Ordering}, time::SystemTime, - usize, }; use super::{BufferManager, PageGuard}; @@ -17,7 +14,7 @@ use crate::innodb::{ InnoDBError, }; use anyhow::{anyhow, Result}; -use tracing::{trace, warn}; +use tracing::trace; const LRU_PAGE_COUNT: usize = 16; @@ -52,7 +49,7 @@ impl LRUBufferManager { .lru_list .borrow_mut() .resize(LRU_PAGE_COUNT, 0); - return buffer_manager; + buffer_manager } pub fn find_free(&self) -> usize { @@ -74,15 +71,13 @@ impl LRUBufferManager { let ((space_id, offset), _) = borrowed_pin_map .iter() .find(|(_, val)| **val == result_frame) - .expect(&format!( - "can't find the frame({result_frame}), {:#?}, pinmap: {:#?}", - self, borrowed_pin_map - )) + .unwrap_or_else(|| panic!("can't find the frame({result_frame}), {:#?}, pinmap: {:#?}", + self, borrowed_pin_map)) .to_owned(); let (space_id, offset) = (*space_id, *offset); borrowed_pin_map.remove(&(space_id, offset)); self.lru_list.borrow_mut()[result_frame] = 0; - return result_frame; + result_frame } else { panic!("pin too many pages, \nState: {:#?}", self); } @@ -127,7 +122,7 @@ impl BufferManager for LRUBufferManager { })?; // Validate page *FIRST* - let page = Page::from_bytes(&self.backing_store[free_frame as usize])?; + let page = Page::from_bytes(&self.backing_store[free_frame])?; if page.header.space_id == 0 && page.header.offset == 0 { return Err(anyhow!(InnoDBError::PageNotFound)); } diff --git a/src/innodb/page/mod.rs b/src/innodb/page/mod.rs index 901df66..0ccb3b0 100644 --- a/src/innodb/page/mod.rs +++ b/src/innodb/page/mod.rs @@ -69,7 +69,7 @@ impl<'a> Page<'a> { Ok(Page { // space_id: header.space_id, - header: header, + header, trailer: FILTrailer::from_bytes(&buf[(FIL_PAGE_SIZE - FIL_TRAILER_SIZE)..])?, raw_data: buf, }) diff --git a/src/innodb/table/field.rs b/src/innodb/table/field.rs index 5ecc5ed..422f7f4 100644 --- a/src/innodb/table/field.rs +++ b/src/innodb/table/field.rs @@ -1,6 +1,5 @@ -use std::u64; - use crate::innodb::charset::InnoDBCharset; +use chrono::DateTime; use tracing::{info, trace}; #[derive(Debug, Clone, PartialEq, Eq)] @@ -18,20 +17,15 @@ pub enum FieldType { Char(usize, InnoDBCharset), Date, + DateTime, + Timestamp, } impl FieldType { // Returns how many bytes does the "length" metadata takes up pub fn is_variable(&self) -> bool { match self { - FieldType::TinyInt(_) - | FieldType::SmallInt(_) - | FieldType::MediumInt(_) - | FieldType::Int(_) - | FieldType::Int6(_) - | FieldType::BigInt(_) => false, - FieldType::Enum(_) | FieldType::Date => false, - FieldType::Char(_, _) => false, FieldType::Text(_, _) => true, + _ => false, } } @@ -45,6 +39,8 @@ impl FieldType { FieldType::BigInt(_) => 8, FieldType::Enum(_) => 2, FieldType::Date => 3, + FieldType::DateTime => 8, + FieldType::Timestamp => 4, FieldType::Text(len, charset) => (*len as u64) * charset.max_len(), FieldType::Char(len, charset) => (*len as u64) * charset.max_len(), } @@ -77,47 +73,55 @@ impl Field { } } - fn parse_int(&self, buf: &[u8], len: usize, signed: bool) -> FieldValue { + fn parse_uint(&self, buf: &[u8], len: usize) -> u64 { assert!(len <= 8, "Currently only support upto u64"); assert!(buf.len() >= len, "buf not long enough"); let mut num = 0u64; for byte in buf[0..len].iter().cloned() { num = (num << 8) | (byte as u64); } + num + } + + fn parse_signed_int(&self, buf: &[u8], len: usize) -> i64 { + let mut num = self.parse_uint(buf, len); + num ^= 1u64 << (len * 8 - 1); // Filp the sign bit -- I don`t know why but it works + + let signed_value; + if (num & (1u64 << (len * 8 - 1))) != 0 { + num = !(num - 1); + num &= (1u64 << (len * 8)) - 1; // Clear other bits + signed_value = -(num as i64); + } else { + signed_value = num as i64; + } + signed_value + } + + fn parse_int_field(&self, buf: &[u8], len: usize, signed: bool) -> FieldValue { if signed { - num ^= 1u64 << (len * 8 - 1); // Filp the sign bit -- I don`t know why but it works - - let signed_value; - if (num & (1u64 << (len * 8 - 1))) != 0 { - num = !(num - 1); - num &= (1u64 << (len * 8)) - 1; // Clear other bits - signed_value = -(num as i64); - } else { - signed_value = num as i64; - } - FieldValue::SignedInt(signed_value) + FieldValue::SignedInt(self.parse_signed_int(buf, len)) } else { - assert!(len == 8 || num < (1 << (len * 8))); - FieldValue::UnsignedInt(num) + FieldValue::UnsignedInt(self.parse_uint(buf, len)) } } pub fn parse(&self, buf: &[u8], length_opt: Option) -> (FieldValue, usize) { let (val, len) = match self.field_type { - FieldType::TinyInt(signed) => (self.parse_int(buf, 1, signed), 1), - FieldType::SmallInt(signed) => (self.parse_int(buf, 2, signed), 2), - FieldType::MediumInt(signed) => (self.parse_int(buf, 3, signed), 3), - FieldType::Int(signed) => (self.parse_int(buf, 4, signed), 4), - FieldType::Int6(signed) => (self.parse_int(buf, 6, signed), 6), - FieldType::BigInt(signed) => (self.parse_int(buf, 8, signed), 8), + FieldType::TinyInt(signed) => (self.parse_int_field(buf, 1, signed), 1), + FieldType::SmallInt(signed) => (self.parse_int_field(buf, 2, signed), 2), + FieldType::MediumInt(signed) => (self.parse_int_field(buf, 3, signed), 3), + FieldType::Int(signed) => (self.parse_int_field(buf, 4, signed), 4), + FieldType::Int6(signed) => (self.parse_int_field(buf, 6, signed), 6), + FieldType::BigInt(signed) => (self.parse_int_field(buf, 8, signed), 8), FieldType::Char(len, _) => ( FieldValue::String( - String::from_utf8(buf[0..len as usize].into()) + String::from_utf8(buf[0..len].into()) .expect("Failed parsing UTF-8") .trim_end() .to_string(), ), - len as usize, + len, ), FieldType::Text(max_len, _) => match length_opt { None => (FieldValue::Null, 0), @@ -137,15 +141,45 @@ impl Field { } }, FieldType::Date => { - if let FieldValue::SignedInt(date_num) = self.parse_int(buf, 3, true) { - let day = date_num & 0x1F; - let month = (date_num >> 5) & 0xF; - let year = date_num >> 9; - (FieldValue::String(format!("{:04}-{:02}-{:02}", year, month, day)), 3) + let date_num = self.parse_signed_int(buf, 3); + let day = date_num & 0x1F; + let month = (date_num >> 5) & 0xF; + let year = date_num >> 9; + ( + FieldValue::String(format!("{:04}-{:02}-{:02}", year, month, day)), + 3, + ) + } + FieldType::DateTime => { + let datetime = self.parse_signed_int(buf, 8) as u64; + let yd = datetime >> 46; + let year = yd / 13; + let month = yd - year * 13; + let day = (datetime >> 41) & 0b11111; + let hour = (datetime >> 36) & 0b11111; + let min = (datetime >> 30) & 0b111111; + let sec = (datetime >> 24) & 0b111111; + ( + FieldValue::String(format!( + "{:04}-{:02}-{:02} {:02}:{:02}:{:02}", + year, month, day, hour, min, sec + )), + 8, + ) + } + FieldType::Timestamp => { + let ts = self.parse_uint(buf, 4); + if ts == 0 { + (FieldValue::String("0000-00-00 00:00:00".to_owned()), 4) } else { - panic!("Can't parse int"); + let datetime = + DateTime::from_timestamp(ts as i64, 0).expect("Out of range Datetime"); + ( + FieldValue::String(format!("{}", datetime.format("%Y-%m-%d %H:%M:%S"))), + 4, + ) } - }, + } FieldType::Enum(ref values) => { let len = if values.len() <= u8::MAX as usize { 1 @@ -153,15 +187,12 @@ impl Field { 2 }; - if let FieldValue::UnsignedInt(num) = self.parse_int(buf, len, false) { - assert!( - (num as usize) < values.len(), - "Enum Value is larger than expected?" - ); - (FieldValue::String(values[num as usize].clone()), len) - } else { - panic!("Unexpected Enum Parsing Failure"); - } + let num = self.parse_uint(buf, len); + assert!( + (num as usize) < values.len(), + "Enum Value is larger than expected?" + ); + (FieldValue::String(values[num as usize].clone()), len) } #[allow(unreachable_patterns)] _ => { @@ -186,7 +217,7 @@ mod test { field_type: FieldType::MediumInt(true), nullable: false, }; - let result = field.parse_int(&buf, 3, true); + let result = field.parse_int_field(&buf, 3, true); match result { super::FieldValue::SignedInt(val) => assert_eq!(val, 0), _ => unreachable!(), @@ -201,7 +232,7 @@ mod test { field_type: FieldType::TinyInt(true), nullable: false, }; - let result = field.parse_int(&buf, 1, true); + let result = field.parse_int_field(&buf, 1, true); match result { super::FieldValue::SignedInt(val) => assert_eq!(val, -1), _ => unreachable!(), diff --git a/src/innodb/table/mod.rs b/src/innodb/table/mod.rs index f0baef1..78fbec5 100644 --- a/src/innodb/table/mod.rs +++ b/src/innodb/table/mod.rs @@ -92,10 +92,10 @@ impl TableDefinition { "longtext" => FieldType::Text((1 << 32) - 1, charset), _ => unimplemented!("Custom: {} unhandled", name.0[0].value), }, - DataType::Enum(values) => { - FieldType::Enum(values.clone()) - }, + DataType::Enum(values) => FieldType::Enum(values.clone()), DataType::Date => FieldType::Date, + DataType::Datetime(_)=> FieldType::DateTime, + DataType::Timestamp(_,_) => FieldType::Timestamp, _ => unimplemented!("mapping of {:?}", column.data_type), }; @@ -107,7 +107,7 @@ impl TableDefinition { let field = Field { name: column.name.value.clone(), field_type: f_type, - nullable: nullable, + nullable, }; parsed_fields.push(field); @@ -194,7 +194,7 @@ impl TableDefinition { } assert!( - table_def.cluster_columns.len() > 0, + !table_def.cluster_columns.is_empty(), "Table must have at least 1 cluster column" ); @@ -260,7 +260,7 @@ mod test { let field1 = def.get_field("field1").unwrap(); assert_eq!(field1.name, "field1"); assert_eq!(field1.field_type, FieldType::Int(false)); - assert_eq!(field1.nullable, false); + assert!(!field1.nullable); } #[test] diff --git a/src/innodb/table/row.rs b/src/innodb/table/row.rs index 3573600..513446a 100644 --- a/src/innodb/table/row.rs +++ b/src/innodb/table/row.rs @@ -6,8 +6,7 @@ use std::{ }; use crate::innodb::{ - buffer_manager::{self, BufferManager}, - file_list::FileListInnerNode, + buffer_manager::{BufferManager}, page::{ index::record::{Record, RECORD_HEADER_FIXED_LENGTH}, lob::{data_page::LobData, LobFirst, LobIndexEntry}, @@ -22,7 +21,7 @@ use super::{ }; use anyhow::{anyhow, Result}; -use tracing::{debug, info, trace, warn}; +use tracing::{trace, warn}; pub struct Row<'a> { td: Arc, @@ -229,7 +228,7 @@ impl<'a> Row<'a> { trace!("Extern Header: {:?}", &extern_header); ( self.parse_extern_field(f, &extern_header, buf_mgr), - len as usize, + len, ) } else { let (value, len) = f.parse(buf, self.field_len_map.get(&idx).cloned());