diff --git a/src/innodb/buffer_manager/lru.rs b/src/innodb/buffer_manager/lru.rs index 85a71c2..d67dece 100644 --- a/src/innodb/buffer_manager/lru.rs +++ b/src/innodb/buffer_manager/lru.rs @@ -1,11 +1,11 @@ -use super::BufferManager; +use super::{BufferManager, PageGuard}; use anyhow::{anyhow, Result}; use crate::innodb::page::Page; pub struct LRUBufferManager {} impl BufferManager for LRUBufferManager { - fn open_page(&self, space_id: u32, offset: u32) -> Result { + fn open_page(&self, space_id: u32, offset: u32) -> Result { todo!() } diff --git a/src/innodb/buffer_manager/mod.rs b/src/innodb/buffer_manager/mod.rs index af6cff9..9917737 100644 --- a/src/innodb/buffer_manager/mod.rs +++ b/src/innodb/buffer_manager/mod.rs @@ -7,16 +7,25 @@ pub mod lru; pub mod simple; pub trait BufferManager { - fn open_page(&self, space_id: u32, offset: u32) -> Result; + fn open_page(&self, space_id: u32, offset: u32) -> Result; fn close_page(&self, page: Page); } -pub struct PageGuard<'a, B: BufferManager> { +pub struct PageGuard<'a> { page: Page<'a>, - buffer_manager: &'a B + buffer_manager: &'a dyn BufferManager } -impl <'a, B: BufferManager> Deref for PageGuard<'a, B> { +impl <'a> PageGuard<'a> { + pub fn new(page: Page<'a>, buffer_manager: &'a dyn BufferManager) -> Self { + PageGuard { + page, + buffer_manager, + } + } +} + +impl <'a> Deref for PageGuard<'a> { type Target = Page<'a>; fn deref(&self) -> &Self::Target { @@ -24,7 +33,7 @@ impl <'a, B: BufferManager> Deref for PageGuard<'a, B> { } } -impl <'a, B: BufferManager> Drop for PageGuard<'a, B> { +impl <'a> Drop for PageGuard<'a> { fn drop(&mut self) { self.buffer_manager.close_page(std::mem::take(&mut self.page)); } @@ -33,11 +42,11 @@ impl <'a, B: BufferManager> Drop for PageGuard<'a, B> { pub struct DummyBufferMangaer; impl BufferManager for DummyBufferMangaer { - fn open_page(&self, _space_id: u32, _offset: u32) -> Result { + fn open_page(&self, _space_id: u32, _offset: u32) -> Result { Err(anyhow!("Dummy buffer manager can't open")) } fn close_page(&self, _: Page) { panic!("This doens't open how can we close"); } -} \ No newline at end of file +} diff --git a/src/innodb/buffer_manager/simple.rs b/src/innodb/buffer_manager/simple.rs index a1ac2fe..8a7a4d9 100644 --- a/src/innodb/buffer_manager/simple.rs +++ b/src/innodb/buffer_manager/simple.rs @@ -1,4 +1,5 @@ use anyhow::Result; +use tracing::trace; use std::{ cell::RefCell, collections::HashMap, @@ -10,7 +11,7 @@ use std::{ use crate::innodb::page::{Page, FIL_PAGE_SIZE}; -use super::BufferManager; +use super::{BufferManager, PageGuard}; pub struct SimpleBufferManager { page_directory: PathBuf, @@ -52,12 +53,13 @@ impl SimpleBufferManager { } impl BufferManager for SimpleBufferManager { - fn open_page(&self, space_id: u32, offset: u32) -> Result { + fn open_page(&self, space_id: u32, offset: u32) -> Result { let buf = self.get_page(space_id, offset)?; - Page::from_bytes(buf) + trace!("Opened ({}, {})", space_id, offset); + Ok(PageGuard::new(Page::from_bytes(buf)?, self)) } fn close_page(&self, page: Page) { - // Nothing + trace!("Closed ({:?}, {})", page.space_id, page.header.offset); } } diff --git a/src/innodb/file_list.rs b/src/innodb/file_list.rs index a423bc9..4b2cd46 100644 --- a/src/innodb/file_list.rs +++ b/src/innodb/file_list.rs @@ -1,12 +1,48 @@ -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, Ok, Result}; + +pub const FIL_NULL: u32 = 0xFFFF_FFFF; + +#[derive(Debug, Clone, Copy)] +pub struct FileAddress { + pub page_number: u32, + pub offset: u16, +} + +impl FileAddress { + pub fn new(page_number: u32, offset: u16) -> Self { + FileAddress { + page_number, + offset, + } + } + + pub fn try_from_bytes(buf: &[u8]) -> Result { + if buf.len() < 6 { + return Err(anyhow!("Buffer is too small")); + } + + let page_number = u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]]); + let offset = u16::from_be_bytes([buf[4], buf[5]]); + Ok(FileAddress { + page_number, + offset, + }) + } + + pub fn is_null(&self) -> bool { + self.page_number == FIL_NULL + } + + fn size() -> usize { + 6 + } +} #[derive(Debug, Clone)] pub struct FileListBaseNode { pub list_len: u32, - pub first_node_page_number: u32, - pub first_node_offset: u16, - pub last_node_page_number: u32, - pub last_node_offset: u16, + pub first_node: FileAddress, + pub last_node: FileAddress, } impl FileListBaseNode { @@ -16,27 +52,25 @@ impl FileListBaseNode { } let list_len = u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]]); - let first_node_page_number = u32::from_be_bytes([buf[4], buf[5], buf[6], buf[7]]); - let first_node_offset = u16::from_be_bytes([buf[8], buf[9]]); - let last_node_page_number = u32::from_be_bytes([buf[10], buf[11], buf[12], buf[13]]); - let last_node_offset = u16::from_be_bytes([buf[14], buf[15]]); + let first_node = FileAddress::try_from_bytes(&buf[4..10])?; + let last_node = FileAddress::try_from_bytes(&buf[10..16])?; Ok(FileListBaseNode { list_len, - first_node_page_number, - first_node_offset, - last_node_page_number, - last_node_offset, + first_node, + last_node }) } + + pub(crate) fn size() -> usize { + 4 + FileAddress::size() + FileAddress::size() + } } #[derive(Debug, Clone)] pub struct FileListInnerNode { - pub prev_node_page_number: u32, - pub prev_node_offset: u16, - pub next_node_page_number: u32, - pub next_node_offset: u16, + pub prev: FileAddress, + pub next: FileAddress, } impl FileListInnerNode { @@ -45,16 +79,17 @@ impl FileListInnerNode { return Err(anyhow!("Buffer is too small")); } - let prev_node_page_number = u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]]); - let prev_node_offset = u16::from_be_bytes([buf[4], buf[5]]); - let next_node_page_number = u32::from_be_bytes([buf[6], buf[7], buf[8], buf[9]]); - let next_node_offset = u16::from_be_bytes([buf[10], buf[11]]); + + let prev = FileAddress::try_from_bytes(&buf[0..6])?; + let next = FileAddress::try_from_bytes(&buf[6..12])?; Ok(FileListInnerNode { - prev_node_page_number, - prev_node_offset, - next_node_page_number, - next_node_offset, + prev, + next }) } + + pub fn size() -> usize { + FileAddress::size() + FileAddress::size() + } } diff --git a/src/innodb/page/lob/mod.rs b/src/innodb/page/lob/mod.rs index 58c5ee5..ef7c121 100644 --- a/src/innodb/page/lob/mod.rs +++ b/src/innodb/page/lob/mod.rs @@ -1,8 +1,27 @@ -use crate::innodb::{file_list::FileListBaseNode, InnoDBError}; +use crate::innodb::{ + file_list::{FileListBaseNode, FileListInnerNode}, + InnoDBError, +}; use anyhow::{anyhow, Ok, Result}; use super::{Page, PageType}; +/* + * General Flow for reading extern records + * + * First Obtain `ExternalReference` from cluster index + * Load the page number from that ext ref. + * + * Load this page: + * + * If Type is BLOB or SDI BLOB you have a great time. + * TODO: Document this "easy route" + * + * If Type is LOB_FIRST (assert on this, it gotta be): + * + * + */ + #[derive(Debug, Clone)] pub struct LobFirstHeader { pub version: u8, @@ -16,8 +35,6 @@ pub struct LobFirstHeader { pub free_list_head: FileListBaseNode, } -pub const LOB_FIRST_HEADER_SIZE: usize = 54; - impl LobFirstHeader { pub fn try_from_bytes(buf: &[u8]) -> Result { if buf.len() < 54 { @@ -54,16 +71,25 @@ impl LobFirstHeader { free_list_head, }) } + + pub fn size() -> usize { + 1 + // Version + 1 + // Flags + 4 + // LOB Version + 6 + // trx id + 4 + // undo id + 4 + 6 + FileListBaseNode::size() + FileListBaseNode::size() + } } #[derive(Debug)] pub struct LobFirst<'a> { - pub page: Page<'a>, + pub page: &'a Page<'a>, pub header: LobFirstHeader, } impl<'a> LobFirst<'a> { - pub fn try_from_page(p: Page<'a>) -> Result { + pub fn try_from_page(p: &'a Page<'a>) -> Result { match p.header.page_type { PageType::LobFirst => Ok(LobFirst { header: LobFirstHeader::try_from_bytes(p.body())?, @@ -76,7 +102,139 @@ impl<'a> LobFirst<'a> { } } - pub fn data(&self) -> &[u8] { - &self.page.body()[LOB_FIRST_HEADER_SIZE..] + pub fn read(&self, offset: usize, buf: &mut [u8]) -> usize { + let index_array_size = LobIndexEntry::size() * 10; // Hardcoded? somehow see mysql: lob0first.h::node_count() + let data_len = self.header.data_length as usize; + let data = &self.body()[index_array_size..][..data_len]; + assert!(offset < data.len(), "offset too large"); + let data = &data[offset..]; + assert!(buf.len() <= data.len(), "Not implemented multi page extern expecting: {} have {}", buf.len(), data.len()); + let bytes_to_copy = std::cmp::min(buf.len(), data.len()); + buf[..bytes_to_copy].copy_from_slice(&data[..bytes_to_copy]); + bytes_to_copy + } + + pub fn body(&self) -> &[u8] { + &self.page.body()[LobFirstHeader::size()..] + } +} + +#[derive(Debug, Clone)] +pub struct LobIndexEntry { + pub file_list_node: FileListInnerNode, + pub version_list: FileListBaseNode, + pub creation_transaction_id: u64, // 6 bytes + pub modify_transaction_id: u64, // 6 bytes + pub undo_number: u32, // Undo Number for the creation transation + pub undo_number_modify: u32, // Undo Number for the modify transaction + pub page_number: u32, + pub data_length: u16, + // Two byte gap here + pub lob_version: u32, +} + +impl LobIndexEntry { + pub fn try_from_bytes(bytes: &[u8]) -> Result { + if bytes.len() < Self::size() { + return Err(anyhow!("Buffer not long enough")); + } + + let mut offset = 0; + + // Parse FileListInnerNode + let file_list_node_size = FileListInnerNode::size(); + let file_list_node = + FileListInnerNode::try_from_bytes(&bytes[offset..offset + file_list_node_size])?; + offset += file_list_node_size; + + // Parse FileListBaseNode + let version_list_size = FileListBaseNode::size(); + let version_list = + FileListBaseNode::try_from_bytes(&bytes[offset..offset + version_list_size])?; + offset += version_list_size; + + // Parse creation_transaction_id (6 bytes) + let creation_transaction_id = u64::from_be_bytes([ + 0, + 0, + bytes[offset], + bytes[offset + 1], + bytes[offset + 2], + bytes[offset + 3], + bytes[offset + 4], + bytes[offset + 5], + ]); + offset += 6; + + // Parse modify_transaction_id (6 bytes) + let modify_transaction_id = u64::from_be_bytes([ + 0, + 0, + bytes[offset], + bytes[offset + 1], + bytes[offset + 2], + bytes[offset + 3], + bytes[offset + 4], + bytes[offset + 5], + ]); + offset += 6; + + // Parse undo_number (4 bytes) + let undo_number = u32::from_be_bytes([ + bytes[offset], + bytes[offset + 1], + bytes[offset + 2], + bytes[offset + 3], + ]); + offset += 4; + + // Parse undo_number (4 bytes) + let undo_number_modify = u32::from_be_bytes([ + bytes[offset], + bytes[offset + 1], + bytes[offset + 2], + bytes[offset + 3], + ]); + offset += 4; + + // Parse page_number (4 bytes) + let page_number = u32::from_be_bytes([ + bytes[offset], + bytes[offset + 1], + bytes[offset + 2], + bytes[offset + 3], + ]); + offset += 4; + + // Parse data_length (4 bytes) + let data_length = u16::from_be_bytes([bytes[offset], bytes[offset + 1]]); + offset += 2; + + // Gap of 2 byte??? Why + offset += 2; + + let lob_version = u32::from_be_bytes([ + bytes[offset], + bytes[offset + 1], + bytes[offset + 2], + bytes[offset + 3], + ]); + offset += 4; + + Ok(LobIndexEntry { + file_list_node, + version_list, + creation_transaction_id, + modify_transaction_id, + undo_number, + undo_number_modify, + page_number, + data_length, + lob_version, + }) + } + + pub fn size() -> usize { + 60 } } diff --git a/src/innodb/table/blob_header.rs b/src/innodb/table/blob_header.rs index 5fb0fe4..85468f7 100644 --- a/src/innodb/table/blob_header.rs +++ b/src/innodb/table/blob_header.rs @@ -1,7 +1,7 @@ use anyhow::Result; #[derive(Debug, Clone)] -pub struct BlobHeader { +pub struct ExternReference { pub space_id: u32, pub page_number: u32, pub offset: u32, @@ -10,8 +10,9 @@ pub struct BlobHeader { pub length: u64, } -impl BlobHeader { - pub fn from_bytes(bytes: &[u8]) -> Result { +/// B-Tree Extern Reference +impl ExternReference { + pub fn from_bytes(bytes: &[u8]) -> Result { if bytes.len() < 20 { anyhow::bail!("Insufficient bytes to construct BlobHeader"); } @@ -33,13 +34,14 @@ impl BlobHeader { bytes[16], bytes[17], bytes[18], bytes[19] ]); - Ok(BlobHeader { + Ok(ExternReference { space_id, page_number, offset, - owner: (length & 0x8000_0000_0000_0000u64) != 0, + owner: (length & 0x8000_0000_0000_0000u64) == 0, inherit: (length & 0x4000_0000_0000_0000u64) != 0, - length: length & 0x3FFF_FFFF_FFFF_FFFFu64, + // There's technically a is being modified bit, idgaf + length: length & 0x0FFF_FFFF_FFFF_FFFFu64, }) } } diff --git a/src/innodb/table/row.rs b/src/innodb/table/row.rs index 6519444..c1521ae 100644 --- a/src/innodb/table/row.rs +++ b/src/innodb/table/row.rs @@ -1,17 +1,12 @@ use std::{ - collections::{HashMap, HashSet}, - fmt::Debug, - sync::Arc, + collections::{HashMap, HashSet}, fmt::Debug, ops::Deref, sync::Arc }; use crate::innodb::{ - buffer_manager::BufferManager, - page::{ + buffer_manager::BufferManager, file_list::FileListInnerNode, page::{ index::record::{Record, RECORD_HEADER_FIXED_LENGTH}, - lob::LobFirst, - }, - table::blob_header::BlobHeader, - InnoDBError, + lob::{LobFirst, LobIndexEntry}, + }, table::blob_header::ExternReference, InnoDBError }; use super::{ @@ -134,38 +129,57 @@ impl<'a> Row<'a> { fn load_extern( &self, - extern_header: &BlobHeader, + extern_header: &ExternReference, buffer_mgr: &mut dyn BufferManager, ) -> Result> { + let first_page_number = extern_header.page_number; let lob_first_page = - buffer_mgr.open_page(extern_header.space_id, extern_header.page_number)?; + buffer_mgr.open_page(extern_header.space_id, first_page_number)?; if lob_first_page.header.offset != extern_header.page_number { - buffer_mgr.close_page(lob_first_page); return Err(anyhow!(InnoDBError::InvalidPage)); } - let lob_first = LobFirst::try_from_page(lob_first_page)?; + let lob_first = LobFirst::try_from_page(lob_first_page.deref())?; + let index_list = &lob_first.header.index_list_head; trace!("LOB First: {:#?}", lob_first); - // Current page - if lob_first.header.index_list_head.first_node_page_number == lob_first.page.header.offset { - return Ok(Box::from( - &lob_first.data() - [lob_first.header.index_list_head.first_node_offset as usize..] - [..extern_header.length as usize], - )); + + let mut node_location = index_list.first_node; + let mut page_offset = 0; + + let mut output_buffer = Vec::::new(); + let mut filled = 0usize; + output_buffer.resize(extern_header.length as usize, 0); + + while !node_location.is_null() { + trace!("Inspecting Node at offset {}", node_location.offset); + assert_eq!(index_list.first_node.page_number, lob_first.page.header.offset, "assumption"); + let buf = &lob_first.page.raw_data[node_location.offset as usize..]; + let node = LobIndexEntry::try_from_bytes(buf)?; + trace!("Index Node: {:#?}", node); + + if node_location.page_number == first_page_number { + let bytes_read = lob_first.read(page_offset, &mut output_buffer[filled..]); + filled += bytes_read; + page_offset = page_offset.saturating_sub(bytes_read); + trace!("Read {} bytes from first page, expecting {} bytes", bytes_read, output_buffer.len()); + } + + node_location = node.file_list_node.next; } - unimplemented!(); + + assert_eq!(filled, output_buffer.len(), "Read incomplete"); + + Ok(output_buffer.into()) } fn parse_extern_field( &self, f: &Field, - extern_header: &BlobHeader, + extern_header: &ExternReference, buffer_mgr: &mut dyn BufferManager, ) -> FieldValue { // Load a page match self.load_extern(extern_header, buffer_mgr) { Ok(buf) => { - trace!("Returned buffer: {}", pretty_hex::pretty_hex(&buf)); f.parse(&buf, Some(extern_header.length)).0 }, Err(err) => { @@ -189,7 +203,7 @@ impl<'a> Row<'a> { let len = *self.field_len_map.get(&idx).unwrap() as usize; assert_eq!(len, 20, "Extern header should be 20 bytes long"); let extern_header = - BlobHeader::from_bytes(&buf[0..len]).expect("Can't make blob header"); + ExternReference::from_bytes(&buf[0..len]).expect("Can't make blob header"); trace!("Extern Header: {:?}", &extern_header); ( self.parse_extern_field(f, &extern_header, buf_mgr),