From 7419412be47f273e78fdc872c9f816ab75129968 Mon Sep 17 00:00:00 2001 From: Nicholas Novak <34256932+NickyBoy89@users.noreply.github.com> Date: Sat, 28 Oct 2023 21:39:45 -0700 Subject: [PATCH] progress: Work on serializing and deserializing data structures, as well as disk storage --- Cargo.lock | 171 +++++++++++++++++++++++++++++++++++- Cargo.toml | 2 + src/main.rs | 4 + src/simple_server/server.rs | 63 ++++--------- src/storage/disk_storage.rs | 90 ++++++++++++++----- src/storage/mod.rs | 2 +- src/storage/world.rs | 61 +++++++------ src/storage_server.rs | 2 +- 8 files changed, 295 insertions(+), 100 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 37eff11..0685da7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -120,7 +120,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "hashbrown", + "hashbrown 0.14.0", "num", ] @@ -355,6 +355,7 @@ dependencies = [ "android-tzdata", "iana-time-zone", "num-traits", + "serde", "windows-targets", ] @@ -447,6 +448,57 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +[[package]] +name = "darling" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "deranged" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3" +dependencies = [ + "powerfmt", + "serde", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "flatbuffers" version = "23.5.26" @@ -543,6 +595,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.0" @@ -561,6 +619,12 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "http" version = "0.2.9" @@ -641,6 +705,34 @@ dependencies = [ "cc", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad227c3af19d4914570ad36d30409928b75967c298feb9ea1969db3a610bb14e" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", + "serde", +] + [[package]] name = "integer-encoding" version = "3.0.4" @@ -941,7 +1033,7 @@ dependencies = [ "bytes", "chrono", "flate2", - "hashbrown", + "hashbrown 0.14.0", "lz4", "num", "num-bigint", @@ -1003,6 +1095,12 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -1111,6 +1209,15 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-big-array" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f" +dependencies = [ + "serde", +] + [[package]] name = "serde_arrays" version = "0.1.0" @@ -1164,6 +1271,35 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64cd236ccc1b7a29e7e2739f27c0b2dd199804abc4290e32f59f3b68d6405c23" +dependencies = [ + "base64", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.0.1", + "serde", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93634eb5f75a2323b16de4748022ac4297f9e76b6dced2be287a099f41b5e788" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "snap" version = "1.1.0" @@ -1200,8 +1336,10 @@ dependencies = [ "parquet", "rand", "serde", + "serde-big-array", "serde_arrays", "serde_json", + "serde_with", "tokio", ] @@ -1245,6 +1383,35 @@ dependencies = [ "ordered-float", ] +[[package]] +name = "time" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" +dependencies = [ + "deranged", + "itoa", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" +dependencies = [ + "time-core", +] + [[package]] name = "tiny-keccak" version = "2.0.2" diff --git a/Cargo.toml b/Cargo.toml index 8f4af9d..ce36e98 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,8 @@ clap = { version = "4.4.5", features = ["derive"] } parquet = "47.0.0" rand = "0.8.5" serde = { version = "1.0.189", features = ["derive"] } +serde-big-array = "0.5.1" serde_arrays = "0.1.0" serde_json = "1.0.107" +serde_with = "3.4.0" tokio = { version = "1.32.0", features = ["macros", "rt-multi-thread"] } diff --git a/src/main.rs b/src/main.rs index dd3d483..e71ef9f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,9 @@ #![feature(test)] +extern crate serde; +#[macro_use] +extern crate serde_big_array; + mod simple_server; mod storage; mod storage_server; diff --git a/src/simple_server/server.rs b/src/simple_server/server.rs index 7ef868f..ed5737b 100644 --- a/src/simple_server/server.rs +++ b/src/simple_server/server.rs @@ -1,3 +1,4 @@ +use crate::storage::disk_storage::ChunkStorageCache; use crate::storage::world::{BlockID, BlockPos, BlockRange, ChunkData, ChunkPos}; use crate::storage_server::StorageServer; @@ -9,66 +10,49 @@ struct MultipleBlocks { #[derive(Debug)] pub struct SimpleServer { - chunks: Vec, - // block_ranges: Vec, + chunk_storage: ChunkStorageCache, } impl SimpleServer { pub fn new() -> Self { SimpleServer { - chunks: Vec::new(), - // block_ranges: Vec::new(), + chunk_storage: ChunkStorageCache::new(), } } pub fn num_chunks(&self) -> usize { - self.chunks.len() + unimplemented!() } - fn chunk_at_block_mut(&mut self, block_pos: &BlockPos) -> Option<&mut ChunkData> { - // Find what chunk the block is in + fn chunk_at(&mut self, block_pos: &BlockPos) -> Option { let chunk_pos = ChunkPos::from(block_pos); - // Find the chunk with the correct index - for chunk in self.chunks.iter_mut() { - if chunk.pos == chunk_pos { - return Some(chunk); - } - } + let chunk = self + .chunk_storage + .fetch_chunk_by_pos(&chunk_pos) + .expect("Finding chunk failed"); - None - } - - fn chunk_at(&self, block_pos: &BlockPos) -> Option<&ChunkData> { - let chunk_pos = ChunkPos::from(block_pos); - - for chunk in self.chunks.iter() { - if chunk.pos == chunk_pos { - return Some(chunk); - } - } - - None + Some(chunk) } fn create_chunk_at(&mut self, chunk_pos: &ChunkPos) { - let new_chunk = ChunkData::new(chunk_pos); - - self.chunks.push(new_chunk); + self.chunk_storage + .fetch_chunk_by_pos(&chunk_pos) + .expect("Creatinc chunk failed"); } } impl StorageServer for SimpleServer { fn change_block(&mut self, target_state: BlockID, world_position: &BlockPos) { - let mut chunk = self.chunk_at_block_mut(world_position); + let mut chunk = self.chunk_at(world_position); // Test if there is a chunk that already exists if chunk.is_none() { self.create_chunk_at(&ChunkPos::from(world_position)); - chunk = self.chunk_at_block_mut(world_position); + chunk = self.chunk_at(world_position); } - let chunk = chunk.expect("Could not find chunk"); + let mut chunk = chunk.expect("Could not find chunk"); // Find the section that the block is located in let current_section = &mut chunk.sections[world_position.y % 16]; @@ -79,16 +63,9 @@ impl StorageServer for SimpleServer { fn change_block_range(&mut self, target_stage: BlockID, start: &BlockPos, end: &BlockPos) { unimplemented!() - // self.block_ranges.push(MultipleBlocks { - // id: target_stage, - // range: BlockRange { - // start: start.clone(), - // end: end.clone(), - // }, - // }) } - fn read_block_at(&self, pos: &BlockPos) -> BlockID { + fn read_block_at(&mut self, pos: &BlockPos) -> BlockID { let chunk = self.chunk_at(pos); if let Some(chunk) = chunk { @@ -97,12 +74,6 @@ impl StorageServer for SimpleServer { return chunk_section.get_block_at_index(pos).clone(); } - // for blocks in self.block_ranges.iter() { - // if blocks.range.within_range(&pos) { - // return blocks.id.clone(); - // } - // } - BlockID::Empty } } diff --git a/src/storage/disk_storage.rs b/src/storage/disk_storage.rs index 95c6449..84c6cf5 100644 --- a/src/storage/disk_storage.rs +++ b/src/storage/disk_storage.rs @@ -1,19 +1,18 @@ -use super::world::{BlockID, ChunkData, ChunkPos}; +use super::world::{ChunkData, ChunkPos}; use std::cmp::Ordering; +use std::error::Error; +use std::io::{BufReader, ErrorKind, Write}; use std::{collections::HashMap, fs::File, time::Instant}; -const DATABASE_FILE_LOCATION: &str = "./persistence"; - -struct ChunkFile {} - const CACHED_CHUNK_FILES: usize = 1; /// `ChunkStorageCache` caches a list of the most recently used file handles /// where chunks are stored from, and allows for faster accessing of the data /// from chunks -struct ChunkStorageCache { +#[derive(Debug)] +pub struct ChunkStorageCache { // `cached_chunk_files` is a vector of cached file handles that are already open - cached_chunk_files: [File; CACHED_CHUNK_FILES], + cached_chunk_files: [Option; CACHED_CHUNK_FILES], // `cached_file_names` is a list of all the filenames that are contained // within the cache cached_file_names: HashMap, @@ -21,8 +20,46 @@ struct ChunkStorageCache { } impl ChunkStorageCache { - fn load_chunk_file(&mut self, file_name: &str) -> &File { - let chunk_file = File::open(file_name).expect("Opening file for chunk failed"); + pub fn new() -> Self { + ChunkStorageCache { + cached_chunk_files: [None; CACHED_CHUNK_FILES], + cached_file_names: HashMap::new(), + last_used_times: [Instant::now(); CACHED_CHUNK_FILES], + } + } + + /// `load_chunk_file` is called whenever a file is missing in the file cache + /// and needs to be loaded from disk + /// + /// This replaces a slot for another file in the cache, according to the + /// caching strategy + fn load_chunk_file(&mut self, chunk_pos: &ChunkPos, file_name: &str) -> &File { + let chunk_file = File::options().write(true).read(true).open(file_name); + + let chunk_file = match chunk_file { + Ok(file) => file, + Err(err) => match err.kind() { + ErrorKind::NotFound => { + let mut new_chunk_file = File::options() + .write(true) + .read(true) + .create(true) + .open(file_name) + .expect("Opening new chunk file failed"); + + let blank_chunk = ChunkData::new(chunk_pos); + + let encoded_chunk = serde_json::to_string(&blank_chunk).unwrap(); + + new_chunk_file + .write_all(encoded_chunk.as_bytes()) + .expect("Error writing data to chunk"); + + new_chunk_file + } + err => panic!("Opening new file for chunk failed with: {:?}", err), + }, + }; // Add the newly opened file to the cache @@ -45,21 +82,23 @@ impl ChunkStorageCache { // * Replace the last used time with the curent time // * Replace the open file with the current one - // Find the name of the previous entry - let (previous_file_name, _) = self - .cached_file_names - .iter() - .find(|(_, &array_index)| array_index == last_used_index) - .expect("The last used index should always have a name"); + if !self.cached_file_names.is_empty() { + // Find the name of the previous entry + let (previous_file_name, _) = self + .cached_file_names + .iter() + .find(|(_, &array_index)| array_index == last_used_index) + .expect("The last used index should always have a name"); - self.cached_file_names.remove(&previous_file_name.clone()); + self.cached_file_names.remove(&previous_file_name.clone()); + } self.cached_file_names .insert(file_name.to_string(), last_used_index); // Replace the timestamp with the new timestamp self.last_used_times[last_used_index] = Instant::now(); - self.cached_chunk_files[last_used_index] = chunk_file; + self.cached_chunk_files[last_used_index] = Some(chunk_file); - &self.cached_chunk_files[last_used_index] + self.cached_chunk_files[last_used_index].as_ref().unwrap() } /// `fetch_chunk_by_pos` takes in the position of a chunk, and returns the @@ -67,16 +106,23 @@ impl ChunkStorageCache { /// /// This operation is cached, if possible, so that subsequent accesses to /// the same chunk are handled by the same file - pub fn fetch_chunk_by_pos(&mut self, pos: &ChunkPos) -> ChunkData { + pub fn fetch_chunk_by_pos(&mut self, pos: &ChunkPos) -> Result> { let file_name = pos.storage_file_name(); let file_index = self.cached_file_names.get(file_name.as_str()); let chunk_file = match file_index { - Some(index) => &self.cached_chunk_files[*index], - None => self.load_chunk_file(file_name.as_str()), + Some(index) => self.cached_chunk_files[*index].as_ref().unwrap(), + None => self.load_chunk_file(pos, file_name.as_str()), }; - panic!("Yes"); + let file_contents = std::io::read_to_string(chunk_file)?; + + let read_data: ChunkData = serde_json::from_str(file_contents.as_str())?; + + // let read_data: ChunkData = serde_json::from_reader(&mut file_buffer)?; + // let read_data = ChunkData::new(&ChunkPos { x: 0, z: 0 }); + + Ok(read_data) } } diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 107a43d..0565773 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -1,3 +1,3 @@ mod chunk_compression; -mod disk_storage; +pub mod disk_storage; pub mod world; diff --git a/src/storage/world.rs b/src/storage/world.rs index a2c8dd4..0c49169 100644 --- a/src/storage/world.rs +++ b/src/storage/world.rs @@ -1,17 +1,21 @@ use core::fmt; use serde::ser; use serde::ser::{SerializeSeq, SerializeStruct, Serializer}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use std::{ cmp::{max, min}, fmt::Debug, fs::File, }; +use serde_big_array::BigArray; + const SECTIONS_PER_CHUNK: usize = 16; const SLICE_SIZE: usize = 16 * 16; -#[derive(Debug, Clone, PartialEq, Serialize)] +const DATABASE_FILE_LOCATION: &str = "./persistence"; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct ChunkPos { pub x: isize, pub z: isize, @@ -28,29 +32,30 @@ impl From<&BlockPos> for ChunkPos { impl ChunkPos { pub fn storage_file_name(&self) -> String { - format!("{}.{}.chunk", self.x, self.z) + format!("{DATABASE_FILE_LOCATION}/{}.{}.chunk", self.x, self.z) } } -#[derive(Debug)] +#[derive(Debug, Serialize, Deserialize)] pub struct ChunkData { pub pos: ChunkPos, + #[serde(with = "serde_arrays")] pub sections: [ChunkSection; SECTIONS_PER_CHUNK], } -impl Serialize for ChunkData { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - let mut seq = serializer.serialize_seq(Some(self.sections.len()))?; - - for section in self.sections { - seq.serialize_element(§ion)?; - } - seq.end() - } -} +// impl Serialize for ChunkData { +// fn serialize(&self, serializer: S) -> Result +// where +// S: serde::Serializer, +// { +// let mut seq = serializer.serialize_seq(Some(self.sections.len()))?; +// +// for section in self.sections { +// seq.serialize_element(§ion)?; +// } +// seq.end() +// } +// } impl ChunkData { pub fn new(pos: &ChunkPos) -> Self { @@ -74,7 +79,7 @@ impl ChunkData { } // https://wiki.vg/Chunk_Format -#[derive(Clone, Copy, Serialize)] +#[derive(Clone, Copy, Serialize, Deserialize, Debug)] pub struct ChunkSection { /// The number of non-empty blocks in the section. If completely full, the /// section contains a 16 x 16 x 16 cube of blocks = 4096 blocks @@ -87,15 +92,15 @@ pub struct ChunkSection { block_states: [BlockID; 16 * 16 * 16], } -impl Debug for ChunkSection { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "ChunkSection {{ blocks: {}, states: ", self.block_count)?; - if self.block_count > 0 { - write!(f, "{:?}", self.block_states)?; - } - write!(f, " }}") - } -} +// impl Debug for ChunkSection { +// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +// write!(f, "ChunkSection {{ blocks: {}, states: ", self.block_count)?; +// if self.block_count > 0 { +// write!(f, "{:?}", self.block_states)?; +// } +// write!(f, " }}") +// } +// } impl ChunkSection { pub fn new() -> Self { @@ -201,7 +206,7 @@ impl BlockRange { /// BlockID represents the type of block stored #[repr(u8)] -#[derive(Debug, Clone, Copy, PartialEq, Serialize)] +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] pub enum BlockID { Empty, Generic, diff --git a/src/storage_server.rs b/src/storage_server.rs index 6f60dae..233e321 100644 --- a/src/storage_server.rs +++ b/src/storage_server.rs @@ -9,7 +9,7 @@ pub trait StorageServer { /// `read_block_at` returns the id of the block at the location specified /// If no block is present, the returned id will be of the empty type - fn read_block_at(&self, pos: &BlockPos) -> BlockID; + fn read_block_at(&mut self, pos: &BlockPos) -> BlockID; } #[tokio::main]