progress: Work on serializing and deserializing data structures, as well as disk storage

Cargo.lock

@@ -120,7 +120,7 @@ dependencies = [
  "arrow-schema",
  "chrono",
  "half",
- "hashbrown",
+ "hashbrown 0.14.0",
  "num",
 ]
 
@@ -355,6 +355,7 @@ dependencies = [
  "android-tzdata",
  "iana-time-zone",
  "num-traits",
+ "serde",
  "windows-targets",
 ]
 
@@ -447,6 +448,57 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
 
+[[package]]
+name = "darling"
+version = "0.20.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e"
+dependencies = [
+ "darling_core",
+ "darling_macro",
+]
+
+[[package]]
+name = "darling_core"
+version = "0.20.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621"
+dependencies = [
+ "fnv",
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim",
+ "syn",
+]
+
+[[package]]
+name = "darling_macro"
+version = "0.20.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
+dependencies = [
+ "darling_core",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "deranged"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3"
+dependencies = [
+ "powerfmt",
+ "serde",
+]
+
+[[package]]
+name = "equivalent"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
+
 [[package]]
 name = "flatbuffers"
 version = "23.5.26"
@@ -543,6 +595,12 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "hashbrown"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+
 [[package]]
 name = "hashbrown"
 version = "0.14.0"
@@ -561,6 +619,12 @@ version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
 
+[[package]]
+name = "hex"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
+
 [[package]]
 name = "http"
 version = "0.2.9"
@@ -641,6 +705,34 @@ dependencies = [
  "cc",
 ]
 
+[[package]]
+name = "ident_case"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
+
+[[package]]
+name = "indexmap"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
+dependencies = [
+ "autocfg",
+ "hashbrown 0.12.3",
+ "serde",
+]
+
+[[package]]
+name = "indexmap"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad227c3af19d4914570ad36d30409928b75967c298feb9ea1969db3a610bb14e"
+dependencies = [
+ "equivalent",
+ "hashbrown 0.14.0",
+ "serde",
+]
+
 [[package]]
 name = "integer-encoding"
 version = "3.0.4"
@@ -941,7 +1033,7 @@ dependencies = [
  "bytes",
  "chrono",
  "flate2",
- "hashbrown",
+ "hashbrown 0.14.0",
  "lz4",
  "num",
  "num-bigint",
@@ -1003,6 +1095,12 @@ version = "0.3.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
 
+[[package]]
+name = "powerfmt"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.17"
@@ -1111,6 +1209,15 @@ dependencies = [
  "serde_derive",
 ]
 
+[[package]]
+name = "serde-big-array"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "serde_arrays"
 version = "0.1.0"
@@ -1164,6 +1271,35 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "serde_with"
+version = "3.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64cd236ccc1b7a29e7e2739f27c0b2dd199804abc4290e32f59f3b68d6405c23"
+dependencies = [
+ "base64",
+ "chrono",
+ "hex",
+ "indexmap 1.9.3",
+ "indexmap 2.0.1",
+ "serde",
+ "serde_json",
+ "serde_with_macros",
+ "time",
+]
+
+[[package]]
+name = "serde_with_macros"
+version = "3.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93634eb5f75a2323b16de4748022ac4297f9e76b6dced2be287a099f41b5e788"
+dependencies = [
+ "darling",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "snap"
 version = "1.1.0"
@@ -1200,8 +1336,10 @@ dependencies = [
  "parquet",
  "rand",
  "serde",
+ "serde-big-array",
  "serde_arrays",
  "serde_json",
+ "serde_with",
  "tokio",
 ]
 
@@ -1245,6 +1383,35 @@ dependencies = [
  "ordered-float",
 ]
 
+[[package]]
+name = "time"
+version = "0.3.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5"
+dependencies = [
+ "deranged",
+ "itoa",
+ "powerfmt",
+ "serde",
+ "time-core",
+ "time-macros",
+]
+
+[[package]]
+name = "time-core"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
+
+[[package]]
+name = "time-macros"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20"
+dependencies = [
+ "time-core",
+]
+
 [[package]]
 name = "tiny-keccak"
 version = "2.0.2"

Cargo.toml

@@ -12,6 +12,8 @@ clap = { version = "4.4.5", features = ["derive"] }
 parquet = "47.0.0"
 rand = "0.8.5"
 serde = { version = "1.0.189", features = ["derive"] }
+serde-big-array = "0.5.1"
 serde_arrays = "0.1.0"
 serde_json = "1.0.107"
+serde_with = "3.4.0"
 tokio = { version = "1.32.0", features = ["macros", "rt-multi-thread"] }

src/main.rs

@@ -1,5 +1,9 @@
 #![feature(test)]
 
+extern crate serde;
+#[macro_use]
+extern crate serde_big_array;
+
 mod simple_server;
 mod storage;
 mod storage_server;

src/simple_server/server.rs

@@ -1,3 +1,4 @@
+use crate::storage::disk_storage::ChunkStorageCache;
 use crate::storage::world::{BlockID, BlockPos, BlockRange, ChunkData, ChunkPos};
 use crate::storage_server::StorageServer;
 
@@ -9,66 +10,49 @@ struct MultipleBlocks {
 
 #[derive(Debug)]
 pub struct SimpleServer {
-    chunks: Vec<ChunkData>,
+    chunk_storage: ChunkStorageCache,
-    // block_ranges: Vec<MultipleBlocks>,
 }
 
 impl SimpleServer {
     pub fn new() -> Self {
         SimpleServer {
-            chunks: Vec::new(),
+            chunk_storage: ChunkStorageCache::new(),
-            // block_ranges: Vec::new(),
         }
     }
 
     pub fn num_chunks(&self) -> usize {
-        self.chunks.len()
+        unimplemented!()
     }
 
-    fn chunk_at_block_mut(&mut self, block_pos: &BlockPos) -> Option<&mut ChunkData> {
+    fn chunk_at(&mut self, block_pos: &BlockPos) -> Option<ChunkData> {
-        // Find what chunk the block is in
         let chunk_pos = ChunkPos::from(block_pos);
 
-        // Find the chunk with the correct index
+        let chunk = self
-        for chunk in self.chunks.iter_mut() {
+            .chunk_storage
-            if chunk.pos == chunk_pos {
+            .fetch_chunk_by_pos(&chunk_pos)
-                return Some(chunk);
+            .expect("Finding chunk failed");
-            }
-        }
 
-        None
+        Some(chunk)
-    }
-
-    fn chunk_at(&self, block_pos: &BlockPos) -> Option<&ChunkData> {
-        let chunk_pos = ChunkPos::from(block_pos);
-
-        for chunk in self.chunks.iter() {
-            if chunk.pos == chunk_pos {
-                return Some(chunk);
-            }
-        }
-
-        None
     }
 
     fn create_chunk_at(&mut self, chunk_pos: &ChunkPos) {
-        let new_chunk = ChunkData::new(chunk_pos);
+        self.chunk_storage
-
+            .fetch_chunk_by_pos(&chunk_pos)
-        self.chunks.push(new_chunk);
+            .expect("Creatinc chunk failed");
     }
 }
 
 impl StorageServer for SimpleServer {
     fn change_block(&mut self, target_state: BlockID, world_position: &BlockPos) {
-        let mut chunk = self.chunk_at_block_mut(world_position);
+        let mut chunk = self.chunk_at(world_position);
 
         // Test if there is a chunk that already exists
         if chunk.is_none() {
             self.create_chunk_at(&ChunkPos::from(world_position));
-            chunk = self.chunk_at_block_mut(world_position);
+            chunk = self.chunk_at(world_position);
         }
 
-        let chunk = chunk.expect("Could not find chunk");
+        let mut chunk = chunk.expect("Could not find chunk");
 
         // Find the section that the block is located in
         let current_section = &mut chunk.sections[world_position.y % 16];
@@ -79,16 +63,9 @@ impl StorageServer for SimpleServer {
 
     fn change_block_range(&mut self, target_stage: BlockID, start: &BlockPos, end: &BlockPos) {
         unimplemented!()
-        // self.block_ranges.push(MultipleBlocks {
-        //     id: target_stage,
-        //     range: BlockRange {
-        //         start: start.clone(),
-        //         end: end.clone(),
-        //     },
-        // })
     }
 
-    fn read_block_at(&self, pos: &BlockPos) -> BlockID {
+    fn read_block_at(&mut self, pos: &BlockPos) -> BlockID {
         let chunk = self.chunk_at(pos);
 
         if let Some(chunk) = chunk {
@@ -97,12 +74,6 @@ impl StorageServer for SimpleServer {
             return chunk_section.get_block_at_index(pos).clone();
         }
 
-        // for blocks in self.block_ranges.iter() {
-        //     if blocks.range.within_range(&pos) {
-        //         return blocks.id.clone();
-        //     }
-        // }
-
         BlockID::Empty
     }
 }

src/storage/disk_storage.rs

@@ -1,19 +1,18 @@
-use super::world::{BlockID, ChunkData, ChunkPos};
+use super::world::{ChunkData, ChunkPos};
 use std::cmp::Ordering;
+use std::error::Error;
+use std::io::{BufReader, ErrorKind, Write};
 use std::{collections::HashMap, fs::File, time::Instant};
 
-const DATABASE_FILE_LOCATION: &str = "./persistence";
-
-struct ChunkFile {}
-
 const CACHED_CHUNK_FILES: usize = 1;
 
 /// `ChunkStorageCache` caches a list of the most recently used file handles
 /// where chunks are stored from, and allows for faster accessing of the data
 /// from chunks
-struct ChunkStorageCache {
+#[derive(Debug)]
+pub struct ChunkStorageCache {
     // `cached_chunk_files` is a vector of cached file handles that are already open
-    cached_chunk_files: [File; CACHED_CHUNK_FILES],
+    cached_chunk_files: [Option<File>; CACHED_CHUNK_FILES],
     // `cached_file_names` is a list of all the filenames that are contained
     // within the cache
     cached_file_names: HashMap<String, usize>,
@@ -21,8 +20,46 @@ struct ChunkStorageCache {
 }
 
 impl ChunkStorageCache {
-    fn load_chunk_file(&mut self, file_name: &str) -> &File {
+    pub fn new() -> Self {
-        let chunk_file = File::open(file_name).expect("Opening file for chunk failed");
+        ChunkStorageCache {
+            cached_chunk_files: [None; CACHED_CHUNK_FILES],
+            cached_file_names: HashMap::new(),
+            last_used_times: [Instant::now(); CACHED_CHUNK_FILES],
+        }
+    }
+
+    /// `load_chunk_file` is called whenever a file is missing in the file cache
+    /// and needs to be loaded from disk
+    ///
+    /// This replaces a slot for another file in the cache, according to the
+    /// caching strategy
+    fn load_chunk_file(&mut self, chunk_pos: &ChunkPos, file_name: &str) -> &File {
+        let chunk_file = File::options().write(true).read(true).open(file_name);
+
+        let chunk_file = match chunk_file {
+            Ok(file) => file,
+            Err(err) => match err.kind() {
+                ErrorKind::NotFound => {
+                    let mut new_chunk_file = File::options()
+                        .write(true)
+                        .read(true)
+                        .create(true)
+                        .open(file_name)
+                        .expect("Opening new chunk file failed");
+
+                    let blank_chunk = ChunkData::new(chunk_pos);
+
+                    let encoded_chunk = serde_json::to_string(&blank_chunk).unwrap();
+
+                    new_chunk_file
+                        .write_all(encoded_chunk.as_bytes())
+                        .expect("Error writing data to chunk");
+
+                    new_chunk_file
+                }
+                err => panic!("Opening new file for chunk failed with: {:?}", err),
+            },
+        };
 
         // Add the newly opened file to the cache
 
@@ -45,6 +82,7 @@ impl ChunkStorageCache {
         // * Replace the last used time with the curent time
         // * Replace the open file with the current one
 
+        if !self.cached_file_names.is_empty() {
             // Find the name of the previous entry
             let (previous_file_name, _) = self
                 .cached_file_names
@@ -53,13 +91,14 @@ impl ChunkStorageCache {
                 .expect("The last used index should always have a name");
 
             self.cached_file_names.remove(&previous_file_name.clone());
+        }
         self.cached_file_names
             .insert(file_name.to_string(), last_used_index);
         // Replace the timestamp with the new timestamp
         self.last_used_times[last_used_index] = Instant::now();
-        self.cached_chunk_files[last_used_index] = chunk_file;
+        self.cached_chunk_files[last_used_index] = Some(chunk_file);
 
-        &self.cached_chunk_files[last_used_index]
+        self.cached_chunk_files[last_used_index].as_ref().unwrap()
     }
 
     /// `fetch_chunk_by_pos` takes in the position of a chunk, and returns the
@@ -67,16 +106,23 @@ impl ChunkStorageCache {
     ///
     /// This operation is cached, if possible, so that subsequent accesses to
     /// the same chunk are handled by the same file
-    pub fn fetch_chunk_by_pos(&mut self, pos: &ChunkPos) -> ChunkData {
+    pub fn fetch_chunk_by_pos(&mut self, pos: &ChunkPos) -> Result<ChunkData, Box<dyn Error>> {
         let file_name = pos.storage_file_name();
 
         let file_index = self.cached_file_names.get(file_name.as_str());
 
         let chunk_file = match file_index {
-            Some(index) => &self.cached_chunk_files[*index],
+            Some(index) => self.cached_chunk_files[*index].as_ref().unwrap(),
-            None => self.load_chunk_file(file_name.as_str()),
+            None => self.load_chunk_file(pos, file_name.as_str()),
         };
 
-        panic!("Yes");
+        let file_contents = std::io::read_to_string(chunk_file)?;
+
+        let read_data: ChunkData = serde_json::from_str(file_contents.as_str())?;
+
+        // let read_data: ChunkData = serde_json::from_reader(&mut file_buffer)?;
+        // let read_data = ChunkData::new(&ChunkPos { x: 0, z: 0 });
+
+        Ok(read_data)
     }
 }

src/storage/mod.rs

@@ -1,3 +1,3 @@
 mod chunk_compression;
-mod disk_storage;
+pub mod disk_storage;
 pub mod world;

src/storage/world.rs

@@ -1,17 +1,21 @@
 use core::fmt;
 use serde::ser;
 use serde::ser::{SerializeSeq, SerializeStruct, Serializer};
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
 use std::{
     cmp::{max, min},
     fmt::Debug,
     fs::File,
 };
 
+use serde_big_array::BigArray;
+
 const SECTIONS_PER_CHUNK: usize = 16;
 const SLICE_SIZE: usize = 16 * 16;
 
-#[derive(Debug, Clone, PartialEq, Serialize)]
+const DATABASE_FILE_LOCATION: &str = "./persistence";
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct ChunkPos {
     pub x: isize,
     pub z: isize,
@@ -28,29 +32,30 @@ impl From<&BlockPos> for ChunkPos {
 
 impl ChunkPos {
     pub fn storage_file_name(&self) -> String {
-        format!("{}.{}.chunk", self.x, self.z)
+        format!("{DATABASE_FILE_LOCATION}/{}.{}.chunk", self.x, self.z)
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
 pub struct ChunkData {
     pub pos: ChunkPos,
+    #[serde(with = "serde_arrays")]
     pub sections: [ChunkSection; SECTIONS_PER_CHUNK],
 }
 
-impl Serialize for ChunkData {
+// impl Serialize for ChunkData {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+//     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
+//     where
-        S: serde::Serializer,
+//         S: serde::Serializer,
-    {
+//     {
-        let mut seq = serializer.serialize_seq(Some(self.sections.len()))?;
+//         let mut seq = serializer.serialize_seq(Some(self.sections.len()))?;
-
+//
-        for section in self.sections {
+//         for section in self.sections {
-            seq.serialize_element(&section)?;
+//             seq.serialize_element(&section)?;
-        }
+//         }
-        seq.end()
+//         seq.end()
-    }
+//     }
-}
+// }
 
 impl ChunkData {
     pub fn new(pos: &ChunkPos) -> Self {
@@ -74,7 +79,7 @@ impl ChunkData {
 }
 
 // https://wiki.vg/Chunk_Format
-#[derive(Clone, Copy, Serialize)]
+#[derive(Clone, Copy, Serialize, Deserialize, Debug)]
 pub struct ChunkSection {
     /// The number of non-empty blocks in the section. If completely full, the
     /// section contains a 16 x 16 x 16 cube of blocks = 4096 blocks
@@ -87,15 +92,15 @@ pub struct ChunkSection {
     block_states: [BlockID; 16 * 16 * 16],
 }
 
-impl Debug for ChunkSection {
+// impl Debug for ChunkSection {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+//     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "ChunkSection {{ blocks: {}, states: ", self.block_count)?;
+//         write!(f, "ChunkSection {{ blocks: {}, states: ", self.block_count)?;
-        if self.block_count > 0 {
+//         if self.block_count > 0 {
-            write!(f, "{:?}", self.block_states)?;
+//             write!(f, "{:?}", self.block_states)?;
-        }
+//         }
-        write!(f, " }}")
+//         write!(f, " }}")
-    }
+//     }
-}
+// }
 
 impl ChunkSection {
     pub fn new() -> Self {
@@ -201,7 +206,7 @@ impl BlockRange {
 
 /// BlockID represents the type of block stored
 #[repr(u8)]
-#[derive(Debug, Clone, Copy, PartialEq, Serialize)]
+#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
 pub enum BlockID {
     Empty,
     Generic,

src/storage_server.rs

@@ -9,7 +9,7 @@ pub trait StorageServer {
 
     /// `read_block_at` returns the id of the block at the location specified
     /// If no block is present, the returned id will be of the empty type
-    fn read_block_at(&self, pos: &BlockPos) -> BlockID;
+    fn read_block_at(&mut self, pos: &BlockPos) -> BlockID;
 }
 
 #[tokio::main]