nnovak/spatial-db
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

progress: Work on serializing and deserializing data structures, as well as disk storage

Browse Source
This commit is contained in:
Nicholas Novak 2023-10-28 21:39:45 -07:00
parent 63ff6e7c2c
commit 7419412be4
8 changed files with 295 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

171
Cargo.lock generated
View File

@ -120,7 +120,7 @@ dependencies = [
"arrow-schema",
"chrono",
"half",
"hashbrown",
"hashbrown 0.14.0",
"num",
]
@ -355,6 +355,7 @@ dependencies = [
"android-tzdata",
"iana-time-zone",
"num-traits",
"serde",
"windows-targets",
]
@ -447,6 +448,57 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
[[package]]
name = "darling"
version = "0.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e"
dependencies = [
"darling_core",
"darling_macro",
]
[[package]]
name = "darling_core"
version = "0.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
"strsim",
"syn",
]
[[package]]
name = "darling_macro"
version = "0.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5"
dependencies = [
"darling_core",
"quote",
"syn",
]
[[package]]
name = "deranged"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f32d04922c60427da6f9fef14d042d9edddef64cb9d4ce0d64d0685fbeb1fd3"
dependencies = [
"powerfmt",
"serde",
]
[[package]]
name = "equivalent"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
[[package]]
name = "flatbuffers"
version = "23.5.26"
@ -543,6 +595,12 @@ dependencies = [
"num-traits",
]
[[package]]
name = "hashbrown"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]]
name = "hashbrown"
version = "0.14.0"
@ -561,6 +619,12 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
[[package]]
name = "hex"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]]
name = "http"
version = "0.2.9"
@ -641,6 +705,34 @@ dependencies = [
"cc",
]
[[package]]
name = "ident_case"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
[[package]]
name = "indexmap"
version = "1.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
dependencies = [
"autocfg",
"hashbrown 0.12.3",
"serde",
]
[[package]]
name = "indexmap"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad227c3af19d4914570ad36d30409928b75967c298feb9ea1969db3a610bb14e"
dependencies = [
"equivalent",
"hashbrown 0.14.0",
"serde",
]
[[package]]
name = "integer-encoding"
version = "3.0.4"
@ -941,7 +1033,7 @@ dependencies = [
"bytes",
"chrono",
"flate2",
"hashbrown",
"hashbrown 0.14.0",
"lz4",
"num",
"num-bigint",
@ -1003,6 +1095,12 @@ version = "0.3.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
[[package]]
name = "powerfmt"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
[[package]]
name = "ppv-lite86"
version = "0.2.17"
@ -1111,6 +1209,15 @@ dependencies = [
"serde_derive",
]
[[package]]
name = "serde-big-array"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f"
dependencies = [
"serde",
]
[[package]]
name = "serde_arrays"
version = "0.1.0"
@ -1164,6 +1271,35 @@ dependencies = [
"serde",
]
[[package]]
name = "serde_with"
version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64cd236ccc1b7a29e7e2739f27c0b2dd199804abc4290e32f59f3b68d6405c23"
dependencies = [
"base64",
"chrono",
"hex",
"indexmap 1.9.3",
"indexmap 2.0.1",
"serde",
"serde_json",
"serde_with_macros",
"time",
]
[[package]]
name = "serde_with_macros"
version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93634eb5f75a2323b16de4748022ac4297f9e76b6dced2be287a099f41b5e788"
dependencies = [
"darling",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "snap"
version = "1.1.0"
@ -1200,8 +1336,10 @@ dependencies = [
"parquet",
"rand",
"serde",
"serde-big-array",
"serde_arrays",
"serde_json",
"serde_with",
"tokio",
]
@ -1245,6 +1383,35 @@ dependencies = [
"ordered-float",
]
[[package]]
name = "time"
version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5"
dependencies = [
"deranged",
"itoa",
"powerfmt",
"serde",
"time-core",
"time-macros",
]
[[package]]
name = "time-core"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
[[package]]
name = "time-macros"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20"
dependencies = [
"time-core",
]
[[package]]
name = "tiny-keccak"
version = "2.0.2"

2
Cargo.toml
View File

@ -12,6 +12,8 @@ clap = { version = "4.4.5", features = ["derive"] }
parquet = "47.0.0"
rand = "0.8.5"
serde = { version = "1.0.189", features = ["derive"] }
serde-big-array = "0.5.1"
serde_arrays = "0.1.0"
serde_json = "1.0.107"
serde_with = "3.4.0"
tokio = { version = "1.32.0", features = ["macros", "rt-multi-thread"] }

4
src/main.rs
View File

@ -1,5 +1,9 @@
#![feature(test)]
extern crate serde;
#[macro_use]
extern crate serde_big_array;
mod simple_server;
mod storage;
mod storage_server;

63
src/simple_server/server.rs
View File

@ -1,3 +1,4 @@
use crate::storage::disk_storage::ChunkStorageCache;
use crate::storage::world::{BlockID, BlockPos, BlockRange, ChunkData, ChunkPos};
use crate::storage_server::StorageServer;
@ -9,66 +10,49 @@ struct MultipleBlocks {
#[derive(Debug)]
pub struct SimpleServer {
chunks: Vec<ChunkData>,
// block_ranges: Vec<MultipleBlocks>,
chunk_storage: ChunkStorageCache,
}
impl SimpleServer {
pub fn new() -> Self {
SimpleServer {
chunks: Vec::new(),
// block_ranges: Vec::new(),
chunk_storage: ChunkStorageCache::new(),
}
}
pub fn num_chunks(&self) -> usize {
self.chunks.len()
unimplemented!()
}
fn chunk_at_block_mut(&mut self, block_pos: &BlockPos) -> Option<&mut ChunkData> {
// Find what chunk the block is in
fn chunk_at(&mut self, block_pos: &BlockPos) -> Option<ChunkData> {
let chunk_pos = ChunkPos::from(block_pos);
// Find the chunk with the correct index
for chunk in self.chunks.iter_mut() {
if chunk.pos == chunk_pos {
return Some(chunk);
}
}
let chunk = self
.chunk_storage
.fetch_chunk_by_pos(&chunk_pos)
.expect("Finding chunk failed");
None
}
fn chunk_at(&self, block_pos: &BlockPos) -> Option<&ChunkData> {
let chunk_pos = ChunkPos::from(block_pos);
for chunk in self.chunks.iter() {
if chunk.pos == chunk_pos {
return Some(chunk);
}
}
None
Some(chunk)
}
fn create_chunk_at(&mut self, chunk_pos: &ChunkPos) {
let new_chunk = ChunkData::new(chunk_pos);
self.chunks.push(new_chunk);
self.chunk_storage
.fetch_chunk_by_pos(&chunk_pos)
.expect("Creatinc chunk failed");
}
}
impl StorageServer for SimpleServer {
fn change_block(&mut self, target_state: BlockID, world_position: &BlockPos) {
let mut chunk = self.chunk_at_block_mut(world_position);
let mut chunk = self.chunk_at(world_position);
// Test if there is a chunk that already exists
if chunk.is_none() {
self.create_chunk_at(&ChunkPos::from(world_position));
chunk = self.chunk_at_block_mut(world_position);
chunk = self.chunk_at(world_position);
}
let chunk = chunk.expect("Could not find chunk");
let mut chunk = chunk.expect("Could not find chunk");
// Find the section that the block is located in
let current_section = &mut chunk.sections[world_position.y % 16];
@ -79,16 +63,9 @@ impl StorageServer for SimpleServer {
fn change_block_range(&mut self, target_stage: BlockID, start: &BlockPos, end: &BlockPos) {
unimplemented!()
// self.block_ranges.push(MultipleBlocks {
// id: target_stage,
// range: BlockRange {
// start: start.clone(),
// end: end.clone(),
// },
// })
}
fn read_block_at(&self, pos: &BlockPos) -> BlockID {
fn read_block_at(&mut self, pos: &BlockPos) -> BlockID {
let chunk = self.chunk_at(pos);
if let Some(chunk) = chunk {
@ -97,12 +74,6 @@ impl StorageServer for SimpleServer {
return chunk_section.get_block_at_index(pos).clone();
}
// for blocks in self.block_ranges.iter() {
// if blocks.range.within_range(&pos) {
// return blocks.id.clone();
// }
// }
BlockID::Empty
}
}

90
src/storage/disk_storage.rs
View File

@ -1,19 +1,18 @@
use super::world::{BlockID, ChunkData, ChunkPos};
use super::world::{ChunkData, ChunkPos};
use std::cmp::Ordering;
use std::error::Error;
use std::io::{BufReader, ErrorKind, Write};
use std::{collections::HashMap, fs::File, time::Instant};
const DATABASE_FILE_LOCATION: &str = "./persistence";
struct ChunkFile {}
const CACHED_CHUNK_FILES: usize = 1;
/// `ChunkStorageCache` caches a list of the most recently used file handles
/// where chunks are stored from, and allows for faster accessing of the data
/// from chunks
struct ChunkStorageCache {
#[derive(Debug)]
pub struct ChunkStorageCache {
// `cached_chunk_files` is a vector of cached file handles that are already open
cached_chunk_files: [File; CACHED_CHUNK_FILES],
cached_chunk_files: [Option<File>; CACHED_CHUNK_FILES],
// `cached_file_names` is a list of all the filenames that are contained
// within the cache
cached_file_names: HashMap<String, usize>,
@ -21,8 +20,46 @@ struct ChunkStorageCache {
}
impl ChunkStorageCache {
fn load_chunk_file(&mut self, file_name: &str) -> &File {
let chunk_file = File::open(file_name).expect("Opening file for chunk failed");
pub fn new() -> Self {
ChunkStorageCache {
cached_chunk_files: [None; CACHED_CHUNK_FILES],
cached_file_names: HashMap::new(),
last_used_times: [Instant::now(); CACHED_CHUNK_FILES],
}
}
/// `load_chunk_file` is called whenever a file is missing in the file cache
/// and needs to be loaded from disk
///
/// This replaces a slot for another file in the cache, according to the
/// caching strategy
fn load_chunk_file(&mut self, chunk_pos: &ChunkPos, file_name: &str) -> &File {
let chunk_file = File::options().write(true).read(true).open(file_name);
let chunk_file = match chunk_file {
Ok(file) => file,
Err(err) => match err.kind() {
ErrorKind::NotFound => {
let mut new_chunk_file = File::options()
.write(true)
.read(true)
.create(true)
.open(file_name)
.expect("Opening new chunk file failed");
let blank_chunk = ChunkData::new(chunk_pos);
let encoded_chunk = serde_json::to_string(&blank_chunk).unwrap();
new_chunk_file
.write_all(encoded_chunk.as_bytes())
.expect("Error writing data to chunk");
new_chunk_file
}
err => panic!("Opening new file for chunk failed with: {:?}", err),
},
};
// Add the newly opened file to the cache
@ -45,21 +82,23 @@ impl ChunkStorageCache {
// * Replace the last used time with the curent time
// * Replace the open file with the current one
// Find the name of the previous entry
let (previous_file_name, _) = self
.cached_file_names
.iter()
.find(|(_, &array_index)| array_index == last_used_index)
.expect("The last used index should always have a name");
if !self.cached_file_names.is_empty() {
// Find the name of the previous entry
let (previous_file_name, _) = self
.cached_file_names
.iter()
.find(|(_, &array_index)| array_index == last_used_index)
.expect("The last used index should always have a name");
self.cached_file_names.remove(&previous_file_name.clone());
self.cached_file_names.remove(&previous_file_name.clone());
}
self.cached_file_names
.insert(file_name.to_string(), last_used_index);
// Replace the timestamp with the new timestamp
self.last_used_times[last_used_index] = Instant::now();
self.cached_chunk_files[last_used_index] = chunk_file;
self.cached_chunk_files[last_used_index] = Some(chunk_file);
&self.cached_chunk_files[last_used_index]
self.cached_chunk_files[last_used_index].as_ref().unwrap()
}
/// `fetch_chunk_by_pos` takes in the position of a chunk, and returns the
@ -67,16 +106,23 @@ impl ChunkStorageCache {
///
/// This operation is cached, if possible, so that subsequent accesses to
/// the same chunk are handled by the same file
pub fn fetch_chunk_by_pos(&mut self, pos: &ChunkPos) -> ChunkData {
pub fn fetch_chunk_by_pos(&mut self, pos: &ChunkPos) -> Result<ChunkData, Box<dyn Error>> {
let file_name = pos.storage_file_name();
let file_index = self.cached_file_names.get(file_name.as_str());
let chunk_file = match file_index {
Some(index) => &self.cached_chunk_files[*index],
None => self.load_chunk_file(file_name.as_str()),
Some(index) => self.cached_chunk_files[*index].as_ref().unwrap(),
None => self.load_chunk_file(pos, file_name.as_str()),
};
panic!("Yes");
let file_contents = std::io::read_to_string(chunk_file)?;
let read_data: ChunkData = serde_json::from_str(file_contents.as_str())?;
// let read_data: ChunkData = serde_json::from_reader(&mut file_buffer)?;
// let read_data = ChunkData::new(&ChunkPos { x: 0, z: 0 });
Ok(read_data)
}
}

2
src/storage/mod.rs
View File

@ -1,3 +1,3 @@
mod chunk_compression;
mod disk_storage;
pub mod disk_storage;
pub mod world;

61
src/storage/world.rs
View File

@ -1,17 +1,21 @@
use core::fmt;
use serde::ser;
use serde::ser::{SerializeSeq, SerializeStruct, Serializer};
use serde::Serialize;
use serde::{Deserialize, Serialize};
use std::{
cmp::{max, min},
fmt::Debug,
fs::File,
};
use serde_big_array::BigArray;
const SECTIONS_PER_CHUNK: usize = 16;
const SLICE_SIZE: usize = 16 * 16;
#[derive(Debug, Clone, PartialEq, Serialize)]
const DATABASE_FILE_LOCATION: &str = "./persistence";
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct ChunkPos {
pub x: isize,
pub z: isize,
@ -28,29 +32,30 @@ impl From<&BlockPos> for ChunkPos {
impl ChunkPos {
pub fn storage_file_name(&self) -> String {
format!("{}.{}.chunk", self.x, self.z)
format!("{DATABASE_FILE_LOCATION}/{}.{}.chunk", self.x, self.z)
}
}
#[derive(Debug)]
#[derive(Debug, Serialize, Deserialize)]
pub struct ChunkData {
pub pos: ChunkPos,
#[serde(with = "serde_arrays")]
pub sections: [ChunkSection; SECTIONS_PER_CHUNK],
}
impl Serialize for ChunkData {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
let mut seq = serializer.serialize_seq(Some(self.sections.len()))?;
for section in self.sections {
seq.serialize_element(&section)?;
}
seq.end()
}
}
// impl Serialize for ChunkData {
// fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
// where
// S: serde::Serializer,
// {
// let mut seq = serializer.serialize_seq(Some(self.sections.len()))?;
//
// for section in self.sections {
// seq.serialize_element(&section)?;
// }
// seq.end()
// }
// }
impl ChunkData {
pub fn new(pos: &ChunkPos) -> Self {
@ -74,7 +79,7 @@ impl ChunkData {
}
// https://wiki.vg/Chunk_Format
#[derive(Clone, Copy, Serialize)]
#[derive(Clone, Copy, Serialize, Deserialize, Debug)]
pub struct ChunkSection {
/// The number of non-empty blocks in the section. If completely full, the
/// section contains a 16 x 16 x 16 cube of blocks = 4096 blocks
@ -87,15 +92,15 @@ pub struct ChunkSection {
block_states: [BlockID; 16 * 16 * 16],
}
impl Debug for ChunkSection {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "ChunkSection {{ blocks: {}, states: ", self.block_count)?;
if self.block_count > 0 {
write!(f, "{:?}", self.block_states)?;
}
write!(f, " }}")
}
}
// impl Debug for ChunkSection {
// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
// write!(f, "ChunkSection {{ blocks: {}, states: ", self.block_count)?;
// if self.block_count > 0 {
// write!(f, "{:?}", self.block_states)?;
// }
// write!(f, " }}")
// }
// }
impl ChunkSection {
pub fn new() -> Self {
@ -201,7 +206,7 @@ impl BlockRange {
/// BlockID represents the type of block stored
#[repr(u8)]
#[derive(Debug, Clone, Copy, PartialEq, Serialize)]
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
pub enum BlockID {
Empty,
Generic,

2
src/storage_server.rs
View File

@ -9,7 +9,7 @@ pub trait StorageServer {
/// `read_block_at` returns the id of the block at the location specified
/// If no block is present, the returned id will be of the empty type
fn read_block_at(&self, pos: &BlockPos) -> BlockID;
fn read_block_at(&mut self, pos: &BlockPos) -> BlockID;
}
#[tokio::main]