feat: Moved over all of the servers to use unity files

Makefile

@@ -5,15 +5,12 @@ SAVES += witchcraft-save
 all: $(SAVES)
 
 imperial-save: compile
-	mkdir imperial-save
 	./spatial-db load worldsave "saves/Imperialcity v14.1/region" --output "imperial-save"
 
 skygrid-save: compile
-	mkdir skygrid-save
 	./spatial-db load worldsave "saves/SkyGrid/region" --output "skygrid-save"
 
 witchcraft-save: compile
-	mkdir witchcraft-save
 	./spatial-db load worldsave "saves/Witchcraft/region" --output "witchcraft-save"
 
 .PHONY: compile

loading/load_save.go

@@ -1,11 +1,11 @@
 package loading
 
 import (
-	"encoding/json"
 	"os"
 	"path/filepath"
 	"strings"
 
+	"git.nicholasnovak.io/nnovak/spatial-db/storage"
 	log "github.com/sirupsen/logrus"
 
 	"github.com/spf13/cobra"
@@ -32,6 +32,13 @@ var LoadSaveDirCommand = &cobra.Command{
 
 		log.Infof("Loading save directory of %s", args[0])
 
+		u, err := storage.CreateUnityFile(saveOutputDir)
+		if err != nil {
+			return err
+		}
+		defer u.Close()
+		defer u.WriteMetadataFile(saveOutputDir + ".metadata")
+
 		for regionIndex, regionFile := range regionFiles {
 			if regionFile.IsDir() {
 				continue
@@ -58,22 +65,9 @@ var LoadSaveDirCommand = &cobra.Command{
 
 			// Save each chunk to a separate file
 			for _, chunk := range chunks {
-				chunkFilename := chunk.Pos.ToFileName()
+				if err := u.WriteChunk(chunk); err != nil {
-
-				outfile, err := os.OpenFile(
-					filepath.Join(saveOutputDir, chunkFilename),
-					os.O_WRONLY|os.O_CREATE|os.O_APPEND,
-					0664,
-				)
-				if err != nil {
 					return err
 				}
-
-				if err := json.NewEncoder(outfile).Encode(chunk); err != nil {
-					return err
-				}
-
-				outfile.Close()
 			}
 		}
 

server/hashserver.go

@@ -13,7 +13,13 @@ type HashServer struct {
 func (hs *HashServer) SetStorageRoot(path string) {
 	hs.blocks = make(map[world.BlockPos]world.BlockID)
 
-	chunks, err := storage.ReadParallelFromDirectory(path)
+	u, err := storage.OpenUnityFile(path, path+".metadata")
+	if err != nil {
+		panic(err)
+	}
+	defer u.Close()
+
+	chunks, err := u.ReadAllChunks()
 	if err != nil {
 		panic(err)
 	}

server/inmemory_server.go

@@ -18,7 +18,13 @@ type InMemoryServer struct {
 func (s *InMemoryServer) SetStorageRoot(path string) {
 	s.StorageDir = path
 
-	chunks, err := storage.ReadParallelFromDirectory(s.StorageDir)
+	u, err := storage.OpenUnityFile(s.StorageDir, s.StorageDir+".metadata")
+	if err != nil {
+		panic(err)
+	}
+	defer u.Close()
+
+	chunks, err := u.ReadAllChunks()
 	if err != nil {
 		panic(err)
 	}

server/simple_server.go

@@ -15,12 +15,17 @@ const fileCacheSize = 8
 
 type SimpleServer struct {
 	StorageDir     string
-	cache      storage.FileCache
+	storageBackend storage.UnityFile
 }
 
 func (s *SimpleServer) SetStorageRoot(path string) {
 	s.StorageDir = path
-	s.cache = storage.NewFileCache(256)
+
+	var err error
+	s.storageBackend, err = storage.OpenUnityFile(path, path+".metadata")
+	if err != nil {
+		panic(err)
+	}
 }
 
 // Filesystem operations
@@ -58,11 +63,7 @@ func (s *SimpleServer) FetchOrCreateChunk(pos world.ChunkPos) (world.ChunkData,
 
 // `FetchChunk' fetches the chunk's data, given the chunk's position
 func (s *SimpleServer) FetchChunk(pos world.ChunkPos) (world.ChunkData, error) {
-	chunkFileName := filepath.Join(s.StorageDir, pos.ToFileName())
+	chunkData, err := s.storageBackend.ReadChunk(pos)
-
-	var chunkData world.ChunkData
-
-	chunkFile, err := s.cache.FetchFile(chunkFileName)
 	if err != nil {
 		if errors.Is(err, fs.ErrNotExist) {
 			return chunkData, storage.ChunkNotFoundError
@@ -71,7 +72,7 @@ func (s *SimpleServer) FetchChunk(pos world.ChunkPos) (world.ChunkData, error) {
 		}
 	}
 
-	return storage.ReadChunkFromFile(chunkFile)
+	return chunkData, nil
 }
 
 // Voxel server implementation

storage/unity_file.go

@@ -21,8 +21,8 @@ type UnityFile struct {
 }
 
 type fileMetadata struct {
-	startOffset int
+	StartOffset int `json:"start_offset"`
-	fileSize    int
+	FileSize    int `json:"file_size"`
 }
 
 func CreateUnityFile(fileName string) (UnityFile, error) {
@@ -39,6 +39,23 @@ func CreateUnityFile(fileName string) (UnityFile, error) {
 	return u, nil
 }
 
+func OpenUnityFile(fileName, metadataName string) (UnityFile, error) {
+	var u UnityFile
+
+	// Read the file
+	f, err := os.Open(fileName)
+	if err != nil {
+		return u, err
+	}
+	u.fd = f
+
+	if err := u.ReadMetadataFile(metadataName); err != nil {
+		return u, err
+	}
+
+	return u, nil
+}
+
 func (u UnityFile) Size() int {
 	return u.fileSize
 }
@@ -54,7 +71,7 @@ func (u *UnityFile) WriteChunk(data world.ChunkData) error {
 	encodedSize := encoded.Len()
 
 	// Go to the end of the file
-	u.fd.Seek(0, u.fileSize)
+	u.fd.Seek(int64(u.fileSize), io.SeekStart)
 	// Write the encoded contents to the file
 	if _, err := u.fd.Write(encoded.Bytes()); err != nil {
 		return err
@@ -62,8 +79,8 @@ func (u *UnityFile) WriteChunk(data world.ChunkData) error {
 
 	// Update the metadata with the new file
 	u.metadata[data.Pos] = fileMetadata{
-		startOffset: u.fileSize,
+		StartOffset: u.fileSize,
-		fileSize:    encodedSize,
+		FileSize:    encodedSize,
 	}
 	u.fileSize += encodedSize
 
@@ -100,9 +117,9 @@ func (u *UnityFile) ReadMetadataFile(fileName string) error {
 func (u UnityFile) ReadChunk(pos world.ChunkPos) (world.ChunkData, error) {
 	m := u.metadata[pos]
 
-	u.fd.Seek(0, m.startOffset)
+	u.fd.Seek(int64(m.StartOffset), io.SeekStart)
 
-	fileReader := io.LimitReader(u.fd, int64(m.fileSize))
+	fileReader := io.LimitReader(u.fd, int64(m.FileSize))
 
 	var data world.ChunkData
 	if err := json.NewDecoder(fileReader).Decode(&data); err != nil {
@@ -111,3 +128,21 @@ func (u UnityFile) ReadChunk(pos world.ChunkPos) (world.ChunkData, error) {
 
 	return data, nil
 }
+
+func (u UnityFile) ReadAllChunks() ([]world.ChunkData, error) {
+	chunks := []world.ChunkData{}
+
+	for pos := range u.metadata {
+		chunk, err := u.ReadChunk(pos)
+		if err != nil {
+			return nil, err
+		}
+		chunks = append(chunks, chunk)
+	}
+
+	return chunks, nil
+}
+
+func (u *UnityFile) Close() error {
+	return u.fd.Close()
+}

storage/unity_file_test.go

@@ -1,6 +1,7 @@
 package storage
 
 import (
+	"fmt"
 	"os"
 	"path"
 	"reflect"
@@ -69,6 +70,7 @@ func TestWriteMultipleFiles(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Error creating temporary directory: %v", err)
 	}
+	fmt.Println(tempDir)
 	defer os.RemoveAll(tempDir)
 
 	u, err := CreateUnityFile(path.Join(tempDir, "test-unity"))
@@ -119,3 +121,60 @@ func TestWriteMultipleFiles(t *testing.T) {
 		}
 	}
 }
+
+func TestReadAllChunks(t *testing.T) {
+	tempDir, err := os.MkdirTemp("", "unity")
+	if err != nil {
+		t.Fatalf("Error creating temporary directory: %v", err)
+	}
+	fmt.Println(tempDir)
+	defer os.RemoveAll(tempDir)
+
+	u, err := CreateUnityFile(path.Join(tempDir, "test-unity"))
+	if err != nil {
+		t.Fatalf("Error creating unity file: %v", err)
+	}
+
+	var (
+		chunk1 world.ChunkData
+		chunk2 world.ChunkData
+		chunk3 world.ChunkData
+	)
+	chunk1.Pos = world.ChunkPos{
+		X: 0,
+		Z: 0,
+	}
+	chunk1.Sections[0].BlockStates[0] = 2
+	chunk2.Sections[0].BlockStates[0] = 3
+	chunk2.Pos = world.ChunkPos{
+		X: 1,
+		Z: 0,
+	}
+	chunk3.Sections[0].BlockStates[0] = 4
+	chunk3.Pos = world.ChunkPos{
+		X: 2,
+		Z: 0,
+	}
+
+	chunks := []world.ChunkData{chunk1, chunk2, chunk3}
+
+	// Write all chunks
+	for _, data := range chunks {
+		if err := u.WriteChunk(data); err != nil {
+			t.Fatalf("Error writing chunk: %v", err)
+		}
+	}
+
+	t.Log(chunks)
+
+	readChunks, err := u.ReadAllChunks()
+	if err != nil {
+		t.Fatalf("Error reading chunks: %v", err)
+	}
+
+	for index, chunk := range readChunks {
+		if !reflect.DeepEqual(chunk, chunks[index]) {
+			t.Fatalf("Chunks were not equal")
+		}
+	}
+}