feat: Added the generate command to load existing world saves

connector/listen_port.go

@@ -75,7 +75,7 @@ func handleConn(clientConn net.Conn) {
 				if errors.Is(err, io.EOF) {
 					return
 				}
-				panic(err)
+				log.Errorf("Error reading packet from server: %v", err)
 			}
 
 			// log.Infof("Received packet with id %.2x", p.ID)
@@ -103,6 +103,6 @@ func handleConn(clientConn net.Conn) {
 	// Now, we need to copy the network data from the client to the server
 	log.Info("Copying data from the client to the server")
 	if _, err := io.Copy(wrappedServerConn, clientConn); err != nil {
-		panic(err)
+		log.Errorf("Error copying data to the server: %v", err)
 	}
 }

go.sum

@@ -10,6 +10,7 @@ github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 h1:q2hJAaP1k2
 github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -22,8 +23,6 @@ github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D
 github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
 github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
 github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
-github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
-github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
 github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b h1:1XF24mVaiu7u+CFywTdcDo2ie1pzzhwjt6RHqzpMU34=
@@ -32,10 +31,9 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU
 github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
 github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=
 github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8=
-github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs=
-github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ=
 github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo=
 github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
@@ -48,12 +46,12 @@ github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRM
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -63,4 +61,5 @@ golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"git.nicholasnovak.io/nnovak/spatial-db/connector"
+	"git.nicholasnovak.io/nnovak/spatial-db/loading"
 	"git.nicholasnovak.io/nnovak/spatial-db/visualization"
 	"github.com/spf13/cobra"
 )
@@ -13,6 +14,7 @@ func main() {
 
 	rootCmd.AddCommand(visualization.VisualizeCommand)
 	rootCmd.AddCommand(connector.ProxyPortCommand)
+	rootCmd.AddCommand(loading.LoadRegionFileCommand)
 
 	if err := rootCmd.Execute(); err != nil {
 		panic(err)

scaling_test.go

@@ -17,6 +17,7 @@ var (
 	largeDenseDir  = "./lg-dense"
 )
 
+// Point densities
 const (
 	sparse = 1_000
 	dense  = 10_000
@@ -37,10 +38,11 @@ func init() {
 	}
 }
 
+// insertPointTemplate inserts a configurable variety of points into the server
 func insertPointTemplate(testDir string, b *testing.B) {
 	var server storage.SimpleServer
 
-	server.StorageDir = testDir
+	server.SetStorageRoot(testDir)
 
 	b.ResetTimer()
 
@@ -52,26 +54,69 @@ func insertPointTemplate(testDir string, b *testing.B) {
 	}
 }
 
-func BenchmarkSmallSparse(b *testing.B) {
+func fetchChunkTemplate(testDir string, b *testing.B) {
+	var server storage.SimpleServer
+
+	server.SetStorageRoot(testDir)
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		pos := world.RandomBlockPosWithRange(2048).ToChunkPos()
+		if _, err := server.ReadChunkAt(pos); err != nil {
+			b.Error(err)
+		}
+	}
+}
+
+// Insert blocks
+
+func BenchmarkInsertSmallSparse(b *testing.B) {
 	insertPointTemplate(smallSparseDir, b)
 }
 
-func BenchmarkMedSparse(b *testing.B) {
+func BenchmarkInsertMedSparse(b *testing.B) {
 	insertPointTemplate(medSparseDir, b)
 }
 
-func BenchmarkLgSparse(b *testing.B) {
+func BenchmarkInsertLgSparse(b *testing.B) {
 	insertPointTemplate(largeSparseDir, b)
 }
 
-func BenchmarkSmallDense(b *testing.B) {
+func BenchmarkInsertSmallDense(b *testing.B) {
 	insertPointTemplate(smallDenseDir, b)
 }
 
-func BenchmarkMedDense(b *testing.B) {
+func BenchmarkInsertMedDense(b *testing.B) {
 	insertPointTemplate(medDenseDir, b)
 }
 
-func BenchmarkLgDense(b *testing.B) {
+func BenchmarkInsertLgDense(b *testing.B) {
 	insertPointTemplate(largeDenseDir, b)
 }
+
+// Fetch chunks
+
+func BenchmarkFetchChunkSmallSparse(b *testing.B) {
+	fetchChunkTemplate(smallSparseDir, b)
+}
+
+func BenchmarkFetchChunkMedSparse(b *testing.B) {
+	fetchChunkTemplate(medSparseDir, b)
+}
+
+func BenchmarkFetchChunkLgSparse(b *testing.B) {
+	fetchChunkTemplate(largeSparseDir, b)
+}
+
+func BenchmarkFetchChunkSmallDense(b *testing.B) {
+	fetchChunkTemplate(smallDenseDir, b)
+}
+
+func BenchmarkFetchChunkMedDense(b *testing.B) {
+	fetchChunkTemplate(medDenseDir, b)
+}
+
+func BenchmarkFetchChunkLgDense(b *testing.B) {
+	fetchChunkTemplate(largeDenseDir, b)
+}

storage/hashserver.go

@@ -0,0 +1,47 @@
+package storage
+
+import "git.nicholasnovak.io/nnovak/spatial-db/world"
+
+type HashServer struct {
+	blocks map[world.BlockPos]world.BlockID
+}
+
+func (hs *HashServer) SetStorageRoot(path string) {
+	hs.blocks = make(map[world.BlockPos]world.BlockID)
+}
+
+func (hs *HashServer) FetchChunk(pos world.ChunkPos) (world.ChunkData, error) {
+	panic("Unimplemented")
+}
+
+func (hs *HashServer) ChangeBlock(
+	worldPosition world.BlockPos,
+	targetState world.BlockID,
+) error {
+	hs.blocks[worldPosition] = targetState
+	return nil
+}
+
+func (hs *HashServer) ChangeBlockRange(
+	targetState world.BlockID,
+	start, end world.BlockPos,
+) error {
+	panic("Unimplemented")
+}
+
+func (hs *HashServer) ReadBlockAt(pos world.BlockPos) (world.BlockID, error) {
+	panic("Unimplemented")
+}
+
+func (hs *HashServer) ReadChunkAt(pos world.ChunkPos) (world.ChunkData, error) {
+	var data world.ChunkData
+	data.Pos = pos
+	for blockPos, state := range hs.blocks {
+		if blockPos.ToChunkPos() == pos {
+			sec := data.SectionFor(blockPos)
+			sec.UpdateBlock(blockPos, state)
+		}
+	}
+
+	return data, nil
+}

storage/simple_server.go

@@ -20,6 +20,10 @@ type SimpleServer struct {
 	StorageDir string
 }
 
+func (s *SimpleServer) SetStorageRoot(path string) {
+	s.StorageDir = path
+}
+
 // Filesystem operations
 
 func (s *SimpleServer) FetchOrCreateChunk(pos world.ChunkPos) (world.ChunkData, error) {

storage/storage_server.go

@@ -3,6 +3,9 @@ package storage
 import "git.nicholasnovak.io/nnovak/spatial-db/world"
 
 type StorageServer interface {
+	// Individual operations
+	SetStorageRoot(path string)
+
 	// Individual block-level interactions
 	ChangeBlock(targetState world.BlockID, world_position world.BlockPos) error
 	ChangeBlockRange(targetState world.BlockID, start, end world.BlockPos) error

world/chunk_data.go

@@ -0,0 +1,75 @@
+package world
+
+import (
+	"github.com/Tnze/go-mc/save"
+)
+
+const ChunkSectionCount = 16
+
+type ChunkData struct {
+	Pos      ChunkPos                        `json:"pos"`
+	Sections [ChunkSectionCount]ChunkSection `json:"sections"`
+}
+
+func (cd *ChunkData) SectionFor(pos BlockPos) *ChunkSection {
+	return &cd.Sections[pos.Y%ChunkSectionCount]
+}
+
+func extractPaletteIndexes(compressed int64) [16]byte {
+	var outputs [16]byte
+	var outputIndex int
+
+	for index := 0; index < 64; index += 4 {
+		shifted := compressed >> index
+		// Mask off the lowest four bits
+		shifted &= 0xf
+		outputs[outputIndex] = byte(shifted)
+		outputIndex += 1
+	}
+
+	return outputs
+}
+
+func (cd *ChunkData) FromMCAChunk(other save.Chunk) {
+	// Load the chunk's position
+	cd.Pos = ChunkPos{X: int(other.XPos), Z: int(other.ZPos)}
+
+	// Load the data from the chunk
+	for sectionIndex, section := range other.Sections {
+		// TODO: Enable chunks to have more than 16 sections
+		if sectionIndex >= ChunkSectionCount {
+			break
+		}
+
+		var currentSection ChunkSection
+
+		paletteIndexes := []int{}
+		for _, compress := range section.BlockStates.Data {
+			indexes := extractPaletteIndexes(int64(compress))
+
+			converted := make([]int, 16)
+			for i := 0; i < 16; i++ {
+				converted[i] = int(indexes[i])
+			}
+
+			paletteIndexes = append(paletteIndexes, converted...)
+		}
+
+		for blockIndex, paletteIndex := range paletteIndexes {
+			var state BlockID
+			if section.BlockStates.Palette[paletteIndex].Name == "minecraft:air" {
+				state = Empty
+			} else {
+				cd.Sections[sectionIndex].BlockCount += 1
+				state = Generic
+			}
+
+			// TODO: Remove this workaround for larger bit sizes in palettes
+			if blockIndex < 4096 {
+				currentSection.BlockStates[blockIndex] = state
+			}
+		}
+
+		cd.Sections[sectionIndex] = currentSection
+	}
+}

world/data_format.go

@@ -31,15 +31,6 @@ func (b BlockPos) ToChunkPos() ChunkPos {
 	}
 }
 
-type ChunkData struct {
-	Pos      ChunkPos         `json:"pos"`
-	Sections [16]ChunkSection `json:"sections"`
-}
-
-func (cd *ChunkData) SectionFor(pos BlockPos) *ChunkSection {
-	return &cd.Sections[pos.Y%16]
-}
-
 type ChunkPos struct {
 	X int `json:"x"`
 	Z int `json:"z"`