From 89a6ddc9af0a47c29f420bccf850f9f205e60504 Mon Sep 17 00:00:00 2001 From: Nicholas Novak <34256932+NickyBoy89@users.noreply.github.com> Date: Mon, 13 Nov 2023 16:22:46 -0800 Subject: [PATCH] feat: Added the generate command to load existing world saves --- connector/listen_port.go | 4 +-- go.sum | 9 +++-- main.go | 2 ++ scaling_test.go | 59 ++++++++++++++++++++++++++---- storage/hashserver.go | 47 ++++++++++++++++++++++++ storage/simple_server.go | 4 +++ storage/storage_server.go | 3 ++ world/chunk_data.go | 75 +++++++++++++++++++++++++++++++++++++++ world/data_format.go | 9 ----- 9 files changed, 189 insertions(+), 23 deletions(-) create mode 100644 storage/hashserver.go create mode 100644 world/chunk_data.go diff --git a/connector/listen_port.go b/connector/listen_port.go index d04c711..60045a2 100644 --- a/connector/listen_port.go +++ b/connector/listen_port.go @@ -75,7 +75,7 @@ func handleConn(clientConn net.Conn) { if errors.Is(err, io.EOF) { return } - panic(err) + log.Errorf("Error reading packet from server: %v", err) } // log.Infof("Received packet with id %.2x", p.ID) @@ -103,6 +103,6 @@ func handleConn(clientConn net.Conn) { // Now, we need to copy the network data from the client to the server log.Info("Copying data from the client to the server") if _, err := io.Copy(wrappedServerConn, clientConn); err != nil { - panic(err) + log.Errorf("Error copying data to the server: %v", err) } } diff --git a/go.sum b/go.sum index 21ba377..c93994f 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,7 @@ github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 h1:q2hJAaP1k2 github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -22,8 +23,6 @@ github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= -github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= -github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b h1:1XF24mVaiu7u+CFywTdcDo2ie1pzzhwjt6RHqzpMU34= @@ -32,10 +31,9 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= -github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs= -github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ= github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= @@ -48,12 +46,12 @@ github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRM github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -63,4 +61,5 @@ golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go index 4e47741..0e3e28b 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,7 @@ package main import ( "git.nicholasnovak.io/nnovak/spatial-db/connector" + "git.nicholasnovak.io/nnovak/spatial-db/loading" "git.nicholasnovak.io/nnovak/spatial-db/visualization" "github.com/spf13/cobra" ) @@ -13,6 +14,7 @@ func main() { rootCmd.AddCommand(visualization.VisualizeCommand) rootCmd.AddCommand(connector.ProxyPortCommand) + rootCmd.AddCommand(loading.LoadRegionFileCommand) if err := rootCmd.Execute(); err != nil { panic(err) diff --git a/scaling_test.go b/scaling_test.go index b18a64d..85d5a4d 100644 --- a/scaling_test.go +++ b/scaling_test.go @@ -17,6 +17,7 @@ var ( largeDenseDir = "./lg-dense" ) +// Point densities const ( sparse = 1_000 dense = 10_000 @@ -37,10 +38,11 @@ func init() { } } +// insertPointTemplate inserts a configurable variety of points into the server func insertPointTemplate(testDir string, b *testing.B) { var server storage.SimpleServer - server.StorageDir = testDir + server.SetStorageRoot(testDir) b.ResetTimer() @@ -52,26 +54,69 @@ func insertPointTemplate(testDir string, b *testing.B) { } } -func BenchmarkSmallSparse(b *testing.B) { +func fetchChunkTemplate(testDir string, b *testing.B) { + var server storage.SimpleServer + + server.SetStorageRoot(testDir) + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + pos := world.RandomBlockPosWithRange(2048).ToChunkPos() + if _, err := server.ReadChunkAt(pos); err != nil { + b.Error(err) + } + } +} + +// Insert blocks + +func BenchmarkInsertSmallSparse(b *testing.B) { insertPointTemplate(smallSparseDir, b) } -func BenchmarkMedSparse(b *testing.B) { +func BenchmarkInsertMedSparse(b *testing.B) { insertPointTemplate(medSparseDir, b) } -func BenchmarkLgSparse(b *testing.B) { +func BenchmarkInsertLgSparse(b *testing.B) { insertPointTemplate(largeSparseDir, b) } -func BenchmarkSmallDense(b *testing.B) { +func BenchmarkInsertSmallDense(b *testing.B) { insertPointTemplate(smallDenseDir, b) } -func BenchmarkMedDense(b *testing.B) { +func BenchmarkInsertMedDense(b *testing.B) { insertPointTemplate(medDenseDir, b) } -func BenchmarkLgDense(b *testing.B) { +func BenchmarkInsertLgDense(b *testing.B) { insertPointTemplate(largeDenseDir, b) } + +// Fetch chunks + +func BenchmarkFetchChunkSmallSparse(b *testing.B) { + fetchChunkTemplate(smallSparseDir, b) +} + +func BenchmarkFetchChunkMedSparse(b *testing.B) { + fetchChunkTemplate(medSparseDir, b) +} + +func BenchmarkFetchChunkLgSparse(b *testing.B) { + fetchChunkTemplate(largeSparseDir, b) +} + +func BenchmarkFetchChunkSmallDense(b *testing.B) { + fetchChunkTemplate(smallDenseDir, b) +} + +func BenchmarkFetchChunkMedDense(b *testing.B) { + fetchChunkTemplate(medDenseDir, b) +} + +func BenchmarkFetchChunkLgDense(b *testing.B) { + fetchChunkTemplate(largeDenseDir, b) +} diff --git a/storage/hashserver.go b/storage/hashserver.go new file mode 100644 index 0000000..17134a2 --- /dev/null +++ b/storage/hashserver.go @@ -0,0 +1,47 @@ +package storage + +import "git.nicholasnovak.io/nnovak/spatial-db/world" + +type HashServer struct { + blocks map[world.BlockPos]world.BlockID +} + +func (hs *HashServer) SetStorageRoot(path string) { + hs.blocks = make(map[world.BlockPos]world.BlockID) +} + +func (hs *HashServer) FetchChunk(pos world.ChunkPos) (world.ChunkData, error) { + panic("Unimplemented") +} + +func (hs *HashServer) ChangeBlock( + worldPosition world.BlockPos, + targetState world.BlockID, +) error { + hs.blocks[worldPosition] = targetState + return nil +} + +func (hs *HashServer) ChangeBlockRange( + targetState world.BlockID, + start, end world.BlockPos, +) error { + panic("Unimplemented") +} + +func (hs *HashServer) ReadBlockAt(pos world.BlockPos) (world.BlockID, error) { + panic("Unimplemented") +} + +func (hs *HashServer) ReadChunkAt(pos world.ChunkPos) (world.ChunkData, error) { + var data world.ChunkData + data.Pos = pos + for blockPos, state := range hs.blocks { + if blockPos.ToChunkPos() == pos { + sec := data.SectionFor(blockPos) + sec.UpdateBlock(blockPos, state) + } + } + + return data, nil +} diff --git a/storage/simple_server.go b/storage/simple_server.go index 9301ac3..d28d2d2 100644 --- a/storage/simple_server.go +++ b/storage/simple_server.go @@ -20,6 +20,10 @@ type SimpleServer struct { StorageDir string } +func (s *SimpleServer) SetStorageRoot(path string) { + s.StorageDir = path +} + // Filesystem operations func (s *SimpleServer) FetchOrCreateChunk(pos world.ChunkPos) (world.ChunkData, error) { diff --git a/storage/storage_server.go b/storage/storage_server.go index ce83281..23852c3 100644 --- a/storage/storage_server.go +++ b/storage/storage_server.go @@ -3,6 +3,9 @@ package storage import "git.nicholasnovak.io/nnovak/spatial-db/world" type StorageServer interface { + // Individual operations + SetStorageRoot(path string) + // Individual block-level interactions ChangeBlock(targetState world.BlockID, world_position world.BlockPos) error ChangeBlockRange(targetState world.BlockID, start, end world.BlockPos) error diff --git a/world/chunk_data.go b/world/chunk_data.go new file mode 100644 index 0000000..ba19723 --- /dev/null +++ b/world/chunk_data.go @@ -0,0 +1,75 @@ +package world + +import ( + "github.com/Tnze/go-mc/save" +) + +const ChunkSectionCount = 16 + +type ChunkData struct { + Pos ChunkPos `json:"pos"` + Sections [ChunkSectionCount]ChunkSection `json:"sections"` +} + +func (cd *ChunkData) SectionFor(pos BlockPos) *ChunkSection { + return &cd.Sections[pos.Y%ChunkSectionCount] +} + +func extractPaletteIndexes(compressed int64) [16]byte { + var outputs [16]byte + var outputIndex int + + for index := 0; index < 64; index += 4 { + shifted := compressed >> index + // Mask off the lowest four bits + shifted &= 0xf + outputs[outputIndex] = byte(shifted) + outputIndex += 1 + } + + return outputs +} + +func (cd *ChunkData) FromMCAChunk(other save.Chunk) { + // Load the chunk's position + cd.Pos = ChunkPos{X: int(other.XPos), Z: int(other.ZPos)} + + // Load the data from the chunk + for sectionIndex, section := range other.Sections { + // TODO: Enable chunks to have more than 16 sections + if sectionIndex >= ChunkSectionCount { + break + } + + var currentSection ChunkSection + + paletteIndexes := []int{} + for _, compress := range section.BlockStates.Data { + indexes := extractPaletteIndexes(int64(compress)) + + converted := make([]int, 16) + for i := 0; i < 16; i++ { + converted[i] = int(indexes[i]) + } + + paletteIndexes = append(paletteIndexes, converted...) + } + + for blockIndex, paletteIndex := range paletteIndexes { + var state BlockID + if section.BlockStates.Palette[paletteIndex].Name == "minecraft:air" { + state = Empty + } else { + cd.Sections[sectionIndex].BlockCount += 1 + state = Generic + } + + // TODO: Remove this workaround for larger bit sizes in palettes + if blockIndex < 4096 { + currentSection.BlockStates[blockIndex] = state + } + } + + cd.Sections[sectionIndex] = currentSection + } +} diff --git a/world/data_format.go b/world/data_format.go index ef3fd40..9b4c5e7 100644 --- a/world/data_format.go +++ b/world/data_format.go @@ -31,15 +31,6 @@ func (b BlockPos) ToChunkPos() ChunkPos { } } -type ChunkData struct { - Pos ChunkPos `json:"pos"` - Sections [16]ChunkSection `json:"sections"` -} - -func (cd *ChunkData) SectionFor(pos BlockPos) *ChunkSection { - return &cd.Sections[pos.Y%16] -} - type ChunkPos struct { X int `json:"x"` Z int `json:"z"`