From 76038020f0f0bddda1c67599f37663742e3e6eae Mon Sep 17 00:00:00 2001 From: Nicholas Novak <34256932+NickyBoy89@users.noreply.github.com> Date: Sun, 10 Dec 2023 21:46:15 -0800 Subject: [PATCH] feat: Refactored all of the scaling tests and separated them out --- basic_functionality_test.go | 4 +-- hash_read_test.go | 39 ++++++++++++++++++++++++++ inmemory_read_test.go | 39 ++++++++++++++++++++++++++ ondisk_read_test.go | 39 ++++++++++++++++++++++++++ scaling_test.go | 56 ------------------------------------- server/hashserver.go | 5 +++- templates_test.go | 29 +++++++++++++++++++ 7 files changed, 152 insertions(+), 59 deletions(-) create mode 100644 hash_read_test.go create mode 100644 inmemory_read_test.go create mode 100644 ondisk_read_test.go delete mode 100644 scaling_test.go create mode 100644 templates_test.go diff --git a/basic_functionality_test.go b/basic_functionality_test.go index 42f0a3d..05fb112 100644 --- a/basic_functionality_test.go +++ b/basic_functionality_test.go @@ -9,7 +9,7 @@ import ( log "github.com/sirupsen/logrus" - "git.nicholasnovak.io/nnovak/spatial-db/storage" + "git.nicholasnovak.io/nnovak/spatial-db/server" "git.nicholasnovak.io/nnovak/spatial-db/world" ) @@ -36,7 +36,7 @@ func populateStorageDir( return } - var server storage.SimpleServer + var server server.SimpleServer server.StorageDir = dirName if cleanup { diff --git a/hash_read_test.go b/hash_read_test.go new file mode 100644 index 0000000..ab984bc --- /dev/null +++ b/hash_read_test.go @@ -0,0 +1,39 @@ +package main + +import ( + "fmt" + "testing" + + "git.nicholasnovak.io/nnovak/spatial-db/server" +) + +var hash server.HashServer +var hashInit bool + +func initHash() { + if !hashInit { + fmt.Println("Initializing hash server") + hash.SetStorageRoot("skygrid-save") + hashInit = true + } +} + +func BenchmarkReadWithin128Hashtable(b *testing.B) { + initHash() + readBlockTemplate(&hash, b, 128) +} + +func BenchmarkReadWithin512Hashtable(b *testing.B) { + initHash() + readBlockTemplate(&hash, b, 512) +} + +func BenchmarkReadWithin2048Hashtable(b *testing.B) { + initHash() + readBlockTemplate(&hash, b, 2048) +} + +func BenchmarkReadWithin65536Hashtable(b *testing.B) { + initHash() + readBlockTemplate(&hash, b, 65536) +} diff --git a/inmemory_read_test.go b/inmemory_read_test.go new file mode 100644 index 0000000..ff271c6 --- /dev/null +++ b/inmemory_read_test.go @@ -0,0 +1,39 @@ +package main + +import ( + "fmt" + "testing" + + "git.nicholasnovak.io/nnovak/spatial-db/server" +) + +var inmemory server.InMemoryServer +var inmemoryInit bool + +func initInMemory() { + if !inmemoryInit { + fmt.Println("Initializing in-memory server") + inmemory.SetStorageRoot("skygrid-save") + inmemoryInit = true + } +} + +func BenchmarkReadWithin128InMemory(b *testing.B) { + initInMemory() + readBlockTemplate(&inmemory, b, 128) +} + +func BenchmarkReadWithin512InMemory(b *testing.B) { + initInMemory() + readBlockTemplate(&inmemory, b, 512) +} + +func BenchmarkReadWithin2048InMemory(b *testing.B) { + initInMemory() + readBlockTemplate(&inmemory, b, 2048) +} + +func BenchmarkReadWithin65536InMemory(b *testing.B) { + initInMemory() + readBlockTemplate(&inmemory, b, 65536) +} diff --git a/ondisk_read_test.go b/ondisk_read_test.go new file mode 100644 index 0000000..9978fe1 --- /dev/null +++ b/ondisk_read_test.go @@ -0,0 +1,39 @@ +package main + +import ( + "fmt" + "testing" + + "git.nicholasnovak.io/nnovak/spatial-db/server" +) + +var disk server.SimpleServer +var diskInit bool + +func initDisk() { + if !diskInit { + fmt.Println("Initializing disk server") + disk.SetStorageRoot("skygrid-save") + diskInit = true + } +} + +func BenchmarkReadWithin128OnDisk(b *testing.B) { + initDisk() + readBlockTemplate(&disk, b, 128) +} + +func BenchmarkReadWithin512OnDisk(b *testing.B) { + initDisk() + readBlockTemplate(&disk, b, 512) +} + +func BenchmarkReadWithin2048OnDisk(b *testing.B) { + initDisk() + readBlockTemplate(&disk, b, 2048) +} + +func BenchmarkReadWithin65536OnDisk(b *testing.B) { + initDisk() + readBlockTemplate(&disk, b, 65536) +} diff --git a/scaling_test.go b/scaling_test.go deleted file mode 100644 index 5100b51..0000000 --- a/scaling_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package main - -import ( - "errors" - "io" - "testing" - - "git.nicholasnovak.io/nnovak/spatial-db/storage" - "git.nicholasnovak.io/nnovak/spatial-db/world" -) - -var server storage.InMemoryServer - -func init() { - server.SetStorageRoot("skygrid-save") -} - -func readBlockTemplate(rootDir string, b *testing.B, pointSpread int) { - b.ResetTimer() - - for i := 0; i < b.N; i++ { - pos := world.RandomBlockPosWithRange(float64(pointSpread)) - if _, err := server.ReadBlockAt(pos); err != nil { - if errors.Is(err, storage.ChunkNotFoundError) || errors.Is(err, io.EOF) { - continue - } else { - b.Error(err) - } - } - } -} - -func fetchChunkTemplate(testDir string, b *testing.B) { - b.ResetTimer() - - for i := 0; i < b.N; i++ { - pos := world.RandomBlockPosWithRange(2048).ToChunkPos() - if _, err := server.ReadChunkAt(pos); err != nil { - b.Error(err) - } - } -} - -// Insert blocks - -func BenchmarkReadClusteredPoints(b *testing.B) { - readBlockTemplate("skygrid-test", b, 128) -} - -func BenchmarkReadSparserPoints(b *testing.B) { - readBlockTemplate("skygrid-test", b, 2048) -} - -func BenchmarkReadSparserPoints1(b *testing.B) { - readBlockTemplate("skygrid-test", b, 65536) -} diff --git a/server/hashserver.go b/server/hashserver.go index 1073658..dafdedc 100644 --- a/server/hashserver.go +++ b/server/hashserver.go @@ -3,6 +3,7 @@ package server import ( "git.nicholasnovak.io/nnovak/spatial-db/storage" "git.nicholasnovak.io/nnovak/spatial-db/world" + log "github.com/sirupsen/logrus" ) type HashServer struct { @@ -17,8 +18,10 @@ func (hs *HashServer) SetStorageRoot(path string) { panic(err) } - for _, data := range chunks { + for chunkIndex, data := range chunks { // Load in each data point from disk + log.Infof("Reading in chunk %d of %d", chunkIndex, len(chunks)) + for _, section := range data.Sections { for blockIndex, blockState := range section.BlockStates { pos := data.IndexToBlockPos(blockIndex) diff --git a/templates_test.go b/templates_test.go new file mode 100644 index 0000000..609b6d9 --- /dev/null +++ b/templates_test.go @@ -0,0 +1,29 @@ +package main + +import ( + "errors" + "io" + "testing" + + "git.nicholasnovak.io/nnovak/spatial-db/storage" + "git.nicholasnovak.io/nnovak/spatial-db/world" +) + +func readBlockTemplate( + storageServer storage.StorageServer, + b *testing.B, + pointSpread int, +) { + b.ResetTimer() + + for i := 0; i < b.N; i++ { + pos := world.RandomBlockPosWithRange(float64(pointSpread)) + if _, err := storageServer.ReadBlockAt(pos); err != nil { + if errors.Is(err, storage.ChunkNotFoundError) || errors.Is(err, io.EOF) { + continue + } else { + b.Error(err) + } + } + } +}