spatial-db/storage/file_operations.go

package storage

import (
	"encoding/json"
	"io/fs"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"git.nicholasnovak.io/nnovak/spatial-db/world"
)

func ReadChunkFromFile(chunkFile *os.File) (world.ChunkData, error) {
	var chunkData world.ChunkData

	if err := json.NewDecoder(chunkFile).Decode(&chunkData); err != nil {
		return chunkData, err
	}

	return chunkData, nil
}

func ReadParallelFromDirectory(dirName string) ([]world.ChunkData, error) {
	chunkFiles, err := os.ReadDir(dirName)
	if err != nil {
		panic(err)
	}

	// Filter invalid chunks

	validChunkFiles := []fs.DirEntry{}
	for _, chunkFile := range chunkFiles {
		if chunkFile.IsDir() || !strings.HasSuffix(chunkFile.Name(), ".chunk") {
			continue
		}
		validChunkFiles = append(validChunkFiles, chunkFile)
	}

	chunks := make([]world.ChunkData, len(validChunkFiles))

	var wg sync.WaitGroup
	wg.Add(len(validChunkFiles))

	for fileIndex, chunkFile := range validChunkFiles {
		// Avoid implicit copies
		chunkFile := chunkFile
		fileIndex := fileIndex

		go func() {
			defer wg.Done()
			file, err := os.Open(filepath.Join(dirName, chunkFile.Name()))
			if err != nil {
				panic(err)
			}

			chunkData, err := ReadChunkFromFile(file)
			if err != nil {
				panic(err)
			}

			file.Close()

			chunks[fileIndex] = chunkData
		}()
	}

	wg.Wait()

	return chunks, nil
}