feat: ChunkStreamer load path with target-set, dispatch, and diff application

VoxelEngine.Tests/Streaming/ChunkStreamerTests.cs

@@ -0,0 +1,173 @@
+using OpenTK.Mathematics;
+using VoxelEngine.Core;
+using VoxelEngine.World;
+using VoxelEngine.World.Blocks;
+using VoxelEngine.World.Chunk;
+using VoxelEngine.World.Persistence;
+using VoxelEngine.World.Streaming;
+using Xunit;
+using VoxelChunk = VoxelEngine.World.Chunk.Chunk;
+
+namespace VoxelEngine.Tests.Streaming;
+
+// Synchronous executor: runs the submitted action on the calling thread.
+internal sealed class SynchronousJobExecutor : IJobExecutor
+{
+    public void Submit(Action work) => work();
+}
+
+// Fake renderer: never touches GL. Just records lifecycle.
+internal sealed class FakeRenderer : IChunkRenderer
+{
+    public bool Disposed { get; private set; }
+    public int DrawCount { get; private set; }
+    public float[] Mesh { get; }
+
+    public FakeRenderer(float[] mesh) { this.Mesh = mesh; }
+    public void Draw() => this.DrawCount++;
+    public void Dispose() => this.Disposed = true;
+}
+
+// Minimal mesher returning a fixed-length non-empty mesh.
+internal sealed class FakeMesher : IChunkMesher
+{
+    public float[] Build(VoxelChunk chunk, BlockRegistry registry, Func<int, int, int, bool>? neighborOpaqueOutside = null)
+        => new float[6]; // one vertex worth
+}
+
+public class ChunkStreamerTests
+{
+    private static (ChunkStreamer streamer, World.World world, List<FakeRenderer> created) CreateStreamer(Configuration cfg)
+    {
+        var registry = new BlockRegistry();
+        var store = new InMemoryChunkModificationStore();
+        var world = new World.World(registry, store);
+        var mesher = new FakeMesher();
+        var executor = new SynchronousJobExecutor();
+        var created = new List<FakeRenderer>();
+        Func<float[], IChunkRenderer> factory = m =>
+        {
+            var r = new FakeRenderer(m);
+            created.Add(r);
+            return r;
+        };
+        Func<Vector3i, VoxelChunk> generate = _ => new VoxelChunk();
+        var streamer = new ChunkStreamer(cfg, world, generate, mesher, registry, executor, factory);
+        return (streamer, world, created);
+    }
+
+    [Fact]
+    public void Tick_FirstCall_LoadsAllChunksInLoadBox()
+    {
+        var cfg = new Configuration
+        {
+            RenderDistanceHorizontal = 1,
+            RenderDistanceVertical = 0,
+            MaxGenerateJobsInFlight = 100,
+            MaxMeshUploadsPerFrame = 100,
+        };
+        var (streamer, world, created) = CreateStreamer(cfg);
+
+        streamer.Tick(new Vector3i(0, 0, 0));
+
+        // Load box H=1, V=0 → 3 × 1 × 3 = 9 chunks
+        Assert.Equal(9, created.Count);
+        for (int x = -1; x <= 1; x++)
+        for (int z = -1; z <= 1; z++)
+        {
+            var pos = new Vector3i(x, 0, z);
+            Assert.NotNull(world.GetChunk(pos));
+            Assert.True(streamer.IsReady(pos));
+        }
+    }
+
+    [Fact]
+    public void Tick_PlayerMoves_LoadsNewChunksOnly()
+    {
+        var cfg = new Configuration
+        {
+            RenderDistanceHorizontal = 1,
+            RenderDistanceVertical = 0,
+            MaxGenerateJobsInFlight = 100,
+            MaxMeshUploadsPerFrame = 100,
+        };
+        var (streamer, world, created) = CreateStreamer(cfg);
+
+        streamer.Tick(new Vector3i(0, 0, 0));     // loads 9
+        int firstWave = created.Count;
+        streamer.Tick(new Vector3i(1, 0, 0));     // loads only the 3 new column to +x
+
+        Assert.Equal(firstWave + 3, created.Count);
+    }
+
+    [Fact]
+    public void Tick_RespectsMaxMeshUploadsPerFrame()
+    {
+        var cfg = new Configuration
+        {
+            RenderDistanceHorizontal = 1,
+            RenderDistanceVertical = 0,
+            MaxGenerateJobsInFlight = 100,
+            MaxMeshUploadsPerFrame = 2,   // cap drains per tick
+        };
+        var (streamer, _, created) = CreateStreamer(cfg);
+
+        // With sync executor every submitted job completes immediately and
+        // posts into _completed. The cap throttles how many drain per Tick().
+        streamer.Tick(new Vector3i(0, 0, 0));   // dispatch fills the queue; drain ≤ 2
+        Assert.Equal(2, created.Count);
+
+        streamer.Tick(new Vector3i(0, 0, 0));   // same player chunk, drains 2 more
+        Assert.Equal(4, created.Count);
+    }
+
+    [Fact]
+    public void Tick_ClampsYToWorldBounds()
+    {
+        var cfg = new Configuration
+        {
+            RenderDistanceHorizontal = 0,
+            RenderDistanceVertical = 2,
+            MaxGenerateJobsInFlight = 100,
+            MaxMeshUploadsPerFrame = 100,
+        };
+        var (streamer, world, created) = CreateStreamer(cfg);
+
+        // Player at y=0 (chunk Y = 0). V=2 wants Y ∈ [-2, 2], but valid Y is [0, 8).
+        // So only Y ∈ {0, 1, 2} qualifies → 1 × 3 × 1 = 3 chunks.
+        streamer.Tick(new Vector3i(0, 0, 0));
+        Assert.Equal(3, created.Count);
+        Assert.NotNull(world.GetChunk(new Vector3i(0, 0, 0)));
+        Assert.NotNull(world.GetChunk(new Vector3i(0, 1, 0)));
+        Assert.NotNull(world.GetChunk(new Vector3i(0, 2, 0)));
+    }
+
+    [Fact]
+    public void Tick_AppliesModificationDiffOnLoad()
+    {
+        var cfg = new Configuration
+        {
+            RenderDistanceHorizontal = 0,
+            RenderDistanceVertical = 0,
+            MaxGenerateJobsInFlight = 100,
+            MaxMeshUploadsPerFrame = 100,
+        };
+        var registry = new BlockRegistry();
+        var store = new InMemoryChunkModificationStore();
+        var world = new World.World(registry, store);
+        var mesher = new FakeMesher();
+        var executor = new SynchronousJobExecutor();
+        Func<float[], IChunkRenderer> factory = m => new FakeRenderer(m);
+        Func<Vector3i, VoxelChunk> generate = _ => new VoxelChunk();  // all air
+
+        // Pre-record an edit before the chunk loads.
+        var pos = new Vector3i(0, 0, 0);
+        int localIndex = (5 * VoxelChunk.Size + 6) * VoxelChunk.Size + 7;
+        store.RecordEdit(pos, localIndex, registry.Stone.Id);
+
+        var streamer = new ChunkStreamer(cfg, world, generate, mesher, registry, executor, factory);
+        streamer.Tick(pos);
+
+        Assert.Equal(registry.Stone.Id, world.GetBlock(5, 6, 7).Id);
+    }
+}

World/Streaming/ChunkStreamer.cs

@@ -0,0 +1,215 @@
+using System.Collections.Concurrent;
+using OpenTK.Mathematics;
+using Serilog;
+using VoxelEngine.Core;
+using VoxelEngine.World.Blocks;
+using VoxelEngine.World.Chunk;
+using VoxelEngine.World.WorldGen;
+using VoxelChunk = VoxelEngine.World.Chunk.Chunk;
+
+namespace VoxelEngine.World.Streaming;
+
+public sealed class ChunkStreamer : IDisposable
+{
+    private static readonly ILogger Log = Serilog.Log.ForContext<ChunkStreamer>();
+
+    private readonly Configuration _cfg;
+    private readonly IWorld _world;
+    private readonly Func<Vector3i, VoxelChunk> _generate;
+    private readonly IChunkMesher _mesher;
+    private readonly BlockRegistry _registry;
+    private readonly IJobExecutor _executor;
+    private readonly Func<float[], IChunkRenderer> _rendererFactory;
+
+    private readonly Dictionary<Vector3i, IChunkRenderer> _renderers = new();
+    private readonly HashSet<Vector3i> _tracked = new();
+    private readonly HashSet<Vector3i> _ready = new();
+    private readonly PriorityQueue<Vector3i, int> _pendingLoad = new();
+    private readonly ConcurrentQueue<ChunkJobResult> _completed = new();
+
+    private Vector3i _lastPlayerChunk = new(int.MinValue, int.MinValue, int.MinValue);
+    private int _inFlight;
+
+    public int RendererCount => this._renderers.Count;
+    public bool IsReady(Vector3i pos) => this._ready.Contains(pos);
+
+    public ChunkStreamer(
+        Configuration cfg,
+        IWorld world,
+        Func<Vector3i, VoxelChunk> generate,
+        IChunkMesher mesher,
+        BlockRegistry registry,
+        IJobExecutor executor,
+        Func<float[], IChunkRenderer> rendererFactory)
+    {
+        this._cfg = cfg;
+        this._world = world;
+        this._generate = generate;
+        this._mesher = mesher;
+        this._registry = registry;
+        this._executor = executor;
+        this._rendererFactory = rendererFactory;
+    }
+
+    public void Tick(Vector3i playerChunkPos)
+    {
+        if (playerChunkPos != this._lastPlayerChunk)
+        {
+            this.RecomputeTargetSet(playerChunkPos);
+            this._lastPlayerChunk = playerChunkPos;
+        }
+
+        // Dispatch BEFORE drain: with a synchronous executor (tests) work
+        // completes before Submit() returns, so we want to pick up those
+        // results in this same tick. With the production threadpool executor,
+        // jobs don't finish synchronously — Drain harvests whatever previous
+        // ticks' workers completed in the meantime. Either way, this order is
+        // correct.
+        this.DispatchJobs();
+        this.DrainCompleted();
+    }
+
+    public void DrawAll(Action<Vector3i, IChunkRenderer> drawCallback)
+    {
+        foreach (var kvp in this._renderers) drawCallback(kvp.Key, kvp.Value);
+    }
+
+    public void Dispose()
+    {
+        foreach (var r in this._renderers.Values) r.Dispose();
+        this._renderers.Clear();
+    }
+
+    private void RecomputeTargetSet(Vector3i p)
+    {
+        int h = this._cfg.RenderDistanceHorizontal;
+        int v = this._cfg.RenderDistanceVertical;
+
+        for (int dx = -h; dx <= h; dx++)
+        for (int dy = -v; dy <= v; dy++)
+        for (int dz = -h; dz <= h; dz++)
+        {
+            int cy = p.Y + dy;
+            if (cy < 0 || cy >= WorldConstants.WorldHeightChunks) continue;
+
+            var pos = new Vector3i(p.X + dx, cy, p.Z + dz);
+            if (this._tracked.Contains(pos)) continue;
+
+            this._tracked.Add(pos);
+            int priority = dx * dx + dy * dy + dz * dz;
+            this._pendingLoad.Enqueue(pos, priority);
+        }
+    }
+
+    private void DrainCompleted()
+    {
+        int budget = this._cfg.MaxMeshUploadsPerFrame;
+        while (budget > 0 && this._completed.TryDequeue(out ChunkJobResult? result))
+        {
+            this._inFlight--;
+            budget--;
+            this.InstallResult(result);
+        }
+    }
+
+    private void InstallResult(ChunkJobResult result)
+    {
+        if (result.Error != null || result.Mesh == null || result.Chunk == null)
+        {
+            Log.Error(result.Error, "Chunk {Pos} failed to load", result.Pos);
+            return;
+        }
+
+        this._world.SetChunk(result.Pos, result.Chunk);
+
+        if (this._renderers.TryGetValue(result.Pos, out IChunkRenderer? old))
+        {
+            old.Dispose();
+        }
+        this._renderers[result.Pos] = this._rendererFactory(result.Mesh);
+        this._ready.Add(result.Pos);
+
+        Log.Debug("Loaded {Pos}", result.Pos);
+    }
+
+    private void DispatchJobs()
+    {
+        while (this._inFlight < this._cfg.MaxGenerateJobsInFlight && this._pendingLoad.Count > 0)
+        {
+            Vector3i pos = this._pendingLoad.Dequeue();
+            this.DispatchLoad(pos);
+        }
+    }
+
+    private void DispatchLoad(Vector3i pos)
+    {
+        NeighborSnapshots neighbors = this.SnapshotNeighbors(pos);
+        IReadOnlyDictionary<int, ushort>? diff = null;
+        if (this._world.ModificationStore.TryGetDiff(pos, out var d) && d != null)
+        {
+            // Copy: TryGetDiff returns the live internal dictionary, which the
+            // main thread may mutate via World.SetBlock while the worker iterates.
+            diff = new Dictionary<int, ushort>(d);
+        }
+
+        var job = new ChunkJob(pos, ExistingChunk: null, neighbors, diff);
+        this._inFlight++;
+
+        IChunkMesher mesher = this._mesher;
+        BlockRegistry registry = this._registry;
+        var generate = this._generate;
+        var completed = this._completed;
+
+        this._executor.Submit(() =>
+        {
+            try
+            {
+                VoxelChunk chunk = generate(job.Pos);
+                if (job.Diff != null)
+                {
+                    foreach (var (idx, id) in job.Diff)
+                    {
+                        int lx = idx / (VoxelChunk.Size * VoxelChunk.Size);
+                        int ly = (idx / VoxelChunk.Size) % VoxelChunk.Size;
+                        int lz = idx % VoxelChunk.Size;
+                        chunk[lx, ly, lz] = id;
+                    }
+                }
+
+                float[] mesh = mesher.Build(chunk, registry, (nx, ny, nz) =>
+                    job.Neighbors.IsOpaqueOutside(nx, ny, nz, id => registry.Get(id).IsOpaque));
+
+                completed.Enqueue(new ChunkJobResult(job.Pos, chunk, mesh, Error: null, IsRemeshOnly: false));
+            }
+            catch (Exception ex)
+            {
+                completed.Enqueue(new ChunkJobResult(job.Pos, Chunk: null, Mesh: null, Error: ex, IsRemeshOnly: false));
+            }
+        });
+    }
+
+    private NeighborSnapshots SnapshotNeighbors(Vector3i pos)
+    {
+        ushort[]? Snap(Vector3i p)
+        {
+            VoxelChunk? c = this._world.GetChunk(p);
+            if (c == null) return null;
+            var copy = new ushort[VoxelChunk.Size * VoxelChunk.Size * VoxelChunk.Size];
+            for (int x = 0; x < VoxelChunk.Size; x++)
+            for (int y = 0; y < VoxelChunk.Size; y++)
+            for (int z = 0; z < VoxelChunk.Size; z++)
+            {
+                copy[(x * VoxelChunk.Size + y) * VoxelChunk.Size + z] = c[x, y, z];
+            }
+            return copy;
+        }
+
+        return new NeighborSnapshots(
+            negX: Snap(pos + new Vector3i(-1, 0, 0)),
+            posX: Snap(pos + new Vector3i(1, 0, 0)),
+            negY: Snap(pos + new Vector3i(0, -1, 0)),
+            posY: Snap(pos + new Vector3i(0, 1, 0)),
+            negZ: Snap(pos + new Vector3i(0, 0, -1)),
+            posZ: Snap(pos + new Vector3i(0, 0, 1)));
+    }
+}