de-epsilon geometry for translucent sorting

src/main/java/me/jellysquid/mods/sodium/client/render/chunk/compile/pipeline/FluidRenderer.java

@@ -47,9 +47,9 @@
  * and outside quads for each face of the block.
  */
 public class FluidRenderer {
-    // TODO: allow this to be changed by vertex format
+    // TODO: allow this to be changed by vertex format, WARNING: make sure TranslucentGeometryCollector knows about EPSILON
     // TODO: move fluid rendering to a separate render pass and control glPolygonOffset and glDepthFunc to fix this properly
-    private static final float EPSILON = 0.001f;
+    public static final float EPSILON = 0.001f;
 
     private final BlockPos.Mutable scratchPos = new BlockPos.Mutable();
     private final MutableFloat scratchHeight = new MutableFloat(0);

src/main/java/me/jellysquid/mods/sodium/client/render/chunk/gfni/ComplexSorting.java

@@ -120,9 +120,9 @@ private static boolean orthogonalQuadVisibleThrough(TQuad halfspace, TQuad other
         var otherSign = otherQuad.facing().getSign();
 
         // A: test that the other quad has an extent within this quad's halfspace
-        return hSign * halfspace.extents()[hd] - QUERY_EPSILON > hSign * otherQuad.extents()[hdOpposite]
+        return hSign * halfspace.extents()[hd] > hSign * otherQuad.extents()[hdOpposite]
                 // B: test that this quad is not fully within the other quad's halfspace
-                && !(otherSign * otherQuad.extents()[od] - QUERY_EPSILON >= otherSign * halfspace.extents()[od]);
+                && !(otherSign * otherQuad.extents()[od] >= otherSign * halfspace.extents()[od]);
     }
 
     /**
@@ -331,15 +331,13 @@ public static boolean topoSortAlignedScanningCyclic(IntBuffer indexBuffer, TQuad
         return true;
     }
 
-    private static final float QUERY_EPSILON = 0.0011f;
-
     private static boolean testSeparatorRange(Object2ReferenceOpenHashMap<Vector3fc, double[]> distancesByNormal,
             Vector3fc normal, float start, float end) {
         var distances = distancesByNormal.get(normal);
         if (distances == null) {
             return false;
         }
-        return Group.queryRange(distances, start - QUERY_EPSILON, end + QUERY_EPSILON);
+        return Group.queryRange(distances, start, end);
     }
 
     private static boolean visibilityWithSeparator(TQuad quadA, TQuad quadB,

src/main/java/me/jellysquid/mods/sodium/client/render/chunk/gfni/GFNI.java

@@ -25,10 +25,6 @@
  * may result in many sections suddenly needing sorting when the camera moves.
  * Maybe it's better to schedule them to be sorted gradually even if not
  * visible, if there are idle threads.
- * - De-epsilon all the geometry by snapping to multiples of 0.005 or sth like
- * that. Would simplify the ComplexSorting code so that it doesn't need to deal
- * with the existence of error margins (epsilons in the trigger distances and
- * the centers).
  * - Groups of quads that form convex shapes in a single plane without holes can
  * be sorted as one "quad". Their internal sorting can be arbitrary. Detecting
  * and grouping/ungrouping them might prove difficult. Finding all quad groups

src/main/java/me/jellysquid/mods/sodium/client/render/chunk/gfni/TranslucentGeometryCollector.java

@@ -9,6 +9,7 @@
 import me.jellysquid.mods.sodium.client.gui.SodiumGameOptions.SortBehavior;
 import me.jellysquid.mods.sodium.client.model.quad.ModelQuadView;
 import me.jellysquid.mods.sodium.client.model.quad.properties.ModelQuadFacing;
+import me.jellysquid.mods.sodium.client.render.chunk.compile.pipeline.FluidRenderer;
 import me.jellysquid.mods.sodium.client.render.chunk.data.BuiltSectionMeshParts;
 import me.jellysquid.mods.sodium.client.render.chunk.vertex.format.ChunkVertexEncoder;
 import net.minecraft.util.math.ChunkSectionPos;
@@ -50,12 +51,26 @@ public TranslucentGeometryCollector(ChunkSectionPos sectionPos) {
         this.sectionPos = sectionPos;
     }
 
+    private static final float INV_QUANTIZE_EPSILON = 256f;
+    private static final float QUANTIZE_EPSILON = 1f / INV_QUANTIZE_EPSILON;
+
+    static {
+        // ensure it fits with the fluid renderer epsilon and that it's a power-of-two fraction
+        var targetEpsilon = FluidRenderer.EPSILON * 2.1f;
+        if (QUANTIZE_EPSILON <= targetEpsilon && Integer.bitCount((int) INV_QUANTIZE_EPSILON) == 1) {
+            throw new RuntimeException("epsilon is invalid: " + QUANTIZE_EPSILON);
+        }
+    }
+
+    private static float quantizeEpsilon(float value) {
+        return (float) Math.floor(value * INV_QUANTIZE_EPSILON + 0.5) * QUANTIZE_EPSILON;
+    }
+
     public void appendQuad(ModelQuadView quadView, ChunkVertexEncoder.Vertex[] vertices, ModelQuadFacing facing) {
         float xSum = 0;
         float ySum = 0;
         float zSum = 0;
 
-
         // keep track of distinct vertices to compute the center accurately for
         // degenerate quads
         float lastX = vertices[3].x;
@@ -105,7 +120,17 @@ public void appendQuad(ModelQuadView quadView, ChunkVertexEncoder.Vertex[] verti
             }
         }
 
-        var center = new Vector3f(xSum / uniqueQuads, ySum / uniqueQuads, zSum / uniqueQuads);
+        var centerX = quantizeEpsilon(xSum / uniqueQuads);
+        var centerY = quantizeEpsilon(ySum / uniqueQuads);
+        var centerZ = quantizeEpsilon(zSum / uniqueQuads);
+        var center = new Vector3f(centerX, centerY, centerZ);
+
+        negXExtent = quantizeEpsilon(negXExtent);
+        negYExtent = quantizeEpsilon(negYExtent);
+        negZExtent = quantizeEpsilon(negZExtent);
+        posXExtent = quantizeEpsilon(posXExtent);
+        posYExtent = quantizeEpsilon(posYExtent);
+        posZExtent = quantizeEpsilon(posZExtent);
 
         if (facing != ModelQuadFacing.UNASSIGNED && this.unalignedDistances == null) {
             minBounds.x = Math.min(minBounds.x, negXExtent);