batch persistent data reading properly

kuzudb · Aug 7, 2024 · 8fca246 · 8fca246
1 parent d652bfe
commit 8fca246
Show file tree

Hide file tree

Showing 3 changed files with 10 additions and 6 deletions.
diff --git a/src/include/storage/store/csr_node_group.h b/src/include/storage/store/csr_node_group.h
@@ -114,7 +114,7 @@ struct CSRNodeGroupScanState final : NodeGroupScanState {
     std::unique_ptr<ChunkedCSRHeader> csrHeader;
     std::vector<csr_list_t> persistentCSRLists;
     NodeCSRIndex inMemCSRList;
-    // position in vector of either csr list/index vector
+    // position in vector of csr list vector
     uint32_t nextCSRToScan;
 
     bool persistentInitialized = false;

diff --git a/src/storage/store/csr_node_group.cpp b/src/storage/store/csr_node_group.cpp
@@ -156,13 +156,15 @@ NodeGroupScanResult CSRNodeGroup::scanCommittedPersistent(const Transaction* tra
     while (nodeGroupScanState.nextCSRToScan < nodeGroupScanState.persistentCSRLists.size()) {
         auto& csrList = nodeGroupScanState.persistentCSRLists[nodeGroupScanState.nextCSRToScan];
         const auto startRow = csrList.startRow + nodeGroupScanState.nextRowToScan;
-        const auto numToScan = std::min(csrList.length - nodeGroupScanState.nextRowToScan,
-            DEFAULT_VECTOR_CAPACITY - result.numRows);
         if (result.startRow == INVALID_ROW_IDX) {
             result.startRow = startRow;
+        } else if (startRow - result.startRow >= DEFAULT_VECTOR_CAPACITY) {
+            break;
         }
-        persistentChunkGroup->scan(transaction, tableState, nodeGroupScanState, startRow,
-            numToScan);
+        // accomodate for any gaps in between the current and previous csr
+        result.numRows = startRow - result.startRow;
+        const auto numToScan = std::min(csrList.length - nodeGroupScanState.nextRowToScan,
+            DEFAULT_VECTOR_CAPACITY - result.numRows);
         result.numRows += numToScan;
         if (numToScan == csrList.length - nodeGroupScanState.nextRowToScan) {
             nodeGroupScanState.nextCSRToScan++;
@@ -173,6 +175,8 @@ NodeGroupScanResult CSRNodeGroup::scanCommittedPersistent(const Transaction* tra
             break;
         }
     }
+    persistentChunkGroup->scan(transaction, tableState, nodeGroupScanState, result.startRow,
+        result.numRows);
     return result;
 }
 

diff --git a/src/storage/store/rel_table.cpp b/src/storage/store/rel_table.cpp
@@ -100,7 +100,7 @@ bool RelTable::scanInternal(Transaction* transaction, TableScanState& scanState)
     if (relScanState.currNodeIdx == relScanState.totalNodeIdx) {
         return false;
     }
-    // We need to reinitialize per node group
+    // We reinitialize per node for in memory and local table data
     if (relScanState.currNodeIdx == relScanState.endNodeIdx) {
         if (relScanState.resetCommitted) {
             // reset to read committed, in memory data