FileStorage: fix rare data corruption when using restore after multiple undos

CHANGES.rst

@@ -11,6 +11,9 @@
 
 - Fix sorting issue in ``scripts/space.py``.
 
+- FileStorage: fix a rare data corruption when using restore after multiple undos.
+  For details see `#395 <https://github.com/zopefoundation/ZODB/pull/395>`_.
+
 - Fix exit code of ``repozo`` script in case of verification error.
   For details see `#396 <https://github.com/zopefoundation/ZODB/pull/396>`_.
 

src/ZODB/FileStorage/FileStorage.py

@@ -663,6 +663,10 @@ def _data_find(self, tpos, oid, data):
         # Else oid's data record contains the data, and the file offset of
         # oid's data record is returned.  This data record should contain
         # a pickle identical to the 'data' argument.
+        # When looking for oid's data record we scan all data records in
+        # the transaction till the end looking for the latest record with oid,
+        # even if there are multiple such records. This matches load behaviour
+        # which uses the data record created by last store.
 
         # Unclear:  If the length of the stored data doesn't match len(data),
         # an exception is raised.  If the lengths match but the data isn't
@@ -672,28 +676,38 @@ def _data_find(self, tpos, oid, data):
         tid, tl, status, ul, dl, el = unpack(TRANS_HDR, h)
         status = as_text(status)
         self._file.read(ul + dl + el)
-        tend = tpos + tl + 8
+        tend = tpos + tl
         pos = self._file.tell()
+        data_hdr = None
+        data_pos = 0
+        # scan all data records in this transaction looking for the latest
+        # record with our oid
         while pos < tend:
             h = self._read_data_header(pos)
             if h.oid == oid:
-                # Make sure this looks like the right data record
-                if h.plen == 0:
-                    # This is also a backpointer.  Gotta trust it.
-                    return pos
-                if h.plen != len(data):
-                    # The expected data doesn't match what's in the
-                    # backpointer.  Something is wrong.
-                    logger.error("Mismatch between data and"
-                                 " backpointer at %d", pos)
-                    return 0
-                _data = self._file.read(h.plen)
-                if data != _data:
-                    return 0
-                return pos
+                data_hdr = h
+                data_pos = pos
             pos += h.recordlen()
             self._file.seek(pos)
-        return 0
+        if data_hdr is None:
+            return 0
+
+        # return position of found data record, but make sure this looks like
+        # the right data record to return.
+        if data_hdr.plen == 0:
+            # This is also a backpointer,  Gotta trust it.
+            return data_pos
+        else:
+            if data_hdr.plen != len(data):
+                # The expected data doesn't match what's in the
+                # backpointer.  Something is wrong.
+                logger.error("Mismatch between data and"
+                             " backpointer at %d", pos)
+                return 0
+            _data = self._file.read(data_hdr.plen)
+            if data != _data:
+                return 0
+            return data_pos
 
     def restore(self, oid, serial, data, version, prev_txn, transaction):
         # A lot like store() but without all the consistency checks.  This

src/ZODB/tests/RecoveryStorage.py

@@ -199,3 +199,43 @@ def testRestoreWithMultipleObjectsInUndoRedo(self):
         # part of the test.
         self._dst.copyTransactionsFrom(self._storage)
         self.compare(self._storage, self._dst)
+
+    def testRestoreWithMultipleUndoRedo(self):
+        db = DB(self._storage)
+        c = db.open()
+        r = c.root()
+
+        # TO
+        r["obj"] = MinPO(0)
+        transaction.commit()
+        c.close()
+
+        # T1: do 1
+        r = db.open().root()
+        r["obj"].value = 1
+        transaction.commit()
+
+        # T2: do 2
+        r = db.open().root()
+        r["obj"].value = 2
+        transaction.commit()
+
+        # T3: undo T1 and T2
+        db.undoMultiple([u["id"] for u in db.undoLog(0, 2)])
+        transaction.commit()
+        # obj will be a backpointer to T0
+
+        # T4: do 3
+        r = db.open().root()
+        r["obj"].value = 3
+        transaction.commit()
+
+        # T4: undo
+        self._undo(self._storage.undoInfo()[0]['id'])
+        # obj will be a backpointer to T3, which is a backpointer to T0
+
+        r = db.open().root()
+        self.assertEqual(r["obj"].value, 0)
+
+        self._dst.copyTransactionsFrom(self._storage)
+        self.compare(self._storage, self._dst)

src/ZODB/tests/testFileStorage.py

@@ -456,6 +456,10 @@ def testRestoreAcrossPack(self):
         # Skip this check as it calls restore directly.
         pass
 
+    def testRestoreWithMultipleUndoRedo(self):
+        # This case is only supported with restore, not with store "emulation"
+        pass
+
 
 class AnalyzeDotPyTest(StorageTestBase.StorageTestBase):