SF-3166 Fix crash while retrieving history during sync

src/SIL.XForge.Scripture/Models/TextData.cs

@@ -11,6 +11,9 @@ public class TextData : Delta, IIdentifiable
     public static string GetTextDocId(string projectId, int book, int chapter) =>
         $"{projectId}:{Canon.BookNumberToId(book)}:{chapter}:target";
 
+    public static string GetTextDocId(string projectId, string book, int chapter) =>
+        $"{projectId}:{book}:{chapter}:target";
+
     public TextData() { }
 
     public TextData(Delta delta)

src/SIL.XForge.Scripture/Services/ParatextService.cs

@@ -1910,8 +1910,8 @@ DateTime timestamp
         using ScrText scrText = GetScrText(userSecret, ptProjectId);
         VerseRef verseRef = new VerseRef($"{book} {chapter}:0");
 
-        TextSnapshot ret = null;
-        string id = $"{sfProjectId}:{book}:{chapter}:target";
+        TextSnapshot ret;
+        string id = TextData.GetTextDocId(sfProjectId, book, chapter);
         Snapshot<TextData> snapshot = await connection.FetchSnapshotAsync<TextData>(id, timestamp);
 
         if (snapshot.Data is null)
@@ -1924,10 +1924,10 @@ DateTime timestamp
             HgRevisionCollection revisionCollection = HgRevisionCollection.Get(scrText);
             DateTimeOffset timeStampOffset = new DateTimeOffset(timestamp, TimeSpan.Zero);
             HgRevision? revision = revisionCollection
-                .MutableCollection.Where(r => r.CommitTimeStamp <= timeStampOffset)
+                .FilterRevisions(r => r.CommitTimeStamp <= timeStampOffset)
                 .MaxBy(r => r.CommitTimeStamp);
 
-            // No revision was before than the timestamp, so get the first revision
+            // No revision was before the timestamp, so get the first revision
             revision ??= revisionCollection.MinBy(r => r.LocalRevisionNumber);
             if (revision is null)
             {
@@ -1988,7 +1988,7 @@ int chapter
             throw new ForbiddenException();
         }
 
-        string id = $"{sfProjectId}:{book}:{chapter}:target";
+        string id = TextData.GetTextDocId(sfProjectId, book, chapter);
         Op[] ops = await connection.GetOpsAsync<TextData>(id);
 
         // Iterate over the ops in reverse order, returning a milestone at least every 15 minutes
@@ -2062,11 +2062,11 @@ int chapter
         // Note: The following code is not testable due to ParatextData limitations
         // Iterate over the Paratext commits earlier than the earliest MongoOp
         DateTimeOffset timeStampOffset = new DateTimeOffset(milestonePeriod, TimeSpan.Zero);
-        HgRevisionCollection revisionCollection = HgRevisionCollection.Get(scrText);
+        HgRevisionCollection revisionCollection = HgRevisionCollection
+            .Get(scrText)
+            .FilterRevisions(r => r.CommitTimeStamp <= timeStampOffset);
         int bookNum = Canon.BookIdToNumber(book);
-        foreach (
-            HgRevision revision in revisionCollection.MutableCollection.Where(r => r.CommitTimeStamp <= timeStampOffset)
-        )
+        foreach (HgRevision revision in revisionCollection)
         {
             // Get the revision summary to see if the book and chapter has changed
             RevisionChangeInfo revisionSummary = revisionCollection.GetSummaryFor(revision);

src/SIL.XForge.Scripture/Services/SFScrTextCollection.cs

@@ -21,6 +21,13 @@ public class SFScrTextCollection : ScrTextCollection
     /// </summary>
     public static string ResourcesByIdDirectory => Path.Combine(SettingsDirectory, "_resourcesById");
 
+    /// <summary>
+    /// Adds a ScrText to the internal index. This should only be used for testing purposes!
+    /// </summary>
+    /// <param name="scrText">The scripture text to add to the index.</param>
+    public void AddToInternalIndex(ScrText scrText) =>
+        AddInternal(scrText, skipChangeNotify: false, checkAlreadyExists: false);
+
     protected override string DictionariesDirectoryInternal => null;
 
     protected override void InitializeInternal(string settingsDir, bool allowMigration)

src/SIL.XForge/DataAccess/MemoryRepository.cs

@@ -8,6 +8,7 @@
 using MongoDB.Bson;
 using Newtonsoft.Json;
 using SIL.XForge.Models;
+using SIL.XForge.Realtime;
 using SIL.XForge.Utils;
 
 namespace SIL.XForge.DataAccess;
@@ -23,7 +24,8 @@ public class MemoryRepository<T> : IRepository<T>
         Converters = [new BsonValueConverter()],
     };
 
-    private readonly ConcurrentDictionary<string, string> _entities;
+    private readonly ConcurrentDictionary<string, string> _entities = [];
+    private readonly ConcurrentDictionary<string, Op[]> _entityOps = [];
     private readonly Func<T, object>[] _uniqueKeySelectors;
     private readonly HashSet<object>[] _uniqueKeys;
 
@@ -37,7 +39,6 @@ public MemoryRepository(IEnumerable<Func<T, object>>? uniqueKeySelectors = null,
         for (int i = 0; i < _uniqueKeys.Length; i++)
             _uniqueKeys[i] = [];
 
-        _entities = new ConcurrentDictionary<string, string>();
         if (entities != null)
             Add(entities);
     }
@@ -86,6 +87,10 @@ public void Replace(T entity)
 
     public T Get(string id) => DeserializeEntity(id, _entities[id]);
 
+    public Op[] GetOps(string id) => _entityOps.TryGetValue(id, out Op[] ops) ? ops : [];
+
+    public void SetOps(string id, Op[] ops) => _entityOps[id] = ops;
+
     public IQueryable<T> Query() => _entities.Select(kvp => DeserializeEntity(kvp.Key, kvp.Value)).AsQueryable();
 
     public Task InsertAsync(T entity)

src/SIL.XForge/Realtime/MemoryConnection.cs

@@ -103,39 +103,9 @@ public async Task<Snapshot<T>> FetchSnapshotAsync<T>(string id, DateTime timesta
     /// <summary>
     /// Gets the ops for a document.
     /// </summary>
-    /// <returns>A default Op array for test purposes.</returns>
+    /// <returns>An Op array.</returns>
     public Task<Op[]> GetOpsAsync<T>(string id)
-        where T : IIdentifiable =>
-        Task.FromResult(
-            new Op[]
-            {
-                new Op
-                {
-                    Metadata = new OpMetadata { Timestamp = DateTime.UtcNow.AddMinutes(-30) },
-                    Version = 1,
-                },
-                new Op
-                {
-                    Metadata = new OpMetadata { Timestamp = DateTime.UtcNow.AddMinutes(-10) },
-                    Version = 2,
-                },
-                new Op
-                {
-                    Metadata = new OpMetadata { Timestamp = DateTime.UtcNow.AddMinutes(-1) },
-                    Version = 3,
-                },
-                new Op
-                {
-                    Metadata = new OpMetadata
-                    {
-                        Timestamp = DateTime.UtcNow,
-                        UserId = "user01",
-                        Source = OpSource.Draft,
-                    },
-                    Version = 4,
-                },
-            }
-        );
+        where T : IIdentifiable => Task.FromResult(_realtimeService.GetRepository<T>().GetOps(id));
 
     public IDocument<T> Get<T>(string id)
         where T : IIdentifiable

src/SIL.XForge/Realtime/MemoryRealtimeService.cs

@@ -30,14 +30,8 @@ private static INodeJSService CreateNodeJSService()
         return sp.GetRequiredService<INodeJSService>();
     }
 
-    private readonly Dictionary<Type, object> _repos;
-    private readonly Dictionary<Type, DocConfig> _docConfigs;
-
-    public MemoryRealtimeService()
-    {
-        _repos = [];
-        _docConfigs = [];
-    }
+    private readonly Dictionary<Type, object> _repos = [];
+    private readonly Dictionary<Type, DocConfig> _docConfigs = [];
 
     /// <summary>
     /// Gets or sets the last modified user identifier.

test/SIL.XForge.Scripture.Tests/Services/MachineApiServiceTests.cs

@@ -1204,7 +1204,7 @@ public async Task GetPreTranslationDeltaAsync_Success()
             CancellationToken.None
         );
         Assert.AreEqual(expected.Ops[0], actual.Data.Ops[0]);
-        Assert.AreEqual($"{Project01}:MAT:1:target", actual.Id);
+        Assert.AreEqual(TextData.GetTextDocId(Project01, "MAT", 1), actual.Id);
     }
 
     [Test]

test/SIL.XForge.Scripture.Tests/Services/MockHg.cs

@@ -0,0 +1,28 @@
+using System.Collections.Generic;
+using Paratext.Data.Repository;
+
+namespace SIL.XForge.Scripture.Services;
+
+internal class MockHg : Hg
+{
+    /// <summary>
+    /// A log containing a revision from Paratext
+    /// </summary>
+    public readonly List<HgRevision> Log =
+    [
+        new HgRevision
+        {
+            Id = "2",
+            CommitTimeStampString = "2024-11-13T11:43:01+13:00",
+            Filenames = "08RUT.SFM",
+            ParentsString = "1",
+            UserEscaped = "pt_username",
+        },
+    ];
+
+    public override bool IsRepository(string repository) => true;
+
+    public override List<HgRevision> GetLog(string repository, string startRev, string endRev) => Log;
+
+    public override List<HgRevision> GetLogWithLimit(string repository, int limit) => Log;
+}

test/SIL.XForge.Scripture.Tests/Services/MockHgRunner.cs

@@ -0,0 +1,59 @@
+using Paratext.Data;
+
+namespace SIL.XForge.Scripture.Services;
+
+internal class MockHgRunner() : HgExeRunner(null, null, null)
+{
+    private bool _appendCallCount;
+    private int _callCount;
+    private string _standardOutput = string.Empty;
+
+    /// <summary>
+    /// Mocks running a Mercurial command by incrementing a call count.
+    /// </summary>
+    /// <param name="args"></param>
+    public override void RunHg(string args) => _callCount++;
+
+    /// <summary>
+    /// Gets a success exit code.
+    /// </summary>
+    public override int ExitCode => 0;
+
+    /// <summary>
+    /// Gets the standard error, which is always empty.
+    /// </summary>
+    public override string StandardError => string.Empty;
+
+    /// <summary>
+    /// Gets the standard output.
+    /// </summary>
+    public override string StandardOutput =>
+        _standardOutput + (_appendCallCount ? _callCount.ToString() : string.Empty);
+
+    /// <summary>
+    /// Gets the standard output.
+    /// </summary>
+    public override string StandardOutputNoTrim => StandardOutput;
+
+    /// <summary>
+    /// Initialize the HgRunner (i.e. do nothing)
+    /// </summary>
+    public override void Initialize()
+    {
+        // Do nothing
+    }
+
+    /// <summary>
+    /// Sets the standard output
+    /// </summary>
+    /// <param name="standardOutput">The standard output.</param>
+    /// <param name="appendCallCountToOutput">
+    /// Adds the call count to the end of the output.
+    /// This is useful to simulate the last verse changing in hg cat output.
+    /// </param>
+    public void SetStandardOutput(string standardOutput, bool appendCallCountToOutput)
+    {
+        _standardOutput = standardOutput;
+        _appendCallCount = appendCallCountToOutput;
+    }
+}