Avoid loading all data into memory when downloading/processing.

Ability to specify output filename when processing.
jdpurcell · Mar 18, 2018 · 09ac8bf · 09ac8bf
1 parent 1475b1d
commit 09ac8bf
Show file tree

Hide file tree

Showing 4 changed files with 45 additions and 30 deletions.
diff --git a/RechatTool/Program.cs b/RechatTool/Program.cs
@@ -9,7 +9,7 @@
 
 namespace RechatTool {
 	internal class Program {
-		public const string Version = "1.4.0.0";
+		public const string Version = "1.5.0.0";
 
 		private static int Main(string[] args) {
 			int iArg = 0;
@@ -26,7 +26,7 @@ string GetArg(bool optional = false) =>
 					long videoId = videoIdStr.TryParseInt64() ??
 						TryParseVideoIdFromUrl(videoIdStr) ??
 						throw new InvalidArgumentException();
-					string path = PeekArg()?.StartsWith("-") == false ? GetArg() : $"{videoId}.json";
+					string path = PeekArg()?.StartsWith("-", StringComparison.Ordinal) == false ? GetArg() : $"{videoId}.json";
 					bool overwrite = false;
 					while ((arg = GetArg(true)) != null) {
 						if (arg == "-o") {
@@ -48,9 +48,13 @@ void UpdateProgress(int downloaded) {
 				}
 				else if (arg == "-p") {
 					string[] paths = { GetArg() };
+					string outputPath = null;
 					if (paths[0].IndexOfAny(new[] { '*', '?'}) != -1) {
 						paths = Directory.GetFiles(Path.GetDirectoryName(paths[0]), Path.GetFileName(paths[0]));
 					}
+					else if (PeekArg()?.StartsWith("-", StringComparison.Ordinal) == false) {
+						outputPath = GetArg();
+					}
 					bool overwrite = false;
 					bool showBadges = false;
 					while ((arg = GetArg(true)) != null) {
@@ -66,7 +70,7 @@ void UpdateProgress(int downloaded) {
 					}
 					foreach (string p in paths) {
 						Console.WriteLine("Processing " + Path.GetFileName(p));
-						Rechat.ProcessFile(p, overwrite: overwrite, showBadges: showBadges);
+						Rechat.ProcessFile(p, pathOut: outputPath, overwrite: overwrite, showBadges: showBadges);
 					}
 					Console.WriteLine("Done!");
 				}
@@ -80,16 +84,20 @@ void UpdateProgress(int downloaded) {
 				Console.WriteLine($"RechatTool v{new Version(Version).ToDisplayString()}");
 				Console.WriteLine();
 				Console.WriteLine("Modes:");
-				Console.WriteLine("   -d videoid [path] [-o]");
-				Console.WriteLine("      Downloads chat replay for the specified videoid. If path is not");
-				Console.WriteLine("      specified, output is saved to the current directory. -o overwrites");
-				Console.WriteLine("      existing output file.");
+				Console.WriteLine("   -d videoid [filename] [-o]");
+				Console.WriteLine("      Downloads chat replay for the specified videoid.");
+				Console.WriteLine("        filename: Output location as relative or absolute filename, otherwise");
+				Console.WriteLine("          defaults to the current directory and named as videoid.json.");
+				Console.WriteLine("        -o: Overwrite the existing output file.");
 				Console.WriteLine("   -D (same parameters as -d)");
 				Console.WriteLine("      Downloads and processes chat replay (combines -d and -p).");
-				Console.WriteLine("   -p path [-o]");
+				Console.WriteLine("   -p filename [output_filename] [-o] [-b]");
 				Console.WriteLine("      Processes a JSON chat replay file and outputs a human-readable text file.");
-				Console.WriteLine("      Output is written to same folder as the input file with the extension");
-				Console.WriteLine("      changed to .txt.");
+				Console.WriteLine("        output_filename: Output location as relative or absolute filename,");
+				Console.WriteLine("            otherwise defaults to the same location as the input file with the");
+				Console.WriteLine("            extension changed to .txt.");
+				Console.WriteLine("        -o: Overwrite the existing output file. ");
+				Console.WriteLine("        -b: Show user badges (e.g. moderator/subscriber).");
 				return 1;
 			}
 			catch (Exception ex) {

diff --git a/RechatTool/Rechat.cs b/RechatTool/Rechat.cs
@@ -19,20 +19,25 @@ public static void DownloadFile(long videoId, string path, bool overwrite = fals
 				throw new Exception("Output file already exists.");
 			}
 			string baseUrl = $"{"https"}://api.twitch.tv/v5/videos/{videoId}/comments";
-			var segments = new List<JArray>();
 			string nextCursor = null;
-			do {
-				string url = nextCursor == null ?
-					$"{baseUrl}?content_offset_seconds=0" :
-					$"{baseUrl}?cursor={nextCursor}";
-				JObject response = JObject.Parse(DownloadUrlAsString(url, withRequest: AddTwitchApiHeaders));
-				segments.Add((JArray)response["comments"]);
-				nextCursor = (string)response["_next"];
-				progressCallback?.Invoke(segments.Count);
+			int segmentCount = 0;
+			using (var writer = new JsonTextWriter(new StreamWriter(path, false, new UTF8Encoding(true)))) {
+				writer.WriteStartArray();
+				do {
+					string url = nextCursor == null ?
+						$"{baseUrl}?content_offset_seconds=0" :
+						$"{baseUrl}?cursor={nextCursor}";
+					JObject response = JObject.Parse(DownloadUrlAsString(url, withRequest: AddTwitchApiHeaders));
+					foreach (JObject comment in (JArray)response["comments"]) {
+						comment.WriteTo(writer);
+					}
+					nextCursor = (string)response["_next"];
+					segmentCount++;
+					progressCallback?.Invoke(segmentCount);
+				}
+				while (nextCursor != null);
+				writer.WriteEndArray();
 			}
-			while (nextCursor != null);
-			JArray combined = new JArray(segments.SelectMany(s => s));
-			File.WriteAllText(path, combined.ToString(Formatting.None), new UTF8Encoding(true));
 		}
 
 		private static string DownloadUrlAsString(string url, Action<HttpWebRequest> withRequest = null) {
@@ -65,11 +70,13 @@ public static void ProcessFile(string pathIn, string pathOut = null, bool overwr
 			File.WriteAllLines(pathOut, lines, new UTF8Encoding(true));
 		}
 
-		public static List<RechatMessage> ParseMessages(string path) {
-			return JArray.Parse(File.ReadAllText(path))
-				.Cast<JObject>()
-				.Select(n => new RechatMessage(n))
-				.ToList();
+		public static IEnumerable<RechatMessage> ParseMessages(string path) {
+			using (var reader = new JsonTextReader(File.OpenText(path))) {
+				while (reader.Read()) {
+					if (reader.TokenType != JsonToken.StartObject) continue;
+					yield return new RechatMessage(JObject.Load(reader));
+				}
+			}
 		}
 
 		private static string ToReadableString(RechatMessage m, bool showBadges) {

diff --git a/RechatTool/RechatTool.csproj b/RechatTool/RechatTool.csproj
@@ -34,8 +34,8 @@
     <Prefer32Bit>false</Prefer32Bit>
   </PropertyGroup>
   <ItemGroup>
-    <Reference Include="Newtonsoft.Json, Version=10.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
-      <HintPath>..\packages\Newtonsoft.Json.10.0.3\lib\net45\Newtonsoft.Json.dll</HintPath>
+    <Reference Include="Newtonsoft.Json, Version=11.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
+      <HintPath>..\packages\Newtonsoft.Json.11.0.1\lib\net45\Newtonsoft.Json.dll</HintPath>
     </Reference>
     <Reference Include="System" />
     <Reference Include="System.Core" />

diff --git a/RechatTool/packages.config b/RechatTool/packages.config
@@ -1,4 +1,4 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="Newtonsoft.Json" version="10.0.3" targetFramework="net45" />
+  <package id="Newtonsoft.Json" version="11.0.1" targetFramework="net46" />
 </packages>