Merge pull request #268 from Mbucari/master

Address issues in 263

Source/AppScaffolding/LibationScaffolding.cs

@@ -59,6 +59,7 @@ public static void RunPostConfigMigrations(Configuration config)
 			//
 
 			Migrations.migrate_to_v6_6_9(config);
+			Migrations.migrate_from_7_10_1(config);
 		}
 
 		public static void PopulateMissingConfigValues(Configuration config)
@@ -401,5 +402,28 @@ public static void migrate_to_v6_6_9(Configuration config)
 				UNSAFE_MigrationHelper.Settings_AddUniqueToArray("Serilog.Enrich", "WithExceptionDetails");
 			}
 		}
+
+		public static void migrate_from_7_10_1(Configuration config)
+		{
+			//This migration removes books and series with SERIES_ prefix that were created
+			//as a hack workaround in 7.10.1. Said workaround was removed in 7.10.2
+
+			var migrated = config.GetNonString<bool>(nameof(migrate_from_7_10_1));
+
+			if (migrated) return;
+
+			using var context = DbContexts.GetContext();
+
+			var booksToRemove = context.Books.Where(b => b.AudibleProductId.StartsWith("SERIES_")).ToArray();
+			var seriesToRemove = context.Series.Where(s => s.AudibleSeriesId.StartsWith("SERIES_")).ToArray();
+			var lbToRemove = context.LibraryBooks.Where(lb => booksToRemove.Any(b => b == lb.Book)).ToArray();
+
+			context.LibraryBooks.RemoveRange(lbToRemove);
+			context.Books.RemoveRange(booksToRemove);
+			context.Series.RemoveRange(seriesToRemove);
+
+			LibraryCommands.SaveContext(context);
+			config.SetObject(nameof(migrate_from_7_10_1), true);
+		}
 	}
 }

Source/ApplicationServices/DbContexts.cs

@@ -12,10 +12,10 @@ public static LibationContext GetContext()
 			=> LibationContext.Create(SqliteStorage.ConnectionString);
 
 		/// <summary>Use for full library querying. No lazy loading</summary>
-		public static List<LibraryBook> GetLibrary_Flat_NoTracking()
+		public static List<LibraryBook> GetLibrary_Flat_NoTracking(bool includeParents = false)
 		{
 			using var context = GetContext();
-			return context.GetLibrary_Flat_NoTracking();
+			return context.GetLibrary_Flat_NoTracking(includeParents);
 		}
 	}
 }

Source/ApplicationServices/LibraryCommands.cs

@@ -200,7 +200,7 @@ private static async Task<int> importIntoDbAsync(List<ImportItem> importItems)
             var libraryBookImporter = new LibraryBookImporter(context);
             var newCount = await Task.Run(() => libraryBookImporter.Import(importItems));
             logTime("importIntoDbAsync -- post Import()");
-            int qtyChanges = saveChanges(context);
+            int qtyChanges = SaveContext(context);
             logTime("importIntoDbAsync -- post SaveChanges");
 
 			// this is any changes at all to the database, not just new books
@@ -211,7 +211,7 @@ private static async Task<int> importIntoDbAsync(List<ImportItem> importItems)
             return newCount;
         }
 
-        private static int saveChanges(LibationContext context)
+        public static int SaveContext(LibationContext context)
 		{
 			try
 			{

Source/ApplicationServices/SearchEngineCommands.cs

@@ -29,7 +29,7 @@ private static T performSafeQuery<T>(Func<SearchEngine, T> func)
 		}
 		#endregion
 
-		public static EventHandler SearchEngineUpdated;
+		public static event EventHandler SearchEngineUpdated;
 
 		#region Update
 		private static bool isUpdating;

Source/AudibleUtilities/ApiExtended.cs

@@ -1,11 +1,14 @@
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using AudibleApi;
 using AudibleApi.Common;
 using Dinah.Core;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
 using Polly;
 using Polly.Retry;
 
@@ -129,7 +132,7 @@ private async Task<List<Item>> getItemsAsync(LibraryOptions libraryOptions, bool
 
 			await foreach (var item in Api.GetLibraryItemAsyncEnumerable(libraryOptions))
 			{
-				if (item.IsEpisodes && importEpisodes)
+				if ((item.IsEpisodes || item.IsSeriesParent) && importEpisodes)
 				{
 					//Get child episodes asynchronously and await all at the end
 					getChildEpisodesTasks.Add(getChildEpisodesAsync(concurrencySemaphore, item));
@@ -173,17 +176,66 @@ private async Task<List<Item>> getChildEpisodesAsync(SemaphoreSlim concurrencySe
 			{
 				Serilog.Log.Logger.Debug("Beginning episode scan for {parent}", parent);
 
-				var children = await getEpisodeChildrenAsync(parent);
+				List<Item> children;
 
-				if (!children.Any())
+				if (parent.IsEpisodes)
 				{
-					//The parent is the only episode in the podcase series,
-					//so the parent is its own child.
-					parent.Series = new Series[] { new Series { Asin = parent.Asin, Sequence = RelationshipToProduct.Parent, Title = parent.TitleWithSubtitle } };
-					children.Add(parent);
-					return children;
+					//The 'parent' is a single episode that was added to the library.
+					//Get the episode's parent and add it to the database.
+
+					Serilog.Log.Logger.Debug("Supplied Parent is an episode. Beginning parent scan for {parent}", parent);
+
+					children = new() { parent };
+
+					var parentAsins = parent.Relationships
+						.Where(r => r.RelationshipToProduct == RelationshipToProduct.Parent)
+						.Select(p => p.Asin);
+
+					var seriesParents = await Api.GetCatalogProductsAsync(parentAsins, CatalogOptions.ResponseGroupOptions.ALL_OPTIONS);
+
+					int numSeriesParents = seriesParents.Count(p => p.IsSeriesParent);
+					if (numSeriesParents != 1)
+					{
+						//There should only ever be 1 top-level parent per episode. If not, log
+						//and throw so we can figure out what to do about those special cases.
+						JsonSerializerSettings Settings = new()
+						{
+							MetadataPropertyHandling = MetadataPropertyHandling.Ignore,
+							DateParseHandling = DateParseHandling.None,
+							Converters =
+							{
+								new IsoDateTimeConverter { DateTimeStyles = DateTimeStyles.AssumeUniversal }
+							},
+						};
+						var ex = new ApplicationException($"Found {numSeriesParents} parents for {parent.Asin}");
+						Serilog.Log.Logger.Error(ex, $"Episode Product:\r\n{JsonConvert.SerializeObject(parent, Formatting.None, Settings)}");
+						throw ex;
+					}
+
+					var realParent = seriesParents.Single(p => p.IsSeriesParent);
+					realParent.PurchaseDate = parent.PurchaseDate;
+
+					Serilog.Log.Logger.Debug("Completed parent scan for {parent}", parent); 
+					parent = realParent;
+				}
+				else
+				{
+					children = await getEpisodeChildrenAsync(parent);
+					if (!children.Any())
+						return new();
 				}
 
+				//A series parent will always have exactly 1 Series
+				parent.Series = new Series[]
+				{
+					new Series
+					{
+						Asin = parent.Asin,
+						Sequence = "-1",
+						Title = parent.TitleWithSubtitle
+					}
+				};
+
 				foreach (var child in children)
 				{
 					// use parent's 'DateAdded'. DateAdded is just a convenience prop for: PurchaseDate.UtcDateTime
@@ -199,17 +251,10 @@ private async Task<List<Item>> getChildEpisodesAsync(SemaphoreSlim concurrencySe
 							Title = parent.TitleWithSubtitle
 						}
 					};
-					// overload (read: abuse) IsEpisodes flag
-					child.Relationships = new Relationship[]
-					{
-						new Relationship
-						{
-							RelationshipToProduct = RelationshipToProduct.Child,
-							RelationshipType = RelationshipType.Episode
-						}
-					};
 				}
 
+				children.Add(parent);
+
 				Serilog.Log.Logger.Debug("Completed episode scan for {parent}", parent);
 
 				return children;

Source/AudibleUtilities/AudibleUtilities.csproj

@@ -5,7 +5,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="AudibleApi" Version="3.0.2.1" />
+    <PackageReference Include="AudibleApi" Version="3.1.0.1" />
   </ItemGroup>
 
   <ItemGroup>

Source/DataLayer/EfClasses/Book.cs

@@ -16,8 +16,14 @@ public AudibleProductId(string id)
         }
     }
 
-    // enum will be easier than bool to extend later
-    public enum ContentType { Unknown = 0, Product = 1, Episode = 2 }
+    // enum will be easier than bool to extend later. 
+    public enum ContentType 
+    { 
+        Unknown = 0,
+        Product = 1,
+        Episode = 2,
+        Parent = 4,
+    }
 
     public class Book
     {

Source/DataLayer/QueryObjects/BookQueries.cs

@@ -35,5 +35,15 @@ public static IQueryable<Book> GetBooks(this IQueryable<Book> books)
                 .Include(b => b.SeriesLink).ThenInclude(sb => sb.Series)
                 .Include(b => b.ContributorsLink).ThenInclude(c => c.Contributor)
                 .Include(b => b.Category).ThenInclude(c => c.ParentCategory);
+
+        public static bool IsProduct(this Book book)
+            => book.ContentType is not ContentType.Episode and not ContentType.Parent;
+
+        public static bool IsEpisodeChild(this Book book)
+            => book.ContentType is ContentType.Episode;
+
+        public static bool IsEpisodeParent(this Book book)
+            => book.ContentType is ContentType.Parent;
+
     }
 }

Source/DataLayer/QueryObjects/LibraryBookQueries.cs

@@ -15,11 +15,13 @@ public static class LibraryBookQueries
 		//		.GetLibrary()
 		//		.ToList();
 
-		public static List<LibraryBook> GetLibrary_Flat_NoTracking(this LibationContext context)
+		public static List<LibraryBook> GetLibrary_Flat_NoTracking(this LibationContext context, bool includeParents = false)
             => context
                 .LibraryBooks
                 .AsNoTrackingWithIdentityResolution()
                 .GetLibrary()
+                .AsEnumerable()
+                .Where(lb => !lb.Book.IsEpisodeParent() || includeParents)
                 .ToList();
 
         public static LibraryBook GetLibraryBook_Flat_NoTracking(this LibationContext context, string productId)
@@ -40,5 +42,32 @@ public static IQueryable<LibraryBook> GetLibrary(this IQueryable<LibraryBook> li
                 .Include(le => le.Book).ThenInclude(b => b.SeriesLink).ThenInclude(sb => sb.Series)
                 .Include(le => le.Book).ThenInclude(b => b.ContributorsLink).ThenInclude(c => c.Contributor)
                 .Include(le => le.Book).ThenInclude(b => b.Category).ThenInclude(c => c.ParentCategory);
+
+#nullable enable
+        public static LibraryBook? FindSeriesParent(this IEnumerable<LibraryBook> libraryBooks, LibraryBook seriesEpisode)
+        {
+            if (seriesEpisode.Book.SeriesLink is null) return null;
+
+            //Parent books will always have exactly 1 SeriesBook due to how
+            //they are imported in ApiExtended.getChildEpisodesAsync()
+            return libraryBooks.FirstOrDefault(
+                lb =>
+                lb.Book.IsEpisodeParent() &&
+                seriesEpisode.Book.SeriesLink.Any(
+                    s => s.Series.AudibleSeriesId == lb.Book.SeriesLink.Single().Series.AudibleSeriesId));
+        }
+#nullable disable
+
+        public static IEnumerable<LibraryBook> FindChildren(this IEnumerable<LibraryBook> bookList, LibraryBook parent)
+            => bookList
+            .Where(
+                lb => 
+                lb.Book.IsEpisodeChild() && 
+                lb.Book.SeriesLink?
+                    .Any(
+                        s => 
+                        s.Series.AudibleSeriesId == parent.Book.AudibleProductId
+                        ) == true
+                    ).ToList();
     }
 }

Source/DtoImporterService/BookImporter.cs

@@ -75,7 +75,7 @@ private Book createNewBook(ImportItem importItem)
 		{
 			var item = importItem.DtoItem;
 
-			var contentType = item.IsEpisodes ? DataLayer.ContentType.Episode : DataLayer.ContentType.Product;
+			var contentType = GetContentType(item);
 
 			// absence of authors is very rare, but possible
 			if (!item.Authors?.Any() ?? true)
@@ -184,5 +184,15 @@ private void updateBook(ImportItem importItem, Book book)
 				}
 			}
 		}
+
+		private static DataLayer.ContentType GetContentType(Item item)
+		{
+			if (item.IsEpisodes)
+				return DataLayer.ContentType.Episode;
+			else if (item.IsSeriesParent)
+				return DataLayer.ContentType.Parent;
+			else 
+				return DataLayer.ContentType.Product;			
+		}
 	}
 }

Source/FileLiberator/Processable.cs

@@ -29,7 +29,7 @@ public abstract class Processable : Streamable
         public IEnumerable<LibraryBook> GetValidLibraryBooks(IEnumerable<LibraryBook> library)
             => library.Where(libraryBook =>
                 Validate(libraryBook)
-                && (libraryBook.Book.ContentType != ContentType.Episode || LibationFileManager.Configuration.Instance.DownloadEpisodes)
+                && (!libraryBook.Book.IsEpisodeChild() || Configuration.Instance.DownloadEpisodes)
                 );
 
         public async Task<StatusHandler> ProcessSingleAsync(LibraryBook libraryBook, bool validate)

Source/LibationSearchEngine/SearchEngine.cs

@@ -121,12 +121,12 @@ public class SearchEngine
                     ["Liberated"] = lb => isLiberated(lb.Book),
                     ["LiberatedError"] = lb => liberatedError(lb.Book),
 
-                    ["Podcast"] = lb => lb.Book.ContentType == ContentType.Episode,
-                    ["Podcasts"] = lb => lb.Book.ContentType == ContentType.Episode,
-                    ["IsPodcast"] = lb => lb.Book.ContentType == ContentType.Episode,
-                    ["Episode"] = lb => lb.Book.ContentType == ContentType.Episode,
-                    ["Episodes"] = lb => lb.Book.ContentType == ContentType.Episode,
-                    ["IsEpisode"] = lb => lb.Book.ContentType == ContentType.Episode,
+                    ["Podcast"] = lb => lb.Book.IsEpisodeChild(),
+                    ["Podcasts"] = lb => lb.Book.IsEpisodeChild(),
+                    ["IsPodcast"] = lb => lb.Book.IsEpisodeChild(),
+                    ["Episode"] = lb => lb.Book.IsEpisodeChild(),
+                    ["Episodes"] = lb => lb.Book.IsEpisodeChild(),
+                    ["IsEpisode"] = lb => lb.Book.IsEpisodeChild(),
                 }
                 );