Merge pull request #549 from Mbucari/master

Lots of Bug Fixes and 2 New Features.

Source/ApplicationServices/LibraryCommands.cs

@@ -12,6 +12,7 @@
 using FileManager;
 using LibationFileManager;
 using Newtonsoft.Json.Linq;
+using NPOI.OpenXmlFormats.Spreadsheet;
 using Serilog;
 using static DtoImporterService.PerfLogger;
 
@@ -171,11 +172,64 @@ public static async Task<List<LibraryBook>> FindInactiveBooks(Func<Account, Task
             }
         }
 
-        private static async Task<List<ImportItem>> scanAccountsAsync(Func<Account, Task<ApiExtended>> apiExtendedfunc, Account[] accounts, LibraryOptions libraryOptions)
+        public static async Task<int> ImportSingleToDbAsync(AudibleApi.Common.Item item, string accountId, string localeName)
+        {
+            ArgumentValidator.EnsureNotNull(item, "item");
+            ArgumentValidator.EnsureNotNull(accountId, "accountId");
+            ArgumentValidator.EnsureNotNull(localeName, "localeName");
+
+            var importItem = new ImportItem
+            {
+                DtoItem = item,
+                AccountId = accountId,
+                LocaleName = localeName
+            };
+
+            var importItems = new List<ImportItem> { importItem };
+            var validator = new LibraryValidator();
+            var exceptions = validator.Validate(importItems.Select(i => i.DtoItem));
+
+            if (exceptions?.Any() ?? false)
+            {
+                Log.Logger.Error(new AggregateException(exceptions), "Error validating library book. {@DebugInfo}", new { item, accountId, localeName });
+                return 0;
+            }
+
+            using var context = DbContexts.GetContext();
+
+            var bookImporter = new BookImporter(context);
+            await Task.Run(() => bookImporter.Import(importItems));
+            var book = await Task.Run(() => context.LibraryBooks.FirstOrDefault(lb => lb.Book.AudibleProductId == importItem.DtoItem.ProductId));
+
+            if (book is null)
+            {
+                book = new LibraryBook(bookImporter.Cache[importItem.DtoItem.ProductId], importItem.DtoItem.DateAdded, importItem.AccountId);
+                context.LibraryBooks.Add(book);
+            }
+            else
+            {
+                book.AbsentFromLastScan = false;
+            }
+
+            try
+            {
+                int qtyChanged = await Task.Run(() => SaveContext(context));
+                if (qtyChanged > 0)
+                    await Task.Run(finalizeLibrarySizeChange);
+                return qtyChanged;
+            }
+            catch (Exception ex)
+            {
+                Log.Logger.Error(ex, "Error adding single library book to DB. {@DebugInfo}", new { item, accountId, localeName });
+                return 0;
+            }
+        }
+
+		private static async Task<List<ImportItem>> scanAccountsAsync(Func<Account, Task<ApiExtended>> apiExtendedfunc, Account[] accounts, LibraryOptions libraryOptions)
         {
             var tasks = new List<Task<List<ImportItem>>>();
 
-            await using LogArchiver archiver
+           await using LogArchiver archiver
                 = Log.Logger.IsDebugEnabled()
                 ? new LogArchiver(System.IO.Path.Combine(Configuration.Instance.LibationFiles, "LibraryScans.zip"))
                 : default;
@@ -184,16 +238,24 @@ private static async Task<List<ImportItem>> scanAccountsAsync(Func<Account, Task
 
 			foreach (var account in accounts)
             {
-                // get APIs in serial b/c of logins. do NOT move inside of parallel (Task.WhenAll)
-                var apiExtended = await apiExtendedfunc(account);
+                try
+                {
+                    // get APIs in serial b/c of logins. do NOT move inside of parallel (Task.WhenAll)
+                    var apiExtended = await apiExtendedfunc(account);
 
-                // add scanAccountAsync as a TASK: do not await
-                tasks.Add(scanAccountAsync(apiExtended, account, libraryOptions, archiver));
+                    // add scanAccountAsync as a TASK: do not await
+                    tasks.Add(scanAccountAsync(apiExtended, account, libraryOptions, archiver));
+                }
+                catch(Exception ex)
+                {
+                    //Catch to allow other accounts to continue scanning.
+                    Log.Logger.Error(ex, "Failed to scan account");
+                }
             }
 
             // import library in parallel
-            var arrayOfLists = await Task.WhenAll(tasks);
-            var importItems = arrayOfLists.SelectMany(a => a).ToList();
+            var arrayOfLists = await Task.WhenAll(tasks);            
+			var importItems = arrayOfLists.SelectMany(a => a).ToList();
             return importItems;
         }
 
@@ -208,26 +270,43 @@ private static async Task<List<ImportItem>> scanAccountAsync(ApiExtended apiExte
 
             logTime($"pre scanAccountAsync {account.AccountName}");
 
-            var dtoItems = await apiExtended.GetLibraryValidatedAsync(libraryOptions, Configuration.Instance.ImportEpisodes);
+            try
+            {
+                var dtoItems = await apiExtended.GetLibraryValidatedAsync(libraryOptions, Configuration.Instance.ImportEpisodes);
 
-            if (archiver is not null)
+                logTime($"post scanAccountAsync {account.AccountName} qty: {dtoItems.Count}");
+
+                await logDtoItemsAsync(dtoItems);
+
+				return dtoItems.Select(d => new ImportItem { DtoItem = d, AccountId = account.AccountId, LocaleName = account.Locale?.Name }).ToList();
+            }
+            catch(ImportValidationException ex)
+			{
+				await logDtoItemsAsync(ex.Items, ex.InnerExceptions.ToArray());
+				throw;
+            }
+
+            async Task logDtoItemsAsync(IEnumerable<AudibleApi.Common.Item> dtoItems, IEnumerable<Exception> exceptions = null)
             {
-                var fileName = $"{DateTime.Now:u} {account.MaskedLogEntry}.json";
-                var items = await Task.Run(() => JArray.FromObject(dtoItems.Select(i => i.SourceJson)));
+				if (archiver is not null)
+				{
+					var fileName = $"{DateTime.Now:u} {account.MaskedLogEntry}.json";
+					var items = await Task.Run(() => JArray.FromObject(dtoItems.Select(i => i.SourceJson)));
 
-				var scanFile = new JObject
-                {
-                    { "Account", account.MaskedLogEntry },
-                    { "ScannedDateTime", DateTime.Now.ToString("u") },
-                    { "Items",  items}
-                };
+					var scanFile = new JObject
+				    {
+					    { "Account", account.MaskedLogEntry },
+					    { "ScannedDateTime", DateTime.Now.ToString("u") },
+				    };
 
-				await archiver.AddFileAsync(fileName, scanFile);
-			}
+					if (exceptions?.Any() is true)
+                        scanFile.Add("Exceptions", JArray.FromObject(exceptions));
 
-            logTime($"post scanAccountAsync {account.AccountName} qty: {dtoItems.Count}");
+					scanFile.Add("Items", items);
 
-            return dtoItems.Select(d => new ImportItem { DtoItem = d, AccountId = account.AccountId, LocaleName = account.Locale?.Name }).ToList();
+					await archiver.AddFileAsync(fileName, scanFile);
+				}
+			}
         }
 
         private static async Task<int> importIntoDbAsync(List<ImportItem> importItems)

Source/AudibleUtilities/ApiExtended.cs

@@ -149,33 +149,19 @@ private async Task<List<Item>> getItemsAsync(LibraryOptions libraryOptions, bool
 			foreach (var parent in items.Where(i => i.IsSeriesParent))
 			{
 				var children = items.Where(i => i.IsEpisodes && i.Relationships.Any(r => r.Asin == parent.Asin));
-				setSeries(parent, children);
+				SetSeries(parent, children);
 			}
 
 			sw.Stop();
 			totalTime += sw.Elapsed;
 			Serilog.Log.Logger.Information("Completed indexing series episodes after {elappsed_ms} ms.", sw.ElapsedMilliseconds);
 			Serilog.Log.Logger.Information($"Completed library scan in {totalTime.TotalMilliseconds:F0} ms.");
 
-			var validators = new List<IValidator>();
-			validators.AddRange(getValidators());
-			foreach (var v in validators)
-			{
-				var exceptions = v.Validate(items);
-				if (exceptions is not null && exceptions.Any())
-					throw new AggregateException(exceptions);
-			}
-			return items;
-		}
+			var allExceptions = IValidator.GetAllValidators().SelectMany(v => v.Validate(items));
+			if (allExceptions?.Any() is true)
+				throw new ImportValidationException(items, allExceptions);
 
-		private static List<IValidator> getValidators()
-		{
-			var type = typeof(IValidator);
-			var types = AppDomain.CurrentDomain.GetAssemblies()
-				.SelectMany(s => s.GetTypes())
-				.Where(p => type.IsAssignableFrom(p) && !p.IsInterface);
-
-			return types.Select(t => Activator.CreateInstance(t) as IValidator).ToList();
+			return items;
 		}
 
 		#region episodes and podcasts
@@ -232,8 +218,11 @@ private async Task<List<Item>> getProductsAsync(int batchNum, List<string> asins
 			finally { semaphore.Release(); }
 		}
 
-		private static void setSeries(Item parent, IEnumerable<Item> children)
+		public static void SetSeries(Item parent, IEnumerable<Item> children)
 		{
+			ArgumentValidator.EnsureNotNull(parent, nameof(parent));
+			ArgumentValidator.EnsureNotNull(children, nameof(children));
+
 			//A series parent will always have exactly 1 Series
 			parent.Series = new[]
 			{
@@ -246,7 +235,15 @@ private static void setSeries(Item parent, IEnumerable<Item> children)
 			};
 
 			if (parent.PurchaseDate == default)
-				parent.PurchaseDate = children.Select(c => c.PurchaseDate).Order().First();
+			{
+				parent.PurchaseDate = children.Select(c => c.PurchaseDate).Order().FirstOrDefault(d => d != default);
+
+				if (parent.PurchaseDate == default)
+				{
+					Serilog.Log.Logger.Warning("{series} doesn't have a purchase date. Using UtcNow", parent);
+					parent.PurchaseDate = DateTimeOffset.UtcNow;
+				}
+			}
 
 			foreach (var child in children)
 			{
@@ -267,4 +264,4 @@ private static void setSeries(Item parent, IEnumerable<Item> children)
 		}
 		#endregion
 	}
-}
+}

Source/AudibleUtilities/AudibleApiValidators.cs

@@ -8,7 +8,18 @@ namespace AudibleUtilities
 	public interface IValidator
 	{
 		IEnumerable<Exception> Validate(IEnumerable<Item> items);
+
+		public static IValidator[] GetAllValidators()
+			=> new IValidator[]
+			{
+				new LibraryValidator(),
+				new BookValidator(),
+				new CategoryValidator(),
+				new ContributorValidator(),
+				new SeriesValidator(),
+			};
 	}
+
 	public class LibraryValidator : IValidator
 	{
 		public IEnumerable<Exception> Validate(IEnumerable<Item> items)

Source/AudibleUtilities/ImportValidationException.cs

@@ -0,0 +1,15 @@
+using AudibleApi.Common;
+using System;
+using System.Collections.Generic;
+
+namespace AudibleUtilities
+{
+	public class ImportValidationException : AggregateException
+	{
+		public List<Item> Items { get; }
+		public ImportValidationException(List<Item> items, IEnumerable<Exception> exceptions) : base(exceptions)
+		{
+			Items = items;
+		}
+	}
+}

Source/DataLayer/EfClasses/LibraryBook.cs

@@ -25,6 +25,8 @@ public LibraryBook(Book book, DateTime dateAdded, string account)
             Account = account;
         }
 
-        public override string ToString() => $"{DateAdded:d} {Book}";
+        public void SetAccount(string account) => Account = account;
+
+		public override string ToString() => $"{DateAdded:d} {Book}";
 	}
 }

Source/DataLayer/EntityExtensions.cs

@@ -18,17 +18,17 @@ public static class EntityExtensions
         /// <summary>True if exists and IsLiberated. Else false</summary>
         public static bool PDF_Exists(this Book book) => book.UserDefinedItem.PdfStatus == LiberatedStatus.Liberated;
 
-        public static string SeriesSortable(this Book book) => Formatters.GetSortName(book.SeriesNames());
+        public static string SeriesSortable(this Book book) => Formatters.GetSortName(book.SeriesNames(true));
 		public static bool HasPdf(this Book book) => book.Supplements.Any();
-        public static string SeriesNames(this Book book)
+        public static string SeriesNames(this Book book, bool includeIndex = false)
         {
             if (book.SeriesLink is null)
                 return "";
 
             // first: alphabetical by name
             var withNames = book.SeriesLink
                 .Where(s => !string.IsNullOrWhiteSpace(s.Series.Name))
-                .Select(s => s.Series.Name)
+                .Select(getSeriesNameString)
                 .OrderBy(a => a)
                 .ToList();
             // then un-named are alpha by series id
@@ -40,7 +40,12 @@ public static string SeriesNames(this Book book)
 
             var all = withNames.Union(nullNames).ToList();
             return string.Join(", ", all);
-        }
+
+            string getSeriesNameString(SeriesBook sb)
+                => includeIndex && !string.IsNullOrWhiteSpace(sb.Order) && sb.Order != "-1"
+                ? $"{sb.Series.Name} (#{sb.Order})"
+                : sb.Series.Name;
+		}
         public static string[] CategoriesNames(this Book book)
             => book.Category is null ? new string[0]
             : book.Category.ParentCategory is null ? new[] { book.Category.Name }