diff --git a/aspnetcore/src/api/Controllers/OrcidController.cs b/aspnetcore/src/api/Controllers/OrcidController.cs index 72ef439a..5283003c 100644 --- a/aspnetcore/src/api/Controllers/OrcidController.cs +++ b/aspnetcore/src/api/Controllers/OrcidController.cs @@ -1,6 +1,6 @@ -using api.Services; -using api.Models.Api; -using api.Models.Orcid; +using api.Services; +using api.Models.Api; +using api.Models.Orcid; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using System.Threading.Tasks; @@ -209,7 +209,7 @@ public async Task Get() action: LogContent.Action.ORCID_RECORD_IMPORT, state: LogContent.ActionState.START)); - await _orcidImportService.ImportOrcidRecordJsonIntoUserProfile(userprofileId, orcidRecordJson); + await _orcidImportService.ImportOrcidRecordJsonIntoUserProfile(userprofileId, orcidRecordJson, logUserIdentification); importSuccess = true; _logger.LogInformation( @@ -264,7 +264,7 @@ await _taskQueue.QueueBackgroundWorkItemAsync(async token => .Include(ffv => ffv.DimProfileOnlyFundingDecision) .Include(ffv => ffv.DimPidIdOrcidPutCodeNavigation).ToListAsync(); - await _orcidImportService.ImportAdditionalData(factFieldValues: ffvs, orcidAccessToken: orcidTokens.AccessToken, useOrcidPublicApi: useOrcidPublicApi); + await _orcidImportService.ImportAdditionalData(factFieldValues: ffvs, orcidAccessToken: orcidTokens.AccessToken, logUserIdentification: logUserIdentification, useOrcidPublicApi: useOrcidPublicApi); await localTtvContext.SaveChangesAsync(); _logger.LogInformation( diff --git a/aspnetcore/src/api/Controllers/WebhookController.cs b/aspnetcore/src/api/Controllers/WebhookController.cs index d75cc98e..b462291a 100644 --- a/aspnetcore/src/api/Controllers/WebhookController.cs +++ b/aspnetcore/src/api/Controllers/WebhookController.cs @@ -159,7 +159,7 @@ await _taskQueue.QueueBackgroundWorkItemAsync(async token => action: LogContent.Action.ORCID_RECORD_IMPORT, state: LogContent.ActionState.START)); - importSuccess = await localOrcidImportService.ImportOrcidRecordJsonIntoUserProfile(dimUserprofileId, orcidRecordJson); + importSuccess = await localOrcidImportService.ImportOrcidRecordJsonIntoUserProfile(dimUserprofileId, orcidRecordJson, logUserIdentification); _logger.LogInformation( LogContent.MESSAGE_TEMPLATE, @@ -202,7 +202,7 @@ await _taskQueue.QueueBackgroundWorkItemAsync(async token => .Include(ffv => ffv.DimProfileOnlyFundingDecision) .Include(ffv => ffv.DimPidIdOrcidPutCodeNavigation).ToListAsync(); - await localOrcidImportService.ImportAdditionalData(factFieldValues: ffvs, orcidAccessToken: orcidAccessToken); + await localOrcidImportService.ImportAdditionalData(factFieldValues: ffvs, orcidAccessToken: orcidAccessToken, logUserIdentification: logUserIdentification); await localTtvContext.SaveChangesAsync(); _logger.LogInformation( diff --git a/aspnetcore/src/api/Models/StructuredLog/LogContent.cs b/aspnetcore/src/api/Models/StructuredLog/LogContent.cs index 99baa2a4..6a16c57a 100644 --- a/aspnetcore/src/api/Models/StructuredLog/LogContent.cs +++ b/aspnetcore/src/api/Models/StructuredLog/LogContent.cs @@ -35,6 +35,7 @@ public static class Action public const string ORCID_RECORD_GET_PUBLIC_API = "ORCID: record: get from public API"; public const string ORCID_RECORD_IMPORT = "ORCID: record: import"; public const string ORCID_RECORD_IMPORT_ADDITIONAL = "ORCID: record: import additional"; + public const string ORCID_RECORD_PARSE = "ORCID: record: parse"; public const string ORCID_REVOKE_TOKEN = "ORCID: revoke token"; public const string ORCID_WEBHOOK_REGISTER = "ORCID: webhook: register"; public const string ORCID_WEBHOOK_UNREGISTER = "ORCID: webhook: unregister"; diff --git a/aspnetcore/src/api/Models/Ttv/DimPublication.cs b/aspnetcore/src/api/Models/Ttv/DimPublication.cs index d9ab8db2..8d394452 100644 --- a/aspnetcore/src/api/Models/Ttv/DimPublication.cs +++ b/aspnetcore/src/api/Models/Ttv/DimPublication.cs @@ -29,10 +29,6 @@ public partial class DimPublication public string Isbn2 { get; set; } - public string JufoCode { get; set; } - - public string JufoClassCode { get; set; } - public int PublicationCountryCode { get; set; } public string JournalName { get; set; } @@ -69,8 +65,6 @@ public partial class DimPublication public int LanguageCode { get; set; } - public string OpenAccessCode { get; set; } - public bool SpecialStateSubsidy { get; set; } public bool? BusinessCollaboration { get; set; } @@ -119,16 +113,22 @@ public partial class DimPublication public string OpenAccess { get; set; } - public string PublisherOpenAccessCode { get; set; } + public int PublisherOpenAccessCode { get; set; } public string Abstract { get; set; } + public int DimPublicationChannelId { get; set; } + + public int JufoClass { get; set; } + public virtual DimReferencedatum ArticleTypeCodeNavigation { get; set; } public virtual ICollection DimLocallyReportedPubInfos { get; set; } = new List(); public virtual ICollection DimPids { get; set; } = new List(); + public virtual DimPublicationChannel DimPublicationChannel { get; set; } + public virtual DimRegisteredDataSource DimRegisteredDataSource { get; set; } public virtual ICollection FactContributions { get; set; } = new List(); @@ -137,6 +137,8 @@ public partial class DimPublication public virtual ICollection FactFieldValues { get; set; } = new List(); + public virtual DimReferencedatum JufoClassNavigation { get; set; } + public virtual DimReferencedatum LanguageCodeNavigation { get; set; } public virtual DimReferencedatum LicenseCodeNavigation { get; set; } @@ -149,6 +151,8 @@ public partial class DimPublication public virtual DimReferencedatum PublicationTypeCodeNavigation { get; set; } + public virtual DimReferencedatum PublisherOpenAccessCodeNavigation { get; set; } + public virtual DimReferencedatum TargetAudienceCodeNavigation { get; set; } public virtual DimReferencedatum ThesisTypeCodeNavigation { get; set; } diff --git a/aspnetcore/src/api/Models/Ttv/DimPublicationChannel.cs b/aspnetcore/src/api/Models/Ttv/DimPublicationChannel.cs index 2b455cb7..204b82f9 100644 --- a/aspnetcore/src/api/Models/Ttv/DimPublicationChannel.cs +++ b/aspnetcore/src/api/Models/Ttv/DimPublicationChannel.cs @@ -13,9 +13,17 @@ public partial class DimPublicationChannel public string PublisherNameText { get; set; } + public string SourceId { get; set; } + + public string SourceDescription { get; set; } + + public DateTime? Created { get; set; } + + public DateTime? Modified { get; set; } + public virtual ICollection DimPids { get; set; } = new List(); - public virtual ICollection DimResearchActivities { get; set; } = new List(); + public virtual ICollection DimPublications { get; set; } = new List(); - public virtual ICollection FactJufoClassCodesForPubChannels { get; set; } = new List(); + public virtual ICollection DimResearchActivities { get; set; } = new List(); } diff --git a/aspnetcore/src/api/Models/Ttv/DimReferencedatum.cs b/aspnetcore/src/api/Models/Ttv/DimReferencedatum.cs index c7318ec1..a9491ee7 100644 --- a/aspnetcore/src/api/Models/Ttv/DimReferencedatum.cs +++ b/aspnetcore/src/api/Models/Ttv/DimReferencedatum.cs @@ -63,6 +63,8 @@ public partial class DimReferencedatum public virtual ICollection DimPublicationArticleTypeCodeNavigations { get; set; } = new List(); + public virtual ICollection DimPublicationJufoClassNavigations { get; set; } = new List(); + public virtual ICollection DimPublicationLanguageCodeNavigations { get; set; } = new List(); public virtual ICollection DimPublicationLicenseCodeNavigations { get; set; } = new List(); @@ -75,6 +77,8 @@ public partial class DimReferencedatum public virtual ICollection DimPublicationPublicationTypeCodeNavigations { get; set; } = new List(); + public virtual ICollection DimPublicationPublisherOpenAccessCodeNavigations { get; set; } = new List(); + public virtual ICollection DimPublicationTargetAudienceCodeNavigations { get; set; } = new List(); public virtual ICollection DimPublicationThesisTypeCodeNavigations { get; set; } = new List(); @@ -95,8 +99,6 @@ public partial class DimReferencedatum public virtual ICollection FactFieldValueDimReferencedataFieldOfSciences { get; set; } = new List(); - public virtual ICollection FactJufoClassCodesForPubChannels { get; set; } = new List(); - public virtual ICollection InverseDimReferencedata { get; set; } = new List(); public virtual ICollection DimCallProgrammes { get; set; } = new List(); diff --git a/aspnetcore/src/api/Models/Ttv/FactJufoClassCodesForPubChannel.cs b/aspnetcore/src/api/Models/Ttv/FactJufoClassCodesForPubChannel.cs index 704d4916..a4ab2f58 100644 --- a/aspnetcore/src/api/Models/Ttv/FactJufoClassCodesForPubChannel.cs +++ b/aspnetcore/src/api/Models/Ttv/FactJufoClassCodesForPubChannel.cs @@ -7,11 +7,19 @@ public partial class FactJufoClassCodesForPubChannel { public int DimPublicationChannelId { get; set; } - public int DimReferencedataId { get; set; } + public int JufoClasses { get; set; } public int Year { get; set; } + public string SourceId { get; set; } + + public string SourceDescription { get; set; } + + public DateTime? Created { get; set; } + + public DateTime? Modified { get; set; } + public virtual DimPublicationChannel DimPublicationChannel { get; set; } - public virtual DimReferencedatum DimReferencedata { get; set; } + public virtual DimReferencedatum JufoClassesNavigation { get; set; } } diff --git a/aspnetcore/src/api/Models/Ttv/TtvContext.cs b/aspnetcore/src/api/Models/Ttv/TtvContext.cs index fa42bfab..da572e3b 100644 --- a/aspnetcore/src/api/Models/Ttv/TtvContext.cs +++ b/aspnetcore/src/api/Models/Ttv/TtvContext.cs @@ -137,8 +137,6 @@ public TtvContext(DbContextOptions options) public virtual DbSet FactInfraKeywords { get; set; } - public virtual DbSet FactJufoClassCodesForPubChannels { get; set; } - public virtual DbSet FactUpkeeps { get; set; } protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) @@ -2430,6 +2428,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) entity.Property(e => e.Created) .HasColumnType("datetime") .HasColumnName("created"); + entity.Property(e => e.DimPublicationChannelId).HasColumnName("dim_publication_channel_id"); entity.Property(e => e.DimRegisteredDataSourceId).HasColumnName("dim_registered_data_source_id"); entity.Property(e => e.Doi) .HasMaxLength(4000) @@ -2459,12 +2458,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) entity.Property(e => e.JournalName) .HasMaxLength(4000) .HasColumnName("journal_name"); - entity.Property(e => e.JufoClassCode) - .HasMaxLength(255) - .HasColumnName("jufo_class_code"); - entity.Property(e => e.JufoCode) - .HasMaxLength(255) - .HasColumnName("jufo_code"); + entity.Property(e => e.JufoClass).HasColumnName("jufo_class"); entity.Property(e => e.JuuliAddress) .HasMaxLength(4000) .HasColumnName("juuli_address"); @@ -2477,9 +2471,6 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) entity.Property(e => e.OpenAccess) .HasMaxLength(255) .HasColumnName("open_access"); - entity.Property(e => e.OpenAccessCode) - .HasMaxLength(255) - .HasColumnName("open_access_code"); entity.Property(e => e.OriginalPublicationId) .HasMaxLength(255) .HasColumnName("original_publication_id"); @@ -2520,9 +2511,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) entity.Property(e => e.PublisherName) .HasMaxLength(4000) .HasColumnName("publisher_name"); - entity.Property(e => e.PublisherOpenAccessCode) - .HasMaxLength(255) - .HasColumnName("publisher_open_access_code"); + entity.Property(e => e.PublisherOpenAccessCode).HasColumnName("publisher_open_access_code"); entity.Property(e => e.Report).HasColumnName("report"); entity.Property(e => e.ReportingYear).HasColumnName("reporting_year"); entity.Property(e => e.SelfArchivedCode).HasColumnName("self_archived_code"); @@ -2544,11 +2533,21 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .HasForeignKey(d => d.ArticleTypeCode) .HasConstraintName("article_type_code"); + entity.HasOne(d => d.DimPublicationChannel).WithMany(p => p.DimPublications) + .HasForeignKey(d => d.DimPublicationChannelId) + .OnDelete(DeleteBehavior.ClientSetNull) + .HasConstraintName("publication_channel"); + entity.HasOne(d => d.DimRegisteredDataSource).WithMany(p => p.DimPublications) .HasForeignKey(d => d.DimRegisteredDataSourceId) .OnDelete(DeleteBehavior.ClientSetNull) .HasConstraintName("FKdim_public896887"); + entity.HasOne(d => d.JufoClassNavigation).WithMany(p => p.DimPublicationJufoClassNavigations) + .HasForeignKey(d => d.JufoClass) + .OnDelete(DeleteBehavior.ClientSetNull) + .HasConstraintName("jufo_class"); + entity.HasOne(d => d.LanguageCodeNavigation).WithMany(p => p.DimPublicationLanguageCodeNavigations) .HasForeignKey(d => d.LanguageCode) .OnDelete(DeleteBehavior.ClientSetNull) @@ -2577,6 +2576,11 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .HasForeignKey(d => d.PublicationTypeCode2) .HasConstraintName("publication_type_code2"); + entity.HasOne(d => d.PublisherOpenAccessCodeNavigation).WithMany(p => p.DimPublicationPublisherOpenAccessCodeNavigations) + .HasForeignKey(d => d.PublisherOpenAccessCode) + .OnDelete(DeleteBehavior.ClientSetNull) + .HasConstraintName("publisher_open_access"); + entity.HasOne(d => d.TargetAudienceCodeNavigation).WithMany(p => p.DimPublicationTargetAudienceCodeNavigations) .HasForeignKey(d => d.TargetAudienceCode) .HasConstraintName("target_audience_code"); @@ -2616,12 +2620,25 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) entity.Property(e => e.ChannelNameAnylang) .HasMaxLength(4000) .HasColumnName("channel_name_anylang"); + entity.Property(e => e.Created) + .HasColumnType("datetime") + .HasColumnName("created"); entity.Property(e => e.JufoCode) .HasMaxLength(255) .HasColumnName("jufo_code"); + entity.Property(e => e.Modified) + .HasColumnType("datetime") + .HasColumnName("modified"); entity.Property(e => e.PublisherNameText) .HasMaxLength(4000) .HasColumnName("publisher_name_text"); + entity.Property(e => e.SourceDescription) + .HasMaxLength(255) + .HasColumnName("source_description"); + entity.Property(e => e.SourceId) + .IsRequired() + .HasMaxLength(255) + .HasColumnName("source_id"); }); modelBuilder.Entity(entity => @@ -4020,27 +4037,6 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .HasConstraintName("FKfact_infra619117"); }); - modelBuilder.Entity(entity => - { - entity.HasKey(e => new { e.DimPublicationChannelId, e.DimReferencedataId, e.Year }).HasName("PK__fact_juf__0E099E4B39813744"); - - entity.ToTable("fact_jufo_class_codes_for_pub_channels"); - - entity.Property(e => e.DimPublicationChannelId).HasColumnName("dim_publication_channel_id"); - entity.Property(e => e.DimReferencedataId).HasColumnName("dim_referencedata_id"); - entity.Property(e => e.Year).HasColumnName("year"); - - entity.HasOne(d => d.DimPublicationChannel).WithMany(p => p.FactJufoClassCodesForPubChannels) - .HasForeignKey(d => d.DimPublicationChannelId) - .OnDelete(DeleteBehavior.ClientSetNull) - .HasConstraintName("jufo_classes"); - - entity.HasOne(d => d.DimReferencedata).WithMany(p => p.FactJufoClassCodesForPubChannels) - .HasForeignKey(d => d.DimReferencedataId) - .OnDelete(DeleteBehavior.ClientSetNull) - .HasConstraintName("FKfact_jufo_876058"); - }); - modelBuilder.Entity(entity => { entity.HasKey(e => new { e.DimOrganizationId, e.DimGeoId, e.DimInfrastructureId, e.DimServiceId, e.DimServicePointId, e.DimDateIdStart, e.DimDateIdEnd }).HasName("PK__fact_upk__850A8E30119C2EB4"); diff --git a/aspnetcore/src/api/Services/IOrcidImportService.cs b/aspnetcore/src/api/Services/IOrcidImportService.cs index 213fadba..af636c7f 100644 --- a/aspnetcore/src/api/Services/IOrcidImportService.cs +++ b/aspnetcore/src/api/Services/IOrcidImportService.cs @@ -1,13 +1,14 @@ -using System; +using System; using System.Collections.Generic; using System.Threading.Tasks; +using api.Models.Log; using api.Models.Ttv; namespace api.Services { public interface IOrcidImportService { - Task ImportOrcidRecordJsonIntoUserProfile(int userprofileId, string json); - Task ImportAdditionalData(List factFieldValues, String orcidAccessToken, bool useOrcidPublicApi = false); + Task ImportOrcidRecordJsonIntoUserProfile(int userprofileId, string json, LogUserIdentification logUserIdentification); + Task ImportAdditionalData(List factFieldValues, String orcidAccessToken, LogUserIdentification logUserIdentification, bool useOrcidPublicApi = false); } } \ No newline at end of file diff --git a/aspnetcore/src/api/Services/OrcidImportService.cs b/aspnetcore/src/api/Services/OrcidImportService.cs index 2fd7aa2e..0df82d9b 100644 --- a/aspnetcore/src/api/Services/OrcidImportService.cs +++ b/aspnetcore/src/api/Services/OrcidImportService.cs @@ -1,20 +1,12 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Net.Mail; using System.Threading.Tasks; -using api.Controllers; -using api.Models.Common; +using api.Models.Log; using api.Models.Orcid; using api.Models.Ttv; using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.ChangeTracking; -using Microsoft.EntityFrameworkCore.Storage; using Microsoft.Extensions.Logging; -using Nest; -using Serilog.Core; -using static Microsoft.EntityFrameworkCore.DbLoggerCategory; -using static Microsoft.Extensions.Logging.EventSource.LoggingEventSource; using Constants = api.Models.Common.Constants; namespace api.Services @@ -31,10 +23,11 @@ public class OrcidImportService : IOrcidImportService private readonly IOrganizationHandlerService _organizationHandlerService; private readonly IDataSourceHelperService _dataSourceHelperService; private readonly IUtilityService _utilityService; + private readonly ILogger _logger; public OrcidImportService( TtvContext ttvContext, IUserProfileService userProfileService, IOrcidApiService orcidApiService, IOrcidJsonParserService orcidJsonParserService, - IOrganizationHandlerService organizationHandlerService, IUtilityService utilityService, IDataSourceHelperService dataSourceHelperService) + IOrganizationHandlerService organizationHandlerService, IUtilityService utilityService, IDataSourceHelperService dataSourceHelperService, ILogger logger) { _ttvContext = ttvContext; _userProfileService = userProfileService; @@ -43,16 +36,32 @@ public OrcidImportService( _organizationHandlerService = organizationHandlerService; _utilityService = utilityService; _dataSourceHelperService = dataSourceHelperService; + _logger = logger; } /* * Add DimDates entities needed in ORCID record. */ - public async Task AddDimDates(string orcidRecordJson, DateTime currentDateTime) + public async Task AddDimDates(string orcidRecordJson, DateTime currentDateTime, LogUserIdentification logUserIdentification) { // Education DimDates - List educations = _orcidJsonParserService.GetEducations(orcidRecordJson); + List educations = new(); + try + { + educations = _orcidJsonParserService.GetEducations(orcidRecordJson); + } + catch (Exception ex) + { + _logger.LogError( + LogContent.MESSAGE_TEMPLATE, + logUserIdentification, + new LogApiInfo( + action: LogContent.Action.ORCID_RECORD_PARSE, + state: LogContent.ActionState.IN_PROGRESS, + error: true, + message: $"{ex.ToString()}")); + } foreach (OrcidEducation education in educations) { // Start date @@ -101,7 +110,22 @@ await _ttvContext.DimDates.FirstOrDefaultAsync( } // Employment DimDates - List employments = _orcidJsonParserService.GetEmployments(orcidRecordJson); + List employments = new(); + try + { + employments = _orcidJsonParserService.GetEmployments(orcidRecordJson); + } + catch (Exception ex) + { + _logger.LogError( + LogContent.MESSAGE_TEMPLATE, + logUserIdentification, + new LogApiInfo( + action: LogContent.Action.ORCID_RECORD_PARSE, + state: LogContent.ActionState.IN_PROGRESS, + error: true, + message: $"{ex.ToString()}")); + } foreach (OrcidEmployment employment in employments) { // Start date @@ -149,7 +173,22 @@ await _ttvContext.DimDates.FirstOrDefaultAsync( } // Funding DimDates - List fundings = _orcidJsonParserService.GetFundings(orcidRecordJson); + List fundings = new(); + try + { + fundings = _orcidJsonParserService.GetFundings(orcidRecordJson); + } + catch (Exception ex) + { + _logger.LogError( + LogContent.MESSAGE_TEMPLATE, + logUserIdentification, + new LogApiInfo( + action: LogContent.Action.ORCID_RECORD_PARSE, + state: LogContent.ActionState.IN_PROGRESS, + error: true, + message: $"{ex.ToString()}")); + } foreach (OrcidFunding funding in fundings) { // Start data @@ -198,10 +237,40 @@ await _ttvContext.DimDates.FirstOrDefaultAsync( // Research activity DimDates - List orcidResearchActivity_invitedPositionsAndDistinctionsMembershipsServices = - _orcidJsonParserService.GetProfileOnlyResearchActivityItems(orcidRecordJson); - List orcidResearchActivity_works = - _orcidJsonParserService.GetWorks(orcidRecordJson, processOnlyResearchActivities: true).ResearchActivities; + List orcidResearchActivity_invitedPositionsAndDistinctionsMembershipsServices = new(); + try + { + orcidResearchActivity_invitedPositionsAndDistinctionsMembershipsServices = _orcidJsonParserService.GetProfileOnlyResearchActivityItems(orcidRecordJson); + } + catch (Exception ex) + { + _logger.LogError( + LogContent.MESSAGE_TEMPLATE, + logUserIdentification, + new LogApiInfo( + action: LogContent.Action.ORCID_RECORD_PARSE, + state: LogContent.ActionState.IN_PROGRESS, + error: true, + message: $"{ex.ToString()}")); + } + + List orcidResearchActivity_works = new(); + try + { + orcidResearchActivity_works = _orcidJsonParserService.GetWorks(orcidRecordJson, processOnlyResearchActivities: true).ResearchActivities; + } + catch (Exception ex) + { + _logger.LogError( + LogContent.MESSAGE_TEMPLATE, + logUserIdentification, + new LogApiInfo( + action: LogContent.Action.ORCID_RECORD_PARSE, + state: LogContent.ActionState.IN_PROGRESS, + error: true, + message: $"{ex.ToString()}")); + } + foreach (OrcidResearchActivity researchActivity in orcidResearchActivity_invitedPositionsAndDistinctionsMembershipsServices.Concat(orcidResearchActivity_works)) { // Start date @@ -281,7 +350,7 @@ private DimWebLink GetDimWebLink( /* * Import ORCID record json into user profile. */ - public async Task ImportOrcidRecordJsonIntoUserProfile(int userprofileId, string orcidRecordJson) + public async Task ImportOrcidRecordJsonIntoUserProfile(int userprofileId, string orcidRecordJson, LogUserIdentification logUserIdentification) { // Get ORCID registered data source id. int orcidRegisteredDataSourceId = _dataSourceHelperService.DimRegisteredDataSourceId_ORCID; @@ -373,7 +442,7 @@ public async Task ImportOrcidRecordJsonIntoUserProfile(int userprofileId, DateTime currentDateTime = _utilityService.GetCurrentDateTime(); // Add DimDates. - await AddDimDates(orcidRecordJson, currentDateTime); + await AddDimDates(orcidRecordJson, currentDateTime, logUserIdentification); // Helper object to store processed IDs, used when deciding what data needs to be removed. OrcidImportHelper orcidImportHelper = new(); @@ -424,7 +493,22 @@ public async Task ImportOrcidRecordJsonIntoUserProfile(int userprofileId, // Other names - List otherNames = _orcidJsonParserService.GetOtherNames(orcidRecordJson); + List otherNames = new(); + try + { + otherNames = _orcidJsonParserService.GetOtherNames(orcidRecordJson); + } + catch (Exception ex) + { + _logger.LogError( + LogContent.MESSAGE_TEMPLATE, + logUserIdentification, + new LogApiInfo( + action: LogContent.Action.ORCID_RECORD_PARSE, + state: LogContent.ActionState.IN_PROGRESS, + error: true, + message: $"{ex.ToString()}")); + } // Get DimFieldDisplaySettings for other name DimFieldDisplaySetting dimFieldDisplaySettingsOtherName = dimUserProfile.DimFieldDisplaySettings.FirstOrDefault(dfdsOtherName => dfdsOtherName.FieldIdentifier == Constants.FieldIdentifiers.PERSON_OTHER_NAMES); @@ -483,7 +567,22 @@ public async Task ImportOrcidRecordJsonIntoUserProfile(int userprofileId, } // Researcher urls - List researcherUrls = _orcidJsonParserService.GetResearcherUrls(orcidRecordJson); + List researcherUrls = new(); + try + { + researcherUrls = _orcidJsonParserService.GetResearcherUrls(orcidRecordJson); + } + catch (Exception ex) + { + _logger.LogError( + LogContent.MESSAGE_TEMPLATE, + logUserIdentification, + new LogApiInfo( + action: LogContent.Action.ORCID_RECORD_PARSE, + state: LogContent.ActionState.IN_PROGRESS, + error: true, + message: $"{ex.ToString()}")); + } // Get DimFieldDisplaySettings for weblink DimFieldDisplaySetting dimFieldDisplaySettingsWebLink = dimUserProfile.DimFieldDisplaySettings.FirstOrDefault(dfdsWebLink => dfdsWebLink.FieldIdentifier == Constants.FieldIdentifiers.PERSON_WEB_LINK); @@ -539,7 +638,22 @@ public async Task ImportOrcidRecordJsonIntoUserProfile(int userprofileId, // Researcher description - OrcidBiography biography = _orcidJsonParserService.GetBiography(orcidRecordJson); + OrcidBiography biography = null; + try + { + biography = _orcidJsonParserService.GetBiography(orcidRecordJson); + } + catch (Exception ex) + { + _logger.LogError( + LogContent.MESSAGE_TEMPLATE, + logUserIdentification, + new LogApiInfo( + action: LogContent.Action.ORCID_RECORD_PARSE, + state: LogContent.ActionState.IN_PROGRESS, + error: true, + message: $"{ex.ToString()}")); + } // Get DimFieldDisplaySettings for researcher description DimFieldDisplaySetting dimFieldDisplaySettingsResearcherDescription = dimUserProfile.DimFieldDisplaySettings.FirstOrDefault( @@ -591,7 +705,22 @@ public async Task ImportOrcidRecordJsonIntoUserProfile(int userprofileId, // Email - List emails = _orcidJsonParserService.GetEmails(orcidRecordJson); + List emails = new(); + try + { + emails = _orcidJsonParserService.GetEmails(orcidRecordJson); + } + catch (Exception ex) + { + _logger.LogError( + LogContent.MESSAGE_TEMPLATE, + logUserIdentification, + new LogApiInfo( + action: LogContent.Action.ORCID_RECORD_PARSE, + state: LogContent.ActionState.IN_PROGRESS, + error: true, + message: $"{ex.ToString()}")); + } // Email: DimFieldDisplaySettings DimFieldDisplaySetting dimFieldDisplaySettingsEmailAddress = dimUserProfile.DimFieldDisplaySettings.FirstOrDefault( @@ -655,7 +784,22 @@ public async Task ImportOrcidRecordJsonIntoUserProfile(int userprofileId, // Keyword - List keywords = _orcidJsonParserService.GetKeywords(orcidRecordJson); + List keywords = new(); + try + { + keywords = _orcidJsonParserService.GetKeywords(orcidRecordJson); + } + catch (Exception ex) + { + _logger.LogError( + LogContent.MESSAGE_TEMPLATE, + logUserIdentification, + new LogApiInfo( + action: LogContent.Action.ORCID_RECORD_PARSE, + state: LogContent.ActionState.IN_PROGRESS, + error: true, + message: $"{ex.ToString()}")); + } // Get DimFieldDisplaySettings for keyword DimFieldDisplaySetting dimFieldDisplaySettingsKeyword = dimUserProfile.DimFieldDisplaySettings.FirstOrDefault(dfdsKeyword => dfdsKeyword.FieldIdentifier == Constants.FieldIdentifiers.PERSON_KEYWORD); @@ -718,7 +862,22 @@ public async Task ImportOrcidRecordJsonIntoUserProfile(int userprofileId, // External identifier (=DimPid) - List externalIdentifiers = _orcidJsonParserService.GetExternalIdentifiers(orcidRecordJson); + List externalIdentifiers = new(); + try + { + externalIdentifiers = _orcidJsonParserService.GetExternalIdentifiers(orcidRecordJson); + } + catch (Exception ex) + { + _logger.LogError( + LogContent.MESSAGE_TEMPLATE, + logUserIdentification, + new LogApiInfo( + action: LogContent.Action.ORCID_RECORD_PARSE, + state: LogContent.ActionState.IN_PROGRESS, + error: true, + message: $"{ex.ToString()}")); + } // Get DimFieldDisplaySettings for keyword DimFieldDisplaySetting dimFieldDisplaySettingsExternalIdentifier = dimUserProfile.DimFieldDisplaySettings.FirstOrDefault(dfdsKeyword => dfdsKeyword.FieldIdentifier == Constants.FieldIdentifiers.PERSON_EXTERNAL_IDENTIFIER); @@ -774,7 +933,22 @@ public async Task ImportOrcidRecordJsonIntoUserProfile(int userprofileId, // Education - List educations = _orcidJsonParserService.GetEducations(orcidRecordJson); + List educations = new(); + try + { + educations = _orcidJsonParserService.GetEducations(orcidRecordJson); + } + catch (Exception ex) + { + _logger.LogError( + LogContent.MESSAGE_TEMPLATE, + logUserIdentification, + new LogApiInfo( + action: LogContent.Action.ORCID_RECORD_PARSE, + state: LogContent.ActionState.IN_PROGRESS, + error: true, + message: $"{ex.ToString()}")); + } // Get DimFieldDisplaySettings for education DimFieldDisplaySetting dimFieldDisplaySettingsEducation = dimUserProfile.DimFieldDisplaySettings.FirstOrDefault(dfdsEducation => dfdsEducation.FieldIdentifier == Constants.FieldIdentifiers.ACTIVITY_EDUCATION); @@ -849,7 +1023,22 @@ public async Task ImportOrcidRecordJsonIntoUserProfile(int userprofileId, // Employment (Affiliation in Ttv database) - List employments = _orcidJsonParserService.GetEmployments(orcidRecordJson); + List employments = new(); + try + { + employments = _orcidJsonParserService.GetEmployments(orcidRecordJson); + } + catch (Exception ex) + { + _logger.LogError( + LogContent.MESSAGE_TEMPLATE, + logUserIdentification, + new LogApiInfo( + action: LogContent.Action.ORCID_RECORD_PARSE, + state: LogContent.ActionState.IN_PROGRESS, + error: true, + message: $"{ex.ToString()}")); + } // Get DimFieldDisplaySettings for affiliation DimFieldDisplaySetting dimFieldDisplaySettingsAffiliation = dimUserProfile.DimFieldDisplaySettings.FirstOrDefault(dfdsAffiliation => dfdsAffiliation.FieldIdentifier == Constants.FieldIdentifiers.ACTIVITY_AFFILIATION); @@ -1071,7 +1260,22 @@ public async Task ImportOrcidRecordJsonIntoUserProfile(int userprofileId, // Works - OrcidWorks orcidWorks = _orcidJsonParserService.GetWorks(orcidRecordJson); + OrcidWorks orcidWorks = new(); + try + { + orcidWorks = _orcidJsonParserService.GetWorks(orcidRecordJson); + } + catch (Exception ex) + { + _logger.LogError( + LogContent.MESSAGE_TEMPLATE, + logUserIdentification, + new LogApiInfo( + action: LogContent.Action.ORCID_RECORD_PARSE, + state: LogContent.ActionState.IN_PROGRESS, + error: true, + message: $"{ex.ToString()}")); + } // Works => Publication if (orcidWorks.Publications.Count > 0) @@ -1256,7 +1460,22 @@ public async Task ImportOrcidRecordJsonIntoUserProfile(int userprofileId, // Funding - List orcidFundings = _orcidJsonParserService.GetFundings(orcidRecordJson); + List orcidFundings = new(); + try + { + orcidFundings = _orcidJsonParserService.GetFundings(orcidRecordJson); + } + catch (Exception ex) + { + _logger.LogError( + LogContent.MESSAGE_TEMPLATE, + logUserIdentification, + new LogApiInfo( + action: LogContent.Action.ORCID_RECORD_PARSE, + state: LogContent.ActionState.IN_PROGRESS, + error: true, + message: $"{ex.ToString()}")); + } // Get DimFieldDisplaySettings for orcid publication DimFieldDisplaySetting dimFieldDisplaySettingsOrcidFunding = dimUserProfile.DimFieldDisplaySettings.FirstOrDefault(dfdsPublication => dfdsPublication.FieldIdentifier == Constants.FieldIdentifiers.ACTIVITY_FUNDING_DECISION); @@ -1298,7 +1517,7 @@ public async Task ImportOrcidRecordJsonIntoUserProfile(int userprofileId, DimProfileOnlyFundingDecision dimProfileOnlyFundingDecision_existing = factFieldValuesProfileOnlyFundingDecision.DimProfileOnlyFundingDecision; dimProfileOnlyFundingDecision_existing.NameEn = orcidFunding.Name; dimProfileOnlyFundingDecision_existing.DimDateIdStartNavigation = fundingStartDate; - dimProfileOnlyFundingDecision_existing.DimDateIdStartNavigation = fundingEndDate; + dimProfileOnlyFundingDecision_existing.DimDateIdEndNavigation = fundingEndDate; dimProfileOnlyFundingDecision_existing.SourceDescription = orcidFunding.Path; // Related DimWebLink if (!string.IsNullOrWhiteSpace(orcidFunding.Url)) @@ -1513,7 +1732,22 @@ public async Task ImportOrcidRecordJsonIntoUserProfile(int userprofileId, dimUserProfile.DimFieldDisplaySettings.FirstOrDefault(dfdsResearchActivity => dfdsResearchActivity.FieldIdentifier == Constants.FieldIdentifiers.ACTIVITY_RESEARCH_ACTIVITY); // Invited positions, distinctions, memberships and services => Research activity - List orcidResearchActivities = _orcidJsonParserService.GetProfileOnlyResearchActivityItems(orcidRecordJson); + List orcidResearchActivities = new(); + try + { + orcidResearchActivities = _orcidJsonParserService.GetProfileOnlyResearchActivityItems(orcidRecordJson); + } + catch (Exception ex) + { + _logger.LogError( + LogContent.MESSAGE_TEMPLATE, + logUserIdentification, + new LogApiInfo( + action: LogContent.Action.ORCID_RECORD_PARSE, + state: LogContent.ActionState.IN_PROGRESS, + error: true, + message: $"{ex.ToString()}")); + } foreach (OrcidResearchActivity orcidResearchActivity in orcidResearchActivities.Concat(orcidWorks.ResearchActivities)) { @@ -1977,7 +2211,7 @@ public async Task ImportOrcidRecordJsonIntoUserProfile(int userprofileId, * Implemented for: * - fundings */ - public async Task ImportAdditionalData(List factFieldValues, String orcidAccessToken, bool useOrcidPublicApi=false) + public async Task ImportAdditionalData(List factFieldValues, String orcidAccessToken, LogUserIdentification logUserIdentification, bool useOrcidPublicApi=false) { foreach (FactFieldValue ffv in factFieldValues) { @@ -1992,19 +2226,32 @@ public async Task ImportAdditionalData(List factFieldValue { result = await _orcidApiService.GetDataFromMemberApi(path: ffv.DimProfileOnlyFundingDecision.SourceDescription, orcidAccessToken: orcidAccessToken); } - OrcidFunding orcidFunding = _orcidJsonParserService.GetFundingDetail(fundingDetailJson: result); - - ffv.DimProfileOnlyFundingDecision.OrcidWorkType = orcidFunding.Type; - ffv.DimProfileOnlyFundingDecision.NameEn = orcidFunding.Name; - ffv.DimProfileOnlyFundingDecision.DescriptionEn = orcidFunding.Description; - ffv.DimProfileOnlyFundingDecision.AmountInFundingDecisionCurrency = - _utilityService.StringToNullableDecimal(orcidFunding.Amount); - ffv.DimProfileOnlyFundingDecision.FundingDecisionCurrencyAbbreviation = orcidFunding.CurrencyCode; - - // Set EUR value - if (orcidFunding.CurrencyCode == "EUR" && ffv.DimProfileOnlyFundingDecision.AmountInFundingDecisionCurrency != null) + try + { + OrcidFunding orcidFunding = _orcidJsonParserService.GetFundingDetail(fundingDetailJson: result); + ffv.DimProfileOnlyFundingDecision.OrcidWorkType = orcidFunding.Type; + ffv.DimProfileOnlyFundingDecision.NameEn = orcidFunding.Name; + ffv.DimProfileOnlyFundingDecision.DescriptionEn = orcidFunding.Description; + ffv.DimProfileOnlyFundingDecision.AmountInFundingDecisionCurrency = + _utilityService.StringToNullableDecimal(orcidFunding.Amount); + ffv.DimProfileOnlyFundingDecision.FundingDecisionCurrencyAbbreviation = orcidFunding.CurrencyCode; + + // Set EUR value + if (orcidFunding.CurrencyCode == "EUR" && ffv.DimProfileOnlyFundingDecision.AmountInFundingDecisionCurrency != null) + { + ffv.DimProfileOnlyFundingDecision.AmountInEur = (decimal)ffv.DimProfileOnlyFundingDecision.AmountInFundingDecisionCurrency; + } + } + catch (Exception ex) { - ffv.DimProfileOnlyFundingDecision.AmountInEur = (decimal)ffv.DimProfileOnlyFundingDecision.AmountInFundingDecisionCurrency; + _logger.LogError( + LogContent.MESSAGE_TEMPLATE, + logUserIdentification, + new LogApiInfo( + action: LogContent.Action.ORCID_RECORD_PARSE, + state: LogContent.ActionState.IN_PROGRESS, + error: true, + message: $"{ex.ToString()}")); } } } diff --git a/aspnetcore/src/api/Services/OrcidJsonParserService.cs b/aspnetcore/src/api/Services/OrcidJsonParserService.cs index 032af12c..5d671d82 100644 --- a/aspnetcore/src/api/Services/OrcidJsonParserService.cs +++ b/aspnetcore/src/api/Services/OrcidJsonParserService.cs @@ -615,10 +615,13 @@ public OrcidWorks GetWorks(String json, bool processOnlyResearchActivities=false */ if (IsPublication(orcidWorkType) && !processOnlyResearchActivities) { + string title = (workElement.GetProperty("title").ValueKind == JsonValueKind.Null) ? + "" : workElement.GetProperty("title").GetProperty("title").GetProperty("value").GetString(); + orcidWorks.Publications.Add( new OrcidPublication() { - PublicationName = workElement.GetProperty("title").GetProperty("title").GetProperty("value").GetString(), + PublicationName = title, Doi = DOI, PublicationYear = this.GetPublicationYear(workElement), Type = orcidWorkType, @@ -635,13 +638,15 @@ public OrcidWorks GetWorks(String json, bool processOnlyResearchActivities=false */ if (IsDataset(orcidWorkType) && !processOnlyResearchActivities) { + string title = (workElement.GetProperty("title").ValueKind == JsonValueKind.Null) ? + "" : workElement.GetProperty("title").GetProperty("title").GetProperty("value").GetString(); string url = (workElement.GetProperty("url").ValueKind == JsonValueKind.Null) ? "" : workElement.GetProperty("url").GetProperty("value").GetString(); orcidWorks.Datasets.Add( new OrcidDataset() { - DatasetName = workElement.GetProperty("title").GetProperty("title").GetProperty("value").GetString(), + DatasetName = title, DatasetDate = GetOrcidDate(workElement.GetProperty("publication-date")), Type = orcidWorkType, PutCode = this.GetOrcidPutCode(workElement), diff --git a/aspnetcore/src/api/Services/UserProfileService.cs b/aspnetcore/src/api/Services/UserProfileService.cs index dc24addc..e59696c7 100644 --- a/aspnetcore/src/api/Services/UserProfileService.cs +++ b/aspnetcore/src/api/Services/UserProfileService.cs @@ -10,16 +10,7 @@ using api.Models.Orcid; using api.Models.Log; using Dapper; -using System.Transactions; -using api.Controllers; using Microsoft.Extensions.Logging; -using api.Models.Elasticsearch; -using Elasticsearch.Net; -using api.Models.Api; -using Serilog; -using System.Text.Json; -using static api.Models.Common.Constants; -using System.Security.Cryptography; namespace api.Services { @@ -839,6 +830,8 @@ public async Task AddTtvDataToUserProfile(DimKnownPerson dimKnownPerson, DimUser factFieldValuePublication.DimPublicationId = fc.DimPublicationId; factFieldValuePublication.DimRegisteredDataSourceId = dimName.DimRegisteredDataSourceId; _ttvContext.FactFieldValues.Add(factFieldValuePublication); + // Prevent duplicate key error with publications + existingPublicationIds.Add(fc.DimPublicationId); } // research activity @@ -850,6 +843,8 @@ public async Task AddTtvDataToUserProfile(DimKnownPerson dimKnownPerson, DimUser factFieldValueResearchActivity.DimResearchActivityId = fc.DimResearchActivityId; factFieldValueResearchActivity.DimRegisteredDataSourceId = dimName.DimRegisteredDataSourceId; _ttvContext.FactFieldValues.Add(factFieldValueResearchActivity); + // Prevent duplicate key error with research activities + existingResearchActivityIds.Add(fc.DimResearchActivityId); } // research dataset @@ -861,6 +856,8 @@ public async Task AddTtvDataToUserProfile(DimKnownPerson dimKnownPerson, DimUser factFieldValueResearchDataset.DimResearchDatasetId = fc.DimResearchDatasetId; factFieldValueResearchDataset.DimRegisteredDataSourceId = dimName.DimRegisteredDataSourceId; _ttvContext.FactFieldValues.Add(factFieldValueResearchDataset); + // Prevent duplicate key error with research datasets + existingResearchDatasetIds.Add(fc.DimResearchDatasetId); } } } diff --git a/keycloak/openshift/Dockerfile b/keycloak/openshift/Dockerfile index 58708701..29c0c141 100644 --- a/keycloak/openshift/Dockerfile +++ b/keycloak/openshift/Dockerfile @@ -1,7 +1,7 @@ # # Build container # -FROM quay.io/keycloak/keycloak:19.0.3 as builder +FROM --platform=linux/amd64 quay.io/keycloak/keycloak:20.0.5 as builder ENV KC_HEALTH_ENABLED=true ENV KC_METRICS_ENABLED=true ENV KC_FEATURES=token-exchange @@ -17,7 +17,7 @@ RUN /opt/keycloak/bin/kc.sh build --transaction-xa-enabled=false # # Optimized container # -FROM quay.io/keycloak/keycloak:19.0.3 +FROM --platform=linux/amd64 quay.io/keycloak/keycloak:20.0.5 COPY --from=builder /opt/keycloak/ /opt/keycloak/ WORKDIR /opt/keycloak