CSCTTV-4081 Add updated field in Elasticsearch person index (#249)

* CSCTTV-4081 set dim_user_profile.modified . Add field updated to Elasticsearch data. * Move timestamp update logic to userProfileService. Add missing update operations.
CSCfi · Nov 7, 2024 · d5eb17a · d5eb17a
1 parent a58ffc6
commit d5eb17a
Show file tree

Hide file tree

Showing 15 changed files with 85 additions and 7 deletions.
diff --git a/aspnetcore/src/api.Tests/Services_Tests/TtvSqlServiceTest.cs b/aspnetcore/src/api.Tests/Services_Tests/TtvSqlServiceTest.cs
@@ -901,11 +901,23 @@ public void GetSqlQuery_Select_GetHiddenInUserprofile()
         {
             // Arrange
             TtvSqlService ttvSqlService = new();
-            string expectedSqlString = $"SELECT hidden as 'Hidden' FROM dim_user_profile WHERE id={53445623}";
+            string expectedSqlString = $"SELECT hidden as 'Hidden' FROM dim_user_profile WHERE id=53445623";
             // Act
             string actualSqlString = ttvSqlService.GetSqlQuery_Select_GetHiddenInUserprofile(53445623);
             // Assert
             Assert.Equal(expectedSqlString, actualSqlString);
         }
+
+        [Fact(DisplayName = "Get SQL query for setting 'modified' timestamp in userprofile")]
+        public void GetSqlQuery_Update_DimUserProfile_Modified()
+        {
+            // Arrange
+            TtvSqlService ttvSqlService = new();
+            string expectedSqlString = $"UPDATE dim_user_profile SET modified=GETDATE() WHERE id=445566778";
+            // Act
+            string actualSqlString = ttvSqlService.GetSqlQuery_Update_DimUserProfile_Modified(445566778);
+            // Assert
+            Assert.Equal(expectedSqlString, actualSqlString);
+        }
     }
 }
diff --git a/aspnetcore/src/api/Background/BackgroundProfileData.cs b/aspnetcore/src/api/Background/BackgroundProfileData.cs
@@ -1,6 +1,7 @@
 using System.Threading.Tasks;
 using api.Models.Log;
 using api.Models.Ttv;
+using api.Models.Common;
 using api.Models.Elasticsearch;
 using Microsoft.Extensions.DependencyInjection;
 using api.Models.ProfileEditor.Items;
@@ -58,6 +59,13 @@ await localUserProfileService.GetProfileDataAsync(
                 .Where(duc => duc.DimUserProfileId == userprofileId && duc.UserChoiceValue == true).AsNoTracking().ProjectTo<ElasticsearchCooperation>(mapper.ConfigurationProvider).ToListAsync();
             elasticsearchPerson.cooperation.AddRange(userChoices);
 
+            // Add updated timestamp
+            DateTimeDTO userProfileModified = await localTtvContext.DimUserProfiles.Where(dup => dup.Id == userprofileId).AsNoTracking().Select(dimUserProfile => new DateTimeDTO()  
+                {  
+                    Value = dimUserProfile.Modified
+                }).FirstOrDefaultAsync();
+            elasticsearchPerson.updated = userProfileModified.Value;
+
             return elasticsearchPerson;
         }
     }

diff --git a/aspnetcore/src/api/Controllers/CooperationChoicesController.cs b/aspnetcore/src/api/Controllers/CooperationChoicesController.cs
@@ -147,9 +147,11 @@ public async Task<IActionResult> PatchMany([FromBody] List<ProfileEditorCooperat
                     dimUserChoice.UserChoiceValue = profileEditorCooperationItem.Selected;
                 }
             }
-
             await _ttvContext.SaveChangesAsync();
 
+            // Refresh 'modified' timestamp in user profile
+            await _userProfileService.SetModifiedTimestampInUserProfile(userprofileId);
+
             return Ok(new ApiResponse(success: true));
         }
     }

diff --git a/aspnetcore/src/api/Controllers/FundingDecisionController.cs b/aspnetcore/src/api/Controllers/FundingDecisionController.cs
@@ -163,6 +163,9 @@ public async Task<IActionResult> PostMany([FromBody] List<ProfileEditorFundingDe
                 }
             }
 
+            // Refresh 'modified' timestamp in user profile
+            await _userProfileService.SetModifiedTimestampInUserProfile(userprofileId);
+
             // Update Elasticsearch index.
             LogUserIdentification logUserIdentification = this.GetLogUserIdentification();
             await _userProfileService.UpdateProfileInElasticsearch(
@@ -230,6 +233,9 @@ public async Task<IActionResult> RemoveMany([FromBody] List<int> projectIds)
             }
             await _ttvContext.SaveChangesAsync();
 
+            // Refresh 'modified' timestamp in user profile
+            await _userProfileService.SetModifiedTimestampInUserProfile(userprofileId);
+
             // Update Elasticsearch index.
             LogUserIdentification logUserIdentification = this.GetLogUserIdentification();
             await _userProfileService.UpdateProfileInElasticsearch(

diff --git a/aspnetcore/src/api/Controllers/ProfileDataController.cs b/aspnetcore/src/api/Controllers/ProfileDataController.cs
@@ -119,6 +119,9 @@ public async Task<IActionResult> PatchMany([FromBody] ProfileEditorDataModificat
                 profileEditorDataModificationResponse.items.Add(profileEditorItemMeta);
             }
 
+            // Refresh 'modified' timestamp in user profile
+            await _userProfileService.SetModifiedTimestampInUserProfile(userprofileId);
+
             // Update Elasticsearch index.
             LogUserIdentification logUserIdentification = this.GetLogUserIdentification();
             await _userProfileService.UpdateProfileInElasticsearch(

diff --git a/aspnetcore/src/api/Controllers/PublicationController.cs b/aspnetcore/src/api/Controllers/PublicationController.cs
@@ -184,6 +184,9 @@ public async Task<IActionResult> PostMany([FromBody] List<ProfileEditorPublicati
                 }
             }
 
+            // Refresh 'modified' timestamp in user profile
+            await _userProfileService.SetModifiedTimestampInUserProfile(userprofileId);
+
             // Update Elasticsearch index.
             LogUserIdentification logUserIdentification = this.GetLogUserIdentification();
             await _userProfileService.UpdateProfileInElasticsearch(
@@ -247,6 +250,8 @@ public async Task<IActionResult> RemoveMany([FromBody] List<string> publicationI
             }
             await _ttvContext.SaveChangesAsync();
 
+            // Refresh 'modified' timestamp in user profile
+            await _userProfileService.SetModifiedTimestampInUserProfile(userprofileId);
 
             // Update Elasticsearch index.
             LogUserIdentification logUserIdentification = this.GetLogUserIdentification();

diff --git a/aspnetcore/src/api/Controllers/ResearchDatasetController.cs b/aspnetcore/src/api/Controllers/ResearchDatasetController.cs
@@ -161,6 +161,9 @@ public async Task<IActionResult> PostMany([FromBody] List<ProfileEditorResearchD
                 }
             }
 
+            // Refresh 'modified' timestamp in user profile
+            await _userProfileService.SetModifiedTimestampInUserProfile(userprofileId);
+
             // Update Elasticsearch index
             LogUserIdentification logUserIdentification = this.GetLogUserIdentification();
             await _userProfileService.UpdateProfileInElasticsearch(
@@ -228,6 +231,8 @@ public async Task<IActionResult> RemoveMany([FromBody] List<string> localIdentif
             }
             await _ttvContext.SaveChangesAsync();
 
+            // Refresh 'modified' timestamp in user profile
+            await _userProfileService.SetModifiedTimestampInUserProfile(userprofileId);
 
             // Update Elasticsearch index.
             LogUserIdentification logUserIdentification = this.GetLogUserIdentification();

diff --git a/aspnetcore/src/api/Controllers/WebhookController.cs b/aspnetcore/src/api/Controllers/WebhookController.cs
@@ -26,12 +26,14 @@ namespace api.Controllers
     public class WebhookController : ControllerBase
     {
         private readonly IUserProfileService _userProfileService;
+        private readonly ITtvSqlService _ttvSqlService;
         private readonly ILogger<OrcidController> _logger;
         private readonly IBackgroundTaskQueue _taskQueue;
         private readonly IServiceScopeFactory _serviceScopeFactory;
         private readonly IMemoryCache _cache;
 
         public WebhookController(IUserProfileService userProfileService,
+            ITtvSqlService ttvSqlService,
             ILogger<OrcidController> logger,
             IBackgroundTaskQueue taskQueue,
             IServiceScopeFactory serviceScopeFactory,
@@ -40,6 +42,7 @@ public WebhookController(IUserProfileService userProfileService,
             IMemoryCache memoryCache)
         {
             _userProfileService = userProfileService;
+            _ttvSqlService = ttvSqlService;
             _logger = logger;
             _taskQueue = taskQueue;
             _serviceScopeFactory = serviceScopeFactory;
@@ -91,6 +94,10 @@ public async Task<IActionResult> HandleOrcidWebhook(string webhookOrcidId)
             // Remove cached profile data response. Cache key is ORCID ID.
             _cache.Remove(webhookOrcidId);
 
+            // Refresh 'modified' timestamp in table dim_user_profile
+            string setModifiedSql = _ttvSqlService.GetSqlQuery_Update_DimUserProfile_Modified(dimUserProfile.Id);
+            await _userProfileService.ExecuteRawSql(setModifiedSql); 
+
             // Store values to be used in background task.
             orcidAccessToken = dimUserProfile.OrcidAccessToken;
             dimUserprofileId = dimUserProfile.Id;

diff --git a/aspnetcore/src/api/Models/Common/DateTimeDTO.cs b/aspnetcore/src/api/Models/Common/DateTimeDTO.cs
@@ -0,0 +1,9 @@
+using System;
+
+namespace api.Models.Common
+{
+    public partial class DateTimeDTO
+    {
+        public DateTime? Value { get; set; }
+    }
+}
diff --git a/aspnetcore/src/api/Models/Common/IntegerDTO.cs b/aspnetcore/src/api/Models/Common/IntegerDTO.cs
@@ -2,6 +2,6 @@
 {
     public partial class IntegerDTO
     {
-        public int Integer { get; set; }
+        public int Value { get; set; }
     }
 }
diff --git a/aspnetcore/src/api/Models/Elasticsearch/ElasticsearchPerson.cs b/aspnetcore/src/api/Models/Elasticsearch/ElasticsearchPerson.cs
@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using Nest;
 
 namespace api.Models.Elasticsearch
@@ -30,5 +31,6 @@ public ElasticsearchPerson(string orcidId)
         public ElasticsearchActivity activity { get; set; }
         public List<ElasticsearchCooperation> cooperation { get; set; }
         public List<ElasticsearchSource> uniqueDataSources { get; set; }
+        public DateTime? updated { get; set; }
     }
 }
diff --git a/aspnetcore/src/api/Services/ITtvSqlService.cs b/aspnetcore/src/api/Services/ITtvSqlService.cs
@@ -48,5 +48,6 @@ public interface ITtvSqlService
         string GetSqlQuery_Select_FactFieldValues(int userprofileId);
         string GetSqlQuery_Select_GetHiddenInUserprofile(int dimUserProfileId);
         string GetSqlQuery_Update_FactFieldValues(int dimUserProfileId, ProfileEditorItemMeta profileEditorItemMeta);
+        string GetSqlQuery_Update_DimUserProfile_Modified(int dimUserProfileId);
     }
 }
diff --git a/aspnetcore/src/api/Services/IUserProfileService.cs b/aspnetcore/src/api/Services/IUserProfileService.cs
@@ -47,5 +47,6 @@ public interface IUserProfileService
         string GetCMemoryCacheKey_SharePurposes();
         string GetCMemoryCacheKey_SharePermissions();
         string GetCMemoryCacheKey_GivenPermissions(string orcidId);
+        Task SetModifiedTimestampInUserProfile(int Id);
     }
 }
diff --git a/aspnetcore/src/api/Services/TtvSqlService.cs b/aspnetcore/src/api/Services/TtvSqlService.cs
@@ -95,6 +95,12 @@ public string GetSqlQuery_Update_FactFieldValues(int dimUserProfileId, ProfileEd
                             {fk_column_name}={profileEditorItemMeta.Id}";
         }
 
+        // Return SQL update statement for setting 'modified' timestamp in userprofile.
+        public string GetSqlQuery_Update_DimUserProfile_Modified(int dimUserProfileId)
+        {
+            return $@"UPDATE dim_user_profile SET modified=GETDATE() WHERE id={dimUserProfileId}";
+        }
+
         // Convert list of integers into a comma separated string
         public string ConvertListOfIntsToCommaSeparatedString(List<int> listOfInts)
         {

diff --git a/aspnetcore/src/api/Services/UserProfileService.cs b/aspnetcore/src/api/Services/UserProfileService.cs
@@ -156,16 +156,26 @@ public async Task<DimUserProfile> GetUserprofileById(int Id)
             IntegerDTO dimUserProfileIdDTO = await _ttvContext.DimUserProfiles.Where(dup => dup.OrcidId == orcidId).AsNoTracking()
                 .Select(dimUserProfile => new IntegerDTO()  
                 {  
-                    Integer = dimUserProfile.Id
+                    Value = dimUserProfile.Id
                 }).FirstOrDefaultAsync();
-            if (dimUserProfileIdDTO == null || dimUserProfileIdDTO.Integer < 0) {
+            if (dimUserProfileIdDTO == null || dimUserProfileIdDTO.Value < 0) {
                 return (UserProfileExists: false,  UserProfileId: -1);
             }
             else {
-                return (UserProfileExists: true,  UserProfileId: dimUserProfileIdDTO.Integer);
+                return (UserProfileExists: true,  UserProfileId: dimUserProfileIdDTO.Value);
             }
         }
 
+        /*
+         * Set 'modified' timestamp in user profile
+         */
+        public async Task SetModifiedTimestampInUserProfile(int Id)
+        {
+            await ExecuteRawSql(
+                _ttvSqlService.GetSqlQuery_Update_DimUserProfile_Modified(Id)
+            ); 
+        }
+
         /*
          * Add or update DimName.
          */
@@ -1009,6 +1019,7 @@ public async Task CreateProfile(string orcidId, LogUserIdentification logUserIde
                     SourceId = Constants.SourceIdentifiers.PROFILE_API,
                     SourceDescription = Constants.SourceDescriptions.PROFILE_API,
                     Created = currentDateTime,
+                    Modified = currentDateTime,
                     AllowAllSubscriptions = false,
                     Hidden = false,
                     PublishNewOrcidData = false