Use merge TVPs in reindex (update search parameters)

src/Microsoft.Health.Fhir.Core/Features/Persistence/IFhirDataStore.cs

@@ -23,7 +23,7 @@ public interface IFhirDataStore
 
         Task BulkUpdateSearchParameterIndicesAsync(IReadOnlyCollection<ResourceWrapper> resources, CancellationToken cancellationToken);
 
-        Task<ResourceWrapper> UpdateSearchParameterIndicesAsync(ResourceWrapper resourceWrapper, WeakETag weakETag, CancellationToken cancellationToken);
+        Task<ResourceWrapper> UpdateSearchParameterIndicesAsync(ResourceWrapper resourceWrapper, CancellationToken cancellationToken);
 
         Task<int?> GetProvisionedDataStoreCapacityAsync(CancellationToken cancellationToken);
     }

src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosFhirDataStore.cs

@@ -455,27 +455,26 @@ public async Task BulkUpdateSearchParameterIndicesAsync(IReadOnlyCollection<Reso
             // this is a place holder update until we batch update resources
             foreach (var resource in resources)
             {
-                await UpdateSearchParameterIndicesAsync(resource, WeakETag.FromVersionId(resource.Version), cancellationToken);
+                await UpdateSearchParameterIndicesAsync(resource, cancellationToken);
             }
         }
 
-        public async Task<ResourceWrapper> UpdateSearchParameterIndicesAsync(ResourceWrapper resourceWrapper, WeakETag weakETag, CancellationToken cancellationToken)
+        public async Task<ResourceWrapper> UpdateSearchParameterIndicesAsync(ResourceWrapper resourceWrapper, CancellationToken cancellationToken)
         {
             EnsureArg.IsNotNull(resourceWrapper, nameof(resourceWrapper));
-            EnsureArg.IsNotNull(weakETag, nameof(weakETag));
 
             var cosmosWrapper = new FhirCosmosResourceWrapper(resourceWrapper);
             UpdateSortIndex(cosmosWrapper);
 
             try
             {
-                _logger.LogDebug("Replacing {ResourceType}/{Id}, ETag: \"{Tag}\"", resourceWrapper.ResourceTypeName, resourceWrapper.ResourceId, weakETag.VersionId);
+                _logger.LogDebug("Replacing {ResourceType}/{Id}/{Version}", resourceWrapper.ResourceTypeName, resourceWrapper.ResourceId, resourceWrapper.Version);
 
                 FhirCosmosResourceWrapper response = await _retryExceptionPolicyFactory.RetryPolicy.ExecuteAsync(
                     async ct => await _replaceSingleResource.Execute(
                         _containerScope.Value.Scripts,
                         cosmosWrapper,
-                        weakETag.VersionId,
+                        resourceWrapper.Version,
                         ct),
                     cancellationToken);
 
@@ -487,8 +486,8 @@ public async Task<ResourceWrapper> UpdateSearchParameterIndicesAsync(ResourceWra
                 switch (exception.GetSubStatusCode())
                 {
                     case HttpStatusCode.PreconditionFailed:
-                        _logger.LogError(string.Format(Core.Resources.ResourceVersionConflict, weakETag));
-                        throw new PreconditionFailedException(string.Format(Core.Resources.ResourceVersionConflict, weakETag));
+                        _logger.LogError(string.Format(Core.Resources.ResourceVersionConflict, WeakETag.FromVersionId(resourceWrapper.Version)));
+                        throw new PreconditionFailedException(string.Format(Core.Resources.ResourceVersionConflict, WeakETag.FromVersionId(resourceWrapper.Version)));
 
                     case HttpStatusCode.ServiceUnavailable:
                         _logger.LogError("Failed to reindex resource because the Cosmos service was unavailable.");

src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/Reindex/ReindexSingleResourceRequestHandlerTests.cs

@@ -207,11 +207,11 @@ private async Task ValidateUpdateCallBasedOnHttpMethodType(string httpMethodName
         {
             if (httpMethodName == HttpPostName)
             {
-                await _fhirDataStore.Received().UpdateSearchParameterIndicesAsync(Arg.Any<ResourceWrapper>(), Arg.Any<WeakETag>(), _cancellationToken);
+                await _fhirDataStore.Received().UpdateSearchParameterIndicesAsync(Arg.Any<ResourceWrapper>(), _cancellationToken);
             }
             else if (httpMethodName == HttpGetName)
             {
-                await _fhirDataStore.DidNotReceive().UpdateSearchParameterIndicesAsync(Arg.Any<ResourceWrapper>(), Arg.Any<WeakETag>(), _cancellationToken);
+                await _fhirDataStore.DidNotReceive().UpdateSearchParameterIndicesAsync(Arg.Any<ResourceWrapper>(), _cancellationToken);
             }
         }
     }

src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Reindex/ReindexSingleResourceRequestHandler.cs

@@ -117,9 +117,10 @@ private async System.Threading.Tasks.Task ProcessPostReindexSingleResourceReques
                 searchIndices,
                 originalResource.CompartmentIndices,
                 originalResource.LastModifiedClaims,
-                hashValue);
+                hashValue,
+                originalResource.ResourceSurrogateId);
 
-            await _fhirDataStore.UpdateSearchParameterIndicesAsync(updatedResource, WeakETag.FromVersionId(originalResource.Version), CancellationToken.None);
+            await _fhirDataStore.UpdateSearchParameterIndicesAsync(updatedResource, CancellationToken.None);
         }
     }
 }

src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/62.diff.sql

@@ -0,0 +1,143 @@
+CREATE OR ALTER PROCEDURE dbo.UpdateResourceSearchParams
+    @FailedResources int = 0 OUT
+   ,@Resources dbo.ResourceList READONLY
+   ,@ResourceWriteClaims dbo.ResourceWriteClaimList READONLY
+   ,@ReferenceSearchParams dbo.ReferenceSearchParamList READONLY
+   ,@TokenSearchParams dbo.TokenSearchParamList READONLY
+   ,@TokenTexts dbo.TokenTextList READONLY
+   ,@StringSearchParams dbo.StringSearchParamList READONLY
+   ,@UriSearchParams dbo.UriSearchParamList READONLY
+   ,@NumberSearchParams dbo.NumberSearchParamList READONLY
+   ,@QuantitySearchParams dbo.QuantitySearchParamList READONLY
+   ,@DateTimeSearchParams dbo.DateTimeSearchParamList READONLY
+   ,@ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY
+   ,@TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY
+   ,@TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY
+   ,@TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY
+   ,@TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY
+   ,@TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY
+AS
+set nocount on
+DECLARE @st datetime = getUTCdate()
+       ,@SP varchar(100) = object_name(@@procid)
+       ,@Mode varchar(200) = isnull((SELECT 'RT=['+convert(varchar,min(ResourceTypeId))+','+convert(varchar,max(ResourceTypeId))+'] Sur=['+convert(varchar,min(ResourceSurrogateId))+','+convert(varchar,max(ResourceSurrogateId))+'] V='+convert(varchar,max(Version))+' Rows='+convert(varchar,count(*)) FROM @Resources),'Input=Empty')
+       ,@Rows int
+
+BEGIN TRY
+  DECLARE @Ids TABLE (ResourceTypeId smallint NOT NULL, ResourceSurrogateId bigint NOT NULL)
+
+  BEGIN TRANSACTION
+
+  -- Update the search parameter hash value in the main resource table
+  UPDATE B
+    SET SearchParamHash = A.SearchParamHash
+    OUTPUT deleted.ResourceTypeId, deleted.ResourceSurrogateId INTO @Ids 
+    FROM @Resources A JOIN dbo.Resource B ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId
+    WHERE B.IsHistory = 0
+  SET @Rows = @@rowcount
+
+  -- First, delete all the search params of the resources to reindex.
+  DELETE FROM B FROM @Ids A JOIN dbo.ResourceWriteClaim B ON B.ResourceSurrogateId = A.ResourceSurrogateId
+  DELETE FROM B FROM @Ids A JOIN dbo.ReferenceSearchParam B ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId
+  DELETE FROM B FROM @Ids A JOIN dbo.TokenSearchParam B ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId
+  DELETE FROM B FROM @Ids A JOIN dbo.TokenText B ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId
+  DELETE FROM B FROM @Ids A JOIN dbo.StringSearchParam B ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId
+  DELETE FROM B FROM @Ids A JOIN dbo.UriSearchParam B ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId
+  DELETE FROM B FROM @Ids A JOIN dbo.NumberSearchParam B ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId
+  DELETE FROM B FROM @Ids A JOIN dbo.QuantitySearchParam B ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId
+  DELETE FROM B FROM @Ids A JOIN dbo.DateTimeSearchParam B ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId
+  DELETE FROM B FROM @Ids A JOIN dbo.ReferenceTokenCompositeSearchParam B ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId
+  DELETE FROM B FROM @Ids A JOIN dbo.TokenTokenCompositeSearchParam B ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId
+  DELETE FROM B FROM @Ids A JOIN dbo.TokenDateTimeCompositeSearchParam B ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId
+  DELETE FROM B FROM @Ids A JOIN dbo.TokenQuantityCompositeSearchParam B ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId
+  DELETE FROM B FROM @Ids A JOIN dbo.TokenStringCompositeSearchParam B ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId
+  DELETE FROM B FROM @Ids A JOIN dbo.TokenNumberNumberCompositeSearchParam B ON B.ResourceTypeId = A.ResourceTypeId AND B.ResourceSurrogateId = A.ResourceSurrogateId
+
+  -- Next, insert all the new search params.
+  INSERT INTO dbo.ResourceWriteClaim 
+         ( ResourceSurrogateId, ClaimTypeId, ClaimValue )
+    SELECT ResourceSurrogateId, ClaimTypeId, ClaimValue
+      FROM @ResourceWriteClaims
+
+  INSERT INTO dbo.ReferenceSearchParam 
+         ( ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion )
+    SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion
+      FROM @ReferenceSearchParams
+
+  INSERT INTO dbo.TokenSearchParam 
+         ( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow )
+    SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow
+      FROM @TokenSearchParams
+
+  INSERT INTO dbo.TokenText 
+         ( ResourceTypeId, ResourceSurrogateId, SearchParamId, Text )
+    SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, Text
+      FROM @TokenTexts
+
+  INSERT INTO dbo.StringSearchParam 
+         ( ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax )
+    SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax
+      FROM @StringSearchParams
+
+  INSERT INTO dbo.UriSearchParam 
+         ( ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri )
+    SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri
+      FROM @UriSearchParams
+
+  INSERT INTO dbo.NumberSearchParam 
+         ( ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue )
+    SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue
+      FROM @NumberSearchParams
+
+  INSERT INTO dbo.QuantitySearchParam 
+         ( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue )
+    SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue
+      FROM @QuantitySearchParams
+
+  INSERT INTO dbo.DateTimeSearchParam 
+         ( ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax )
+    SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax
+      FROM @DateTimeSearchParams
+
+  INSERT INTO dbo.ReferenceTokenCompositeSearchParam 
+         ( ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2 )
+    SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2
+      FROM @ReferenceTokenCompositeSearchParams
+
+  INSERT INTO dbo.TokenTokenCompositeSearchParam 
+         ( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2 )
+    SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2
+      FROM @TokenTokenCompositeSearchParams
+
+  INSERT INTO dbo.TokenDateTimeCompositeSearchParam 
+         ( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2 )
+    SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2
+      FROM @TokenDateTimeCompositeSearchParams
+
+  INSERT INTO dbo.TokenQuantityCompositeSearchParam 
+         ( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2 )
+    SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2
+      FROM @TokenQuantityCompositeSearchParams
+
+  INSERT INTO dbo.TokenStringCompositeSearchParam 
+         ( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2 )
+    SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2
+      FROM @TokenStringCompositeSearchParams
+
+  INSERT INTO dbo.TokenNumberNumberCompositeSearchParam 
+         ( ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange )
+    SELECT ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange
+      FROM @TokenNumberNumberCompositeSearchParams
+
+  COMMIT TRANSACTION
+
+  SET @FailedResources = (SELECT count(*) FROM @Resources) - @Rows
+
+  EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@st,@Rows=@Rows
+END TRY
+BEGIN CATCH
+  IF @@trancount > 0 ROLLBACK TRANSACTION
+  EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Error',@Start=@st;
+  THROW
+END CATCH
+GO