From 43b972a32eed422f50f87170d4ca5a7a3375dc3f Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Wed, 6 Sep 2023 21:40:57 -0700 Subject: [PATCH 01/58] Added initial logic for Cosmos history/delete in $export --- .../Extensions/ExportMediatorExtensions.cs | 19 +++++++- .../Features/KnownQueryParameterNames.cs | 4 ++ .../Export/CreateExportRequestHandler.cs | 46 +++++++++---------- .../Export/Models/ExportJobRecord.cs | 11 +++++ .../Operations/JobRecordProperties.cs | 4 ++ .../Features/Search/SearchOptions.cs | 4 ++ .../Messages/Export/CreateExportRequest.cs | 8 ++++ .../Export/CosmosExportOrchestratorJob.cs | 10 ++-- .../Features/Search/Queries/QueryBuilder.cs | 20 ++++++-- .../Controllers/ExportController.cs | 6 +++ .../Features/Search/SearchOptionsFactory.cs | 3 ++ 11 files changed, 102 insertions(+), 33 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core/Extensions/ExportMediatorExtensions.cs b/src/Microsoft.Health.Fhir.Core/Extensions/ExportMediatorExtensions.cs index c13ce5e623..2804a6a963 100644 --- a/src/Microsoft.Health.Fhir.Core/Extensions/ExportMediatorExtensions.cs +++ b/src/Microsoft.Health.Fhir.Core/Extensions/ExportMediatorExtensions.cs @@ -28,6 +28,8 @@ public static async Task ExportAsync( string containerName, string formatName, bool isParallel, + bool includeDeleted, + bool includeHistory, string anonymizationConfigurationCollectionReference, string anonymizationConfigLocation, string anonymizationConfigFileETag, @@ -36,7 +38,22 @@ public static async Task ExportAsync( EnsureArg.IsNotNull(mediator, nameof(mediator)); EnsureArg.IsNotNull(requestUri, nameof(requestUri)); - var request = new CreateExportRequest(requestUri, requestType, resourceType, since, till, filters, groupId, containerName, formatName, isParallel, anonymizationConfigurationCollectionReference, anonymizationConfigLocation, anonymizationConfigFileETag); + var request = new CreateExportRequest( + requestUri, + requestType, + resourceType, + since, + till, + filters, + groupId, + containerName, + formatName, + isParallel, + includeDeleted, + includeHistory, + anonymizationConfigurationCollectionReference, + anonymizationConfigLocation, + anonymizationConfigFileETag); CreateExportResponse response = await mediator.Send(request, cancellationToken); return response; diff --git a/src/Microsoft.Health.Fhir.Core/Features/KnownQueryParameterNames.cs b/src/Microsoft.Health.Fhir.Core/Features/KnownQueryParameterNames.cs index 12ba5455a5..9f299cf61f 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/KnownQueryParameterNames.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/KnownQueryParameterNames.cs @@ -98,5 +98,9 @@ public static class KnownQueryParameterNames public const string HardDelete = "_hardDelete"; public const string PurgeHistory = "_purgeHistory"; + + public const string IncludeHistory = "_includeHistory"; + + public const string IncludeDeleted = "_includeDeleted"; } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/CreateExportRequestHandler.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/CreateExportRequestHandler.cs index d4c1f61c8f..086cf42de2 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/CreateExportRequestHandler.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/CreateExportRequestHandler.cs @@ -76,30 +76,28 @@ public async Task Handle(CreateExportRequest request, Canc ExportJobFormatConfiguration formatConfiguration = ParseFormat(request.FormatName, request.ContainerName != null); var jobRecord = new ExportJobRecord( - request.RequestUri, - request.RequestType, - formatConfiguration.Format, - request.ResourceType, - filters, - "N/A", - _exportJobConfiguration.RollingFileSizeInMB, - requestorClaims, - request.Since, - request.Till, - null, - null, - null, - null, - request.GroupId, - storageAccountConnectionHash, - _exportJobConfiguration.StorageAccountUri, - request.AnonymizationConfigurationCollectionReference, - request.AnonymizationConfigurationLocation, - request.AnonymizationConfigurationFileETag, - _exportJobConfiguration.MaximumNumberOfResourcesPerQuery, - _exportJobConfiguration.NumberOfPagesPerCommit, - request.ContainerName, - request.IsParallel, + requestUri: request.RequestUri, + exportType: request.RequestType, + exportFormat: formatConfiguration.Format, + resourceType: request.ResourceType, + filters: filters, + hash: "N/A", + rollingFileSizeInMB: _exportJobConfiguration.RollingFileSizeInMB, + requestorClaims: requestorClaims, + since: request.Since, + till: request.Till, + groupId: request.GroupId, + storageAccountConnectionHash: storageAccountConnectionHash, + storageAccountUri: _exportJobConfiguration.StorageAccountUri, + anonymizationConfigurationCollectionReference: request.AnonymizationConfigurationCollectionReference, + anonymizationConfigurationLocation: request.AnonymizationConfigurationLocation, + anonymizationConfigurationFileETag: request.AnonymizationConfigurationFileETag, + maximumNumberOfResourcesPerQuery: _exportJobConfiguration.MaximumNumberOfResourcesPerQuery, + numberOfPagesPerCommit: _exportJobConfiguration.NumberOfPagesPerCommit, + storageAccountContainerName: request.ContainerName, + isParallel: request.IsParallel, + includeHistory: request.IncludeHistory, + includeDeleted: request.IncludeDeleted, smartRequest: _contextAccessor?.RequestContext?.AccessControlContext?.ApplyFineGrainedAccessControl == true); var outcome = await _fhirOperationDataStore.CreateExportJobAsync(jobRecord, cancellationToken); diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/Models/ExportJobRecord.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/Models/ExportJobRecord.cs index bf4b6879e7..93453faa66 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/Models/ExportJobRecord.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/Models/ExportJobRecord.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; +using DotLiquid.Tags; using EnsureThat; using Microsoft.Health.Core; using Microsoft.Health.Fhir.Core.Models; @@ -43,6 +44,8 @@ public ExportJobRecord( uint numberOfPagesPerCommit = 10, string storageAccountContainerName = null, bool isParallel = true, + bool includeHistory = false, + bool includeDeleted = false, int schemaVersion = 2, int typeId = (int)JobType.ExportOrchestrator, bool smartRequest = false) @@ -69,6 +72,8 @@ public ExportJobRecord( RestartCount = 0; TypeId = typeId; IsParallel = isParallel; + IncludeHistory = includeHistory; + IncludeDeleted = includeDeleted; AnonymizationConfigurationCollectionReference = anonymizationConfigurationCollectionReference; AnonymizationConfigurationLocation = anonymizationConfigurationLocation; @@ -200,6 +205,12 @@ protected ExportJobRecord() [JsonProperty(JobRecordProperties.IsParallel)] public bool IsParallel { get; private set; } + [JsonProperty(JobRecordProperties.IncludeHistory)] + public bool IncludeHistory { get; private set; } + + [JsonProperty(JobRecordProperties.IncludeDeleted)] + public bool IncludeDeleted { get; private set; } + [JsonProperty(JobRecordProperties.SmartRequest)] public bool SmartRequest { get; private set; } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/JobRecordProperties.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/JobRecordProperties.cs index f378c69486..80b68036fd 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/JobRecordProperties.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/JobRecordProperties.cs @@ -157,6 +157,10 @@ public static class JobRecordProperties public const string IsParallel = "isParallel"; + public const string IncludeHistory = "includeHistory"; + + public const string IncludeDeleted = "includeDeleted"; + public const string SmartRequest = "smartRequest"; public const string DeleteOperation = "deleteOperation"; diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SearchOptions.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SearchOptions.cs index 75bf95548b..7e9639e41f 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/SearchOptions.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SearchOptions.cs @@ -111,6 +111,10 @@ internal set } } + public bool IncludeHistory { get; internal set; } = false; + + public bool IncludeDeleted { get; internal set; } = false; + /// /// Gets the search expression. /// diff --git a/src/Microsoft.Health.Fhir.Core/Messages/Export/CreateExportRequest.cs b/src/Microsoft.Health.Fhir.Core/Messages/Export/CreateExportRequest.cs index 46255c8fcd..6236c339c2 100644 --- a/src/Microsoft.Health.Fhir.Core/Messages/Export/CreateExportRequest.cs +++ b/src/Microsoft.Health.Fhir.Core/Messages/Export/CreateExportRequest.cs @@ -24,6 +24,8 @@ public CreateExportRequest( string containerName = null, string formatName = null, bool isParallel = true, + bool includeHistory = false, + bool includeDeleted = false, string anonymizationConfigurationCollectionReference = null, string anonymizationConfigurationLocation = null, string anonymizationConfigurationFileETag = null) @@ -44,6 +46,8 @@ public CreateExportRequest( ContainerName = containerName; FormatName = formatName; IsParallel = isParallel; + IncludeHistory = includeHistory; + IncludeDeleted = includeDeleted; } public Uri RequestUri { get; } @@ -71,5 +75,9 @@ public CreateExportRequest( public string FormatName { get; } public bool IsParallel { get; } + + public bool IncludeHistory { get; } + + public bool IncludeDeleted { get; } } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Operations/Export/CosmosExportOrchestratorJob.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Operations/Export/CosmosExportOrchestratorJob.cs index b5a8202129..4e4fac74e9 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Operations/Export/CosmosExportOrchestratorJob.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Operations/Export/CosmosExportOrchestratorJob.cs @@ -85,12 +85,16 @@ private static ExportJobRecord CreateExportRecord(ExportJobRecord record, long g record.NumberOfPagesPerCommit, container, record.IsParallel, + record.IncludeHistory, + record.IncludeDeleted, record.SchemaVersion, (int)JobType.ExportProcessing, - record.SmartRequest); + record.SmartRequest) + { + Id = string.Empty, + QueuedTime = record.QueuedTime, // preserve create date of coordinator job in form of queued time for all children, so same time is used on file names. + }; - rec.Id = string.Empty; - rec.QueuedTime = record.QueuedTime; // preserve create date of coordinator job in form of queued time for all children, so same time is used on file names. return rec; } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/QueryBuilder.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/QueryBuilder.cs index d326334184..6593e51817 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/QueryBuilder.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/QueryBuilder.cs @@ -83,11 +83,21 @@ public QueryDefinition BuildSqlQuerySpec(SearchOptions searchOptions, QueryBuild searchOptions.Expression.AcceptVisitor(expressionQueryBuilder); } - AppendFilterCondition( - "AND", - true, - (KnownResourceWrapperProperties.IsHistory, false), - (KnownResourceWrapperProperties.IsDeleted, false)); + if (searchOptions.IncludeHistory is false) + { + AppendFilterCondition( + "AND", + true, + (KnownResourceWrapperProperties.IsHistory, false)); + } + + if (searchOptions.IncludeDeleted is false) + { + AppendFilterCondition( + "AND", + true, + (KnownResourceWrapperProperties.IsDeleted, false)); + } if (!searchOptions.CountOnly) { diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs b/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs index aca8144e30..9955a7f22e 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs @@ -97,6 +97,8 @@ public async Task Export( [FromQuery(Name = KnownQueryParameterNames.TypeFilter)] string typeFilter, [FromQuery(Name = KnownQueryParameterNames.Format)] string formatName, [FromQuery(Name = KnownQueryParameterNames.IsParallel)] bool isParallel = true, + [FromQuery(Name = KnownQueryParameterNames.IncludeDeleted)] bool includeDeleted = false, + [FromQuery(Name = KnownQueryParameterNames.IncludeHistory)] bool includeHistory = false, [FromQuery(Name = KnownQueryParameterNames.AnonymizationConfigurationCollectionReference)] string anonymizationConfigCollectionReference = null, [FromQuery(Name = KnownQueryParameterNames.AnonymizationConfigurationLocation)] string anonymizationConfigLocation = null, [FromQuery(Name = KnownQueryParameterNames.AnonymizationConfigurationFileEtag)] string anonymizationConfigFileETag = null) @@ -244,6 +246,8 @@ private async Task SendExportRequest( string containerName = null, string formatName = null, bool isParallel = true, + bool includeHistory = false, + bool includeDeleted = false, string anonymizationConfigCollectionReference = null, string anonymizationConfigLocation = null, string anonymizationConfigFileETag = null) @@ -259,6 +263,8 @@ private async Task SendExportRequest( containerName, formatName, isParallel, + includeDeleted, + includeHistory, anonymizationConfigCollectionReference, anonymizationConfigLocation, anonymizationConfigFileETag, diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SearchOptionsFactory.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SearchOptionsFactory.cs index ddd4e186d7..caf07a7c79 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SearchOptionsFactory.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SearchOptionsFactory.cs @@ -286,6 +286,9 @@ public SearchOptions Create( searchExpressions.Add(Expression.SearchParameter(_resourceTypeSearchParameter, Expression.StringEquals(FieldName.TokenCode, null, resourceType, false))); } + searchOptions.IncludeHistory = queryParameters.Any(_ => _.Item1 == KnownQueryParameterNames.IncludeHistory && string.Equals(_.Item2, true.ToString(), StringComparison.OrdinalIgnoreCase)); + searchOptions.IncludeDeleted = queryParameters.Any(_ => _.Item1 == KnownQueryParameterNames.IncludeDeleted && string.Equals(_.Item2, true.ToString(), StringComparison.OrdinalIgnoreCase)); + CheckFineGrainedAccessControl(searchExpressions); var resourceTypesString = parsedResourceTypes.Select(x => x.ToString()).ToArray(); From 9468cf1287705b130c7c739c68015e0b4ebc3317 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Wed, 6 Sep 2023 22:08:07 -0700 Subject: [PATCH 02/58] Fixed build issue with SQL export orchestrator --- .../Export/SqlExportOrchestratorJob.cs | 56 ++++++++++--------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/Export/SqlExportOrchestratorJob.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/Export/SqlExportOrchestratorJob.cs index 09cd20fd2c..0e3409d1ac 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/Export/SqlExportOrchestratorJob.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/Export/SqlExportOrchestratorJob.cs @@ -140,33 +140,35 @@ private static ExportJobRecord CreateExportRecord(ExportJobRecord record, long g } var rec = new ExportJobRecord( - record.RequestUri, - record.ExportType, - format, - string.IsNullOrEmpty(resourceType) ? record.ResourceType : resourceType, - record.Filters, - record.Hash, - record.RollingFileSizeInMB, - record.RequestorClaims, - since == null ? record.Since : since, - till == null ? record.Till : till, - startSurrogateId, - endSurrogateId, - globalStartSurrogateId, - globalEndSurrogateId, - record.GroupId, - record.StorageAccountConnectionHash, - record.StorageAccountUri, - record.AnonymizationConfigurationCollectionReference, - record.AnonymizationConfigurationLocation, - record.AnonymizationConfigurationFileETag, - record.MaximumNumberOfResourcesPerQuery, - record.NumberOfPagesPerCommit, - container, - record.IsParallel, - record.SchemaVersion, - (int)JobType.ExportProcessing, - record.SmartRequest); + requestUri: record.RequestUri, + exportType: record.ExportType, + exportFormat: format, + resourceType: string.IsNullOrEmpty(resourceType) ? record.ResourceType : resourceType, + filters: record.Filters, + hash: record.Hash, + rollingFileSizeInMB: record.RollingFileSizeInMB, + requestorClaims: record.RequestorClaims, + since: since == null ? record.Since : since, + till: till == null ? record.Till : till, + startSurrogateId: startSurrogateId, + endSurrogateId: endSurrogateId, + globalStartSurrogateId: globalStartSurrogateId, + globalEndSurrogateId: globalEndSurrogateId, + groupId: record.GroupId, + storageAccountConnectionHash: record.StorageAccountConnectionHash, + storageAccountUri: record.StorageAccountUri, + anonymizationConfigurationCollectionReference: record.AnonymizationConfigurationCollectionReference, + anonymizationConfigurationLocation: record.AnonymizationConfigurationLocation, + anonymizationConfigurationFileETag: record.AnonymizationConfigurationFileETag, + maximumNumberOfResourcesPerQuery: record.MaximumNumberOfResourcesPerQuery, + numberOfPagesPerCommit: record.NumberOfPagesPerCommit, + storageAccountContainerName: container, + isParallel: record.IsParallel, + includeHistory: record.IncludeHistory, + includeDeleted: record.IncludeDeleted, + schemaVersion: record.SchemaVersion, + typeId: (int)JobType.ExportProcessing, + smartRequest: record.SmartRequest); rec.Id = string.Empty; rec.QueuedTime = record.QueuedTime; // preserve create date of coordinator job in form of queued time for all children, so same time is used on file names. return rec; From cba2aa143f485c5100de70f2eea008c6ed778a78 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 7 Sep 2023 16:58:29 -0700 Subject: [PATCH 03/58] Fixed cosmos export bugs found via manual testing --- .../ValidateExportRequestFilterAttribute.cs | 2 ++ .../Extensions/ExportMediatorExtensions.cs | 30 +++++++++---------- .../Operations/Export/ExportJobTask.cs | 10 +++---- .../Controllers/ExportController.cs | 2 ++ 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Api/Features/Filters/ValidateExportRequestFilterAttribute.cs b/src/Microsoft.Health.Fhir.Api/Features/Filters/ValidateExportRequestFilterAttribute.cs index d6619cfc9f..6e96abe8e5 100644 --- a/src/Microsoft.Health.Fhir.Api/Features/Filters/ValidateExportRequestFilterAttribute.cs +++ b/src/Microsoft.Health.Fhir.Api/Features/Filters/ValidateExportRequestFilterAttribute.cs @@ -45,6 +45,8 @@ public ValidateExportRequestFilterAttribute() KnownQueryParameterNames.Format, KnownQueryParameterNames.TypeFilter, KnownQueryParameterNames.IsParallel, + KnownQueryParameterNames.IncludeHistory, + KnownQueryParameterNames.IncludeDeleted, KnownQueryParameterNames.AnonymizationConfigurationCollectionReference, KnownQueryParameterNames.AnonymizationConfigurationLocation, KnownQueryParameterNames.AnonymizationConfigurationFileEtag, diff --git a/src/Microsoft.Health.Fhir.Core/Extensions/ExportMediatorExtensions.cs b/src/Microsoft.Health.Fhir.Core/Extensions/ExportMediatorExtensions.cs index 2804a6a963..1cc9847e08 100644 --- a/src/Microsoft.Health.Fhir.Core/Extensions/ExportMediatorExtensions.cs +++ b/src/Microsoft.Health.Fhir.Core/Extensions/ExportMediatorExtensions.cs @@ -39,21 +39,21 @@ public static async Task ExportAsync( EnsureArg.IsNotNull(requestUri, nameof(requestUri)); var request = new CreateExportRequest( - requestUri, - requestType, - resourceType, - since, - till, - filters, - groupId, - containerName, - formatName, - isParallel, - includeDeleted, - includeHistory, - anonymizationConfigurationCollectionReference, - anonymizationConfigLocation, - anonymizationConfigFileETag); + requestUri: requestUri, + requestType: requestType, + resourceType: resourceType, + since: since, + till: till, + filters: filters, + groupId: groupId, + containerName: containerName, + formatName: formatName, + isParallel: isParallel, + includeDeleted: includeDeleted, + includeHistory: includeHistory, + anonymizationConfigurationCollectionReference: anonymizationConfigurationCollectionReference, + anonymizationConfigurationLocation: anonymizationConfigLocation, + anonymizationConfigurationFileETag: anonymizationConfigFileETag); CreateExportResponse response = await mediator.Send(request, cancellationToken); return response; diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs index ce3ade91e2..cc213fc12e 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs @@ -169,11 +169,11 @@ public async Task ExecuteAsync(ExportJobRecord exportJobRecord, WeakETag weakETa // from the search result. // As Till is a new property QueuedTime is being used as a backup incase Till doesn't exist in the job record. var tillTime = _exportJobRecord.Till != null ? _exportJobRecord.Till : new PartialDateTime(_exportJobRecord.QueuedTime); - var queryParametersList = new List>() - { - Tuple.Create(KnownQueryParameterNames.Count, _exportJobRecord.MaximumNumberOfResourcesPerQuery.ToString(CultureInfo.InvariantCulture)), - Tuple.Create(KnownQueryParameterNames.LastUpdated, $"le{tillTime}"), - }; + List> queryParametersList = new(); + queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.Count, _exportJobRecord.MaximumNumberOfResourcesPerQuery.ToString(CultureInfo.InvariantCulture))); + queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.LastUpdated, $"le{tillTime.ToString()}")); + queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.IncludeHistory, _exportJobRecord.IncludeHistory.ToString(CultureInfo.InvariantCulture))); + queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.IncludeDeleted, _exportJobRecord.IncludeDeleted.ToString(CultureInfo.InvariantCulture))); if (_exportJobRecord.GlobalEndSurrogateId != null) // no need to check individually as they all should have values if anyone does { diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs b/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs index 9955a7f22e..dc0e4da119 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs @@ -115,6 +115,8 @@ public async Task Export( containerName: containerName, formatName: formatName, isParallel: isParallel, + includeHistory: includeHistory, + includeDeleted: includeDeleted, anonymizationConfigCollectionReference: anonymizationConfigCollectionReference, anonymizationConfigLocation: anonymizationConfigLocation, anonymizationConfigFileETag: anonymizationConfigFileETag); From 6f6e1d27745a6d61ad42d20705459b14d6390bd3 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 7 Sep 2023 22:02:43 -0700 Subject: [PATCH 04/58] Added soft delete extension to export --- .../Features/Operations/Export/ExportJobTask.cs | 11 +++++++++-- .../Export/IResourceToByteArraySerializer.cs | 2 +- .../Export/ResourceToNdjsonBytesSerializer.cs | 8 +++++++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs index cc213fc12e..d790cc6fa3 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs @@ -675,6 +675,7 @@ private void ProcessSearchResults(IEnumerable searchResults, { ResourceWrapper resourceWrapper = result.Resource; var data = result.Resource.RawResource.Data; + var addSoftDeletedExtension = resourceWrapper.IsDeleted && _exportJobRecord.IncludeDeleted; if (anonymizer != null) { @@ -689,13 +690,19 @@ private void ProcessSearchResults(IEnumerable searchResults, } // Serialize into NDJson and write to the file. - data = _resourceToByteArraySerializer.StringSerialize(element); + data = _resourceToByteArraySerializer.StringSerialize(element, addSoftDeletedExtension); } else if (!resourceWrapper.RawResource.IsMetaSet) { // For older records in Cosmos the metadata isn't included in the raw resource ResourceElement element = _resourceDeserializer.Deserialize(resourceWrapper); - data = _resourceToByteArraySerializer.StringSerialize(element); + data = _resourceToByteArraySerializer.StringSerialize(element, addSoftDeletedExtension); + } + else if (addSoftDeletedExtension) + { + // If the resource is deleted and we aren't anonymizing it, we need to add the soft delete extension + ResourceElement element = _resourceDeserializer.Deserialize(resourceWrapper); + data = _resourceToByteArraySerializer.StringSerialize(element, addSoftDeletedExtension); } _fileManager.WriteToFile(resourceWrapper.ResourceTypeName, data); diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/IResourceToByteArraySerializer.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/IResourceToByteArraySerializer.cs index 960633d03c..78ef0ad7f1 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/IResourceToByteArraySerializer.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/IResourceToByteArraySerializer.cs @@ -19,6 +19,6 @@ public interface IResourceToByteArraySerializer /// The serialized bytes. byte[] Serialize(ResourceElement resourceElement); - string StringSerialize(ResourceElement resourceElement); + string StringSerialize(ResourceElement resourceElement, bool addSoftDeletedExtension = false); } } diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Export/ResourceToNdjsonBytesSerializer.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Export/ResourceToNdjsonBytesSerializer.cs index 60b96ae1e5..68c9b0549c 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Export/ResourceToNdjsonBytesSerializer.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Export/ResourceToNdjsonBytesSerializer.cs @@ -6,6 +6,7 @@ using System.Text; using EnsureThat; using Hl7.Fhir.Serialization; +using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Models; @@ -28,10 +29,15 @@ public byte[] Serialize(ResourceElement resourceElement) return bytesToWrite; } - public string StringSerialize(ResourceElement resourceElement) + public string StringSerialize(ResourceElement resourceElement, bool addSoftDeletedExtension = false) { EnsureArg.IsNotNull(resourceElement, nameof(resourceElement)); + if (addSoftDeletedExtension) + { + resourceElement.TryAddSoftDeletedExtension(); + } + return resourceElement.Instance.ToJson(); } } From 2f2bd6bd242338d65f9e3648e644e88a1fa22f7c Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 8 Sep 2023 14:20:35 -0700 Subject: [PATCH 05/58] Add first pass on Export E2E Tests --- .../Features/Search/SearchOptionsFactory.cs | 11 +- .../Rest/ExportDataValidationTests.cs | 31 ++-- .../Rest/ExportLongRunningTests.cs | 145 +++++++++++++++--- .../Rest/ExportTestHelper.cs | 98 ++++++++---- 4 files changed, 223 insertions(+), 62 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SearchOptionsFactory.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SearchOptionsFactory.cs index caf07a7c79..42ea055386 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SearchOptionsFactory.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SearchOptionsFactory.cs @@ -184,6 +184,14 @@ public SearchOptions Create( throw new BadRequestException(string.Format(Core.Resources.InvalidTotalParameter, query.Item2, SupportedTotalTypes)); } } + else if (string.Equals(query.Item1, KnownQueryParameterNames.IncludeHistory, StringComparison.OrdinalIgnoreCase) && bool.TryParse(query.Item2, out bool includeHistory)) + { + searchOptions.IncludeHistory = includeHistory; + } + else if (string.Equals(query.Item1, KnownQueryParameterNames.IncludeDeleted, StringComparison.OrdinalIgnoreCase) && bool.TryParse(query.Item2, out bool includeDeleted)) + { + searchOptions.IncludeDeleted = includeDeleted; + } else { // Parse the search parameters. @@ -286,9 +294,6 @@ public SearchOptions Create( searchExpressions.Add(Expression.SearchParameter(_resourceTypeSearchParameter, Expression.StringEquals(FieldName.TokenCode, null, resourceType, false))); } - searchOptions.IncludeHistory = queryParameters.Any(_ => _.Item1 == KnownQueryParameterNames.IncludeHistory && string.Equals(_.Item2, true.ToString(), StringComparison.OrdinalIgnoreCase)); - searchOptions.IncludeDeleted = queryParameters.Any(_ => _.Item1 == KnownQueryParameterNames.IncludeDeleted && string.Equals(_.Item2, true.ToString(), StringComparison.OrdinalIgnoreCase)); - CheckFineGrainedAccessControl(searchExpressions); var resourceTypesString = parsedResourceTypes.Select(x => x.ToString()).ToArray(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportDataValidationTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportDataValidationTests.cs index c9ddbac0c2..b54bdb28aa 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportDataValidationTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportDataValidationTests.cs @@ -43,7 +43,7 @@ public ExportDataValidationTests(ExportTestFixture fixture, ITestOutputHelper te [Fact] public async Task GivenFhirServer_WhenGroupDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() { - // NOTE: Azure Storage Emulator is required to run these tests locally. + // NOTE: Azure Storage Emulator or Azurite is required to run these tests locally. // Add data for test var (dataInFhirServer, groupId) = await CreateGroupWithPatient(true); @@ -53,7 +53,7 @@ public async Task GivenFhirServer_WhenGroupDataIsExported_ThenExportedDataIsSame IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); // Download exported data from storage account - Dictionary<(string resourceType, string resourceId), Resource> dataFromExport = + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); // Assert both sets of data are equal @@ -63,7 +63,7 @@ public async Task GivenFhirServer_WhenGroupDataIsExported_ThenExportedDataIsSame [Fact] public async Task GivenFhirServer_WhenGroupDataIsExportedWithTypeParameter_ThenExportedDataIsSameAsDataInFhirServer() { - // NOTE: Azure Storage Emulator is required to run these tests locally. + // NOTE: Azure Storage Emulator or Azurite is required to run these tests locally. // Add data for test var (dataInFhirServer, groupId) = await CreateGroupWithPatient(false); @@ -73,7 +73,7 @@ public async Task GivenFhirServer_WhenGroupDataIsExportedWithTypeParameter_ThenE IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); // Download exported data from storage account - Dictionary<(string resourceType, string resourceId), Resource> dataFromExport = + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); // Assert both sets of data are equal @@ -83,7 +83,7 @@ public async Task GivenFhirServer_WhenGroupDataIsExportedWithTypeParameter_ThenE [Fact] public async Task GivenFhirServer_WhenGroupDataWithNoMemberPatientIdIsExported_ThenNoDataIsExported() { - // NOTE: Azure Storage Emulator is required to run these tests locally. + // NOTE: Azure Storage Emulator or Azurite is required to run these tests locally. // Add data for test string groupId = await CreateGroupWithoutPatientIds(); @@ -110,7 +110,7 @@ async Task CreateGroupWithoutPatientIds() [Fact] public async Task GivenFhirServer_WhenDataIsExported_ThenExportTaskMetricsNotificationShouldBePosted() { - // NOTE: Azure Storage Emulator is required to run these tests locally. + // NOTE: Azure Storage Emulator or Azurite is required to run these tests locally. if (!_fixture.IsUsingInProcTestServer) { @@ -132,12 +132,13 @@ public async Task GivenFhirServer_WhenDataIsExported_ThenExportTaskMetricsNotifi Assert.Single(_fixture.MetricHandler.NotificationMapping[typeof(ExportTaskMetricsNotification)]); } - private async Task<(Dictionary<(string resourceType, string resourceId), Resource> serverData, string groupId)> CreateGroupWithPatient(bool includeAllResources) + private async Task<(Dictionary<(string resourceType, string resourceId, string versionId), Resource> serverData, string groupId)> CreateGroupWithPatient(bool includeAllResources) { // Add data for test var patient = new Patient(); var patientResponse = await _testFhirClient.CreateAsync(patient); var patientId = patientResponse.Resource.Id; + var patientVersionId = patientResponse.Resource.VersionId; var relative = new RelatedPerson() { @@ -146,6 +147,7 @@ public async Task GivenFhirServer_WhenDataIsExported_ThenExportTaskMetricsNotifi var relativeResponse = await _testFhirClient.CreateAsync(relative); var relativeId = relativeResponse.Resource.Id; + var relativeVersionId = relativeResponse.Resource.VersionId; var encounter = new Encounter() { @@ -159,6 +161,7 @@ public async Task GivenFhirServer_WhenDataIsExported_ThenExportTaskMetricsNotifi var encounterResponse = await _testFhirClient.CreateAsync(encounter); var encounterId = encounterResponse.Resource.Id; + var encounterVersionId = encounterResponse.Resource.VersionId; var observation = new Observation() { @@ -178,6 +181,7 @@ public async Task GivenFhirServer_WhenDataIsExported_ThenExportTaskMetricsNotifi var observationResponse = await _testFhirClient.CreateAsync(observation); var observationId = observationResponse.Resource.Id; + var observationVersionId = observationResponse.Resource.VersionId; var group = new FhirGroup() { @@ -194,16 +198,17 @@ public async Task GivenFhirServer_WhenDataIsExported_ThenExportTaskMetricsNotifi var groupResponse = await _testFhirClient.CreateAsync(group); var groupId = groupResponse.Resource.Id; + var groupVersionId = groupResponse.Resource.VersionId; - var resourceDictionary = new Dictionary<(string resourceType, string resourceId), Resource>(); - resourceDictionary.Add((KnownResourceTypes.RelatedPerson, relativeId), relativeResponse.Resource); - resourceDictionary.Add((KnownResourceTypes.Encounter, encounterId), encounterResponse.Resource); + var resourceDictionary = new Dictionary<(string resourceType, string resourceId, string versionId), Resource>(); + resourceDictionary.Add((KnownResourceTypes.RelatedPerson, relativeId, relativeVersionId), relativeResponse.Resource); + resourceDictionary.Add((KnownResourceTypes.Encounter, encounterId, encounterVersionId), encounterResponse.Resource); if (includeAllResources) { - resourceDictionary.Add((KnownResourceTypes.Patient, patientId), patientResponse.Resource); - resourceDictionary.Add((KnownResourceTypes.Observation, observationId), observationResponse.Resource); - resourceDictionary.Add((KnownResourceTypes.Group, groupId), groupResponse.Resource); + resourceDictionary.Add((KnownResourceTypes.Patient, patientId, patientVersionId), patientResponse.Resource); + resourceDictionary.Add((KnownResourceTypes.Observation, observationId, observationVersionId), observationResponse.Resource); + resourceDictionary.Add((KnownResourceTypes.Group, groupId, groupVersionId), groupResponse.Resource); } return (resourceDictionary, groupId); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportLongRunningTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportLongRunningTests.cs index 5fad2be130..730934923f 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportLongRunningTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportLongRunningTests.cs @@ -8,6 +8,7 @@ using System.Linq; using Hl7.Fhir.Model; using Hl7.Fhir.Serialization; +using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Microsoft.Health.Fhir.Tests.E2E.Common; @@ -28,6 +29,7 @@ public class ExportLongRunningTests : IClassFixture private readonly ITestOutputHelper _outputHelper; private readonly FhirJsonParser _fhirJsonParser; private readonly ExportTestFixture _fixture; + private readonly Guid _resourceTag; public ExportLongRunningTests(ExportTestFixture fixture, ITestOutputHelper testOutputHelper) { @@ -35,23 +37,24 @@ public ExportLongRunningTests(ExportTestFixture fixture, ITestOutputHelper testO _outputHelper = testOutputHelper; _fhirJsonParser = new FhirJsonParser(); _fixture = fixture; + _resourceTag = Guid.NewGuid(); } [Fact] public async Task GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() { - // NOTE: Azure Storage Emulator is required to run these tests locally. + // NOTE: Azure Storage Emulator or Azurite is required to run these tests locally. // Trigger export request and check for export status Uri contentLocation = await _testFhirClient.ExportAsync(); IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); // Download exported data from storage account - Dictionary<(string resourceType, string resourceId), Resource> dataFromExport = + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); // Download all resources from fhir server - Dictionary<(string resourceType, string resourceId), Resource> dataFromFhirServer = + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromFhirServer = await ExportTestHelper.GetResourcesFromFhirServer(_testFhirClient, _testFhirClient.HttpClient.BaseAddress, _fhirJsonParser, _outputHelper); // Assert both data are equal @@ -61,22 +64,24 @@ public async Task GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAs [Fact] public async Task GivenFhirServer_WhenPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() { - // NOTE: Azure Storage Emulator is required to run these tests locally. + // NOTE: Azure Storage Emulator or Azurite is required to run these tests locally. // Trigger export request and check for export status Uri contentLocation = await _testFhirClient.ExportAsync("Patient/"); IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); // Download exported data from storage account - Dictionary<(string resourceType, string resourceId), Resource> dataFromExport = + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); // Download resources from fhir server Uri address = new Uri(_testFhirClient.HttpClient.BaseAddress, "Patient/"); - Dictionary<(string resourceType, string resourceId), Resource> dataFromFhirServer = await ExportTestHelper.GetResourcesFromFhirServer(_testFhirClient, address, _fhirJsonParser, _outputHelper); + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromFhirServer = + await ExportTestHelper.GetResourcesFromFhirServer(_testFhirClient, address, _fhirJsonParser, _outputHelper); - Dictionary<(string resourceType, string resourceId), Resource> compartmentData = new Dictionary<(string resourceType, string resourceId), Resource>(); - foreach ((string resourceType, string resourceId) key in dataFromFhirServer.Keys) + Dictionary<(string resourceType, string resourceId, string versionId), Resource> compartmentData = new(); + + foreach ((string resourceType, string resourceId, string versionId) key in dataFromFhirServer.Keys) { address = new Uri(_testFhirClient.HttpClient.BaseAddress, "Patient/" + key.resourceId + "/*"); @@ -96,19 +101,19 @@ public async Task GivenFhirServer_WhenPatientDataIsExported_ThenExportedDataIsSa [Fact] public async Task GivenFhirServer_WhenAllObservationAndPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() { - // NOTE: Azure Storage Emulator is required to run these tests locally. + // NOTE: Azure Storage Emulator or Azurite is required to run these tests locally. // Trigger export request and check for export status Uri contentLocation = await _testFhirClient.ExportAsync(string.Empty, "_type=Observation,Patient"); IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); // Download exported data from storage account - Dictionary<(string resourceType, string resourceId), Resource> dataFromExport = + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); // Download resources from fhir server Uri address = new Uri(_testFhirClient.HttpClient.BaseAddress, "?_type=Observation,Patient"); - Dictionary<(string resourceType, string resourceId), Resource> dataFromFhirServer = + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromFhirServer = await ExportTestHelper.GetResourcesFromFhirServer(_testFhirClient, address, _fhirJsonParser, _outputHelper); // Assert both data are equal @@ -118,23 +123,23 @@ public async Task GivenFhirServer_WhenAllObservationAndPatientDataIsExported_The [Fact] public async Task GivenFhirServer_WhenPatientObservationDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() { - // NOTE: Azure Storage Emulator is required to run these tests locally. + // NOTE: Azure Storage Emulator or Azurite is required to run these tests locally. // Trigger export request and check for export status Uri contentLocation = await _testFhirClient.ExportAsync("Patient/", "_type=Observation"); IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); // Download exported data from storage account - Dictionary<(string resourceType, string resourceId), Resource> dataFromExport = + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); // Download resources from fhir server Uri address = new Uri(_testFhirClient.HttpClient.BaseAddress, "Patient/"); - Dictionary<(string resourceType, string resourceId), Resource> patientData = + Dictionary<(string resourceType, string resourceId, string versionId), Resource> patientData = await ExportTestHelper.GetResourcesFromFhirServer(_testFhirClient, address, _fhirJsonParser, _outputHelper); - Dictionary<(string resourceType, string resourceId), Resource> compartmentData = new Dictionary<(string resourceType, string resourceId), Resource>(); - foreach ((string resourceType, string resourceId) key in patientData.Keys) + Dictionary<(string resourceType, string resourceId, string versionId), Resource> compartmentData = new(); + foreach ((string resourceType, string resourceId, string versionId) key in patientData.Keys) { address = new Uri(_testFhirClient.HttpClient.BaseAddress, "Patient/" + key.resourceId + "/Observation"); @@ -154,7 +159,7 @@ public async Task GivenFhirServer_WhenPatientObservationDataIsExported_ThenExpor [Fact] public async Task GivenFhirServer_WhenAllDataIsExportedToASpecificContainer_ThenExportedDataIsInTheSpecifiedContianer() { - // NOTE: Azure Storage Emulator is required to run these tests locally. + // NOTE: Azure Storage Emulator or Azurite is required to run these tests locally. string testContainer = "test-container"; @@ -163,16 +168,118 @@ public async Task GivenFhirServer_WhenAllDataIsExportedToASpecificContainer_Then IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); // Download exported data from storage account - Dictionary<(string resourceType, string resourceId), Resource> dataFromExport = + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); // Download all resources from fhir server - Dictionary<(string resourceType, string resourceId), Resource> dataFromFhirServer = + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromFhirServer = await ExportTestHelper.GetResourcesFromFhirServer(_testFhirClient, _testFhirClient.HttpClient.BaseAddress, _fhirJsonParser, _outputHelper); // Assert both data are equal Assert.True(ExportTestHelper.ValidateDataFromBothSources(dataFromFhirServer, dataFromExport, _outputHelper)); Assert.True(blobUris.All((url) => url.OriginalString.Contains(testContainer))); } + + [Fact] + public async Task GivenFhirServer_WhenDataIsExportedWithHistoryAndSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer() + { + // NOTE: Azure Storage Emulator or Azurite is required to run these tests locally. + var resourceDictionary = await CreatePatientWithObservations(true, true); + + // NOTE: Azure Storage Emulator is required to run these tests locally. + + // Trigger export request and check for export status + Uri contentLocation = await _testFhirClient.ExportAsync(); + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); + + // Download exported data from storage account + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = + await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); + + // Download all resources from fhir server + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromFhirServer = + await ExportTestHelper.GetResourcesWithHistoryFromFhirServer(_testFhirClient, new Uri($"{_testFhirClient.HttpClient.BaseAddress}/?_tag={_resourceTag}"), _fhirJsonParser, _outputHelper); + + // Assert both data are equal + Assert.True(ExportTestHelper.ValidateDataFromBothSources(dataFromFhirServer, dataFromExport, _outputHelper)); + } + + private async System.Threading.Tasks.Task> CreatePatientWithObservations(bool addVersion, bool softDelete) + { + // Add data for test + var patient = new Patient() + { + Meta = new Meta + { + Tag = new List + { + new("http://e2e-test", _resourceTag.ToString()), + }, + }, + }; + + var patientResponse = await _testFhirClient.CreateAsync(patient); + var patientId = patientResponse.Resource.Id; + var patientVersionId = patientResponse.Resource.VersionId; + + var observation = new Observation() + { + Meta = new Meta + { + Tag = new List + { + new("http://e2e-test", _resourceTag.ToString()), + }, + }, + Status = ObservationStatus.Final, + Code = new CodeableConcept("http://loinc.org", "12345-6"), + Subject = new ResourceReference($"{KnownResourceTypes.Patient}/{patientId}"), + }; + + var observationResponse = await _testFhirClient.CreateAsync(observation); + var observationId = observationResponse.Resource.Id; + var observationVersionId = observationResponse.Resource.VersionId; + + var resourceDictionary = new Dictionary<(string resourceType, string resourceId, string versionId), Resource> + { + { (KnownResourceTypes.Patient, patientId, patientVersionId), patientResponse.Resource }, + { (KnownResourceTypes.Observation, observationId, observationVersionId), observationResponse.Resource }, + }; + + if (addVersion) + { + patient.BirthDate = "2000-01-01"; + patientResponse = await _testFhirClient.UpdateAsync(patient); + patientVersionId = patientResponse.Resource.VersionId; + + observation.Issued = new DateTimeOffset(new DateTime(2020, 07, 05)); + observationResponse = await _testFhirClient.UpdateAsync(observation); + observationVersionId = observationResponse.Resource.VersionId; + + resourceDictionary = new Dictionary<(string resourceType, string resourceId, string versionId), Resource> + { + { (KnownResourceTypes.Patient, patientId, patientVersionId), patientResponse.Resource }, + { (KnownResourceTypes.Observation, observationId, observationVersionId), observationResponse.Resource }, + }; + } + + if (softDelete) + { + // The response does not give us the version of the deleted resource, so we need to get it from the headers. + // [2..^1] removes the starting W" and end " + var patientDeleteResponse = await _testFhirClient.DeleteAsync(patient); + var patientDeleteVersion = patientDeleteResponse.Response.Headers.ETag.Tag[2..^1]; + var observationDeleteResponse = await _testFhirClient.DeleteAsync(observation); + var observationDeleteVersion = observationDeleteResponse.Response.Headers.ETag.Tag[2..^1]; + + resourceDictionary = new Dictionary<(string resourceType, string resourceId, string versionId), Resource> + { + { (KnownResourceTypes.Patient, patientId, patientDeleteVersion), null }, + { (KnownResourceTypes.Observation, observationId, observationDeleteVersion), null }, + }; + } + + return resourceDictionary; + } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestHelper.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestHelper.cs index 4dd79eb599..9a4f1355e4 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestHelper.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestHelper.cs @@ -63,14 +63,69 @@ internal static async Task> CheckExportStatus(TestFhirClient testFhir return exportJobResult.Output.Select(x => x.FileUri).ToList(); } - internal static async Task> GetResourcesFromFhirServer( + internal static async Task> GetResourcesFromFhirServer( TestFhirClient testFhirClient, Uri requestUri, FhirJsonParser fhirJsonParser, ITestOutputHelper outputHelper) { - var resourceIdToResourceMapping = new Dictionary<(string resourceType, string resourceId), Resource>(); + var resourceIdToResourceMapping = new Dictionary<(string resourceType, string resourceId, string versionId), Resource>(); + try + { + await foreach (Resource resource in GetResourceListFromFhirServer(testFhirClient, requestUri, fhirJsonParser)) + { + resourceIdToResourceMapping.TryAdd((resource.TypeName, resource.Id, resource.VersionId), resource); + } + } + catch (Exception ex) + { + outputHelper.WriteLine($"Unable to parse response into bundle: {ex}"); + return resourceIdToResourceMapping; + } + + return resourceIdToResourceMapping; + } + + internal static async Task> GetResourcesWithHistoryFromFhirServer( + TestFhirClient testFhirClient, + Uri requestUri, + FhirJsonParser fhirJsonParser, + ITestOutputHelper outputHelper) + { + var resourceIdToResourceMapping = new Dictionary<(string resourceType, string resourceId, string versionId), Resource>(); + + try + { + await foreach (Resource resource in GetResourceListFromFhirServer(testFhirClient, requestUri, fhirJsonParser)) + { + string resourceWithHistoryUriString = $"{testFhirClient.HttpClient.BaseAddress}/{resource.TypeName}/{resource.Id}/_history"; + + if (requestUri.Query is not null) + { + resourceWithHistoryUriString += requestUri.Query; + } + + await foreach (Resource historyResource in GetResourceListFromFhirServer(testFhirClient, new Uri(resourceWithHistoryUriString), fhirJsonParser)) + { + resourceIdToResourceMapping.TryAdd((historyResource.TypeName, historyResource.Id, historyResource.VersionId), historyResource); + } + } + } + catch (Exception ex) + { + outputHelper.WriteLine($"Unable to parse response into bundle: {ex}"); + return resourceIdToResourceMapping; + } + + return resourceIdToResourceMapping; + } + + private static async IAsyncEnumerable GetResourceListFromFhirServer( + TestFhirClient testFhirClient, + Uri requestUri, + FhirJsonParser fhirJsonParser) + { while (requestUri != null) { HttpRequestMessage request = new HttpRequestMessage() @@ -82,28 +137,17 @@ internal static async Task> CheckExportStatus(TestFhirClient testFhir using HttpResponseMessage response = await testFhirClient.HttpClient.SendAsync(request); var responseString = await response.Content.ReadAsStringAsync(); - Bundle searchResults; - try - { - searchResults = fhirJsonParser.Parse(responseString); - } - catch (Exception ex) - { - outputHelper.WriteLine($"Unable to parse response into bundle: {ex}"); - return resourceIdToResourceMapping; - } - - foreach (Bundle.EntryComponent entry in searchResults.Entry) - { - resourceIdToResourceMapping.TryAdd((entry.Resource.TypeName, entry.Resource.Id), entry.Resource); - } + Bundle searchResults = fhirJsonParser.Parse(responseString); // Look at whether a continuation token has been returned. string nextLink = searchResults.NextLink?.ToString(); requestUri = nextLink == null ? null : new Uri(nextLink); - } - return resourceIdToResourceMapping; + foreach (Bundle.EntryComponent entry in searchResults.Entry) + { + yield return entry.Resource; + } + } } internal static (StorageSharedKeyCredential credential, string connectionString) GetStorageCredentialOrConnectionString(Uri storageServiceUri) @@ -152,14 +196,14 @@ internal static (StorageSharedKeyCredential credential, string connectionString) return (storageCredential, null); } - internal static async Task> DownloadBlobAndParse( + internal static async Task> DownloadBlobAndParse( IList blobUri, FhirJsonParser fhirJsonParser, ITestOutputHelper outputHelper) { if (blobUri == null || blobUri.Count == 0) { - return new Dictionary<(string resourceType, string resourceId), Resource>(); + return new Dictionary<(string resourceType, string resourceId, string versionId), Resource>(); } // Extract storage account name from blob uri in order to get corresponding access token. @@ -167,7 +211,7 @@ internal static (StorageSharedKeyCredential credential, string connectionString) Uri storageServiceUri = new UriBuilder(sampleUri.Scheme, sampleUri.Host).Uri; (StorageSharedKeyCredential credential, string connectionString) = GetStorageCredentialOrConnectionString(storageServiceUri); - var resourceIdToResourceMapping = new Dictionary<(string resourceType, string resourceId), Resource>(); + var resourceIdToResourceMapping = new Dictionary<(string resourceType, string resourceId, string versionId), Resource>(); var localRun = credential == null; foreach (Uri uri in blobUri) @@ -197,7 +241,7 @@ internal static (StorageSharedKeyCredential credential, string connectionString) // Ideally this should just be Add, but until we prevent duplicates from being added to the server // there is a chance the same resource being added multiple times resulting in a key conflict. - resourceIdToResourceMapping.TryAdd((resource.TypeName, resource.Id), resource); + resourceIdToResourceMapping.TryAdd((resource.TypeName, resource.Id, resource.VersionId), resource); } } @@ -205,8 +249,8 @@ internal static (StorageSharedKeyCredential credential, string connectionString) } internal static bool ValidateDataFromBothSources( - Dictionary<(string resourceType, string resourceId), Resource> dataFromServer, - Dictionary<(string resourceType, string resourceId), Resource> dataFromStorageAccount, + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromServer, + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromStorageAccount, ITestOutputHelper outputHelper) { bool result = true; @@ -216,7 +260,7 @@ internal static bool ValidateDataFromBothSources( outputHelper.WriteLine($"Count differs. Exported data count: {dataFromStorageAccount.Count} Fhir Server Count: {dataFromServer.Count}"); result = false; - foreach (KeyValuePair<(string resourceType, string resourceId), Resource> kvp in dataFromStorageAccount) + foreach (KeyValuePair<(string resourceType, string resourceId, string versionId), Resource> kvp in dataFromStorageAccount) { if (!dataFromServer.ContainsKey(kvp.Key)) { @@ -235,7 +279,7 @@ internal static bool ValidateDataFromBothSources( */ int wrongCount = 0; - foreach (KeyValuePair<(string resourceType, string resourceId), Resource> kvp in dataFromServer) + foreach (KeyValuePair<(string resourceType, string resourceId, string versionId), Resource> kvp in dataFromServer) { if (!dataFromStorageAccount.ContainsKey(kvp.Key)) { From 0e3f95ecdcf056b0857d8f793affcce80b759660 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Mon, 11 Sep 2023 10:36:44 -0700 Subject: [PATCH 06/58] Updated export E2E tests to use test data from fixture. --- .../Resources.Designer.cs | 9 + src/Microsoft.Health.Fhir.Api/Resources.resx | 3 + .../Controllers/ExportControllerTests.cs | 22 +++ .../Controllers/ExportController.cs | 16 +- .../Rest/ExportLongRunningTests.cs | 181 +++--------------- .../Rest/ExportTestFixture.cs | 100 ++++++++++ 6 files changed, 171 insertions(+), 160 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Api/Resources.Designer.cs b/src/Microsoft.Health.Fhir.Api/Resources.Designer.cs index 0e81b2c21e..987404cd81 100644 --- a/src/Microsoft.Health.Fhir.Api/Resources.Designer.cs +++ b/src/Microsoft.Health.Fhir.Api/Resources.Designer.cs @@ -806,5 +806,14 @@ public static string WelcomeTitle { return ResourceManager.GetString("WelcomeTitle", resourceCulture); } } + + /// + /// Looks up a localized string similar to The request "_typeFilter" cannot be used with an export request with historical or soft deleted resources. + /// + public static string TypeFilterNotSupportedWithHistoryOrDeletedExport { + get { + return ResourceManager.GetString("TypeFilterNotSupportedWithHistoryOrDeletedExport", resourceCulture); + } + } } } diff --git a/src/Microsoft.Health.Fhir.Api/Resources.resx b/src/Microsoft.Health.Fhir.Api/Resources.resx index 4746b029eb..f345ac9dad 100644 --- a/src/Microsoft.Health.Fhir.Api/Resources.resx +++ b/src/Microsoft.Health.Fhir.Api/Resources.resx @@ -408,4 +408,7 @@ Invalid combination of processing logic and bundle type: {0} and {1}. Error message when there is a invalid/unknown combination of a bundle type and a processing logic. + + The request "_typeFilter" cannot be used with an export request with historical or soft deleted resources. + \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/ExportControllerTests.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/ExportControllerTests.cs index 750c2bccf3..b678860e43 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/ExportControllerTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/ExportControllerTests.cs @@ -206,6 +206,28 @@ await Assert.ThrowsAsync(() => exportController.Export typeParameter: ResourceType.Patient.ToString())); } + [Fact] + public async Task GivenAnExportRequestWithHistoryOrDeletedIncluded_WhenHasTypeFilter_ThenRequestNotValidExceptionShouldBeThrown() + { + await Assert.ThrowsAsync(() => _exportEnabledController.Export( + since: null, + till: null, + resourceType: ResourceType.Patient.ToString(), + containerName: null, + formatName: null, + typeFilter: "Patient%3Factive%3Dtrue", + includeHistory: true)); + + await Assert.ThrowsAsync(() => _exportEnabledController.Export( + since: null, + till: null, + resourceType: ResourceType.Patient.ToString(), + containerName: null, + formatName: null, + typeFilter: "Patient%3Factive%3Dtrue", + includeDeleted: true)); + } + // We can configure OciArtifacts through three fields: LoginServer, ImageName and Digest // If ImageName and Digest are null, all images under the specified LoginSever are allowed to be used. // Similarly, if LoginSever and ImageName are specified and Digest is empty, all digests under the specified ImageName are allowed to be used. diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs b/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs index dc0e4da119..506241b9f2 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs @@ -105,6 +105,7 @@ public async Task Export( { CheckIfExportIsEnabled(); ValidateForAnonymizedExport(containerName, anonymizationConfigCollectionReference, anonymizationConfigLocation, anonymizationConfigFileETag); + ValidateForHistoryOrSoftDeletedExport(includeHistory, includeDeleted, typeFilter); return await SendExportRequest( exportType: ExportJobType.All, @@ -300,7 +301,7 @@ private void ValidateForAnonymizedExport(string containerName, string anonymizat { CheckReferenceAndETagParameterConflictForAnonymizedExport(anonymizationConfigCollectionReference, anonymizationConfigFileETag); CheckConfigCollectionReferenceIsValid(anonymizationConfigCollectionReference); - CheckIfConfigCollectionReferencIsConfigured(anonymizationConfigCollectionReference); + CheckIfConfigCollectionReferenceIsConfigured(anonymizationConfigCollectionReference); } } } @@ -334,7 +335,7 @@ private static void CheckReferenceAndETagParameterConflictForAnonymizedExport(st } } - private void CheckIfConfigCollectionReferencIsConfigured(string anonymizationConfigCollectionReference) + private void CheckIfConfigCollectionReferenceIsConfigured(string anonymizationConfigCollectionReference) { var ociImage = ImageInfo.CreateFromImageReference(anonymizationConfigCollectionReference); @@ -352,5 +353,16 @@ private void CheckIfConfigCollectionReferencIsConfigured(string anonymizationCon throw new RequestNotValidException(string.Format(Resources.AnonymizationConfigCollectionNotConfigured, anonymizationConfigCollectionReference)); } } + + private static void ValidateForHistoryOrSoftDeletedExport(bool includeHistory, bool includeDeleted, string typeFilter) + { + if (includeHistory || includeDeleted) + { + if (!string.IsNullOrWhiteSpace(typeFilter)) + { + throw new RequestNotValidException(Resources.TypeFilterNotSupportedWithHistoryOrDeletedExport); + } + } + } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportLongRunningTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportLongRunningTests.cs index 730934923f..b7bf8d8cdd 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportLongRunningTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportLongRunningTests.cs @@ -29,7 +29,6 @@ public class ExportLongRunningTests : IClassFixture private readonly ITestOutputHelper _outputHelper; private readonly FhirJsonParser _fhirJsonParser; private readonly ExportTestFixture _fixture; - private readonly Guid _resourceTag; public ExportLongRunningTests(ExportTestFixture fixture, ITestOutputHelper testOutputHelper) { @@ -37,249 +36,115 @@ public ExportLongRunningTests(ExportTestFixture fixture, ITestOutputHelper testO _outputHelper = testOutputHelper; _fhirJsonParser = new FhirJsonParser(); _fixture = fixture; - _resourceTag = Guid.NewGuid(); } [Fact] public async Task GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() { - // NOTE: Azure Storage Emulator or Azurite is required to run these tests locally. + // NOTE: Azurite is required to run these tests locally. // Trigger export request and check for export status - Uri contentLocation = await _testFhirClient.ExportAsync(); + Uri contentLocation = await _testFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:o}"); IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); - // Download all resources from fhir server - Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromFhirServer = - await ExportTestHelper.GetResourcesFromFhirServer(_testFhirClient, _testFhirClient.HttpClient.BaseAddress, _fhirJsonParser, _outputHelper); - // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(dataFromFhirServer, dataFromExport, _outputHelper)); + Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResources, dataFromExport, _outputHelper)); } [Fact] public async Task GivenFhirServer_WhenPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() { - // NOTE: Azure Storage Emulator or Azurite is required to run these tests locally. + // NOTE: Azurite is required to run these tests locally. // Trigger export request and check for export status - Uri contentLocation = await _testFhirClient.ExportAsync("Patient/"); + Uri contentLocation = await _testFhirClient.ExportAsync(path: "Patient/", parameters: _fixture.ExportTestResourcesQueryParameters); IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); - // Download resources from fhir server - Uri address = new Uri(_testFhirClient.HttpClient.BaseAddress, "Patient/"); - Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromFhirServer = - await ExportTestHelper.GetResourcesFromFhirServer(_testFhirClient, address, _fhirJsonParser, _outputHelper); - - Dictionary<(string resourceType, string resourceId, string versionId), Resource> compartmentData = new(); - - foreach ((string resourceType, string resourceId, string versionId) key in dataFromFhirServer.Keys) - { - address = new Uri(_testFhirClient.HttpClient.BaseAddress, "Patient/" + key.resourceId + "/*"); - - // copies all the new values into the compartment data dictionary - (await ExportTestHelper.GetResourcesFromFhirServer(_testFhirClient, address, _fhirJsonParser, _outputHelper)) - .ToList() - .ForEach(x => compartmentData.TryAdd(x.Key, x.Value)); - } - - compartmentData.ToList().ForEach(x => dataFromFhirServer.TryAdd(x.Key, x.Value)); - dataFromFhirServer.Union(compartmentData); - // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(dataFromFhirServer, dataFromExport, _outputHelper)); + Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResources, dataFromExport, _outputHelper)); } [Fact] public async Task GivenFhirServer_WhenAllObservationAndPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() { - // NOTE: Azure Storage Emulator or Azurite is required to run these tests locally. + // NOTE: Azurite is required to run these tests locally. // Trigger export request and check for export status - Uri contentLocation = await _testFhirClient.ExportAsync(string.Empty, "_type=Observation,Patient"); + Uri contentLocation = await _testFhirClient.ExportAsync(parameters: $"_type=Observation,Patient&{_fixture.ExportTestResourcesQueryParameters}"); IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); - // Download resources from fhir server - Uri address = new Uri(_testFhirClient.HttpClient.BaseAddress, "?_type=Observation,Patient"); - Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromFhirServer = - await ExportTestHelper.GetResourcesFromFhirServer(_testFhirClient, address, _fhirJsonParser, _outputHelper); - // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(dataFromFhirServer, dataFromExport, _outputHelper)); + Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResources, dataFromExport, _outputHelper)); } [Fact] public async Task GivenFhirServer_WhenPatientObservationDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() { - // NOTE: Azure Storage Emulator or Azurite is required to run these tests locally. + // NOTE: Azurite is required to run these tests locally. // Trigger export request and check for export status - Uri contentLocation = await _testFhirClient.ExportAsync("Patient/", "_type=Observation"); + Uri contentLocation = await _testFhirClient.ExportAsync("Patient/", $"_type=Observation&_typeFilter=Observation%3F_tag%3D{_fixture.FixtureTag}"); IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); - // Download resources from fhir server - Uri address = new Uri(_testFhirClient.HttpClient.BaseAddress, "Patient/"); - Dictionary<(string resourceType, string resourceId, string versionId), Resource> patientData = - await ExportTestHelper.GetResourcesFromFhirServer(_testFhirClient, address, _fhirJsonParser, _outputHelper); + var expectedResources = _fixture.TestResources + .Where(r => r.Key.resourceType == ResourceType.Observation.ToString()) + .ToDictionary(x => x.Key, x => x.Value); - Dictionary<(string resourceType, string resourceId, string versionId), Resource> compartmentData = new(); - foreach ((string resourceType, string resourceId, string versionId) key in patientData.Keys) - { - address = new Uri(_testFhirClient.HttpClient.BaseAddress, "Patient/" + key.resourceId + "/Observation"); - - // copies all the new values into the compartment data dictionary - (await ExportTestHelper.GetResourcesFromFhirServer(_testFhirClient, address, _fhirJsonParser, _outputHelper)) - .ToList() - .ForEach(x => compartmentData.TryAdd(x.Key, x.Value)); - } - - compartmentData.ToList().ForEach(x => patientData.TryAdd(x.Key, x.Value)); - patientData.Union(compartmentData); - - // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(compartmentData, dataFromExport, _outputHelper)); + // Assert both data are equal. Only Observation data is expected due to the type query parameter. + Assert.True(ExportTestHelper.ValidateDataFromBothSources(expectedResources, dataFromExport, _outputHelper)); } [Fact] public async Task GivenFhirServer_WhenAllDataIsExportedToASpecificContainer_ThenExportedDataIsInTheSpecifiedContianer() { - // NOTE: Azure Storage Emulator or Azurite is required to run these tests locally. + // NOTE: Azurite is required to run these tests locally. string testContainer = "test-container"; // Trigger export request and check for export status - Uri contentLocation = await _testFhirClient.ExportAsync(parameters: $"_container={testContainer}"); + Uri contentLocation = await _testFhirClient.ExportAsync(parameters: $"_container={testContainer}&{_fixture.ExportTestResourcesQueryParameters}"); IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); - // Download all resources from fhir server - Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromFhirServer = - await ExportTestHelper.GetResourcesFromFhirServer(_testFhirClient, _testFhirClient.HttpClient.BaseAddress, _fhirJsonParser, _outputHelper); - // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(dataFromFhirServer, dataFromExport, _outputHelper)); + Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResources, dataFromExport, _outputHelper)); Assert.True(blobUris.All((url) => url.OriginalString.Contains(testContainer))); } [Fact] public async Task GivenFhirServer_WhenDataIsExportedWithHistoryAndSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer() { - // NOTE: Azure Storage Emulator or Azurite is required to run these tests locally. - var resourceDictionary = await CreatePatientWithObservations(true, true); - - // NOTE: Azure Storage Emulator is required to run these tests locally. + // NOTE: Azurite is required to run these tests locally. - // Trigger export request and check for export status - Uri contentLocation = await _testFhirClient.ExportAsync(); + // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. + Uri contentLocation = await _testFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:O}&_includeHistory=true&_includeDeleted=true"); IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); - // Download all resources from fhir server - Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromFhirServer = - await ExportTestHelper.GetResourcesWithHistoryFromFhirServer(_testFhirClient, new Uri($"{_testFhirClient.HttpClient.BaseAddress}/?_tag={_resourceTag}"), _fhirJsonParser, _outputHelper); - // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(dataFromFhirServer, dataFromExport, _outputHelper)); - } - - private async System.Threading.Tasks.Task> CreatePatientWithObservations(bool addVersion, bool softDelete) - { - // Add data for test - var patient = new Patient() - { - Meta = new Meta - { - Tag = new List - { - new("http://e2e-test", _resourceTag.ToString()), - }, - }, - }; - - var patientResponse = await _testFhirClient.CreateAsync(patient); - var patientId = patientResponse.Resource.Id; - var patientVersionId = patientResponse.Resource.VersionId; - - var observation = new Observation() - { - Meta = new Meta - { - Tag = new List - { - new("http://e2e-test", _resourceTag.ToString()), - }, - }, - Status = ObservationStatus.Final, - Code = new CodeableConcept("http://loinc.org", "12345-6"), - Subject = new ResourceReference($"{KnownResourceTypes.Patient}/{patientId}"), - }; - - var observationResponse = await _testFhirClient.CreateAsync(observation); - var observationId = observationResponse.Resource.Id; - var observationVersionId = observationResponse.Resource.VersionId; - - var resourceDictionary = new Dictionary<(string resourceType, string resourceId, string versionId), Resource> - { - { (KnownResourceTypes.Patient, patientId, patientVersionId), patientResponse.Resource }, - { (KnownResourceTypes.Observation, observationId, observationVersionId), observationResponse.Resource }, - }; - - if (addVersion) - { - patient.BirthDate = "2000-01-01"; - patientResponse = await _testFhirClient.UpdateAsync(patient); - patientVersionId = patientResponse.Resource.VersionId; - - observation.Issued = new DateTimeOffset(new DateTime(2020, 07, 05)); - observationResponse = await _testFhirClient.UpdateAsync(observation); - observationVersionId = observationResponse.Resource.VersionId; - - resourceDictionary = new Dictionary<(string resourceType, string resourceId, string versionId), Resource> - { - { (KnownResourceTypes.Patient, patientId, patientVersionId), patientResponse.Resource }, - { (KnownResourceTypes.Observation, observationId, observationVersionId), observationResponse.Resource }, - }; - } - - if (softDelete) - { - // The response does not give us the version of the deleted resource, so we need to get it from the headers. - // [2..^1] removes the starting W" and end " - var patientDeleteResponse = await _testFhirClient.DeleteAsync(patient); - var patientDeleteVersion = patientDeleteResponse.Response.Headers.ETag.Tag[2..^1]; - var observationDeleteResponse = await _testFhirClient.DeleteAsync(observation); - var observationDeleteVersion = observationDeleteResponse.Response.Headers.ETag.Tag[2..^1]; - - resourceDictionary = new Dictionary<(string resourceType, string resourceId, string versionId), Resource> - { - { (KnownResourceTypes.Patient, patientId, patientDeleteVersion), null }, - { (KnownResourceTypes.Observation, observationId, observationDeleteVersion), null }, - }; - } - - return resourceDictionary; + Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResourcesWithHistory, dataFromExport, _outputHelper)); } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestFixture.cs index 70fa434a8a..d4fc7f8846 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestFixture.cs @@ -3,11 +3,16 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System; +using System.Collections.Generic; +using System.Linq; +using Hl7.Fhir.Model; using MediatR; using Microsoft.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Core.Features.Operations.Export; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Microsoft.Health.Fhir.Tests.E2E.Rest.Metric; +using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest { @@ -24,5 +29,100 @@ public MetricHandler MetricHandler { get => _metricHandler ?? (_metricHandler = (MetricHandler)(TestFhirServer as InProcTestFhirServer)?.Server.Host.Services.GetRequiredService>()); } + + public Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestResourcesWithHistory { get; } = new(); + + public Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestResources => TestResourcesWithHistory + .GroupBy(entry => entry.Key.resourceId) + .Select(group => group.OrderByDescending(entry => entry.Value.Meta.LastUpdated).First()) + .Where(entry => !entry.Value.Meta.Extension.Any(extension => + extension.Url == "http://azurehealthcareapis.com/data-extensions/deleted-state" + && ((FhirString)extension.Value).Value == "soft-deleted")) + .ToDictionary(entry => entry.Key, entry => entry.Value); + + public string FixtureTag { get; } = Guid.NewGuid().ToString(); + + public DateTime TestDataInsertionTime { get; } = DateTime.UtcNow; + + public string ExportTestResourcesQueryParameters => $"_type=Patient,Observation&_typeFilter=Patient%3F_tag%3D{FixtureTag},Observation%3F_tag%3D{FixtureTag}"; + + protected override async Task OnInitializedAsync() + { + string NewGuidString() => Guid.NewGuid().ToString(); + + void AddResourceToTestResources(Resource resource) => + TestResourcesWithHistory[(resource.TypeName, resource.Id, resource.VersionId)] = resource; + + Meta testDataMeta = new Meta() + { + Tag = new List() + { + new Coding("http://e2e-test", FixtureTag), + }, + }; + + List<(bool updateResource, bool deleteResource)> testResourceData = new() + { + (false, false), + (false, true), + (true, false), + (true, true), + }; + + foreach (var d in testResourceData) + { + var patient = new Patient() + { + Id = NewGuidString(), + Meta = testDataMeta, + }; + + var observation = new Observation() + { + Id = NewGuidString(), + Meta = testDataMeta, + Status = ObservationStatus.Final, + Code = new CodeableConcept("http://loinc.org", "12345-6"), + Subject = new ResourceReference($"Patient/{patient.Id}"), + }; + + var patientResponse = await TestFhirClient.UpdateAsync(patient); + var observationResponse = await TestFhirClient.UpdateAsync(observation); + + AddResourceToTestResources(patientResponse.Resource); + AddResourceToTestResources(observationResponse.Resource); + + if (d.updateResource) + { + patient = patientResponse.Resource.DeepCopy() as Patient; + patient.Name.Add(new HumanName() { Family = "Updated" }); + + observation = observationResponse.Resource.DeepCopy() as Observation; + observation.Code.Coding.Add(new Coding("http://loinc.org", "12345-7")); + + patientResponse = await TestFhirClient.UpdateAsync(patient); + observationResponse = await TestFhirClient.UpdateAsync(observation); + + AddResourceToTestResources(patientResponse.Resource); + AddResourceToTestResources(observationResponse.Resource); + } + + if (d.deleteResource) + { + await TestFhirClient.DeleteAsync($"Patient/{patient.Id}"); + await TestFhirClient.DeleteAsync($"Observation/{observation.Id}"); + + // Since we don't get the version id back from the delete response, we need to pull + // this data from the server. + foreach (var resourceInfo in new List<(string Type, string Id)> { ("Patient", patient.Id), ("Observation", observation.Id) }) + { + var allResourcesWithDeleted = await TestFhirClient.SearchAsync($"{resourceInfo.Type}/{resourceInfo.Id}/_history"); + var deletedResource = allResourcesWithDeleted.Resource.Entry.OrderByDescending(x => x.Resource.Meta.LastUpdated).First().Resource; + deletedResource.Meta.Extension.Add(new Extension("http://azurehealthcareapis.com/data-extensions/deleted-state", new FhirString("soft-deleted"))); + AddResourceToTestResources(deletedResource); + } + } + } + } } } From 6809ccfc9798a664b0fad25205446c90c68271fd Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Mon, 11 Sep 2023 11:01:41 -0700 Subject: [PATCH 07/58] Updated long export tests for individual history/delete scenarios --- .../Rest/ExportLongRunningTests.cs | 36 ++++++++++++++++++- .../Rest/ExportTestFixture.cs | 16 ++++++--- 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportLongRunningTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportLongRunningTests.cs index b7bf8d8cdd..f9beacce09 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportLongRunningTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportLongRunningTests.cs @@ -130,6 +130,40 @@ public async Task GivenFhirServer_WhenAllDataIsExportedToASpecificContainer_Then Assert.True(blobUris.All((url) => url.OriginalString.Contains(testContainer))); } + [Fact] + public async Task GivenFhirServer_WhenDataIsExportedWithHistory_ThenExportedDataIsSameAsDataInFhirServer() + { + // NOTE: Azurite is required to run these tests locally. + + // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. + Uri contentLocation = await _testFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:O}&_includeHistory=true"); + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); + + // Download exported data from storage account + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = + await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); + + // Assert both data are equal + Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResourcesWithHistory, dataFromExport, _outputHelper)); + } + + [Fact] + public async Task GivenFhirServer_WhenDataIsExportedWithSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer() + { + // NOTE: Azurite is required to run these tests locally. + + // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. + Uri contentLocation = await _testFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:O}&_includeDeleted=true"); + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); + + // Download exported data from storage account + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = + await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); + + // Assert both data are equal + Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResourcesWithDeletes, dataFromExport, _outputHelper)); + } + [Fact] public async Task GivenFhirServer_WhenDataIsExportedWithHistoryAndSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer() { @@ -144,7 +178,7 @@ public async Task GivenFhirServer_WhenDataIsExportedWithHistoryAndSoftDeletes_Th await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResourcesWithHistory, dataFromExport, _outputHelper)); + Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResourcesWithHistoryAndDeletes, dataFromExport, _outputHelper)); } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestFixture.cs index d4fc7f8846..4239c19f1f 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestFixture.cs @@ -30,16 +30,22 @@ public MetricHandler MetricHandler get => _metricHandler ?? (_metricHandler = (MetricHandler)(TestFhirServer as InProcTestFhirServer)?.Server.Host.Services.GetRequiredService>()); } - public Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestResourcesWithHistory { get; } = new(); + public Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestResourcesWithHistoryAndDeletes { get; } = new(); - public Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestResources => TestResourcesWithHistory - .GroupBy(entry => entry.Key.resourceId) - .Select(group => group.OrderByDescending(entry => entry.Value.Meta.LastUpdated).First()) + public Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestResourcesWithHistory => TestResourcesWithHistory .Where(entry => !entry.Value.Meta.Extension.Any(extension => extension.Url == "http://azurehealthcareapis.com/data-extensions/deleted-state" && ((FhirString)extension.Value).Value == "soft-deleted")) .ToDictionary(entry => entry.Key, entry => entry.Value); + public Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestResourcesWithDeletes => TestResourcesWithHistoryAndDeletes + .GroupBy(entry => entry.Key.resourceId) + .Select(group => group.OrderByDescending(entry => entry.Value.Meta.LastUpdated).First()) + .ToDictionary(entry => entry.Key, entry => entry.Value); + + public Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestResources => + TestResourcesWithHistory.Where(pair => TestResourcesWithDeletes.ContainsKey(pair.Key)).ToDictionary(pair => pair.Key, pair => pair.Value); + public string FixtureTag { get; } = Guid.NewGuid().ToString(); public DateTime TestDataInsertionTime { get; } = DateTime.UtcNow; @@ -51,7 +57,7 @@ protected override async Task OnInitializedAsync() string NewGuidString() => Guid.NewGuid().ToString(); void AddResourceToTestResources(Resource resource) => - TestResourcesWithHistory[(resource.TypeName, resource.Id, resource.VersionId)] = resource; + TestResourcesWithHistoryAndDeletes[(resource.TypeName, resource.Id, resource.VersionId)] = resource; Meta testDataMeta = new Meta() { From 2762608c315f32706c38f7f0ccbdb27617a2bf73 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Mon, 11 Sep 2023 11:31:06 -0700 Subject: [PATCH 08/58] Updated gitignore to ignore azurite temp files --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 6f9ce8e1c8..6db6704fc4 100644 --- a/.gitignore +++ b/.gitignore @@ -292,3 +292,6 @@ install-aso/ # Generated C# code *.Generated.cs *.Generated.*.cs + +# Azurite temporary files +__* \ No newline at end of file From ab207e4f8facc85c642d5f988e26d57da4acc632 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Mon, 11 Sep 2023 11:32:25 -0700 Subject: [PATCH 09/58] Small azure storage explorer / azurite comment update for export tests --- .../Rest/ExportLongRunningTests.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportLongRunningTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportLongRunningTests.cs index f9beacce09..d101f646ab 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportLongRunningTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportLongRunningTests.cs @@ -41,7 +41,7 @@ public ExportLongRunningTests(ExportTestFixture fixture, ITestOutputHelper testO [Fact] public async Task GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() { - // NOTE: Azurite is required to run these tests locally. + // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. // Trigger export request and check for export status Uri contentLocation = await _testFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:o}"); @@ -58,7 +58,7 @@ public async Task GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAs [Fact] public async Task GivenFhirServer_WhenPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() { - // NOTE: Azurite is required to run these tests locally. + // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. // Trigger export request and check for export status Uri contentLocation = await _testFhirClient.ExportAsync(path: "Patient/", parameters: _fixture.ExportTestResourcesQueryParameters); @@ -75,7 +75,7 @@ public async Task GivenFhirServer_WhenPatientDataIsExported_ThenExportedDataIsSa [Fact] public async Task GivenFhirServer_WhenAllObservationAndPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() { - // NOTE: Azurite is required to run these tests locally. + // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. // Trigger export request and check for export status Uri contentLocation = await _testFhirClient.ExportAsync(parameters: $"_type=Observation,Patient&{_fixture.ExportTestResourcesQueryParameters}"); @@ -92,7 +92,7 @@ public async Task GivenFhirServer_WhenAllObservationAndPatientDataIsExported_The [Fact] public async Task GivenFhirServer_WhenPatientObservationDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() { - // NOTE: Azurite is required to run these tests locally. + // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. // Trigger export request and check for export status Uri contentLocation = await _testFhirClient.ExportAsync("Patient/", $"_type=Observation&_typeFilter=Observation%3F_tag%3D{_fixture.FixtureTag}"); @@ -113,7 +113,7 @@ public async Task GivenFhirServer_WhenPatientObservationDataIsExported_ThenExpor [Fact] public async Task GivenFhirServer_WhenAllDataIsExportedToASpecificContainer_ThenExportedDataIsInTheSpecifiedContianer() { - // NOTE: Azurite is required to run these tests locally. + // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. string testContainer = "test-container"; @@ -133,7 +133,7 @@ public async Task GivenFhirServer_WhenAllDataIsExportedToASpecificContainer_Then [Fact] public async Task GivenFhirServer_WhenDataIsExportedWithHistory_ThenExportedDataIsSameAsDataInFhirServer() { - // NOTE: Azurite is required to run these tests locally. + // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. Uri contentLocation = await _testFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:O}&_includeHistory=true"); @@ -150,7 +150,7 @@ public async Task GivenFhirServer_WhenDataIsExportedWithHistory_ThenExportedData [Fact] public async Task GivenFhirServer_WhenDataIsExportedWithSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer() { - // NOTE: Azurite is required to run these tests locally. + // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. Uri contentLocation = await _testFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:O}&_includeDeleted=true"); @@ -167,7 +167,7 @@ public async Task GivenFhirServer_WhenDataIsExportedWithSoftDeletes_ThenExported [Fact] public async Task GivenFhirServer_WhenDataIsExportedWithHistoryAndSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer() { - // NOTE: Azurite is required to run these tests locally. + // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. Uri contentLocation = await _testFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:O}&_includeHistory=true&_includeDeleted=true"); From 3c1970cc57c03aa00875f35b87c20862b5e54b61 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Mon, 11 Sep 2023 14:39:47 -0700 Subject: [PATCH 10/58] Fixed ExportJobTaskTests to not rely on position of queryParameterList --- .../Operations/Export/ExportJobTaskTests.cs | 75 ++++++++++++------- 1 file changed, 50 insertions(+), 25 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Export/ExportJobTaskTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Export/ExportJobTaskTests.cs index 722d77c1b7..6276bf0352 100644 --- a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Export/ExportJobTaskTests.cs +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Export/ExportJobTaskTests.cs @@ -6,6 +6,7 @@ using System; using System.Collections.Generic; using System.Globalization; +using System.Linq; using System.Linq.Expressions; using System.Net; using System.Threading; @@ -20,6 +21,7 @@ using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Core.Configs; using Microsoft.Health.Fhir.Core.Exceptions; +using Microsoft.Health.Fhir.Core.Features; using Microsoft.Health.Fhir.Core.Features.Context; using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.Core.Features.Operations.Export; @@ -324,37 +326,37 @@ public async Task GivenThereAreMultiplePagesOfSearchResultsWithSinceParameter_Wh private Expression>>> CreateQueryParametersExpression(string resourceType) { return arg => arg != null && - Tuple.Create("_count", "1").Equals(arg[0]) && - Tuple.Create("_lastUpdated", $"le{_exportJobRecord.Till}").Equals(arg[1]) && - Tuple.Create("_type", resourceType).Equals(arg[2]); + arg.Any(x => x.Item1 == "_count" && x.Item2 == "1") && + arg.Any(x => x.Item1 == "_lastUpdated" && x.Item2 == $"le{_exportJobRecord.Till}") && + arg.Any(x => x.Item1 == "_type" && x.Item2 == resourceType); } private Expression>>> CreateQueryParametersExpression(PartialDateTime since, string resourceType) { return arg => arg != null && - Tuple.Create("_count", "1").Equals(arg[0]) && - Tuple.Create("_lastUpdated", $"le{_exportJobRecord.Till}").Equals(arg[1]) && - Tuple.Create("_lastUpdated", $"ge{since}").Equals(arg[2]) && - Tuple.Create("_type", resourceType).Equals(arg[3]); + arg.Any(x => x.Item1 == "_count" && x.Item2 == "1") && + arg.Any(x => x.Item1 == "_lastUpdated" && x.Item2 == $"le{_exportJobRecord.Till}") && + arg.Any(x => x.Item1 == "_lastUpdated" && x.Item2 == $"ge{since}") && + arg.Any(x => x.Item1 == "_type" && x.Item2 == resourceType); } private Expression>>> CreateQueryParametersExpressionWithContinuationToken(string continuationToken, string resourceType) { return arg => arg != null && - Tuple.Create("_count", "1").Equals(arg[0]) && - Tuple.Create("_lastUpdated", $"le{_exportJobRecord.Till}").Equals(arg[1]) && - Tuple.Create("_type", resourceType).Equals(arg[2]) && - Tuple.Create("ct", continuationToken).Equals(arg[3]); + arg.Any(x => x.Item1 == "_count" && x.Item2 == "1") && + arg.Any(x => x.Item1 == "_lastUpdated" && x.Item2 == $"le{_exportJobRecord.Till}") && + arg.Any(x => x.Item1 == "_type" && x.Item2 == resourceType) && + arg.Any(x => x.Item1 == "ct" && x.Item2 == continuationToken); } private Expression>>> CreateQueryParametersExpressionWithContinuationToken(string continuationToken, PartialDateTime since, string resourceType) { return arg => arg != null && - Tuple.Create("_count", "1").Equals(arg[0]) && - Tuple.Create("_lastUpdated", $"le{_exportJobRecord.Till}").Equals(arg[1]) && - Tuple.Create("_lastUpdated", $"ge{since}").Equals(arg[2]) && - Tuple.Create("_type", resourceType).Equals(arg[3]) && - Tuple.Create("ct", continuationToken).Equals(arg[4]); + arg.Any(x => x.Item1 == "_count" && x.Item2 == "1") && + arg.Any(x => x.Item1 == "_lastUpdated" && x.Item2 == $"le{_exportJobRecord.Till}") && + arg.Any(x => x.Item1 == "_lastUpdated" && x.Item2 == $"ge{since}") && + arg.Any(x => x.Item1 == "_type" && x.Item2 == resourceType) && + arg.Any(x => x.Item1 == "ct" && x.Item2 == continuationToken); } [Fact] @@ -834,7 +836,10 @@ public async Task GivenAnExportJobWithTheTypeParameter_WhenExecuted_ThenOnlyReso true) .Returns(x => { - string[] types = x.ArgAt>>(1)[3].Item2.Split(','); + // string[] types = x.ArgAt>>(1)[3].Item2.Split(','); + string[] types = x.ArgAt>>(1) + .Where(x => x.Item1 == Core.Features.KnownQueryParameterNames.Type) + .Select(x => x.Item2).First().Split(','); SearchResultEntry[] entries = new SearchResultEntry[types.Length]; for (int index = 0; index < types.Length; index++) @@ -889,7 +894,10 @@ public async Task GivenAPatientExportJobWithTheTypeParameter_WhenExecuted_ThenOn true) .Returns(x => { - string[] types = x.ArgAt>>(3)[3].Item2.Split(','); + // string[] types = x.ArgAt>>(3)[3].Item2.Split(','); + string[] types = x.ArgAt>>(3) + .Where(x => x.Item1 == Core.Features.KnownQueryParameterNames.Type) + .Select(x => x.Item2).First().Split(','); SearchResultEntry[] entries = new SearchResultEntry[types.Length]; for (int index = 0; index < types.Length; index++) @@ -1107,7 +1115,9 @@ public async Task GivenAGroupExportJob_WhenExecuted_ThenAllPatientResourcesInThe true) .Returns(x => { - string[] ids = x.ArgAt>>(1)[2].Item2.Split(','); + string[] ids = x.ArgAt>>(1) + .Where(x => x.Item1 == Core.Features.KnownQueryParameterNames.Id) + .Select(x => x.Item2).First().Split(','); SearchResultEntry[] entries = new SearchResultEntry[ids.Length]; for (int index = 0; index < ids.Length; index++) @@ -1175,7 +1185,10 @@ public async Task GivenAGroupExportJobWithMultiplePagesOfPatients_WhenExecuted_T true) .Returns(x => { - string[] ids = x.ArgAt>>(1)[2].Item2.Split(','); + // string[] ids = x.ArgAt>>(1)[2].Item2.Split(','); + string[] ids = x.ArgAt>>(1) + .Where(x => x.Item1 == Core.Features.KnownQueryParameterNames.Id) + .Select(x => x.Item2).First().Split(','); countOfSearches++; @@ -1237,7 +1250,7 @@ public async Task GivenAGroupExportJobToResume_WhenExecuted_ThenAllPatientResour }); int countOfSearches = 0; - _searchService.SearchAsync( + _ = _searchService.SearchAsync( KnownResourceTypes.Patient, Arg.Any>>(), _cancellationToken, @@ -1250,7 +1263,10 @@ public async Task GivenAGroupExportJobToResume_WhenExecuted_ThenAllPatientResour if (countOfSearches == 1) { - ids = x.ArgAt>>(1)[2].Item2.Split(','); + // ids = x.ArgAt>>(1)[2].Item2.Split(','); + ids = x.ArgAt>>(1) + .Where(x => x.Item1 == Core.Features.KnownQueryParameterNames.Id) + .Select(x => x.Item2).First().Split(','); continuationTokenIndex = 0; } else if (countOfSearches == 2) @@ -1261,7 +1277,10 @@ public async Task GivenAGroupExportJobToResume_WhenExecuted_ThenAllPatientResour { // The ids aren't in the query parameters because of the reset ids = new string[] { "1", "2", "3" }; - continuationTokenIndex = int.Parse(ContinuationTokenConverter.Decode(x.ArgAt>>(1)[2].Item2).Substring(2)); + continuationTokenIndex = int.Parse(ContinuationTokenConverter.Decode( + x.ArgAt>>(1) + .Where(x => x.Item1 == Core.Features.KnownQueryParameterNames.ContinuationToken) + .Select(x => x.Item2).First())[2..]); } return CreateSearchResult( @@ -1342,7 +1361,11 @@ public async Task GivenAGroupExportJobWithTheTypeParameter_WhenExecuted_ThenAllP true) .Returns(x => { - string[] ids = x.ArgAt>>(1)[2].Item2.Split(','); + // string[] ids = x.ArgAt>>(1)[2].Item2.Split(','); + string[] ids = x.ArgAt>>(1) + .Where(x => x.Item1 == Core.Features.KnownQueryParameterNames.Id) + .Select(x => x.Item2).First().Split(','); + SearchResultEntry[] entries = new SearchResultEntry[ids.Length]; for (int index = 0; index < ids.Length; index++) @@ -1363,7 +1386,9 @@ public async Task GivenAGroupExportJobWithTheTypeParameter_WhenExecuted_ThenAllP .Returns(x => { string parentId = x.ArgAt(1); - string[] resourceTypes = x.ArgAt>>(3)[2].Item2.Split(','); + string[] resourceTypes = x.ArgAt>>(3) + .Where(x => x.Item1 == Core.Features.KnownQueryParameterNames.Type) + .Select(x => x.Item2).First().Split(','); SearchResultEntry[] entries = new SearchResultEntry[resourceTypes.Length]; From c713863176ff83ec85b59ef2fc034ce891aae98f Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Mon, 11 Sep 2023 15:17:12 -0700 Subject: [PATCH 11/58] Added ExportJobTask unit test for history/delete Also cleaned up some left over comments from previous commit --- .vscode/launch.json | 17 ++++-- .vscode/settings.json | 7 +++ .../Operations/Export/ExportJobTaskTests.cs | 57 ++++++++++++++----- 3 files changed, 62 insertions(+), 19 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/launch.json b/.vscode/launch.json index 1e6ec2f2a7..ce4fdd4bf9 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -10,7 +10,7 @@ "request": "launch", "preLaunchTask": "buildR4", // If you have changed target frameworks, make sure to update the program path. - "program": "${workspaceFolder}/src/Microsoft.Health.Fhir.R4.Web/bin/Debug/net6.0/Microsoft.Health.Fhir.R4.Web.dll", + "program": "${workspaceFolder}/src/Microsoft.Health.Fhir.R4.Web/bin/Debug/net7.0/Microsoft.Health.Fhir.R4.Web.dll", "args": [], "cwd": "${workspaceFolder}/src/Microsoft.Health.Fhir.R4.Web", "stopAtEntry": false, @@ -22,6 +22,11 @@ // https://docs.microsoft.com/en-us/azure/cosmos-db/local-emulator?tabs=ssl-netstd21#authenticate-requests "CosmosDb:Host": "https://localhost:8081/", "CosmosDb:Key": "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==", + "TaskHosting:Enabled": "true", + "TaskHosting:MaxRunningTaskCount": "2", + "FhirServer:Operations:Export:StorageAccountConnection": "UseDevelopmentStorage=true", + "FhirServer:Operations:IntegrationDataStore:StorageAccountConnection": "UseDevelopmentStorage=true", + "FhirServer:Operations:Import:Enabled": "true" }, "sourceFileMap": { "/Views": "${workspaceFolder}/Views" @@ -33,7 +38,7 @@ "request": "launch", "preLaunchTask": "buildR4", // If you have changed target frameworks, make sure to update the program path. - "program": "${workspaceFolder}/src/Microsoft.Health.Fhir.R4.Web/bin/Debug/net6.0/Microsoft.Health.Fhir.R4.Web.dll", + "program": "${workspaceFolder}/src/Microsoft.Health.Fhir.R4.Web/bin/Debug/net7.0/Microsoft.Health.Fhir.R4.Web.dll", "args": [], "cwd": "${workspaceFolder}/src/Microsoft.Health.Fhir.R4.Web", "stopAtEntry": false, @@ -57,7 +62,7 @@ "request": "launch", "preLaunchTask": "buildSTU3", // If you have changed target frameworks, make sure to update the program path. - "program": "${workspaceFolder}/src/Microsoft.Health.Fhir.Stu3.Web/bin/Debug/net6.0/Microsoft.Health.Fhir.Stu3.Web.dll", + "program": "${workspaceFolder}/src/Microsoft.Health.Fhir.Stu3.Web/bin/Debug/net7.0/Microsoft.Health.Fhir.Stu3.Web.dll", "args": [], "cwd": "${workspaceFolder}/src/Microsoft.Health.Fhir.Stu3.Web", "stopAtEntry": false, @@ -80,7 +85,7 @@ "request": "launch", "preLaunchTask": "buildSTU3", // If you have changed target frameworks, make sure to update the program path. - "program": "${workspaceFolder}/src/Microsoft.Health.Fhir.Stu3.Web/bin/Debug/net6.0/Microsoft.Health.Fhir.Stu3.Web.dll", + "program": "${workspaceFolder}/src/Microsoft.Health.Fhir.Stu3.Web/bin/Debug/net7.0/Microsoft.Health.Fhir.Stu3.Web.dll", "args": [], "cwd": "${workspaceFolder}/src/Microsoft.Health.Fhir.Stu3.Web", "stopAtEntry": false, @@ -104,7 +109,7 @@ "request": "launch", "preLaunchTask": "buildR5", // If you have changed target frameworks, make sure to update the program path. - "program": "${workspaceFolder}/src/Microsoft.Health.Fhir.R5.Web/bin/Debug/net6.0/Microsoft.Health.Fhir.R5.Web.dll", + "program": "${workspaceFolder}/src/Microsoft.Health.Fhir.R5.Web/bin/Debug/net7.0/Microsoft.Health.Fhir.R5.Web.dll", "args": [], "cwd": "${workspaceFolder}/src/Microsoft.Health.Fhir.R5.Web", "stopAtEntry": false, @@ -127,7 +132,7 @@ "request": "launch", "preLaunchTask": "buildR5", // If you have changed target frameworks, make sure to update the program path. - "program": "${workspaceFolder}/src/Microsoft.Health.Fhir.R5.Web/bin/Debug/net6.0/Microsoft.Health.Fhir.R5.Web.dll", + "program": "${workspaceFolder}/src/Microsoft.Health.Fhir.R5.Web/bin/Debug/net7.0/Microsoft.Health.Fhir.R5.Web.dll", "args": [], "cwd": "${workspaceFolder}/src/Microsoft.Health.Fhir.R5.Web", "stopAtEntry": false, diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000000..1cc39f5057 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,7 @@ +{ + "csharp.debug.justMyCode": true, + "csharp.debug.symbolOptions": { + "searchMicrosoftSymbolServer": true, + "searchNuGetOrgSymbolServer": true, + } +} \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Export/ExportJobTaskTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Export/ExportJobTaskTests.cs index 6276bf0352..3b796bc1d8 100644 --- a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Export/ExportJobTaskTests.cs +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Export/ExportJobTaskTests.cs @@ -323,12 +323,43 @@ public async Task GivenThereAreMultiplePagesOfSearchResultsWithSinceParameter_Wh Assert.True(secondCapturedSearch); } - private Expression>>> CreateQueryParametersExpression(string resourceType) + [Fact] + public async Task GivenAnExportJobWithHistoryAndSoftDeletes_WhenExecuted_ThenAllResourcesAreExportedToTheProperLocation() + { + bool capturedSearch = false; + + var exportJobRecordIncludeHistory = CreateExportJobRecord( + exportJobType: ExportJobType.Patient, + includeHistory: true, + includeDeleted: true, + maximumNumberOfResourcesPerQuery: 1); + SetupExportJobRecordAndOperationDataStore(exportJobRecordIncludeHistory); + + _searchService.SearchAsync( + null, + Arg.Is(CreateQueryParametersExpression(KnownResourceTypes.Patient, includeHistory: true, includeDeleted: true)), + _cancellationToken, + true) + .Returns(x => + { + capturedSearch = true; + + return CreateSearchResult(); + }); + + await _exportJobTask.ExecuteAsync(_exportJobRecord, _weakETag, _cancellationToken); + + Assert.True(capturedSearch); + } + + private Expression>>> CreateQueryParametersExpression(string resourceType, bool includeHistory = false, bool includeDeleted = false) { return arg => arg != null && arg.Any(x => x.Item1 == "_count" && x.Item2 == "1") && arg.Any(x => x.Item1 == "_lastUpdated" && x.Item2 == $"le{_exportJobRecord.Till}") && - arg.Any(x => x.Item1 == "_type" && x.Item2 == resourceType); + arg.Any(x => x.Item1 == "_type" && x.Item2 == resourceType) && + arg.Any(x => x.Item1 == "_includeHistory" && x.Item2 == includeHistory.ToString()) && + arg.Any(x => x.Item1 == "_includeDeleted" && x.Item2 == includeDeleted.ToString()); } private Expression>>> CreateQueryParametersExpression(PartialDateTime since, string resourceType) @@ -836,9 +867,8 @@ public async Task GivenAnExportJobWithTheTypeParameter_WhenExecuted_ThenOnlyReso true) .Returns(x => { - // string[] types = x.ArgAt>>(1)[3].Item2.Split(','); string[] types = x.ArgAt>>(1) - .Where(x => x.Item1 == Core.Features.KnownQueryParameterNames.Type) + .Where(x => x.Item1 == KnownQueryParameterNames.Type) .Select(x => x.Item2).First().Split(','); SearchResultEntry[] entries = new SearchResultEntry[types.Length]; @@ -894,9 +924,8 @@ public async Task GivenAPatientExportJobWithTheTypeParameter_WhenExecuted_ThenOn true) .Returns(x => { - // string[] types = x.ArgAt>>(3)[3].Item2.Split(','); string[] types = x.ArgAt>>(3) - .Where(x => x.Item1 == Core.Features.KnownQueryParameterNames.Type) + .Where(x => x.Item1 == KnownQueryParameterNames.Type) .Select(x => x.Item2).First().Split(','); SearchResultEntry[] entries = new SearchResultEntry[types.Length]; @@ -1185,9 +1214,8 @@ public async Task GivenAGroupExportJobWithMultiplePagesOfPatients_WhenExecuted_T true) .Returns(x => { - // string[] ids = x.ArgAt>>(1)[2].Item2.Split(','); string[] ids = x.ArgAt>>(1) - .Where(x => x.Item1 == Core.Features.KnownQueryParameterNames.Id) + .Where(x => x.Item1 == KnownQueryParameterNames.Id) .Select(x => x.Item2).First().Split(','); countOfSearches++; @@ -1361,9 +1389,8 @@ public async Task GivenAGroupExportJobWithTheTypeParameter_WhenExecuted_ThenAllP true) .Returns(x => { - // string[] ids = x.ArgAt>>(1)[2].Item2.Split(','); string[] ids = x.ArgAt>>(1) - .Where(x => x.Item1 == Core.Features.KnownQueryParameterNames.Id) + .Where(x => x.Item1 == KnownQueryParameterNames.Id) .Select(x => x.Item2).First().Split(','); SearchResultEntry[] entries = new SearchResultEntry[ids.Length]; @@ -1387,7 +1414,7 @@ public async Task GivenAGroupExportJobWithTheTypeParameter_WhenExecuted_ThenAllP { string parentId = x.ArgAt(1); string[] resourceTypes = x.ArgAt>>(3) - .Where(x => x.Item1 == Core.Features.KnownQueryParameterNames.Type) + .Where(x => x.Item1 == KnownQueryParameterNames.Type) .Select(x => x.Item2).First().Split(','); SearchResultEntry[] entries = new SearchResultEntry[resourceTypes.Length]; @@ -2101,7 +2128,9 @@ private ExportJobRecord CreateExportJobRecord( uint numberOfPagesPerCommit = 0, string containerName = null, string anonymizationConfigurationLocation = null, - string anonymizationConfigurationFileEtag = null) + string anonymizationConfigurationFileEtag = null, + bool includeHistory = false, + bool includeDeleted = false) { return new ExportJobRecord( new Uri(requestEndpoint), @@ -2119,7 +2148,9 @@ private ExportJobRecord CreateExportJobRecord( numberOfPagesPerCommit: numberOfPagesPerCommit == 0 ? _exportJobConfiguration.NumberOfPagesPerCommit : numberOfPagesPerCommit, storageAccountContainerName: containerName, anonymizationConfigurationLocation: anonymizationConfigurationLocation, - anonymizationConfigurationFileETag: anonymizationConfigurationFileEtag); + anonymizationConfigurationFileETag: anonymizationConfigurationFileEtag, + includeHistory: includeHistory, + includeDeleted: includeDeleted); } private ExportJobTask CreateExportJobTask( From a2b6c44e51d1e86758ae67e5c3b0b06ffe548de9 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Mon, 11 Sep 2023 20:45:35 -0700 Subject: [PATCH 12/58] Removing changed settings.json file --- .vscode/settings.json | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 1cc39f5057..0000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "csharp.debug.justMyCode": true, - "csharp.debug.symbolOptions": { - "searchMicrosoftSymbolServer": true, - "searchNuGetOrgSymbolServer": true, - } -} \ No newline at end of file From d13941908b4f7223789b69f7c978c84c546942e1 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Mon, 11 Sep 2023 21:06:00 -0700 Subject: [PATCH 13/58] Added historical and soft delete export to rest file --- docs/rest/ExportRequests.http | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/rest/ExportRequests.http b/docs/rest/ExportRequests.http index 2be1596599..db354ec4d7 100644 --- a/docs/rest/ExportRequests.http +++ b/docs/rest/ExportRequests.http @@ -71,6 +71,13 @@ Accept: application/fhir+json Prefer: respond-async Authorization: Bearer {{bearer.response.body.access_token}} +### Export with history and soft deleted records +# @name export +GET https://{{hostname}}/$export?_includeHistory=true&includeDeleted=true +Accept: application/fhir+json +Prefer: respond-async +Authorization: Bearer {{bearer.response.body.access_token}} + ### Get Export request GET {{exportLocation}} Authorization: Bearer {{bearer.response.body.access_token}} From bc04057af34847d00f7d9eed59441cf9b460085a Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 14 Sep 2023 21:01:26 -0700 Subject: [PATCH 14/58] Added import of soft deleted resources --- .../Import/ImportResourceLoaderTests.cs | 2 +- .../Operations/Import/ImportResource.cs | 11 ++- .../Import/ImportResourceParserTests.cs | 77 +++++++++++++++++++ ...ealth.Fhir.Shared.Core.UnitTests.projitems | 1 + .../Operations/Import/ImportResourceParser.cs | 14 +++- .../Import/ImportProcessingJobTests.cs | 2 +- 6 files changed, 101 insertions(+), 6 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/Import/ImportResourceParserTests.cs diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Import/ImportResourceLoaderTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Import/ImportResourceLoaderTests.cs index be81cffdb7..d70738e873 100644 --- a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Import/ImportResourceLoaderTests.cs +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Import/ImportResourceLoaderTests.cs @@ -243,7 +243,7 @@ private async Task VerifyResourceLoaderAsync(int resourcCount, int batchSize, lo null, null, "SearchParam"); - return new ImportResource(index, 0, 0, false, false, resourceWrapper); + return new ImportResource(index, 0, 0, false, false, false, resourceWrapper); }); IImportErrorSerializer serializer = Substitute.For(); diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Import/ImportResource.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Import/ImportResource.cs index 83078344bc..5598d739c4 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Import/ImportResource.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Import/ImportResource.cs @@ -9,18 +9,19 @@ namespace Microsoft.Health.Fhir.Core.Features.Operations.Import { public class ImportResource { - public ImportResource(long index, long offset, int length, bool keepLastUpdated, bool keepVersion, ResourceWrapper resourceWrapper) + public ImportResource(long index, long offset, int length, bool keepLastUpdated, bool keepVersion, bool isDeleted, ResourceWrapper resourceWrapper) { Index = index; Offset = offset; Length = length; KeepLastUpdated = keepLastUpdated; KeepVersion = keepVersion; + IsDeleted = isDeleted; ResourceWrapper = resourceWrapper; } public ImportResource(ResourceWrapper resource) - : this(0, 0, 0, false, false, resource) + : this(0, 0, 0, false, false, false, resource) { } @@ -31,6 +32,7 @@ public ImportResource(long index, long offset, string importError) Length = 0; KeepLastUpdated = false; KeepVersion = false; + IsDeleted = false; ImportError = importError; } @@ -59,6 +61,11 @@ public ImportResource(long index, long offset, string importError) /// public bool KeepVersion { get; set; } + /// + /// Flag indicating whether deletion state was provided on input + /// + public bool IsDeleted { get; set; } + /// /// Resource wrapper from raw content /// diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/Import/ImportResourceParserTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/Import/ImportResourceParserTests.cs new file mode 100644 index 0000000000..82706bb17d --- /dev/null +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/Import/ImportResourceParserTests.cs @@ -0,0 +1,77 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Hl7.Fhir.Model; +using Hl7.Fhir.Serialization; +using Microsoft.Health.Core.Features.Context; +using Microsoft.Health.Core.Features.Security; +using Microsoft.Health.Fhir.Core.Features.Compartment; +using Microsoft.Health.Fhir.Core.Features.Context; +using Microsoft.Health.Fhir.Core.Features.Definition; +using Microsoft.Health.Fhir.Core.Features.Operations.Import; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.Shared.Core.Features.Operations.Import; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using NSubstitute; +using Xunit; + +namespace Microsoft.Health.Fhir.Shared.Core.UnitTests.Features.Operations.Import +{ + [Trait(Traits.OwningTeam, OwningTeam.FhirImport)] + [Trait(Traits.Category, Categories.Import)] + public class ImportResourceParserTests + { + private readonly FhirJsonSerializer _jsonSerializer = new FhirJsonSerializer(); + + private readonly FhirJsonParser _jsonParser = new(); + + private readonly ResourceWrapperFactory _wrapperFactory; + + private readonly ImportResourceParser _importResourceParser; + + public ImportResourceParserTests() + { + var requestContextAccessor = Substitute.For>(); + + requestContextAccessor.RequestContext.Method.Returns("PUT"); + requestContextAccessor.RequestContext.Uri.Returns(new Uri("https://unittest/Patient/123")); + + _wrapperFactory = new ResourceWrapperFactory( + new RawResourceFactory(new FhirJsonSerializer()), + requestContextAccessor, + Substitute.For(), + Substitute.For(), + Substitute.For(), + Substitute.For(), + Deserializers.ResourceDeserializer); + + _importResourceParser = new(_jsonParser, _wrapperFactory); + } + + [Fact] + public void GivenImportWithSoftDeletedFile_WhenParsed_DeletedExtensionShouldBeRemoved() + { + Patient patient = new Patient(); + patient.Id = Guid.NewGuid().ToString(); + patient.Name.Add(HumanName.ForFamily("Test")); + patient.Meta = new() + { + LastUpdated = DateTimeOffset.UtcNow, + VersionId = "3", + }; + patient.Meta.AddExtension(KnownFhirPaths.AzureSoftDeletedExtensionUrl, new FhirString("soft-deleted")); + + string patientAsString = _jsonSerializer.SerializeToString(patient); + var importResource = _importResourceParser.Parse(0, 0, 0, patientAsString, ImportMode.IncrementalLoad); + + Assert.DoesNotContain(KnownFhirPaths.AzureSoftDeletedExtensionUrl, importResource.ResourceWrapper.RawResource.Data); + Assert.DoesNotContain("soft-deleted", importResource.ResourceWrapper.RawResource.Data); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems index d492ee7645..18e06f63a5 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems @@ -21,6 +21,7 @@ + diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Import/ImportResourceParser.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Import/ImportResourceParser.cs index 7d9a1bc765..632cc4e8f9 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Import/ImportResourceParser.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Import/ImportResourceParser.cs @@ -6,12 +6,14 @@ using System; using System.Collections.Generic; using EnsureThat; +using Hl7.Fhir.FhirPath; using Hl7.Fhir.Model; using Hl7.Fhir.Serialization; using Microsoft.Health.Core; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Resources; +using Microsoft.Health.Fhir.Core.Models; namespace Microsoft.Health.Fhir.Core.Features.Operations.Import { @@ -50,9 +52,17 @@ public ImportResource Parse(long index, long offset, int length, string rawResou } var resourceElement = resource.ToResourceElement(); - var resourceWapper = _resourceFactory.Create(resourceElement, false, true, keepVersion); - return new ImportResource(index, offset, length, !lastUpdatedIsNull, keepVersion, resourceWapper); + var isDeleted = resourceElement.IsSoftDeleted(); + + if (isDeleted) + { + resource.Meta.RemoveExtension(KnownFhirPaths.AzureSoftDeletedExtensionUrl); + } + + var resourceWapper = _resourceFactory.Create(resourceElement, isDeleted, true, keepVersion); + + return new ImportResource(index, offset, length, !lastUpdatedIsNull, keepVersion, isDeleted, resourceWapper); } private static void CheckConditionalReferenceInResource(Resource resource, ImportMode importMode) diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Operations/Import/ImportProcessingJobTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Operations/Import/ImportProcessingJobTests.cs index add95eb58e..bf6d25d65c 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Operations/Import/ImportProcessingJobTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Operations/Import/ImportProcessingJobTests.cs @@ -152,7 +152,7 @@ private static async Task VerifyCommonImportAsync(ImportProcessingJobDefinition null, "SearchParam"); - await resourceChannel.Writer.WriteAsync(new ImportResource(0, 0, 0, false, false, resourceWrapper)); + await resourceChannel.Writer.WriteAsync(new ImportResource(0, 0, 0, false, false, false, resourceWrapper)); await resourceChannel.Writer.WriteAsync(new ImportResource(1, 0, "Error")); resourceChannel.Writer.Complete(); }); From ac395eddac048d05cd96ed74888460477513bbeb Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Wed, 20 Sep 2023 06:16:21 -0700 Subject: [PATCH 15/58] Updates per PR review --- .vscode/launch.json | 23 ++++++++-------- docs/rest/ExportRequests.http | 2 +- .../Operations/Export/ExportJobTask.cs | 26 +++++++++---------- .../Export/Models/ExportJobRecord.cs | 1 - .../Features/Search/SearchOptions.cs | 8 ++++++ .../Rest/ExportDataValidationTests.cs | 8 +++--- 6 files changed, 36 insertions(+), 32 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index ce4fdd4bf9..4509e916c5 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -10,7 +10,7 @@ "request": "launch", "preLaunchTask": "buildR4", // If you have changed target frameworks, make sure to update the program path. - "program": "${workspaceFolder}/src/Microsoft.Health.Fhir.R4.Web/bin/Debug/net7.0/Microsoft.Health.Fhir.R4.Web.dll", + "program": "${workspaceFolder}/src/Microsoft.Health.Fhir.R4.Web/bin/Debug/net6.0/Microsoft.Health.Fhir.R4.Web.dll", "args": [], "cwd": "${workspaceFolder}/src/Microsoft.Health.Fhir.R4.Web", "stopAtEntry": false, @@ -22,11 +22,6 @@ // https://docs.microsoft.com/en-us/azure/cosmos-db/local-emulator?tabs=ssl-netstd21#authenticate-requests "CosmosDb:Host": "https://localhost:8081/", "CosmosDb:Key": "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==", - "TaskHosting:Enabled": "true", - "TaskHosting:MaxRunningTaskCount": "2", - "FhirServer:Operations:Export:StorageAccountConnection": "UseDevelopmentStorage=true", - "FhirServer:Operations:IntegrationDataStore:StorageAccountConnection": "UseDevelopmentStorage=true", - "FhirServer:Operations:Import:Enabled": "true" }, "sourceFileMap": { "/Views": "${workspaceFolder}/Views" @@ -38,7 +33,7 @@ "request": "launch", "preLaunchTask": "buildR4", // If you have changed target frameworks, make sure to update the program path. - "program": "${workspaceFolder}/src/Microsoft.Health.Fhir.R4.Web/bin/Debug/net7.0/Microsoft.Health.Fhir.R4.Web.dll", + "program": "${workspaceFolder}/src/Microsoft.Health.Fhir.R4.Web/bin/Debug/net6.0/Microsoft.Health.Fhir.R4.Web.dll", "args": [], "cwd": "${workspaceFolder}/src/Microsoft.Health.Fhir.R4.Web", "stopAtEntry": false, @@ -46,11 +41,15 @@ "ASPNETCORE_ENVIRONMENT": "Development", "DataStore": "SqlServer", "SqlServer:AllowDatabaseCreation": "true", - "SqlServer:ConnectionString": "server=(local);Initial Catalog=FHIR_R4;Integrated Security=true", + "SqlServer:ConnectionString": "Server=localhost,1433;Persist Security Info=False;TrustServerCertificate=true;User ID=sa;Password=localDev1!;Initial Catalog=FHIR_R4", "SqlServer:Initialize": "true", "SqlServer:SchemaOptions:AutomaticUpdatesEnabled": "true", "TestAuthEnvironment:FilePath": "..//..//testauthenvironment.json", "FhirServer:Security:Enabled": "false", + "TaskHosting:Enabled": "true", + "TaskHosting:MaxRunningTaskCount": "2", + "FhirServer:Operations:IntegrationDataStore:StorageAccountConnection": "UseDevelopmentStorage=true", + "FhirServer:Operations:Import:Enabled": "true" }, "sourceFileMap": { "/Views": "${workspaceFolder}/Views" @@ -62,7 +61,7 @@ "request": "launch", "preLaunchTask": "buildSTU3", // If you have changed target frameworks, make sure to update the program path. - "program": "${workspaceFolder}/src/Microsoft.Health.Fhir.Stu3.Web/bin/Debug/net7.0/Microsoft.Health.Fhir.Stu3.Web.dll", + "program": "${workspaceFolder}/src/Microsoft.Health.Fhir.Stu3.Web/bin/Debug/net6.0/Microsoft.Health.Fhir.Stu3.Web.dll", "args": [], "cwd": "${workspaceFolder}/src/Microsoft.Health.Fhir.Stu3.Web", "stopAtEntry": false, @@ -85,7 +84,7 @@ "request": "launch", "preLaunchTask": "buildSTU3", // If you have changed target frameworks, make sure to update the program path. - "program": "${workspaceFolder}/src/Microsoft.Health.Fhir.Stu3.Web/bin/Debug/net7.0/Microsoft.Health.Fhir.Stu3.Web.dll", + "program": "${workspaceFolder}/src/Microsoft.Health.Fhir.Stu3.Web/bin/Debug/net6.0/Microsoft.Health.Fhir.Stu3.Web.dll", "args": [], "cwd": "${workspaceFolder}/src/Microsoft.Health.Fhir.Stu3.Web", "stopAtEntry": false, @@ -109,7 +108,7 @@ "request": "launch", "preLaunchTask": "buildR5", // If you have changed target frameworks, make sure to update the program path. - "program": "${workspaceFolder}/src/Microsoft.Health.Fhir.R5.Web/bin/Debug/net7.0/Microsoft.Health.Fhir.R5.Web.dll", + "program": "${workspaceFolder}/src/Microsoft.Health.Fhir.R5.Web/bin/Debug/net6.0/Microsoft.Health.Fhir.R5.Web.dll", "args": [], "cwd": "${workspaceFolder}/src/Microsoft.Health.Fhir.R5.Web", "stopAtEntry": false, @@ -132,7 +131,7 @@ "request": "launch", "preLaunchTask": "buildR5", // If you have changed target frameworks, make sure to update the program path. - "program": "${workspaceFolder}/src/Microsoft.Health.Fhir.R5.Web/bin/Debug/net7.0/Microsoft.Health.Fhir.R5.Web.dll", + "program": "${workspaceFolder}/src/Microsoft.Health.Fhir.R5.Web/bin/Debug/net6.0/Microsoft.Health.Fhir.R5.Web.dll", "args": [], "cwd": "${workspaceFolder}/src/Microsoft.Health.Fhir.R5.Web", "stopAtEntry": false, diff --git a/docs/rest/ExportRequests.http b/docs/rest/ExportRequests.http index db354ec4d7..9d434846cb 100644 --- a/docs/rest/ExportRequests.http +++ b/docs/rest/ExportRequests.http @@ -73,7 +73,7 @@ Authorization: Bearer {{bearer.response.body.access_token}} ### Export with history and soft deleted records # @name export -GET https://{{hostname}}/$export?_includeHistory=true&includeDeleted=true +GET https://{{hostname}}/$export?_includeHistory=true&_includeDeleted=true Accept: application/fhir+json Prefer: respond-async Authorization: Bearer {{bearer.response.body.access_token}} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs index d790cc6fa3..d75d3c5b6f 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs @@ -674,38 +674,36 @@ private void ProcessSearchResults(IEnumerable searchResults, foreach (SearchResultEntry result in searchResults) { ResourceWrapper resourceWrapper = result.Resource; - var data = result.Resource.RawResource.Data; + ResourceElement overrideDataElement = null; var addSoftDeletedExtension = resourceWrapper.IsDeleted && _exportJobRecord.IncludeDeleted; if (anonymizer != null) { - ResourceElement element = _resourceDeserializer.Deserialize(resourceWrapper); + overrideDataElement = _resourceDeserializer.Deserialize(resourceWrapper); try { - element = anonymizer.Anonymize(element); + overrideDataElement = anonymizer.Anonymize(overrideDataElement); } catch (Exception ex) { throw new FailedToAnonymizeResourceException(ex.Message, ex); } - - // Serialize into NDJson and write to the file. - data = _resourceToByteArraySerializer.StringSerialize(element, addSoftDeletedExtension); } - else if (!resourceWrapper.RawResource.IsMetaSet) + else if (!resourceWrapper.RawResource.IsMetaSet || addSoftDeletedExtension) { // For older records in Cosmos the metadata isn't included in the raw resource - ResourceElement element = _resourceDeserializer.Deserialize(resourceWrapper); - data = _resourceToByteArraySerializer.StringSerialize(element, addSoftDeletedExtension); + overrideDataElement = _resourceDeserializer.Deserialize(resourceWrapper); } - else if (addSoftDeletedExtension) + + var outputData = result.Resource.RawResource.Data; + + // If any modifications were made to the resource / are needed, serialize the element instead of using the raw data string. + if (overrideDataElement is not null) { - // If the resource is deleted and we aren't anonymizing it, we need to add the soft delete extension - ResourceElement element = _resourceDeserializer.Deserialize(resourceWrapper); - data = _resourceToByteArraySerializer.StringSerialize(element, addSoftDeletedExtension); + outputData = _resourceToByteArraySerializer.StringSerialize(overrideDataElement, addSoftDeletedExtension); } - _fileManager.WriteToFile(resourceWrapper.ResourceTypeName, data); + _fileManager.WriteToFile(resourceWrapper.ResourceTypeName, outputData); } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/Models/ExportJobRecord.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/Models/ExportJobRecord.cs index 93453faa66..18ff1e12f4 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/Models/ExportJobRecord.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/Models/ExportJobRecord.cs @@ -5,7 +5,6 @@ using System; using System.Collections.Generic; -using DotLiquid.Tags; using EnsureThat; using Microsoft.Health.Core; using Microsoft.Health.Fhir.Core.Models; diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SearchOptions.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SearchOptions.cs index 7e9639e41f..12a156bb8f 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/SearchOptions.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SearchOptions.cs @@ -111,8 +111,16 @@ internal set } } + /// + /// Gets a value indicating whether to include history resources in the search result, + /// as long as they match the other search parameters as well. + /// public bool IncludeHistory { get; internal set; } = false; + /// + /// Gets a value indicating whether to include deleted resources in the search result, + /// as long as they match the other search parameters as well. + /// public bool IncludeDeleted { get; internal set; } = false; /// diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportDataValidationTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportDataValidationTests.cs index b54bdb28aa..db656a5a00 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportDataValidationTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportDataValidationTests.cs @@ -43,7 +43,7 @@ public ExportDataValidationTests(ExportTestFixture fixture, ITestOutputHelper te [Fact] public async Task GivenFhirServer_WhenGroupDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() { - // NOTE: Azure Storage Emulator or Azurite is required to run these tests locally. + // NOTE: Azurite is required to run these tests locally. // Add data for test var (dataInFhirServer, groupId) = await CreateGroupWithPatient(true); @@ -63,7 +63,7 @@ public async Task GivenFhirServer_WhenGroupDataIsExported_ThenExportedDataIsSame [Fact] public async Task GivenFhirServer_WhenGroupDataIsExportedWithTypeParameter_ThenExportedDataIsSameAsDataInFhirServer() { - // NOTE: Azure Storage Emulator or Azurite is required to run these tests locally. + // NOTE: Azurite is required to run these tests locally. // Add data for test var (dataInFhirServer, groupId) = await CreateGroupWithPatient(false); @@ -83,7 +83,7 @@ public async Task GivenFhirServer_WhenGroupDataIsExportedWithTypeParameter_ThenE [Fact] public async Task GivenFhirServer_WhenGroupDataWithNoMemberPatientIdIsExported_ThenNoDataIsExported() { - // NOTE: Azure Storage Emulator or Azurite is required to run these tests locally. + // NOTE: Azurite is required to run these tests locally. // Add data for test string groupId = await CreateGroupWithoutPatientIds(); @@ -110,7 +110,7 @@ async Task CreateGroupWithoutPatientIds() [Fact] public async Task GivenFhirServer_WhenDataIsExported_ThenExportTaskMetricsNotificationShouldBePosted() { - // NOTE: Azure Storage Emulator or Azurite is required to run these tests locally. + // NOTE: Azurite is required to run these tests locally. if (!_fixture.IsUsingInProcTestServer) { From eba2976e797c628eaa11ff486d46a0698f358466 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Wed, 20 Sep 2023 08:33:43 -0700 Subject: [PATCH 16/58] Removed import changes - going to another PR --- .vscode/launch.json | 2 +- .../Operations/Export/ExportJobTaskTests.cs | 3 +- .../Import/ImportResourceLoaderTests.cs | 2 +- .../Operations/Import/ImportResource.cs | 11 +-- .../Import/ImportResourceParserTests.cs | 77 ------------------- ...ealth.Fhir.Shared.Core.UnitTests.projitems | 1 - .../Operations/Import/ImportResourceParser.cs | 11 +-- .../Import/ImportProcessingJobTests.cs | 2 +- 8 files changed, 8 insertions(+), 101 deletions(-) delete mode 100644 src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/Import/ImportResourceParserTests.cs diff --git a/.vscode/launch.json b/.vscode/launch.json index 4509e916c5..3a1f5d9283 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -41,7 +41,7 @@ "ASPNETCORE_ENVIRONMENT": "Development", "DataStore": "SqlServer", "SqlServer:AllowDatabaseCreation": "true", - "SqlServer:ConnectionString": "Server=localhost,1433;Persist Security Info=False;TrustServerCertificate=true;User ID=sa;Password=localDev1!;Initial Catalog=FHIR_R4", + "SqlServer:ConnectionString": "server=(local);Initial Catalog=FHIR_R4;Integrated Security=true", "SqlServer:Initialize": "true", "SqlServer:SchemaOptions:AutomaticUpdatesEnabled": "true", "TestAuthEnvironment:FilePath": "..//..//testauthenvironment.json", diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Export/ExportJobTaskTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Export/ExportJobTaskTests.cs index 3b796bc1d8..2e5bef4790 100644 --- a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Export/ExportJobTaskTests.cs +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Export/ExportJobTaskTests.cs @@ -1278,7 +1278,7 @@ public async Task GivenAGroupExportJobToResume_WhenExecuted_ThenAllPatientResour }); int countOfSearches = 0; - _ = _searchService.SearchAsync( + _searchService.SearchAsync( KnownResourceTypes.Patient, Arg.Any>>(), _cancellationToken, @@ -1291,7 +1291,6 @@ public async Task GivenAGroupExportJobToResume_WhenExecuted_ThenAllPatientResour if (countOfSearches == 1) { - // ids = x.ArgAt>>(1)[2].Item2.Split(','); ids = x.ArgAt>>(1) .Where(x => x.Item1 == Core.Features.KnownQueryParameterNames.Id) .Select(x => x.Item2).First().Split(','); diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Import/ImportResourceLoaderTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Import/ImportResourceLoaderTests.cs index d70738e873..be81cffdb7 100644 --- a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Import/ImportResourceLoaderTests.cs +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Import/ImportResourceLoaderTests.cs @@ -243,7 +243,7 @@ private async Task VerifyResourceLoaderAsync(int resourcCount, int batchSize, lo null, null, "SearchParam"); - return new ImportResource(index, 0, 0, false, false, false, resourceWrapper); + return new ImportResource(index, 0, 0, false, false, resourceWrapper); }); IImportErrorSerializer serializer = Substitute.For(); diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Import/ImportResource.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Import/ImportResource.cs index 5598d739c4..83078344bc 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Import/ImportResource.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Import/ImportResource.cs @@ -9,19 +9,18 @@ namespace Microsoft.Health.Fhir.Core.Features.Operations.Import { public class ImportResource { - public ImportResource(long index, long offset, int length, bool keepLastUpdated, bool keepVersion, bool isDeleted, ResourceWrapper resourceWrapper) + public ImportResource(long index, long offset, int length, bool keepLastUpdated, bool keepVersion, ResourceWrapper resourceWrapper) { Index = index; Offset = offset; Length = length; KeepLastUpdated = keepLastUpdated; KeepVersion = keepVersion; - IsDeleted = isDeleted; ResourceWrapper = resourceWrapper; } public ImportResource(ResourceWrapper resource) - : this(0, 0, 0, false, false, false, resource) + : this(0, 0, 0, false, false, resource) { } @@ -32,7 +31,6 @@ public ImportResource(long index, long offset, string importError) Length = 0; KeepLastUpdated = false; KeepVersion = false; - IsDeleted = false; ImportError = importError; } @@ -61,11 +59,6 @@ public ImportResource(long index, long offset, string importError) /// public bool KeepVersion { get; set; } - /// - /// Flag indicating whether deletion state was provided on input - /// - public bool IsDeleted { get; set; } - /// /// Resource wrapper from raw content /// diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/Import/ImportResourceParserTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/Import/ImportResourceParserTests.cs deleted file mode 100644 index 82706bb17d..0000000000 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Operations/Import/ImportResourceParserTests.cs +++ /dev/null @@ -1,77 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System; -using Hl7.Fhir.Model; -using Hl7.Fhir.Serialization; -using Microsoft.Health.Core.Features.Context; -using Microsoft.Health.Core.Features.Security; -using Microsoft.Health.Fhir.Core.Features.Compartment; -using Microsoft.Health.Fhir.Core.Features.Context; -using Microsoft.Health.Fhir.Core.Features.Definition; -using Microsoft.Health.Fhir.Core.Features.Operations.Import; -using Microsoft.Health.Fhir.Core.Features.Persistence; -using Microsoft.Health.Fhir.Core.Features.Search; -using Microsoft.Health.Fhir.Core.Models; -using Microsoft.Health.Fhir.Shared.Core.Features.Operations.Import; -using Microsoft.Health.Fhir.Tests.Common; -using Microsoft.Health.Test.Utilities; -using NSubstitute; -using Xunit; - -namespace Microsoft.Health.Fhir.Shared.Core.UnitTests.Features.Operations.Import -{ - [Trait(Traits.OwningTeam, OwningTeam.FhirImport)] - [Trait(Traits.Category, Categories.Import)] - public class ImportResourceParserTests - { - private readonly FhirJsonSerializer _jsonSerializer = new FhirJsonSerializer(); - - private readonly FhirJsonParser _jsonParser = new(); - - private readonly ResourceWrapperFactory _wrapperFactory; - - private readonly ImportResourceParser _importResourceParser; - - public ImportResourceParserTests() - { - var requestContextAccessor = Substitute.For>(); - - requestContextAccessor.RequestContext.Method.Returns("PUT"); - requestContextAccessor.RequestContext.Uri.Returns(new Uri("https://unittest/Patient/123")); - - _wrapperFactory = new ResourceWrapperFactory( - new RawResourceFactory(new FhirJsonSerializer()), - requestContextAccessor, - Substitute.For(), - Substitute.For(), - Substitute.For(), - Substitute.For(), - Deserializers.ResourceDeserializer); - - _importResourceParser = new(_jsonParser, _wrapperFactory); - } - - [Fact] - public void GivenImportWithSoftDeletedFile_WhenParsed_DeletedExtensionShouldBeRemoved() - { - Patient patient = new Patient(); - patient.Id = Guid.NewGuid().ToString(); - patient.Name.Add(HumanName.ForFamily("Test")); - patient.Meta = new() - { - LastUpdated = DateTimeOffset.UtcNow, - VersionId = "3", - }; - patient.Meta.AddExtension(KnownFhirPaths.AzureSoftDeletedExtensionUrl, new FhirString("soft-deleted")); - - string patientAsString = _jsonSerializer.SerializeToString(patient); - var importResource = _importResourceParser.Parse(0, 0, 0, patientAsString, ImportMode.IncrementalLoad); - - Assert.DoesNotContain(KnownFhirPaths.AzureSoftDeletedExtensionUrl, importResource.ResourceWrapper.RawResource.Data); - Assert.DoesNotContain("soft-deleted", importResource.ResourceWrapper.RawResource.Data); - } - } -} diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems index 18e06f63a5..d492ee7645 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems @@ -21,7 +21,6 @@ - diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Import/ImportResourceParser.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Import/ImportResourceParser.cs index 632cc4e8f9..c495159eb0 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Import/ImportResourceParser.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Import/ImportResourceParser.cs @@ -53,16 +53,9 @@ public ImportResource Parse(long index, long offset, int length, string rawResou var resourceElement = resource.ToResourceElement(); - var isDeleted = resourceElement.IsSoftDeleted(); + var resourceWapper = _resourceFactory.Create(resourceElement, true, keepVersion); - if (isDeleted) - { - resource.Meta.RemoveExtension(KnownFhirPaths.AzureSoftDeletedExtensionUrl); - } - - var resourceWapper = _resourceFactory.Create(resourceElement, isDeleted, true, keepVersion); - - return new ImportResource(index, offset, length, !lastUpdatedIsNull, keepVersion, isDeleted, resourceWapper); + return new ImportResource(index, offset, length, !lastUpdatedIsNull, keepVersion, resourceWapper); } private static void CheckConditionalReferenceInResource(Resource resource, ImportMode importMode) diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Operations/Import/ImportProcessingJobTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Operations/Import/ImportProcessingJobTests.cs index bf6d25d65c..add95eb58e 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Operations/Import/ImportProcessingJobTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Operations/Import/ImportProcessingJobTests.cs @@ -152,7 +152,7 @@ private static async Task VerifyCommonImportAsync(ImportProcessingJobDefinition null, "SearchParam"); - await resourceChannel.Writer.WriteAsync(new ImportResource(0, 0, 0, false, false, false, resourceWrapper)); + await resourceChannel.Writer.WriteAsync(new ImportResource(0, 0, 0, false, false, resourceWrapper)); await resourceChannel.Writer.WriteAsync(new ImportResource(1, 0, "Error")); resourceChannel.Writer.Complete(); }); From 59b0d059dd3d271662557c2e48bb344ba7c3b5a3 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Wed, 20 Sep 2023 09:12:59 -0700 Subject: [PATCH 17/58] Removed launch.json changes accidentally committed --- .vscode/launch.json | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 3a1f5d9283..1e6ec2f2a7 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -46,10 +46,6 @@ "SqlServer:SchemaOptions:AutomaticUpdatesEnabled": "true", "TestAuthEnvironment:FilePath": "..//..//testauthenvironment.json", "FhirServer:Security:Enabled": "false", - "TaskHosting:Enabled": "true", - "TaskHosting:MaxRunningTaskCount": "2", - "FhirServer:Operations:IntegrationDataStore:StorageAccountConnection": "UseDevelopmentStorage=true", - "FhirServer:Operations:Import:Enabled": "true" }, "sourceFileMap": { "/Views": "${workspaceFolder}/Views" From 60b4fbccdb877ad21820ffcb80d7ebc1f17b5c98 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Wed, 20 Sep 2023 15:35:41 -0700 Subject: [PATCH 18/58] Added initial SQL export with soft delete / history --- .../Visitors/QueryGenerators/SqlQueryGenerator.cs | 7 ++++--- .../Features/Search/SqlSearchType.cs | 9 ++++++--- .../Features/Search/SqlServerSearchService.cs | 12 +++++++++--- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs index 2c1501a667..c5b4004c1d 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs @@ -192,7 +192,8 @@ public override object VisitSqlRoot(SqlRootExpression expression, SearchOptions if (_schemaInfo.Current < SchemaVersionConstants.PartitionedTables && expression.SearchParamTableExpressions.Count == 0 && - !_searchType.HasFlag(SqlSearchType.History) && + !_searchType.HasFlag(SqlSearchType.IncludeHistory) && + !_searchType.HasFlag(SqlSearchType.IncludeDeleted) && expression.ResourceTableExpressions.Any(e => e.AcceptVisitor(ExpressionContainsParameterVisitor.Instance, SearchParameterNames.ResourceType)) && !expression.ResourceTableExpressions.Any(e => e.AcceptVisitor(ExpressionContainsParameterVisitor.Instance, SearchParameterNames.Id))) { @@ -1316,7 +1317,7 @@ int FindImpl(int currentIndex) private void AppendDeletedClause(in IndentedStringBuilder.DelimitedScope delimited, string tableAlias = null) { - if (!_searchType.HasFlag(SqlSearchType.History)) + if (!_searchType.HasFlag(SqlSearchType.IncludeDeleted)) { delimited.BeginDelimitedElement().Append(VLatest.Resource.IsDeleted, tableAlias).Append(" = 0"); } @@ -1324,7 +1325,7 @@ private void AppendDeletedClause(in IndentedStringBuilder.DelimitedScope delimit private void AppendHistoryClause(in IndentedStringBuilder.DelimitedScope delimited, string tableAlias = null) { - if (!_searchType.HasFlag(SqlSearchType.History)) + if (!_searchType.HasFlag(SqlSearchType.IncludeHistory)) { delimited.BeginDelimitedElement(); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlSearchType.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlSearchType.cs index 6760be6329..dacbc31368 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlSearchType.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlSearchType.cs @@ -16,13 +16,16 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Search [Flags] internal enum SqlSearchType { - // Set if we do not need to consider history or reindexing + // Set if we do not need to consider history, deleted resources, or re-indexing Default = 0, - // Set if we are including previous resource versions or deleted resources - History = 1 << 0, + // Set if we are including previous resource versions without deleted resources + IncludeHistory = 1 << 0, // Set if the search parameter hash value needs to be considered in a search Reindex = 1 << 1, + + // Set if we are including deleted resource versions + IncludeDeleted = 1 << 2, } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs index 6aac0c4f0e..7c08174306 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs @@ -122,7 +122,12 @@ public SqlServerSearchService( public override async Task SearchAsync(SearchOptions searchOptions, CancellationToken cancellationToken) { SqlSearchOptions sqlSearchOptions = new SqlSearchOptions(searchOptions); - SearchResult searchResult = await SearchImpl(sqlSearchOptions, SqlSearchType.Default, null, cancellationToken); + + SqlSearchType searchType = SqlSearchType.Default; + searchType |= sqlSearchOptions.IncludeHistory ? SqlSearchType.IncludeHistory : searchType; + searchType |= sqlSearchOptions.IncludeDeleted ? SqlSearchType.IncludeDeleted : searchType; + + SearchResult searchResult = await SearchImpl(sqlSearchOptions, searchType, null, cancellationToken); int resultCount = searchResult.Results.Count(); if (!sqlSearchOptions.IsSortWithFilter && searchResult.ContinuationToken == null && @@ -203,7 +208,7 @@ public override async Task SearchAsync(SearchOptions searchOptions protected override async Task SearchHistoryInternalAsync(SearchOptions searchOptions, CancellationToken cancellationToken) { SqlSearchOptions sqlSearchOptions = new SqlSearchOptions(searchOptions); - return await SearchImpl(sqlSearchOptions, SqlSearchType.History, null, cancellationToken); + return await SearchImpl(sqlSearchOptions, SqlSearchType.IncludeHistory & SqlSearchType.IncludeDeleted, null, cancellationToken); } private async Task SearchImpl(SqlSearchOptions sqlSearchOptions, SqlSearchType searchType, string currentSearchParameterHash, CancellationToken cancellationToken) @@ -282,6 +287,7 @@ private async Task SearchImpl(SqlSearchOptions sqlSearchOptions, S searchExpression = searchExpression?.AcceptVisitor(RemoveIncludesRewriter.Instance); } + // ! - Trace SqlRootExpression expression = (SqlRootExpression)searchExpression ?.AcceptVisitor(LastUpdatedToResourceSurrogateIdRewriter.Instance) .AcceptVisitor(_compartmentSearchRewriter) @@ -694,7 +700,7 @@ public override async Task> GetUsedResourceTypes(Cancellat private SqlSearchOptions UpdateSort(SqlSearchOptions searchOptions, Expression searchExpression, SqlSearchType sqlSearchType) { SqlSearchOptions newSearchOptions = searchOptions; - if (sqlSearchType == SqlSearchType.History) + if (sqlSearchType.HasFlag(SqlSearchType.IncludeHistory)) { // history is always sorted by _lastUpdated. newSearchOptions = searchOptions.CloneSqlSearchOptions(); From e40516d4144f17de46a8b1e776f1beff9fe6481e Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Wed, 20 Sep 2023 20:52:49 -0700 Subject: [PATCH 19/58] oopsies undo change to importresourceprocessor breaking tests --- .../Features/Operations/Import/ImportResourceParser.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Import/ImportResourceParser.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Import/ImportResourceParser.cs index c495159eb0..bdfd341be4 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Import/ImportResourceParser.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Import/ImportResourceParser.cs @@ -52,8 +52,7 @@ public ImportResource Parse(long index, long offset, int length, string rawResou } var resourceElement = resource.ToResourceElement(); - - var resourceWapper = _resourceFactory.Create(resourceElement, true, keepVersion); + var resourceWapper = _resourceFactory.Create(resourceElement, false, true, keepVersion); return new ImportResource(index, offset, length, !lastUpdatedIsNull, keepVersion, resourceWapper); } From f6c05384b6cb4cbdbbd1531e0b0390570fd27f00 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Wed, 20 Sep 2023 22:35:52 -0700 Subject: [PATCH 20/58] Fixed logical error in new SQL historical search --- .../Features/Search/SqlServerSearchService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs index 7c08174306..faf8121e89 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs @@ -208,7 +208,7 @@ public override async Task SearchAsync(SearchOptions searchOptions protected override async Task SearchHistoryInternalAsync(SearchOptions searchOptions, CancellationToken cancellationToken) { SqlSearchOptions sqlSearchOptions = new SqlSearchOptions(searchOptions); - return await SearchImpl(sqlSearchOptions, SqlSearchType.IncludeHistory & SqlSearchType.IncludeDeleted, null, cancellationToken); + return await SearchImpl(sqlSearchOptions, SqlSearchType.IncludeHistory | SqlSearchType.IncludeDeleted, null, cancellationToken); } private async Task SearchImpl(SqlSearchOptions sqlSearchOptions, SqlSearchType searchType, string currentSearchParameterHash, CancellationToken cancellationToken) From 5d4b5fd15c46e659601e197f8949adbb687c20d6 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 21 Sep 2023 06:32:47 -0700 Subject: [PATCH 21/58] Added searchoptionsfactory test for include history/deleted --- .../Search/SearchOptionsFactoryTests.cs | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs index 41dfd22282..e32603e74b 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs @@ -487,6 +487,36 @@ public void GivenSearchParameterText_WhenCreated_ThenSearchParameterShouldBeAdde Assert.Single(options.UnsupportedSearchParams); } + [Theory] + [InlineData("false", "false")] + [InlineData("true", "false")] + [InlineData("false", "true")] + [InlineData("true", "true")] + [InlineData("TRUE", "TRUE")] + [InlineData("bad", "bad")] + public void GivenIncludeHistoryAndDeletedParameters_WhenCreated_ThenSearchParametersShouldMatchInput(string includeHistory, string includeDeleted) + { + var queryParameters = new[] + { + Tuple.Create(KnownQueryParameterNames.IncludeHistory, includeHistory), + Tuple.Create(KnownQueryParameterNames.IncludeDeleted, includeDeleted), + }; + + SearchOptions options = CreateSearchOptions(ResourceType.Patient.ToString(), queryParameters); + Assert.NotNull(options); + + bool includeHistoryBool = false; + bool includeDeletedBool = false; + + bool.TryParse(includeHistory, out includeHistoryBool); + bool.TryParse(includeDeleted, out includeDeletedBool); + + Assert.Equal(includeHistoryBool, options.IncludeHistory); + Assert.Equal(includeDeletedBool, options.IncludeDeleted); + + Assert.Empty(options.UnsupportedSearchParams); + } + private SearchOptions CreateSearchOptions( string resourceType = DefaultResourceType, IReadOnlyList> queryParameters = null, From ba97d8885e4e5cd3984f0d2e3916b95f86d6aa5a Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Tue, 26 Sep 2023 22:51:04 -0700 Subject: [PATCH 22/58] Fixed failing export test failure. --- .../Rest/ExportTestFixture.cs | 239 ++++++++++++++---- 1 file changed, 190 insertions(+), 49 deletions(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestFixture.cs index 4239c19f1f..0754c0e976 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestFixture.cs @@ -9,7 +9,9 @@ using Hl7.Fhir.Model; using MediatR; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Core.Features.Operations.Export; +using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Microsoft.Health.Fhir.Tests.E2E.Rest.Metric; using Task = System.Threading.Tasks.Task; @@ -32,7 +34,7 @@ public MetricHandler MetricHandler public Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestResourcesWithHistoryAndDeletes { get; } = new(); - public Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestResourcesWithHistory => TestResourcesWithHistory + public Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestResourcesWithHistory => TestResourcesWithHistoryAndDeletes .Where(entry => !entry.Value.Meta.Extension.Any(extension => extension.Url == "http://azurehealthcareapis.com/data-extensions/deleted-state" && ((FhirString)extension.Value).Value == "soft-deleted")) @@ -54,81 +56,220 @@ public MetricHandler MetricHandler protected override async Task OnInitializedAsync() { - string NewGuidString() => Guid.NewGuid().ToString(); - void AddResourceToTestResources(Resource resource) => TestResourcesWithHistoryAndDeletes[(resource.TypeName, resource.Id, resource.VersionId)] = resource; - Meta testDataMeta = new Meta() + void AddResourcesToTestResources(List resources) => resources.ForEach(AddResourceToTestResources); + + var testResourcesInfo = GenerateTestResources().Select(x => (x, false)).ToList(); + + while (testResourcesInfo.Count > 0) { - Tag = new List() + var testResourceResponse = await SaveResourceListToServer(testResourcesInfo); + + try { - new Coding("http://e2e-test", FixtureTag), - }, - }; + AddResourcesToTestResources(testResourceResponse); + + testResourcesInfo = new(); + + for (int i = 0; i < testResourceResponse.Count; i++) + { + var resource = testResourceResponse[i]; + + if (resource.Meta.Extension.Any(x => x.Url == KnownFhirPaths.AzureSoftDeletedExtensionUrl)) + { + // Skip already deleted resources for now. + // TODO - add un-deletes in here. + continue; + } + else if (i % 10 == 0) + { + Practitioner deletedResource = resource.DeepCopy() as Practitioner; + testResourcesInfo.Add((deletedResource, true)); + } + else if (i % 4 == 1) + { + Practitioner updatedResource = resource.DeepCopy() as Practitioner; + updatedResource.Name.Add(new() + { + Given = updatedResource.Name.First().Given, + Family = $"UpdatedFromVersion{updatedResource.Meta.VersionId}", + }); + + testResourcesInfo.Add((updatedResource, false)); + } + } + } + catch (Exception ex) + { + Console.WriteLine(ex); + } + } + + Console.WriteLine($"Generated {TestResourcesWithHistoryAndDeletes.Count} resources."); + } + + private List GenerateTestResources(int numberOfResources = 100) + { + var resources = new List(); + + for (int i = 0; i < numberOfResources; i++) + { + resources.Add(new Practitioner + { + Id = Guid.NewGuid().ToString("N"), + Meta = new() { Tag = new List() { new Coding("http://e2e-test", FixtureTag) } }, + Active = true, + Name = new List() { new HumanName() { Family = $"Test{i}", Given = new List { "Export", "History", "SoftDelete" } } }, + }); + } + + return resources; + } + + private async System.Threading.Tasks.Task> SaveResourceListToServer(List<(Resource resource, bool delete)> entries) + { + if (entries.Count > 500) + { + throw new ArgumentException("The number of resources to save must be less than or equal to 500."); + } - List<(bool updateResource, bool deleteResource)> testResourceData = new() + var bundle = new Bundle { - (false, false), - (false, true), - (true, false), - (true, true), + Type = Bundle.BundleType.Transaction, + Entry = new List(), }; - foreach (var d in testResourceData) + foreach (var entry in entries) { - var patient = new Patient() + bundle.Entry.Add(new Bundle.EntryComponent { - Id = NewGuidString(), - Meta = testDataMeta, - }; + Resource = entry.resource, + Request = new Bundle.RequestComponent + { + Method = entry.delete ? Bundle.HTTPVerb.DELETE : Bundle.HTTPVerb.PUT, + Url = $"{entry.resource.TypeName}/{entry.resource.Id}", + }, + FullUrl = $"{TestFhirClient.HttpClient.BaseAddress}{entry.resource.TypeName}/{entry.resource.Id}", + }); + } + + FhirResponse response = await TestFhirClient.PostBundleAsync(bundle); + + if (response.StatusCode != System.Net.HttpStatusCode.OK) + { + throw new Exception("Could not save resources to server."); + } + + List rtn = new(); - var observation = new Observation() + for (int i = 0; i < response.Resource.Entry.Count; i++) + { + var inputResource = entries[i].resource; + var responseEntry = response.Resource.Entry[i]; + + if (responseEntry.Resource is not null) { - Id = NewGuidString(), - Meta = testDataMeta, - Status = ObservationStatus.Final, - Code = new CodeableConcept("http://loinc.org", "12345-6"), - Subject = new ResourceReference($"Patient/{patient.Id}"), - }; + rtn.Add(responseEntry.Resource); + } + else + { + var allResourcesWithDeleted = await TestFhirClient.SearchAsync($"{inputResource.TypeName}/{inputResource.Id}/_history"); + var deletedResource = allResourcesWithDeleted.Resource.Entry.OrderByDescending(x => x.Resource.Meta.LastUpdated).First().Resource; + deletedResource.Meta.Extension.Add(new Extension(KnownFhirPaths.AzureSoftDeletedExtensionUrl, new FhirString("soft-deleted"))); + rtn.Add(deletedResource); + } + } + + return rtn; + } + + /* + private static List GenerateTestResources(int numberOfPatients = 20) + { + string[] firstNames = { "John", "Jane", "Robert", "Emily", "Michael", "Sarah", "William", "Anna", "James", "Laura" }; + + string[] lastNames = { "Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia", "Miller", "Davis", "Rodriguez", "Martinez" }; + + static string RandomBirthdate() + { + DateTime startDate = new DateTime(1970, 1, 1); + int range = (DateTime.Today - startDate).Days; + return startDate.AddDays(_random.Next(range)).ToString("yyyy-MM-dd"); + } - var patientResponse = await TestFhirClient.UpdateAsync(patient); - var observationResponse = await TestFhirClient.UpdateAsync(observation); + string RandomLastName() => lastNames[_random.Next(lastNames.Length)]; - AddResourceToTestResources(patientResponse.Resource); - AddResourceToTestResources(observationResponse.Resource); + string RandomFirstName() => firstNames[_random.Next(firstNames.Length)]; - if (d.updateResource) + Encounter CreateEncounter(string patientId) + { + return new() { - patient = patientResponse.Resource.DeepCopy() as Patient; - patient.Name.Add(new HumanName() { Family = "Updated" }); + Id = Guid.NewGuid().ToString("N"), + Meta = new() { Tag = new List() { new Coding("http://e2e-test", FixtureTag) }}, + Status = Encounter.EncounterStatus.Planned, + Subject = new ResourceReference($"Patient/{patientId}"), + }; + } - observation = observationResponse.Resource.DeepCopy() as Observation; - observation.Code.Coding.Add(new Coding("http://loinc.org", "12345-7")); + Observation CreateObservation(string patientId, string encounterId) + { + return new() + { + Id = Guid.NewGuid().ToString("N"), + Meta = new() { Tag = new List() { new Coding("http://e2e-test", FixtureTag) }}, + Status = ObservationStatus.Preliminary, + Code = new CodeableConcept("http://loinc.org", "12345-6"), + Subject = new ResourceReference($"Patient/{patientId}"), + Encounter = new ResourceReference($"Encounter/{encounterId}"), + }; + } - patientResponse = await TestFhirClient.UpdateAsync(patient); - observationResponse = await TestFhirClient.UpdateAsync(observation); + var resources = new List(); - AddResourceToTestResources(patientResponse.Resource); - AddResourceToTestResources(observationResponse.Resource); - } + for (int i = 0; i < numberOfPatients; i++) + { + var patient = new Patient() + { + Id = Guid.NewGuid().ToString("N"), + Meta = new() { Tag = new List() { new Coding("http://e2e-test", FixtureTag) }}, + Active = true, + Name = new List() { new HumanName() { Family = RandomLastName(), Given = new List { RandomFirstName() }} }, + BirthDate = RandomBirthdate(), + }; + resources.Add(patient); - if (d.deleteResource) + for (int j = 0; j < _random.Next(1, 5); j++) { - await TestFhirClient.DeleteAsync($"Patient/{patient.Id}"); - await TestFhirClient.DeleteAsync($"Observation/{observation.Id}"); + var encounter = CreateEncounter(patient.Id); + resources.Add(encounter); - // Since we don't get the version id back from the delete response, we need to pull - // this data from the server. - foreach (var resourceInfo in new List<(string Type, string Id)> { ("Patient", patient.Id), ("Observation", observation.Id) }) + for (int k = 0; k < _random.Next(1, 5); k++) { - var allResourcesWithDeleted = await TestFhirClient.SearchAsync($"{resourceInfo.Type}/{resourceInfo.Id}/_history"); - var deletedResource = allResourcesWithDeleted.Resource.Entry.OrderByDescending(x => x.Resource.Meta.LastUpdated).First().Resource; - deletedResource.Meta.Extension.Add(new Extension("http://azurehealthcareapis.com/data-extensions/deleted-state", new FhirString("soft-deleted"))); - AddResourceToTestResources(deletedResource); + var observation = CreateObservation(patient.Id, encounter.Id); + resources.Add(observation); + + while (observation.Id.GetHashCode() % 4 != 0) + { + observation = observation.DeepCopy() as Observation; + observation.Code.Coding.Add(new Coding("http://loinc.org", "12345-1")); + resources.Add(observation); + } + } + + while (encounter.Id.GetHashCode() % 4 != 0) + { + encounter = encounter.DeepCopy() as Encounter; + encounter.ClassHistory.Add(new Encounter.ClassHistoryComponent() { Class = new Coding("http://hl7.org/fhir/v3/ActCode", "EMER") }); + resources.Add(encounter); } } } + + return resources; } + */ } } From 87d7dc6863b8bfaac72668b2fc8826bd859634c1 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Wed, 27 Sep 2023 06:41:58 -0700 Subject: [PATCH 23/58] Removed long running export flag to see if it runs in pipeline --- .../Rest/ExportLongRunningTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportLongRunningTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportLongRunningTests.cs index d101f646ab..eda78d7143 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportLongRunningTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportLongRunningTests.cs @@ -21,7 +21,6 @@ namespace Microsoft.Health.Fhir.Tests.E2E.Rest { [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.Export)] - [Trait(Traits.Category, Categories.ExportLongRunning)] [HttpIntegrationFixtureArgumentSets(DataStore.All, Format.Json)] public class ExportLongRunningTests : IClassFixture { From 83914451e25602b798873085773f14b434b3178f Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 28 Sep 2023 08:53:40 -0700 Subject: [PATCH 24/58] restructured export test location for readability --- .../Features/Schema/Migrations/64.diff.sql | 75 + .../Features/Schema/Migrations/64.sql | 6750 +++++++++++++++++ .../Features/Schema/SchemaVersion.cs | 1 + .../Features/Schema/SchemaVersionConstants.cs | 2 +- .../TransactionCheckWithInitialiScript.sql | 2 +- .../GetResourcesByTypeAndSurrogateIdRange.sql | 14 +- .../Features/Search/SqlServerSearchService.cs | 12 +- .../Microsoft.Health.Fhir.SqlServer.csproj | 2 +- .../Microsoft.Health.Fhir.R4.Tests.E2E.csproj | 4 +- ...oft.Health.Fhir.Shared.Tests.E2E.projitems | 12 +- ...rosoft.Health.Fhir.Shared.Tests.E2E.shproj | 4 +- .../{ => Export}/AnonymizedExportTests.cs | 6 +- .../AnonymizedExportUsingAcrTests.cs | 6 +- .../ExportDataTestFixture.cs} | 155 +- .../ExportDataTests.cs} | 75 +- .../{ => Export}/ExportDataValidationTests.cs | 8 +- .../Rest/{ => Export}/ExportTestHelper.cs | 19 +- .../Rest/{ => Export}/ExportTests.cs | 2 +- .../StartupForExportTestProvider.cs | 2 +- ...icrosoft.Health.Fhir.Stu3.Tests.E2E.csproj | 4 +- 20 files changed, 6991 insertions(+), 164 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/64.diff.sql create mode 100644 src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/64.sql rename test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/{ => Export}/AnonymizedExportTests.cs (99%) rename test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/{ => Export}/AnonymizedExportUsingAcrTests.cs (99%) rename test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/{ExportTestFixture.cs => Export/ExportDataTestFixture.cs} (62%) rename test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/{ExportLongRunningTests.cs => Export/ExportDataTests.cs} (78%) rename test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/{ => Export}/ExportDataValidationTests.cs (97%) rename test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/{ => Export}/ExportTestHelper.cs (95%) rename test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/{ => Export}/ExportTests.cs (99%) rename test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/{ => Export}/StartupForExportTestProvider.cs (96%) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/64.diff.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/64.diff.sql new file mode 100644 index 0000000000..196fdf6905 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/64.diff.sql @@ -0,0 +1,75 @@ +CREATE OR ALTER PROCEDURE dbo.GetResourcesByTypeAndSurrogateIdRange @ResourceTypeId smallint, @StartId bigint, @EndId bigint, @GlobalStartId bigint = NULL, @GlobalEndId bigint = NULL, @IncludeHistory bit = 0, @IncludeDeleted bit = 0 +AS +set nocount on +DECLARE @SP varchar(100) = 'GetResourcesByTypeAndSurrogateIdRange' + ,@Mode varchar(100) = 'RT='+isnull(convert(varchar,@ResourceTypeId),'NULL') + +' S='+isnull(convert(varchar,@StartId),'NULL') + +' E='+isnull(convert(varchar,@EndId),'NULL') + +' GS='+isnull(convert(varchar,@GlobalStartId),'NULL') -- Is global start id needed? I'm not seeing a usecase for setting it. + +' GE='+isnull(convert(varchar,@GlobalEndId),'NULL') -- Could this just be a boolean for if historical records should be returned? GlobalEndId should equal EndId in all cases I can think of. + +' HI='+isnull(convert(varchar,@IncludeHistory),'NULL') + +' DE'+isnull(convert(varchar,@IncludeDeleted),'NULL') + ,@st datetime = getUTCdate() + +BEGIN TRY + DECLARE @ResourceIds TABLE (ResourceId varchar(64) COLLATE Latin1_General_100_CS_AS, ResourceSurrogateId bigint, RowId int, PRIMARY KEY (ResourceId, RowId)) + + IF @GlobalStartId IS NULL -- export from time zero (no lower boundary) + SET @GlobalStartId = 0 + + IF @GlobalEndId IS NOT NULL -- snapshot view + INSERT INTO @ResourceIds + SELECT ResourceId, ResourceSurrogateId, RowId = row_number() OVER (PARTITION BY ResourceId ORDER BY ResourceSurrogateId DESC) + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceId IN (SELECT DISTINCT ResourceId + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + AND IsHistory = 1 + AND (IsDeleted = 0 OR @IncludeDeleted = 1) -- TBD if this is needed. + ) + AND ResourceSurrogateId BETWEEN @GlobalStartId AND @GlobalEndId + + IF EXISTS (SELECT * FROM @ResourceIds) + BEGIN + DECLARE @SurrogateIdMap TABLE (MaxSurrogateId bigint PRIMARY KEY) + INSERT INTO @SurrogateIdMap + SELECT MaxSurrogateId = A.ResourceSurrogateId + FROM (SELECT * FROM @ResourceIds WHERE RowId = 1 AND ResourceSurrogateId BETWEEN @StartId AND @EndId) A + + SELECT @ResourceTypeId + ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.ResourceId ELSE A.ResourceId END + ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.Version ELSE A.Version END + ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.IsDeleted ELSE A.IsDeleted END + ,isnull(C.ResourceSurrogateId, A.ResourceSurrogateId) + ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.RequestMethod ELSE A.RequestMethod END + ,IsMatch = convert(bit,1) + ,IsPartial = convert(bit,0) + ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.IsRawResourceMetaSet ELSE A.IsRawResourceMetaSet END + ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.SearchParamHash ELSE A.SearchParamHash END + ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.RawResource ELSE A.RawResource END + FROM dbo.Resource A + LEFT OUTER JOIN @SurrogateIdMap B ON B.MaxSurrogateId = A.ResourceSurrogateId + LEFT OUTER JOIN dbo.Resource C ON C.ResourceTypeId = @ResourceTypeId AND C.ResourceSurrogateId = MaxSurrogateId + WHERE A.ResourceTypeId = @ResourceTypeId + AND A.ResourceSurrogateId BETWEEN @StartId AND @EndId + AND (A.IsHistory = 0 OR MaxSurrogateId IS NOT NULL OR @IncludeHistory = 1) + AND (A.IsDeleted = 0 OR @IncludeDeleted = 1) + END + ELSE + SELECT ResourceTypeId, ResourceId, Version, IsDeleted, ResourceSurrogateId, RequestMethod, IsMatch = convert(bit,1), IsPartial = convert(bit,0), IsRawResourceMetaSet, SearchParamHash, RawResource + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + AND (IsHistory = 0 OR @IncludeHistory = 1) + AND (IsDeleted = 0 OR @IncludeDeleted = 1) + + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@st,@Rows=@@rowcount +END TRY +BEGIN CATCH + IF error_number() = 1750 THROW -- Real error is before 1750, cannot trap in SQL. + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Error'; + THROW +END CATCH +GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/64.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/64.sql new file mode 100644 index 0000000000..ab1df0985b --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/64.sql @@ -0,0 +1,6750 @@ + +/************************************************************************************************* + Auto-Generated from Sql build task. Do not manually edit it. +**************************************************************************************************/ +SET XACT_ABORT ON +BEGIN TRAN +IF EXISTS (SELECT * + FROM sys.tables + WHERE name = 'ClaimType') + BEGIN + ROLLBACK; + RETURN; + END + + +GO +INSERT INTO dbo.SchemaVersion +VALUES (64, 'started'); + +CREATE PARTITION FUNCTION PartitionFunction_ResourceTypeId(SMALLINT) + AS RANGE RIGHT + FOR VALUES (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150); + +CREATE PARTITION SCHEME PartitionScheme_ResourceTypeId + AS PARTITION PartitionFunction_ResourceTypeId + ALL TO ([PRIMARY]); + + +GO +CREATE PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp(DATETIME2 (7)) + AS RANGE RIGHT + FOR VALUES (N'1970-01-01T00:00:00.0000000'); + +CREATE PARTITION SCHEME PartitionScheme_ResourceChangeData_Timestamp + AS PARTITION PartitionFunction_ResourceChangeData_Timestamp + ALL TO ([PRIMARY]); + +DECLARE @numberOfHistoryPartitions AS INT = 48; + +DECLARE @numberOfFuturePartitions AS INT = 720; + +DECLARE @rightPartitionBoundary AS DATETIME2 (7); + +DECLARE @currentDateTime AS DATETIME2 (7) = sysutcdatetime(); + +WHILE @numberOfHistoryPartitions >= -@numberOfFuturePartitions + BEGIN + SET @rightPartitionBoundary = DATEADD(hour, DATEDIFF(hour, 0, @currentDateTime) - @numberOfHistoryPartitions, 0); + ALTER PARTITION SCHEME PartitionScheme_ResourceChangeData_Timestamp NEXT USED [Primary]; + ALTER PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp( ) + SPLIT RANGE (@rightPartitionBoundary); + SET @numberOfHistoryPartitions -= 1; + END + +CREATE SEQUENCE dbo.ResourceSurrogateIdUniquifierSequence + AS INT + START WITH 0 + INCREMENT BY 1 + MINVALUE 0 + MAXVALUE 79999 + CYCLE + CACHE 1000000; + +CREATE TYPE dbo.BigintList AS TABLE ( + Id BIGINT NOT NULL PRIMARY KEY); + +CREATE TYPE dbo.CompartmentAssignmentList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + CompartmentTypeId TINYINT NOT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL PRIMARY KEY (ResourceTypeId, ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId)); + +CREATE TYPE dbo.DateTimeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + StartDateTime DATETIMEOFFSET (7) NOT NULL, + EndDateTime DATETIMEOFFSET (7) NOT NULL, + IsLongerThanADay BIT NOT NULL, + IsMin BIT NOT NULL, + IsMax BIT NOT NULL UNIQUE (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax)); + +CREATE TYPE dbo.NumberSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SingleValue DECIMAL (36, 18) NULL, + LowValue DECIMAL (36, 18) NULL, + HighValue DECIMAL (36, 18) NULL UNIQUE (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue)); + +CREATE TYPE dbo.QuantitySearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + QuantityCodeId INT NULL, + SingleValue DECIMAL (36, 18) NULL, + LowValue DECIMAL (36, 18) NULL, + HighValue DECIMAL (36, 18) NULL UNIQUE (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue)); + +CREATE TYPE dbo.ReferenceSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId SMALLINT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion INT NULL UNIQUE (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId)); + +CREATE TYPE dbo.ReferenceTokenCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId1 SMALLINT NULL, + ReferenceResourceId1 VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion1 INT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.ResourceDateKeyList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ResourceSurrogateId BIGINT NOT NULL PRIMARY KEY (ResourceTypeId, ResourceId, ResourceSurrogateId)); + +CREATE TYPE dbo.ResourceKeyList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Version INT NULL UNIQUE (ResourceTypeId, ResourceId, Version)); + +CREATE TYPE dbo.ResourceList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Version INT NOT NULL, + HasVersionToCompare BIT NOT NULL, + IsDeleted BIT NOT NULL, + IsHistory BIT NOT NULL, + KeepHistory BIT NOT NULL, + RawResource VARBINARY (MAX) NOT NULL, + IsRawResourceMetaSet BIT NOT NULL, + RequestMethod VARCHAR (10) NULL, + SearchParamHash VARCHAR (64) NULL PRIMARY KEY (ResourceTypeId, ResourceSurrogateId), + UNIQUE (ResourceTypeId, ResourceId, Version)); + +CREATE TYPE dbo.ResourceWriteClaimList AS TABLE ( + ResourceSurrogateId BIGINT NOT NULL, + ClaimTypeId TINYINT NOT NULL, + ClaimValue NVARCHAR (128) NOT NULL); + +CREATE TYPE dbo.StringList AS TABLE ( + String VARCHAR (MAX)); + +CREATE TYPE dbo.StringSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL, + IsMin BIT NOT NULL, + IsMax BIT NOT NULL); + +CREATE TYPE dbo.TokenDateTimeCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + StartDateTime2 DATETIMEOFFSET (7) NOT NULL, + EndDateTime2 DATETIMEOFFSET (7) NOT NULL, + IsLongerThanADay2 BIT NOT NULL); + +CREATE TYPE dbo.TokenNumberNumberCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SingleValue2 DECIMAL (36, 18) NULL, + LowValue2 DECIMAL (36, 18) NULL, + HighValue2 DECIMAL (36, 18) NULL, + SingleValue3 DECIMAL (36, 18) NULL, + LowValue3 DECIMAL (36, 18) NULL, + HighValue3 DECIMAL (36, 18) NULL, + HasRange BIT NOT NULL); + +CREATE TYPE dbo.TokenQuantityCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SystemId2 INT NULL, + QuantityCodeId2 INT NULL, + SingleValue2 DECIMAL (36, 18) NULL, + LowValue2 DECIMAL (36, 18) NULL, + HighValue2 DECIMAL (36, 18) NULL); + +CREATE TYPE dbo.TokenSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + Code VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.TokenStringCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + Text2 NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow2 NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL); + +CREATE TYPE dbo.TokenTextList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (400) COLLATE Latin1_General_CI_AI NOT NULL); + +CREATE TYPE dbo.TokenTokenCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.BulkResourceWriteClaimTableType_1 AS TABLE ( + Offset INT NOT NULL, + ClaimTypeId TINYINT NOT NULL, + ClaimValue NVARCHAR (128) NOT NULL); + +CREATE TYPE dbo.BulkCompartmentAssignmentTableType_1 AS TABLE ( + Offset INT NOT NULL, + CompartmentTypeId TINYINT NOT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL); + +CREATE TYPE dbo.BulkReferenceSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId SMALLINT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion INT NULL); + +CREATE TYPE dbo.BulkTokenSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + Code VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL); + +CREATE TYPE dbo.BulkTokenSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + Code VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.BulkTokenTextTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (400) COLLATE Latin1_General_CI_AI NOT NULL); + +CREATE TYPE dbo.BulkStringSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL); + +CREATE TYPE dbo.BulkStringSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL, + IsMin BIT NOT NULL, + IsMax BIT NOT NULL); + +CREATE TYPE dbo.BulkUriSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Uri VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL); + +CREATE TYPE dbo.BulkNumberSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SingleValue DECIMAL (18, 6) NULL, + LowValue DECIMAL (18, 6) NULL, + HighValue DECIMAL (18, 6) NULL); + +CREATE TYPE dbo.BulkQuantitySearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + QuantityCodeId INT NULL, + SingleValue DECIMAL (18, 6) NULL, + LowValue DECIMAL (18, 6) NULL, + HighValue DECIMAL (18, 6) NULL); + +CREATE TYPE dbo.BulkDateTimeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + StartDateTime DATETIMEOFFSET (7) NOT NULL, + EndDateTime DATETIMEOFFSET (7) NOT NULL, + IsLongerThanADay BIT NOT NULL); + +CREATE TYPE dbo.BulkDateTimeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + StartDateTime DATETIMEOFFSET (7) NOT NULL, + EndDateTime DATETIMEOFFSET (7) NOT NULL, + IsLongerThanADay BIT NOT NULL, + IsMin BIT NOT NULL, + IsMax BIT NOT NULL); + +CREATE TYPE dbo.BulkReferenceTokenCompositeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId1 SMALLINT NULL, + ReferenceResourceId1 VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion1 INT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL); + +CREATE TYPE dbo.BulkReferenceTokenCompositeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId1 SMALLINT NULL, + ReferenceResourceId1 VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion1 INT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.BulkTokenTokenCompositeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL); + +CREATE TYPE dbo.BulkTokenTokenCompositeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.BulkTokenDateTimeCompositeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + StartDateTime2 DATETIMEOFFSET (7) NOT NULL, + EndDateTime2 DATETIMEOFFSET (7) NOT NULL, + IsLongerThanADay2 BIT NOT NULL); + +CREATE TYPE dbo.BulkTokenDateTimeCompositeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + StartDateTime2 DATETIMEOFFSET (7) NOT NULL, + EndDateTime2 DATETIMEOFFSET (7) NOT NULL, + IsLongerThanADay2 BIT NOT NULL); + +CREATE TYPE dbo.BulkTokenQuantityCompositeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + SystemId2 INT NULL, + QuantityCodeId2 INT NULL, + SingleValue2 DECIMAL (18, 6) NULL, + LowValue2 DECIMAL (18, 6) NULL, + HighValue2 DECIMAL (18, 6) NULL); + +CREATE TYPE dbo.BulkTokenQuantityCompositeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SystemId2 INT NULL, + QuantityCodeId2 INT NULL, + SingleValue2 DECIMAL (18, 6) NULL, + LowValue2 DECIMAL (18, 6) NULL, + HighValue2 DECIMAL (18, 6) NULL); + +CREATE TYPE dbo.BulkTokenStringCompositeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + Text2 NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow2 NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL); + +CREATE TYPE dbo.BulkTokenStringCompositeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + Text2 NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow2 NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL); + +CREATE TYPE dbo.BulkTokenNumberNumberCompositeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + SingleValue2 DECIMAL (18, 6) NULL, + LowValue2 DECIMAL (18, 6) NULL, + HighValue2 DECIMAL (18, 6) NULL, + SingleValue3 DECIMAL (18, 6) NULL, + LowValue3 DECIMAL (18, 6) NULL, + HighValue3 DECIMAL (18, 6) NULL, + HasRange BIT NOT NULL); + +CREATE TYPE dbo.BulkTokenNumberNumberCompositeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SingleValue2 DECIMAL (18, 6) NULL, + LowValue2 DECIMAL (18, 6) NULL, + HighValue2 DECIMAL (18, 6) NULL, + SingleValue3 DECIMAL (18, 6) NULL, + LowValue3 DECIMAL (18, 6) NULL, + HighValue3 DECIMAL (18, 6) NULL, + HasRange BIT NOT NULL); + +CREATE TYPE dbo.SearchParamTableType_1 AS TABLE ( + Uri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + Status VARCHAR (10) NOT NULL, + IsPartiallySupported BIT NOT NULL); + +CREATE TYPE dbo.SearchParamTableType_2 AS TABLE ( + Uri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + Status VARCHAR (20) NOT NULL, + IsPartiallySupported BIT NOT NULL); + +CREATE TYPE dbo.BulkReindexResourceTableType_1 AS TABLE ( + Offset INT NOT NULL, + ResourceTypeId SMALLINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ETag INT NULL, + SearchParamHash VARCHAR (64) NOT NULL); + +CREATE TYPE dbo.BulkImportResourceType_1 AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Version INT NOT NULL, + IsHistory BIT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + IsDeleted BIT NOT NULL, + RequestMethod VARCHAR (10) NULL, + RawResource VARBINARY (MAX) NOT NULL, + IsRawResourceMetaSet BIT DEFAULT 0 NOT NULL, + SearchParamHash VARCHAR (64) NULL); + +CREATE TYPE dbo.UriSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Uri VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL PRIMARY KEY (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri)); + +CREATE TABLE dbo.ClaimType ( + ClaimTypeId TINYINT IDENTITY (1, 1) NOT NULL, + Name VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + CONSTRAINT UQ_ClaimType_ClaimTypeId UNIQUE (ClaimTypeId), + CONSTRAINT PKC_ClaimType PRIMARY KEY CLUSTERED (Name) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE dbo.CompartmentAssignment ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + CompartmentTypeId TINYINT NOT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + IsHistory BIT NOT NULL, + CONSTRAINT PKC_CompartmentAssignment PRIMARY KEY CLUSTERED (ResourceTypeId, ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId) WITH (DATA_COMPRESSION = PAGE) ON PartitionScheme_ResourceTypeId (ResourceTypeId) +); + + +GO +ALTER TABLE dbo.CompartmentAssignment + ADD CONSTRAINT DF_CompartmentAssignment_IsHistory DEFAULT 0 FOR IsHistory; + + +GO +ALTER TABLE dbo.CompartmentAssignment SET (LOCK_ESCALATION = AUTO); + + +GO +CREATE NONCLUSTERED INDEX IX_CompartmentAssignment_CompartmentTypeId_ReferenceResourceId + ON dbo.CompartmentAssignment(ResourceTypeId, CompartmentTypeId, ReferenceResourceId, ResourceSurrogateId) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.CompartmentType ( + CompartmentTypeId TINYINT IDENTITY (1, 1) NOT NULL, + Name VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + CONSTRAINT UQ_CompartmentType_CompartmentTypeId UNIQUE (CompartmentTypeId), + CONSTRAINT PKC_CompartmentType PRIMARY KEY CLUSTERED (Name) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE dbo.DateTimeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + StartDateTime DATETIME2 (7) NOT NULL, + EndDateTime DATETIME2 (7) NOT NULL, + IsLongerThanADay BIT NOT NULL, + IsHistory BIT NOT NULL, + IsMin BIT CONSTRAINT date_IsMin_Constraint DEFAULT 0 NOT NULL, + IsMax BIT CONSTRAINT date_IsMax_Constraint DEFAULT 0 NOT NULL +); + +ALTER TABLE dbo.DateTimeSearchParam + ADD CONSTRAINT DF_DateTimeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.DateTimeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_DateTimeSearchParam + ON dbo.DateTimeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_DateTimeSearchParam_SearchParamId_StartDateTime_EndDateTime + ON dbo.DateTimeSearchParam(ResourceTypeId, SearchParamId, StartDateTime, EndDateTime, ResourceSurrogateId) + INCLUDE(IsLongerThanADay, IsMin, IsMax) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_DateTimeSearchParam_SearchParamId_EndDateTime_StartDateTime + ON dbo.DateTimeSearchParam(ResourceTypeId, SearchParamId, EndDateTime, StartDateTime, ResourceSurrogateId) + INCLUDE(IsLongerThanADay, IsMin, IsMax) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_DateTimeSearchParam_SearchParamId_StartDateTime_EndDateTime_Long + ON dbo.DateTimeSearchParam(ResourceTypeId, SearchParamId, StartDateTime, EndDateTime, ResourceSurrogateId) + INCLUDE(IsMin, IsMax) WHERE IsHistory = 0 + AND IsLongerThanADay = 1 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_DateTimeSearchParam_SearchParamId_EndDateTime_StartDateTime_Long + ON dbo.DateTimeSearchParam(ResourceTypeId, SearchParamId, EndDateTime, StartDateTime, ResourceSurrogateId) + INCLUDE(IsMin, IsMax) WHERE IsHistory = 0 + AND IsLongerThanADay = 1 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +IF NOT EXISTS (SELECT 1 + FROM sys.tables + WHERE name = 'EventAgentCheckpoint') + BEGIN + CREATE TABLE dbo.EventAgentCheckpoint ( + CheckpointId VARCHAR (64) NOT NULL, + LastProcessedDateTime DATETIMEOFFSET (7), + LastProcessedIdentifier VARCHAR (64) , + UpdatedOn DATETIME2 (7) DEFAULT sysutcdatetime() NOT NULL, + CONSTRAINT PK_EventAgentCheckpoint PRIMARY KEY CLUSTERED (CheckpointId) + ) ON [PRIMARY]; + END + +CREATE PARTITION FUNCTION EventLogPartitionFunction(TINYINT) + AS RANGE RIGHT + FOR VALUES (0, 1, 2, 3, 4, 5, 6, 7); + + +GO +CREATE PARTITION SCHEME EventLogPartitionScheme + AS PARTITION EventLogPartitionFunction + ALL TO ([PRIMARY]); + + +GO +CREATE TABLE dbo.EventLog ( + PartitionId AS isnull(CONVERT (TINYINT, EventId % 8), 0) PERSISTED, + EventId BIGINT IDENTITY (1, 1) NOT NULL, + EventDate DATETIME NOT NULL, + Process VARCHAR (100) NOT NULL, + Status VARCHAR (10) NOT NULL, + Mode VARCHAR (200) NULL, + Action VARCHAR (20) NULL, + Target VARCHAR (100) NULL, + Rows BIGINT NULL, + Milliseconds INT NULL, + EventText NVARCHAR (3500) NULL, + SPID SMALLINT NOT NULL, + HostName VARCHAR (64) NOT NULL CONSTRAINT PKC_EventLog_EventDate_EventId_PartitionId PRIMARY KEY CLUSTERED (EventDate, EventId, PartitionId) ON EventLogPartitionScheme (PartitionId) +); + +CREATE TABLE dbo.ExportJob ( + Id VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Hash VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Status VARCHAR (10) NOT NULL, + HeartbeatDateTime DATETIME2 (7) NULL, + RawJobRecord VARCHAR (MAX) NOT NULL, + JobVersion ROWVERSION NOT NULL, + CONSTRAINT PKC_ExportJob PRIMARY KEY CLUSTERED (Id) +); + +CREATE UNIQUE NONCLUSTERED INDEX IX_ExportJob_Hash_Status_HeartbeatDateTime + ON dbo.ExportJob(Hash, Status, HeartbeatDateTime); + +CREATE TABLE dbo.IndexProperties ( + TableName VARCHAR (100) NOT NULL, + IndexName VARCHAR (200) NOT NULL, + PropertyName VARCHAR (100) NOT NULL, + PropertyValue VARCHAR (100) NOT NULL, + CreateDate DATETIME CONSTRAINT DF_IndexProperties_CreateDate DEFAULT getUTCdate() NOT NULL CONSTRAINT PKC_IndexProperties_TableName_IndexName_PropertyName PRIMARY KEY CLUSTERED (TableName, IndexName, PropertyName) +); + +CREATE PARTITION FUNCTION TinyintPartitionFunction(TINYINT) + AS RANGE RIGHT + FOR VALUES (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255); + + +GO +CREATE PARTITION SCHEME TinyintPartitionScheme + AS PARTITION TinyintPartitionFunction + ALL TO ([PRIMARY]); + + +GO +CREATE TABLE dbo.JobQueue ( + QueueType TINYINT NOT NULL, + GroupId BIGINT NOT NULL, + JobId BIGINT NOT NULL, + PartitionId AS CONVERT (TINYINT, JobId % 16) PERSISTED, + Definition VARCHAR (MAX) NOT NULL, + DefinitionHash VARBINARY (20) NOT NULL, + Version BIGINT CONSTRAINT DF_JobQueue_Version DEFAULT datediff_big(millisecond, '0001-01-01', getUTCdate()) NOT NULL, + Status TINYINT CONSTRAINT DF_JobQueue_Status DEFAULT 0 NOT NULL, + Priority TINYINT CONSTRAINT DF_JobQueue_Priority DEFAULT 100 NOT NULL, + Data BIGINT NULL, + Result VARCHAR (MAX) NULL, + CreateDate DATETIME CONSTRAINT DF_JobQueue_CreateDate DEFAULT getUTCdate() NOT NULL, + StartDate DATETIME NULL, + EndDate DATETIME NULL, + HeartbeatDate DATETIME CONSTRAINT DF_JobQueue_HeartbeatDate DEFAULT getUTCdate() NOT NULL, + Worker VARCHAR (100) NULL, + Info VARCHAR (1000) NULL, + CancelRequested BIT CONSTRAINT DF_JobQueue_CancelRequested DEFAULT 0 NOT NULL CONSTRAINT PKC_JobQueue_QueueType_PartitionId_JobId PRIMARY KEY CLUSTERED (QueueType, PartitionId, JobId) ON TinyintPartitionScheme (QueueType), + CONSTRAINT U_JobQueue_QueueType_JobId UNIQUE (QueueType, JobId) +); + + +GO +CREATE INDEX IX_QueueType_PartitionId_Status_Priority + ON dbo.JobQueue(PartitionId, Status, Priority) + ON TinyintPartitionScheme (QueueType); + + +GO +CREATE INDEX IX_QueueType_GroupId + ON dbo.JobQueue(QueueType, GroupId) + ON TinyintPartitionScheme (QueueType); + + +GO +CREATE INDEX IX_QueueType_DefinitionHash + ON dbo.JobQueue(QueueType, DefinitionHash) + ON TinyintPartitionScheme (QueueType); + +CREATE TABLE dbo.NumberSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SingleValue DECIMAL (36, 18) NULL, + LowValue DECIMAL (36, 18) NOT NULL, + HighValue DECIMAL (36, 18) NOT NULL, + IsHistory BIT NOT NULL +); + +ALTER TABLE dbo.NumberSearchParam + ADD CONSTRAINT DF_NumberSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.NumberSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_NumberSearchParam + ON dbo.NumberSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_NumberSearchParam_SearchParamId_SingleValue + ON dbo.NumberSearchParam(ResourceTypeId, SearchParamId, SingleValue, ResourceSurrogateId) WHERE IsHistory = 0 + AND SingleValue IS NOT NULL + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_NumberSearchParam_SearchParamId_LowValue_HighValue + ON dbo.NumberSearchParam(ResourceTypeId, SearchParamId, LowValue, HighValue, ResourceSurrogateId) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_NumberSearchParam_SearchParamId_HighValue_LowValue + ON dbo.NumberSearchParam(ResourceTypeId, SearchParamId, HighValue, LowValue, ResourceSurrogateId) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.Parameters ( + Id VARCHAR (100) NOT NULL, + Date DATETIME NULL, + Number FLOAT NULL, + Bigint BIGINT NULL, + Char VARCHAR (4000) NULL, + Binary VARBINARY (MAX) NULL, + UpdatedDate DATETIME NULL, + UpdatedBy NVARCHAR (255) NULL CONSTRAINT PKC_Parameters_Id PRIMARY KEY CLUSTERED (Id) WITH (IGNORE_DUP_KEY = ON) +); + + +GO +CREATE TABLE dbo.ParametersHistory ( + ChangeId INT IDENTITY (1, 1) NOT NULL, + Id VARCHAR (100) NOT NULL, + Date DATETIME NULL, + Number FLOAT NULL, + Bigint BIGINT NULL, + Char VARCHAR (4000) NULL, + Binary VARBINARY (MAX) NULL, + UpdatedDate DATETIME NULL, + UpdatedBy NVARCHAR (255) NULL +); + +CREATE TABLE dbo.QuantityCode ( + QuantityCodeId INT IDENTITY (1, 1) NOT NULL, + Value NVARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CONSTRAINT UQ_QuantityCode_QuantityCodeId UNIQUE (QuantityCodeId), + CONSTRAINT PKC_QuantityCode PRIMARY KEY CLUSTERED (Value) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE dbo.QuantitySearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + QuantityCodeId INT NULL, + SingleValue DECIMAL (36, 18) NULL, + LowValue DECIMAL (36, 18) NOT NULL, + HighValue DECIMAL (36, 18) NOT NULL, + IsHistory BIT NOT NULL +); + +ALTER TABLE dbo.QuantitySearchParam + ADD CONSTRAINT DF_QuantitySearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.QuantitySearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_QuantitySearchParam + ON dbo.QuantitySearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_QuantitySearchParam_SearchParamId_QuantityCodeId_SingleValue + ON dbo.QuantitySearchParam(ResourceTypeId, SearchParamId, QuantityCodeId, SingleValue, ResourceSurrogateId) + INCLUDE(SystemId) WHERE IsHistory = 0 + AND SingleValue IS NOT NULL + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_QuantitySearchParam_SearchParamId_QuantityCodeId_LowValue_HighValue + ON dbo.QuantitySearchParam(ResourceTypeId, SearchParamId, QuantityCodeId, LowValue, HighValue, ResourceSurrogateId) + INCLUDE(SystemId) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_QuantitySearchParam_SearchParamId_QuantityCodeId_HighValue_LowValue + ON dbo.QuantitySearchParam(ResourceTypeId, SearchParamId, QuantityCodeId, HighValue, LowValue, ResourceSurrogateId) + INCLUDE(SystemId) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.ReferenceSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId SMALLINT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion INT NULL, + IsHistory BIT NOT NULL +); + +ALTER TABLE dbo.ReferenceSearchParam + ADD CONSTRAINT DF_ReferenceSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.ReferenceSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_ReferenceSearchParam + ON dbo.ReferenceSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_ReferenceSearchParam_SearchParamId_ReferenceResourceTypeId_ReferenceResourceId_BaseUri_ReferenceResourceVersion + ON dbo.ReferenceSearchParam(ResourceTypeId, SearchParamId, ReferenceResourceId, ReferenceResourceTypeId, BaseUri, ResourceSurrogateId) + INCLUDE(ReferenceResourceVersion) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.ReferenceTokenCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId1 SMALLINT NULL, + ReferenceResourceId1 VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion1 INT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + IsHistory BIT NOT NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.ReferenceTokenCompositeSearchParam + ADD CONSTRAINT DF_ReferenceTokenCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.ReferenceTokenCompositeSearchParam + ADD CONSTRAINT CHK_ReferenceTokenCompositeSearchParam_CodeOverflow2 CHECK (LEN(Code2) = 256 + OR CodeOverflow2 IS NULL); + +ALTER TABLE dbo.ReferenceTokenCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_ReferenceTokenCompositeSearchParam + ON dbo.ReferenceTokenCompositeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_ReferenceTokenCompositeSearchParam_ReferenceResourceId1_Code2 + ON dbo.ReferenceTokenCompositeSearchParam(ResourceTypeId, SearchParamId, ReferenceResourceId1, Code2, ResourceSurrogateId) + INCLUDE(ReferenceResourceTypeId1, BaseUri1, SystemId2) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.ReindexJob ( + Id VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Status VARCHAR (10) NOT NULL, + HeartbeatDateTime DATETIME2 (7) NULL, + RawJobRecord VARCHAR (MAX) NOT NULL, + JobVersion ROWVERSION NOT NULL, + CONSTRAINT PKC_ReindexJob PRIMARY KEY CLUSTERED (Id) +); + +CREATE TABLE dbo.Resource ( + ResourceTypeId SMALLINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Version INT NOT NULL, + IsHistory BIT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + IsDeleted BIT NOT NULL, + RequestMethod VARCHAR (10) NULL, + RawResource VARBINARY (MAX) NOT NULL, + IsRawResourceMetaSet BIT DEFAULT 0 NOT NULL, + SearchParamHash VARCHAR (64) NULL, + TransactionId BIGINT NULL, + HistoryTransactionId BIGINT NULL CONSTRAINT PKC_Resource PRIMARY KEY CLUSTERED (ResourceTypeId, ResourceSurrogateId) WITH (DATA_COMPRESSION = PAGE) ON PartitionScheme_ResourceTypeId (ResourceTypeId), + CONSTRAINT CH_Resource_RawResource_Length CHECK (RawResource > 0x0) +); + +ALTER TABLE dbo.Resource SET (LOCK_ESCALATION = AUTO); + +CREATE INDEX IX_ResourceTypeId_TransactionId + ON dbo.Resource(ResourceTypeId, TransactionId) WHERE TransactionId IS NOT NULL + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_ResourceTypeId_HistoryTransactionId + ON dbo.Resource(ResourceTypeId, HistoryTransactionId) WHERE HistoryTransactionId IS NOT NULL + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE UNIQUE NONCLUSTERED INDEX IX_Resource_ResourceTypeId_ResourceId_Version + ON dbo.Resource(ResourceTypeId, ResourceId, Version) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE UNIQUE NONCLUSTERED INDEX IX_Resource_ResourceTypeId_ResourceId + ON dbo.Resource(ResourceTypeId, ResourceId) + INCLUDE(Version, IsDeleted) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE UNIQUE NONCLUSTERED INDEX IX_Resource_ResourceTypeId_ResourceSurrgateId + ON dbo.Resource(ResourceTypeId, ResourceSurrogateId) WHERE IsHistory = 0 + AND IsDeleted = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.ResourceChangeData ( + Id BIGINT IDENTITY (1, 1) NOT NULL, + Timestamp DATETIME2 (7) CONSTRAINT DF_ResourceChangeData_Timestamp DEFAULT sysutcdatetime() NOT NULL, + ResourceId VARCHAR (64) NOT NULL, + ResourceTypeId SMALLINT NOT NULL, + ResourceVersion INT NOT NULL, + ResourceChangeTypeId TINYINT NOT NULL +) ON PartitionScheme_ResourceChangeData_Timestamp (Timestamp); + +CREATE CLUSTERED INDEX IXC_ResourceChangeData + ON dbo.ResourceChangeData(Id ASC) WITH (ONLINE = ON) + ON PartitionScheme_ResourceChangeData_Timestamp (Timestamp); + +CREATE TABLE dbo.ResourceChangeDataStaging ( + Id BIGINT IDENTITY (1, 1) NOT NULL, + Timestamp DATETIME2 (7) CONSTRAINT DF_ResourceChangeDataStaging_Timestamp DEFAULT sysutcdatetime() NOT NULL, + ResourceId VARCHAR (64) NOT NULL, + ResourceTypeId SMALLINT NOT NULL, + ResourceVersion INT NOT NULL, + ResourceChangeTypeId TINYINT NOT NULL +) ON [PRIMARY]; + +CREATE CLUSTERED INDEX IXC_ResourceChangeDataStaging + ON dbo.ResourceChangeDataStaging(Id ASC, Timestamp ASC) WITH (ONLINE = ON) + ON [PRIMARY]; + +ALTER TABLE dbo.ResourceChangeDataStaging WITH CHECK + ADD CONSTRAINT CHK_ResourceChangeDataStaging_partition CHECK (Timestamp < CONVERT (DATETIME2 (7), N'9999-12-31 23:59:59.9999999')); + +ALTER TABLE dbo.ResourceChangeDataStaging CHECK CONSTRAINT CHK_ResourceChangeDataStaging_partition; + +CREATE TABLE dbo.ResourceChangeType ( + ResourceChangeTypeId TINYINT NOT NULL, + Name NVARCHAR (50) NOT NULL, + CONSTRAINT PK_ResourceChangeType PRIMARY KEY CLUSTERED (ResourceChangeTypeId), + CONSTRAINT UQ_ResourceChangeType_Name UNIQUE NONCLUSTERED (Name) +) ON [PRIMARY]; + + +GO +INSERT dbo.ResourceChangeType (ResourceChangeTypeId, Name) +VALUES (0, N'Creation'); + +INSERT dbo.ResourceChangeType (ResourceChangeTypeId, Name) +VALUES (1, N'Update'); + +INSERT dbo.ResourceChangeType (ResourceChangeTypeId, Name) +VALUES (2, N'Deletion'); + +CREATE TABLE dbo.ResourceType ( + ResourceTypeId SMALLINT IDENTITY (1, 1) NOT NULL, + Name NVARCHAR (50) COLLATE Latin1_General_100_CS_AS NOT NULL, + CONSTRAINT UQ_ResourceType_ResourceTypeId UNIQUE (ResourceTypeId), + CONSTRAINT PKC_ResourceType PRIMARY KEY CLUSTERED (Name) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE dbo.ResourceWriteClaim ( + ResourceSurrogateId BIGINT NOT NULL, + ClaimTypeId TINYINT NOT NULL, + ClaimValue NVARCHAR (128) NOT NULL +) +WITH (DATA_COMPRESSION = PAGE); + +CREATE CLUSTERED INDEX IXC_ResourceWriteClaim + ON dbo.ResourceWriteClaim(ResourceSurrogateId, ClaimTypeId); + +CREATE TABLE dbo.SchemaMigrationProgress ( + Timestamp DATETIME2 (3) DEFAULT CURRENT_TIMESTAMP, + Message NVARCHAR (MAX) +); + +CREATE TABLE dbo.SearchParam ( + SearchParamId SMALLINT IDENTITY (1, 1) NOT NULL, + Uri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + Status VARCHAR (20) NULL, + LastUpdated DATETIMEOFFSET (7) NULL, + IsPartiallySupported BIT NULL, + CONSTRAINT UQ_SearchParam_SearchParamId UNIQUE (SearchParamId), + CONSTRAINT PKC_SearchParam PRIMARY KEY CLUSTERED (Uri) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE dbo.StringSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL, + IsHistory BIT NOT NULL, + IsMin BIT CONSTRAINT string_IsMin_Constraint DEFAULT 0 NOT NULL, + IsMax BIT CONSTRAINT string_IsMax_Constraint DEFAULT 0 NOT NULL +); + +ALTER TABLE dbo.StringSearchParam + ADD CONSTRAINT DF_StringSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.StringSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_StringSearchParam + ON dbo.StringSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_StringSearchParam_SearchParamId_Text + ON dbo.StringSearchParam(ResourceTypeId, SearchParamId, Text, ResourceSurrogateId) + INCLUDE(TextOverflow, IsMin, IsMax) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_StringSearchParam_SearchParamId_TextWithOverflow + ON dbo.StringSearchParam(ResourceTypeId, SearchParamId, Text, ResourceSurrogateId) + INCLUDE(IsMin, IsMax) WHERE IsHistory = 0 + AND TextOverflow IS NOT NULL WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.System ( + SystemId INT IDENTITY (1, 1) NOT NULL, + Value NVARCHAR (256) NOT NULL, + CONSTRAINT UQ_System_SystemId UNIQUE (SystemId), + CONSTRAINT PKC_System PRIMARY KEY CLUSTERED (Value) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE [dbo].[TaskInfo] ( + [TaskId] VARCHAR (64) NOT NULL, + [QueueId] VARCHAR (64) NOT NULL, + [Status] SMALLINT NOT NULL, + [TaskTypeId] SMALLINT NOT NULL, + [RunId] VARCHAR (50) NULL, + [IsCanceled] BIT NOT NULL, + [RetryCount] SMALLINT NOT NULL, + [MaxRetryCount] SMALLINT NOT NULL, + [HeartbeatDateTime] DATETIME2 (7) NULL, + [InputData] VARCHAR (MAX) NOT NULL, + [TaskContext] VARCHAR (MAX) NULL, + [Result] VARCHAR (MAX) NULL, + [CreateDateTime] DATETIME2 (7) CONSTRAINT DF_TaskInfo_CreateDate DEFAULT SYSUTCDATETIME() NOT NULL, + [StartDateTime] DATETIME2 (7) NULL, + [EndDateTime] DATETIME2 (7) NULL, + [Worker] VARCHAR (100) NULL, + [RestartInfo] VARCHAR (MAX) NULL, + [ParentTaskId] VARCHAR (64) NULL, + CONSTRAINT PKC_TaskInfo PRIMARY KEY CLUSTERED (TaskId) WITH (DATA_COMPRESSION = PAGE) +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]; + + +GO +CREATE NONCLUSTERED INDEX IX_QueueId_Status + ON dbo.TaskInfo(QueueId, Status); + + +GO +CREATE NONCLUSTERED INDEX IX_QueueId_ParentTaskId + ON dbo.TaskInfo(QueueId, ParentTaskId); + +CREATE TABLE dbo.TokenDateTimeCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + StartDateTime2 DATETIME2 (7) NOT NULL, + EndDateTime2 DATETIME2 (7) NOT NULL, + IsLongerThanADay2 BIT NOT NULL, + IsHistory BIT NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenDateTimeCompositeSearchParam + ADD CONSTRAINT DF_TokenDateTimeCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenDateTimeCompositeSearchParam + ADD CONSTRAINT CHK_TokenDateTimeCompositeSearchParam_CodeOverflow1 CHECK (LEN(Code1) = 256 + OR CodeOverflow1 IS NULL); + +ALTER TABLE dbo.TokenDateTimeCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenDateTimeCompositeSearchParam + ON dbo.TokenDateTimeCompositeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenDateTimeCompositeSearchParam_Code1_StartDateTime2_EndDateTime2 + ON dbo.TokenDateTimeCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, StartDateTime2, EndDateTime2, ResourceSurrogateId) + INCLUDE(SystemId1, IsLongerThanADay2) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenDateTimeCompositeSearchParam_Code1_EndDateTime2_StartDateTime2 + ON dbo.TokenDateTimeCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, EndDateTime2, StartDateTime2, ResourceSurrogateId) + INCLUDE(SystemId1, IsLongerThanADay2) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenDateTimeCompositeSearchParam_Code1_StartDateTime2_EndDateTime2_Long + ON dbo.TokenDateTimeCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, StartDateTime2, EndDateTime2, ResourceSurrogateId) + INCLUDE(SystemId1) WHERE IsHistory = 0 + AND IsLongerThanADay2 = 1 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenDateTimeCompositeSearchParam_Code1_EndDateTime2_StartDateTime2_Long + ON dbo.TokenDateTimeCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, EndDateTime2, StartDateTime2, ResourceSurrogateId) + INCLUDE(SystemId1) WHERE IsHistory = 0 + AND IsLongerThanADay2 = 1 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenNumberNumberCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + SingleValue2 DECIMAL (36, 18) NULL, + LowValue2 DECIMAL (36, 18) NULL, + HighValue2 DECIMAL (36, 18) NULL, + SingleValue3 DECIMAL (36, 18) NULL, + LowValue3 DECIMAL (36, 18) NULL, + HighValue3 DECIMAL (36, 18) NULL, + HasRange BIT NOT NULL, + IsHistory BIT NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenNumberNumberCompositeSearchParam + ADD CONSTRAINT DF_TokenNumberNumberCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenNumberNumberCompositeSearchParam + ADD CONSTRAINT CHK_TokenNumberNumberCompositeSearchParam_CodeOverflow1 CHECK (LEN(Code1) = 256 + OR CodeOverflow1 IS NULL); + +ALTER TABLE dbo.TokenNumberNumberCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenNumberNumberCompositeSearchParam + ON dbo.TokenNumberNumberCompositeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenNumberNumberCompositeSearchParam_SearchParamId_Code1_Text2 + ON dbo.TokenNumberNumberCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, SingleValue2, SingleValue3, ResourceSurrogateId) + INCLUDE(SystemId1) WHERE IsHistory = 0 + AND HasRange = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenNumberNumberCompositeSearchParam_SearchParamId_Code1_LowValue2_HighValue2_LowValue3_HighValue3 + ON dbo.TokenNumberNumberCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, LowValue2, HighValue2, LowValue3, HighValue3, ResourceSurrogateId) + INCLUDE(SystemId1) WHERE IsHistory = 0 + AND HasRange = 1 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenQuantityCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + SystemId2 INT NULL, + QuantityCodeId2 INT NULL, + SingleValue2 DECIMAL (36, 18) NULL, + LowValue2 DECIMAL (36, 18) NULL, + HighValue2 DECIMAL (36, 18) NULL, + IsHistory BIT NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenQuantityCompositeSearchParam + ADD CONSTRAINT DF_TokenQuantityCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenQuantityCompositeSearchParam + ADD CONSTRAINT CHK_TokenQuantityCompositeSearchParam_CodeOverflow1 CHECK (LEN(Code1) = 256 + OR CodeOverflow1 IS NULL); + +ALTER TABLE dbo.TokenQuantityCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenQuantityCompositeSearchParam + ON dbo.TokenQuantityCompositeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenQuantityCompositeSearchParam_SearchParamId_Code1_QuantityCodeId2_SingleValue2 + ON dbo.TokenQuantityCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, SingleValue2, ResourceSurrogateId) + INCLUDE(QuantityCodeId2, SystemId1, SystemId2) WHERE IsHistory = 0 + AND SingleValue2 IS NOT NULL WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenQuantityCompositeSearchParam_SearchParamId_Code1_QuantityCodeId2_LowValue2_HighValue2 + ON dbo.TokenQuantityCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, LowValue2, HighValue2, ResourceSurrogateId) + INCLUDE(QuantityCodeId2, SystemId1, SystemId2) WHERE IsHistory = 0 + AND LowValue2 IS NOT NULL WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenQuantityCompositeSearchParam_SearchParamId_Code1_QuantityCodeId2_HighValue2_LowValue2 + ON dbo.TokenQuantityCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, HighValue2, LowValue2, ResourceSurrogateId) + INCLUDE(QuantityCodeId2, SystemId1, SystemId2) WHERE IsHistory = 0 + AND LowValue2 IS NOT NULL WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + Code VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + IsHistory BIT NOT NULL, + CodeOverflow VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenSearchParam + ADD CONSTRAINT DF_TokenSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenSearchParam + ADD CONSTRAINT CHK_TokenSearchParam_CodeOverflow CHECK (LEN(Code) = 256 + OR CodeOverflow IS NULL); + +ALTER TABLE dbo.TokenSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenSearchParam + ON dbo.TokenSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenSeachParam_SearchParamId_Code_SystemId + ON dbo.TokenSearchParam(ResourceTypeId, SearchParamId, Code, ResourceSurrogateId) + INCLUDE(SystemId) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenStringCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + Text2 NVARCHAR (256) COLLATE Latin1_General_CI_AI NOT NULL, + TextOverflow2 NVARCHAR (MAX) COLLATE Latin1_General_CI_AI NULL, + IsHistory BIT NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenStringCompositeSearchParam + ADD CONSTRAINT DF_TokenStringCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenStringCompositeSearchParam + ADD CONSTRAINT CHK_TokenStringCompositeSearchParam_CodeOverflow1 CHECK (LEN(Code1) = 256 + OR CodeOverflow1 IS NULL); + +ALTER TABLE dbo.TokenStringCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenStringCompositeSearchParam + ON dbo.TokenStringCompositeSearchParam(ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenStringCompositeSearchParam_SearchParamId_Code1_Text2 + ON dbo.TokenStringCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, Text2, ResourceSurrogateId) + INCLUDE(SystemId1, TextOverflow2) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenStringCompositeSearchParam_SearchParamId_Code1_Text2WithOverflow + ON dbo.TokenStringCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, Text2, ResourceSurrogateId) + INCLUDE(SystemId1) WHERE IsHistory = 0 + AND TextOverflow2 IS NOT NULL WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenText ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (400) COLLATE Latin1_General_CI_AI NOT NULL, + IsHistory BIT NOT NULL +); + +ALTER TABLE dbo.TokenText + ADD CONSTRAINT DF_TokenText_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenText SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenText + ON dbo.TokenText(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenText_SearchParamId_Text + ON dbo.TokenText(ResourceTypeId, SearchParamId, Text, ResourceSurrogateId) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenTokenCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + IsHistory BIT NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenTokenCompositeSearchParam + ADD CONSTRAINT DF_TokenTokenCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenTokenCompositeSearchParam + ADD CONSTRAINT CHK_TokenTokenCompositeSearchParam_CodeOverflow1 CHECK (LEN(Code1) = 256 + OR CodeOverflow1 IS NULL); + +ALTER TABLE dbo.TokenTokenCompositeSearchParam + ADD CONSTRAINT CHK_TokenTokenCompositeSearchParam_CodeOverflow2 CHECK (LEN(Code2) = 256 + OR CodeOverflow2 IS NULL); + +ALTER TABLE dbo.TokenTokenCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenTokenCompositeSearchParam + ON dbo.TokenTokenCompositeSearchParam(ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenTokenCompositeSearchParam_Code1_Code2 + ON dbo.TokenTokenCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, Code2, ResourceSurrogateId) + INCLUDE(SystemId1, SystemId2) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.Transactions ( + SurrogateIdRangeFirstValue BIGINT NOT NULL, + SurrogateIdRangeLastValue BIGINT NOT NULL, + Definition VARCHAR (2000) NULL, + IsCompleted BIT CONSTRAINT DF_Transactions_IsCompleted DEFAULT 0 NOT NULL, + IsSuccess BIT CONSTRAINT DF_Transactions_IsSuccess DEFAULT 0 NOT NULL, + IsVisible BIT CONSTRAINT DF_Transactions_IsVisible DEFAULT 0 NOT NULL, + IsHistoryMoved BIT CONSTRAINT DF_Transactions_IsHistoryMoved DEFAULT 0 NOT NULL, + CreateDate DATETIME CONSTRAINT DF_Transactions_CreateDate DEFAULT getUTCdate() NOT NULL, + EndDate DATETIME NULL, + VisibleDate DATETIME NULL, + HistoryMovedDate DATETIME NULL, + HeartbeatDate DATETIME CONSTRAINT DF_Transactions_HeartbeatDate DEFAULT getUTCdate() NOT NULL, + FailureReason VARCHAR (MAX) NULL, + IsControlledByClient BIT CONSTRAINT DF_Transactions_IsControlledByClient DEFAULT 1 NOT NULL, + InvisibleHistoryRemovedDate DATETIME NULL CONSTRAINT PKC_Transactions_SurrogateIdRangeFirstValue PRIMARY KEY CLUSTERED (SurrogateIdRangeFirstValue) +); + +CREATE INDEX IX_IsVisible + ON dbo.Transactions(IsVisible); + +CREATE TABLE dbo.UriSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Uri VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + IsHistory BIT NOT NULL +); + +ALTER TABLE dbo.UriSearchParam + ADD CONSTRAINT DF_UriSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.UriSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_UriSearchParam + ON dbo.UriSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_UriSearchParam_SearchParamId_Uri + ON dbo.UriSearchParam(ResourceTypeId, SearchParamId, Uri, ResourceSurrogateId) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.WatchdogLeases ( + Watchdog VARCHAR (100) NOT NULL, + LeaseHolder VARCHAR (100) CONSTRAINT DF_WatchdogLeases_LeaseHolder DEFAULT '' NOT NULL, + LeaseEndTime DATETIME CONSTRAINT DF_WatchdogLeases_LeaseEndTime DEFAULT 0 NOT NULL, + RemainingLeaseTimeSec AS datediff(second, getUTCdate(), LeaseEndTime), + LeaseRequestor VARCHAR (100) CONSTRAINT DF_WatchdogLeases_LeaseRequestor DEFAULT '' NOT NULL, + LeaseRequestTime DATETIME CONSTRAINT DF_WatchdogLeases_LeaseRequestTime DEFAULT 0 NOT NULL CONSTRAINT PKC_WatchdogLeases_Watchdog PRIMARY KEY CLUSTERED (Watchdog) +); + +COMMIT +GO +CREATE PROCEDURE dbo.AcquireExportJobs +@jobHeartbeatTimeoutThresholdInSeconds BIGINT, @maximumNumberOfConcurrentJobsAllowed INT +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DECLARE @expirationDateTime AS DATETIME2 (7); +SELECT @expirationDateTime = DATEADD(second, -@jobHeartbeatTimeoutThresholdInSeconds, SYSUTCDATETIME()); +DECLARE @numberOfRunningJobs AS INT; +SELECT @numberOfRunningJobs = COUNT(*) +FROM dbo.ExportJob WITH (TABLOCKX) +WHERE Status = 'Running' + AND HeartbeatDateTime > @expirationDateTime; +DECLARE @limit AS INT = @maximumNumberOfConcurrentJobsAllowed - @numberOfRunningJobs; +IF (@limit > 0) + BEGIN + DECLARE @availableJobs TABLE ( + Id VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + JobVersion BINARY (8) NOT NULL); + INSERT INTO @availableJobs + SELECT TOP (@limit) Id, + JobVersion + FROM dbo.ExportJob + WHERE (Status = 'Queued' + OR (Status = 'Running' + AND HeartbeatDateTime <= @expirationDateTime)) + ORDER BY HeartbeatDateTime; + DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); + UPDATE dbo.ExportJob + SET Status = 'Running', + HeartbeatDateTime = @heartbeatDateTime, + RawJobRecord = JSON_MODIFY(RawJobRecord, '$.status', 'Running') + OUTPUT inserted.RawJobRecord, inserted.JobVersion + FROM dbo.ExportJob AS job + INNER JOIN + @availableJobs AS availableJob + ON job.Id = availableJob.Id + AND job.JobVersion = availableJob.JobVersion; + END +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.AcquireReindexJobs +@jobHeartbeatTimeoutThresholdInSeconds BIGINT, @maximumNumberOfConcurrentJobsAllowed INT +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DECLARE @expirationDateTime AS DATETIME2 (7); +SELECT @expirationDateTime = DATEADD(second, -@jobHeartbeatTimeoutThresholdInSeconds, SYSUTCDATETIME()); +DECLARE @numberOfRunningJobs AS INT; +SELECT @numberOfRunningJobs = COUNT(*) +FROM dbo.ReindexJob WITH (TABLOCKX) +WHERE Status = 'Running' + AND HeartbeatDateTime > @expirationDateTime; +DECLARE @limit AS INT = @maximumNumberOfConcurrentJobsAllowed - @numberOfRunningJobs; +IF (@limit > 0) + BEGIN + DECLARE @availableJobs TABLE ( + Id VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + JobVersion BINARY (8) NOT NULL); + INSERT INTO @availableJobs + SELECT TOP (@limit) Id, + JobVersion + FROM dbo.ReindexJob + WHERE (Status = 'Queued' + OR (Status = 'Running' + AND HeartbeatDateTime <= @expirationDateTime)) + ORDER BY HeartbeatDateTime; + DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); + UPDATE dbo.ReindexJob + SET Status = 'Running', + HeartbeatDateTime = @heartbeatDateTime, + RawJobRecord = JSON_MODIFY(RawJobRecord, '$.status', 'Running') + OUTPUT inserted.RawJobRecord, inserted.JobVersion + FROM dbo.ReindexJob AS job + INNER JOIN + @availableJobs AS availableJob + ON job.Id = availableJob.Id + AND job.JobVersion = availableJob.JobVersion; + END +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.AcquireWatchdogLease +@Watchdog VARCHAR (100), @Worker VARCHAR (100), @AllowRebalance BIT=1, @ForceAcquire BIT=0, @LeasePeriodSec FLOAT, @WorkerIsRunning BIT=0, @LeaseEndTime DATETIME OUTPUT, @IsAcquired BIT OUTPUT, @CurrentLeaseHolder VARCHAR (100)=NULL OUTPUT +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +DECLARE @SP AS VARCHAR (100) = 'AcquireWatchdogLease', @Mode AS VARCHAR (100), @msg AS VARCHAR (1000), @MyLeasesNumber AS INT, @OtherValidRequestsOrLeasesNumber AS INT, @MyValidRequestsOrLeasesNumber AS INT, @DesiredLeasesNumber AS INT, @NotLeasedWatchdogNumber AS INT, @WatchdogNumber AS INT, @Now AS DATETIME, @MyLastChangeTime AS DATETIME, @PreviousLeaseHolder AS VARCHAR (100), @Rows AS INT = 0, @NumberOfWorkers AS INT, @st AS DATETIME = getUTCdate(), @RowsInt AS INT, @Pattern AS VARCHAR (100); +BEGIN TRY + SET @Mode = 'R=' + isnull(@Watchdog, 'NULL') + ' W=' + isnull(@Worker, 'NULL') + ' F=' + isnull(CONVERT (VARCHAR, @ForceAcquire), 'NULL') + ' LP=' + isnull(CONVERT (VARCHAR, @LeasePeriodSec), 'NULL'); + SET @CurrentLeaseHolder = ''; + SET @IsAcquired = 0; + SET @Now = getUTCdate(); + SET @LeaseEndTime = @Now; + SET @Pattern = NULLIF ((SELECT Char + FROM dbo.Parameters + WHERE Id = 'WatchdogLeaseHolderIncludePatternFor' + @Watchdog), ''); + IF @Pattern IS NULL + SET @Pattern = NULLIF ((SELECT Char + FROM dbo.Parameters + WHERE Id = 'WatchdogLeaseHolderIncludePattern'), ''); + IF @Pattern IS NOT NULL + AND @Worker NOT LIKE @Pattern + BEGIN + SET @msg = 'Worker does not match include pattern=' + @Pattern; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows, @Text = @msg; + SET @CurrentLeaseHolder = isnull((SELECT LeaseHolder + FROM dbo.WatchdogLeases + WHERE Watchdog = @Watchdog), ''); + RETURN; + END + SET @Pattern = NULLIF ((SELECT Char + FROM dbo.Parameters + WHERE Id = 'WatchdogLeaseHolderExcludePatternFor' + @Watchdog), ''); + IF @Pattern IS NULL + SET @Pattern = NULLIF ((SELECT Char + FROM dbo.Parameters + WHERE Id = 'WatchdogLeaseHolderExcludePattern'), ''); + IF @Pattern IS NOT NULL + AND @Worker LIKE @Pattern + BEGIN + SET @msg = 'Worker matches exclude pattern=' + @Pattern; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows, @Text = @msg; + SET @CurrentLeaseHolder = isnull((SELECT LeaseHolder + FROM dbo.WatchdogLeases + WHERE Watchdog = @Watchdog), ''); + RETURN; + END + DECLARE @Watchdogs TABLE ( + Watchdog VARCHAR (100) PRIMARY KEY); + INSERT INTO @Watchdogs + SELECT Watchdog + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE RemainingLeaseTimeSec * (-1) > 10 * @LeasePeriodSec + OR @ForceAcquire = 1 + AND Watchdog = @Watchdog + AND LeaseHolder <> @Worker; + IF @@rowcount > 0 + BEGIN + DELETE dbo.WatchdogLeases + WHERE Watchdog IN (SELECT Watchdog + FROM @Watchdogs); + SET @Rows += @@rowcount; + IF @Rows > 0 + BEGIN + SET @msg = ''; + SELECT @msg = CONVERT (VARCHAR (1000), @msg + CASE WHEN @msg = '' THEN '' ELSE ',' END + Watchdog) + FROM @Watchdogs; + SET @msg = CONVERT (VARCHAR (1000), 'Remove old/forced leases:' + @msg); + EXECUTE dbo.LogEvent @Process = 'AcquireWatchdogLease', @Status = 'Info', @Mode = @Mode, @Target = 'WatchdogLeases', @Action = 'Delete', @Rows = @Rows, @Text = @msg; + END + END + SET @NumberOfWorkers = 1 + (SELECT count(*) + FROM (SELECT LeaseHolder + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseHolder <> @Worker + UNION + SELECT LeaseRequestor + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseRequestor <> @Worker + AND LeaseRequestor <> '') AS A); + SET @Mode = CONVERT (VARCHAR (100), @Mode + ' N=' + CONVERT (VARCHAR (10), @NumberOfWorkers)); + IF NOT EXISTS (SELECT * + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE Watchdog = @Watchdog) + INSERT INTO dbo.WatchdogLeases (Watchdog, LeaseEndTime, LeaseRequestTime) + SELECT @Watchdog, + dateadd(day, -10, @Now), + dateadd(day, -10, @Now) + WHERE NOT EXISTS (SELECT * + FROM dbo.WatchdogLeases WITH (TABLOCKX) + WHERE Watchdog = @Watchdog); + SET @LeaseEndTime = dateadd(second, @LeasePeriodSec, @Now); + SET @WatchdogNumber = (SELECT count(*) + FROM dbo.WatchdogLeases WITH (NOLOCK)); + SET @NotLeasedWatchdogNumber = (SELECT count(*) + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseHolder = '' + OR LeaseEndTime < @Now); + SET @MyLeasesNumber = (SELECT count(*) + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseHolder = @Worker + AND LeaseEndTime > @Now); + SET @OtherValidRequestsOrLeasesNumber = (SELECT count(*) + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseHolder <> @Worker + AND LeaseEndTime > @Now + OR LeaseRequestor <> @Worker + AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec); + SET @MyValidRequestsOrLeasesNumber = (SELECT count(*) + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseHolder = @Worker + AND LeaseEndTime > @Now + OR LeaseRequestor = @Worker + AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec); + SET @DesiredLeasesNumber = ceiling(1.0 * @WatchdogNumber / @NumberOfWorkers); + IF @DesiredLeasesNumber = 0 + SET @DesiredLeasesNumber = 1; + IF @DesiredLeasesNumber = 1 + AND @OtherValidRequestsOrLeasesNumber = 1 + AND @WatchdogNumber = 1 + SET @DesiredLeasesNumber = 0; + IF @MyValidRequestsOrLeasesNumber = floor(1.0 * @WatchdogNumber / @NumberOfWorkers) + AND @OtherValidRequestsOrLeasesNumber + @MyValidRequestsOrLeasesNumber = @WatchdogNumber + SET @DesiredLeasesNumber = @DesiredLeasesNumber - 1; + UPDATE dbo.WatchdogLeases + SET LeaseHolder = @Worker, + LeaseEndTime = @LeaseEndTime, + LeaseRequestor = '', + @PreviousLeaseHolder = LeaseHolder + WHERE Watchdog = @Watchdog + AND NOT (LeaseRequestor <> @Worker + AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec) + AND (LeaseHolder = @Worker + AND (LeaseEndTime > @Now + OR @WorkerIsRunning = 1) + OR LeaseEndTime < @Now + AND (@DesiredLeasesNumber > @MyLeasesNumber + OR @OtherValidRequestsOrLeasesNumber < @WatchdogNumber)); + IF @@rowcount > 0 + BEGIN + SET @IsAcquired = 1; + SET @msg = 'Lease holder changed from [' + isnull(@PreviousLeaseHolder, '') + '] to [' + @Worker + ']'; + IF @PreviousLeaseHolder <> @Worker + EXECUTE dbo.LogEvent @Process = 'AcquireWatchdogLease', @Status = 'Info', @Mode = @Mode, @Text = @msg; + END + ELSE + IF @AllowRebalance = 1 + BEGIN + SET @CurrentLeaseHolder = (SELECT LeaseHolder + FROM dbo.WatchdogLeases + WHERE Watchdog = @Watchdog); + UPDATE dbo.WatchdogLeases + SET LeaseRequestTime = @Now + WHERE Watchdog = @Watchdog + AND LeaseRequestor = @Worker + AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec; + IF @DesiredLeasesNumber > @MyValidRequestsOrLeasesNumber + BEGIN + UPDATE A + SET LeaseRequestor = @Worker, + LeaseRequestTime = @Now + FROM dbo.WatchdogLeases AS A + WHERE Watchdog = @Watchdog + AND NOT (LeaseRequestor <> @Worker + AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec) + AND @NotLeasedWatchdogNumber = 0 + AND (SELECT count(*) + FROM dbo.WatchdogLeases AS B + WHERE B.LeaseHolder = A.LeaseHolder + AND datediff(second, B.LeaseEndTime, @Now) < @LeasePeriodSec) > @DesiredLeasesNumber; + SET @RowsInt = @@rowcount; + SET @msg = '@DesiredLeasesNumber=[' + CONVERT (VARCHAR (10), @DesiredLeasesNumber) + '] > @MyValidRequestsOrLeasesNumber=[' + CONVERT (VARCHAR (10), @MyValidRequestsOrLeasesNumber) + ']'; + EXECUTE dbo.LogEvent @Process = 'AcquireWatchdogLease', @Status = 'Info', @Mode = @Mode, @Rows = @RowsInt, @Text = @msg; + END + END + SET @Mode = CONVERT (VARCHAR (100), @Mode + ' A=' + CONVERT (VARCHAR (1), @IsAcquired)); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = 'AcquireWatchdogLease', @Status = 'Error', @Mode = @Mode; + THROW; +END CATCH + +GO +CREATE OR ALTER PROCEDURE dbo.AddPartitionOnResourceChanges +@partitionBoundary DATETIME2 (7) OUTPUT +AS +BEGIN + SET XACT_ABORT ON; + BEGIN TRANSACTION; + DECLARE @rightPartitionBoundary AS DATETIME2 (7) = CAST ((SELECT TOP (1) value + FROM sys.partition_range_values AS prv + INNER JOIN + sys.partition_functions AS pf + ON pf.function_id = prv.function_id + WHERE pf.name = N'PartitionFunction_ResourceChangeData_Timestamp' + ORDER BY prv.boundary_id DESC) AS DATETIME2 (7)); + DECLARE @timestamp AS DATETIME2 (7) = DATEADD(hour, DATEDIFF(hour, 0, sysutcdatetime()), 0); + IF (@rightPartitionBoundary < @timestamp) + BEGIN + SET @rightPartitionBoundary = @timestamp; + END + SET @rightPartitionBoundary = DATEADD(hour, 1, @rightPartitionBoundary); + ALTER PARTITION SCHEME PartitionScheme_ResourceChangeData_Timestamp NEXT USED [Primary]; + ALTER PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp( ) + SPLIT RANGE (@rightPartitionBoundary); + SET @partitionBoundary = @rightPartitionBoundary; + COMMIT TRANSACTION; +END + +GO +CREATE PROCEDURE dbo.ArchiveJobs +@QueueType TINYINT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'ArchiveJobs', @Mode AS VARCHAR (100) = '', @st AS DATETIME = getUTCdate(), @Rows AS INT = 0, @PartitionId AS TINYINT, @MaxPartitions AS TINYINT = 16, @LookedAtPartitions AS TINYINT = 0, @InflightRows AS INT = 0, @Lock AS VARCHAR (100) = 'DequeueJob_' + CONVERT (VARCHAR, @QueueType); +BEGIN TRY + SET @PartitionId = @MaxPartitions * rand(); + BEGIN TRANSACTION; + EXECUTE sp_getapplock @Lock, 'Exclusive'; + WHILE @LookedAtPartitions <= @MaxPartitions + BEGIN + SET @InflightRows += (SELECT count(*) + FROM dbo.JobQueue + WHERE PartitionId = @PartitionId + AND QueueType = @QueueType + AND Status IN (0, 1)); + SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; + SET @LookedAtPartitions = @LookedAtPartitions + 1; + END + IF @InflightRows = 0 + BEGIN + SET @LookedAtPartitions = 0; + WHILE @LookedAtPartitions <= @MaxPartitions + BEGIN + UPDATE dbo.JobQueue + SET Status = 5 + WHERE PartitionId = @PartitionId + AND QueueType = @QueueType + AND Status IN (2, 3, 4); + SET @Rows += @@rowcount; + SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; + SET @LookedAtPartitions = @LookedAtPartitions + 1; + END + END + COMMIT TRANSACTION; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.BatchDeleteResourceParams +@tableName NVARCHAR (128), @resourceTypeId SMALLINT, @startResourceSurrogateId BIGINT, @endResourceSurrogateId BIGINT, @batchSize INT +AS +SET XACT_ABORT ON; +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DECLARE @Sql AS NVARCHAR (MAX); +DECLARE @ParmDefinition AS NVARCHAR (512); +IF OBJECT_ID(@tableName) IS NOT NULL + BEGIN + SET @sql = N'DELETE TOP(@BatchSizeParam) FROM ' + @tableName + N' WITH (TABLOCK) WHERE ResourceTypeId = @ResourceTypeIdParam AND ResourceSurrogateId >= @StartResourceSurrogateIdParam AND ResourceSurrogateId < @EndResourceSurrogateIdParam'; + SET @parmDefinition = N'@BatchSizeParam int, @ResourceTypeIdParam smallint, @StartResourceSurrogateIdParam bigint, @EndResourceSurrogateIdParam bigint'; + EXECUTE sp_executesql @sql, @parmDefinition, @BatchSizeParam = @batchSize, @ResourceTypeIdParam = @resourceTypeId, @StartResourceSurrogateIdParam = @startResourceSurrogateId, @EndResourceSurrogateIdParam = @endResourceSurrogateId; + END +COMMIT TRANSACTION; +RETURN @@rowcount; + +GO +CREATE PROCEDURE dbo.BatchDeleteResources +@resourceTypeId SMALLINT, @startResourceSurrogateId BIGINT, @endResourceSurrogateId BIGINT, @batchSize INT +AS +SET XACT_ABORT ON; +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DELETE TOP (@batchSize) + dbo.Resource WITH (TABLOCK) +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId >= @startResourceSurrogateId + AND ResourceSurrogateId < @endResourceSurrogateId; +COMMIT TRANSACTION; +RETURN @@rowcount; + +GO +CREATE PROCEDURE dbo.BatchDeleteResourceWriteClaims +@startResourceSurrogateId BIGINT, @endResourceSurrogateId BIGINT, @batchSize INT +AS +SET XACT_ABORT ON; +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DELETE TOP (@batchSize) + dbo.ResourceWriteClaim WITH (TABLOCK) +WHERE ResourceSurrogateId >= @startResourceSurrogateId + AND ResourceSurrogateId < @endResourceSurrogateId; +COMMIT TRANSACTION; +RETURN @@rowcount; + +GO +CREATE PROCEDURE dbo.BulkMergeResource +@resources dbo.BulkImportResourceType_1 READONLY +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +MERGE INTO [dbo].[Resource] WITH (ROWLOCK, INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) + AS target +USING @resources AS source ON source.[ResourceTypeId] = target.[ResourceTypeId] + AND source.[ResourceId] = target.[ResourceId] + AND source.[Version] = target.[Version] +WHEN NOT MATCHED BY TARGET THEN INSERT ([ResourceTypeId], [ResourceId], [Version], [IsHistory], [ResourceSurrogateId], [IsDeleted], [RequestMethod], [RawResource], [IsRawResourceMetaSet], [SearchParamHash]) VALUES ([ResourceTypeId], [ResourceId], [Version], [IsHistory], [ResourceSurrogateId], [IsDeleted], [RequestMethod], [RawResource], [IsRawResourceMetaSet], [SearchParamHash]) OUTPUT Inserted.[ResourceSurrogateId]; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.BulkReindexResources_2 +@resourcesToReindex dbo.BulkReindexResourceTableType_1 READONLY, @resourceWriteClaims dbo.BulkResourceWriteClaimTableType_1 READONLY, @compartmentAssignments dbo.BulkCompartmentAssignmentTableType_1 READONLY, @referenceSearchParams dbo.BulkReferenceSearchParamTableType_1 READONLY, @tokenSearchParams dbo.BulkTokenSearchParamTableType_2 READONLY, @tokenTextSearchParams dbo.BulkTokenTextTableType_1 READONLY, @stringSearchParams dbo.BulkStringSearchParamTableType_2 READONLY, @numberSearchParams dbo.BulkNumberSearchParamTableType_1 READONLY, @quantitySearchParams dbo.BulkQuantitySearchParamTableType_1 READONLY, @uriSearchParams dbo.BulkUriSearchParamTableType_1 READONLY, @dateTimeSearchParms dbo.BulkDateTimeSearchParamTableType_2 READONLY, @referenceTokenCompositeSearchParams dbo.BulkReferenceTokenCompositeSearchParamTableType_2 READONLY, @tokenTokenCompositeSearchParams dbo.BulkTokenTokenCompositeSearchParamTableType_2 READONLY, @tokenDateTimeCompositeSearchParams dbo.BulkTokenDateTimeCompositeSearchParamTableType_2 READONLY, @tokenQuantityCompositeSearchParams dbo.BulkTokenQuantityCompositeSearchParamTableType_2 READONLY, @tokenStringCompositeSearchParams dbo.BulkTokenStringCompositeSearchParamTableType_2 READONLY, @tokenNumberNumberCompositeSearchParams dbo.BulkTokenNumberNumberCompositeSearchParamTableType_2 READONLY +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @computedValues TABLE ( + Offset INT NOT NULL, + ResourceTypeId SMALLINT NOT NULL, + VersionProvided BIGINT NULL, + SearchParamHash VARCHAR (64) NOT NULL, + ResourceSurrogateId BIGINT NULL, + VersionInDatabase BIGINT NULL); +INSERT INTO @computedValues +SELECT resourceToReindex.Offset, + resourceToReindex.ResourceTypeId, + resourceToReindex.ETag, + resourceToReindex.SearchParamHash, + resourceInDB.ResourceSurrogateId, + resourceInDB.Version +FROM @resourcesToReindex AS resourceToReindex + LEFT OUTER JOIN + dbo.Resource AS resourceInDB WITH (UPDLOCK, INDEX (IX_Resource_ResourceTypeId_ResourceId)) + ON resourceInDB.ResourceTypeId = resourceToReindex.ResourceTypeId + AND resourceInDB.ResourceId = resourceToReindex.ResourceId + AND resourceInDB.IsHistory = 0; +DECLARE @versionDiff AS INT; +SET @versionDiff = (SELECT COUNT(*) + FROM @computedValues + WHERE VersionProvided IS NOT NULL + AND VersionProvided <> VersionInDatabase); +IF (@versionDiff > 0) + BEGIN + DELETE @computedValues + WHERE VersionProvided IS NOT NULL + AND VersionProvided <> VersionInDatabase; + END +UPDATE resourceInDB +SET resourceInDB.SearchParamHash = resourceToReindex.SearchParamHash +FROM @computedValues AS resourceToReindex + INNER JOIN + dbo.Resource AS resourceInDB + ON resourceInDB.ResourceTypeId = resourceToReindex.ResourceTypeId + AND resourceInDB.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.ResourceWriteClaim AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.CompartmentAssignment AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.ReferenceSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenText AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.StringSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.UriSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.NumberSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.QuantitySearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.DateTimeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.ReferenceTokenCompositeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenTokenCompositeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenDateTimeCompositeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenQuantityCompositeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenStringCompositeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenNumberNumberCompositeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) +SELECT DISTINCT resourceToReindex.ResourceSurrogateId, + searchIndex.ClaimTypeId, + searchIndex.ClaimValue +FROM @resourceWriteClaims AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.CompartmentAssignment (ResourceTypeId, ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.CompartmentTypeId, + searchIndex.ReferenceResourceId, + 0 +FROM @compartmentAssignments AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.BaseUri, + searchIndex.ReferenceResourceTypeId, + searchIndex.ReferenceResourceId, + searchIndex.ReferenceResourceVersion, + 0 +FROM @referenceSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId, + searchIndex.Code, + searchIndex.CodeOverflow, + 0 +FROM @tokenSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.Text, + 0 +FROM @tokenTextSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsHistory, IsMin, IsMax) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.Text, + searchIndex.TextOverflow, + 0, + searchIndex.IsMin, + searchIndex.IsMax +FROM @stringSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.Uri, + 0 +FROM @uriSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SingleValue, + searchIndex.LowValue, + searchIndex.HighValue, + 0 +FROM @numberSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId, + searchIndex.QuantityCodeId, + searchIndex.SingleValue, + searchIndex.LowValue, + searchIndex.HighValue, + 0 +FROM @quantitySearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsHistory, IsMin, IsMax) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.StartDateTime, + searchIndex.EndDateTime, + searchIndex.IsLongerThanADay, + 0, + searchIndex.IsMin, + searchIndex.IsMax +FROM @dateTimeSearchParms AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.BaseUri1, + searchIndex.ReferenceResourceTypeId1, + searchIndex.ReferenceResourceId1, + searchIndex.ReferenceResourceVersion1, + searchIndex.SystemId2, + searchIndex.Code2, + searchIndex.CodeOverflow2, + 0 +FROM @referenceTokenCompositeSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId1, + searchIndex.Code1, + searchIndex.CodeOverflow1, + searchIndex.SystemId2, + searchIndex.Code2, + searchIndex.CodeOverflow2, + 0 +FROM @tokenTokenCompositeSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId1, + searchIndex.Code1, + searchIndex.CodeOverflow1, + searchIndex.StartDateTime2, + searchIndex.EndDateTime2, + searchIndex.IsLongerThanADay2, + 0 +FROM @tokenDateTimeCompositeSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId1, + searchIndex.Code1, + searchIndex.CodeOverflow1, + searchIndex.SingleValue2, + searchIndex.SystemId2, + searchIndex.QuantityCodeId2, + searchIndex.LowValue2, + searchIndex.HighValue2, + 0 +FROM @tokenQuantityCompositeSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId1, + searchIndex.Code1, + searchIndex.CodeOverflow1, + searchIndex.Text2, + searchIndex.TextOverflow2, + 0 +FROM @tokenStringCompositeSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId1, + searchIndex.Code1, + searchIndex.CodeOverflow1, + searchIndex.SingleValue2, + searchIndex.LowValue2, + searchIndex.HighValue2, + searchIndex.SingleValue3, + searchIndex.LowValue3, + searchIndex.HighValue3, + searchIndex.HasRange, + 0 +FROM @tokenNumberNumberCompositeSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +SELECT @versionDiff; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE [dbo].[CancelTask] +@taskId VARCHAR (64) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +IF NOT EXISTS (SELECT * + FROM [dbo].[TaskInfo] + WHERE TaskId = @taskId) + BEGIN + THROW 50404, 'Task not exist', 1; + END +UPDATE dbo.TaskInfo +SET IsCanceled = 1, + HeartbeatDateTime = @heartbeatDateTime +WHERE TaskId = @taskId; +SELECT TaskId, + QueueId, + Status, + TaskTypeId, + RunId, + IsCanceled, + RetryCount, + MaxRetryCount, + HeartbeatDateTime, + InputData, + TaskContext, + Result +FROM [dbo].[TaskInfo] +WHERE TaskId = @taskId; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.CaptureResourceChanges +@isDeleted BIT, @version INT, @resourceId VARCHAR (64), @resourceTypeId SMALLINT +AS +BEGIN + DECLARE @changeType AS SMALLINT; + IF (@isDeleted = 1) + BEGIN + SET @changeType = 2; + END + ELSE + BEGIN + IF (@version = 1) + BEGIN + SET @changeType = 0; + END + ELSE + BEGIN + SET @changeType = 1; + END + END + INSERT INTO dbo.ResourceChangeData (ResourceId, ResourceTypeId, ResourceVersion, ResourceChangeTypeId) + VALUES (@resourceId, @resourceTypeId, @version, @changeType); +END + +GO +CREATE PROCEDURE dbo.CaptureResourceIdsForChanges +@Resources dbo.ResourceList READONLY +AS +SET NOCOUNT ON; +INSERT INTO dbo.ResourceChangeData (ResourceId, ResourceTypeId, ResourceVersion, ResourceChangeTypeId) +SELECT ResourceId, + ResourceTypeId, + Version, + CASE WHEN IsDeleted = 1 THEN 2 WHEN Version > 1 THEN 1 ELSE 0 END +FROM @Resources +WHERE IsHistory = 0; + +GO +CREATE PROCEDURE dbo.CheckActiveReindexJobs +AS +SET NOCOUNT ON; +SELECT Id +FROM dbo.ReindexJob +WHERE Status = 'Running' + OR Status = 'Queued' + OR Status = 'Paused'; + +GO +CREATE PROCEDURE dbo.CleanupEventLog +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'CleanupEventLog', @Mode AS VARCHAR (100) = '', @MaxDeleteRows AS INT, @MaxAllowedRows AS BIGINT, @RetentionPeriodSecond AS INT, @DeletedRows AS INT, @TotalDeletedRows AS INT = 0, @TotalRows AS INT, @Now AS DATETIME = getUTCdate(); +EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; +BEGIN TRY + SET @MaxDeleteRows = (SELECT Number + FROM dbo.Parameters + WHERE Id = 'CleanupEventLog.DeleteBatchSize'); + IF @MaxDeleteRows IS NULL + RAISERROR ('Cannot get Parameter.CleanupEventLog.DeleteBatchSize', 18, 127); + SET @MaxAllowedRows = (SELECT Number + FROM dbo.Parameters + WHERE Id = 'CleanupEventLog.AllowedRows'); + IF @MaxAllowedRows IS NULL + RAISERROR ('Cannot get Parameter.CleanupEventLog.AllowedRows', 18, 127); + SET @RetentionPeriodSecond = (SELECT Number * 24 * 60 * 60 + FROM dbo.Parameters + WHERE Id = 'CleanupEventLog.RetentionPeriodDay'); + IF @RetentionPeriodSecond IS NULL + RAISERROR ('Cannot get Parameter.CleanupEventLog.RetentionPeriodDay', 18, 127); + SET @TotalRows = (SELECT sum(row_count) + FROM sys.dm_db_partition_stats + WHERE object_id = object_id('EventLog') + AND index_id IN (0, 1)); + SET @DeletedRows = 1; + WHILE @DeletedRows > 0 + AND EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = 'CleanupEventLog.IsEnabled' + AND Number = 1) + BEGIN + SET @DeletedRows = 0; + IF @TotalRows - @TotalDeletedRows > @MaxAllowedRows + BEGIN + DELETE TOP (@MaxDeleteRows) + dbo.EventLog WITH (PAGLOCK) + WHERE EventDate <= dateadd(second, -@RetentionPeriodSecond, @Now); + SET @DeletedRows = @@rowcount; + SET @TotalDeletedRows += @DeletedRows; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'EventLog', @Action = 'Delete', @Rows = @DeletedRows, @Text = @TotalDeletedRows; + END + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @Now; +END TRY +BEGIN CATCH + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.CompleteTask +@taskId VARCHAR (64), @taskResult VARCHAR (MAX), @runId VARCHAR (50) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +IF NOT EXISTS (SELECT * + FROM [dbo].[TaskInfo] + WHERE TaskId = @taskId + AND RunId = @runId) + BEGIN + THROW 50404, 'Task not exist or runid not match', 1; + END +UPDATE dbo.TaskInfo +SET Status = 3, + EndDateTime = SYSUTCDATETIME(), + Result = @taskResult +WHERE TaskId = @taskId; +COMMIT TRANSACTION; +EXECUTE dbo.GetTaskDetails @TaskId = @taskId; + +GO +CREATE OR ALTER PROCEDURE dbo.ConfigurePartitionOnResourceChanges +@numberOfFuturePartitionsToAdd INT +AS +BEGIN + SET XACT_ABORT ON; + BEGIN TRANSACTION; + DECLARE @partitionBoundary AS DATETIME2 (7) = DATEADD(hour, DATEDIFF(hour, 0, sysutcdatetime()), 0); + DECLARE @startingRightPartitionBoundary AS DATETIME2 (7) = CAST ((SELECT TOP (1) value + FROM sys.partition_range_values AS prv + INNER JOIN + sys.partition_functions AS pf + ON pf.function_id = prv.function_id + WHERE pf.name = N'PartitionFunction_ResourceChangeData_Timestamp' + ORDER BY prv.boundary_id DESC) AS DATETIME2 (7)); + DECLARE @numberOfPartitionsToAdd AS INT = @numberOfFuturePartitionsToAdd + 1; + WHILE @numberOfPartitionsToAdd > 0 + BEGIN + IF (@startingRightPartitionBoundary < @partitionBoundary) + BEGIN + ALTER PARTITION SCHEME PartitionScheme_ResourceChangeData_Timestamp NEXT USED [PRIMARY]; + ALTER PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp( ) + SPLIT RANGE (@partitionBoundary); + END + SET @partitionBoundary = DATEADD(hour, 1, @partitionBoundary); + SET @numberOfPartitionsToAdd -= 1; + END + COMMIT TRANSACTION; +END + +GO +CREATE PROCEDURE dbo.CreateExportJob +@id VARCHAR (64), @hash VARCHAR (64), @status VARCHAR (10), @rawJobRecord VARCHAR (MAX) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +INSERT INTO dbo.ExportJob (Id, Hash, Status, HeartbeatDateTime, RawJobRecord) +VALUES (@id, @hash, @status, @heartbeatDateTime, @rawJobRecord); +SELECT CAST (MIN_ACTIVE_ROWVERSION() AS INT); +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.CreateReindexJob +@id VARCHAR (64), @status VARCHAR (10), @rawJobRecord VARCHAR (MAX) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +INSERT INTO dbo.ReindexJob (Id, Status, HeartbeatDateTime, RawJobRecord) +VALUES (@id, @status, @heartbeatDateTime, @rawJobRecord); +SELECT CAST (MIN_ACTIVE_ROWVERSION() AS INT); +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE [dbo].[CreateTask_3] +@taskId VARCHAR (64), @queueId VARCHAR (64), @taskTypeId SMALLINT, @parentTaskId VARCHAR (64), @maxRetryCount SMALLINT=3, @inputData VARCHAR (MAX), @isUniqueTaskByType BIT +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +DECLARE @status AS SMALLINT = 1; +DECLARE @retryCount AS SMALLINT = 0; +DECLARE @isCanceled AS BIT = 0; +IF (@isUniqueTaskByType = 1) + BEGIN + IF EXISTS (SELECT * + FROM [dbo].[TaskInfo] + WHERE TaskId = @taskId + OR (TaskTypeId = @taskTypeId + AND Status <> 3)) + BEGIN + THROW 50409, 'Task already existed', 1; + END + END +ELSE + BEGIN + IF EXISTS (SELECT * + FROM [dbo].[TaskInfo] + WHERE TaskId = @taskId) + BEGIN + THROW 50409, 'Task already existed', 1; + END + END +INSERT INTO [dbo].[TaskInfo] (TaskId, QueueId, Status, TaskTypeId, IsCanceled, RetryCount, MaxRetryCount, HeartbeatDateTime, InputData, ParentTaskId) +VALUES (@taskId, @queueId, @status, @taskTypeId, @isCanceled, @retryCount, @maxRetryCount, @heartbeatDateTime, @inputData, @parentTaskId); +EXECUTE dbo.GetTaskDetails @TaskId = @taskId; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.Defrag +@TableName VARCHAR (100), @IndexName VARCHAR (200), @PartitionNumber INT, @IsPartitioned BIT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'Defrag', @Mode AS VARCHAR (200) = @TableName + '.' + @IndexName + '.' + CONVERT (VARCHAR, @PartitionNumber) + '.' + CONVERT (VARCHAR, @IsPartitioned), @st AS DATETIME = getUTCdate(), @SQL AS VARCHAR (3500), @msg AS VARCHAR (1000), @SizeBefore AS FLOAT, @SizeAfter AS FLOAT, @IndexId AS INT; +BEGIN TRY + SET @IndexId = (SELECT index_id + FROM sys.indexes + WHERE object_id = object_id(@TableName) + AND name = @IndexName); + SET @SizeBefore = (SELECT sum(reserved_page_count) + FROM sys.dm_db_partition_stats + WHERE object_id = object_id(@TableName) + AND index_id = @IndexId) * 8.0 / 1024 / 1024; + SET @msg = 'Size[GB] before=' + CONVERT (VARCHAR, @SizeBefore); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start', @Text = @msg; + SET @Sql = 'ALTER INDEX ' + quotename(@IndexName) + ' ON dbo.' + quotename(@TableName) + ' REORGANIZE' + CASE WHEN @IsPartitioned = 1 THEN ' PARTITION = ' + CONVERT (VARCHAR, @PartitionNumber) ELSE '' END; + BEGIN TRY + EXECUTE (@Sql); + SET @SizeAfter = (SELECT sum(reserved_page_count) + FROM sys.dm_db_partition_stats + WHERE object_id = object_id(@TableName) + AND index_id = @IndexId) * 8.0 / 1024 / 1024; + SET @msg = 'Size[GB] before=' + CONVERT (VARCHAR, @SizeBefore) + ', after=' + CONVERT (VARCHAR, @SizeAfter) + ', reduced by=' + CONVERT (VARCHAR, @SizeBefore - @SizeAfter); + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'End', @Mode = @Mode, @Action = 'Reorganize', @Start = @st, @Text = @msg; + END TRY + BEGIN CATCH + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'Error', @Mode = @Mode, @Action = 'Reorganize', @Start = @st, @ReRaisError = 0; + END CATCH +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.DefragChangeDatabaseSettings +@IsOn BIT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'DefragChangeDatabaseSettings', @Mode AS VARCHAR (200) = 'On=' + CONVERT (VARCHAR, @IsOn), @st AS DATETIME = getUTCdate(), @SQL AS VARCHAR (3500); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'Start', @Mode = @Mode; + SET @SQL = 'ALTER DATABASE CURRENT SET AUTO_UPDATE_STATISTICS ' + CASE WHEN @IsOn = 1 THEN 'ON' ELSE 'OFF' END; + EXECUTE (@SQL); + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'Run', @Mode = @Mode, @Text = @SQL; + SET @SQL = 'ALTER DATABASE CURRENT SET AUTO_CREATE_STATISTICS ' + CASE WHEN @IsOn = 1 THEN 'ON' ELSE 'OFF' END; + EXECUTE (@SQL); + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'End', @Mode = @Mode, @Start = @st, @Text = @SQL; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.DeleteHistory +@DeleteResources BIT=0, @Reset BIT=0, @DisableLogEvent BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'DeleteHistory', @Mode AS VARCHAR (100) = 'D=' + isnull(CONVERT (VARCHAR, @DeleteResources), 'NULL') + ' R=' + isnull(CONVERT (VARCHAR, @Reset), 'NULL'), @st AS DATETIME = getUTCdate(), @Id AS VARCHAR (100) = 'DeleteHistory.LastProcessed.TypeId.SurrogateId', @ResourceTypeId AS SMALLINT, @SurrogateId AS BIGINT, @RowsToProcess AS INT, @ProcessedResources AS INT = 0, @DeletedResources AS INT = 0, @DeletedSearchParams AS INT = 0, @ReportDate AS DATETIME = getUTCdate(); +BEGIN TRY + IF @DisableLogEvent = 0 + INSERT INTO dbo.Parameters (Id, Char) + SELECT @SP, + 'LogEvent'; + ELSE + DELETE dbo.Parameters + WHERE Id = @SP; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + INSERT INTO dbo.Parameters (Id, Char) + SELECT @Id, + '0.0' + WHERE NOT EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = @Id); + DECLARE @LastProcessed AS VARCHAR (100) = CASE WHEN @Reset = 0 THEN (SELECT Char + FROM dbo.Parameters + WHERE Id = @Id) ELSE '0.0' END; + DECLARE @Types TABLE ( + ResourceTypeId SMALLINT PRIMARY KEY, + Name VARCHAR (100)); + DECLARE @SurrogateIds TABLE ( + ResourceSurrogateId BIGINT PRIMARY KEY, + IsHistory BIT ); + INSERT INTO @Types + EXECUTE dbo.GetUsedResourceTypes ; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = '@Types', @Action = 'Insert', @Rows = @@rowcount; + SET @ResourceTypeId = substring(@LastProcessed, 1, charindex('.', @LastProcessed) - 1); + SET @SurrogateId = substring(@LastProcessed, charindex('.', @LastProcessed) + 1, 255); + DELETE @Types + WHERE ResourceTypeId < @ResourceTypeId; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = '@Types', @Action = 'Delete', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Types) + BEGIN + SET @ResourceTypeId = (SELECT TOP 1 ResourceTypeId + FROM @Types + ORDER BY ResourceTypeId); + SET @ProcessedResources = 0; + SET @DeletedResources = 0; + SET @DeletedSearchParams = 0; + SET @RowsToProcess = 1; + WHILE @RowsToProcess > 0 + BEGIN + DELETE @SurrogateIds; + INSERT INTO @SurrogateIds + SELECT TOP 10000 ResourceSurrogateId, + IsHistory + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId > @SurrogateId + ORDER BY ResourceSurrogateId; + SET @RowsToProcess = @@rowcount; + SET @ProcessedResources += @RowsToProcess; + IF @RowsToProcess > 0 + SET @SurrogateId = (SELECT max(ResourceSurrogateId) + FROM @SurrogateIds); + SET @LastProcessed = CONVERT (VARCHAR, @ResourceTypeId) + '.' + CONVERT (VARCHAR, @SurrogateId); + DELETE @SurrogateIds + WHERE IsHistory = 0; + IF EXISTS (SELECT * + FROM @SurrogateIds) + BEGIN + DELETE dbo.ResourceWriteClaim + WHERE ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.CompartmentAssignment + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.ReferenceSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenText + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.StringSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.UriSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.NumberSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.QuantitySearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.DateTimeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.ReferenceTokenCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenTokenCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenDateTimeCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenQuantityCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenStringCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenNumberNumberCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + IF @DeleteResources = 1 + BEGIN + DELETE dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedResources += @@rowcount; + END + END + UPDATE dbo.Parameters + SET Char = @LastProcessed + WHERE Id = @Id; + IF datediff(second, @ReportDate, getUTCdate()) > 60 + BEGIN + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'Resource', @Action = 'Select', @Rows = @ProcessedResources, @Text = @LastProcessed; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = '*SearchParam', @Action = 'Delete', @Rows = @DeletedSearchParams, @Text = @LastProcessed; + IF @DeleteResources = 1 + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'Resource', @Action = 'Delete', @Rows = @DeletedResources, @Text = @LastProcessed; + SET @ReportDate = getUTCdate(); + SET @ProcessedResources = 0; + SET @DeletedSearchParams = 0; + SET @DeletedResources = 0; + END + END + DELETE @Types + WHERE ResourceTypeId = @ResourceTypeId; + SET @SurrogateId = 0; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'Resource', @Action = 'Select', @Rows = @ProcessedResources, @Text = @LastProcessed; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = '*SearchParam', @Action = 'Delete', @Rows = @DeletedSearchParams, @Text = @LastProcessed; + IF @DeleteResources = 1 + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'Resource', @Action = 'Delete', @Rows = @DeletedResources, @Text = @LastProcessed; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.DequeueJob +@QueueType TINYINT, @Worker VARCHAR (100), @HeartbeatTimeoutSec INT, @InputJobId BIGINT=NULL, @CheckTimeoutJobs BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'DequeueJob', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' H=' + isnull(CONVERT (VARCHAR, @HeartbeatTimeoutSec), 'NULL') + ' W=' + isnull(@Worker, 'NULL') + ' IJ=' + isnull(CONVERT (VARCHAR, @InputJobId), 'NULL') + ' T=' + isnull(CONVERT (VARCHAR, @CheckTimeoutJobs), 'NULL'), @Rows AS INT = 0, @st AS DATETIME = getUTCdate(), @JobId AS BIGINT, @msg AS VARCHAR (100), @Lock AS VARCHAR (100), @PartitionId AS TINYINT, @MaxPartitions AS TINYINT = 16, @LookedAtPartitions AS TINYINT = 0; +BEGIN TRY + IF EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = 'DequeueJobStop' + AND Number = 1) + BEGIN + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = 0, @Text = 'Skipped'; + RETURN; + END + IF @InputJobId IS NULL + SET @PartitionId = @MaxPartitions * rand(); + ELSE + SET @PartitionId = @InputJobId % 16; + SET TRANSACTION ISOLATION LEVEL READ COMMITTED; + WHILE @InputJobId IS NULL + AND @JobId IS NULL + AND @LookedAtPartitions <= @MaxPartitions + AND @CheckTimeoutJobs = 0 + BEGIN + SET @Lock = 'DequeueJob_' + CONVERT (VARCHAR, @QueueType) + '_' + CONVERT (VARCHAR, @PartitionId); + BEGIN TRANSACTION; + EXECUTE sp_getapplock @Lock, 'Exclusive'; + UPDATE T + SET StartDate = getUTCdate(), + HeartbeatDate = getUTCdate(), + Worker = @Worker, + Status = 1, + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()), + @JobId = T.JobId + FROM dbo.JobQueue AS T WITH (PAGLOCK) + INNER JOIN + (SELECT TOP 1 JobId + FROM dbo.JobQueue WITH (INDEX (IX_QueueType_PartitionId_Status_Priority)) + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND Status = 0 + ORDER BY Priority, JobId) AS S + ON QueueType = @QueueType + AND PartitionId = @PartitionId + AND T.JobId = S.JobId; + SET @Rows += @@rowcount; + COMMIT TRANSACTION; + IF @JobId IS NULL + BEGIN + SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; + SET @LookedAtPartitions = @LookedAtPartitions + 1; + END + END + SET @LookedAtPartitions = 0; + WHILE @InputJobId IS NULL + AND @JobId IS NULL + AND @LookedAtPartitions <= @MaxPartitions + BEGIN + SET @Lock = 'DequeueStoreCopyWorkUnit_' + CONVERT (VARCHAR, @PartitionId); + BEGIN TRANSACTION; + EXECUTE sp_getapplock @Lock, 'Exclusive'; + UPDATE T + SET StartDate = getUTCdate(), + HeartbeatDate = getUTCdate(), + Worker = @Worker, + Status = CASE WHEN CancelRequested = 0 THEN 1 ELSE 4 END, + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()), + @JobId = CASE WHEN CancelRequested = 0 THEN T.JobId END, + Info = CONVERT (VARCHAR (1000), isnull(Info, '') + ' Prev: Worker=' + Worker + ' Start=' + CONVERT (VARCHAR, StartDate, 121)) + FROM dbo.JobQueue AS T WITH (PAGLOCK) + INNER JOIN + (SELECT TOP 1 JobId + FROM dbo.JobQueue WITH (INDEX (IX_QueueType_PartitionId_Status_Priority)) + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND Status = 1 + AND datediff(second, HeartbeatDate, getUTCdate()) > @HeartbeatTimeoutSec + ORDER BY Priority, JobId) AS S + ON QueueType = @QueueType + AND PartitionId = @PartitionId + AND T.JobId = S.JobId; + SET @Rows += @@rowcount; + COMMIT TRANSACTION; + IF @JobId IS NULL + BEGIN + SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; + SET @LookedAtPartitions = @LookedAtPartitions + 1; + END + END + IF @InputJobId IS NOT NULL + BEGIN + UPDATE dbo.JobQueue WITH (PAGLOCK) + SET StartDate = getUTCdate(), + HeartbeatDate = getUTCdate(), + Worker = @Worker, + Status = 1, + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()), + @JobId = JobId + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND Status = 0 + AND JobId = @InputJobId; + SET @Rows += @@rowcount; + IF @JobId IS NULL + BEGIN + UPDATE dbo.JobQueue WITH (PAGLOCK) + SET StartDate = getUTCdate(), + HeartbeatDate = getUTCdate(), + Worker = @Worker, + Status = 1, + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()), + @JobId = JobId + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND Status = 1 + AND JobId = @InputJobId + AND datediff(second, HeartbeatDate, getUTCdate()) > @HeartbeatTimeoutSec; + SET @Rows += @@rowcount; + END + END + IF @JobId IS NOT NULL + EXECUTE dbo.GetJobs @QueueType = @QueueType, @JobId = @JobId; + SET @msg = 'J=' + isnull(CONVERT (VARCHAR, @JobId), 'NULL') + ' P=' + CONVERT (VARCHAR, @PartitionId); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows, @Text = @msg; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.DisableIndex +@tableName NVARCHAR (128), @indexName NVARCHAR (128) +WITH EXECUTE AS 'dbo' +AS +DECLARE @errorTxt AS VARCHAR (1000), @sql AS NVARCHAR (1000), @isDisabled AS BIT; +IF object_id(@tableName) IS NULL + BEGIN + SET @errorTxt = @tableName + ' does not exist or you don''t have permissions.'; + RAISERROR (@errorTxt, 18, 127); + END +SET @isDisabled = (SELECT is_disabled + FROM sys.indexes + WHERE object_id = object_id(@tableName) + AND name = @indexName); +IF @isDisabled IS NULL + BEGIN + SET @errorTxt = @indexName + ' does not exist or you don''t have permissions.'; + RAISERROR (@errorTxt, 18, 127); + END +IF @isDisabled = 0 + BEGIN + SET @sql = N'ALTER INDEX ' + QUOTENAME(@indexName) + N' on ' + @tableName + ' Disable'; + EXECUTE sp_executesql @sql; + END + +GO +CREATE PROCEDURE dbo.DisableIndexes +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'DisableIndexes', @Mode AS VARCHAR (200) = '', @st AS DATETIME = getUTCdate(), @Tbl AS VARCHAR (100), @Ind AS VARCHAR (200), @Txt AS VARCHAR (4000); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + DECLARE @Tables TABLE ( + Tbl VARCHAR (100) PRIMARY KEY, + Supported BIT ); + INSERT INTO @Tables + EXECUTE dbo.GetPartitionedTables @IncludeNotDisabled = 1, @IncludeNotSupported = 0; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Tables', @Action = 'Insert', @Rows = @@rowcount; + DECLARE @Indexes TABLE ( + Tbl VARCHAR (100), + Ind VARCHAR (200), + TblId INT , + IndId INT PRIMARY KEY (Tbl, Ind)); + INSERT INTO @Indexes + SELECT Tbl, + I.Name, + TblId, + I.index_id + FROM (SELECT object_id(Tbl) AS TblId, + Tbl + FROM @Tables) AS O + INNER JOIN + sys.indexes AS I + ON I.object_id = TblId; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Insert', @Rows = @@rowcount; + INSERT INTO dbo.IndexProperties (TableName, IndexName, PropertyName, PropertyValue) + SELECT Tbl, + Ind, + 'DATA_COMPRESSION', + data_comp + FROM (SELECT Tbl, + Ind, + isnull((SELECT TOP 1 CASE WHEN data_compression_desc = 'PAGE' THEN 'PAGE' END + FROM sys.partitions + WHERE object_id = TblId + AND index_id = IndId), 'NONE') AS data_comp + FROM @Indexes) AS A + WHERE NOT EXISTS (SELECT * + FROM dbo.IndexProperties + WHERE TableName = Tbl + AND IndexName = Ind); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = 'IndexProperties', @Action = 'Insert', @Rows = @@rowcount; + DELETE @Indexes + WHERE Tbl = 'Resource' + OR IndId = 1; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Delete', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Indexes) + BEGIN + SELECT TOP 1 @Tbl = Tbl, + @Ind = Ind + FROM @Indexes; + SET @Txt = 'ALTER INDEX ' + @Ind + ' ON dbo.' + @Tbl + ' DISABLE'; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Ind, @Action = 'Disable', @Text = @Txt; + DELETE @Indexes + WHERE Tbl = @Tbl + AND Ind = @Ind; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.EnqueueJobs +@QueueType TINYINT, @Definitions StringList READONLY, @GroupId BIGINT=NULL, @ForceOneActiveJobGroup BIT=1, @IsCompleted BIT=NULL, @ReturnJobs BIT=1 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'EnqueueJobs', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' D=' + CONVERT (VARCHAR, (SELECT count(*) + FROM @Definitions)) + ' G=' + isnull(CONVERT (VARCHAR, @GroupId), 'NULL') + ' F=' + isnull(CONVERT (VARCHAR, @ForceOneActiveJobGroup), 'NULL') + ' C=' + isnull(CONVERT (VARCHAR, @IsCompleted), 'NULL'), @st AS DATETIME = getUTCdate(), @Lock AS VARCHAR (100) = 'EnqueueJobs_' + CONVERT (VARCHAR, @QueueType), @MaxJobId AS BIGINT, @Rows AS INT, @msg AS VARCHAR (1000), @JobIds AS BigintList, @InputRows AS INT; +BEGIN TRY + DECLARE @Input TABLE ( + DefinitionHash VARBINARY (20) PRIMARY KEY, + Definition VARCHAR (MAX) ); + INSERT INTO @Input + SELECT hashbytes('SHA1', String) AS DefinitionHash, + String AS Definition + FROM @Definitions; + SET @InputRows = @@rowcount; + INSERT INTO @JobIds + SELECT JobId + FROM @Input AS A + INNER JOIN + dbo.JobQueue AS B + ON B.QueueType = @QueueType + AND B.DefinitionHash = A.DefinitionHash + AND B.Status <> 5; + IF @@rowcount < @InputRows + BEGIN + BEGIN TRANSACTION; + EXECUTE sp_getapplock @Lock, 'Exclusive'; + IF @ForceOneActiveJobGroup = 1 + AND EXISTS (SELECT * + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND Status IN (0, 1) + AND (@GroupId IS NULL + OR GroupId <> @GroupId)) + RAISERROR ('There are other active job groups', 18, 127); + SET @MaxJobId = isnull((SELECT TOP 1 JobId + FROM dbo.JobQueue + WHERE QueueType = @QueueType + ORDER BY JobId DESC), 0); + INSERT INTO dbo.JobQueue (QueueType, GroupId, JobId, Definition, DefinitionHash, Status) + OUTPUT inserted.JobId INTO @JobIds + SELECT @QueueType, + isnull(@GroupId, @MaxJobId + 1) AS GroupId, + JobId, + Definition, + DefinitionHash, + CASE WHEN @IsCompleted = 1 THEN 2 ELSE 0 END AS Status + FROM (SELECT @MaxJobId + row_number() OVER (ORDER BY Dummy) AS JobId, + * + FROM (SELECT *, + 0 AS Dummy + FROM @Input) AS A) AS A + WHERE NOT EXISTS (SELECT * + FROM dbo.JobQueue AS B + WHERE B.QueueType = @QueueType + AND B.DefinitionHash = A.DefinitionHash + AND B.Status <> 5); + SET @Rows = @@rowcount; + COMMIT TRANSACTION; + END + IF @ReturnJobs = 1 + EXECUTE dbo.GetJobs @QueueType = @QueueType, @JobIds = @JobIds; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.ExecuteCommandForRebuildIndexes +@Tbl VARCHAR (100), @Ind VARCHAR (1000), @Cmd VARCHAR (MAX) +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'ExecuteCommandForRebuildIndexes', @Mode AS VARCHAR (200) = 'Tbl=' + isnull(@Tbl, 'NULL'), @st AS DATETIME, @Retries AS INT = 0, @Action AS VARCHAR (100), @msg AS VARCHAR (1000); +RetryOnTempdbError: +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start', @Text = @Cmd; + SET @st = getUTCdate(); + IF @Tbl IS NULL + RAISERROR ('@Tbl IS NULL', 18, 127); + IF @Cmd IS NULL + RAISERROR ('@Cmd IS NULL', 18, 127); + SET @Action = CASE WHEN @Cmd LIKE 'UPDATE STAT%' THEN 'Update statistics' WHEN @Cmd LIKE 'CREATE%INDEX%' THEN 'Create Index' WHEN @Cmd LIKE 'ALTER%INDEX%REBUILD%' THEN 'Rebuild Index' WHEN @Cmd LIKE 'ALTER%TABLE%ADD%' THEN 'Add Constraint' END; + IF @Action IS NULL + BEGIN + SET @msg = 'Not supported command = ' + CONVERT (VARCHAR (900), @Cmd); + RAISERROR (@msg, 18, 127); + END + IF @Action = 'Create Index' + WAITFOR DELAY '00:00:05'; + EXECUTE (@Cmd); + SELECT @Ind; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Action = @Action, @Status = 'End', @Start = @st, @Text = @Cmd; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + IF error_number() = 40544 + BEGIN + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st, @Retry = @Retries; + SET @Retries = @Retries + 1; + IF @Tbl = 'TokenText_96' + WAITFOR DELAY '01:00:00'; + ELSE + WAITFOR DELAY '00:10:00'; + GOTO RetryOnTempdbError; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE OR ALTER PROCEDURE dbo.FetchEventAgentCheckpoint +@CheckpointId VARCHAR (64) +AS +BEGIN + SELECT TOP (1) CheckpointId, + LastProcessedDateTime, + LastProcessedIdentifier + FROM dbo.EventAgentCheckpoint + WHERE CheckpointId = @CheckpointId; +END + +GO +CREATE PROCEDURE dbo.FetchResourceChanges_3 +@startId BIGINT, @lastProcessedUtcDateTime DATETIME2 (7), @pageSize SMALLINT +AS +BEGIN + SET NOCOUNT ON; + DECLARE @precedingPartitionBoundary AS DATETIME2 (7) = (SELECT TOP (1) CAST (prv.value AS DATETIME2 (7)) AS value + FROM sys.partition_range_values AS prv WITH (NOLOCK) + INNER JOIN + sys.partition_functions AS pf WITH (NOLOCK) + ON pf.function_id = prv.function_id + WHERE pf.name = N'PartitionFunction_ResourceChangeData_Timestamp' + AND SQL_VARIANT_PROPERTY(prv.Value, 'BaseType') = 'datetime2' + AND CAST (prv.value AS DATETIME2 (7)) < DATEADD(HOUR, DATEDIFF(HOUR, 0, @lastProcessedUtcDateTime), 0) + ORDER BY prv.boundary_id DESC); + IF (@precedingPartitionBoundary IS NULL) + BEGIN + SET @precedingPartitionBoundary = CONVERT (DATETIME2 (7), N'1970-01-01T00:00:00.0000000'); + END + DECLARE @endDateTimeToFilter AS DATETIME2 (7) = DATEADD(HOUR, 1, SYSUTCDATETIME()); + WITH PartitionBoundaries + AS (SELECT CAST (prv.value AS DATETIME2 (7)) AS PartitionBoundary + FROM sys.partition_range_values AS prv WITH (NOLOCK) + INNER JOIN + sys.partition_functions AS pf WITH (NOLOCK) + ON pf.function_id = prv.function_id + WHERE pf.name = N'PartitionFunction_ResourceChangeData_Timestamp' + AND SQL_VARIANT_PROPERTY(prv.Value, 'BaseType') = 'datetime2' + AND CAST (prv.value AS DATETIME2 (7)) BETWEEN @precedingPartitionBoundary AND @endDateTimeToFilter) + SELECT TOP (@pageSize) Id, + Timestamp, + ResourceId, + ResourceTypeId, + ResourceVersion, + ResourceChangeTypeId + FROM PartitionBoundaries AS p CROSS APPLY (SELECT TOP (@pageSize) Id, + Timestamp, + ResourceId, + ResourceTypeId, + ResourceVersion, + ResourceChangeTypeId + FROM dbo.ResourceChangeData WITH (TABLOCK, HOLDLOCK) + WHERE Id >= @startId + AND $PARTITION.PartitionFunction_ResourceChangeData_Timestamp (Timestamp) = $PARTITION.PartitionFunction_ResourceChangeData_Timestamp (p.PartitionBoundary) + ORDER BY Id ASC) AS rcd + ORDER BY rcd.Id ASC; +END + +GO +CREATE PROCEDURE dbo.GetActiveJobs +@QueueType TINYINT, @GroupId BIGINT=NULL +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetActiveJobs', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' G=' + isnull(CONVERT (VARCHAR, @GroupId), 'NULL'), @st AS DATETIME = getUTCdate(), @JobIds AS BigintList, @PartitionId AS TINYINT, @MaxPartitions AS TINYINT = 16, @LookedAtPartitions AS TINYINT = 0, @Rows AS INT = 0; +BEGIN TRY + SET @PartitionId = @MaxPartitions * rand(); + WHILE @LookedAtPartitions <= @MaxPartitions + BEGIN + IF @GroupId IS NULL + INSERT INTO @JobIds + SELECT JobId + FROM dbo.JobQueue + WHERE PartitionId = @PartitionId + AND QueueType = @QueueType + AND Status IN (0, 1); + ELSE + INSERT INTO @JobIds + SELECT JobId + FROM dbo.JobQueue + WHERE PartitionId = @PartitionId + AND QueueType = @QueueType + AND GroupId = @GroupId + AND Status IN (0, 1); + SET @Rows += @@rowcount; + SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; + SET @LookedAtPartitions += 1; + END + IF @Rows > 0 + EXECUTE dbo.GetJobs @QueueType = @QueueType, @JobIds = @JobIds; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetCommandsForRebuildIndexes +@RebuildClustered BIT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetCommandsForRebuildIndexes', @Mode AS VARCHAR (200) = 'PS=PartitionScheme_ResourceTypeId RC=' + isnull(CONVERT (VARCHAR, @RebuildClustered), 'NULL'), @st AS DATETIME = getUTCdate(), @Tbl AS VARCHAR (100), @TblInt AS VARCHAR (100), @Ind AS VARCHAR (200), @IndId AS INT, @Supported AS BIT, @Txt AS VARCHAR (MAX), @Rows AS BIGINT, @Pages AS BIGINT, @ResourceTypeId AS SMALLINT, @IndexesCnt AS INT, @DataComp AS VARCHAR (100); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + DECLARE @Commands TABLE ( + Tbl VARCHAR (100), + Ind VARCHAR (200), + Txt VARCHAR (MAX), + Pages BIGINT ); + DECLARE @ResourceTypes TABLE ( + ResourceTypeId SMALLINT PRIMARY KEY); + DECLARE @Indexes TABLE ( + Ind VARCHAR (200) PRIMARY KEY, + IndId INT ); + DECLARE @Tables TABLE ( + name VARCHAR (100) PRIMARY KEY, + Supported BIT ); + INSERT INTO @Tables + EXECUTE dbo.GetPartitionedTables @IncludeNotDisabled = 1, @IncludeNotSupported = 1; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Tables', @Action = 'Insert', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Tables) + BEGIN + SELECT TOP 1 @Tbl = name, + @Supported = Supported + FROM @Tables + ORDER BY name; + IF @Supported = 0 + BEGIN + INSERT INTO @Commands + SELECT @Tbl, + name, + 'ALTER INDEX ' + name + ' ON dbo.' + @Tbl + ' REBUILD' + CASE WHEN (SELECT PropertyValue + FROM dbo.IndexProperties + WHERE TableName = @Tbl + AND IndexName = name) = 'PAGE' THEN ' PARTITION = ALL WITH (DATA_COMPRESSION = PAGE)' ELSE '' END, + CONVERT (BIGINT, 9e18) + FROM sys.indexes + WHERE object_id = object_id(@Tbl) + AND (is_disabled = 1 + AND index_id > 1 + AND @RebuildClustered = 0 + OR index_id = 1 + AND @RebuildClustered = 1); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Commands', @Action = 'Insert', @Rows = @@rowcount, @Text = 'Not supported tables with disabled indexes'; + END + ELSE + BEGIN + DELETE @ResourceTypes; + INSERT INTO @ResourceTypes + SELECT CONVERT (SMALLINT, substring(name, charindex('_', name) + 1, 6)) AS ResourceTypeId + FROM sys.sysobjects + WHERE name LIKE @Tbl + '[_]%'; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@ResourceTypes', @Action = 'Insert', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @ResourceTypes) + BEGIN + SET @ResourceTypeId = (SELECT TOP 1 ResourceTypeId + FROM @ResourceTypes + ORDER BY ResourceTypeId); + SET @TblInt = @Tbl + '_' + CONVERT (VARCHAR, @ResourceTypeId); + SET @Pages = (SELECT dpages + FROM sysindexes + WHERE id = object_id(@TblInt) + AND indid IN (0, 1)); + DELETE @Indexes; + INSERT INTO @Indexes + SELECT name, + index_id + FROM sys.indexes + WHERE object_id = object_id(@Tbl) + AND (index_id > 1 + AND @RebuildClustered = 0 + OR index_id = 1 + AND @RebuildClustered = 1); + SET @IndexesCnt = 0; + WHILE EXISTS (SELECT * + FROM @Indexes) + BEGIN + SELECT TOP 1 @Ind = Ind, + @IndId = IndId + FROM @Indexes + ORDER BY Ind; + IF @IndId = 1 + BEGIN + SET @Txt = 'ALTER INDEX ' + @Ind + ' ON dbo.' + @TblInt + ' REBUILD' + CASE WHEN (SELECT PropertyValue + FROM dbo.IndexProperties + WHERE TableName = @Tbl + AND IndexName = @Ind) = 'PAGE' THEN ' PARTITION = ALL WITH (DATA_COMPRESSION = PAGE)' ELSE '' END; + INSERT INTO @Commands + SELECT @TblInt, + @Ind, + @Txt, + @Pages; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Add command', @Rows = @@rowcount, @Text = @Txt; + END + ELSE + IF NOT EXISTS (SELECT * + FROM sys.indexes + WHERE object_id = object_id(@TblInt) + AND name = @Ind) + BEGIN + EXECUTE dbo.GetIndexCommands @Tbl = @Tbl, @Ind = @Ind, @AddPartClause = 0, @IncludeClustered = 0, @Txt = @Txt OUTPUT; + SET @Txt = replace(@Txt, '[' + @Tbl + ']', @TblInt); + IF @Txt IS NOT NULL + BEGIN + SET @IndexesCnt = @IndexesCnt + 1; + INSERT INTO @Commands + SELECT @TblInt, + @Ind, + @Txt, + @Pages; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Add command', @Rows = @@rowcount, @Text = @Txt; + END + END + DELETE @Indexes + WHERE Ind = @Ind; + END + IF @IndexesCnt > 1 + BEGIN + INSERT INTO @Commands + SELECT @TblInt, + 'UPDATE STAT', + 'UPDATE STATISTICS dbo.' + @TblInt, + @Pages; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Add command', @Rows = @@rowcount, @Text = 'Add stats update'; + END + DELETE @ResourceTypes + WHERE ResourceTypeId = @ResourceTypeId; + END + END + DELETE @Tables + WHERE name = @Tbl; + END + SELECT Tbl, + Ind, + Txt + FROM @Commands + ORDER BY Pages DESC, Tbl, CASE WHEN Txt LIKE 'UPDATE STAT%' THEN 0 ELSE 1 END; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Commands', @Action = 'Select', @Rows = @@rowcount; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetExportJobByHash +@hash VARCHAR (64) +AS +SET NOCOUNT ON; +SELECT TOP (1) RawJobRecord, + JobVersion +FROM dbo.ExportJob +WHERE Hash = @hash + AND (Status = 'Queued' + OR Status = 'Running') +ORDER BY HeartbeatDateTime ASC; + +GO +CREATE PROCEDURE dbo.GetExportJobById +@id VARCHAR (64) +AS +SET NOCOUNT ON; +SELECT RawJobRecord, + JobVersion +FROM dbo.ExportJob +WHERE Id = @id; + +GO +CREATE PROCEDURE [dbo].[GetImportProcessingTaskResult] +@queueId VARCHAR (64), @importTaskId VARCHAR (64) +AS +SET NOCOUNT ON; +SELECT Result +FROM [dbo].[TaskInfo] WITH (INDEX (IX_QueueId_ParentTaskId)) +WHERE ParentTaskId = @importTaskId + AND TaskTypeId = 1 + AND Status = 3; + +GO +CREATE PROCEDURE dbo.GetIndexCommands +@Tbl VARCHAR (100), @Ind VARCHAR (200), @AddPartClause BIT, @IncludeClustered BIT, @Txt VARCHAR (MAX)=NULL OUTPUT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetIndexCommands', @Mode AS VARCHAR (200) = 'Tbl=' + isnull(@Tbl, 'NULL') + ' Ind=' + isnull(@Ind, 'NULL'), @st AS DATETIME = getUTCdate(); +DECLARE @Indexes TABLE ( + Ind VARCHAR (200) PRIMARY KEY, + Txt VARCHAR (MAX)); +BEGIN TRY + IF @Tbl IS NULL + RAISERROR ('@Tbl IS NULL', 18, 127); + INSERT INTO @Indexes + SELECT Ind, + CASE WHEN is_primary_key = 1 THEN 'ALTER TABLE dbo.[' + Tbl + '] ADD PRIMARY KEY ' + CASE WHEN type = 1 THEN ' CLUSTERED' ELSE '' END ELSE 'CREATE' + CASE WHEN is_unique = 1 THEN ' UNIQUE' ELSE '' END + CASE WHEN type = 1 THEN ' CLUSTERED' ELSE '' END + ' INDEX ' + Ind + ' ON dbo.[' + Tbl + ']' END + ' (' + KeyCols + ')' + IncClause + CASE WHEN filter_def IS NOT NULL THEN ' WHERE ' + filter_def ELSE '' END + CASE WHEN data_comp IS NOT NULL THEN ' WITH (DATA_COMPRESSION = ' + data_comp + ')' ELSE '' END + CASE WHEN @AddPartClause = 1 THEN PartClause ELSE '' END + FROM (SELECT O.Name AS Tbl, + I.Name AS Ind, + isnull((SELECT TOP 1 CASE WHEN data_compression_desc = 'PAGE' THEN 'PAGE' END + FROM sys.partitions AS P + WHERE P.object_id = I.object_id + AND I.index_id = P.index_id), (SELECT NULLIF (PropertyValue, 'NONE') + FROM dbo.IndexProperties + WHERE TableName = O.Name + AND IndexName = I.Name + AND PropertyName = 'DATA_COMPRESSION')) AS data_comp, + replace(replace(replace(replace(I.filter_definition, '[', ''), ']', ''), '(', ''), ')', '') AS filter_def, + I.is_unique, + I.is_primary_key, + I.type, + KeyCols, + CASE WHEN IncCols IS NOT NULL THEN ' INCLUDE (' + IncCols + ')' ELSE '' END AS IncClause, + CASE WHEN EXISTS (SELECT * + FROM sys.partition_schemes AS S + WHERE S.data_space_id = I.data_space_id + AND name = 'PartitionScheme_ResourceTypeId') THEN ' ON PartitionScheme_ResourceTypeId (ResourceTypeId)' ELSE '' END AS PartClause + FROM sys.indexes AS I + INNER JOIN + sys.objects AS O + ON O.object_id = I.object_id CROSS APPLY (SELECT string_agg(CASE WHEN IC.key_ordinal > 0 + AND IC.is_included_column = 0 THEN C.name END, ',') WITHIN GROUP (ORDER BY key_ordinal) AS KeyCols, + string_agg(CASE WHEN IC.is_included_column = 1 THEN C.name END, ',') WITHIN GROUP (ORDER BY key_ordinal) AS IncCols + FROM sys.index_columns AS IC + INNER JOIN + sys.columns AS C + ON C.object_id = IC.object_id + AND C.column_id = IC.column_id + WHERE IC.object_id = I.object_id + AND IC.index_id = I.index_id + GROUP BY IC.object_id, IC.index_id) AS IC + WHERE O.name = @Tbl + AND (@Ind IS NULL + OR I.name = @Ind) + AND (@IncludeClustered = 1 + OR index_id > 1)) AS A; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Insert', @Rows = @@rowcount; + IF @Ind IS NULL + SELECT Ind, + Txt + FROM @Indexes; + ELSE + SET @Txt = (SELECT Txt + FROM @Indexes); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Text = @Txt; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetJobs +@QueueType TINYINT, @JobId BIGINT=NULL, @JobIds BigintList READONLY, @GroupId BIGINT=NULL, @ReturnDefinition BIT=1 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetJobs', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' J=' + isnull(CONVERT (VARCHAR, @JobId), 'NULL') + ' G=' + isnull(CONVERT (VARCHAR, @GroupId), 'NULL'), @st AS DATETIME = getUTCdate(), @PartitionId AS TINYINT = @JobId % 16; +BEGIN TRY + IF @JobId IS NULL + AND @GroupId IS NULL + AND NOT EXISTS (SELECT * + FROM @JobIds) + RAISERROR ('@JobId = NULL and @GroupId = NULL and @JobIds is empty', 18, 127); + IF @JobId IS NOT NULL + SELECT GroupId, + JobId, + CASE WHEN @ReturnDefinition = 1 THEN Definition ELSE NULL END AS Definition, + Version, + Status, + Priority, + Data, + Result, + CreateDate, + StartDate, + EndDate, + HeartbeatDate, + CancelRequested + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = isnull(@JobId, -1) + AND Status <> 5; + ELSE + IF @GroupId IS NOT NULL + SELECT GroupId, + JobId, + CASE WHEN @ReturnDefinition = 1 THEN Definition ELSE NULL END AS Definition, + Version, + Status, + Priority, + Data, + Result, + CreateDate, + StartDate, + EndDate, + HeartbeatDate, + CancelRequested + FROM dbo.JobQueue WITH (INDEX (IX_QueueType_GroupId)) + WHERE QueueType = @QueueType + AND GroupId = isnull(@GroupId, -1) + AND Status <> 5; + ELSE + SELECT GroupId, + JobId, + CASE WHEN @ReturnDefinition = 1 THEN Definition ELSE NULL END AS Definition, + Version, + Status, + Priority, + Data, + Result, + CreateDate, + StartDate, + EndDate, + HeartbeatDate, + CancelRequested + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND JobId IN (SELECT Id + FROM @JobIds) + AND PartitionId = JobId % 16 + AND Status <> 5; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetNextTask_3 +@queueId VARCHAR (64), @taskHeartbeatTimeoutThresholdInSeconds INT=600 +AS +SET NOCOUNT ON; +DECLARE @lock AS VARCHAR (200) = 'GetNextTask_Q=' + @queueId, @taskId AS VARCHAR (64) = NULL, @expirationDateTime AS DATETIME2 (7), @startDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +SET @expirationDateTime = DATEADD(second, -@taskHeartbeatTimeoutThresholdInSeconds, @startDateTime); +BEGIN TRY + BEGIN TRANSACTION; + EXECUTE sp_getapplock @lock, 'Exclusive'; + UPDATE T + SET Status = 2, + StartDateTime = @startDateTime, + HeartbeatDateTime = @startDateTime, + Worker = host_name(), + RunId = NEWID(), + @taskId = T.TaskId + FROM dbo.TaskInfo AS T WITH (PAGLOCK) + INNER JOIN + (SELECT TOP 1 TaskId + FROM dbo.TaskInfo WITH (INDEX (IX_QueueId_Status)) + WHERE QueueId = @queueId + AND Status = 1 + ORDER BY TaskId) AS S + ON T.QueueId = @queueId + AND T.TaskId = S.TaskId; + IF @taskId IS NULL + UPDATE T + SET StartDateTime = @startDateTime, + HeartbeatDateTime = @startDateTime, + Worker = host_name(), + RunId = NEWID(), + @taskId = T.TaskId, + RestartInfo = ISNULL(RestartInfo, '') + ' Prev: Worker=' + Worker + ' Start=' + CONVERT (VARCHAR, @startDateTime, 121) + FROM dbo.TaskInfo AS T WITH (PAGLOCK) + INNER JOIN + (SELECT TOP 1 TaskId + FROM dbo.TaskInfo WITH (INDEX (IX_QueueId_Status)) + WHERE QueueId = @queueId + AND Status = 2 + AND HeartbeatDateTime <= @expirationDateTime + ORDER BY TaskId) AS S + ON T.QueueId = @queueId + AND T.TaskId = S.TaskId; + COMMIT TRANSACTION; + EXECUTE dbo.GetTaskDetails @TaskId = @taskId; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK TRANSACTION THROW; +END CATCH + +GO +CREATE OR ALTER PROCEDURE dbo.GetNonCompletedJobCountOfSpecificQueueType +@queueType TINYINT +AS +BEGIN + SET NOCOUNT ON; + SELECT COUNT(*) + FROM dbo.JobQueue + WHERE QueueType = @queueType + AND (Status = 0 + OR Status = 1); +END + +GO +CREATE PROCEDURE dbo.GetPartitionedTables +@IncludeNotDisabled BIT, @IncludeNotSupported BIT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetPartitionedTables', @Mode AS VARCHAR (200) = 'PS=PartitionScheme_ResourceTypeId D=' + isnull(CONVERT (VARCHAR, @IncludeNotDisabled), 'NULL') + ' S=' + isnull(CONVERT (VARCHAR, @IncludeNotSupported), 'NULL'), @st AS DATETIME = getUTCdate(); +DECLARE @NotSupportedTables TABLE ( + id INT PRIMARY KEY); +BEGIN TRY + INSERT INTO @NotSupportedTables + SELECT DISTINCT O.object_id + FROM sys.indexes AS I + INNER JOIN + sys.objects AS O + ON O.object_id = I.object_id + WHERE O.type = 'u' + AND EXISTS (SELECT * + FROM sys.partition_schemes AS PS + WHERE PS.data_space_id = I.data_space_id + AND name = 'PartitionScheme_ResourceTypeId') + AND (NOT EXISTS (SELECT * + FROM sys.index_columns AS IC + INNER JOIN + sys.columns AS C + ON C.object_id = IC.object_id + AND C.column_id = IC.column_id + WHERE IC.object_id = I.object_id + AND IC.index_id = I.index_id + AND IC.key_ordinal > 0 + AND IC.is_included_column = 0 + AND C.name = 'ResourceTypeId') + OR EXISTS (SELECT * + FROM sys.indexes AS NSI + WHERE NSI.object_id = O.object_id + AND NOT EXISTS (SELECT * + FROM sys.partition_schemes AS PS + WHERE PS.data_space_id = NSI.data_space_id + AND name = 'PartitionScheme_ResourceTypeId'))); + SELECT CONVERT (VARCHAR (100), O.name), + CONVERT (BIT, CASE WHEN EXISTS (SELECT * + FROM @NotSupportedTables AS NSI + WHERE NSI.id = O.object_id) THEN 0 ELSE 1 END) + FROM sys.indexes AS I + INNER JOIN + sys.objects AS O + ON O.object_id = I.object_id + WHERE O.type = 'u' + AND I.index_id IN (0, 1) + AND EXISTS (SELECT * + FROM sys.partition_schemes AS PS + WHERE PS.data_space_id = I.data_space_id + AND name = 'PartitionScheme_ResourceTypeId') + AND EXISTS (SELECT * + FROM sys.index_columns AS IC + INNER JOIN + sys.columns AS C + ON C.object_id = I.object_id + AND C.column_id = IC.column_id + AND IC.is_included_column = 0 + AND C.name = 'ResourceTypeId') + AND (@IncludeNotSupported = 1 + OR NOT EXISTS (SELECT * + FROM @NotSupportedTables AS NSI + WHERE NSI.id = O.object_id)) + AND (@IncludeNotDisabled = 1 + OR EXISTS (SELECT * + FROM sys.indexes AS D + WHERE D.object_id = O.object_id + AND D.is_disabled = 1)) + ORDER BY 1; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetReindexJobById +@id VARCHAR (64) +AS +SET NOCOUNT ON; +SELECT RawJobRecord, + JobVersion +FROM dbo.ReindexJob +WHERE Id = @id; + +GO +CREATE PROCEDURE dbo.GetResources +@ResourceKeys dbo.ResourceKeyList READONLY +AS +SET NOCOUNT ON; +DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = 'GetResources', @InputRows AS INT, @DummyTop AS BIGINT = 9223372036854775807, @NotNullVersionExists AS BIT, @NullVersionExists AS BIT, @MinRT AS SMALLINT, @MaxRT AS SMALLINT; +SELECT @MinRT = min(ResourceTypeId), + @MaxRT = max(ResourceTypeId), + @InputRows = count(*), + @NotNullVersionExists = max(CASE WHEN Version IS NOT NULL THEN 1 ELSE 0 END), + @NullVersionExists = max(CASE WHEN Version IS NULL THEN 1 ELSE 0 END) +FROM @ResourceKeys; +DECLARE @Mode AS VARCHAR (100) = 'RT=[' + CONVERT (VARCHAR, @MinRT) + ',' + CONVERT (VARCHAR, @MaxRT) + '] Cnt=' + CONVERT (VARCHAR, @InputRows) + ' NNVE=' + CONVERT (VARCHAR, @NotNullVersionExists) + ' NVE=' + CONVERT (VARCHAR, @NullVersionExists); +BEGIN TRY + IF @NotNullVersionExists = 1 + IF @NullVersionExists = 0 + SELECT B.ResourceTypeId, + B.ResourceId, + ResourceSurrogateId, + B.Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceKeys) AS A + INNER JOIN + dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.Version = A.Version + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + ELSE + SELECT * + FROM (SELECT B.ResourceTypeId, + B.ResourceId, + ResourceSurrogateId, + B.Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceKeys + WHERE Version IS NOT NULL) AS A + INNER JOIN + dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.Version = A.Version + UNION ALL + SELECT B.ResourceTypeId, + B.ResourceId, + ResourceSurrogateId, + B.Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceKeys + WHERE Version IS NULL) AS A + INNER JOIN + dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId)) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + WHERE IsHistory = 0) AS A + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + ELSE + SELECT B.ResourceTypeId, + B.ResourceId, + ResourceSurrogateId, + B.Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceKeys) AS A + INNER JOIN + dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId)) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + WHERE IsHistory = 0 + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetResourcesByTransactionId +@TransactionId BIGINT, @IncludeHistory BIT=0, @ReturnResourceKeysOnly BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'T=' + CONVERT (VARCHAR, @TransactionId) + ' H=' + CONVERT (VARCHAR, @IncludeHistory), @st AS DATETIME = getUTCdate(), @DummyTop AS BIGINT = 9223372036854775807, @TypeId AS SMALLINT; +BEGIN TRY + DECLARE @Types TABLE ( + TypeId SMALLINT PRIMARY KEY, + Name VARCHAR (100)); + INSERT INTO @Types + EXECUTE dbo.GetUsedResourceTypes ; + DECLARE @Keys TABLE ( + TypeId SMALLINT, + SurrogateId BIGINT PRIMARY KEY (TypeId, SurrogateId)); + WHILE EXISTS (SELECT * + FROM @Types) + BEGIN + SET @TypeId = (SELECT TOP 1 TypeId + FROM @Types + ORDER BY TypeId); + INSERT INTO @Keys + SELECT @TypeId, + ResourceSurrogateId + FROM dbo.Resource + WHERE ResourceTypeId = @TypeId + AND TransactionId = @TransactionId; + DELETE @Types + WHERE TypeId = @TypeId; + END + IF @ReturnResourceKeysOnly = 0 + SELECT ResourceTypeId, + ResourceId, + ResourceSurrogateId, + Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash, + RequestMethod + FROM (SELECT TOP (@DummyTop) * + FROM @Keys) AS A + INNER JOIN + dbo.Resource AS B + ON ResourceTypeId = TypeId + AND ResourceSurrogateId = SurrogateId + WHERE IsHistory = 0 + OR @IncludeHistory = 1 + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + ELSE + SELECT ResourceTypeId, + ResourceId, + ResourceSurrogateId, + Version, + IsDeleted + FROM (SELECT TOP (@DummyTop) * + FROM @Keys) AS A + INNER JOIN + dbo.Resource AS B + ON ResourceTypeId = TypeId + AND ResourceSurrogateId = SurrogateId + WHERE IsHistory = 0 + OR @IncludeHistory = 1 + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetResourcesByTypeAndSurrogateIdRange +@ResourceTypeId SMALLINT, @StartId BIGINT, @EndId BIGINT, @GlobalStartId BIGINT=NULL, @GlobalEndId BIGINT=NULL, @IncludeHistory BIT=0, @IncludeDeleted BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetResourcesByTypeAndSurrogateIdRange', @Mode AS VARCHAR (100) = 'RT=' + isnull(CONVERT (VARCHAR, @ResourceTypeId), 'NULL') + ' S=' + isnull(CONVERT (VARCHAR, @StartId), 'NULL') + ' E=' + isnull(CONVERT (VARCHAR, @EndId), 'NULL') + ' GS=' + isnull(CONVERT (VARCHAR, @GlobalStartId), 'NULL') + ' GE=' + isnull(CONVERT (VARCHAR, @GlobalEndId), 'NULL') + ' HI=' + isnull(CONVERT (VARCHAR, @IncludeHistory), 'NULL') + ' DE' + isnull(CONVERT (VARCHAR, @IncludeDeleted), 'NULL'), @st AS DATETIME = getUTCdate(); +BEGIN TRY + DECLARE @ResourceIds TABLE ( + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS, + ResourceSurrogateId BIGINT , + RowId INT , + PRIMARY KEY (ResourceId, RowId)); + IF @GlobalStartId IS NULL + SET @GlobalStartId = 0; + IF @GlobalEndId IS NOT NULL + INSERT INTO @ResourceIds + SELECT ResourceId, + ResourceSurrogateId, + row_number() OVER (PARTITION BY ResourceId ORDER BY ResourceSurrogateId DESC) AS RowId + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceId IN (SELECT DISTINCT ResourceId + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + AND IsHistory = 1 + AND (IsDeleted = 0 + OR @IncludeDeleted = 1)) + AND ResourceSurrogateId BETWEEN @GlobalStartId AND @GlobalEndId; + IF EXISTS (SELECT * + FROM @ResourceIds) + BEGIN + DECLARE @SurrogateIdMap TABLE ( + MaxSurrogateId BIGINT PRIMARY KEY); + INSERT INTO @SurrogateIdMap + SELECT A.ResourceSurrogateId AS MaxSurrogateId + FROM (SELECT * + FROM @ResourceIds + WHERE RowId = 1 + AND ResourceSurrogateId BETWEEN @StartId AND @EndId) AS A; + SELECT @ResourceTypeId, + CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.ResourceId ELSE A.ResourceId END, + CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.Version ELSE A.Version END, + CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.IsDeleted ELSE A.IsDeleted END, + isnull(C.ResourceSurrogateId, A.ResourceSurrogateId), + CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.RequestMethod ELSE A.RequestMethod END, + CONVERT (BIT, 1) AS IsMatch, + CONVERT (BIT, 0) AS IsPartial, + CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.IsRawResourceMetaSet ELSE A.IsRawResourceMetaSet END, + CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.SearchParamHash ELSE A.SearchParamHash END, + CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.RawResource ELSE A.RawResource END + FROM dbo.Resource AS A + LEFT OUTER JOIN + @SurrogateIdMap AS B + ON B.MaxSurrogateId = A.ResourceSurrogateId + LEFT OUTER JOIN + dbo.Resource AS C + ON C.ResourceTypeId = @ResourceTypeId + AND C.ResourceSurrogateId = MaxSurrogateId + WHERE A.ResourceTypeId = @ResourceTypeId + AND A.ResourceSurrogateId BETWEEN @StartId AND @EndId + AND (A.IsHistory = 0 + OR MaxSurrogateId IS NOT NULL + OR @IncludeHistory = 1) + AND (A.IsDeleted = 0 + OR @IncludeDeleted = 1); + END + ELSE + SELECT ResourceTypeId, + ResourceId, + Version, + IsDeleted, + ResourceSurrogateId, + RequestMethod, + CONVERT (BIT, 1) AS IsMatch, + CONVERT (BIT, 0) AS IsPartial, + IsRawResourceMetaSet, + SearchParamHash, + RawResource + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + AND (IsHistory = 0 + OR @IncludeHistory = 1) + AND (IsDeleted = 0 + OR @IncludeDeleted = 1); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetResourceSurrogateIdRanges +@ResourceTypeId SMALLINT, @StartId BIGINT, @EndId BIGINT, @RangeSize INT, @NumberOfRanges INT=100, @Up BIT=1 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetResourceSurrogateIdRanges', @Mode AS VARCHAR (100) = 'RT=' + isnull(CONVERT (VARCHAR, @ResourceTypeId), 'NULL') + ' S=' + isnull(CONVERT (VARCHAR, @StartId), 'NULL') + ' E=' + isnull(CONVERT (VARCHAR, @EndId), 'NULL') + ' R=' + isnull(CONVERT (VARCHAR, @RangeSize), 'NULL') + ' UP=' + isnull(CONVERT (VARCHAR, @Up), 'NULL'), @st AS DATETIME = getUTCdate(); +BEGIN TRY + IF @Up = 1 + SELECT RangeId, + min(ResourceSurrogateId), + max(ResourceSurrogateId), + count(*) + FROM (SELECT isnull(CONVERT (INT, (row_number() OVER (ORDER BY ResourceSurrogateId) - 1) / @RangeSize), 0) AS RangeId, + ResourceSurrogateId + FROM (SELECT TOP (@RangeSize * @NumberOfRanges) ResourceSurrogateId + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId >= @StartId + AND ResourceSurrogateId <= @EndId + ORDER BY ResourceSurrogateId) AS A) AS A + GROUP BY RangeId + OPTION (MAXDOP 1); + ELSE + SELECT RangeId, + min(ResourceSurrogateId), + max(ResourceSurrogateId), + count(*) + FROM (SELECT isnull(CONVERT (INT, (row_number() OVER (ORDER BY ResourceSurrogateId) - 1) / @RangeSize), 0) AS RangeId, + ResourceSurrogateId + FROM (SELECT TOP (@RangeSize * @NumberOfRanges) ResourceSurrogateId + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId >= @StartId + AND ResourceSurrogateId <= @EndId + ORDER BY ResourceSurrogateId DESC) AS A) AS A + GROUP BY RangeId + OPTION (MAXDOP 1); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetResourceVersions +@ResourceDateKeys dbo.ResourceDateKeyList READONLY +AS +SET NOCOUNT ON; +DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = 'GetResourceVersions', @Mode AS VARCHAR (100) = 'Rows=' + CONVERT (VARCHAR, (SELECT count(*) + FROM @ResourceDateKeys)), @DummyTop AS BIGINT = 9223372036854775807; +BEGIN TRY + SELECT A.ResourceTypeId, + A.ResourceId, + A.ResourceSurrogateId, + CASE WHEN EXISTS (SELECT * + FROM dbo.Resource AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.ResourceSurrogateId = A.ResourceSurrogateId) THEN 0 WHEN isnull(U.Version, 1) - isnull(L.Version, 0) > 1 THEN isnull(U.Version, 1) - 1 ELSE 0 END AS Version + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceDateKeys) AS A OUTER APPLY (SELECT TOP 1 * + FROM dbo.Resource AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.ResourceSurrogateId < A.ResourceSurrogateId + ORDER BY B.ResourceSurrogateId DESC) AS L OUTER APPLY (SELECT TOP 1 * + FROM dbo.Resource AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.ResourceSurrogateId > A.ResourceSurrogateId + ORDER BY B.ResourceSurrogateId) AS U + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetSearchParamStatuses +AS +SET NOCOUNT ON; +SELECT SearchParamId, + Uri, + Status, + LastUpdated, + IsPartiallySupported +FROM dbo.SearchParam; + +GO +CREATE PROCEDURE [dbo].[GetTaskDetails] +@taskId VARCHAR (64) +AS +SET NOCOUNT ON; +SELECT TaskId, + QueueId, + Status, + TaskTypeId, + RunId, + IsCanceled, + RetryCount, + MaxRetryCount, + HeartbeatDateTime, + InputData, + TaskContext, + Result, + ParentTaskId +FROM [dbo].[TaskInfo] +WHERE TaskId = @taskId; + +GO +CREATE PROCEDURE dbo.GetTransactions +@StartNotInclusiveTranId BIGINT, @EndInclusiveTranId BIGINT, @EndDate DATETIME=NULL +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'ST=' + CONVERT (VARCHAR, @StartNotInclusiveTranId) + ' ET=' + CONVERT (VARCHAR, @EndInclusiveTranId) + ' ED=' + isnull(CONVERT (VARCHAR, @EndDate, 121), 'NULL'), @st AS DATETIME = getUTCdate(); +IF @EndDate IS NULL + SET @EndDate = getUTCdate(); +SELECT SurrogateIdRangeFirstValue, + VisibleDate, + InvisibleHistoryRemovedDate +FROM dbo.Transactions +WHERE SurrogateIdRangeFirstValue > @StartNotInclusiveTranId + AND SurrogateIdRangeFirstValue <= @EndInclusiveTranId + AND EndDate <= @EndDate +ORDER BY SurrogateIdRangeFirstValue; +EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; + +GO +CREATE PROCEDURE dbo.GetUsedResourceTypes +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetUsedResourceTypes', @Mode AS VARCHAR (100) = '', @st AS DATETIME = getUTCdate(); +BEGIN TRY + SELECT ResourceTypeId, + Name + FROM dbo.ResourceType AS A + WHERE EXISTS (SELECT * + FROM dbo.Resource AS B + WHERE B.ResourceTypeId = A.ResourceTypeId); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.HardDeleteResource +@ResourceTypeId SMALLINT, @ResourceId VARCHAR (64), @KeepCurrentVersion BIT, @IsResourceChangeCaptureEnabled BIT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (200) = 'RT=' + CONVERT (VARCHAR, @ResourceTypeId) + ' R=' + @ResourceId + ' V=' + CONVERT (VARCHAR, @KeepCurrentVersion) + ' CC=' + CONVERT (VARCHAR, @IsResourceChangeCaptureEnabled), @st AS DATETIME = getUTCdate(), @TransactionId AS BIGINT; +BEGIN TRY + IF @IsResourceChangeCaptureEnabled = 1 + EXECUTE dbo.MergeResourcesBeginTransaction @Count = 1, @TransactionId = @TransactionId OUTPUT; + IF @KeepCurrentVersion = 0 + BEGIN TRANSACTION; + DECLARE @SurrogateIds TABLE ( + ResourceSurrogateId BIGINT NOT NULL); + IF @IsResourceChangeCaptureEnabled = 1 + AND EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = 'InvisibleHistory.IsEnabled' + AND Number = 1) + UPDATE dbo.Resource + SET IsHistory = 1, + RawResource = 0xF, + SearchParamHash = NULL, + HistoryTransactionId = @TransactionId + OUTPUT deleted.ResourceSurrogateId INTO @SurrogateIds + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceId = @ResourceId + AND (@KeepCurrentVersion = 0 + OR IsHistory = 1) + AND RawResource <> 0xF; + ELSE + DELETE dbo.Resource + OUTPUT deleted.ResourceSurrogateId INTO @SurrogateIds + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceId = @ResourceId + AND (@KeepCurrentVersion = 0 + OR IsHistory = 1) + AND RawResource <> 0xF; + IF @KeepCurrentVersion = 0 + BEGIN + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.ResourceWriteClaim AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.ReferenceSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenText AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.StringSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.UriSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.NumberSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.QuantitySearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.DateTimeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.ReferenceTokenCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenTokenCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenDateTimeCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenQuantityCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenStringCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenNumberNumberCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + END + IF @@trancount > 0 + COMMIT TRANSACTION; + IF @IsResourceChangeCaptureEnabled = 1 + EXECUTE dbo.MergeResourcesCommitTransaction @TransactionId; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.InitDefrag +@QueueType TINYINT, @GroupId BIGINT, @DefragItems INT=NULL OUTPUT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'InitDefrag', @st AS DATETIME = getUTCdate(), @ObjectId AS INT, @msg AS VARCHAR (1000), @Rows AS INT, @MinFragPct AS INT = isnull((SELECT Number + FROM dbo.Parameters + WHERE Id = 'Defrag.MinFragPct'), 10), @MinSizeGB AS FLOAT = isnull((SELECT Number + FROM dbo.Parameters + WHERE Id = 'Defrag.MinSizeGB'), 0.1), @DefinitionsSorted AS StringList; +DECLARE @Mode AS VARCHAR (200) = 'G=' + CONVERT (VARCHAR, @GroupId) + ' MF=' + CONVERT (VARCHAR, @MinFragPct) + ' MS=' + CONVERT (VARCHAR, @MinSizeGB); +DECLARE @Definitions AS TABLE ( + Def VARCHAR (900) PRIMARY KEY, + FragGB FLOAT ); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + SELECT * + INTO #filter + FROM (SELECT object_id, + sum(reserved_page_count * 8.0 / 1024 / 1024) AS ReservedGB + FROM sys.dm_db_partition_stats AS A + WHERE object_id IN (SELECT object_id + FROM sys.objects + WHERE type = 'U' + AND name NOT IN ('EventLog')) + GROUP BY object_id) AS A + WHERE ReservedGB > @MinSizeGB; + WHILE EXISTS (SELECT * + FROM #filter) + BEGIN + SET @ObjectId = (SELECT TOP 1 object_id + FROM #filter + ORDER BY ReservedGB DESC); + INSERT INTO @Definitions + SELECT object_name(@ObjectId) + ';' + I.name + ';' + CONVERT (VARCHAR, partition_number) + ';' + CONVERT (VARCHAR, CASE WHEN EXISTS (SELECT * + FROM sys.partition_schemes AS PS + WHERE PS.data_space_id = I.data_space_id) THEN 1 ELSE 0 END) + ';' + CONVERT (VARCHAR, (SELECT sum(reserved_page_count) + FROM sys.dm_db_partition_stats AS S + WHERE S.object_id = A.object_id + AND S.index_id = A.index_id + AND S.partition_number = A.partition_number) * 8.0 / 1024 / 1024), + FragGB + FROM (SELECT object_id, + index_id, + partition_number, + A.avg_fragmentation_in_percent * A.page_count * 8.0 / 1024 / 1024 / 100 AS FragGB + FROM sys.dm_db_index_physical_stats(db_id(), @ObjectId, NULL, NULL, 'LIMITED') AS A + WHERE index_id > 0 + AND avg_fragmentation_in_percent >= @MinFragPct + AND A.page_count > 500) AS A + INNER JOIN + sys.indexes AS I + ON I.object_id = A.object_id + AND I.index_id = A.index_id; + SET @Rows = @@rowcount; + SET @msg = object_name(@ObjectId); + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'Run', @Mode = @Mode, @Target = '@Definitions', @Action = 'Insert', @Rows = @Rows, @Text = @msg; + DELETE #filter + WHERE object_id = @ObjectId; + END + INSERT INTO @DefinitionsSorted + SELECT Def + ';' + CONVERT (VARCHAR, FragGB) + FROM @Definitions + ORDER BY FragGB DESC; + SET @DefragItems = @@rowcount; + IF @DefragItems > 0 + EXECUTE dbo.EnqueueJobs @QueueType = @QueueType, @Definitions = @DefinitionsSorted, @GroupId = @GroupId, @ForceOneActiveJobGroup = 1; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.InitializeIndexProperties +AS +SET NOCOUNT ON; +INSERT INTO dbo.IndexProperties (TableName, IndexName, PropertyName, PropertyValue) +SELECT Tbl, + Ind, + 'DATA_COMPRESSION', + isnull(data_comp, 'NONE') +FROM (SELECT O.Name AS Tbl, + I.Name AS Ind, + (SELECT TOP 1 CASE WHEN data_compression_desc = 'PAGE' THEN 'PAGE' END + FROM sys.partitions AS P + WHERE P.object_id = I.object_id + AND I.index_id = P.index_id) AS data_comp + FROM sys.indexes AS I + INNER JOIN + sys.objects AS O + ON O.object_id = I.object_id + WHERE O.type = 'u' + AND EXISTS (SELECT * + FROM sys.partition_schemes AS PS + WHERE PS.data_space_id = I.data_space_id + AND name = 'PartitionScheme_ResourceTypeId')) AS A +WHERE NOT EXISTS (SELECT * + FROM dbo.IndexProperties + WHERE TableName = Tbl + AND IndexName = Ind); + +GO +CREATE PROCEDURE dbo.LogEvent +@Process VARCHAR (100), @Status VARCHAR (10), @Mode VARCHAR (200)=NULL, @Action VARCHAR (20)=NULL, @Target VARCHAR (100)=NULL, @Rows BIGINT=NULL, @Start DATETIME=NULL, @Text NVARCHAR (3500)=NULL, @EventId BIGINT=NULL OUTPUT, @Retry INT=NULL +AS +SET NOCOUNT ON; +DECLARE @ErrorNumber AS INT = error_number(), @ErrorMessage AS VARCHAR (1000) = '', @TranCount AS INT = @@trancount, @DoWork AS BIT = 0, @NumberAdded AS BIT; +IF @ErrorNumber IS NOT NULL + OR @Status IN ('Warn', 'Error') + SET @DoWork = 1; +IF @DoWork = 0 + SET @DoWork = CASE WHEN EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = isnull(@Process, '') + AND Char = 'LogEvent') THEN 1 ELSE 0 END; +IF @DoWork = 0 + RETURN; +IF @ErrorNumber IS NOT NULL + SET @ErrorMessage = CASE WHEN @Retry IS NOT NULL THEN 'Retry ' + CONVERT (VARCHAR, @Retry) + ', ' ELSE '' END + 'Error ' + CONVERT (VARCHAR, error_number()) + ': ' + CONVERT (VARCHAR (1000), error_message()) + ', Level ' + CONVERT (VARCHAR, error_severity()) + ', State ' + CONVERT (VARCHAR, error_state()) + CASE WHEN error_procedure() IS NOT NULL THEN ', Procedure ' + error_procedure() ELSE '' END + ', Line ' + CONVERT (VARCHAR, error_line()); +IF @TranCount > 0 + AND @ErrorNumber IS NOT NULL + ROLLBACK; +IF databasepropertyex(db_name(), 'UpdateAbility') = 'READ_WRITE' + BEGIN + INSERT INTO dbo.EventLog (Process, Status, Mode, Action, Target, Rows, Milliseconds, EventDate, EventText, SPID, HostName) + SELECT @Process, + @Status, + @Mode, + @Action, + @Target, + @Rows, + datediff(millisecond, @Start, getUTCdate()), + getUTCdate() AS EventDate, + CASE WHEN @ErrorNumber IS NULL THEN @Text ELSE @ErrorMessage + CASE WHEN isnull(@Text, '') <> '' THEN '. ' + @Text ELSE '' END END AS Text, + @@SPID, + host_name() AS HostName; + SET @EventId = scope_identity(); + END +IF @TranCount > 0 + AND @ErrorNumber IS NOT NULL + BEGIN TRANSACTION; + +GO +CREATE PROCEDURE dbo.LogSchemaMigrationProgress +@message VARCHAR (MAX) +AS +INSERT INTO dbo.SchemaMigrationProgress (Message) +VALUES (@message); + +GO +CREATE PROCEDURE dbo.MergeResources +@AffectedRows INT=0 OUTPUT, @RaiseExceptionOnConflict BIT=1, @IsResourceChangeCaptureEnabled BIT=0, @TransactionId BIGINT=NULL, @SingleTransaction BIT=1, @Resources dbo.ResourceList READONLY, @ResourceWriteClaims dbo.ResourceWriteClaimList READONLY, @CompartmentAssignments dbo.CompartmentAssignmentList READONLY, @ReferenceSearchParams dbo.ReferenceSearchParamList READONLY, @TokenSearchParams dbo.TokenSearchParamList READONLY, @TokenTexts dbo.TokenTextList READONLY, @StringSearchParams dbo.StringSearchParamList READONLY, @UriSearchParams dbo.UriSearchParamList READONLY, @NumberSearchParams dbo.NumberSearchParamList READONLY, @QuantitySearchParams dbo.QuantitySearchParamList READONLY, @DateTimeSearchParms dbo.DateTimeSearchParamList READONLY, @ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY, @TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY, @TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY, @TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY, @TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY, @TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY +AS +SET NOCOUNT ON; +DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = object_name(@@procid), @DummyTop AS BIGINT = 9223372036854775807, @InitialTranCount AS INT = @@trancount, @IsRetry AS BIT = 0; +DECLARE @Mode AS VARCHAR (200) = isnull((SELECT 'RT=[' + CONVERT (VARCHAR, min(ResourceTypeId)) + ',' + CONVERT (VARCHAR, max(ResourceTypeId)) + '] Sur=[' + CONVERT (VARCHAR, min(ResourceSurrogateId)) + ',' + CONVERT (VARCHAR, max(ResourceSurrogateId)) + '] V=' + CONVERT (VARCHAR, max(Version)) + ' Rows=' + CONVERT (VARCHAR, count(*)) + FROM @Resources), 'Input=Empty'); +SET @Mode += ' E=' + CONVERT (VARCHAR, @RaiseExceptionOnConflict) + ' CC=' + CONVERT (VARCHAR, @IsResourceChangeCaptureEnabled) + ' IT=' + CONVERT (VARCHAR, @InitialTranCount) + ' T=' + isnull(CONVERT (VARCHAR, @TransactionId), 'NULL'); +SET @AffectedRows = 0; +BEGIN TRY + DECLARE @Existing AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + SurrogateId BIGINT NOT NULL PRIMARY KEY (ResourceTypeId, SurrogateId)); + DECLARE @ResourceInfos AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + SurrogateId BIGINT NOT NULL, + Version INT NOT NULL, + KeepHistory BIT NOT NULL, + PreviousVersion INT NULL, + PreviousSurrogateId BIGINT NULL PRIMARY KEY (ResourceTypeId, SurrogateId)); + DECLARE @PreviousSurrogateIds AS TABLE ( + TypeId SMALLINT NOT NULL, + SurrogateId BIGINT NOT NULL PRIMARY KEY (TypeId, SurrogateId), + KeepHistory BIT ); + IF @SingleTransaction = 0 + AND isnull((SELECT Number + FROM dbo.Parameters + WHERE Id = 'MergeResources.NoTransaction.IsEnabled'), 0) = 0 + SET @SingleTransaction = 1; + SET @Mode += ' ST=' + CONVERT (VARCHAR, @SingleTransaction); + IF @InitialTranCount = 0 + BEGIN + IF EXISTS (SELECT * + FROM @Resources AS A + INNER JOIN + dbo.Resource AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + WHERE B.IsHistory = 0) + BEGIN + BEGIN TRANSACTION; + INSERT INTO @Existing (ResourceTypeId, SurrogateId) + SELECT B.ResourceTypeId, + B.ResourceSurrogateId + FROM (SELECT TOP (@DummyTop) * + FROM @Resources) AS A + INNER JOIN + dbo.Resource AS B WITH (ROWLOCK, HOLDLOCK) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + WHERE B.IsHistory = 0 + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + IF @@rowcount > 0 + SET @IsRetry = 1; + IF @IsRetry = 0 + COMMIT TRANSACTION; + END + END + SET @Mode += ' R=' + CONVERT (VARCHAR, @IsRetry); + IF @SingleTransaction = 1 + AND @@trancount = 0 + BEGIN TRANSACTION; + IF @IsRetry = 0 + BEGIN + INSERT INTO @ResourceInfos (ResourceTypeId, SurrogateId, Version, KeepHistory, PreviousVersion, PreviousSurrogateId) + SELECT A.ResourceTypeId, + A.ResourceSurrogateId, + A.Version, + A.KeepHistory, + B.Version, + B.ResourceSurrogateId + FROM (SELECT TOP (@DummyTop) * + FROM @Resources + WHERE HasVersionToCompare = 1) AS A + LEFT OUTER JOIN + dbo.Resource AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.IsHistory = 0 + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + IF @RaiseExceptionOnConflict = 1 + AND EXISTS (SELECT * + FROM @ResourceInfos + WHERE PreviousVersion IS NOT NULL + AND Version <> PreviousVersion + 1) + THROW 50409, 'Resource has been recently updated or added, please compare the resource content in code for any duplicate updates', 1; + INSERT INTO @PreviousSurrogateIds + SELECT ResourceTypeId, + PreviousSurrogateId, + KeepHistory + FROM @ResourceInfos + WHERE PreviousSurrogateId IS NOT NULL; + IF @@rowcount > 0 + BEGIN + UPDATE dbo.Resource + SET IsHistory = 1 + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId + AND KeepHistory = 1); + SET @AffectedRows += @@rowcount; + IF @IsResourceChangeCaptureEnabled = 1 + AND EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = 'InvisibleHistory.IsEnabled' + AND Number = 1) + UPDATE dbo.Resource + SET IsHistory = 1, + RawResource = 0xF, + SearchParamHash = NULL, + HistoryTransactionId = @TransactionId + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId + AND KeepHistory = 0); + ELSE + DELETE dbo.Resource + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId + AND KeepHistory = 0); + SET @AffectedRows += @@rowcount; + DELETE dbo.ResourceWriteClaim + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.ReferenceSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenText + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.StringSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.UriSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.NumberSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.QuantitySearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.DateTimeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.ReferenceTokenCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenTokenCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenDateTimeCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenQuantityCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenStringCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenNumberNumberCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + END + INSERT INTO dbo.Resource (ResourceTypeId, ResourceId, Version, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, RawResource, IsRawResourceMetaSet, SearchParamHash, TransactionId) + SELECT ResourceTypeId, + ResourceId, + Version, + IsHistory, + ResourceSurrogateId, + IsDeleted, + RequestMethod, + RawResource, + IsRawResourceMetaSet, + SearchParamHash, + @TransactionId + FROM @Resources; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) + SELECT ResourceSurrogateId, + ClaimTypeId, + ClaimValue + FROM @ResourceWriteClaims; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri, + ReferenceResourceTypeId, + ReferenceResourceId, + ReferenceResourceVersion + FROM @ReferenceSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + Code, + CodeOverflow + FROM @TokenSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text + FROM @TokenTexts; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text, + TextOverflow, + IsMin, + IsMax + FROM @StringSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Uri + FROM @UriSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SingleValue, + LowValue, + HighValue + FROM @NumberSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + QuantityCodeId, + SingleValue, + LowValue, + HighValue + FROM @QuantitySearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + StartDateTime, + EndDateTime, + IsLongerThanADay, + IsMin, + IsMax + FROM @DateTimeSearchParms; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri1, + ReferenceResourceTypeId1, + ReferenceResourceId1, + ReferenceResourceVersion1, + SystemId2, + Code2, + CodeOverflow2 + FROM @ReferenceTokenCompositeSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SystemId2, + Code2, + CodeOverflow2 + FROM @TokenTokenCompositeSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + StartDateTime2, + EndDateTime2, + IsLongerThanADay2 + FROM @TokenDateTimeCompositeSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + SystemId2, + QuantityCodeId2, + LowValue2, + HighValue2 + FROM @TokenQuantityCompositeSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + Text2, + TextOverflow2 + FROM @TokenStringCompositeSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + LowValue2, + HighValue2, + SingleValue3, + LowValue3, + HighValue3, + HasRange + FROM @TokenNumberNumberCompositeSearchParams; + SET @AffectedRows += @@rowcount; + END + ELSE + BEGIN + INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) + SELECT ResourceSurrogateId, + ClaimTypeId, + ClaimValue + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceWriteClaims) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.ResourceWriteClaim AS C + WHERE C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri, + ReferenceResourceTypeId, + ReferenceResourceId, + ReferenceResourceVersion + FROM (SELECT TOP (@DummyTop) * + FROM @ReferenceSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.ReferenceSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + Code, + CodeOverflow + FROM (SELECT TOP (@DummyTop) * + FROM @TokenSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text + FROM (SELECT TOP (@DummyTop) * + FROM @TokenTexts) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text, + TextOverflow, + IsMin, + IsMax + FROM (SELECT TOP (@DummyTop) * + FROM @StringSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenText AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Uri + FROM (SELECT TOP (@DummyTop) * + FROM @UriSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.UriSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SingleValue, + LowValue, + HighValue + FROM (SELECT TOP (@DummyTop) * + FROM @NumberSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.NumberSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + QuantityCodeId, + SingleValue, + LowValue, + HighValue + FROM (SELECT TOP (@DummyTop) * + FROM @QuantitySearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.QuantitySearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + StartDateTime, + EndDateTime, + IsLongerThanADay, + IsMin, + IsMax + FROM (SELECT TOP (@DummyTop) * + FROM @DateTimeSearchParms) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri1, + ReferenceResourceTypeId1, + ReferenceResourceId1, + ReferenceResourceVersion1, + SystemId2, + Code2, + CodeOverflow2 + FROM (SELECT TOP (@DummyTop) * + FROM @ReferenceTokenCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.DateTimeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SystemId2, + Code2, + CodeOverflow2 + FROM (SELECT TOP (@DummyTop) * + FROM @TokenTokenCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenTokenCompositeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + StartDateTime2, + EndDateTime2, + IsLongerThanADay2 + FROM (SELECT TOP (@DummyTop) * + FROM @TokenDateTimeCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenDateTimeCompositeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + SystemId2, + QuantityCodeId2, + LowValue2, + HighValue2 + FROM (SELECT TOP (@DummyTop) * + FROM @TokenQuantityCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenQuantityCompositeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + Text2, + TextOverflow2 + FROM (SELECT TOP (@DummyTop) * + FROM @TokenStringCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenStringCompositeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + LowValue2, + HighValue2, + SingleValue3, + LowValue3, + HighValue3, + HasRange + FROM (SELECT TOP (@DummyTop) * + FROM @TokenNumberNumberCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenNumberNumberCompositeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + END + IF @IsResourceChangeCaptureEnabled = 1 + EXECUTE dbo.CaptureResourceIdsForChanges @Resources; + IF @TransactionId IS NOT NULL + EXECUTE dbo.MergeResourcesCommitTransaction @TransactionId; + IF @InitialTranCount = 0 + AND @@trancount > 0 + COMMIT TRANSACTION; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @AffectedRows; +END TRY +BEGIN CATCH + IF @InitialTranCount = 0 + AND @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + IF @RaiseExceptionOnConflict = 1 + AND error_number() IN (2601, 2627) + AND error_message() LIKE '%''dbo.Resource''%' + THROW 50409, 'Resource has been recently updated or added, please compare the resource content in code for any duplicate updates', 1; + ELSE + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesAdvanceTransactionVisibility +@AffectedRows INT=0 OUTPUT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = '', @st AS DATETIME = getUTCdate(), @msg AS VARCHAR (1000), @MaxTransactionId AS BIGINT, @MinTransactionId AS BIGINT, @MinNotCompletedTransactionId AS BIGINT, @CurrentTransactionId AS BIGINT; +SET @AffectedRows = 0; +BEGIN TRY + EXECUTE dbo.MergeResourcesGetTransactionVisibility @MinTransactionId OUTPUT; + SET @MinTransactionId += 1; + SET @CurrentTransactionId = (SELECT TOP 1 SurrogateIdRangeFirstValue + FROM dbo.Transactions + ORDER BY SurrogateIdRangeFirstValue DESC); + SET @MinNotCompletedTransactionId = isnull((SELECT TOP 1 SurrogateIdRangeFirstValue + FROM dbo.Transactions + WHERE IsCompleted = 0 + AND SurrogateIdRangeFirstValue BETWEEN @MinTransactionId AND @CurrentTransactionId + ORDER BY SurrogateIdRangeFirstValue), @CurrentTransactionId + 1); + SET @MaxTransactionId = (SELECT TOP 1 SurrogateIdRangeFirstValue + FROM dbo.Transactions + WHERE IsCompleted = 1 + AND SurrogateIdRangeFirstValue BETWEEN @MinTransactionId AND @CurrentTransactionId + AND SurrogateIdRangeFirstValue < @MinNotCompletedTransactionId + ORDER BY SurrogateIdRangeFirstValue DESC); + IF @MaxTransactionId >= @MinTransactionId + BEGIN + UPDATE A + SET IsVisible = 1, + VisibleDate = getUTCdate() + FROM dbo.Transactions AS A WITH (INDEX (1)) + WHERE SurrogateIdRangeFirstValue BETWEEN @MinTransactionId AND @CurrentTransactionId + AND SurrogateIdRangeFirstValue <= @MaxTransactionId; + SET @AffectedRows += @@rowcount; + END + SET @msg = 'Min=' + CONVERT (VARCHAR, @MinTransactionId) + ' C=' + CONVERT (VARCHAR, @CurrentTransactionId) + ' MinNC=' + CONVERT (VARCHAR, @MinNotCompletedTransactionId) + ' Max=' + CONVERT (VARCHAR, @MaxTransactionId); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @AffectedRows, @Text = @msg; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesBeginTransaction +@Count INT, @TransactionId BIGINT=0 OUTPUT, @SurrogateIdRangeFirstValue BIGINT=0 OUTPUT, @SequenceRangeFirstValue INT=0 OUTPUT, @HeartbeatDate DATETIME=NULL +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'MergeResourcesBeginTransaction', @Mode AS VARCHAR (200) = 'Cnt=' + CONVERT (VARCHAR, @Count), @st AS DATETIME = getUTCdate(), @FirstValueVar AS SQL_VARIANT, @LastValueVar AS SQL_VARIANT; +BEGIN TRY + SET @TransactionId = NULL; + IF @@trancount > 0 + RAISERROR ('MergeResourcesBeginTransaction cannot be called inside outer transaction.', 18, 127); + SET @FirstValueVar = NULL; + WHILE @FirstValueVar IS NULL + BEGIN + EXECUTE sys.sp_sequence_get_range @sequence_name = 'dbo.ResourceSurrogateIdUniquifierSequence', @range_size = @Count, @range_first_value = @FirstValueVar OUTPUT, @range_last_value = @LastValueVar OUTPUT; + SET @SequenceRangeFirstValue = CONVERT (INT, @FirstValueVar); + IF @SequenceRangeFirstValue > CONVERT (INT, @LastValueVar) + SET @FirstValueVar = NULL; + END + SET @SurrogateIdRangeFirstValue = datediff_big(millisecond, '0001-01-01', sysUTCdatetime()) * 80000 + @SequenceRangeFirstValue; + INSERT INTO dbo.Transactions (SurrogateIdRangeFirstValue, SurrogateIdRangeLastValue, HeartbeatDate) + SELECT @SurrogateIdRangeFirstValue, + @SurrogateIdRangeFirstValue + @Count - 1, + isnull(@HeartbeatDate, getUTCdate()); + SET @TransactionId = @SurrogateIdRangeFirstValue; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + IF @@trancount > 0 + ROLLBACK; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesCommitTransaction +@TransactionId BIGINT=NULL, @FailureReason VARCHAR (MAX)=NULL, @OverrideIsControlledByClientCheck BIT=0, @SurrogateIdRangeFirstValue BIGINT=NULL +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'MergeResourcesCommitTransaction', @st AS DATETIME = getUTCdate(), @InitialTranCount AS INT = @@trancount, @IsCompletedBefore AS BIT, @Rows AS INT, @msg AS VARCHAR (1000); +SET @TransactionId = isnull(@TransactionId, @SurrogateIdRangeFirstValue); +DECLARE @Mode AS VARCHAR (200) = 'TR=' + CONVERT (VARCHAR, @TransactionId) + ' OC=' + isnull(CONVERT (VARCHAR, @OverrideIsControlledByClientCheck), 'NULL'); +BEGIN TRY + IF @InitialTranCount = 0 + BEGIN TRANSACTION; + UPDATE dbo.Transactions + SET IsCompleted = 1, + @IsCompletedBefore = IsCompleted, + EndDate = getUTCdate(), + IsSuccess = CASE WHEN @FailureReason IS NULL THEN 1 ELSE 0 END, + FailureReason = @FailureReason + WHERE SurrogateIdRangeFirstValue = @TransactionId + AND (IsControlledByClient = 1 + OR @OverrideIsControlledByClientCheck = 1); + SET @Rows = @@rowcount; + IF @Rows = 0 + BEGIN + SET @msg = 'Transaction [' + CONVERT (VARCHAR (20), @TransactionId) + '] is not controlled by client or does not exist.'; + RAISERROR (@msg, 18, 127); + END + IF @IsCompletedBefore = 1 + BEGIN + IF @InitialTranCount = 0 + ROLLBACK; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows, @Target = '@IsCompletedBefore', @Text = '=1'; + RETURN; + END + IF @InitialTranCount = 0 + COMMIT TRANSACTION; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF @InitialTranCount = 0 + AND @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesDeleteInvisibleHistory +@TransactionId BIGINT, @AffectedRows INT=NULL OUTPUT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'T=' + CONVERT (VARCHAR, @TransactionId), @st AS DATETIME = getUTCdate(), @TypeId AS SMALLINT; +SET @AffectedRows = 0; +BEGIN TRY + DECLARE @Types TABLE ( + TypeId SMALLINT PRIMARY KEY, + Name VARCHAR (100)); + INSERT INTO @Types + EXECUTE dbo.GetUsedResourceTypes ; + WHILE EXISTS (SELECT * + FROM @Types) + BEGIN + SET @TypeId = (SELECT TOP 1 TypeId + FROM @Types + ORDER BY TypeId); + DELETE dbo.Resource + WHERE ResourceTypeId = @TypeId + AND HistoryTransactionId = @TransactionId + AND IsHistory = 1 + AND RawResource = 0xF; + SET @AffectedRows += @@rowcount; + DELETE @Types + WHERE TypeId = @TypeId; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @AffectedRows; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesGetTimeoutTransactions +@TimeoutSec INT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'T=' + CONVERT (VARCHAR, @TimeoutSec), @st AS DATETIME = getUTCdate(), @MinTransactionId AS BIGINT; +BEGIN TRY + EXECUTE dbo.MergeResourcesGetTransactionVisibility @MinTransactionId OUTPUT; + SELECT SurrogateIdRangeFirstValue + FROM dbo.Transactions + WHERE SurrogateIdRangeFirstValue > @MinTransactionId + AND IsCompleted = 0 + AND datediff(second, HeartbeatDate, getUTCdate()) > @TimeoutSec + ORDER BY SurrogateIdRangeFirstValue; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesGetTransactionVisibility +@TransactionId BIGINT OUTPUT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = '', @st AS DATETIME = getUTCdate(); +SET @TransactionId = isnull((SELECT TOP 1 SurrogateIdRangeFirstValue + FROM dbo.Transactions + WHERE IsVisible = 1 + ORDER BY SurrogateIdRangeFirstValue DESC), -1); +EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount, @Text = @TransactionId; + +GO +CREATE PROCEDURE dbo.MergeResourcesPutTransactionHeartbeat +@TransactionId BIGINT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'MergeResourcesPutTransactionHeartbeat', @Mode AS VARCHAR (100) = 'TR=' + CONVERT (VARCHAR, @TransactionId); +BEGIN TRY + UPDATE dbo.Transactions + SET HeartbeatDate = getUTCdate() + WHERE SurrogateIdRangeFirstValue = @TransactionId + AND IsControlledByClient = 1; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesPutTransactionInvisibleHistory +@TransactionId BIGINT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'TR=' + CONVERT (VARCHAR, @TransactionId), @st AS DATETIME = getUTCdate(); +BEGIN TRY + UPDATE dbo.Transactions + SET InvisibleHistoryRemovedDate = getUTCdate() + WHERE SurrogateIdRangeFirstValue = @TransactionId + AND InvisibleHistoryRemovedDate IS NULL; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.PutJobCancelation +@QueueType TINYINT, @GroupId BIGINT=NULL, @JobId BIGINT=NULL +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'PutJobCancelation', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' G=' + isnull(CONVERT (VARCHAR, @GroupId), 'NULL') + ' J=' + isnull(CONVERT (VARCHAR, @JobId), 'NULL'), @st AS DATETIME = getUTCdate(), @Rows AS INT, @PartitionId AS TINYINT = @JobId % 16; +BEGIN TRY + IF @JobId IS NULL + AND @GroupId IS NULL + RAISERROR ('@JobId = NULL and @GroupId = NULL', 18, 127); + IF @JobId IS NOT NULL + BEGIN + UPDATE dbo.JobQueue + SET Status = 4, + EndDate = getUTCdate(), + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()) + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Status = 0; + SET @Rows = @@rowcount; + IF @Rows = 0 + BEGIN + UPDATE dbo.JobQueue + SET CancelRequested = 1 + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Status = 1; + SET @Rows = @@rowcount; + END + END + ELSE + BEGIN + UPDATE dbo.JobQueue + SET Status = 4, + EndDate = getUTCdate(), + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()) + WHERE QueueType = @QueueType + AND GroupId = @GroupId + AND Status = 0; + SET @Rows = @@rowcount; + UPDATE dbo.JobQueue + SET CancelRequested = 1 + WHERE QueueType = @QueueType + AND GroupId = @GroupId + AND Status = 1; + SET @Rows += @@rowcount; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.PutJobHeartbeat +@QueueType TINYINT, @JobId BIGINT, @Version BIGINT, @Data BIGINT=NULL, @CurrentResult VARCHAR (MAX)=NULL, @CancelRequested BIT=0 OUTPUT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'PutJobHeartbeat', @Mode AS VARCHAR (100), @st AS DATETIME = getUTCdate(), @Rows AS INT = 0, @PartitionId AS TINYINT = @JobId % 16; +SET @Mode = 'Q=' + CONVERT (VARCHAR, @QueueType) + ' J=' + CONVERT (VARCHAR, @JobId) + ' P=' + CONVERT (VARCHAR, @PartitionId) + ' V=' + CONVERT (VARCHAR, @Version) + ' D=' + isnull(CONVERT (VARCHAR, @Data), 'NULL'); +BEGIN TRY + IF @CurrentResult IS NULL + UPDATE dbo.JobQueue + SET @CancelRequested = CancelRequested, + HeartbeatDate = getUTCdate(), + Data = isnull(@Data, Data) + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Status = 1 + AND Version = @Version; + ELSE + UPDATE dbo.JobQueue + SET @CancelRequested = CancelRequested, + HeartbeatDate = getUTCdate(), + Data = isnull(@Data, Data), + Result = @CurrentResult + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Status = 1 + AND Version = @Version; + SET @Rows = @@rowcount; + IF @Rows = 0 + AND NOT EXISTS (SELECT * + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Version = @Version + AND Status IN (2, 3, 4)) + BEGIN + IF EXISTS (SELECT * + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId) + THROW 50412, 'Precondition failed', 1; + ELSE + THROW 50404, 'Job record not found', 1; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.PutJobStatus +@QueueType TINYINT, @JobId BIGINT, @Version BIGINT, @Failed BIT, @Data BIGINT, @FinalResult VARCHAR (MAX), @RequestCancellationOnFailure BIT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'PutJobStatus', @Mode AS VARCHAR (100), @st AS DATETIME = getUTCdate(), @Rows AS INT = 0, @PartitionId AS TINYINT = @JobId % 16, @GroupId AS BIGINT; +SET @Mode = 'Q=' + CONVERT (VARCHAR, @QueueType) + ' J=' + CONVERT (VARCHAR, @JobId) + ' P=' + CONVERT (VARCHAR, @PartitionId) + ' V=' + CONVERT (VARCHAR, @Version) + ' F=' + CONVERT (VARCHAR, @Failed) + ' R=' + isnull(@FinalResult, 'NULL'); +BEGIN TRY + UPDATE dbo.JobQueue + SET EndDate = getUTCdate(), + Status = CASE WHEN @Failed = 1 THEN 3 WHEN CancelRequested = 1 THEN 4 ELSE 2 END, + Data = @Data, + Result = @FinalResult, + @GroupId = GroupId + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Status = 1 + AND Version = @Version; + SET @Rows = @@rowcount; + IF @Rows = 0 + BEGIN + SET @GroupId = (SELECT GroupId + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Version = @Version + AND Status IN (2, 3, 4)); + IF @GroupId IS NULL + IF EXISTS (SELECT * + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId) + THROW 50412, 'Precondition failed', 1; + ELSE + THROW 50404, 'Job record not found', 1; + END + IF @Failed = 1 + AND @RequestCancellationOnFailure = 1 + EXECUTE dbo.PutJobCancelation @QueueType = @QueueType, @GroupId = @GroupId; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.ReadResource +@resourceTypeId SMALLINT, @resourceId VARCHAR (64), @version INT=NULL +AS +SET NOCOUNT ON; +IF (@version IS NULL) + BEGIN + SELECT ResourceSurrogateId, + Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM dbo.Resource + WHERE ResourceTypeId = @resourceTypeId + AND ResourceId = @resourceId + AND IsHistory = 0; + END +ELSE + BEGIN + SELECT ResourceSurrogateId, + Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM dbo.Resource + WHERE ResourceTypeId = @resourceTypeId + AND ResourceId = @resourceId + AND Version = @version; + END + +GO +CREATE PROCEDURE dbo.ReindexResource_2 +@resourceTypeId SMALLINT, @resourceId VARCHAR (64), @eTag INT=NULL, @searchParamHash VARCHAR (64), @resourceWriteClaims dbo.BulkResourceWriteClaimTableType_1 READONLY, @compartmentAssignments dbo.BulkCompartmentAssignmentTableType_1 READONLY, @referenceSearchParams dbo.BulkReferenceSearchParamTableType_1 READONLY, @tokenSearchParams dbo.BulkTokenSearchParamTableType_2 READONLY, @tokenTextSearchParams dbo.BulkTokenTextTableType_1 READONLY, @stringSearchParams dbo.BulkStringSearchParamTableType_2 READONLY, @numberSearchParams dbo.BulkNumberSearchParamTableType_1 READONLY, @quantitySearchParams dbo.BulkQuantitySearchParamTableType_1 READONLY, @uriSearchParams dbo.BulkUriSearchParamTableType_1 READONLY, @dateTimeSearchParms dbo.BulkDateTimeSearchParamTableType_2 READONLY, @referenceTokenCompositeSearchParams dbo.BulkReferenceTokenCompositeSearchParamTableType_2 READONLY, @tokenTokenCompositeSearchParams dbo.BulkTokenTokenCompositeSearchParamTableType_2 READONLY, @tokenDateTimeCompositeSearchParams dbo.BulkTokenDateTimeCompositeSearchParamTableType_2 READONLY, @tokenQuantityCompositeSearchParams dbo.BulkTokenQuantityCompositeSearchParamTableType_2 READONLY, @tokenStringCompositeSearchParams dbo.BulkTokenStringCompositeSearchParamTableType_2 READONLY, @tokenNumberNumberCompositeSearchParams dbo.BulkTokenNumberNumberCompositeSearchParamTableType_2 READONLY +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @resourceSurrogateId AS BIGINT; +DECLARE @version AS BIGINT; +SELECT @resourceSurrogateId = ResourceSurrogateId, + @version = Version +FROM dbo.Resource WITH (UPDLOCK, HOLDLOCK) +WHERE ResourceTypeId = @resourceTypeId + AND ResourceId = @resourceId + AND IsHistory = 0; +IF (@etag IS NOT NULL + AND @etag <> @version) + BEGIN + THROW 50412, 'Precondition failed', 1; + END +UPDATE dbo.Resource +SET SearchParamHash = @searchParamHash +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.ResourceWriteClaim +WHERE ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.CompartmentAssignment +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.ReferenceSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenText +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.StringSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.UriSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.NumberSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.QuantitySearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.DateTimeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.ReferenceTokenCompositeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenTokenCompositeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenDateTimeCompositeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenQuantityCompositeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenStringCompositeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenNumberNumberCompositeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) +SELECT @resourceSurrogateId, + ClaimTypeId, + ClaimValue +FROM @resourceWriteClaims; +INSERT INTO dbo.CompartmentAssignment (ResourceTypeId, ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + CompartmentTypeId, + ReferenceResourceId, + 0 +FROM @compartmentAssignments; +INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + BaseUri, + ReferenceResourceTypeId, + ReferenceResourceId, + ReferenceResourceVersion, + 0 +FROM @referenceSearchParams; +INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId, + Code, + CodeOverflow, + 0 +FROM @tokenSearchParams; +INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + Text, + 0 +FROM @tokenTextSearchParams; +INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsHistory, IsMin, IsMax) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + Text, + TextOverflow, + 0, + IsMin, + IsMax +FROM @stringSearchParams; +INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + Uri, + 0 +FROM @uriSearchParams; +INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SingleValue, + LowValue, + HighValue, + 0 +FROM @numberSearchParams; +INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId, + QuantityCodeId, + SingleValue, + LowValue, + HighValue, + 0 +FROM @quantitySearchParams; +INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsHistory, IsMin, IsMax) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + StartDateTime, + EndDateTime, + IsLongerThanADay, + 0, + IsMin, + IsMax +FROM @dateTimeSearchParms; +INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + BaseUri1, + ReferenceResourceTypeId1, + ReferenceResourceId1, + ReferenceResourceVersion1, + SystemId2, + Code2, + CodeOverflow2, + 0 +FROM @referenceTokenCompositeSearchParams; +INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SystemId2, + Code2, + CodeOverflow2, + 0 +FROM @tokenTokenCompositeSearchParams; +INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + StartDateTime2, + EndDateTime2, + IsLongerThanADay2, + 0 +FROM @tokenDateTimeCompositeSearchParams; +INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + SystemId2, + QuantityCodeId2, + LowValue2, + HighValue2, + 0 +FROM @tokenQuantityCompositeSearchParams; +INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + Text2, + TextOverflow2, + 0 +FROM @tokenStringCompositeSearchParams; +INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + LowValue2, + HighValue2, + SingleValue3, + LowValue3, + HighValue3, + HasRange, + 0 +FROM @tokenNumberNumberCompositeSearchParams; +COMMIT TRANSACTION; + +GO +CREATE OR ALTER PROCEDURE dbo.RemovePartitionFromResourceChanges_2 +@partitionNumberToSwitchOut INT, @partitionBoundaryToMerge DATETIME2 (7) +AS +BEGIN + TRUNCATE TABLE dbo.ResourceChangeDataStaging; + ALTER TABLE dbo.ResourceChangeData SWITCH PARTITION @partitionNumberToSwitchOut TO dbo.ResourceChangeDataStaging; + ALTER PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp( ) + MERGE RANGE (@partitionBoundaryToMerge); + TRUNCATE TABLE dbo.ResourceChangeDataStaging; +END + +GO +CREATE PROCEDURE dbo.ResetTask_2 +@taskId VARCHAR (64), @runId VARCHAR (50), @result VARCHAR (MAX) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +DECLARE @retryCount AS SMALLINT = NULL; +IF NOT EXISTS (SELECT * + FROM dbo.TaskInfo + WHERE TaskId = @taskId + AND RunId = @runId) + BEGIN + THROW 50404, 'Task not exist or runid not match', 1; + END +UPDATE dbo.TaskInfo +SET Status = 3, + EndDateTime = SYSUTCDATETIME(), + Result = @result, + @retryCount = retryCount +WHERE TaskId = @taskId + AND RunId = @runId + AND (MaxRetryCount <> -1 + AND RetryCount >= MaxRetryCount); +IF @retryCount IS NULL + UPDATE dbo.TaskInfo + SET Status = 1, + Result = @result, + RetryCount = RetryCount + 1, + RestartInfo = ISNULL(RestartInfo, '') + ' Prev: Worker=' + Worker + ' Start=' + CONVERT (VARCHAR, StartDateTime, 121) + WHERE TaskId = @taskId + AND RunId = @runId + AND Status <> 3 + AND (MaxRetryCount = -1 + OR RetryCount < MaxRetryCount); +EXECUTE dbo.GetTaskDetails @TaskId = @taskId; + +GO +CREATE PROCEDURE dbo.SwitchPartitionsIn +@Tbl VARCHAR (100) +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'SwitchPartitionsIn', @Mode AS VARCHAR (200) = 'Tbl=' + isnull(@Tbl, 'NULL'), @st AS DATETIME = getUTCdate(), @ResourceTypeId AS SMALLINT, @Rows AS BIGINT, @Txt AS VARCHAR (1000), @TblInt AS VARCHAR (100), @Ind AS VARCHAR (200), @IndId AS INT, @DataComp AS VARCHAR (100); +DECLARE @Indexes TABLE ( + IndId INT PRIMARY KEY, + name VARCHAR (200)); +DECLARE @ResourceTypes TABLE ( + ResourceTypeId SMALLINT PRIMARY KEY); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + IF @Tbl IS NULL + RAISERROR ('@Tbl IS NULL', 18, 127); + INSERT INTO @Indexes + SELECT index_id, + name + FROM sys.indexes + WHERE object_id = object_id(@Tbl) + AND is_disabled = 1; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Insert', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Indexes) + BEGIN + SELECT TOP 1 @IndId = IndId, + @Ind = name + FROM @Indexes + ORDER BY IndId; + SET @DataComp = CASE WHEN (SELECT PropertyValue + FROM dbo.IndexProperties + WHERE TableName = @Tbl + AND IndexName = @Ind) = 'PAGE' THEN ' PARTITION = ALL WITH (DATA_COMPRESSION = PAGE)' ELSE '' END; + SET @Txt = 'IF EXISTS (SELECT * FROM sys.indexes WHERE object_id = object_id(''' + @Tbl + ''') AND name = ''' + @Ind + ''' AND is_disabled = 1) ALTER INDEX ' + @Ind + ' ON dbo.' + @Tbl + ' REBUILD' + @DataComp; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Ind, @Action = 'Rebuild', @Text = @Txt; + DELETE @Indexes + WHERE IndId = @IndId; + END + INSERT INTO @ResourceTypes + SELECT CONVERT (SMALLINT, substring(name, charindex('_', name) + 1, 6)) AS ResourceTypeId + FROM sys.objects AS O + WHERE name LIKE @Tbl + '[_]%' + AND EXISTS (SELECT * + FROM sysindexes + WHERE id = O.object_id + AND indid IN (0, 1) + AND rows > 0); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '#ResourceTypes', @Action = 'Select Into', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @ResourceTypes) + BEGIN + SET @ResourceTypeId = (SELECT TOP 1 ResourceTypeId + FROM @ResourceTypes); + SET @TblInt = @Tbl + '_' + CONVERT (VARCHAR, @ResourceTypeId); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt; + SET @Txt = 'ALTER TABLE dbo.' + @TblInt + ' SWITCH TO dbo.' + @Tbl + ' PARTITION $partition.PartitionFunction_ResourceTypeId(' + CONVERT (VARCHAR, @ResourceTypeId) + ')'; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Switch in start', @Text = @Txt; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Switch in', @Text = @Txt; + IF EXISTS (SELECT * + FROM sysindexes + WHERE id = object_id(@TblInt) + AND rows > 0) + BEGIN + SET @Txt = @TblInt + ' is not empty after switch'; + RAISERROR (@Txt, 18, 127); + END + EXECUTE ('DROP TABLE dbo.' + @TblInt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Drop'; + DELETE @ResourceTypes + WHERE ResourceTypeId = @ResourceTypeId; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.SwitchPartitionsInAllTables +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'SwitchPartitionsInAllTables', @Mode AS VARCHAR (200) = 'PS=PartitionScheme_ResourceTypeId', @st AS DATETIME = getUTCdate(), @Tbl AS VARCHAR (100); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + DECLARE @Tables TABLE ( + name VARCHAR (100) PRIMARY KEY, + supported BIT ); + INSERT INTO @Tables + EXECUTE dbo.GetPartitionedTables @IncludeNotDisabled = 1, @IncludeNotSupported = 0; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Tables', @Action = 'Insert', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Tables) + BEGIN + SET @Tbl = (SELECT TOP 1 name + FROM @Tables + ORDER BY name); + EXECUTE dbo.SwitchPartitionsIn @Tbl = @Tbl; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = 'SwitchPartitionsIn', @Action = 'Execute', @Text = @Tbl; + DELETE @Tables + WHERE name = @Tbl; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.SwitchPartitionsOut +@Tbl VARCHAR (100), @RebuildClustered BIT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'SwitchPartitionsOut', @Mode AS VARCHAR (200) = 'Tbl=' + isnull(@Tbl, 'NULL') + ' ND=' + isnull(CONVERT (VARCHAR, @RebuildClustered), 'NULL'), @st AS DATETIME = getUTCdate(), @ResourceTypeId AS SMALLINT, @Rows AS BIGINT, @Txt AS VARCHAR (MAX), @TblInt AS VARCHAR (100), @IndId AS INT, @Ind AS VARCHAR (200), @Name AS VARCHAR (100), @checkName AS VARCHAR (200), @definition AS VARCHAR (200); +DECLARE @Indexes TABLE ( + IndId INT PRIMARY KEY, + name VARCHAR (200), + IsDisabled BIT ); +DECLARE @IndexesRT TABLE ( + IndId INT PRIMARY KEY, + name VARCHAR (200), + IsDisabled BIT ); +DECLARE @ResourceTypes TABLE ( + ResourceTypeId SMALLINT PRIMARY KEY, + partition_number_roundtrip INT , + partition_number INT , + row_count BIGINT ); +DECLARE @Names TABLE ( + name VARCHAR (100) PRIMARY KEY); +DECLARE @CheckConstraints TABLE ( + CheckName VARCHAR (200), + CheckDefinition VARCHAR (200)); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + IF @Tbl IS NULL + RAISERROR ('@Tbl IS NULL', 18, 127); + IF @RebuildClustered IS NULL + RAISERROR ('@RebuildClustered IS NULL', 18, 127); + INSERT INTO @Indexes + SELECT index_id, + name, + is_disabled + FROM sys.indexes + WHERE object_id = object_id(@Tbl) + AND (is_disabled = 0 + OR @RebuildClustered = 1); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Insert', @Rows = @@rowcount; + INSERT INTO @ResourceTypes + SELECT partition_number - 1 AS ResourceTypeId, + $PARTITION.PartitionFunction_ResourceTypeId (partition_number - 1) AS partition_number_roundtrip, + partition_number, + row_count + FROM sys.dm_db_partition_stats + WHERE object_id = object_id(@Tbl) + AND index_id = 1 + AND row_count > 0; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@ResourceTypes', @Action = 'Insert', @Rows = @@rowcount, @Text = 'For partition switch'; + IF EXISTS (SELECT * + FROM @ResourceTypes + WHERE partition_number_roundtrip <> partition_number) + RAISERROR ('Partition sanity check failed', 18, 127); + WHILE EXISTS (SELECT * + FROM @ResourceTypes) + BEGIN + SELECT TOP 1 @ResourceTypeId = ResourceTypeId, + @Rows = row_count + FROM @ResourceTypes + ORDER BY ResourceTypeId; + SET @TblInt = @Tbl + '_' + CONVERT (VARCHAR, @ResourceTypeId); + SET @Txt = 'Starting @ResourceTypeId=' + CONVERT (VARCHAR, @ResourceTypeId) + ' row_count=' + CONVERT (VARCHAR, @Rows); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Text = @Txt; + IF NOT EXISTS (SELECT * + FROM sysindexes + WHERE id = object_id(@TblInt) + AND rows > 0) + BEGIN + IF object_id(@TblInt) IS NOT NULL + BEGIN + EXECUTE ('DROP TABLE dbo.' + @TblInt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Drop'; + END + EXECUTE ('SELECT * INTO dbo.' + @TblInt + ' FROM dbo.' + @Tbl + ' WHERE 1 = 2'); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Select Into', @Rows = @@rowcount; + DELETE @CheckConstraints; + INSERT INTO @CheckConstraints + SELECT name, + definition + FROM sys.check_constraints + WHERE parent_object_id = object_id(@Tbl); + WHILE EXISTS (SELECT * + FROM @CheckConstraints) + BEGIN + SELECT TOP 1 @checkName = CheckName, + @definition = CheckDefinition + FROM @CheckConstraints; + SET @Txt = 'ALTER TABLE ' + @TblInt + ' ADD CHECK ' + @definition; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'ALTER', @Text = @Txt; + DELETE @CheckConstraints + WHERE CheckName = @checkName; + END + DELETE @Names; + INSERT INTO @Names + SELECT name + FROM sys.columns + WHERE object_id = object_id(@Tbl) + AND is_sparse = 1; + WHILE EXISTS (SELECT * + FROM @Names) + BEGIN + SET @Name = (SELECT TOP 1 name + FROM @Names + ORDER BY name); + SET @Txt = (SELECT 'ALTER TABLE dbo.' + @TblInt + ' ALTER COLUMN ' + @Name + ' ' + T.name + '(' + CONVERT (VARCHAR, C.precision) + ',' + CONVERT (VARCHAR, C.scale) + ') SPARSE NULL' + FROM sys.types AS T + INNER JOIN + sys.columns AS C + ON C.system_type_id = T.system_type_id + WHERE C.object_id = object_id(@Tbl) + AND C.name = @Name); + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'ALTER', @Text = @Txt; + DELETE @Names + WHERE name = @Name; + END + END + INSERT INTO @IndexesRT + SELECT * + FROM @Indexes + WHERE IsDisabled = 0; + WHILE EXISTS (SELECT * + FROM @IndexesRT) + BEGIN + SELECT TOP 1 @IndId = IndId, + @Ind = name + FROM @IndexesRT + ORDER BY IndId; + IF NOT EXISTS (SELECT * + FROM sys.indexes + WHERE object_id = object_id(@TblInt) + AND name = @Ind) + BEGIN + EXECUTE dbo.GetIndexCommands @Tbl = @Tbl, @Ind = @Ind, @AddPartClause = 0, @IncludeClustered = 1, @Txt = @Txt OUTPUT; + SET @Txt = replace(@Txt, '[' + @Tbl + ']', @TblInt); + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Create Index', @Text = @Txt; + END + DELETE @IndexesRT + WHERE IndId = @IndId; + END + SET @Txt = 'ALTER TABLE dbo.' + @TblInt + ' ADD CHECK (ResourceTypeId >= ' + CONVERT (VARCHAR, @ResourceTypeId) + ' AND ResourceTypeId < ' + CONVERT (VARCHAR, @ResourceTypeId) + ' + 1)'; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Add check', @Text = @Txt; + SET @Txt = 'ALTER TABLE dbo.' + @Tbl + ' SWITCH PARTITION $partition.PartitionFunction_ResourceTypeId(' + CONVERT (VARCHAR, @ResourceTypeId) + ') TO dbo.' + @TblInt; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Switch out start', @Text = @Txt; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Switch out end', @Text = @Txt; + DELETE @ResourceTypes + WHERE ResourceTypeId = @ResourceTypeId; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.SwitchPartitionsOutAllTables +@RebuildClustered BIT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'SwitchPartitionsOutAllTables', @Mode AS VARCHAR (200) = 'PS=PartitionScheme_ResourceTypeId ND=' + isnull(CONVERT (VARCHAR, @RebuildClustered), 'NULL'), @st AS DATETIME = getUTCdate(), @Tbl AS VARCHAR (100); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + DECLARE @Tables TABLE ( + name VARCHAR (100) PRIMARY KEY, + supported BIT ); + INSERT INTO @Tables + EXECUTE dbo.GetPartitionedTables @IncludeNotDisabled = @RebuildClustered, @IncludeNotSupported = 0; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Tables', @Action = 'Insert', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Tables) + BEGIN + SET @Tbl = (SELECT TOP 1 name + FROM @Tables + ORDER BY name); + EXECUTE dbo.SwitchPartitionsOut @Tbl = @Tbl, @RebuildClustered = @RebuildClustered; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = 'SwitchPartitionsOut', @Action = 'Execute', @Text = @Tbl; + DELETE @Tables + WHERE name = @Tbl; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE [dbo].[TaskKeepAlive] +@taskId VARCHAR (64), @runId VARCHAR (50) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +IF NOT EXISTS (SELECT * + FROM [dbo].[TaskInfo] + WHERE TaskId = @taskId + AND RunId = @runId) + BEGIN + THROW 50404, 'Task not exist or runid not match', 1; + END +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +UPDATE dbo.TaskInfo +SET HeartbeatDateTime = @heartbeatDateTime +WHERE TaskId = @taskId; +SELECT TaskId, + QueueId, + Status, + TaskTypeId, + RunId, + IsCanceled, + RetryCount, + MaxRetryCount, + HeartbeatDateTime, + InputData, + TaskContext, + Result +FROM [dbo].[TaskInfo] +WHERE TaskId = @taskId; +COMMIT TRANSACTION; + +GO +CREATE OR ALTER PROCEDURE dbo.UpdateEventAgentCheckpoint +@CheckpointId VARCHAR (64), @LastProcessedDateTime DATETIMEOFFSET (7)=NULL, @LastProcessedIdentifier VARCHAR (64)=NULL +AS +BEGIN + IF EXISTS (SELECT * + FROM dbo.EventAgentCheckpoint + WHERE CheckpointId = @CheckpointId) + UPDATE dbo.EventAgentCheckpoint + SET CheckpointId = @CheckpointId, + LastProcessedDateTime = @LastProcessedDateTime, + LastProcessedIdentifier = @LastProcessedIdentifier, + UpdatedOn = sysutcdatetime() + WHERE CheckpointId = @CheckpointId; + ELSE + INSERT INTO dbo.EventAgentCheckpoint (CheckpointId, LastProcessedDateTime, LastProcessedIdentifier, UpdatedOn) + VALUES (@CheckpointId, @LastProcessedDateTime, @LastProcessedIdentifier, sysutcdatetime()); +END + +GO +CREATE PROCEDURE dbo.UpdateExportJob +@id VARCHAR (64), @status VARCHAR (10), @rawJobRecord VARCHAR (MAX), @jobVersion BINARY (8) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @currentJobVersion AS BINARY (8); +SELECT @currentJobVersion = JobVersion +FROM dbo.ExportJob WITH (UPDLOCK, HOLDLOCK) +WHERE Id = @id; +IF (@currentJobVersion IS NULL) + BEGIN + THROW 50404, 'Export job record not found', 1; + END +IF (@jobVersion <> @currentJobVersion) + BEGIN + THROW 50412, 'Precondition failed', 1; + END +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +UPDATE dbo.ExportJob +SET Status = @status, + HeartbeatDateTime = @heartbeatDateTime, + RawJobRecord = @rawJobRecord +WHERE Id = @id; +SELECT @@DBTS; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.UpdateReindexJob +@id VARCHAR (64), @status VARCHAR (10), @rawJobRecord VARCHAR (MAX), @jobVersion BINARY (8) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @currentJobVersion AS BINARY (8); +SELECT @currentJobVersion = JobVersion +FROM dbo.ReindexJob WITH (UPDLOCK, HOLDLOCK) +WHERE Id = @id; +IF (@currentJobVersion IS NULL) + BEGIN + THROW 50404, 'Reindex job record not found', 1; + END +IF (@jobVersion <> @currentJobVersion) + BEGIN + THROW 50412, 'Precondition failed', 1; + END +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +UPDATE dbo.ReindexJob +SET Status = @status, + HeartbeatDateTime = @heartbeatDateTime, + RawJobRecord = @rawJobRecord +WHERE Id = @id; +SELECT @@DBTS; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.UpdateResourceSearchParams +@FailedResources INT=0 OUTPUT, @Resources dbo.ResourceList READONLY, @ResourceWriteClaims dbo.ResourceWriteClaimList READONLY, @ReferenceSearchParams dbo.ReferenceSearchParamList READONLY, @TokenSearchParams dbo.TokenSearchParamList READONLY, @TokenTexts dbo.TokenTextList READONLY, @StringSearchParams dbo.StringSearchParamList READONLY, @UriSearchParams dbo.UriSearchParamList READONLY, @NumberSearchParams dbo.NumberSearchParamList READONLY, @QuantitySearchParams dbo.QuantitySearchParamList READONLY, @DateTimeSearchParams dbo.DateTimeSearchParamList READONLY, @ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY, @TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY, @TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY, @TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY, @TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY, @TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY +AS +SET NOCOUNT ON; +DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (200) = isnull((SELECT 'RT=[' + CONVERT (VARCHAR, min(ResourceTypeId)) + ',' + CONVERT (VARCHAR, max(ResourceTypeId)) + '] Sur=[' + CONVERT (VARCHAR, min(ResourceSurrogateId)) + ',' + CONVERT (VARCHAR, max(ResourceSurrogateId)) + '] V=' + CONVERT (VARCHAR, max(Version)) + ' Rows=' + CONVERT (VARCHAR, count(*)) + FROM @Resources), 'Input=Empty'), @Rows AS INT; +BEGIN TRY + DECLARE @Ids TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL); + BEGIN TRANSACTION; + UPDATE B + SET SearchParamHash = A.SearchParamHash + OUTPUT deleted.ResourceTypeId, deleted.ResourceSurrogateId INTO @Ids + FROM @Resources AS A + INNER JOIN + dbo.Resource AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + WHERE B.IsHistory = 0; + SET @Rows = @@rowcount; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.ResourceWriteClaim AS B + ON B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.ReferenceSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenText AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.StringSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.UriSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.NumberSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.QuantitySearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.DateTimeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.ReferenceTokenCompositeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenTokenCompositeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenDateTimeCompositeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenQuantityCompositeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenStringCompositeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenNumberNumberCompositeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) + SELECT ResourceSurrogateId, + ClaimTypeId, + ClaimValue + FROM @ResourceWriteClaims; + INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri, + ReferenceResourceTypeId, + ReferenceResourceId, + ReferenceResourceVersion + FROM @ReferenceSearchParams; + INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + Code, + CodeOverflow + FROM @TokenSearchParams; + INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text + FROM @TokenTexts; + INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text, + TextOverflow, + IsMin, + IsMax + FROM @StringSearchParams; + INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Uri + FROM @UriSearchParams; + INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SingleValue, + LowValue, + HighValue + FROM @NumberSearchParams; + INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + QuantityCodeId, + SingleValue, + LowValue, + HighValue + FROM @QuantitySearchParams; + INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + StartDateTime, + EndDateTime, + IsLongerThanADay, + IsMin, + IsMax + FROM @DateTimeSearchParams; + INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri1, + ReferenceResourceTypeId1, + ReferenceResourceId1, + ReferenceResourceVersion1, + SystemId2, + Code2, + CodeOverflow2 + FROM @ReferenceTokenCompositeSearchParams; + INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SystemId2, + Code2, + CodeOverflow2 + FROM @TokenTokenCompositeSearchParams; + INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + StartDateTime2, + EndDateTime2, + IsLongerThanADay2 + FROM @TokenDateTimeCompositeSearchParams; + INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + SystemId2, + QuantityCodeId2, + LowValue2, + HighValue2 + FROM @TokenQuantityCompositeSearchParams; + INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + Text2, + TextOverflow2 + FROM @TokenStringCompositeSearchParams; + INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + LowValue2, + HighValue2, + SingleValue3, + LowValue3, + HighValue3, + HasRange + FROM @TokenNumberNumberCompositeSearchParams; + COMMIT TRANSACTION; + SET @FailedResources = (SELECT count(*) + FROM @Resources) - @Rows; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE [dbo].[UpdateTaskContext] +@taskId VARCHAR (64), @taskContext VARCHAR (MAX), @runId VARCHAR (50) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +IF NOT EXISTS (SELECT * + FROM [dbo].[TaskInfo] + WHERE TaskId = @taskId + AND RunId = @runId) + BEGIN + THROW 50404, 'Task not exist or runid not match', 1; + END +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +UPDATE dbo.TaskInfo +SET HeartbeatDateTime = @heartbeatDateTime, + TaskContext = @taskContext +WHERE TaskId = @taskId; +SELECT TaskId, + QueueId, + Status, + TaskTypeId, + RunId, + IsCanceled, + RetryCount, + MaxRetryCount, + HeartbeatDateTime, + InputData, + TaskContext, + Result +FROM [dbo].[TaskInfo] +WHERE TaskId = @taskId; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.UpsertResource_7 +@baseResourceSurrogateId BIGINT, @resourceTypeId SMALLINT, @resourceId VARCHAR (64), @eTag INT=NULL, @allowCreate BIT, @isDeleted BIT, @keepHistory BIT, @requireETagOnUpdate BIT, @requestMethod VARCHAR (10), @searchParamHash VARCHAR (64), @rawResource VARBINARY (MAX), @resourceWriteClaims dbo.BulkResourceWriteClaimTableType_1 READONLY, @compartmentAssignments dbo.BulkCompartmentAssignmentTableType_1 READONLY, @referenceSearchParams dbo.BulkReferenceSearchParamTableType_1 READONLY, @tokenSearchParams dbo.BulkTokenSearchParamTableType_2 READONLY, @tokenTextSearchParams dbo.BulkTokenTextTableType_1 READONLY, @stringSearchParams dbo.BulkStringSearchParamTableType_2 READONLY, @numberSearchParams dbo.BulkNumberSearchParamTableType_1 READONLY, @quantitySearchParams dbo.BulkQuantitySearchParamTableType_1 READONLY, @uriSearchParams dbo.BulkUriSearchParamTableType_1 READONLY, @dateTimeSearchParms dbo.BulkDateTimeSearchParamTableType_2 READONLY, @referenceTokenCompositeSearchParams dbo.BulkReferenceTokenCompositeSearchParamTableType_2 READONLY, @tokenTokenCompositeSearchParams dbo.BulkTokenTokenCompositeSearchParamTableType_2 READONLY, @tokenDateTimeCompositeSearchParams dbo.BulkTokenDateTimeCompositeSearchParamTableType_2 READONLY, @tokenQuantityCompositeSearchParams dbo.BulkTokenQuantityCompositeSearchParamTableType_2 READONLY, @tokenStringCompositeSearchParams dbo.BulkTokenStringCompositeSearchParamTableType_2 READONLY, @tokenNumberNumberCompositeSearchParams dbo.BulkTokenNumberNumberCompositeSearchParamTableType_2 READONLY, @isResourceChangeCaptureEnabled BIT=0, @comparedVersion INT=NULL +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +DECLARE @previousResourceSurrogateId AS BIGINT, @previousVersion AS BIGINT, @previousIsDeleted AS BIT, @version AS INT, @resourceSurrogateId AS BIGINT, @InitialTranCount AS INT = @@trancount; +IF @InitialTranCount = 0 + BEGIN TRANSACTION; +SELECT @previousResourceSurrogateId = ResourceSurrogateId, + @previousVersion = Version, + @previousIsDeleted = IsDeleted +FROM dbo.Resource WITH (UPDLOCK, HOLDLOCK) +WHERE ResourceTypeId = @resourceTypeId + AND ResourceId = @resourceId + AND IsHistory = 0; +IF @previousResourceSurrogateId IS NULL + SET @version = 1; +ELSE + BEGIN + IF @isDeleted = 0 + BEGIN + IF @comparedVersion IS NULL + OR @comparedVersion <> @previousVersion + BEGIN + THROW 50409, 'Resource has been recently updated or added, please compare the resource content in code for any duplicate updates', 1; + END + END + SET @version = @previousVersion + 1; + IF @keepHistory = 1 + UPDATE dbo.Resource + SET IsHistory = 1 + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + ELSE + DELETE dbo.Resource + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.ResourceWriteClaim + WHERE ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.CompartmentAssignment + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.ReferenceSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenText + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.StringSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.UriSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.NumberSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.QuantitySearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.DateTimeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.ReferenceTokenCompositeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenTokenCompositeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenDateTimeCompositeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenQuantityCompositeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenStringCompositeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenNumberNumberCompositeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + END +SET @resourceSurrogateId = @baseResourceSurrogateId + ( NEXT VALUE FOR ResourceSurrogateIdUniquifierSequence); +INSERT INTO dbo.Resource (ResourceTypeId, ResourceId, Version, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, RawResource, IsRawResourceMetaSet, SearchParamHash) +SELECT @resourceTypeId, + @resourceId, + @version, + 0, + @resourceSurrogateId, + @isDeleted, + @requestMethod, + @rawResource, + CASE WHEN @version = 1 THEN 1 ELSE 0 END, + @searchParamHash; +INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) +SELECT @resourceSurrogateId, + ClaimTypeId, + ClaimValue +FROM @resourceWriteClaims; +INSERT INTO dbo.CompartmentAssignment (ResourceTypeId, ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + CompartmentTypeId, + ReferenceResourceId, + 0 +FROM @compartmentAssignments; +INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + BaseUri, + ReferenceResourceTypeId, + ReferenceResourceId, + ReferenceResourceVersion, + 0 +FROM @referenceSearchParams; +INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId, + Code, + CodeOverflow, + 0 +FROM @tokenSearchParams; +INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + Text, + 0 +FROM @tokenTextSearchParams; +INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsHistory, IsMin, IsMax) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + Text, + TextOverflow, + 0, + IsMin, + IsMax +FROM @stringSearchParams; +INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + Uri, + 0 +FROM @uriSearchParams; +INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SingleValue, + LowValue, + HighValue, + 0 +FROM @numberSearchParams; +INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId, + QuantityCodeId, + SingleValue, + LowValue, + HighValue, + 0 +FROM @quantitySearchParams; +INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsHistory, IsMin, IsMax) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + StartDateTime, + EndDateTime, + IsLongerThanADay, + 0, + IsMin, + IsMax +FROM @dateTimeSearchParms; +INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + BaseUri1, + ReferenceResourceTypeId1, + ReferenceResourceId1, + ReferenceResourceVersion1, + SystemId2, + Code2, + CodeOverflow2, + 0 +FROM @referenceTokenCompositeSearchParams; +INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SystemId2, + Code2, + CodeOverflow2, + 0 +FROM @tokenTokenCompositeSearchParams; +INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + StartDateTime2, + EndDateTime2, + IsLongerThanADay2, + 0 +FROM @tokenDateTimeCompositeSearchParams; +INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + SystemId2, + QuantityCodeId2, + LowValue2, + HighValue2, + 0 +FROM @tokenQuantityCompositeSearchParams; +INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + Text2, + TextOverflow2, + 0 +FROM @tokenStringCompositeSearchParams; +INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + LowValue2, + HighValue2, + SingleValue3, + LowValue3, + HighValue3, + HasRange, + 0 +FROM @tokenNumberNumberCompositeSearchParams; +SELECT @version; +IF @isResourceChangeCaptureEnabled = 1 + EXECUTE dbo.CaptureResourceChanges @isDeleted = @isDeleted, @version = @version, @resourceId = @resourceId, @resourceTypeId = @resourceTypeId; +IF @InitialTranCount = 0 + COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.UpsertSearchParams +@searchParams dbo.SearchParamTableType_2 READONLY +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DECLARE @lastUpdated AS DATETIMEOFFSET (7) = SYSDATETIMEOFFSET(); +DECLARE @summaryOfChanges TABLE ( + Uri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + Action VARCHAR (20) NOT NULL); +MERGE INTO dbo.SearchParam WITH (TABLOCKX) + AS target +USING @searchParams AS source ON target.Uri = source.Uri +WHEN MATCHED THEN UPDATE +SET Status = source.Status, + LastUpdated = @lastUpdated, + IsPartiallySupported = source.IsPartiallySupported +WHEN NOT MATCHED BY TARGET THEN INSERT (Uri, Status, LastUpdated, IsPartiallySupported) VALUES (source.Uri, source.Status, @lastUpdated, source.IsPartiallySupported) +OUTPUT source.Uri, $ACTION INTO @summaryOfChanges; +SELECT SearchParamId, + SearchParam.Uri +FROM dbo.SearchParam AS searchParam + INNER JOIN + @summaryOfChanges AS upsertedSearchParam + ON searchParam.Uri = upsertedSearchParam.Uri +WHERE upsertedSearchParam.Action = 'INSERT'; +COMMIT TRANSACTION; + +GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs index 4af385dc63..dfe4a32a05 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs @@ -73,5 +73,6 @@ public enum SchemaVersion V61 = 61, V62 = 62, V63 = 63, + V64 = 64, } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs index dc37798414..766dd6520f 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs @@ -8,7 +8,7 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Schema public static class SchemaVersionConstants { public const int Min = (int)SchemaVersion.V63; - public const int Max = (int)SchemaVersion.V63; + public const int Max = (int)SchemaVersion.V64; public const int MinForUpgrade = (int)SchemaVersion.V58; // this is used for upgrade tests only public const int SearchParameterStatusSchemaVersion = (int)SchemaVersion.V6; public const int SupportForReferencesWithMissingTypeVersion = (int)SchemaVersion.V7; diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/TransactionCheckWithInitialiScript.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/TransactionCheckWithInitialiScript.sql index f531515aab..36f2010660 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/TransactionCheckWithInitialiScript.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/TransactionCheckWithInitialiScript.sql @@ -19,6 +19,6 @@ Go INSERT INTO dbo.SchemaVersion VALUES - (63, 'started') + (64, 'started') Go diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetResourcesByTypeAndSurrogateIdRange.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetResourcesByTypeAndSurrogateIdRange.sql index be6b269fa6..6847aa6b56 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetResourcesByTypeAndSurrogateIdRange.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetResourcesByTypeAndSurrogateIdRange.sql @@ -1,6 +1,6 @@ --DROP PROCEDURE dbo.GetResourcesByTypeAndSurrogateIdRange GO -CREATE PROCEDURE dbo.GetResourcesByTypeAndSurrogateIdRange @ResourceTypeId smallint, @StartId bigint, @EndId bigint, @GlobalStartId bigint = NULL, @GlobalEndId bigint = NULL +CREATE PROCEDURE dbo.GetResourcesByTypeAndSurrogateIdRange @ResourceTypeId smallint, @StartId bigint, @EndId bigint, @GlobalStartId bigint = NULL, @GlobalEndId bigint = NULL, @IncludeHistory bit = 0, @IncludeDeleted bit = 0 AS set nocount on DECLARE @SP varchar(100) = 'GetResourcesByTypeAndSurrogateIdRange' @@ -9,6 +9,8 @@ DECLARE @SP varchar(100) = 'GetResourcesByTypeAndSurrogateIdRange' +' E='+isnull(convert(varchar,@EndId),'NULL') +' GS='+isnull(convert(varchar,@GlobalStartId),'NULL') -- Is global start id needed? I'm not seeing a usecase for setting it. +' GE='+isnull(convert(varchar,@GlobalEndId),'NULL') -- Could this just be a boolean for if historical records should be returned? GlobalEndId should equal EndId in all cases I can think of. + +' HI='+isnull(convert(varchar,@IncludeHistory),'NULL') + +' DE'+isnull(convert(varchar,@IncludeDeleted),'NULL') ,@st datetime = getUTCdate() BEGIN TRY @@ -27,7 +29,7 @@ BEGIN TRY WHERE ResourceTypeId = @ResourceTypeId AND ResourceSurrogateId BETWEEN @StartId AND @EndId AND IsHistory = 1 - AND IsDeleted = 0 + AND (IsDeleted = 0 OR @IncludeDeleted = 1) -- TBD if this is needed. ) AND ResourceSurrogateId BETWEEN @GlobalStartId AND @GlobalEndId @@ -54,16 +56,16 @@ BEGIN TRY LEFT OUTER JOIN dbo.Resource C ON C.ResourceTypeId = @ResourceTypeId AND C.ResourceSurrogateId = MaxSurrogateId WHERE A.ResourceTypeId = @ResourceTypeId AND A.ResourceSurrogateId BETWEEN @StartId AND @EndId - AND (A.IsHistory = 0 OR MaxSurrogateId IS NOT NULL) - AND A.IsDeleted = 0 + AND (A.IsHistory = 0 OR MaxSurrogateId IS NOT NULL OR @IncludeHistory = 1) + AND (A.IsDeleted = 0 OR @IncludeDeleted = 1) END ELSE SELECT ResourceTypeId, ResourceId, Version, IsDeleted, ResourceSurrogateId, RequestMethod, IsMatch = convert(bit,1), IsPartial = convert(bit,0), IsRawResourceMetaSet, SearchParamHash, RawResource FROM dbo.Resource WHERE ResourceTypeId = @ResourceTypeId AND ResourceSurrogateId BETWEEN @StartId AND @EndId - AND IsHistory = 0 - AND IsDeleted = 0 + AND (IsHistory = 0 OR @IncludeHistory = 1) + AND (IsDeleted = 0 OR @IncludeDeleted = 1) EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@st,@Rows=@@rowcount END TRY diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs index faf8121e89..850e10e619 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs @@ -555,10 +555,10 @@ private void PopulateSqlCommandFromQueryHints(SqlSearchOptions options, SqlComma var globalStartId = long.Parse(hints.First(_ => _.Param == KnownQueryParameterNames.GlobalStartSurrogateId).Value); var globalEndId = long.Parse(hints.First(_ => _.Param == KnownQueryParameterNames.GlobalEndSurrogateId).Value); - PopulateSqlCommandFromQueryHints(command, resourceTypeId, startId, endId, globalStartId, globalEndId); + PopulateSqlCommandFromQueryHints(command, resourceTypeId, startId, endId, globalStartId, globalEndId, options.IncludeHistory, options.IncludeDeleted); } - private static void PopulateSqlCommandFromQueryHints(SqlCommand command, short resourceTypeId, long startId, long endId, long? globalStartId, long? globalEndId) + private static void PopulateSqlCommandFromQueryHints(SqlCommand command, short resourceTypeId, long startId, long endId, long? globalStartId, long? globalEndId, bool? includeHistory, bool? includeDeleted) { command.CommandType = CommandType.StoredProcedure; command.CommandText = "dbo.GetResourcesByTypeAndSurrogateIdRange"; @@ -567,6 +567,8 @@ private static void PopulateSqlCommandFromQueryHints(SqlCommand command, short r command.Parameters.AddWithValue("@EndId", endId); command.Parameters.AddWithValue("@GlobalStartId", globalStartId); command.Parameters.AddWithValue("@GlobalEndId", globalEndId); + command.Parameters.AddWithValue("@IncludeHistory", includeHistory); + command.Parameters.AddWithValue("@IncludeDeleted", includeDeleted); } /// @@ -579,13 +581,15 @@ private static void PopulateSqlCommandFromQueryHints(SqlCommand command, short r /// The upper bound for the window of time to consider for historical records /// Cancellation token /// When not null then we filter using the searchParameterHash + /// Return historical records that match the other parameters. + /// Return deleted records that match the other parameters. /// All resources with surrogate ids greater than or equal to startId and less than or equal to endId. If windowEndId is set it will return the most recent version of a resource that was created before windowEndId that is within the range of startId to endId. - public async Task SearchBySurrogateIdRange(string resourceType, long startId, long endId, long? windowStartId, long? windowEndId, CancellationToken cancellationToken, string searchParamHashFilter = null) + public async Task SearchBySurrogateIdRange(string resourceType, long startId, long endId, long? windowStartId, long? windowEndId, CancellationToken cancellationToken, string searchParamHashFilter = null, bool includeHistory = false, bool includeDeleted = false) { var resourceTypeId = _model.GetResourceTypeId(resourceType); using var sqlCommand = new SqlCommand(); sqlCommand.CommandTimeout = GetReindexCommandTimeout(); - PopulateSqlCommandFromQueryHints(sqlCommand, resourceTypeId, startId, endId, windowStartId, windowEndId); + PopulateSqlCommandFromQueryHints(sqlCommand, resourceTypeId, startId, endId, windowStartId, windowEndId, includeHistory, includeDeleted); LogSqlCommand(sqlCommand); List resources = null; await _sqlRetryService.ExecuteSql( diff --git a/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj b/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj index ec66fb40ce..cd24927e23 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj +++ b/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj @@ -1,7 +1,7 @@  - 63 + 64 Features\Schema\Migrations\$(LatestSchemaVersion).sql diff --git a/test/Microsoft.Health.Fhir.R4.Tests.E2E/Microsoft.Health.Fhir.R4.Tests.E2E.csproj b/test/Microsoft.Health.Fhir.R4.Tests.E2E/Microsoft.Health.Fhir.R4.Tests.E2E.csproj index a78523efaa..0abc057fbf 100644 --- a/test/Microsoft.Health.Fhir.R4.Tests.E2E/Microsoft.Health.Fhir.R4.Tests.E2E.csproj +++ b/test/Microsoft.Health.Fhir.R4.Tests.E2E/Microsoft.Health.Fhir.R4.Tests.E2E.csproj @@ -15,8 +15,8 @@ - - + + diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.projitems b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.projitems index e7e642d77a..b59ab3cf2e 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.projitems +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.projitems @@ -32,10 +32,10 @@ - - - - + + + + @@ -66,7 +66,7 @@ - + @@ -114,7 +114,7 @@ - + diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.shproj b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.shproj index e97728cced..174ed2ed52 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.shproj +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.shproj @@ -9,8 +9,8 @@ - - + + diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/AnonymizedExportTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/AnonymizedExportTests.cs similarity index 99% rename from test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/AnonymizedExportTests.cs rename to test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/AnonymizedExportTests.cs index 4aadd3e39a..de197526da 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/AnonymizedExportTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/AnonymizedExportTests.cs @@ -31,12 +31,12 @@ using FhirGroup = Hl7.Fhir.Model.Group; using Task = System.Threading.Tasks.Task; -namespace Microsoft.Health.Fhir.Tests.E2E.Rest +namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Export { [Trait(Traits.OwningTeam, OwningTeam.FhirImport)] [Trait(Traits.Category, Categories.AnonymizedExport)] [HttpIntegrationFixtureArgumentSets(DataStore.All, Format.Json)] - public class AnonymizedExportTests : IClassFixture + public class AnonymizedExportTests : IClassFixture { private const string TestExportStoreUriEnvironmentVariableName = "TestExportStoreUri"; private const string TestExportStoreKeyEnvironmentVariableName = "TestExportStoreKey"; @@ -52,7 +52,7 @@ public class AnonymizedExportTests : IClassFixture ] }"; - public AnonymizedExportTests(ExportTestFixture fixture) + public AnonymizedExportTests(ExportDataTestFixture fixture) { _isUsingInProcTestServer = fixture.IsUsingInProcTestServer; _testFhirClient = fixture.TestFhirClient; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/AnonymizedExportUsingAcrTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/AnonymizedExportUsingAcrTests.cs similarity index 99% rename from test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/AnonymizedExportUsingAcrTests.cs rename to test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/AnonymizedExportUsingAcrTests.cs index c4b1252c93..08360f9087 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/AnonymizedExportUsingAcrTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/AnonymizedExportUsingAcrTests.cs @@ -35,12 +35,12 @@ using FhirGroup = Hl7.Fhir.Model.Group; using Task = System.Threading.Tasks.Task; -namespace Microsoft.Health.Fhir.Tests.E2E.Rest +namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Export { [Trait(Traits.OwningTeam, OwningTeam.FhirImport)] [Trait(Traits.Category, Categories.AnonymizedExport)] [HttpIntegrationFixtureArgumentSets(DataStore.All, Format.Json)] - public class AnonymizedExportUsingAcrTests : IClassFixture + public class AnonymizedExportUsingAcrTests : IClassFixture { private const string TestRepositoryName = "testanonymizationconfigs"; private const string TestConfigName = "testconfigname.json"; @@ -60,7 +60,7 @@ public class AnonymizedExportUsingAcrTests : IClassFixture ] }"; - public AnonymizedExportUsingAcrTests(ExportTestFixture fixture) + public AnonymizedExportUsingAcrTests(ExportDataTestFixture fixture) { _isUsingInProcTestServer = fixture.IsUsingInProcTestServer; _testFhirClient = fixture.TestFhirClient; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs similarity index 62% rename from test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestFixture.cs rename to test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs index 0754c0e976..f68fcd2540 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs @@ -16,13 +16,13 @@ using Microsoft.Health.Fhir.Tests.E2E.Rest.Metric; using Task = System.Threading.Tasks.Task; -namespace Microsoft.Health.Fhir.Tests.E2E.Rest +namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Export { - public class ExportTestFixture : HttpIntegrationTestFixture + public class ExportDataTestFixture : HttpIntegrationTestFixture { private MetricHandler _metricHandler; - public ExportTestFixture(DataStore dataStore, Format format, TestFhirServerFactory testFhirServerFactory) + public ExportDataTestFixture(DataStore dataStore, Format format, TestFhirServerFactory testFhirServerFactory) : base(dataStore, format, testFhirServerFactory) { } @@ -48,6 +48,8 @@ public MetricHandler MetricHandler public Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestResources => TestResourcesWithHistory.Where(pair => TestResourcesWithDeletes.ContainsKey(pair.Key)).ToDictionary(pair => pair.Key, pair => pair.Value); + public Dictionary ExportTestCasesContentUrls { get; } = new(); + public string FixtureTag { get; } = Guid.NewGuid().ToString(); public DateTime TestDataInsertionTime { get; } = DateTime.UtcNow; @@ -55,6 +57,11 @@ public MetricHandler MetricHandler public string ExportTestResourcesQueryParameters => $"_type=Patient,Observation&_typeFilter=Patient%3F_tag%3D{FixtureTag},Observation%3F_tag%3D{FixtureTag}"; protected override async Task OnInitializedAsync() + { + await SaveTestResourcesToServer(); + } + + private async Task SaveTestResourcesToServer() { void AddResourceToTestResources(Resource resource) => TestResourcesWithHistoryAndDeletes[(resource.TypeName, resource.Id, resource.VersionId)] = resource; @@ -85,19 +92,36 @@ void AddResourceToTestResources(Resource resource) => } else if (i % 10 == 0) { - Practitioner deletedResource = resource.DeepCopy() as Practitioner; - testResourcesInfo.Add((deletedResource, true)); + testResourcesInfo.Add((resource.DeepCopy() as Resource, true)); } else if (i % 4 == 1) { - Practitioner updatedResource = resource.DeepCopy() as Practitioner; - updatedResource.Name.Add(new() + Resource updatedResource = resource.DeepCopy() as Resource; + + if (updatedResource is Patient) + { + Patient updatedPatient = updatedResource as Patient; + updatedPatient.Name.Add(new() + { + Given = updatedPatient.Name.First().Given, + Family = $"UpdatedFromVersion{updatedResource.Meta.VersionId}", + }); + testResourcesInfo.Add((updatedPatient, false)); + } + + if (updatedResource is Encounter) { - Given = updatedResource.Name.First().Given, - Family = $"UpdatedFromVersion{updatedResource.Meta.VersionId}", - }); + Encounter updatedEncounter = updatedResource as Encounter; + updatedEncounter.Type.Add(new CodeableConcept("http://e2e-test", $"UpdatedFromVersion{updatedResource.Meta.VersionId}")); + testResourcesInfo.Add((updatedEncounter, false)); + } - testResourcesInfo.Add((updatedResource, false)); + if (updatedResource is Observation) + { + Observation updatedObservation = updatedResource as Observation; + updatedObservation.Category.Add(new CodeableConcept("http://e2e-test", $"UpdatedFromVersion{updatedResource.Meta.VersionId}")); + testResourcesInfo.Add((updatedObservation, false)); + } } } } @@ -110,24 +134,6 @@ void AddResourceToTestResources(Resource resource) => Console.WriteLine($"Generated {TestResourcesWithHistoryAndDeletes.Count} resources."); } - private List GenerateTestResources(int numberOfResources = 100) - { - var resources = new List(); - - for (int i = 0; i < numberOfResources; i++) - { - resources.Add(new Practitioner - { - Id = Guid.NewGuid().ToString("N"), - Meta = new() { Tag = new List() { new Coding("http://e2e-test", FixtureTag) } }, - Active = true, - Name = new List() { new HumanName() { Family = $"Test{i}", Given = new List { "Export", "History", "SoftDelete" } } }, - }); - } - - return resources; - } - private async System.Threading.Tasks.Task> SaveResourceListToServer(List<(Resource resource, bool delete)> entries) { if (entries.Count > 500) @@ -157,10 +163,7 @@ private async System.Threading.Tasks.Task> SaveResourceListToServ FhirResponse response = await TestFhirClient.PostBundleAsync(bundle); - if (response.StatusCode != System.Net.HttpStatusCode.OK) - { - throw new Exception("Could not save resources to server."); - } + response.Response.EnsureSuccessStatusCode(); List rtn = new(); @@ -185,48 +188,8 @@ private async System.Threading.Tasks.Task> SaveResourceListToServ return rtn; } - /* - private static List GenerateTestResources(int numberOfPatients = 20) + private List GenerateTestResources(int numberOfPatients = 5, int numberOfEncountersPerPatient = 1, int numberOfObservationsPerEncounter = 2) { - string[] firstNames = { "John", "Jane", "Robert", "Emily", "Michael", "Sarah", "William", "Anna", "James", "Laura" }; - - string[] lastNames = { "Smith", "Johnson", "Williams", "Brown", "Jones", "Garcia", "Miller", "Davis", "Rodriguez", "Martinez" }; - - static string RandomBirthdate() - { - DateTime startDate = new DateTime(1970, 1, 1); - int range = (DateTime.Today - startDate).Days; - return startDate.AddDays(_random.Next(range)).ToString("yyyy-MM-dd"); - } - - string RandomLastName() => lastNames[_random.Next(lastNames.Length)]; - - string RandomFirstName() => firstNames[_random.Next(firstNames.Length)]; - - Encounter CreateEncounter(string patientId) - { - return new() - { - Id = Guid.NewGuid().ToString("N"), - Meta = new() { Tag = new List() { new Coding("http://e2e-test", FixtureTag) }}, - Status = Encounter.EncounterStatus.Planned, - Subject = new ResourceReference($"Patient/{patientId}"), - }; - } - - Observation CreateObservation(string patientId, string encounterId) - { - return new() - { - Id = Guid.NewGuid().ToString("N"), - Meta = new() { Tag = new List() { new Coding("http://e2e-test", FixtureTag) }}, - Status = ObservationStatus.Preliminary, - Code = new CodeableConcept("http://loinc.org", "12345-6"), - Subject = new ResourceReference($"Patient/{patientId}"), - Encounter = new ResourceReference($"Encounter/{encounterId}"), - }; - } - var resources = new List(); for (int i = 0; i < numberOfPatients; i++) @@ -236,40 +199,42 @@ Observation CreateObservation(string patientId, string encounterId) Id = Guid.NewGuid().ToString("N"), Meta = new() { Tag = new List() { new Coding("http://e2e-test", FixtureTag) }}, Active = true, - Name = new List() { new HumanName() { Family = RandomLastName(), Given = new List { RandomFirstName() }} }, - BirthDate = RandomBirthdate(), + Name = new List() { new HumanName() { Family = $"Test{i}", Given = new List { "Export", "History", "SoftDelete" } } }, }; resources.Add(patient); - for (int j = 0; j < _random.Next(1, 5); j++) + for (int j = 0; j < numberOfEncountersPerPatient; j++) { - var encounter = CreateEncounter(patient.Id); + Encounter encounter = new() + { + Id = Guid.NewGuid().ToString("N"), + Meta = new() { Tag = new List() { new Coding("http://e2e-test", FixtureTag) } }, + Status = Encounter.EncounterStatus.Planned, + Type = new() { new CodeableConcept("http://e2e-test", $"Test{i}") }, + Class = new Coding("http://e2e-test", $"Test{i}"), + Subject = new ResourceReference($"Patient/{patient.Id}"), + }; resources.Add(encounter); - for (int k = 0; k < _random.Next(1, 5); k++) + for (int k = 0; k < numberOfObservationsPerEncounter; k++) { - var observation = CreateObservation(patient.Id, encounter.Id); - resources.Add(observation); - - while (observation.Id.GetHashCode() % 4 != 0) + Observation observation = new() { - observation = observation.DeepCopy() as Observation; - observation.Code.Coding.Add(new Coding("http://loinc.org", "12345-1")); - resources.Add(observation); - } - } - - while (encounter.Id.GetHashCode() % 4 != 0) - { - encounter = encounter.DeepCopy() as Encounter; - encounter.ClassHistory.Add(new Encounter.ClassHistoryComponent() { Class = new Coding("http://hl7.org/fhir/v3/ActCode", "EMER") }); - resources.Add(encounter); + Id = Guid.NewGuid().ToString("N"), + Meta = new() { Tag = new List() { new Coding("http://e2e-test", FixtureTag) } }, + Status = ObservationStatus.Preliminary, + Category = new() { new CodeableConcept("http://e2e-test", $"Test{i}") }, + Code = new CodeableConcept("http://e2e-test", $"Test{i}"), + Subject = new ResourceReference($"Patient/{patient.Id}"), + + // Encounter = new ResourceReference($"Encounter/{encounter.Id}"), + }; + resources.Add(observation); } } } return resources; } - */ } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportLongRunningTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs similarity index 78% rename from test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportLongRunningTests.cs rename to test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs index eda78d7143..faf29be2f2 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportLongRunningTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs @@ -17,19 +17,19 @@ using Xunit.Abstractions; using Task = System.Threading.Tasks.Task; -namespace Microsoft.Health.Fhir.Tests.E2E.Rest +namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Export { [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.Export)] [HttpIntegrationFixtureArgumentSets(DataStore.All, Format.Json)] - public class ExportLongRunningTests : IClassFixture + public class ExportDataTests : IClassFixture { private readonly TestFhirClient _testFhirClient; private readonly ITestOutputHelper _outputHelper; private readonly FhirJsonParser _fhirJsonParser; - private readonly ExportTestFixture _fixture; + private readonly ExportDataTestFixture _fixture; - public ExportLongRunningTests(ExportTestFixture fixture, ITestOutputHelper testOutputHelper) + public ExportDataTests(ExportDataTestFixture fixture, ITestOutputHelper testOutputHelper) { _testFhirClient = fixture.TestFhirClient; _outputHelper = testOutputHelper; @@ -43,15 +43,18 @@ public async Task GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAs // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. // Trigger export request and check for export status - Uri contentLocation = await _testFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:o}"); - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); + Uri contentLocation = await ExportTestHelper.StartExportAsync( + _fixture.TestFhirClient, + parameters: $"_since={_fixture.TestDataInsertionTime:o}"); + + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResources, dataFromExport, _outputHelper)); + Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResources, dataFromExport, _outputHelper, true)); } [Fact] @@ -60,8 +63,12 @@ public async Task GivenFhirServer_WhenPatientDataIsExported_ThenExportedDataIsSa // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. // Trigger export request and check for export status - Uri contentLocation = await _testFhirClient.ExportAsync(path: "Patient/", parameters: _fixture.ExportTestResourcesQueryParameters); - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); + Uri contentLocation = await ExportTestHelper.StartExportAsync( + _fixture.TestFhirClient, + path: "Patient/", + parameters: _fixture.ExportTestResourcesQueryParameters); + + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = @@ -77,8 +84,11 @@ public async Task GivenFhirServer_WhenAllObservationAndPatientDataIsExported_The // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. // Trigger export request and check for export status - Uri contentLocation = await _testFhirClient.ExportAsync(parameters: $"_type=Observation,Patient&{_fixture.ExportTestResourcesQueryParameters}"); - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); + Uri contentLocation = await ExportTestHelper.StartExportAsync( + _fixture.TestFhirClient, + parameters: $"_type=Observation,Patient&{_fixture.ExportTestResourcesQueryParameters}"); + + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = @@ -94,8 +104,12 @@ public async Task GivenFhirServer_WhenPatientObservationDataIsExported_ThenExpor // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. // Trigger export request and check for export status - Uri contentLocation = await _testFhirClient.ExportAsync("Patient/", $"_type=Observation&_typeFilter=Observation%3F_tag%3D{_fixture.FixtureTag}"); - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); + Uri contentLocation = await ExportTestHelper.StartExportAsync( + _fixture.TestFhirClient, + path: "Patient/", + parameters: $"_type=Observation&_typeFilter=Observation%3F_tag%3D{_fixture.FixtureTag}"); + + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 3); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = @@ -113,12 +127,14 @@ public async Task GivenFhirServer_WhenPatientObservationDataIsExported_ThenExpor public async Task GivenFhirServer_WhenAllDataIsExportedToASpecificContainer_ThenExportedDataIsInTheSpecifiedContianer() { // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. - string testContainer = "test-container"; // Trigger export request and check for export status - Uri contentLocation = await _testFhirClient.ExportAsync(parameters: $"_container={testContainer}&{_fixture.ExportTestResourcesQueryParameters}"); - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); + Uri contentLocation = await ExportTestHelper.StartExportAsync( + _fixture.TestFhirClient, + parameters: $"_container={testContainer}&{_fixture.ExportTestResourcesQueryParameters}"); + + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 3); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = @@ -135,15 +151,18 @@ public async Task GivenFhirServer_WhenDataIsExportedWithHistory_ThenExportedData // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. - Uri contentLocation = await _testFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:O}&_includeHistory=true"); - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); + Uri contentLocation = await ExportTestHelper.StartExportAsync( + _fixture.TestFhirClient, + parameters: $"_since={_fixture.TestDataInsertionTime:O}&_includeHistory=true"); + + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 3); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResourcesWithHistory, dataFromExport, _outputHelper)); + Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResourcesWithHistory, dataFromExport, _outputHelper, true)); } [Fact] @@ -152,15 +171,18 @@ public async Task GivenFhirServer_WhenDataIsExportedWithSoftDeletes_ThenExported // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. - Uri contentLocation = await _testFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:O}&_includeDeleted=true"); - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); + Uri contentLocation = await ExportTestHelper.StartExportAsync( + _fixture.TestFhirClient, + parameters: $"_since={_fixture.TestDataInsertionTime:O}&_includeDeleted=true"); + + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 3); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResourcesWithDeletes, dataFromExport, _outputHelper)); + Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResourcesWithDeletes, dataFromExport, _outputHelper, true)); } [Fact] @@ -169,15 +191,18 @@ public async Task GivenFhirServer_WhenDataIsExportedWithHistoryAndSoftDeletes_Th // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. - Uri contentLocation = await _testFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:O}&_includeHistory=true&_includeDeleted=true"); - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); + Uri contentLocation = await ExportTestHelper.StartExportAsync( + _fixture.TestFhirClient, + parameters: $"_since={_fixture.TestDataInsertionTime:O}&_includeHistory=true&_includeDeleted=true"); + + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 3); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResourcesWithHistoryAndDeletes, dataFromExport, _outputHelper)); + Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResourcesWithHistoryAndDeletes, dataFromExport, _outputHelper, true)); } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportDataValidationTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataValidationTests.cs similarity index 97% rename from test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportDataValidationTests.cs rename to test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataValidationTests.cs index db656a5a00..468220bccc 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportDataValidationTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataValidationTests.cs @@ -19,20 +19,20 @@ using FhirGroup = Hl7.Fhir.Model.Group; using Task = System.Threading.Tasks.Task; -namespace Microsoft.Health.Fhir.Tests.E2E.Rest +namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Export { [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.Export)] [Trait(Traits.Category, Categories.ExportDataValidation)] [HttpIntegrationFixtureArgumentSets(DataStore.All, Format.Json)] - public class ExportDataValidationTests : IClassFixture + public class ExportDataValidationTests : IClassFixture { private readonly TestFhirClient _testFhirClient; private readonly ITestOutputHelper _outputHelper; private readonly FhirJsonParser _fhirJsonParser; - private readonly ExportTestFixture _fixture; + private readonly ExportDataTestFixture _fixture; - public ExportDataValidationTests(ExportTestFixture fixture, ITestOutputHelper testOutputHelper) + public ExportDataValidationTests(ExportDataTestFixture fixture, ITestOutputHelper testOutputHelper) { _testFhirClient = fixture.TestFhirClient; _outputHelper = testOutputHelper; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestHelper.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs similarity index 95% rename from test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestHelper.cs rename to test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs index 9a4f1355e4..a6562dfe1d 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestHelper.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs @@ -19,7 +19,7 @@ using Xunit.Abstractions; using Task = System.Threading.Tasks.Task; -namespace Microsoft.Health.Fhir.Tests.E2E.Rest +namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Export { internal static class ExportTestHelper { @@ -251,11 +251,12 @@ internal static (StorageSharedKeyCredential credential, string connectionString) internal static bool ValidateDataFromBothSources( Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromServer, Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromStorageAccount, - ITestOutputHelper outputHelper) + ITestOutputHelper outputHelper, + bool allowDataFromServerToBeSubsetOfExportData = false) { bool result = true; - if (dataFromStorageAccount.Count != dataFromServer.Count) + if (dataFromStorageAccount.Count != dataFromServer.Count && (!allowDataFromServerToBeSubsetOfExportData || dataFromServer.Count < dataFromStorageAccount.Count)) { outputHelper.WriteLine($"Count differs. Exported data count: {dataFromStorageAccount.Count} Fhir Server Count: {dataFromServer.Count}"); result = false; @@ -270,13 +271,11 @@ internal static bool ValidateDataFromBothSources( } // Enable this check when creating/updating data validation tests to ensure there is data to export - /* if (dataFromStorageAccount.Count == 0) { - _outputHelper.WriteLine("No data exported. This test expects data to be present."); + outputHelper.WriteLine("No data exported. This test expects data to be present."); return false; } - */ int wrongCount = 0; foreach (KeyValuePair<(string resourceType, string resourceId, string versionId), Resource> kvp in dataFromServer) @@ -300,8 +299,14 @@ internal static bool ValidateDataFromBothSources( } } - outputHelper.WriteLine($"Missing or wrong match count: {wrongCount}"); return result; } + + internal static async Task StartExportAsync(TestFhirClient testFhirClient, string path = "", string parameters = "") + { + Uri contentLocation = await testFhirClient.ExportAsync(path: path, parameters: parameters); + + return contentLocation; + } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTests.cs similarity index 99% rename from test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTests.cs rename to test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTests.cs index c59d5ff679..6fdd1382bb 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTests.cs @@ -18,7 +18,7 @@ using Microsoft.Net.Http.Headers; using Xunit; -namespace Microsoft.Health.Fhir.Tests.E2E.Rest +namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Export { [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.Export)] diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/StartupForExportTestProvider.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/StartupForExportTestProvider.cs similarity index 96% rename from test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/StartupForExportTestProvider.cs rename to test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/StartupForExportTestProvider.cs index bb9be374ec..018228e45b 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/StartupForExportTestProvider.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/StartupForExportTestProvider.cs @@ -13,7 +13,7 @@ using Microsoft.Health.Fhir.Core.Features.Operations.Export; using Microsoft.Health.Fhir.Tests.E2E.Rest.Metric; -namespace Microsoft.Health.Fhir.Tests.E2E.Rest +namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Export { [RequiresIsolatedDatabase] public class StartupForExportTestProvider : StartupBaseForCustomProviders diff --git a/test/Microsoft.Health.Fhir.Stu3.Tests.E2E/Microsoft.Health.Fhir.Stu3.Tests.E2E.csproj b/test/Microsoft.Health.Fhir.Stu3.Tests.E2E/Microsoft.Health.Fhir.Stu3.Tests.E2E.csproj index 3143ad0c2a..45a4767080 100644 --- a/test/Microsoft.Health.Fhir.Stu3.Tests.E2E/Microsoft.Health.Fhir.Stu3.Tests.E2E.csproj +++ b/test/Microsoft.Health.Fhir.Stu3.Tests.E2E/Microsoft.Health.Fhir.Stu3.Tests.E2E.csproj @@ -14,8 +14,8 @@ - - + + From ab46e87bf259630fbcbdc42ad15c65c6a686b230 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 28 Sep 2023 21:37:22 -0700 Subject: [PATCH 25/58] Fixed export issues found in testing --- .../Export/Models/ExportJobFilter.cs | 60 +++++++ .../Features/Search/SearchOptions.cs | 3 + .../QueryGenerators/SqlQueryGenerator.cs | 6 +- .../Features/Search/SqlSearchOptions.cs | 17 ++ .../Features/Search/SqlServerSearchService.cs | 8 +- .../Rest/Export/ExportDataTestFixture.cs | 110 +++++++------ .../Rest/Export/ExportDataTests.cs | 147 ++++++++++++++---- .../Rest/Export/ExportTestHelper.cs | 2 +- 8 files changed, 266 insertions(+), 87 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/Models/ExportJobFilter.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/Models/ExportJobFilter.cs index 617dc2cde2..a45f6cb59a 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/Models/ExportJobFilter.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/Models/ExportJobFilter.cs @@ -31,5 +31,65 @@ public ExportJobFilter() [JsonProperty(JobRecordProperties.SearchParams)] public IList> Parameters { get; private set; } + + public static bool operator ==(ExportJobFilter left, ExportJobFilter right) + { + return object.Equals(left, right); + } + + public static bool operator !=(ExportJobFilter left, ExportJobFilter right) + { + return !object.Equals(left, right); + } + + public override bool Equals(object obj) + { + if (ReferenceEquals(this, obj)) + { + return true; + } + + if (obj == null || GetType() != obj.GetType()) + { + return false; + } + + ExportJobFilter other = (ExportJobFilter)obj; + return ResourceType == other.ResourceType && AreParametersEqual(Parameters, other.Parameters); + } + + public override int GetHashCode() + { + unchecked + { + int hash = 17; + hash = (hash * 23) + (ResourceType?.GetHashCode(StringComparison.InvariantCulture) ?? 0); + + foreach (var param in Parameters) + { + hash = (hash * 23) + (param?.GetHashCode() ?? 0); + } + + return hash; + } + } + + private static bool AreParametersEqual(IList> a, IList> b) + { + if (a.Count != b.Count) + { + return false; + } + + for (int i = 0; i < a.Count; i++) + { + if (a[i].Item1 != b[i].Item1 || a[i].Item2 != b[i].Item2) + { + return false; + } + } + + return true; + } } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SearchOptions.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SearchOptions.cs index 12a156bb8f..c49c137cbe 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/SearchOptions.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SearchOptions.cs @@ -48,6 +48,9 @@ internal SearchOptions(SearchOptions other) } QueryHints = other.QueryHints; + + IncludeHistory = other.IncludeHistory; + IncludeDeleted = other.IncludeDeleted; } /// diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs index c5b4004c1d..c5bd5b9f56 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs @@ -1319,7 +1319,9 @@ private void AppendDeletedClause(in IndentedStringBuilder.DelimitedScope delimit { if (!_searchType.HasFlag(SqlSearchType.IncludeDeleted)) { - delimited.BeginDelimitedElement().Append(VLatest.Resource.IsDeleted, tableAlias).Append(" = 0"); + delimited.BeginDelimitedElement(); + + StringBuilder.Append(VLatest.Resource.IsDeleted, tableAlias).Append(" = 0 "); } } @@ -1329,7 +1331,7 @@ private void AppendHistoryClause(in IndentedStringBuilder.DelimitedScope delimit { delimited.BeginDelimitedElement(); - StringBuilder.Append(VLatest.Resource.IsHistory, tableAlias).Append(" = 0"); + StringBuilder.Append(VLatest.Resource.IsHistory, tableAlias).Append(" = 0 "); } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlSearchOptions.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlSearchOptions.cs index 0dae4ca022..f4dbcfac6d 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlSearchOptions.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlSearchOptions.cs @@ -34,5 +34,22 @@ public SqlSearchOptions(SearchOptions searchOptions) /// Performs a shallow clone of this instance /// public SqlSearchOptions CloneSqlSearchOptions() => (SqlSearchOptions)MemberwiseClone(); + + internal SqlSearchType GetSearchTypeFromOptions() + { + var searchType = SqlSearchType.Default; + + if (IncludeHistory) + { + searchType |= SqlSearchType.IncludeHistory; + } + + if (IncludeDeleted) + { + searchType |= SqlSearchType.IncludeDeleted; + } + + return searchType; + } } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs index 850e10e619..e66f29c6c8 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs @@ -123,9 +123,7 @@ public override async Task SearchAsync(SearchOptions searchOptions { SqlSearchOptions sqlSearchOptions = new SqlSearchOptions(searchOptions); - SqlSearchType searchType = SqlSearchType.Default; - searchType |= sqlSearchOptions.IncludeHistory ? SqlSearchType.IncludeHistory : searchType; - searchType |= sqlSearchOptions.IncludeDeleted ? SqlSearchType.IncludeDeleted : searchType; + SqlSearchType searchType = sqlSearchOptions.GetSearchTypeFromOptions(); SearchResult searchResult = await SearchImpl(sqlSearchOptions, searchType, null, cancellationToken); int resultCount = searchResult.Results.Count(); @@ -704,9 +702,9 @@ public override async Task> GetUsedResourceTypes(Cancellat private SqlSearchOptions UpdateSort(SqlSearchOptions searchOptions, Expression searchExpression, SqlSearchType sqlSearchType) { SqlSearchOptions newSearchOptions = searchOptions; - if (sqlSearchType.HasFlag(SqlSearchType.IncludeHistory)) + if (sqlSearchType.HasFlag(SqlSearchType.IncludeHistory) && searchOptions.Sort.Any()) { - // history is always sorted by _lastUpdated. + // history is always sorted by _lastUpdated (except for export). newSearchOptions = searchOptions.CloneSqlSearchOptions(); return newSearchOptions; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs index f68fcd2540..5832acc029 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs @@ -6,6 +6,8 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text.RegularExpressions; +using Antlr4.Runtime.Atn; using Hl7.Fhir.Model; using MediatR; using Microsoft.Extensions.DependencyInjection; @@ -48,13 +50,29 @@ public MetricHandler MetricHandler public Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestResources => TestResourcesWithHistory.Where(pair => TestResourcesWithDeletes.ContainsKey(pair.Key)).ToDictionary(pair => pair.Key, pair => pair.Value); + // If the patient is deleted but the child resources are not, they should not be returned in patient centric exports. + public Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestPatientCompartmentResources => TestResources + .Where(x => x.Key.resourceType != "Encounter" || TestResources.Keys.Any(pat => pat.resourceType == "Patient" && pat.resourceId == (x.Value as Encounter).Subject.Reference.Split("/")[1])) + .Where(x => x.Key.resourceType != "Observation" || TestResources.Keys.Any(pat => pat.resourceType == "Patient" && pat.resourceId == (x.Value as Observation).Subject.Reference.Split("/")[1])) + .ToDictionary(pair => pair.Key, pair => pair.Value); + public Dictionary ExportTestCasesContentUrls { get; } = new(); public string FixtureTag { get; } = Guid.NewGuid().ToString(); public DateTime TestDataInsertionTime { get; } = DateTime.UtcNow; - public string ExportTestResourcesQueryParameters => $"_type=Patient,Observation&_typeFilter=Patient%3F_tag%3D{FixtureTag},Observation%3F_tag%3D{FixtureTag}"; + public string ExportTestFilterQueryParameters(params string[] uniqueResourceTypes) + { + if (uniqueResourceTypes.Length == 0) + { + uniqueResourceTypes = TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct().ToArray(); + } + + var typeFilterPart = string.Join(',', uniqueResourceTypes.Select(rt => $"{rt}%3F_tag%3D{FixtureTag}")); + + return $"_type={string.Join(',', uniqueResourceTypes)}&_typeFilter={typeFilterPart}"; + } protected override async Task OnInitializedAsync() { @@ -74,64 +92,55 @@ void AddResourceToTestResources(Resource resource) => { var testResourceResponse = await SaveResourceListToServer(testResourcesInfo); - try - { - AddResourcesToTestResources(testResourceResponse); + AddResourcesToTestResources(testResourceResponse); - testResourcesInfo = new(); + testResourcesInfo = new(); - for (int i = 0; i < testResourceResponse.Count; i++) + for (int i = 0; i < testResourceResponse.Count; i++) + { + var resource = testResourceResponse[i]; + + if (resource.Meta.Extension.Any(x => x.Url == KnownFhirPaths.AzureSoftDeletedExtensionUrl)) + { + // Skip already deleted resources for now. + // TODO - add un-deletes in here. + continue; + } + else if (i % 10 == 0) { - var resource = testResourceResponse[i]; + testResourcesInfo.Add((resource.DeepCopy() as Resource, true)); + } + else if (i % 4 == 1) + { + Resource updatedResource = resource.DeepCopy() as Resource; - if (resource.Meta.Extension.Any(x => x.Url == KnownFhirPaths.AzureSoftDeletedExtensionUrl)) + if (updatedResource is Patient) { - // Skip already deleted resources for now. - // TODO - add un-deletes in here. - continue; + Patient updatedPatient = updatedResource as Patient; + updatedPatient.Name.Add(new() + { + Given = updatedPatient.Name.First().Given, + Family = $"UpdatedFromVersion{updatedResource.Meta.VersionId}", + }); + testResourcesInfo.Add((updatedPatient, false)); } - else if (i % 10 == 0) + + if (updatedResource is Encounter) { - testResourcesInfo.Add((resource.DeepCopy() as Resource, true)); + Encounter updatedEncounter = updatedResource as Encounter; + updatedEncounter.Type.Add(new CodeableConcept("http://e2e-test", $"UpdatedFromVersion{updatedResource.Meta.VersionId}")); + testResourcesInfo.Add((updatedEncounter, false)); } - else if (i % 4 == 1) - { - Resource updatedResource = resource.DeepCopy() as Resource; - - if (updatedResource is Patient) - { - Patient updatedPatient = updatedResource as Patient; - updatedPatient.Name.Add(new() - { - Given = updatedPatient.Name.First().Given, - Family = $"UpdatedFromVersion{updatedResource.Meta.VersionId}", - }); - testResourcesInfo.Add((updatedPatient, false)); - } - - if (updatedResource is Encounter) - { - Encounter updatedEncounter = updatedResource as Encounter; - updatedEncounter.Type.Add(new CodeableConcept("http://e2e-test", $"UpdatedFromVersion{updatedResource.Meta.VersionId}")); - testResourcesInfo.Add((updatedEncounter, false)); - } - if (updatedResource is Observation) - { - Observation updatedObservation = updatedResource as Observation; - updatedObservation.Category.Add(new CodeableConcept("http://e2e-test", $"UpdatedFromVersion{updatedResource.Meta.VersionId}")); - testResourcesInfo.Add((updatedObservation, false)); - } + if (updatedResource is Observation) + { + Observation updatedObservation = updatedResource as Observation; + updatedObservation.Category.Add(new CodeableConcept("http://e2e-test", $"UpdatedFromVersion{updatedResource.Meta.VersionId}")); + testResourcesInfo.Add((updatedObservation, false)); } } } - catch (Exception ex) - { - Console.WriteLine(ex); - } } - - Console.WriteLine($"Generated {TestResourcesWithHistoryAndDeletes.Count} resources."); } private async System.Threading.Tasks.Task> SaveResourceListToServer(List<(Resource resource, bool delete)> entries) @@ -181,6 +190,15 @@ private async System.Threading.Tasks.Task> SaveResourceListToServ var allResourcesWithDeleted = await TestFhirClient.SearchAsync($"{inputResource.TypeName}/{inputResource.Id}/_history"); var deletedResource = allResourcesWithDeleted.Resource.Entry.OrderByDescending(x => x.Resource.Meta.LastUpdated).First().Resource; deletedResource.Meta.Extension.Add(new Extension(KnownFhirPaths.AzureSoftDeletedExtensionUrl, new FhirString("soft-deleted"))); + + // The history endpoint does not return the version id in the resource, so we need to parse it from the etag. + var etagVersionMatch = Regex.Match(responseEntry.Response.Etag, @"\d+"); + + if (deletedResource.Meta.VersionId is null && etagVersionMatch.Success) + { + deletedResource.Meta.VersionId = etagVersionMatch.Value; + } + rtn.Add(deletedResource); } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs index faf29be2f2..b599c54ac1 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs @@ -37,15 +37,31 @@ public ExportDataTests(ExportDataTestFixture fixture, ITestOutputHelper testOutp _fixture = fixture; } - [Fact] - public async Task GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() + [Theory] + [InlineData("since")] + [InlineData("tag")] + public async Task GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAsDataInFhirServer(string parametersKey) { // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. + string parameters = "not set"; + bool allowDataFromServerToBeSubsetOfExportData = false; + + if (parametersKey == "tag") + { + parameters = _fixture.ExportTestFilterQueryParameters(); + } + else if (parametersKey == "since") + { + var uniqueFixtureResources = string.Join(',', _fixture.TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); + parameters = $"_since={_fixture.TestDataInsertionTime:o}&_type={uniqueFixtureResources}&_isParallel=false"; + allowDataFromServerToBeSubsetOfExportData = true; + } + // Trigger export request and check for export status Uri contentLocation = await ExportTestHelper.StartExportAsync( _fixture.TestFhirClient, - parameters: $"_since={_fixture.TestDataInsertionTime:o}"); + parameters: parameters); IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); @@ -54,19 +70,35 @@ public async Task GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAs await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResources, dataFromExport, _outputHelper, true)); + Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResources, dataFromExport, _outputHelper, allowDataFromServerToBeSubsetOfExportData)); } - [Fact] - public async Task GivenFhirServer_WhenPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() + [Theory] + [InlineData("since")] + [InlineData("tag")] + public async Task GivenFhirServer_WhenPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer(string parametersKey) { // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. + string parameters = "not set"; + bool allowDataFromServerToBeSubsetOfExportData = false; + + if (parametersKey == "tag") + { + parameters = _fixture.ExportTestFilterQueryParameters(); + } + else if (parametersKey == "since") + { + var uniqueFixtureResources = string.Join(',', _fixture.TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); + parameters = $"_since={_fixture.TestDataInsertionTime:o}&_type={uniqueFixtureResources}"; + allowDataFromServerToBeSubsetOfExportData = true; + } + // Trigger export request and check for export status Uri contentLocation = await ExportTestHelper.StartExportAsync( _fixture.TestFhirClient, path: "Patient/", - parameters: _fixture.ExportTestResourcesQueryParameters); + parameters: parameters); IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); @@ -75,18 +107,34 @@ public async Task GivenFhirServer_WhenPatientDataIsExported_ThenExportedDataIsSa await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResources, dataFromExport, _outputHelper)); + Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestPatientCompartmentResources, dataFromExport, _outputHelper, allowDataFromServerToBeSubsetOfExportData)); } - [Fact] - public async Task GivenFhirServer_WhenAllObservationAndPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() + [Theory] + [InlineData("since")] + [InlineData("tag")] + public async Task GivenFhirServer_WhenAllObservationAndPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer(string parametersKey) { // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. + string[] testResorceTypes = { "Observation", "Patient" }; + string parameters = "not set"; + bool allowDataFromServerToBeSubsetOfExportData = false; + + if (parametersKey == "tag") + { + parameters = _fixture.ExportTestFilterQueryParameters(testResorceTypes); + } + else if (parametersKey == "since") + { + parameters = $"_since={_fixture.TestDataInsertionTime:o}&_type={string.Join(',', testResorceTypes)}"; + allowDataFromServerToBeSubsetOfExportData = true; + } + // Trigger export request and check for export status Uri contentLocation = await ExportTestHelper.StartExportAsync( _fixture.TestFhirClient, - parameters: $"_type=Observation,Patient&{_fixture.ExportTestResourcesQueryParameters}"); + parameters: parameters); IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); @@ -94,35 +142,55 @@ public async Task GivenFhirServer_WhenAllObservationAndPatientDataIsExported_The Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); + var expectedResources = _fixture.TestResources + .Where(r => testResorceTypes.Contains(r.Key.resourceType)) + .ToDictionary(x => x.Key, x => x.Value); + // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResources, dataFromExport, _outputHelper)); + Assert.True(ExportTestHelper.ValidateDataFromBothSources(expectedResources, dataFromExport, _outputHelper, allowDataFromServerToBeSubsetOfExportData)); } - [Fact] - public async Task GivenFhirServer_WhenPatientObservationDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() + [Theory] + [InlineData("since")] + [InlineData("tag")] + public async Task GivenFhirServer_WhenPatientObservationDataIsExported_ThenExportedDataIsSameAsDataInFhirServer(string parametersKey) { // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. + string parameters = "not set"; + bool allowDataFromServerToBeSubsetOfExportData = false; + + if (parametersKey == "tag") + { + parameters = _fixture.ExportTestFilterQueryParameters("Observation"); + } + else if (parametersKey == "since") + { + parameters = $"_since={_fixture.TestDataInsertionTime:o}&type=Observation"; + allowDataFromServerToBeSubsetOfExportData = true; + } + // Trigger export request and check for export status Uri contentLocation = await ExportTestHelper.StartExportAsync( _fixture.TestFhirClient, path: "Patient/", - parameters: $"_type=Observation&_typeFilter=Observation%3F_tag%3D{_fixture.FixtureTag}"); + parameters: _fixture.ExportTestFilterQueryParameters("Observation")); - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 3); + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); - var expectedResources = _fixture.TestResources - .Where(r => r.Key.resourceType == ResourceType.Observation.ToString()) + var expectedResources = _fixture.TestPatientCompartmentResources + .Where(r => r.Key.resourceType == "Observation") .ToDictionary(x => x.Key, x => x.Value); // Assert both data are equal. Only Observation data is expected due to the type query parameter. - Assert.True(ExportTestHelper.ValidateDataFromBothSources(expectedResources, dataFromExport, _outputHelper)); + Assert.True(ExportTestHelper.ValidateDataFromBothSources(expectedResources, dataFromExport, _outputHelper, allowDataFromServerToBeSubsetOfExportData)); } + // No need to test both code paths for testing container is written to. [Fact] public async Task GivenFhirServer_WhenAllDataIsExportedToASpecificContainer_ThenExportedDataIsInTheSpecifiedContianer() { @@ -132,9 +200,9 @@ public async Task GivenFhirServer_WhenAllDataIsExportedToASpecificContainer_Then // Trigger export request and check for export status Uri contentLocation = await ExportTestHelper.StartExportAsync( _fixture.TestFhirClient, - parameters: $"_container={testContainer}&{_fixture.ExportTestResourcesQueryParameters}"); + parameters: $"_container={testContainer}&{_fixture.ExportTestFilterQueryParameters()}"); - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 3); + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = @@ -145,37 +213,47 @@ public async Task GivenFhirServer_WhenAllDataIsExportedToASpecificContainer_Then Assert.True(blobUris.All((url) => url.OriginalString.Contains(testContainer))); } - [Fact] - public async Task GivenFhirServer_WhenDataIsExportedWithHistory_ThenExportedDataIsSameAsDataInFhirServer() + // _tag filter cannot be used with history or deleted export. Using isParallel to test both SQL code paths. + [Theory] + [InlineData("_isParallel=true")] + [InlineData("_isParallel=false")] + public async Task GivenFhirServer_WhenDataIsExportedWithHistory_ThenExportedDataIsSameAsDataInFhirServer(string parallelQueryParam) { // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. Uri contentLocation = await ExportTestHelper.StartExportAsync( _fixture.TestFhirClient, - parameters: $"_since={_fixture.TestDataInsertionTime:O}&_includeHistory=true"); + parameters: $"_since={_fixture.TestDataInsertionTime:O}&_includeHistory=true&{parallelQueryParam}"); - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 3); + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); + // removeme + var missing = _fixture.TestResourcesWithHistory.Where(x => !dataFromExport.Any(y => x.Key.resourceId == y.Key.resourceId && x.Key.versionId == y.Key.versionId)); + var missingLastUpdated = missing.Select(x => x.Value.Meta.LastUpdated).ToList(); + // Assert both data are equal Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResourcesWithHistory, dataFromExport, _outputHelper, true)); } - [Fact] - public async Task GivenFhirServer_WhenDataIsExportedWithSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer() + // _tag filter cannot be used with history or deleted export. Using isParallel to test both SQL code paths. + [Theory] + [InlineData("_isParallel=true")] + [InlineData("_isParallel=false")] + public async Task GivenFhirServer_WhenDataIsExportedWithSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer(string parallelQueryParam) { // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. Uri contentLocation = await ExportTestHelper.StartExportAsync( _fixture.TestFhirClient, - parameters: $"_since={_fixture.TestDataInsertionTime:O}&_includeDeleted=true"); + parameters: $"_since={_fixture.TestDataInsertionTime:O}&_includeDeleted=true&{parallelQueryParam}"); - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 3); + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = @@ -185,17 +263,20 @@ public async Task GivenFhirServer_WhenDataIsExportedWithSoftDeletes_ThenExported Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResourcesWithDeletes, dataFromExport, _outputHelper, true)); } - [Fact] - public async Task GivenFhirServer_WhenDataIsExportedWithHistoryAndSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer() + // _tag filter cannot be used with history or deleted export. Using isParallel to test both SQL code paths. + [Theory] + [InlineData("_isParallel=true")] + [InlineData("_isParallel=false")] + public async Task GivenFhirServer_WhenDataIsExportedWithHistoryAndSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer(string parallelQueryParam) { // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. Uri contentLocation = await ExportTestHelper.StartExportAsync( _fixture.TestFhirClient, - parameters: $"_since={_fixture.TestDataInsertionTime:O}&_includeHistory=true&_includeDeleted=true"); + parameters: $"_since={_fixture.TestDataInsertionTime:O}&_includeHistory=true&_includeDeleted=true&{parallelQueryParam}"); - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 3); + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs index a6562dfe1d..228586d816 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs @@ -256,7 +256,7 @@ internal static bool ValidateDataFromBothSources( { bool result = true; - if (dataFromStorageAccount.Count != dataFromServer.Count && (!allowDataFromServerToBeSubsetOfExportData || dataFromServer.Count < dataFromStorageAccount.Count)) + if (dataFromStorageAccount.Count != dataFromServer.Count && !(allowDataFromServerToBeSubsetOfExportData && dataFromServer.Count < dataFromStorageAccount.Count)) { outputHelper.WriteLine($"Count differs. Exported data count: {dataFromStorageAccount.Count} Fhir Server Count: {dataFromServer.Count}"); result = false; From b81d871a597fc02d098a3f117abb7fbd9fedd4ef Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 29 Sep 2023 06:31:23 -0700 Subject: [PATCH 26/58] Changed SQL script version for merge --- .../Features/Schema/Migrations/64.sql | 6750 ----------------- .../Migrations/{64.diff.sql => 65.diff.sql} | 0 2 files changed, 6750 deletions(-) delete mode 100644 src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/64.sql rename src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/{64.diff.sql => 65.diff.sql} (100%) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/64.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/64.sql deleted file mode 100644 index ab1df0985b..0000000000 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/64.sql +++ /dev/null @@ -1,6750 +0,0 @@ - -/************************************************************************************************* - Auto-Generated from Sql build task. Do not manually edit it. -**************************************************************************************************/ -SET XACT_ABORT ON -BEGIN TRAN -IF EXISTS (SELECT * - FROM sys.tables - WHERE name = 'ClaimType') - BEGIN - ROLLBACK; - RETURN; - END - - -GO -INSERT INTO dbo.SchemaVersion -VALUES (64, 'started'); - -CREATE PARTITION FUNCTION PartitionFunction_ResourceTypeId(SMALLINT) - AS RANGE RIGHT - FOR VALUES (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150); - -CREATE PARTITION SCHEME PartitionScheme_ResourceTypeId - AS PARTITION PartitionFunction_ResourceTypeId - ALL TO ([PRIMARY]); - - -GO -CREATE PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp(DATETIME2 (7)) - AS RANGE RIGHT - FOR VALUES (N'1970-01-01T00:00:00.0000000'); - -CREATE PARTITION SCHEME PartitionScheme_ResourceChangeData_Timestamp - AS PARTITION PartitionFunction_ResourceChangeData_Timestamp - ALL TO ([PRIMARY]); - -DECLARE @numberOfHistoryPartitions AS INT = 48; - -DECLARE @numberOfFuturePartitions AS INT = 720; - -DECLARE @rightPartitionBoundary AS DATETIME2 (7); - -DECLARE @currentDateTime AS DATETIME2 (7) = sysutcdatetime(); - -WHILE @numberOfHistoryPartitions >= -@numberOfFuturePartitions - BEGIN - SET @rightPartitionBoundary = DATEADD(hour, DATEDIFF(hour, 0, @currentDateTime) - @numberOfHistoryPartitions, 0); - ALTER PARTITION SCHEME PartitionScheme_ResourceChangeData_Timestamp NEXT USED [Primary]; - ALTER PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp( ) - SPLIT RANGE (@rightPartitionBoundary); - SET @numberOfHistoryPartitions -= 1; - END - -CREATE SEQUENCE dbo.ResourceSurrogateIdUniquifierSequence - AS INT - START WITH 0 - INCREMENT BY 1 - MINVALUE 0 - MAXVALUE 79999 - CYCLE - CACHE 1000000; - -CREATE TYPE dbo.BigintList AS TABLE ( - Id BIGINT NOT NULL PRIMARY KEY); - -CREATE TYPE dbo.CompartmentAssignmentList AS TABLE ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - CompartmentTypeId TINYINT NOT NULL, - ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL PRIMARY KEY (ResourceTypeId, ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId)); - -CREATE TYPE dbo.DateTimeSearchParamList AS TABLE ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - StartDateTime DATETIMEOFFSET (7) NOT NULL, - EndDateTime DATETIMEOFFSET (7) NOT NULL, - IsLongerThanADay BIT NOT NULL, - IsMin BIT NOT NULL, - IsMax BIT NOT NULL UNIQUE (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax)); - -CREATE TYPE dbo.NumberSearchParamList AS TABLE ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SingleValue DECIMAL (36, 18) NULL, - LowValue DECIMAL (36, 18) NULL, - HighValue DECIMAL (36, 18) NULL UNIQUE (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue)); - -CREATE TYPE dbo.QuantitySearchParamList AS TABLE ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId INT NULL, - QuantityCodeId INT NULL, - SingleValue DECIMAL (36, 18) NULL, - LowValue DECIMAL (36, 18) NULL, - HighValue DECIMAL (36, 18) NULL UNIQUE (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue)); - -CREATE TYPE dbo.ReferenceSearchParamList AS TABLE ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - BaseUri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, - ReferenceResourceTypeId SMALLINT NULL, - ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, - ReferenceResourceVersion INT NULL UNIQUE (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId)); - -CREATE TYPE dbo.ReferenceTokenCompositeSearchParamList AS TABLE ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - BaseUri1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, - ReferenceResourceTypeId1 SMALLINT NULL, - ReferenceResourceId1 VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, - ReferenceResourceVersion1 INT NULL, - SystemId2 INT NULL, - Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, - CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); - -CREATE TYPE dbo.ResourceDateKeyList AS TABLE ( - ResourceTypeId SMALLINT NOT NULL, - ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, - ResourceSurrogateId BIGINT NOT NULL PRIMARY KEY (ResourceTypeId, ResourceId, ResourceSurrogateId)); - -CREATE TYPE dbo.ResourceKeyList AS TABLE ( - ResourceTypeId SMALLINT NOT NULL, - ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, - Version INT NULL UNIQUE (ResourceTypeId, ResourceId, Version)); - -CREATE TYPE dbo.ResourceList AS TABLE ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, - Version INT NOT NULL, - HasVersionToCompare BIT NOT NULL, - IsDeleted BIT NOT NULL, - IsHistory BIT NOT NULL, - KeepHistory BIT NOT NULL, - RawResource VARBINARY (MAX) NOT NULL, - IsRawResourceMetaSet BIT NOT NULL, - RequestMethod VARCHAR (10) NULL, - SearchParamHash VARCHAR (64) NULL PRIMARY KEY (ResourceTypeId, ResourceSurrogateId), - UNIQUE (ResourceTypeId, ResourceId, Version)); - -CREATE TYPE dbo.ResourceWriteClaimList AS TABLE ( - ResourceSurrogateId BIGINT NOT NULL, - ClaimTypeId TINYINT NOT NULL, - ClaimValue NVARCHAR (128) NOT NULL); - -CREATE TYPE dbo.StringList AS TABLE ( - String VARCHAR (MAX)); - -CREATE TYPE dbo.StringSearchParamList AS TABLE ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - Text NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, - TextOverflow NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL, - IsMin BIT NOT NULL, - IsMax BIT NOT NULL); - -CREATE TYPE dbo.TokenDateTimeCompositeSearchParamList AS TABLE ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId1 INT NULL, - Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, - CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, - StartDateTime2 DATETIMEOFFSET (7) NOT NULL, - EndDateTime2 DATETIMEOFFSET (7) NOT NULL, - IsLongerThanADay2 BIT NOT NULL); - -CREATE TYPE dbo.TokenNumberNumberCompositeSearchParamList AS TABLE ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId1 INT NULL, - Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, - CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, - SingleValue2 DECIMAL (36, 18) NULL, - LowValue2 DECIMAL (36, 18) NULL, - HighValue2 DECIMAL (36, 18) NULL, - SingleValue3 DECIMAL (36, 18) NULL, - LowValue3 DECIMAL (36, 18) NULL, - HighValue3 DECIMAL (36, 18) NULL, - HasRange BIT NOT NULL); - -CREATE TYPE dbo.TokenQuantityCompositeSearchParamList AS TABLE ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId1 INT NULL, - Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, - CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, - SystemId2 INT NULL, - QuantityCodeId2 INT NULL, - SingleValue2 DECIMAL (36, 18) NULL, - LowValue2 DECIMAL (36, 18) NULL, - HighValue2 DECIMAL (36, 18) NULL); - -CREATE TYPE dbo.TokenSearchParamList AS TABLE ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId INT NULL, - Code VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, - CodeOverflow VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); - -CREATE TYPE dbo.TokenStringCompositeSearchParamList AS TABLE ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId1 INT NULL, - Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, - CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, - Text2 NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, - TextOverflow2 NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL); - -CREATE TYPE dbo.TokenTextList AS TABLE ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - Text NVARCHAR (400) COLLATE Latin1_General_CI_AI NOT NULL); - -CREATE TYPE dbo.TokenTokenCompositeSearchParamList AS TABLE ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId1 INT NULL, - Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, - CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, - SystemId2 INT NULL, - Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, - CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); - -CREATE TYPE dbo.BulkResourceWriteClaimTableType_1 AS TABLE ( - Offset INT NOT NULL, - ClaimTypeId TINYINT NOT NULL, - ClaimValue NVARCHAR (128) NOT NULL); - -CREATE TYPE dbo.BulkCompartmentAssignmentTableType_1 AS TABLE ( - Offset INT NOT NULL, - CompartmentTypeId TINYINT NOT NULL, - ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL); - -CREATE TYPE dbo.BulkReferenceSearchParamTableType_1 AS TABLE ( - Offset INT NOT NULL, - SearchParamId SMALLINT NOT NULL, - BaseUri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, - ReferenceResourceTypeId SMALLINT NULL, - ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, - ReferenceResourceVersion INT NULL); - -CREATE TYPE dbo.BulkTokenSearchParamTableType_1 AS TABLE ( - Offset INT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId INT NULL, - Code VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL); - -CREATE TYPE dbo.BulkTokenSearchParamTableType_2 AS TABLE ( - Offset INT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId INT NULL, - Code VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, - CodeOverflow VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); - -CREATE TYPE dbo.BulkTokenTextTableType_1 AS TABLE ( - Offset INT NOT NULL, - SearchParamId SMALLINT NOT NULL, - Text NVARCHAR (400) COLLATE Latin1_General_CI_AI NOT NULL); - -CREATE TYPE dbo.BulkStringSearchParamTableType_1 AS TABLE ( - Offset INT NOT NULL, - SearchParamId SMALLINT NOT NULL, - Text NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, - TextOverflow NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL); - -CREATE TYPE dbo.BulkStringSearchParamTableType_2 AS TABLE ( - Offset INT NOT NULL, - SearchParamId SMALLINT NOT NULL, - Text NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, - TextOverflow NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL, - IsMin BIT NOT NULL, - IsMax BIT NOT NULL); - -CREATE TYPE dbo.BulkUriSearchParamTableType_1 AS TABLE ( - Offset INT NOT NULL, - SearchParamId SMALLINT NOT NULL, - Uri VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL); - -CREATE TYPE dbo.BulkNumberSearchParamTableType_1 AS TABLE ( - Offset INT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SingleValue DECIMAL (18, 6) NULL, - LowValue DECIMAL (18, 6) NULL, - HighValue DECIMAL (18, 6) NULL); - -CREATE TYPE dbo.BulkQuantitySearchParamTableType_1 AS TABLE ( - Offset INT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId INT NULL, - QuantityCodeId INT NULL, - SingleValue DECIMAL (18, 6) NULL, - LowValue DECIMAL (18, 6) NULL, - HighValue DECIMAL (18, 6) NULL); - -CREATE TYPE dbo.BulkDateTimeSearchParamTableType_1 AS TABLE ( - Offset INT NOT NULL, - SearchParamId SMALLINT NOT NULL, - StartDateTime DATETIMEOFFSET (7) NOT NULL, - EndDateTime DATETIMEOFFSET (7) NOT NULL, - IsLongerThanADay BIT NOT NULL); - -CREATE TYPE dbo.BulkDateTimeSearchParamTableType_2 AS TABLE ( - Offset INT NOT NULL, - SearchParamId SMALLINT NOT NULL, - StartDateTime DATETIMEOFFSET (7) NOT NULL, - EndDateTime DATETIMEOFFSET (7) NOT NULL, - IsLongerThanADay BIT NOT NULL, - IsMin BIT NOT NULL, - IsMax BIT NOT NULL); - -CREATE TYPE dbo.BulkReferenceTokenCompositeSearchParamTableType_1 AS TABLE ( - Offset INT NOT NULL, - SearchParamId SMALLINT NOT NULL, - BaseUri1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, - ReferenceResourceTypeId1 SMALLINT NULL, - ReferenceResourceId1 VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, - ReferenceResourceVersion1 INT NULL, - SystemId2 INT NULL, - Code2 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL); - -CREATE TYPE dbo.BulkReferenceTokenCompositeSearchParamTableType_2 AS TABLE ( - Offset INT NOT NULL, - SearchParamId SMALLINT NOT NULL, - BaseUri1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, - ReferenceResourceTypeId1 SMALLINT NULL, - ReferenceResourceId1 VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, - ReferenceResourceVersion1 INT NULL, - SystemId2 INT NULL, - Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, - CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); - -CREATE TYPE dbo.BulkTokenTokenCompositeSearchParamTableType_1 AS TABLE ( - Offset INT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId1 INT NULL, - Code1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, - SystemId2 INT NULL, - Code2 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL); - -CREATE TYPE dbo.BulkTokenTokenCompositeSearchParamTableType_2 AS TABLE ( - Offset INT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId1 INT NULL, - Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, - CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, - SystemId2 INT NULL, - Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, - CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); - -CREATE TYPE dbo.BulkTokenDateTimeCompositeSearchParamTableType_1 AS TABLE ( - Offset INT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId1 INT NULL, - Code1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, - StartDateTime2 DATETIMEOFFSET (7) NOT NULL, - EndDateTime2 DATETIMEOFFSET (7) NOT NULL, - IsLongerThanADay2 BIT NOT NULL); - -CREATE TYPE dbo.BulkTokenDateTimeCompositeSearchParamTableType_2 AS TABLE ( - Offset INT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId1 INT NULL, - Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, - CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, - StartDateTime2 DATETIMEOFFSET (7) NOT NULL, - EndDateTime2 DATETIMEOFFSET (7) NOT NULL, - IsLongerThanADay2 BIT NOT NULL); - -CREATE TYPE dbo.BulkTokenQuantityCompositeSearchParamTableType_1 AS TABLE ( - Offset INT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId1 INT NULL, - Code1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, - SystemId2 INT NULL, - QuantityCodeId2 INT NULL, - SingleValue2 DECIMAL (18, 6) NULL, - LowValue2 DECIMAL (18, 6) NULL, - HighValue2 DECIMAL (18, 6) NULL); - -CREATE TYPE dbo.BulkTokenQuantityCompositeSearchParamTableType_2 AS TABLE ( - Offset INT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId1 INT NULL, - Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, - CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, - SystemId2 INT NULL, - QuantityCodeId2 INT NULL, - SingleValue2 DECIMAL (18, 6) NULL, - LowValue2 DECIMAL (18, 6) NULL, - HighValue2 DECIMAL (18, 6) NULL); - -CREATE TYPE dbo.BulkTokenStringCompositeSearchParamTableType_1 AS TABLE ( - Offset INT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId1 INT NULL, - Code1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, - Text2 NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, - TextOverflow2 NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL); - -CREATE TYPE dbo.BulkTokenStringCompositeSearchParamTableType_2 AS TABLE ( - Offset INT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId1 INT NULL, - Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, - CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, - Text2 NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, - TextOverflow2 NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL); - -CREATE TYPE dbo.BulkTokenNumberNumberCompositeSearchParamTableType_1 AS TABLE ( - Offset INT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId1 INT NULL, - Code1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, - SingleValue2 DECIMAL (18, 6) NULL, - LowValue2 DECIMAL (18, 6) NULL, - HighValue2 DECIMAL (18, 6) NULL, - SingleValue3 DECIMAL (18, 6) NULL, - LowValue3 DECIMAL (18, 6) NULL, - HighValue3 DECIMAL (18, 6) NULL, - HasRange BIT NOT NULL); - -CREATE TYPE dbo.BulkTokenNumberNumberCompositeSearchParamTableType_2 AS TABLE ( - Offset INT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId1 INT NULL, - Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, - CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, - SingleValue2 DECIMAL (18, 6) NULL, - LowValue2 DECIMAL (18, 6) NULL, - HighValue2 DECIMAL (18, 6) NULL, - SingleValue3 DECIMAL (18, 6) NULL, - LowValue3 DECIMAL (18, 6) NULL, - HighValue3 DECIMAL (18, 6) NULL, - HasRange BIT NOT NULL); - -CREATE TYPE dbo.SearchParamTableType_1 AS TABLE ( - Uri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, - Status VARCHAR (10) NOT NULL, - IsPartiallySupported BIT NOT NULL); - -CREATE TYPE dbo.SearchParamTableType_2 AS TABLE ( - Uri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, - Status VARCHAR (20) NOT NULL, - IsPartiallySupported BIT NOT NULL); - -CREATE TYPE dbo.BulkReindexResourceTableType_1 AS TABLE ( - Offset INT NOT NULL, - ResourceTypeId SMALLINT NOT NULL, - ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, - ETag INT NULL, - SearchParamHash VARCHAR (64) NOT NULL); - -CREATE TYPE dbo.BulkImportResourceType_1 AS TABLE ( - ResourceTypeId SMALLINT NOT NULL, - ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, - Version INT NOT NULL, - IsHistory BIT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - IsDeleted BIT NOT NULL, - RequestMethod VARCHAR (10) NULL, - RawResource VARBINARY (MAX) NOT NULL, - IsRawResourceMetaSet BIT DEFAULT 0 NOT NULL, - SearchParamHash VARCHAR (64) NULL); - -CREATE TYPE dbo.UriSearchParamList AS TABLE ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - Uri VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL PRIMARY KEY (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri)); - -CREATE TABLE dbo.ClaimType ( - ClaimTypeId TINYINT IDENTITY (1, 1) NOT NULL, - Name VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, - CONSTRAINT UQ_ClaimType_ClaimTypeId UNIQUE (ClaimTypeId), - CONSTRAINT PKC_ClaimType PRIMARY KEY CLUSTERED (Name) WITH (DATA_COMPRESSION = PAGE) -); - -CREATE TABLE dbo.CompartmentAssignment ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - CompartmentTypeId TINYINT NOT NULL, - ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, - IsHistory BIT NOT NULL, - CONSTRAINT PKC_CompartmentAssignment PRIMARY KEY CLUSTERED (ResourceTypeId, ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId) WITH (DATA_COMPRESSION = PAGE) ON PartitionScheme_ResourceTypeId (ResourceTypeId) -); - - -GO -ALTER TABLE dbo.CompartmentAssignment - ADD CONSTRAINT DF_CompartmentAssignment_IsHistory DEFAULT 0 FOR IsHistory; - - -GO -ALTER TABLE dbo.CompartmentAssignment SET (LOCK_ESCALATION = AUTO); - - -GO -CREATE NONCLUSTERED INDEX IX_CompartmentAssignment_CompartmentTypeId_ReferenceResourceId - ON dbo.CompartmentAssignment(ResourceTypeId, CompartmentTypeId, ReferenceResourceId, ResourceSurrogateId) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE TABLE dbo.CompartmentType ( - CompartmentTypeId TINYINT IDENTITY (1, 1) NOT NULL, - Name VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, - CONSTRAINT UQ_CompartmentType_CompartmentTypeId UNIQUE (CompartmentTypeId), - CONSTRAINT PKC_CompartmentType PRIMARY KEY CLUSTERED (Name) WITH (DATA_COMPRESSION = PAGE) -); - -CREATE TABLE dbo.DateTimeSearchParam ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - StartDateTime DATETIME2 (7) NOT NULL, - EndDateTime DATETIME2 (7) NOT NULL, - IsLongerThanADay BIT NOT NULL, - IsHistory BIT NOT NULL, - IsMin BIT CONSTRAINT date_IsMin_Constraint DEFAULT 0 NOT NULL, - IsMax BIT CONSTRAINT date_IsMax_Constraint DEFAULT 0 NOT NULL -); - -ALTER TABLE dbo.DateTimeSearchParam - ADD CONSTRAINT DF_DateTimeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; - -ALTER TABLE dbo.DateTimeSearchParam SET (LOCK_ESCALATION = AUTO); - -CREATE CLUSTERED INDEX IXC_DateTimeSearchParam - ON dbo.DateTimeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_DateTimeSearchParam_SearchParamId_StartDateTime_EndDateTime - ON dbo.DateTimeSearchParam(ResourceTypeId, SearchParamId, StartDateTime, EndDateTime, ResourceSurrogateId) - INCLUDE(IsLongerThanADay, IsMin, IsMax) WHERE IsHistory = 0 - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_DateTimeSearchParam_SearchParamId_EndDateTime_StartDateTime - ON dbo.DateTimeSearchParam(ResourceTypeId, SearchParamId, EndDateTime, StartDateTime, ResourceSurrogateId) - INCLUDE(IsLongerThanADay, IsMin, IsMax) WHERE IsHistory = 0 - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_DateTimeSearchParam_SearchParamId_StartDateTime_EndDateTime_Long - ON dbo.DateTimeSearchParam(ResourceTypeId, SearchParamId, StartDateTime, EndDateTime, ResourceSurrogateId) - INCLUDE(IsMin, IsMax) WHERE IsHistory = 0 - AND IsLongerThanADay = 1 - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_DateTimeSearchParam_SearchParamId_EndDateTime_StartDateTime_Long - ON dbo.DateTimeSearchParam(ResourceTypeId, SearchParamId, EndDateTime, StartDateTime, ResourceSurrogateId) - INCLUDE(IsMin, IsMax) WHERE IsHistory = 0 - AND IsLongerThanADay = 1 - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -IF NOT EXISTS (SELECT 1 - FROM sys.tables - WHERE name = 'EventAgentCheckpoint') - BEGIN - CREATE TABLE dbo.EventAgentCheckpoint ( - CheckpointId VARCHAR (64) NOT NULL, - LastProcessedDateTime DATETIMEOFFSET (7), - LastProcessedIdentifier VARCHAR (64) , - UpdatedOn DATETIME2 (7) DEFAULT sysutcdatetime() NOT NULL, - CONSTRAINT PK_EventAgentCheckpoint PRIMARY KEY CLUSTERED (CheckpointId) - ) ON [PRIMARY]; - END - -CREATE PARTITION FUNCTION EventLogPartitionFunction(TINYINT) - AS RANGE RIGHT - FOR VALUES (0, 1, 2, 3, 4, 5, 6, 7); - - -GO -CREATE PARTITION SCHEME EventLogPartitionScheme - AS PARTITION EventLogPartitionFunction - ALL TO ([PRIMARY]); - - -GO -CREATE TABLE dbo.EventLog ( - PartitionId AS isnull(CONVERT (TINYINT, EventId % 8), 0) PERSISTED, - EventId BIGINT IDENTITY (1, 1) NOT NULL, - EventDate DATETIME NOT NULL, - Process VARCHAR (100) NOT NULL, - Status VARCHAR (10) NOT NULL, - Mode VARCHAR (200) NULL, - Action VARCHAR (20) NULL, - Target VARCHAR (100) NULL, - Rows BIGINT NULL, - Milliseconds INT NULL, - EventText NVARCHAR (3500) NULL, - SPID SMALLINT NOT NULL, - HostName VARCHAR (64) NOT NULL CONSTRAINT PKC_EventLog_EventDate_EventId_PartitionId PRIMARY KEY CLUSTERED (EventDate, EventId, PartitionId) ON EventLogPartitionScheme (PartitionId) -); - -CREATE TABLE dbo.ExportJob ( - Id VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, - Hash VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, - Status VARCHAR (10) NOT NULL, - HeartbeatDateTime DATETIME2 (7) NULL, - RawJobRecord VARCHAR (MAX) NOT NULL, - JobVersion ROWVERSION NOT NULL, - CONSTRAINT PKC_ExportJob PRIMARY KEY CLUSTERED (Id) -); - -CREATE UNIQUE NONCLUSTERED INDEX IX_ExportJob_Hash_Status_HeartbeatDateTime - ON dbo.ExportJob(Hash, Status, HeartbeatDateTime); - -CREATE TABLE dbo.IndexProperties ( - TableName VARCHAR (100) NOT NULL, - IndexName VARCHAR (200) NOT NULL, - PropertyName VARCHAR (100) NOT NULL, - PropertyValue VARCHAR (100) NOT NULL, - CreateDate DATETIME CONSTRAINT DF_IndexProperties_CreateDate DEFAULT getUTCdate() NOT NULL CONSTRAINT PKC_IndexProperties_TableName_IndexName_PropertyName PRIMARY KEY CLUSTERED (TableName, IndexName, PropertyName) -); - -CREATE PARTITION FUNCTION TinyintPartitionFunction(TINYINT) - AS RANGE RIGHT - FOR VALUES (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255); - - -GO -CREATE PARTITION SCHEME TinyintPartitionScheme - AS PARTITION TinyintPartitionFunction - ALL TO ([PRIMARY]); - - -GO -CREATE TABLE dbo.JobQueue ( - QueueType TINYINT NOT NULL, - GroupId BIGINT NOT NULL, - JobId BIGINT NOT NULL, - PartitionId AS CONVERT (TINYINT, JobId % 16) PERSISTED, - Definition VARCHAR (MAX) NOT NULL, - DefinitionHash VARBINARY (20) NOT NULL, - Version BIGINT CONSTRAINT DF_JobQueue_Version DEFAULT datediff_big(millisecond, '0001-01-01', getUTCdate()) NOT NULL, - Status TINYINT CONSTRAINT DF_JobQueue_Status DEFAULT 0 NOT NULL, - Priority TINYINT CONSTRAINT DF_JobQueue_Priority DEFAULT 100 NOT NULL, - Data BIGINT NULL, - Result VARCHAR (MAX) NULL, - CreateDate DATETIME CONSTRAINT DF_JobQueue_CreateDate DEFAULT getUTCdate() NOT NULL, - StartDate DATETIME NULL, - EndDate DATETIME NULL, - HeartbeatDate DATETIME CONSTRAINT DF_JobQueue_HeartbeatDate DEFAULT getUTCdate() NOT NULL, - Worker VARCHAR (100) NULL, - Info VARCHAR (1000) NULL, - CancelRequested BIT CONSTRAINT DF_JobQueue_CancelRequested DEFAULT 0 NOT NULL CONSTRAINT PKC_JobQueue_QueueType_PartitionId_JobId PRIMARY KEY CLUSTERED (QueueType, PartitionId, JobId) ON TinyintPartitionScheme (QueueType), - CONSTRAINT U_JobQueue_QueueType_JobId UNIQUE (QueueType, JobId) -); - - -GO -CREATE INDEX IX_QueueType_PartitionId_Status_Priority - ON dbo.JobQueue(PartitionId, Status, Priority) - ON TinyintPartitionScheme (QueueType); - - -GO -CREATE INDEX IX_QueueType_GroupId - ON dbo.JobQueue(QueueType, GroupId) - ON TinyintPartitionScheme (QueueType); - - -GO -CREATE INDEX IX_QueueType_DefinitionHash - ON dbo.JobQueue(QueueType, DefinitionHash) - ON TinyintPartitionScheme (QueueType); - -CREATE TABLE dbo.NumberSearchParam ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SingleValue DECIMAL (36, 18) NULL, - LowValue DECIMAL (36, 18) NOT NULL, - HighValue DECIMAL (36, 18) NOT NULL, - IsHistory BIT NOT NULL -); - -ALTER TABLE dbo.NumberSearchParam - ADD CONSTRAINT DF_NumberSearchParam_IsHistory DEFAULT 0 FOR IsHistory; - -ALTER TABLE dbo.NumberSearchParam SET (LOCK_ESCALATION = AUTO); - -CREATE CLUSTERED INDEX IXC_NumberSearchParam - ON dbo.NumberSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_NumberSearchParam_SearchParamId_SingleValue - ON dbo.NumberSearchParam(ResourceTypeId, SearchParamId, SingleValue, ResourceSurrogateId) WHERE IsHistory = 0 - AND SingleValue IS NOT NULL - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_NumberSearchParam_SearchParamId_LowValue_HighValue - ON dbo.NumberSearchParam(ResourceTypeId, SearchParamId, LowValue, HighValue, ResourceSurrogateId) WHERE IsHistory = 0 - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_NumberSearchParam_SearchParamId_HighValue_LowValue - ON dbo.NumberSearchParam(ResourceTypeId, SearchParamId, HighValue, LowValue, ResourceSurrogateId) WHERE IsHistory = 0 - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE TABLE dbo.Parameters ( - Id VARCHAR (100) NOT NULL, - Date DATETIME NULL, - Number FLOAT NULL, - Bigint BIGINT NULL, - Char VARCHAR (4000) NULL, - Binary VARBINARY (MAX) NULL, - UpdatedDate DATETIME NULL, - UpdatedBy NVARCHAR (255) NULL CONSTRAINT PKC_Parameters_Id PRIMARY KEY CLUSTERED (Id) WITH (IGNORE_DUP_KEY = ON) -); - - -GO -CREATE TABLE dbo.ParametersHistory ( - ChangeId INT IDENTITY (1, 1) NOT NULL, - Id VARCHAR (100) NOT NULL, - Date DATETIME NULL, - Number FLOAT NULL, - Bigint BIGINT NULL, - Char VARCHAR (4000) NULL, - Binary VARBINARY (MAX) NULL, - UpdatedDate DATETIME NULL, - UpdatedBy NVARCHAR (255) NULL -); - -CREATE TABLE dbo.QuantityCode ( - QuantityCodeId INT IDENTITY (1, 1) NOT NULL, - Value NVARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, - CONSTRAINT UQ_QuantityCode_QuantityCodeId UNIQUE (QuantityCodeId), - CONSTRAINT PKC_QuantityCode PRIMARY KEY CLUSTERED (Value) WITH (DATA_COMPRESSION = PAGE) -); - -CREATE TABLE dbo.QuantitySearchParam ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId INT NULL, - QuantityCodeId INT NULL, - SingleValue DECIMAL (36, 18) NULL, - LowValue DECIMAL (36, 18) NOT NULL, - HighValue DECIMAL (36, 18) NOT NULL, - IsHistory BIT NOT NULL -); - -ALTER TABLE dbo.QuantitySearchParam - ADD CONSTRAINT DF_QuantitySearchParam_IsHistory DEFAULT 0 FOR IsHistory; - -ALTER TABLE dbo.QuantitySearchParam SET (LOCK_ESCALATION = AUTO); - -CREATE CLUSTERED INDEX IXC_QuantitySearchParam - ON dbo.QuantitySearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_QuantitySearchParam_SearchParamId_QuantityCodeId_SingleValue - ON dbo.QuantitySearchParam(ResourceTypeId, SearchParamId, QuantityCodeId, SingleValue, ResourceSurrogateId) - INCLUDE(SystemId) WHERE IsHistory = 0 - AND SingleValue IS NOT NULL - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_QuantitySearchParam_SearchParamId_QuantityCodeId_LowValue_HighValue - ON dbo.QuantitySearchParam(ResourceTypeId, SearchParamId, QuantityCodeId, LowValue, HighValue, ResourceSurrogateId) - INCLUDE(SystemId) WHERE IsHistory = 0 - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_QuantitySearchParam_SearchParamId_QuantityCodeId_HighValue_LowValue - ON dbo.QuantitySearchParam(ResourceTypeId, SearchParamId, QuantityCodeId, HighValue, LowValue, ResourceSurrogateId) - INCLUDE(SystemId) WHERE IsHistory = 0 - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE TABLE dbo.ReferenceSearchParam ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - BaseUri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, - ReferenceResourceTypeId SMALLINT NULL, - ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, - ReferenceResourceVersion INT NULL, - IsHistory BIT NOT NULL -); - -ALTER TABLE dbo.ReferenceSearchParam - ADD CONSTRAINT DF_ReferenceSearchParam_IsHistory DEFAULT 0 FOR IsHistory; - -ALTER TABLE dbo.ReferenceSearchParam SET (LOCK_ESCALATION = AUTO); - -CREATE CLUSTERED INDEX IXC_ReferenceSearchParam - ON dbo.ReferenceSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_ReferenceSearchParam_SearchParamId_ReferenceResourceTypeId_ReferenceResourceId_BaseUri_ReferenceResourceVersion - ON dbo.ReferenceSearchParam(ResourceTypeId, SearchParamId, ReferenceResourceId, ReferenceResourceTypeId, BaseUri, ResourceSurrogateId) - INCLUDE(ReferenceResourceVersion) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE TABLE dbo.ReferenceTokenCompositeSearchParam ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - BaseUri1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, - ReferenceResourceTypeId1 SMALLINT NULL, - ReferenceResourceId1 VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, - ReferenceResourceVersion1 INT NULL, - SystemId2 INT NULL, - Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, - IsHistory BIT NOT NULL, - CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL -); - -ALTER TABLE dbo.ReferenceTokenCompositeSearchParam - ADD CONSTRAINT DF_ReferenceTokenCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; - -ALTER TABLE dbo.ReferenceTokenCompositeSearchParam - ADD CONSTRAINT CHK_ReferenceTokenCompositeSearchParam_CodeOverflow2 CHECK (LEN(Code2) = 256 - OR CodeOverflow2 IS NULL); - -ALTER TABLE dbo.ReferenceTokenCompositeSearchParam SET (LOCK_ESCALATION = AUTO); - -CREATE CLUSTERED INDEX IXC_ReferenceTokenCompositeSearchParam - ON dbo.ReferenceTokenCompositeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_ReferenceTokenCompositeSearchParam_ReferenceResourceId1_Code2 - ON dbo.ReferenceTokenCompositeSearchParam(ResourceTypeId, SearchParamId, ReferenceResourceId1, Code2, ResourceSurrogateId) - INCLUDE(ReferenceResourceTypeId1, BaseUri1, SystemId2) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE TABLE dbo.ReindexJob ( - Id VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, - Status VARCHAR (10) NOT NULL, - HeartbeatDateTime DATETIME2 (7) NULL, - RawJobRecord VARCHAR (MAX) NOT NULL, - JobVersion ROWVERSION NOT NULL, - CONSTRAINT PKC_ReindexJob PRIMARY KEY CLUSTERED (Id) -); - -CREATE TABLE dbo.Resource ( - ResourceTypeId SMALLINT NOT NULL, - ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, - Version INT NOT NULL, - IsHistory BIT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - IsDeleted BIT NOT NULL, - RequestMethod VARCHAR (10) NULL, - RawResource VARBINARY (MAX) NOT NULL, - IsRawResourceMetaSet BIT DEFAULT 0 NOT NULL, - SearchParamHash VARCHAR (64) NULL, - TransactionId BIGINT NULL, - HistoryTransactionId BIGINT NULL CONSTRAINT PKC_Resource PRIMARY KEY CLUSTERED (ResourceTypeId, ResourceSurrogateId) WITH (DATA_COMPRESSION = PAGE) ON PartitionScheme_ResourceTypeId (ResourceTypeId), - CONSTRAINT CH_Resource_RawResource_Length CHECK (RawResource > 0x0) -); - -ALTER TABLE dbo.Resource SET (LOCK_ESCALATION = AUTO); - -CREATE INDEX IX_ResourceTypeId_TransactionId - ON dbo.Resource(ResourceTypeId, TransactionId) WHERE TransactionId IS NOT NULL - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE INDEX IX_ResourceTypeId_HistoryTransactionId - ON dbo.Resource(ResourceTypeId, HistoryTransactionId) WHERE HistoryTransactionId IS NOT NULL - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE UNIQUE NONCLUSTERED INDEX IX_Resource_ResourceTypeId_ResourceId_Version - ON dbo.Resource(ResourceTypeId, ResourceId, Version) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE UNIQUE NONCLUSTERED INDEX IX_Resource_ResourceTypeId_ResourceId - ON dbo.Resource(ResourceTypeId, ResourceId) - INCLUDE(Version, IsDeleted) WHERE IsHistory = 0 - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE UNIQUE NONCLUSTERED INDEX IX_Resource_ResourceTypeId_ResourceSurrgateId - ON dbo.Resource(ResourceTypeId, ResourceSurrogateId) WHERE IsHistory = 0 - AND IsDeleted = 0 - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE TABLE dbo.ResourceChangeData ( - Id BIGINT IDENTITY (1, 1) NOT NULL, - Timestamp DATETIME2 (7) CONSTRAINT DF_ResourceChangeData_Timestamp DEFAULT sysutcdatetime() NOT NULL, - ResourceId VARCHAR (64) NOT NULL, - ResourceTypeId SMALLINT NOT NULL, - ResourceVersion INT NOT NULL, - ResourceChangeTypeId TINYINT NOT NULL -) ON PartitionScheme_ResourceChangeData_Timestamp (Timestamp); - -CREATE CLUSTERED INDEX IXC_ResourceChangeData - ON dbo.ResourceChangeData(Id ASC) WITH (ONLINE = ON) - ON PartitionScheme_ResourceChangeData_Timestamp (Timestamp); - -CREATE TABLE dbo.ResourceChangeDataStaging ( - Id BIGINT IDENTITY (1, 1) NOT NULL, - Timestamp DATETIME2 (7) CONSTRAINT DF_ResourceChangeDataStaging_Timestamp DEFAULT sysutcdatetime() NOT NULL, - ResourceId VARCHAR (64) NOT NULL, - ResourceTypeId SMALLINT NOT NULL, - ResourceVersion INT NOT NULL, - ResourceChangeTypeId TINYINT NOT NULL -) ON [PRIMARY]; - -CREATE CLUSTERED INDEX IXC_ResourceChangeDataStaging - ON dbo.ResourceChangeDataStaging(Id ASC, Timestamp ASC) WITH (ONLINE = ON) - ON [PRIMARY]; - -ALTER TABLE dbo.ResourceChangeDataStaging WITH CHECK - ADD CONSTRAINT CHK_ResourceChangeDataStaging_partition CHECK (Timestamp < CONVERT (DATETIME2 (7), N'9999-12-31 23:59:59.9999999')); - -ALTER TABLE dbo.ResourceChangeDataStaging CHECK CONSTRAINT CHK_ResourceChangeDataStaging_partition; - -CREATE TABLE dbo.ResourceChangeType ( - ResourceChangeTypeId TINYINT NOT NULL, - Name NVARCHAR (50) NOT NULL, - CONSTRAINT PK_ResourceChangeType PRIMARY KEY CLUSTERED (ResourceChangeTypeId), - CONSTRAINT UQ_ResourceChangeType_Name UNIQUE NONCLUSTERED (Name) -) ON [PRIMARY]; - - -GO -INSERT dbo.ResourceChangeType (ResourceChangeTypeId, Name) -VALUES (0, N'Creation'); - -INSERT dbo.ResourceChangeType (ResourceChangeTypeId, Name) -VALUES (1, N'Update'); - -INSERT dbo.ResourceChangeType (ResourceChangeTypeId, Name) -VALUES (2, N'Deletion'); - -CREATE TABLE dbo.ResourceType ( - ResourceTypeId SMALLINT IDENTITY (1, 1) NOT NULL, - Name NVARCHAR (50) COLLATE Latin1_General_100_CS_AS NOT NULL, - CONSTRAINT UQ_ResourceType_ResourceTypeId UNIQUE (ResourceTypeId), - CONSTRAINT PKC_ResourceType PRIMARY KEY CLUSTERED (Name) WITH (DATA_COMPRESSION = PAGE) -); - -CREATE TABLE dbo.ResourceWriteClaim ( - ResourceSurrogateId BIGINT NOT NULL, - ClaimTypeId TINYINT NOT NULL, - ClaimValue NVARCHAR (128) NOT NULL -) -WITH (DATA_COMPRESSION = PAGE); - -CREATE CLUSTERED INDEX IXC_ResourceWriteClaim - ON dbo.ResourceWriteClaim(ResourceSurrogateId, ClaimTypeId); - -CREATE TABLE dbo.SchemaMigrationProgress ( - Timestamp DATETIME2 (3) DEFAULT CURRENT_TIMESTAMP, - Message NVARCHAR (MAX) -); - -CREATE TABLE dbo.SearchParam ( - SearchParamId SMALLINT IDENTITY (1, 1) NOT NULL, - Uri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, - Status VARCHAR (20) NULL, - LastUpdated DATETIMEOFFSET (7) NULL, - IsPartiallySupported BIT NULL, - CONSTRAINT UQ_SearchParam_SearchParamId UNIQUE (SearchParamId), - CONSTRAINT PKC_SearchParam PRIMARY KEY CLUSTERED (Uri) WITH (DATA_COMPRESSION = PAGE) -); - -CREATE TABLE dbo.StringSearchParam ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - Text NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, - TextOverflow NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL, - IsHistory BIT NOT NULL, - IsMin BIT CONSTRAINT string_IsMin_Constraint DEFAULT 0 NOT NULL, - IsMax BIT CONSTRAINT string_IsMax_Constraint DEFAULT 0 NOT NULL -); - -ALTER TABLE dbo.StringSearchParam - ADD CONSTRAINT DF_StringSearchParam_IsHistory DEFAULT 0 FOR IsHistory; - -ALTER TABLE dbo.StringSearchParam SET (LOCK_ESCALATION = AUTO); - -CREATE CLUSTERED INDEX IXC_StringSearchParam - ON dbo.StringSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_StringSearchParam_SearchParamId_Text - ON dbo.StringSearchParam(ResourceTypeId, SearchParamId, Text, ResourceSurrogateId) - INCLUDE(TextOverflow, IsMin, IsMax) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_StringSearchParam_SearchParamId_TextWithOverflow - ON dbo.StringSearchParam(ResourceTypeId, SearchParamId, Text, ResourceSurrogateId) - INCLUDE(IsMin, IsMax) WHERE IsHistory = 0 - AND TextOverflow IS NOT NULL WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE TABLE dbo.System ( - SystemId INT IDENTITY (1, 1) NOT NULL, - Value NVARCHAR (256) NOT NULL, - CONSTRAINT UQ_System_SystemId UNIQUE (SystemId), - CONSTRAINT PKC_System PRIMARY KEY CLUSTERED (Value) WITH (DATA_COMPRESSION = PAGE) -); - -CREATE TABLE [dbo].[TaskInfo] ( - [TaskId] VARCHAR (64) NOT NULL, - [QueueId] VARCHAR (64) NOT NULL, - [Status] SMALLINT NOT NULL, - [TaskTypeId] SMALLINT NOT NULL, - [RunId] VARCHAR (50) NULL, - [IsCanceled] BIT NOT NULL, - [RetryCount] SMALLINT NOT NULL, - [MaxRetryCount] SMALLINT NOT NULL, - [HeartbeatDateTime] DATETIME2 (7) NULL, - [InputData] VARCHAR (MAX) NOT NULL, - [TaskContext] VARCHAR (MAX) NULL, - [Result] VARCHAR (MAX) NULL, - [CreateDateTime] DATETIME2 (7) CONSTRAINT DF_TaskInfo_CreateDate DEFAULT SYSUTCDATETIME() NOT NULL, - [StartDateTime] DATETIME2 (7) NULL, - [EndDateTime] DATETIME2 (7) NULL, - [Worker] VARCHAR (100) NULL, - [RestartInfo] VARCHAR (MAX) NULL, - [ParentTaskId] VARCHAR (64) NULL, - CONSTRAINT PKC_TaskInfo PRIMARY KEY CLUSTERED (TaskId) WITH (DATA_COMPRESSION = PAGE) -) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]; - - -GO -CREATE NONCLUSTERED INDEX IX_QueueId_Status - ON dbo.TaskInfo(QueueId, Status); - - -GO -CREATE NONCLUSTERED INDEX IX_QueueId_ParentTaskId - ON dbo.TaskInfo(QueueId, ParentTaskId); - -CREATE TABLE dbo.TokenDateTimeCompositeSearchParam ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId1 INT NULL, - Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, - StartDateTime2 DATETIME2 (7) NOT NULL, - EndDateTime2 DATETIME2 (7) NOT NULL, - IsLongerThanADay2 BIT NOT NULL, - IsHistory BIT NOT NULL, - CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL -); - -ALTER TABLE dbo.TokenDateTimeCompositeSearchParam - ADD CONSTRAINT DF_TokenDateTimeCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; - -ALTER TABLE dbo.TokenDateTimeCompositeSearchParam - ADD CONSTRAINT CHK_TokenDateTimeCompositeSearchParam_CodeOverflow1 CHECK (LEN(Code1) = 256 - OR CodeOverflow1 IS NULL); - -ALTER TABLE dbo.TokenDateTimeCompositeSearchParam SET (LOCK_ESCALATION = AUTO); - -CREATE CLUSTERED INDEX IXC_TokenDateTimeCompositeSearchParam - ON dbo.TokenDateTimeCompositeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_TokenDateTimeCompositeSearchParam_Code1_StartDateTime2_EndDateTime2 - ON dbo.TokenDateTimeCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, StartDateTime2, EndDateTime2, ResourceSurrogateId) - INCLUDE(SystemId1, IsLongerThanADay2) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_TokenDateTimeCompositeSearchParam_Code1_EndDateTime2_StartDateTime2 - ON dbo.TokenDateTimeCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, EndDateTime2, StartDateTime2, ResourceSurrogateId) - INCLUDE(SystemId1, IsLongerThanADay2) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_TokenDateTimeCompositeSearchParam_Code1_StartDateTime2_EndDateTime2_Long - ON dbo.TokenDateTimeCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, StartDateTime2, EndDateTime2, ResourceSurrogateId) - INCLUDE(SystemId1) WHERE IsHistory = 0 - AND IsLongerThanADay2 = 1 WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_TokenDateTimeCompositeSearchParam_Code1_EndDateTime2_StartDateTime2_Long - ON dbo.TokenDateTimeCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, EndDateTime2, StartDateTime2, ResourceSurrogateId) - INCLUDE(SystemId1) WHERE IsHistory = 0 - AND IsLongerThanADay2 = 1 WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE TABLE dbo.TokenNumberNumberCompositeSearchParam ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId1 INT NULL, - Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, - SingleValue2 DECIMAL (36, 18) NULL, - LowValue2 DECIMAL (36, 18) NULL, - HighValue2 DECIMAL (36, 18) NULL, - SingleValue3 DECIMAL (36, 18) NULL, - LowValue3 DECIMAL (36, 18) NULL, - HighValue3 DECIMAL (36, 18) NULL, - HasRange BIT NOT NULL, - IsHistory BIT NOT NULL, - CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL -); - -ALTER TABLE dbo.TokenNumberNumberCompositeSearchParam - ADD CONSTRAINT DF_TokenNumberNumberCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; - -ALTER TABLE dbo.TokenNumberNumberCompositeSearchParam - ADD CONSTRAINT CHK_TokenNumberNumberCompositeSearchParam_CodeOverflow1 CHECK (LEN(Code1) = 256 - OR CodeOverflow1 IS NULL); - -ALTER TABLE dbo.TokenNumberNumberCompositeSearchParam SET (LOCK_ESCALATION = AUTO); - -CREATE CLUSTERED INDEX IXC_TokenNumberNumberCompositeSearchParam - ON dbo.TokenNumberNumberCompositeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_TokenNumberNumberCompositeSearchParam_SearchParamId_Code1_Text2 - ON dbo.TokenNumberNumberCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, SingleValue2, SingleValue3, ResourceSurrogateId) - INCLUDE(SystemId1) WHERE IsHistory = 0 - AND HasRange = 0 WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_TokenNumberNumberCompositeSearchParam_SearchParamId_Code1_LowValue2_HighValue2_LowValue3_HighValue3 - ON dbo.TokenNumberNumberCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, LowValue2, HighValue2, LowValue3, HighValue3, ResourceSurrogateId) - INCLUDE(SystemId1) WHERE IsHistory = 0 - AND HasRange = 1 WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE TABLE dbo.TokenQuantityCompositeSearchParam ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId1 INT NULL, - Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, - SystemId2 INT NULL, - QuantityCodeId2 INT NULL, - SingleValue2 DECIMAL (36, 18) NULL, - LowValue2 DECIMAL (36, 18) NULL, - HighValue2 DECIMAL (36, 18) NULL, - IsHistory BIT NOT NULL, - CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL -); - -ALTER TABLE dbo.TokenQuantityCompositeSearchParam - ADD CONSTRAINT DF_TokenQuantityCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; - -ALTER TABLE dbo.TokenQuantityCompositeSearchParam - ADD CONSTRAINT CHK_TokenQuantityCompositeSearchParam_CodeOverflow1 CHECK (LEN(Code1) = 256 - OR CodeOverflow1 IS NULL); - -ALTER TABLE dbo.TokenQuantityCompositeSearchParam SET (LOCK_ESCALATION = AUTO); - -CREATE CLUSTERED INDEX IXC_TokenQuantityCompositeSearchParam - ON dbo.TokenQuantityCompositeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_TokenQuantityCompositeSearchParam_SearchParamId_Code1_QuantityCodeId2_SingleValue2 - ON dbo.TokenQuantityCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, SingleValue2, ResourceSurrogateId) - INCLUDE(QuantityCodeId2, SystemId1, SystemId2) WHERE IsHistory = 0 - AND SingleValue2 IS NOT NULL WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_TokenQuantityCompositeSearchParam_SearchParamId_Code1_QuantityCodeId2_LowValue2_HighValue2 - ON dbo.TokenQuantityCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, LowValue2, HighValue2, ResourceSurrogateId) - INCLUDE(QuantityCodeId2, SystemId1, SystemId2) WHERE IsHistory = 0 - AND LowValue2 IS NOT NULL WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_TokenQuantityCompositeSearchParam_SearchParamId_Code1_QuantityCodeId2_HighValue2_LowValue2 - ON dbo.TokenQuantityCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, HighValue2, LowValue2, ResourceSurrogateId) - INCLUDE(QuantityCodeId2, SystemId1, SystemId2) WHERE IsHistory = 0 - AND LowValue2 IS NOT NULL WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE TABLE dbo.TokenSearchParam ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId INT NULL, - Code VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, - IsHistory BIT NOT NULL, - CodeOverflow VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL -); - -ALTER TABLE dbo.TokenSearchParam - ADD CONSTRAINT DF_TokenSearchParam_IsHistory DEFAULT 0 FOR IsHistory; - -ALTER TABLE dbo.TokenSearchParam - ADD CONSTRAINT CHK_TokenSearchParam_CodeOverflow CHECK (LEN(Code) = 256 - OR CodeOverflow IS NULL); - -ALTER TABLE dbo.TokenSearchParam SET (LOCK_ESCALATION = AUTO); - -CREATE CLUSTERED INDEX IXC_TokenSearchParam - ON dbo.TokenSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_TokenSeachParam_SearchParamId_Code_SystemId - ON dbo.TokenSearchParam(ResourceTypeId, SearchParamId, Code, ResourceSurrogateId) - INCLUDE(SystemId) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE TABLE dbo.TokenStringCompositeSearchParam ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId1 INT NULL, - Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, - Text2 NVARCHAR (256) COLLATE Latin1_General_CI_AI NOT NULL, - TextOverflow2 NVARCHAR (MAX) COLLATE Latin1_General_CI_AI NULL, - IsHistory BIT NOT NULL, - CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL -); - -ALTER TABLE dbo.TokenStringCompositeSearchParam - ADD CONSTRAINT DF_TokenStringCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; - -ALTER TABLE dbo.TokenStringCompositeSearchParam - ADD CONSTRAINT CHK_TokenStringCompositeSearchParam_CodeOverflow1 CHECK (LEN(Code1) = 256 - OR CodeOverflow1 IS NULL); - -ALTER TABLE dbo.TokenStringCompositeSearchParam SET (LOCK_ESCALATION = AUTO); - -CREATE CLUSTERED INDEX IXC_TokenStringCompositeSearchParam - ON dbo.TokenStringCompositeSearchParam(ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_TokenStringCompositeSearchParam_SearchParamId_Code1_Text2 - ON dbo.TokenStringCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, Text2, ResourceSurrogateId) - INCLUDE(SystemId1, TextOverflow2) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_TokenStringCompositeSearchParam_SearchParamId_Code1_Text2WithOverflow - ON dbo.TokenStringCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, Text2, ResourceSurrogateId) - INCLUDE(SystemId1) WHERE IsHistory = 0 - AND TextOverflow2 IS NOT NULL WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE TABLE dbo.TokenText ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - Text NVARCHAR (400) COLLATE Latin1_General_CI_AI NOT NULL, - IsHistory BIT NOT NULL -); - -ALTER TABLE dbo.TokenText - ADD CONSTRAINT DF_TokenText_IsHistory DEFAULT 0 FOR IsHistory; - -ALTER TABLE dbo.TokenText SET (LOCK_ESCALATION = AUTO); - -CREATE CLUSTERED INDEX IXC_TokenText - ON dbo.TokenText(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_TokenText_SearchParamId_Text - ON dbo.TokenText(ResourceTypeId, SearchParamId, Text, ResourceSurrogateId) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE TABLE dbo.TokenTokenCompositeSearchParam ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - SystemId1 INT NULL, - Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, - SystemId2 INT NULL, - Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, - IsHistory BIT NOT NULL, - CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, - CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL -); - -ALTER TABLE dbo.TokenTokenCompositeSearchParam - ADD CONSTRAINT DF_TokenTokenCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; - -ALTER TABLE dbo.TokenTokenCompositeSearchParam - ADD CONSTRAINT CHK_TokenTokenCompositeSearchParam_CodeOverflow1 CHECK (LEN(Code1) = 256 - OR CodeOverflow1 IS NULL); - -ALTER TABLE dbo.TokenTokenCompositeSearchParam - ADD CONSTRAINT CHK_TokenTokenCompositeSearchParam_CodeOverflow2 CHECK (LEN(Code2) = 256 - OR CodeOverflow2 IS NULL); - -ALTER TABLE dbo.TokenTokenCompositeSearchParam SET (LOCK_ESCALATION = AUTO); - -CREATE CLUSTERED INDEX IXC_TokenTokenCompositeSearchParam - ON dbo.TokenTokenCompositeSearchParam(ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_TokenTokenCompositeSearchParam_Code1_Code2 - ON dbo.TokenTokenCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, Code2, ResourceSurrogateId) - INCLUDE(SystemId1, SystemId2) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE TABLE dbo.Transactions ( - SurrogateIdRangeFirstValue BIGINT NOT NULL, - SurrogateIdRangeLastValue BIGINT NOT NULL, - Definition VARCHAR (2000) NULL, - IsCompleted BIT CONSTRAINT DF_Transactions_IsCompleted DEFAULT 0 NOT NULL, - IsSuccess BIT CONSTRAINT DF_Transactions_IsSuccess DEFAULT 0 NOT NULL, - IsVisible BIT CONSTRAINT DF_Transactions_IsVisible DEFAULT 0 NOT NULL, - IsHistoryMoved BIT CONSTRAINT DF_Transactions_IsHistoryMoved DEFAULT 0 NOT NULL, - CreateDate DATETIME CONSTRAINT DF_Transactions_CreateDate DEFAULT getUTCdate() NOT NULL, - EndDate DATETIME NULL, - VisibleDate DATETIME NULL, - HistoryMovedDate DATETIME NULL, - HeartbeatDate DATETIME CONSTRAINT DF_Transactions_HeartbeatDate DEFAULT getUTCdate() NOT NULL, - FailureReason VARCHAR (MAX) NULL, - IsControlledByClient BIT CONSTRAINT DF_Transactions_IsControlledByClient DEFAULT 1 NOT NULL, - InvisibleHistoryRemovedDate DATETIME NULL CONSTRAINT PKC_Transactions_SurrogateIdRangeFirstValue PRIMARY KEY CLUSTERED (SurrogateIdRangeFirstValue) -); - -CREATE INDEX IX_IsVisible - ON dbo.Transactions(IsVisible); - -CREATE TABLE dbo.UriSearchParam ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL, - SearchParamId SMALLINT NOT NULL, - Uri VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, - IsHistory BIT NOT NULL -); - -ALTER TABLE dbo.UriSearchParam - ADD CONSTRAINT DF_UriSearchParam_IsHistory DEFAULT 0 FOR IsHistory; - -ALTER TABLE dbo.UriSearchParam SET (LOCK_ESCALATION = AUTO); - -CREATE CLUSTERED INDEX IXC_UriSearchParam - ON dbo.UriSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE NONCLUSTERED INDEX IX_UriSearchParam_SearchParamId_Uri - ON dbo.UriSearchParam(ResourceTypeId, SearchParamId, Uri, ResourceSurrogateId) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) - ON PartitionScheme_ResourceTypeId (ResourceTypeId); - -CREATE TABLE dbo.WatchdogLeases ( - Watchdog VARCHAR (100) NOT NULL, - LeaseHolder VARCHAR (100) CONSTRAINT DF_WatchdogLeases_LeaseHolder DEFAULT '' NOT NULL, - LeaseEndTime DATETIME CONSTRAINT DF_WatchdogLeases_LeaseEndTime DEFAULT 0 NOT NULL, - RemainingLeaseTimeSec AS datediff(second, getUTCdate(), LeaseEndTime), - LeaseRequestor VARCHAR (100) CONSTRAINT DF_WatchdogLeases_LeaseRequestor DEFAULT '' NOT NULL, - LeaseRequestTime DATETIME CONSTRAINT DF_WatchdogLeases_LeaseRequestTime DEFAULT 0 NOT NULL CONSTRAINT PKC_WatchdogLeases_Watchdog PRIMARY KEY CLUSTERED (Watchdog) -); - -COMMIT -GO -CREATE PROCEDURE dbo.AcquireExportJobs -@jobHeartbeatTimeoutThresholdInSeconds BIGINT, @maximumNumberOfConcurrentJobsAllowed INT -AS -SET NOCOUNT ON; -SET XACT_ABORT ON; -SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; -BEGIN TRANSACTION; -DECLARE @expirationDateTime AS DATETIME2 (7); -SELECT @expirationDateTime = DATEADD(second, -@jobHeartbeatTimeoutThresholdInSeconds, SYSUTCDATETIME()); -DECLARE @numberOfRunningJobs AS INT; -SELECT @numberOfRunningJobs = COUNT(*) -FROM dbo.ExportJob WITH (TABLOCKX) -WHERE Status = 'Running' - AND HeartbeatDateTime > @expirationDateTime; -DECLARE @limit AS INT = @maximumNumberOfConcurrentJobsAllowed - @numberOfRunningJobs; -IF (@limit > 0) - BEGIN - DECLARE @availableJobs TABLE ( - Id VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, - JobVersion BINARY (8) NOT NULL); - INSERT INTO @availableJobs - SELECT TOP (@limit) Id, - JobVersion - FROM dbo.ExportJob - WHERE (Status = 'Queued' - OR (Status = 'Running' - AND HeartbeatDateTime <= @expirationDateTime)) - ORDER BY HeartbeatDateTime; - DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); - UPDATE dbo.ExportJob - SET Status = 'Running', - HeartbeatDateTime = @heartbeatDateTime, - RawJobRecord = JSON_MODIFY(RawJobRecord, '$.status', 'Running') - OUTPUT inserted.RawJobRecord, inserted.JobVersion - FROM dbo.ExportJob AS job - INNER JOIN - @availableJobs AS availableJob - ON job.Id = availableJob.Id - AND job.JobVersion = availableJob.JobVersion; - END -COMMIT TRANSACTION; - -GO -CREATE PROCEDURE dbo.AcquireReindexJobs -@jobHeartbeatTimeoutThresholdInSeconds BIGINT, @maximumNumberOfConcurrentJobsAllowed INT -AS -SET NOCOUNT ON; -SET XACT_ABORT ON; -SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; -BEGIN TRANSACTION; -DECLARE @expirationDateTime AS DATETIME2 (7); -SELECT @expirationDateTime = DATEADD(second, -@jobHeartbeatTimeoutThresholdInSeconds, SYSUTCDATETIME()); -DECLARE @numberOfRunningJobs AS INT; -SELECT @numberOfRunningJobs = COUNT(*) -FROM dbo.ReindexJob WITH (TABLOCKX) -WHERE Status = 'Running' - AND HeartbeatDateTime > @expirationDateTime; -DECLARE @limit AS INT = @maximumNumberOfConcurrentJobsAllowed - @numberOfRunningJobs; -IF (@limit > 0) - BEGIN - DECLARE @availableJobs TABLE ( - Id VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, - JobVersion BINARY (8) NOT NULL); - INSERT INTO @availableJobs - SELECT TOP (@limit) Id, - JobVersion - FROM dbo.ReindexJob - WHERE (Status = 'Queued' - OR (Status = 'Running' - AND HeartbeatDateTime <= @expirationDateTime)) - ORDER BY HeartbeatDateTime; - DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); - UPDATE dbo.ReindexJob - SET Status = 'Running', - HeartbeatDateTime = @heartbeatDateTime, - RawJobRecord = JSON_MODIFY(RawJobRecord, '$.status', 'Running') - OUTPUT inserted.RawJobRecord, inserted.JobVersion - FROM dbo.ReindexJob AS job - INNER JOIN - @availableJobs AS availableJob - ON job.Id = availableJob.Id - AND job.JobVersion = availableJob.JobVersion; - END -COMMIT TRANSACTION; - -GO -CREATE PROCEDURE dbo.AcquireWatchdogLease -@Watchdog VARCHAR (100), @Worker VARCHAR (100), @AllowRebalance BIT=1, @ForceAcquire BIT=0, @LeasePeriodSec FLOAT, @WorkerIsRunning BIT=0, @LeaseEndTime DATETIME OUTPUT, @IsAcquired BIT OUTPUT, @CurrentLeaseHolder VARCHAR (100)=NULL OUTPUT -AS -SET NOCOUNT ON; -SET XACT_ABORT ON; -DECLARE @SP AS VARCHAR (100) = 'AcquireWatchdogLease', @Mode AS VARCHAR (100), @msg AS VARCHAR (1000), @MyLeasesNumber AS INT, @OtherValidRequestsOrLeasesNumber AS INT, @MyValidRequestsOrLeasesNumber AS INT, @DesiredLeasesNumber AS INT, @NotLeasedWatchdogNumber AS INT, @WatchdogNumber AS INT, @Now AS DATETIME, @MyLastChangeTime AS DATETIME, @PreviousLeaseHolder AS VARCHAR (100), @Rows AS INT = 0, @NumberOfWorkers AS INT, @st AS DATETIME = getUTCdate(), @RowsInt AS INT, @Pattern AS VARCHAR (100); -BEGIN TRY - SET @Mode = 'R=' + isnull(@Watchdog, 'NULL') + ' W=' + isnull(@Worker, 'NULL') + ' F=' + isnull(CONVERT (VARCHAR, @ForceAcquire), 'NULL') + ' LP=' + isnull(CONVERT (VARCHAR, @LeasePeriodSec), 'NULL'); - SET @CurrentLeaseHolder = ''; - SET @IsAcquired = 0; - SET @Now = getUTCdate(); - SET @LeaseEndTime = @Now; - SET @Pattern = NULLIF ((SELECT Char - FROM dbo.Parameters - WHERE Id = 'WatchdogLeaseHolderIncludePatternFor' + @Watchdog), ''); - IF @Pattern IS NULL - SET @Pattern = NULLIF ((SELECT Char - FROM dbo.Parameters - WHERE Id = 'WatchdogLeaseHolderIncludePattern'), ''); - IF @Pattern IS NOT NULL - AND @Worker NOT LIKE @Pattern - BEGIN - SET @msg = 'Worker does not match include pattern=' + @Pattern; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows, @Text = @msg; - SET @CurrentLeaseHolder = isnull((SELECT LeaseHolder - FROM dbo.WatchdogLeases - WHERE Watchdog = @Watchdog), ''); - RETURN; - END - SET @Pattern = NULLIF ((SELECT Char - FROM dbo.Parameters - WHERE Id = 'WatchdogLeaseHolderExcludePatternFor' + @Watchdog), ''); - IF @Pattern IS NULL - SET @Pattern = NULLIF ((SELECT Char - FROM dbo.Parameters - WHERE Id = 'WatchdogLeaseHolderExcludePattern'), ''); - IF @Pattern IS NOT NULL - AND @Worker LIKE @Pattern - BEGIN - SET @msg = 'Worker matches exclude pattern=' + @Pattern; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows, @Text = @msg; - SET @CurrentLeaseHolder = isnull((SELECT LeaseHolder - FROM dbo.WatchdogLeases - WHERE Watchdog = @Watchdog), ''); - RETURN; - END - DECLARE @Watchdogs TABLE ( - Watchdog VARCHAR (100) PRIMARY KEY); - INSERT INTO @Watchdogs - SELECT Watchdog - FROM dbo.WatchdogLeases WITH (NOLOCK) - WHERE RemainingLeaseTimeSec * (-1) > 10 * @LeasePeriodSec - OR @ForceAcquire = 1 - AND Watchdog = @Watchdog - AND LeaseHolder <> @Worker; - IF @@rowcount > 0 - BEGIN - DELETE dbo.WatchdogLeases - WHERE Watchdog IN (SELECT Watchdog - FROM @Watchdogs); - SET @Rows += @@rowcount; - IF @Rows > 0 - BEGIN - SET @msg = ''; - SELECT @msg = CONVERT (VARCHAR (1000), @msg + CASE WHEN @msg = '' THEN '' ELSE ',' END + Watchdog) - FROM @Watchdogs; - SET @msg = CONVERT (VARCHAR (1000), 'Remove old/forced leases:' + @msg); - EXECUTE dbo.LogEvent @Process = 'AcquireWatchdogLease', @Status = 'Info', @Mode = @Mode, @Target = 'WatchdogLeases', @Action = 'Delete', @Rows = @Rows, @Text = @msg; - END - END - SET @NumberOfWorkers = 1 + (SELECT count(*) - FROM (SELECT LeaseHolder - FROM dbo.WatchdogLeases WITH (NOLOCK) - WHERE LeaseHolder <> @Worker - UNION - SELECT LeaseRequestor - FROM dbo.WatchdogLeases WITH (NOLOCK) - WHERE LeaseRequestor <> @Worker - AND LeaseRequestor <> '') AS A); - SET @Mode = CONVERT (VARCHAR (100), @Mode + ' N=' + CONVERT (VARCHAR (10), @NumberOfWorkers)); - IF NOT EXISTS (SELECT * - FROM dbo.WatchdogLeases WITH (NOLOCK) - WHERE Watchdog = @Watchdog) - INSERT INTO dbo.WatchdogLeases (Watchdog, LeaseEndTime, LeaseRequestTime) - SELECT @Watchdog, - dateadd(day, -10, @Now), - dateadd(day, -10, @Now) - WHERE NOT EXISTS (SELECT * - FROM dbo.WatchdogLeases WITH (TABLOCKX) - WHERE Watchdog = @Watchdog); - SET @LeaseEndTime = dateadd(second, @LeasePeriodSec, @Now); - SET @WatchdogNumber = (SELECT count(*) - FROM dbo.WatchdogLeases WITH (NOLOCK)); - SET @NotLeasedWatchdogNumber = (SELECT count(*) - FROM dbo.WatchdogLeases WITH (NOLOCK) - WHERE LeaseHolder = '' - OR LeaseEndTime < @Now); - SET @MyLeasesNumber = (SELECT count(*) - FROM dbo.WatchdogLeases WITH (NOLOCK) - WHERE LeaseHolder = @Worker - AND LeaseEndTime > @Now); - SET @OtherValidRequestsOrLeasesNumber = (SELECT count(*) - FROM dbo.WatchdogLeases WITH (NOLOCK) - WHERE LeaseHolder <> @Worker - AND LeaseEndTime > @Now - OR LeaseRequestor <> @Worker - AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec); - SET @MyValidRequestsOrLeasesNumber = (SELECT count(*) - FROM dbo.WatchdogLeases WITH (NOLOCK) - WHERE LeaseHolder = @Worker - AND LeaseEndTime > @Now - OR LeaseRequestor = @Worker - AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec); - SET @DesiredLeasesNumber = ceiling(1.0 * @WatchdogNumber / @NumberOfWorkers); - IF @DesiredLeasesNumber = 0 - SET @DesiredLeasesNumber = 1; - IF @DesiredLeasesNumber = 1 - AND @OtherValidRequestsOrLeasesNumber = 1 - AND @WatchdogNumber = 1 - SET @DesiredLeasesNumber = 0; - IF @MyValidRequestsOrLeasesNumber = floor(1.0 * @WatchdogNumber / @NumberOfWorkers) - AND @OtherValidRequestsOrLeasesNumber + @MyValidRequestsOrLeasesNumber = @WatchdogNumber - SET @DesiredLeasesNumber = @DesiredLeasesNumber - 1; - UPDATE dbo.WatchdogLeases - SET LeaseHolder = @Worker, - LeaseEndTime = @LeaseEndTime, - LeaseRequestor = '', - @PreviousLeaseHolder = LeaseHolder - WHERE Watchdog = @Watchdog - AND NOT (LeaseRequestor <> @Worker - AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec) - AND (LeaseHolder = @Worker - AND (LeaseEndTime > @Now - OR @WorkerIsRunning = 1) - OR LeaseEndTime < @Now - AND (@DesiredLeasesNumber > @MyLeasesNumber - OR @OtherValidRequestsOrLeasesNumber < @WatchdogNumber)); - IF @@rowcount > 0 - BEGIN - SET @IsAcquired = 1; - SET @msg = 'Lease holder changed from [' + isnull(@PreviousLeaseHolder, '') + '] to [' + @Worker + ']'; - IF @PreviousLeaseHolder <> @Worker - EXECUTE dbo.LogEvent @Process = 'AcquireWatchdogLease', @Status = 'Info', @Mode = @Mode, @Text = @msg; - END - ELSE - IF @AllowRebalance = 1 - BEGIN - SET @CurrentLeaseHolder = (SELECT LeaseHolder - FROM dbo.WatchdogLeases - WHERE Watchdog = @Watchdog); - UPDATE dbo.WatchdogLeases - SET LeaseRequestTime = @Now - WHERE Watchdog = @Watchdog - AND LeaseRequestor = @Worker - AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec; - IF @DesiredLeasesNumber > @MyValidRequestsOrLeasesNumber - BEGIN - UPDATE A - SET LeaseRequestor = @Worker, - LeaseRequestTime = @Now - FROM dbo.WatchdogLeases AS A - WHERE Watchdog = @Watchdog - AND NOT (LeaseRequestor <> @Worker - AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec) - AND @NotLeasedWatchdogNumber = 0 - AND (SELECT count(*) - FROM dbo.WatchdogLeases AS B - WHERE B.LeaseHolder = A.LeaseHolder - AND datediff(second, B.LeaseEndTime, @Now) < @LeasePeriodSec) > @DesiredLeasesNumber; - SET @RowsInt = @@rowcount; - SET @msg = '@DesiredLeasesNumber=[' + CONVERT (VARCHAR (10), @DesiredLeasesNumber) + '] > @MyValidRequestsOrLeasesNumber=[' + CONVERT (VARCHAR (10), @MyValidRequestsOrLeasesNumber) + ']'; - EXECUTE dbo.LogEvent @Process = 'AcquireWatchdogLease', @Status = 'Info', @Mode = @Mode, @Rows = @RowsInt, @Text = @msg; - END - END - SET @Mode = CONVERT (VARCHAR (100), @Mode + ' A=' + CONVERT (VARCHAR (1), @IsAcquired)); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; -END TRY -BEGIN CATCH - IF @@trancount > 0 - ROLLBACK; - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = 'AcquireWatchdogLease', @Status = 'Error', @Mode = @Mode; - THROW; -END CATCH - -GO -CREATE OR ALTER PROCEDURE dbo.AddPartitionOnResourceChanges -@partitionBoundary DATETIME2 (7) OUTPUT -AS -BEGIN - SET XACT_ABORT ON; - BEGIN TRANSACTION; - DECLARE @rightPartitionBoundary AS DATETIME2 (7) = CAST ((SELECT TOP (1) value - FROM sys.partition_range_values AS prv - INNER JOIN - sys.partition_functions AS pf - ON pf.function_id = prv.function_id - WHERE pf.name = N'PartitionFunction_ResourceChangeData_Timestamp' - ORDER BY prv.boundary_id DESC) AS DATETIME2 (7)); - DECLARE @timestamp AS DATETIME2 (7) = DATEADD(hour, DATEDIFF(hour, 0, sysutcdatetime()), 0); - IF (@rightPartitionBoundary < @timestamp) - BEGIN - SET @rightPartitionBoundary = @timestamp; - END - SET @rightPartitionBoundary = DATEADD(hour, 1, @rightPartitionBoundary); - ALTER PARTITION SCHEME PartitionScheme_ResourceChangeData_Timestamp NEXT USED [Primary]; - ALTER PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp( ) - SPLIT RANGE (@rightPartitionBoundary); - SET @partitionBoundary = @rightPartitionBoundary; - COMMIT TRANSACTION; -END - -GO -CREATE PROCEDURE dbo.ArchiveJobs -@QueueType TINYINT -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'ArchiveJobs', @Mode AS VARCHAR (100) = '', @st AS DATETIME = getUTCdate(), @Rows AS INT = 0, @PartitionId AS TINYINT, @MaxPartitions AS TINYINT = 16, @LookedAtPartitions AS TINYINT = 0, @InflightRows AS INT = 0, @Lock AS VARCHAR (100) = 'DequeueJob_' + CONVERT (VARCHAR, @QueueType); -BEGIN TRY - SET @PartitionId = @MaxPartitions * rand(); - BEGIN TRANSACTION; - EXECUTE sp_getapplock @Lock, 'Exclusive'; - WHILE @LookedAtPartitions <= @MaxPartitions - BEGIN - SET @InflightRows += (SELECT count(*) - FROM dbo.JobQueue - WHERE PartitionId = @PartitionId - AND QueueType = @QueueType - AND Status IN (0, 1)); - SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; - SET @LookedAtPartitions = @LookedAtPartitions + 1; - END - IF @InflightRows = 0 - BEGIN - SET @LookedAtPartitions = 0; - WHILE @LookedAtPartitions <= @MaxPartitions - BEGIN - UPDATE dbo.JobQueue - SET Status = 5 - WHERE PartitionId = @PartitionId - AND QueueType = @QueueType - AND Status IN (2, 3, 4); - SET @Rows += @@rowcount; - SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; - SET @LookedAtPartitions = @LookedAtPartitions + 1; - END - END - COMMIT TRANSACTION; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; -END TRY -BEGIN CATCH - IF @@trancount > 0 - ROLLBACK; - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.BatchDeleteResourceParams -@tableName NVARCHAR (128), @resourceTypeId SMALLINT, @startResourceSurrogateId BIGINT, @endResourceSurrogateId BIGINT, @batchSize INT -AS -SET XACT_ABORT ON; -SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; -BEGIN TRANSACTION; -DECLARE @Sql AS NVARCHAR (MAX); -DECLARE @ParmDefinition AS NVARCHAR (512); -IF OBJECT_ID(@tableName) IS NOT NULL - BEGIN - SET @sql = N'DELETE TOP(@BatchSizeParam) FROM ' + @tableName + N' WITH (TABLOCK) WHERE ResourceTypeId = @ResourceTypeIdParam AND ResourceSurrogateId >= @StartResourceSurrogateIdParam AND ResourceSurrogateId < @EndResourceSurrogateIdParam'; - SET @parmDefinition = N'@BatchSizeParam int, @ResourceTypeIdParam smallint, @StartResourceSurrogateIdParam bigint, @EndResourceSurrogateIdParam bigint'; - EXECUTE sp_executesql @sql, @parmDefinition, @BatchSizeParam = @batchSize, @ResourceTypeIdParam = @resourceTypeId, @StartResourceSurrogateIdParam = @startResourceSurrogateId, @EndResourceSurrogateIdParam = @endResourceSurrogateId; - END -COMMIT TRANSACTION; -RETURN @@rowcount; - -GO -CREATE PROCEDURE dbo.BatchDeleteResources -@resourceTypeId SMALLINT, @startResourceSurrogateId BIGINT, @endResourceSurrogateId BIGINT, @batchSize INT -AS -SET XACT_ABORT ON; -SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; -BEGIN TRANSACTION; -DELETE TOP (@batchSize) - dbo.Resource WITH (TABLOCK) -WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId >= @startResourceSurrogateId - AND ResourceSurrogateId < @endResourceSurrogateId; -COMMIT TRANSACTION; -RETURN @@rowcount; - -GO -CREATE PROCEDURE dbo.BatchDeleteResourceWriteClaims -@startResourceSurrogateId BIGINT, @endResourceSurrogateId BIGINT, @batchSize INT -AS -SET XACT_ABORT ON; -SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; -BEGIN TRANSACTION; -DELETE TOP (@batchSize) - dbo.ResourceWriteClaim WITH (TABLOCK) -WHERE ResourceSurrogateId >= @startResourceSurrogateId - AND ResourceSurrogateId < @endResourceSurrogateId; -COMMIT TRANSACTION; -RETURN @@rowcount; - -GO -CREATE PROCEDURE dbo.BulkMergeResource -@resources dbo.BulkImportResourceType_1 READONLY -AS -SET NOCOUNT ON; -SET XACT_ABORT ON; -BEGIN TRANSACTION; -MERGE INTO [dbo].[Resource] WITH (ROWLOCK, INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) - AS target -USING @resources AS source ON source.[ResourceTypeId] = target.[ResourceTypeId] - AND source.[ResourceId] = target.[ResourceId] - AND source.[Version] = target.[Version] -WHEN NOT MATCHED BY TARGET THEN INSERT ([ResourceTypeId], [ResourceId], [Version], [IsHistory], [ResourceSurrogateId], [IsDeleted], [RequestMethod], [RawResource], [IsRawResourceMetaSet], [SearchParamHash]) VALUES ([ResourceTypeId], [ResourceId], [Version], [IsHistory], [ResourceSurrogateId], [IsDeleted], [RequestMethod], [RawResource], [IsRawResourceMetaSet], [SearchParamHash]) OUTPUT Inserted.[ResourceSurrogateId]; -COMMIT TRANSACTION; - -GO -CREATE PROCEDURE dbo.BulkReindexResources_2 -@resourcesToReindex dbo.BulkReindexResourceTableType_1 READONLY, @resourceWriteClaims dbo.BulkResourceWriteClaimTableType_1 READONLY, @compartmentAssignments dbo.BulkCompartmentAssignmentTableType_1 READONLY, @referenceSearchParams dbo.BulkReferenceSearchParamTableType_1 READONLY, @tokenSearchParams dbo.BulkTokenSearchParamTableType_2 READONLY, @tokenTextSearchParams dbo.BulkTokenTextTableType_1 READONLY, @stringSearchParams dbo.BulkStringSearchParamTableType_2 READONLY, @numberSearchParams dbo.BulkNumberSearchParamTableType_1 READONLY, @quantitySearchParams dbo.BulkQuantitySearchParamTableType_1 READONLY, @uriSearchParams dbo.BulkUriSearchParamTableType_1 READONLY, @dateTimeSearchParms dbo.BulkDateTimeSearchParamTableType_2 READONLY, @referenceTokenCompositeSearchParams dbo.BulkReferenceTokenCompositeSearchParamTableType_2 READONLY, @tokenTokenCompositeSearchParams dbo.BulkTokenTokenCompositeSearchParamTableType_2 READONLY, @tokenDateTimeCompositeSearchParams dbo.BulkTokenDateTimeCompositeSearchParamTableType_2 READONLY, @tokenQuantityCompositeSearchParams dbo.BulkTokenQuantityCompositeSearchParamTableType_2 READONLY, @tokenStringCompositeSearchParams dbo.BulkTokenStringCompositeSearchParamTableType_2 READONLY, @tokenNumberNumberCompositeSearchParams dbo.BulkTokenNumberNumberCompositeSearchParamTableType_2 READONLY -AS -SET NOCOUNT ON; -SET XACT_ABORT ON; -BEGIN TRANSACTION; -DECLARE @computedValues TABLE ( - Offset INT NOT NULL, - ResourceTypeId SMALLINT NOT NULL, - VersionProvided BIGINT NULL, - SearchParamHash VARCHAR (64) NOT NULL, - ResourceSurrogateId BIGINT NULL, - VersionInDatabase BIGINT NULL); -INSERT INTO @computedValues -SELECT resourceToReindex.Offset, - resourceToReindex.ResourceTypeId, - resourceToReindex.ETag, - resourceToReindex.SearchParamHash, - resourceInDB.ResourceSurrogateId, - resourceInDB.Version -FROM @resourcesToReindex AS resourceToReindex - LEFT OUTER JOIN - dbo.Resource AS resourceInDB WITH (UPDLOCK, INDEX (IX_Resource_ResourceTypeId_ResourceId)) - ON resourceInDB.ResourceTypeId = resourceToReindex.ResourceTypeId - AND resourceInDB.ResourceId = resourceToReindex.ResourceId - AND resourceInDB.IsHistory = 0; -DECLARE @versionDiff AS INT; -SET @versionDiff = (SELECT COUNT(*) - FROM @computedValues - WHERE VersionProvided IS NOT NULL - AND VersionProvided <> VersionInDatabase); -IF (@versionDiff > 0) - BEGIN - DELETE @computedValues - WHERE VersionProvided IS NOT NULL - AND VersionProvided <> VersionInDatabase; - END -UPDATE resourceInDB -SET resourceInDB.SearchParamHash = resourceToReindex.SearchParamHash -FROM @computedValues AS resourceToReindex - INNER JOIN - dbo.Resource AS resourceInDB - ON resourceInDB.ResourceTypeId = resourceToReindex.ResourceTypeId - AND resourceInDB.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; -DELETE searchIndex -FROM dbo.ResourceWriteClaim AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; -DELETE searchIndex -FROM dbo.CompartmentAssignment AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId - AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; -DELETE searchIndex -FROM dbo.ReferenceSearchParam AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId - AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; -DELETE searchIndex -FROM dbo.TokenSearchParam AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId - AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; -DELETE searchIndex -FROM dbo.TokenText AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId - AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; -DELETE searchIndex -FROM dbo.StringSearchParam AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId - AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; -DELETE searchIndex -FROM dbo.UriSearchParam AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId - AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; -DELETE searchIndex -FROM dbo.NumberSearchParam AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId - AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; -DELETE searchIndex -FROM dbo.QuantitySearchParam AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId - AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; -DELETE searchIndex -FROM dbo.DateTimeSearchParam AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId - AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; -DELETE searchIndex -FROM dbo.ReferenceTokenCompositeSearchParam AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId - AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; -DELETE searchIndex -FROM dbo.TokenTokenCompositeSearchParam AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId - AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; -DELETE searchIndex -FROM dbo.TokenDateTimeCompositeSearchParam AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId - AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; -DELETE searchIndex -FROM dbo.TokenQuantityCompositeSearchParam AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId - AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; -DELETE searchIndex -FROM dbo.TokenStringCompositeSearchParam AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId - AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; -DELETE searchIndex -FROM dbo.TokenNumberNumberCompositeSearchParam AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId - AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; -INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) -SELECT DISTINCT resourceToReindex.ResourceSurrogateId, - searchIndex.ClaimTypeId, - searchIndex.ClaimValue -FROM @resourceWriteClaims AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.Offset = resourceToReindex.Offset; -INSERT INTO dbo.CompartmentAssignment (ResourceTypeId, ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId, IsHistory) -SELECT DISTINCT resourceToReindex.ResourceTypeId, - resourceToReindex.ResourceSurrogateId, - searchIndex.CompartmentTypeId, - searchIndex.ReferenceResourceId, - 0 -FROM @compartmentAssignments AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.Offset = resourceToReindex.Offset; -INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion, IsHistory) -SELECT DISTINCT resourceToReindex.ResourceTypeId, - resourceToReindex.ResourceSurrogateId, - searchIndex.SearchParamId, - searchIndex.BaseUri, - searchIndex.ReferenceResourceTypeId, - searchIndex.ReferenceResourceId, - searchIndex.ReferenceResourceVersion, - 0 -FROM @referenceSearchParams AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.Offset = resourceToReindex.Offset; -INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow, IsHistory) -SELECT DISTINCT resourceToReindex.ResourceTypeId, - resourceToReindex.ResourceSurrogateId, - searchIndex.SearchParamId, - searchIndex.SystemId, - searchIndex.Code, - searchIndex.CodeOverflow, - 0 -FROM @tokenSearchParams AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.Offset = resourceToReindex.Offset; -INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, IsHistory) -SELECT DISTINCT resourceToReindex.ResourceTypeId, - resourceToReindex.ResourceSurrogateId, - searchIndex.SearchParamId, - searchIndex.Text, - 0 -FROM @tokenTextSearchParams AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.Offset = resourceToReindex.Offset; -INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsHistory, IsMin, IsMax) -SELECT DISTINCT resourceToReindex.ResourceTypeId, - resourceToReindex.ResourceSurrogateId, - searchIndex.SearchParamId, - searchIndex.Text, - searchIndex.TextOverflow, - 0, - searchIndex.IsMin, - searchIndex.IsMax -FROM @stringSearchParams AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.Offset = resourceToReindex.Offset; -INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri, IsHistory) -SELECT DISTINCT resourceToReindex.ResourceTypeId, - resourceToReindex.ResourceSurrogateId, - searchIndex.SearchParamId, - searchIndex.Uri, - 0 -FROM @uriSearchParams AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.Offset = resourceToReindex.Offset; -INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue, IsHistory) -SELECT DISTINCT resourceToReindex.ResourceTypeId, - resourceToReindex.ResourceSurrogateId, - searchIndex.SearchParamId, - searchIndex.SingleValue, - searchIndex.LowValue, - searchIndex.HighValue, - 0 -FROM @numberSearchParams AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.Offset = resourceToReindex.Offset; -INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue, IsHistory) -SELECT DISTINCT resourceToReindex.ResourceTypeId, - resourceToReindex.ResourceSurrogateId, - searchIndex.SearchParamId, - searchIndex.SystemId, - searchIndex.QuantityCodeId, - searchIndex.SingleValue, - searchIndex.LowValue, - searchIndex.HighValue, - 0 -FROM @quantitySearchParams AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.Offset = resourceToReindex.Offset; -INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsHistory, IsMin, IsMax) -SELECT DISTINCT resourceToReindex.ResourceTypeId, - resourceToReindex.ResourceSurrogateId, - searchIndex.SearchParamId, - searchIndex.StartDateTime, - searchIndex.EndDateTime, - searchIndex.IsLongerThanADay, - 0, - searchIndex.IsMin, - searchIndex.IsMax -FROM @dateTimeSearchParms AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.Offset = resourceToReindex.Offset; -INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2, IsHistory) -SELECT DISTINCT resourceToReindex.ResourceTypeId, - resourceToReindex.ResourceSurrogateId, - searchIndex.SearchParamId, - searchIndex.BaseUri1, - searchIndex.ReferenceResourceTypeId1, - searchIndex.ReferenceResourceId1, - searchIndex.ReferenceResourceVersion1, - searchIndex.SystemId2, - searchIndex.Code2, - searchIndex.CodeOverflow2, - 0 -FROM @referenceTokenCompositeSearchParams AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.Offset = resourceToReindex.Offset; -INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2, IsHistory) -SELECT DISTINCT resourceToReindex.ResourceTypeId, - resourceToReindex.ResourceSurrogateId, - searchIndex.SearchParamId, - searchIndex.SystemId1, - searchIndex.Code1, - searchIndex.CodeOverflow1, - searchIndex.SystemId2, - searchIndex.Code2, - searchIndex.CodeOverflow2, - 0 -FROM @tokenTokenCompositeSearchParams AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.Offset = resourceToReindex.Offset; -INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2, IsHistory) -SELECT DISTINCT resourceToReindex.ResourceTypeId, - resourceToReindex.ResourceSurrogateId, - searchIndex.SearchParamId, - searchIndex.SystemId1, - searchIndex.Code1, - searchIndex.CodeOverflow1, - searchIndex.StartDateTime2, - searchIndex.EndDateTime2, - searchIndex.IsLongerThanADay2, - 0 -FROM @tokenDateTimeCompositeSearchParams AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.Offset = resourceToReindex.Offset; -INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2, IsHistory) -SELECT DISTINCT resourceToReindex.ResourceTypeId, - resourceToReindex.ResourceSurrogateId, - searchIndex.SearchParamId, - searchIndex.SystemId1, - searchIndex.Code1, - searchIndex.CodeOverflow1, - searchIndex.SingleValue2, - searchIndex.SystemId2, - searchIndex.QuantityCodeId2, - searchIndex.LowValue2, - searchIndex.HighValue2, - 0 -FROM @tokenQuantityCompositeSearchParams AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.Offset = resourceToReindex.Offset; -INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2, IsHistory) -SELECT DISTINCT resourceToReindex.ResourceTypeId, - resourceToReindex.ResourceSurrogateId, - searchIndex.SearchParamId, - searchIndex.SystemId1, - searchIndex.Code1, - searchIndex.CodeOverflow1, - searchIndex.Text2, - searchIndex.TextOverflow2, - 0 -FROM @tokenStringCompositeSearchParams AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.Offset = resourceToReindex.Offset; -INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange, IsHistory) -SELECT DISTINCT resourceToReindex.ResourceTypeId, - resourceToReindex.ResourceSurrogateId, - searchIndex.SearchParamId, - searchIndex.SystemId1, - searchIndex.Code1, - searchIndex.CodeOverflow1, - searchIndex.SingleValue2, - searchIndex.LowValue2, - searchIndex.HighValue2, - searchIndex.SingleValue3, - searchIndex.LowValue3, - searchIndex.HighValue3, - searchIndex.HasRange, - 0 -FROM @tokenNumberNumberCompositeSearchParams AS searchIndex - INNER JOIN - @computedValues AS resourceToReindex - ON searchIndex.Offset = resourceToReindex.Offset; -SELECT @versionDiff; -COMMIT TRANSACTION; - -GO -CREATE PROCEDURE [dbo].[CancelTask] -@taskId VARCHAR (64) -AS -SET NOCOUNT ON; -SET XACT_ABORT ON; -BEGIN TRANSACTION; -DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); -IF NOT EXISTS (SELECT * - FROM [dbo].[TaskInfo] - WHERE TaskId = @taskId) - BEGIN - THROW 50404, 'Task not exist', 1; - END -UPDATE dbo.TaskInfo -SET IsCanceled = 1, - HeartbeatDateTime = @heartbeatDateTime -WHERE TaskId = @taskId; -SELECT TaskId, - QueueId, - Status, - TaskTypeId, - RunId, - IsCanceled, - RetryCount, - MaxRetryCount, - HeartbeatDateTime, - InputData, - TaskContext, - Result -FROM [dbo].[TaskInfo] -WHERE TaskId = @taskId; -COMMIT TRANSACTION; - -GO -CREATE PROCEDURE dbo.CaptureResourceChanges -@isDeleted BIT, @version INT, @resourceId VARCHAR (64), @resourceTypeId SMALLINT -AS -BEGIN - DECLARE @changeType AS SMALLINT; - IF (@isDeleted = 1) - BEGIN - SET @changeType = 2; - END - ELSE - BEGIN - IF (@version = 1) - BEGIN - SET @changeType = 0; - END - ELSE - BEGIN - SET @changeType = 1; - END - END - INSERT INTO dbo.ResourceChangeData (ResourceId, ResourceTypeId, ResourceVersion, ResourceChangeTypeId) - VALUES (@resourceId, @resourceTypeId, @version, @changeType); -END - -GO -CREATE PROCEDURE dbo.CaptureResourceIdsForChanges -@Resources dbo.ResourceList READONLY -AS -SET NOCOUNT ON; -INSERT INTO dbo.ResourceChangeData (ResourceId, ResourceTypeId, ResourceVersion, ResourceChangeTypeId) -SELECT ResourceId, - ResourceTypeId, - Version, - CASE WHEN IsDeleted = 1 THEN 2 WHEN Version > 1 THEN 1 ELSE 0 END -FROM @Resources -WHERE IsHistory = 0; - -GO -CREATE PROCEDURE dbo.CheckActiveReindexJobs -AS -SET NOCOUNT ON; -SELECT Id -FROM dbo.ReindexJob -WHERE Status = 'Running' - OR Status = 'Queued' - OR Status = 'Paused'; - -GO -CREATE PROCEDURE dbo.CleanupEventLog -WITH EXECUTE AS 'dbo' -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'CleanupEventLog', @Mode AS VARCHAR (100) = '', @MaxDeleteRows AS INT, @MaxAllowedRows AS BIGINT, @RetentionPeriodSecond AS INT, @DeletedRows AS INT, @TotalDeletedRows AS INT = 0, @TotalRows AS INT, @Now AS DATETIME = getUTCdate(); -EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; -BEGIN TRY - SET @MaxDeleteRows = (SELECT Number - FROM dbo.Parameters - WHERE Id = 'CleanupEventLog.DeleteBatchSize'); - IF @MaxDeleteRows IS NULL - RAISERROR ('Cannot get Parameter.CleanupEventLog.DeleteBatchSize', 18, 127); - SET @MaxAllowedRows = (SELECT Number - FROM dbo.Parameters - WHERE Id = 'CleanupEventLog.AllowedRows'); - IF @MaxAllowedRows IS NULL - RAISERROR ('Cannot get Parameter.CleanupEventLog.AllowedRows', 18, 127); - SET @RetentionPeriodSecond = (SELECT Number * 24 * 60 * 60 - FROM dbo.Parameters - WHERE Id = 'CleanupEventLog.RetentionPeriodDay'); - IF @RetentionPeriodSecond IS NULL - RAISERROR ('Cannot get Parameter.CleanupEventLog.RetentionPeriodDay', 18, 127); - SET @TotalRows = (SELECT sum(row_count) - FROM sys.dm_db_partition_stats - WHERE object_id = object_id('EventLog') - AND index_id IN (0, 1)); - SET @DeletedRows = 1; - WHILE @DeletedRows > 0 - AND EXISTS (SELECT * - FROM dbo.Parameters - WHERE Id = 'CleanupEventLog.IsEnabled' - AND Number = 1) - BEGIN - SET @DeletedRows = 0; - IF @TotalRows - @TotalDeletedRows > @MaxAllowedRows - BEGIN - DELETE TOP (@MaxDeleteRows) - dbo.EventLog WITH (PAGLOCK) - WHERE EventDate <= dateadd(second, -@RetentionPeriodSecond, @Now); - SET @DeletedRows = @@rowcount; - SET @TotalDeletedRows += @DeletedRows; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'EventLog', @Action = 'Delete', @Rows = @DeletedRows, @Text = @TotalDeletedRows; - END - END - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @Now; -END TRY -BEGIN CATCH - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.CompleteTask -@taskId VARCHAR (64), @taskResult VARCHAR (MAX), @runId VARCHAR (50) -AS -SET NOCOUNT ON; -SET XACT_ABORT ON; -BEGIN TRANSACTION; -IF NOT EXISTS (SELECT * - FROM [dbo].[TaskInfo] - WHERE TaskId = @taskId - AND RunId = @runId) - BEGIN - THROW 50404, 'Task not exist or runid not match', 1; - END -UPDATE dbo.TaskInfo -SET Status = 3, - EndDateTime = SYSUTCDATETIME(), - Result = @taskResult -WHERE TaskId = @taskId; -COMMIT TRANSACTION; -EXECUTE dbo.GetTaskDetails @TaskId = @taskId; - -GO -CREATE OR ALTER PROCEDURE dbo.ConfigurePartitionOnResourceChanges -@numberOfFuturePartitionsToAdd INT -AS -BEGIN - SET XACT_ABORT ON; - BEGIN TRANSACTION; - DECLARE @partitionBoundary AS DATETIME2 (7) = DATEADD(hour, DATEDIFF(hour, 0, sysutcdatetime()), 0); - DECLARE @startingRightPartitionBoundary AS DATETIME2 (7) = CAST ((SELECT TOP (1) value - FROM sys.partition_range_values AS prv - INNER JOIN - sys.partition_functions AS pf - ON pf.function_id = prv.function_id - WHERE pf.name = N'PartitionFunction_ResourceChangeData_Timestamp' - ORDER BY prv.boundary_id DESC) AS DATETIME2 (7)); - DECLARE @numberOfPartitionsToAdd AS INT = @numberOfFuturePartitionsToAdd + 1; - WHILE @numberOfPartitionsToAdd > 0 - BEGIN - IF (@startingRightPartitionBoundary < @partitionBoundary) - BEGIN - ALTER PARTITION SCHEME PartitionScheme_ResourceChangeData_Timestamp NEXT USED [PRIMARY]; - ALTER PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp( ) - SPLIT RANGE (@partitionBoundary); - END - SET @partitionBoundary = DATEADD(hour, 1, @partitionBoundary); - SET @numberOfPartitionsToAdd -= 1; - END - COMMIT TRANSACTION; -END - -GO -CREATE PROCEDURE dbo.CreateExportJob -@id VARCHAR (64), @hash VARCHAR (64), @status VARCHAR (10), @rawJobRecord VARCHAR (MAX) -AS -SET NOCOUNT ON; -SET XACT_ABORT ON; -BEGIN TRANSACTION; -DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); -INSERT INTO dbo.ExportJob (Id, Hash, Status, HeartbeatDateTime, RawJobRecord) -VALUES (@id, @hash, @status, @heartbeatDateTime, @rawJobRecord); -SELECT CAST (MIN_ACTIVE_ROWVERSION() AS INT); -COMMIT TRANSACTION; - -GO -CREATE PROCEDURE dbo.CreateReindexJob -@id VARCHAR (64), @status VARCHAR (10), @rawJobRecord VARCHAR (MAX) -AS -SET NOCOUNT ON; -SET XACT_ABORT ON; -BEGIN TRANSACTION; -DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); -INSERT INTO dbo.ReindexJob (Id, Status, HeartbeatDateTime, RawJobRecord) -VALUES (@id, @status, @heartbeatDateTime, @rawJobRecord); -SELECT CAST (MIN_ACTIVE_ROWVERSION() AS INT); -COMMIT TRANSACTION; - -GO -CREATE PROCEDURE [dbo].[CreateTask_3] -@taskId VARCHAR (64), @queueId VARCHAR (64), @taskTypeId SMALLINT, @parentTaskId VARCHAR (64), @maxRetryCount SMALLINT=3, @inputData VARCHAR (MAX), @isUniqueTaskByType BIT -AS -SET NOCOUNT ON; -SET XACT_ABORT ON; -BEGIN TRANSACTION; -DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); -DECLARE @status AS SMALLINT = 1; -DECLARE @retryCount AS SMALLINT = 0; -DECLARE @isCanceled AS BIT = 0; -IF (@isUniqueTaskByType = 1) - BEGIN - IF EXISTS (SELECT * - FROM [dbo].[TaskInfo] - WHERE TaskId = @taskId - OR (TaskTypeId = @taskTypeId - AND Status <> 3)) - BEGIN - THROW 50409, 'Task already existed', 1; - END - END -ELSE - BEGIN - IF EXISTS (SELECT * - FROM [dbo].[TaskInfo] - WHERE TaskId = @taskId) - BEGIN - THROW 50409, 'Task already existed', 1; - END - END -INSERT INTO [dbo].[TaskInfo] (TaskId, QueueId, Status, TaskTypeId, IsCanceled, RetryCount, MaxRetryCount, HeartbeatDateTime, InputData, ParentTaskId) -VALUES (@taskId, @queueId, @status, @taskTypeId, @isCanceled, @retryCount, @maxRetryCount, @heartbeatDateTime, @inputData, @parentTaskId); -EXECUTE dbo.GetTaskDetails @TaskId = @taskId; -COMMIT TRANSACTION; - -GO -CREATE PROCEDURE dbo.Defrag -@TableName VARCHAR (100), @IndexName VARCHAR (200), @PartitionNumber INT, @IsPartitioned BIT -WITH EXECUTE AS 'dbo' -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'Defrag', @Mode AS VARCHAR (200) = @TableName + '.' + @IndexName + '.' + CONVERT (VARCHAR, @PartitionNumber) + '.' + CONVERT (VARCHAR, @IsPartitioned), @st AS DATETIME = getUTCdate(), @SQL AS VARCHAR (3500), @msg AS VARCHAR (1000), @SizeBefore AS FLOAT, @SizeAfter AS FLOAT, @IndexId AS INT; -BEGIN TRY - SET @IndexId = (SELECT index_id - FROM sys.indexes - WHERE object_id = object_id(@TableName) - AND name = @IndexName); - SET @SizeBefore = (SELECT sum(reserved_page_count) - FROM sys.dm_db_partition_stats - WHERE object_id = object_id(@TableName) - AND index_id = @IndexId) * 8.0 / 1024 / 1024; - SET @msg = 'Size[GB] before=' + CONVERT (VARCHAR, @SizeBefore); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start', @Text = @msg; - SET @Sql = 'ALTER INDEX ' + quotename(@IndexName) + ' ON dbo.' + quotename(@TableName) + ' REORGANIZE' + CASE WHEN @IsPartitioned = 1 THEN ' PARTITION = ' + CONVERT (VARCHAR, @PartitionNumber) ELSE '' END; - BEGIN TRY - EXECUTE (@Sql); - SET @SizeAfter = (SELECT sum(reserved_page_count) - FROM sys.dm_db_partition_stats - WHERE object_id = object_id(@TableName) - AND index_id = @IndexId) * 8.0 / 1024 / 1024; - SET @msg = 'Size[GB] before=' + CONVERT (VARCHAR, @SizeBefore) + ', after=' + CONVERT (VARCHAR, @SizeAfter) + ', reduced by=' + CONVERT (VARCHAR, @SizeBefore - @SizeAfter); - EXECUTE dbo.LogEvent @Process = @SP, @Status = 'End', @Mode = @Mode, @Action = 'Reorganize', @Start = @st, @Text = @msg; - END TRY - BEGIN CATCH - EXECUTE dbo.LogEvent @Process = @SP, @Status = 'Error', @Mode = @Mode, @Action = 'Reorganize', @Start = @st, @ReRaisError = 0; - END CATCH -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.DefragChangeDatabaseSettings -@IsOn BIT -WITH EXECUTE AS 'dbo' -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'DefragChangeDatabaseSettings', @Mode AS VARCHAR (200) = 'On=' + CONVERT (VARCHAR, @IsOn), @st AS DATETIME = getUTCdate(), @SQL AS VARCHAR (3500); -BEGIN TRY - EXECUTE dbo.LogEvent @Process = @SP, @Status = 'Start', @Mode = @Mode; - SET @SQL = 'ALTER DATABASE CURRENT SET AUTO_UPDATE_STATISTICS ' + CASE WHEN @IsOn = 1 THEN 'ON' ELSE 'OFF' END; - EXECUTE (@SQL); - EXECUTE dbo.LogEvent @Process = @SP, @Status = 'Run', @Mode = @Mode, @Text = @SQL; - SET @SQL = 'ALTER DATABASE CURRENT SET AUTO_CREATE_STATISTICS ' + CASE WHEN @IsOn = 1 THEN 'ON' ELSE 'OFF' END; - EXECUTE (@SQL); - EXECUTE dbo.LogEvent @Process = @SP, @Status = 'End', @Mode = @Mode, @Start = @st, @Text = @SQL; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.DeleteHistory -@DeleteResources BIT=0, @Reset BIT=0, @DisableLogEvent BIT=0 -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'DeleteHistory', @Mode AS VARCHAR (100) = 'D=' + isnull(CONVERT (VARCHAR, @DeleteResources), 'NULL') + ' R=' + isnull(CONVERT (VARCHAR, @Reset), 'NULL'), @st AS DATETIME = getUTCdate(), @Id AS VARCHAR (100) = 'DeleteHistory.LastProcessed.TypeId.SurrogateId', @ResourceTypeId AS SMALLINT, @SurrogateId AS BIGINT, @RowsToProcess AS INT, @ProcessedResources AS INT = 0, @DeletedResources AS INT = 0, @DeletedSearchParams AS INT = 0, @ReportDate AS DATETIME = getUTCdate(); -BEGIN TRY - IF @DisableLogEvent = 0 - INSERT INTO dbo.Parameters (Id, Char) - SELECT @SP, - 'LogEvent'; - ELSE - DELETE dbo.Parameters - WHERE Id = @SP; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; - INSERT INTO dbo.Parameters (Id, Char) - SELECT @Id, - '0.0' - WHERE NOT EXISTS (SELECT * - FROM dbo.Parameters - WHERE Id = @Id); - DECLARE @LastProcessed AS VARCHAR (100) = CASE WHEN @Reset = 0 THEN (SELECT Char - FROM dbo.Parameters - WHERE Id = @Id) ELSE '0.0' END; - DECLARE @Types TABLE ( - ResourceTypeId SMALLINT PRIMARY KEY, - Name VARCHAR (100)); - DECLARE @SurrogateIds TABLE ( - ResourceSurrogateId BIGINT PRIMARY KEY, - IsHistory BIT ); - INSERT INTO @Types - EXECUTE dbo.GetUsedResourceTypes ; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = '@Types', @Action = 'Insert', @Rows = @@rowcount; - SET @ResourceTypeId = substring(@LastProcessed, 1, charindex('.', @LastProcessed) - 1); - SET @SurrogateId = substring(@LastProcessed, charindex('.', @LastProcessed) + 1, 255); - DELETE @Types - WHERE ResourceTypeId < @ResourceTypeId; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = '@Types', @Action = 'Delete', @Rows = @@rowcount; - WHILE EXISTS (SELECT * - FROM @Types) - BEGIN - SET @ResourceTypeId = (SELECT TOP 1 ResourceTypeId - FROM @Types - ORDER BY ResourceTypeId); - SET @ProcessedResources = 0; - SET @DeletedResources = 0; - SET @DeletedSearchParams = 0; - SET @RowsToProcess = 1; - WHILE @RowsToProcess > 0 - BEGIN - DELETE @SurrogateIds; - INSERT INTO @SurrogateIds - SELECT TOP 10000 ResourceSurrogateId, - IsHistory - FROM dbo.Resource - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId > @SurrogateId - ORDER BY ResourceSurrogateId; - SET @RowsToProcess = @@rowcount; - SET @ProcessedResources += @RowsToProcess; - IF @RowsToProcess > 0 - SET @SurrogateId = (SELECT max(ResourceSurrogateId) - FROM @SurrogateIds); - SET @LastProcessed = CONVERT (VARCHAR, @ResourceTypeId) + '.' + CONVERT (VARCHAR, @SurrogateId); - DELETE @SurrogateIds - WHERE IsHistory = 0; - IF EXISTS (SELECT * - FROM @SurrogateIds) - BEGIN - DELETE dbo.ResourceWriteClaim - WHERE ResourceSurrogateId IN (SELECT ResourceSurrogateId - FROM @SurrogateIds); - SET @DeletedSearchParams += @@rowcount; - DELETE dbo.CompartmentAssignment - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId IN (SELECT ResourceSurrogateId - FROM @SurrogateIds); - SET @DeletedSearchParams += @@rowcount; - DELETE dbo.ReferenceSearchParam - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId IN (SELECT ResourceSurrogateId - FROM @SurrogateIds); - SET @DeletedSearchParams += @@rowcount; - DELETE dbo.TokenSearchParam - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId IN (SELECT ResourceSurrogateId - FROM @SurrogateIds); - SET @DeletedSearchParams += @@rowcount; - DELETE dbo.TokenText - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId IN (SELECT ResourceSurrogateId - FROM @SurrogateIds); - SET @DeletedSearchParams += @@rowcount; - DELETE dbo.StringSearchParam - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId IN (SELECT ResourceSurrogateId - FROM @SurrogateIds); - SET @DeletedSearchParams += @@rowcount; - DELETE dbo.UriSearchParam - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId IN (SELECT ResourceSurrogateId - FROM @SurrogateIds); - SET @DeletedSearchParams += @@rowcount; - DELETE dbo.NumberSearchParam - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId IN (SELECT ResourceSurrogateId - FROM @SurrogateIds); - SET @DeletedSearchParams += @@rowcount; - DELETE dbo.QuantitySearchParam - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId IN (SELECT ResourceSurrogateId - FROM @SurrogateIds); - SET @DeletedSearchParams += @@rowcount; - DELETE dbo.DateTimeSearchParam - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId IN (SELECT ResourceSurrogateId - FROM @SurrogateIds); - SET @DeletedSearchParams += @@rowcount; - DELETE dbo.ReferenceTokenCompositeSearchParam - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId IN (SELECT ResourceSurrogateId - FROM @SurrogateIds); - SET @DeletedSearchParams += @@rowcount; - DELETE dbo.TokenTokenCompositeSearchParam - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId IN (SELECT ResourceSurrogateId - FROM @SurrogateIds); - SET @DeletedSearchParams += @@rowcount; - DELETE dbo.TokenDateTimeCompositeSearchParam - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId IN (SELECT ResourceSurrogateId - FROM @SurrogateIds); - SET @DeletedSearchParams += @@rowcount; - DELETE dbo.TokenQuantityCompositeSearchParam - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId IN (SELECT ResourceSurrogateId - FROM @SurrogateIds); - SET @DeletedSearchParams += @@rowcount; - DELETE dbo.TokenStringCompositeSearchParam - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId IN (SELECT ResourceSurrogateId - FROM @SurrogateIds); - SET @DeletedSearchParams += @@rowcount; - DELETE dbo.TokenNumberNumberCompositeSearchParam - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId IN (SELECT ResourceSurrogateId - FROM @SurrogateIds); - SET @DeletedSearchParams += @@rowcount; - IF @DeleteResources = 1 - BEGIN - DELETE dbo.Resource - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId IN (SELECT ResourceSurrogateId - FROM @SurrogateIds); - SET @DeletedResources += @@rowcount; - END - END - UPDATE dbo.Parameters - SET Char = @LastProcessed - WHERE Id = @Id; - IF datediff(second, @ReportDate, getUTCdate()) > 60 - BEGIN - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'Resource', @Action = 'Select', @Rows = @ProcessedResources, @Text = @LastProcessed; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = '*SearchParam', @Action = 'Delete', @Rows = @DeletedSearchParams, @Text = @LastProcessed; - IF @DeleteResources = 1 - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'Resource', @Action = 'Delete', @Rows = @DeletedResources, @Text = @LastProcessed; - SET @ReportDate = getUTCdate(); - SET @ProcessedResources = 0; - SET @DeletedSearchParams = 0; - SET @DeletedResources = 0; - END - END - DELETE @Types - WHERE ResourceTypeId = @ResourceTypeId; - SET @SurrogateId = 0; - END - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'Resource', @Action = 'Select', @Rows = @ProcessedResources, @Text = @LastProcessed; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = '*SearchParam', @Action = 'Delete', @Rows = @DeletedSearchParams, @Text = @LastProcessed; - IF @DeleteResources = 1 - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'Resource', @Action = 'Delete', @Rows = @DeletedResources, @Text = @LastProcessed; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.DequeueJob -@QueueType TINYINT, @Worker VARCHAR (100), @HeartbeatTimeoutSec INT, @InputJobId BIGINT=NULL, @CheckTimeoutJobs BIT=0 -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'DequeueJob', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' H=' + isnull(CONVERT (VARCHAR, @HeartbeatTimeoutSec), 'NULL') + ' W=' + isnull(@Worker, 'NULL') + ' IJ=' + isnull(CONVERT (VARCHAR, @InputJobId), 'NULL') + ' T=' + isnull(CONVERT (VARCHAR, @CheckTimeoutJobs), 'NULL'), @Rows AS INT = 0, @st AS DATETIME = getUTCdate(), @JobId AS BIGINT, @msg AS VARCHAR (100), @Lock AS VARCHAR (100), @PartitionId AS TINYINT, @MaxPartitions AS TINYINT = 16, @LookedAtPartitions AS TINYINT = 0; -BEGIN TRY - IF EXISTS (SELECT * - FROM dbo.Parameters - WHERE Id = 'DequeueJobStop' - AND Number = 1) - BEGIN - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = 0, @Text = 'Skipped'; - RETURN; - END - IF @InputJobId IS NULL - SET @PartitionId = @MaxPartitions * rand(); - ELSE - SET @PartitionId = @InputJobId % 16; - SET TRANSACTION ISOLATION LEVEL READ COMMITTED; - WHILE @InputJobId IS NULL - AND @JobId IS NULL - AND @LookedAtPartitions <= @MaxPartitions - AND @CheckTimeoutJobs = 0 - BEGIN - SET @Lock = 'DequeueJob_' + CONVERT (VARCHAR, @QueueType) + '_' + CONVERT (VARCHAR, @PartitionId); - BEGIN TRANSACTION; - EXECUTE sp_getapplock @Lock, 'Exclusive'; - UPDATE T - SET StartDate = getUTCdate(), - HeartbeatDate = getUTCdate(), - Worker = @Worker, - Status = 1, - Version = datediff_big(millisecond, '0001-01-01', getUTCdate()), - @JobId = T.JobId - FROM dbo.JobQueue AS T WITH (PAGLOCK) - INNER JOIN - (SELECT TOP 1 JobId - FROM dbo.JobQueue WITH (INDEX (IX_QueueType_PartitionId_Status_Priority)) - WHERE QueueType = @QueueType - AND PartitionId = @PartitionId - AND Status = 0 - ORDER BY Priority, JobId) AS S - ON QueueType = @QueueType - AND PartitionId = @PartitionId - AND T.JobId = S.JobId; - SET @Rows += @@rowcount; - COMMIT TRANSACTION; - IF @JobId IS NULL - BEGIN - SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; - SET @LookedAtPartitions = @LookedAtPartitions + 1; - END - END - SET @LookedAtPartitions = 0; - WHILE @InputJobId IS NULL - AND @JobId IS NULL - AND @LookedAtPartitions <= @MaxPartitions - BEGIN - SET @Lock = 'DequeueStoreCopyWorkUnit_' + CONVERT (VARCHAR, @PartitionId); - BEGIN TRANSACTION; - EXECUTE sp_getapplock @Lock, 'Exclusive'; - UPDATE T - SET StartDate = getUTCdate(), - HeartbeatDate = getUTCdate(), - Worker = @Worker, - Status = CASE WHEN CancelRequested = 0 THEN 1 ELSE 4 END, - Version = datediff_big(millisecond, '0001-01-01', getUTCdate()), - @JobId = CASE WHEN CancelRequested = 0 THEN T.JobId END, - Info = CONVERT (VARCHAR (1000), isnull(Info, '') + ' Prev: Worker=' + Worker + ' Start=' + CONVERT (VARCHAR, StartDate, 121)) - FROM dbo.JobQueue AS T WITH (PAGLOCK) - INNER JOIN - (SELECT TOP 1 JobId - FROM dbo.JobQueue WITH (INDEX (IX_QueueType_PartitionId_Status_Priority)) - WHERE QueueType = @QueueType - AND PartitionId = @PartitionId - AND Status = 1 - AND datediff(second, HeartbeatDate, getUTCdate()) > @HeartbeatTimeoutSec - ORDER BY Priority, JobId) AS S - ON QueueType = @QueueType - AND PartitionId = @PartitionId - AND T.JobId = S.JobId; - SET @Rows += @@rowcount; - COMMIT TRANSACTION; - IF @JobId IS NULL - BEGIN - SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; - SET @LookedAtPartitions = @LookedAtPartitions + 1; - END - END - IF @InputJobId IS NOT NULL - BEGIN - UPDATE dbo.JobQueue WITH (PAGLOCK) - SET StartDate = getUTCdate(), - HeartbeatDate = getUTCdate(), - Worker = @Worker, - Status = 1, - Version = datediff_big(millisecond, '0001-01-01', getUTCdate()), - @JobId = JobId - WHERE QueueType = @QueueType - AND PartitionId = @PartitionId - AND Status = 0 - AND JobId = @InputJobId; - SET @Rows += @@rowcount; - IF @JobId IS NULL - BEGIN - UPDATE dbo.JobQueue WITH (PAGLOCK) - SET StartDate = getUTCdate(), - HeartbeatDate = getUTCdate(), - Worker = @Worker, - Status = 1, - Version = datediff_big(millisecond, '0001-01-01', getUTCdate()), - @JobId = JobId - WHERE QueueType = @QueueType - AND PartitionId = @PartitionId - AND Status = 1 - AND JobId = @InputJobId - AND datediff(second, HeartbeatDate, getUTCdate()) > @HeartbeatTimeoutSec; - SET @Rows += @@rowcount; - END - END - IF @JobId IS NOT NULL - EXECUTE dbo.GetJobs @QueueType = @QueueType, @JobId = @JobId; - SET @msg = 'J=' + isnull(CONVERT (VARCHAR, @JobId), 'NULL') + ' P=' + CONVERT (VARCHAR, @PartitionId); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows, @Text = @msg; -END TRY -BEGIN CATCH - IF @@trancount > 0 - ROLLBACK; - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.DisableIndex -@tableName NVARCHAR (128), @indexName NVARCHAR (128) -WITH EXECUTE AS 'dbo' -AS -DECLARE @errorTxt AS VARCHAR (1000), @sql AS NVARCHAR (1000), @isDisabled AS BIT; -IF object_id(@tableName) IS NULL - BEGIN - SET @errorTxt = @tableName + ' does not exist or you don''t have permissions.'; - RAISERROR (@errorTxt, 18, 127); - END -SET @isDisabled = (SELECT is_disabled - FROM sys.indexes - WHERE object_id = object_id(@tableName) - AND name = @indexName); -IF @isDisabled IS NULL - BEGIN - SET @errorTxt = @indexName + ' does not exist or you don''t have permissions.'; - RAISERROR (@errorTxt, 18, 127); - END -IF @isDisabled = 0 - BEGIN - SET @sql = N'ALTER INDEX ' + QUOTENAME(@indexName) + N' on ' + @tableName + ' Disable'; - EXECUTE sp_executesql @sql; - END - -GO -CREATE PROCEDURE dbo.DisableIndexes -WITH EXECUTE AS 'dbo' -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'DisableIndexes', @Mode AS VARCHAR (200) = '', @st AS DATETIME = getUTCdate(), @Tbl AS VARCHAR (100), @Ind AS VARCHAR (200), @Txt AS VARCHAR (4000); -BEGIN TRY - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; - DECLARE @Tables TABLE ( - Tbl VARCHAR (100) PRIMARY KEY, - Supported BIT ); - INSERT INTO @Tables - EXECUTE dbo.GetPartitionedTables @IncludeNotDisabled = 1, @IncludeNotSupported = 0; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Tables', @Action = 'Insert', @Rows = @@rowcount; - DECLARE @Indexes TABLE ( - Tbl VARCHAR (100), - Ind VARCHAR (200), - TblId INT , - IndId INT PRIMARY KEY (Tbl, Ind)); - INSERT INTO @Indexes - SELECT Tbl, - I.Name, - TblId, - I.index_id - FROM (SELECT object_id(Tbl) AS TblId, - Tbl - FROM @Tables) AS O - INNER JOIN - sys.indexes AS I - ON I.object_id = TblId; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Insert', @Rows = @@rowcount; - INSERT INTO dbo.IndexProperties (TableName, IndexName, PropertyName, PropertyValue) - SELECT Tbl, - Ind, - 'DATA_COMPRESSION', - data_comp - FROM (SELECT Tbl, - Ind, - isnull((SELECT TOP 1 CASE WHEN data_compression_desc = 'PAGE' THEN 'PAGE' END - FROM sys.partitions - WHERE object_id = TblId - AND index_id = IndId), 'NONE') AS data_comp - FROM @Indexes) AS A - WHERE NOT EXISTS (SELECT * - FROM dbo.IndexProperties - WHERE TableName = Tbl - AND IndexName = Ind); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = 'IndexProperties', @Action = 'Insert', @Rows = @@rowcount; - DELETE @Indexes - WHERE Tbl = 'Resource' - OR IndId = 1; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Delete', @Rows = @@rowcount; - WHILE EXISTS (SELECT * - FROM @Indexes) - BEGIN - SELECT TOP 1 @Tbl = Tbl, - @Ind = Ind - FROM @Indexes; - SET @Txt = 'ALTER INDEX ' + @Ind + ' ON dbo.' + @Tbl + ' DISABLE'; - EXECUTE (@Txt); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Ind, @Action = 'Disable', @Text = @Txt; - DELETE @Indexes - WHERE Tbl = @Tbl - AND Ind = @Ind; - END - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.EnqueueJobs -@QueueType TINYINT, @Definitions StringList READONLY, @GroupId BIGINT=NULL, @ForceOneActiveJobGroup BIT=1, @IsCompleted BIT=NULL, @ReturnJobs BIT=1 -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'EnqueueJobs', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' D=' + CONVERT (VARCHAR, (SELECT count(*) - FROM @Definitions)) + ' G=' + isnull(CONVERT (VARCHAR, @GroupId), 'NULL') + ' F=' + isnull(CONVERT (VARCHAR, @ForceOneActiveJobGroup), 'NULL') + ' C=' + isnull(CONVERT (VARCHAR, @IsCompleted), 'NULL'), @st AS DATETIME = getUTCdate(), @Lock AS VARCHAR (100) = 'EnqueueJobs_' + CONVERT (VARCHAR, @QueueType), @MaxJobId AS BIGINT, @Rows AS INT, @msg AS VARCHAR (1000), @JobIds AS BigintList, @InputRows AS INT; -BEGIN TRY - DECLARE @Input TABLE ( - DefinitionHash VARBINARY (20) PRIMARY KEY, - Definition VARCHAR (MAX) ); - INSERT INTO @Input - SELECT hashbytes('SHA1', String) AS DefinitionHash, - String AS Definition - FROM @Definitions; - SET @InputRows = @@rowcount; - INSERT INTO @JobIds - SELECT JobId - FROM @Input AS A - INNER JOIN - dbo.JobQueue AS B - ON B.QueueType = @QueueType - AND B.DefinitionHash = A.DefinitionHash - AND B.Status <> 5; - IF @@rowcount < @InputRows - BEGIN - BEGIN TRANSACTION; - EXECUTE sp_getapplock @Lock, 'Exclusive'; - IF @ForceOneActiveJobGroup = 1 - AND EXISTS (SELECT * - FROM dbo.JobQueue - WHERE QueueType = @QueueType - AND Status IN (0, 1) - AND (@GroupId IS NULL - OR GroupId <> @GroupId)) - RAISERROR ('There are other active job groups', 18, 127); - SET @MaxJobId = isnull((SELECT TOP 1 JobId - FROM dbo.JobQueue - WHERE QueueType = @QueueType - ORDER BY JobId DESC), 0); - INSERT INTO dbo.JobQueue (QueueType, GroupId, JobId, Definition, DefinitionHash, Status) - OUTPUT inserted.JobId INTO @JobIds - SELECT @QueueType, - isnull(@GroupId, @MaxJobId + 1) AS GroupId, - JobId, - Definition, - DefinitionHash, - CASE WHEN @IsCompleted = 1 THEN 2 ELSE 0 END AS Status - FROM (SELECT @MaxJobId + row_number() OVER (ORDER BY Dummy) AS JobId, - * - FROM (SELECT *, - 0 AS Dummy - FROM @Input) AS A) AS A - WHERE NOT EXISTS (SELECT * - FROM dbo.JobQueue AS B - WHERE B.QueueType = @QueueType - AND B.DefinitionHash = A.DefinitionHash - AND B.Status <> 5); - SET @Rows = @@rowcount; - COMMIT TRANSACTION; - END - IF @ReturnJobs = 1 - EXECUTE dbo.GetJobs @QueueType = @QueueType, @JobIds = @JobIds; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; -END TRY -BEGIN CATCH - IF @@trancount > 0 - ROLLBACK; - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.ExecuteCommandForRebuildIndexes -@Tbl VARCHAR (100), @Ind VARCHAR (1000), @Cmd VARCHAR (MAX) -WITH EXECUTE AS 'dbo' -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'ExecuteCommandForRebuildIndexes', @Mode AS VARCHAR (200) = 'Tbl=' + isnull(@Tbl, 'NULL'), @st AS DATETIME, @Retries AS INT = 0, @Action AS VARCHAR (100), @msg AS VARCHAR (1000); -RetryOnTempdbError: -BEGIN TRY - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start', @Text = @Cmd; - SET @st = getUTCdate(); - IF @Tbl IS NULL - RAISERROR ('@Tbl IS NULL', 18, 127); - IF @Cmd IS NULL - RAISERROR ('@Cmd IS NULL', 18, 127); - SET @Action = CASE WHEN @Cmd LIKE 'UPDATE STAT%' THEN 'Update statistics' WHEN @Cmd LIKE 'CREATE%INDEX%' THEN 'Create Index' WHEN @Cmd LIKE 'ALTER%INDEX%REBUILD%' THEN 'Rebuild Index' WHEN @Cmd LIKE 'ALTER%TABLE%ADD%' THEN 'Add Constraint' END; - IF @Action IS NULL - BEGIN - SET @msg = 'Not supported command = ' + CONVERT (VARCHAR (900), @Cmd); - RAISERROR (@msg, 18, 127); - END - IF @Action = 'Create Index' - WAITFOR DELAY '00:00:05'; - EXECUTE (@Cmd); - SELECT @Ind; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Action = @Action, @Status = 'End', @Start = @st, @Text = @Cmd; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - IF error_number() = 40544 - BEGIN - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st, @Retry = @Retries; - SET @Retries = @Retries + 1; - IF @Tbl = 'TokenText_96' - WAITFOR DELAY '01:00:00'; - ELSE - WAITFOR DELAY '00:10:00'; - GOTO RetryOnTempdbError; - END - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; - THROW; -END CATCH - -GO -CREATE OR ALTER PROCEDURE dbo.FetchEventAgentCheckpoint -@CheckpointId VARCHAR (64) -AS -BEGIN - SELECT TOP (1) CheckpointId, - LastProcessedDateTime, - LastProcessedIdentifier - FROM dbo.EventAgentCheckpoint - WHERE CheckpointId = @CheckpointId; -END - -GO -CREATE PROCEDURE dbo.FetchResourceChanges_3 -@startId BIGINT, @lastProcessedUtcDateTime DATETIME2 (7), @pageSize SMALLINT -AS -BEGIN - SET NOCOUNT ON; - DECLARE @precedingPartitionBoundary AS DATETIME2 (7) = (SELECT TOP (1) CAST (prv.value AS DATETIME2 (7)) AS value - FROM sys.partition_range_values AS prv WITH (NOLOCK) - INNER JOIN - sys.partition_functions AS pf WITH (NOLOCK) - ON pf.function_id = prv.function_id - WHERE pf.name = N'PartitionFunction_ResourceChangeData_Timestamp' - AND SQL_VARIANT_PROPERTY(prv.Value, 'BaseType') = 'datetime2' - AND CAST (prv.value AS DATETIME2 (7)) < DATEADD(HOUR, DATEDIFF(HOUR, 0, @lastProcessedUtcDateTime), 0) - ORDER BY prv.boundary_id DESC); - IF (@precedingPartitionBoundary IS NULL) - BEGIN - SET @precedingPartitionBoundary = CONVERT (DATETIME2 (7), N'1970-01-01T00:00:00.0000000'); - END - DECLARE @endDateTimeToFilter AS DATETIME2 (7) = DATEADD(HOUR, 1, SYSUTCDATETIME()); - WITH PartitionBoundaries - AS (SELECT CAST (prv.value AS DATETIME2 (7)) AS PartitionBoundary - FROM sys.partition_range_values AS prv WITH (NOLOCK) - INNER JOIN - sys.partition_functions AS pf WITH (NOLOCK) - ON pf.function_id = prv.function_id - WHERE pf.name = N'PartitionFunction_ResourceChangeData_Timestamp' - AND SQL_VARIANT_PROPERTY(prv.Value, 'BaseType') = 'datetime2' - AND CAST (prv.value AS DATETIME2 (7)) BETWEEN @precedingPartitionBoundary AND @endDateTimeToFilter) - SELECT TOP (@pageSize) Id, - Timestamp, - ResourceId, - ResourceTypeId, - ResourceVersion, - ResourceChangeTypeId - FROM PartitionBoundaries AS p CROSS APPLY (SELECT TOP (@pageSize) Id, - Timestamp, - ResourceId, - ResourceTypeId, - ResourceVersion, - ResourceChangeTypeId - FROM dbo.ResourceChangeData WITH (TABLOCK, HOLDLOCK) - WHERE Id >= @startId - AND $PARTITION.PartitionFunction_ResourceChangeData_Timestamp (Timestamp) = $PARTITION.PartitionFunction_ResourceChangeData_Timestamp (p.PartitionBoundary) - ORDER BY Id ASC) AS rcd - ORDER BY rcd.Id ASC; -END - -GO -CREATE PROCEDURE dbo.GetActiveJobs -@QueueType TINYINT, @GroupId BIGINT=NULL -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'GetActiveJobs', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' G=' + isnull(CONVERT (VARCHAR, @GroupId), 'NULL'), @st AS DATETIME = getUTCdate(), @JobIds AS BigintList, @PartitionId AS TINYINT, @MaxPartitions AS TINYINT = 16, @LookedAtPartitions AS TINYINT = 0, @Rows AS INT = 0; -BEGIN TRY - SET @PartitionId = @MaxPartitions * rand(); - WHILE @LookedAtPartitions <= @MaxPartitions - BEGIN - IF @GroupId IS NULL - INSERT INTO @JobIds - SELECT JobId - FROM dbo.JobQueue - WHERE PartitionId = @PartitionId - AND QueueType = @QueueType - AND Status IN (0, 1); - ELSE - INSERT INTO @JobIds - SELECT JobId - FROM dbo.JobQueue - WHERE PartitionId = @PartitionId - AND QueueType = @QueueType - AND GroupId = @GroupId - AND Status IN (0, 1); - SET @Rows += @@rowcount; - SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; - SET @LookedAtPartitions += 1; - END - IF @Rows > 0 - EXECUTE dbo.GetJobs @QueueType = @QueueType, @JobIds = @JobIds; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.GetCommandsForRebuildIndexes -@RebuildClustered BIT -WITH EXECUTE AS 'dbo' -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'GetCommandsForRebuildIndexes', @Mode AS VARCHAR (200) = 'PS=PartitionScheme_ResourceTypeId RC=' + isnull(CONVERT (VARCHAR, @RebuildClustered), 'NULL'), @st AS DATETIME = getUTCdate(), @Tbl AS VARCHAR (100), @TblInt AS VARCHAR (100), @Ind AS VARCHAR (200), @IndId AS INT, @Supported AS BIT, @Txt AS VARCHAR (MAX), @Rows AS BIGINT, @Pages AS BIGINT, @ResourceTypeId AS SMALLINT, @IndexesCnt AS INT, @DataComp AS VARCHAR (100); -BEGIN TRY - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; - DECLARE @Commands TABLE ( - Tbl VARCHAR (100), - Ind VARCHAR (200), - Txt VARCHAR (MAX), - Pages BIGINT ); - DECLARE @ResourceTypes TABLE ( - ResourceTypeId SMALLINT PRIMARY KEY); - DECLARE @Indexes TABLE ( - Ind VARCHAR (200) PRIMARY KEY, - IndId INT ); - DECLARE @Tables TABLE ( - name VARCHAR (100) PRIMARY KEY, - Supported BIT ); - INSERT INTO @Tables - EXECUTE dbo.GetPartitionedTables @IncludeNotDisabled = 1, @IncludeNotSupported = 1; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Tables', @Action = 'Insert', @Rows = @@rowcount; - WHILE EXISTS (SELECT * - FROM @Tables) - BEGIN - SELECT TOP 1 @Tbl = name, - @Supported = Supported - FROM @Tables - ORDER BY name; - IF @Supported = 0 - BEGIN - INSERT INTO @Commands - SELECT @Tbl, - name, - 'ALTER INDEX ' + name + ' ON dbo.' + @Tbl + ' REBUILD' + CASE WHEN (SELECT PropertyValue - FROM dbo.IndexProperties - WHERE TableName = @Tbl - AND IndexName = name) = 'PAGE' THEN ' PARTITION = ALL WITH (DATA_COMPRESSION = PAGE)' ELSE '' END, - CONVERT (BIGINT, 9e18) - FROM sys.indexes - WHERE object_id = object_id(@Tbl) - AND (is_disabled = 1 - AND index_id > 1 - AND @RebuildClustered = 0 - OR index_id = 1 - AND @RebuildClustered = 1); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Commands', @Action = 'Insert', @Rows = @@rowcount, @Text = 'Not supported tables with disabled indexes'; - END - ELSE - BEGIN - DELETE @ResourceTypes; - INSERT INTO @ResourceTypes - SELECT CONVERT (SMALLINT, substring(name, charindex('_', name) + 1, 6)) AS ResourceTypeId - FROM sys.sysobjects - WHERE name LIKE @Tbl + '[_]%'; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@ResourceTypes', @Action = 'Insert', @Rows = @@rowcount; - WHILE EXISTS (SELECT * - FROM @ResourceTypes) - BEGIN - SET @ResourceTypeId = (SELECT TOP 1 ResourceTypeId - FROM @ResourceTypes - ORDER BY ResourceTypeId); - SET @TblInt = @Tbl + '_' + CONVERT (VARCHAR, @ResourceTypeId); - SET @Pages = (SELECT dpages - FROM sysindexes - WHERE id = object_id(@TblInt) - AND indid IN (0, 1)); - DELETE @Indexes; - INSERT INTO @Indexes - SELECT name, - index_id - FROM sys.indexes - WHERE object_id = object_id(@Tbl) - AND (index_id > 1 - AND @RebuildClustered = 0 - OR index_id = 1 - AND @RebuildClustered = 1); - SET @IndexesCnt = 0; - WHILE EXISTS (SELECT * - FROM @Indexes) - BEGIN - SELECT TOP 1 @Ind = Ind, - @IndId = IndId - FROM @Indexes - ORDER BY Ind; - IF @IndId = 1 - BEGIN - SET @Txt = 'ALTER INDEX ' + @Ind + ' ON dbo.' + @TblInt + ' REBUILD' + CASE WHEN (SELECT PropertyValue - FROM dbo.IndexProperties - WHERE TableName = @Tbl - AND IndexName = @Ind) = 'PAGE' THEN ' PARTITION = ALL WITH (DATA_COMPRESSION = PAGE)' ELSE '' END; - INSERT INTO @Commands - SELECT @TblInt, - @Ind, - @Txt, - @Pages; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Add command', @Rows = @@rowcount, @Text = @Txt; - END - ELSE - IF NOT EXISTS (SELECT * - FROM sys.indexes - WHERE object_id = object_id(@TblInt) - AND name = @Ind) - BEGIN - EXECUTE dbo.GetIndexCommands @Tbl = @Tbl, @Ind = @Ind, @AddPartClause = 0, @IncludeClustered = 0, @Txt = @Txt OUTPUT; - SET @Txt = replace(@Txt, '[' + @Tbl + ']', @TblInt); - IF @Txt IS NOT NULL - BEGIN - SET @IndexesCnt = @IndexesCnt + 1; - INSERT INTO @Commands - SELECT @TblInt, - @Ind, - @Txt, - @Pages; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Add command', @Rows = @@rowcount, @Text = @Txt; - END - END - DELETE @Indexes - WHERE Ind = @Ind; - END - IF @IndexesCnt > 1 - BEGIN - INSERT INTO @Commands - SELECT @TblInt, - 'UPDATE STAT', - 'UPDATE STATISTICS dbo.' + @TblInt, - @Pages; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Add command', @Rows = @@rowcount, @Text = 'Add stats update'; - END - DELETE @ResourceTypes - WHERE ResourceTypeId = @ResourceTypeId; - END - END - DELETE @Tables - WHERE name = @Tbl; - END - SELECT Tbl, - Ind, - Txt - FROM @Commands - ORDER BY Pages DESC, Tbl, CASE WHEN Txt LIKE 'UPDATE STAT%' THEN 0 ELSE 1 END; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Commands', @Action = 'Select', @Rows = @@rowcount; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.GetExportJobByHash -@hash VARCHAR (64) -AS -SET NOCOUNT ON; -SELECT TOP (1) RawJobRecord, - JobVersion -FROM dbo.ExportJob -WHERE Hash = @hash - AND (Status = 'Queued' - OR Status = 'Running') -ORDER BY HeartbeatDateTime ASC; - -GO -CREATE PROCEDURE dbo.GetExportJobById -@id VARCHAR (64) -AS -SET NOCOUNT ON; -SELECT RawJobRecord, - JobVersion -FROM dbo.ExportJob -WHERE Id = @id; - -GO -CREATE PROCEDURE [dbo].[GetImportProcessingTaskResult] -@queueId VARCHAR (64), @importTaskId VARCHAR (64) -AS -SET NOCOUNT ON; -SELECT Result -FROM [dbo].[TaskInfo] WITH (INDEX (IX_QueueId_ParentTaskId)) -WHERE ParentTaskId = @importTaskId - AND TaskTypeId = 1 - AND Status = 3; - -GO -CREATE PROCEDURE dbo.GetIndexCommands -@Tbl VARCHAR (100), @Ind VARCHAR (200), @AddPartClause BIT, @IncludeClustered BIT, @Txt VARCHAR (MAX)=NULL OUTPUT -WITH EXECUTE AS 'dbo' -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'GetIndexCommands', @Mode AS VARCHAR (200) = 'Tbl=' + isnull(@Tbl, 'NULL') + ' Ind=' + isnull(@Ind, 'NULL'), @st AS DATETIME = getUTCdate(); -DECLARE @Indexes TABLE ( - Ind VARCHAR (200) PRIMARY KEY, - Txt VARCHAR (MAX)); -BEGIN TRY - IF @Tbl IS NULL - RAISERROR ('@Tbl IS NULL', 18, 127); - INSERT INTO @Indexes - SELECT Ind, - CASE WHEN is_primary_key = 1 THEN 'ALTER TABLE dbo.[' + Tbl + '] ADD PRIMARY KEY ' + CASE WHEN type = 1 THEN ' CLUSTERED' ELSE '' END ELSE 'CREATE' + CASE WHEN is_unique = 1 THEN ' UNIQUE' ELSE '' END + CASE WHEN type = 1 THEN ' CLUSTERED' ELSE '' END + ' INDEX ' + Ind + ' ON dbo.[' + Tbl + ']' END + ' (' + KeyCols + ')' + IncClause + CASE WHEN filter_def IS NOT NULL THEN ' WHERE ' + filter_def ELSE '' END + CASE WHEN data_comp IS NOT NULL THEN ' WITH (DATA_COMPRESSION = ' + data_comp + ')' ELSE '' END + CASE WHEN @AddPartClause = 1 THEN PartClause ELSE '' END - FROM (SELECT O.Name AS Tbl, - I.Name AS Ind, - isnull((SELECT TOP 1 CASE WHEN data_compression_desc = 'PAGE' THEN 'PAGE' END - FROM sys.partitions AS P - WHERE P.object_id = I.object_id - AND I.index_id = P.index_id), (SELECT NULLIF (PropertyValue, 'NONE') - FROM dbo.IndexProperties - WHERE TableName = O.Name - AND IndexName = I.Name - AND PropertyName = 'DATA_COMPRESSION')) AS data_comp, - replace(replace(replace(replace(I.filter_definition, '[', ''), ']', ''), '(', ''), ')', '') AS filter_def, - I.is_unique, - I.is_primary_key, - I.type, - KeyCols, - CASE WHEN IncCols IS NOT NULL THEN ' INCLUDE (' + IncCols + ')' ELSE '' END AS IncClause, - CASE WHEN EXISTS (SELECT * - FROM sys.partition_schemes AS S - WHERE S.data_space_id = I.data_space_id - AND name = 'PartitionScheme_ResourceTypeId') THEN ' ON PartitionScheme_ResourceTypeId (ResourceTypeId)' ELSE '' END AS PartClause - FROM sys.indexes AS I - INNER JOIN - sys.objects AS O - ON O.object_id = I.object_id CROSS APPLY (SELECT string_agg(CASE WHEN IC.key_ordinal > 0 - AND IC.is_included_column = 0 THEN C.name END, ',') WITHIN GROUP (ORDER BY key_ordinal) AS KeyCols, - string_agg(CASE WHEN IC.is_included_column = 1 THEN C.name END, ',') WITHIN GROUP (ORDER BY key_ordinal) AS IncCols - FROM sys.index_columns AS IC - INNER JOIN - sys.columns AS C - ON C.object_id = IC.object_id - AND C.column_id = IC.column_id - WHERE IC.object_id = I.object_id - AND IC.index_id = I.index_id - GROUP BY IC.object_id, IC.index_id) AS IC - WHERE O.name = @Tbl - AND (@Ind IS NULL - OR I.name = @Ind) - AND (@IncludeClustered = 1 - OR index_id > 1)) AS A; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Insert', @Rows = @@rowcount; - IF @Ind IS NULL - SELECT Ind, - Txt - FROM @Indexes; - ELSE - SET @Txt = (SELECT Txt - FROM @Indexes); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Text = @Txt; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.GetJobs -@QueueType TINYINT, @JobId BIGINT=NULL, @JobIds BigintList READONLY, @GroupId BIGINT=NULL, @ReturnDefinition BIT=1 -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'GetJobs', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' J=' + isnull(CONVERT (VARCHAR, @JobId), 'NULL') + ' G=' + isnull(CONVERT (VARCHAR, @GroupId), 'NULL'), @st AS DATETIME = getUTCdate(), @PartitionId AS TINYINT = @JobId % 16; -BEGIN TRY - IF @JobId IS NULL - AND @GroupId IS NULL - AND NOT EXISTS (SELECT * - FROM @JobIds) - RAISERROR ('@JobId = NULL and @GroupId = NULL and @JobIds is empty', 18, 127); - IF @JobId IS NOT NULL - SELECT GroupId, - JobId, - CASE WHEN @ReturnDefinition = 1 THEN Definition ELSE NULL END AS Definition, - Version, - Status, - Priority, - Data, - Result, - CreateDate, - StartDate, - EndDate, - HeartbeatDate, - CancelRequested - FROM dbo.JobQueue - WHERE QueueType = @QueueType - AND PartitionId = @PartitionId - AND JobId = isnull(@JobId, -1) - AND Status <> 5; - ELSE - IF @GroupId IS NOT NULL - SELECT GroupId, - JobId, - CASE WHEN @ReturnDefinition = 1 THEN Definition ELSE NULL END AS Definition, - Version, - Status, - Priority, - Data, - Result, - CreateDate, - StartDate, - EndDate, - HeartbeatDate, - CancelRequested - FROM dbo.JobQueue WITH (INDEX (IX_QueueType_GroupId)) - WHERE QueueType = @QueueType - AND GroupId = isnull(@GroupId, -1) - AND Status <> 5; - ELSE - SELECT GroupId, - JobId, - CASE WHEN @ReturnDefinition = 1 THEN Definition ELSE NULL END AS Definition, - Version, - Status, - Priority, - Data, - Result, - CreateDate, - StartDate, - EndDate, - HeartbeatDate, - CancelRequested - FROM dbo.JobQueue - WHERE QueueType = @QueueType - AND JobId IN (SELECT Id - FROM @JobIds) - AND PartitionId = JobId % 16 - AND Status <> 5; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.GetNextTask_3 -@queueId VARCHAR (64), @taskHeartbeatTimeoutThresholdInSeconds INT=600 -AS -SET NOCOUNT ON; -DECLARE @lock AS VARCHAR (200) = 'GetNextTask_Q=' + @queueId, @taskId AS VARCHAR (64) = NULL, @expirationDateTime AS DATETIME2 (7), @startDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); -SET @expirationDateTime = DATEADD(second, -@taskHeartbeatTimeoutThresholdInSeconds, @startDateTime); -BEGIN TRY - BEGIN TRANSACTION; - EXECUTE sp_getapplock @lock, 'Exclusive'; - UPDATE T - SET Status = 2, - StartDateTime = @startDateTime, - HeartbeatDateTime = @startDateTime, - Worker = host_name(), - RunId = NEWID(), - @taskId = T.TaskId - FROM dbo.TaskInfo AS T WITH (PAGLOCK) - INNER JOIN - (SELECT TOP 1 TaskId - FROM dbo.TaskInfo WITH (INDEX (IX_QueueId_Status)) - WHERE QueueId = @queueId - AND Status = 1 - ORDER BY TaskId) AS S - ON T.QueueId = @queueId - AND T.TaskId = S.TaskId; - IF @taskId IS NULL - UPDATE T - SET StartDateTime = @startDateTime, - HeartbeatDateTime = @startDateTime, - Worker = host_name(), - RunId = NEWID(), - @taskId = T.TaskId, - RestartInfo = ISNULL(RestartInfo, '') + ' Prev: Worker=' + Worker + ' Start=' + CONVERT (VARCHAR, @startDateTime, 121) - FROM dbo.TaskInfo AS T WITH (PAGLOCK) - INNER JOIN - (SELECT TOP 1 TaskId - FROM dbo.TaskInfo WITH (INDEX (IX_QueueId_Status)) - WHERE QueueId = @queueId - AND Status = 2 - AND HeartbeatDateTime <= @expirationDateTime - ORDER BY TaskId) AS S - ON T.QueueId = @queueId - AND T.TaskId = S.TaskId; - COMMIT TRANSACTION; - EXECUTE dbo.GetTaskDetails @TaskId = @taskId; -END TRY -BEGIN CATCH - IF @@trancount > 0 - ROLLBACK TRANSACTION THROW; -END CATCH - -GO -CREATE OR ALTER PROCEDURE dbo.GetNonCompletedJobCountOfSpecificQueueType -@queueType TINYINT -AS -BEGIN - SET NOCOUNT ON; - SELECT COUNT(*) - FROM dbo.JobQueue - WHERE QueueType = @queueType - AND (Status = 0 - OR Status = 1); -END - -GO -CREATE PROCEDURE dbo.GetPartitionedTables -@IncludeNotDisabled BIT, @IncludeNotSupported BIT -WITH EXECUTE AS 'dbo' -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'GetPartitionedTables', @Mode AS VARCHAR (200) = 'PS=PartitionScheme_ResourceTypeId D=' + isnull(CONVERT (VARCHAR, @IncludeNotDisabled), 'NULL') + ' S=' + isnull(CONVERT (VARCHAR, @IncludeNotSupported), 'NULL'), @st AS DATETIME = getUTCdate(); -DECLARE @NotSupportedTables TABLE ( - id INT PRIMARY KEY); -BEGIN TRY - INSERT INTO @NotSupportedTables - SELECT DISTINCT O.object_id - FROM sys.indexes AS I - INNER JOIN - sys.objects AS O - ON O.object_id = I.object_id - WHERE O.type = 'u' - AND EXISTS (SELECT * - FROM sys.partition_schemes AS PS - WHERE PS.data_space_id = I.data_space_id - AND name = 'PartitionScheme_ResourceTypeId') - AND (NOT EXISTS (SELECT * - FROM sys.index_columns AS IC - INNER JOIN - sys.columns AS C - ON C.object_id = IC.object_id - AND C.column_id = IC.column_id - WHERE IC.object_id = I.object_id - AND IC.index_id = I.index_id - AND IC.key_ordinal > 0 - AND IC.is_included_column = 0 - AND C.name = 'ResourceTypeId') - OR EXISTS (SELECT * - FROM sys.indexes AS NSI - WHERE NSI.object_id = O.object_id - AND NOT EXISTS (SELECT * - FROM sys.partition_schemes AS PS - WHERE PS.data_space_id = NSI.data_space_id - AND name = 'PartitionScheme_ResourceTypeId'))); - SELECT CONVERT (VARCHAR (100), O.name), - CONVERT (BIT, CASE WHEN EXISTS (SELECT * - FROM @NotSupportedTables AS NSI - WHERE NSI.id = O.object_id) THEN 0 ELSE 1 END) - FROM sys.indexes AS I - INNER JOIN - sys.objects AS O - ON O.object_id = I.object_id - WHERE O.type = 'u' - AND I.index_id IN (0, 1) - AND EXISTS (SELECT * - FROM sys.partition_schemes AS PS - WHERE PS.data_space_id = I.data_space_id - AND name = 'PartitionScheme_ResourceTypeId') - AND EXISTS (SELECT * - FROM sys.index_columns AS IC - INNER JOIN - sys.columns AS C - ON C.object_id = I.object_id - AND C.column_id = IC.column_id - AND IC.is_included_column = 0 - AND C.name = 'ResourceTypeId') - AND (@IncludeNotSupported = 1 - OR NOT EXISTS (SELECT * - FROM @NotSupportedTables AS NSI - WHERE NSI.id = O.object_id)) - AND (@IncludeNotDisabled = 1 - OR EXISTS (SELECT * - FROM sys.indexes AS D - WHERE D.object_id = O.object_id - AND D.is_disabled = 1)) - ORDER BY 1; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.GetReindexJobById -@id VARCHAR (64) -AS -SET NOCOUNT ON; -SELECT RawJobRecord, - JobVersion -FROM dbo.ReindexJob -WHERE Id = @id; - -GO -CREATE PROCEDURE dbo.GetResources -@ResourceKeys dbo.ResourceKeyList READONLY -AS -SET NOCOUNT ON; -DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = 'GetResources', @InputRows AS INT, @DummyTop AS BIGINT = 9223372036854775807, @NotNullVersionExists AS BIT, @NullVersionExists AS BIT, @MinRT AS SMALLINT, @MaxRT AS SMALLINT; -SELECT @MinRT = min(ResourceTypeId), - @MaxRT = max(ResourceTypeId), - @InputRows = count(*), - @NotNullVersionExists = max(CASE WHEN Version IS NOT NULL THEN 1 ELSE 0 END), - @NullVersionExists = max(CASE WHEN Version IS NULL THEN 1 ELSE 0 END) -FROM @ResourceKeys; -DECLARE @Mode AS VARCHAR (100) = 'RT=[' + CONVERT (VARCHAR, @MinRT) + ',' + CONVERT (VARCHAR, @MaxRT) + '] Cnt=' + CONVERT (VARCHAR, @InputRows) + ' NNVE=' + CONVERT (VARCHAR, @NotNullVersionExists) + ' NVE=' + CONVERT (VARCHAR, @NullVersionExists); -BEGIN TRY - IF @NotNullVersionExists = 1 - IF @NullVersionExists = 0 - SELECT B.ResourceTypeId, - B.ResourceId, - ResourceSurrogateId, - B.Version, - IsDeleted, - IsHistory, - RawResource, - IsRawResourceMetaSet, - SearchParamHash - FROM (SELECT TOP (@DummyTop) * - FROM @ResourceKeys) AS A - INNER JOIN - dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) - ON B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceId = A.ResourceId - AND B.Version = A.Version - OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); - ELSE - SELECT * - FROM (SELECT B.ResourceTypeId, - B.ResourceId, - ResourceSurrogateId, - B.Version, - IsDeleted, - IsHistory, - RawResource, - IsRawResourceMetaSet, - SearchParamHash - FROM (SELECT TOP (@DummyTop) * - FROM @ResourceKeys - WHERE Version IS NOT NULL) AS A - INNER JOIN - dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) - ON B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceId = A.ResourceId - AND B.Version = A.Version - UNION ALL - SELECT B.ResourceTypeId, - B.ResourceId, - ResourceSurrogateId, - B.Version, - IsDeleted, - IsHistory, - RawResource, - IsRawResourceMetaSet, - SearchParamHash - FROM (SELECT TOP (@DummyTop) * - FROM @ResourceKeys - WHERE Version IS NULL) AS A - INNER JOIN - dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId)) - ON B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceId = A.ResourceId - WHERE IsHistory = 0) AS A - OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); - ELSE - SELECT B.ResourceTypeId, - B.ResourceId, - ResourceSurrogateId, - B.Version, - IsDeleted, - IsHistory, - RawResource, - IsRawResourceMetaSet, - SearchParamHash - FROM (SELECT TOP (@DummyTop) * - FROM @ResourceKeys) AS A - INNER JOIN - dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId)) - ON B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceId = A.ResourceId - WHERE IsHistory = 0 - OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.GetResourcesByTransactionId -@TransactionId BIGINT, @IncludeHistory BIT=0, @ReturnResourceKeysOnly BIT=0 -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'T=' + CONVERT (VARCHAR, @TransactionId) + ' H=' + CONVERT (VARCHAR, @IncludeHistory), @st AS DATETIME = getUTCdate(), @DummyTop AS BIGINT = 9223372036854775807, @TypeId AS SMALLINT; -BEGIN TRY - DECLARE @Types TABLE ( - TypeId SMALLINT PRIMARY KEY, - Name VARCHAR (100)); - INSERT INTO @Types - EXECUTE dbo.GetUsedResourceTypes ; - DECLARE @Keys TABLE ( - TypeId SMALLINT, - SurrogateId BIGINT PRIMARY KEY (TypeId, SurrogateId)); - WHILE EXISTS (SELECT * - FROM @Types) - BEGIN - SET @TypeId = (SELECT TOP 1 TypeId - FROM @Types - ORDER BY TypeId); - INSERT INTO @Keys - SELECT @TypeId, - ResourceSurrogateId - FROM dbo.Resource - WHERE ResourceTypeId = @TypeId - AND TransactionId = @TransactionId; - DELETE @Types - WHERE TypeId = @TypeId; - END - IF @ReturnResourceKeysOnly = 0 - SELECT ResourceTypeId, - ResourceId, - ResourceSurrogateId, - Version, - IsDeleted, - IsHistory, - RawResource, - IsRawResourceMetaSet, - SearchParamHash, - RequestMethod - FROM (SELECT TOP (@DummyTop) * - FROM @Keys) AS A - INNER JOIN - dbo.Resource AS B - ON ResourceTypeId = TypeId - AND ResourceSurrogateId = SurrogateId - WHERE IsHistory = 0 - OR @IncludeHistory = 1 - OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); - ELSE - SELECT ResourceTypeId, - ResourceId, - ResourceSurrogateId, - Version, - IsDeleted - FROM (SELECT TOP (@DummyTop) * - FROM @Keys) AS A - INNER JOIN - dbo.Resource AS B - ON ResourceTypeId = TypeId - AND ResourceSurrogateId = SurrogateId - WHERE IsHistory = 0 - OR @IncludeHistory = 1 - OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.GetResourcesByTypeAndSurrogateIdRange -@ResourceTypeId SMALLINT, @StartId BIGINT, @EndId BIGINT, @GlobalStartId BIGINT=NULL, @GlobalEndId BIGINT=NULL, @IncludeHistory BIT=0, @IncludeDeleted BIT=0 -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'GetResourcesByTypeAndSurrogateIdRange', @Mode AS VARCHAR (100) = 'RT=' + isnull(CONVERT (VARCHAR, @ResourceTypeId), 'NULL') + ' S=' + isnull(CONVERT (VARCHAR, @StartId), 'NULL') + ' E=' + isnull(CONVERT (VARCHAR, @EndId), 'NULL') + ' GS=' + isnull(CONVERT (VARCHAR, @GlobalStartId), 'NULL') + ' GE=' + isnull(CONVERT (VARCHAR, @GlobalEndId), 'NULL') + ' HI=' + isnull(CONVERT (VARCHAR, @IncludeHistory), 'NULL') + ' DE' + isnull(CONVERT (VARCHAR, @IncludeDeleted), 'NULL'), @st AS DATETIME = getUTCdate(); -BEGIN TRY - DECLARE @ResourceIds TABLE ( - ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS, - ResourceSurrogateId BIGINT , - RowId INT , - PRIMARY KEY (ResourceId, RowId)); - IF @GlobalStartId IS NULL - SET @GlobalStartId = 0; - IF @GlobalEndId IS NOT NULL - INSERT INTO @ResourceIds - SELECT ResourceId, - ResourceSurrogateId, - row_number() OVER (PARTITION BY ResourceId ORDER BY ResourceSurrogateId DESC) AS RowId - FROM dbo.Resource - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceId IN (SELECT DISTINCT ResourceId - FROM dbo.Resource - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId BETWEEN @StartId AND @EndId - AND IsHistory = 1 - AND (IsDeleted = 0 - OR @IncludeDeleted = 1)) - AND ResourceSurrogateId BETWEEN @GlobalStartId AND @GlobalEndId; - IF EXISTS (SELECT * - FROM @ResourceIds) - BEGIN - DECLARE @SurrogateIdMap TABLE ( - MaxSurrogateId BIGINT PRIMARY KEY); - INSERT INTO @SurrogateIdMap - SELECT A.ResourceSurrogateId AS MaxSurrogateId - FROM (SELECT * - FROM @ResourceIds - WHERE RowId = 1 - AND ResourceSurrogateId BETWEEN @StartId AND @EndId) AS A; - SELECT @ResourceTypeId, - CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.ResourceId ELSE A.ResourceId END, - CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.Version ELSE A.Version END, - CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.IsDeleted ELSE A.IsDeleted END, - isnull(C.ResourceSurrogateId, A.ResourceSurrogateId), - CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.RequestMethod ELSE A.RequestMethod END, - CONVERT (BIT, 1) AS IsMatch, - CONVERT (BIT, 0) AS IsPartial, - CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.IsRawResourceMetaSet ELSE A.IsRawResourceMetaSet END, - CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.SearchParamHash ELSE A.SearchParamHash END, - CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.RawResource ELSE A.RawResource END - FROM dbo.Resource AS A - LEFT OUTER JOIN - @SurrogateIdMap AS B - ON B.MaxSurrogateId = A.ResourceSurrogateId - LEFT OUTER JOIN - dbo.Resource AS C - ON C.ResourceTypeId = @ResourceTypeId - AND C.ResourceSurrogateId = MaxSurrogateId - WHERE A.ResourceTypeId = @ResourceTypeId - AND A.ResourceSurrogateId BETWEEN @StartId AND @EndId - AND (A.IsHistory = 0 - OR MaxSurrogateId IS NOT NULL - OR @IncludeHistory = 1) - AND (A.IsDeleted = 0 - OR @IncludeDeleted = 1); - END - ELSE - SELECT ResourceTypeId, - ResourceId, - Version, - IsDeleted, - ResourceSurrogateId, - RequestMethod, - CONVERT (BIT, 1) AS IsMatch, - CONVERT (BIT, 0) AS IsPartial, - IsRawResourceMetaSet, - SearchParamHash, - RawResource - FROM dbo.Resource - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId BETWEEN @StartId AND @EndId - AND (IsHistory = 0 - OR @IncludeHistory = 1) - AND (IsDeleted = 0 - OR @IncludeDeleted = 1); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.GetResourceSurrogateIdRanges -@ResourceTypeId SMALLINT, @StartId BIGINT, @EndId BIGINT, @RangeSize INT, @NumberOfRanges INT=100, @Up BIT=1 -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'GetResourceSurrogateIdRanges', @Mode AS VARCHAR (100) = 'RT=' + isnull(CONVERT (VARCHAR, @ResourceTypeId), 'NULL') + ' S=' + isnull(CONVERT (VARCHAR, @StartId), 'NULL') + ' E=' + isnull(CONVERT (VARCHAR, @EndId), 'NULL') + ' R=' + isnull(CONVERT (VARCHAR, @RangeSize), 'NULL') + ' UP=' + isnull(CONVERT (VARCHAR, @Up), 'NULL'), @st AS DATETIME = getUTCdate(); -BEGIN TRY - IF @Up = 1 - SELECT RangeId, - min(ResourceSurrogateId), - max(ResourceSurrogateId), - count(*) - FROM (SELECT isnull(CONVERT (INT, (row_number() OVER (ORDER BY ResourceSurrogateId) - 1) / @RangeSize), 0) AS RangeId, - ResourceSurrogateId - FROM (SELECT TOP (@RangeSize * @NumberOfRanges) ResourceSurrogateId - FROM dbo.Resource - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId >= @StartId - AND ResourceSurrogateId <= @EndId - ORDER BY ResourceSurrogateId) AS A) AS A - GROUP BY RangeId - OPTION (MAXDOP 1); - ELSE - SELECT RangeId, - min(ResourceSurrogateId), - max(ResourceSurrogateId), - count(*) - FROM (SELECT isnull(CONVERT (INT, (row_number() OVER (ORDER BY ResourceSurrogateId) - 1) / @RangeSize), 0) AS RangeId, - ResourceSurrogateId - FROM (SELECT TOP (@RangeSize * @NumberOfRanges) ResourceSurrogateId - FROM dbo.Resource - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId >= @StartId - AND ResourceSurrogateId <= @EndId - ORDER BY ResourceSurrogateId DESC) AS A) AS A - GROUP BY RangeId - OPTION (MAXDOP 1); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.GetResourceVersions -@ResourceDateKeys dbo.ResourceDateKeyList READONLY -AS -SET NOCOUNT ON; -DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = 'GetResourceVersions', @Mode AS VARCHAR (100) = 'Rows=' + CONVERT (VARCHAR, (SELECT count(*) - FROM @ResourceDateKeys)), @DummyTop AS BIGINT = 9223372036854775807; -BEGIN TRY - SELECT A.ResourceTypeId, - A.ResourceId, - A.ResourceSurrogateId, - CASE WHEN EXISTS (SELECT * - FROM dbo.Resource AS B - WHERE B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceId = A.ResourceId - AND B.ResourceSurrogateId = A.ResourceSurrogateId) THEN 0 WHEN isnull(U.Version, 1) - isnull(L.Version, 0) > 1 THEN isnull(U.Version, 1) - 1 ELSE 0 END AS Version - FROM (SELECT TOP (@DummyTop) * - FROM @ResourceDateKeys) AS A OUTER APPLY (SELECT TOP 1 * - FROM dbo.Resource AS B - WHERE B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceId = A.ResourceId - AND B.ResourceSurrogateId < A.ResourceSurrogateId - ORDER BY B.ResourceSurrogateId DESC) AS L OUTER APPLY (SELECT TOP 1 * - FROM dbo.Resource AS B - WHERE B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceId = A.ResourceId - AND B.ResourceSurrogateId > A.ResourceSurrogateId - ORDER BY B.ResourceSurrogateId) AS U - OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.GetSearchParamStatuses -AS -SET NOCOUNT ON; -SELECT SearchParamId, - Uri, - Status, - LastUpdated, - IsPartiallySupported -FROM dbo.SearchParam; - -GO -CREATE PROCEDURE [dbo].[GetTaskDetails] -@taskId VARCHAR (64) -AS -SET NOCOUNT ON; -SELECT TaskId, - QueueId, - Status, - TaskTypeId, - RunId, - IsCanceled, - RetryCount, - MaxRetryCount, - HeartbeatDateTime, - InputData, - TaskContext, - Result, - ParentTaskId -FROM [dbo].[TaskInfo] -WHERE TaskId = @taskId; - -GO -CREATE PROCEDURE dbo.GetTransactions -@StartNotInclusiveTranId BIGINT, @EndInclusiveTranId BIGINT, @EndDate DATETIME=NULL -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'ST=' + CONVERT (VARCHAR, @StartNotInclusiveTranId) + ' ET=' + CONVERT (VARCHAR, @EndInclusiveTranId) + ' ED=' + isnull(CONVERT (VARCHAR, @EndDate, 121), 'NULL'), @st AS DATETIME = getUTCdate(); -IF @EndDate IS NULL - SET @EndDate = getUTCdate(); -SELECT SurrogateIdRangeFirstValue, - VisibleDate, - InvisibleHistoryRemovedDate -FROM dbo.Transactions -WHERE SurrogateIdRangeFirstValue > @StartNotInclusiveTranId - AND SurrogateIdRangeFirstValue <= @EndInclusiveTranId - AND EndDate <= @EndDate -ORDER BY SurrogateIdRangeFirstValue; -EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; - -GO -CREATE PROCEDURE dbo.GetUsedResourceTypes -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'GetUsedResourceTypes', @Mode AS VARCHAR (100) = '', @st AS DATETIME = getUTCdate(); -BEGIN TRY - SELECT ResourceTypeId, - Name - FROM dbo.ResourceType AS A - WHERE EXISTS (SELECT * - FROM dbo.Resource AS B - WHERE B.ResourceTypeId = A.ResourceTypeId); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.HardDeleteResource -@ResourceTypeId SMALLINT, @ResourceId VARCHAR (64), @KeepCurrentVersion BIT, @IsResourceChangeCaptureEnabled BIT -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (200) = 'RT=' + CONVERT (VARCHAR, @ResourceTypeId) + ' R=' + @ResourceId + ' V=' + CONVERT (VARCHAR, @KeepCurrentVersion) + ' CC=' + CONVERT (VARCHAR, @IsResourceChangeCaptureEnabled), @st AS DATETIME = getUTCdate(), @TransactionId AS BIGINT; -BEGIN TRY - IF @IsResourceChangeCaptureEnabled = 1 - EXECUTE dbo.MergeResourcesBeginTransaction @Count = 1, @TransactionId = @TransactionId OUTPUT; - IF @KeepCurrentVersion = 0 - BEGIN TRANSACTION; - DECLARE @SurrogateIds TABLE ( - ResourceSurrogateId BIGINT NOT NULL); - IF @IsResourceChangeCaptureEnabled = 1 - AND EXISTS (SELECT * - FROM dbo.Parameters - WHERE Id = 'InvisibleHistory.IsEnabled' - AND Number = 1) - UPDATE dbo.Resource - SET IsHistory = 1, - RawResource = 0xF, - SearchParamHash = NULL, - HistoryTransactionId = @TransactionId - OUTPUT deleted.ResourceSurrogateId INTO @SurrogateIds - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceId = @ResourceId - AND (@KeepCurrentVersion = 0 - OR IsHistory = 1) - AND RawResource <> 0xF; - ELSE - DELETE dbo.Resource - OUTPUT deleted.ResourceSurrogateId INTO @SurrogateIds - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceId = @ResourceId - AND (@KeepCurrentVersion = 0 - OR IsHistory = 1) - AND RawResource <> 0xF; - IF @KeepCurrentVersion = 0 - BEGIN - DELETE B - FROM @SurrogateIds AS A - INNER LOOP JOIN - dbo.ResourceWriteClaim AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) - ON B.ResourceSurrogateId = A.ResourceSurrogateId - OPTION (MAXDOP 1); - DELETE B - FROM @SurrogateIds AS A - INNER LOOP JOIN - dbo.ReferenceSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) - ON B.ResourceTypeId = @ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId - OPTION (MAXDOP 1); - DELETE B - FROM @SurrogateIds AS A - INNER LOOP JOIN - dbo.TokenSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) - ON B.ResourceTypeId = @ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId - OPTION (MAXDOP 1); - DELETE B - FROM @SurrogateIds AS A - INNER LOOP JOIN - dbo.TokenText AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) - ON B.ResourceTypeId = @ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId - OPTION (MAXDOP 1); - DELETE B - FROM @SurrogateIds AS A - INNER LOOP JOIN - dbo.StringSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) - ON B.ResourceTypeId = @ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId - OPTION (MAXDOP 1); - DELETE B - FROM @SurrogateIds AS A - INNER LOOP JOIN - dbo.UriSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) - ON B.ResourceTypeId = @ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId - OPTION (MAXDOP 1); - DELETE B - FROM @SurrogateIds AS A - INNER LOOP JOIN - dbo.NumberSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) - ON B.ResourceTypeId = @ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId - OPTION (MAXDOP 1); - DELETE B - FROM @SurrogateIds AS A - INNER LOOP JOIN - dbo.QuantitySearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) - ON B.ResourceTypeId = @ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId - OPTION (MAXDOP 1); - DELETE B - FROM @SurrogateIds AS A - INNER LOOP JOIN - dbo.DateTimeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) - ON B.ResourceTypeId = @ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId - OPTION (MAXDOP 1); - DELETE B - FROM @SurrogateIds AS A - INNER LOOP JOIN - dbo.ReferenceTokenCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) - ON B.ResourceTypeId = @ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId - OPTION (MAXDOP 1); - DELETE B - FROM @SurrogateIds AS A - INNER LOOP JOIN - dbo.TokenTokenCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) - ON B.ResourceTypeId = @ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId - OPTION (MAXDOP 1); - DELETE B - FROM @SurrogateIds AS A - INNER LOOP JOIN - dbo.TokenDateTimeCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) - ON B.ResourceTypeId = @ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId - OPTION (MAXDOP 1); - DELETE B - FROM @SurrogateIds AS A - INNER LOOP JOIN - dbo.TokenQuantityCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) - ON B.ResourceTypeId = @ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId - OPTION (MAXDOP 1); - DELETE B - FROM @SurrogateIds AS A - INNER LOOP JOIN - dbo.TokenStringCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) - ON B.ResourceTypeId = @ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId - OPTION (MAXDOP 1); - DELETE B - FROM @SurrogateIds AS A - INNER LOOP JOIN - dbo.TokenNumberNumberCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) - ON B.ResourceTypeId = @ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId - OPTION (MAXDOP 1); - END - IF @@trancount > 0 - COMMIT TRANSACTION; - IF @IsResourceChangeCaptureEnabled = 1 - EXECUTE dbo.MergeResourcesCommitTransaction @TransactionId; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; -END TRY -BEGIN CATCH - IF @@trancount > 0 - ROLLBACK; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.InitDefrag -@QueueType TINYINT, @GroupId BIGINT, @DefragItems INT=NULL OUTPUT -WITH EXECUTE AS 'dbo' -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'InitDefrag', @st AS DATETIME = getUTCdate(), @ObjectId AS INT, @msg AS VARCHAR (1000), @Rows AS INT, @MinFragPct AS INT = isnull((SELECT Number - FROM dbo.Parameters - WHERE Id = 'Defrag.MinFragPct'), 10), @MinSizeGB AS FLOAT = isnull((SELECT Number - FROM dbo.Parameters - WHERE Id = 'Defrag.MinSizeGB'), 0.1), @DefinitionsSorted AS StringList; -DECLARE @Mode AS VARCHAR (200) = 'G=' + CONVERT (VARCHAR, @GroupId) + ' MF=' + CONVERT (VARCHAR, @MinFragPct) + ' MS=' + CONVERT (VARCHAR, @MinSizeGB); -DECLARE @Definitions AS TABLE ( - Def VARCHAR (900) PRIMARY KEY, - FragGB FLOAT ); -BEGIN TRY - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; - SELECT * - INTO #filter - FROM (SELECT object_id, - sum(reserved_page_count * 8.0 / 1024 / 1024) AS ReservedGB - FROM sys.dm_db_partition_stats AS A - WHERE object_id IN (SELECT object_id - FROM sys.objects - WHERE type = 'U' - AND name NOT IN ('EventLog')) - GROUP BY object_id) AS A - WHERE ReservedGB > @MinSizeGB; - WHILE EXISTS (SELECT * - FROM #filter) - BEGIN - SET @ObjectId = (SELECT TOP 1 object_id - FROM #filter - ORDER BY ReservedGB DESC); - INSERT INTO @Definitions - SELECT object_name(@ObjectId) + ';' + I.name + ';' + CONVERT (VARCHAR, partition_number) + ';' + CONVERT (VARCHAR, CASE WHEN EXISTS (SELECT * - FROM sys.partition_schemes AS PS - WHERE PS.data_space_id = I.data_space_id) THEN 1 ELSE 0 END) + ';' + CONVERT (VARCHAR, (SELECT sum(reserved_page_count) - FROM sys.dm_db_partition_stats AS S - WHERE S.object_id = A.object_id - AND S.index_id = A.index_id - AND S.partition_number = A.partition_number) * 8.0 / 1024 / 1024), - FragGB - FROM (SELECT object_id, - index_id, - partition_number, - A.avg_fragmentation_in_percent * A.page_count * 8.0 / 1024 / 1024 / 100 AS FragGB - FROM sys.dm_db_index_physical_stats(db_id(), @ObjectId, NULL, NULL, 'LIMITED') AS A - WHERE index_id > 0 - AND avg_fragmentation_in_percent >= @MinFragPct - AND A.page_count > 500) AS A - INNER JOIN - sys.indexes AS I - ON I.object_id = A.object_id - AND I.index_id = A.index_id; - SET @Rows = @@rowcount; - SET @msg = object_name(@ObjectId); - EXECUTE dbo.LogEvent @Process = @SP, @Status = 'Run', @Mode = @Mode, @Target = '@Definitions', @Action = 'Insert', @Rows = @Rows, @Text = @msg; - DELETE #filter - WHERE object_id = @ObjectId; - END - INSERT INTO @DefinitionsSorted - SELECT Def + ';' + CONVERT (VARCHAR, FragGB) - FROM @Definitions - ORDER BY FragGB DESC; - SET @DefragItems = @@rowcount; - IF @DefragItems > 0 - EXECUTE dbo.EnqueueJobs @QueueType = @QueueType, @Definitions = @DefinitionsSorted, @GroupId = @GroupId, @ForceOneActiveJobGroup = 1; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.InitializeIndexProperties -AS -SET NOCOUNT ON; -INSERT INTO dbo.IndexProperties (TableName, IndexName, PropertyName, PropertyValue) -SELECT Tbl, - Ind, - 'DATA_COMPRESSION', - isnull(data_comp, 'NONE') -FROM (SELECT O.Name AS Tbl, - I.Name AS Ind, - (SELECT TOP 1 CASE WHEN data_compression_desc = 'PAGE' THEN 'PAGE' END - FROM sys.partitions AS P - WHERE P.object_id = I.object_id - AND I.index_id = P.index_id) AS data_comp - FROM sys.indexes AS I - INNER JOIN - sys.objects AS O - ON O.object_id = I.object_id - WHERE O.type = 'u' - AND EXISTS (SELECT * - FROM sys.partition_schemes AS PS - WHERE PS.data_space_id = I.data_space_id - AND name = 'PartitionScheme_ResourceTypeId')) AS A -WHERE NOT EXISTS (SELECT * - FROM dbo.IndexProperties - WHERE TableName = Tbl - AND IndexName = Ind); - -GO -CREATE PROCEDURE dbo.LogEvent -@Process VARCHAR (100), @Status VARCHAR (10), @Mode VARCHAR (200)=NULL, @Action VARCHAR (20)=NULL, @Target VARCHAR (100)=NULL, @Rows BIGINT=NULL, @Start DATETIME=NULL, @Text NVARCHAR (3500)=NULL, @EventId BIGINT=NULL OUTPUT, @Retry INT=NULL -AS -SET NOCOUNT ON; -DECLARE @ErrorNumber AS INT = error_number(), @ErrorMessage AS VARCHAR (1000) = '', @TranCount AS INT = @@trancount, @DoWork AS BIT = 0, @NumberAdded AS BIT; -IF @ErrorNumber IS NOT NULL - OR @Status IN ('Warn', 'Error') - SET @DoWork = 1; -IF @DoWork = 0 - SET @DoWork = CASE WHEN EXISTS (SELECT * - FROM dbo.Parameters - WHERE Id = isnull(@Process, '') - AND Char = 'LogEvent') THEN 1 ELSE 0 END; -IF @DoWork = 0 - RETURN; -IF @ErrorNumber IS NOT NULL - SET @ErrorMessage = CASE WHEN @Retry IS NOT NULL THEN 'Retry ' + CONVERT (VARCHAR, @Retry) + ', ' ELSE '' END + 'Error ' + CONVERT (VARCHAR, error_number()) + ': ' + CONVERT (VARCHAR (1000), error_message()) + ', Level ' + CONVERT (VARCHAR, error_severity()) + ', State ' + CONVERT (VARCHAR, error_state()) + CASE WHEN error_procedure() IS NOT NULL THEN ', Procedure ' + error_procedure() ELSE '' END + ', Line ' + CONVERT (VARCHAR, error_line()); -IF @TranCount > 0 - AND @ErrorNumber IS NOT NULL - ROLLBACK; -IF databasepropertyex(db_name(), 'UpdateAbility') = 'READ_WRITE' - BEGIN - INSERT INTO dbo.EventLog (Process, Status, Mode, Action, Target, Rows, Milliseconds, EventDate, EventText, SPID, HostName) - SELECT @Process, - @Status, - @Mode, - @Action, - @Target, - @Rows, - datediff(millisecond, @Start, getUTCdate()), - getUTCdate() AS EventDate, - CASE WHEN @ErrorNumber IS NULL THEN @Text ELSE @ErrorMessage + CASE WHEN isnull(@Text, '') <> '' THEN '. ' + @Text ELSE '' END END AS Text, - @@SPID, - host_name() AS HostName; - SET @EventId = scope_identity(); - END -IF @TranCount > 0 - AND @ErrorNumber IS NOT NULL - BEGIN TRANSACTION; - -GO -CREATE PROCEDURE dbo.LogSchemaMigrationProgress -@message VARCHAR (MAX) -AS -INSERT INTO dbo.SchemaMigrationProgress (Message) -VALUES (@message); - -GO -CREATE PROCEDURE dbo.MergeResources -@AffectedRows INT=0 OUTPUT, @RaiseExceptionOnConflict BIT=1, @IsResourceChangeCaptureEnabled BIT=0, @TransactionId BIGINT=NULL, @SingleTransaction BIT=1, @Resources dbo.ResourceList READONLY, @ResourceWriteClaims dbo.ResourceWriteClaimList READONLY, @CompartmentAssignments dbo.CompartmentAssignmentList READONLY, @ReferenceSearchParams dbo.ReferenceSearchParamList READONLY, @TokenSearchParams dbo.TokenSearchParamList READONLY, @TokenTexts dbo.TokenTextList READONLY, @StringSearchParams dbo.StringSearchParamList READONLY, @UriSearchParams dbo.UriSearchParamList READONLY, @NumberSearchParams dbo.NumberSearchParamList READONLY, @QuantitySearchParams dbo.QuantitySearchParamList READONLY, @DateTimeSearchParms dbo.DateTimeSearchParamList READONLY, @ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY, @TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY, @TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY, @TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY, @TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY, @TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY -AS -SET NOCOUNT ON; -DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = object_name(@@procid), @DummyTop AS BIGINT = 9223372036854775807, @InitialTranCount AS INT = @@trancount, @IsRetry AS BIT = 0; -DECLARE @Mode AS VARCHAR (200) = isnull((SELECT 'RT=[' + CONVERT (VARCHAR, min(ResourceTypeId)) + ',' + CONVERT (VARCHAR, max(ResourceTypeId)) + '] Sur=[' + CONVERT (VARCHAR, min(ResourceSurrogateId)) + ',' + CONVERT (VARCHAR, max(ResourceSurrogateId)) + '] V=' + CONVERT (VARCHAR, max(Version)) + ' Rows=' + CONVERT (VARCHAR, count(*)) - FROM @Resources), 'Input=Empty'); -SET @Mode += ' E=' + CONVERT (VARCHAR, @RaiseExceptionOnConflict) + ' CC=' + CONVERT (VARCHAR, @IsResourceChangeCaptureEnabled) + ' IT=' + CONVERT (VARCHAR, @InitialTranCount) + ' T=' + isnull(CONVERT (VARCHAR, @TransactionId), 'NULL'); -SET @AffectedRows = 0; -BEGIN TRY - DECLARE @Existing AS TABLE ( - ResourceTypeId SMALLINT NOT NULL, - SurrogateId BIGINT NOT NULL PRIMARY KEY (ResourceTypeId, SurrogateId)); - DECLARE @ResourceInfos AS TABLE ( - ResourceTypeId SMALLINT NOT NULL, - SurrogateId BIGINT NOT NULL, - Version INT NOT NULL, - KeepHistory BIT NOT NULL, - PreviousVersion INT NULL, - PreviousSurrogateId BIGINT NULL PRIMARY KEY (ResourceTypeId, SurrogateId)); - DECLARE @PreviousSurrogateIds AS TABLE ( - TypeId SMALLINT NOT NULL, - SurrogateId BIGINT NOT NULL PRIMARY KEY (TypeId, SurrogateId), - KeepHistory BIT ); - IF @SingleTransaction = 0 - AND isnull((SELECT Number - FROM dbo.Parameters - WHERE Id = 'MergeResources.NoTransaction.IsEnabled'), 0) = 0 - SET @SingleTransaction = 1; - SET @Mode += ' ST=' + CONVERT (VARCHAR, @SingleTransaction); - IF @InitialTranCount = 0 - BEGIN - IF EXISTS (SELECT * - FROM @Resources AS A - INNER JOIN - dbo.Resource AS B - ON B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId - WHERE B.IsHistory = 0) - BEGIN - BEGIN TRANSACTION; - INSERT INTO @Existing (ResourceTypeId, SurrogateId) - SELECT B.ResourceTypeId, - B.ResourceSurrogateId - FROM (SELECT TOP (@DummyTop) * - FROM @Resources) AS A - INNER JOIN - dbo.Resource AS B WITH (ROWLOCK, HOLDLOCK) - ON B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId - WHERE B.IsHistory = 0 - OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); - IF @@rowcount > 0 - SET @IsRetry = 1; - IF @IsRetry = 0 - COMMIT TRANSACTION; - END - END - SET @Mode += ' R=' + CONVERT (VARCHAR, @IsRetry); - IF @SingleTransaction = 1 - AND @@trancount = 0 - BEGIN TRANSACTION; - IF @IsRetry = 0 - BEGIN - INSERT INTO @ResourceInfos (ResourceTypeId, SurrogateId, Version, KeepHistory, PreviousVersion, PreviousSurrogateId) - SELECT A.ResourceTypeId, - A.ResourceSurrogateId, - A.Version, - A.KeepHistory, - B.Version, - B.ResourceSurrogateId - FROM (SELECT TOP (@DummyTop) * - FROM @Resources - WHERE HasVersionToCompare = 1) AS A - LEFT OUTER JOIN - dbo.Resource AS B - ON B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceId = A.ResourceId - AND B.IsHistory = 0 - OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); - IF @RaiseExceptionOnConflict = 1 - AND EXISTS (SELECT * - FROM @ResourceInfos - WHERE PreviousVersion IS NOT NULL - AND Version <> PreviousVersion + 1) - THROW 50409, 'Resource has been recently updated or added, please compare the resource content in code for any duplicate updates', 1; - INSERT INTO @PreviousSurrogateIds - SELECT ResourceTypeId, - PreviousSurrogateId, - KeepHistory - FROM @ResourceInfos - WHERE PreviousSurrogateId IS NOT NULL; - IF @@rowcount > 0 - BEGIN - UPDATE dbo.Resource - SET IsHistory = 1 - WHERE EXISTS (SELECT * - FROM @PreviousSurrogateIds - WHERE TypeId = ResourceTypeId - AND SurrogateId = ResourceSurrogateId - AND KeepHistory = 1); - SET @AffectedRows += @@rowcount; - IF @IsResourceChangeCaptureEnabled = 1 - AND EXISTS (SELECT * - FROM dbo.Parameters - WHERE Id = 'InvisibleHistory.IsEnabled' - AND Number = 1) - UPDATE dbo.Resource - SET IsHistory = 1, - RawResource = 0xF, - SearchParamHash = NULL, - HistoryTransactionId = @TransactionId - WHERE EXISTS (SELECT * - FROM @PreviousSurrogateIds - WHERE TypeId = ResourceTypeId - AND SurrogateId = ResourceSurrogateId - AND KeepHistory = 0); - ELSE - DELETE dbo.Resource - WHERE EXISTS (SELECT * - FROM @PreviousSurrogateIds - WHERE TypeId = ResourceTypeId - AND SurrogateId = ResourceSurrogateId - AND KeepHistory = 0); - SET @AffectedRows += @@rowcount; - DELETE dbo.ResourceWriteClaim - WHERE EXISTS (SELECT * - FROM @PreviousSurrogateIds - WHERE SurrogateId = ResourceSurrogateId); - SET @AffectedRows += @@rowcount; - DELETE dbo.ReferenceSearchParam - WHERE EXISTS (SELECT * - FROM @PreviousSurrogateIds - WHERE TypeId = ResourceTypeId - AND SurrogateId = ResourceSurrogateId); - SET @AffectedRows += @@rowcount; - DELETE dbo.TokenSearchParam - WHERE EXISTS (SELECT * - FROM @PreviousSurrogateIds - WHERE TypeId = ResourceTypeId - AND SurrogateId = ResourceSurrogateId); - SET @AffectedRows += @@rowcount; - DELETE dbo.TokenText - WHERE EXISTS (SELECT * - FROM @PreviousSurrogateIds - WHERE TypeId = ResourceTypeId - AND SurrogateId = ResourceSurrogateId); - SET @AffectedRows += @@rowcount; - DELETE dbo.StringSearchParam - WHERE EXISTS (SELECT * - FROM @PreviousSurrogateIds - WHERE TypeId = ResourceTypeId - AND SurrogateId = ResourceSurrogateId); - SET @AffectedRows += @@rowcount; - DELETE dbo.UriSearchParam - WHERE EXISTS (SELECT * - FROM @PreviousSurrogateIds - WHERE TypeId = ResourceTypeId - AND SurrogateId = ResourceSurrogateId); - SET @AffectedRows += @@rowcount; - DELETE dbo.NumberSearchParam - WHERE EXISTS (SELECT * - FROM @PreviousSurrogateIds - WHERE TypeId = ResourceTypeId - AND SurrogateId = ResourceSurrogateId); - SET @AffectedRows += @@rowcount; - DELETE dbo.QuantitySearchParam - WHERE EXISTS (SELECT * - FROM @PreviousSurrogateIds - WHERE TypeId = ResourceTypeId - AND SurrogateId = ResourceSurrogateId); - SET @AffectedRows += @@rowcount; - DELETE dbo.DateTimeSearchParam - WHERE EXISTS (SELECT * - FROM @PreviousSurrogateIds - WHERE TypeId = ResourceTypeId - AND SurrogateId = ResourceSurrogateId); - SET @AffectedRows += @@rowcount; - DELETE dbo.ReferenceTokenCompositeSearchParam - WHERE EXISTS (SELECT * - FROM @PreviousSurrogateIds - WHERE TypeId = ResourceTypeId - AND SurrogateId = ResourceSurrogateId); - SET @AffectedRows += @@rowcount; - DELETE dbo.TokenTokenCompositeSearchParam - WHERE EXISTS (SELECT * - FROM @PreviousSurrogateIds - WHERE TypeId = ResourceTypeId - AND SurrogateId = ResourceSurrogateId); - SET @AffectedRows += @@rowcount; - DELETE dbo.TokenDateTimeCompositeSearchParam - WHERE EXISTS (SELECT * - FROM @PreviousSurrogateIds - WHERE TypeId = ResourceTypeId - AND SurrogateId = ResourceSurrogateId); - SET @AffectedRows += @@rowcount; - DELETE dbo.TokenQuantityCompositeSearchParam - WHERE EXISTS (SELECT * - FROM @PreviousSurrogateIds - WHERE TypeId = ResourceTypeId - AND SurrogateId = ResourceSurrogateId); - SET @AffectedRows += @@rowcount; - DELETE dbo.TokenStringCompositeSearchParam - WHERE EXISTS (SELECT * - FROM @PreviousSurrogateIds - WHERE TypeId = ResourceTypeId - AND SurrogateId = ResourceSurrogateId); - SET @AffectedRows += @@rowcount; - DELETE dbo.TokenNumberNumberCompositeSearchParam - WHERE EXISTS (SELECT * - FROM @PreviousSurrogateIds - WHERE TypeId = ResourceTypeId - AND SurrogateId = ResourceSurrogateId); - SET @AffectedRows += @@rowcount; - END - INSERT INTO dbo.Resource (ResourceTypeId, ResourceId, Version, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, RawResource, IsRawResourceMetaSet, SearchParamHash, TransactionId) - SELECT ResourceTypeId, - ResourceId, - Version, - IsHistory, - ResourceSurrogateId, - IsDeleted, - RequestMethod, - RawResource, - IsRawResourceMetaSet, - SearchParamHash, - @TransactionId - FROM @Resources; - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) - SELECT ResourceSurrogateId, - ClaimTypeId, - ClaimValue - FROM @ResourceWriteClaims; - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - BaseUri, - ReferenceResourceTypeId, - ReferenceResourceId, - ReferenceResourceVersion - FROM @ReferenceSearchParams; - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - SystemId, - Code, - CodeOverflow - FROM @TokenSearchParams; - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - Text - FROM @TokenTexts; - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - Text, - TextOverflow, - IsMin, - IsMax - FROM @StringSearchParams; - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - Uri - FROM @UriSearchParams; - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - SingleValue, - LowValue, - HighValue - FROM @NumberSearchParams; - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - SystemId, - QuantityCodeId, - SingleValue, - LowValue, - HighValue - FROM @QuantitySearchParams; - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - StartDateTime, - EndDateTime, - IsLongerThanADay, - IsMin, - IsMax - FROM @DateTimeSearchParms; - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - BaseUri1, - ReferenceResourceTypeId1, - ReferenceResourceId1, - ReferenceResourceVersion1, - SystemId2, - Code2, - CodeOverflow2 - FROM @ReferenceTokenCompositeSearchParams; - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - SystemId1, - Code1, - CodeOverflow1, - SystemId2, - Code2, - CodeOverflow2 - FROM @TokenTokenCompositeSearchParams; - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - SystemId1, - Code1, - CodeOverflow1, - StartDateTime2, - EndDateTime2, - IsLongerThanADay2 - FROM @TokenDateTimeCompositeSearchParams; - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - SystemId1, - Code1, - CodeOverflow1, - SingleValue2, - SystemId2, - QuantityCodeId2, - LowValue2, - HighValue2 - FROM @TokenQuantityCompositeSearchParams; - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - SystemId1, - Code1, - CodeOverflow1, - Text2, - TextOverflow2 - FROM @TokenStringCompositeSearchParams; - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - SystemId1, - Code1, - CodeOverflow1, - SingleValue2, - LowValue2, - HighValue2, - SingleValue3, - LowValue3, - HighValue3, - HasRange - FROM @TokenNumberNumberCompositeSearchParams; - SET @AffectedRows += @@rowcount; - END - ELSE - BEGIN - INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) - SELECT ResourceSurrogateId, - ClaimTypeId, - ClaimValue - FROM (SELECT TOP (@DummyTop) * - FROM @ResourceWriteClaims) AS A - WHERE EXISTS (SELECT * - FROM @Existing AS B - WHERE B.SurrogateId = A.ResourceSurrogateId) - AND NOT EXISTS (SELECT * - FROM dbo.ResourceWriteClaim AS C - WHERE C.ResourceSurrogateId = A.ResourceSurrogateId) - OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - BaseUri, - ReferenceResourceTypeId, - ReferenceResourceId, - ReferenceResourceVersion - FROM (SELECT TOP (@DummyTop) * - FROM @ReferenceSearchParams) AS A - WHERE EXISTS (SELECT * - FROM @Existing AS B - WHERE B.ResourceTypeId = A.ResourceTypeId - AND B.SurrogateId = A.ResourceSurrogateId) - AND NOT EXISTS (SELECT * - FROM dbo.ReferenceSearchParam AS C - WHERE C.ResourceTypeId = A.ResourceTypeId - AND C.ResourceSurrogateId = A.ResourceSurrogateId) - OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - SystemId, - Code, - CodeOverflow - FROM (SELECT TOP (@DummyTop) * - FROM @TokenSearchParams) AS A - WHERE EXISTS (SELECT * - FROM @Existing AS B - WHERE B.ResourceTypeId = A.ResourceTypeId - AND B.SurrogateId = A.ResourceSurrogateId) - AND NOT EXISTS (SELECT * - FROM dbo.TokenSearchParam AS C - WHERE C.ResourceTypeId = A.ResourceTypeId - AND C.ResourceSurrogateId = A.ResourceSurrogateId) - OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - Text - FROM (SELECT TOP (@DummyTop) * - FROM @TokenTexts) AS A - WHERE EXISTS (SELECT * - FROM @Existing AS B - WHERE B.ResourceTypeId = A.ResourceTypeId - AND B.SurrogateId = A.ResourceSurrogateId) - AND NOT EXISTS (SELECT * - FROM dbo.TokenSearchParam AS C - WHERE C.ResourceTypeId = A.ResourceTypeId - AND C.ResourceSurrogateId = A.ResourceSurrogateId) - OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - Text, - TextOverflow, - IsMin, - IsMax - FROM (SELECT TOP (@DummyTop) * - FROM @StringSearchParams) AS A - WHERE EXISTS (SELECT * - FROM @Existing AS B - WHERE B.ResourceTypeId = A.ResourceTypeId - AND B.SurrogateId = A.ResourceSurrogateId) - AND NOT EXISTS (SELECT * - FROM dbo.TokenText AS C - WHERE C.ResourceTypeId = A.ResourceTypeId - AND C.ResourceSurrogateId = A.ResourceSurrogateId) - OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - Uri - FROM (SELECT TOP (@DummyTop) * - FROM @UriSearchParams) AS A - WHERE EXISTS (SELECT * - FROM @Existing AS B - WHERE B.ResourceTypeId = A.ResourceTypeId - AND B.SurrogateId = A.ResourceSurrogateId) - AND NOT EXISTS (SELECT * - FROM dbo.UriSearchParam AS C - WHERE C.ResourceTypeId = A.ResourceTypeId - AND C.ResourceSurrogateId = A.ResourceSurrogateId) - OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - SingleValue, - LowValue, - HighValue - FROM (SELECT TOP (@DummyTop) * - FROM @NumberSearchParams) AS A - WHERE EXISTS (SELECT * - FROM @Existing AS B - WHERE B.ResourceTypeId = A.ResourceTypeId - AND B.SurrogateId = A.ResourceSurrogateId) - AND NOT EXISTS (SELECT * - FROM dbo.NumberSearchParam AS C - WHERE C.ResourceTypeId = A.ResourceTypeId - AND C.ResourceSurrogateId = A.ResourceSurrogateId) - OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - SystemId, - QuantityCodeId, - SingleValue, - LowValue, - HighValue - FROM (SELECT TOP (@DummyTop) * - FROM @QuantitySearchParams) AS A - WHERE EXISTS (SELECT * - FROM @Existing AS B - WHERE B.ResourceTypeId = A.ResourceTypeId - AND B.SurrogateId = A.ResourceSurrogateId) - AND NOT EXISTS (SELECT * - FROM dbo.QuantitySearchParam AS C - WHERE C.ResourceTypeId = A.ResourceTypeId - AND C.ResourceSurrogateId = A.ResourceSurrogateId) - OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - StartDateTime, - EndDateTime, - IsLongerThanADay, - IsMin, - IsMax - FROM (SELECT TOP (@DummyTop) * - FROM @DateTimeSearchParms) AS A - WHERE EXISTS (SELECT * - FROM @Existing AS B - WHERE B.ResourceTypeId = A.ResourceTypeId - AND B.SurrogateId = A.ResourceSurrogateId) - AND NOT EXISTS (SELECT * - FROM dbo.TokenSearchParam AS C - WHERE C.ResourceTypeId = A.ResourceTypeId - AND C.ResourceSurrogateId = A.ResourceSurrogateId) - OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - BaseUri1, - ReferenceResourceTypeId1, - ReferenceResourceId1, - ReferenceResourceVersion1, - SystemId2, - Code2, - CodeOverflow2 - FROM (SELECT TOP (@DummyTop) * - FROM @ReferenceTokenCompositeSearchParams) AS A - WHERE EXISTS (SELECT * - FROM @Existing AS B - WHERE B.ResourceTypeId = A.ResourceTypeId - AND B.SurrogateId = A.ResourceSurrogateId) - AND NOT EXISTS (SELECT * - FROM dbo.DateTimeSearchParam AS C - WHERE C.ResourceTypeId = A.ResourceTypeId - AND C.ResourceSurrogateId = A.ResourceSurrogateId) - OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - SystemId1, - Code1, - CodeOverflow1, - SystemId2, - Code2, - CodeOverflow2 - FROM (SELECT TOP (@DummyTop) * - FROM @TokenTokenCompositeSearchParams) AS A - WHERE EXISTS (SELECT * - FROM @Existing AS B - WHERE B.ResourceTypeId = A.ResourceTypeId - AND B.SurrogateId = A.ResourceSurrogateId) - AND NOT EXISTS (SELECT * - FROM dbo.TokenTokenCompositeSearchParam AS C - WHERE C.ResourceTypeId = A.ResourceTypeId - AND C.ResourceSurrogateId = A.ResourceSurrogateId) - OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - SystemId1, - Code1, - CodeOverflow1, - StartDateTime2, - EndDateTime2, - IsLongerThanADay2 - FROM (SELECT TOP (@DummyTop) * - FROM @TokenDateTimeCompositeSearchParams) AS A - WHERE EXISTS (SELECT * - FROM @Existing AS B - WHERE B.ResourceTypeId = A.ResourceTypeId - AND B.SurrogateId = A.ResourceSurrogateId) - AND NOT EXISTS (SELECT * - FROM dbo.TokenDateTimeCompositeSearchParam AS C - WHERE C.ResourceTypeId = A.ResourceTypeId - AND C.ResourceSurrogateId = A.ResourceSurrogateId) - OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - SystemId1, - Code1, - CodeOverflow1, - SingleValue2, - SystemId2, - QuantityCodeId2, - LowValue2, - HighValue2 - FROM (SELECT TOP (@DummyTop) * - FROM @TokenQuantityCompositeSearchParams) AS A - WHERE EXISTS (SELECT * - FROM @Existing AS B - WHERE B.ResourceTypeId = A.ResourceTypeId - AND B.SurrogateId = A.ResourceSurrogateId) - AND NOT EXISTS (SELECT * - FROM dbo.TokenQuantityCompositeSearchParam AS C - WHERE C.ResourceTypeId = A.ResourceTypeId - AND C.ResourceSurrogateId = A.ResourceSurrogateId) - OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - SystemId1, - Code1, - CodeOverflow1, - Text2, - TextOverflow2 - FROM (SELECT TOP (@DummyTop) * - FROM @TokenStringCompositeSearchParams) AS A - WHERE EXISTS (SELECT * - FROM @Existing AS B - WHERE B.ResourceTypeId = A.ResourceTypeId - AND B.SurrogateId = A.ResourceSurrogateId) - AND NOT EXISTS (SELECT * - FROM dbo.TokenStringCompositeSearchParam AS C - WHERE C.ResourceTypeId = A.ResourceTypeId - AND C.ResourceSurrogateId = A.ResourceSurrogateId) - OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); - SET @AffectedRows += @@rowcount; - INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - SystemId1, - Code1, - CodeOverflow1, - SingleValue2, - LowValue2, - HighValue2, - SingleValue3, - LowValue3, - HighValue3, - HasRange - FROM (SELECT TOP (@DummyTop) * - FROM @TokenNumberNumberCompositeSearchParams) AS A - WHERE EXISTS (SELECT * - FROM @Existing AS B - WHERE B.ResourceTypeId = A.ResourceTypeId - AND B.SurrogateId = A.ResourceSurrogateId) - AND NOT EXISTS (SELECT * - FROM dbo.TokenNumberNumberCompositeSearchParam AS C - WHERE C.ResourceTypeId = A.ResourceTypeId - AND C.ResourceSurrogateId = A.ResourceSurrogateId) - OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); - SET @AffectedRows += @@rowcount; - END - IF @IsResourceChangeCaptureEnabled = 1 - EXECUTE dbo.CaptureResourceIdsForChanges @Resources; - IF @TransactionId IS NOT NULL - EXECUTE dbo.MergeResourcesCommitTransaction @TransactionId; - IF @InitialTranCount = 0 - AND @@trancount > 0 - COMMIT TRANSACTION; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @AffectedRows; -END TRY -BEGIN CATCH - IF @InitialTranCount = 0 - AND @@trancount > 0 - ROLLBACK; - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; - IF @RaiseExceptionOnConflict = 1 - AND error_number() IN (2601, 2627) - AND error_message() LIKE '%''dbo.Resource''%' - THROW 50409, 'Resource has been recently updated or added, please compare the resource content in code for any duplicate updates', 1; - ELSE - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.MergeResourcesAdvanceTransactionVisibility -@AffectedRows INT=0 OUTPUT -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = '', @st AS DATETIME = getUTCdate(), @msg AS VARCHAR (1000), @MaxTransactionId AS BIGINT, @MinTransactionId AS BIGINT, @MinNotCompletedTransactionId AS BIGINT, @CurrentTransactionId AS BIGINT; -SET @AffectedRows = 0; -BEGIN TRY - EXECUTE dbo.MergeResourcesGetTransactionVisibility @MinTransactionId OUTPUT; - SET @MinTransactionId += 1; - SET @CurrentTransactionId = (SELECT TOP 1 SurrogateIdRangeFirstValue - FROM dbo.Transactions - ORDER BY SurrogateIdRangeFirstValue DESC); - SET @MinNotCompletedTransactionId = isnull((SELECT TOP 1 SurrogateIdRangeFirstValue - FROM dbo.Transactions - WHERE IsCompleted = 0 - AND SurrogateIdRangeFirstValue BETWEEN @MinTransactionId AND @CurrentTransactionId - ORDER BY SurrogateIdRangeFirstValue), @CurrentTransactionId + 1); - SET @MaxTransactionId = (SELECT TOP 1 SurrogateIdRangeFirstValue - FROM dbo.Transactions - WHERE IsCompleted = 1 - AND SurrogateIdRangeFirstValue BETWEEN @MinTransactionId AND @CurrentTransactionId - AND SurrogateIdRangeFirstValue < @MinNotCompletedTransactionId - ORDER BY SurrogateIdRangeFirstValue DESC); - IF @MaxTransactionId >= @MinTransactionId - BEGIN - UPDATE A - SET IsVisible = 1, - VisibleDate = getUTCdate() - FROM dbo.Transactions AS A WITH (INDEX (1)) - WHERE SurrogateIdRangeFirstValue BETWEEN @MinTransactionId AND @CurrentTransactionId - AND SurrogateIdRangeFirstValue <= @MaxTransactionId; - SET @AffectedRows += @@rowcount; - END - SET @msg = 'Min=' + CONVERT (VARCHAR, @MinTransactionId) + ' C=' + CONVERT (VARCHAR, @CurrentTransactionId) + ' MinNC=' + CONVERT (VARCHAR, @MinNotCompletedTransactionId) + ' Max=' + CONVERT (VARCHAR, @MaxTransactionId); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @AffectedRows, @Text = @msg; -END TRY -BEGIN CATCH - IF @@trancount > 0 - ROLLBACK; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.MergeResourcesBeginTransaction -@Count INT, @TransactionId BIGINT=0 OUTPUT, @SurrogateIdRangeFirstValue BIGINT=0 OUTPUT, @SequenceRangeFirstValue INT=0 OUTPUT, @HeartbeatDate DATETIME=NULL -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'MergeResourcesBeginTransaction', @Mode AS VARCHAR (200) = 'Cnt=' + CONVERT (VARCHAR, @Count), @st AS DATETIME = getUTCdate(), @FirstValueVar AS SQL_VARIANT, @LastValueVar AS SQL_VARIANT; -BEGIN TRY - SET @TransactionId = NULL; - IF @@trancount > 0 - RAISERROR ('MergeResourcesBeginTransaction cannot be called inside outer transaction.', 18, 127); - SET @FirstValueVar = NULL; - WHILE @FirstValueVar IS NULL - BEGIN - EXECUTE sys.sp_sequence_get_range @sequence_name = 'dbo.ResourceSurrogateIdUniquifierSequence', @range_size = @Count, @range_first_value = @FirstValueVar OUTPUT, @range_last_value = @LastValueVar OUTPUT; - SET @SequenceRangeFirstValue = CONVERT (INT, @FirstValueVar); - IF @SequenceRangeFirstValue > CONVERT (INT, @LastValueVar) - SET @FirstValueVar = NULL; - END - SET @SurrogateIdRangeFirstValue = datediff_big(millisecond, '0001-01-01', sysUTCdatetime()) * 80000 + @SequenceRangeFirstValue; - INSERT INTO dbo.Transactions (SurrogateIdRangeFirstValue, SurrogateIdRangeLastValue, HeartbeatDate) - SELECT @SurrogateIdRangeFirstValue, - @SurrogateIdRangeFirstValue + @Count - 1, - isnull(@HeartbeatDate, getUTCdate()); - SET @TransactionId = @SurrogateIdRangeFirstValue; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - IF @@trancount > 0 - ROLLBACK; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.MergeResourcesCommitTransaction -@TransactionId BIGINT=NULL, @FailureReason VARCHAR (MAX)=NULL, @OverrideIsControlledByClientCheck BIT=0, @SurrogateIdRangeFirstValue BIGINT=NULL -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'MergeResourcesCommitTransaction', @st AS DATETIME = getUTCdate(), @InitialTranCount AS INT = @@trancount, @IsCompletedBefore AS BIT, @Rows AS INT, @msg AS VARCHAR (1000); -SET @TransactionId = isnull(@TransactionId, @SurrogateIdRangeFirstValue); -DECLARE @Mode AS VARCHAR (200) = 'TR=' + CONVERT (VARCHAR, @TransactionId) + ' OC=' + isnull(CONVERT (VARCHAR, @OverrideIsControlledByClientCheck), 'NULL'); -BEGIN TRY - IF @InitialTranCount = 0 - BEGIN TRANSACTION; - UPDATE dbo.Transactions - SET IsCompleted = 1, - @IsCompletedBefore = IsCompleted, - EndDate = getUTCdate(), - IsSuccess = CASE WHEN @FailureReason IS NULL THEN 1 ELSE 0 END, - FailureReason = @FailureReason - WHERE SurrogateIdRangeFirstValue = @TransactionId - AND (IsControlledByClient = 1 - OR @OverrideIsControlledByClientCheck = 1); - SET @Rows = @@rowcount; - IF @Rows = 0 - BEGIN - SET @msg = 'Transaction [' + CONVERT (VARCHAR (20), @TransactionId) + '] is not controlled by client or does not exist.'; - RAISERROR (@msg, 18, 127); - END - IF @IsCompletedBefore = 1 - BEGIN - IF @InitialTranCount = 0 - ROLLBACK; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows, @Target = '@IsCompletedBefore', @Text = '=1'; - RETURN; - END - IF @InitialTranCount = 0 - COMMIT TRANSACTION; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; -END TRY -BEGIN CATCH - IF @InitialTranCount = 0 - AND @@trancount > 0 - ROLLBACK; - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.MergeResourcesDeleteInvisibleHistory -@TransactionId BIGINT, @AffectedRows INT=NULL OUTPUT -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'T=' + CONVERT (VARCHAR, @TransactionId), @st AS DATETIME = getUTCdate(), @TypeId AS SMALLINT; -SET @AffectedRows = 0; -BEGIN TRY - DECLARE @Types TABLE ( - TypeId SMALLINT PRIMARY KEY, - Name VARCHAR (100)); - INSERT INTO @Types - EXECUTE dbo.GetUsedResourceTypes ; - WHILE EXISTS (SELECT * - FROM @Types) - BEGIN - SET @TypeId = (SELECT TOP 1 TypeId - FROM @Types - ORDER BY TypeId); - DELETE dbo.Resource - WHERE ResourceTypeId = @TypeId - AND HistoryTransactionId = @TransactionId - AND IsHistory = 1 - AND RawResource = 0xF; - SET @AffectedRows += @@rowcount; - DELETE @Types - WHERE TypeId = @TypeId; - END - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @AffectedRows; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.MergeResourcesGetTimeoutTransactions -@TimeoutSec INT -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'T=' + CONVERT (VARCHAR, @TimeoutSec), @st AS DATETIME = getUTCdate(), @MinTransactionId AS BIGINT; -BEGIN TRY - EXECUTE dbo.MergeResourcesGetTransactionVisibility @MinTransactionId OUTPUT; - SELECT SurrogateIdRangeFirstValue - FROM dbo.Transactions - WHERE SurrogateIdRangeFirstValue > @MinTransactionId - AND IsCompleted = 0 - AND datediff(second, HeartbeatDate, getUTCdate()) > @TimeoutSec - ORDER BY SurrogateIdRangeFirstValue; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; -END TRY -BEGIN CATCH - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.MergeResourcesGetTransactionVisibility -@TransactionId BIGINT OUTPUT -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = '', @st AS DATETIME = getUTCdate(); -SET @TransactionId = isnull((SELECT TOP 1 SurrogateIdRangeFirstValue - FROM dbo.Transactions - WHERE IsVisible = 1 - ORDER BY SurrogateIdRangeFirstValue DESC), -1); -EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount, @Text = @TransactionId; - -GO -CREATE PROCEDURE dbo.MergeResourcesPutTransactionHeartbeat -@TransactionId BIGINT -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'MergeResourcesPutTransactionHeartbeat', @Mode AS VARCHAR (100) = 'TR=' + CONVERT (VARCHAR, @TransactionId); -BEGIN TRY - UPDATE dbo.Transactions - SET HeartbeatDate = getUTCdate() - WHERE SurrogateIdRangeFirstValue = @TransactionId - AND IsControlledByClient = 1; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.MergeResourcesPutTransactionInvisibleHistory -@TransactionId BIGINT -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'TR=' + CONVERT (VARCHAR, @TransactionId), @st AS DATETIME = getUTCdate(); -BEGIN TRY - UPDATE dbo.Transactions - SET InvisibleHistoryRemovedDate = getUTCdate() - WHERE SurrogateIdRangeFirstValue = @TransactionId - AND InvisibleHistoryRemovedDate IS NULL; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.PutJobCancelation -@QueueType TINYINT, @GroupId BIGINT=NULL, @JobId BIGINT=NULL -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'PutJobCancelation', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' G=' + isnull(CONVERT (VARCHAR, @GroupId), 'NULL') + ' J=' + isnull(CONVERT (VARCHAR, @JobId), 'NULL'), @st AS DATETIME = getUTCdate(), @Rows AS INT, @PartitionId AS TINYINT = @JobId % 16; -BEGIN TRY - IF @JobId IS NULL - AND @GroupId IS NULL - RAISERROR ('@JobId = NULL and @GroupId = NULL', 18, 127); - IF @JobId IS NOT NULL - BEGIN - UPDATE dbo.JobQueue - SET Status = 4, - EndDate = getUTCdate(), - Version = datediff_big(millisecond, '0001-01-01', getUTCdate()) - WHERE QueueType = @QueueType - AND PartitionId = @PartitionId - AND JobId = @JobId - AND Status = 0; - SET @Rows = @@rowcount; - IF @Rows = 0 - BEGIN - UPDATE dbo.JobQueue - SET CancelRequested = 1 - WHERE QueueType = @QueueType - AND PartitionId = @PartitionId - AND JobId = @JobId - AND Status = 1; - SET @Rows = @@rowcount; - END - END - ELSE - BEGIN - UPDATE dbo.JobQueue - SET Status = 4, - EndDate = getUTCdate(), - Version = datediff_big(millisecond, '0001-01-01', getUTCdate()) - WHERE QueueType = @QueueType - AND GroupId = @GroupId - AND Status = 0; - SET @Rows = @@rowcount; - UPDATE dbo.JobQueue - SET CancelRequested = 1 - WHERE QueueType = @QueueType - AND GroupId = @GroupId - AND Status = 1; - SET @Rows += @@rowcount; - END - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.PutJobHeartbeat -@QueueType TINYINT, @JobId BIGINT, @Version BIGINT, @Data BIGINT=NULL, @CurrentResult VARCHAR (MAX)=NULL, @CancelRequested BIT=0 OUTPUT -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'PutJobHeartbeat', @Mode AS VARCHAR (100), @st AS DATETIME = getUTCdate(), @Rows AS INT = 0, @PartitionId AS TINYINT = @JobId % 16; -SET @Mode = 'Q=' + CONVERT (VARCHAR, @QueueType) + ' J=' + CONVERT (VARCHAR, @JobId) + ' P=' + CONVERT (VARCHAR, @PartitionId) + ' V=' + CONVERT (VARCHAR, @Version) + ' D=' + isnull(CONVERT (VARCHAR, @Data), 'NULL'); -BEGIN TRY - IF @CurrentResult IS NULL - UPDATE dbo.JobQueue - SET @CancelRequested = CancelRequested, - HeartbeatDate = getUTCdate(), - Data = isnull(@Data, Data) - WHERE QueueType = @QueueType - AND PartitionId = @PartitionId - AND JobId = @JobId - AND Status = 1 - AND Version = @Version; - ELSE - UPDATE dbo.JobQueue - SET @CancelRequested = CancelRequested, - HeartbeatDate = getUTCdate(), - Data = isnull(@Data, Data), - Result = @CurrentResult - WHERE QueueType = @QueueType - AND PartitionId = @PartitionId - AND JobId = @JobId - AND Status = 1 - AND Version = @Version; - SET @Rows = @@rowcount; - IF @Rows = 0 - AND NOT EXISTS (SELECT * - FROM dbo.JobQueue - WHERE QueueType = @QueueType - AND PartitionId = @PartitionId - AND JobId = @JobId - AND Version = @Version - AND Status IN (2, 3, 4)) - BEGIN - IF EXISTS (SELECT * - FROM dbo.JobQueue - WHERE QueueType = @QueueType - AND PartitionId = @PartitionId - AND JobId = @JobId) - THROW 50412, 'Precondition failed', 1; - ELSE - THROW 50404, 'Job record not found', 1; - END - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.PutJobStatus -@QueueType TINYINT, @JobId BIGINT, @Version BIGINT, @Failed BIT, @Data BIGINT, @FinalResult VARCHAR (MAX), @RequestCancellationOnFailure BIT -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'PutJobStatus', @Mode AS VARCHAR (100), @st AS DATETIME = getUTCdate(), @Rows AS INT = 0, @PartitionId AS TINYINT = @JobId % 16, @GroupId AS BIGINT; -SET @Mode = 'Q=' + CONVERT (VARCHAR, @QueueType) + ' J=' + CONVERT (VARCHAR, @JobId) + ' P=' + CONVERT (VARCHAR, @PartitionId) + ' V=' + CONVERT (VARCHAR, @Version) + ' F=' + CONVERT (VARCHAR, @Failed) + ' R=' + isnull(@FinalResult, 'NULL'); -BEGIN TRY - UPDATE dbo.JobQueue - SET EndDate = getUTCdate(), - Status = CASE WHEN @Failed = 1 THEN 3 WHEN CancelRequested = 1 THEN 4 ELSE 2 END, - Data = @Data, - Result = @FinalResult, - @GroupId = GroupId - WHERE QueueType = @QueueType - AND PartitionId = @PartitionId - AND JobId = @JobId - AND Status = 1 - AND Version = @Version; - SET @Rows = @@rowcount; - IF @Rows = 0 - BEGIN - SET @GroupId = (SELECT GroupId - FROM dbo.JobQueue - WHERE QueueType = @QueueType - AND PartitionId = @PartitionId - AND JobId = @JobId - AND Version = @Version - AND Status IN (2, 3, 4)); - IF @GroupId IS NULL - IF EXISTS (SELECT * - FROM dbo.JobQueue - WHERE QueueType = @QueueType - AND PartitionId = @PartitionId - AND JobId = @JobId) - THROW 50412, 'Precondition failed', 1; - ELSE - THROW 50404, 'Job record not found', 1; - END - IF @Failed = 1 - AND @RequestCancellationOnFailure = 1 - EXECUTE dbo.PutJobCancelation @QueueType = @QueueType, @GroupId = @GroupId; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.ReadResource -@resourceTypeId SMALLINT, @resourceId VARCHAR (64), @version INT=NULL -AS -SET NOCOUNT ON; -IF (@version IS NULL) - BEGIN - SELECT ResourceSurrogateId, - Version, - IsDeleted, - IsHistory, - RawResource, - IsRawResourceMetaSet, - SearchParamHash - FROM dbo.Resource - WHERE ResourceTypeId = @resourceTypeId - AND ResourceId = @resourceId - AND IsHistory = 0; - END -ELSE - BEGIN - SELECT ResourceSurrogateId, - Version, - IsDeleted, - IsHistory, - RawResource, - IsRawResourceMetaSet, - SearchParamHash - FROM dbo.Resource - WHERE ResourceTypeId = @resourceTypeId - AND ResourceId = @resourceId - AND Version = @version; - END - -GO -CREATE PROCEDURE dbo.ReindexResource_2 -@resourceTypeId SMALLINT, @resourceId VARCHAR (64), @eTag INT=NULL, @searchParamHash VARCHAR (64), @resourceWriteClaims dbo.BulkResourceWriteClaimTableType_1 READONLY, @compartmentAssignments dbo.BulkCompartmentAssignmentTableType_1 READONLY, @referenceSearchParams dbo.BulkReferenceSearchParamTableType_1 READONLY, @tokenSearchParams dbo.BulkTokenSearchParamTableType_2 READONLY, @tokenTextSearchParams dbo.BulkTokenTextTableType_1 READONLY, @stringSearchParams dbo.BulkStringSearchParamTableType_2 READONLY, @numberSearchParams dbo.BulkNumberSearchParamTableType_1 READONLY, @quantitySearchParams dbo.BulkQuantitySearchParamTableType_1 READONLY, @uriSearchParams dbo.BulkUriSearchParamTableType_1 READONLY, @dateTimeSearchParms dbo.BulkDateTimeSearchParamTableType_2 READONLY, @referenceTokenCompositeSearchParams dbo.BulkReferenceTokenCompositeSearchParamTableType_2 READONLY, @tokenTokenCompositeSearchParams dbo.BulkTokenTokenCompositeSearchParamTableType_2 READONLY, @tokenDateTimeCompositeSearchParams dbo.BulkTokenDateTimeCompositeSearchParamTableType_2 READONLY, @tokenQuantityCompositeSearchParams dbo.BulkTokenQuantityCompositeSearchParamTableType_2 READONLY, @tokenStringCompositeSearchParams dbo.BulkTokenStringCompositeSearchParamTableType_2 READONLY, @tokenNumberNumberCompositeSearchParams dbo.BulkTokenNumberNumberCompositeSearchParamTableType_2 READONLY -AS -SET NOCOUNT ON; -SET XACT_ABORT ON; -BEGIN TRANSACTION; -DECLARE @resourceSurrogateId AS BIGINT; -DECLARE @version AS BIGINT; -SELECT @resourceSurrogateId = ResourceSurrogateId, - @version = Version -FROM dbo.Resource WITH (UPDLOCK, HOLDLOCK) -WHERE ResourceTypeId = @resourceTypeId - AND ResourceId = @resourceId - AND IsHistory = 0; -IF (@etag IS NOT NULL - AND @etag <> @version) - BEGIN - THROW 50412, 'Precondition failed', 1; - END -UPDATE dbo.Resource -SET SearchParamHash = @searchParamHash -WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @resourceSurrogateId; -DELETE dbo.ResourceWriteClaim -WHERE ResourceSurrogateId = @resourceSurrogateId; -DELETE dbo.CompartmentAssignment -WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @resourceSurrogateId; -DELETE dbo.ReferenceSearchParam -WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @resourceSurrogateId; -DELETE dbo.TokenSearchParam -WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @resourceSurrogateId; -DELETE dbo.TokenText -WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @resourceSurrogateId; -DELETE dbo.StringSearchParam -WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @resourceSurrogateId; -DELETE dbo.UriSearchParam -WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @resourceSurrogateId; -DELETE dbo.NumberSearchParam -WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @resourceSurrogateId; -DELETE dbo.QuantitySearchParam -WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @resourceSurrogateId; -DELETE dbo.DateTimeSearchParam -WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @resourceSurrogateId; -DELETE dbo.ReferenceTokenCompositeSearchParam -WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @resourceSurrogateId; -DELETE dbo.TokenTokenCompositeSearchParam -WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @resourceSurrogateId; -DELETE dbo.TokenDateTimeCompositeSearchParam -WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @resourceSurrogateId; -DELETE dbo.TokenQuantityCompositeSearchParam -WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @resourceSurrogateId; -DELETE dbo.TokenStringCompositeSearchParam -WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @resourceSurrogateId; -DELETE dbo.TokenNumberNumberCompositeSearchParam -WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @resourceSurrogateId; -INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) -SELECT @resourceSurrogateId, - ClaimTypeId, - ClaimValue -FROM @resourceWriteClaims; -INSERT INTO dbo.CompartmentAssignment (ResourceTypeId, ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId, IsHistory) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - CompartmentTypeId, - ReferenceResourceId, - 0 -FROM @compartmentAssignments; -INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion, IsHistory) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - BaseUri, - ReferenceResourceTypeId, - ReferenceResourceId, - ReferenceResourceVersion, - 0 -FROM @referenceSearchParams; -INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow, IsHistory) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - SystemId, - Code, - CodeOverflow, - 0 -FROM @tokenSearchParams; -INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, IsHistory) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - Text, - 0 -FROM @tokenTextSearchParams; -INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsHistory, IsMin, IsMax) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - Text, - TextOverflow, - 0, - IsMin, - IsMax -FROM @stringSearchParams; -INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri, IsHistory) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - Uri, - 0 -FROM @uriSearchParams; -INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue, IsHistory) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - SingleValue, - LowValue, - HighValue, - 0 -FROM @numberSearchParams; -INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue, IsHistory) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - SystemId, - QuantityCodeId, - SingleValue, - LowValue, - HighValue, - 0 -FROM @quantitySearchParams; -INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsHistory, IsMin, IsMax) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - StartDateTime, - EndDateTime, - IsLongerThanADay, - 0, - IsMin, - IsMax -FROM @dateTimeSearchParms; -INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2, IsHistory) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - BaseUri1, - ReferenceResourceTypeId1, - ReferenceResourceId1, - ReferenceResourceVersion1, - SystemId2, - Code2, - CodeOverflow2, - 0 -FROM @referenceTokenCompositeSearchParams; -INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2, IsHistory) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - SystemId1, - Code1, - CodeOverflow1, - SystemId2, - Code2, - CodeOverflow2, - 0 -FROM @tokenTokenCompositeSearchParams; -INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2, IsHistory) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - SystemId1, - Code1, - CodeOverflow1, - StartDateTime2, - EndDateTime2, - IsLongerThanADay2, - 0 -FROM @tokenDateTimeCompositeSearchParams; -INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2, IsHistory) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - SystemId1, - Code1, - CodeOverflow1, - SingleValue2, - SystemId2, - QuantityCodeId2, - LowValue2, - HighValue2, - 0 -FROM @tokenQuantityCompositeSearchParams; -INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2, IsHistory) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - SystemId1, - Code1, - CodeOverflow1, - Text2, - TextOverflow2, - 0 -FROM @tokenStringCompositeSearchParams; -INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange, IsHistory) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - SystemId1, - Code1, - CodeOverflow1, - SingleValue2, - LowValue2, - HighValue2, - SingleValue3, - LowValue3, - HighValue3, - HasRange, - 0 -FROM @tokenNumberNumberCompositeSearchParams; -COMMIT TRANSACTION; - -GO -CREATE OR ALTER PROCEDURE dbo.RemovePartitionFromResourceChanges_2 -@partitionNumberToSwitchOut INT, @partitionBoundaryToMerge DATETIME2 (7) -AS -BEGIN - TRUNCATE TABLE dbo.ResourceChangeDataStaging; - ALTER TABLE dbo.ResourceChangeData SWITCH PARTITION @partitionNumberToSwitchOut TO dbo.ResourceChangeDataStaging; - ALTER PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp( ) - MERGE RANGE (@partitionBoundaryToMerge); - TRUNCATE TABLE dbo.ResourceChangeDataStaging; -END - -GO -CREATE PROCEDURE dbo.ResetTask_2 -@taskId VARCHAR (64), @runId VARCHAR (50), @result VARCHAR (MAX) -AS -SET NOCOUNT ON; -SET XACT_ABORT ON; -DECLARE @retryCount AS SMALLINT = NULL; -IF NOT EXISTS (SELECT * - FROM dbo.TaskInfo - WHERE TaskId = @taskId - AND RunId = @runId) - BEGIN - THROW 50404, 'Task not exist or runid not match', 1; - END -UPDATE dbo.TaskInfo -SET Status = 3, - EndDateTime = SYSUTCDATETIME(), - Result = @result, - @retryCount = retryCount -WHERE TaskId = @taskId - AND RunId = @runId - AND (MaxRetryCount <> -1 - AND RetryCount >= MaxRetryCount); -IF @retryCount IS NULL - UPDATE dbo.TaskInfo - SET Status = 1, - Result = @result, - RetryCount = RetryCount + 1, - RestartInfo = ISNULL(RestartInfo, '') + ' Prev: Worker=' + Worker + ' Start=' + CONVERT (VARCHAR, StartDateTime, 121) - WHERE TaskId = @taskId - AND RunId = @runId - AND Status <> 3 - AND (MaxRetryCount = -1 - OR RetryCount < MaxRetryCount); -EXECUTE dbo.GetTaskDetails @TaskId = @taskId; - -GO -CREATE PROCEDURE dbo.SwitchPartitionsIn -@Tbl VARCHAR (100) -WITH EXECUTE AS 'dbo' -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'SwitchPartitionsIn', @Mode AS VARCHAR (200) = 'Tbl=' + isnull(@Tbl, 'NULL'), @st AS DATETIME = getUTCdate(), @ResourceTypeId AS SMALLINT, @Rows AS BIGINT, @Txt AS VARCHAR (1000), @TblInt AS VARCHAR (100), @Ind AS VARCHAR (200), @IndId AS INT, @DataComp AS VARCHAR (100); -DECLARE @Indexes TABLE ( - IndId INT PRIMARY KEY, - name VARCHAR (200)); -DECLARE @ResourceTypes TABLE ( - ResourceTypeId SMALLINT PRIMARY KEY); -BEGIN TRY - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; - IF @Tbl IS NULL - RAISERROR ('@Tbl IS NULL', 18, 127); - INSERT INTO @Indexes - SELECT index_id, - name - FROM sys.indexes - WHERE object_id = object_id(@Tbl) - AND is_disabled = 1; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Insert', @Rows = @@rowcount; - WHILE EXISTS (SELECT * - FROM @Indexes) - BEGIN - SELECT TOP 1 @IndId = IndId, - @Ind = name - FROM @Indexes - ORDER BY IndId; - SET @DataComp = CASE WHEN (SELECT PropertyValue - FROM dbo.IndexProperties - WHERE TableName = @Tbl - AND IndexName = @Ind) = 'PAGE' THEN ' PARTITION = ALL WITH (DATA_COMPRESSION = PAGE)' ELSE '' END; - SET @Txt = 'IF EXISTS (SELECT * FROM sys.indexes WHERE object_id = object_id(''' + @Tbl + ''') AND name = ''' + @Ind + ''' AND is_disabled = 1) ALTER INDEX ' + @Ind + ' ON dbo.' + @Tbl + ' REBUILD' + @DataComp; - EXECUTE (@Txt); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Ind, @Action = 'Rebuild', @Text = @Txt; - DELETE @Indexes - WHERE IndId = @IndId; - END - INSERT INTO @ResourceTypes - SELECT CONVERT (SMALLINT, substring(name, charindex('_', name) + 1, 6)) AS ResourceTypeId - FROM sys.objects AS O - WHERE name LIKE @Tbl + '[_]%' - AND EXISTS (SELECT * - FROM sysindexes - WHERE id = O.object_id - AND indid IN (0, 1) - AND rows > 0); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '#ResourceTypes', @Action = 'Select Into', @Rows = @@rowcount; - WHILE EXISTS (SELECT * - FROM @ResourceTypes) - BEGIN - SET @ResourceTypeId = (SELECT TOP 1 ResourceTypeId - FROM @ResourceTypes); - SET @TblInt = @Tbl + '_' + CONVERT (VARCHAR, @ResourceTypeId); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt; - SET @Txt = 'ALTER TABLE dbo.' + @TblInt + ' SWITCH TO dbo.' + @Tbl + ' PARTITION $partition.PartitionFunction_ResourceTypeId(' + CONVERT (VARCHAR, @ResourceTypeId) + ')'; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Switch in start', @Text = @Txt; - EXECUTE (@Txt); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Switch in', @Text = @Txt; - IF EXISTS (SELECT * - FROM sysindexes - WHERE id = object_id(@TblInt) - AND rows > 0) - BEGIN - SET @Txt = @TblInt + ' is not empty after switch'; - RAISERROR (@Txt, 18, 127); - END - EXECUTE ('DROP TABLE dbo.' + @TblInt); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Drop'; - DELETE @ResourceTypes - WHERE ResourceTypeId = @ResourceTypeId; - END - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.SwitchPartitionsInAllTables -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'SwitchPartitionsInAllTables', @Mode AS VARCHAR (200) = 'PS=PartitionScheme_ResourceTypeId', @st AS DATETIME = getUTCdate(), @Tbl AS VARCHAR (100); -BEGIN TRY - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; - DECLARE @Tables TABLE ( - name VARCHAR (100) PRIMARY KEY, - supported BIT ); - INSERT INTO @Tables - EXECUTE dbo.GetPartitionedTables @IncludeNotDisabled = 1, @IncludeNotSupported = 0; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Tables', @Action = 'Insert', @Rows = @@rowcount; - WHILE EXISTS (SELECT * - FROM @Tables) - BEGIN - SET @Tbl = (SELECT TOP 1 name - FROM @Tables - ORDER BY name); - EXECUTE dbo.SwitchPartitionsIn @Tbl = @Tbl; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = 'SwitchPartitionsIn', @Action = 'Execute', @Text = @Tbl; - DELETE @Tables - WHERE name = @Tbl; - END - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.SwitchPartitionsOut -@Tbl VARCHAR (100), @RebuildClustered BIT -WITH EXECUTE AS 'dbo' -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'SwitchPartitionsOut', @Mode AS VARCHAR (200) = 'Tbl=' + isnull(@Tbl, 'NULL') + ' ND=' + isnull(CONVERT (VARCHAR, @RebuildClustered), 'NULL'), @st AS DATETIME = getUTCdate(), @ResourceTypeId AS SMALLINT, @Rows AS BIGINT, @Txt AS VARCHAR (MAX), @TblInt AS VARCHAR (100), @IndId AS INT, @Ind AS VARCHAR (200), @Name AS VARCHAR (100), @checkName AS VARCHAR (200), @definition AS VARCHAR (200); -DECLARE @Indexes TABLE ( - IndId INT PRIMARY KEY, - name VARCHAR (200), - IsDisabled BIT ); -DECLARE @IndexesRT TABLE ( - IndId INT PRIMARY KEY, - name VARCHAR (200), - IsDisabled BIT ); -DECLARE @ResourceTypes TABLE ( - ResourceTypeId SMALLINT PRIMARY KEY, - partition_number_roundtrip INT , - partition_number INT , - row_count BIGINT ); -DECLARE @Names TABLE ( - name VARCHAR (100) PRIMARY KEY); -DECLARE @CheckConstraints TABLE ( - CheckName VARCHAR (200), - CheckDefinition VARCHAR (200)); -BEGIN TRY - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; - IF @Tbl IS NULL - RAISERROR ('@Tbl IS NULL', 18, 127); - IF @RebuildClustered IS NULL - RAISERROR ('@RebuildClustered IS NULL', 18, 127); - INSERT INTO @Indexes - SELECT index_id, - name, - is_disabled - FROM sys.indexes - WHERE object_id = object_id(@Tbl) - AND (is_disabled = 0 - OR @RebuildClustered = 1); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Insert', @Rows = @@rowcount; - INSERT INTO @ResourceTypes - SELECT partition_number - 1 AS ResourceTypeId, - $PARTITION.PartitionFunction_ResourceTypeId (partition_number - 1) AS partition_number_roundtrip, - partition_number, - row_count - FROM sys.dm_db_partition_stats - WHERE object_id = object_id(@Tbl) - AND index_id = 1 - AND row_count > 0; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@ResourceTypes', @Action = 'Insert', @Rows = @@rowcount, @Text = 'For partition switch'; - IF EXISTS (SELECT * - FROM @ResourceTypes - WHERE partition_number_roundtrip <> partition_number) - RAISERROR ('Partition sanity check failed', 18, 127); - WHILE EXISTS (SELECT * - FROM @ResourceTypes) - BEGIN - SELECT TOP 1 @ResourceTypeId = ResourceTypeId, - @Rows = row_count - FROM @ResourceTypes - ORDER BY ResourceTypeId; - SET @TblInt = @Tbl + '_' + CONVERT (VARCHAR, @ResourceTypeId); - SET @Txt = 'Starting @ResourceTypeId=' + CONVERT (VARCHAR, @ResourceTypeId) + ' row_count=' + CONVERT (VARCHAR, @Rows); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Text = @Txt; - IF NOT EXISTS (SELECT * - FROM sysindexes - WHERE id = object_id(@TblInt) - AND rows > 0) - BEGIN - IF object_id(@TblInt) IS NOT NULL - BEGIN - EXECUTE ('DROP TABLE dbo.' + @TblInt); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Drop'; - END - EXECUTE ('SELECT * INTO dbo.' + @TblInt + ' FROM dbo.' + @Tbl + ' WHERE 1 = 2'); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Select Into', @Rows = @@rowcount; - DELETE @CheckConstraints; - INSERT INTO @CheckConstraints - SELECT name, - definition - FROM sys.check_constraints - WHERE parent_object_id = object_id(@Tbl); - WHILE EXISTS (SELECT * - FROM @CheckConstraints) - BEGIN - SELECT TOP 1 @checkName = CheckName, - @definition = CheckDefinition - FROM @CheckConstraints; - SET @Txt = 'ALTER TABLE ' + @TblInt + ' ADD CHECK ' + @definition; - EXECUTE (@Txt); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'ALTER', @Text = @Txt; - DELETE @CheckConstraints - WHERE CheckName = @checkName; - END - DELETE @Names; - INSERT INTO @Names - SELECT name - FROM sys.columns - WHERE object_id = object_id(@Tbl) - AND is_sparse = 1; - WHILE EXISTS (SELECT * - FROM @Names) - BEGIN - SET @Name = (SELECT TOP 1 name - FROM @Names - ORDER BY name); - SET @Txt = (SELECT 'ALTER TABLE dbo.' + @TblInt + ' ALTER COLUMN ' + @Name + ' ' + T.name + '(' + CONVERT (VARCHAR, C.precision) + ',' + CONVERT (VARCHAR, C.scale) + ') SPARSE NULL' - FROM sys.types AS T - INNER JOIN - sys.columns AS C - ON C.system_type_id = T.system_type_id - WHERE C.object_id = object_id(@Tbl) - AND C.name = @Name); - EXECUTE (@Txt); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'ALTER', @Text = @Txt; - DELETE @Names - WHERE name = @Name; - END - END - INSERT INTO @IndexesRT - SELECT * - FROM @Indexes - WHERE IsDisabled = 0; - WHILE EXISTS (SELECT * - FROM @IndexesRT) - BEGIN - SELECT TOP 1 @IndId = IndId, - @Ind = name - FROM @IndexesRT - ORDER BY IndId; - IF NOT EXISTS (SELECT * - FROM sys.indexes - WHERE object_id = object_id(@TblInt) - AND name = @Ind) - BEGIN - EXECUTE dbo.GetIndexCommands @Tbl = @Tbl, @Ind = @Ind, @AddPartClause = 0, @IncludeClustered = 1, @Txt = @Txt OUTPUT; - SET @Txt = replace(@Txt, '[' + @Tbl + ']', @TblInt); - EXECUTE (@Txt); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Create Index', @Text = @Txt; - END - DELETE @IndexesRT - WHERE IndId = @IndId; - END - SET @Txt = 'ALTER TABLE dbo.' + @TblInt + ' ADD CHECK (ResourceTypeId >= ' + CONVERT (VARCHAR, @ResourceTypeId) + ' AND ResourceTypeId < ' + CONVERT (VARCHAR, @ResourceTypeId) + ' + 1)'; - EXECUTE (@Txt); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Add check', @Text = @Txt; - SET @Txt = 'ALTER TABLE dbo.' + @Tbl + ' SWITCH PARTITION $partition.PartitionFunction_ResourceTypeId(' + CONVERT (VARCHAR, @ResourceTypeId) + ') TO dbo.' + @TblInt; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Switch out start', @Text = @Txt; - EXECUTE (@Txt); - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Switch out end', @Text = @Txt; - DELETE @ResourceTypes - WHERE ResourceTypeId = @ResourceTypeId; - END - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; - THROW; -END CATCH - -GO -CREATE PROCEDURE dbo.SwitchPartitionsOutAllTables -@RebuildClustered BIT -AS -SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'SwitchPartitionsOutAllTables', @Mode AS VARCHAR (200) = 'PS=PartitionScheme_ResourceTypeId ND=' + isnull(CONVERT (VARCHAR, @RebuildClustered), 'NULL'), @st AS DATETIME = getUTCdate(), @Tbl AS VARCHAR (100); -BEGIN TRY - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; - DECLARE @Tables TABLE ( - name VARCHAR (100) PRIMARY KEY, - supported BIT ); - INSERT INTO @Tables - EXECUTE dbo.GetPartitionedTables @IncludeNotDisabled = @RebuildClustered, @IncludeNotSupported = 0; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Tables', @Action = 'Insert', @Rows = @@rowcount; - WHILE EXISTS (SELECT * - FROM @Tables) - BEGIN - SET @Tbl = (SELECT TOP 1 name - FROM @Tables - ORDER BY name); - EXECUTE dbo.SwitchPartitionsOut @Tbl = @Tbl, @RebuildClustered = @RebuildClustered; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = 'SwitchPartitionsOut', @Action = 'Execute', @Text = @Tbl; - DELETE @Tables - WHERE name = @Tbl; - END - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; -END TRY -BEGIN CATCH - IF error_number() = 1750 - THROW; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; - THROW; -END CATCH - -GO -CREATE PROCEDURE [dbo].[TaskKeepAlive] -@taskId VARCHAR (64), @runId VARCHAR (50) -AS -SET NOCOUNT ON; -SET XACT_ABORT ON; -BEGIN TRANSACTION; -IF NOT EXISTS (SELECT * - FROM [dbo].[TaskInfo] - WHERE TaskId = @taskId - AND RunId = @runId) - BEGIN - THROW 50404, 'Task not exist or runid not match', 1; - END -DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); -UPDATE dbo.TaskInfo -SET HeartbeatDateTime = @heartbeatDateTime -WHERE TaskId = @taskId; -SELECT TaskId, - QueueId, - Status, - TaskTypeId, - RunId, - IsCanceled, - RetryCount, - MaxRetryCount, - HeartbeatDateTime, - InputData, - TaskContext, - Result -FROM [dbo].[TaskInfo] -WHERE TaskId = @taskId; -COMMIT TRANSACTION; - -GO -CREATE OR ALTER PROCEDURE dbo.UpdateEventAgentCheckpoint -@CheckpointId VARCHAR (64), @LastProcessedDateTime DATETIMEOFFSET (7)=NULL, @LastProcessedIdentifier VARCHAR (64)=NULL -AS -BEGIN - IF EXISTS (SELECT * - FROM dbo.EventAgentCheckpoint - WHERE CheckpointId = @CheckpointId) - UPDATE dbo.EventAgentCheckpoint - SET CheckpointId = @CheckpointId, - LastProcessedDateTime = @LastProcessedDateTime, - LastProcessedIdentifier = @LastProcessedIdentifier, - UpdatedOn = sysutcdatetime() - WHERE CheckpointId = @CheckpointId; - ELSE - INSERT INTO dbo.EventAgentCheckpoint (CheckpointId, LastProcessedDateTime, LastProcessedIdentifier, UpdatedOn) - VALUES (@CheckpointId, @LastProcessedDateTime, @LastProcessedIdentifier, sysutcdatetime()); -END - -GO -CREATE PROCEDURE dbo.UpdateExportJob -@id VARCHAR (64), @status VARCHAR (10), @rawJobRecord VARCHAR (MAX), @jobVersion BINARY (8) -AS -SET NOCOUNT ON; -SET XACT_ABORT ON; -BEGIN TRANSACTION; -DECLARE @currentJobVersion AS BINARY (8); -SELECT @currentJobVersion = JobVersion -FROM dbo.ExportJob WITH (UPDLOCK, HOLDLOCK) -WHERE Id = @id; -IF (@currentJobVersion IS NULL) - BEGIN - THROW 50404, 'Export job record not found', 1; - END -IF (@jobVersion <> @currentJobVersion) - BEGIN - THROW 50412, 'Precondition failed', 1; - END -DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); -UPDATE dbo.ExportJob -SET Status = @status, - HeartbeatDateTime = @heartbeatDateTime, - RawJobRecord = @rawJobRecord -WHERE Id = @id; -SELECT @@DBTS; -COMMIT TRANSACTION; - -GO -CREATE PROCEDURE dbo.UpdateReindexJob -@id VARCHAR (64), @status VARCHAR (10), @rawJobRecord VARCHAR (MAX), @jobVersion BINARY (8) -AS -SET NOCOUNT ON; -SET XACT_ABORT ON; -BEGIN TRANSACTION; -DECLARE @currentJobVersion AS BINARY (8); -SELECT @currentJobVersion = JobVersion -FROM dbo.ReindexJob WITH (UPDLOCK, HOLDLOCK) -WHERE Id = @id; -IF (@currentJobVersion IS NULL) - BEGIN - THROW 50404, 'Reindex job record not found', 1; - END -IF (@jobVersion <> @currentJobVersion) - BEGIN - THROW 50412, 'Precondition failed', 1; - END -DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); -UPDATE dbo.ReindexJob -SET Status = @status, - HeartbeatDateTime = @heartbeatDateTime, - RawJobRecord = @rawJobRecord -WHERE Id = @id; -SELECT @@DBTS; -COMMIT TRANSACTION; - -GO -CREATE PROCEDURE dbo.UpdateResourceSearchParams -@FailedResources INT=0 OUTPUT, @Resources dbo.ResourceList READONLY, @ResourceWriteClaims dbo.ResourceWriteClaimList READONLY, @ReferenceSearchParams dbo.ReferenceSearchParamList READONLY, @TokenSearchParams dbo.TokenSearchParamList READONLY, @TokenTexts dbo.TokenTextList READONLY, @StringSearchParams dbo.StringSearchParamList READONLY, @UriSearchParams dbo.UriSearchParamList READONLY, @NumberSearchParams dbo.NumberSearchParamList READONLY, @QuantitySearchParams dbo.QuantitySearchParamList READONLY, @DateTimeSearchParams dbo.DateTimeSearchParamList READONLY, @ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY, @TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY, @TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY, @TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY, @TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY, @TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY -AS -SET NOCOUNT ON; -DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (200) = isnull((SELECT 'RT=[' + CONVERT (VARCHAR, min(ResourceTypeId)) + ',' + CONVERT (VARCHAR, max(ResourceTypeId)) + '] Sur=[' + CONVERT (VARCHAR, min(ResourceSurrogateId)) + ',' + CONVERT (VARCHAR, max(ResourceSurrogateId)) + '] V=' + CONVERT (VARCHAR, max(Version)) + ' Rows=' + CONVERT (VARCHAR, count(*)) - FROM @Resources), 'Input=Empty'), @Rows AS INT; -BEGIN TRY - DECLARE @Ids TABLE ( - ResourceTypeId SMALLINT NOT NULL, - ResourceSurrogateId BIGINT NOT NULL); - BEGIN TRANSACTION; - UPDATE B - SET SearchParamHash = A.SearchParamHash - OUTPUT deleted.ResourceTypeId, deleted.ResourceSurrogateId INTO @Ids - FROM @Resources AS A - INNER JOIN - dbo.Resource AS B - ON B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId - WHERE B.IsHistory = 0; - SET @Rows = @@rowcount; - DELETE B - FROM @Ids AS A - INNER JOIN - dbo.ResourceWriteClaim AS B - ON B.ResourceSurrogateId = A.ResourceSurrogateId; - DELETE B - FROM @Ids AS A - INNER JOIN - dbo.ReferenceSearchParam AS B - ON B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId; - DELETE B - FROM @Ids AS A - INNER JOIN - dbo.TokenSearchParam AS B - ON B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId; - DELETE B - FROM @Ids AS A - INNER JOIN - dbo.TokenText AS B - ON B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId; - DELETE B - FROM @Ids AS A - INNER JOIN - dbo.StringSearchParam AS B - ON B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId; - DELETE B - FROM @Ids AS A - INNER JOIN - dbo.UriSearchParam AS B - ON B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId; - DELETE B - FROM @Ids AS A - INNER JOIN - dbo.NumberSearchParam AS B - ON B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId; - DELETE B - FROM @Ids AS A - INNER JOIN - dbo.QuantitySearchParam AS B - ON B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId; - DELETE B - FROM @Ids AS A - INNER JOIN - dbo.DateTimeSearchParam AS B - ON B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId; - DELETE B - FROM @Ids AS A - INNER JOIN - dbo.ReferenceTokenCompositeSearchParam AS B - ON B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId; - DELETE B - FROM @Ids AS A - INNER JOIN - dbo.TokenTokenCompositeSearchParam AS B - ON B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId; - DELETE B - FROM @Ids AS A - INNER JOIN - dbo.TokenDateTimeCompositeSearchParam AS B - ON B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId; - DELETE B - FROM @Ids AS A - INNER JOIN - dbo.TokenQuantityCompositeSearchParam AS B - ON B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId; - DELETE B - FROM @Ids AS A - INNER JOIN - dbo.TokenStringCompositeSearchParam AS B - ON B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId; - DELETE B - FROM @Ids AS A - INNER JOIN - dbo.TokenNumberNumberCompositeSearchParam AS B - ON B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId; - INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) - SELECT ResourceSurrogateId, - ClaimTypeId, - ClaimValue - FROM @ResourceWriteClaims; - INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - BaseUri, - ReferenceResourceTypeId, - ReferenceResourceId, - ReferenceResourceVersion - FROM @ReferenceSearchParams; - INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - SystemId, - Code, - CodeOverflow - FROM @TokenSearchParams; - INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - Text - FROM @TokenTexts; - INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - Text, - TextOverflow, - IsMin, - IsMax - FROM @StringSearchParams; - INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - Uri - FROM @UriSearchParams; - INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - SingleValue, - LowValue, - HighValue - FROM @NumberSearchParams; - INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - SystemId, - QuantityCodeId, - SingleValue, - LowValue, - HighValue - FROM @QuantitySearchParams; - INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - StartDateTime, - EndDateTime, - IsLongerThanADay, - IsMin, - IsMax - FROM @DateTimeSearchParams; - INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - BaseUri1, - ReferenceResourceTypeId1, - ReferenceResourceId1, - ReferenceResourceVersion1, - SystemId2, - Code2, - CodeOverflow2 - FROM @ReferenceTokenCompositeSearchParams; - INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - SystemId1, - Code1, - CodeOverflow1, - SystemId2, - Code2, - CodeOverflow2 - FROM @TokenTokenCompositeSearchParams; - INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - SystemId1, - Code1, - CodeOverflow1, - StartDateTime2, - EndDateTime2, - IsLongerThanADay2 - FROM @TokenDateTimeCompositeSearchParams; - INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - SystemId1, - Code1, - CodeOverflow1, - SingleValue2, - SystemId2, - QuantityCodeId2, - LowValue2, - HighValue2 - FROM @TokenQuantityCompositeSearchParams; - INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - SystemId1, - Code1, - CodeOverflow1, - Text2, - TextOverflow2 - FROM @TokenStringCompositeSearchParams; - INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange) - SELECT ResourceTypeId, - ResourceSurrogateId, - SearchParamId, - SystemId1, - Code1, - CodeOverflow1, - SingleValue2, - LowValue2, - HighValue2, - SingleValue3, - LowValue3, - HighValue3, - HasRange - FROM @TokenNumberNumberCompositeSearchParams; - COMMIT TRANSACTION; - SET @FailedResources = (SELECT count(*) - FROM @Resources) - @Rows; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; -END TRY -BEGIN CATCH - IF @@trancount > 0 - ROLLBACK; - EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; - THROW; -END CATCH - -GO -CREATE PROCEDURE [dbo].[UpdateTaskContext] -@taskId VARCHAR (64), @taskContext VARCHAR (MAX), @runId VARCHAR (50) -AS -SET NOCOUNT ON; -SET XACT_ABORT ON; -BEGIN TRANSACTION; -IF NOT EXISTS (SELECT * - FROM [dbo].[TaskInfo] - WHERE TaskId = @taskId - AND RunId = @runId) - BEGIN - THROW 50404, 'Task not exist or runid not match', 1; - END -DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); -UPDATE dbo.TaskInfo -SET HeartbeatDateTime = @heartbeatDateTime, - TaskContext = @taskContext -WHERE TaskId = @taskId; -SELECT TaskId, - QueueId, - Status, - TaskTypeId, - RunId, - IsCanceled, - RetryCount, - MaxRetryCount, - HeartbeatDateTime, - InputData, - TaskContext, - Result -FROM [dbo].[TaskInfo] -WHERE TaskId = @taskId; -COMMIT TRANSACTION; - -GO -CREATE PROCEDURE dbo.UpsertResource_7 -@baseResourceSurrogateId BIGINT, @resourceTypeId SMALLINT, @resourceId VARCHAR (64), @eTag INT=NULL, @allowCreate BIT, @isDeleted BIT, @keepHistory BIT, @requireETagOnUpdate BIT, @requestMethod VARCHAR (10), @searchParamHash VARCHAR (64), @rawResource VARBINARY (MAX), @resourceWriteClaims dbo.BulkResourceWriteClaimTableType_1 READONLY, @compartmentAssignments dbo.BulkCompartmentAssignmentTableType_1 READONLY, @referenceSearchParams dbo.BulkReferenceSearchParamTableType_1 READONLY, @tokenSearchParams dbo.BulkTokenSearchParamTableType_2 READONLY, @tokenTextSearchParams dbo.BulkTokenTextTableType_1 READONLY, @stringSearchParams dbo.BulkStringSearchParamTableType_2 READONLY, @numberSearchParams dbo.BulkNumberSearchParamTableType_1 READONLY, @quantitySearchParams dbo.BulkQuantitySearchParamTableType_1 READONLY, @uriSearchParams dbo.BulkUriSearchParamTableType_1 READONLY, @dateTimeSearchParms dbo.BulkDateTimeSearchParamTableType_2 READONLY, @referenceTokenCompositeSearchParams dbo.BulkReferenceTokenCompositeSearchParamTableType_2 READONLY, @tokenTokenCompositeSearchParams dbo.BulkTokenTokenCompositeSearchParamTableType_2 READONLY, @tokenDateTimeCompositeSearchParams dbo.BulkTokenDateTimeCompositeSearchParamTableType_2 READONLY, @tokenQuantityCompositeSearchParams dbo.BulkTokenQuantityCompositeSearchParamTableType_2 READONLY, @tokenStringCompositeSearchParams dbo.BulkTokenStringCompositeSearchParamTableType_2 READONLY, @tokenNumberNumberCompositeSearchParams dbo.BulkTokenNumberNumberCompositeSearchParamTableType_2 READONLY, @isResourceChangeCaptureEnabled BIT=0, @comparedVersion INT=NULL -AS -SET NOCOUNT ON; -SET XACT_ABORT ON; -DECLARE @previousResourceSurrogateId AS BIGINT, @previousVersion AS BIGINT, @previousIsDeleted AS BIT, @version AS INT, @resourceSurrogateId AS BIGINT, @InitialTranCount AS INT = @@trancount; -IF @InitialTranCount = 0 - BEGIN TRANSACTION; -SELECT @previousResourceSurrogateId = ResourceSurrogateId, - @previousVersion = Version, - @previousIsDeleted = IsDeleted -FROM dbo.Resource WITH (UPDLOCK, HOLDLOCK) -WHERE ResourceTypeId = @resourceTypeId - AND ResourceId = @resourceId - AND IsHistory = 0; -IF @previousResourceSurrogateId IS NULL - SET @version = 1; -ELSE - BEGIN - IF @isDeleted = 0 - BEGIN - IF @comparedVersion IS NULL - OR @comparedVersion <> @previousVersion - BEGIN - THROW 50409, 'Resource has been recently updated or added, please compare the resource content in code for any duplicate updates', 1; - END - END - SET @version = @previousVersion + 1; - IF @keepHistory = 1 - UPDATE dbo.Resource - SET IsHistory = 1 - WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @previousResourceSurrogateId; - ELSE - DELETE dbo.Resource - WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @previousResourceSurrogateId; - DELETE dbo.ResourceWriteClaim - WHERE ResourceSurrogateId = @previousResourceSurrogateId; - DELETE dbo.CompartmentAssignment - WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @previousResourceSurrogateId; - DELETE dbo.ReferenceSearchParam - WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @previousResourceSurrogateId; - DELETE dbo.TokenSearchParam - WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @previousResourceSurrogateId; - DELETE dbo.TokenText - WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @previousResourceSurrogateId; - DELETE dbo.StringSearchParam - WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @previousResourceSurrogateId; - DELETE dbo.UriSearchParam - WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @previousResourceSurrogateId; - DELETE dbo.NumberSearchParam - WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @previousResourceSurrogateId; - DELETE dbo.QuantitySearchParam - WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @previousResourceSurrogateId; - DELETE dbo.DateTimeSearchParam - WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @previousResourceSurrogateId; - DELETE dbo.ReferenceTokenCompositeSearchParam - WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @previousResourceSurrogateId; - DELETE dbo.TokenTokenCompositeSearchParam - WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @previousResourceSurrogateId; - DELETE dbo.TokenDateTimeCompositeSearchParam - WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @previousResourceSurrogateId; - DELETE dbo.TokenQuantityCompositeSearchParam - WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @previousResourceSurrogateId; - DELETE dbo.TokenStringCompositeSearchParam - WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @previousResourceSurrogateId; - DELETE dbo.TokenNumberNumberCompositeSearchParam - WHERE ResourceTypeId = @resourceTypeId - AND ResourceSurrogateId = @previousResourceSurrogateId; - END -SET @resourceSurrogateId = @baseResourceSurrogateId + ( NEXT VALUE FOR ResourceSurrogateIdUniquifierSequence); -INSERT INTO dbo.Resource (ResourceTypeId, ResourceId, Version, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, RawResource, IsRawResourceMetaSet, SearchParamHash) -SELECT @resourceTypeId, - @resourceId, - @version, - 0, - @resourceSurrogateId, - @isDeleted, - @requestMethod, - @rawResource, - CASE WHEN @version = 1 THEN 1 ELSE 0 END, - @searchParamHash; -INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) -SELECT @resourceSurrogateId, - ClaimTypeId, - ClaimValue -FROM @resourceWriteClaims; -INSERT INTO dbo.CompartmentAssignment (ResourceTypeId, ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId, IsHistory) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - CompartmentTypeId, - ReferenceResourceId, - 0 -FROM @compartmentAssignments; -INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion, IsHistory) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - BaseUri, - ReferenceResourceTypeId, - ReferenceResourceId, - ReferenceResourceVersion, - 0 -FROM @referenceSearchParams; -INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow, IsHistory) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - SystemId, - Code, - CodeOverflow, - 0 -FROM @tokenSearchParams; -INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, IsHistory) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - Text, - 0 -FROM @tokenTextSearchParams; -INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsHistory, IsMin, IsMax) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - Text, - TextOverflow, - 0, - IsMin, - IsMax -FROM @stringSearchParams; -INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri, IsHistory) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - Uri, - 0 -FROM @uriSearchParams; -INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue, IsHistory) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - SingleValue, - LowValue, - HighValue, - 0 -FROM @numberSearchParams; -INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue, IsHistory) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - SystemId, - QuantityCodeId, - SingleValue, - LowValue, - HighValue, - 0 -FROM @quantitySearchParams; -INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsHistory, IsMin, IsMax) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - StartDateTime, - EndDateTime, - IsLongerThanADay, - 0, - IsMin, - IsMax -FROM @dateTimeSearchParms; -INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2, IsHistory) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - BaseUri1, - ReferenceResourceTypeId1, - ReferenceResourceId1, - ReferenceResourceVersion1, - SystemId2, - Code2, - CodeOverflow2, - 0 -FROM @referenceTokenCompositeSearchParams; -INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2, IsHistory) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - SystemId1, - Code1, - CodeOverflow1, - SystemId2, - Code2, - CodeOverflow2, - 0 -FROM @tokenTokenCompositeSearchParams; -INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2, IsHistory) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - SystemId1, - Code1, - CodeOverflow1, - StartDateTime2, - EndDateTime2, - IsLongerThanADay2, - 0 -FROM @tokenDateTimeCompositeSearchParams; -INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2, IsHistory) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - SystemId1, - Code1, - CodeOverflow1, - SingleValue2, - SystemId2, - QuantityCodeId2, - LowValue2, - HighValue2, - 0 -FROM @tokenQuantityCompositeSearchParams; -INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2, IsHistory) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - SystemId1, - Code1, - CodeOverflow1, - Text2, - TextOverflow2, - 0 -FROM @tokenStringCompositeSearchParams; -INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange, IsHistory) -SELECT DISTINCT @resourceTypeId, - @resourceSurrogateId, - SearchParamId, - SystemId1, - Code1, - CodeOverflow1, - SingleValue2, - LowValue2, - HighValue2, - SingleValue3, - LowValue3, - HighValue3, - HasRange, - 0 -FROM @tokenNumberNumberCompositeSearchParams; -SELECT @version; -IF @isResourceChangeCaptureEnabled = 1 - EXECUTE dbo.CaptureResourceChanges @isDeleted = @isDeleted, @version = @version, @resourceId = @resourceId, @resourceTypeId = @resourceTypeId; -IF @InitialTranCount = 0 - COMMIT TRANSACTION; - -GO -CREATE PROCEDURE dbo.UpsertSearchParams -@searchParams dbo.SearchParamTableType_2 READONLY -AS -SET NOCOUNT ON; -SET XACT_ABORT ON; -SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; -BEGIN TRANSACTION; -DECLARE @lastUpdated AS DATETIMEOFFSET (7) = SYSDATETIMEOFFSET(); -DECLARE @summaryOfChanges TABLE ( - Uri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, - Action VARCHAR (20) NOT NULL); -MERGE INTO dbo.SearchParam WITH (TABLOCKX) - AS target -USING @searchParams AS source ON target.Uri = source.Uri -WHEN MATCHED THEN UPDATE -SET Status = source.Status, - LastUpdated = @lastUpdated, - IsPartiallySupported = source.IsPartiallySupported -WHEN NOT MATCHED BY TARGET THEN INSERT (Uri, Status, LastUpdated, IsPartiallySupported) VALUES (source.Uri, source.Status, @lastUpdated, source.IsPartiallySupported) -OUTPUT source.Uri, $ACTION INTO @summaryOfChanges; -SELECT SearchParamId, - SearchParam.Uri -FROM dbo.SearchParam AS searchParam - INNER JOIN - @summaryOfChanges AS upsertedSearchParam - ON searchParam.Uri = upsertedSearchParam.Uri -WHERE upsertedSearchParam.Action = 'INSERT'; -COMMIT TRANSACTION; - -GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/64.diff.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/65.diff.sql similarity index 100% rename from src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/64.diff.sql rename to src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/65.diff.sql From da05a52d98efc40a0098762e16b789d1892c028f Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 29 Sep 2023 06:36:06 -0700 Subject: [PATCH 27/58] Updated SQL schema version --- .../Features/Schema/SchemaVersion.cs | 1 + .../Features/Schema/SchemaVersionConstants.cs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs index dfe4a32a05..fdd74278b4 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs @@ -74,5 +74,6 @@ public enum SchemaVersion V62 = 62, V63 = 63, V64 = 64, + V65 = 65, } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs index 2a68fa3068..8174a4dc2c 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs @@ -8,7 +8,7 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Schema public static class SchemaVersionConstants { public const int Min = (int)SchemaVersion.V63; - public const int Max = (int)SchemaVersion.V64; + public const int Max = (int)SchemaVersion.V65; public const int MinForUpgrade = (int)SchemaVersion.V60; // this is used for upgrade tests only public const int SearchParameterStatusSchemaVersion = (int)SchemaVersion.V6; public const int SupportForReferencesWithMissingTypeVersion = (int)SchemaVersion.V7; From 4708e4b175be0ba402d8b6a0cd09136b25201e5f Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 29 Sep 2023 09:25:21 -0700 Subject: [PATCH 28/58] Code style cleanup --- .../Operations/Export/ExportJobTask.cs | 12 ++-- .../TransactionCheckWithInitialiScript.sql | 2 +- .../Features/Search/SqlServerSearchService.cs | 2 +- .../Rest/Export/ExportDataTests.cs | 71 ++++++------------- .../Rest/Export/ExportTestHelper.cs | 7 -- 5 files changed, 29 insertions(+), 65 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs index b109bfe4e5..1511525295 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs @@ -169,11 +169,13 @@ public async Task ExecuteAsync(ExportJobRecord exportJobRecord, WeakETag weakETa // from the search result. // As Till is a new property QueuedTime is being used as a backup incase Till doesn't exist in the job record. var tillTime = _exportJobRecord.Till != null ? _exportJobRecord.Till : new PartialDateTime(_exportJobRecord.QueuedTime); - List> queryParametersList = new(); - queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.Count, _exportJobRecord.MaximumNumberOfResourcesPerQuery.ToString(CultureInfo.InvariantCulture))); - queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.LastUpdated, $"le{tillTime.ToString()}")); - queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.IncludeHistory, _exportJobRecord.IncludeHistory.ToString(CultureInfo.InvariantCulture))); - queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.IncludeDeleted, _exportJobRecord.IncludeDeleted.ToString(CultureInfo.InvariantCulture))); + List> queryParametersList = new() + { + Tuple.Create(KnownQueryParameterNames.Count, _exportJobRecord.MaximumNumberOfResourcesPerQuery.ToString(CultureInfo.InvariantCulture)), + Tuple.Create(KnownQueryParameterNames.LastUpdated, $"le{tillTime}"), + Tuple.Create(KnownQueryParameterNames.IncludeHistory, _exportJobRecord.IncludeHistory.ToString(CultureInfo.InvariantCulture)), + Tuple.Create(KnownQueryParameterNames.IncludeDeleted, _exportJobRecord.IncludeDeleted.ToString(CultureInfo.InvariantCulture)), + } if (_exportJobRecord.GlobalEndSurrogateId != null) // no need to check individually as they all should have values if anyone does { diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/TransactionCheckWithInitialiScript.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/TransactionCheckWithInitialiScript.sql index 36f2010660..42e8869131 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/TransactionCheckWithInitialiScript.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/TransactionCheckWithInitialiScript.sql @@ -19,6 +19,6 @@ Go INSERT INTO dbo.SchemaVersion VALUES - (64, 'started') + (65, 'started') Go diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs index 8e1ffad698..5e27471a3a 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs @@ -122,11 +122,11 @@ public SqlServerSearchService( public override async Task SearchAsync(SearchOptions searchOptions, CancellationToken cancellationToken) { SqlSearchOptions sqlSearchOptions = new SqlSearchOptions(searchOptions); - SqlSearchType searchType = sqlSearchOptions.GetSearchTypeFromOptions(); SearchResult searchResult = await SearchImpl(sqlSearchOptions, searchType, null, cancellationToken); int resultCount = searchResult.Results.Count(); + if (!sqlSearchOptions.IsSortWithFilter && searchResult.ContinuationToken == null && resultCount <= sqlSearchOptions.MaxItemCount && diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs index b599c54ac1..26be2fb62b 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs @@ -42,10 +42,8 @@ public ExportDataTests(ExportDataTestFixture fixture, ITestOutputHelper testOutp [InlineData("tag")] public async Task GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAsDataInFhirServer(string parametersKey) { - // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. - + // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally.\ string parameters = "not set"; - bool allowDataFromServerToBeSubsetOfExportData = false; if (parametersKey == "tag") { @@ -54,14 +52,11 @@ public async Task GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAs else if (parametersKey == "since") { var uniqueFixtureResources = string.Join(',', _fixture.TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); - parameters = $"_since={_fixture.TestDataInsertionTime:o}&_type={uniqueFixtureResources}&_isParallel=false"; - allowDataFromServerToBeSubsetOfExportData = true; + parameters = $"_since={_fixture.TestDataInsertionTime:o}&_type={uniqueFixtureResources}"; } // Trigger export request and check for export status - Uri contentLocation = await ExportTestHelper.StartExportAsync( - _fixture.TestFhirClient, - parameters: parameters); + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: parameters) IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); @@ -70,7 +65,7 @@ public async Task GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAs await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResources, dataFromExport, _outputHelper, allowDataFromServerToBeSubsetOfExportData)); + Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResources, dataFromExport, _outputHelper)); } [Theory] @@ -79,9 +74,7 @@ public async Task GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAs public async Task GivenFhirServer_WhenPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer(string parametersKey) { // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. - string parameters = "not set"; - bool allowDataFromServerToBeSubsetOfExportData = false; if (parametersKey == "tag") { @@ -91,14 +84,10 @@ public async Task GivenFhirServer_WhenPatientDataIsExported_ThenExportedDataIsSa { var uniqueFixtureResources = string.Join(',', _fixture.TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); parameters = $"_since={_fixture.TestDataInsertionTime:o}&_type={uniqueFixtureResources}"; - allowDataFromServerToBeSubsetOfExportData = true; } // Trigger export request and check for export status - Uri contentLocation = await ExportTestHelper.StartExportAsync( - _fixture.TestFhirClient, - path: "Patient/", - parameters: parameters); + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(path: "Patient/", parameters: parameters); IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); @@ -107,7 +96,7 @@ public async Task GivenFhirServer_WhenPatientDataIsExported_ThenExportedDataIsSa await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestPatientCompartmentResources, dataFromExport, _outputHelper, allowDataFromServerToBeSubsetOfExportData)); + Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestPatientCompartmentResources, dataFromExport, _outputHelper)); } [Theory] @@ -116,10 +105,8 @@ public async Task GivenFhirServer_WhenPatientDataIsExported_ThenExportedDataIsSa public async Task GivenFhirServer_WhenAllObservationAndPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer(string parametersKey) { // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. - string[] testResorceTypes = { "Observation", "Patient" }; string parameters = "not set"; - bool allowDataFromServerToBeSubsetOfExportData = false; if (parametersKey == "tag") { @@ -128,13 +115,10 @@ public async Task GivenFhirServer_WhenAllObservationAndPatientDataIsExported_The else if (parametersKey == "since") { parameters = $"_since={_fixture.TestDataInsertionTime:o}&_type={string.Join(',', testResorceTypes)}"; - allowDataFromServerToBeSubsetOfExportData = true; } // Trigger export request and check for export status - Uri contentLocation = await ExportTestHelper.StartExportAsync( - _fixture.TestFhirClient, - parameters: parameters); + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: parameters) IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); @@ -147,7 +131,7 @@ public async Task GivenFhirServer_WhenAllObservationAndPatientDataIsExported_The .ToDictionary(x => x.Key, x => x.Value); // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(expectedResources, dataFromExport, _outputHelper, allowDataFromServerToBeSubsetOfExportData)); + Assert.True(ExportTestHelper.ValidateDataFromBothSources(expectedResources, dataFromExport, _outputHelper)); } [Theory] @@ -156,9 +140,7 @@ public async Task GivenFhirServer_WhenAllObservationAndPatientDataIsExported_The public async Task GivenFhirServer_WhenPatientObservationDataIsExported_ThenExportedDataIsSameAsDataInFhirServer(string parametersKey) { // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. - string parameters = "not set"; - bool allowDataFromServerToBeSubsetOfExportData = false; if (parametersKey == "tag") { @@ -167,14 +149,10 @@ public async Task GivenFhirServer_WhenPatientObservationDataIsExported_ThenExpor else if (parametersKey == "since") { parameters = $"_since={_fixture.TestDataInsertionTime:o}&type=Observation"; - allowDataFromServerToBeSubsetOfExportData = true; } // Trigger export request and check for export status - Uri contentLocation = await ExportTestHelper.StartExportAsync( - _fixture.TestFhirClient, - path: "Patient/", - parameters: _fixture.ExportTestFilterQueryParameters("Observation")); + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(path: "Patient/", parameters: _fixture.ExportTestFilterQueryParameters("Observation")); IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); @@ -187,7 +165,7 @@ public async Task GivenFhirServer_WhenPatientObservationDataIsExported_ThenExpor .ToDictionary(x => x.Key, x => x.Value); // Assert both data are equal. Only Observation data is expected due to the type query parameter. - Assert.True(ExportTestHelper.ValidateDataFromBothSources(expectedResources, dataFromExport, _outputHelper, allowDataFromServerToBeSubsetOfExportData)); + Assert.True(ExportTestHelper.ValidateDataFromBothSources(expectedResources, dataFromExport, _outputHelper)); } // No need to test both code paths for testing container is written to. @@ -198,9 +176,7 @@ public async Task GivenFhirServer_WhenAllDataIsExportedToASpecificContainer_Then string testContainer = "test-container"; // Trigger export request and check for export status - Uri contentLocation = await ExportTestHelper.StartExportAsync( - _fixture.TestFhirClient, - parameters: $"_container={testContainer}&{_fixture.ExportTestFilterQueryParameters()}"); + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: $"_container={testContainer}&{_fixture.ExportTestFilterQueryParameters()}"); IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); @@ -220,11 +196,10 @@ public async Task GivenFhirServer_WhenAllDataIsExportedToASpecificContainer_Then public async Task GivenFhirServer_WhenDataIsExportedWithHistory_ThenExportedDataIsSameAsDataInFhirServer(string parallelQueryParam) { // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. + var uniqueFixtureResources = string.Join(',', _fixture.TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. - Uri contentLocation = await ExportTestHelper.StartExportAsync( - _fixture.TestFhirClient, - parameters: $"_since={_fixture.TestDataInsertionTime:O}&_includeHistory=true&{parallelQueryParam}"); + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:O}&_type={uniqueFixtureResources}&_includeHistory=true&{parallelQueryParam}"); IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); @@ -232,12 +207,8 @@ public async Task GivenFhirServer_WhenDataIsExportedWithHistory_ThenExportedData Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); - // removeme - var missing = _fixture.TestResourcesWithHistory.Where(x => !dataFromExport.Any(y => x.Key.resourceId == y.Key.resourceId && x.Key.versionId == y.Key.versionId)); - var missingLastUpdated = missing.Select(x => x.Value.Meta.LastUpdated).ToList(); - // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResourcesWithHistory, dataFromExport, _outputHelper, true)); + Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResourcesWithHistory, dataFromExport, _outputHelper)); } // _tag filter cannot be used with history or deleted export. Using isParallel to test both SQL code paths. @@ -247,11 +218,10 @@ public async Task GivenFhirServer_WhenDataIsExportedWithHistory_ThenExportedData public async Task GivenFhirServer_WhenDataIsExportedWithSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer(string parallelQueryParam) { // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. + var uniqueFixtureResources = string.Join(',', _fixture.TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. - Uri contentLocation = await ExportTestHelper.StartExportAsync( - _fixture.TestFhirClient, - parameters: $"_since={_fixture.TestDataInsertionTime:O}&_includeDeleted=true&{parallelQueryParam}"); + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync($"_since={_fixture.TestDataInsertionTime:O}&_type={uniqueFixtureResources}&_includeDeleted=true&{parallelQueryParam}"); IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); @@ -260,7 +230,7 @@ public async Task GivenFhirServer_WhenDataIsExportedWithSoftDeletes_ThenExported await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResourcesWithDeletes, dataFromExport, _outputHelper, true)); + Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResourcesWithDeletes, dataFromExport, _outputHelper)); } // _tag filter cannot be used with history or deleted export. Using isParallel to test both SQL code paths. @@ -270,11 +240,10 @@ public async Task GivenFhirServer_WhenDataIsExportedWithSoftDeletes_ThenExported public async Task GivenFhirServer_WhenDataIsExportedWithHistoryAndSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer(string parallelQueryParam) { // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. + var uniqueFixtureResources = string.Join(',', _fixture.TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. - Uri contentLocation = await ExportTestHelper.StartExportAsync( - _fixture.TestFhirClient, - parameters: $"_since={_fixture.TestDataInsertionTime:O}&_includeHistory=true&_includeDeleted=true&{parallelQueryParam}"); + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync($"_since={_fixture.TestDataInsertionTime:O}&_type={uniqueFixtureResources}&_includeHistory=true&_includeDeleted=true&{parallelQueryParam}"); IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); @@ -283,7 +252,7 @@ public async Task GivenFhirServer_WhenDataIsExportedWithHistoryAndSoftDeletes_Th await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResourcesWithHistoryAndDeletes, dataFromExport, _outputHelper, true)); + Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResourcesWithHistoryAndDeletes, dataFromExport, _outputHelper)); } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs index 228586d816..d630d7ac55 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs @@ -301,12 +301,5 @@ internal static bool ValidateDataFromBothSources( return result; } - - internal static async Task StartExportAsync(TestFhirClient testFhirClient, string path = "", string parameters = "") - { - Uri contentLocation = await testFhirClient.ExportAsync(path: path, parameters: parameters); - - return contentLocation; - } } } From ccc95cbbad6c9fc01b6604515762f265c88aa4fd Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Wed, 4 Oct 2023 19:37:59 -0700 Subject: [PATCH 29/58] Updated export history/deleted query params --- docs/rest/ExportRequests.http | 2 +- .../ValidateExportRequestFilterAttribute.cs | 3 +- .../Resources.Designer.cs | 29 ++++++++++------ src/Microsoft.Health.Fhir.Api/Resources.resx | 3 ++ .../Features/KnownQueryParameterNames.cs | 5 +++ .../Operations/Export/ExportJobTask.cs | 2 +- .../Controllers/ExportControllerTests.cs | 26 +++++++++++++-- .../Controllers/ExportController.cs | 33 ++++++++++++++++--- .../Rest/Export/ExportDataTests.cs | 4 +-- 9 files changed, 85 insertions(+), 22 deletions(-) diff --git a/docs/rest/ExportRequests.http b/docs/rest/ExportRequests.http index 9d434846cb..9beb5e9c73 100644 --- a/docs/rest/ExportRequests.http +++ b/docs/rest/ExportRequests.http @@ -73,7 +73,7 @@ Authorization: Bearer {{bearer.response.body.access_token}} ### Export with history and soft deleted records # @name export -GET https://{{hostname}}/$export?_includeHistory=true&_includeDeleted=true +GET https://{{hostname}}/$export?includeAssociatedData=_history,_deleted Accept: application/fhir+json Prefer: respond-async Authorization: Bearer {{bearer.response.body.access_token}} diff --git a/src/Microsoft.Health.Fhir.Api/Features/Filters/ValidateExportRequestFilterAttribute.cs b/src/Microsoft.Health.Fhir.Api/Features/Filters/ValidateExportRequestFilterAttribute.cs index 6e96abe8e5..c379854f80 100644 --- a/src/Microsoft.Health.Fhir.Api/Features/Filters/ValidateExportRequestFilterAttribute.cs +++ b/src/Microsoft.Health.Fhir.Api/Features/Filters/ValidateExportRequestFilterAttribute.cs @@ -45,8 +45,7 @@ public ValidateExportRequestFilterAttribute() KnownQueryParameterNames.Format, KnownQueryParameterNames.TypeFilter, KnownQueryParameterNames.IsParallel, - KnownQueryParameterNames.IncludeHistory, - KnownQueryParameterNames.IncludeDeleted, + KnownQueryParameterNames.IncludeAssociatedData, KnownQueryParameterNames.AnonymizationConfigurationCollectionReference, KnownQueryParameterNames.AnonymizationConfigurationLocation, KnownQueryParameterNames.AnonymizationConfigurationFileEtag, diff --git a/src/Microsoft.Health.Fhir.Api/Resources.Designer.cs b/src/Microsoft.Health.Fhir.Api/Resources.Designer.cs index 987404cd81..d47d1321ad 100644 --- a/src/Microsoft.Health.Fhir.Api/Resources.Designer.cs +++ b/src/Microsoft.Health.Fhir.Api/Resources.Designer.cs @@ -383,7 +383,16 @@ public static string InvalidElementsParameter { return ResourceManager.GetString("InvalidElementsParameter", resourceCulture); } } - + + /// + /// Looks up a localized string similar to invalid parametr given for export associated data. + /// + public static string InvalidExportAssociatedDataParameter { + get { + return ResourceManager.GetString("InvalidExportAssociatedDataParameter", resourceCulture); + } + } + /// /// Looks up a localized string similar to Invalid launch context parameters.. /// @@ -708,6 +717,15 @@ public static string TransactionFailed { } } + /// + /// Looks up a localized string similar to The request "_typeFilter" cannot be used with an export request with historical or soft deleted resources.. + /// + public static string TypeFilterNotSupportedWithHistoryOrDeletedExport { + get { + return ResourceManager.GetString("TypeFilterNotSupportedWithHistoryOrDeletedExport", resourceCulture); + } + } + /// /// Looks up a localized string similar to The _type parameter must be included when using the _typeFilter parameter. . /// @@ -806,14 +824,5 @@ public static string WelcomeTitle { return ResourceManager.GetString("WelcomeTitle", resourceCulture); } } - - /// - /// Looks up a localized string similar to The request "_typeFilter" cannot be used with an export request with historical or soft deleted resources. - /// - public static string TypeFilterNotSupportedWithHistoryOrDeletedExport { - get { - return ResourceManager.GetString("TypeFilterNotSupportedWithHistoryOrDeletedExport", resourceCulture); - } - } } } diff --git a/src/Microsoft.Health.Fhir.Api/Resources.resx b/src/Microsoft.Health.Fhir.Api/Resources.resx index f345ac9dad..edc916b613 100644 --- a/src/Microsoft.Health.Fhir.Api/Resources.resx +++ b/src/Microsoft.Health.Fhir.Api/Resources.resx @@ -411,4 +411,7 @@ The request "_typeFilter" cannot be used with an export request with historical or soft deleted resources. + + The export parameter "associatedData" contains an invalid parameter. + \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.Core/Features/KnownQueryParameterNames.cs b/src/Microsoft.Health.Fhir.Core/Features/KnownQueryParameterNames.cs index 9f299cf61f..fa3a41ffc9 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/KnownQueryParameterNames.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/KnownQueryParameterNames.cs @@ -102,5 +102,10 @@ public static class KnownQueryParameterNames public const string IncludeHistory = "_includeHistory"; public const string IncludeDeleted = "_includeDeleted"; + + /// + /// Used by $export as a comma-separated list of parameters instructing which initial data should be included. + /// + public const string IncludeAssociatedData = "includeAssociatedData"; } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs index 1511525295..1e73c7e9c8 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs @@ -175,7 +175,7 @@ public async Task ExecuteAsync(ExportJobRecord exportJobRecord, WeakETag weakETa Tuple.Create(KnownQueryParameterNames.LastUpdated, $"le{tillTime}"), Tuple.Create(KnownQueryParameterNames.IncludeHistory, _exportJobRecord.IncludeHistory.ToString(CultureInfo.InvariantCulture)), Tuple.Create(KnownQueryParameterNames.IncludeDeleted, _exportJobRecord.IncludeDeleted.ToString(CultureInfo.InvariantCulture)), - } + }; if (_exportJobRecord.GlobalEndSurrogateId != null) // no need to check individually as they all should have values if anyone does { diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/ExportControllerTests.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/ExportControllerTests.cs index b678860e43..873052ef17 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/ExportControllerTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/ExportControllerTests.cs @@ -216,7 +216,7 @@ await Assert.ThrowsAsync(() => _exportEnabledControlle containerName: null, formatName: null, typeFilter: "Patient%3Factive%3Dtrue", - includeHistory: true)); + includeAssociatedData: "_history")); await Assert.ThrowsAsync(() => _exportEnabledController.Export( since: null, @@ -225,7 +225,29 @@ await Assert.ThrowsAsync(() => _exportEnabledControlle containerName: null, formatName: null, typeFilter: "Patient%3Factive%3Dtrue", - includeDeleted: true)); + includeAssociatedData: "_deleted")); + } + + [Fact] + public async Task GivenAnExportRequestWithDataIncluded_WhenKeyIsInvalid_ThenRequestNotValidExceptionShouldBeThrown() + { + await Assert.ThrowsAsync(() => _exportEnabledController.Export( + since: null, + till: null, + resourceType: null, + containerName: null, + formatName: null, + typeFilter: null, + includeAssociatedData: "_test")); + + await Assert.ThrowsAsync(() => _exportEnabledController.Export( + since: null, + till: null, + resourceType: null, + containerName: null, + formatName: null, + typeFilter: null, + includeAssociatedData: ",")); } // We can configure OciArtifacts through three fields: LoginServer, ImageName and Digest diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs b/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs index 506241b9f2..97d7cdb311 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs @@ -4,11 +4,13 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Collections.Generic; using System.Linq; using System.Net; using System.Threading.Tasks; using EnsureThat; using Hl7.Fhir.Model; +using ICSharpCode.SharpZipLib; using MediatR; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; @@ -32,6 +34,7 @@ using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.TemplateManagement.Models; using Microsoft.Health.Fhir.ValueSets; +using static ICSharpCode.SharpZipLib.Zip.ExtendedUnixData; namespace Microsoft.Health.Fhir.Api.Controllers { @@ -97,15 +100,14 @@ public async Task Export( [FromQuery(Name = KnownQueryParameterNames.TypeFilter)] string typeFilter, [FromQuery(Name = KnownQueryParameterNames.Format)] string formatName, [FromQuery(Name = KnownQueryParameterNames.IsParallel)] bool isParallel = true, - [FromQuery(Name = KnownQueryParameterNames.IncludeDeleted)] bool includeDeleted = false, - [FromQuery(Name = KnownQueryParameterNames.IncludeHistory)] bool includeHistory = false, + [FromQuery(Name = KnownQueryParameterNames.IncludeAssociatedData)] string includeAssociatedData = null, [FromQuery(Name = KnownQueryParameterNames.AnonymizationConfigurationCollectionReference)] string anonymizationConfigCollectionReference = null, [FromQuery(Name = KnownQueryParameterNames.AnonymizationConfigurationLocation)] string anonymizationConfigLocation = null, [FromQuery(Name = KnownQueryParameterNames.AnonymizationConfigurationFileEtag)] string anonymizationConfigFileETag = null) { CheckIfExportIsEnabled(); ValidateForAnonymizedExport(containerName, anonymizationConfigCollectionReference, anonymizationConfigLocation, anonymizationConfigFileETag); - ValidateForHistoryOrSoftDeletedExport(includeHistory, includeDeleted, typeFilter); + (bool includeHistory, bool includeDeleted) = ValidateAndParseIncludeAssociatedData(includeAssociatedData, typeFilter); return await SendExportRequest( exportType: ExportJobType.All, @@ -354,8 +356,29 @@ private void CheckIfConfigCollectionReferenceIsConfigured(string anonymizationCo } } - private static void ValidateForHistoryOrSoftDeletedExport(bool includeHistory, bool includeDeleted, string typeFilter) + private static (bool includeHistory, bool includeDeleted) ValidateAndParseIncludeAssociatedData(string associatedDataa, string typeFilter) { + var associatedDataParams = associatedDataa.Split(',', StringSplitOptions.RemoveEmptyEntries); + + bool includeHistory = false; + bool includeDeleted = false; + + foreach (var associatedDataParam in associatedDataParams) + { + if (associatedDataParam == "_deleted") + { + includeDeleted = true; + } + else if (associatedDataParam == "_history") + { + includeHistory = true; + } + else + { + throw new RequestNotValidException(string.Format(Resources.InvalidExportAssociatedDataParameter, associatedDataParam)); + } + } + if (includeHistory || includeDeleted) { if (!string.IsNullOrWhiteSpace(typeFilter)) @@ -363,6 +386,8 @@ private static void ValidateForHistoryOrSoftDeletedExport(bool includeHistory, b throw new RequestNotValidException(Resources.TypeFilterNotSupportedWithHistoryOrDeletedExport); } } + + return (includeHistory, includeDeleted); } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs index 26be2fb62b..ac231f806c 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs @@ -56,7 +56,7 @@ public async Task GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAs } // Trigger export request and check for export status - Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: parameters) + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: parameters); IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); @@ -118,7 +118,7 @@ public async Task GivenFhirServer_WhenAllObservationAndPatientDataIsExported_The } // Trigger export request and check for export status - Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: parameters) + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: parameters); IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); From 224d153bbe84180e0a1fbd92a4a72d01526656fa Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 5 Oct 2023 05:42:10 -0700 Subject: [PATCH 30/58] Fixed export included data test --- .../Controllers/ExportControllerTests.cs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/ExportControllerTests.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/ExportControllerTests.cs index 873052ef17..4ff9a7a4b6 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/ExportControllerTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/ExportControllerTests.cs @@ -239,15 +239,6 @@ await Assert.ThrowsAsync(() => _exportEnabledControlle formatName: null, typeFilter: null, includeAssociatedData: "_test")); - - await Assert.ThrowsAsync(() => _exportEnabledController.Export( - since: null, - till: null, - resourceType: null, - containerName: null, - formatName: null, - typeFilter: null, - includeAssociatedData: ",")); } // We can configure OciArtifacts through three fields: LoginServer, ImageName and Digest From 8cb91acbd223e290ebbfc04e973c0e4f552a03d4 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 6 Oct 2023 11:13:02 -0700 Subject: [PATCH 31/58] Changed SQL exporter to use export configuration vs magic numbers in code --- .../Configs/ExportJobConfiguration.cs | 10 ++++++++++ .../Export/ExportOrchestratorJobTests.cs | 11 +++++++---- .../Export/SqlExportOrchestratorJob.cs | 16 +++++++++++----- .../Operations/Export/SqlServerExportTests.cs | 10 +++++++--- 4 files changed, 35 insertions(+), 12 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core/Configs/ExportJobConfiguration.cs b/src/Microsoft.Health.Fhir.Core/Configs/ExportJobConfiguration.cs index aa3a55dcad..b1648525dc 100644 --- a/src/Microsoft.Health.Fhir.Core/Configs/ExportJobConfiguration.cs +++ b/src/Microsoft.Health.Fhir.Core/Configs/ExportJobConfiguration.cs @@ -42,6 +42,16 @@ public class ExportJobConfiguration /// public uint MaximumNumberOfResourcesPerQuery { get; set; } = 10000; + /// + /// For SQL export, controlls the number of parallel id ranges to gather to be used for parallel export. + /// + public int NumberOfParallelRecordRanges { get; set; } = 100; + + /// + /// For SQL export, controlls the DOP (degree of parallelization) used by the coordinator to build sub-jobs. + /// + public int CoordinatorMaxDegreeOfParallelization { get; set; } = 4; + /// /// Number of pages to be iterated before committing the export progress. /// diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Operations/Export/ExportOrchestratorJobTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Operations/Export/ExportOrchestratorJobTests.cs index 46edd2c78f..47ca591812 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Operations/Export/ExportOrchestratorJobTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Operations/Export/ExportOrchestratorJobTests.cs @@ -8,7 +8,9 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Options; using Microsoft.Health.Core.Extensions; +using Microsoft.Health.Fhir.Core.Configs; using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.Core.Features.Operations.Export; using Microsoft.Health.Fhir.Core.Features.Operations.Export.Models; @@ -30,6 +32,7 @@ public class ExportOrchestratorJobTests { private ISearchService _mockSearchService = Substitute.For(); private IQueueClient _mockQueueClient = Substitute.For(); + private IOptions _exportJobConfiguration = Options.Create(new ExportJobConfiguration()); [Theory] [InlineData(ExportJobType.Patient)] @@ -42,7 +45,7 @@ public async Task GivenANonSystemLevelExportJob_WhenRun_ThenOneProcessingJobShou SetupMockQueue(numExpectedJobs, orchestratorJobId); var orchestratorJob = GetJobInfoArray(0, orchestratorJobId, false, orchestratorJobId, isParallel: true, exportJobType: exportJobType)[0]; - var exportOrchestratorJob = new SqlExportOrchestratorJob(_mockQueueClient, _mockSearchService); + var exportOrchestratorJob = new SqlExportOrchestratorJob(_mockQueueClient, _mockSearchService, _exportJobConfiguration); var result = await exportOrchestratorJob.ExecuteAsync(orchestratorJob, new Progress((result) => { }), CancellationToken.None); var jobResult = JsonConvert.DeserializeObject(result); @@ -58,7 +61,7 @@ public async Task GivenAnExportJobWithIsParallelSetToFalse_WhenRun_ThenOneProces SetupMockQueue(numExpectedJobs, orchestratorJobId); var orchestratorJob = GetJobInfoArray(0, orchestratorJobId, false, orchestratorJobId, isParallel: false).First(); - var exportOrchestratorJob = new SqlExportOrchestratorJob(_mockQueueClient, _mockSearchService); + var exportOrchestratorJob = new SqlExportOrchestratorJob(_mockQueueClient, _mockSearchService, _exportJobConfiguration); var result = await exportOrchestratorJob.ExecuteAsync(orchestratorJob, new Progress((result) => { }), CancellationToken.None); var jobResult = JsonConvert.DeserializeObject(result); Assert.Equal(OperationStatus.Completed, jobResult.Status); @@ -75,7 +78,7 @@ public async Task GivenAnExportJobWithNoTypeRestriction_WhenRun_ThenMultipleProc SetupMockQueue(numExpectedJobsPerResourceType, orchestratorJobId); var orchestratorJob = GetJobInfoArray(0, orchestratorJobId, false, orchestratorJobId, isParallel: true).First(); - var exportOrchestratorJob = new SqlExportOrchestratorJob(_mockQueueClient, _mockSearchService); + var exportOrchestratorJob = new SqlExportOrchestratorJob(_mockQueueClient, _mockSearchService, _exportJobConfiguration); var result = await exportOrchestratorJob.ExecuteAsync(orchestratorJob, new Progress((result) => { }), CancellationToken.None); var jobResult = JsonConvert.DeserializeObject(result); Assert.Equal(OperationStatus.Completed, jobResult.Status); @@ -92,7 +95,7 @@ public async Task GivenAnExportJobWithTypeRestrictions_WhenRun_ThenProcessingJob SetupMockQueue(numExpectedJobs, orchestratorJobId); JobInfo orchestratorJob = GetJobInfoArray(0, orchestratorJobId, false, orchestratorJobId, isParallel: true, typeFilter: "Patient,Observation").First(); - var exportOrchestratorJob = new SqlExportOrchestratorJob(_mockQueueClient, _mockSearchService); + var exportOrchestratorJob = new SqlExportOrchestratorJob(_mockQueueClient, _mockSearchService, _exportJobConfiguration); string result = await exportOrchestratorJob.ExecuteAsync(orchestratorJob, new Progress(_ => { }), CancellationToken.None); ExportJobRecord jobResult = JsonConvert.DeserializeObject(result); Assert.Equal(OperationStatus.Completed, jobResult.Status); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/Export/SqlExportOrchestratorJob.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/Export/SqlExportOrchestratorJob.cs index 0e3409d1ac..3b0b853ec9 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/Export/SqlExportOrchestratorJob.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/Export/SqlExportOrchestratorJob.cs @@ -9,6 +9,8 @@ using System.Threading; using System.Threading.Tasks; using EnsureThat; +using Microsoft.Extensions.Options; +using Microsoft.Health.Fhir.Core.Configs; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.Core.Features.Operations.Export; @@ -23,23 +25,27 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Operations.Export [JobTypeId((int)JobType.ExportOrchestrator)] public class SqlExportOrchestratorJob : IJob { - private const int DefaultNumberOfSurrogateIdRanges = 100; - private IQueueClient _queueClient; private ISearchService _searchService; + private readonly ExportJobConfiguration _exportJobConfiguration; public SqlExportOrchestratorJob( IQueueClient queueClient, - ISearchService searchService) + ISearchService searchService, + IOptions exportJobConfiguration) { EnsureArg.IsNotNull(queueClient, nameof(queueClient)); EnsureArg.IsNotNull(searchService, nameof(searchService)); + EnsureArg.IsNotNull(exportJobConfiguration, nameof(exportJobConfiguration)); _queueClient = queueClient; _searchService = searchService; + _exportJobConfiguration = exportJobConfiguration.Value; + + NumberOfSurrogateIdRanges = _exportJobConfiguration.NumberOfParallelRecordRanges; } - internal int NumberOfSurrogateIdRanges { get; set; } = DefaultNumberOfSurrogateIdRanges; + internal int NumberOfSurrogateIdRanges { get; set; } public async Task ExecuteAsync(JobInfo jobInfo, IProgress progress, CancellationToken cancellationToken) { @@ -73,7 +79,7 @@ public async Task ExecuteAsync(JobInfo jobInfo, IProgress progre .GroupBy(x => x.ResourceType) .ToDictionary(x => x.Key, x => x.Max(r => long.Parse(r.EndSurrogateId))); - await Parallel.ForEachAsync(resourceTypes, new ParallelOptions { MaxDegreeOfParallelism = 4, CancellationToken = cancellationToken }, async (type, cancel) => + await Parallel.ForEachAsync(resourceTypes, new ParallelOptions { MaxDegreeOfParallelism = _exportJobConfiguration.CoordinatorMaxDegreeOfParallelization, CancellationToken = cancellationToken }, async (type, cancel) => { var startId = globalStartId; if (enqueued.TryGetValue(type, out var max)) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/SqlServerExportTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/SqlServerExportTests.cs index 4cfb725478..a6aee70ae9 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/SqlServerExportTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/SqlServerExportTests.cs @@ -9,6 +9,8 @@ using System.Threading.Tasks; using Microsoft.Data.SqlClient; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Microsoft.Health.Fhir.Core.Configs; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.Core.Features.Operations.Export; @@ -36,6 +38,10 @@ public class SqlServerExportTests : IClassFixture _exportJobConfiguration = Options.Create(new ExportJobConfiguration() { NumberOfParallelRecordRanges = 5 }); + public SqlServerExportTests(SqlServerFhirStorageTestsFixture fixture, ITestOutputHelper testOutputHelper) { _fixture = fixture; @@ -52,9 +58,7 @@ public async Task ExportWorkRegistration() { PrepareData(); // 1000 patients + 1000 observations + 1000 claims. !!! RawResource is invalid. - var coordJob = new SqlExportOrchestratorJob(_queueClient, _searchService); - //// surrogate id range size is set via max number of resources per query on coord record - coordJob.NumberOfSurrogateIdRanges = 5; // 100*5=500 is 50% of 1000, so there are 2 insert transactions in JobQueue per each resource type + var coordJob = new SqlExportOrchestratorJob(_queueClient, _searchService, _exportJobConfiguration); await RunExport(null, coordJob, 31, 6); // 31=coord+3*1000/SurrogateIdRangeSize 6=coord+100*5/SurrogateIdRangeSize From b156374934dc2389ba43814835c1973c4346e6ba Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 12 Oct 2023 09:26:22 -0700 Subject: [PATCH 32/58] Rolling back unneeded SQL changes --- .../Features/Schema/SchemaVersion.cs | 3 +-- .../Features/Schema/SchemaVersionConstants.cs | 2 +- .../TransactionCheckWithInitialiScript.sql | 2 +- .../GetResourcesByTypeAndSurrogateIdRange.sql | 19 +++++++------------ 4 files changed, 10 insertions(+), 16 deletions(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs index fdd74278b4..a35062b9ae 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs @@ -73,7 +73,6 @@ public enum SchemaVersion V61 = 61, V62 = 62, V63 = 63, - V64 = 64, - V65 = 65, + V64 = 64 } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs index 8174a4dc2c..2a68fa3068 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs @@ -8,7 +8,7 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Schema public static class SchemaVersionConstants { public const int Min = (int)SchemaVersion.V63; - public const int Max = (int)SchemaVersion.V65; + public const int Max = (int)SchemaVersion.V64; public const int MinForUpgrade = (int)SchemaVersion.V60; // this is used for upgrade tests only public const int SearchParameterStatusSchemaVersion = (int)SchemaVersion.V6; public const int SupportForReferencesWithMissingTypeVersion = (int)SchemaVersion.V7; diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/TransactionCheckWithInitialiScript.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/TransactionCheckWithInitialiScript.sql index 42e8869131..36f2010660 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/TransactionCheckWithInitialiScript.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/TransactionCheckWithInitialiScript.sql @@ -19,6 +19,6 @@ Go INSERT INTO dbo.SchemaVersion VALUES - (65, 'started') + (64, 'started') Go diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetResourcesByTypeAndSurrogateIdRange.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetResourcesByTypeAndSurrogateIdRange.sql index 6847aa6b56..ebdbcd413c 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetResourcesByTypeAndSurrogateIdRange.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetResourcesByTypeAndSurrogateIdRange.sql @@ -1,6 +1,6 @@ --DROP PROCEDURE dbo.GetResourcesByTypeAndSurrogateIdRange GO -CREATE PROCEDURE dbo.GetResourcesByTypeAndSurrogateIdRange @ResourceTypeId smallint, @StartId bigint, @EndId bigint, @GlobalStartId bigint = NULL, @GlobalEndId bigint = NULL, @IncludeHistory bit = 0, @IncludeDeleted bit = 0 +CREATE PROCEDURE dbo.GetResourcesByTypeAndSurrogateIdRange @ResourceTypeId smallint, @StartId bigint, @EndId bigint, @GlobalStartId bigint = NULL, @GlobalEndId bigint = NULL AS set nocount on DECLARE @SP varchar(100) = 'GetResourcesByTypeAndSurrogateIdRange' @@ -9,16 +9,12 @@ DECLARE @SP varchar(100) = 'GetResourcesByTypeAndSurrogateIdRange' +' E='+isnull(convert(varchar,@EndId),'NULL') +' GS='+isnull(convert(varchar,@GlobalStartId),'NULL') -- Is global start id needed? I'm not seeing a usecase for setting it. +' GE='+isnull(convert(varchar,@GlobalEndId),'NULL') -- Could this just be a boolean for if historical records should be returned? GlobalEndId should equal EndId in all cases I can think of. - +' HI='+isnull(convert(varchar,@IncludeHistory),'NULL') - +' DE'+isnull(convert(varchar,@IncludeDeleted),'NULL') ,@st datetime = getUTCdate() BEGIN TRY DECLARE @ResourceIds TABLE (ResourceId varchar(64) COLLATE Latin1_General_100_CS_AS, ResourceSurrogateId bigint, RowId int, PRIMARY KEY (ResourceId, RowId)) - IF @GlobalStartId IS NULL -- export from time zero (no lower boundary) SET @GlobalStartId = 0 - IF @GlobalEndId IS NOT NULL -- snapshot view INSERT INTO @ResourceIds SELECT ResourceId, ResourceSurrogateId, RowId = row_number() OVER (PARTITION BY ResourceId ORDER BY ResourceSurrogateId DESC) @@ -29,17 +25,16 @@ BEGIN TRY WHERE ResourceTypeId = @ResourceTypeId AND ResourceSurrogateId BETWEEN @StartId AND @EndId AND IsHistory = 1 - AND (IsDeleted = 0 OR @IncludeDeleted = 1) -- TBD if this is needed. + AND IsDeleted = 0 ) AND ResourceSurrogateId BETWEEN @GlobalStartId AND @GlobalEndId - + IF EXISTS (SELECT * FROM @ResourceIds) BEGIN DECLARE @SurrogateIdMap TABLE (MaxSurrogateId bigint PRIMARY KEY) INSERT INTO @SurrogateIdMap SELECT MaxSurrogateId = A.ResourceSurrogateId FROM (SELECT * FROM @ResourceIds WHERE RowId = 1 AND ResourceSurrogateId BETWEEN @StartId AND @EndId) A - SELECT @ResourceTypeId ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.ResourceId ELSE A.ResourceId END ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.Version ELSE A.Version END @@ -56,16 +51,16 @@ BEGIN TRY LEFT OUTER JOIN dbo.Resource C ON C.ResourceTypeId = @ResourceTypeId AND C.ResourceSurrogateId = MaxSurrogateId WHERE A.ResourceTypeId = @ResourceTypeId AND A.ResourceSurrogateId BETWEEN @StartId AND @EndId - AND (A.IsHistory = 0 OR MaxSurrogateId IS NOT NULL OR @IncludeHistory = 1) - AND (A.IsDeleted = 0 OR @IncludeDeleted = 1) + AND (A.IsHistory = 0 OR MaxSurrogateId IS NOT NULL) + AND A.IsDeleted = 0 END ELSE SELECT ResourceTypeId, ResourceId, Version, IsDeleted, ResourceSurrogateId, RequestMethod, IsMatch = convert(bit,1), IsPartial = convert(bit,0), IsRawResourceMetaSet, SearchParamHash, RawResource FROM dbo.Resource WHERE ResourceTypeId = @ResourceTypeId AND ResourceSurrogateId BETWEEN @StartId AND @EndId - AND (IsHistory = 0 OR @IncludeHistory = 1) - AND (IsDeleted = 0 OR @IncludeDeleted = 1) + AND IsHistory = 0 + AND IsDeleted = 0 EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@st,@Rows=@@rowcount END TRY From 7233c58687c8fc5eff4e8660242ba78289691265 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 12 Oct 2023 12:42:36 -0700 Subject: [PATCH 33/58] Fixed parallel export with history/soft delete --- src/Microsoft.Health.Fhir.Api/Resources.resx | 2 +- .../Controllers/ExportController.cs | 34 +- .../Features/Schema/Migrations/65.diff.sql | 104 +- .../Features/Schema/Migrations/65.sql | 6739 +++++++++++++++++ .../Features/Schema/SchemaVersion.cs | 3 +- .../Features/Schema/SchemaVersionConstants.cs | 2 +- .../TransactionCheckWithInitialiScript.sql | 2 +- .../GetResourcesByTypeAndSurrogateIdRange.sql | 88 +- .../Features/Search/SqlServerSearchService.cs | 1 - .../Microsoft.Health.Fhir.SqlServer.csproj | 2 +- .../Rest/Export/ExportDataTests.cs | 6 +- 11 files changed, 6855 insertions(+), 128 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/65.sql diff --git a/src/Microsoft.Health.Fhir.Api/Resources.resx b/src/Microsoft.Health.Fhir.Api/Resources.resx index edc916b613..5b9e0f61ec 100644 --- a/src/Microsoft.Health.Fhir.Api/Resources.resx +++ b/src/Microsoft.Health.Fhir.Api/Resources.resx @@ -412,6 +412,6 @@ The request "_typeFilter" cannot be used with an export request with historical or soft deleted resources. - The export parameter "associatedData" contains an invalid parameter. + The export parameter "includeAssociatedData" contains an invalid parameter. \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs b/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs index 97d7cdb311..175b1b28f1 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs @@ -356,35 +356,23 @@ private void CheckIfConfigCollectionReferenceIsConfigured(string anonymizationCo } } - private static (bool includeHistory, bool includeDeleted) ValidateAndParseIncludeAssociatedData(string associatedDataa, string typeFilter) + private static (bool includeHistory, bool includeDeleted) ValidateAndParseIncludeAssociatedData(string associatedData, string typeFilter) { - var associatedDataParams = associatedDataa.Split(',', StringSplitOptions.RemoveEmptyEntries); + var associatedDataParams = (associatedData ?? string.Empty).Split(',', StringSplitOptions.RemoveEmptyEntries).ToList(); + var possibleParams = new List { "_history", "_deleted" }; + var invalidParams = associatedDataParams.Where(param => !possibleParams.Contains(param)).ToList(); - bool includeHistory = false; - bool includeDeleted = false; - - foreach (var associatedDataParam in associatedDataParams) + if (invalidParams.Any()) { - if (associatedDataParam == "_deleted") - { - includeDeleted = true; - } - else if (associatedDataParam == "_history") - { - includeHistory = true; - } - else - { - throw new RequestNotValidException(string.Format(Resources.InvalidExportAssociatedDataParameter, associatedDataParam)); - } + throw new RequestNotValidException(string.Format(Resources.InvalidExportAssociatedDataParameter, string.Join(',', invalidParams))); } - if (includeHistory || includeDeleted) + bool includeHistory = associatedDataParams.Contains("_history"); + bool includeDeleted = associatedDataParams.Contains("_deleted"); + + if ((includeHistory || includeDeleted) && !string.IsNullOrWhiteSpace(typeFilter)) { - if (!string.IsNullOrWhiteSpace(typeFilter)) - { - throw new RequestNotValidException(Resources.TypeFilterNotSupportedWithHistoryOrDeletedExport); - } + throw new RequestNotValidException(Resources.TypeFilterNotSupportedWithHistoryOrDeletedExport); } return (includeHistory, includeDeleted); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/65.diff.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/65.diff.sql index 196fdf6905..947ba59e22 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/65.diff.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/65.diff.sql @@ -1,69 +1,58 @@ -CREATE OR ALTER PROCEDURE dbo.GetResourcesByTypeAndSurrogateIdRange @ResourceTypeId smallint, @StartId bigint, @EndId bigint, @GlobalStartId bigint = NULL, @GlobalEndId bigint = NULL, @IncludeHistory bit = 0, @IncludeDeleted bit = 0 +CREATE OR ALTER PROCEDURE dbo.GetResourcesByTypeAndSurrogateIdRange @ResourceTypeId smallint, @StartId bigint, @EndId bigint, @GlobalEndId bigint = NULL, @IncludeHistory bit = 0, @IncludeDeleted bit = 0 AS set nocount on DECLARE @SP varchar(100) = 'GetResourcesByTypeAndSurrogateIdRange' ,@Mode varchar(100) = 'RT='+isnull(convert(varchar,@ResourceTypeId),'NULL') +' S='+isnull(convert(varchar,@StartId),'NULL') +' E='+isnull(convert(varchar,@EndId),'NULL') - +' GS='+isnull(convert(varchar,@GlobalStartId),'NULL') -- Is global start id needed? I'm not seeing a usecase for setting it. +' GE='+isnull(convert(varchar,@GlobalEndId),'NULL') -- Could this just be a boolean for if historical records should be returned? GlobalEndId should equal EndId in all cases I can think of. +' HI='+isnull(convert(varchar,@IncludeHistory),'NULL') +' DE'+isnull(convert(varchar,@IncludeDeleted),'NULL') ,@st datetime = getUTCdate() + ,@DummyTop bigint = 9223372036854775807 BEGIN TRY - DECLARE @ResourceIds TABLE (ResourceId varchar(64) COLLATE Latin1_General_100_CS_AS, ResourceSurrogateId bigint, RowId int, PRIMARY KEY (ResourceId, RowId)) + DECLARE @ResourceIds TABLE (ResourceId varchar(64) COLLATE Latin1_General_100_CS_AS PRIMARY KEY) + DECLARE @SurrogateIds TABLE (MaxSurrogateId bigint PRIMARY KEY) - IF @GlobalStartId IS NULL -- export from time zero (no lower boundary) - SET @GlobalStartId = 0 - - IF @GlobalEndId IS NOT NULL -- snapshot view + IF @GlobalEndId IS NOT NULL AND @IncludeHistory = 0 -- snapshot view + BEGIN INSERT INTO @ResourceIds - SELECT ResourceId, ResourceSurrogateId, RowId = row_number() OVER (PARTITION BY ResourceId ORDER BY ResourceSurrogateId DESC) + SELECT DISTINCT ResourceId FROM dbo.Resource - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceId IN (SELECT DISTINCT ResourceId - FROM dbo.Resource - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId BETWEEN @StartId AND @EndId - AND IsHistory = 1 - AND (IsDeleted = 0 OR @IncludeDeleted = 1) -- TBD if this is needed. - ) - AND ResourceSurrogateId BETWEEN @GlobalStartId AND @GlobalEndId - - IF EXISTS (SELECT * FROM @ResourceIds) - BEGIN - DECLARE @SurrogateIdMap TABLE (MaxSurrogateId bigint PRIMARY KEY) - INSERT INTO @SurrogateIdMap - SELECT MaxSurrogateId = A.ResourceSurrogateId - FROM (SELECT * FROM @ResourceIds WHERE RowId = 1 AND ResourceSurrogateId BETWEEN @StartId AND @EndId) A + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + AND IsHistory = 1 + AND (IsDeleted = 0 OR @IncludeDeleted = 1) + OPTION (MAXDOP 1) - SELECT @ResourceTypeId - ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.ResourceId ELSE A.ResourceId END - ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.Version ELSE A.Version END - ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.IsDeleted ELSE A.IsDeleted END - ,isnull(C.ResourceSurrogateId, A.ResourceSurrogateId) - ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.RequestMethod ELSE A.RequestMethod END - ,IsMatch = convert(bit,1) - ,IsPartial = convert(bit,0) - ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.IsRawResourceMetaSet ELSE A.IsRawResourceMetaSet END - ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.SearchParamHash ELSE A.SearchParamHash END - ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.RawResource ELSE A.RawResource END - FROM dbo.Resource A - LEFT OUTER JOIN @SurrogateIdMap B ON B.MaxSurrogateId = A.ResourceSurrogateId - LEFT OUTER JOIN dbo.Resource C ON C.ResourceTypeId = @ResourceTypeId AND C.ResourceSurrogateId = MaxSurrogateId - WHERE A.ResourceTypeId = @ResourceTypeId - AND A.ResourceSurrogateId BETWEEN @StartId AND @EndId - AND (A.IsHistory = 0 OR MaxSurrogateId IS NOT NULL OR @IncludeHistory = 1) - AND (A.IsDeleted = 0 OR @IncludeDeleted = 1) + IF @@rowcount > 0 + INSERT INTO @SurrogateIds + SELECT ResourceSurrogateId + FROM (SELECT ResourceId, ResourceSurrogateId, RowId = row_number() OVER (PARTITION BY ResourceId ORDER BY ResourceSurrogateId DESC) + FROM dbo.Resource WITH (INDEX = IX_Resource_ResourceTypeId_ResourceId_Version) -- w/o hint access to Resource table is inefficient when many versions are present. Hint is ignored if Resource is a view. + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceId IN (SELECT TOP (@DummyTop) ResourceId FROM @ResourceIds) + AND ResourceSurrogateId BETWEEN @StartId AND @GlobalEndId + ) A + WHERE RowId = 1 + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)) END - ELSE - SELECT ResourceTypeId, ResourceId, Version, IsDeleted, ResourceSurrogateId, RequestMethod, IsMatch = convert(bit,1), IsPartial = convert(bit,0), IsRawResourceMetaSet, SearchParamHash, RawResource - FROM dbo.Resource - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId BETWEEN @StartId AND @EndId - AND (IsHistory = 0 OR @IncludeHistory = 1) - AND (IsDeleted = 0 OR @IncludeDeleted = 1) + + SELECT ResourceTypeId, ResourceId, Version, IsDeleted, ResourceSurrogateId, RequestMethod, IsMatch = convert(bit,1), IsPartial = convert(bit,0), IsRawResourceMetaSet, SearchParamHash, RawResource + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + AND (IsHistory = 0 OR @IncludeHistory = 1) + AND (IsDeleted = 0 OR @IncludeDeleted = 1) + UNION ALL + SELECT ResourceTypeId, ResourceId, Version, IsDeleted, ResourceSurrogateId, RequestMethod, IsMatch = convert(bit,1), IsPartial = convert(bit,0), IsRawResourceMetaSet, SearchParamHash, RawResource + FROM @SurrogateIds + JOIN dbo.Resource ON ResourceTypeId = @ResourceTypeId AND ResourceSurrogateId = MaxSurrogateId + WHERE IsHistory = 1 + AND (IsDeleted = 0 OR @IncludeDeleted = 1) + OPTION (MAXDOP 1) EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@st,@Rows=@@rowcount END TRY @@ -73,3 +62,20 @@ BEGIN CATCH THROW END CATCH GO +--set nocount on +--DECLARE @Ranges TABLE (UnitId int PRIMARY KEY, MinId bigint, MaxId bigint, Cnt int) +--INSERT INTO @Ranges +-- EXECUTE dbo.GetResourceSurrogateIdRanges 96, 0, 9e18, 90000, 10 +--SELECT count(*) FROM @Ranges +--DECLARE @UnitId int +-- ,@MinId bigint +-- ,@MaxId bigint +--DECLARE @Resources TABLE (RawResource varbinary(max)) +--WHILE EXISTS (SELECT * FROM @Ranges) +--BEGIN +-- SELECT TOP 1 @UnitId = UnitId, @MinId = MinId, @MaxId = MaxId FROM @Ranges ORDER BY UnitId +-- INSERT INTO @Resources +-- EXECUTE dbo.GetResourcesByTypeAndSurrogateIdRange 96, @MinId, @MaxId, NULL, @MaxId -- last is to invoke snapshot logic +-- DELETE FROM @Resources +-- DELETE FROM @Ranges WHERE UnitId = @UnitId +--END diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/65.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/65.sql new file mode 100644 index 0000000000..d63159a283 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/65.sql @@ -0,0 +1,6739 @@ + +/************************************************************************************************* + Auto-Generated from Sql build task. Do not manually edit it. +**************************************************************************************************/ +SET XACT_ABORT ON +BEGIN TRAN +IF EXISTS (SELECT * + FROM sys.tables + WHERE name = 'ClaimType') + BEGIN + ROLLBACK; + RETURN; + END + + +GO +INSERT INTO dbo.SchemaVersion +VALUES (65, 'started'); + +CREATE PARTITION FUNCTION PartitionFunction_ResourceTypeId(SMALLINT) + AS RANGE RIGHT + FOR VALUES (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150); + +CREATE PARTITION SCHEME PartitionScheme_ResourceTypeId + AS PARTITION PartitionFunction_ResourceTypeId + ALL TO ([PRIMARY]); + + +GO +CREATE PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp(DATETIME2 (7)) + AS RANGE RIGHT + FOR VALUES (N'1970-01-01T00:00:00.0000000'); + +CREATE PARTITION SCHEME PartitionScheme_ResourceChangeData_Timestamp + AS PARTITION PartitionFunction_ResourceChangeData_Timestamp + ALL TO ([PRIMARY]); + +DECLARE @numberOfHistoryPartitions AS INT = 48; + +DECLARE @numberOfFuturePartitions AS INT = 720; + +DECLARE @rightPartitionBoundary AS DATETIME2 (7); + +DECLARE @currentDateTime AS DATETIME2 (7) = sysutcdatetime(); + +WHILE @numberOfHistoryPartitions >= -@numberOfFuturePartitions + BEGIN + SET @rightPartitionBoundary = DATEADD(hour, DATEDIFF(hour, 0, @currentDateTime) - @numberOfHistoryPartitions, 0); + ALTER PARTITION SCHEME PartitionScheme_ResourceChangeData_Timestamp NEXT USED [Primary]; + ALTER PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp( ) + SPLIT RANGE (@rightPartitionBoundary); + SET @numberOfHistoryPartitions -= 1; + END + +CREATE SEQUENCE dbo.ResourceSurrogateIdUniquifierSequence + AS INT + START WITH 0 + INCREMENT BY 1 + MINVALUE 0 + MAXVALUE 79999 + CYCLE + CACHE 1000000; + +CREATE TYPE dbo.BigintList AS TABLE ( + Id BIGINT NOT NULL PRIMARY KEY); + +CREATE TYPE dbo.CompartmentAssignmentList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + CompartmentTypeId TINYINT NOT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL PRIMARY KEY (ResourceTypeId, ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId)); + +CREATE TYPE dbo.DateTimeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + StartDateTime DATETIMEOFFSET (7) NOT NULL, + EndDateTime DATETIMEOFFSET (7) NOT NULL, + IsLongerThanADay BIT NOT NULL, + IsMin BIT NOT NULL, + IsMax BIT NOT NULL UNIQUE (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax)); + +CREATE TYPE dbo.NumberSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SingleValue DECIMAL (36, 18) NULL, + LowValue DECIMAL (36, 18) NULL, + HighValue DECIMAL (36, 18) NULL UNIQUE (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue)); + +CREATE TYPE dbo.QuantitySearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + QuantityCodeId INT NULL, + SingleValue DECIMAL (36, 18) NULL, + LowValue DECIMAL (36, 18) NULL, + HighValue DECIMAL (36, 18) NULL UNIQUE (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue)); + +CREATE TYPE dbo.ReferenceSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId SMALLINT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion INT NULL UNIQUE (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId)); + +CREATE TYPE dbo.ReferenceTokenCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId1 SMALLINT NULL, + ReferenceResourceId1 VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion1 INT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.ResourceDateKeyList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ResourceSurrogateId BIGINT NOT NULL PRIMARY KEY (ResourceTypeId, ResourceId, ResourceSurrogateId)); + +CREATE TYPE dbo.ResourceKeyList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Version INT NULL UNIQUE (ResourceTypeId, ResourceId, Version)); + +CREATE TYPE dbo.ResourceList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Version INT NOT NULL, + HasVersionToCompare BIT NOT NULL, + IsDeleted BIT NOT NULL, + IsHistory BIT NOT NULL, + KeepHistory BIT NOT NULL, + RawResource VARBINARY (MAX) NOT NULL, + IsRawResourceMetaSet BIT NOT NULL, + RequestMethod VARCHAR (10) NULL, + SearchParamHash VARCHAR (64) NULL PRIMARY KEY (ResourceTypeId, ResourceSurrogateId), + UNIQUE (ResourceTypeId, ResourceId, Version)); + +CREATE TYPE dbo.ResourceWriteClaimList AS TABLE ( + ResourceSurrogateId BIGINT NOT NULL, + ClaimTypeId TINYINT NOT NULL, + ClaimValue NVARCHAR (128) NOT NULL); + +CREATE TYPE dbo.StringList AS TABLE ( + String VARCHAR (MAX)); + +CREATE TYPE dbo.StringSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL, + IsMin BIT NOT NULL, + IsMax BIT NOT NULL); + +CREATE TYPE dbo.TokenDateTimeCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + StartDateTime2 DATETIMEOFFSET (7) NOT NULL, + EndDateTime2 DATETIMEOFFSET (7) NOT NULL, + IsLongerThanADay2 BIT NOT NULL); + +CREATE TYPE dbo.TokenNumberNumberCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SingleValue2 DECIMAL (36, 18) NULL, + LowValue2 DECIMAL (36, 18) NULL, + HighValue2 DECIMAL (36, 18) NULL, + SingleValue3 DECIMAL (36, 18) NULL, + LowValue3 DECIMAL (36, 18) NULL, + HighValue3 DECIMAL (36, 18) NULL, + HasRange BIT NOT NULL); + +CREATE TYPE dbo.TokenQuantityCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SystemId2 INT NULL, + QuantityCodeId2 INT NULL, + SingleValue2 DECIMAL (36, 18) NULL, + LowValue2 DECIMAL (36, 18) NULL, + HighValue2 DECIMAL (36, 18) NULL); + +CREATE TYPE dbo.TokenSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + Code VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.TokenStringCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + Text2 NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow2 NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL); + +CREATE TYPE dbo.TokenTextList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (400) COLLATE Latin1_General_CI_AI NOT NULL); + +CREATE TYPE dbo.TokenTokenCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.BulkResourceWriteClaimTableType_1 AS TABLE ( + Offset INT NOT NULL, + ClaimTypeId TINYINT NOT NULL, + ClaimValue NVARCHAR (128) NOT NULL); + +CREATE TYPE dbo.BulkCompartmentAssignmentTableType_1 AS TABLE ( + Offset INT NOT NULL, + CompartmentTypeId TINYINT NOT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL); + +CREATE TYPE dbo.BulkReferenceSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId SMALLINT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion INT NULL); + +CREATE TYPE dbo.BulkTokenSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + Code VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL); + +CREATE TYPE dbo.BulkTokenSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + Code VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.BulkTokenTextTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (400) COLLATE Latin1_General_CI_AI NOT NULL); + +CREATE TYPE dbo.BulkStringSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL); + +CREATE TYPE dbo.BulkStringSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL, + IsMin BIT NOT NULL, + IsMax BIT NOT NULL); + +CREATE TYPE dbo.BulkUriSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Uri VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL); + +CREATE TYPE dbo.BulkNumberSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SingleValue DECIMAL (18, 6) NULL, + LowValue DECIMAL (18, 6) NULL, + HighValue DECIMAL (18, 6) NULL); + +CREATE TYPE dbo.BulkQuantitySearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + QuantityCodeId INT NULL, + SingleValue DECIMAL (18, 6) NULL, + LowValue DECIMAL (18, 6) NULL, + HighValue DECIMAL (18, 6) NULL); + +CREATE TYPE dbo.BulkDateTimeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + StartDateTime DATETIMEOFFSET (7) NOT NULL, + EndDateTime DATETIMEOFFSET (7) NOT NULL, + IsLongerThanADay BIT NOT NULL); + +CREATE TYPE dbo.BulkDateTimeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + StartDateTime DATETIMEOFFSET (7) NOT NULL, + EndDateTime DATETIMEOFFSET (7) NOT NULL, + IsLongerThanADay BIT NOT NULL, + IsMin BIT NOT NULL, + IsMax BIT NOT NULL); + +CREATE TYPE dbo.BulkReferenceTokenCompositeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId1 SMALLINT NULL, + ReferenceResourceId1 VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion1 INT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL); + +CREATE TYPE dbo.BulkReferenceTokenCompositeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId1 SMALLINT NULL, + ReferenceResourceId1 VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion1 INT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.BulkTokenTokenCompositeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL); + +CREATE TYPE dbo.BulkTokenTokenCompositeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.BulkTokenDateTimeCompositeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + StartDateTime2 DATETIMEOFFSET (7) NOT NULL, + EndDateTime2 DATETIMEOFFSET (7) NOT NULL, + IsLongerThanADay2 BIT NOT NULL); + +CREATE TYPE dbo.BulkTokenDateTimeCompositeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + StartDateTime2 DATETIMEOFFSET (7) NOT NULL, + EndDateTime2 DATETIMEOFFSET (7) NOT NULL, + IsLongerThanADay2 BIT NOT NULL); + +CREATE TYPE dbo.BulkTokenQuantityCompositeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + SystemId2 INT NULL, + QuantityCodeId2 INT NULL, + SingleValue2 DECIMAL (18, 6) NULL, + LowValue2 DECIMAL (18, 6) NULL, + HighValue2 DECIMAL (18, 6) NULL); + +CREATE TYPE dbo.BulkTokenQuantityCompositeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SystemId2 INT NULL, + QuantityCodeId2 INT NULL, + SingleValue2 DECIMAL (18, 6) NULL, + LowValue2 DECIMAL (18, 6) NULL, + HighValue2 DECIMAL (18, 6) NULL); + +CREATE TYPE dbo.BulkTokenStringCompositeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + Text2 NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow2 NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL); + +CREATE TYPE dbo.BulkTokenStringCompositeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + Text2 NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow2 NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL); + +CREATE TYPE dbo.BulkTokenNumberNumberCompositeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + SingleValue2 DECIMAL (18, 6) NULL, + LowValue2 DECIMAL (18, 6) NULL, + HighValue2 DECIMAL (18, 6) NULL, + SingleValue3 DECIMAL (18, 6) NULL, + LowValue3 DECIMAL (18, 6) NULL, + HighValue3 DECIMAL (18, 6) NULL, + HasRange BIT NOT NULL); + +CREATE TYPE dbo.BulkTokenNumberNumberCompositeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SingleValue2 DECIMAL (18, 6) NULL, + LowValue2 DECIMAL (18, 6) NULL, + HighValue2 DECIMAL (18, 6) NULL, + SingleValue3 DECIMAL (18, 6) NULL, + LowValue3 DECIMAL (18, 6) NULL, + HighValue3 DECIMAL (18, 6) NULL, + HasRange BIT NOT NULL); + +CREATE TYPE dbo.SearchParamTableType_1 AS TABLE ( + Uri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + Status VARCHAR (10) NOT NULL, + IsPartiallySupported BIT NOT NULL); + +CREATE TYPE dbo.SearchParamTableType_2 AS TABLE ( + Uri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + Status VARCHAR (20) NOT NULL, + IsPartiallySupported BIT NOT NULL); + +CREATE TYPE dbo.BulkReindexResourceTableType_1 AS TABLE ( + Offset INT NOT NULL, + ResourceTypeId SMALLINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ETag INT NULL, + SearchParamHash VARCHAR (64) NOT NULL); + +CREATE TYPE dbo.BulkImportResourceType_1 AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Version INT NOT NULL, + IsHistory BIT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + IsDeleted BIT NOT NULL, + RequestMethod VARCHAR (10) NULL, + RawResource VARBINARY (MAX) NOT NULL, + IsRawResourceMetaSet BIT DEFAULT 0 NOT NULL, + SearchParamHash VARCHAR (64) NULL); + +CREATE TYPE dbo.UriSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Uri VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL PRIMARY KEY (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri)); + +CREATE TABLE dbo.ClaimType ( + ClaimTypeId TINYINT IDENTITY (1, 1) NOT NULL, + Name VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + CONSTRAINT UQ_ClaimType_ClaimTypeId UNIQUE (ClaimTypeId), + CONSTRAINT PKC_ClaimType PRIMARY KEY CLUSTERED (Name) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE dbo.CompartmentAssignment ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + CompartmentTypeId TINYINT NOT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + IsHistory BIT NOT NULL, + CONSTRAINT PKC_CompartmentAssignment PRIMARY KEY CLUSTERED (ResourceTypeId, ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId) WITH (DATA_COMPRESSION = PAGE) ON PartitionScheme_ResourceTypeId (ResourceTypeId) +); + + +GO +ALTER TABLE dbo.CompartmentAssignment + ADD CONSTRAINT DF_CompartmentAssignment_IsHistory DEFAULT 0 FOR IsHistory; + + +GO +ALTER TABLE dbo.CompartmentAssignment SET (LOCK_ESCALATION = AUTO); + + +GO +CREATE NONCLUSTERED INDEX IX_CompartmentAssignment_CompartmentTypeId_ReferenceResourceId + ON dbo.CompartmentAssignment(ResourceTypeId, CompartmentTypeId, ReferenceResourceId, ResourceSurrogateId) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.CompartmentType ( + CompartmentTypeId TINYINT IDENTITY (1, 1) NOT NULL, + Name VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + CONSTRAINT UQ_CompartmentType_CompartmentTypeId UNIQUE (CompartmentTypeId), + CONSTRAINT PKC_CompartmentType PRIMARY KEY CLUSTERED (Name) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE dbo.DateTimeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + StartDateTime DATETIME2 (7) NOT NULL, + EndDateTime DATETIME2 (7) NOT NULL, + IsLongerThanADay BIT NOT NULL, + IsHistory BIT NOT NULL, + IsMin BIT CONSTRAINT date_IsMin_Constraint DEFAULT 0 NOT NULL, + IsMax BIT CONSTRAINT date_IsMax_Constraint DEFAULT 0 NOT NULL +); + +ALTER TABLE dbo.DateTimeSearchParam + ADD CONSTRAINT DF_DateTimeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.DateTimeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_DateTimeSearchParam + ON dbo.DateTimeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_DateTimeSearchParam_SearchParamId_StartDateTime_EndDateTime + ON dbo.DateTimeSearchParam(ResourceTypeId, SearchParamId, StartDateTime, EndDateTime, ResourceSurrogateId) + INCLUDE(IsLongerThanADay, IsMin, IsMax) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_DateTimeSearchParam_SearchParamId_EndDateTime_StartDateTime + ON dbo.DateTimeSearchParam(ResourceTypeId, SearchParamId, EndDateTime, StartDateTime, ResourceSurrogateId) + INCLUDE(IsLongerThanADay, IsMin, IsMax) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_DateTimeSearchParam_SearchParamId_StartDateTime_EndDateTime_Long + ON dbo.DateTimeSearchParam(ResourceTypeId, SearchParamId, StartDateTime, EndDateTime, ResourceSurrogateId) + INCLUDE(IsMin, IsMax) WHERE IsHistory = 0 + AND IsLongerThanADay = 1 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_DateTimeSearchParam_SearchParamId_EndDateTime_StartDateTime_Long + ON dbo.DateTimeSearchParam(ResourceTypeId, SearchParamId, EndDateTime, StartDateTime, ResourceSurrogateId) + INCLUDE(IsMin, IsMax) WHERE IsHistory = 0 + AND IsLongerThanADay = 1 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +IF NOT EXISTS (SELECT 1 + FROM sys.tables + WHERE name = 'EventAgentCheckpoint') + BEGIN + CREATE TABLE dbo.EventAgentCheckpoint ( + CheckpointId VARCHAR (64) NOT NULL, + LastProcessedDateTime DATETIMEOFFSET (7), + LastProcessedIdentifier VARCHAR (64) , + UpdatedOn DATETIME2 (7) DEFAULT sysutcdatetime() NOT NULL, + CONSTRAINT PK_EventAgentCheckpoint PRIMARY KEY CLUSTERED (CheckpointId) + ) ON [PRIMARY]; + END + +CREATE PARTITION FUNCTION EventLogPartitionFunction(TINYINT) + AS RANGE RIGHT + FOR VALUES (0, 1, 2, 3, 4, 5, 6, 7); + + +GO +CREATE PARTITION SCHEME EventLogPartitionScheme + AS PARTITION EventLogPartitionFunction + ALL TO ([PRIMARY]); + + +GO +CREATE TABLE dbo.EventLog ( + PartitionId AS isnull(CONVERT (TINYINT, EventId % 8), 0) PERSISTED, + EventId BIGINT IDENTITY (1, 1) NOT NULL, + EventDate DATETIME NOT NULL, + Process VARCHAR (100) NOT NULL, + Status VARCHAR (10) NOT NULL, + Mode VARCHAR (200) NULL, + Action VARCHAR (20) NULL, + Target VARCHAR (100) NULL, + Rows BIGINT NULL, + Milliseconds INT NULL, + EventText NVARCHAR (3500) NULL, + SPID SMALLINT NOT NULL, + HostName VARCHAR (64) NOT NULL CONSTRAINT PKC_EventLog_EventDate_EventId_PartitionId PRIMARY KEY CLUSTERED (EventDate, EventId, PartitionId) ON EventLogPartitionScheme (PartitionId) +); + +CREATE TABLE dbo.ExportJob ( + Id VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Hash VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Status VARCHAR (10) NOT NULL, + HeartbeatDateTime DATETIME2 (7) NULL, + RawJobRecord VARCHAR (MAX) NOT NULL, + JobVersion ROWVERSION NOT NULL, + CONSTRAINT PKC_ExportJob PRIMARY KEY CLUSTERED (Id) +); + +CREATE UNIQUE NONCLUSTERED INDEX IX_ExportJob_Hash_Status_HeartbeatDateTime + ON dbo.ExportJob(Hash, Status, HeartbeatDateTime); + +CREATE TABLE dbo.IndexProperties ( + TableName VARCHAR (100) NOT NULL, + IndexName VARCHAR (200) NOT NULL, + PropertyName VARCHAR (100) NOT NULL, + PropertyValue VARCHAR (100) NOT NULL, + CreateDate DATETIME CONSTRAINT DF_IndexProperties_CreateDate DEFAULT getUTCdate() NOT NULL CONSTRAINT PKC_IndexProperties_TableName_IndexName_PropertyName PRIMARY KEY CLUSTERED (TableName, IndexName, PropertyName) +); + +CREATE PARTITION FUNCTION TinyintPartitionFunction(TINYINT) + AS RANGE RIGHT + FOR VALUES (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255); + + +GO +CREATE PARTITION SCHEME TinyintPartitionScheme + AS PARTITION TinyintPartitionFunction + ALL TO ([PRIMARY]); + + +GO +CREATE TABLE dbo.JobQueue ( + QueueType TINYINT NOT NULL, + GroupId BIGINT NOT NULL, + JobId BIGINT NOT NULL, + PartitionId AS CONVERT (TINYINT, JobId % 16) PERSISTED, + Definition VARCHAR (MAX) NOT NULL, + DefinitionHash VARBINARY (20) NOT NULL, + Version BIGINT CONSTRAINT DF_JobQueue_Version DEFAULT datediff_big(millisecond, '0001-01-01', getUTCdate()) NOT NULL, + Status TINYINT CONSTRAINT DF_JobQueue_Status DEFAULT 0 NOT NULL, + Priority TINYINT CONSTRAINT DF_JobQueue_Priority DEFAULT 100 NOT NULL, + Data BIGINT NULL, + Result VARCHAR (MAX) NULL, + CreateDate DATETIME CONSTRAINT DF_JobQueue_CreateDate DEFAULT getUTCdate() NOT NULL, + StartDate DATETIME NULL, + EndDate DATETIME NULL, + HeartbeatDate DATETIME CONSTRAINT DF_JobQueue_HeartbeatDate DEFAULT getUTCdate() NOT NULL, + Worker VARCHAR (100) NULL, + Info VARCHAR (1000) NULL, + CancelRequested BIT CONSTRAINT DF_JobQueue_CancelRequested DEFAULT 0 NOT NULL CONSTRAINT PKC_JobQueue_QueueType_PartitionId_JobId PRIMARY KEY CLUSTERED (QueueType, PartitionId, JobId) ON TinyintPartitionScheme (QueueType), + CONSTRAINT U_JobQueue_QueueType_JobId UNIQUE (QueueType, JobId) +); + + +GO +CREATE INDEX IX_QueueType_PartitionId_Status_Priority + ON dbo.JobQueue(PartitionId, Status, Priority) + ON TinyintPartitionScheme (QueueType); + + +GO +CREATE INDEX IX_QueueType_GroupId + ON dbo.JobQueue(QueueType, GroupId) + ON TinyintPartitionScheme (QueueType); + + +GO +CREATE INDEX IX_QueueType_DefinitionHash + ON dbo.JobQueue(QueueType, DefinitionHash) + ON TinyintPartitionScheme (QueueType); + +CREATE TABLE dbo.NumberSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SingleValue DECIMAL (36, 18) NULL, + LowValue DECIMAL (36, 18) NOT NULL, + HighValue DECIMAL (36, 18) NOT NULL, + IsHistory BIT NOT NULL +); + +ALTER TABLE dbo.NumberSearchParam + ADD CONSTRAINT DF_NumberSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.NumberSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_NumberSearchParam + ON dbo.NumberSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_NumberSearchParam_SearchParamId_SingleValue + ON dbo.NumberSearchParam(ResourceTypeId, SearchParamId, SingleValue, ResourceSurrogateId) WHERE IsHistory = 0 + AND SingleValue IS NOT NULL + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_NumberSearchParam_SearchParamId_LowValue_HighValue + ON dbo.NumberSearchParam(ResourceTypeId, SearchParamId, LowValue, HighValue, ResourceSurrogateId) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_NumberSearchParam_SearchParamId_HighValue_LowValue + ON dbo.NumberSearchParam(ResourceTypeId, SearchParamId, HighValue, LowValue, ResourceSurrogateId) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.Parameters ( + Id VARCHAR (100) NOT NULL, + Date DATETIME NULL, + Number FLOAT NULL, + Bigint BIGINT NULL, + Char VARCHAR (4000) NULL, + Binary VARBINARY (MAX) NULL, + UpdatedDate DATETIME NULL, + UpdatedBy NVARCHAR (255) NULL CONSTRAINT PKC_Parameters_Id PRIMARY KEY CLUSTERED (Id) WITH (IGNORE_DUP_KEY = ON) +); + + +GO +CREATE TABLE dbo.ParametersHistory ( + ChangeId INT IDENTITY (1, 1) NOT NULL, + Id VARCHAR (100) NOT NULL, + Date DATETIME NULL, + Number FLOAT NULL, + Bigint BIGINT NULL, + Char VARCHAR (4000) NULL, + Binary VARBINARY (MAX) NULL, + UpdatedDate DATETIME NULL, + UpdatedBy NVARCHAR (255) NULL +); + +CREATE TABLE dbo.QuantityCode ( + QuantityCodeId INT IDENTITY (1, 1) NOT NULL, + Value NVARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CONSTRAINT UQ_QuantityCode_QuantityCodeId UNIQUE (QuantityCodeId), + CONSTRAINT PKC_QuantityCode PRIMARY KEY CLUSTERED (Value) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE dbo.QuantitySearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + QuantityCodeId INT NULL, + SingleValue DECIMAL (36, 18) NULL, + LowValue DECIMAL (36, 18) NOT NULL, + HighValue DECIMAL (36, 18) NOT NULL, + IsHistory BIT NOT NULL +); + +ALTER TABLE dbo.QuantitySearchParam + ADD CONSTRAINT DF_QuantitySearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.QuantitySearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_QuantitySearchParam + ON dbo.QuantitySearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_QuantitySearchParam_SearchParamId_QuantityCodeId_SingleValue + ON dbo.QuantitySearchParam(ResourceTypeId, SearchParamId, QuantityCodeId, SingleValue, ResourceSurrogateId) + INCLUDE(SystemId) WHERE IsHistory = 0 + AND SingleValue IS NOT NULL + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_QuantitySearchParam_SearchParamId_QuantityCodeId_LowValue_HighValue + ON dbo.QuantitySearchParam(ResourceTypeId, SearchParamId, QuantityCodeId, LowValue, HighValue, ResourceSurrogateId) + INCLUDE(SystemId) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_QuantitySearchParam_SearchParamId_QuantityCodeId_HighValue_LowValue + ON dbo.QuantitySearchParam(ResourceTypeId, SearchParamId, QuantityCodeId, HighValue, LowValue, ResourceSurrogateId) + INCLUDE(SystemId) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.ReferenceSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId SMALLINT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion INT NULL, + IsHistory BIT NOT NULL +); + +ALTER TABLE dbo.ReferenceSearchParam + ADD CONSTRAINT DF_ReferenceSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.ReferenceSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_ReferenceSearchParam + ON dbo.ReferenceSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_ReferenceSearchParam_SearchParamId_ReferenceResourceTypeId_ReferenceResourceId_BaseUri_ReferenceResourceVersion + ON dbo.ReferenceSearchParam(ResourceTypeId, SearchParamId, ReferenceResourceId, ReferenceResourceTypeId, BaseUri, ResourceSurrogateId) + INCLUDE(ReferenceResourceVersion) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.ReferenceTokenCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId1 SMALLINT NULL, + ReferenceResourceId1 VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion1 INT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + IsHistory BIT NOT NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.ReferenceTokenCompositeSearchParam + ADD CONSTRAINT DF_ReferenceTokenCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.ReferenceTokenCompositeSearchParam + ADD CONSTRAINT CHK_ReferenceTokenCompositeSearchParam_CodeOverflow2 CHECK (LEN(Code2) = 256 + OR CodeOverflow2 IS NULL); + +ALTER TABLE dbo.ReferenceTokenCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_ReferenceTokenCompositeSearchParam + ON dbo.ReferenceTokenCompositeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_ReferenceTokenCompositeSearchParam_ReferenceResourceId1_Code2 + ON dbo.ReferenceTokenCompositeSearchParam(ResourceTypeId, SearchParamId, ReferenceResourceId1, Code2, ResourceSurrogateId) + INCLUDE(ReferenceResourceTypeId1, BaseUri1, SystemId2) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.ReindexJob ( + Id VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Status VARCHAR (10) NOT NULL, + HeartbeatDateTime DATETIME2 (7) NULL, + RawJobRecord VARCHAR (MAX) NOT NULL, + JobVersion ROWVERSION NOT NULL, + CONSTRAINT PKC_ReindexJob PRIMARY KEY CLUSTERED (Id) +); + +CREATE TABLE dbo.Resource ( + ResourceTypeId SMALLINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Version INT NOT NULL, + IsHistory BIT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + IsDeleted BIT NOT NULL, + RequestMethod VARCHAR (10) NULL, + RawResource VARBINARY (MAX) NOT NULL, + IsRawResourceMetaSet BIT DEFAULT 0 NOT NULL, + SearchParamHash VARCHAR (64) NULL, + TransactionId BIGINT NULL, + HistoryTransactionId BIGINT NULL CONSTRAINT PKC_Resource PRIMARY KEY CLUSTERED (ResourceTypeId, ResourceSurrogateId) WITH (DATA_COMPRESSION = PAGE) ON PartitionScheme_ResourceTypeId (ResourceTypeId), + CONSTRAINT CH_Resource_RawResource_Length CHECK (RawResource > 0x0) +); + +ALTER TABLE dbo.Resource SET (LOCK_ESCALATION = AUTO); + +CREATE INDEX IX_ResourceTypeId_TransactionId + ON dbo.Resource(ResourceTypeId, TransactionId) WHERE TransactionId IS NOT NULL + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_ResourceTypeId_HistoryTransactionId + ON dbo.Resource(ResourceTypeId, HistoryTransactionId) WHERE HistoryTransactionId IS NOT NULL + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE UNIQUE NONCLUSTERED INDEX IX_Resource_ResourceTypeId_ResourceId_Version + ON dbo.Resource(ResourceTypeId, ResourceId, Version) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE UNIQUE NONCLUSTERED INDEX IX_Resource_ResourceTypeId_ResourceId + ON dbo.Resource(ResourceTypeId, ResourceId) + INCLUDE(Version, IsDeleted) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE UNIQUE NONCLUSTERED INDEX IX_Resource_ResourceTypeId_ResourceSurrgateId + ON dbo.Resource(ResourceTypeId, ResourceSurrogateId) WHERE IsHistory = 0 + AND IsDeleted = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.ResourceChangeData ( + Id BIGINT IDENTITY (1, 1) NOT NULL, + Timestamp DATETIME2 (7) CONSTRAINT DF_ResourceChangeData_Timestamp DEFAULT sysutcdatetime() NOT NULL, + ResourceId VARCHAR (64) NOT NULL, + ResourceTypeId SMALLINT NOT NULL, + ResourceVersion INT NOT NULL, + ResourceChangeTypeId TINYINT NOT NULL +) ON PartitionScheme_ResourceChangeData_Timestamp (Timestamp); + +CREATE CLUSTERED INDEX IXC_ResourceChangeData + ON dbo.ResourceChangeData(Id ASC) WITH (ONLINE = ON) + ON PartitionScheme_ResourceChangeData_Timestamp (Timestamp); + +CREATE TABLE dbo.ResourceChangeDataStaging ( + Id BIGINT IDENTITY (1, 1) NOT NULL, + Timestamp DATETIME2 (7) CONSTRAINT DF_ResourceChangeDataStaging_Timestamp DEFAULT sysutcdatetime() NOT NULL, + ResourceId VARCHAR (64) NOT NULL, + ResourceTypeId SMALLINT NOT NULL, + ResourceVersion INT NOT NULL, + ResourceChangeTypeId TINYINT NOT NULL +) ON [PRIMARY]; + +CREATE CLUSTERED INDEX IXC_ResourceChangeDataStaging + ON dbo.ResourceChangeDataStaging(Id ASC, Timestamp ASC) WITH (ONLINE = ON) + ON [PRIMARY]; + +ALTER TABLE dbo.ResourceChangeDataStaging WITH CHECK + ADD CONSTRAINT CHK_ResourceChangeDataStaging_partition CHECK (Timestamp < CONVERT (DATETIME2 (7), N'9999-12-31 23:59:59.9999999')); + +ALTER TABLE dbo.ResourceChangeDataStaging CHECK CONSTRAINT CHK_ResourceChangeDataStaging_partition; + +CREATE TABLE dbo.ResourceChangeType ( + ResourceChangeTypeId TINYINT NOT NULL, + Name NVARCHAR (50) NOT NULL, + CONSTRAINT PK_ResourceChangeType PRIMARY KEY CLUSTERED (ResourceChangeTypeId), + CONSTRAINT UQ_ResourceChangeType_Name UNIQUE NONCLUSTERED (Name) +) ON [PRIMARY]; + + +GO +INSERT dbo.ResourceChangeType (ResourceChangeTypeId, Name) +VALUES (0, N'Creation'); + +INSERT dbo.ResourceChangeType (ResourceChangeTypeId, Name) +VALUES (1, N'Update'); + +INSERT dbo.ResourceChangeType (ResourceChangeTypeId, Name) +VALUES (2, N'Deletion'); + +CREATE TABLE dbo.ResourceType ( + ResourceTypeId SMALLINT IDENTITY (1, 1) NOT NULL, + Name NVARCHAR (50) COLLATE Latin1_General_100_CS_AS NOT NULL, + CONSTRAINT UQ_ResourceType_ResourceTypeId UNIQUE (ResourceTypeId), + CONSTRAINT PKC_ResourceType PRIMARY KEY CLUSTERED (Name) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE dbo.ResourceWriteClaim ( + ResourceSurrogateId BIGINT NOT NULL, + ClaimTypeId TINYINT NOT NULL, + ClaimValue NVARCHAR (128) NOT NULL +) +WITH (DATA_COMPRESSION = PAGE); + +CREATE CLUSTERED INDEX IXC_ResourceWriteClaim + ON dbo.ResourceWriteClaim(ResourceSurrogateId, ClaimTypeId); + +CREATE TABLE dbo.SchemaMigrationProgress ( + Timestamp DATETIME2 (3) DEFAULT CURRENT_TIMESTAMP, + Message NVARCHAR (MAX) +); + +CREATE TABLE dbo.SearchParam ( + SearchParamId SMALLINT IDENTITY (1, 1) NOT NULL, + Uri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + Status VARCHAR (20) NULL, + LastUpdated DATETIMEOFFSET (7) NULL, + IsPartiallySupported BIT NULL, + CONSTRAINT UQ_SearchParam_SearchParamId UNIQUE (SearchParamId), + CONSTRAINT PKC_SearchParam PRIMARY KEY CLUSTERED (Uri) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE dbo.StringSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL, + IsHistory BIT NOT NULL, + IsMin BIT CONSTRAINT string_IsMin_Constraint DEFAULT 0 NOT NULL, + IsMax BIT CONSTRAINT string_IsMax_Constraint DEFAULT 0 NOT NULL +); + +ALTER TABLE dbo.StringSearchParam + ADD CONSTRAINT DF_StringSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.StringSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_StringSearchParam + ON dbo.StringSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_StringSearchParam_SearchParamId_Text + ON dbo.StringSearchParam(ResourceTypeId, SearchParamId, Text, ResourceSurrogateId) + INCLUDE(TextOverflow, IsMin, IsMax) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_StringSearchParam_SearchParamId_TextWithOverflow + ON dbo.StringSearchParam(ResourceTypeId, SearchParamId, Text, ResourceSurrogateId) + INCLUDE(IsMin, IsMax) WHERE IsHistory = 0 + AND TextOverflow IS NOT NULL WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.System ( + SystemId INT IDENTITY (1, 1) NOT NULL, + Value NVARCHAR (256) NOT NULL, + CONSTRAINT UQ_System_SystemId UNIQUE (SystemId), + CONSTRAINT PKC_System PRIMARY KEY CLUSTERED (Value) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE [dbo].[TaskInfo] ( + [TaskId] VARCHAR (64) NOT NULL, + [QueueId] VARCHAR (64) NOT NULL, + [Status] SMALLINT NOT NULL, + [TaskTypeId] SMALLINT NOT NULL, + [RunId] VARCHAR (50) NULL, + [IsCanceled] BIT NOT NULL, + [RetryCount] SMALLINT NOT NULL, + [MaxRetryCount] SMALLINT NOT NULL, + [HeartbeatDateTime] DATETIME2 (7) NULL, + [InputData] VARCHAR (MAX) NOT NULL, + [TaskContext] VARCHAR (MAX) NULL, + [Result] VARCHAR (MAX) NULL, + [CreateDateTime] DATETIME2 (7) CONSTRAINT DF_TaskInfo_CreateDate DEFAULT SYSUTCDATETIME() NOT NULL, + [StartDateTime] DATETIME2 (7) NULL, + [EndDateTime] DATETIME2 (7) NULL, + [Worker] VARCHAR (100) NULL, + [RestartInfo] VARCHAR (MAX) NULL, + [ParentTaskId] VARCHAR (64) NULL, + CONSTRAINT PKC_TaskInfo PRIMARY KEY CLUSTERED (TaskId) WITH (DATA_COMPRESSION = PAGE) +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]; + + +GO +CREATE NONCLUSTERED INDEX IX_QueueId_Status + ON dbo.TaskInfo(QueueId, Status); + + +GO +CREATE NONCLUSTERED INDEX IX_QueueId_ParentTaskId + ON dbo.TaskInfo(QueueId, ParentTaskId); + +CREATE TABLE dbo.TokenDateTimeCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + StartDateTime2 DATETIME2 (7) NOT NULL, + EndDateTime2 DATETIME2 (7) NOT NULL, + IsLongerThanADay2 BIT NOT NULL, + IsHistory BIT NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenDateTimeCompositeSearchParam + ADD CONSTRAINT DF_TokenDateTimeCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenDateTimeCompositeSearchParam + ADD CONSTRAINT CHK_TokenDateTimeCompositeSearchParam_CodeOverflow1 CHECK (LEN(Code1) = 256 + OR CodeOverflow1 IS NULL); + +ALTER TABLE dbo.TokenDateTimeCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenDateTimeCompositeSearchParam + ON dbo.TokenDateTimeCompositeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenDateTimeCompositeSearchParam_Code1_StartDateTime2_EndDateTime2 + ON dbo.TokenDateTimeCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, StartDateTime2, EndDateTime2, ResourceSurrogateId) + INCLUDE(SystemId1, IsLongerThanADay2) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenDateTimeCompositeSearchParam_Code1_EndDateTime2_StartDateTime2 + ON dbo.TokenDateTimeCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, EndDateTime2, StartDateTime2, ResourceSurrogateId) + INCLUDE(SystemId1, IsLongerThanADay2) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenDateTimeCompositeSearchParam_Code1_StartDateTime2_EndDateTime2_Long + ON dbo.TokenDateTimeCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, StartDateTime2, EndDateTime2, ResourceSurrogateId) + INCLUDE(SystemId1) WHERE IsHistory = 0 + AND IsLongerThanADay2 = 1 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenDateTimeCompositeSearchParam_Code1_EndDateTime2_StartDateTime2_Long + ON dbo.TokenDateTimeCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, EndDateTime2, StartDateTime2, ResourceSurrogateId) + INCLUDE(SystemId1) WHERE IsHistory = 0 + AND IsLongerThanADay2 = 1 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenNumberNumberCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + SingleValue2 DECIMAL (36, 18) NULL, + LowValue2 DECIMAL (36, 18) NULL, + HighValue2 DECIMAL (36, 18) NULL, + SingleValue3 DECIMAL (36, 18) NULL, + LowValue3 DECIMAL (36, 18) NULL, + HighValue3 DECIMAL (36, 18) NULL, + HasRange BIT NOT NULL, + IsHistory BIT NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenNumberNumberCompositeSearchParam + ADD CONSTRAINT DF_TokenNumberNumberCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenNumberNumberCompositeSearchParam + ADD CONSTRAINT CHK_TokenNumberNumberCompositeSearchParam_CodeOverflow1 CHECK (LEN(Code1) = 256 + OR CodeOverflow1 IS NULL); + +ALTER TABLE dbo.TokenNumberNumberCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenNumberNumberCompositeSearchParam + ON dbo.TokenNumberNumberCompositeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenNumberNumberCompositeSearchParam_SearchParamId_Code1_Text2 + ON dbo.TokenNumberNumberCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, SingleValue2, SingleValue3, ResourceSurrogateId) + INCLUDE(SystemId1) WHERE IsHistory = 0 + AND HasRange = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenNumberNumberCompositeSearchParam_SearchParamId_Code1_LowValue2_HighValue2_LowValue3_HighValue3 + ON dbo.TokenNumberNumberCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, LowValue2, HighValue2, LowValue3, HighValue3, ResourceSurrogateId) + INCLUDE(SystemId1) WHERE IsHistory = 0 + AND HasRange = 1 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenQuantityCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + SystemId2 INT NULL, + QuantityCodeId2 INT NULL, + SingleValue2 DECIMAL (36, 18) NULL, + LowValue2 DECIMAL (36, 18) NULL, + HighValue2 DECIMAL (36, 18) NULL, + IsHistory BIT NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenQuantityCompositeSearchParam + ADD CONSTRAINT DF_TokenQuantityCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenQuantityCompositeSearchParam + ADD CONSTRAINT CHK_TokenQuantityCompositeSearchParam_CodeOverflow1 CHECK (LEN(Code1) = 256 + OR CodeOverflow1 IS NULL); + +ALTER TABLE dbo.TokenQuantityCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenQuantityCompositeSearchParam + ON dbo.TokenQuantityCompositeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenQuantityCompositeSearchParam_SearchParamId_Code1_QuantityCodeId2_SingleValue2 + ON dbo.TokenQuantityCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, SingleValue2, ResourceSurrogateId) + INCLUDE(QuantityCodeId2, SystemId1, SystemId2) WHERE IsHistory = 0 + AND SingleValue2 IS NOT NULL WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenQuantityCompositeSearchParam_SearchParamId_Code1_QuantityCodeId2_LowValue2_HighValue2 + ON dbo.TokenQuantityCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, LowValue2, HighValue2, ResourceSurrogateId) + INCLUDE(QuantityCodeId2, SystemId1, SystemId2) WHERE IsHistory = 0 + AND LowValue2 IS NOT NULL WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenQuantityCompositeSearchParam_SearchParamId_Code1_QuantityCodeId2_HighValue2_LowValue2 + ON dbo.TokenQuantityCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, HighValue2, LowValue2, ResourceSurrogateId) + INCLUDE(QuantityCodeId2, SystemId1, SystemId2) WHERE IsHistory = 0 + AND LowValue2 IS NOT NULL WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + Code VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + IsHistory BIT NOT NULL, + CodeOverflow VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenSearchParam + ADD CONSTRAINT DF_TokenSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenSearchParam + ADD CONSTRAINT CHK_TokenSearchParam_CodeOverflow CHECK (LEN(Code) = 256 + OR CodeOverflow IS NULL); + +ALTER TABLE dbo.TokenSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenSearchParam + ON dbo.TokenSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenSeachParam_SearchParamId_Code_SystemId + ON dbo.TokenSearchParam(ResourceTypeId, SearchParamId, Code, ResourceSurrogateId) + INCLUDE(SystemId) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenStringCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + Text2 NVARCHAR (256) COLLATE Latin1_General_CI_AI NOT NULL, + TextOverflow2 NVARCHAR (MAX) COLLATE Latin1_General_CI_AI NULL, + IsHistory BIT NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenStringCompositeSearchParam + ADD CONSTRAINT DF_TokenStringCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenStringCompositeSearchParam + ADD CONSTRAINT CHK_TokenStringCompositeSearchParam_CodeOverflow1 CHECK (LEN(Code1) = 256 + OR CodeOverflow1 IS NULL); + +ALTER TABLE dbo.TokenStringCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenStringCompositeSearchParam + ON dbo.TokenStringCompositeSearchParam(ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenStringCompositeSearchParam_SearchParamId_Code1_Text2 + ON dbo.TokenStringCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, Text2, ResourceSurrogateId) + INCLUDE(SystemId1, TextOverflow2) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenStringCompositeSearchParam_SearchParamId_Code1_Text2WithOverflow + ON dbo.TokenStringCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, Text2, ResourceSurrogateId) + INCLUDE(SystemId1) WHERE IsHistory = 0 + AND TextOverflow2 IS NOT NULL WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenText ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (400) COLLATE Latin1_General_CI_AI NOT NULL, + IsHistory BIT NOT NULL +); + +ALTER TABLE dbo.TokenText + ADD CONSTRAINT DF_TokenText_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenText SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenText + ON dbo.TokenText(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenText_SearchParamId_Text + ON dbo.TokenText(ResourceTypeId, SearchParamId, Text, ResourceSurrogateId) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenTokenCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + IsHistory BIT NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenTokenCompositeSearchParam + ADD CONSTRAINT DF_TokenTokenCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenTokenCompositeSearchParam + ADD CONSTRAINT CHK_TokenTokenCompositeSearchParam_CodeOverflow1 CHECK (LEN(Code1) = 256 + OR CodeOverflow1 IS NULL); + +ALTER TABLE dbo.TokenTokenCompositeSearchParam + ADD CONSTRAINT CHK_TokenTokenCompositeSearchParam_CodeOverflow2 CHECK (LEN(Code2) = 256 + OR CodeOverflow2 IS NULL); + +ALTER TABLE dbo.TokenTokenCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenTokenCompositeSearchParam + ON dbo.TokenTokenCompositeSearchParam(ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenTokenCompositeSearchParam_Code1_Code2 + ON dbo.TokenTokenCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, Code2, ResourceSurrogateId) + INCLUDE(SystemId1, SystemId2) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.Transactions ( + SurrogateIdRangeFirstValue BIGINT NOT NULL, + SurrogateIdRangeLastValue BIGINT NOT NULL, + Definition VARCHAR (2000) NULL, + IsCompleted BIT CONSTRAINT DF_Transactions_IsCompleted DEFAULT 0 NOT NULL, + IsSuccess BIT CONSTRAINT DF_Transactions_IsSuccess DEFAULT 0 NOT NULL, + IsVisible BIT CONSTRAINT DF_Transactions_IsVisible DEFAULT 0 NOT NULL, + IsHistoryMoved BIT CONSTRAINT DF_Transactions_IsHistoryMoved DEFAULT 0 NOT NULL, + CreateDate DATETIME CONSTRAINT DF_Transactions_CreateDate DEFAULT getUTCdate() NOT NULL, + EndDate DATETIME NULL, + VisibleDate DATETIME NULL, + HistoryMovedDate DATETIME NULL, + HeartbeatDate DATETIME CONSTRAINT DF_Transactions_HeartbeatDate DEFAULT getUTCdate() NOT NULL, + FailureReason VARCHAR (MAX) NULL, + IsControlledByClient BIT CONSTRAINT DF_Transactions_IsControlledByClient DEFAULT 1 NOT NULL, + InvisibleHistoryRemovedDate DATETIME NULL CONSTRAINT PKC_Transactions_SurrogateIdRangeFirstValue PRIMARY KEY CLUSTERED (SurrogateIdRangeFirstValue) +); + +CREATE INDEX IX_IsVisible + ON dbo.Transactions(IsVisible); + +CREATE TABLE dbo.UriSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Uri VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + IsHistory BIT NOT NULL +); + +ALTER TABLE dbo.UriSearchParam + ADD CONSTRAINT DF_UriSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.UriSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_UriSearchParam + ON dbo.UriSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_UriSearchParam_SearchParamId_Uri + ON dbo.UriSearchParam(ResourceTypeId, SearchParamId, Uri, ResourceSurrogateId) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.WatchdogLeases ( + Watchdog VARCHAR (100) NOT NULL, + LeaseHolder VARCHAR (100) CONSTRAINT DF_WatchdogLeases_LeaseHolder DEFAULT '' NOT NULL, + LeaseEndTime DATETIME CONSTRAINT DF_WatchdogLeases_LeaseEndTime DEFAULT 0 NOT NULL, + RemainingLeaseTimeSec AS datediff(second, getUTCdate(), LeaseEndTime), + LeaseRequestor VARCHAR (100) CONSTRAINT DF_WatchdogLeases_LeaseRequestor DEFAULT '' NOT NULL, + LeaseRequestTime DATETIME CONSTRAINT DF_WatchdogLeases_LeaseRequestTime DEFAULT 0 NOT NULL CONSTRAINT PKC_WatchdogLeases_Watchdog PRIMARY KEY CLUSTERED (Watchdog) +); + +COMMIT +GO +CREATE PROCEDURE dbo.AcquireExportJobs +@jobHeartbeatTimeoutThresholdInSeconds BIGINT, @maximumNumberOfConcurrentJobsAllowed INT +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DECLARE @expirationDateTime AS DATETIME2 (7); +SELECT @expirationDateTime = DATEADD(second, -@jobHeartbeatTimeoutThresholdInSeconds, SYSUTCDATETIME()); +DECLARE @numberOfRunningJobs AS INT; +SELECT @numberOfRunningJobs = COUNT(*) +FROM dbo.ExportJob WITH (TABLOCKX) +WHERE Status = 'Running' + AND HeartbeatDateTime > @expirationDateTime; +DECLARE @limit AS INT = @maximumNumberOfConcurrentJobsAllowed - @numberOfRunningJobs; +IF (@limit > 0) + BEGIN + DECLARE @availableJobs TABLE ( + Id VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + JobVersion BINARY (8) NOT NULL); + INSERT INTO @availableJobs + SELECT TOP (@limit) Id, + JobVersion + FROM dbo.ExportJob + WHERE (Status = 'Queued' + OR (Status = 'Running' + AND HeartbeatDateTime <= @expirationDateTime)) + ORDER BY HeartbeatDateTime; + DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); + UPDATE dbo.ExportJob + SET Status = 'Running', + HeartbeatDateTime = @heartbeatDateTime, + RawJobRecord = JSON_MODIFY(RawJobRecord, '$.status', 'Running') + OUTPUT inserted.RawJobRecord, inserted.JobVersion + FROM dbo.ExportJob AS job + INNER JOIN + @availableJobs AS availableJob + ON job.Id = availableJob.Id + AND job.JobVersion = availableJob.JobVersion; + END +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.AcquireReindexJobs +@jobHeartbeatTimeoutThresholdInSeconds BIGINT, @maximumNumberOfConcurrentJobsAllowed INT +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DECLARE @expirationDateTime AS DATETIME2 (7); +SELECT @expirationDateTime = DATEADD(second, -@jobHeartbeatTimeoutThresholdInSeconds, SYSUTCDATETIME()); +DECLARE @numberOfRunningJobs AS INT; +SELECT @numberOfRunningJobs = COUNT(*) +FROM dbo.ReindexJob WITH (TABLOCKX) +WHERE Status = 'Running' + AND HeartbeatDateTime > @expirationDateTime; +DECLARE @limit AS INT = @maximumNumberOfConcurrentJobsAllowed - @numberOfRunningJobs; +IF (@limit > 0) + BEGIN + DECLARE @availableJobs TABLE ( + Id VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + JobVersion BINARY (8) NOT NULL); + INSERT INTO @availableJobs + SELECT TOP (@limit) Id, + JobVersion + FROM dbo.ReindexJob + WHERE (Status = 'Queued' + OR (Status = 'Running' + AND HeartbeatDateTime <= @expirationDateTime)) + ORDER BY HeartbeatDateTime; + DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); + UPDATE dbo.ReindexJob + SET Status = 'Running', + HeartbeatDateTime = @heartbeatDateTime, + RawJobRecord = JSON_MODIFY(RawJobRecord, '$.status', 'Running') + OUTPUT inserted.RawJobRecord, inserted.JobVersion + FROM dbo.ReindexJob AS job + INNER JOIN + @availableJobs AS availableJob + ON job.Id = availableJob.Id + AND job.JobVersion = availableJob.JobVersion; + END +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.AcquireWatchdogLease +@Watchdog VARCHAR (100), @Worker VARCHAR (100), @AllowRebalance BIT=1, @ForceAcquire BIT=0, @LeasePeriodSec FLOAT, @WorkerIsRunning BIT=0, @LeaseEndTime DATETIME OUTPUT, @IsAcquired BIT OUTPUT, @CurrentLeaseHolder VARCHAR (100)=NULL OUTPUT +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +DECLARE @SP AS VARCHAR (100) = 'AcquireWatchdogLease', @Mode AS VARCHAR (100), @msg AS VARCHAR (1000), @MyLeasesNumber AS INT, @OtherValidRequestsOrLeasesNumber AS INT, @MyValidRequestsOrLeasesNumber AS INT, @DesiredLeasesNumber AS INT, @NotLeasedWatchdogNumber AS INT, @WatchdogNumber AS INT, @Now AS DATETIME, @MyLastChangeTime AS DATETIME, @PreviousLeaseHolder AS VARCHAR (100), @Rows AS INT = 0, @NumberOfWorkers AS INT, @st AS DATETIME = getUTCdate(), @RowsInt AS INT, @Pattern AS VARCHAR (100); +BEGIN TRY + SET @Mode = 'R=' + isnull(@Watchdog, 'NULL') + ' W=' + isnull(@Worker, 'NULL') + ' F=' + isnull(CONVERT (VARCHAR, @ForceAcquire), 'NULL') + ' LP=' + isnull(CONVERT (VARCHAR, @LeasePeriodSec), 'NULL'); + SET @CurrentLeaseHolder = ''; + SET @IsAcquired = 0; + SET @Now = getUTCdate(); + SET @LeaseEndTime = @Now; + SET @Pattern = NULLIF ((SELECT Char + FROM dbo.Parameters + WHERE Id = 'WatchdogLeaseHolderIncludePatternFor' + @Watchdog), ''); + IF @Pattern IS NULL + SET @Pattern = NULLIF ((SELECT Char + FROM dbo.Parameters + WHERE Id = 'WatchdogLeaseHolderIncludePattern'), ''); + IF @Pattern IS NOT NULL + AND @Worker NOT LIKE @Pattern + BEGIN + SET @msg = 'Worker does not match include pattern=' + @Pattern; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows, @Text = @msg; + SET @CurrentLeaseHolder = isnull((SELECT LeaseHolder + FROM dbo.WatchdogLeases + WHERE Watchdog = @Watchdog), ''); + RETURN; + END + SET @Pattern = NULLIF ((SELECT Char + FROM dbo.Parameters + WHERE Id = 'WatchdogLeaseHolderExcludePatternFor' + @Watchdog), ''); + IF @Pattern IS NULL + SET @Pattern = NULLIF ((SELECT Char + FROM dbo.Parameters + WHERE Id = 'WatchdogLeaseHolderExcludePattern'), ''); + IF @Pattern IS NOT NULL + AND @Worker LIKE @Pattern + BEGIN + SET @msg = 'Worker matches exclude pattern=' + @Pattern; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows, @Text = @msg; + SET @CurrentLeaseHolder = isnull((SELECT LeaseHolder + FROM dbo.WatchdogLeases + WHERE Watchdog = @Watchdog), ''); + RETURN; + END + DECLARE @Watchdogs TABLE ( + Watchdog VARCHAR (100) PRIMARY KEY); + INSERT INTO @Watchdogs + SELECT Watchdog + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE RemainingLeaseTimeSec * (-1) > 10 * @LeasePeriodSec + OR @ForceAcquire = 1 + AND Watchdog = @Watchdog + AND LeaseHolder <> @Worker; + IF @@rowcount > 0 + BEGIN + DELETE dbo.WatchdogLeases + WHERE Watchdog IN (SELECT Watchdog + FROM @Watchdogs); + SET @Rows += @@rowcount; + IF @Rows > 0 + BEGIN + SET @msg = ''; + SELECT @msg = CONVERT (VARCHAR (1000), @msg + CASE WHEN @msg = '' THEN '' ELSE ',' END + Watchdog) + FROM @Watchdogs; + SET @msg = CONVERT (VARCHAR (1000), 'Remove old/forced leases:' + @msg); + EXECUTE dbo.LogEvent @Process = 'AcquireWatchdogLease', @Status = 'Info', @Mode = @Mode, @Target = 'WatchdogLeases', @Action = 'Delete', @Rows = @Rows, @Text = @msg; + END + END + SET @NumberOfWorkers = 1 + (SELECT count(*) + FROM (SELECT LeaseHolder + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseHolder <> @Worker + UNION + SELECT LeaseRequestor + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseRequestor <> @Worker + AND LeaseRequestor <> '') AS A); + SET @Mode = CONVERT (VARCHAR (100), @Mode + ' N=' + CONVERT (VARCHAR (10), @NumberOfWorkers)); + IF NOT EXISTS (SELECT * + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE Watchdog = @Watchdog) + INSERT INTO dbo.WatchdogLeases (Watchdog, LeaseEndTime, LeaseRequestTime) + SELECT @Watchdog, + dateadd(day, -10, @Now), + dateadd(day, -10, @Now) + WHERE NOT EXISTS (SELECT * + FROM dbo.WatchdogLeases WITH (TABLOCKX) + WHERE Watchdog = @Watchdog); + SET @LeaseEndTime = dateadd(second, @LeasePeriodSec, @Now); + SET @WatchdogNumber = (SELECT count(*) + FROM dbo.WatchdogLeases WITH (NOLOCK)); + SET @NotLeasedWatchdogNumber = (SELECT count(*) + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseHolder = '' + OR LeaseEndTime < @Now); + SET @MyLeasesNumber = (SELECT count(*) + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseHolder = @Worker + AND LeaseEndTime > @Now); + SET @OtherValidRequestsOrLeasesNumber = (SELECT count(*) + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseHolder <> @Worker + AND LeaseEndTime > @Now + OR LeaseRequestor <> @Worker + AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec); + SET @MyValidRequestsOrLeasesNumber = (SELECT count(*) + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseHolder = @Worker + AND LeaseEndTime > @Now + OR LeaseRequestor = @Worker + AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec); + SET @DesiredLeasesNumber = ceiling(1.0 * @WatchdogNumber / @NumberOfWorkers); + IF @DesiredLeasesNumber = 0 + SET @DesiredLeasesNumber = 1; + IF @DesiredLeasesNumber = 1 + AND @OtherValidRequestsOrLeasesNumber = 1 + AND @WatchdogNumber = 1 + SET @DesiredLeasesNumber = 0; + IF @MyValidRequestsOrLeasesNumber = floor(1.0 * @WatchdogNumber / @NumberOfWorkers) + AND @OtherValidRequestsOrLeasesNumber + @MyValidRequestsOrLeasesNumber = @WatchdogNumber + SET @DesiredLeasesNumber = @DesiredLeasesNumber - 1; + UPDATE dbo.WatchdogLeases + SET LeaseHolder = @Worker, + LeaseEndTime = @LeaseEndTime, + LeaseRequestor = '', + @PreviousLeaseHolder = LeaseHolder + WHERE Watchdog = @Watchdog + AND NOT (LeaseRequestor <> @Worker + AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec) + AND (LeaseHolder = @Worker + AND (LeaseEndTime > @Now + OR @WorkerIsRunning = 1) + OR LeaseEndTime < @Now + AND (@DesiredLeasesNumber > @MyLeasesNumber + OR @OtherValidRequestsOrLeasesNumber < @WatchdogNumber)); + IF @@rowcount > 0 + BEGIN + SET @IsAcquired = 1; + SET @msg = 'Lease holder changed from [' + isnull(@PreviousLeaseHolder, '') + '] to [' + @Worker + ']'; + IF @PreviousLeaseHolder <> @Worker + EXECUTE dbo.LogEvent @Process = 'AcquireWatchdogLease', @Status = 'Info', @Mode = @Mode, @Text = @msg; + END + ELSE + IF @AllowRebalance = 1 + BEGIN + SET @CurrentLeaseHolder = (SELECT LeaseHolder + FROM dbo.WatchdogLeases + WHERE Watchdog = @Watchdog); + UPDATE dbo.WatchdogLeases + SET LeaseRequestTime = @Now + WHERE Watchdog = @Watchdog + AND LeaseRequestor = @Worker + AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec; + IF @DesiredLeasesNumber > @MyValidRequestsOrLeasesNumber + BEGIN + UPDATE A + SET LeaseRequestor = @Worker, + LeaseRequestTime = @Now + FROM dbo.WatchdogLeases AS A + WHERE Watchdog = @Watchdog + AND NOT (LeaseRequestor <> @Worker + AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec) + AND @NotLeasedWatchdogNumber = 0 + AND (SELECT count(*) + FROM dbo.WatchdogLeases AS B + WHERE B.LeaseHolder = A.LeaseHolder + AND datediff(second, B.LeaseEndTime, @Now) < @LeasePeriodSec) > @DesiredLeasesNumber; + SET @RowsInt = @@rowcount; + SET @msg = '@DesiredLeasesNumber=[' + CONVERT (VARCHAR (10), @DesiredLeasesNumber) + '] > @MyValidRequestsOrLeasesNumber=[' + CONVERT (VARCHAR (10), @MyValidRequestsOrLeasesNumber) + ']'; + EXECUTE dbo.LogEvent @Process = 'AcquireWatchdogLease', @Status = 'Info', @Mode = @Mode, @Rows = @RowsInt, @Text = @msg; + END + END + SET @Mode = CONVERT (VARCHAR (100), @Mode + ' A=' + CONVERT (VARCHAR (1), @IsAcquired)); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = 'AcquireWatchdogLease', @Status = 'Error', @Mode = @Mode; + THROW; +END CATCH + +GO +CREATE OR ALTER PROCEDURE dbo.AddPartitionOnResourceChanges +@partitionBoundary DATETIME2 (7) OUTPUT +AS +BEGIN + SET XACT_ABORT ON; + BEGIN TRANSACTION; + DECLARE @rightPartitionBoundary AS DATETIME2 (7) = CAST ((SELECT TOP (1) value + FROM sys.partition_range_values AS prv + INNER JOIN + sys.partition_functions AS pf + ON pf.function_id = prv.function_id + WHERE pf.name = N'PartitionFunction_ResourceChangeData_Timestamp' + ORDER BY prv.boundary_id DESC) AS DATETIME2 (7)); + DECLARE @timestamp AS DATETIME2 (7) = DATEADD(hour, DATEDIFF(hour, 0, sysutcdatetime()), 0); + IF (@rightPartitionBoundary < @timestamp) + BEGIN + SET @rightPartitionBoundary = @timestamp; + END + SET @rightPartitionBoundary = DATEADD(hour, 1, @rightPartitionBoundary); + ALTER PARTITION SCHEME PartitionScheme_ResourceChangeData_Timestamp NEXT USED [Primary]; + ALTER PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp( ) + SPLIT RANGE (@rightPartitionBoundary); + SET @partitionBoundary = @rightPartitionBoundary; + COMMIT TRANSACTION; +END + +GO +CREATE PROCEDURE dbo.ArchiveJobs +@QueueType TINYINT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'ArchiveJobs', @Mode AS VARCHAR (100) = '', @st AS DATETIME = getUTCdate(), @Rows AS INT = 0, @PartitionId AS TINYINT, @MaxPartitions AS TINYINT = 16, @LookedAtPartitions AS TINYINT = 0, @InflightRows AS INT = 0, @Lock AS VARCHAR (100) = 'DequeueJob_' + CONVERT (VARCHAR, @QueueType); +BEGIN TRY + SET @PartitionId = @MaxPartitions * rand(); + BEGIN TRANSACTION; + EXECUTE sp_getapplock @Lock, 'Exclusive'; + WHILE @LookedAtPartitions <= @MaxPartitions + BEGIN + SET @InflightRows += (SELECT count(*) + FROM dbo.JobQueue + WHERE PartitionId = @PartitionId + AND QueueType = @QueueType + AND Status IN (0, 1)); + SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; + SET @LookedAtPartitions = @LookedAtPartitions + 1; + END + IF @InflightRows = 0 + BEGIN + SET @LookedAtPartitions = 0; + WHILE @LookedAtPartitions <= @MaxPartitions + BEGIN + UPDATE dbo.JobQueue + SET Status = 5 + WHERE PartitionId = @PartitionId + AND QueueType = @QueueType + AND Status IN (2, 3, 4); + SET @Rows += @@rowcount; + SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; + SET @LookedAtPartitions = @LookedAtPartitions + 1; + END + END + COMMIT TRANSACTION; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.BatchDeleteResourceParams +@tableName NVARCHAR (128), @resourceTypeId SMALLINT, @startResourceSurrogateId BIGINT, @endResourceSurrogateId BIGINT, @batchSize INT +AS +SET XACT_ABORT ON; +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DECLARE @Sql AS NVARCHAR (MAX); +DECLARE @ParmDefinition AS NVARCHAR (512); +IF OBJECT_ID(@tableName) IS NOT NULL + BEGIN + SET @sql = N'DELETE TOP(@BatchSizeParam) FROM ' + @tableName + N' WITH (TABLOCK) WHERE ResourceTypeId = @ResourceTypeIdParam AND ResourceSurrogateId >= @StartResourceSurrogateIdParam AND ResourceSurrogateId < @EndResourceSurrogateIdParam'; + SET @parmDefinition = N'@BatchSizeParam int, @ResourceTypeIdParam smallint, @StartResourceSurrogateIdParam bigint, @EndResourceSurrogateIdParam bigint'; + EXECUTE sp_executesql @sql, @parmDefinition, @BatchSizeParam = @batchSize, @ResourceTypeIdParam = @resourceTypeId, @StartResourceSurrogateIdParam = @startResourceSurrogateId, @EndResourceSurrogateIdParam = @endResourceSurrogateId; + END +COMMIT TRANSACTION; +RETURN @@rowcount; + +GO +CREATE PROCEDURE dbo.BatchDeleteResources +@resourceTypeId SMALLINT, @startResourceSurrogateId BIGINT, @endResourceSurrogateId BIGINT, @batchSize INT +AS +SET XACT_ABORT ON; +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DELETE TOP (@batchSize) + dbo.Resource WITH (TABLOCK) +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId >= @startResourceSurrogateId + AND ResourceSurrogateId < @endResourceSurrogateId; +COMMIT TRANSACTION; +RETURN @@rowcount; + +GO +CREATE PROCEDURE dbo.BatchDeleteResourceWriteClaims +@startResourceSurrogateId BIGINT, @endResourceSurrogateId BIGINT, @batchSize INT +AS +SET XACT_ABORT ON; +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DELETE TOP (@batchSize) + dbo.ResourceWriteClaim WITH (TABLOCK) +WHERE ResourceSurrogateId >= @startResourceSurrogateId + AND ResourceSurrogateId < @endResourceSurrogateId; +COMMIT TRANSACTION; +RETURN @@rowcount; + +GO +CREATE PROCEDURE dbo.BulkMergeResource +@resources dbo.BulkImportResourceType_1 READONLY +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +MERGE INTO [dbo].[Resource] WITH (ROWLOCK, INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) + AS target +USING @resources AS source ON source.[ResourceTypeId] = target.[ResourceTypeId] + AND source.[ResourceId] = target.[ResourceId] + AND source.[Version] = target.[Version] +WHEN NOT MATCHED BY TARGET THEN INSERT ([ResourceTypeId], [ResourceId], [Version], [IsHistory], [ResourceSurrogateId], [IsDeleted], [RequestMethod], [RawResource], [IsRawResourceMetaSet], [SearchParamHash]) VALUES ([ResourceTypeId], [ResourceId], [Version], [IsHistory], [ResourceSurrogateId], [IsDeleted], [RequestMethod], [RawResource], [IsRawResourceMetaSet], [SearchParamHash]) OUTPUT Inserted.[ResourceSurrogateId]; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.BulkReindexResources_2 +@resourcesToReindex dbo.BulkReindexResourceTableType_1 READONLY, @resourceWriteClaims dbo.BulkResourceWriteClaimTableType_1 READONLY, @compartmentAssignments dbo.BulkCompartmentAssignmentTableType_1 READONLY, @referenceSearchParams dbo.BulkReferenceSearchParamTableType_1 READONLY, @tokenSearchParams dbo.BulkTokenSearchParamTableType_2 READONLY, @tokenTextSearchParams dbo.BulkTokenTextTableType_1 READONLY, @stringSearchParams dbo.BulkStringSearchParamTableType_2 READONLY, @numberSearchParams dbo.BulkNumberSearchParamTableType_1 READONLY, @quantitySearchParams dbo.BulkQuantitySearchParamTableType_1 READONLY, @uriSearchParams dbo.BulkUriSearchParamTableType_1 READONLY, @dateTimeSearchParms dbo.BulkDateTimeSearchParamTableType_2 READONLY, @referenceTokenCompositeSearchParams dbo.BulkReferenceTokenCompositeSearchParamTableType_2 READONLY, @tokenTokenCompositeSearchParams dbo.BulkTokenTokenCompositeSearchParamTableType_2 READONLY, @tokenDateTimeCompositeSearchParams dbo.BulkTokenDateTimeCompositeSearchParamTableType_2 READONLY, @tokenQuantityCompositeSearchParams dbo.BulkTokenQuantityCompositeSearchParamTableType_2 READONLY, @tokenStringCompositeSearchParams dbo.BulkTokenStringCompositeSearchParamTableType_2 READONLY, @tokenNumberNumberCompositeSearchParams dbo.BulkTokenNumberNumberCompositeSearchParamTableType_2 READONLY +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @computedValues TABLE ( + Offset INT NOT NULL, + ResourceTypeId SMALLINT NOT NULL, + VersionProvided BIGINT NULL, + SearchParamHash VARCHAR (64) NOT NULL, + ResourceSurrogateId BIGINT NULL, + VersionInDatabase BIGINT NULL); +INSERT INTO @computedValues +SELECT resourceToReindex.Offset, + resourceToReindex.ResourceTypeId, + resourceToReindex.ETag, + resourceToReindex.SearchParamHash, + resourceInDB.ResourceSurrogateId, + resourceInDB.Version +FROM @resourcesToReindex AS resourceToReindex + LEFT OUTER JOIN + dbo.Resource AS resourceInDB WITH (UPDLOCK, INDEX (IX_Resource_ResourceTypeId_ResourceId)) + ON resourceInDB.ResourceTypeId = resourceToReindex.ResourceTypeId + AND resourceInDB.ResourceId = resourceToReindex.ResourceId + AND resourceInDB.IsHistory = 0; +DECLARE @versionDiff AS INT; +SET @versionDiff = (SELECT COUNT(*) + FROM @computedValues + WHERE VersionProvided IS NOT NULL + AND VersionProvided <> VersionInDatabase); +IF (@versionDiff > 0) + BEGIN + DELETE @computedValues + WHERE VersionProvided IS NOT NULL + AND VersionProvided <> VersionInDatabase; + END +UPDATE resourceInDB +SET resourceInDB.SearchParamHash = resourceToReindex.SearchParamHash +FROM @computedValues AS resourceToReindex + INNER JOIN + dbo.Resource AS resourceInDB + ON resourceInDB.ResourceTypeId = resourceToReindex.ResourceTypeId + AND resourceInDB.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.ResourceWriteClaim AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.CompartmentAssignment AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.ReferenceSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenText AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.StringSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.UriSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.NumberSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.QuantitySearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.DateTimeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.ReferenceTokenCompositeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenTokenCompositeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenDateTimeCompositeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenQuantityCompositeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenStringCompositeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenNumberNumberCompositeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) +SELECT DISTINCT resourceToReindex.ResourceSurrogateId, + searchIndex.ClaimTypeId, + searchIndex.ClaimValue +FROM @resourceWriteClaims AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.CompartmentAssignment (ResourceTypeId, ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.CompartmentTypeId, + searchIndex.ReferenceResourceId, + 0 +FROM @compartmentAssignments AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.BaseUri, + searchIndex.ReferenceResourceTypeId, + searchIndex.ReferenceResourceId, + searchIndex.ReferenceResourceVersion, + 0 +FROM @referenceSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId, + searchIndex.Code, + searchIndex.CodeOverflow, + 0 +FROM @tokenSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.Text, + 0 +FROM @tokenTextSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsHistory, IsMin, IsMax) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.Text, + searchIndex.TextOverflow, + 0, + searchIndex.IsMin, + searchIndex.IsMax +FROM @stringSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.Uri, + 0 +FROM @uriSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SingleValue, + searchIndex.LowValue, + searchIndex.HighValue, + 0 +FROM @numberSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId, + searchIndex.QuantityCodeId, + searchIndex.SingleValue, + searchIndex.LowValue, + searchIndex.HighValue, + 0 +FROM @quantitySearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsHistory, IsMin, IsMax) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.StartDateTime, + searchIndex.EndDateTime, + searchIndex.IsLongerThanADay, + 0, + searchIndex.IsMin, + searchIndex.IsMax +FROM @dateTimeSearchParms AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.BaseUri1, + searchIndex.ReferenceResourceTypeId1, + searchIndex.ReferenceResourceId1, + searchIndex.ReferenceResourceVersion1, + searchIndex.SystemId2, + searchIndex.Code2, + searchIndex.CodeOverflow2, + 0 +FROM @referenceTokenCompositeSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId1, + searchIndex.Code1, + searchIndex.CodeOverflow1, + searchIndex.SystemId2, + searchIndex.Code2, + searchIndex.CodeOverflow2, + 0 +FROM @tokenTokenCompositeSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId1, + searchIndex.Code1, + searchIndex.CodeOverflow1, + searchIndex.StartDateTime2, + searchIndex.EndDateTime2, + searchIndex.IsLongerThanADay2, + 0 +FROM @tokenDateTimeCompositeSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId1, + searchIndex.Code1, + searchIndex.CodeOverflow1, + searchIndex.SingleValue2, + searchIndex.SystemId2, + searchIndex.QuantityCodeId2, + searchIndex.LowValue2, + searchIndex.HighValue2, + 0 +FROM @tokenQuantityCompositeSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId1, + searchIndex.Code1, + searchIndex.CodeOverflow1, + searchIndex.Text2, + searchIndex.TextOverflow2, + 0 +FROM @tokenStringCompositeSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId1, + searchIndex.Code1, + searchIndex.CodeOverflow1, + searchIndex.SingleValue2, + searchIndex.LowValue2, + searchIndex.HighValue2, + searchIndex.SingleValue3, + searchIndex.LowValue3, + searchIndex.HighValue3, + searchIndex.HasRange, + 0 +FROM @tokenNumberNumberCompositeSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +SELECT @versionDiff; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE [dbo].[CancelTask] +@taskId VARCHAR (64) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +IF NOT EXISTS (SELECT * + FROM [dbo].[TaskInfo] + WHERE TaskId = @taskId) + BEGIN + THROW 50404, 'Task not exist', 1; + END +UPDATE dbo.TaskInfo +SET IsCanceled = 1, + HeartbeatDateTime = @heartbeatDateTime +WHERE TaskId = @taskId; +SELECT TaskId, + QueueId, + Status, + TaskTypeId, + RunId, + IsCanceled, + RetryCount, + MaxRetryCount, + HeartbeatDateTime, + InputData, + TaskContext, + Result +FROM [dbo].[TaskInfo] +WHERE TaskId = @taskId; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.CaptureResourceChanges +@isDeleted BIT, @version INT, @resourceId VARCHAR (64), @resourceTypeId SMALLINT +AS +BEGIN + DECLARE @changeType AS SMALLINT; + IF (@isDeleted = 1) + BEGIN + SET @changeType = 2; + END + ELSE + BEGIN + IF (@version = 1) + BEGIN + SET @changeType = 0; + END + ELSE + BEGIN + SET @changeType = 1; + END + END + INSERT INTO dbo.ResourceChangeData (ResourceId, ResourceTypeId, ResourceVersion, ResourceChangeTypeId) + VALUES (@resourceId, @resourceTypeId, @version, @changeType); +END + +GO +CREATE PROCEDURE dbo.CaptureResourceIdsForChanges +@Resources dbo.ResourceList READONLY +AS +SET NOCOUNT ON; +INSERT INTO dbo.ResourceChangeData (ResourceId, ResourceTypeId, ResourceVersion, ResourceChangeTypeId) +SELECT ResourceId, + ResourceTypeId, + Version, + CASE WHEN IsDeleted = 1 THEN 2 WHEN Version > 1 THEN 1 ELSE 0 END +FROM @Resources +WHERE IsHistory = 0; + +GO +CREATE PROCEDURE dbo.CheckActiveReindexJobs +AS +SET NOCOUNT ON; +SELECT Id +FROM dbo.ReindexJob +WHERE Status = 'Running' + OR Status = 'Queued' + OR Status = 'Paused'; + +GO +CREATE PROCEDURE dbo.CleanupEventLog +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'CleanupEventLog', @Mode AS VARCHAR (100) = '', @MaxDeleteRows AS INT, @MaxAllowedRows AS BIGINT, @RetentionPeriodSecond AS INT, @DeletedRows AS INT, @TotalDeletedRows AS INT = 0, @TotalRows AS INT, @Now AS DATETIME = getUTCdate(); +EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; +BEGIN TRY + SET @MaxDeleteRows = (SELECT Number + FROM dbo.Parameters + WHERE Id = 'CleanupEventLog.DeleteBatchSize'); + IF @MaxDeleteRows IS NULL + RAISERROR ('Cannot get Parameter.CleanupEventLog.DeleteBatchSize', 18, 127); + SET @MaxAllowedRows = (SELECT Number + FROM dbo.Parameters + WHERE Id = 'CleanupEventLog.AllowedRows'); + IF @MaxAllowedRows IS NULL + RAISERROR ('Cannot get Parameter.CleanupEventLog.AllowedRows', 18, 127); + SET @RetentionPeriodSecond = (SELECT Number * 24 * 60 * 60 + FROM dbo.Parameters + WHERE Id = 'CleanupEventLog.RetentionPeriodDay'); + IF @RetentionPeriodSecond IS NULL + RAISERROR ('Cannot get Parameter.CleanupEventLog.RetentionPeriodDay', 18, 127); + SET @TotalRows = (SELECT sum(row_count) + FROM sys.dm_db_partition_stats + WHERE object_id = object_id('EventLog') + AND index_id IN (0, 1)); + SET @DeletedRows = 1; + WHILE @DeletedRows > 0 + AND EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = 'CleanupEventLog.IsEnabled' + AND Number = 1) + BEGIN + SET @DeletedRows = 0; + IF @TotalRows - @TotalDeletedRows > @MaxAllowedRows + BEGIN + DELETE TOP (@MaxDeleteRows) + dbo.EventLog WITH (PAGLOCK) + WHERE EventDate <= dateadd(second, -@RetentionPeriodSecond, @Now); + SET @DeletedRows = @@rowcount; + SET @TotalDeletedRows += @DeletedRows; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'EventLog', @Action = 'Delete', @Rows = @DeletedRows, @Text = @TotalDeletedRows; + END + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @Now; +END TRY +BEGIN CATCH + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.CompleteTask +@taskId VARCHAR (64), @taskResult VARCHAR (MAX), @runId VARCHAR (50) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +IF NOT EXISTS (SELECT * + FROM [dbo].[TaskInfo] + WHERE TaskId = @taskId + AND RunId = @runId) + BEGIN + THROW 50404, 'Task not exist or runid not match', 1; + END +UPDATE dbo.TaskInfo +SET Status = 3, + EndDateTime = SYSUTCDATETIME(), + Result = @taskResult +WHERE TaskId = @taskId; +COMMIT TRANSACTION; +EXECUTE dbo.GetTaskDetails @TaskId = @taskId; + +GO +CREATE OR ALTER PROCEDURE dbo.ConfigurePartitionOnResourceChanges +@numberOfFuturePartitionsToAdd INT +AS +BEGIN + SET XACT_ABORT ON; + BEGIN TRANSACTION; + DECLARE @partitionBoundary AS DATETIME2 (7) = DATEADD(hour, DATEDIFF(hour, 0, sysutcdatetime()), 0); + DECLARE @startingRightPartitionBoundary AS DATETIME2 (7) = CAST ((SELECT TOP (1) value + FROM sys.partition_range_values AS prv + INNER JOIN + sys.partition_functions AS pf + ON pf.function_id = prv.function_id + WHERE pf.name = N'PartitionFunction_ResourceChangeData_Timestamp' + ORDER BY prv.boundary_id DESC) AS DATETIME2 (7)); + DECLARE @numberOfPartitionsToAdd AS INT = @numberOfFuturePartitionsToAdd + 1; + WHILE @numberOfPartitionsToAdd > 0 + BEGIN + IF (@startingRightPartitionBoundary < @partitionBoundary) + BEGIN + ALTER PARTITION SCHEME PartitionScheme_ResourceChangeData_Timestamp NEXT USED [PRIMARY]; + ALTER PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp( ) + SPLIT RANGE (@partitionBoundary); + END + SET @partitionBoundary = DATEADD(hour, 1, @partitionBoundary); + SET @numberOfPartitionsToAdd -= 1; + END + COMMIT TRANSACTION; +END + +GO +CREATE PROCEDURE dbo.CreateExportJob +@id VARCHAR (64), @hash VARCHAR (64), @status VARCHAR (10), @rawJobRecord VARCHAR (MAX) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +INSERT INTO dbo.ExportJob (Id, Hash, Status, HeartbeatDateTime, RawJobRecord) +VALUES (@id, @hash, @status, @heartbeatDateTime, @rawJobRecord); +SELECT CAST (MIN_ACTIVE_ROWVERSION() AS INT); +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.CreateReindexJob +@id VARCHAR (64), @status VARCHAR (10), @rawJobRecord VARCHAR (MAX) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +INSERT INTO dbo.ReindexJob (Id, Status, HeartbeatDateTime, RawJobRecord) +VALUES (@id, @status, @heartbeatDateTime, @rawJobRecord); +SELECT CAST (MIN_ACTIVE_ROWVERSION() AS INT); +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE [dbo].[CreateTask_3] +@taskId VARCHAR (64), @queueId VARCHAR (64), @taskTypeId SMALLINT, @parentTaskId VARCHAR (64), @maxRetryCount SMALLINT=3, @inputData VARCHAR (MAX), @isUniqueTaskByType BIT +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +DECLARE @status AS SMALLINT = 1; +DECLARE @retryCount AS SMALLINT = 0; +DECLARE @isCanceled AS BIT = 0; +IF (@isUniqueTaskByType = 1) + BEGIN + IF EXISTS (SELECT * + FROM [dbo].[TaskInfo] + WHERE TaskId = @taskId + OR (TaskTypeId = @taskTypeId + AND Status <> 3)) + BEGIN + THROW 50409, 'Task already existed', 1; + END + END +ELSE + BEGIN + IF EXISTS (SELECT * + FROM [dbo].[TaskInfo] + WHERE TaskId = @taskId) + BEGIN + THROW 50409, 'Task already existed', 1; + END + END +INSERT INTO [dbo].[TaskInfo] (TaskId, QueueId, Status, TaskTypeId, IsCanceled, RetryCount, MaxRetryCount, HeartbeatDateTime, InputData, ParentTaskId) +VALUES (@taskId, @queueId, @status, @taskTypeId, @isCanceled, @retryCount, @maxRetryCount, @heartbeatDateTime, @inputData, @parentTaskId); +EXECUTE dbo.GetTaskDetails @TaskId = @taskId; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.Defrag +@TableName VARCHAR (100), @IndexName VARCHAR (200), @PartitionNumber INT, @IsPartitioned BIT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'Defrag', @Mode AS VARCHAR (200) = @TableName + '.' + @IndexName + '.' + CONVERT (VARCHAR, @PartitionNumber) + '.' + CONVERT (VARCHAR, @IsPartitioned), @st AS DATETIME = getUTCdate(), @SQL AS VARCHAR (3500), @msg AS VARCHAR (1000), @SizeBefore AS FLOAT, @SizeAfter AS FLOAT, @IndexId AS INT; +BEGIN TRY + SET @IndexId = (SELECT index_id + FROM sys.indexes + WHERE object_id = object_id(@TableName) + AND name = @IndexName); + SET @SizeBefore = (SELECT sum(reserved_page_count) + FROM sys.dm_db_partition_stats + WHERE object_id = object_id(@TableName) + AND index_id = @IndexId) * 8.0 / 1024 / 1024; + SET @msg = 'Size[GB] before=' + CONVERT (VARCHAR, @SizeBefore); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start', @Text = @msg; + SET @Sql = 'ALTER INDEX ' + quotename(@IndexName) + ' ON dbo.' + quotename(@TableName) + ' REORGANIZE' + CASE WHEN @IsPartitioned = 1 THEN ' PARTITION = ' + CONVERT (VARCHAR, @PartitionNumber) ELSE '' END; + BEGIN TRY + EXECUTE (@Sql); + SET @SizeAfter = (SELECT sum(reserved_page_count) + FROM sys.dm_db_partition_stats + WHERE object_id = object_id(@TableName) + AND index_id = @IndexId) * 8.0 / 1024 / 1024; + SET @msg = 'Size[GB] before=' + CONVERT (VARCHAR, @SizeBefore) + ', after=' + CONVERT (VARCHAR, @SizeAfter) + ', reduced by=' + CONVERT (VARCHAR, @SizeBefore - @SizeAfter); + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'End', @Mode = @Mode, @Action = 'Reorganize', @Start = @st, @Text = @msg; + END TRY + BEGIN CATCH + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'Error', @Mode = @Mode, @Action = 'Reorganize', @Start = @st, @ReRaisError = 0; + END CATCH +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.DefragChangeDatabaseSettings +@IsOn BIT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'DefragChangeDatabaseSettings', @Mode AS VARCHAR (200) = 'On=' + CONVERT (VARCHAR, @IsOn), @st AS DATETIME = getUTCdate(), @SQL AS VARCHAR (3500); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'Start', @Mode = @Mode; + SET @SQL = 'ALTER DATABASE CURRENT SET AUTO_UPDATE_STATISTICS ' + CASE WHEN @IsOn = 1 THEN 'ON' ELSE 'OFF' END; + EXECUTE (@SQL); + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'Run', @Mode = @Mode, @Text = @SQL; + SET @SQL = 'ALTER DATABASE CURRENT SET AUTO_CREATE_STATISTICS ' + CASE WHEN @IsOn = 1 THEN 'ON' ELSE 'OFF' END; + EXECUTE (@SQL); + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'End', @Mode = @Mode, @Start = @st, @Text = @SQL; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.DeleteHistory +@DeleteResources BIT=0, @Reset BIT=0, @DisableLogEvent BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'DeleteHistory', @Mode AS VARCHAR (100) = 'D=' + isnull(CONVERT (VARCHAR, @DeleteResources), 'NULL') + ' R=' + isnull(CONVERT (VARCHAR, @Reset), 'NULL'), @st AS DATETIME = getUTCdate(), @Id AS VARCHAR (100) = 'DeleteHistory.LastProcessed.TypeId.SurrogateId', @ResourceTypeId AS SMALLINT, @SurrogateId AS BIGINT, @RowsToProcess AS INT, @ProcessedResources AS INT = 0, @DeletedResources AS INT = 0, @DeletedSearchParams AS INT = 0, @ReportDate AS DATETIME = getUTCdate(); +BEGIN TRY + IF @DisableLogEvent = 0 + INSERT INTO dbo.Parameters (Id, Char) + SELECT @SP, + 'LogEvent'; + ELSE + DELETE dbo.Parameters + WHERE Id = @SP; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + INSERT INTO dbo.Parameters (Id, Char) + SELECT @Id, + '0.0' + WHERE NOT EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = @Id); + DECLARE @LastProcessed AS VARCHAR (100) = CASE WHEN @Reset = 0 THEN (SELECT Char + FROM dbo.Parameters + WHERE Id = @Id) ELSE '0.0' END; + DECLARE @Types TABLE ( + ResourceTypeId SMALLINT PRIMARY KEY, + Name VARCHAR (100)); + DECLARE @SurrogateIds TABLE ( + ResourceSurrogateId BIGINT PRIMARY KEY, + IsHistory BIT ); + INSERT INTO @Types + EXECUTE dbo.GetUsedResourceTypes ; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = '@Types', @Action = 'Insert', @Rows = @@rowcount; + SET @ResourceTypeId = substring(@LastProcessed, 1, charindex('.', @LastProcessed) - 1); + SET @SurrogateId = substring(@LastProcessed, charindex('.', @LastProcessed) + 1, 255); + DELETE @Types + WHERE ResourceTypeId < @ResourceTypeId; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = '@Types', @Action = 'Delete', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Types) + BEGIN + SET @ResourceTypeId = (SELECT TOP 1 ResourceTypeId + FROM @Types + ORDER BY ResourceTypeId); + SET @ProcessedResources = 0; + SET @DeletedResources = 0; + SET @DeletedSearchParams = 0; + SET @RowsToProcess = 1; + WHILE @RowsToProcess > 0 + BEGIN + DELETE @SurrogateIds; + INSERT INTO @SurrogateIds + SELECT TOP 10000 ResourceSurrogateId, + IsHistory + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId > @SurrogateId + ORDER BY ResourceSurrogateId; + SET @RowsToProcess = @@rowcount; + SET @ProcessedResources += @RowsToProcess; + IF @RowsToProcess > 0 + SET @SurrogateId = (SELECT max(ResourceSurrogateId) + FROM @SurrogateIds); + SET @LastProcessed = CONVERT (VARCHAR, @ResourceTypeId) + '.' + CONVERT (VARCHAR, @SurrogateId); + DELETE @SurrogateIds + WHERE IsHistory = 0; + IF EXISTS (SELECT * + FROM @SurrogateIds) + BEGIN + DELETE dbo.ResourceWriteClaim + WHERE ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.CompartmentAssignment + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.ReferenceSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenText + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.StringSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.UriSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.NumberSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.QuantitySearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.DateTimeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.ReferenceTokenCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenTokenCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenDateTimeCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenQuantityCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenStringCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenNumberNumberCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + IF @DeleteResources = 1 + BEGIN + DELETE dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedResources += @@rowcount; + END + END + UPDATE dbo.Parameters + SET Char = @LastProcessed + WHERE Id = @Id; + IF datediff(second, @ReportDate, getUTCdate()) > 60 + BEGIN + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'Resource', @Action = 'Select', @Rows = @ProcessedResources, @Text = @LastProcessed; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = '*SearchParam', @Action = 'Delete', @Rows = @DeletedSearchParams, @Text = @LastProcessed; + IF @DeleteResources = 1 + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'Resource', @Action = 'Delete', @Rows = @DeletedResources, @Text = @LastProcessed; + SET @ReportDate = getUTCdate(); + SET @ProcessedResources = 0; + SET @DeletedSearchParams = 0; + SET @DeletedResources = 0; + END + END + DELETE @Types + WHERE ResourceTypeId = @ResourceTypeId; + SET @SurrogateId = 0; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'Resource', @Action = 'Select', @Rows = @ProcessedResources, @Text = @LastProcessed; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = '*SearchParam', @Action = 'Delete', @Rows = @DeletedSearchParams, @Text = @LastProcessed; + IF @DeleteResources = 1 + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'Resource', @Action = 'Delete', @Rows = @DeletedResources, @Text = @LastProcessed; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.DequeueJob +@QueueType TINYINT, @Worker VARCHAR (100), @HeartbeatTimeoutSec INT, @InputJobId BIGINT=NULL, @CheckTimeoutJobs BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'DequeueJob', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' H=' + isnull(CONVERT (VARCHAR, @HeartbeatTimeoutSec), 'NULL') + ' W=' + isnull(@Worker, 'NULL') + ' IJ=' + isnull(CONVERT (VARCHAR, @InputJobId), 'NULL') + ' T=' + isnull(CONVERT (VARCHAR, @CheckTimeoutJobs), 'NULL'), @Rows AS INT = 0, @st AS DATETIME = getUTCdate(), @JobId AS BIGINT, @msg AS VARCHAR (100), @Lock AS VARCHAR (100), @PartitionId AS TINYINT, @MaxPartitions AS TINYINT = 16, @LookedAtPartitions AS TINYINT = 0; +BEGIN TRY + IF EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = 'DequeueJobStop' + AND Number = 1) + BEGIN + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = 0, @Text = 'Skipped'; + RETURN; + END + IF @InputJobId IS NULL + SET @PartitionId = @MaxPartitions * rand(); + ELSE + SET @PartitionId = @InputJobId % 16; + SET TRANSACTION ISOLATION LEVEL READ COMMITTED; + WHILE @InputJobId IS NULL + AND @JobId IS NULL + AND @LookedAtPartitions <= @MaxPartitions + AND @CheckTimeoutJobs = 0 + BEGIN + SET @Lock = 'DequeueJob_' + CONVERT (VARCHAR, @QueueType) + '_' + CONVERT (VARCHAR, @PartitionId); + BEGIN TRANSACTION; + EXECUTE sp_getapplock @Lock, 'Exclusive'; + UPDATE T + SET StartDate = getUTCdate(), + HeartbeatDate = getUTCdate(), + Worker = @Worker, + Status = 1, + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()), + @JobId = T.JobId + FROM dbo.JobQueue AS T WITH (PAGLOCK) + INNER JOIN + (SELECT TOP 1 JobId + FROM dbo.JobQueue WITH (INDEX (IX_QueueType_PartitionId_Status_Priority)) + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND Status = 0 + ORDER BY Priority, JobId) AS S + ON QueueType = @QueueType + AND PartitionId = @PartitionId + AND T.JobId = S.JobId; + SET @Rows += @@rowcount; + COMMIT TRANSACTION; + IF @JobId IS NULL + BEGIN + SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; + SET @LookedAtPartitions = @LookedAtPartitions + 1; + END + END + SET @LookedAtPartitions = 0; + WHILE @InputJobId IS NULL + AND @JobId IS NULL + AND @LookedAtPartitions <= @MaxPartitions + BEGIN + SET @Lock = 'DequeueStoreCopyWorkUnit_' + CONVERT (VARCHAR, @PartitionId); + BEGIN TRANSACTION; + EXECUTE sp_getapplock @Lock, 'Exclusive'; + UPDATE T + SET StartDate = getUTCdate(), + HeartbeatDate = getUTCdate(), + Worker = @Worker, + Status = CASE WHEN CancelRequested = 0 THEN 1 ELSE 4 END, + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()), + @JobId = CASE WHEN CancelRequested = 0 THEN T.JobId END, + Info = CONVERT (VARCHAR (1000), isnull(Info, '') + ' Prev: Worker=' + Worker + ' Start=' + CONVERT (VARCHAR, StartDate, 121)) + FROM dbo.JobQueue AS T WITH (PAGLOCK) + INNER JOIN + (SELECT TOP 1 JobId + FROM dbo.JobQueue WITH (INDEX (IX_QueueType_PartitionId_Status_Priority)) + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND Status = 1 + AND datediff(second, HeartbeatDate, getUTCdate()) > @HeartbeatTimeoutSec + ORDER BY Priority, JobId) AS S + ON QueueType = @QueueType + AND PartitionId = @PartitionId + AND T.JobId = S.JobId; + SET @Rows += @@rowcount; + COMMIT TRANSACTION; + IF @JobId IS NULL + BEGIN + SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; + SET @LookedAtPartitions = @LookedAtPartitions + 1; + END + END + IF @InputJobId IS NOT NULL + BEGIN + UPDATE dbo.JobQueue WITH (PAGLOCK) + SET StartDate = getUTCdate(), + HeartbeatDate = getUTCdate(), + Worker = @Worker, + Status = 1, + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()), + @JobId = JobId + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND Status = 0 + AND JobId = @InputJobId; + SET @Rows += @@rowcount; + IF @JobId IS NULL + BEGIN + UPDATE dbo.JobQueue WITH (PAGLOCK) + SET StartDate = getUTCdate(), + HeartbeatDate = getUTCdate(), + Worker = @Worker, + Status = 1, + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()), + @JobId = JobId + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND Status = 1 + AND JobId = @InputJobId + AND datediff(second, HeartbeatDate, getUTCdate()) > @HeartbeatTimeoutSec; + SET @Rows += @@rowcount; + END + END + IF @JobId IS NOT NULL + EXECUTE dbo.GetJobs @QueueType = @QueueType, @JobId = @JobId; + SET @msg = 'J=' + isnull(CONVERT (VARCHAR, @JobId), 'NULL') + ' P=' + CONVERT (VARCHAR, @PartitionId); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows, @Text = @msg; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.DisableIndex +@tableName NVARCHAR (128), @indexName NVARCHAR (128) +WITH EXECUTE AS 'dbo' +AS +DECLARE @errorTxt AS VARCHAR (1000), @sql AS NVARCHAR (1000), @isDisabled AS BIT; +IF object_id(@tableName) IS NULL + BEGIN + SET @errorTxt = @tableName + ' does not exist or you don''t have permissions.'; + RAISERROR (@errorTxt, 18, 127); + END +SET @isDisabled = (SELECT is_disabled + FROM sys.indexes + WHERE object_id = object_id(@tableName) + AND name = @indexName); +IF @isDisabled IS NULL + BEGIN + SET @errorTxt = @indexName + ' does not exist or you don''t have permissions.'; + RAISERROR (@errorTxt, 18, 127); + END +IF @isDisabled = 0 + BEGIN + SET @sql = N'ALTER INDEX ' + QUOTENAME(@indexName) + N' on ' + @tableName + ' Disable'; + EXECUTE sp_executesql @sql; + END + +GO +CREATE PROCEDURE dbo.DisableIndexes +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'DisableIndexes', @Mode AS VARCHAR (200) = '', @st AS DATETIME = getUTCdate(), @Tbl AS VARCHAR (100), @Ind AS VARCHAR (200), @Txt AS VARCHAR (4000); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + DECLARE @Tables TABLE ( + Tbl VARCHAR (100) PRIMARY KEY, + Supported BIT ); + INSERT INTO @Tables + EXECUTE dbo.GetPartitionedTables @IncludeNotDisabled = 1, @IncludeNotSupported = 0; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Tables', @Action = 'Insert', @Rows = @@rowcount; + DECLARE @Indexes TABLE ( + Tbl VARCHAR (100), + Ind VARCHAR (200), + TblId INT , + IndId INT PRIMARY KEY (Tbl, Ind)); + INSERT INTO @Indexes + SELECT Tbl, + I.Name, + TblId, + I.index_id + FROM (SELECT object_id(Tbl) AS TblId, + Tbl + FROM @Tables) AS O + INNER JOIN + sys.indexes AS I + ON I.object_id = TblId; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Insert', @Rows = @@rowcount; + INSERT INTO dbo.IndexProperties (TableName, IndexName, PropertyName, PropertyValue) + SELECT Tbl, + Ind, + 'DATA_COMPRESSION', + data_comp + FROM (SELECT Tbl, + Ind, + isnull((SELECT TOP 1 CASE WHEN data_compression_desc = 'PAGE' THEN 'PAGE' END + FROM sys.partitions + WHERE object_id = TblId + AND index_id = IndId), 'NONE') AS data_comp + FROM @Indexes) AS A + WHERE NOT EXISTS (SELECT * + FROM dbo.IndexProperties + WHERE TableName = Tbl + AND IndexName = Ind); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = 'IndexProperties', @Action = 'Insert', @Rows = @@rowcount; + DELETE @Indexes + WHERE Tbl = 'Resource' + OR IndId = 1; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Delete', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Indexes) + BEGIN + SELECT TOP 1 @Tbl = Tbl, + @Ind = Ind + FROM @Indexes; + SET @Txt = 'ALTER INDEX ' + @Ind + ' ON dbo.' + @Tbl + ' DISABLE'; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Ind, @Action = 'Disable', @Text = @Txt; + DELETE @Indexes + WHERE Tbl = @Tbl + AND Ind = @Ind; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.EnqueueJobs +@QueueType TINYINT, @Definitions StringList READONLY, @GroupId BIGINT=NULL, @ForceOneActiveJobGroup BIT=1, @IsCompleted BIT=NULL, @ReturnJobs BIT=1 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'EnqueueJobs', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' D=' + CONVERT (VARCHAR, (SELECT count(*) + FROM @Definitions)) + ' G=' + isnull(CONVERT (VARCHAR, @GroupId), 'NULL') + ' F=' + isnull(CONVERT (VARCHAR, @ForceOneActiveJobGroup), 'NULL') + ' C=' + isnull(CONVERT (VARCHAR, @IsCompleted), 'NULL'), @st AS DATETIME = getUTCdate(), @Lock AS VARCHAR (100) = 'EnqueueJobs_' + CONVERT (VARCHAR, @QueueType), @MaxJobId AS BIGINT, @Rows AS INT, @msg AS VARCHAR (1000), @JobIds AS BigintList, @InputRows AS INT; +BEGIN TRY + DECLARE @Input TABLE ( + DefinitionHash VARBINARY (20) PRIMARY KEY, + Definition VARCHAR (MAX) ); + INSERT INTO @Input + SELECT hashbytes('SHA1', String) AS DefinitionHash, + String AS Definition + FROM @Definitions; + SET @InputRows = @@rowcount; + INSERT INTO @JobIds + SELECT JobId + FROM @Input AS A + INNER JOIN + dbo.JobQueue AS B + ON B.QueueType = @QueueType + AND B.DefinitionHash = A.DefinitionHash + AND B.Status <> 5; + IF @@rowcount < @InputRows + BEGIN + BEGIN TRANSACTION; + EXECUTE sp_getapplock @Lock, 'Exclusive'; + IF @ForceOneActiveJobGroup = 1 + AND EXISTS (SELECT * + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND Status IN (0, 1) + AND (@GroupId IS NULL + OR GroupId <> @GroupId)) + RAISERROR ('There are other active job groups', 18, 127); + SET @MaxJobId = isnull((SELECT TOP 1 JobId + FROM dbo.JobQueue + WHERE QueueType = @QueueType + ORDER BY JobId DESC), 0); + INSERT INTO dbo.JobQueue (QueueType, GroupId, JobId, Definition, DefinitionHash, Status) + OUTPUT inserted.JobId INTO @JobIds + SELECT @QueueType, + isnull(@GroupId, @MaxJobId + 1) AS GroupId, + JobId, + Definition, + DefinitionHash, + CASE WHEN @IsCompleted = 1 THEN 2 ELSE 0 END AS Status + FROM (SELECT @MaxJobId + row_number() OVER (ORDER BY Dummy) AS JobId, + * + FROM (SELECT *, + 0 AS Dummy + FROM @Input) AS A) AS A + WHERE NOT EXISTS (SELECT * + FROM dbo.JobQueue AS B + WHERE B.QueueType = @QueueType + AND B.DefinitionHash = A.DefinitionHash + AND B.Status <> 5); + SET @Rows = @@rowcount; + COMMIT TRANSACTION; + END + IF @ReturnJobs = 1 + EXECUTE dbo.GetJobs @QueueType = @QueueType, @JobIds = @JobIds; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.ExecuteCommandForRebuildIndexes +@Tbl VARCHAR (100), @Ind VARCHAR (1000), @Cmd VARCHAR (MAX) +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'ExecuteCommandForRebuildIndexes', @Mode AS VARCHAR (200) = 'Tbl=' + isnull(@Tbl, 'NULL'), @st AS DATETIME, @Retries AS INT = 0, @Action AS VARCHAR (100), @msg AS VARCHAR (1000); +RetryOnTempdbError: +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start', @Text = @Cmd; + SET @st = getUTCdate(); + IF @Tbl IS NULL + RAISERROR ('@Tbl IS NULL', 18, 127); + IF @Cmd IS NULL + RAISERROR ('@Cmd IS NULL', 18, 127); + SET @Action = CASE WHEN @Cmd LIKE 'UPDATE STAT%' THEN 'Update statistics' WHEN @Cmd LIKE 'CREATE%INDEX%' THEN 'Create Index' WHEN @Cmd LIKE 'ALTER%INDEX%REBUILD%' THEN 'Rebuild Index' WHEN @Cmd LIKE 'ALTER%TABLE%ADD%' THEN 'Add Constraint' END; + IF @Action IS NULL + BEGIN + SET @msg = 'Not supported command = ' + CONVERT (VARCHAR (900), @Cmd); + RAISERROR (@msg, 18, 127); + END + IF @Action = 'Create Index' + WAITFOR DELAY '00:00:05'; + EXECUTE (@Cmd); + SELECT @Ind; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Action = @Action, @Status = 'End', @Start = @st, @Text = @Cmd; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + IF error_number() = 40544 + BEGIN + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st, @Retry = @Retries; + SET @Retries = @Retries + 1; + IF @Tbl = 'TokenText_96' + WAITFOR DELAY '01:00:00'; + ELSE + WAITFOR DELAY '00:10:00'; + GOTO RetryOnTempdbError; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE OR ALTER PROCEDURE dbo.FetchEventAgentCheckpoint +@CheckpointId VARCHAR (64) +AS +BEGIN + SELECT TOP (1) CheckpointId, + LastProcessedDateTime, + LastProcessedIdentifier + FROM dbo.EventAgentCheckpoint + WHERE CheckpointId = @CheckpointId; +END + +GO +CREATE PROCEDURE dbo.FetchResourceChanges_3 +@startId BIGINT, @lastProcessedUtcDateTime DATETIME2 (7), @pageSize SMALLINT +AS +BEGIN + SET NOCOUNT ON; + DECLARE @precedingPartitionBoundary AS DATETIME2 (7) = (SELECT TOP (1) CAST (prv.value AS DATETIME2 (7)) AS value + FROM sys.partition_range_values AS prv WITH (NOLOCK) + INNER JOIN + sys.partition_functions AS pf WITH (NOLOCK) + ON pf.function_id = prv.function_id + WHERE pf.name = N'PartitionFunction_ResourceChangeData_Timestamp' + AND SQL_VARIANT_PROPERTY(prv.Value, 'BaseType') = 'datetime2' + AND CAST (prv.value AS DATETIME2 (7)) < DATEADD(HOUR, DATEDIFF(HOUR, 0, @lastProcessedUtcDateTime), 0) + ORDER BY prv.boundary_id DESC); + IF (@precedingPartitionBoundary IS NULL) + BEGIN + SET @precedingPartitionBoundary = CONVERT (DATETIME2 (7), N'1970-01-01T00:00:00.0000000'); + END + DECLARE @endDateTimeToFilter AS DATETIME2 (7) = DATEADD(HOUR, 1, SYSUTCDATETIME()); + WITH PartitionBoundaries + AS (SELECT CAST (prv.value AS DATETIME2 (7)) AS PartitionBoundary + FROM sys.partition_range_values AS prv WITH (NOLOCK) + INNER JOIN + sys.partition_functions AS pf WITH (NOLOCK) + ON pf.function_id = prv.function_id + WHERE pf.name = N'PartitionFunction_ResourceChangeData_Timestamp' + AND SQL_VARIANT_PROPERTY(prv.Value, 'BaseType') = 'datetime2' + AND CAST (prv.value AS DATETIME2 (7)) BETWEEN @precedingPartitionBoundary AND @endDateTimeToFilter) + SELECT TOP (@pageSize) Id, + Timestamp, + ResourceId, + ResourceTypeId, + ResourceVersion, + ResourceChangeTypeId + FROM PartitionBoundaries AS p CROSS APPLY (SELECT TOP (@pageSize) Id, + Timestamp, + ResourceId, + ResourceTypeId, + ResourceVersion, + ResourceChangeTypeId + FROM dbo.ResourceChangeData WITH (TABLOCK, HOLDLOCK) + WHERE Id >= @startId + AND $PARTITION.PartitionFunction_ResourceChangeData_Timestamp (Timestamp) = $PARTITION.PartitionFunction_ResourceChangeData_Timestamp (p.PartitionBoundary) + ORDER BY Id ASC) AS rcd + ORDER BY rcd.Id ASC; +END + +GO +CREATE PROCEDURE dbo.GetActiveJobs +@QueueType TINYINT, @GroupId BIGINT=NULL +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetActiveJobs', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' G=' + isnull(CONVERT (VARCHAR, @GroupId), 'NULL'), @st AS DATETIME = getUTCdate(), @JobIds AS BigintList, @PartitionId AS TINYINT, @MaxPartitions AS TINYINT = 16, @LookedAtPartitions AS TINYINT = 0, @Rows AS INT = 0; +BEGIN TRY + SET @PartitionId = @MaxPartitions * rand(); + WHILE @LookedAtPartitions <= @MaxPartitions + BEGIN + IF @GroupId IS NULL + INSERT INTO @JobIds + SELECT JobId + FROM dbo.JobQueue + WHERE PartitionId = @PartitionId + AND QueueType = @QueueType + AND Status IN (0, 1); + ELSE + INSERT INTO @JobIds + SELECT JobId + FROM dbo.JobQueue + WHERE PartitionId = @PartitionId + AND QueueType = @QueueType + AND GroupId = @GroupId + AND Status IN (0, 1); + SET @Rows += @@rowcount; + SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; + SET @LookedAtPartitions += 1; + END + IF @Rows > 0 + EXECUTE dbo.GetJobs @QueueType = @QueueType, @JobIds = @JobIds; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetCommandsForRebuildIndexes +@RebuildClustered BIT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetCommandsForRebuildIndexes', @Mode AS VARCHAR (200) = 'PS=PartitionScheme_ResourceTypeId RC=' + isnull(CONVERT (VARCHAR, @RebuildClustered), 'NULL'), @st AS DATETIME = getUTCdate(), @Tbl AS VARCHAR (100), @TblInt AS VARCHAR (100), @Ind AS VARCHAR (200), @IndId AS INT, @Supported AS BIT, @Txt AS VARCHAR (MAX), @Rows AS BIGINT, @Pages AS BIGINT, @ResourceTypeId AS SMALLINT, @IndexesCnt AS INT, @DataComp AS VARCHAR (100); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + DECLARE @Commands TABLE ( + Tbl VARCHAR (100), + Ind VARCHAR (200), + Txt VARCHAR (MAX), + Pages BIGINT ); + DECLARE @ResourceTypes TABLE ( + ResourceTypeId SMALLINT PRIMARY KEY); + DECLARE @Indexes TABLE ( + Ind VARCHAR (200) PRIMARY KEY, + IndId INT ); + DECLARE @Tables TABLE ( + name VARCHAR (100) PRIMARY KEY, + Supported BIT ); + INSERT INTO @Tables + EXECUTE dbo.GetPartitionedTables @IncludeNotDisabled = 1, @IncludeNotSupported = 1; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Tables', @Action = 'Insert', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Tables) + BEGIN + SELECT TOP 1 @Tbl = name, + @Supported = Supported + FROM @Tables + ORDER BY name; + IF @Supported = 0 + BEGIN + INSERT INTO @Commands + SELECT @Tbl, + name, + 'ALTER INDEX ' + name + ' ON dbo.' + @Tbl + ' REBUILD' + CASE WHEN (SELECT PropertyValue + FROM dbo.IndexProperties + WHERE TableName = @Tbl + AND IndexName = name) = 'PAGE' THEN ' PARTITION = ALL WITH (DATA_COMPRESSION = PAGE)' ELSE '' END, + CONVERT (BIGINT, 9e18) + FROM sys.indexes + WHERE object_id = object_id(@Tbl) + AND (is_disabled = 1 + AND index_id > 1 + AND @RebuildClustered = 0 + OR index_id = 1 + AND @RebuildClustered = 1); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Commands', @Action = 'Insert', @Rows = @@rowcount, @Text = 'Not supported tables with disabled indexes'; + END + ELSE + BEGIN + DELETE @ResourceTypes; + INSERT INTO @ResourceTypes + SELECT CONVERT (SMALLINT, substring(name, charindex('_', name) + 1, 6)) AS ResourceTypeId + FROM sys.sysobjects + WHERE name LIKE @Tbl + '[_]%'; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@ResourceTypes', @Action = 'Insert', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @ResourceTypes) + BEGIN + SET @ResourceTypeId = (SELECT TOP 1 ResourceTypeId + FROM @ResourceTypes + ORDER BY ResourceTypeId); + SET @TblInt = @Tbl + '_' + CONVERT (VARCHAR, @ResourceTypeId); + SET @Pages = (SELECT dpages + FROM sysindexes + WHERE id = object_id(@TblInt) + AND indid IN (0, 1)); + DELETE @Indexes; + INSERT INTO @Indexes + SELECT name, + index_id + FROM sys.indexes + WHERE object_id = object_id(@Tbl) + AND (index_id > 1 + AND @RebuildClustered = 0 + OR index_id = 1 + AND @RebuildClustered = 1); + SET @IndexesCnt = 0; + WHILE EXISTS (SELECT * + FROM @Indexes) + BEGIN + SELECT TOP 1 @Ind = Ind, + @IndId = IndId + FROM @Indexes + ORDER BY Ind; + IF @IndId = 1 + BEGIN + SET @Txt = 'ALTER INDEX ' + @Ind + ' ON dbo.' + @TblInt + ' REBUILD' + CASE WHEN (SELECT PropertyValue + FROM dbo.IndexProperties + WHERE TableName = @Tbl + AND IndexName = @Ind) = 'PAGE' THEN ' PARTITION = ALL WITH (DATA_COMPRESSION = PAGE)' ELSE '' END; + INSERT INTO @Commands + SELECT @TblInt, + @Ind, + @Txt, + @Pages; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Add command', @Rows = @@rowcount, @Text = @Txt; + END + ELSE + IF NOT EXISTS (SELECT * + FROM sys.indexes + WHERE object_id = object_id(@TblInt) + AND name = @Ind) + BEGIN + EXECUTE dbo.GetIndexCommands @Tbl = @Tbl, @Ind = @Ind, @AddPartClause = 0, @IncludeClustered = 0, @Txt = @Txt OUTPUT; + SET @Txt = replace(@Txt, '[' + @Tbl + ']', @TblInt); + IF @Txt IS NOT NULL + BEGIN + SET @IndexesCnt = @IndexesCnt + 1; + INSERT INTO @Commands + SELECT @TblInt, + @Ind, + @Txt, + @Pages; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Add command', @Rows = @@rowcount, @Text = @Txt; + END + END + DELETE @Indexes + WHERE Ind = @Ind; + END + IF @IndexesCnt > 1 + BEGIN + INSERT INTO @Commands + SELECT @TblInt, + 'UPDATE STAT', + 'UPDATE STATISTICS dbo.' + @TblInt, + @Pages; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Add command', @Rows = @@rowcount, @Text = 'Add stats update'; + END + DELETE @ResourceTypes + WHERE ResourceTypeId = @ResourceTypeId; + END + END + DELETE @Tables + WHERE name = @Tbl; + END + SELECT Tbl, + Ind, + Txt + FROM @Commands + ORDER BY Pages DESC, Tbl, CASE WHEN Txt LIKE 'UPDATE STAT%' THEN 0 ELSE 1 END; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Commands', @Action = 'Select', @Rows = @@rowcount; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetExportJobByHash +@hash VARCHAR (64) +AS +SET NOCOUNT ON; +SELECT TOP (1) RawJobRecord, + JobVersion +FROM dbo.ExportJob +WHERE Hash = @hash + AND (Status = 'Queued' + OR Status = 'Running') +ORDER BY HeartbeatDateTime ASC; + +GO +CREATE PROCEDURE dbo.GetExportJobById +@id VARCHAR (64) +AS +SET NOCOUNT ON; +SELECT RawJobRecord, + JobVersion +FROM dbo.ExportJob +WHERE Id = @id; + +GO +CREATE PROCEDURE [dbo].[GetImportProcessingTaskResult] +@queueId VARCHAR (64), @importTaskId VARCHAR (64) +AS +SET NOCOUNT ON; +SELECT Result +FROM [dbo].[TaskInfo] WITH (INDEX (IX_QueueId_ParentTaskId)) +WHERE ParentTaskId = @importTaskId + AND TaskTypeId = 1 + AND Status = 3; + +GO +CREATE PROCEDURE dbo.GetIndexCommands +@Tbl VARCHAR (100), @Ind VARCHAR (200), @AddPartClause BIT, @IncludeClustered BIT, @Txt VARCHAR (MAX)=NULL OUTPUT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetIndexCommands', @Mode AS VARCHAR (200) = 'Tbl=' + isnull(@Tbl, 'NULL') + ' Ind=' + isnull(@Ind, 'NULL'), @st AS DATETIME = getUTCdate(); +DECLARE @Indexes TABLE ( + Ind VARCHAR (200) PRIMARY KEY, + Txt VARCHAR (MAX)); +BEGIN TRY + IF @Tbl IS NULL + RAISERROR ('@Tbl IS NULL', 18, 127); + INSERT INTO @Indexes + SELECT Ind, + CASE WHEN is_primary_key = 1 THEN 'ALTER TABLE dbo.[' + Tbl + '] ADD PRIMARY KEY ' + CASE WHEN type = 1 THEN ' CLUSTERED' ELSE '' END ELSE 'CREATE' + CASE WHEN is_unique = 1 THEN ' UNIQUE' ELSE '' END + CASE WHEN type = 1 THEN ' CLUSTERED' ELSE '' END + ' INDEX ' + Ind + ' ON dbo.[' + Tbl + ']' END + ' (' + KeyCols + ')' + IncClause + CASE WHEN filter_def IS NOT NULL THEN ' WHERE ' + filter_def ELSE '' END + CASE WHEN data_comp IS NOT NULL THEN ' WITH (DATA_COMPRESSION = ' + data_comp + ')' ELSE '' END + CASE WHEN @AddPartClause = 1 THEN PartClause ELSE '' END + FROM (SELECT O.Name AS Tbl, + I.Name AS Ind, + isnull((SELECT TOP 1 CASE WHEN data_compression_desc = 'PAGE' THEN 'PAGE' END + FROM sys.partitions AS P + WHERE P.object_id = I.object_id + AND I.index_id = P.index_id), (SELECT NULLIF (PropertyValue, 'NONE') + FROM dbo.IndexProperties + WHERE TableName = O.Name + AND IndexName = I.Name + AND PropertyName = 'DATA_COMPRESSION')) AS data_comp, + replace(replace(replace(replace(I.filter_definition, '[', ''), ']', ''), '(', ''), ')', '') AS filter_def, + I.is_unique, + I.is_primary_key, + I.type, + KeyCols, + CASE WHEN IncCols IS NOT NULL THEN ' INCLUDE (' + IncCols + ')' ELSE '' END AS IncClause, + CASE WHEN EXISTS (SELECT * + FROM sys.partition_schemes AS S + WHERE S.data_space_id = I.data_space_id + AND name = 'PartitionScheme_ResourceTypeId') THEN ' ON PartitionScheme_ResourceTypeId (ResourceTypeId)' ELSE '' END AS PartClause + FROM sys.indexes AS I + INNER JOIN + sys.objects AS O + ON O.object_id = I.object_id CROSS APPLY (SELECT string_agg(CASE WHEN IC.key_ordinal > 0 + AND IC.is_included_column = 0 THEN C.name END, ',') WITHIN GROUP (ORDER BY key_ordinal) AS KeyCols, + string_agg(CASE WHEN IC.is_included_column = 1 THEN C.name END, ',') WITHIN GROUP (ORDER BY key_ordinal) AS IncCols + FROM sys.index_columns AS IC + INNER JOIN + sys.columns AS C + ON C.object_id = IC.object_id + AND C.column_id = IC.column_id + WHERE IC.object_id = I.object_id + AND IC.index_id = I.index_id + GROUP BY IC.object_id, IC.index_id) AS IC + WHERE O.name = @Tbl + AND (@Ind IS NULL + OR I.name = @Ind) + AND (@IncludeClustered = 1 + OR index_id > 1)) AS A; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Insert', @Rows = @@rowcount; + IF @Ind IS NULL + SELECT Ind, + Txt + FROM @Indexes; + ELSE + SET @Txt = (SELECT Txt + FROM @Indexes); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Text = @Txt; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetJobs +@QueueType TINYINT, @JobId BIGINT=NULL, @JobIds BigintList READONLY, @GroupId BIGINT=NULL, @ReturnDefinition BIT=1 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetJobs', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' J=' + isnull(CONVERT (VARCHAR, @JobId), 'NULL') + ' G=' + isnull(CONVERT (VARCHAR, @GroupId), 'NULL'), @st AS DATETIME = getUTCdate(), @PartitionId AS TINYINT = @JobId % 16; +BEGIN TRY + IF @JobId IS NULL + AND @GroupId IS NULL + AND NOT EXISTS (SELECT * + FROM @JobIds) + RAISERROR ('@JobId = NULL and @GroupId = NULL and @JobIds is empty', 18, 127); + IF @JobId IS NOT NULL + SELECT GroupId, + JobId, + CASE WHEN @ReturnDefinition = 1 THEN Definition ELSE NULL END AS Definition, + Version, + Status, + Priority, + Data, + Result, + CreateDate, + StartDate, + EndDate, + HeartbeatDate, + CancelRequested + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = isnull(@JobId, -1) + AND Status <> 5; + ELSE + IF @GroupId IS NOT NULL + SELECT GroupId, + JobId, + CASE WHEN @ReturnDefinition = 1 THEN Definition ELSE NULL END AS Definition, + Version, + Status, + Priority, + Data, + Result, + CreateDate, + StartDate, + EndDate, + HeartbeatDate, + CancelRequested + FROM dbo.JobQueue WITH (INDEX (IX_QueueType_GroupId)) + WHERE QueueType = @QueueType + AND GroupId = isnull(@GroupId, -1) + AND Status <> 5; + ELSE + SELECT GroupId, + JobId, + CASE WHEN @ReturnDefinition = 1 THEN Definition ELSE NULL END AS Definition, + Version, + Status, + Priority, + Data, + Result, + CreateDate, + StartDate, + EndDate, + HeartbeatDate, + CancelRequested + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND JobId IN (SELECT Id + FROM @JobIds) + AND PartitionId = JobId % 16 + AND Status <> 5; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetNextTask_3 +@queueId VARCHAR (64), @taskHeartbeatTimeoutThresholdInSeconds INT=600 +AS +SET NOCOUNT ON; +DECLARE @lock AS VARCHAR (200) = 'GetNextTask_Q=' + @queueId, @taskId AS VARCHAR (64) = NULL, @expirationDateTime AS DATETIME2 (7), @startDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +SET @expirationDateTime = DATEADD(second, -@taskHeartbeatTimeoutThresholdInSeconds, @startDateTime); +BEGIN TRY + BEGIN TRANSACTION; + EXECUTE sp_getapplock @lock, 'Exclusive'; + UPDATE T + SET Status = 2, + StartDateTime = @startDateTime, + HeartbeatDateTime = @startDateTime, + Worker = host_name(), + RunId = NEWID(), + @taskId = T.TaskId + FROM dbo.TaskInfo AS T WITH (PAGLOCK) + INNER JOIN + (SELECT TOP 1 TaskId + FROM dbo.TaskInfo WITH (INDEX (IX_QueueId_Status)) + WHERE QueueId = @queueId + AND Status = 1 + ORDER BY TaskId) AS S + ON T.QueueId = @queueId + AND T.TaskId = S.TaskId; + IF @taskId IS NULL + UPDATE T + SET StartDateTime = @startDateTime, + HeartbeatDateTime = @startDateTime, + Worker = host_name(), + RunId = NEWID(), + @taskId = T.TaskId, + RestartInfo = ISNULL(RestartInfo, '') + ' Prev: Worker=' + Worker + ' Start=' + CONVERT (VARCHAR, @startDateTime, 121) + FROM dbo.TaskInfo AS T WITH (PAGLOCK) + INNER JOIN + (SELECT TOP 1 TaskId + FROM dbo.TaskInfo WITH (INDEX (IX_QueueId_Status)) + WHERE QueueId = @queueId + AND Status = 2 + AND HeartbeatDateTime <= @expirationDateTime + ORDER BY TaskId) AS S + ON T.QueueId = @queueId + AND T.TaskId = S.TaskId; + COMMIT TRANSACTION; + EXECUTE dbo.GetTaskDetails @TaskId = @taskId; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK TRANSACTION THROW; +END CATCH + +GO +CREATE OR ALTER PROCEDURE dbo.GetNonCompletedJobCountOfSpecificQueueType +@queueType TINYINT +AS +BEGIN + SET NOCOUNT ON; + SELECT COUNT(*) + FROM dbo.JobQueue + WHERE QueueType = @queueType + AND (Status = 0 + OR Status = 1); +END + +GO +CREATE PROCEDURE dbo.GetPartitionedTables +@IncludeNotDisabled BIT, @IncludeNotSupported BIT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetPartitionedTables', @Mode AS VARCHAR (200) = 'PS=PartitionScheme_ResourceTypeId D=' + isnull(CONVERT (VARCHAR, @IncludeNotDisabled), 'NULL') + ' S=' + isnull(CONVERT (VARCHAR, @IncludeNotSupported), 'NULL'), @st AS DATETIME = getUTCdate(); +DECLARE @NotSupportedTables TABLE ( + id INT PRIMARY KEY); +BEGIN TRY + INSERT INTO @NotSupportedTables + SELECT DISTINCT O.object_id + FROM sys.indexes AS I + INNER JOIN + sys.objects AS O + ON O.object_id = I.object_id + WHERE O.type = 'u' + AND EXISTS (SELECT * + FROM sys.partition_schemes AS PS + WHERE PS.data_space_id = I.data_space_id + AND name = 'PartitionScheme_ResourceTypeId') + AND (NOT EXISTS (SELECT * + FROM sys.index_columns AS IC + INNER JOIN + sys.columns AS C + ON C.object_id = IC.object_id + AND C.column_id = IC.column_id + WHERE IC.object_id = I.object_id + AND IC.index_id = I.index_id + AND IC.key_ordinal > 0 + AND IC.is_included_column = 0 + AND C.name = 'ResourceTypeId') + OR EXISTS (SELECT * + FROM sys.indexes AS NSI + WHERE NSI.object_id = O.object_id + AND NOT EXISTS (SELECT * + FROM sys.partition_schemes AS PS + WHERE PS.data_space_id = NSI.data_space_id + AND name = 'PartitionScheme_ResourceTypeId'))); + SELECT CONVERT (VARCHAR (100), O.name), + CONVERT (BIT, CASE WHEN EXISTS (SELECT * + FROM @NotSupportedTables AS NSI + WHERE NSI.id = O.object_id) THEN 0 ELSE 1 END) + FROM sys.indexes AS I + INNER JOIN + sys.objects AS O + ON O.object_id = I.object_id + WHERE O.type = 'u' + AND I.index_id IN (0, 1) + AND EXISTS (SELECT * + FROM sys.partition_schemes AS PS + WHERE PS.data_space_id = I.data_space_id + AND name = 'PartitionScheme_ResourceTypeId') + AND EXISTS (SELECT * + FROM sys.index_columns AS IC + INNER JOIN + sys.columns AS C + ON C.object_id = I.object_id + AND C.column_id = IC.column_id + AND IC.is_included_column = 0 + AND C.name = 'ResourceTypeId') + AND (@IncludeNotSupported = 1 + OR NOT EXISTS (SELECT * + FROM @NotSupportedTables AS NSI + WHERE NSI.id = O.object_id)) + AND (@IncludeNotDisabled = 1 + OR EXISTS (SELECT * + FROM sys.indexes AS D + WHERE D.object_id = O.object_id + AND D.is_disabled = 1)) + ORDER BY 1; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetReindexJobById +@id VARCHAR (64) +AS +SET NOCOUNT ON; +SELECT RawJobRecord, + JobVersion +FROM dbo.ReindexJob +WHERE Id = @id; + +GO +CREATE PROCEDURE dbo.GetResources +@ResourceKeys dbo.ResourceKeyList READONLY +AS +SET NOCOUNT ON; +DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = 'GetResources', @InputRows AS INT, @DummyTop AS BIGINT = 9223372036854775807, @NotNullVersionExists AS BIT, @NullVersionExists AS BIT, @MinRT AS SMALLINT, @MaxRT AS SMALLINT; +SELECT @MinRT = min(ResourceTypeId), + @MaxRT = max(ResourceTypeId), + @InputRows = count(*), + @NotNullVersionExists = max(CASE WHEN Version IS NOT NULL THEN 1 ELSE 0 END), + @NullVersionExists = max(CASE WHEN Version IS NULL THEN 1 ELSE 0 END) +FROM @ResourceKeys; +DECLARE @Mode AS VARCHAR (100) = 'RT=[' + CONVERT (VARCHAR, @MinRT) + ',' + CONVERT (VARCHAR, @MaxRT) + '] Cnt=' + CONVERT (VARCHAR, @InputRows) + ' NNVE=' + CONVERT (VARCHAR, @NotNullVersionExists) + ' NVE=' + CONVERT (VARCHAR, @NullVersionExists); +BEGIN TRY + IF @NotNullVersionExists = 1 + IF @NullVersionExists = 0 + SELECT B.ResourceTypeId, + B.ResourceId, + ResourceSurrogateId, + B.Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceKeys) AS A + INNER JOIN + dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.Version = A.Version + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + ELSE + SELECT * + FROM (SELECT B.ResourceTypeId, + B.ResourceId, + ResourceSurrogateId, + B.Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceKeys + WHERE Version IS NOT NULL) AS A + INNER JOIN + dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.Version = A.Version + UNION ALL + SELECT B.ResourceTypeId, + B.ResourceId, + ResourceSurrogateId, + B.Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceKeys + WHERE Version IS NULL) AS A + INNER JOIN + dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId)) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + WHERE IsHistory = 0) AS A + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + ELSE + SELECT B.ResourceTypeId, + B.ResourceId, + ResourceSurrogateId, + B.Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceKeys) AS A + INNER JOIN + dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId)) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + WHERE IsHistory = 0 + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetResourcesByTransactionId +@TransactionId BIGINT, @IncludeHistory BIT=0, @ReturnResourceKeysOnly BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'T=' + CONVERT (VARCHAR, @TransactionId) + ' H=' + CONVERT (VARCHAR, @IncludeHistory), @st AS DATETIME = getUTCdate(), @DummyTop AS BIGINT = 9223372036854775807, @TypeId AS SMALLINT; +BEGIN TRY + DECLARE @Types TABLE ( + TypeId SMALLINT PRIMARY KEY, + Name VARCHAR (100)); + INSERT INTO @Types + EXECUTE dbo.GetUsedResourceTypes ; + DECLARE @Keys TABLE ( + TypeId SMALLINT, + SurrogateId BIGINT PRIMARY KEY (TypeId, SurrogateId)); + WHILE EXISTS (SELECT * + FROM @Types) + BEGIN + SET @TypeId = (SELECT TOP 1 TypeId + FROM @Types + ORDER BY TypeId); + INSERT INTO @Keys + SELECT @TypeId, + ResourceSurrogateId + FROM dbo.Resource + WHERE ResourceTypeId = @TypeId + AND TransactionId = @TransactionId; + DELETE @Types + WHERE TypeId = @TypeId; + END + IF @ReturnResourceKeysOnly = 0 + SELECT ResourceTypeId, + ResourceId, + ResourceSurrogateId, + Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash, + RequestMethod + FROM (SELECT TOP (@DummyTop) * + FROM @Keys) AS A + INNER JOIN + dbo.Resource AS B + ON ResourceTypeId = TypeId + AND ResourceSurrogateId = SurrogateId + WHERE IsHistory = 0 + OR @IncludeHistory = 1 + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + ELSE + SELECT ResourceTypeId, + ResourceId, + ResourceSurrogateId, + Version, + IsDeleted + FROM (SELECT TOP (@DummyTop) * + FROM @Keys) AS A + INNER JOIN + dbo.Resource AS B + ON ResourceTypeId = TypeId + AND ResourceSurrogateId = SurrogateId + WHERE IsHistory = 0 + OR @IncludeHistory = 1 + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetResourcesByTypeAndSurrogateIdRange +@ResourceTypeId SMALLINT, @StartId BIGINT, @EndId BIGINT, @GlobalEndId BIGINT=NULL, @IncludeHistory BIT=0, @IncludeDeleted BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetResourcesByTypeAndSurrogateIdRange', @Mode AS VARCHAR (100) = 'RT=' + isnull(CONVERT (VARCHAR, @ResourceTypeId), 'NULL') + ' S=' + isnull(CONVERT (VARCHAR, @StartId), 'NULL') + ' E=' + isnull(CONVERT (VARCHAR, @EndId), 'NULL') + ' GE=' + isnull(CONVERT (VARCHAR, @GlobalEndId), 'NULL') + ' HI=' + isnull(CONVERT (VARCHAR, @IncludeHistory), 'NULL') + ' DE' + isnull(CONVERT (VARCHAR, @IncludeDeleted), 'NULL'), @st AS DATETIME = getUTCdate(), @DummyTop AS BIGINT = 9223372036854775807; +BEGIN TRY + DECLARE @ResourceIds TABLE ( + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS PRIMARY KEY); + DECLARE @SurrogateIds TABLE ( + MaxSurrogateId BIGINT PRIMARY KEY); + IF @GlobalEndId IS NOT NULL + AND @IncludeHistory = 0 + BEGIN + INSERT INTO @ResourceIds + SELECT DISTINCT ResourceId + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + AND IsHistory = 1 + AND (IsDeleted = 0 + OR @IncludeDeleted = 1) + OPTION (MAXDOP 1); + IF @@rowcount > 0 + INSERT INTO @SurrogateIds + SELECT ResourceSurrogateId + FROM (SELECT ResourceId, + ResourceSurrogateId, + row_number() OVER (PARTITION BY ResourceId ORDER BY ResourceSurrogateId DESC) AS RowId + FROM dbo.Resource WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceId IN (SELECT TOP (@DummyTop) ResourceId + FROM @ResourceIds) + AND ResourceSurrogateId BETWEEN @StartId AND @GlobalEndId) AS A + WHERE RowId = 1 + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + END + SELECT ResourceTypeId, + ResourceId, + Version, + IsDeleted, + ResourceSurrogateId, + RequestMethod, + CONVERT (BIT, 1) AS IsMatch, + CONVERT (BIT, 0) AS IsPartial, + IsRawResourceMetaSet, + SearchParamHash, + RawResource + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + AND (IsHistory = 0 + OR @IncludeHistory = 1) + AND (IsDeleted = 0 + OR @IncludeDeleted = 1) + UNION ALL + SELECT ResourceTypeId, + ResourceId, + Version, + IsDeleted, + ResourceSurrogateId, + RequestMethod, + CONVERT (BIT, 1) AS IsMatch, + CONVERT (BIT, 0) AS IsPartial, + IsRawResourceMetaSet, + SearchParamHash, + RawResource + FROM @SurrogateIds + INNER JOIN + dbo.Resource + ON ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId = MaxSurrogateId + WHERE IsHistory = 1 + AND (IsDeleted = 0 + OR @IncludeDeleted = 1) + OPTION (MAXDOP 1); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetResourceSurrogateIdRanges +@ResourceTypeId SMALLINT, @StartId BIGINT, @EndId BIGINT, @RangeSize INT, @NumberOfRanges INT=100, @Up BIT=1 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetResourceSurrogateIdRanges', @Mode AS VARCHAR (100) = 'RT=' + isnull(CONVERT (VARCHAR, @ResourceTypeId), 'NULL') + ' S=' + isnull(CONVERT (VARCHAR, @StartId), 'NULL') + ' E=' + isnull(CONVERT (VARCHAR, @EndId), 'NULL') + ' R=' + isnull(CONVERT (VARCHAR, @RangeSize), 'NULL') + ' UP=' + isnull(CONVERT (VARCHAR, @Up), 'NULL'), @st AS DATETIME = getUTCdate(); +BEGIN TRY + IF @Up = 1 + SELECT RangeId, + min(ResourceSurrogateId), + max(ResourceSurrogateId), + count(*) + FROM (SELECT isnull(CONVERT (INT, (row_number() OVER (ORDER BY ResourceSurrogateId) - 1) / @RangeSize), 0) AS RangeId, + ResourceSurrogateId + FROM (SELECT TOP (@RangeSize * @NumberOfRanges) ResourceSurrogateId + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId >= @StartId + AND ResourceSurrogateId <= @EndId + ORDER BY ResourceSurrogateId) AS A) AS A + GROUP BY RangeId + OPTION (MAXDOP 1); + ELSE + SELECT RangeId, + min(ResourceSurrogateId), + max(ResourceSurrogateId), + count(*) + FROM (SELECT isnull(CONVERT (INT, (row_number() OVER (ORDER BY ResourceSurrogateId) - 1) / @RangeSize), 0) AS RangeId, + ResourceSurrogateId + FROM (SELECT TOP (@RangeSize * @NumberOfRanges) ResourceSurrogateId + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId >= @StartId + AND ResourceSurrogateId <= @EndId + ORDER BY ResourceSurrogateId DESC) AS A) AS A + GROUP BY RangeId + OPTION (MAXDOP 1); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetResourceVersions +@ResourceDateKeys dbo.ResourceDateKeyList READONLY +AS +SET NOCOUNT ON; +DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = 'GetResourceVersions', @Mode AS VARCHAR (100) = 'Rows=' + CONVERT (VARCHAR, (SELECT count(*) + FROM @ResourceDateKeys)), @DummyTop AS BIGINT = 9223372036854775807; +BEGIN TRY + SELECT A.ResourceTypeId, + A.ResourceId, + A.ResourceSurrogateId, + CASE WHEN EXISTS (SELECT * + FROM dbo.Resource AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.ResourceSurrogateId = A.ResourceSurrogateId) THEN 0 WHEN isnull(U.Version, 1) - isnull(L.Version, 0) > 1 THEN isnull(U.Version, 1) - 1 ELSE 0 END AS Version + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceDateKeys) AS A OUTER APPLY (SELECT TOP 1 * + FROM dbo.Resource AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.ResourceSurrogateId < A.ResourceSurrogateId + ORDER BY B.ResourceSurrogateId DESC) AS L OUTER APPLY (SELECT TOP 1 * + FROM dbo.Resource AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.ResourceSurrogateId > A.ResourceSurrogateId + ORDER BY B.ResourceSurrogateId) AS U + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetSearchParamStatuses +AS +SET NOCOUNT ON; +SELECT SearchParamId, + Uri, + Status, + LastUpdated, + IsPartiallySupported +FROM dbo.SearchParam; + +GO +CREATE PROCEDURE [dbo].[GetTaskDetails] +@taskId VARCHAR (64) +AS +SET NOCOUNT ON; +SELECT TaskId, + QueueId, + Status, + TaskTypeId, + RunId, + IsCanceled, + RetryCount, + MaxRetryCount, + HeartbeatDateTime, + InputData, + TaskContext, + Result, + ParentTaskId +FROM [dbo].[TaskInfo] +WHERE TaskId = @taskId; + +GO +CREATE PROCEDURE dbo.GetTransactions +@StartNotInclusiveTranId BIGINT, @EndInclusiveTranId BIGINT, @EndDate DATETIME=NULL +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'ST=' + CONVERT (VARCHAR, @StartNotInclusiveTranId) + ' ET=' + CONVERT (VARCHAR, @EndInclusiveTranId) + ' ED=' + isnull(CONVERT (VARCHAR, @EndDate, 121), 'NULL'), @st AS DATETIME = getUTCdate(); +IF @EndDate IS NULL + SET @EndDate = getUTCdate(); +SELECT SurrogateIdRangeFirstValue, + VisibleDate, + InvisibleHistoryRemovedDate +FROM dbo.Transactions +WHERE SurrogateIdRangeFirstValue > @StartNotInclusiveTranId + AND SurrogateIdRangeFirstValue <= @EndInclusiveTranId + AND EndDate <= @EndDate +ORDER BY SurrogateIdRangeFirstValue; +EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; + +GO +CREATE PROCEDURE dbo.GetUsedResourceTypes +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetUsedResourceTypes', @Mode AS VARCHAR (100) = '', @st AS DATETIME = getUTCdate(); +BEGIN TRY + SELECT ResourceTypeId, + Name + FROM dbo.ResourceType AS A + WHERE EXISTS (SELECT * + FROM dbo.Resource AS B + WHERE B.ResourceTypeId = A.ResourceTypeId); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.HardDeleteResource +@ResourceTypeId SMALLINT, @ResourceId VARCHAR (64), @KeepCurrentVersion BIT, @IsResourceChangeCaptureEnabled BIT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (200) = 'RT=' + CONVERT (VARCHAR, @ResourceTypeId) + ' R=' + @ResourceId + ' V=' + CONVERT (VARCHAR, @KeepCurrentVersion) + ' CC=' + CONVERT (VARCHAR, @IsResourceChangeCaptureEnabled), @st AS DATETIME = getUTCdate(), @TransactionId AS BIGINT; +BEGIN TRY + IF @IsResourceChangeCaptureEnabled = 1 + EXECUTE dbo.MergeResourcesBeginTransaction @Count = 1, @TransactionId = @TransactionId OUTPUT; + IF @KeepCurrentVersion = 0 + BEGIN TRANSACTION; + DECLARE @SurrogateIds TABLE ( + ResourceSurrogateId BIGINT NOT NULL); + IF @IsResourceChangeCaptureEnabled = 1 + AND NOT EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = 'InvisibleHistory.IsEnabled' + AND Number = 0) + UPDATE dbo.Resource + SET IsHistory = 1, + RawResource = 0xF, + SearchParamHash = NULL, + HistoryTransactionId = @TransactionId + OUTPUT deleted.ResourceSurrogateId INTO @SurrogateIds + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceId = @ResourceId + AND (@KeepCurrentVersion = 0 + OR IsHistory = 1) + AND RawResource <> 0xF; + ELSE + DELETE dbo.Resource + OUTPUT deleted.ResourceSurrogateId INTO @SurrogateIds + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceId = @ResourceId + AND (@KeepCurrentVersion = 0 + OR IsHistory = 1) + AND RawResource <> 0xF; + IF @KeepCurrentVersion = 0 + BEGIN + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.ResourceWriteClaim AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.ReferenceSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenText AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.StringSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.UriSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.NumberSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.QuantitySearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.DateTimeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.ReferenceTokenCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenTokenCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenDateTimeCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenQuantityCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenStringCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenNumberNumberCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + END + IF @@trancount > 0 + COMMIT TRANSACTION; + IF @IsResourceChangeCaptureEnabled = 1 + EXECUTE dbo.MergeResourcesCommitTransaction @TransactionId; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.InitDefrag +@QueueType TINYINT, @GroupId BIGINT, @DefragItems INT=NULL OUTPUT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'InitDefrag', @st AS DATETIME = getUTCdate(), @ObjectId AS INT, @msg AS VARCHAR (1000), @Rows AS INT, @MinFragPct AS INT = isnull((SELECT Number + FROM dbo.Parameters + WHERE Id = 'Defrag.MinFragPct'), 10), @MinSizeGB AS FLOAT = isnull((SELECT Number + FROM dbo.Parameters + WHERE Id = 'Defrag.MinSizeGB'), 0.1), @DefinitionsSorted AS StringList; +DECLARE @Mode AS VARCHAR (200) = 'G=' + CONVERT (VARCHAR, @GroupId) + ' MF=' + CONVERT (VARCHAR, @MinFragPct) + ' MS=' + CONVERT (VARCHAR, @MinSizeGB); +DECLARE @Definitions AS TABLE ( + Def VARCHAR (900) PRIMARY KEY, + FragGB FLOAT ); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + SELECT * + INTO #filter + FROM (SELECT object_id, + sum(reserved_page_count * 8.0 / 1024 / 1024) AS ReservedGB + FROM sys.dm_db_partition_stats AS A + WHERE object_id IN (SELECT object_id + FROM sys.objects + WHERE type = 'U' + AND name NOT IN ('EventLog')) + GROUP BY object_id) AS A + WHERE ReservedGB > @MinSizeGB; + WHILE EXISTS (SELECT * + FROM #filter) + BEGIN + SET @ObjectId = (SELECT TOP 1 object_id + FROM #filter + ORDER BY ReservedGB DESC); + INSERT INTO @Definitions + SELECT object_name(@ObjectId) + ';' + I.name + ';' + CONVERT (VARCHAR, partition_number) + ';' + CONVERT (VARCHAR, CASE WHEN EXISTS (SELECT * + FROM sys.partition_schemes AS PS + WHERE PS.data_space_id = I.data_space_id) THEN 1 ELSE 0 END) + ';' + CONVERT (VARCHAR, (SELECT sum(reserved_page_count) + FROM sys.dm_db_partition_stats AS S + WHERE S.object_id = A.object_id + AND S.index_id = A.index_id + AND S.partition_number = A.partition_number) * 8.0 / 1024 / 1024), + FragGB + FROM (SELECT object_id, + index_id, + partition_number, + A.avg_fragmentation_in_percent * A.page_count * 8.0 / 1024 / 1024 / 100 AS FragGB + FROM sys.dm_db_index_physical_stats(db_id(), @ObjectId, NULL, NULL, 'LIMITED') AS A + WHERE index_id > 0 + AND avg_fragmentation_in_percent >= @MinFragPct + AND A.page_count > 500) AS A + INNER JOIN + sys.indexes AS I + ON I.object_id = A.object_id + AND I.index_id = A.index_id; + SET @Rows = @@rowcount; + SET @msg = object_name(@ObjectId); + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'Run', @Mode = @Mode, @Target = '@Definitions', @Action = 'Insert', @Rows = @Rows, @Text = @msg; + DELETE #filter + WHERE object_id = @ObjectId; + END + INSERT INTO @DefinitionsSorted + SELECT Def + ';' + CONVERT (VARCHAR, FragGB) + FROM @Definitions + ORDER BY FragGB DESC; + SET @DefragItems = @@rowcount; + IF @DefragItems > 0 + EXECUTE dbo.EnqueueJobs @QueueType = @QueueType, @Definitions = @DefinitionsSorted, @GroupId = @GroupId, @ForceOneActiveJobGroup = 1; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.InitializeIndexProperties +AS +SET NOCOUNT ON; +INSERT INTO dbo.IndexProperties (TableName, IndexName, PropertyName, PropertyValue) +SELECT Tbl, + Ind, + 'DATA_COMPRESSION', + isnull(data_comp, 'NONE') +FROM (SELECT O.Name AS Tbl, + I.Name AS Ind, + (SELECT TOP 1 CASE WHEN data_compression_desc = 'PAGE' THEN 'PAGE' END + FROM sys.partitions AS P + WHERE P.object_id = I.object_id + AND I.index_id = P.index_id) AS data_comp + FROM sys.indexes AS I + INNER JOIN + sys.objects AS O + ON O.object_id = I.object_id + WHERE O.type = 'u' + AND EXISTS (SELECT * + FROM sys.partition_schemes AS PS + WHERE PS.data_space_id = I.data_space_id + AND name = 'PartitionScheme_ResourceTypeId')) AS A +WHERE NOT EXISTS (SELECT * + FROM dbo.IndexProperties + WHERE TableName = Tbl + AND IndexName = Ind); + +GO +CREATE PROCEDURE dbo.LogEvent +@Process VARCHAR (100), @Status VARCHAR (10), @Mode VARCHAR (200)=NULL, @Action VARCHAR (20)=NULL, @Target VARCHAR (100)=NULL, @Rows BIGINT=NULL, @Start DATETIME=NULL, @Text NVARCHAR (3500)=NULL, @EventId BIGINT=NULL OUTPUT, @Retry INT=NULL +AS +SET NOCOUNT ON; +DECLARE @ErrorNumber AS INT = error_number(), @ErrorMessage AS VARCHAR (1000) = '', @TranCount AS INT = @@trancount, @DoWork AS BIT = 0, @NumberAdded AS BIT; +IF @ErrorNumber IS NOT NULL + OR @Status IN ('Warn', 'Error') + SET @DoWork = 1; +IF @DoWork = 0 + SET @DoWork = CASE WHEN EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = isnull(@Process, '') + AND Char = 'LogEvent') THEN 1 ELSE 0 END; +IF @DoWork = 0 + RETURN; +IF @ErrorNumber IS NOT NULL + SET @ErrorMessage = CASE WHEN @Retry IS NOT NULL THEN 'Retry ' + CONVERT (VARCHAR, @Retry) + ', ' ELSE '' END + 'Error ' + CONVERT (VARCHAR, error_number()) + ': ' + CONVERT (VARCHAR (1000), error_message()) + ', Level ' + CONVERT (VARCHAR, error_severity()) + ', State ' + CONVERT (VARCHAR, error_state()) + CASE WHEN error_procedure() IS NOT NULL THEN ', Procedure ' + error_procedure() ELSE '' END + ', Line ' + CONVERT (VARCHAR, error_line()); +IF @TranCount > 0 + AND @ErrorNumber IS NOT NULL + ROLLBACK; +IF databasepropertyex(db_name(), 'UpdateAbility') = 'READ_WRITE' + BEGIN + INSERT INTO dbo.EventLog (Process, Status, Mode, Action, Target, Rows, Milliseconds, EventDate, EventText, SPID, HostName) + SELECT @Process, + @Status, + @Mode, + @Action, + @Target, + @Rows, + datediff(millisecond, @Start, getUTCdate()), + getUTCdate() AS EventDate, + CASE WHEN @ErrorNumber IS NULL THEN @Text ELSE @ErrorMessage + CASE WHEN isnull(@Text, '') <> '' THEN '. ' + @Text ELSE '' END END AS Text, + @@SPID, + host_name() AS HostName; + SET @EventId = scope_identity(); + END +IF @TranCount > 0 + AND @ErrorNumber IS NOT NULL + BEGIN TRANSACTION; + +GO +CREATE PROCEDURE dbo.LogSchemaMigrationProgress +@message VARCHAR (MAX) +AS +INSERT INTO dbo.SchemaMigrationProgress (Message) +VALUES (@message); + +GO +CREATE PROCEDURE dbo.MergeResources +@AffectedRows INT=0 OUTPUT, @RaiseExceptionOnConflict BIT=1, @IsResourceChangeCaptureEnabled BIT=0, @TransactionId BIGINT=NULL, @SingleTransaction BIT=1, @Resources dbo.ResourceList READONLY, @ResourceWriteClaims dbo.ResourceWriteClaimList READONLY, @CompartmentAssignments dbo.CompartmentAssignmentList READONLY, @ReferenceSearchParams dbo.ReferenceSearchParamList READONLY, @TokenSearchParams dbo.TokenSearchParamList READONLY, @TokenTexts dbo.TokenTextList READONLY, @StringSearchParams dbo.StringSearchParamList READONLY, @UriSearchParams dbo.UriSearchParamList READONLY, @NumberSearchParams dbo.NumberSearchParamList READONLY, @QuantitySearchParams dbo.QuantitySearchParamList READONLY, @DateTimeSearchParms dbo.DateTimeSearchParamList READONLY, @ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY, @TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY, @TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY, @TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY, @TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY, @TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY +AS +SET NOCOUNT ON; +DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = object_name(@@procid), @DummyTop AS BIGINT = 9223372036854775807, @InitialTranCount AS INT = @@trancount, @IsRetry AS BIT = 0; +DECLARE @Mode AS VARCHAR (200) = isnull((SELECT 'RT=[' + CONVERT (VARCHAR, min(ResourceTypeId)) + ',' + CONVERT (VARCHAR, max(ResourceTypeId)) + '] Sur=[' + CONVERT (VARCHAR, min(ResourceSurrogateId)) + ',' + CONVERT (VARCHAR, max(ResourceSurrogateId)) + '] V=' + CONVERT (VARCHAR, max(Version)) + ' Rows=' + CONVERT (VARCHAR, count(*)) + FROM @Resources), 'Input=Empty'); +SET @Mode += ' E=' + CONVERT (VARCHAR, @RaiseExceptionOnConflict) + ' CC=' + CONVERT (VARCHAR, @IsResourceChangeCaptureEnabled) + ' IT=' + CONVERT (VARCHAR, @InitialTranCount) + ' T=' + isnull(CONVERT (VARCHAR, @TransactionId), 'NULL'); +SET @AffectedRows = 0; +BEGIN TRY + DECLARE @Existing AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + SurrogateId BIGINT NOT NULL PRIMARY KEY (ResourceTypeId, SurrogateId)); + DECLARE @ResourceInfos AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + SurrogateId BIGINT NOT NULL, + Version INT NOT NULL, + KeepHistory BIT NOT NULL, + PreviousVersion INT NULL, + PreviousSurrogateId BIGINT NULL PRIMARY KEY (ResourceTypeId, SurrogateId)); + DECLARE @PreviousSurrogateIds AS TABLE ( + TypeId SMALLINT NOT NULL, + SurrogateId BIGINT NOT NULL PRIMARY KEY (TypeId, SurrogateId), + KeepHistory BIT ); + IF @SingleTransaction = 0 + AND isnull((SELECT Number + FROM dbo.Parameters + WHERE Id = 'MergeResources.NoTransaction.IsEnabled'), 0) = 0 + SET @SingleTransaction = 1; + SET @Mode += ' ST=' + CONVERT (VARCHAR, @SingleTransaction); + IF @InitialTranCount = 0 + BEGIN + IF EXISTS (SELECT * + FROM @Resources AS A + INNER JOIN + dbo.Resource AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + WHERE B.IsHistory = 0) + BEGIN + BEGIN TRANSACTION; + INSERT INTO @Existing (ResourceTypeId, SurrogateId) + SELECT B.ResourceTypeId, + B.ResourceSurrogateId + FROM (SELECT TOP (@DummyTop) * + FROM @Resources) AS A + INNER JOIN + dbo.Resource AS B WITH (ROWLOCK, HOLDLOCK) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + WHERE B.IsHistory = 0 + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + IF @@rowcount > 0 + SET @IsRetry = 1; + IF @IsRetry = 0 + COMMIT TRANSACTION; + END + END + SET @Mode += ' R=' + CONVERT (VARCHAR, @IsRetry); + IF @SingleTransaction = 1 + AND @@trancount = 0 + BEGIN TRANSACTION; + IF @IsRetry = 0 + BEGIN + INSERT INTO @ResourceInfos (ResourceTypeId, SurrogateId, Version, KeepHistory, PreviousVersion, PreviousSurrogateId) + SELECT A.ResourceTypeId, + A.ResourceSurrogateId, + A.Version, + A.KeepHistory, + B.Version, + B.ResourceSurrogateId + FROM (SELECT TOP (@DummyTop) * + FROM @Resources + WHERE HasVersionToCompare = 1) AS A + LEFT OUTER JOIN + dbo.Resource AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.IsHistory = 0 + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + IF @RaiseExceptionOnConflict = 1 + AND EXISTS (SELECT * + FROM @ResourceInfos + WHERE PreviousVersion IS NOT NULL + AND Version <> PreviousVersion + 1) + THROW 50409, 'Resource has been recently updated or added, please compare the resource content in code for any duplicate updates', 1; + INSERT INTO @PreviousSurrogateIds + SELECT ResourceTypeId, + PreviousSurrogateId, + KeepHistory + FROM @ResourceInfos + WHERE PreviousSurrogateId IS NOT NULL; + IF @@rowcount > 0 + BEGIN + UPDATE dbo.Resource + SET IsHistory = 1 + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId + AND KeepHistory = 1); + SET @AffectedRows += @@rowcount; + IF @IsResourceChangeCaptureEnabled = 1 + AND NOT EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = 'InvisibleHistory.IsEnabled' + AND Number = 0) + UPDATE dbo.Resource + SET IsHistory = 1, + RawResource = 0xF, + SearchParamHash = NULL, + HistoryTransactionId = @TransactionId + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId + AND KeepHistory = 0); + ELSE + DELETE dbo.Resource + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId + AND KeepHistory = 0); + SET @AffectedRows += @@rowcount; + DELETE dbo.ResourceWriteClaim + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.ReferenceSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenText + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.StringSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.UriSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.NumberSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.QuantitySearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.DateTimeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.ReferenceTokenCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenTokenCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenDateTimeCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenQuantityCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenStringCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenNumberNumberCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + END + INSERT INTO dbo.Resource (ResourceTypeId, ResourceId, Version, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, RawResource, IsRawResourceMetaSet, SearchParamHash, TransactionId) + SELECT ResourceTypeId, + ResourceId, + Version, + IsHistory, + ResourceSurrogateId, + IsDeleted, + RequestMethod, + RawResource, + IsRawResourceMetaSet, + SearchParamHash, + @TransactionId + FROM @Resources; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) + SELECT ResourceSurrogateId, + ClaimTypeId, + ClaimValue + FROM @ResourceWriteClaims; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri, + ReferenceResourceTypeId, + ReferenceResourceId, + ReferenceResourceVersion + FROM @ReferenceSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + Code, + CodeOverflow + FROM @TokenSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text + FROM @TokenTexts; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text, + TextOverflow, + IsMin, + IsMax + FROM @StringSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Uri + FROM @UriSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SingleValue, + LowValue, + HighValue + FROM @NumberSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + QuantityCodeId, + SingleValue, + LowValue, + HighValue + FROM @QuantitySearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + StartDateTime, + EndDateTime, + IsLongerThanADay, + IsMin, + IsMax + FROM @DateTimeSearchParms; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri1, + ReferenceResourceTypeId1, + ReferenceResourceId1, + ReferenceResourceVersion1, + SystemId2, + Code2, + CodeOverflow2 + FROM @ReferenceTokenCompositeSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SystemId2, + Code2, + CodeOverflow2 + FROM @TokenTokenCompositeSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + StartDateTime2, + EndDateTime2, + IsLongerThanADay2 + FROM @TokenDateTimeCompositeSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + SystemId2, + QuantityCodeId2, + LowValue2, + HighValue2 + FROM @TokenQuantityCompositeSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + Text2, + TextOverflow2 + FROM @TokenStringCompositeSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + LowValue2, + HighValue2, + SingleValue3, + LowValue3, + HighValue3, + HasRange + FROM @TokenNumberNumberCompositeSearchParams; + SET @AffectedRows += @@rowcount; + END + ELSE + BEGIN + INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) + SELECT ResourceSurrogateId, + ClaimTypeId, + ClaimValue + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceWriteClaims) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.ResourceWriteClaim AS C + WHERE C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri, + ReferenceResourceTypeId, + ReferenceResourceId, + ReferenceResourceVersion + FROM (SELECT TOP (@DummyTop) * + FROM @ReferenceSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.ReferenceSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + Code, + CodeOverflow + FROM (SELECT TOP (@DummyTop) * + FROM @TokenSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text + FROM (SELECT TOP (@DummyTop) * + FROM @TokenTexts) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text, + TextOverflow, + IsMin, + IsMax + FROM (SELECT TOP (@DummyTop) * + FROM @StringSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenText AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Uri + FROM (SELECT TOP (@DummyTop) * + FROM @UriSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.UriSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SingleValue, + LowValue, + HighValue + FROM (SELECT TOP (@DummyTop) * + FROM @NumberSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.NumberSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + QuantityCodeId, + SingleValue, + LowValue, + HighValue + FROM (SELECT TOP (@DummyTop) * + FROM @QuantitySearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.QuantitySearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + StartDateTime, + EndDateTime, + IsLongerThanADay, + IsMin, + IsMax + FROM (SELECT TOP (@DummyTop) * + FROM @DateTimeSearchParms) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri1, + ReferenceResourceTypeId1, + ReferenceResourceId1, + ReferenceResourceVersion1, + SystemId2, + Code2, + CodeOverflow2 + FROM (SELECT TOP (@DummyTop) * + FROM @ReferenceTokenCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.DateTimeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SystemId2, + Code2, + CodeOverflow2 + FROM (SELECT TOP (@DummyTop) * + FROM @TokenTokenCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenTokenCompositeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + StartDateTime2, + EndDateTime2, + IsLongerThanADay2 + FROM (SELECT TOP (@DummyTop) * + FROM @TokenDateTimeCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenDateTimeCompositeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + SystemId2, + QuantityCodeId2, + LowValue2, + HighValue2 + FROM (SELECT TOP (@DummyTop) * + FROM @TokenQuantityCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenQuantityCompositeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + Text2, + TextOverflow2 + FROM (SELECT TOP (@DummyTop) * + FROM @TokenStringCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenStringCompositeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + LowValue2, + HighValue2, + SingleValue3, + LowValue3, + HighValue3, + HasRange + FROM (SELECT TOP (@DummyTop) * + FROM @TokenNumberNumberCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenNumberNumberCompositeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + END + IF @IsResourceChangeCaptureEnabled = 1 + EXECUTE dbo.CaptureResourceIdsForChanges @Resources; + IF @TransactionId IS NOT NULL + EXECUTE dbo.MergeResourcesCommitTransaction @TransactionId; + IF @InitialTranCount = 0 + AND @@trancount > 0 + COMMIT TRANSACTION; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @AffectedRows; +END TRY +BEGIN CATCH + IF @InitialTranCount = 0 + AND @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + IF @RaiseExceptionOnConflict = 1 + AND error_number() IN (2601, 2627) + AND error_message() LIKE '%''dbo.Resource''%' + THROW 50409, 'Resource has been recently updated or added, please compare the resource content in code for any duplicate updates', 1; + ELSE + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesAdvanceTransactionVisibility +@AffectedRows INT=0 OUTPUT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = '', @st AS DATETIME = getUTCdate(), @msg AS VARCHAR (1000), @MaxTransactionId AS BIGINT, @MinTransactionId AS BIGINT, @MinNotCompletedTransactionId AS BIGINT, @CurrentTransactionId AS BIGINT; +SET @AffectedRows = 0; +BEGIN TRY + EXECUTE dbo.MergeResourcesGetTransactionVisibility @MinTransactionId OUTPUT; + SET @MinTransactionId += 1; + SET @CurrentTransactionId = (SELECT TOP 1 SurrogateIdRangeFirstValue + FROM dbo.Transactions + ORDER BY SurrogateIdRangeFirstValue DESC); + SET @MinNotCompletedTransactionId = isnull((SELECT TOP 1 SurrogateIdRangeFirstValue + FROM dbo.Transactions + WHERE IsCompleted = 0 + AND SurrogateIdRangeFirstValue BETWEEN @MinTransactionId AND @CurrentTransactionId + ORDER BY SurrogateIdRangeFirstValue), @CurrentTransactionId + 1); + SET @MaxTransactionId = (SELECT TOP 1 SurrogateIdRangeFirstValue + FROM dbo.Transactions + WHERE IsCompleted = 1 + AND SurrogateIdRangeFirstValue BETWEEN @MinTransactionId AND @CurrentTransactionId + AND SurrogateIdRangeFirstValue < @MinNotCompletedTransactionId + ORDER BY SurrogateIdRangeFirstValue DESC); + IF @MaxTransactionId >= @MinTransactionId + BEGIN + UPDATE A + SET IsVisible = 1, + VisibleDate = getUTCdate() + FROM dbo.Transactions AS A WITH (INDEX (1)) + WHERE SurrogateIdRangeFirstValue BETWEEN @MinTransactionId AND @CurrentTransactionId + AND SurrogateIdRangeFirstValue <= @MaxTransactionId; + SET @AffectedRows += @@rowcount; + END + SET @msg = 'Min=' + CONVERT (VARCHAR, @MinTransactionId) + ' C=' + CONVERT (VARCHAR, @CurrentTransactionId) + ' MinNC=' + CONVERT (VARCHAR, @MinNotCompletedTransactionId) + ' Max=' + CONVERT (VARCHAR, @MaxTransactionId); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @AffectedRows, @Text = @msg; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesBeginTransaction +@Count INT, @TransactionId BIGINT OUTPUT, @SequenceRangeFirstValue INT=NULL OUTPUT, @HeartbeatDate DATETIME=NULL +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'MergeResourcesBeginTransaction', @Mode AS VARCHAR (200) = 'Cnt=' + CONVERT (VARCHAR, @Count), @st AS DATETIME = getUTCdate(), @FirstValueVar AS SQL_VARIANT, @LastValueVar AS SQL_VARIANT; +BEGIN TRY + SET @TransactionId = NULL; + IF @@trancount > 0 + RAISERROR ('MergeResourcesBeginTransaction cannot be called inside outer transaction.', 18, 127); + SET @FirstValueVar = NULL; + WHILE @FirstValueVar IS NULL + BEGIN + EXECUTE sys.sp_sequence_get_range @sequence_name = 'dbo.ResourceSurrogateIdUniquifierSequence', @range_size = @Count, @range_first_value = @FirstValueVar OUTPUT, @range_last_value = @LastValueVar OUTPUT; + SET @SequenceRangeFirstValue = CONVERT (INT, @FirstValueVar); + IF @SequenceRangeFirstValue > CONVERT (INT, @LastValueVar) + SET @FirstValueVar = NULL; + END + SET @TransactionId = datediff_big(millisecond, '0001-01-01', sysUTCdatetime()) * 80000 + @SequenceRangeFirstValue; + INSERT INTO dbo.Transactions (SurrogateIdRangeFirstValue, SurrogateIdRangeLastValue, HeartbeatDate) + SELECT @TransactionId, + @TransactionId + @Count - 1, + isnull(@HeartbeatDate, getUTCdate()); +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + IF @@trancount > 0 + ROLLBACK; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesCommitTransaction +@TransactionId BIGINT, @FailureReason VARCHAR (MAX)=NULL, @OverrideIsControlledByClientCheck BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'MergeResourcesCommitTransaction', @st AS DATETIME = getUTCdate(), @InitialTranCount AS INT = @@trancount, @IsCompletedBefore AS BIT, @Rows AS INT, @msg AS VARCHAR (1000); +DECLARE @Mode AS VARCHAR (200) = 'TR=' + CONVERT (VARCHAR, @TransactionId) + ' OC=' + isnull(CONVERT (VARCHAR, @OverrideIsControlledByClientCheck), 'NULL'); +BEGIN TRY + IF @InitialTranCount = 0 + BEGIN TRANSACTION; + UPDATE dbo.Transactions + SET IsCompleted = 1, + @IsCompletedBefore = IsCompleted, + EndDate = getUTCdate(), + IsSuccess = CASE WHEN @FailureReason IS NULL THEN 1 ELSE 0 END, + FailureReason = @FailureReason + WHERE SurrogateIdRangeFirstValue = @TransactionId + AND (IsControlledByClient = 1 + OR @OverrideIsControlledByClientCheck = 1); + SET @Rows = @@rowcount; + IF @Rows = 0 + BEGIN + SET @msg = 'Transaction [' + CONVERT (VARCHAR (20), @TransactionId) + '] is not controlled by client or does not exist.'; + RAISERROR (@msg, 18, 127); + END + IF @IsCompletedBefore = 1 + BEGIN + IF @InitialTranCount = 0 + ROLLBACK; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows, @Target = '@IsCompletedBefore', @Text = '=1'; + RETURN; + END + IF @InitialTranCount = 0 + COMMIT TRANSACTION; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF @InitialTranCount = 0 + AND @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesDeleteInvisibleHistory +@TransactionId BIGINT, @AffectedRows INT=NULL OUTPUT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'T=' + CONVERT (VARCHAR, @TransactionId), @st AS DATETIME = getUTCdate(), @TypeId AS SMALLINT; +SET @AffectedRows = 0; +BEGIN TRY + DECLARE @Types TABLE ( + TypeId SMALLINT PRIMARY KEY, + Name VARCHAR (100)); + INSERT INTO @Types + EXECUTE dbo.GetUsedResourceTypes ; + WHILE EXISTS (SELECT * + FROM @Types) + BEGIN + SET @TypeId = (SELECT TOP 1 TypeId + FROM @Types + ORDER BY TypeId); + DELETE dbo.Resource + WHERE ResourceTypeId = @TypeId + AND HistoryTransactionId = @TransactionId + AND IsHistory = 1 + AND RawResource = 0xF; + SET @AffectedRows += @@rowcount; + DELETE @Types + WHERE TypeId = @TypeId; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @AffectedRows; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesGetTimeoutTransactions +@TimeoutSec INT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'T=' + CONVERT (VARCHAR, @TimeoutSec), @st AS DATETIME = getUTCdate(), @MinTransactionId AS BIGINT; +BEGIN TRY + EXECUTE dbo.MergeResourcesGetTransactionVisibility @MinTransactionId OUTPUT; + SELECT SurrogateIdRangeFirstValue + FROM dbo.Transactions + WHERE SurrogateIdRangeFirstValue > @MinTransactionId + AND IsCompleted = 0 + AND datediff(second, HeartbeatDate, getUTCdate()) > @TimeoutSec + ORDER BY SurrogateIdRangeFirstValue; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesGetTransactionVisibility +@TransactionId BIGINT OUTPUT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = '', @st AS DATETIME = getUTCdate(); +SET @TransactionId = isnull((SELECT TOP 1 SurrogateIdRangeFirstValue + FROM dbo.Transactions + WHERE IsVisible = 1 + ORDER BY SurrogateIdRangeFirstValue DESC), -1); +EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount, @Text = @TransactionId; + +GO +CREATE PROCEDURE dbo.MergeResourcesPutTransactionHeartbeat +@TransactionId BIGINT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'MergeResourcesPutTransactionHeartbeat', @Mode AS VARCHAR (100) = 'TR=' + CONVERT (VARCHAR, @TransactionId); +BEGIN TRY + UPDATE dbo.Transactions + SET HeartbeatDate = getUTCdate() + WHERE SurrogateIdRangeFirstValue = @TransactionId + AND IsControlledByClient = 1; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesPutTransactionInvisibleHistory +@TransactionId BIGINT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'TR=' + CONVERT (VARCHAR, @TransactionId), @st AS DATETIME = getUTCdate(); +BEGIN TRY + UPDATE dbo.Transactions + SET InvisibleHistoryRemovedDate = getUTCdate() + WHERE SurrogateIdRangeFirstValue = @TransactionId + AND InvisibleHistoryRemovedDate IS NULL; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.PutJobCancelation +@QueueType TINYINT, @GroupId BIGINT=NULL, @JobId BIGINT=NULL +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'PutJobCancelation', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' G=' + isnull(CONVERT (VARCHAR, @GroupId), 'NULL') + ' J=' + isnull(CONVERT (VARCHAR, @JobId), 'NULL'), @st AS DATETIME = getUTCdate(), @Rows AS INT, @PartitionId AS TINYINT = @JobId % 16; +BEGIN TRY + IF @JobId IS NULL + AND @GroupId IS NULL + RAISERROR ('@JobId = NULL and @GroupId = NULL', 18, 127); + IF @JobId IS NOT NULL + BEGIN + UPDATE dbo.JobQueue + SET Status = 4, + EndDate = getUTCdate(), + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()) + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Status = 0; + SET @Rows = @@rowcount; + IF @Rows = 0 + BEGIN + UPDATE dbo.JobQueue + SET CancelRequested = 1 + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Status = 1; + SET @Rows = @@rowcount; + END + END + ELSE + BEGIN + UPDATE dbo.JobQueue + SET Status = 4, + EndDate = getUTCdate(), + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()) + WHERE QueueType = @QueueType + AND GroupId = @GroupId + AND Status = 0; + SET @Rows = @@rowcount; + UPDATE dbo.JobQueue + SET CancelRequested = 1 + WHERE QueueType = @QueueType + AND GroupId = @GroupId + AND Status = 1; + SET @Rows += @@rowcount; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.PutJobHeartbeat +@QueueType TINYINT, @JobId BIGINT, @Version BIGINT, @Data BIGINT=NULL, @CurrentResult VARCHAR (MAX)=NULL, @CancelRequested BIT=0 OUTPUT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'PutJobHeartbeat', @Mode AS VARCHAR (100), @st AS DATETIME = getUTCdate(), @Rows AS INT = 0, @PartitionId AS TINYINT = @JobId % 16; +SET @Mode = 'Q=' + CONVERT (VARCHAR, @QueueType) + ' J=' + CONVERT (VARCHAR, @JobId) + ' P=' + CONVERT (VARCHAR, @PartitionId) + ' V=' + CONVERT (VARCHAR, @Version) + ' D=' + isnull(CONVERT (VARCHAR, @Data), 'NULL'); +BEGIN TRY + IF @CurrentResult IS NULL + UPDATE dbo.JobQueue + SET @CancelRequested = CancelRequested, + HeartbeatDate = getUTCdate(), + Data = isnull(@Data, Data) + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Status = 1 + AND Version = @Version; + ELSE + UPDATE dbo.JobQueue + SET @CancelRequested = CancelRequested, + HeartbeatDate = getUTCdate(), + Data = isnull(@Data, Data), + Result = @CurrentResult + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Status = 1 + AND Version = @Version; + SET @Rows = @@rowcount; + IF @Rows = 0 + AND NOT EXISTS (SELECT * + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Version = @Version + AND Status IN (2, 3, 4)) + BEGIN + IF EXISTS (SELECT * + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId) + THROW 50412, 'Precondition failed', 1; + ELSE + THROW 50404, 'Job record not found', 1; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.PutJobStatus +@QueueType TINYINT, @JobId BIGINT, @Version BIGINT, @Failed BIT, @Data BIGINT, @FinalResult VARCHAR (MAX), @RequestCancellationOnFailure BIT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'PutJobStatus', @Mode AS VARCHAR (100), @st AS DATETIME = getUTCdate(), @Rows AS INT = 0, @PartitionId AS TINYINT = @JobId % 16, @GroupId AS BIGINT; +SET @Mode = 'Q=' + CONVERT (VARCHAR, @QueueType) + ' J=' + CONVERT (VARCHAR, @JobId) + ' P=' + CONVERT (VARCHAR, @PartitionId) + ' V=' + CONVERT (VARCHAR, @Version) + ' F=' + CONVERT (VARCHAR, @Failed) + ' R=' + isnull(@FinalResult, 'NULL'); +BEGIN TRY + UPDATE dbo.JobQueue + SET EndDate = getUTCdate(), + Status = CASE WHEN @Failed = 1 THEN 3 WHEN CancelRequested = 1 THEN 4 ELSE 2 END, + Data = @Data, + Result = @FinalResult, + @GroupId = GroupId + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Status = 1 + AND Version = @Version; + SET @Rows = @@rowcount; + IF @Rows = 0 + BEGIN + SET @GroupId = (SELECT GroupId + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Version = @Version + AND Status IN (2, 3, 4)); + IF @GroupId IS NULL + IF EXISTS (SELECT * + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId) + THROW 50412, 'Precondition failed', 1; + ELSE + THROW 50404, 'Job record not found', 1; + END + IF @Failed = 1 + AND @RequestCancellationOnFailure = 1 + EXECUTE dbo.PutJobCancelation @QueueType = @QueueType, @GroupId = @GroupId; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.ReadResource +@resourceTypeId SMALLINT, @resourceId VARCHAR (64), @version INT=NULL +AS +SET NOCOUNT ON; +IF (@version IS NULL) + BEGIN + SELECT ResourceSurrogateId, + Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM dbo.Resource + WHERE ResourceTypeId = @resourceTypeId + AND ResourceId = @resourceId + AND IsHistory = 0; + END +ELSE + BEGIN + SELECT ResourceSurrogateId, + Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM dbo.Resource + WHERE ResourceTypeId = @resourceTypeId + AND ResourceId = @resourceId + AND Version = @version; + END + +GO +CREATE PROCEDURE dbo.ReindexResource_2 +@resourceTypeId SMALLINT, @resourceId VARCHAR (64), @eTag INT=NULL, @searchParamHash VARCHAR (64), @resourceWriteClaims dbo.BulkResourceWriteClaimTableType_1 READONLY, @compartmentAssignments dbo.BulkCompartmentAssignmentTableType_1 READONLY, @referenceSearchParams dbo.BulkReferenceSearchParamTableType_1 READONLY, @tokenSearchParams dbo.BulkTokenSearchParamTableType_2 READONLY, @tokenTextSearchParams dbo.BulkTokenTextTableType_1 READONLY, @stringSearchParams dbo.BulkStringSearchParamTableType_2 READONLY, @numberSearchParams dbo.BulkNumberSearchParamTableType_1 READONLY, @quantitySearchParams dbo.BulkQuantitySearchParamTableType_1 READONLY, @uriSearchParams dbo.BulkUriSearchParamTableType_1 READONLY, @dateTimeSearchParms dbo.BulkDateTimeSearchParamTableType_2 READONLY, @referenceTokenCompositeSearchParams dbo.BulkReferenceTokenCompositeSearchParamTableType_2 READONLY, @tokenTokenCompositeSearchParams dbo.BulkTokenTokenCompositeSearchParamTableType_2 READONLY, @tokenDateTimeCompositeSearchParams dbo.BulkTokenDateTimeCompositeSearchParamTableType_2 READONLY, @tokenQuantityCompositeSearchParams dbo.BulkTokenQuantityCompositeSearchParamTableType_2 READONLY, @tokenStringCompositeSearchParams dbo.BulkTokenStringCompositeSearchParamTableType_2 READONLY, @tokenNumberNumberCompositeSearchParams dbo.BulkTokenNumberNumberCompositeSearchParamTableType_2 READONLY +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @resourceSurrogateId AS BIGINT; +DECLARE @version AS BIGINT; +SELECT @resourceSurrogateId = ResourceSurrogateId, + @version = Version +FROM dbo.Resource WITH (UPDLOCK, HOLDLOCK) +WHERE ResourceTypeId = @resourceTypeId + AND ResourceId = @resourceId + AND IsHistory = 0; +IF (@etag IS NOT NULL + AND @etag <> @version) + BEGIN + THROW 50412, 'Precondition failed', 1; + END +UPDATE dbo.Resource +SET SearchParamHash = @searchParamHash +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.ResourceWriteClaim +WHERE ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.CompartmentAssignment +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.ReferenceSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenText +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.StringSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.UriSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.NumberSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.QuantitySearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.DateTimeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.ReferenceTokenCompositeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenTokenCompositeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenDateTimeCompositeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenQuantityCompositeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenStringCompositeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenNumberNumberCompositeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) +SELECT @resourceSurrogateId, + ClaimTypeId, + ClaimValue +FROM @resourceWriteClaims; +INSERT INTO dbo.CompartmentAssignment (ResourceTypeId, ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + CompartmentTypeId, + ReferenceResourceId, + 0 +FROM @compartmentAssignments; +INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + BaseUri, + ReferenceResourceTypeId, + ReferenceResourceId, + ReferenceResourceVersion, + 0 +FROM @referenceSearchParams; +INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId, + Code, + CodeOverflow, + 0 +FROM @tokenSearchParams; +INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + Text, + 0 +FROM @tokenTextSearchParams; +INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsHistory, IsMin, IsMax) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + Text, + TextOverflow, + 0, + IsMin, + IsMax +FROM @stringSearchParams; +INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + Uri, + 0 +FROM @uriSearchParams; +INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SingleValue, + LowValue, + HighValue, + 0 +FROM @numberSearchParams; +INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId, + QuantityCodeId, + SingleValue, + LowValue, + HighValue, + 0 +FROM @quantitySearchParams; +INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsHistory, IsMin, IsMax) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + StartDateTime, + EndDateTime, + IsLongerThanADay, + 0, + IsMin, + IsMax +FROM @dateTimeSearchParms; +INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + BaseUri1, + ReferenceResourceTypeId1, + ReferenceResourceId1, + ReferenceResourceVersion1, + SystemId2, + Code2, + CodeOverflow2, + 0 +FROM @referenceTokenCompositeSearchParams; +INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SystemId2, + Code2, + CodeOverflow2, + 0 +FROM @tokenTokenCompositeSearchParams; +INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + StartDateTime2, + EndDateTime2, + IsLongerThanADay2, + 0 +FROM @tokenDateTimeCompositeSearchParams; +INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + SystemId2, + QuantityCodeId2, + LowValue2, + HighValue2, + 0 +FROM @tokenQuantityCompositeSearchParams; +INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + Text2, + TextOverflow2, + 0 +FROM @tokenStringCompositeSearchParams; +INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + LowValue2, + HighValue2, + SingleValue3, + LowValue3, + HighValue3, + HasRange, + 0 +FROM @tokenNumberNumberCompositeSearchParams; +COMMIT TRANSACTION; + +GO +CREATE OR ALTER PROCEDURE dbo.RemovePartitionFromResourceChanges_2 +@partitionNumberToSwitchOut INT, @partitionBoundaryToMerge DATETIME2 (7) +AS +BEGIN + TRUNCATE TABLE dbo.ResourceChangeDataStaging; + ALTER TABLE dbo.ResourceChangeData SWITCH PARTITION @partitionNumberToSwitchOut TO dbo.ResourceChangeDataStaging; + ALTER PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp( ) + MERGE RANGE (@partitionBoundaryToMerge); + TRUNCATE TABLE dbo.ResourceChangeDataStaging; +END + +GO +CREATE PROCEDURE dbo.ResetTask_2 +@taskId VARCHAR (64), @runId VARCHAR (50), @result VARCHAR (MAX) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +DECLARE @retryCount AS SMALLINT = NULL; +IF NOT EXISTS (SELECT * + FROM dbo.TaskInfo + WHERE TaskId = @taskId + AND RunId = @runId) + BEGIN + THROW 50404, 'Task not exist or runid not match', 1; + END +UPDATE dbo.TaskInfo +SET Status = 3, + EndDateTime = SYSUTCDATETIME(), + Result = @result, + @retryCount = retryCount +WHERE TaskId = @taskId + AND RunId = @runId + AND (MaxRetryCount <> -1 + AND RetryCount >= MaxRetryCount); +IF @retryCount IS NULL + UPDATE dbo.TaskInfo + SET Status = 1, + Result = @result, + RetryCount = RetryCount + 1, + RestartInfo = ISNULL(RestartInfo, '') + ' Prev: Worker=' + Worker + ' Start=' + CONVERT (VARCHAR, StartDateTime, 121) + WHERE TaskId = @taskId + AND RunId = @runId + AND Status <> 3 + AND (MaxRetryCount = -1 + OR RetryCount < MaxRetryCount); +EXECUTE dbo.GetTaskDetails @TaskId = @taskId; + +GO +CREATE PROCEDURE dbo.SwitchPartitionsIn +@Tbl VARCHAR (100) +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'SwitchPartitionsIn', @Mode AS VARCHAR (200) = 'Tbl=' + isnull(@Tbl, 'NULL'), @st AS DATETIME = getUTCdate(), @ResourceTypeId AS SMALLINT, @Rows AS BIGINT, @Txt AS VARCHAR (1000), @TblInt AS VARCHAR (100), @Ind AS VARCHAR (200), @IndId AS INT, @DataComp AS VARCHAR (100); +DECLARE @Indexes TABLE ( + IndId INT PRIMARY KEY, + name VARCHAR (200)); +DECLARE @ResourceTypes TABLE ( + ResourceTypeId SMALLINT PRIMARY KEY); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + IF @Tbl IS NULL + RAISERROR ('@Tbl IS NULL', 18, 127); + INSERT INTO @Indexes + SELECT index_id, + name + FROM sys.indexes + WHERE object_id = object_id(@Tbl) + AND is_disabled = 1; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Insert', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Indexes) + BEGIN + SELECT TOP 1 @IndId = IndId, + @Ind = name + FROM @Indexes + ORDER BY IndId; + SET @DataComp = CASE WHEN (SELECT PropertyValue + FROM dbo.IndexProperties + WHERE TableName = @Tbl + AND IndexName = @Ind) = 'PAGE' THEN ' PARTITION = ALL WITH (DATA_COMPRESSION = PAGE)' ELSE '' END; + SET @Txt = 'IF EXISTS (SELECT * FROM sys.indexes WHERE object_id = object_id(''' + @Tbl + ''') AND name = ''' + @Ind + ''' AND is_disabled = 1) ALTER INDEX ' + @Ind + ' ON dbo.' + @Tbl + ' REBUILD' + @DataComp; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Ind, @Action = 'Rebuild', @Text = @Txt; + DELETE @Indexes + WHERE IndId = @IndId; + END + INSERT INTO @ResourceTypes + SELECT CONVERT (SMALLINT, substring(name, charindex('_', name) + 1, 6)) AS ResourceTypeId + FROM sys.objects AS O + WHERE name LIKE @Tbl + '[_]%' + AND EXISTS (SELECT * + FROM sysindexes + WHERE id = O.object_id + AND indid IN (0, 1) + AND rows > 0); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '#ResourceTypes', @Action = 'Select Into', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @ResourceTypes) + BEGIN + SET @ResourceTypeId = (SELECT TOP 1 ResourceTypeId + FROM @ResourceTypes); + SET @TblInt = @Tbl + '_' + CONVERT (VARCHAR, @ResourceTypeId); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt; + SET @Txt = 'ALTER TABLE dbo.' + @TblInt + ' SWITCH TO dbo.' + @Tbl + ' PARTITION $partition.PartitionFunction_ResourceTypeId(' + CONVERT (VARCHAR, @ResourceTypeId) + ')'; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Switch in start', @Text = @Txt; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Switch in', @Text = @Txt; + IF EXISTS (SELECT * + FROM sysindexes + WHERE id = object_id(@TblInt) + AND rows > 0) + BEGIN + SET @Txt = @TblInt + ' is not empty after switch'; + RAISERROR (@Txt, 18, 127); + END + EXECUTE ('DROP TABLE dbo.' + @TblInt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Drop'; + DELETE @ResourceTypes + WHERE ResourceTypeId = @ResourceTypeId; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.SwitchPartitionsInAllTables +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'SwitchPartitionsInAllTables', @Mode AS VARCHAR (200) = 'PS=PartitionScheme_ResourceTypeId', @st AS DATETIME = getUTCdate(), @Tbl AS VARCHAR (100); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + DECLARE @Tables TABLE ( + name VARCHAR (100) PRIMARY KEY, + supported BIT ); + INSERT INTO @Tables + EXECUTE dbo.GetPartitionedTables @IncludeNotDisabled = 1, @IncludeNotSupported = 0; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Tables', @Action = 'Insert', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Tables) + BEGIN + SET @Tbl = (SELECT TOP 1 name + FROM @Tables + ORDER BY name); + EXECUTE dbo.SwitchPartitionsIn @Tbl = @Tbl; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = 'SwitchPartitionsIn', @Action = 'Execute', @Text = @Tbl; + DELETE @Tables + WHERE name = @Tbl; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.SwitchPartitionsOut +@Tbl VARCHAR (100), @RebuildClustered BIT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'SwitchPartitionsOut', @Mode AS VARCHAR (200) = 'Tbl=' + isnull(@Tbl, 'NULL') + ' ND=' + isnull(CONVERT (VARCHAR, @RebuildClustered), 'NULL'), @st AS DATETIME = getUTCdate(), @ResourceTypeId AS SMALLINT, @Rows AS BIGINT, @Txt AS VARCHAR (MAX), @TblInt AS VARCHAR (100), @IndId AS INT, @Ind AS VARCHAR (200), @Name AS VARCHAR (100), @checkName AS VARCHAR (200), @definition AS VARCHAR (200); +DECLARE @Indexes TABLE ( + IndId INT PRIMARY KEY, + name VARCHAR (200), + IsDisabled BIT ); +DECLARE @IndexesRT TABLE ( + IndId INT PRIMARY KEY, + name VARCHAR (200), + IsDisabled BIT ); +DECLARE @ResourceTypes TABLE ( + ResourceTypeId SMALLINT PRIMARY KEY, + partition_number_roundtrip INT , + partition_number INT , + row_count BIGINT ); +DECLARE @Names TABLE ( + name VARCHAR (100) PRIMARY KEY); +DECLARE @CheckConstraints TABLE ( + CheckName VARCHAR (200), + CheckDefinition VARCHAR (200)); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + IF @Tbl IS NULL + RAISERROR ('@Tbl IS NULL', 18, 127); + IF @RebuildClustered IS NULL + RAISERROR ('@RebuildClustered IS NULL', 18, 127); + INSERT INTO @Indexes + SELECT index_id, + name, + is_disabled + FROM sys.indexes + WHERE object_id = object_id(@Tbl) + AND (is_disabled = 0 + OR @RebuildClustered = 1); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Insert', @Rows = @@rowcount; + INSERT INTO @ResourceTypes + SELECT partition_number - 1 AS ResourceTypeId, + $PARTITION.PartitionFunction_ResourceTypeId (partition_number - 1) AS partition_number_roundtrip, + partition_number, + row_count + FROM sys.dm_db_partition_stats + WHERE object_id = object_id(@Tbl) + AND index_id = 1 + AND row_count > 0; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@ResourceTypes', @Action = 'Insert', @Rows = @@rowcount, @Text = 'For partition switch'; + IF EXISTS (SELECT * + FROM @ResourceTypes + WHERE partition_number_roundtrip <> partition_number) + RAISERROR ('Partition sanity check failed', 18, 127); + WHILE EXISTS (SELECT * + FROM @ResourceTypes) + BEGIN + SELECT TOP 1 @ResourceTypeId = ResourceTypeId, + @Rows = row_count + FROM @ResourceTypes + ORDER BY ResourceTypeId; + SET @TblInt = @Tbl + '_' + CONVERT (VARCHAR, @ResourceTypeId); + SET @Txt = 'Starting @ResourceTypeId=' + CONVERT (VARCHAR, @ResourceTypeId) + ' row_count=' + CONVERT (VARCHAR, @Rows); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Text = @Txt; + IF NOT EXISTS (SELECT * + FROM sysindexes + WHERE id = object_id(@TblInt) + AND rows > 0) + BEGIN + IF object_id(@TblInt) IS NOT NULL + BEGIN + EXECUTE ('DROP TABLE dbo.' + @TblInt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Drop'; + END + EXECUTE ('SELECT * INTO dbo.' + @TblInt + ' FROM dbo.' + @Tbl + ' WHERE 1 = 2'); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Select Into', @Rows = @@rowcount; + DELETE @CheckConstraints; + INSERT INTO @CheckConstraints + SELECT name, + definition + FROM sys.check_constraints + WHERE parent_object_id = object_id(@Tbl); + WHILE EXISTS (SELECT * + FROM @CheckConstraints) + BEGIN + SELECT TOP 1 @checkName = CheckName, + @definition = CheckDefinition + FROM @CheckConstraints; + SET @Txt = 'ALTER TABLE ' + @TblInt + ' ADD CHECK ' + @definition; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'ALTER', @Text = @Txt; + DELETE @CheckConstraints + WHERE CheckName = @checkName; + END + DELETE @Names; + INSERT INTO @Names + SELECT name + FROM sys.columns + WHERE object_id = object_id(@Tbl) + AND is_sparse = 1; + WHILE EXISTS (SELECT * + FROM @Names) + BEGIN + SET @Name = (SELECT TOP 1 name + FROM @Names + ORDER BY name); + SET @Txt = (SELECT 'ALTER TABLE dbo.' + @TblInt + ' ALTER COLUMN ' + @Name + ' ' + T.name + '(' + CONVERT (VARCHAR, C.precision) + ',' + CONVERT (VARCHAR, C.scale) + ') SPARSE NULL' + FROM sys.types AS T + INNER JOIN + sys.columns AS C + ON C.system_type_id = T.system_type_id + WHERE C.object_id = object_id(@Tbl) + AND C.name = @Name); + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'ALTER', @Text = @Txt; + DELETE @Names + WHERE name = @Name; + END + END + INSERT INTO @IndexesRT + SELECT * + FROM @Indexes + WHERE IsDisabled = 0; + WHILE EXISTS (SELECT * + FROM @IndexesRT) + BEGIN + SELECT TOP 1 @IndId = IndId, + @Ind = name + FROM @IndexesRT + ORDER BY IndId; + IF NOT EXISTS (SELECT * + FROM sys.indexes + WHERE object_id = object_id(@TblInt) + AND name = @Ind) + BEGIN + EXECUTE dbo.GetIndexCommands @Tbl = @Tbl, @Ind = @Ind, @AddPartClause = 0, @IncludeClustered = 1, @Txt = @Txt OUTPUT; + SET @Txt = replace(@Txt, '[' + @Tbl + ']', @TblInt); + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Create Index', @Text = @Txt; + END + DELETE @IndexesRT + WHERE IndId = @IndId; + END + SET @Txt = 'ALTER TABLE dbo.' + @TblInt + ' ADD CHECK (ResourceTypeId >= ' + CONVERT (VARCHAR, @ResourceTypeId) + ' AND ResourceTypeId < ' + CONVERT (VARCHAR, @ResourceTypeId) + ' + 1)'; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Add check', @Text = @Txt; + SET @Txt = 'ALTER TABLE dbo.' + @Tbl + ' SWITCH PARTITION $partition.PartitionFunction_ResourceTypeId(' + CONVERT (VARCHAR, @ResourceTypeId) + ') TO dbo.' + @TblInt; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Switch out start', @Text = @Txt; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Switch out end', @Text = @Txt; + DELETE @ResourceTypes + WHERE ResourceTypeId = @ResourceTypeId; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.SwitchPartitionsOutAllTables +@RebuildClustered BIT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'SwitchPartitionsOutAllTables', @Mode AS VARCHAR (200) = 'PS=PartitionScheme_ResourceTypeId ND=' + isnull(CONVERT (VARCHAR, @RebuildClustered), 'NULL'), @st AS DATETIME = getUTCdate(), @Tbl AS VARCHAR (100); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + DECLARE @Tables TABLE ( + name VARCHAR (100) PRIMARY KEY, + supported BIT ); + INSERT INTO @Tables + EXECUTE dbo.GetPartitionedTables @IncludeNotDisabled = @RebuildClustered, @IncludeNotSupported = 0; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Tables', @Action = 'Insert', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Tables) + BEGIN + SET @Tbl = (SELECT TOP 1 name + FROM @Tables + ORDER BY name); + EXECUTE dbo.SwitchPartitionsOut @Tbl = @Tbl, @RebuildClustered = @RebuildClustered; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = 'SwitchPartitionsOut', @Action = 'Execute', @Text = @Tbl; + DELETE @Tables + WHERE name = @Tbl; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE [dbo].[TaskKeepAlive] +@taskId VARCHAR (64), @runId VARCHAR (50) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +IF NOT EXISTS (SELECT * + FROM [dbo].[TaskInfo] + WHERE TaskId = @taskId + AND RunId = @runId) + BEGIN + THROW 50404, 'Task not exist or runid not match', 1; + END +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +UPDATE dbo.TaskInfo +SET HeartbeatDateTime = @heartbeatDateTime +WHERE TaskId = @taskId; +SELECT TaskId, + QueueId, + Status, + TaskTypeId, + RunId, + IsCanceled, + RetryCount, + MaxRetryCount, + HeartbeatDateTime, + InputData, + TaskContext, + Result +FROM [dbo].[TaskInfo] +WHERE TaskId = @taskId; +COMMIT TRANSACTION; + +GO +CREATE OR ALTER PROCEDURE dbo.UpdateEventAgentCheckpoint +@CheckpointId VARCHAR (64), @LastProcessedDateTime DATETIMEOFFSET (7)=NULL, @LastProcessedIdentifier VARCHAR (64)=NULL +AS +BEGIN + IF EXISTS (SELECT * + FROM dbo.EventAgentCheckpoint + WHERE CheckpointId = @CheckpointId) + UPDATE dbo.EventAgentCheckpoint + SET CheckpointId = @CheckpointId, + LastProcessedDateTime = @LastProcessedDateTime, + LastProcessedIdentifier = @LastProcessedIdentifier, + UpdatedOn = sysutcdatetime() + WHERE CheckpointId = @CheckpointId; + ELSE + INSERT INTO dbo.EventAgentCheckpoint (CheckpointId, LastProcessedDateTime, LastProcessedIdentifier, UpdatedOn) + VALUES (@CheckpointId, @LastProcessedDateTime, @LastProcessedIdentifier, sysutcdatetime()); +END + +GO +CREATE PROCEDURE dbo.UpdateExportJob +@id VARCHAR (64), @status VARCHAR (10), @rawJobRecord VARCHAR (MAX), @jobVersion BINARY (8) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @currentJobVersion AS BINARY (8); +SELECT @currentJobVersion = JobVersion +FROM dbo.ExportJob WITH (UPDLOCK, HOLDLOCK) +WHERE Id = @id; +IF (@currentJobVersion IS NULL) + BEGIN + THROW 50404, 'Export job record not found', 1; + END +IF (@jobVersion <> @currentJobVersion) + BEGIN + THROW 50412, 'Precondition failed', 1; + END +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +UPDATE dbo.ExportJob +SET Status = @status, + HeartbeatDateTime = @heartbeatDateTime, + RawJobRecord = @rawJobRecord +WHERE Id = @id; +SELECT @@DBTS; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.UpdateReindexJob +@id VARCHAR (64), @status VARCHAR (10), @rawJobRecord VARCHAR (MAX), @jobVersion BINARY (8) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @currentJobVersion AS BINARY (8); +SELECT @currentJobVersion = JobVersion +FROM dbo.ReindexJob WITH (UPDLOCK, HOLDLOCK) +WHERE Id = @id; +IF (@currentJobVersion IS NULL) + BEGIN + THROW 50404, 'Reindex job record not found', 1; + END +IF (@jobVersion <> @currentJobVersion) + BEGIN + THROW 50412, 'Precondition failed', 1; + END +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +UPDATE dbo.ReindexJob +SET Status = @status, + HeartbeatDateTime = @heartbeatDateTime, + RawJobRecord = @rawJobRecord +WHERE Id = @id; +SELECT @@DBTS; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.UpdateResourceSearchParams +@FailedResources INT=0 OUTPUT, @Resources dbo.ResourceList READONLY, @ResourceWriteClaims dbo.ResourceWriteClaimList READONLY, @ReferenceSearchParams dbo.ReferenceSearchParamList READONLY, @TokenSearchParams dbo.TokenSearchParamList READONLY, @TokenTexts dbo.TokenTextList READONLY, @StringSearchParams dbo.StringSearchParamList READONLY, @UriSearchParams dbo.UriSearchParamList READONLY, @NumberSearchParams dbo.NumberSearchParamList READONLY, @QuantitySearchParams dbo.QuantitySearchParamList READONLY, @DateTimeSearchParams dbo.DateTimeSearchParamList READONLY, @ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY, @TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY, @TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY, @TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY, @TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY, @TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY +AS +SET NOCOUNT ON; +DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (200) = isnull((SELECT 'RT=[' + CONVERT (VARCHAR, min(ResourceTypeId)) + ',' + CONVERT (VARCHAR, max(ResourceTypeId)) + '] Sur=[' + CONVERT (VARCHAR, min(ResourceSurrogateId)) + ',' + CONVERT (VARCHAR, max(ResourceSurrogateId)) + '] V=' + CONVERT (VARCHAR, max(Version)) + ' Rows=' + CONVERT (VARCHAR, count(*)) + FROM @Resources), 'Input=Empty'), @Rows AS INT; +BEGIN TRY + DECLARE @Ids TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL); + BEGIN TRANSACTION; + UPDATE B + SET SearchParamHash = A.SearchParamHash + OUTPUT deleted.ResourceTypeId, deleted.ResourceSurrogateId INTO @Ids + FROM @Resources AS A + INNER JOIN + dbo.Resource AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + WHERE B.IsHistory = 0; + SET @Rows = @@rowcount; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.ResourceWriteClaim AS B + ON B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.ReferenceSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenText AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.StringSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.UriSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.NumberSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.QuantitySearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.DateTimeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.ReferenceTokenCompositeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenTokenCompositeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenDateTimeCompositeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenQuantityCompositeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenStringCompositeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenNumberNumberCompositeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) + SELECT ResourceSurrogateId, + ClaimTypeId, + ClaimValue + FROM @ResourceWriteClaims; + INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri, + ReferenceResourceTypeId, + ReferenceResourceId, + ReferenceResourceVersion + FROM @ReferenceSearchParams; + INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + Code, + CodeOverflow + FROM @TokenSearchParams; + INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text + FROM @TokenTexts; + INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text, + TextOverflow, + IsMin, + IsMax + FROM @StringSearchParams; + INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Uri + FROM @UriSearchParams; + INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SingleValue, + LowValue, + HighValue + FROM @NumberSearchParams; + INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + QuantityCodeId, + SingleValue, + LowValue, + HighValue + FROM @QuantitySearchParams; + INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + StartDateTime, + EndDateTime, + IsLongerThanADay, + IsMin, + IsMax + FROM @DateTimeSearchParams; + INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri1, + ReferenceResourceTypeId1, + ReferenceResourceId1, + ReferenceResourceVersion1, + SystemId2, + Code2, + CodeOverflow2 + FROM @ReferenceTokenCompositeSearchParams; + INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SystemId2, + Code2, + CodeOverflow2 + FROM @TokenTokenCompositeSearchParams; + INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + StartDateTime2, + EndDateTime2, + IsLongerThanADay2 + FROM @TokenDateTimeCompositeSearchParams; + INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + SystemId2, + QuantityCodeId2, + LowValue2, + HighValue2 + FROM @TokenQuantityCompositeSearchParams; + INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + Text2, + TextOverflow2 + FROM @TokenStringCompositeSearchParams; + INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + LowValue2, + HighValue2, + SingleValue3, + LowValue3, + HighValue3, + HasRange + FROM @TokenNumberNumberCompositeSearchParams; + COMMIT TRANSACTION; + SET @FailedResources = (SELECT count(*) + FROM @Resources) - @Rows; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE [dbo].[UpdateTaskContext] +@taskId VARCHAR (64), @taskContext VARCHAR (MAX), @runId VARCHAR (50) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +IF NOT EXISTS (SELECT * + FROM [dbo].[TaskInfo] + WHERE TaskId = @taskId + AND RunId = @runId) + BEGIN + THROW 50404, 'Task not exist or runid not match', 1; + END +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +UPDATE dbo.TaskInfo +SET HeartbeatDateTime = @heartbeatDateTime, + TaskContext = @taskContext +WHERE TaskId = @taskId; +SELECT TaskId, + QueueId, + Status, + TaskTypeId, + RunId, + IsCanceled, + RetryCount, + MaxRetryCount, + HeartbeatDateTime, + InputData, + TaskContext, + Result +FROM [dbo].[TaskInfo] +WHERE TaskId = @taskId; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.UpsertResource_7 +@baseResourceSurrogateId BIGINT, @resourceTypeId SMALLINT, @resourceId VARCHAR (64), @eTag INT=NULL, @allowCreate BIT, @isDeleted BIT, @keepHistory BIT, @requireETagOnUpdate BIT, @requestMethod VARCHAR (10), @searchParamHash VARCHAR (64), @rawResource VARBINARY (MAX), @resourceWriteClaims dbo.BulkResourceWriteClaimTableType_1 READONLY, @compartmentAssignments dbo.BulkCompartmentAssignmentTableType_1 READONLY, @referenceSearchParams dbo.BulkReferenceSearchParamTableType_1 READONLY, @tokenSearchParams dbo.BulkTokenSearchParamTableType_2 READONLY, @tokenTextSearchParams dbo.BulkTokenTextTableType_1 READONLY, @stringSearchParams dbo.BulkStringSearchParamTableType_2 READONLY, @numberSearchParams dbo.BulkNumberSearchParamTableType_1 READONLY, @quantitySearchParams dbo.BulkQuantitySearchParamTableType_1 READONLY, @uriSearchParams dbo.BulkUriSearchParamTableType_1 READONLY, @dateTimeSearchParms dbo.BulkDateTimeSearchParamTableType_2 READONLY, @referenceTokenCompositeSearchParams dbo.BulkReferenceTokenCompositeSearchParamTableType_2 READONLY, @tokenTokenCompositeSearchParams dbo.BulkTokenTokenCompositeSearchParamTableType_2 READONLY, @tokenDateTimeCompositeSearchParams dbo.BulkTokenDateTimeCompositeSearchParamTableType_2 READONLY, @tokenQuantityCompositeSearchParams dbo.BulkTokenQuantityCompositeSearchParamTableType_2 READONLY, @tokenStringCompositeSearchParams dbo.BulkTokenStringCompositeSearchParamTableType_2 READONLY, @tokenNumberNumberCompositeSearchParams dbo.BulkTokenNumberNumberCompositeSearchParamTableType_2 READONLY, @isResourceChangeCaptureEnabled BIT=0, @comparedVersion INT=NULL +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +DECLARE @previousResourceSurrogateId AS BIGINT, @previousVersion AS BIGINT, @previousIsDeleted AS BIT, @version AS INT, @resourceSurrogateId AS BIGINT, @InitialTranCount AS INT = @@trancount; +IF @InitialTranCount = 0 + BEGIN TRANSACTION; +SELECT @previousResourceSurrogateId = ResourceSurrogateId, + @previousVersion = Version, + @previousIsDeleted = IsDeleted +FROM dbo.Resource WITH (UPDLOCK, HOLDLOCK) +WHERE ResourceTypeId = @resourceTypeId + AND ResourceId = @resourceId + AND IsHistory = 0; +IF @previousResourceSurrogateId IS NULL + SET @version = 1; +ELSE + BEGIN + IF @isDeleted = 0 + BEGIN + IF @comparedVersion IS NULL + OR @comparedVersion <> @previousVersion + BEGIN + THROW 50409, 'Resource has been recently updated or added, please compare the resource content in code for any duplicate updates', 1; + END + END + SET @version = @previousVersion + 1; + IF @keepHistory = 1 + UPDATE dbo.Resource + SET IsHistory = 1 + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + ELSE + DELETE dbo.Resource + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.ResourceWriteClaim + WHERE ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.CompartmentAssignment + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.ReferenceSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenText + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.StringSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.UriSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.NumberSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.QuantitySearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.DateTimeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.ReferenceTokenCompositeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenTokenCompositeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenDateTimeCompositeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenQuantityCompositeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenStringCompositeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenNumberNumberCompositeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + END +SET @resourceSurrogateId = @baseResourceSurrogateId + ( NEXT VALUE FOR ResourceSurrogateIdUniquifierSequence); +INSERT INTO dbo.Resource (ResourceTypeId, ResourceId, Version, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, RawResource, IsRawResourceMetaSet, SearchParamHash) +SELECT @resourceTypeId, + @resourceId, + @version, + 0, + @resourceSurrogateId, + @isDeleted, + @requestMethod, + @rawResource, + CASE WHEN @version = 1 THEN 1 ELSE 0 END, + @searchParamHash; +INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) +SELECT @resourceSurrogateId, + ClaimTypeId, + ClaimValue +FROM @resourceWriteClaims; +INSERT INTO dbo.CompartmentAssignment (ResourceTypeId, ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + CompartmentTypeId, + ReferenceResourceId, + 0 +FROM @compartmentAssignments; +INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + BaseUri, + ReferenceResourceTypeId, + ReferenceResourceId, + ReferenceResourceVersion, + 0 +FROM @referenceSearchParams; +INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId, + Code, + CodeOverflow, + 0 +FROM @tokenSearchParams; +INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + Text, + 0 +FROM @tokenTextSearchParams; +INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsHistory, IsMin, IsMax) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + Text, + TextOverflow, + 0, + IsMin, + IsMax +FROM @stringSearchParams; +INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + Uri, + 0 +FROM @uriSearchParams; +INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SingleValue, + LowValue, + HighValue, + 0 +FROM @numberSearchParams; +INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId, + QuantityCodeId, + SingleValue, + LowValue, + HighValue, + 0 +FROM @quantitySearchParams; +INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsHistory, IsMin, IsMax) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + StartDateTime, + EndDateTime, + IsLongerThanADay, + 0, + IsMin, + IsMax +FROM @dateTimeSearchParms; +INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + BaseUri1, + ReferenceResourceTypeId1, + ReferenceResourceId1, + ReferenceResourceVersion1, + SystemId2, + Code2, + CodeOverflow2, + 0 +FROM @referenceTokenCompositeSearchParams; +INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SystemId2, + Code2, + CodeOverflow2, + 0 +FROM @tokenTokenCompositeSearchParams; +INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + StartDateTime2, + EndDateTime2, + IsLongerThanADay2, + 0 +FROM @tokenDateTimeCompositeSearchParams; +INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + SystemId2, + QuantityCodeId2, + LowValue2, + HighValue2, + 0 +FROM @tokenQuantityCompositeSearchParams; +INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + Text2, + TextOverflow2, + 0 +FROM @tokenStringCompositeSearchParams; +INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + LowValue2, + HighValue2, + SingleValue3, + LowValue3, + HighValue3, + HasRange, + 0 +FROM @tokenNumberNumberCompositeSearchParams; +SELECT @version; +IF @isResourceChangeCaptureEnabled = 1 + EXECUTE dbo.CaptureResourceChanges @isDeleted = @isDeleted, @version = @version, @resourceId = @resourceId, @resourceTypeId = @resourceTypeId; +IF @InitialTranCount = 0 + COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.UpsertSearchParams +@searchParams dbo.SearchParamTableType_2 READONLY +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DECLARE @lastUpdated AS DATETIMEOFFSET (7) = SYSDATETIMEOFFSET(); +DECLARE @summaryOfChanges TABLE ( + Uri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + Action VARCHAR (20) NOT NULL); +MERGE INTO dbo.SearchParam WITH (TABLOCKX) + AS target +USING @searchParams AS source ON target.Uri = source.Uri +WHEN MATCHED THEN UPDATE +SET Status = source.Status, + LastUpdated = @lastUpdated, + IsPartiallySupported = source.IsPartiallySupported +WHEN NOT MATCHED BY TARGET THEN INSERT (Uri, Status, LastUpdated, IsPartiallySupported) VALUES (source.Uri, source.Status, @lastUpdated, source.IsPartiallySupported) +OUTPUT source.Uri, $ACTION INTO @summaryOfChanges; +SELECT SearchParamId, + SearchParam.Uri +FROM dbo.SearchParam AS searchParam + INNER JOIN + @summaryOfChanges AS upsertedSearchParam + ON searchParam.Uri = upsertedSearchParam.Uri +WHERE upsertedSearchParam.Action = 'INSERT'; +COMMIT TRANSACTION; + +GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs index a35062b9ae..fdd74278b4 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs @@ -73,6 +73,7 @@ public enum SchemaVersion V61 = 61, V62 = 62, V63 = 63, - V64 = 64 + V64 = 64, + V65 = 65, } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs index 2a68fa3068..8174a4dc2c 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs @@ -8,7 +8,7 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Schema public static class SchemaVersionConstants { public const int Min = (int)SchemaVersion.V63; - public const int Max = (int)SchemaVersion.V64; + public const int Max = (int)SchemaVersion.V65; public const int MinForUpgrade = (int)SchemaVersion.V60; // this is used for upgrade tests only public const int SearchParameterStatusSchemaVersion = (int)SchemaVersion.V6; public const int SupportForReferencesWithMissingTypeVersion = (int)SchemaVersion.V7; diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/TransactionCheckWithInitialiScript.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/TransactionCheckWithInitialiScript.sql index 36f2010660..42e8869131 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/TransactionCheckWithInitialiScript.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/TransactionCheckWithInitialiScript.sql @@ -19,6 +19,6 @@ Go INSERT INTO dbo.SchemaVersion VALUES - (64, 'started') + (65, 'started') Go diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetResourcesByTypeAndSurrogateIdRange.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetResourcesByTypeAndSurrogateIdRange.sql index ebdbcd413c..be7eb142e2 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetResourcesByTypeAndSurrogateIdRange.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetResourcesByTypeAndSurrogateIdRange.sql @@ -1,66 +1,60 @@ --DROP PROCEDURE dbo.GetResourcesByTypeAndSurrogateIdRange GO -CREATE PROCEDURE dbo.GetResourcesByTypeAndSurrogateIdRange @ResourceTypeId smallint, @StartId bigint, @EndId bigint, @GlobalStartId bigint = NULL, @GlobalEndId bigint = NULL +CREATE PROCEDURE dbo.GetResourcesByTypeAndSurrogateIdRange @ResourceTypeId smallint, @StartId bigint, @EndId bigint, @GlobalEndId bigint = NULL, @IncludeHistory bit = 0, @IncludeDeleted bit = 0 AS set nocount on DECLARE @SP varchar(100) = 'GetResourcesByTypeAndSurrogateIdRange' ,@Mode varchar(100) = 'RT='+isnull(convert(varchar,@ResourceTypeId),'NULL') +' S='+isnull(convert(varchar,@StartId),'NULL') +' E='+isnull(convert(varchar,@EndId),'NULL') - +' GS='+isnull(convert(varchar,@GlobalStartId),'NULL') -- Is global start id needed? I'm not seeing a usecase for setting it. +' GE='+isnull(convert(varchar,@GlobalEndId),'NULL') -- Could this just be a boolean for if historical records should be returned? GlobalEndId should equal EndId in all cases I can think of. + +' HI='+isnull(convert(varchar,@IncludeHistory),'NULL') + +' DE'+isnull(convert(varchar,@IncludeDeleted),'NULL') ,@st datetime = getUTCdate() + ,@DummyTop bigint = 9223372036854775807 BEGIN TRY - DECLARE @ResourceIds TABLE (ResourceId varchar(64) COLLATE Latin1_General_100_CS_AS, ResourceSurrogateId bigint, RowId int, PRIMARY KEY (ResourceId, RowId)) - IF @GlobalStartId IS NULL -- export from time zero (no lower boundary) - SET @GlobalStartId = 0 - IF @GlobalEndId IS NOT NULL -- snapshot view + DECLARE @ResourceIds TABLE (ResourceId varchar(64) COLLATE Latin1_General_100_CS_AS PRIMARY KEY) + DECLARE @SurrogateIds TABLE (MaxSurrogateId bigint PRIMARY KEY) + + IF @GlobalEndId IS NOT NULL AND @IncludeHistory = 0 -- snapshot view + BEGIN INSERT INTO @ResourceIds - SELECT ResourceId, ResourceSurrogateId, RowId = row_number() OVER (PARTITION BY ResourceId ORDER BY ResourceSurrogateId DESC) + SELECT DISTINCT ResourceId FROM dbo.Resource - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceId IN (SELECT DISTINCT ResourceId - FROM dbo.Resource - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId BETWEEN @StartId AND @EndId - AND IsHistory = 1 - AND IsDeleted = 0 - ) - AND ResourceSurrogateId BETWEEN @GlobalStartId AND @GlobalEndId + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + AND IsHistory = 1 + AND (IsDeleted = 0 OR @IncludeDeleted = 1) + OPTION (MAXDOP 1) - IF EXISTS (SELECT * FROM @ResourceIds) - BEGIN - DECLARE @SurrogateIdMap TABLE (MaxSurrogateId bigint PRIMARY KEY) - INSERT INTO @SurrogateIdMap - SELECT MaxSurrogateId = A.ResourceSurrogateId - FROM (SELECT * FROM @ResourceIds WHERE RowId = 1 AND ResourceSurrogateId BETWEEN @StartId AND @EndId) A - SELECT @ResourceTypeId - ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.ResourceId ELSE A.ResourceId END - ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.Version ELSE A.Version END - ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.IsDeleted ELSE A.IsDeleted END - ,isnull(C.ResourceSurrogateId, A.ResourceSurrogateId) - ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.RequestMethod ELSE A.RequestMethod END - ,IsMatch = convert(bit,1) - ,IsPartial = convert(bit,0) - ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.IsRawResourceMetaSet ELSE A.IsRawResourceMetaSet END - ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.SearchParamHash ELSE A.SearchParamHash END - ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.RawResource ELSE A.RawResource END - FROM dbo.Resource A - LEFT OUTER JOIN @SurrogateIdMap B ON B.MaxSurrogateId = A.ResourceSurrogateId - LEFT OUTER JOIN dbo.Resource C ON C.ResourceTypeId = @ResourceTypeId AND C.ResourceSurrogateId = MaxSurrogateId - WHERE A.ResourceTypeId = @ResourceTypeId - AND A.ResourceSurrogateId BETWEEN @StartId AND @EndId - AND (A.IsHistory = 0 OR MaxSurrogateId IS NOT NULL) - AND A.IsDeleted = 0 + IF @@rowcount > 0 + INSERT INTO @SurrogateIds + SELECT ResourceSurrogateId + FROM (SELECT ResourceId, ResourceSurrogateId, RowId = row_number() OVER (PARTITION BY ResourceId ORDER BY ResourceSurrogateId DESC) + FROM dbo.Resource WITH (INDEX = IX_Resource_ResourceTypeId_ResourceId_Version) -- w/o hint access to Resource table is inefficient when many versions are present. Hint is ignored if Resource is a view. + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceId IN (SELECT TOP (@DummyTop) ResourceId FROM @ResourceIds) + AND ResourceSurrogateId BETWEEN @StartId AND @GlobalEndId + ) A + WHERE RowId = 1 + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)) END - ELSE - SELECT ResourceTypeId, ResourceId, Version, IsDeleted, ResourceSurrogateId, RequestMethod, IsMatch = convert(bit,1), IsPartial = convert(bit,0), IsRawResourceMetaSet, SearchParamHash, RawResource - FROM dbo.Resource - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId BETWEEN @StartId AND @EndId - AND IsHistory = 0 - AND IsDeleted = 0 + + SELECT ResourceTypeId, ResourceId, Version, IsDeleted, ResourceSurrogateId, RequestMethod, IsMatch = convert(bit,1), IsPartial = convert(bit,0), IsRawResourceMetaSet, SearchParamHash, RawResource + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + AND (IsHistory = 0 OR @IncludeHistory = 1) + AND (IsDeleted = 0 OR @IncludeDeleted = 1) + UNION ALL + SELECT ResourceTypeId, ResourceId, Version, IsDeleted, ResourceSurrogateId, RequestMethod, IsMatch = convert(bit,1), IsPartial = convert(bit,0), IsRawResourceMetaSet, SearchParamHash, RawResource + FROM @SurrogateIds + JOIN dbo.Resource ON ResourceTypeId = @ResourceTypeId AND ResourceSurrogateId = MaxSurrogateId + WHERE IsHistory = 1 + AND (IsDeleted = 0 OR @IncludeDeleted = 1) + OPTION (MAXDOP 1) EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@st,@Rows=@@rowcount END TRY diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs index 5e27471a3a..cadf0f0e02 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs @@ -565,7 +565,6 @@ private static void PopulateSqlCommandFromQueryHints(SqlCommand command, short r command.Parameters.AddWithValue("@ResourceTypeId", resourceTypeId); command.Parameters.AddWithValue("@StartId", startId); command.Parameters.AddWithValue("@EndId", endId); - command.Parameters.AddWithValue("@GlobalStartId", globalStartId); command.Parameters.AddWithValue("@GlobalEndId", globalEndId); command.Parameters.AddWithValue("@IncludeHistory", includeHistory); command.Parameters.AddWithValue("@IncludeDeleted", includeDeleted); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj b/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj index cd24927e23..8c734a75ea 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj +++ b/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj @@ -1,7 +1,7 @@  - 64 + 65 Features\Schema\Migrations\$(LatestSchemaVersion).sql diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs index ac231f806c..218d0d32ba 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs @@ -199,7 +199,7 @@ public async Task GivenFhirServer_WhenDataIsExportedWithHistory_ThenExportedData var uniqueFixtureResources = string.Join(',', _fixture.TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. - Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:O}&_type={uniqueFixtureResources}&_includeHistory=true&{parallelQueryParam}"); + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:O}&_type={uniqueFixtureResources}&includeAssociatedData=_history&{parallelQueryParam}"); IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); @@ -221,7 +221,7 @@ public async Task GivenFhirServer_WhenDataIsExportedWithSoftDeletes_ThenExported var uniqueFixtureResources = string.Join(',', _fixture.TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. - Uri contentLocation = await _fixture.TestFhirClient.ExportAsync($"_since={_fixture.TestDataInsertionTime:O}&_type={uniqueFixtureResources}&_includeDeleted=true&{parallelQueryParam}"); + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:O}&_type={uniqueFixtureResources}&includeAssociatedData=_deleted&{parallelQueryParam}"); IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); @@ -243,7 +243,7 @@ public async Task GivenFhirServer_WhenDataIsExportedWithHistoryAndSoftDeletes_Th var uniqueFixtureResources = string.Join(',', _fixture.TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. - Uri contentLocation = await _fixture.TestFhirClient.ExportAsync($"_since={_fixture.TestDataInsertionTime:O}&_type={uniqueFixtureResources}&_includeHistory=true&_includeDeleted=true&{parallelQueryParam}"); + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:O}&_type={uniqueFixtureResources}&includeAssociatedData=_history,_deleted&{parallelQueryParam}"); IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); From db0b66f906333907ebbf0a8fe574990da293faee Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 12 Oct 2023 14:15:35 -0700 Subject: [PATCH 34/58] Updated export E2E tests for parallel export multi-job --- .../Rest/Export/ExportDataTestFixture.cs | 3 ++- .../Rest/Export/ExportTestHelper.cs | 10 +++++----- .../Export/StartupForExportTestProvider.cs | 20 ++++++++++++++++--- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs index 5832acc029..b39ead298e 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs @@ -206,7 +206,8 @@ private async System.Threading.Tasks.Task> SaveResourceListToServ return rtn; } - private List GenerateTestResources(int numberOfPatients = 5, int numberOfEncountersPerPatient = 1, int numberOfObservationsPerEncounter = 2) + // 27 patients, 54 encounters, and, 108 observations. + private List GenerateTestResources(int numberOfPatients = 27, int numberOfEncountersPerPatient = 2, int numberOfObservationsPerEncounter = 2) { var resources = new List(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs index d630d7ac55..f45fe557be 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs @@ -271,11 +271,11 @@ internal static bool ValidateDataFromBothSources( } // Enable this check when creating/updating data validation tests to ensure there is data to export - if (dataFromStorageAccount.Count == 0) - { - outputHelper.WriteLine("No data exported. This test expects data to be present."); - return false; - } + // if (dataFromStorageAccount.Count == 0) + // { + // outputHelper.WriteLine("No data exported. This test expects data to be present."); + // return false; + // } int wrongCount = 0; foreach (KeyValuePair<(string resourceType, string resourceId, string versionId), Resource> kvp in dataFromServer) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/StartupForExportTestProvider.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/StartupForExportTestProvider.cs index 018228e45b..295535f474 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/StartupForExportTestProvider.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/StartupForExportTestProvider.cs @@ -10,6 +10,7 @@ using Microsoft.Extensions.Options; using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Api.Configs; +using Microsoft.Health.Fhir.Core.Configs; using Microsoft.Health.Fhir.Core.Features.Operations.Export; using Microsoft.Health.Fhir.Tests.E2E.Rest.Metric; @@ -18,21 +19,34 @@ namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Export [RequiresIsolatedDatabase] public class StartupForExportTestProvider : StartupBaseForCustomProviders { + private IConfiguration _configuration; + public StartupForExportTestProvider(IConfiguration configuration) : base(configuration) { + _configuration = configuration; } public override void ConfigureServices(IServiceCollection services) { base.ConfigureServices(services); - FeatureConfiguration configuration = new FeatureConfiguration() + FeatureConfiguration featureConfiguration = new() { SupportsAnonymizedExport = true, }; - IOptions options = Options.Create(configuration); - services.Replace(new ServiceDescriptor(typeof(IOptions), options)); + IOptions featureOptions = Options.Create(featureConfiguration); + services.Replace(new ServiceDescriptor(typeof(IOptions), featureOptions)); + + ExportJobConfiguration existingExportOptions = new(); + _configuration.GetSection("FhirServer:Operations:Export").Bind(existingExportOptions); + + // ExportDataTestFixture generates 27 patients, 54 encounters, and, 108 observations with history / deletes. + // We want to test the splitting of jobs and orchestration continuation. Hence this config. + existingExportOptions.MaximumNumberOfResourcesPerQuery = 20; + existingExportOptions.NumberOfParallelRecordRanges = 4; + IOptions exportOptions = Options.Create(existingExportOptions); + services.Replace(new ServiceDescriptor(typeof(IOptions), exportOptions)); services.Add() .Singleton() From d8c9d60e02b331139faa397f0e8ce92c3f0af7b0 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 12 Oct 2023 17:55:00 -0700 Subject: [PATCH 35/58] fixed merge regression --- .../Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs index 98cb8d4641..7f40fe19a0 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs @@ -191,7 +191,7 @@ public override object VisitSqlRoot(SqlRootExpression expression, SearchOptions StringBuilder.Append("FROM ").Append(VLatest.Resource).Append(" ").Append(resourceTableAlias); if (expression.SearchParamTableExpressions.Count == 0 && - !_searchType.HasFlag(SqlSearchType.History) && + !_searchType.HasFlag(SqlSearchType.IncludeHistory) && !_searchType.HasFlag(SqlSearchType.IncludeDeleted) && expression.ResourceTableExpressions.Any(e => e.AcceptVisitor(ExpressionContainsParameterVisitor.Instance, SearchParameterNames.ResourceType)) && !expression.ResourceTableExpressions.Any(e => e.AcceptVisitor(ExpressionContainsParameterVisitor.Instance, SearchParameterNames.Id))) From 4246319280185e2716d51db09403201b72713973 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Mon, 16 Oct 2023 14:15:38 -0700 Subject: [PATCH 36/58] Removed unnecesary usings --- .../Controllers/ExportController.cs | 2 -- .../Features/Schema/Migrations/66.sql | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs b/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs index 175b1b28f1..5010826d28 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs @@ -10,7 +10,6 @@ using System.Threading.Tasks; using EnsureThat; using Hl7.Fhir.Model; -using ICSharpCode.SharpZipLib; using MediatR; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; @@ -34,7 +33,6 @@ using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.TemplateManagement.Models; using Microsoft.Health.Fhir.ValueSets; -using static ICSharpCode.SharpZipLib.Zip.ExtendedUnixData; namespace Microsoft.Health.Fhir.Api.Controllers { diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/66.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/66.sql index 03712c9c52..72acc71117 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/66.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/66.sql @@ -15,7 +15,7 @@ IF EXISTS (SELECT * GO INSERT INTO dbo.SchemaVersion -VALUES (65, 'started'); +VALUES (66, 'started'); CREATE PARTITION FUNCTION PartitionFunction_ResourceTypeId(SMALLINT) AS RANGE RIGHT From 7ec234a962bef0c2103d64d31a65a7d13b55175e Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Tue, 17 Oct 2023 09:29:44 -0700 Subject: [PATCH 37/58] Fixed tx issue in export data tests --- .../Rest/Export/ExportDataTestFixture.cs | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs index b39ead298e..aa07219747 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs @@ -77,6 +77,22 @@ public string ExportTestFilterQueryParameters(params string[] uniqueResourceType protected override async Task OnInitializedAsync() { await SaveTestResourcesToServer(); + + // await StartTestExportOperations(); + } + + private async Task StartTestExportOperations() + { + var uniqueFixtureResources = string.Join(',', TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); + string parameters = $"_since={TestDataInsertionTime:o}&_type={uniqueFixtureResources}"; + + ExportTestCasesContentUrls.Add( + $"{nameof(ExportDataTests.GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAsDataInFhirServer)}-since", + await TestFhirClient.ExportAsync(parameters: parameters)); + + ExportTestCasesContentUrls.Add( + $"{nameof(ExportDataTests.GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAsDataInFhirServer)}-tag", + await TestFhirClient.ExportAsync(parameters: ExportTestFilterQueryParameters())); } private async Task SaveTestResourcesToServer() @@ -152,7 +168,7 @@ private async System.Threading.Tasks.Task> SaveResourceListToServ var bundle = new Bundle { - Type = Bundle.BundleType.Transaction, + Type = Bundle.BundleType.Batch, Entry = new List(), }; From 848bc1284f52667d7f5aef7191b6f7c3649810bc Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Tue, 17 Oct 2023 13:49:35 -0700 Subject: [PATCH 38/58] testing central export perf --- .../Rest/Export/ExportDataTestFixture.cs | 90 +++++++++++++++--- .../Rest/Export/ExportDataTests.cs | 93 ++++--------------- 2 files changed, 95 insertions(+), 88 deletions(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs index aa07219747..82e372f2c9 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs @@ -27,6 +27,7 @@ public class ExportDataTestFixture : HttpIntegrationTestFixture _metricHandler ?? (_metricHandler = (MetricHandler)(TestFhirServer as InProcTestFhirServer)?.Server.Host.Services.GetRequiredService>()); } - public Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestResourcesWithHistoryAndDeletes { get; } = new(); + internal DataStore DataStore { get; private set; } - public Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestResourcesWithHistory => TestResourcesWithHistoryAndDeletes + internal Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestResourcesWithHistoryAndDeletes { get; } = new(); + + internal Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestResourcesWithHistory => TestResourcesWithHistoryAndDeletes .Where(entry => !entry.Value.Meta.Extension.Any(extension => extension.Url == "http://azurehealthcareapis.com/data-extensions/deleted-state" && ((FhirString)extension.Value).Value == "soft-deleted")) .ToDictionary(entry => entry.Key, entry => entry.Value); - public Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestResourcesWithDeletes => TestResourcesWithHistoryAndDeletes + internal Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestResourcesWithDeletes => TestResourcesWithHistoryAndDeletes .GroupBy(entry => entry.Key.resourceId) .Select(group => group.OrderByDescending(entry => entry.Value.Meta.LastUpdated).First()) .ToDictionary(entry => entry.Key, entry => entry.Value); - public Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestResources => + internal Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestResources => TestResourcesWithHistory.Where(pair => TestResourcesWithDeletes.ContainsKey(pair.Key)).ToDictionary(pair => pair.Key, pair => pair.Value); // If the patient is deleted but the child resources are not, they should not be returned in patient centric exports. - public Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestPatientCompartmentResources => TestResources + internal Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestPatientCompartmentResources => TestResources .Where(x => x.Key.resourceType != "Encounter" || TestResources.Keys.Any(pat => pat.resourceType == "Patient" && pat.resourceId == (x.Value as Encounter).Subject.Reference.Split("/")[1])) .Where(x => x.Key.resourceType != "Observation" || TestResources.Keys.Any(pat => pat.resourceType == "Patient" && pat.resourceId == (x.Value as Observation).Subject.Reference.Split("/")[1])) .ToDictionary(pair => pair.Key, pair => pair.Value); - public Dictionary ExportTestCasesContentUrls { get; } = new(); + internal Dictionary ExportTestCasesContentUrls { get; } = new(); - public string FixtureTag { get; } = Guid.NewGuid().ToString(); + internal string FixtureTag { get; } = Guid.NewGuid().ToString(); - public DateTime TestDataInsertionTime { get; } = DateTime.UtcNow; + internal DateTime TestDataInsertionTime { get; } = DateTime.UtcNow; - public string ExportTestFilterQueryParameters(params string[] uniqueResourceTypes) + internal string ExportTestFilterQueryParameters(params string[] uniqueResourceTypes) { if (uniqueResourceTypes.Length == 0) { @@ -78,21 +81,84 @@ protected override async Task OnInitializedAsync() { await SaveTestResourcesToServer(); - // await StartTestExportOperations(); + await StartTestExportOperations(); } private async Task StartTestExportOperations() { + // Shared Parameters var uniqueFixtureResources = string.Join(',', TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); - string parameters = $"_since={TestDataInsertionTime:o}&_type={uniqueFixtureResources}"; + string sinceAndTypeExportParameters = $"_since={TestDataInsertionTime:o}&_type={uniqueFixtureResources}"; + // All data export tests ExportTestCasesContentUrls.Add( $"{nameof(ExportDataTests.GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAsDataInFhirServer)}-since", - await TestFhirClient.ExportAsync(parameters: parameters)); + await TestFhirClient.ExportAsync(parameters: sinceAndTypeExportParameters)); ExportTestCasesContentUrls.Add( $"{nameof(ExportDataTests.GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAsDataInFhirServer)}-tag", await TestFhirClient.ExportAsync(parameters: ExportTestFilterQueryParameters())); + + // Patient centric export tests + ExportTestCasesContentUrls.Add( + $"{nameof(ExportDataTests.GivenFhirServer_WhenPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer)}-since", + await TestFhirClient.ExportAsync(path: "Patient/", parameters: sinceAndTypeExportParameters)); + + ExportTestCasesContentUrls.Add( + $"{nameof(ExportDataTests.GivenFhirServer_WhenPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer)}-tag", + await TestFhirClient.ExportAsync(path: "Patient/", parameters: ExportTestFilterQueryParameters())); + + // Patient/Observation resource system export tests + string[] testResorceTypes = { "Observation", "Patient" }; + ExportTestCasesContentUrls.Add( + $"{nameof(ExportDataTests.GivenFhirServer_WhenAllObservationAndPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer)}-since", + await TestFhirClient.ExportAsync(parameters: $"_since={TestDataInsertionTime:o}&_type={string.Join(',', testResorceTypes)}")); + + ExportTestCasesContentUrls.Add( + $"{nameof(ExportDataTests.GivenFhirServer_WhenAllObservationAndPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer)}-tag", + await TestFhirClient.ExportAsync(parameters: ExportTestFilterQueryParameters(testResorceTypes))); + + // Patient entric Observation resource Patient export tests + ExportTestCasesContentUrls.Add( + $"{nameof(ExportDataTests.GivenFhirServer_WhenPatientObservationDataIsExported_ThenExportedDataIsSameAsDataInFhirServer)}-since", + await TestFhirClient.ExportAsync(path: "Patient/", parameters: $"_since={TestDataInsertionTime:o}&_type=Observation")); + + ExportTestCasesContentUrls.Add( + $"{nameof(ExportDataTests.GivenFhirServer_WhenPatientObservationDataIsExported_ThenExportedDataIsSameAsDataInFhirServer)}-tag", + await TestFhirClient.ExportAsync(path: "Patient/", parameters: ExportTestFilterQueryParameters("Observation"))); + + // Export to specific container + string testContainer = "test-container"; + ExportTestCasesContentUrls.Add( + nameof(ExportDataTests.GivenFhirServer_WhenAllDataIsExportedToASpecificContainer_ThenExportedDataIsInTheSpecifiedContianer), + await TestFhirClient.ExportAsync(parameters: $"_container={testContainer}&{ExportTestFilterQueryParameters()}")); + + // Export with history only + ExportTestCasesContentUrls.Add( + $"{nameof(ExportDataTests.GivenFhirServer_WhenDataIsExportedWithHistory_ThenExportedDataIsSameAsDataInFhirServer)}-_isParallel=true", + await TestFhirClient.ExportAsync(parameters: $"_since={TestDataInsertionTime:O}&_type={uniqueFixtureResources}&includeAssociatedData=_history&_isParallel=true")); + + ExportTestCasesContentUrls.Add( + $"{nameof(ExportDataTests.GivenFhirServer_WhenDataIsExportedWithHistory_ThenExportedDataIsSameAsDataInFhirServer)}-_isParallel=false", + await TestFhirClient.ExportAsync(parameters: $"_since={TestDataInsertionTime:O}&_type={uniqueFixtureResources}&includeAssociatedData=_history&_isParallel=false")); + + // Export with delete only + ExportTestCasesContentUrls.Add( + $"{nameof(ExportDataTests.GivenFhirServer_WhenDataIsExportedWithSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer)}-_isParallel=true", + await TestFhirClient.ExportAsync(parameters: $"_since={TestDataInsertionTime:O}&_type={uniqueFixtureResources}&includeAssociatedData=_deleted&_isParallel=true")); + + ExportTestCasesContentUrls.Add( + $"{nameof(ExportDataTests.GivenFhirServer_WhenDataIsExportedWithSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer)}-_isParallel=false", + await TestFhirClient.ExportAsync(parameters: $"_since={TestDataInsertionTime:O}&_type={uniqueFixtureResources}&includeAssociatedData=_deleted&_isParallel=false")); + + // Export with history and deletes + ExportTestCasesContentUrls.Add( + $"{nameof(ExportDataTests.GivenFhirServer_WhenDataIsExportedWithHistoryAndSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer)}-_isParallel=true", + await TestFhirClient.ExportAsync(parameters: $"_since={TestDataInsertionTime:O}&_type={uniqueFixtureResources}&includeAssociatedData=_history,_deleted&_isParallel=true")); + + ExportTestCasesContentUrls.Add( + $"{nameof(ExportDataTests.GivenFhirServer_WhenDataIsExportedWithHistoryAndSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer)}-_isParallel=false", + await TestFhirClient.ExportAsync(parameters: $"_since={TestDataInsertionTime:O}&_type={uniqueFixtureResources}&includeAssociatedData=_history,_deleted&_isParallel=false")); } private async Task SaveTestResourcesToServer() diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs index 218d0d32ba..360d40ea7e 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs @@ -42,23 +42,12 @@ public ExportDataTests(ExportDataTestFixture fixture, ITestOutputHelper testOutp [InlineData("tag")] public async Task GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAsDataInFhirServer(string parametersKey) { - // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally.\ - string parameters = "not set"; - - if (parametersKey == "tag") - { - parameters = _fixture.ExportTestFilterQueryParameters(); - } - else if (parametersKey == "since") - { - var uniqueFixtureResources = string.Join(',', _fixture.TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); - parameters = $"_since={_fixture.TestDataInsertionTime:o}&_type={uniqueFixtureResources}"; - } + // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. // Trigger export request and check for export status - Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: parameters); + Uri contentLocation = _fixture.ExportTestCasesContentUrls[$"{nameof(GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAsDataInFhirServer)}-{parametersKey}"]; - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, 15); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = @@ -74,22 +63,9 @@ public async Task GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAs public async Task GivenFhirServer_WhenPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer(string parametersKey) { // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. - string parameters = "not set"; - - if (parametersKey == "tag") - { - parameters = _fixture.ExportTestFilterQueryParameters(); - } - else if (parametersKey == "since") - { - var uniqueFixtureResources = string.Join(',', _fixture.TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); - parameters = $"_since={_fixture.TestDataInsertionTime:o}&_type={uniqueFixtureResources}"; - } - - // Trigger export request and check for export status - Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(path: "Patient/", parameters: parameters); + Uri contentLocation = _fixture.ExportTestCasesContentUrls[$"{nameof(GivenFhirServer_WhenPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer)}-{parametersKey}"]; - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, 15); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = @@ -106,21 +82,9 @@ public async Task GivenFhirServer_WhenAllObservationAndPatientDataIsExported_The { // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. string[] testResorceTypes = { "Observation", "Patient" }; - string parameters = "not set"; + Uri contentLocation = _fixture.ExportTestCasesContentUrls[$"{nameof(GivenFhirServer_WhenAllObservationAndPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer)}-{parametersKey}"]; - if (parametersKey == "tag") - { - parameters = _fixture.ExportTestFilterQueryParameters(testResorceTypes); - } - else if (parametersKey == "since") - { - parameters = $"_since={_fixture.TestDataInsertionTime:o}&_type={string.Join(',', testResorceTypes)}"; - } - - // Trigger export request and check for export status - Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: parameters); - - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, 15); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = @@ -140,21 +104,9 @@ public async Task GivenFhirServer_WhenAllObservationAndPatientDataIsExported_The public async Task GivenFhirServer_WhenPatientObservationDataIsExported_ThenExportedDataIsSameAsDataInFhirServer(string parametersKey) { // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. - string parameters = "not set"; - - if (parametersKey == "tag") - { - parameters = _fixture.ExportTestFilterQueryParameters("Observation"); - } - else if (parametersKey == "since") - { - parameters = $"_since={_fixture.TestDataInsertionTime:o}&type=Observation"; - } + Uri contentLocation = _fixture.ExportTestCasesContentUrls[$"{nameof(GivenFhirServer_WhenPatientObservationDataIsExported_ThenExportedDataIsSameAsDataInFhirServer)}-{parametersKey}"]; - // Trigger export request and check for export status - Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(path: "Patient/", parameters: _fixture.ExportTestFilterQueryParameters("Observation")); - - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, 15); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = @@ -174,11 +126,9 @@ public async Task GivenFhirServer_WhenAllDataIsExportedToASpecificContainer_Then { // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. string testContainer = "test-container"; + Uri contentLocation = _fixture.ExportTestCasesContentUrls[nameof(GivenFhirServer_WhenAllDataIsExportedToASpecificContainer_ThenExportedDataIsInTheSpecifiedContianer)]; - // Trigger export request and check for export status - Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: $"_container={testContainer}&{_fixture.ExportTestFilterQueryParameters()}"); - - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, 15); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = @@ -196,12 +146,9 @@ public async Task GivenFhirServer_WhenAllDataIsExportedToASpecificContainer_Then public async Task GivenFhirServer_WhenDataIsExportedWithHistory_ThenExportedDataIsSameAsDataInFhirServer(string parallelQueryParam) { // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. - var uniqueFixtureResources = string.Join(',', _fixture.TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); + Uri contentLocation = _fixture.ExportTestCasesContentUrls[$"{nameof(GivenFhirServer_WhenDataIsExportedWithHistory_ThenExportedDataIsSameAsDataInFhirServer)}-{parallelQueryParam}"]; - // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. - Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:O}&_type={uniqueFixtureResources}&includeAssociatedData=_history&{parallelQueryParam}"); - - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, 15); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = @@ -218,12 +165,9 @@ public async Task GivenFhirServer_WhenDataIsExportedWithHistory_ThenExportedData public async Task GivenFhirServer_WhenDataIsExportedWithSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer(string parallelQueryParam) { // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. - var uniqueFixtureResources = string.Join(',', _fixture.TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); - - // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. - Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:O}&_type={uniqueFixtureResources}&includeAssociatedData=_deleted&{parallelQueryParam}"); + Uri contentLocation = _fixture.ExportTestCasesContentUrls[$"{nameof(GivenFhirServer_WhenDataIsExportedWithSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer)}-{parallelQueryParam}"]; - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, 15); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = @@ -240,12 +184,9 @@ public async Task GivenFhirServer_WhenDataIsExportedWithSoftDeletes_ThenExported public async Task GivenFhirServer_WhenDataIsExportedWithHistoryAndSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer(string parallelQueryParam) { // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. - var uniqueFixtureResources = string.Join(',', _fixture.TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); - - // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. - Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:O}&_type={uniqueFixtureResources}&includeAssociatedData=_history,_deleted&{parallelQueryParam}"); + Uri contentLocation = _fixture.ExportTestCasesContentUrls[$"{nameof(GivenFhirServer_WhenDataIsExportedWithHistoryAndSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer)}-{parallelQueryParam}"]; - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, 15); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = From 1d66c1b528cf846ab4b73b36bba072d339b62629 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Tue, 17 Oct 2023 15:48:04 -0700 Subject: [PATCH 39/58] Rolled back central execution of export tests --- .../Rest/Export/ExportDataTestFixture.cs | 81 --------------- .../Rest/Export/ExportDataTests.cs | 99 ++++++++++++------- 2 files changed, 66 insertions(+), 114 deletions(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs index 82e372f2c9..b288aafd19 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs @@ -59,8 +59,6 @@ public MetricHandler MetricHandler .Where(x => x.Key.resourceType != "Observation" || TestResources.Keys.Any(pat => pat.resourceType == "Patient" && pat.resourceId == (x.Value as Observation).Subject.Reference.Split("/")[1])) .ToDictionary(pair => pair.Key, pair => pair.Value); - internal Dictionary ExportTestCasesContentUrls { get; } = new(); - internal string FixtureTag { get; } = Guid.NewGuid().ToString(); internal DateTime TestDataInsertionTime { get; } = DateTime.UtcNow; @@ -80,85 +78,6 @@ internal string ExportTestFilterQueryParameters(params string[] uniqueResourceTy protected override async Task OnInitializedAsync() { await SaveTestResourcesToServer(); - - await StartTestExportOperations(); - } - - private async Task StartTestExportOperations() - { - // Shared Parameters - var uniqueFixtureResources = string.Join(',', TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); - string sinceAndTypeExportParameters = $"_since={TestDataInsertionTime:o}&_type={uniqueFixtureResources}"; - - // All data export tests - ExportTestCasesContentUrls.Add( - $"{nameof(ExportDataTests.GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAsDataInFhirServer)}-since", - await TestFhirClient.ExportAsync(parameters: sinceAndTypeExportParameters)); - - ExportTestCasesContentUrls.Add( - $"{nameof(ExportDataTests.GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAsDataInFhirServer)}-tag", - await TestFhirClient.ExportAsync(parameters: ExportTestFilterQueryParameters())); - - // Patient centric export tests - ExportTestCasesContentUrls.Add( - $"{nameof(ExportDataTests.GivenFhirServer_WhenPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer)}-since", - await TestFhirClient.ExportAsync(path: "Patient/", parameters: sinceAndTypeExportParameters)); - - ExportTestCasesContentUrls.Add( - $"{nameof(ExportDataTests.GivenFhirServer_WhenPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer)}-tag", - await TestFhirClient.ExportAsync(path: "Patient/", parameters: ExportTestFilterQueryParameters())); - - // Patient/Observation resource system export tests - string[] testResorceTypes = { "Observation", "Patient" }; - ExportTestCasesContentUrls.Add( - $"{nameof(ExportDataTests.GivenFhirServer_WhenAllObservationAndPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer)}-since", - await TestFhirClient.ExportAsync(parameters: $"_since={TestDataInsertionTime:o}&_type={string.Join(',', testResorceTypes)}")); - - ExportTestCasesContentUrls.Add( - $"{nameof(ExportDataTests.GivenFhirServer_WhenAllObservationAndPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer)}-tag", - await TestFhirClient.ExportAsync(parameters: ExportTestFilterQueryParameters(testResorceTypes))); - - // Patient entric Observation resource Patient export tests - ExportTestCasesContentUrls.Add( - $"{nameof(ExportDataTests.GivenFhirServer_WhenPatientObservationDataIsExported_ThenExportedDataIsSameAsDataInFhirServer)}-since", - await TestFhirClient.ExportAsync(path: "Patient/", parameters: $"_since={TestDataInsertionTime:o}&_type=Observation")); - - ExportTestCasesContentUrls.Add( - $"{nameof(ExportDataTests.GivenFhirServer_WhenPatientObservationDataIsExported_ThenExportedDataIsSameAsDataInFhirServer)}-tag", - await TestFhirClient.ExportAsync(path: "Patient/", parameters: ExportTestFilterQueryParameters("Observation"))); - - // Export to specific container - string testContainer = "test-container"; - ExportTestCasesContentUrls.Add( - nameof(ExportDataTests.GivenFhirServer_WhenAllDataIsExportedToASpecificContainer_ThenExportedDataIsInTheSpecifiedContianer), - await TestFhirClient.ExportAsync(parameters: $"_container={testContainer}&{ExportTestFilterQueryParameters()}")); - - // Export with history only - ExportTestCasesContentUrls.Add( - $"{nameof(ExportDataTests.GivenFhirServer_WhenDataIsExportedWithHistory_ThenExportedDataIsSameAsDataInFhirServer)}-_isParallel=true", - await TestFhirClient.ExportAsync(parameters: $"_since={TestDataInsertionTime:O}&_type={uniqueFixtureResources}&includeAssociatedData=_history&_isParallel=true")); - - ExportTestCasesContentUrls.Add( - $"{nameof(ExportDataTests.GivenFhirServer_WhenDataIsExportedWithHistory_ThenExportedDataIsSameAsDataInFhirServer)}-_isParallel=false", - await TestFhirClient.ExportAsync(parameters: $"_since={TestDataInsertionTime:O}&_type={uniqueFixtureResources}&includeAssociatedData=_history&_isParallel=false")); - - // Export with delete only - ExportTestCasesContentUrls.Add( - $"{nameof(ExportDataTests.GivenFhirServer_WhenDataIsExportedWithSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer)}-_isParallel=true", - await TestFhirClient.ExportAsync(parameters: $"_since={TestDataInsertionTime:O}&_type={uniqueFixtureResources}&includeAssociatedData=_deleted&_isParallel=true")); - - ExportTestCasesContentUrls.Add( - $"{nameof(ExportDataTests.GivenFhirServer_WhenDataIsExportedWithSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer)}-_isParallel=false", - await TestFhirClient.ExportAsync(parameters: $"_since={TestDataInsertionTime:O}&_type={uniqueFixtureResources}&includeAssociatedData=_deleted&_isParallel=false")); - - // Export with history and deletes - ExportTestCasesContentUrls.Add( - $"{nameof(ExportDataTests.GivenFhirServer_WhenDataIsExportedWithHistoryAndSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer)}-_isParallel=true", - await TestFhirClient.ExportAsync(parameters: $"_since={TestDataInsertionTime:O}&_type={uniqueFixtureResources}&includeAssociatedData=_history,_deleted&_isParallel=true")); - - ExportTestCasesContentUrls.Add( - $"{nameof(ExportDataTests.GivenFhirServer_WhenDataIsExportedWithHistoryAndSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer)}-_isParallel=false", - await TestFhirClient.ExportAsync(parameters: $"_since={TestDataInsertionTime:O}&_type={uniqueFixtureResources}&includeAssociatedData=_history,_deleted&_isParallel=false")); } private async Task SaveTestResourcesToServer() diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs index 360d40ea7e..7cc65e679b 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs @@ -37,17 +37,16 @@ public ExportDataTests(ExportDataTestFixture fixture, ITestOutputHelper testOutp _fixture = fixture; } - [Theory] - [InlineData("since")] - [InlineData("tag")] - public async Task GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAsDataInFhirServer(string parametersKey) + [Fact] + public async Task GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() { - // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. + // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally.\ + string parameters = _fixture.ExportTestFilterQueryParameters(); // Trigger export request and check for export status - Uri contentLocation = _fixture.ExportTestCasesContentUrls[$"{nameof(GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAsDataInFhirServer)}-{parametersKey}"]; + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: parameters); - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, 15); + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = @@ -57,15 +56,16 @@ public async Task GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAs Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResources, dataFromExport, _outputHelper)); } - [Theory] - [InlineData("since")] - [InlineData("tag")] - public async Task GivenFhirServer_WhenPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer(string parametersKey) + [Fact] + public async Task GivenFhirServer_WhenPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() { // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. - Uri contentLocation = _fixture.ExportTestCasesContentUrls[$"{nameof(GivenFhirServer_WhenPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer)}-{parametersKey}"]; + string parameters = _fixture.ExportTestFilterQueryParameters(); + + // Trigger export request and check for export status + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(path: "Patient/", parameters: parameters); - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, 15); + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = @@ -75,16 +75,17 @@ public async Task GivenFhirServer_WhenPatientDataIsExported_ThenExportedDataIsSa Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestPatientCompartmentResources, dataFromExport, _outputHelper)); } - [Theory] - [InlineData("since")] - [InlineData("tag")] - public async Task GivenFhirServer_WhenAllObservationAndPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer(string parametersKey) + [Fact] + public async Task GivenFhirServer_WhenAllObservationAndPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() { // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. string[] testResorceTypes = { "Observation", "Patient" }; - Uri contentLocation = _fixture.ExportTestCasesContentUrls[$"{nameof(GivenFhirServer_WhenAllObservationAndPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer)}-{parametersKey}"]; + var parameters = _fixture.ExportTestFilterQueryParameters(testResorceTypes); - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, 15); + // Trigger export request and check for export status + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: parameters); + + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = @@ -98,15 +99,16 @@ public async Task GivenFhirServer_WhenAllObservationAndPatientDataIsExported_The Assert.True(ExportTestHelper.ValidateDataFromBothSources(expectedResources, dataFromExport, _outputHelper)); } - [Theory] - [InlineData("since")] - [InlineData("tag")] - public async Task GivenFhirServer_WhenPatientObservationDataIsExported_ThenExportedDataIsSameAsDataInFhirServer(string parametersKey) + [Fact] + public async Task GivenFhirServer_WhenPatientObservationDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() { // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. - Uri contentLocation = _fixture.ExportTestCasesContentUrls[$"{nameof(GivenFhirServer_WhenPatientObservationDataIsExported_ThenExportedDataIsSameAsDataInFhirServer)}-{parametersKey}"]; + var parameters = _fixture.ExportTestFilterQueryParameters("Observation"); + + // Trigger export request and check for export status + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(path: "Patient/", parameters: parameters); - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, 15); + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = @@ -126,9 +128,11 @@ public async Task GivenFhirServer_WhenAllDataIsExportedToASpecificContainer_Then { // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. string testContainer = "test-container"; - Uri contentLocation = _fixture.ExportTestCasesContentUrls[nameof(GivenFhirServer_WhenAllDataIsExportedToASpecificContainer_ThenExportedDataIsInTheSpecifiedContianer)]; - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, 15); + // Trigger export request and check for export status + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: $"_container={testContainer}&{_fixture.ExportTestFilterQueryParameters()}"); + + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = @@ -143,12 +147,22 @@ public async Task GivenFhirServer_WhenAllDataIsExportedToASpecificContainer_Then [Theory] [InlineData("_isParallel=true")] [InlineData("_isParallel=false")] + [Trait(Traits.Category, Categories.ExportLongRunning)] public async Task GivenFhirServer_WhenDataIsExportedWithHistory_ThenExportedDataIsSameAsDataInFhirServer(string parallelQueryParam) { + if (_fixture.DataStore == DataStore.CosmosDb && parallelQueryParam == "_isParallel=true") + { + // CosmosDB does not have parallel export support yet. + return; + } + // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. - Uri contentLocation = _fixture.ExportTestCasesContentUrls[$"{nameof(GivenFhirServer_WhenDataIsExportedWithHistory_ThenExportedDataIsSameAsDataInFhirServer)}-{parallelQueryParam}"]; + var uniqueFixtureResources = string.Join(',', _fixture.TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, 15); + // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:O}&_type={uniqueFixtureResources}&includeAssociatedData=_history&{parallelQueryParam}"); + + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = @@ -162,12 +176,22 @@ public async Task GivenFhirServer_WhenDataIsExportedWithHistory_ThenExportedData [Theory] [InlineData("_isParallel=true")] [InlineData("_isParallel=false")] + [Trait(Traits.Category, Categories.ExportLongRunning)] public async Task GivenFhirServer_WhenDataIsExportedWithSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer(string parallelQueryParam) { + if (_fixture.DataStore == DataStore.CosmosDb && parallelQueryParam == "_isParallel=true") + { + // CosmosDB does not have parallel export support yet. + return; + } + // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. - Uri contentLocation = _fixture.ExportTestCasesContentUrls[$"{nameof(GivenFhirServer_WhenDataIsExportedWithSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer)}-{parallelQueryParam}"]; + var uniqueFixtureResources = string.Join(',', _fixture.TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); + + // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:O}&_type={uniqueFixtureResources}&includeAssociatedData=_deleted&{parallelQueryParam}"); - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, 15); + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = @@ -183,10 +207,19 @@ public async Task GivenFhirServer_WhenDataIsExportedWithSoftDeletes_ThenExported [InlineData("_isParallel=false")] public async Task GivenFhirServer_WhenDataIsExportedWithHistoryAndSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer(string parallelQueryParam) { + if (_fixture.DataStore == DataStore.CosmosDb && parallelQueryParam == "_isParallel=true") + { + // CosmosDB does not have parallel export support yet. + return; + } + // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. - Uri contentLocation = _fixture.ExportTestCasesContentUrls[$"{nameof(GivenFhirServer_WhenDataIsExportedWithHistoryAndSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer)}-{parallelQueryParam}"]; + var uniqueFixtureResources = string.Join(',', _fixture.TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); + + // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:O}&_type={uniqueFixtureResources}&includeAssociatedData=_history,_deleted&{parallelQueryParam}"); - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, 15); + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); // Download exported data from storage account Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = From c26393bd18b62c3ada86a58461c61b5c84053207 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Wed, 18 Oct 2023 09:13:30 -0700 Subject: [PATCH 40/58] Removing exportlongrunning for pipeline perf test --- .../Rest/Export/ExportDataTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs index 7cc65e679b..28e443a049 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs @@ -147,7 +147,7 @@ public async Task GivenFhirServer_WhenAllDataIsExportedToASpecificContainer_Then [Theory] [InlineData("_isParallel=true")] [InlineData("_isParallel=false")] - [Trait(Traits.Category, Categories.ExportLongRunning)] + // [Trait(Traits.Category, Categories.ExportLongRunning)] public async Task GivenFhirServer_WhenDataIsExportedWithHistory_ThenExportedDataIsSameAsDataInFhirServer(string parallelQueryParam) { if (_fixture.DataStore == DataStore.CosmosDb && parallelQueryParam == "_isParallel=true") @@ -176,7 +176,7 @@ public async Task GivenFhirServer_WhenDataIsExportedWithHistory_ThenExportedData [Theory] [InlineData("_isParallel=true")] [InlineData("_isParallel=false")] - [Trait(Traits.Category, Categories.ExportLongRunning)] + // [Trait(Traits.Category, Categories.ExportLongRunning)] public async Task GivenFhirServer_WhenDataIsExportedWithSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer(string parallelQueryParam) { if (_fixture.DataStore == DataStore.CosmosDb && parallelQueryParam == "_isParallel=true") From 8726ebc70a26328aed72da2a1d5a183ceab24cf4 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Wed, 18 Oct 2023 09:47:14 -0700 Subject: [PATCH 41/58] Fixed build --- .../Features/KnownQueryParameterNames.cs | 2 +- .../Features/Operations/Export/CreateExportRequestHandler.cs | 3 +-- .../Rest/Export/ExportDataTests.cs | 2 -- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core/Features/KnownQueryParameterNames.cs b/src/Microsoft.Health.Fhir.Core/Features/KnownQueryParameterNames.cs index 4051e4af54..ceee07c0d0 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/KnownQueryParameterNames.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/KnownQueryParameterNames.cs @@ -98,7 +98,7 @@ public static class KnownQueryParameterNames public const string HardDelete = "_hardDelete"; public const string PurgeHistory = "_purgeHistory"; - + /// /// Used by queries from the $export mediator to signal that history should be included in the export search. /// diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/CreateExportRequestHandler.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/CreateExportRequestHandler.cs index ec9a5e77b5..77f4c506c6 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/CreateExportRequestHandler.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/CreateExportRequestHandler.cs @@ -94,11 +94,10 @@ public async Task Handle(CreateExportRequest request, Canc anonymizationConfigurationCollectionReference: request.AnonymizationConfigurationCollectionReference, anonymizationConfigurationLocation: request.AnonymizationConfigurationLocation, anonymizationConfigurationFileETag: request.AnonymizationConfigurationFileETag, - maximumNumberOfResourcesPerQuery: _exportJobConfiguration.MaximumNumberOfResourcesPerQuery, + maximumNumberOfResourcesPerQuery: maxCount > 0 ? maxCount : _exportJobConfiguration.MaximumNumberOfResourcesPerQuery, numberOfPagesPerCommit: _exportJobConfiguration.NumberOfPagesPerCommit, storageAccountContainerName: request.ContainerName, isParallel: request.IsParallel, - maxCount: maxCount, includeHistory: request.IncludeHistory, includeDeleted: request.IncludeDeleted, smartRequest: _contextAccessor?.RequestContext?.AccessControlContext?.ApplyFineGrainedAccessControl == true); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs index 28e443a049..72074d2797 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs @@ -147,7 +147,6 @@ public async Task GivenFhirServer_WhenAllDataIsExportedToASpecificContainer_Then [Theory] [InlineData("_isParallel=true")] [InlineData("_isParallel=false")] - // [Trait(Traits.Category, Categories.ExportLongRunning)] public async Task GivenFhirServer_WhenDataIsExportedWithHistory_ThenExportedDataIsSameAsDataInFhirServer(string parallelQueryParam) { if (_fixture.DataStore == DataStore.CosmosDb && parallelQueryParam == "_isParallel=true") @@ -176,7 +175,6 @@ public async Task GivenFhirServer_WhenDataIsExportedWithHistory_ThenExportedData [Theory] [InlineData("_isParallel=true")] [InlineData("_isParallel=false")] - // [Trait(Traits.Category, Categories.ExportLongRunning)] public async Task GivenFhirServer_WhenDataIsExportedWithSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer(string parallelQueryParam) { if (_fixture.DataStore == DataStore.CosmosDb && parallelQueryParam == "_isParallel=true") From 6f3550a70e3193abc06ab8f81f8e425cddb26185 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Wed, 18 Oct 2023 12:59:27 -0700 Subject: [PATCH 42/58] Optimized test structure --- .../Rest/Export/ExportDataTests.cs | 97 +++++++++---------- 1 file changed, 46 insertions(+), 51 deletions(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs index 72074d2797..d45703335d 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs @@ -143,51 +143,62 @@ public async Task GivenFhirServer_WhenAllDataIsExportedToASpecificContainer_Then Assert.True(blobUris.All((url) => url.OriginalString.Contains(testContainer))); } - // _tag filter cannot be used with history or deleted export. Using isParallel to test both SQL code paths. - [Theory] - [InlineData("_isParallel=true")] - [InlineData("_isParallel=false")] - public async Task GivenFhirServer_WhenDataIsExportedWithHistory_ThenExportedDataIsSameAsDataInFhirServer(string parallelQueryParam) + [Fact] + [Trait(Traits.Category, Categories.ExportLongRunning)] + public async Task GivenFhirServer_WhenDataIsExportedWithHistoryParallel_ThenExportedDataIsSameAsDataInFhirServer() { - if (_fixture.DataStore == DataStore.CosmosDb && parallelQueryParam == "_isParallel=true") - { - // CosmosDB does not have parallel export support yet. - return; - } + await ExportAndSoftDeleteTestHelper(parallel: true, history: true, deletes: false); + } - // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. - var uniqueFixtureResources = string.Join(',', _fixture.TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); + [Fact] + [Trait(Traits.Category, Categories.ExportLongRunning)] + public async Task GivenFhirServer_WhenDataIsExportedWithHistoryNotParallel_ThenExportedDataIsSameAsDataInFhirServer() + { + await ExportAndSoftDeleteTestHelper(parallel: false, history: true, deletes: false); + } - // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. - Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:O}&_type={uniqueFixtureResources}&includeAssociatedData=_history&{parallelQueryParam}"); + [Fact] + [Trait(Traits.Category, Categories.ExportLongRunning)] + public async Task GivenFhirServer_WhenDataIsExportedWithSoftDeletesParallel_ThenExportedDataIsSameAsDataInFhirServer() + { + await ExportAndSoftDeleteTestHelper(parallel: true, history: false, deletes: true); + } - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); + [Fact] + [Trait(Traits.Category, Categories.ExportLongRunning)] + public async Task GivenFhirServer_WhenDataIsExportedWithSoftDeletesNotParallel_ThenExportedDataIsSameAsDataInFhirServer() + { + await ExportAndSoftDeleteTestHelper(parallel: false, history: false, deletes: true); + } - // Download exported data from storage account - Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = - await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); + [Fact] + public async Task GivenFhirServer_WhenDataIsExportedWithHistoryAndSoftDeletesParallel_ThenExportedDataIsSameAsDataInFhirServer() + { + await ExportAndSoftDeleteTestHelper(parallel: true, history: true, deletes: true); + } - // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResourcesWithHistory, dataFromExport, _outputHelper)); + [Fact] + [Trait(Traits.Category, Categories.ExportLongRunning)] + public async Task GivenFhirServer_WhenDataIsExportedWithHistoryAndSoftDeletesNotParallel_ThenExportedDataIsSameAsDataInFhirServer() + { + await ExportAndSoftDeleteTestHelper(parallel: false, history: true, deletes: true); } // _tag filter cannot be used with history or deleted export. Using isParallel to test both SQL code paths. - [Theory] - [InlineData("_isParallel=true")] - [InlineData("_isParallel=false")] - public async Task GivenFhirServer_WhenDataIsExportedWithSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer(string parallelQueryParam) + private async Task ExportAndSoftDeleteTestHelper(bool parallel, bool history, bool deletes) { - if (_fixture.DataStore == DataStore.CosmosDb && parallelQueryParam == "_isParallel=true") + if (_fixture.DataStore == DataStore.CosmosDb && parallel) { // CosmosDB does not have parallel export support yet. return; } - // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. - var uniqueFixtureResources = string.Join(',', _fixture.TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); + string uniqueFixtureResources = string.Join(',', _fixture.TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); + string includeAssociatedDataParam = (history ? "_history" : string.Empty) + (deletes ? (history ? "," : string.Empty) + "_deleted" : string.Empty); // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. - Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:O}&_type={uniqueFixtureResources}&includeAssociatedData=_deleted&{parallelQueryParam}"); + string parallelQueryParam = $"_isParallel={parallel}"; + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:O}&_type={uniqueFixtureResources}&includeAssociatedData={includeAssociatedDataParam}&{parallelQueryParam}"); IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); @@ -195,33 +206,17 @@ public async Task GivenFhirServer_WhenDataIsExportedWithSoftDeletes_ThenExported Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); - // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResourcesWithDeletes, dataFromExport, _outputHelper)); - } + var expectedResources = _fixture.TestResourcesWithHistoryAndDeletes; - // _tag filter cannot be used with history or deleted export. Using isParallel to test both SQL code paths. - [Theory] - [InlineData("_isParallel=true")] - [InlineData("_isParallel=false")] - public async Task GivenFhirServer_WhenDataIsExportedWithHistoryAndSoftDeletes_ThenExportedDataIsSameAsDataInFhirServer(string parallelQueryParam) - { - if (_fixture.DataStore == DataStore.CosmosDb && parallelQueryParam == "_isParallel=true") + if (!history) { - // CosmosDB does not have parallel export support yet. - return; + expectedResources = _fixture.TestResourcesWithDeletes; } - // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. - var uniqueFixtureResources = string.Join(',', _fixture.TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); - - // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. - Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:O}&_type={uniqueFixtureResources}&includeAssociatedData=_history,_deleted&{parallelQueryParam}"); - - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); - - // Download exported data from storage account - Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = - await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); + if (!deletes) + { + expectedResources = _fixture.TestResourcesWithHistory; + } // Assert both data are equal Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResourcesWithHistoryAndDeletes, dataFromExport, _outputHelper)); From 515e49b9e720b965f8e6d854f3775682299fcf8e Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Tue, 31 Oct 2023 13:46:41 -0700 Subject: [PATCH 43/58] merged from main --- .../Extensions/SearchServiceExtensions.cs | 15 +- .../Features/Schema/Migrations/67.diff.sql | 123 +++++++------- .../Features/Schema/Migrations/67.sql | 157 +++++++++--------- .../Schema/Sql/Sprocs/EnqueueJobs.sql | 2 +- 4 files changed, 148 insertions(+), 149 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core/Extensions/SearchServiceExtensions.cs b/src/Microsoft.Health.Fhir.Core/Extensions/SearchServiceExtensions.cs index a0b502a18e..434ef43e88 100644 --- a/src/Microsoft.Health.Fhir.Core/Extensions/SearchServiceExtensions.cs +++ b/src/Microsoft.Health.Fhir.Core/Extensions/SearchServiceExtensions.cs @@ -62,17 +62,18 @@ public static class SearchServiceExtensions filteredParameters.Add(Tuple.Create(KnownQueryParameterNames.Count, count.ToString())); } - if (!string.IsNullOrEmpty(continuationToken)) - { - filteredParameters.Add(Tuple.Create(KnownQueryParameterNames.ContinuationToken, ContinuationTokenConverter.Encode(continuationToken))); - } - var matchedResults = new List(); - string lastContinuationToken = null; + string lastContinuationToken = continuationToken; do { - SearchResult results = await searchService.SearchAsync(instanceType, filteredParameters.ToImmutableList(), cancellationToken); + var searchParameters = new List>(filteredParameters); + if (!string.IsNullOrEmpty(lastContinuationToken)) + { + searchParameters.Add(Tuple.Create(KnownQueryParameterNames.ContinuationToken, ContinuationTokenConverter.Encode(lastContinuationToken))); + } + + SearchResult results = await searchService.SearchAsync(instanceType, searchParameters.ToImmutableList(), cancellationToken); lastContinuationToken = results?.ContinuationToken; // Check if all parameters passed in were unused, this would result in no search parameters being applied to search results diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/67.diff.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/67.diff.sql index 947ba59e22..65ab98d1b0 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/67.diff.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/67.diff.sql @@ -1,81 +1,78 @@ -CREATE OR ALTER PROCEDURE dbo.GetResourcesByTypeAndSurrogateIdRange @ResourceTypeId smallint, @StartId bigint, @EndId bigint, @GlobalEndId bigint = NULL, @IncludeHistory bit = 0, @IncludeDeleted bit = 0 +--DROP PROCEDURE dbo.EnqueueJobs +GO +ALTER PROCEDURE dbo.EnqueueJobs @QueueType tinyint, @Definitions StringList READONLY, @GroupId bigint = NULL, @ForceOneActiveJobGroup bit = 1, @IsCompleted bit = NULL, @ReturnJobs bit = 1 AS set nocount on -DECLARE @SP varchar(100) = 'GetResourcesByTypeAndSurrogateIdRange' - ,@Mode varchar(100) = 'RT='+isnull(convert(varchar,@ResourceTypeId),'NULL') - +' S='+isnull(convert(varchar,@StartId),'NULL') - +' E='+isnull(convert(varchar,@EndId),'NULL') - +' GE='+isnull(convert(varchar,@GlobalEndId),'NULL') -- Could this just be a boolean for if historical records should be returned? GlobalEndId should equal EndId in all cases I can think of. - +' HI='+isnull(convert(varchar,@IncludeHistory),'NULL') - +' DE'+isnull(convert(varchar,@IncludeDeleted),'NULL') +DECLARE @SP varchar(100) = 'EnqueueJobs' + ,@Mode varchar(100) = 'Q='+isnull(convert(varchar,@QueueType),'NULL') + +' D='+convert(varchar,(SELECT count(*) FROM @Definitions)) + +' G='+isnull(convert(varchar,@GroupId),'NULL') + +' F='+isnull(convert(varchar,@ForceOneActiveJobGroup),'NULL') + +' C='+isnull(convert(varchar,@IsCompleted),'NULL') ,@st datetime = getUTCdate() - ,@DummyTop bigint = 9223372036854775807 + ,@Lock varchar(100) = 'EnqueueJobs_'+convert(varchar,@QueueType) + ,@MaxJobId bigint + ,@Rows int + ,@msg varchar(1000) + ,@JobIds BigintList + ,@InputRows int BEGIN TRY - DECLARE @ResourceIds TABLE (ResourceId varchar(64) COLLATE Latin1_General_100_CS_AS PRIMARY KEY) - DECLARE @SurrogateIds TABLE (MaxSurrogateId bigint PRIMARY KEY) + DECLARE @Input TABLE (DefinitionHash varbinary(20) PRIMARY KEY, Definition varchar(max)) + INSERT INTO @Input SELECT DefinitionHash = hashbytes('SHA1',String), Definition = String FROM @Definitions + SET @InputRows = @@rowcount - IF @GlobalEndId IS NOT NULL AND @IncludeHistory = 0 -- snapshot view + INSERT INTO @JobIds + SELECT JobId + FROM @Input A + JOIN dbo.JobQueue B ON B.QueueType = @QueueType AND B.DefinitionHash = A.DefinitionHash AND B.Status <> 5 + + IF @@rowcount < @InputRows BEGIN - INSERT INTO @ResourceIds - SELECT DISTINCT ResourceId - FROM dbo.Resource - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId BETWEEN @StartId AND @EndId - AND IsHistory = 1 - AND (IsDeleted = 0 OR @IncludeDeleted = 1) - OPTION (MAXDOP 1) + BEGIN TRANSACTION + + EXECUTE sp_getapplock @Lock, 'Exclusive' + + IF @ForceOneActiveJobGroup = 1 AND EXISTS (SELECT * FROM dbo.JobQueue WHERE QueueType = @QueueType AND Status IN (0,1) AND (@GroupId IS NULL OR GroupId <> @GroupId)) + RAISERROR('There are other active job groups',18,127) + + SET @MaxJobId = isnull((SELECT TOP 1 JobId FROM dbo.JobQueue WHERE QueueType = @QueueType ORDER BY JobId DESC),0) + + INSERT INTO dbo.JobQueue + ( + QueueType + ,GroupId + ,JobId + ,Definition + ,DefinitionHash + ,Status + ) + OUTPUT inserted.JobId INTO @JobIds + SELECT @QueueType + ,GroupId = isnull(@GroupId,@MaxJobId+1) + ,JobId + ,Definition + ,DefinitionHash + ,Status = CASE WHEN @IsCompleted = 1 THEN 2 ELSE 0 END + FROM (SELECT JobId = @MaxJobId + row_number() OVER (ORDER BY Dummy), * FROM (SELECT *, Dummy = 0 FROM @Input) A) A -- preserve input order + WHERE NOT EXISTS (SELECT * FROM dbo.JobQueue B WITH (INDEX = IX_QueueType_DefinitionHash) WHERE B.QueueType = @QueueType AND B.DefinitionHash = A.DefinitionHash AND B.Status <> 5) + SET @Rows = @@rowcount - IF @@rowcount > 0 - INSERT INTO @SurrogateIds - SELECT ResourceSurrogateId - FROM (SELECT ResourceId, ResourceSurrogateId, RowId = row_number() OVER (PARTITION BY ResourceId ORDER BY ResourceSurrogateId DESC) - FROM dbo.Resource WITH (INDEX = IX_Resource_ResourceTypeId_ResourceId_Version) -- w/o hint access to Resource table is inefficient when many versions are present. Hint is ignored if Resource is a view. - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceId IN (SELECT TOP (@DummyTop) ResourceId FROM @ResourceIds) - AND ResourceSurrogateId BETWEEN @StartId AND @GlobalEndId - ) A - WHERE RowId = 1 - AND ResourceSurrogateId BETWEEN @StartId AND @EndId - OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)) + COMMIT TRANSACTION END - SELECT ResourceTypeId, ResourceId, Version, IsDeleted, ResourceSurrogateId, RequestMethod, IsMatch = convert(bit,1), IsPartial = convert(bit,0), IsRawResourceMetaSet, SearchParamHash, RawResource - FROM dbo.Resource - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId BETWEEN @StartId AND @EndId - AND (IsHistory = 0 OR @IncludeHistory = 1) - AND (IsDeleted = 0 OR @IncludeDeleted = 1) - UNION ALL - SELECT ResourceTypeId, ResourceId, Version, IsDeleted, ResourceSurrogateId, RequestMethod, IsMatch = convert(bit,1), IsPartial = convert(bit,0), IsRawResourceMetaSet, SearchParamHash, RawResource - FROM @SurrogateIds - JOIN dbo.Resource ON ResourceTypeId = @ResourceTypeId AND ResourceSurrogateId = MaxSurrogateId - WHERE IsHistory = 1 - AND (IsDeleted = 0 OR @IncludeDeleted = 1) - OPTION (MAXDOP 1) + IF @ReturnJobs = 1 + EXECUTE dbo.GetJobs @QueueType = @QueueType, @JobIds = @JobIds - EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@st,@Rows=@@rowcount + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@st,@Rows=@Rows END TRY BEGIN CATCH + IF @@trancount > 0 ROLLBACK TRANSACTION IF error_number() = 1750 THROW -- Real error is before 1750, cannot trap in SQL. EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Error'; THROW END CATCH GO ---set nocount on ---DECLARE @Ranges TABLE (UnitId int PRIMARY KEY, MinId bigint, MaxId bigint, Cnt int) ---INSERT INTO @Ranges --- EXECUTE dbo.GetResourceSurrogateIdRanges 96, 0, 9e18, 90000, 10 ---SELECT count(*) FROM @Ranges ---DECLARE @UnitId int --- ,@MinId bigint --- ,@MaxId bigint ---DECLARE @Resources TABLE (RawResource varbinary(max)) ---WHILE EXISTS (SELECT * FROM @Ranges) ---BEGIN --- SELECT TOP 1 @UnitId = UnitId, @MinId = MinId, @MaxId = MaxId FROM @Ranges ORDER BY UnitId --- INSERT INTO @Resources --- EXECUTE dbo.GetResourcesByTypeAndSurrogateIdRange 96, @MinId, @MaxId, NULL, @MaxId -- last is to invoke snapshot logic --- DELETE FROM @Resources --- DELETE FROM @Ranges WHERE UnitId = @UnitId ---END +--DECLARE @Definitions StringList +--INSERT INTO @Definitions SELECT 'Test' +--EXECUTE dbo.EnqueueJobs 2, @Definitions, @ForceOneActiveJobGroup = 1 diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/67.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/67.sql index 72acc71117..28e15d8c53 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/67.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/67.sql @@ -15,7 +15,7 @@ IF EXISTS (SELECT * GO INSERT INTO dbo.SchemaVersion -VALUES (66, 'started'); +VALUES (67, 'started'); CREATE PARTITION FUNCTION PartitionFunction_ResourceTypeId(SMALLINT) AS RANGE RIGHT @@ -2895,7 +2895,7 @@ BEGIN TRY 0 AS Dummy FROM @Input) AS A) AS A WHERE NOT EXISTS (SELECT * - FROM dbo.JobQueue AS B + FROM dbo.JobQueue AS B WITH (INDEX (IX_QueueType_DefinitionHash)) WHERE B.QueueType = @QueueType AND B.DefinitionHash = A.DefinitionHash AND B.Status <> 5); @@ -3727,81 +3727,85 @@ END CATCH GO CREATE PROCEDURE dbo.GetResourcesByTypeAndSurrogateIdRange -@ResourceTypeId SMALLINT, @StartId BIGINT, @EndId BIGINT, @GlobalEndId BIGINT=NULL, @IncludeHistory BIT=0, @IncludeDeleted BIT=0 +@ResourceTypeId SMALLINT, @StartId BIGINT, @EndId BIGINT, @GlobalStartId BIGINT=NULL, @GlobalEndId BIGINT=NULL AS SET NOCOUNT ON; -DECLARE @SP AS VARCHAR (100) = 'GetResourcesByTypeAndSurrogateIdRange', @Mode AS VARCHAR (100) = 'RT=' + isnull(CONVERT (VARCHAR, @ResourceTypeId), 'NULL') + ' S=' + isnull(CONVERT (VARCHAR, @StartId), 'NULL') + ' E=' + isnull(CONVERT (VARCHAR, @EndId), 'NULL') + ' GE=' + isnull(CONVERT (VARCHAR, @GlobalEndId), 'NULL') + ' HI=' + isnull(CONVERT (VARCHAR, @IncludeHistory), 'NULL') + ' DE' + isnull(CONVERT (VARCHAR, @IncludeDeleted), 'NULL'), @st AS DATETIME = getUTCdate(), @DummyTop AS BIGINT = 9223372036854775807; +DECLARE @SP AS VARCHAR (100) = 'GetResourcesByTypeAndSurrogateIdRange', @Mode AS VARCHAR (100) = 'RT=' + isnull(CONVERT (VARCHAR, @ResourceTypeId), 'NULL') + ' S=' + isnull(CONVERT (VARCHAR, @StartId), 'NULL') + ' E=' + isnull(CONVERT (VARCHAR, @EndId), 'NULL') + ' GS=' + isnull(CONVERT (VARCHAR, @GlobalStartId), 'NULL') + ' GE=' + isnull(CONVERT (VARCHAR, @GlobalEndId), 'NULL'), @st AS DATETIME = getUTCdate(); BEGIN TRY DECLARE @ResourceIds TABLE ( - ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS PRIMARY KEY); - DECLARE @SurrogateIds TABLE ( - MaxSurrogateId BIGINT PRIMARY KEY); + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS, + ResourceSurrogateId BIGINT , + RowId INT , + PRIMARY KEY (ResourceId, RowId)); + IF @GlobalStartId IS NULL + SET @GlobalStartId = 0; IF @GlobalEndId IS NOT NULL - AND @IncludeHistory = 0 + INSERT INTO @ResourceIds + SELECT ResourceId, + ResourceSurrogateId, + row_number() OVER (PARTITION BY ResourceId ORDER BY ResourceSurrogateId DESC) AS RowId + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceId IN (SELECT DISTINCT ResourceId + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + AND IsHistory = 1 + AND IsDeleted = 0) + AND ResourceSurrogateId BETWEEN @GlobalStartId AND @GlobalEndId; + IF EXISTS (SELECT * + FROM @ResourceIds) BEGIN - INSERT INTO @ResourceIds - SELECT DISTINCT ResourceId - FROM dbo.Resource - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId BETWEEN @StartId AND @EndId - AND IsHistory = 1 - AND (IsDeleted = 0 - OR @IncludeDeleted = 1) - OPTION (MAXDOP 1); - IF @@rowcount > 0 - INSERT INTO @SurrogateIds - SELECT ResourceSurrogateId - FROM (SELECT ResourceId, - ResourceSurrogateId, - row_number() OVER (PARTITION BY ResourceId ORDER BY ResourceSurrogateId DESC) AS RowId - FROM dbo.Resource WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceId IN (SELECT TOP (@DummyTop) ResourceId - FROM @ResourceIds) - AND ResourceSurrogateId BETWEEN @StartId AND @GlobalEndId) AS A - WHERE RowId = 1 - AND ResourceSurrogateId BETWEEN @StartId AND @EndId - OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + DECLARE @SurrogateIdMap TABLE ( + MaxSurrogateId BIGINT PRIMARY KEY); + INSERT INTO @SurrogateIdMap + SELECT A.ResourceSurrogateId AS MaxSurrogateId + FROM (SELECT * + FROM @ResourceIds + WHERE RowId = 1 + AND ResourceSurrogateId BETWEEN @StartId AND @EndId) AS A; + SELECT @ResourceTypeId, + CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.ResourceId ELSE A.ResourceId END, + CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.Version ELSE A.Version END, + CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.IsDeleted ELSE A.IsDeleted END, + isnull(C.ResourceSurrogateId, A.ResourceSurrogateId), + CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.RequestMethod ELSE A.RequestMethod END, + CONVERT (BIT, 1) AS IsMatch, + CONVERT (BIT, 0) AS IsPartial, + CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.IsRawResourceMetaSet ELSE A.IsRawResourceMetaSet END, + CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.SearchParamHash ELSE A.SearchParamHash END, + CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.RawResource ELSE A.RawResource END + FROM dbo.Resource AS A + LEFT OUTER JOIN + @SurrogateIdMap AS B + ON B.MaxSurrogateId = A.ResourceSurrogateId + LEFT OUTER JOIN + dbo.Resource AS C + ON C.ResourceTypeId = @ResourceTypeId + AND C.ResourceSurrogateId = MaxSurrogateId + WHERE A.ResourceTypeId = @ResourceTypeId + AND A.ResourceSurrogateId BETWEEN @StartId AND @EndId + AND (A.IsHistory = 0 + OR MaxSurrogateId IS NOT NULL) + AND A.IsDeleted = 0; END - SELECT ResourceTypeId, - ResourceId, - Version, - IsDeleted, - ResourceSurrogateId, - RequestMethod, - CONVERT (BIT, 1) AS IsMatch, - CONVERT (BIT, 0) AS IsPartial, - IsRawResourceMetaSet, - SearchParamHash, - RawResource - FROM dbo.Resource - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId BETWEEN @StartId AND @EndId - AND (IsHistory = 0 - OR @IncludeHistory = 1) - AND (IsDeleted = 0 - OR @IncludeDeleted = 1) - UNION ALL - SELECT ResourceTypeId, - ResourceId, - Version, - IsDeleted, - ResourceSurrogateId, - RequestMethod, - CONVERT (BIT, 1) AS IsMatch, - CONVERT (BIT, 0) AS IsPartial, - IsRawResourceMetaSet, - SearchParamHash, - RawResource - FROM @SurrogateIds - INNER JOIN - dbo.Resource - ON ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId = MaxSurrogateId - WHERE IsHistory = 1 - AND (IsDeleted = 0 - OR @IncludeDeleted = 1) - OPTION (MAXDOP 1); + ELSE + SELECT ResourceTypeId, + ResourceId, + Version, + IsDeleted, + ResourceSurrogateId, + RequestMethod, + CONVERT (BIT, 1) AS IsMatch, + CONVERT (BIT, 0) AS IsPartial, + IsRawResourceMetaSet, + SearchParamHash, + RawResource + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + AND IsHistory = 0 + AND IsDeleted = 0; EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; END TRY BEGIN CATCH @@ -3871,16 +3875,15 @@ BEGIN TRY CASE WHEN EXISTS (SELECT * FROM dbo.Resource AS B WHERE B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceId = A.ResourceId AND B.ResourceSurrogateId = A.ResourceSurrogateId) THEN 0 WHEN isnull(U.Version, 1) - isnull(L.Version, 0) > 1 THEN isnull(U.Version, 1) - 1 ELSE 0 END AS Version FROM (SELECT TOP (@DummyTop) * FROM @ResourceDateKeys) AS A OUTER APPLY (SELECT TOP 1 * - FROM dbo.Resource AS B + FROM dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) WHERE B.ResourceTypeId = A.ResourceTypeId AND B.ResourceId = A.ResourceId AND B.ResourceSurrogateId < A.ResourceSurrogateId ORDER BY B.ResourceSurrogateId DESC) AS L OUTER APPLY (SELECT TOP 1 * - FROM dbo.Resource AS B + FROM dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) WHERE B.ResourceTypeId = A.ResourceTypeId AND B.ResourceId = A.ResourceId AND B.ResourceSurrogateId > A.ResourceSurrogateId @@ -3985,7 +3988,7 @@ BEGIN TRY WHERE Id = 'InvisibleHistory.IsEnabled' AND Number = 0) UPDATE dbo.Resource - SET IsHistory = 1, + SET IsDeleted = 1, RawResource = 0xF, SearchParamHash = NULL, HistoryTransactionId = @TransactionId @@ -4279,7 +4282,7 @@ VALUES (@message); GO CREATE PROCEDURE dbo.MergeResources -@AffectedRows INT=0 OUTPUT, @RaiseExceptionOnConflict BIT=1, @IsResourceChangeCaptureEnabled BIT=0, @TransactionId BIGINT=NULL, @SingleTransaction BIT=1, @Resources dbo.ResourceList READONLY, @ResourceWriteClaims dbo.ResourceWriteClaimList READONLY, @CompartmentAssignments dbo.CompartmentAssignmentList READONLY, @ReferenceSearchParams dbo.ReferenceSearchParamList READONLY, @TokenSearchParams dbo.TokenSearchParamList READONLY, @TokenTexts dbo.TokenTextList READONLY, @StringSearchParams dbo.StringSearchParamList READONLY, @UriSearchParams dbo.UriSearchParamList READONLY, @NumberSearchParams dbo.NumberSearchParamList READONLY, @QuantitySearchParams dbo.QuantitySearchParamList READONLY, @DateTimeSearchParms dbo.DateTimeSearchParamList READONLY, @ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY, @TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY, @TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY, @TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY, @TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY, @TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY +@AffectedRows INT=0 OUTPUT, @RaiseExceptionOnConflict BIT=1, @IsResourceChangeCaptureEnabled BIT=0, @TransactionId BIGINT=NULL, @SingleTransaction BIT=1, @Resources dbo.ResourceList READONLY, @ResourceWriteClaims dbo.ResourceWriteClaimList READONLY, @ReferenceSearchParams dbo.ReferenceSearchParamList READONLY, @TokenSearchParams dbo.TokenSearchParamList READONLY, @TokenTexts dbo.TokenTextList READONLY, @StringSearchParams dbo.StringSearchParamList READONLY, @UriSearchParams dbo.UriSearchParamList READONLY, @NumberSearchParams dbo.NumberSearchParamList READONLY, @QuantitySearchParams dbo.QuantitySearchParamList READONLY, @DateTimeSearchParms dbo.DateTimeSearchParamList READONLY, @ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY, @TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY, @TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY, @TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY, @TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY, @TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY AS SET NOCOUNT ON; DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = object_name(@@procid), @DummyTop AS BIGINT = 9223372036854775807, @InitialTranCount AS INT = @@trancount, @IsRetry AS BIT = 0; @@ -4315,8 +4318,7 @@ BEGIN TRY INNER JOIN dbo.Resource AS B ON B.ResourceTypeId = A.ResourceTypeId - AND B.ResourceSurrogateId = A.ResourceSurrogateId - WHERE B.IsHistory = 0) + AND B.ResourceSurrogateId = A.ResourceSurrogateId) BEGIN BEGIN TRANSACTION; INSERT INTO @Existing (ResourceTypeId, SurrogateId) @@ -5144,7 +5146,6 @@ BEGIN TRY DELETE dbo.Resource WHERE ResourceTypeId = @TypeId AND HistoryTransactionId = @TransactionId - AND IsHistory = 1 AND RawResource = 0xF; SET @AffectedRows += @@rowcount; DELETE @Types diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/EnqueueJobs.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/EnqueueJobs.sql index 02a481219b..e6bc4f0340 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/EnqueueJobs.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/EnqueueJobs.sql @@ -55,7 +55,7 @@ BEGIN TRY ,DefinitionHash ,Status = CASE WHEN @IsCompleted = 1 THEN 2 ELSE 0 END FROM (SELECT JobId = @MaxJobId + row_number() OVER (ORDER BY Dummy), * FROM (SELECT *, Dummy = 0 FROM @Input) A) A -- preserve input order - WHERE NOT EXISTS (SELECT * FROM dbo.JobQueue B WHERE B.QueueType = @QueueType AND B.DefinitionHash = A.DefinitionHash AND B.Status <> 5) + WHERE NOT EXISTS (SELECT * FROM dbo.JobQueue B WITH (INDEX = IX_QueueType_DefinitionHash) WHERE B.QueueType = @QueueType AND B.DefinitionHash = A.DefinitionHash AND B.Status <> 5) SET @Rows = @@rowcount COMMIT TRANSACTION From 4a2b5399d383495dfb5b1ae4457b5932259881f6 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Tue, 31 Oct 2023 13:50:53 -0700 Subject: [PATCH 44/58] Updated schema version to iterate after merge --- .../Features/Schema/Migrations/68.diff.sql | 81 + .../Features/Schema/Migrations/68.sql | 6736 +++++++++++++++++ .../Features/Schema/SchemaVersion.cs | 1 + .../Features/Schema/SchemaVersionConstants.cs | 2 +- .../TransactionCheckWithInitialiScript.sql | 2 +- .../Microsoft.Health.Fhir.SqlServer.csproj | 2 +- 6 files changed, 6821 insertions(+), 3 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/68.diff.sql create mode 100644 src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/68.sql diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/68.diff.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/68.diff.sql new file mode 100644 index 0000000000..947ba59e22 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/68.diff.sql @@ -0,0 +1,81 @@ +CREATE OR ALTER PROCEDURE dbo.GetResourcesByTypeAndSurrogateIdRange @ResourceTypeId smallint, @StartId bigint, @EndId bigint, @GlobalEndId bigint = NULL, @IncludeHistory bit = 0, @IncludeDeleted bit = 0 +AS +set nocount on +DECLARE @SP varchar(100) = 'GetResourcesByTypeAndSurrogateIdRange' + ,@Mode varchar(100) = 'RT='+isnull(convert(varchar,@ResourceTypeId),'NULL') + +' S='+isnull(convert(varchar,@StartId),'NULL') + +' E='+isnull(convert(varchar,@EndId),'NULL') + +' GE='+isnull(convert(varchar,@GlobalEndId),'NULL') -- Could this just be a boolean for if historical records should be returned? GlobalEndId should equal EndId in all cases I can think of. + +' HI='+isnull(convert(varchar,@IncludeHistory),'NULL') + +' DE'+isnull(convert(varchar,@IncludeDeleted),'NULL') + ,@st datetime = getUTCdate() + ,@DummyTop bigint = 9223372036854775807 + +BEGIN TRY + DECLARE @ResourceIds TABLE (ResourceId varchar(64) COLLATE Latin1_General_100_CS_AS PRIMARY KEY) + DECLARE @SurrogateIds TABLE (MaxSurrogateId bigint PRIMARY KEY) + + IF @GlobalEndId IS NOT NULL AND @IncludeHistory = 0 -- snapshot view + BEGIN + INSERT INTO @ResourceIds + SELECT DISTINCT ResourceId + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + AND IsHistory = 1 + AND (IsDeleted = 0 OR @IncludeDeleted = 1) + OPTION (MAXDOP 1) + + IF @@rowcount > 0 + INSERT INTO @SurrogateIds + SELECT ResourceSurrogateId + FROM (SELECT ResourceId, ResourceSurrogateId, RowId = row_number() OVER (PARTITION BY ResourceId ORDER BY ResourceSurrogateId DESC) + FROM dbo.Resource WITH (INDEX = IX_Resource_ResourceTypeId_ResourceId_Version) -- w/o hint access to Resource table is inefficient when many versions are present. Hint is ignored if Resource is a view. + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceId IN (SELECT TOP (@DummyTop) ResourceId FROM @ResourceIds) + AND ResourceSurrogateId BETWEEN @StartId AND @GlobalEndId + ) A + WHERE RowId = 1 + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)) + END + + SELECT ResourceTypeId, ResourceId, Version, IsDeleted, ResourceSurrogateId, RequestMethod, IsMatch = convert(bit,1), IsPartial = convert(bit,0), IsRawResourceMetaSet, SearchParamHash, RawResource + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + AND (IsHistory = 0 OR @IncludeHistory = 1) + AND (IsDeleted = 0 OR @IncludeDeleted = 1) + UNION ALL + SELECT ResourceTypeId, ResourceId, Version, IsDeleted, ResourceSurrogateId, RequestMethod, IsMatch = convert(bit,1), IsPartial = convert(bit,0), IsRawResourceMetaSet, SearchParamHash, RawResource + FROM @SurrogateIds + JOIN dbo.Resource ON ResourceTypeId = @ResourceTypeId AND ResourceSurrogateId = MaxSurrogateId + WHERE IsHistory = 1 + AND (IsDeleted = 0 OR @IncludeDeleted = 1) + OPTION (MAXDOP 1) + + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@st,@Rows=@@rowcount +END TRY +BEGIN CATCH + IF error_number() = 1750 THROW -- Real error is before 1750, cannot trap in SQL. + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Error'; + THROW +END CATCH +GO +--set nocount on +--DECLARE @Ranges TABLE (UnitId int PRIMARY KEY, MinId bigint, MaxId bigint, Cnt int) +--INSERT INTO @Ranges +-- EXECUTE dbo.GetResourceSurrogateIdRanges 96, 0, 9e18, 90000, 10 +--SELECT count(*) FROM @Ranges +--DECLARE @UnitId int +-- ,@MinId bigint +-- ,@MaxId bigint +--DECLARE @Resources TABLE (RawResource varbinary(max)) +--WHILE EXISTS (SELECT * FROM @Ranges) +--BEGIN +-- SELECT TOP 1 @UnitId = UnitId, @MinId = MinId, @MaxId = MaxId FROM @Ranges ORDER BY UnitId +-- INSERT INTO @Resources +-- EXECUTE dbo.GetResourcesByTypeAndSurrogateIdRange 96, @MinId, @MaxId, NULL, @MaxId -- last is to invoke snapshot logic +-- DELETE FROM @Resources +-- DELETE FROM @Ranges WHERE UnitId = @UnitId +--END diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/68.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/68.sql new file mode 100644 index 0000000000..c17b303de7 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/68.sql @@ -0,0 +1,6736 @@ + +/************************************************************************************************* + Auto-Generated from Sql build task. Do not manually edit it. +**************************************************************************************************/ +SET XACT_ABORT ON +BEGIN TRAN +IF EXISTS (SELECT * + FROM sys.tables + WHERE name = 'ClaimType') + BEGIN + ROLLBACK; + RETURN; + END + + +GO +INSERT INTO dbo.SchemaVersion +VALUES (68, 'started'); + +CREATE PARTITION FUNCTION PartitionFunction_ResourceTypeId(SMALLINT) + AS RANGE RIGHT + FOR VALUES (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150); + +CREATE PARTITION SCHEME PartitionScheme_ResourceTypeId + AS PARTITION PartitionFunction_ResourceTypeId + ALL TO ([PRIMARY]); + + +GO +CREATE PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp(DATETIME2 (7)) + AS RANGE RIGHT + FOR VALUES (N'1970-01-01T00:00:00.0000000'); + +CREATE PARTITION SCHEME PartitionScheme_ResourceChangeData_Timestamp + AS PARTITION PartitionFunction_ResourceChangeData_Timestamp + ALL TO ([PRIMARY]); + +DECLARE @numberOfHistoryPartitions AS INT = 48; + +DECLARE @numberOfFuturePartitions AS INT = 720; + +DECLARE @rightPartitionBoundary AS DATETIME2 (7); + +DECLARE @currentDateTime AS DATETIME2 (7) = sysutcdatetime(); + +WHILE @numberOfHistoryPartitions >= -@numberOfFuturePartitions + BEGIN + SET @rightPartitionBoundary = DATEADD(hour, DATEDIFF(hour, 0, @currentDateTime) - @numberOfHistoryPartitions, 0); + ALTER PARTITION SCHEME PartitionScheme_ResourceChangeData_Timestamp NEXT USED [Primary]; + ALTER PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp( ) + SPLIT RANGE (@rightPartitionBoundary); + SET @numberOfHistoryPartitions -= 1; + END + +CREATE SEQUENCE dbo.ResourceSurrogateIdUniquifierSequence + AS INT + START WITH 0 + INCREMENT BY 1 + MINVALUE 0 + MAXVALUE 79999 + CYCLE + CACHE 1000000; + +CREATE TYPE dbo.BigintList AS TABLE ( + Id BIGINT NOT NULL PRIMARY KEY); + +CREATE TYPE dbo.CompartmentAssignmentList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + CompartmentTypeId TINYINT NOT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL PRIMARY KEY (ResourceTypeId, ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId)); + +CREATE TYPE dbo.DateTimeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + StartDateTime DATETIMEOFFSET (7) NOT NULL, + EndDateTime DATETIMEOFFSET (7) NOT NULL, + IsLongerThanADay BIT NOT NULL, + IsMin BIT NOT NULL, + IsMax BIT NOT NULL UNIQUE (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax)); + +CREATE TYPE dbo.NumberSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SingleValue DECIMAL (36, 18) NULL, + LowValue DECIMAL (36, 18) NULL, + HighValue DECIMAL (36, 18) NULL UNIQUE (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue)); + +CREATE TYPE dbo.QuantitySearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + QuantityCodeId INT NULL, + SingleValue DECIMAL (36, 18) NULL, + LowValue DECIMAL (36, 18) NULL, + HighValue DECIMAL (36, 18) NULL UNIQUE (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue)); + +CREATE TYPE dbo.ReferenceSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId SMALLINT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion INT NULL UNIQUE (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId)); + +CREATE TYPE dbo.ReferenceTokenCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId1 SMALLINT NULL, + ReferenceResourceId1 VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion1 INT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.ResourceDateKeyList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ResourceSurrogateId BIGINT NOT NULL PRIMARY KEY (ResourceTypeId, ResourceId, ResourceSurrogateId)); + +CREATE TYPE dbo.ResourceKeyList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Version INT NULL UNIQUE (ResourceTypeId, ResourceId, Version)); + +CREATE TYPE dbo.ResourceList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Version INT NOT NULL, + HasVersionToCompare BIT NOT NULL, + IsDeleted BIT NOT NULL, + IsHistory BIT NOT NULL, + KeepHistory BIT NOT NULL, + RawResource VARBINARY (MAX) NOT NULL, + IsRawResourceMetaSet BIT NOT NULL, + RequestMethod VARCHAR (10) NULL, + SearchParamHash VARCHAR (64) NULL PRIMARY KEY (ResourceTypeId, ResourceSurrogateId), + UNIQUE (ResourceTypeId, ResourceId, Version)); + +CREATE TYPE dbo.ResourceWriteClaimList AS TABLE ( + ResourceSurrogateId BIGINT NOT NULL, + ClaimTypeId TINYINT NOT NULL, + ClaimValue NVARCHAR (128) NOT NULL); + +CREATE TYPE dbo.StringList AS TABLE ( + String VARCHAR (MAX)); + +CREATE TYPE dbo.StringSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL, + IsMin BIT NOT NULL, + IsMax BIT NOT NULL); + +CREATE TYPE dbo.TokenDateTimeCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + StartDateTime2 DATETIMEOFFSET (7) NOT NULL, + EndDateTime2 DATETIMEOFFSET (7) NOT NULL, + IsLongerThanADay2 BIT NOT NULL); + +CREATE TYPE dbo.TokenNumberNumberCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SingleValue2 DECIMAL (36, 18) NULL, + LowValue2 DECIMAL (36, 18) NULL, + HighValue2 DECIMAL (36, 18) NULL, + SingleValue3 DECIMAL (36, 18) NULL, + LowValue3 DECIMAL (36, 18) NULL, + HighValue3 DECIMAL (36, 18) NULL, + HasRange BIT NOT NULL); + +CREATE TYPE dbo.TokenQuantityCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SystemId2 INT NULL, + QuantityCodeId2 INT NULL, + SingleValue2 DECIMAL (36, 18) NULL, + LowValue2 DECIMAL (36, 18) NULL, + HighValue2 DECIMAL (36, 18) NULL); + +CREATE TYPE dbo.TokenSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + Code VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.TokenStringCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + Text2 NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow2 NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL); + +CREATE TYPE dbo.TokenTextList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (400) COLLATE Latin1_General_CI_AI NOT NULL); + +CREATE TYPE dbo.TokenTokenCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.BulkResourceWriteClaimTableType_1 AS TABLE ( + Offset INT NOT NULL, + ClaimTypeId TINYINT NOT NULL, + ClaimValue NVARCHAR (128) NOT NULL); + +CREATE TYPE dbo.BulkCompartmentAssignmentTableType_1 AS TABLE ( + Offset INT NOT NULL, + CompartmentTypeId TINYINT NOT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL); + +CREATE TYPE dbo.BulkReferenceSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId SMALLINT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion INT NULL); + +CREATE TYPE dbo.BulkTokenSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + Code VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL); + +CREATE TYPE dbo.BulkTokenSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + Code VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.BulkTokenTextTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (400) COLLATE Latin1_General_CI_AI NOT NULL); + +CREATE TYPE dbo.BulkStringSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL); + +CREATE TYPE dbo.BulkStringSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL, + IsMin BIT NOT NULL, + IsMax BIT NOT NULL); + +CREATE TYPE dbo.BulkUriSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Uri VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL); + +CREATE TYPE dbo.BulkNumberSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SingleValue DECIMAL (18, 6) NULL, + LowValue DECIMAL (18, 6) NULL, + HighValue DECIMAL (18, 6) NULL); + +CREATE TYPE dbo.BulkQuantitySearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + QuantityCodeId INT NULL, + SingleValue DECIMAL (18, 6) NULL, + LowValue DECIMAL (18, 6) NULL, + HighValue DECIMAL (18, 6) NULL); + +CREATE TYPE dbo.BulkDateTimeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + StartDateTime DATETIMEOFFSET (7) NOT NULL, + EndDateTime DATETIMEOFFSET (7) NOT NULL, + IsLongerThanADay BIT NOT NULL); + +CREATE TYPE dbo.BulkDateTimeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + StartDateTime DATETIMEOFFSET (7) NOT NULL, + EndDateTime DATETIMEOFFSET (7) NOT NULL, + IsLongerThanADay BIT NOT NULL, + IsMin BIT NOT NULL, + IsMax BIT NOT NULL); + +CREATE TYPE dbo.BulkReferenceTokenCompositeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId1 SMALLINT NULL, + ReferenceResourceId1 VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion1 INT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL); + +CREATE TYPE dbo.BulkReferenceTokenCompositeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId1 SMALLINT NULL, + ReferenceResourceId1 VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion1 INT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.BulkTokenTokenCompositeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL); + +CREATE TYPE dbo.BulkTokenTokenCompositeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.BulkTokenDateTimeCompositeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + StartDateTime2 DATETIMEOFFSET (7) NOT NULL, + EndDateTime2 DATETIMEOFFSET (7) NOT NULL, + IsLongerThanADay2 BIT NOT NULL); + +CREATE TYPE dbo.BulkTokenDateTimeCompositeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + StartDateTime2 DATETIMEOFFSET (7) NOT NULL, + EndDateTime2 DATETIMEOFFSET (7) NOT NULL, + IsLongerThanADay2 BIT NOT NULL); + +CREATE TYPE dbo.BulkTokenQuantityCompositeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + SystemId2 INT NULL, + QuantityCodeId2 INT NULL, + SingleValue2 DECIMAL (18, 6) NULL, + LowValue2 DECIMAL (18, 6) NULL, + HighValue2 DECIMAL (18, 6) NULL); + +CREATE TYPE dbo.BulkTokenQuantityCompositeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SystemId2 INT NULL, + QuantityCodeId2 INT NULL, + SingleValue2 DECIMAL (18, 6) NULL, + LowValue2 DECIMAL (18, 6) NULL, + HighValue2 DECIMAL (18, 6) NULL); + +CREATE TYPE dbo.BulkTokenStringCompositeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + Text2 NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow2 NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL); + +CREATE TYPE dbo.BulkTokenStringCompositeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + Text2 NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow2 NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL); + +CREATE TYPE dbo.BulkTokenNumberNumberCompositeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + SingleValue2 DECIMAL (18, 6) NULL, + LowValue2 DECIMAL (18, 6) NULL, + HighValue2 DECIMAL (18, 6) NULL, + SingleValue3 DECIMAL (18, 6) NULL, + LowValue3 DECIMAL (18, 6) NULL, + HighValue3 DECIMAL (18, 6) NULL, + HasRange BIT NOT NULL); + +CREATE TYPE dbo.BulkTokenNumberNumberCompositeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SingleValue2 DECIMAL (18, 6) NULL, + LowValue2 DECIMAL (18, 6) NULL, + HighValue2 DECIMAL (18, 6) NULL, + SingleValue3 DECIMAL (18, 6) NULL, + LowValue3 DECIMAL (18, 6) NULL, + HighValue3 DECIMAL (18, 6) NULL, + HasRange BIT NOT NULL); + +CREATE TYPE dbo.SearchParamTableType_1 AS TABLE ( + Uri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + Status VARCHAR (10) NOT NULL, + IsPartiallySupported BIT NOT NULL); + +CREATE TYPE dbo.SearchParamTableType_2 AS TABLE ( + Uri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + Status VARCHAR (20) NOT NULL, + IsPartiallySupported BIT NOT NULL); + +CREATE TYPE dbo.BulkReindexResourceTableType_1 AS TABLE ( + Offset INT NOT NULL, + ResourceTypeId SMALLINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ETag INT NULL, + SearchParamHash VARCHAR (64) NOT NULL); + +CREATE TYPE dbo.BulkImportResourceType_1 AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Version INT NOT NULL, + IsHistory BIT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + IsDeleted BIT NOT NULL, + RequestMethod VARCHAR (10) NULL, + RawResource VARBINARY (MAX) NOT NULL, + IsRawResourceMetaSet BIT DEFAULT 0 NOT NULL, + SearchParamHash VARCHAR (64) NULL); + +CREATE TYPE dbo.UriSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Uri VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL PRIMARY KEY (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri)); + +CREATE TABLE dbo.ClaimType ( + ClaimTypeId TINYINT IDENTITY (1, 1) NOT NULL, + Name VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + CONSTRAINT UQ_ClaimType_ClaimTypeId UNIQUE (ClaimTypeId), + CONSTRAINT PKC_ClaimType PRIMARY KEY CLUSTERED (Name) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE dbo.CompartmentAssignment ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + CompartmentTypeId TINYINT NOT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + IsHistory BIT NOT NULL, + CONSTRAINT PKC_CompartmentAssignment PRIMARY KEY CLUSTERED (ResourceTypeId, ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId) WITH (DATA_COMPRESSION = PAGE) ON PartitionScheme_ResourceTypeId (ResourceTypeId) +); + + +GO +ALTER TABLE dbo.CompartmentAssignment + ADD CONSTRAINT DF_CompartmentAssignment_IsHistory DEFAULT 0 FOR IsHistory; + + +GO +ALTER TABLE dbo.CompartmentAssignment SET (LOCK_ESCALATION = AUTO); + + +GO +CREATE NONCLUSTERED INDEX IX_CompartmentAssignment_CompartmentTypeId_ReferenceResourceId + ON dbo.CompartmentAssignment(ResourceTypeId, CompartmentTypeId, ReferenceResourceId, ResourceSurrogateId) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.CompartmentType ( + CompartmentTypeId TINYINT IDENTITY (1, 1) NOT NULL, + Name VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + CONSTRAINT UQ_CompartmentType_CompartmentTypeId UNIQUE (CompartmentTypeId), + CONSTRAINT PKC_CompartmentType PRIMARY KEY CLUSTERED (Name) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE dbo.DateTimeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + StartDateTime DATETIME2 (7) NOT NULL, + EndDateTime DATETIME2 (7) NOT NULL, + IsLongerThanADay BIT NOT NULL, + IsHistory BIT NOT NULL, + IsMin BIT CONSTRAINT date_IsMin_Constraint DEFAULT 0 NOT NULL, + IsMax BIT CONSTRAINT date_IsMax_Constraint DEFAULT 0 NOT NULL +); + +ALTER TABLE dbo.DateTimeSearchParam + ADD CONSTRAINT DF_DateTimeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.DateTimeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_DateTimeSearchParam + ON dbo.DateTimeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_DateTimeSearchParam_SearchParamId_StartDateTime_EndDateTime + ON dbo.DateTimeSearchParam(ResourceTypeId, SearchParamId, StartDateTime, EndDateTime, ResourceSurrogateId) + INCLUDE(IsLongerThanADay, IsMin, IsMax) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_DateTimeSearchParam_SearchParamId_EndDateTime_StartDateTime + ON dbo.DateTimeSearchParam(ResourceTypeId, SearchParamId, EndDateTime, StartDateTime, ResourceSurrogateId) + INCLUDE(IsLongerThanADay, IsMin, IsMax) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_DateTimeSearchParam_SearchParamId_StartDateTime_EndDateTime_Long + ON dbo.DateTimeSearchParam(ResourceTypeId, SearchParamId, StartDateTime, EndDateTime, ResourceSurrogateId) + INCLUDE(IsMin, IsMax) WHERE IsHistory = 0 + AND IsLongerThanADay = 1 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_DateTimeSearchParam_SearchParamId_EndDateTime_StartDateTime_Long + ON dbo.DateTimeSearchParam(ResourceTypeId, SearchParamId, EndDateTime, StartDateTime, ResourceSurrogateId) + INCLUDE(IsMin, IsMax) WHERE IsHistory = 0 + AND IsLongerThanADay = 1 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +IF NOT EXISTS (SELECT 1 + FROM sys.tables + WHERE name = 'EventAgentCheckpoint') + BEGIN + CREATE TABLE dbo.EventAgentCheckpoint ( + CheckpointId VARCHAR (64) NOT NULL, + LastProcessedDateTime DATETIMEOFFSET (7), + LastProcessedIdentifier VARCHAR (64) , + UpdatedOn DATETIME2 (7) DEFAULT sysutcdatetime() NOT NULL, + CONSTRAINT PK_EventAgentCheckpoint PRIMARY KEY CLUSTERED (CheckpointId) + ) ON [PRIMARY]; + END + +CREATE PARTITION FUNCTION EventLogPartitionFunction(TINYINT) + AS RANGE RIGHT + FOR VALUES (0, 1, 2, 3, 4, 5, 6, 7); + + +GO +CREATE PARTITION SCHEME EventLogPartitionScheme + AS PARTITION EventLogPartitionFunction + ALL TO ([PRIMARY]); + + +GO +CREATE TABLE dbo.EventLog ( + PartitionId AS isnull(CONVERT (TINYINT, EventId % 8), 0) PERSISTED, + EventId BIGINT IDENTITY (1, 1) NOT NULL, + EventDate DATETIME NOT NULL, + Process VARCHAR (100) NOT NULL, + Status VARCHAR (10) NOT NULL, + Mode VARCHAR (200) NULL, + Action VARCHAR (20) NULL, + Target VARCHAR (100) NULL, + Rows BIGINT NULL, + Milliseconds INT NULL, + EventText NVARCHAR (3500) NULL, + SPID SMALLINT NOT NULL, + HostName VARCHAR (64) NOT NULL CONSTRAINT PKC_EventLog_EventDate_EventId_PartitionId PRIMARY KEY CLUSTERED (EventDate, EventId, PartitionId) ON EventLogPartitionScheme (PartitionId) +); + +CREATE TABLE dbo.ExportJob ( + Id VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Hash VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Status VARCHAR (10) NOT NULL, + HeartbeatDateTime DATETIME2 (7) NULL, + RawJobRecord VARCHAR (MAX) NOT NULL, + JobVersion ROWVERSION NOT NULL, + CONSTRAINT PKC_ExportJob PRIMARY KEY CLUSTERED (Id) +); + +CREATE UNIQUE NONCLUSTERED INDEX IX_ExportJob_Hash_Status_HeartbeatDateTime + ON dbo.ExportJob(Hash, Status, HeartbeatDateTime); + +CREATE TABLE dbo.IndexProperties ( + TableName VARCHAR (100) NOT NULL, + IndexName VARCHAR (200) NOT NULL, + PropertyName VARCHAR (100) NOT NULL, + PropertyValue VARCHAR (100) NOT NULL, + CreateDate DATETIME CONSTRAINT DF_IndexProperties_CreateDate DEFAULT getUTCdate() NOT NULL CONSTRAINT PKC_IndexProperties_TableName_IndexName_PropertyName PRIMARY KEY CLUSTERED (TableName, IndexName, PropertyName) +); + +CREATE PARTITION FUNCTION TinyintPartitionFunction(TINYINT) + AS RANGE RIGHT + FOR VALUES (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255); + + +GO +CREATE PARTITION SCHEME TinyintPartitionScheme + AS PARTITION TinyintPartitionFunction + ALL TO ([PRIMARY]); + + +GO +CREATE TABLE dbo.JobQueue ( + QueueType TINYINT NOT NULL, + GroupId BIGINT NOT NULL, + JobId BIGINT NOT NULL, + PartitionId AS CONVERT (TINYINT, JobId % 16) PERSISTED, + Definition VARCHAR (MAX) NOT NULL, + DefinitionHash VARBINARY (20) NOT NULL, + Version BIGINT CONSTRAINT DF_JobQueue_Version DEFAULT datediff_big(millisecond, '0001-01-01', getUTCdate()) NOT NULL, + Status TINYINT CONSTRAINT DF_JobQueue_Status DEFAULT 0 NOT NULL, + Priority TINYINT CONSTRAINT DF_JobQueue_Priority DEFAULT 100 NOT NULL, + Data BIGINT NULL, + Result VARCHAR (MAX) NULL, + CreateDate DATETIME CONSTRAINT DF_JobQueue_CreateDate DEFAULT getUTCdate() NOT NULL, + StartDate DATETIME NULL, + EndDate DATETIME NULL, + HeartbeatDate DATETIME CONSTRAINT DF_JobQueue_HeartbeatDate DEFAULT getUTCdate() NOT NULL, + Worker VARCHAR (100) NULL, + Info VARCHAR (1000) NULL, + CancelRequested BIT CONSTRAINT DF_JobQueue_CancelRequested DEFAULT 0 NOT NULL CONSTRAINT PKC_JobQueue_QueueType_PartitionId_JobId PRIMARY KEY CLUSTERED (QueueType, PartitionId, JobId) ON TinyintPartitionScheme (QueueType), + CONSTRAINT U_JobQueue_QueueType_JobId UNIQUE (QueueType, JobId) +); + + +GO +CREATE INDEX IX_QueueType_PartitionId_Status_Priority + ON dbo.JobQueue(PartitionId, Status, Priority) + ON TinyintPartitionScheme (QueueType); + + +GO +CREATE INDEX IX_QueueType_GroupId + ON dbo.JobQueue(QueueType, GroupId) + ON TinyintPartitionScheme (QueueType); + + +GO +CREATE INDEX IX_QueueType_DefinitionHash + ON dbo.JobQueue(QueueType, DefinitionHash) + ON TinyintPartitionScheme (QueueType); + +CREATE TABLE dbo.NumberSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SingleValue DECIMAL (36, 18) NULL, + LowValue DECIMAL (36, 18) NOT NULL, + HighValue DECIMAL (36, 18) NOT NULL, + IsHistory BIT NOT NULL +); + +ALTER TABLE dbo.NumberSearchParam + ADD CONSTRAINT DF_NumberSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.NumberSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_NumberSearchParam + ON dbo.NumberSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_NumberSearchParam_SearchParamId_SingleValue + ON dbo.NumberSearchParam(ResourceTypeId, SearchParamId, SingleValue, ResourceSurrogateId) WHERE IsHistory = 0 + AND SingleValue IS NOT NULL + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_NumberSearchParam_SearchParamId_LowValue_HighValue + ON dbo.NumberSearchParam(ResourceTypeId, SearchParamId, LowValue, HighValue, ResourceSurrogateId) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_NumberSearchParam_SearchParamId_HighValue_LowValue + ON dbo.NumberSearchParam(ResourceTypeId, SearchParamId, HighValue, LowValue, ResourceSurrogateId) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.Parameters ( + Id VARCHAR (100) NOT NULL, + Date DATETIME NULL, + Number FLOAT NULL, + Bigint BIGINT NULL, + Char VARCHAR (4000) NULL, + Binary VARBINARY (MAX) NULL, + UpdatedDate DATETIME NULL, + UpdatedBy NVARCHAR (255) NULL CONSTRAINT PKC_Parameters_Id PRIMARY KEY CLUSTERED (Id) WITH (IGNORE_DUP_KEY = ON) +); + + +GO +CREATE TABLE dbo.ParametersHistory ( + ChangeId INT IDENTITY (1, 1) NOT NULL, + Id VARCHAR (100) NOT NULL, + Date DATETIME NULL, + Number FLOAT NULL, + Bigint BIGINT NULL, + Char VARCHAR (4000) NULL, + Binary VARBINARY (MAX) NULL, + UpdatedDate DATETIME NULL, + UpdatedBy NVARCHAR (255) NULL +); + +CREATE TABLE dbo.QuantityCode ( + QuantityCodeId INT IDENTITY (1, 1) NOT NULL, + Value NVARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CONSTRAINT UQ_QuantityCode_QuantityCodeId UNIQUE (QuantityCodeId), + CONSTRAINT PKC_QuantityCode PRIMARY KEY CLUSTERED (Value) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE dbo.QuantitySearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + QuantityCodeId INT NULL, + SingleValue DECIMAL (36, 18) NULL, + LowValue DECIMAL (36, 18) NOT NULL, + HighValue DECIMAL (36, 18) NOT NULL, + IsHistory BIT NOT NULL +); + +ALTER TABLE dbo.QuantitySearchParam + ADD CONSTRAINT DF_QuantitySearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.QuantitySearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_QuantitySearchParam + ON dbo.QuantitySearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_QuantitySearchParam_SearchParamId_QuantityCodeId_SingleValue + ON dbo.QuantitySearchParam(ResourceTypeId, SearchParamId, QuantityCodeId, SingleValue, ResourceSurrogateId) + INCLUDE(SystemId) WHERE IsHistory = 0 + AND SingleValue IS NOT NULL + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_QuantitySearchParam_SearchParamId_QuantityCodeId_LowValue_HighValue + ON dbo.QuantitySearchParam(ResourceTypeId, SearchParamId, QuantityCodeId, LowValue, HighValue, ResourceSurrogateId) + INCLUDE(SystemId) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_QuantitySearchParam_SearchParamId_QuantityCodeId_HighValue_LowValue + ON dbo.QuantitySearchParam(ResourceTypeId, SearchParamId, QuantityCodeId, HighValue, LowValue, ResourceSurrogateId) + INCLUDE(SystemId) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.ReferenceSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId SMALLINT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion INT NULL, + IsHistory BIT NOT NULL +); + +ALTER TABLE dbo.ReferenceSearchParam + ADD CONSTRAINT DF_ReferenceSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.ReferenceSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_ReferenceSearchParam + ON dbo.ReferenceSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_ReferenceSearchParam_SearchParamId_ReferenceResourceTypeId_ReferenceResourceId_BaseUri_ReferenceResourceVersion + ON dbo.ReferenceSearchParam(ResourceTypeId, SearchParamId, ReferenceResourceId, ReferenceResourceTypeId, BaseUri, ResourceSurrogateId) + INCLUDE(ReferenceResourceVersion) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.ReferenceTokenCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId1 SMALLINT NULL, + ReferenceResourceId1 VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion1 INT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + IsHistory BIT NOT NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.ReferenceTokenCompositeSearchParam + ADD CONSTRAINT DF_ReferenceTokenCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.ReferenceTokenCompositeSearchParam + ADD CONSTRAINT CHK_ReferenceTokenCompositeSearchParam_CodeOverflow2 CHECK (LEN(Code2) = 256 + OR CodeOverflow2 IS NULL); + +ALTER TABLE dbo.ReferenceTokenCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_ReferenceTokenCompositeSearchParam + ON dbo.ReferenceTokenCompositeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_ReferenceTokenCompositeSearchParam_ReferenceResourceId1_Code2 + ON dbo.ReferenceTokenCompositeSearchParam(ResourceTypeId, SearchParamId, ReferenceResourceId1, Code2, ResourceSurrogateId) + INCLUDE(ReferenceResourceTypeId1, BaseUri1, SystemId2) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.ReindexJob ( + Id VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Status VARCHAR (10) NOT NULL, + HeartbeatDateTime DATETIME2 (7) NULL, + RawJobRecord VARCHAR (MAX) NOT NULL, + JobVersion ROWVERSION NOT NULL, + CONSTRAINT PKC_ReindexJob PRIMARY KEY CLUSTERED (Id) +); + +CREATE TABLE dbo.Resource ( + ResourceTypeId SMALLINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Version INT NOT NULL, + IsHistory BIT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + IsDeleted BIT NOT NULL, + RequestMethod VARCHAR (10) NULL, + RawResource VARBINARY (MAX) NOT NULL, + IsRawResourceMetaSet BIT DEFAULT 0 NOT NULL, + SearchParamHash VARCHAR (64) NULL, + TransactionId BIGINT NULL, + HistoryTransactionId BIGINT NULL CONSTRAINT PKC_Resource PRIMARY KEY CLUSTERED (ResourceTypeId, ResourceSurrogateId) WITH (DATA_COMPRESSION = PAGE) ON PartitionScheme_ResourceTypeId (ResourceTypeId), + CONSTRAINT CH_Resource_RawResource_Length CHECK (RawResource > 0x0) +); + +ALTER TABLE dbo.Resource SET (LOCK_ESCALATION = AUTO); + +CREATE INDEX IX_ResourceTypeId_TransactionId + ON dbo.Resource(ResourceTypeId, TransactionId) WHERE TransactionId IS NOT NULL + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_ResourceTypeId_HistoryTransactionId + ON dbo.Resource(ResourceTypeId, HistoryTransactionId) WHERE HistoryTransactionId IS NOT NULL + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE UNIQUE NONCLUSTERED INDEX IX_Resource_ResourceTypeId_ResourceId_Version + ON dbo.Resource(ResourceTypeId, ResourceId, Version) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE UNIQUE NONCLUSTERED INDEX IX_Resource_ResourceTypeId_ResourceId + ON dbo.Resource(ResourceTypeId, ResourceId) + INCLUDE(Version, IsDeleted) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE UNIQUE NONCLUSTERED INDEX IX_Resource_ResourceTypeId_ResourceSurrgateId + ON dbo.Resource(ResourceTypeId, ResourceSurrogateId) WHERE IsHistory = 0 + AND IsDeleted = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.ResourceChangeData ( + Id BIGINT IDENTITY (1, 1) NOT NULL, + Timestamp DATETIME2 (7) CONSTRAINT DF_ResourceChangeData_Timestamp DEFAULT sysutcdatetime() NOT NULL, + ResourceId VARCHAR (64) NOT NULL, + ResourceTypeId SMALLINT NOT NULL, + ResourceVersion INT NOT NULL, + ResourceChangeTypeId TINYINT NOT NULL +) ON PartitionScheme_ResourceChangeData_Timestamp (Timestamp); + +CREATE CLUSTERED INDEX IXC_ResourceChangeData + ON dbo.ResourceChangeData(Id ASC) WITH (ONLINE = ON) + ON PartitionScheme_ResourceChangeData_Timestamp (Timestamp); + +CREATE TABLE dbo.ResourceChangeDataStaging ( + Id BIGINT IDENTITY (1, 1) NOT NULL, + Timestamp DATETIME2 (7) CONSTRAINT DF_ResourceChangeDataStaging_Timestamp DEFAULT sysutcdatetime() NOT NULL, + ResourceId VARCHAR (64) NOT NULL, + ResourceTypeId SMALLINT NOT NULL, + ResourceVersion INT NOT NULL, + ResourceChangeTypeId TINYINT NOT NULL +) ON [PRIMARY]; + +CREATE CLUSTERED INDEX IXC_ResourceChangeDataStaging + ON dbo.ResourceChangeDataStaging(Id ASC, Timestamp ASC) WITH (ONLINE = ON) + ON [PRIMARY]; + +ALTER TABLE dbo.ResourceChangeDataStaging WITH CHECK + ADD CONSTRAINT CHK_ResourceChangeDataStaging_partition CHECK (Timestamp < CONVERT (DATETIME2 (7), N'9999-12-31 23:59:59.9999999')); + +ALTER TABLE dbo.ResourceChangeDataStaging CHECK CONSTRAINT CHK_ResourceChangeDataStaging_partition; + +CREATE TABLE dbo.ResourceChangeType ( + ResourceChangeTypeId TINYINT NOT NULL, + Name NVARCHAR (50) NOT NULL, + CONSTRAINT PK_ResourceChangeType PRIMARY KEY CLUSTERED (ResourceChangeTypeId), + CONSTRAINT UQ_ResourceChangeType_Name UNIQUE NONCLUSTERED (Name) +) ON [PRIMARY]; + + +GO +INSERT dbo.ResourceChangeType (ResourceChangeTypeId, Name) +VALUES (0, N'Creation'); + +INSERT dbo.ResourceChangeType (ResourceChangeTypeId, Name) +VALUES (1, N'Update'); + +INSERT dbo.ResourceChangeType (ResourceChangeTypeId, Name) +VALUES (2, N'Deletion'); + +CREATE TABLE dbo.ResourceType ( + ResourceTypeId SMALLINT IDENTITY (1, 1) NOT NULL, + Name NVARCHAR (50) COLLATE Latin1_General_100_CS_AS NOT NULL, + CONSTRAINT UQ_ResourceType_ResourceTypeId UNIQUE (ResourceTypeId), + CONSTRAINT PKC_ResourceType PRIMARY KEY CLUSTERED (Name) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE dbo.ResourceWriteClaim ( + ResourceSurrogateId BIGINT NOT NULL, + ClaimTypeId TINYINT NOT NULL, + ClaimValue NVARCHAR (128) NOT NULL +) +WITH (DATA_COMPRESSION = PAGE); + +CREATE CLUSTERED INDEX IXC_ResourceWriteClaim + ON dbo.ResourceWriteClaim(ResourceSurrogateId, ClaimTypeId); + +CREATE TABLE dbo.SchemaMigrationProgress ( + Timestamp DATETIME2 (3) DEFAULT CURRENT_TIMESTAMP, + Message NVARCHAR (MAX) +); + +CREATE TABLE dbo.SearchParam ( + SearchParamId SMALLINT IDENTITY (1, 1) NOT NULL, + Uri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + Status VARCHAR (20) NULL, + LastUpdated DATETIMEOFFSET (7) NULL, + IsPartiallySupported BIT NULL, + CONSTRAINT UQ_SearchParam_SearchParamId UNIQUE (SearchParamId), + CONSTRAINT PKC_SearchParam PRIMARY KEY CLUSTERED (Uri) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE dbo.StringSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL, + IsHistory BIT NOT NULL, + IsMin BIT CONSTRAINT string_IsMin_Constraint DEFAULT 0 NOT NULL, + IsMax BIT CONSTRAINT string_IsMax_Constraint DEFAULT 0 NOT NULL +); + +ALTER TABLE dbo.StringSearchParam + ADD CONSTRAINT DF_StringSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.StringSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_StringSearchParam + ON dbo.StringSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_StringSearchParam_SearchParamId_Text + ON dbo.StringSearchParam(ResourceTypeId, SearchParamId, Text, ResourceSurrogateId) + INCLUDE(TextOverflow, IsMin, IsMax) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_StringSearchParam_SearchParamId_TextWithOverflow + ON dbo.StringSearchParam(ResourceTypeId, SearchParamId, Text, ResourceSurrogateId) + INCLUDE(IsMin, IsMax) WHERE IsHistory = 0 + AND TextOverflow IS NOT NULL WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.System ( + SystemId INT IDENTITY (1, 1) NOT NULL, + Value NVARCHAR (256) NOT NULL, + CONSTRAINT UQ_System_SystemId UNIQUE (SystemId), + CONSTRAINT PKC_System PRIMARY KEY CLUSTERED (Value) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE [dbo].[TaskInfo] ( + [TaskId] VARCHAR (64) NOT NULL, + [QueueId] VARCHAR (64) NOT NULL, + [Status] SMALLINT NOT NULL, + [TaskTypeId] SMALLINT NOT NULL, + [RunId] VARCHAR (50) NULL, + [IsCanceled] BIT NOT NULL, + [RetryCount] SMALLINT NOT NULL, + [MaxRetryCount] SMALLINT NOT NULL, + [HeartbeatDateTime] DATETIME2 (7) NULL, + [InputData] VARCHAR (MAX) NOT NULL, + [TaskContext] VARCHAR (MAX) NULL, + [Result] VARCHAR (MAX) NULL, + [CreateDateTime] DATETIME2 (7) CONSTRAINT DF_TaskInfo_CreateDate DEFAULT SYSUTCDATETIME() NOT NULL, + [StartDateTime] DATETIME2 (7) NULL, + [EndDateTime] DATETIME2 (7) NULL, + [Worker] VARCHAR (100) NULL, + [RestartInfo] VARCHAR (MAX) NULL, + [ParentTaskId] VARCHAR (64) NULL, + CONSTRAINT PKC_TaskInfo PRIMARY KEY CLUSTERED (TaskId) WITH (DATA_COMPRESSION = PAGE) +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]; + + +GO +CREATE NONCLUSTERED INDEX IX_QueueId_Status + ON dbo.TaskInfo(QueueId, Status); + + +GO +CREATE NONCLUSTERED INDEX IX_QueueId_ParentTaskId + ON dbo.TaskInfo(QueueId, ParentTaskId); + +CREATE TABLE dbo.TokenDateTimeCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + StartDateTime2 DATETIME2 (7) NOT NULL, + EndDateTime2 DATETIME2 (7) NOT NULL, + IsLongerThanADay2 BIT NOT NULL, + IsHistory BIT NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenDateTimeCompositeSearchParam + ADD CONSTRAINT DF_TokenDateTimeCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenDateTimeCompositeSearchParam + ADD CONSTRAINT CHK_TokenDateTimeCompositeSearchParam_CodeOverflow1 CHECK (LEN(Code1) = 256 + OR CodeOverflow1 IS NULL); + +ALTER TABLE dbo.TokenDateTimeCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenDateTimeCompositeSearchParam + ON dbo.TokenDateTimeCompositeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenDateTimeCompositeSearchParam_Code1_StartDateTime2_EndDateTime2 + ON dbo.TokenDateTimeCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, StartDateTime2, EndDateTime2, ResourceSurrogateId) + INCLUDE(SystemId1, IsLongerThanADay2) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenDateTimeCompositeSearchParam_Code1_EndDateTime2_StartDateTime2 + ON dbo.TokenDateTimeCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, EndDateTime2, StartDateTime2, ResourceSurrogateId) + INCLUDE(SystemId1, IsLongerThanADay2) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenDateTimeCompositeSearchParam_Code1_StartDateTime2_EndDateTime2_Long + ON dbo.TokenDateTimeCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, StartDateTime2, EndDateTime2, ResourceSurrogateId) + INCLUDE(SystemId1) WHERE IsHistory = 0 + AND IsLongerThanADay2 = 1 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenDateTimeCompositeSearchParam_Code1_EndDateTime2_StartDateTime2_Long + ON dbo.TokenDateTimeCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, EndDateTime2, StartDateTime2, ResourceSurrogateId) + INCLUDE(SystemId1) WHERE IsHistory = 0 + AND IsLongerThanADay2 = 1 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenNumberNumberCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + SingleValue2 DECIMAL (36, 18) NULL, + LowValue2 DECIMAL (36, 18) NULL, + HighValue2 DECIMAL (36, 18) NULL, + SingleValue3 DECIMAL (36, 18) NULL, + LowValue3 DECIMAL (36, 18) NULL, + HighValue3 DECIMAL (36, 18) NULL, + HasRange BIT NOT NULL, + IsHistory BIT NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenNumberNumberCompositeSearchParam + ADD CONSTRAINT DF_TokenNumberNumberCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenNumberNumberCompositeSearchParam + ADD CONSTRAINT CHK_TokenNumberNumberCompositeSearchParam_CodeOverflow1 CHECK (LEN(Code1) = 256 + OR CodeOverflow1 IS NULL); + +ALTER TABLE dbo.TokenNumberNumberCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenNumberNumberCompositeSearchParam + ON dbo.TokenNumberNumberCompositeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenNumberNumberCompositeSearchParam_SearchParamId_Code1_Text2 + ON dbo.TokenNumberNumberCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, SingleValue2, SingleValue3, ResourceSurrogateId) + INCLUDE(SystemId1) WHERE IsHistory = 0 + AND HasRange = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenNumberNumberCompositeSearchParam_SearchParamId_Code1_LowValue2_HighValue2_LowValue3_HighValue3 + ON dbo.TokenNumberNumberCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, LowValue2, HighValue2, LowValue3, HighValue3, ResourceSurrogateId) + INCLUDE(SystemId1) WHERE IsHistory = 0 + AND HasRange = 1 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenQuantityCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + SystemId2 INT NULL, + QuantityCodeId2 INT NULL, + SingleValue2 DECIMAL (36, 18) NULL, + LowValue2 DECIMAL (36, 18) NULL, + HighValue2 DECIMAL (36, 18) NULL, + IsHistory BIT NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenQuantityCompositeSearchParam + ADD CONSTRAINT DF_TokenQuantityCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenQuantityCompositeSearchParam + ADD CONSTRAINT CHK_TokenQuantityCompositeSearchParam_CodeOverflow1 CHECK (LEN(Code1) = 256 + OR CodeOverflow1 IS NULL); + +ALTER TABLE dbo.TokenQuantityCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenQuantityCompositeSearchParam + ON dbo.TokenQuantityCompositeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenQuantityCompositeSearchParam_SearchParamId_Code1_QuantityCodeId2_SingleValue2 + ON dbo.TokenQuantityCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, SingleValue2, ResourceSurrogateId) + INCLUDE(QuantityCodeId2, SystemId1, SystemId2) WHERE IsHistory = 0 + AND SingleValue2 IS NOT NULL WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenQuantityCompositeSearchParam_SearchParamId_Code1_QuantityCodeId2_LowValue2_HighValue2 + ON dbo.TokenQuantityCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, LowValue2, HighValue2, ResourceSurrogateId) + INCLUDE(QuantityCodeId2, SystemId1, SystemId2) WHERE IsHistory = 0 + AND LowValue2 IS NOT NULL WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenQuantityCompositeSearchParam_SearchParamId_Code1_QuantityCodeId2_HighValue2_LowValue2 + ON dbo.TokenQuantityCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, HighValue2, LowValue2, ResourceSurrogateId) + INCLUDE(QuantityCodeId2, SystemId1, SystemId2) WHERE IsHistory = 0 + AND LowValue2 IS NOT NULL WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + Code VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + IsHistory BIT NOT NULL, + CodeOverflow VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenSearchParam + ADD CONSTRAINT DF_TokenSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenSearchParam + ADD CONSTRAINT CHK_TokenSearchParam_CodeOverflow CHECK (LEN(Code) = 256 + OR CodeOverflow IS NULL); + +ALTER TABLE dbo.TokenSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenSearchParam + ON dbo.TokenSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenSeachParam_SearchParamId_Code_SystemId + ON dbo.TokenSearchParam(ResourceTypeId, SearchParamId, Code, ResourceSurrogateId) + INCLUDE(SystemId) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenStringCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + Text2 NVARCHAR (256) COLLATE Latin1_General_CI_AI NOT NULL, + TextOverflow2 NVARCHAR (MAX) COLLATE Latin1_General_CI_AI NULL, + IsHistory BIT NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenStringCompositeSearchParam + ADD CONSTRAINT DF_TokenStringCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenStringCompositeSearchParam + ADD CONSTRAINT CHK_TokenStringCompositeSearchParam_CodeOverflow1 CHECK (LEN(Code1) = 256 + OR CodeOverflow1 IS NULL); + +ALTER TABLE dbo.TokenStringCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenStringCompositeSearchParam + ON dbo.TokenStringCompositeSearchParam(ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenStringCompositeSearchParam_SearchParamId_Code1_Text2 + ON dbo.TokenStringCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, Text2, ResourceSurrogateId) + INCLUDE(SystemId1, TextOverflow2) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenStringCompositeSearchParam_SearchParamId_Code1_Text2WithOverflow + ON dbo.TokenStringCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, Text2, ResourceSurrogateId) + INCLUDE(SystemId1) WHERE IsHistory = 0 + AND TextOverflow2 IS NOT NULL WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenText ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (400) COLLATE Latin1_General_CI_AI NOT NULL, + IsHistory BIT NOT NULL +); + +ALTER TABLE dbo.TokenText + ADD CONSTRAINT DF_TokenText_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenText SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenText + ON dbo.TokenText(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenText_SearchParamId_Text + ON dbo.TokenText(ResourceTypeId, SearchParamId, Text, ResourceSurrogateId) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenTokenCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + IsHistory BIT NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenTokenCompositeSearchParam + ADD CONSTRAINT DF_TokenTokenCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenTokenCompositeSearchParam + ADD CONSTRAINT CHK_TokenTokenCompositeSearchParam_CodeOverflow1 CHECK (LEN(Code1) = 256 + OR CodeOverflow1 IS NULL); + +ALTER TABLE dbo.TokenTokenCompositeSearchParam + ADD CONSTRAINT CHK_TokenTokenCompositeSearchParam_CodeOverflow2 CHECK (LEN(Code2) = 256 + OR CodeOverflow2 IS NULL); + +ALTER TABLE dbo.TokenTokenCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenTokenCompositeSearchParam + ON dbo.TokenTokenCompositeSearchParam(ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenTokenCompositeSearchParam_Code1_Code2 + ON dbo.TokenTokenCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, Code2, ResourceSurrogateId) + INCLUDE(SystemId1, SystemId2) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.Transactions ( + SurrogateIdRangeFirstValue BIGINT NOT NULL, + SurrogateIdRangeLastValue BIGINT NOT NULL, + Definition VARCHAR (2000) NULL, + IsCompleted BIT CONSTRAINT DF_Transactions_IsCompleted DEFAULT 0 NOT NULL, + IsSuccess BIT CONSTRAINT DF_Transactions_IsSuccess DEFAULT 0 NOT NULL, + IsVisible BIT CONSTRAINT DF_Transactions_IsVisible DEFAULT 0 NOT NULL, + IsHistoryMoved BIT CONSTRAINT DF_Transactions_IsHistoryMoved DEFAULT 0 NOT NULL, + CreateDate DATETIME CONSTRAINT DF_Transactions_CreateDate DEFAULT getUTCdate() NOT NULL, + EndDate DATETIME NULL, + VisibleDate DATETIME NULL, + HistoryMovedDate DATETIME NULL, + HeartbeatDate DATETIME CONSTRAINT DF_Transactions_HeartbeatDate DEFAULT getUTCdate() NOT NULL, + FailureReason VARCHAR (MAX) NULL, + IsControlledByClient BIT CONSTRAINT DF_Transactions_IsControlledByClient DEFAULT 1 NOT NULL, + InvisibleHistoryRemovedDate DATETIME NULL CONSTRAINT PKC_Transactions_SurrogateIdRangeFirstValue PRIMARY KEY CLUSTERED (SurrogateIdRangeFirstValue) +); + +CREATE INDEX IX_IsVisible + ON dbo.Transactions(IsVisible); + +CREATE TABLE dbo.UriSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Uri VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + IsHistory BIT NOT NULL +); + +ALTER TABLE dbo.UriSearchParam + ADD CONSTRAINT DF_UriSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.UriSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_UriSearchParam + ON dbo.UriSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_UriSearchParam_SearchParamId_Uri + ON dbo.UriSearchParam(ResourceTypeId, SearchParamId, Uri, ResourceSurrogateId) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.WatchdogLeases ( + Watchdog VARCHAR (100) NOT NULL, + LeaseHolder VARCHAR (100) CONSTRAINT DF_WatchdogLeases_LeaseHolder DEFAULT '' NOT NULL, + LeaseEndTime DATETIME CONSTRAINT DF_WatchdogLeases_LeaseEndTime DEFAULT 0 NOT NULL, + RemainingLeaseTimeSec AS datediff(second, getUTCdate(), LeaseEndTime), + LeaseRequestor VARCHAR (100) CONSTRAINT DF_WatchdogLeases_LeaseRequestor DEFAULT '' NOT NULL, + LeaseRequestTime DATETIME CONSTRAINT DF_WatchdogLeases_LeaseRequestTime DEFAULT 0 NOT NULL CONSTRAINT PKC_WatchdogLeases_Watchdog PRIMARY KEY CLUSTERED (Watchdog) +); + +COMMIT +GO +CREATE PROCEDURE dbo.AcquireExportJobs +@jobHeartbeatTimeoutThresholdInSeconds BIGINT, @maximumNumberOfConcurrentJobsAllowed INT +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DECLARE @expirationDateTime AS DATETIME2 (7); +SELECT @expirationDateTime = DATEADD(second, -@jobHeartbeatTimeoutThresholdInSeconds, SYSUTCDATETIME()); +DECLARE @numberOfRunningJobs AS INT; +SELECT @numberOfRunningJobs = COUNT(*) +FROM dbo.ExportJob WITH (TABLOCKX) +WHERE Status = 'Running' + AND HeartbeatDateTime > @expirationDateTime; +DECLARE @limit AS INT = @maximumNumberOfConcurrentJobsAllowed - @numberOfRunningJobs; +IF (@limit > 0) + BEGIN + DECLARE @availableJobs TABLE ( + Id VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + JobVersion BINARY (8) NOT NULL); + INSERT INTO @availableJobs + SELECT TOP (@limit) Id, + JobVersion + FROM dbo.ExportJob + WHERE (Status = 'Queued' + OR (Status = 'Running' + AND HeartbeatDateTime <= @expirationDateTime)) + ORDER BY HeartbeatDateTime; + DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); + UPDATE dbo.ExportJob + SET Status = 'Running', + HeartbeatDateTime = @heartbeatDateTime, + RawJobRecord = JSON_MODIFY(RawJobRecord, '$.status', 'Running') + OUTPUT inserted.RawJobRecord, inserted.JobVersion + FROM dbo.ExportJob AS job + INNER JOIN + @availableJobs AS availableJob + ON job.Id = availableJob.Id + AND job.JobVersion = availableJob.JobVersion; + END +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.AcquireReindexJobs +@jobHeartbeatTimeoutThresholdInSeconds BIGINT, @maximumNumberOfConcurrentJobsAllowed INT +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DECLARE @expirationDateTime AS DATETIME2 (7); +SELECT @expirationDateTime = DATEADD(second, -@jobHeartbeatTimeoutThresholdInSeconds, SYSUTCDATETIME()); +DECLARE @numberOfRunningJobs AS INT; +SELECT @numberOfRunningJobs = COUNT(*) +FROM dbo.ReindexJob WITH (TABLOCKX) +WHERE Status = 'Running' + AND HeartbeatDateTime > @expirationDateTime; +DECLARE @limit AS INT = @maximumNumberOfConcurrentJobsAllowed - @numberOfRunningJobs; +IF (@limit > 0) + BEGIN + DECLARE @availableJobs TABLE ( + Id VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + JobVersion BINARY (8) NOT NULL); + INSERT INTO @availableJobs + SELECT TOP (@limit) Id, + JobVersion + FROM dbo.ReindexJob + WHERE (Status = 'Queued' + OR (Status = 'Running' + AND HeartbeatDateTime <= @expirationDateTime)) + ORDER BY HeartbeatDateTime; + DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); + UPDATE dbo.ReindexJob + SET Status = 'Running', + HeartbeatDateTime = @heartbeatDateTime, + RawJobRecord = JSON_MODIFY(RawJobRecord, '$.status', 'Running') + OUTPUT inserted.RawJobRecord, inserted.JobVersion + FROM dbo.ReindexJob AS job + INNER JOIN + @availableJobs AS availableJob + ON job.Id = availableJob.Id + AND job.JobVersion = availableJob.JobVersion; + END +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.AcquireWatchdogLease +@Watchdog VARCHAR (100), @Worker VARCHAR (100), @AllowRebalance BIT=1, @ForceAcquire BIT=0, @LeasePeriodSec FLOAT, @WorkerIsRunning BIT=0, @LeaseEndTime DATETIME OUTPUT, @IsAcquired BIT OUTPUT, @CurrentLeaseHolder VARCHAR (100)=NULL OUTPUT +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +DECLARE @SP AS VARCHAR (100) = 'AcquireWatchdogLease', @Mode AS VARCHAR (100), @msg AS VARCHAR (1000), @MyLeasesNumber AS INT, @OtherValidRequestsOrLeasesNumber AS INT, @MyValidRequestsOrLeasesNumber AS INT, @DesiredLeasesNumber AS INT, @NotLeasedWatchdogNumber AS INT, @WatchdogNumber AS INT, @Now AS DATETIME, @MyLastChangeTime AS DATETIME, @PreviousLeaseHolder AS VARCHAR (100), @Rows AS INT = 0, @NumberOfWorkers AS INT, @st AS DATETIME = getUTCdate(), @RowsInt AS INT, @Pattern AS VARCHAR (100); +BEGIN TRY + SET @Mode = 'R=' + isnull(@Watchdog, 'NULL') + ' W=' + isnull(@Worker, 'NULL') + ' F=' + isnull(CONVERT (VARCHAR, @ForceAcquire), 'NULL') + ' LP=' + isnull(CONVERT (VARCHAR, @LeasePeriodSec), 'NULL'); + SET @CurrentLeaseHolder = ''; + SET @IsAcquired = 0; + SET @Now = getUTCdate(); + SET @LeaseEndTime = @Now; + SET @Pattern = NULLIF ((SELECT Char + FROM dbo.Parameters + WHERE Id = 'WatchdogLeaseHolderIncludePatternFor' + @Watchdog), ''); + IF @Pattern IS NULL + SET @Pattern = NULLIF ((SELECT Char + FROM dbo.Parameters + WHERE Id = 'WatchdogLeaseHolderIncludePattern'), ''); + IF @Pattern IS NOT NULL + AND @Worker NOT LIKE @Pattern + BEGIN + SET @msg = 'Worker does not match include pattern=' + @Pattern; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows, @Text = @msg; + SET @CurrentLeaseHolder = isnull((SELECT LeaseHolder + FROM dbo.WatchdogLeases + WHERE Watchdog = @Watchdog), ''); + RETURN; + END + SET @Pattern = NULLIF ((SELECT Char + FROM dbo.Parameters + WHERE Id = 'WatchdogLeaseHolderExcludePatternFor' + @Watchdog), ''); + IF @Pattern IS NULL + SET @Pattern = NULLIF ((SELECT Char + FROM dbo.Parameters + WHERE Id = 'WatchdogLeaseHolderExcludePattern'), ''); + IF @Pattern IS NOT NULL + AND @Worker LIKE @Pattern + BEGIN + SET @msg = 'Worker matches exclude pattern=' + @Pattern; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows, @Text = @msg; + SET @CurrentLeaseHolder = isnull((SELECT LeaseHolder + FROM dbo.WatchdogLeases + WHERE Watchdog = @Watchdog), ''); + RETURN; + END + DECLARE @Watchdogs TABLE ( + Watchdog VARCHAR (100) PRIMARY KEY); + INSERT INTO @Watchdogs + SELECT Watchdog + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE RemainingLeaseTimeSec * (-1) > 10 * @LeasePeriodSec + OR @ForceAcquire = 1 + AND Watchdog = @Watchdog + AND LeaseHolder <> @Worker; + IF @@rowcount > 0 + BEGIN + DELETE dbo.WatchdogLeases + WHERE Watchdog IN (SELECT Watchdog + FROM @Watchdogs); + SET @Rows += @@rowcount; + IF @Rows > 0 + BEGIN + SET @msg = ''; + SELECT @msg = CONVERT (VARCHAR (1000), @msg + CASE WHEN @msg = '' THEN '' ELSE ',' END + Watchdog) + FROM @Watchdogs; + SET @msg = CONVERT (VARCHAR (1000), 'Remove old/forced leases:' + @msg); + EXECUTE dbo.LogEvent @Process = 'AcquireWatchdogLease', @Status = 'Info', @Mode = @Mode, @Target = 'WatchdogLeases', @Action = 'Delete', @Rows = @Rows, @Text = @msg; + END + END + SET @NumberOfWorkers = 1 + (SELECT count(*) + FROM (SELECT LeaseHolder + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseHolder <> @Worker + UNION + SELECT LeaseRequestor + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseRequestor <> @Worker + AND LeaseRequestor <> '') AS A); + SET @Mode = CONVERT (VARCHAR (100), @Mode + ' N=' + CONVERT (VARCHAR (10), @NumberOfWorkers)); + IF NOT EXISTS (SELECT * + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE Watchdog = @Watchdog) + INSERT INTO dbo.WatchdogLeases (Watchdog, LeaseEndTime, LeaseRequestTime) + SELECT @Watchdog, + dateadd(day, -10, @Now), + dateadd(day, -10, @Now) + WHERE NOT EXISTS (SELECT * + FROM dbo.WatchdogLeases WITH (TABLOCKX) + WHERE Watchdog = @Watchdog); + SET @LeaseEndTime = dateadd(second, @LeasePeriodSec, @Now); + SET @WatchdogNumber = (SELECT count(*) + FROM dbo.WatchdogLeases WITH (NOLOCK)); + SET @NotLeasedWatchdogNumber = (SELECT count(*) + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseHolder = '' + OR LeaseEndTime < @Now); + SET @MyLeasesNumber = (SELECT count(*) + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseHolder = @Worker + AND LeaseEndTime > @Now); + SET @OtherValidRequestsOrLeasesNumber = (SELECT count(*) + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseHolder <> @Worker + AND LeaseEndTime > @Now + OR LeaseRequestor <> @Worker + AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec); + SET @MyValidRequestsOrLeasesNumber = (SELECT count(*) + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseHolder = @Worker + AND LeaseEndTime > @Now + OR LeaseRequestor = @Worker + AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec); + SET @DesiredLeasesNumber = ceiling(1.0 * @WatchdogNumber / @NumberOfWorkers); + IF @DesiredLeasesNumber = 0 + SET @DesiredLeasesNumber = 1; + IF @DesiredLeasesNumber = 1 + AND @OtherValidRequestsOrLeasesNumber = 1 + AND @WatchdogNumber = 1 + SET @DesiredLeasesNumber = 0; + IF @MyValidRequestsOrLeasesNumber = floor(1.0 * @WatchdogNumber / @NumberOfWorkers) + AND @OtherValidRequestsOrLeasesNumber + @MyValidRequestsOrLeasesNumber = @WatchdogNumber + SET @DesiredLeasesNumber = @DesiredLeasesNumber - 1; + UPDATE dbo.WatchdogLeases + SET LeaseHolder = @Worker, + LeaseEndTime = @LeaseEndTime, + LeaseRequestor = '', + @PreviousLeaseHolder = LeaseHolder + WHERE Watchdog = @Watchdog + AND NOT (LeaseRequestor <> @Worker + AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec) + AND (LeaseHolder = @Worker + AND (LeaseEndTime > @Now + OR @WorkerIsRunning = 1) + OR LeaseEndTime < @Now + AND (@DesiredLeasesNumber > @MyLeasesNumber + OR @OtherValidRequestsOrLeasesNumber < @WatchdogNumber)); + IF @@rowcount > 0 + BEGIN + SET @IsAcquired = 1; + SET @msg = 'Lease holder changed from [' + isnull(@PreviousLeaseHolder, '') + '] to [' + @Worker + ']'; + IF @PreviousLeaseHolder <> @Worker + EXECUTE dbo.LogEvent @Process = 'AcquireWatchdogLease', @Status = 'Info', @Mode = @Mode, @Text = @msg; + END + ELSE + IF @AllowRebalance = 1 + BEGIN + SET @CurrentLeaseHolder = (SELECT LeaseHolder + FROM dbo.WatchdogLeases + WHERE Watchdog = @Watchdog); + UPDATE dbo.WatchdogLeases + SET LeaseRequestTime = @Now + WHERE Watchdog = @Watchdog + AND LeaseRequestor = @Worker + AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec; + IF @DesiredLeasesNumber > @MyValidRequestsOrLeasesNumber + BEGIN + UPDATE A + SET LeaseRequestor = @Worker, + LeaseRequestTime = @Now + FROM dbo.WatchdogLeases AS A + WHERE Watchdog = @Watchdog + AND NOT (LeaseRequestor <> @Worker + AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec) + AND @NotLeasedWatchdogNumber = 0 + AND (SELECT count(*) + FROM dbo.WatchdogLeases AS B + WHERE B.LeaseHolder = A.LeaseHolder + AND datediff(second, B.LeaseEndTime, @Now) < @LeasePeriodSec) > @DesiredLeasesNumber; + SET @RowsInt = @@rowcount; + SET @msg = '@DesiredLeasesNumber=[' + CONVERT (VARCHAR (10), @DesiredLeasesNumber) + '] > @MyValidRequestsOrLeasesNumber=[' + CONVERT (VARCHAR (10), @MyValidRequestsOrLeasesNumber) + ']'; + EXECUTE dbo.LogEvent @Process = 'AcquireWatchdogLease', @Status = 'Info', @Mode = @Mode, @Rows = @RowsInt, @Text = @msg; + END + END + SET @Mode = CONVERT (VARCHAR (100), @Mode + ' A=' + CONVERT (VARCHAR (1), @IsAcquired)); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = 'AcquireWatchdogLease', @Status = 'Error', @Mode = @Mode; + THROW; +END CATCH + +GO +CREATE OR ALTER PROCEDURE dbo.AddPartitionOnResourceChanges +@partitionBoundary DATETIME2 (7) OUTPUT +AS +BEGIN + SET XACT_ABORT ON; + BEGIN TRANSACTION; + DECLARE @rightPartitionBoundary AS DATETIME2 (7) = CAST ((SELECT TOP (1) value + FROM sys.partition_range_values AS prv + INNER JOIN + sys.partition_functions AS pf + ON pf.function_id = prv.function_id + WHERE pf.name = N'PartitionFunction_ResourceChangeData_Timestamp' + ORDER BY prv.boundary_id DESC) AS DATETIME2 (7)); + DECLARE @timestamp AS DATETIME2 (7) = DATEADD(hour, DATEDIFF(hour, 0, sysutcdatetime()), 0); + IF (@rightPartitionBoundary < @timestamp) + BEGIN + SET @rightPartitionBoundary = @timestamp; + END + SET @rightPartitionBoundary = DATEADD(hour, 1, @rightPartitionBoundary); + ALTER PARTITION SCHEME PartitionScheme_ResourceChangeData_Timestamp NEXT USED [Primary]; + ALTER PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp( ) + SPLIT RANGE (@rightPartitionBoundary); + SET @partitionBoundary = @rightPartitionBoundary; + COMMIT TRANSACTION; +END + +GO +CREATE PROCEDURE dbo.ArchiveJobs +@QueueType TINYINT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'ArchiveJobs', @Mode AS VARCHAR (100) = '', @st AS DATETIME = getUTCdate(), @Rows AS INT = 0, @PartitionId AS TINYINT, @MaxPartitions AS TINYINT = 16, @LookedAtPartitions AS TINYINT = 0, @InflightRows AS INT = 0, @Lock AS VARCHAR (100) = 'DequeueJob_' + CONVERT (VARCHAR, @QueueType); +BEGIN TRY + SET @PartitionId = @MaxPartitions * rand(); + BEGIN TRANSACTION; + EXECUTE sp_getapplock @Lock, 'Exclusive'; + WHILE @LookedAtPartitions <= @MaxPartitions + BEGIN + SET @InflightRows += (SELECT count(*) + FROM dbo.JobQueue + WHERE PartitionId = @PartitionId + AND QueueType = @QueueType + AND Status IN (0, 1)); + SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; + SET @LookedAtPartitions = @LookedAtPartitions + 1; + END + IF @InflightRows = 0 + BEGIN + SET @LookedAtPartitions = 0; + WHILE @LookedAtPartitions <= @MaxPartitions + BEGIN + UPDATE dbo.JobQueue + SET Status = 5 + WHERE PartitionId = @PartitionId + AND QueueType = @QueueType + AND Status IN (2, 3, 4); + SET @Rows += @@rowcount; + SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; + SET @LookedAtPartitions = @LookedAtPartitions + 1; + END + END + COMMIT TRANSACTION; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.BatchDeleteResourceParams +@tableName NVARCHAR (128), @resourceTypeId SMALLINT, @startResourceSurrogateId BIGINT, @endResourceSurrogateId BIGINT, @batchSize INT +AS +SET XACT_ABORT ON; +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DECLARE @Sql AS NVARCHAR (MAX); +DECLARE @ParmDefinition AS NVARCHAR (512); +IF OBJECT_ID(@tableName) IS NOT NULL + BEGIN + SET @sql = N'DELETE TOP(@BatchSizeParam) FROM ' + @tableName + N' WITH (TABLOCK) WHERE ResourceTypeId = @ResourceTypeIdParam AND ResourceSurrogateId >= @StartResourceSurrogateIdParam AND ResourceSurrogateId < @EndResourceSurrogateIdParam'; + SET @parmDefinition = N'@BatchSizeParam int, @ResourceTypeIdParam smallint, @StartResourceSurrogateIdParam bigint, @EndResourceSurrogateIdParam bigint'; + EXECUTE sp_executesql @sql, @parmDefinition, @BatchSizeParam = @batchSize, @ResourceTypeIdParam = @resourceTypeId, @StartResourceSurrogateIdParam = @startResourceSurrogateId, @EndResourceSurrogateIdParam = @endResourceSurrogateId; + END +COMMIT TRANSACTION; +RETURN @@rowcount; + +GO +CREATE PROCEDURE dbo.BatchDeleteResources +@resourceTypeId SMALLINT, @startResourceSurrogateId BIGINT, @endResourceSurrogateId BIGINT, @batchSize INT +AS +SET XACT_ABORT ON; +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DELETE TOP (@batchSize) + dbo.Resource WITH (TABLOCK) +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId >= @startResourceSurrogateId + AND ResourceSurrogateId < @endResourceSurrogateId; +COMMIT TRANSACTION; +RETURN @@rowcount; + +GO +CREATE PROCEDURE dbo.BatchDeleteResourceWriteClaims +@startResourceSurrogateId BIGINT, @endResourceSurrogateId BIGINT, @batchSize INT +AS +SET XACT_ABORT ON; +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DELETE TOP (@batchSize) + dbo.ResourceWriteClaim WITH (TABLOCK) +WHERE ResourceSurrogateId >= @startResourceSurrogateId + AND ResourceSurrogateId < @endResourceSurrogateId; +COMMIT TRANSACTION; +RETURN @@rowcount; + +GO +CREATE PROCEDURE dbo.BulkMergeResource +@resources dbo.BulkImportResourceType_1 READONLY +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +MERGE INTO [dbo].[Resource] WITH (ROWLOCK, INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) + AS target +USING @resources AS source ON source.[ResourceTypeId] = target.[ResourceTypeId] + AND source.[ResourceId] = target.[ResourceId] + AND source.[Version] = target.[Version] +WHEN NOT MATCHED BY TARGET THEN INSERT ([ResourceTypeId], [ResourceId], [Version], [IsHistory], [ResourceSurrogateId], [IsDeleted], [RequestMethod], [RawResource], [IsRawResourceMetaSet], [SearchParamHash]) VALUES ([ResourceTypeId], [ResourceId], [Version], [IsHistory], [ResourceSurrogateId], [IsDeleted], [RequestMethod], [RawResource], [IsRawResourceMetaSet], [SearchParamHash]) OUTPUT Inserted.[ResourceSurrogateId]; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.BulkReindexResources_2 +@resourcesToReindex dbo.BulkReindexResourceTableType_1 READONLY, @resourceWriteClaims dbo.BulkResourceWriteClaimTableType_1 READONLY, @compartmentAssignments dbo.BulkCompartmentAssignmentTableType_1 READONLY, @referenceSearchParams dbo.BulkReferenceSearchParamTableType_1 READONLY, @tokenSearchParams dbo.BulkTokenSearchParamTableType_2 READONLY, @tokenTextSearchParams dbo.BulkTokenTextTableType_1 READONLY, @stringSearchParams dbo.BulkStringSearchParamTableType_2 READONLY, @numberSearchParams dbo.BulkNumberSearchParamTableType_1 READONLY, @quantitySearchParams dbo.BulkQuantitySearchParamTableType_1 READONLY, @uriSearchParams dbo.BulkUriSearchParamTableType_1 READONLY, @dateTimeSearchParms dbo.BulkDateTimeSearchParamTableType_2 READONLY, @referenceTokenCompositeSearchParams dbo.BulkReferenceTokenCompositeSearchParamTableType_2 READONLY, @tokenTokenCompositeSearchParams dbo.BulkTokenTokenCompositeSearchParamTableType_2 READONLY, @tokenDateTimeCompositeSearchParams dbo.BulkTokenDateTimeCompositeSearchParamTableType_2 READONLY, @tokenQuantityCompositeSearchParams dbo.BulkTokenQuantityCompositeSearchParamTableType_2 READONLY, @tokenStringCompositeSearchParams dbo.BulkTokenStringCompositeSearchParamTableType_2 READONLY, @tokenNumberNumberCompositeSearchParams dbo.BulkTokenNumberNumberCompositeSearchParamTableType_2 READONLY +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @computedValues TABLE ( + Offset INT NOT NULL, + ResourceTypeId SMALLINT NOT NULL, + VersionProvided BIGINT NULL, + SearchParamHash VARCHAR (64) NOT NULL, + ResourceSurrogateId BIGINT NULL, + VersionInDatabase BIGINT NULL); +INSERT INTO @computedValues +SELECT resourceToReindex.Offset, + resourceToReindex.ResourceTypeId, + resourceToReindex.ETag, + resourceToReindex.SearchParamHash, + resourceInDB.ResourceSurrogateId, + resourceInDB.Version +FROM @resourcesToReindex AS resourceToReindex + LEFT OUTER JOIN + dbo.Resource AS resourceInDB WITH (UPDLOCK, INDEX (IX_Resource_ResourceTypeId_ResourceId)) + ON resourceInDB.ResourceTypeId = resourceToReindex.ResourceTypeId + AND resourceInDB.ResourceId = resourceToReindex.ResourceId + AND resourceInDB.IsHistory = 0; +DECLARE @versionDiff AS INT; +SET @versionDiff = (SELECT COUNT(*) + FROM @computedValues + WHERE VersionProvided IS NOT NULL + AND VersionProvided <> VersionInDatabase); +IF (@versionDiff > 0) + BEGIN + DELETE @computedValues + WHERE VersionProvided IS NOT NULL + AND VersionProvided <> VersionInDatabase; + END +UPDATE resourceInDB +SET resourceInDB.SearchParamHash = resourceToReindex.SearchParamHash +FROM @computedValues AS resourceToReindex + INNER JOIN + dbo.Resource AS resourceInDB + ON resourceInDB.ResourceTypeId = resourceToReindex.ResourceTypeId + AND resourceInDB.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.ResourceWriteClaim AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.CompartmentAssignment AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.ReferenceSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenText AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.StringSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.UriSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.NumberSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.QuantitySearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.DateTimeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.ReferenceTokenCompositeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenTokenCompositeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenDateTimeCompositeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenQuantityCompositeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenStringCompositeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenNumberNumberCompositeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) +SELECT DISTINCT resourceToReindex.ResourceSurrogateId, + searchIndex.ClaimTypeId, + searchIndex.ClaimValue +FROM @resourceWriteClaims AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.CompartmentAssignment (ResourceTypeId, ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.CompartmentTypeId, + searchIndex.ReferenceResourceId, + 0 +FROM @compartmentAssignments AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.BaseUri, + searchIndex.ReferenceResourceTypeId, + searchIndex.ReferenceResourceId, + searchIndex.ReferenceResourceVersion, + 0 +FROM @referenceSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId, + searchIndex.Code, + searchIndex.CodeOverflow, + 0 +FROM @tokenSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.Text, + 0 +FROM @tokenTextSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsHistory, IsMin, IsMax) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.Text, + searchIndex.TextOverflow, + 0, + searchIndex.IsMin, + searchIndex.IsMax +FROM @stringSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.Uri, + 0 +FROM @uriSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SingleValue, + searchIndex.LowValue, + searchIndex.HighValue, + 0 +FROM @numberSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId, + searchIndex.QuantityCodeId, + searchIndex.SingleValue, + searchIndex.LowValue, + searchIndex.HighValue, + 0 +FROM @quantitySearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsHistory, IsMin, IsMax) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.StartDateTime, + searchIndex.EndDateTime, + searchIndex.IsLongerThanADay, + 0, + searchIndex.IsMin, + searchIndex.IsMax +FROM @dateTimeSearchParms AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.BaseUri1, + searchIndex.ReferenceResourceTypeId1, + searchIndex.ReferenceResourceId1, + searchIndex.ReferenceResourceVersion1, + searchIndex.SystemId2, + searchIndex.Code2, + searchIndex.CodeOverflow2, + 0 +FROM @referenceTokenCompositeSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId1, + searchIndex.Code1, + searchIndex.CodeOverflow1, + searchIndex.SystemId2, + searchIndex.Code2, + searchIndex.CodeOverflow2, + 0 +FROM @tokenTokenCompositeSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId1, + searchIndex.Code1, + searchIndex.CodeOverflow1, + searchIndex.StartDateTime2, + searchIndex.EndDateTime2, + searchIndex.IsLongerThanADay2, + 0 +FROM @tokenDateTimeCompositeSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId1, + searchIndex.Code1, + searchIndex.CodeOverflow1, + searchIndex.SingleValue2, + searchIndex.SystemId2, + searchIndex.QuantityCodeId2, + searchIndex.LowValue2, + searchIndex.HighValue2, + 0 +FROM @tokenQuantityCompositeSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId1, + searchIndex.Code1, + searchIndex.CodeOverflow1, + searchIndex.Text2, + searchIndex.TextOverflow2, + 0 +FROM @tokenStringCompositeSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId1, + searchIndex.Code1, + searchIndex.CodeOverflow1, + searchIndex.SingleValue2, + searchIndex.LowValue2, + searchIndex.HighValue2, + searchIndex.SingleValue3, + searchIndex.LowValue3, + searchIndex.HighValue3, + searchIndex.HasRange, + 0 +FROM @tokenNumberNumberCompositeSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +SELECT @versionDiff; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE [dbo].[CancelTask] +@taskId VARCHAR (64) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +IF NOT EXISTS (SELECT * + FROM [dbo].[TaskInfo] + WHERE TaskId = @taskId) + BEGIN + THROW 50404, 'Task not exist', 1; + END +UPDATE dbo.TaskInfo +SET IsCanceled = 1, + HeartbeatDateTime = @heartbeatDateTime +WHERE TaskId = @taskId; +SELECT TaskId, + QueueId, + Status, + TaskTypeId, + RunId, + IsCanceled, + RetryCount, + MaxRetryCount, + HeartbeatDateTime, + InputData, + TaskContext, + Result +FROM [dbo].[TaskInfo] +WHERE TaskId = @taskId; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.CaptureResourceChanges +@isDeleted BIT, @version INT, @resourceId VARCHAR (64), @resourceTypeId SMALLINT +AS +BEGIN + DECLARE @changeType AS SMALLINT; + IF (@isDeleted = 1) + BEGIN + SET @changeType = 2; + END + ELSE + BEGIN + IF (@version = 1) + BEGIN + SET @changeType = 0; + END + ELSE + BEGIN + SET @changeType = 1; + END + END + INSERT INTO dbo.ResourceChangeData (ResourceId, ResourceTypeId, ResourceVersion, ResourceChangeTypeId) + VALUES (@resourceId, @resourceTypeId, @version, @changeType); +END + +GO +CREATE PROCEDURE dbo.CaptureResourceIdsForChanges +@Resources dbo.ResourceList READONLY +AS +SET NOCOUNT ON; +INSERT INTO dbo.ResourceChangeData (ResourceId, ResourceTypeId, ResourceVersion, ResourceChangeTypeId) +SELECT ResourceId, + ResourceTypeId, + Version, + CASE WHEN IsDeleted = 1 THEN 2 WHEN Version > 1 THEN 1 ELSE 0 END +FROM @Resources +WHERE IsHistory = 0; + +GO +CREATE PROCEDURE dbo.CheckActiveReindexJobs +AS +SET NOCOUNT ON; +SELECT Id +FROM dbo.ReindexJob +WHERE Status = 'Running' + OR Status = 'Queued' + OR Status = 'Paused'; + +GO +CREATE PROCEDURE dbo.CleanupEventLog +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'CleanupEventLog', @Mode AS VARCHAR (100) = '', @MaxDeleteRows AS INT, @MaxAllowedRows AS BIGINT, @RetentionPeriodSecond AS INT, @DeletedRows AS INT, @TotalDeletedRows AS INT = 0, @TotalRows AS INT, @Now AS DATETIME = getUTCdate(); +EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; +BEGIN TRY + SET @MaxDeleteRows = (SELECT Number + FROM dbo.Parameters + WHERE Id = 'CleanupEventLog.DeleteBatchSize'); + IF @MaxDeleteRows IS NULL + RAISERROR ('Cannot get Parameter.CleanupEventLog.DeleteBatchSize', 18, 127); + SET @MaxAllowedRows = (SELECT Number + FROM dbo.Parameters + WHERE Id = 'CleanupEventLog.AllowedRows'); + IF @MaxAllowedRows IS NULL + RAISERROR ('Cannot get Parameter.CleanupEventLog.AllowedRows', 18, 127); + SET @RetentionPeriodSecond = (SELECT Number * 24 * 60 * 60 + FROM dbo.Parameters + WHERE Id = 'CleanupEventLog.RetentionPeriodDay'); + IF @RetentionPeriodSecond IS NULL + RAISERROR ('Cannot get Parameter.CleanupEventLog.RetentionPeriodDay', 18, 127); + SET @TotalRows = (SELECT sum(row_count) + FROM sys.dm_db_partition_stats + WHERE object_id = object_id('EventLog') + AND index_id IN (0, 1)); + SET @DeletedRows = 1; + WHILE @DeletedRows > 0 + AND EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = 'CleanupEventLog.IsEnabled' + AND Number = 1) + BEGIN + SET @DeletedRows = 0; + IF @TotalRows - @TotalDeletedRows > @MaxAllowedRows + BEGIN + DELETE TOP (@MaxDeleteRows) + dbo.EventLog WITH (PAGLOCK) + WHERE EventDate <= dateadd(second, -@RetentionPeriodSecond, @Now); + SET @DeletedRows = @@rowcount; + SET @TotalDeletedRows += @DeletedRows; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'EventLog', @Action = 'Delete', @Rows = @DeletedRows, @Text = @TotalDeletedRows; + END + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @Now; +END TRY +BEGIN CATCH + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.CompleteTask +@taskId VARCHAR (64), @taskResult VARCHAR (MAX), @runId VARCHAR (50) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +IF NOT EXISTS (SELECT * + FROM [dbo].[TaskInfo] + WHERE TaskId = @taskId + AND RunId = @runId) + BEGIN + THROW 50404, 'Task not exist or runid not match', 1; + END +UPDATE dbo.TaskInfo +SET Status = 3, + EndDateTime = SYSUTCDATETIME(), + Result = @taskResult +WHERE TaskId = @taskId; +COMMIT TRANSACTION; +EXECUTE dbo.GetTaskDetails @TaskId = @taskId; + +GO +CREATE OR ALTER PROCEDURE dbo.ConfigurePartitionOnResourceChanges +@numberOfFuturePartitionsToAdd INT +AS +BEGIN + SET XACT_ABORT ON; + BEGIN TRANSACTION; + DECLARE @partitionBoundary AS DATETIME2 (7) = DATEADD(hour, DATEDIFF(hour, 0, sysutcdatetime()), 0); + DECLARE @startingRightPartitionBoundary AS DATETIME2 (7) = CAST ((SELECT TOP (1) value + FROM sys.partition_range_values AS prv + INNER JOIN + sys.partition_functions AS pf + ON pf.function_id = prv.function_id + WHERE pf.name = N'PartitionFunction_ResourceChangeData_Timestamp' + ORDER BY prv.boundary_id DESC) AS DATETIME2 (7)); + DECLARE @numberOfPartitionsToAdd AS INT = @numberOfFuturePartitionsToAdd + 1; + WHILE @numberOfPartitionsToAdd > 0 + BEGIN + IF (@startingRightPartitionBoundary < @partitionBoundary) + BEGIN + ALTER PARTITION SCHEME PartitionScheme_ResourceChangeData_Timestamp NEXT USED [PRIMARY]; + ALTER PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp( ) + SPLIT RANGE (@partitionBoundary); + END + SET @partitionBoundary = DATEADD(hour, 1, @partitionBoundary); + SET @numberOfPartitionsToAdd -= 1; + END + COMMIT TRANSACTION; +END + +GO +CREATE PROCEDURE dbo.CreateExportJob +@id VARCHAR (64), @hash VARCHAR (64), @status VARCHAR (10), @rawJobRecord VARCHAR (MAX) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +INSERT INTO dbo.ExportJob (Id, Hash, Status, HeartbeatDateTime, RawJobRecord) +VALUES (@id, @hash, @status, @heartbeatDateTime, @rawJobRecord); +SELECT CAST (MIN_ACTIVE_ROWVERSION() AS INT); +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.CreateReindexJob +@id VARCHAR (64), @status VARCHAR (10), @rawJobRecord VARCHAR (MAX) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +INSERT INTO dbo.ReindexJob (Id, Status, HeartbeatDateTime, RawJobRecord) +VALUES (@id, @status, @heartbeatDateTime, @rawJobRecord); +SELECT CAST (MIN_ACTIVE_ROWVERSION() AS INT); +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE [dbo].[CreateTask_3] +@taskId VARCHAR (64), @queueId VARCHAR (64), @taskTypeId SMALLINT, @parentTaskId VARCHAR (64), @maxRetryCount SMALLINT=3, @inputData VARCHAR (MAX), @isUniqueTaskByType BIT +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +DECLARE @status AS SMALLINT = 1; +DECLARE @retryCount AS SMALLINT = 0; +DECLARE @isCanceled AS BIT = 0; +IF (@isUniqueTaskByType = 1) + BEGIN + IF EXISTS (SELECT * + FROM [dbo].[TaskInfo] + WHERE TaskId = @taskId + OR (TaskTypeId = @taskTypeId + AND Status <> 3)) + BEGIN + THROW 50409, 'Task already existed', 1; + END + END +ELSE + BEGIN + IF EXISTS (SELECT * + FROM [dbo].[TaskInfo] + WHERE TaskId = @taskId) + BEGIN + THROW 50409, 'Task already existed', 1; + END + END +INSERT INTO [dbo].[TaskInfo] (TaskId, QueueId, Status, TaskTypeId, IsCanceled, RetryCount, MaxRetryCount, HeartbeatDateTime, InputData, ParentTaskId) +VALUES (@taskId, @queueId, @status, @taskTypeId, @isCanceled, @retryCount, @maxRetryCount, @heartbeatDateTime, @inputData, @parentTaskId); +EXECUTE dbo.GetTaskDetails @TaskId = @taskId; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.Defrag +@TableName VARCHAR (100), @IndexName VARCHAR (200), @PartitionNumber INT, @IsPartitioned BIT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'Defrag', @Mode AS VARCHAR (200) = @TableName + '.' + @IndexName + '.' + CONVERT (VARCHAR, @PartitionNumber) + '.' + CONVERT (VARCHAR, @IsPartitioned), @st AS DATETIME = getUTCdate(), @SQL AS VARCHAR (3500), @msg AS VARCHAR (1000), @SizeBefore AS FLOAT, @SizeAfter AS FLOAT, @IndexId AS INT; +BEGIN TRY + SET @IndexId = (SELECT index_id + FROM sys.indexes + WHERE object_id = object_id(@TableName) + AND name = @IndexName); + SET @SizeBefore = (SELECT sum(reserved_page_count) + FROM sys.dm_db_partition_stats + WHERE object_id = object_id(@TableName) + AND index_id = @IndexId) * 8.0 / 1024 / 1024; + SET @msg = 'Size[GB] before=' + CONVERT (VARCHAR, @SizeBefore); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start', @Text = @msg; + SET @Sql = 'ALTER INDEX ' + quotename(@IndexName) + ' ON dbo.' + quotename(@TableName) + ' REORGANIZE' + CASE WHEN @IsPartitioned = 1 THEN ' PARTITION = ' + CONVERT (VARCHAR, @PartitionNumber) ELSE '' END; + BEGIN TRY + EXECUTE (@Sql); + SET @SizeAfter = (SELECT sum(reserved_page_count) + FROM sys.dm_db_partition_stats + WHERE object_id = object_id(@TableName) + AND index_id = @IndexId) * 8.0 / 1024 / 1024; + SET @msg = 'Size[GB] before=' + CONVERT (VARCHAR, @SizeBefore) + ', after=' + CONVERT (VARCHAR, @SizeAfter) + ', reduced by=' + CONVERT (VARCHAR, @SizeBefore - @SizeAfter); + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'End', @Mode = @Mode, @Action = 'Reorganize', @Start = @st, @Text = @msg; + END TRY + BEGIN CATCH + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'Error', @Mode = @Mode, @Action = 'Reorganize', @Start = @st, @ReRaisError = 0; + END CATCH +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.DefragChangeDatabaseSettings +@IsOn BIT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'DefragChangeDatabaseSettings', @Mode AS VARCHAR (200) = 'On=' + CONVERT (VARCHAR, @IsOn), @st AS DATETIME = getUTCdate(), @SQL AS VARCHAR (3500); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'Start', @Mode = @Mode; + SET @SQL = 'ALTER DATABASE CURRENT SET AUTO_UPDATE_STATISTICS ' + CASE WHEN @IsOn = 1 THEN 'ON' ELSE 'OFF' END; + EXECUTE (@SQL); + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'Run', @Mode = @Mode, @Text = @SQL; + SET @SQL = 'ALTER DATABASE CURRENT SET AUTO_CREATE_STATISTICS ' + CASE WHEN @IsOn = 1 THEN 'ON' ELSE 'OFF' END; + EXECUTE (@SQL); + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'End', @Mode = @Mode, @Start = @st, @Text = @SQL; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.DeleteHistory +@DeleteResources BIT=0, @Reset BIT=0, @DisableLogEvent BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'DeleteHistory', @Mode AS VARCHAR (100) = 'D=' + isnull(CONVERT (VARCHAR, @DeleteResources), 'NULL') + ' R=' + isnull(CONVERT (VARCHAR, @Reset), 'NULL'), @st AS DATETIME = getUTCdate(), @Id AS VARCHAR (100) = 'DeleteHistory.LastProcessed.TypeId.SurrogateId', @ResourceTypeId AS SMALLINT, @SurrogateId AS BIGINT, @RowsToProcess AS INT, @ProcessedResources AS INT = 0, @DeletedResources AS INT = 0, @DeletedSearchParams AS INT = 0, @ReportDate AS DATETIME = getUTCdate(); +BEGIN TRY + IF @DisableLogEvent = 0 + INSERT INTO dbo.Parameters (Id, Char) + SELECT @SP, + 'LogEvent'; + ELSE + DELETE dbo.Parameters + WHERE Id = @SP; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + INSERT INTO dbo.Parameters (Id, Char) + SELECT @Id, + '0.0' + WHERE NOT EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = @Id); + DECLARE @LastProcessed AS VARCHAR (100) = CASE WHEN @Reset = 0 THEN (SELECT Char + FROM dbo.Parameters + WHERE Id = @Id) ELSE '0.0' END; + DECLARE @Types TABLE ( + ResourceTypeId SMALLINT PRIMARY KEY, + Name VARCHAR (100)); + DECLARE @SurrogateIds TABLE ( + ResourceSurrogateId BIGINT PRIMARY KEY, + IsHistory BIT ); + INSERT INTO @Types + EXECUTE dbo.GetUsedResourceTypes ; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = '@Types', @Action = 'Insert', @Rows = @@rowcount; + SET @ResourceTypeId = substring(@LastProcessed, 1, charindex('.', @LastProcessed) - 1); + SET @SurrogateId = substring(@LastProcessed, charindex('.', @LastProcessed) + 1, 255); + DELETE @Types + WHERE ResourceTypeId < @ResourceTypeId; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = '@Types', @Action = 'Delete', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Types) + BEGIN + SET @ResourceTypeId = (SELECT TOP 1 ResourceTypeId + FROM @Types + ORDER BY ResourceTypeId); + SET @ProcessedResources = 0; + SET @DeletedResources = 0; + SET @DeletedSearchParams = 0; + SET @RowsToProcess = 1; + WHILE @RowsToProcess > 0 + BEGIN + DELETE @SurrogateIds; + INSERT INTO @SurrogateIds + SELECT TOP 10000 ResourceSurrogateId, + IsHistory + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId > @SurrogateId + ORDER BY ResourceSurrogateId; + SET @RowsToProcess = @@rowcount; + SET @ProcessedResources += @RowsToProcess; + IF @RowsToProcess > 0 + SET @SurrogateId = (SELECT max(ResourceSurrogateId) + FROM @SurrogateIds); + SET @LastProcessed = CONVERT (VARCHAR, @ResourceTypeId) + '.' + CONVERT (VARCHAR, @SurrogateId); + DELETE @SurrogateIds + WHERE IsHistory = 0; + IF EXISTS (SELECT * + FROM @SurrogateIds) + BEGIN + DELETE dbo.ResourceWriteClaim + WHERE ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.CompartmentAssignment + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.ReferenceSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenText + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.StringSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.UriSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.NumberSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.QuantitySearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.DateTimeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.ReferenceTokenCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenTokenCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenDateTimeCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenQuantityCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenStringCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenNumberNumberCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + IF @DeleteResources = 1 + BEGIN + DELETE dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedResources += @@rowcount; + END + END + UPDATE dbo.Parameters + SET Char = @LastProcessed + WHERE Id = @Id; + IF datediff(second, @ReportDate, getUTCdate()) > 60 + BEGIN + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'Resource', @Action = 'Select', @Rows = @ProcessedResources, @Text = @LastProcessed; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = '*SearchParam', @Action = 'Delete', @Rows = @DeletedSearchParams, @Text = @LastProcessed; + IF @DeleteResources = 1 + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'Resource', @Action = 'Delete', @Rows = @DeletedResources, @Text = @LastProcessed; + SET @ReportDate = getUTCdate(); + SET @ProcessedResources = 0; + SET @DeletedSearchParams = 0; + SET @DeletedResources = 0; + END + END + DELETE @Types + WHERE ResourceTypeId = @ResourceTypeId; + SET @SurrogateId = 0; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'Resource', @Action = 'Select', @Rows = @ProcessedResources, @Text = @LastProcessed; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = '*SearchParam', @Action = 'Delete', @Rows = @DeletedSearchParams, @Text = @LastProcessed; + IF @DeleteResources = 1 + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'Resource', @Action = 'Delete', @Rows = @DeletedResources, @Text = @LastProcessed; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.DequeueJob +@QueueType TINYINT, @Worker VARCHAR (100), @HeartbeatTimeoutSec INT, @InputJobId BIGINT=NULL, @CheckTimeoutJobs BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'DequeueJob', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' H=' + isnull(CONVERT (VARCHAR, @HeartbeatTimeoutSec), 'NULL') + ' W=' + isnull(@Worker, 'NULL') + ' IJ=' + isnull(CONVERT (VARCHAR, @InputJobId), 'NULL') + ' T=' + isnull(CONVERT (VARCHAR, @CheckTimeoutJobs), 'NULL'), @Rows AS INT = 0, @st AS DATETIME = getUTCdate(), @JobId AS BIGINT, @msg AS VARCHAR (100), @Lock AS VARCHAR (100), @PartitionId AS TINYINT, @MaxPartitions AS TINYINT = 16, @LookedAtPartitions AS TINYINT = 0; +BEGIN TRY + IF EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = 'DequeueJobStop' + AND Number = 1) + BEGIN + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = 0, @Text = 'Skipped'; + RETURN; + END + IF @InputJobId IS NULL + SET @PartitionId = @MaxPartitions * rand(); + ELSE + SET @PartitionId = @InputJobId % 16; + SET TRANSACTION ISOLATION LEVEL READ COMMITTED; + WHILE @InputJobId IS NULL + AND @JobId IS NULL + AND @LookedAtPartitions < @MaxPartitions + AND @CheckTimeoutJobs = 0 + BEGIN + SET @Lock = 'DequeueJob_' + CONVERT (VARCHAR, @QueueType) + '_' + CONVERT (VARCHAR, @PartitionId); + BEGIN TRANSACTION; + EXECUTE sp_getapplock @Lock, 'Exclusive'; + UPDATE T + SET StartDate = getUTCdate(), + HeartbeatDate = getUTCdate(), + Worker = @Worker, + Status = 1, + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()), + @JobId = T.JobId + FROM dbo.JobQueue AS T WITH (PAGLOCK) + INNER JOIN + (SELECT TOP 1 JobId + FROM dbo.JobQueue WITH (INDEX (IX_QueueType_PartitionId_Status_Priority)) + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND Status = 0 + ORDER BY Priority, JobId) AS S + ON QueueType = @QueueType + AND PartitionId = @PartitionId + AND T.JobId = S.JobId; + SET @Rows += @@rowcount; + COMMIT TRANSACTION; + IF @JobId IS NULL + BEGIN + SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; + SET @LookedAtPartitions = @LookedAtPartitions + 1; + END + END + SET @LookedAtPartitions = 0; + WHILE @InputJobId IS NULL + AND @JobId IS NULL + AND @LookedAtPartitions < @MaxPartitions + BEGIN + SET @Lock = 'DequeueStoreCopyWorkUnit_' + CONVERT (VARCHAR, @PartitionId); + BEGIN TRANSACTION; + EXECUTE sp_getapplock @Lock, 'Exclusive'; + UPDATE T + SET StartDate = getUTCdate(), + HeartbeatDate = getUTCdate(), + Worker = @Worker, + Status = CASE WHEN CancelRequested = 0 THEN 1 ELSE 4 END, + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()), + @JobId = CASE WHEN CancelRequested = 0 THEN T.JobId END, + Info = CONVERT (VARCHAR (1000), isnull(Info, '') + ' Prev: Worker=' + Worker + ' Start=' + CONVERT (VARCHAR, StartDate, 121)) + FROM dbo.JobQueue AS T WITH (PAGLOCK) + INNER JOIN + (SELECT TOP 1 JobId + FROM dbo.JobQueue WITH (INDEX (IX_QueueType_PartitionId_Status_Priority)) + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND Status = 1 + AND datediff(second, HeartbeatDate, getUTCdate()) > @HeartbeatTimeoutSec + ORDER BY Priority, JobId) AS S + ON QueueType = @QueueType + AND PartitionId = @PartitionId + AND T.JobId = S.JobId; + SET @Rows += @@rowcount; + COMMIT TRANSACTION; + IF @JobId IS NULL + BEGIN + SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; + SET @LookedAtPartitions = @LookedAtPartitions + 1; + END + END + IF @InputJobId IS NOT NULL + BEGIN + UPDATE dbo.JobQueue WITH (PAGLOCK) + SET StartDate = getUTCdate(), + HeartbeatDate = getUTCdate(), + Worker = @Worker, + Status = 1, + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()), + @JobId = JobId + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND Status = 0 + AND JobId = @InputJobId; + SET @Rows += @@rowcount; + IF @JobId IS NULL + BEGIN + UPDATE dbo.JobQueue WITH (PAGLOCK) + SET StartDate = getUTCdate(), + HeartbeatDate = getUTCdate(), + Worker = @Worker, + Status = 1, + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()), + @JobId = JobId + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND Status = 1 + AND JobId = @InputJobId + AND datediff(second, HeartbeatDate, getUTCdate()) > @HeartbeatTimeoutSec; + SET @Rows += @@rowcount; + END + END + IF @JobId IS NOT NULL + EXECUTE dbo.GetJobs @QueueType = @QueueType, @JobId = @JobId; + SET @msg = 'J=' + isnull(CONVERT (VARCHAR, @JobId), 'NULL') + ' P=' + CONVERT (VARCHAR, @PartitionId); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows, @Text = @msg; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.DisableIndex +@tableName NVARCHAR (128), @indexName NVARCHAR (128) +WITH EXECUTE AS 'dbo' +AS +DECLARE @errorTxt AS VARCHAR (1000), @sql AS NVARCHAR (1000), @isDisabled AS BIT; +IF object_id(@tableName) IS NULL + BEGIN + SET @errorTxt = @tableName + ' does not exist or you don''t have permissions.'; + RAISERROR (@errorTxt, 18, 127); + END +SET @isDisabled = (SELECT is_disabled + FROM sys.indexes + WHERE object_id = object_id(@tableName) + AND name = @indexName); +IF @isDisabled IS NULL + BEGIN + SET @errorTxt = @indexName + ' does not exist or you don''t have permissions.'; + RAISERROR (@errorTxt, 18, 127); + END +IF @isDisabled = 0 + BEGIN + SET @sql = N'ALTER INDEX ' + QUOTENAME(@indexName) + N' on ' + @tableName + ' Disable'; + EXECUTE sp_executesql @sql; + END + +GO +CREATE PROCEDURE dbo.DisableIndexes +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'DisableIndexes', @Mode AS VARCHAR (200) = '', @st AS DATETIME = getUTCdate(), @Tbl AS VARCHAR (100), @Ind AS VARCHAR (200), @Txt AS VARCHAR (4000); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + DECLARE @Tables TABLE ( + Tbl VARCHAR (100) PRIMARY KEY, + Supported BIT ); + INSERT INTO @Tables + EXECUTE dbo.GetPartitionedTables @IncludeNotDisabled = 1, @IncludeNotSupported = 0; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Tables', @Action = 'Insert', @Rows = @@rowcount; + DECLARE @Indexes TABLE ( + Tbl VARCHAR (100), + Ind VARCHAR (200), + TblId INT , + IndId INT PRIMARY KEY (Tbl, Ind)); + INSERT INTO @Indexes + SELECT Tbl, + I.Name, + TblId, + I.index_id + FROM (SELECT object_id(Tbl) AS TblId, + Tbl + FROM @Tables) AS O + INNER JOIN + sys.indexes AS I + ON I.object_id = TblId; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Insert', @Rows = @@rowcount; + INSERT INTO dbo.IndexProperties (TableName, IndexName, PropertyName, PropertyValue) + SELECT Tbl, + Ind, + 'DATA_COMPRESSION', + data_comp + FROM (SELECT Tbl, + Ind, + isnull((SELECT TOP 1 CASE WHEN data_compression_desc = 'PAGE' THEN 'PAGE' END + FROM sys.partitions + WHERE object_id = TblId + AND index_id = IndId), 'NONE') AS data_comp + FROM @Indexes) AS A + WHERE NOT EXISTS (SELECT * + FROM dbo.IndexProperties + WHERE TableName = Tbl + AND IndexName = Ind); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = 'IndexProperties', @Action = 'Insert', @Rows = @@rowcount; + DELETE @Indexes + WHERE Tbl = 'Resource' + OR IndId = 1; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Delete', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Indexes) + BEGIN + SELECT TOP 1 @Tbl = Tbl, + @Ind = Ind + FROM @Indexes; + SET @Txt = 'ALTER INDEX ' + @Ind + ' ON dbo.' + @Tbl + ' DISABLE'; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Ind, @Action = 'Disable', @Text = @Txt; + DELETE @Indexes + WHERE Tbl = @Tbl + AND Ind = @Ind; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.EnqueueJobs +@QueueType TINYINT, @Definitions StringList READONLY, @GroupId BIGINT=NULL, @ForceOneActiveJobGroup BIT=1, @IsCompleted BIT=NULL, @ReturnJobs BIT=1 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'EnqueueJobs', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' D=' + CONVERT (VARCHAR, (SELECT count(*) + FROM @Definitions)) + ' G=' + isnull(CONVERT (VARCHAR, @GroupId), 'NULL') + ' F=' + isnull(CONVERT (VARCHAR, @ForceOneActiveJobGroup), 'NULL') + ' C=' + isnull(CONVERT (VARCHAR, @IsCompleted), 'NULL'), @st AS DATETIME = getUTCdate(), @Lock AS VARCHAR (100) = 'EnqueueJobs_' + CONVERT (VARCHAR, @QueueType), @MaxJobId AS BIGINT, @Rows AS INT, @msg AS VARCHAR (1000), @JobIds AS BigintList, @InputRows AS INT; +BEGIN TRY + DECLARE @Input TABLE ( + DefinitionHash VARBINARY (20) PRIMARY KEY, + Definition VARCHAR (MAX) ); + INSERT INTO @Input + SELECT hashbytes('SHA1', String) AS DefinitionHash, + String AS Definition + FROM @Definitions; + SET @InputRows = @@rowcount; + INSERT INTO @JobIds + SELECT JobId + FROM @Input AS A + INNER JOIN + dbo.JobQueue AS B + ON B.QueueType = @QueueType + AND B.DefinitionHash = A.DefinitionHash + AND B.Status <> 5; + IF @@rowcount < @InputRows + BEGIN + BEGIN TRANSACTION; + EXECUTE sp_getapplock @Lock, 'Exclusive'; + IF @ForceOneActiveJobGroup = 1 + AND EXISTS (SELECT * + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND Status IN (0, 1) + AND (@GroupId IS NULL + OR GroupId <> @GroupId)) + RAISERROR ('There are other active job groups', 18, 127); + SET @MaxJobId = isnull((SELECT TOP 1 JobId + FROM dbo.JobQueue + WHERE QueueType = @QueueType + ORDER BY JobId DESC), 0); + INSERT INTO dbo.JobQueue (QueueType, GroupId, JobId, Definition, DefinitionHash, Status) + OUTPUT inserted.JobId INTO @JobIds + SELECT @QueueType, + isnull(@GroupId, @MaxJobId + 1) AS GroupId, + JobId, + Definition, + DefinitionHash, + CASE WHEN @IsCompleted = 1 THEN 2 ELSE 0 END AS Status + FROM (SELECT @MaxJobId + row_number() OVER (ORDER BY Dummy) AS JobId, + * + FROM (SELECT *, + 0 AS Dummy + FROM @Input) AS A) AS A + WHERE NOT EXISTS (SELECT * + FROM dbo.JobQueue AS B WITH (INDEX (IX_QueueType_DefinitionHash)) + WHERE B.QueueType = @QueueType + AND B.DefinitionHash = A.DefinitionHash + AND B.Status <> 5); + SET @Rows = @@rowcount; + COMMIT TRANSACTION; + END + IF @ReturnJobs = 1 + EXECUTE dbo.GetJobs @QueueType = @QueueType, @JobIds = @JobIds; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.ExecuteCommandForRebuildIndexes +@Tbl VARCHAR (100), @Ind VARCHAR (1000), @Cmd VARCHAR (MAX) +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'ExecuteCommandForRebuildIndexes', @Mode AS VARCHAR (200) = 'Tbl=' + isnull(@Tbl, 'NULL'), @st AS DATETIME, @Retries AS INT = 0, @Action AS VARCHAR (100), @msg AS VARCHAR (1000); +RetryOnTempdbError: +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start', @Text = @Cmd; + SET @st = getUTCdate(); + IF @Tbl IS NULL + RAISERROR ('@Tbl IS NULL', 18, 127); + IF @Cmd IS NULL + RAISERROR ('@Cmd IS NULL', 18, 127); + SET @Action = CASE WHEN @Cmd LIKE 'UPDATE STAT%' THEN 'Update statistics' WHEN @Cmd LIKE 'CREATE%INDEX%' THEN 'Create Index' WHEN @Cmd LIKE 'ALTER%INDEX%REBUILD%' THEN 'Rebuild Index' WHEN @Cmd LIKE 'ALTER%TABLE%ADD%' THEN 'Add Constraint' END; + IF @Action IS NULL + BEGIN + SET @msg = 'Not supported command = ' + CONVERT (VARCHAR (900), @Cmd); + RAISERROR (@msg, 18, 127); + END + IF @Action = 'Create Index' + WAITFOR DELAY '00:00:05'; + EXECUTE (@Cmd); + SELECT @Ind; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Action = @Action, @Status = 'End', @Start = @st, @Text = @Cmd; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + IF error_number() = 40544 + BEGIN + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st, @Retry = @Retries; + SET @Retries = @Retries + 1; + IF @Tbl = 'TokenText_96' + WAITFOR DELAY '01:00:00'; + ELSE + WAITFOR DELAY '00:10:00'; + GOTO RetryOnTempdbError; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE OR ALTER PROCEDURE dbo.FetchEventAgentCheckpoint +@CheckpointId VARCHAR (64) +AS +BEGIN + SELECT TOP (1) CheckpointId, + LastProcessedDateTime, + LastProcessedIdentifier + FROM dbo.EventAgentCheckpoint + WHERE CheckpointId = @CheckpointId; +END + +GO +CREATE PROCEDURE dbo.FetchResourceChanges_3 +@startId BIGINT, @lastProcessedUtcDateTime DATETIME2 (7), @pageSize SMALLINT +AS +BEGIN + SET NOCOUNT ON; + DECLARE @precedingPartitionBoundary AS DATETIME2 (7) = (SELECT TOP (1) CAST (prv.value AS DATETIME2 (7)) AS value + FROM sys.partition_range_values AS prv WITH (NOLOCK) + INNER JOIN + sys.partition_functions AS pf WITH (NOLOCK) + ON pf.function_id = prv.function_id + WHERE pf.name = N'PartitionFunction_ResourceChangeData_Timestamp' + AND SQL_VARIANT_PROPERTY(prv.Value, 'BaseType') = 'datetime2' + AND CAST (prv.value AS DATETIME2 (7)) < DATEADD(HOUR, DATEDIFF(HOUR, 0, @lastProcessedUtcDateTime), 0) + ORDER BY prv.boundary_id DESC); + IF (@precedingPartitionBoundary IS NULL) + BEGIN + SET @precedingPartitionBoundary = CONVERT (DATETIME2 (7), N'1970-01-01T00:00:00.0000000'); + END + DECLARE @endDateTimeToFilter AS DATETIME2 (7) = DATEADD(HOUR, 1, SYSUTCDATETIME()); + WITH PartitionBoundaries + AS (SELECT CAST (prv.value AS DATETIME2 (7)) AS PartitionBoundary + FROM sys.partition_range_values AS prv WITH (NOLOCK) + INNER JOIN + sys.partition_functions AS pf WITH (NOLOCK) + ON pf.function_id = prv.function_id + WHERE pf.name = N'PartitionFunction_ResourceChangeData_Timestamp' + AND SQL_VARIANT_PROPERTY(prv.Value, 'BaseType') = 'datetime2' + AND CAST (prv.value AS DATETIME2 (7)) BETWEEN @precedingPartitionBoundary AND @endDateTimeToFilter) + SELECT TOP (@pageSize) Id, + Timestamp, + ResourceId, + ResourceTypeId, + ResourceVersion, + ResourceChangeTypeId + FROM PartitionBoundaries AS p CROSS APPLY (SELECT TOP (@pageSize) Id, + Timestamp, + ResourceId, + ResourceTypeId, + ResourceVersion, + ResourceChangeTypeId + FROM dbo.ResourceChangeData WITH (TABLOCK, HOLDLOCK) + WHERE Id >= @startId + AND $PARTITION.PartitionFunction_ResourceChangeData_Timestamp (Timestamp) = $PARTITION.PartitionFunction_ResourceChangeData_Timestamp (p.PartitionBoundary) + ORDER BY Id ASC) AS rcd + ORDER BY rcd.Id ASC; +END + +GO +CREATE PROCEDURE dbo.GetActiveJobs +@QueueType TINYINT, @GroupId BIGINT=NULL +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetActiveJobs', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' G=' + isnull(CONVERT (VARCHAR, @GroupId), 'NULL'), @st AS DATETIME = getUTCdate(), @JobIds AS BigintList, @PartitionId AS TINYINT, @MaxPartitions AS TINYINT = 16, @LookedAtPartitions AS TINYINT = 0, @Rows AS INT = 0; +BEGIN TRY + SET @PartitionId = @MaxPartitions * rand(); + WHILE @LookedAtPartitions < @MaxPartitions + BEGIN + IF @GroupId IS NULL + INSERT INTO @JobIds + SELECT JobId + FROM dbo.JobQueue + WHERE PartitionId = @PartitionId + AND QueueType = @QueueType + AND Status IN (0, 1); + ELSE + INSERT INTO @JobIds + SELECT JobId + FROM dbo.JobQueue + WHERE PartitionId = @PartitionId + AND QueueType = @QueueType + AND GroupId = @GroupId + AND Status IN (0, 1); + SET @Rows += @@rowcount; + SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; + SET @LookedAtPartitions += 1; + END + IF @Rows > 0 + EXECUTE dbo.GetJobs @QueueType = @QueueType, @JobIds = @JobIds; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetCommandsForRebuildIndexes +@RebuildClustered BIT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetCommandsForRebuildIndexes', @Mode AS VARCHAR (200) = 'PS=PartitionScheme_ResourceTypeId RC=' + isnull(CONVERT (VARCHAR, @RebuildClustered), 'NULL'), @st AS DATETIME = getUTCdate(), @Tbl AS VARCHAR (100), @TblInt AS VARCHAR (100), @Ind AS VARCHAR (200), @IndId AS INT, @Supported AS BIT, @Txt AS VARCHAR (MAX), @Rows AS BIGINT, @Pages AS BIGINT, @ResourceTypeId AS SMALLINT, @IndexesCnt AS INT, @DataComp AS VARCHAR (100); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + DECLARE @Commands TABLE ( + Tbl VARCHAR (100), + Ind VARCHAR (200), + Txt VARCHAR (MAX), + Pages BIGINT ); + DECLARE @ResourceTypes TABLE ( + ResourceTypeId SMALLINT PRIMARY KEY); + DECLARE @Indexes TABLE ( + Ind VARCHAR (200) PRIMARY KEY, + IndId INT ); + DECLARE @Tables TABLE ( + name VARCHAR (100) PRIMARY KEY, + Supported BIT ); + INSERT INTO @Tables + EXECUTE dbo.GetPartitionedTables @IncludeNotDisabled = 1, @IncludeNotSupported = 1; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Tables', @Action = 'Insert', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Tables) + BEGIN + SELECT TOP 1 @Tbl = name, + @Supported = Supported + FROM @Tables + ORDER BY name; + IF @Supported = 0 + BEGIN + INSERT INTO @Commands + SELECT @Tbl, + name, + 'ALTER INDEX ' + name + ' ON dbo.' + @Tbl + ' REBUILD' + CASE WHEN (SELECT PropertyValue + FROM dbo.IndexProperties + WHERE TableName = @Tbl + AND IndexName = name) = 'PAGE' THEN ' PARTITION = ALL WITH (DATA_COMPRESSION = PAGE)' ELSE '' END, + CONVERT (BIGINT, 9e18) + FROM sys.indexes + WHERE object_id = object_id(@Tbl) + AND (is_disabled = 1 + AND index_id > 1 + AND @RebuildClustered = 0 + OR index_id = 1 + AND @RebuildClustered = 1); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Commands', @Action = 'Insert', @Rows = @@rowcount, @Text = 'Not supported tables with disabled indexes'; + END + ELSE + BEGIN + DELETE @ResourceTypes; + INSERT INTO @ResourceTypes + SELECT CONVERT (SMALLINT, substring(name, charindex('_', name) + 1, 6)) AS ResourceTypeId + FROM sys.sysobjects + WHERE name LIKE @Tbl + '[_]%'; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@ResourceTypes', @Action = 'Insert', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @ResourceTypes) + BEGIN + SET @ResourceTypeId = (SELECT TOP 1 ResourceTypeId + FROM @ResourceTypes + ORDER BY ResourceTypeId); + SET @TblInt = @Tbl + '_' + CONVERT (VARCHAR, @ResourceTypeId); + SET @Pages = (SELECT dpages + FROM sysindexes + WHERE id = object_id(@TblInt) + AND indid IN (0, 1)); + DELETE @Indexes; + INSERT INTO @Indexes + SELECT name, + index_id + FROM sys.indexes + WHERE object_id = object_id(@Tbl) + AND (index_id > 1 + AND @RebuildClustered = 0 + OR index_id = 1 + AND @RebuildClustered = 1); + SET @IndexesCnt = 0; + WHILE EXISTS (SELECT * + FROM @Indexes) + BEGIN + SELECT TOP 1 @Ind = Ind, + @IndId = IndId + FROM @Indexes + ORDER BY Ind; + IF @IndId = 1 + BEGIN + SET @Txt = 'ALTER INDEX ' + @Ind + ' ON dbo.' + @TblInt + ' REBUILD' + CASE WHEN (SELECT PropertyValue + FROM dbo.IndexProperties + WHERE TableName = @Tbl + AND IndexName = @Ind) = 'PAGE' THEN ' PARTITION = ALL WITH (DATA_COMPRESSION = PAGE)' ELSE '' END; + INSERT INTO @Commands + SELECT @TblInt, + @Ind, + @Txt, + @Pages; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Add command', @Rows = @@rowcount, @Text = @Txt; + END + ELSE + IF NOT EXISTS (SELECT * + FROM sys.indexes + WHERE object_id = object_id(@TblInt) + AND name = @Ind) + BEGIN + EXECUTE dbo.GetIndexCommands @Tbl = @Tbl, @Ind = @Ind, @AddPartClause = 0, @IncludeClustered = 0, @Txt = @Txt OUTPUT; + SET @Txt = replace(@Txt, '[' + @Tbl + ']', @TblInt); + IF @Txt IS NOT NULL + BEGIN + SET @IndexesCnt = @IndexesCnt + 1; + INSERT INTO @Commands + SELECT @TblInt, + @Ind, + @Txt, + @Pages; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Add command', @Rows = @@rowcount, @Text = @Txt; + END + END + DELETE @Indexes + WHERE Ind = @Ind; + END + IF @IndexesCnt > 1 + BEGIN + INSERT INTO @Commands + SELECT @TblInt, + 'UPDATE STAT', + 'UPDATE STATISTICS dbo.' + @TblInt, + @Pages; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Add command', @Rows = @@rowcount, @Text = 'Add stats update'; + END + DELETE @ResourceTypes + WHERE ResourceTypeId = @ResourceTypeId; + END + END + DELETE @Tables + WHERE name = @Tbl; + END + SELECT Tbl, + Ind, + Txt + FROM @Commands + ORDER BY Pages DESC, Tbl, CASE WHEN Txt LIKE 'UPDATE STAT%' THEN 0 ELSE 1 END; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Commands', @Action = 'Select', @Rows = @@rowcount; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetExportJobByHash +@hash VARCHAR (64) +AS +SET NOCOUNT ON; +SELECT TOP (1) RawJobRecord, + JobVersion +FROM dbo.ExportJob +WHERE Hash = @hash + AND (Status = 'Queued' + OR Status = 'Running') +ORDER BY HeartbeatDateTime ASC; + +GO +CREATE PROCEDURE dbo.GetExportJobById +@id VARCHAR (64) +AS +SET NOCOUNT ON; +SELECT RawJobRecord, + JobVersion +FROM dbo.ExportJob +WHERE Id = @id; + +GO +CREATE PROCEDURE [dbo].[GetImportProcessingTaskResult] +@queueId VARCHAR (64), @importTaskId VARCHAR (64) +AS +SET NOCOUNT ON; +SELECT Result +FROM [dbo].[TaskInfo] WITH (INDEX (IX_QueueId_ParentTaskId)) +WHERE ParentTaskId = @importTaskId + AND TaskTypeId = 1 + AND Status = 3; + +GO +CREATE PROCEDURE dbo.GetIndexCommands +@Tbl VARCHAR (100), @Ind VARCHAR (200), @AddPartClause BIT, @IncludeClustered BIT, @Txt VARCHAR (MAX)=NULL OUTPUT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetIndexCommands', @Mode AS VARCHAR (200) = 'Tbl=' + isnull(@Tbl, 'NULL') + ' Ind=' + isnull(@Ind, 'NULL'), @st AS DATETIME = getUTCdate(); +DECLARE @Indexes TABLE ( + Ind VARCHAR (200) PRIMARY KEY, + Txt VARCHAR (MAX)); +BEGIN TRY + IF @Tbl IS NULL + RAISERROR ('@Tbl IS NULL', 18, 127); + INSERT INTO @Indexes + SELECT Ind, + CASE WHEN is_primary_key = 1 THEN 'ALTER TABLE dbo.[' + Tbl + '] ADD PRIMARY KEY ' + CASE WHEN type = 1 THEN ' CLUSTERED' ELSE '' END ELSE 'CREATE' + CASE WHEN is_unique = 1 THEN ' UNIQUE' ELSE '' END + CASE WHEN type = 1 THEN ' CLUSTERED' ELSE '' END + ' INDEX ' + Ind + ' ON dbo.[' + Tbl + ']' END + ' (' + KeyCols + ')' + IncClause + CASE WHEN filter_def IS NOT NULL THEN ' WHERE ' + filter_def ELSE '' END + CASE WHEN data_comp IS NOT NULL THEN ' WITH (DATA_COMPRESSION = ' + data_comp + ')' ELSE '' END + CASE WHEN @AddPartClause = 1 THEN PartClause ELSE '' END + FROM (SELECT O.Name AS Tbl, + I.Name AS Ind, + isnull((SELECT TOP 1 CASE WHEN data_compression_desc = 'PAGE' THEN 'PAGE' END + FROM sys.partitions AS P + WHERE P.object_id = I.object_id + AND I.index_id = P.index_id), (SELECT NULLIF (PropertyValue, 'NONE') + FROM dbo.IndexProperties + WHERE TableName = O.Name + AND IndexName = I.Name + AND PropertyName = 'DATA_COMPRESSION')) AS data_comp, + replace(replace(replace(replace(I.filter_definition, '[', ''), ']', ''), '(', ''), ')', '') AS filter_def, + I.is_unique, + I.is_primary_key, + I.type, + KeyCols, + CASE WHEN IncCols IS NOT NULL THEN ' INCLUDE (' + IncCols + ')' ELSE '' END AS IncClause, + CASE WHEN EXISTS (SELECT * + FROM sys.partition_schemes AS S + WHERE S.data_space_id = I.data_space_id + AND name = 'PartitionScheme_ResourceTypeId') THEN ' ON PartitionScheme_ResourceTypeId (ResourceTypeId)' ELSE '' END AS PartClause + FROM sys.indexes AS I + INNER JOIN + sys.objects AS O + ON O.object_id = I.object_id CROSS APPLY (SELECT string_agg(CASE WHEN IC.key_ordinal > 0 + AND IC.is_included_column = 0 THEN C.name END, ',') WITHIN GROUP (ORDER BY key_ordinal) AS KeyCols, + string_agg(CASE WHEN IC.is_included_column = 1 THEN C.name END, ',') WITHIN GROUP (ORDER BY key_ordinal) AS IncCols + FROM sys.index_columns AS IC + INNER JOIN + sys.columns AS C + ON C.object_id = IC.object_id + AND C.column_id = IC.column_id + WHERE IC.object_id = I.object_id + AND IC.index_id = I.index_id + GROUP BY IC.object_id, IC.index_id) AS IC + WHERE O.name = @Tbl + AND (@Ind IS NULL + OR I.name = @Ind) + AND (@IncludeClustered = 1 + OR index_id > 1)) AS A; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Insert', @Rows = @@rowcount; + IF @Ind IS NULL + SELECT Ind, + Txt + FROM @Indexes; + ELSE + SET @Txt = (SELECT Txt + FROM @Indexes); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Text = @Txt; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetJobs +@QueueType TINYINT, @JobId BIGINT=NULL, @JobIds BigintList READONLY, @GroupId BIGINT=NULL, @ReturnDefinition BIT=1 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetJobs', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' J=' + isnull(CONVERT (VARCHAR, @JobId), 'NULL') + ' G=' + isnull(CONVERT (VARCHAR, @GroupId), 'NULL'), @st AS DATETIME = getUTCdate(), @PartitionId AS TINYINT = @JobId % 16; +BEGIN TRY + IF @JobId IS NULL + AND @GroupId IS NULL + AND NOT EXISTS (SELECT * + FROM @JobIds) + RAISERROR ('@JobId = NULL and @GroupId = NULL and @JobIds is empty', 18, 127); + IF @JobId IS NOT NULL + SELECT GroupId, + JobId, + CASE WHEN @ReturnDefinition = 1 THEN Definition ELSE NULL END AS Definition, + Version, + Status, + Priority, + Data, + Result, + CreateDate, + StartDate, + EndDate, + HeartbeatDate, + CancelRequested + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = isnull(@JobId, -1) + AND Status <> 5; + ELSE + IF @GroupId IS NOT NULL + SELECT GroupId, + JobId, + CASE WHEN @ReturnDefinition = 1 THEN Definition ELSE NULL END AS Definition, + Version, + Status, + Priority, + Data, + Result, + CreateDate, + StartDate, + EndDate, + HeartbeatDate, + CancelRequested + FROM dbo.JobQueue WITH (INDEX (IX_QueueType_GroupId)) + WHERE QueueType = @QueueType + AND GroupId = isnull(@GroupId, -1) + AND Status <> 5; + ELSE + SELECT GroupId, + JobId, + CASE WHEN @ReturnDefinition = 1 THEN Definition ELSE NULL END AS Definition, + Version, + Status, + Priority, + Data, + Result, + CreateDate, + StartDate, + EndDate, + HeartbeatDate, + CancelRequested + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND JobId IN (SELECT Id + FROM @JobIds) + AND PartitionId = JobId % 16 + AND Status <> 5; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetNextTask_3 +@queueId VARCHAR (64), @taskHeartbeatTimeoutThresholdInSeconds INT=600 +AS +SET NOCOUNT ON; +DECLARE @lock AS VARCHAR (200) = 'GetNextTask_Q=' + @queueId, @taskId AS VARCHAR (64) = NULL, @expirationDateTime AS DATETIME2 (7), @startDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +SET @expirationDateTime = DATEADD(second, -@taskHeartbeatTimeoutThresholdInSeconds, @startDateTime); +BEGIN TRY + BEGIN TRANSACTION; + EXECUTE sp_getapplock @lock, 'Exclusive'; + UPDATE T + SET Status = 2, + StartDateTime = @startDateTime, + HeartbeatDateTime = @startDateTime, + Worker = host_name(), + RunId = NEWID(), + @taskId = T.TaskId + FROM dbo.TaskInfo AS T WITH (PAGLOCK) + INNER JOIN + (SELECT TOP 1 TaskId + FROM dbo.TaskInfo WITH (INDEX (IX_QueueId_Status)) + WHERE QueueId = @queueId + AND Status = 1 + ORDER BY TaskId) AS S + ON T.QueueId = @queueId + AND T.TaskId = S.TaskId; + IF @taskId IS NULL + UPDATE T + SET StartDateTime = @startDateTime, + HeartbeatDateTime = @startDateTime, + Worker = host_name(), + RunId = NEWID(), + @taskId = T.TaskId, + RestartInfo = ISNULL(RestartInfo, '') + ' Prev: Worker=' + Worker + ' Start=' + CONVERT (VARCHAR, @startDateTime, 121) + FROM dbo.TaskInfo AS T WITH (PAGLOCK) + INNER JOIN + (SELECT TOP 1 TaskId + FROM dbo.TaskInfo WITH (INDEX (IX_QueueId_Status)) + WHERE QueueId = @queueId + AND Status = 2 + AND HeartbeatDateTime <= @expirationDateTime + ORDER BY TaskId) AS S + ON T.QueueId = @queueId + AND T.TaskId = S.TaskId; + COMMIT TRANSACTION; + EXECUTE dbo.GetTaskDetails @TaskId = @taskId; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK TRANSACTION THROW; +END CATCH + +GO +CREATE OR ALTER PROCEDURE dbo.GetNonCompletedJobCountOfSpecificQueueType +@queueType TINYINT +AS +BEGIN + SET NOCOUNT ON; + SELECT COUNT(*) + FROM dbo.JobQueue + WHERE QueueType = @queueType + AND (Status = 0 + OR Status = 1); +END + +GO +CREATE PROCEDURE dbo.GetPartitionedTables +@IncludeNotDisabled BIT, @IncludeNotSupported BIT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetPartitionedTables', @Mode AS VARCHAR (200) = 'PS=PartitionScheme_ResourceTypeId D=' + isnull(CONVERT (VARCHAR, @IncludeNotDisabled), 'NULL') + ' S=' + isnull(CONVERT (VARCHAR, @IncludeNotSupported), 'NULL'), @st AS DATETIME = getUTCdate(); +DECLARE @NotSupportedTables TABLE ( + id INT PRIMARY KEY); +BEGIN TRY + INSERT INTO @NotSupportedTables + SELECT DISTINCT O.object_id + FROM sys.indexes AS I + INNER JOIN + sys.objects AS O + ON O.object_id = I.object_id + WHERE O.type = 'u' + AND EXISTS (SELECT * + FROM sys.partition_schemes AS PS + WHERE PS.data_space_id = I.data_space_id + AND name = 'PartitionScheme_ResourceTypeId') + AND (NOT EXISTS (SELECT * + FROM sys.index_columns AS IC + INNER JOIN + sys.columns AS C + ON C.object_id = IC.object_id + AND C.column_id = IC.column_id + WHERE IC.object_id = I.object_id + AND IC.index_id = I.index_id + AND IC.key_ordinal > 0 + AND IC.is_included_column = 0 + AND C.name = 'ResourceTypeId') + OR EXISTS (SELECT * + FROM sys.indexes AS NSI + WHERE NSI.object_id = O.object_id + AND NOT EXISTS (SELECT * + FROM sys.partition_schemes AS PS + WHERE PS.data_space_id = NSI.data_space_id + AND name = 'PartitionScheme_ResourceTypeId'))); + SELECT CONVERT (VARCHAR (100), O.name), + CONVERT (BIT, CASE WHEN EXISTS (SELECT * + FROM @NotSupportedTables AS NSI + WHERE NSI.id = O.object_id) THEN 0 ELSE 1 END) + FROM sys.indexes AS I + INNER JOIN + sys.objects AS O + ON O.object_id = I.object_id + WHERE O.type = 'u' + AND I.index_id IN (0, 1) + AND EXISTS (SELECT * + FROM sys.partition_schemes AS PS + WHERE PS.data_space_id = I.data_space_id + AND name = 'PartitionScheme_ResourceTypeId') + AND EXISTS (SELECT * + FROM sys.index_columns AS IC + INNER JOIN + sys.columns AS C + ON C.object_id = I.object_id + AND C.column_id = IC.column_id + AND IC.is_included_column = 0 + AND C.name = 'ResourceTypeId') + AND (@IncludeNotSupported = 1 + OR NOT EXISTS (SELECT * + FROM @NotSupportedTables AS NSI + WHERE NSI.id = O.object_id)) + AND (@IncludeNotDisabled = 1 + OR EXISTS (SELECT * + FROM sys.indexes AS D + WHERE D.object_id = O.object_id + AND D.is_disabled = 1)) + ORDER BY 1; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetReindexJobById +@id VARCHAR (64) +AS +SET NOCOUNT ON; +SELECT RawJobRecord, + JobVersion +FROM dbo.ReindexJob +WHERE Id = @id; + +GO +CREATE PROCEDURE dbo.GetResources +@ResourceKeys dbo.ResourceKeyList READONLY +AS +SET NOCOUNT ON; +DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = 'GetResources', @InputRows AS INT, @DummyTop AS BIGINT = 9223372036854775807, @NotNullVersionExists AS BIT, @NullVersionExists AS BIT, @MinRT AS SMALLINT, @MaxRT AS SMALLINT; +SELECT @MinRT = min(ResourceTypeId), + @MaxRT = max(ResourceTypeId), + @InputRows = count(*), + @NotNullVersionExists = max(CASE WHEN Version IS NOT NULL THEN 1 ELSE 0 END), + @NullVersionExists = max(CASE WHEN Version IS NULL THEN 1 ELSE 0 END) +FROM @ResourceKeys; +DECLARE @Mode AS VARCHAR (100) = 'RT=[' + CONVERT (VARCHAR, @MinRT) + ',' + CONVERT (VARCHAR, @MaxRT) + '] Cnt=' + CONVERT (VARCHAR, @InputRows) + ' NNVE=' + CONVERT (VARCHAR, @NotNullVersionExists) + ' NVE=' + CONVERT (VARCHAR, @NullVersionExists); +BEGIN TRY + IF @NotNullVersionExists = 1 + IF @NullVersionExists = 0 + SELECT B.ResourceTypeId, + B.ResourceId, + ResourceSurrogateId, + B.Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceKeys) AS A + INNER JOIN + dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.Version = A.Version + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + ELSE + SELECT * + FROM (SELECT B.ResourceTypeId, + B.ResourceId, + ResourceSurrogateId, + B.Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceKeys + WHERE Version IS NOT NULL) AS A + INNER JOIN + dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.Version = A.Version + UNION ALL + SELECT B.ResourceTypeId, + B.ResourceId, + ResourceSurrogateId, + B.Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceKeys + WHERE Version IS NULL) AS A + INNER JOIN + dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId)) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + WHERE IsHistory = 0) AS A + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + ELSE + SELECT B.ResourceTypeId, + B.ResourceId, + ResourceSurrogateId, + B.Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceKeys) AS A + INNER JOIN + dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId)) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + WHERE IsHistory = 0 + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetResourcesByTransactionId +@TransactionId BIGINT, @IncludeHistory BIT=0, @ReturnResourceKeysOnly BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'T=' + CONVERT (VARCHAR, @TransactionId) + ' H=' + CONVERT (VARCHAR, @IncludeHistory), @st AS DATETIME = getUTCdate(), @DummyTop AS BIGINT = 9223372036854775807, @TypeId AS SMALLINT; +BEGIN TRY + DECLARE @Types TABLE ( + TypeId SMALLINT PRIMARY KEY, + Name VARCHAR (100)); + INSERT INTO @Types + EXECUTE dbo.GetUsedResourceTypes ; + DECLARE @Keys TABLE ( + TypeId SMALLINT, + SurrogateId BIGINT PRIMARY KEY (TypeId, SurrogateId)); + WHILE EXISTS (SELECT * + FROM @Types) + BEGIN + SET @TypeId = (SELECT TOP 1 TypeId + FROM @Types + ORDER BY TypeId); + INSERT INTO @Keys + SELECT @TypeId, + ResourceSurrogateId + FROM dbo.Resource + WHERE ResourceTypeId = @TypeId + AND TransactionId = @TransactionId; + DELETE @Types + WHERE TypeId = @TypeId; + END + IF @ReturnResourceKeysOnly = 0 + SELECT ResourceTypeId, + ResourceId, + ResourceSurrogateId, + Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash, + RequestMethod + FROM (SELECT TOP (@DummyTop) * + FROM @Keys) AS A + INNER JOIN + dbo.Resource AS B + ON ResourceTypeId = TypeId + AND ResourceSurrogateId = SurrogateId + WHERE IsHistory = 0 + OR @IncludeHistory = 1 + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + ELSE + SELECT ResourceTypeId, + ResourceId, + ResourceSurrogateId, + Version, + IsDeleted + FROM (SELECT TOP (@DummyTop) * + FROM @Keys) AS A + INNER JOIN + dbo.Resource AS B + ON ResourceTypeId = TypeId + AND ResourceSurrogateId = SurrogateId + WHERE IsHistory = 0 + OR @IncludeHistory = 1 + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetResourcesByTypeAndSurrogateIdRange +@ResourceTypeId SMALLINT, @StartId BIGINT, @EndId BIGINT, @GlobalEndId BIGINT=NULL, @IncludeHistory BIT=0, @IncludeDeleted BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetResourcesByTypeAndSurrogateIdRange', @Mode AS VARCHAR (100) = 'RT=' + isnull(CONVERT (VARCHAR, @ResourceTypeId), 'NULL') + ' S=' + isnull(CONVERT (VARCHAR, @StartId), 'NULL') + ' E=' + isnull(CONVERT (VARCHAR, @EndId), 'NULL') + ' GE=' + isnull(CONVERT (VARCHAR, @GlobalEndId), 'NULL') + ' HI=' + isnull(CONVERT (VARCHAR, @IncludeHistory), 'NULL') + ' DE' + isnull(CONVERT (VARCHAR, @IncludeDeleted), 'NULL'), @st AS DATETIME = getUTCdate(), @DummyTop AS BIGINT = 9223372036854775807; +BEGIN TRY + DECLARE @ResourceIds TABLE ( + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS PRIMARY KEY); + DECLARE @SurrogateIds TABLE ( + MaxSurrogateId BIGINT PRIMARY KEY); + IF @GlobalEndId IS NOT NULL + AND @IncludeHistory = 0 + BEGIN + INSERT INTO @ResourceIds + SELECT DISTINCT ResourceId + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + AND IsHistory = 1 + AND (IsDeleted = 0 + OR @IncludeDeleted = 1) + OPTION (MAXDOP 1); + IF @@rowcount > 0 + INSERT INTO @SurrogateIds + SELECT ResourceSurrogateId + FROM (SELECT ResourceId, + ResourceSurrogateId, + row_number() OVER (PARTITION BY ResourceId ORDER BY ResourceSurrogateId DESC) AS RowId + FROM dbo.Resource WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceId IN (SELECT TOP (@DummyTop) ResourceId + FROM @ResourceIds) + AND ResourceSurrogateId BETWEEN @StartId AND @GlobalEndId) AS A + WHERE RowId = 1 + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + END + SELECT ResourceTypeId, + ResourceId, + Version, + IsDeleted, + ResourceSurrogateId, + RequestMethod, + CONVERT (BIT, 1) AS IsMatch, + CONVERT (BIT, 0) AS IsPartial, + IsRawResourceMetaSet, + SearchParamHash, + RawResource + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + AND (IsHistory = 0 + OR @IncludeHistory = 1) + AND (IsDeleted = 0 + OR @IncludeDeleted = 1) + UNION ALL + SELECT ResourceTypeId, + ResourceId, + Version, + IsDeleted, + ResourceSurrogateId, + RequestMethod, + CONVERT (BIT, 1) AS IsMatch, + CONVERT (BIT, 0) AS IsPartial, + IsRawResourceMetaSet, + SearchParamHash, + RawResource + FROM @SurrogateIds + INNER JOIN + dbo.Resource + ON ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId = MaxSurrogateId + WHERE IsHistory = 1 + AND (IsDeleted = 0 + OR @IncludeDeleted = 1) + OPTION (MAXDOP 1); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetResourceSurrogateIdRanges +@ResourceTypeId SMALLINT, @StartId BIGINT, @EndId BIGINT, @RangeSize INT, @NumberOfRanges INT=100, @Up BIT=1 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetResourceSurrogateIdRanges', @Mode AS VARCHAR (100) = 'RT=' + isnull(CONVERT (VARCHAR, @ResourceTypeId), 'NULL') + ' S=' + isnull(CONVERT (VARCHAR, @StartId), 'NULL') + ' E=' + isnull(CONVERT (VARCHAR, @EndId), 'NULL') + ' R=' + isnull(CONVERT (VARCHAR, @RangeSize), 'NULL') + ' UP=' + isnull(CONVERT (VARCHAR, @Up), 'NULL'), @st AS DATETIME = getUTCdate(); +BEGIN TRY + IF @Up = 1 + SELECT RangeId, + min(ResourceSurrogateId), + max(ResourceSurrogateId), + count(*) + FROM (SELECT isnull(CONVERT (INT, (row_number() OVER (ORDER BY ResourceSurrogateId) - 1) / @RangeSize), 0) AS RangeId, + ResourceSurrogateId + FROM (SELECT TOP (@RangeSize * @NumberOfRanges) ResourceSurrogateId + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId >= @StartId + AND ResourceSurrogateId <= @EndId + ORDER BY ResourceSurrogateId) AS A) AS A + GROUP BY RangeId + OPTION (MAXDOP 1); + ELSE + SELECT RangeId, + min(ResourceSurrogateId), + max(ResourceSurrogateId), + count(*) + FROM (SELECT isnull(CONVERT (INT, (row_number() OVER (ORDER BY ResourceSurrogateId) - 1) / @RangeSize), 0) AS RangeId, + ResourceSurrogateId + FROM (SELECT TOP (@RangeSize * @NumberOfRanges) ResourceSurrogateId + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId >= @StartId + AND ResourceSurrogateId <= @EndId + ORDER BY ResourceSurrogateId DESC) AS A) AS A + GROUP BY RangeId + OPTION (MAXDOP 1); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetResourceVersions +@ResourceDateKeys dbo.ResourceDateKeyList READONLY +AS +SET NOCOUNT ON; +DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = 'GetResourceVersions', @Mode AS VARCHAR (100) = 'Rows=' + CONVERT (VARCHAR, (SELECT count(*) + FROM @ResourceDateKeys)), @DummyTop AS BIGINT = 9223372036854775807; +BEGIN TRY + SELECT A.ResourceTypeId, + A.ResourceId, + A.ResourceSurrogateId, + CASE WHEN EXISTS (SELECT * + FROM dbo.Resource AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId) THEN 0 WHEN isnull(U.Version, 1) - isnull(L.Version, 0) > 1 THEN isnull(U.Version, 1) - 1 ELSE 0 END AS Version + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceDateKeys) AS A OUTER APPLY (SELECT TOP 1 * + FROM dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.ResourceSurrogateId < A.ResourceSurrogateId + ORDER BY B.ResourceSurrogateId DESC) AS L OUTER APPLY (SELECT TOP 1 * + FROM dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.ResourceSurrogateId > A.ResourceSurrogateId + ORDER BY B.ResourceSurrogateId) AS U + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetSearchParamStatuses +AS +SET NOCOUNT ON; +SELECT SearchParamId, + Uri, + Status, + LastUpdated, + IsPartiallySupported +FROM dbo.SearchParam; + +GO +CREATE PROCEDURE [dbo].[GetTaskDetails] +@taskId VARCHAR (64) +AS +SET NOCOUNT ON; +SELECT TaskId, + QueueId, + Status, + TaskTypeId, + RunId, + IsCanceled, + RetryCount, + MaxRetryCount, + HeartbeatDateTime, + InputData, + TaskContext, + Result, + ParentTaskId +FROM [dbo].[TaskInfo] +WHERE TaskId = @taskId; + +GO +CREATE PROCEDURE dbo.GetTransactions +@StartNotInclusiveTranId BIGINT, @EndInclusiveTranId BIGINT, @EndDate DATETIME=NULL +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'ST=' + CONVERT (VARCHAR, @StartNotInclusiveTranId) + ' ET=' + CONVERT (VARCHAR, @EndInclusiveTranId) + ' ED=' + isnull(CONVERT (VARCHAR, @EndDate, 121), 'NULL'), @st AS DATETIME = getUTCdate(); +IF @EndDate IS NULL + SET @EndDate = getUTCdate(); +SELECT SurrogateIdRangeFirstValue, + VisibleDate, + InvisibleHistoryRemovedDate +FROM dbo.Transactions +WHERE SurrogateIdRangeFirstValue > @StartNotInclusiveTranId + AND SurrogateIdRangeFirstValue <= @EndInclusiveTranId + AND EndDate <= @EndDate +ORDER BY SurrogateIdRangeFirstValue; +EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; + +GO +CREATE PROCEDURE dbo.GetUsedResourceTypes +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetUsedResourceTypes', @Mode AS VARCHAR (100) = '', @st AS DATETIME = getUTCdate(); +BEGIN TRY + SELECT ResourceTypeId, + Name + FROM dbo.ResourceType AS A + WHERE EXISTS (SELECT * + FROM dbo.Resource AS B + WHERE B.ResourceTypeId = A.ResourceTypeId); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.HardDeleteResource +@ResourceTypeId SMALLINT, @ResourceId VARCHAR (64), @KeepCurrentVersion BIT, @IsResourceChangeCaptureEnabled BIT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (200) = 'RT=' + CONVERT (VARCHAR, @ResourceTypeId) + ' R=' + @ResourceId + ' V=' + CONVERT (VARCHAR, @KeepCurrentVersion) + ' CC=' + CONVERT (VARCHAR, @IsResourceChangeCaptureEnabled), @st AS DATETIME = getUTCdate(), @TransactionId AS BIGINT; +BEGIN TRY + IF @IsResourceChangeCaptureEnabled = 1 + EXECUTE dbo.MergeResourcesBeginTransaction @Count = 1, @TransactionId = @TransactionId OUTPUT; + IF @KeepCurrentVersion = 0 + BEGIN TRANSACTION; + DECLARE @SurrogateIds TABLE ( + ResourceSurrogateId BIGINT NOT NULL); + IF @IsResourceChangeCaptureEnabled = 1 + AND NOT EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = 'InvisibleHistory.IsEnabled' + AND Number = 0) + UPDATE dbo.Resource + SET IsDeleted = 1, + RawResource = 0xF, + SearchParamHash = NULL, + HistoryTransactionId = @TransactionId + OUTPUT deleted.ResourceSurrogateId INTO @SurrogateIds + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceId = @ResourceId + AND (@KeepCurrentVersion = 0 + OR IsHistory = 1) + AND RawResource <> 0xF; + ELSE + DELETE dbo.Resource + OUTPUT deleted.ResourceSurrogateId INTO @SurrogateIds + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceId = @ResourceId + AND (@KeepCurrentVersion = 0 + OR IsHistory = 1) + AND RawResource <> 0xF; + IF @KeepCurrentVersion = 0 + BEGIN + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.ResourceWriteClaim AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.ReferenceSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenText AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.StringSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.UriSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.NumberSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.QuantitySearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.DateTimeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.ReferenceTokenCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenTokenCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenDateTimeCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenQuantityCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenStringCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenNumberNumberCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + END + IF @@trancount > 0 + COMMIT TRANSACTION; + IF @IsResourceChangeCaptureEnabled = 1 + EXECUTE dbo.MergeResourcesCommitTransaction @TransactionId; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.InitDefrag +@QueueType TINYINT, @GroupId BIGINT, @DefragItems INT=NULL OUTPUT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'InitDefrag', @st AS DATETIME = getUTCdate(), @ObjectId AS INT, @msg AS VARCHAR (1000), @Rows AS INT, @MinFragPct AS INT = isnull((SELECT Number + FROM dbo.Parameters + WHERE Id = 'Defrag.MinFragPct'), 10), @MinSizeGB AS FLOAT = isnull((SELECT Number + FROM dbo.Parameters + WHERE Id = 'Defrag.MinSizeGB'), 0.1), @DefinitionsSorted AS StringList; +DECLARE @Mode AS VARCHAR (200) = 'G=' + CONVERT (VARCHAR, @GroupId) + ' MF=' + CONVERT (VARCHAR, @MinFragPct) + ' MS=' + CONVERT (VARCHAR, @MinSizeGB); +DECLARE @Definitions AS TABLE ( + Def VARCHAR (900) PRIMARY KEY, + FragGB FLOAT ); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + SELECT * + INTO #filter + FROM (SELECT object_id, + sum(reserved_page_count * 8.0 / 1024 / 1024) AS ReservedGB + FROM sys.dm_db_partition_stats AS A + WHERE object_id IN (SELECT object_id + FROM sys.objects + WHERE type = 'U' + AND name NOT IN ('EventLog')) + GROUP BY object_id) AS A + WHERE ReservedGB > @MinSizeGB; + WHILE EXISTS (SELECT * + FROM #filter) + BEGIN + SET @ObjectId = (SELECT TOP 1 object_id + FROM #filter + ORDER BY ReservedGB DESC); + INSERT INTO @Definitions + SELECT object_name(@ObjectId) + ';' + I.name + ';' + CONVERT (VARCHAR, partition_number) + ';' + CONVERT (VARCHAR, CASE WHEN EXISTS (SELECT * + FROM sys.partition_schemes AS PS + WHERE PS.data_space_id = I.data_space_id) THEN 1 ELSE 0 END) + ';' + CONVERT (VARCHAR, (SELECT sum(reserved_page_count) + FROM sys.dm_db_partition_stats AS S + WHERE S.object_id = A.object_id + AND S.index_id = A.index_id + AND S.partition_number = A.partition_number) * 8.0 / 1024 / 1024), + FragGB + FROM (SELECT object_id, + index_id, + partition_number, + A.avg_fragmentation_in_percent * A.page_count * 8.0 / 1024 / 1024 / 100 AS FragGB + FROM sys.dm_db_index_physical_stats(db_id(), @ObjectId, NULL, NULL, 'LIMITED') AS A + WHERE index_id > 0 + AND avg_fragmentation_in_percent >= @MinFragPct + AND A.page_count > 500) AS A + INNER JOIN + sys.indexes AS I + ON I.object_id = A.object_id + AND I.index_id = A.index_id; + SET @Rows = @@rowcount; + SET @msg = object_name(@ObjectId); + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'Run', @Mode = @Mode, @Target = '@Definitions', @Action = 'Insert', @Rows = @Rows, @Text = @msg; + DELETE #filter + WHERE object_id = @ObjectId; + END + INSERT INTO @DefinitionsSorted + SELECT Def + ';' + CONVERT (VARCHAR, FragGB) + FROM @Definitions + ORDER BY FragGB DESC; + SET @DefragItems = @@rowcount; + IF @DefragItems > 0 + EXECUTE dbo.EnqueueJobs @QueueType = @QueueType, @Definitions = @DefinitionsSorted, @GroupId = @GroupId, @ForceOneActiveJobGroup = 1; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.InitializeIndexProperties +AS +SET NOCOUNT ON; +INSERT INTO dbo.IndexProperties (TableName, IndexName, PropertyName, PropertyValue) +SELECT Tbl, + Ind, + 'DATA_COMPRESSION', + isnull(data_comp, 'NONE') +FROM (SELECT O.Name AS Tbl, + I.Name AS Ind, + (SELECT TOP 1 CASE WHEN data_compression_desc = 'PAGE' THEN 'PAGE' END + FROM sys.partitions AS P + WHERE P.object_id = I.object_id + AND I.index_id = P.index_id) AS data_comp + FROM sys.indexes AS I + INNER JOIN + sys.objects AS O + ON O.object_id = I.object_id + WHERE O.type = 'u' + AND EXISTS (SELECT * + FROM sys.partition_schemes AS PS + WHERE PS.data_space_id = I.data_space_id + AND name = 'PartitionScheme_ResourceTypeId')) AS A +WHERE NOT EXISTS (SELECT * + FROM dbo.IndexProperties + WHERE TableName = Tbl + AND IndexName = Ind); + +GO +CREATE PROCEDURE dbo.LogEvent +@Process VARCHAR (100), @Status VARCHAR (10), @Mode VARCHAR (200)=NULL, @Action VARCHAR (20)=NULL, @Target VARCHAR (100)=NULL, @Rows BIGINT=NULL, @Start DATETIME=NULL, @Text NVARCHAR (3500)=NULL, @EventId BIGINT=NULL OUTPUT, @Retry INT=NULL +AS +SET NOCOUNT ON; +DECLARE @ErrorNumber AS INT = error_number(), @ErrorMessage AS VARCHAR (1000) = '', @TranCount AS INT = @@trancount, @DoWork AS BIT = 0, @NumberAdded AS BIT; +IF @ErrorNumber IS NOT NULL + OR @Status IN ('Warn', 'Error') + SET @DoWork = 1; +IF @DoWork = 0 + SET @DoWork = CASE WHEN EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = isnull(@Process, '') + AND Char = 'LogEvent') THEN 1 ELSE 0 END; +IF @DoWork = 0 + RETURN; +IF @ErrorNumber IS NOT NULL + SET @ErrorMessage = CASE WHEN @Retry IS NOT NULL THEN 'Retry ' + CONVERT (VARCHAR, @Retry) + ', ' ELSE '' END + 'Error ' + CONVERT (VARCHAR, error_number()) + ': ' + CONVERT (VARCHAR (1000), error_message()) + ', Level ' + CONVERT (VARCHAR, error_severity()) + ', State ' + CONVERT (VARCHAR, error_state()) + CASE WHEN error_procedure() IS NOT NULL THEN ', Procedure ' + error_procedure() ELSE '' END + ', Line ' + CONVERT (VARCHAR, error_line()); +IF @TranCount > 0 + AND @ErrorNumber IS NOT NULL + ROLLBACK; +IF databasepropertyex(db_name(), 'UpdateAbility') = 'READ_WRITE' + BEGIN + INSERT INTO dbo.EventLog (Process, Status, Mode, Action, Target, Rows, Milliseconds, EventDate, EventText, SPID, HostName) + SELECT @Process, + @Status, + @Mode, + @Action, + @Target, + @Rows, + datediff(millisecond, @Start, getUTCdate()), + getUTCdate() AS EventDate, + CASE WHEN @ErrorNumber IS NULL THEN @Text ELSE @ErrorMessage + CASE WHEN isnull(@Text, '') <> '' THEN '. ' + @Text ELSE '' END END AS Text, + @@SPID, + host_name() AS HostName; + SET @EventId = scope_identity(); + END +IF @TranCount > 0 + AND @ErrorNumber IS NOT NULL + BEGIN TRANSACTION; + +GO +CREATE PROCEDURE dbo.LogSchemaMigrationProgress +@message VARCHAR (MAX) +AS +INSERT INTO dbo.SchemaMigrationProgress (Message) +VALUES (@message); + +GO +CREATE PROCEDURE dbo.MergeResources +@AffectedRows INT=0 OUTPUT, @RaiseExceptionOnConflict BIT=1, @IsResourceChangeCaptureEnabled BIT=0, @TransactionId BIGINT=NULL, @SingleTransaction BIT=1, @Resources dbo.ResourceList READONLY, @ResourceWriteClaims dbo.ResourceWriteClaimList READONLY, @ReferenceSearchParams dbo.ReferenceSearchParamList READONLY, @TokenSearchParams dbo.TokenSearchParamList READONLY, @TokenTexts dbo.TokenTextList READONLY, @StringSearchParams dbo.StringSearchParamList READONLY, @UriSearchParams dbo.UriSearchParamList READONLY, @NumberSearchParams dbo.NumberSearchParamList READONLY, @QuantitySearchParams dbo.QuantitySearchParamList READONLY, @DateTimeSearchParms dbo.DateTimeSearchParamList READONLY, @ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY, @TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY, @TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY, @TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY, @TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY, @TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY +AS +SET NOCOUNT ON; +DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = object_name(@@procid), @DummyTop AS BIGINT = 9223372036854775807, @InitialTranCount AS INT = @@trancount, @IsRetry AS BIT = 0; +DECLARE @Mode AS VARCHAR (200) = isnull((SELECT 'RT=[' + CONVERT (VARCHAR, min(ResourceTypeId)) + ',' + CONVERT (VARCHAR, max(ResourceTypeId)) + '] Sur=[' + CONVERT (VARCHAR, min(ResourceSurrogateId)) + ',' + CONVERT (VARCHAR, max(ResourceSurrogateId)) + '] V=' + CONVERT (VARCHAR, max(Version)) + ' Rows=' + CONVERT (VARCHAR, count(*)) + FROM @Resources), 'Input=Empty'); +SET @Mode += ' E=' + CONVERT (VARCHAR, @RaiseExceptionOnConflict) + ' CC=' + CONVERT (VARCHAR, @IsResourceChangeCaptureEnabled) + ' IT=' + CONVERT (VARCHAR, @InitialTranCount) + ' T=' + isnull(CONVERT (VARCHAR, @TransactionId), 'NULL'); +SET @AffectedRows = 0; +BEGIN TRY + DECLARE @Existing AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + SurrogateId BIGINT NOT NULL PRIMARY KEY (ResourceTypeId, SurrogateId)); + DECLARE @ResourceInfos AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + SurrogateId BIGINT NOT NULL, + Version INT NOT NULL, + KeepHistory BIT NOT NULL, + PreviousVersion INT NULL, + PreviousSurrogateId BIGINT NULL PRIMARY KEY (ResourceTypeId, SurrogateId)); + DECLARE @PreviousSurrogateIds AS TABLE ( + TypeId SMALLINT NOT NULL, + SurrogateId BIGINT NOT NULL PRIMARY KEY (TypeId, SurrogateId), + KeepHistory BIT ); + IF @SingleTransaction = 0 + AND isnull((SELECT Number + FROM dbo.Parameters + WHERE Id = 'MergeResources.NoTransaction.IsEnabled'), 0) = 0 + SET @SingleTransaction = 1; + SET @Mode += ' ST=' + CONVERT (VARCHAR, @SingleTransaction); + IF @InitialTranCount = 0 + BEGIN + IF EXISTS (SELECT * + FROM @Resources AS A + INNER JOIN + dbo.Resource AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId) + BEGIN + BEGIN TRANSACTION; + INSERT INTO @Existing (ResourceTypeId, SurrogateId) + SELECT B.ResourceTypeId, + B.ResourceSurrogateId + FROM (SELECT TOP (@DummyTop) * + FROM @Resources) AS A + INNER JOIN + dbo.Resource AS B WITH (ROWLOCK, HOLDLOCK) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + WHERE B.IsHistory = 0 + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + IF @@rowcount > 0 + SET @IsRetry = 1; + IF @IsRetry = 0 + COMMIT TRANSACTION; + END + END + SET @Mode += ' R=' + CONVERT (VARCHAR, @IsRetry); + IF @SingleTransaction = 1 + AND @@trancount = 0 + BEGIN TRANSACTION; + IF @IsRetry = 0 + BEGIN + INSERT INTO @ResourceInfos (ResourceTypeId, SurrogateId, Version, KeepHistory, PreviousVersion, PreviousSurrogateId) + SELECT A.ResourceTypeId, + A.ResourceSurrogateId, + A.Version, + A.KeepHistory, + B.Version, + B.ResourceSurrogateId + FROM (SELECT TOP (@DummyTop) * + FROM @Resources + WHERE HasVersionToCompare = 1) AS A + LEFT OUTER JOIN + dbo.Resource AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.IsHistory = 0 + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + IF @RaiseExceptionOnConflict = 1 + AND EXISTS (SELECT * + FROM @ResourceInfos + WHERE PreviousVersion IS NOT NULL + AND Version <> PreviousVersion + 1) + THROW 50409, 'Resource has been recently updated or added, please compare the resource content in code for any duplicate updates', 1; + INSERT INTO @PreviousSurrogateIds + SELECT ResourceTypeId, + PreviousSurrogateId, + KeepHistory + FROM @ResourceInfos + WHERE PreviousSurrogateId IS NOT NULL; + IF @@rowcount > 0 + BEGIN + UPDATE dbo.Resource + SET IsHistory = 1 + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId + AND KeepHistory = 1); + SET @AffectedRows += @@rowcount; + IF @IsResourceChangeCaptureEnabled = 1 + AND NOT EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = 'InvisibleHistory.IsEnabled' + AND Number = 0) + UPDATE dbo.Resource + SET IsHistory = 1, + RawResource = 0xF, + SearchParamHash = NULL, + HistoryTransactionId = @TransactionId + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId + AND KeepHistory = 0); + ELSE + DELETE dbo.Resource + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId + AND KeepHistory = 0); + SET @AffectedRows += @@rowcount; + DELETE dbo.ResourceWriteClaim + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.ReferenceSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenText + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.StringSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.UriSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.NumberSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.QuantitySearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.DateTimeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.ReferenceTokenCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenTokenCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenDateTimeCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenQuantityCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenStringCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenNumberNumberCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + END + INSERT INTO dbo.Resource (ResourceTypeId, ResourceId, Version, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, RawResource, IsRawResourceMetaSet, SearchParamHash, TransactionId) + SELECT ResourceTypeId, + ResourceId, + Version, + IsHistory, + ResourceSurrogateId, + IsDeleted, + RequestMethod, + RawResource, + IsRawResourceMetaSet, + SearchParamHash, + @TransactionId + FROM @Resources; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) + SELECT ResourceSurrogateId, + ClaimTypeId, + ClaimValue + FROM @ResourceWriteClaims; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri, + ReferenceResourceTypeId, + ReferenceResourceId, + ReferenceResourceVersion + FROM @ReferenceSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + Code, + CodeOverflow + FROM @TokenSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text + FROM @TokenTexts; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text, + TextOverflow, + IsMin, + IsMax + FROM @StringSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Uri + FROM @UriSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SingleValue, + LowValue, + HighValue + FROM @NumberSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + QuantityCodeId, + SingleValue, + LowValue, + HighValue + FROM @QuantitySearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + StartDateTime, + EndDateTime, + IsLongerThanADay, + IsMin, + IsMax + FROM @DateTimeSearchParms; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri1, + ReferenceResourceTypeId1, + ReferenceResourceId1, + ReferenceResourceVersion1, + SystemId2, + Code2, + CodeOverflow2 + FROM @ReferenceTokenCompositeSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SystemId2, + Code2, + CodeOverflow2 + FROM @TokenTokenCompositeSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + StartDateTime2, + EndDateTime2, + IsLongerThanADay2 + FROM @TokenDateTimeCompositeSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + SystemId2, + QuantityCodeId2, + LowValue2, + HighValue2 + FROM @TokenQuantityCompositeSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + Text2, + TextOverflow2 + FROM @TokenStringCompositeSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + LowValue2, + HighValue2, + SingleValue3, + LowValue3, + HighValue3, + HasRange + FROM @TokenNumberNumberCompositeSearchParams; + SET @AffectedRows += @@rowcount; + END + ELSE + BEGIN + INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) + SELECT ResourceSurrogateId, + ClaimTypeId, + ClaimValue + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceWriteClaims) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.ResourceWriteClaim AS C + WHERE C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri, + ReferenceResourceTypeId, + ReferenceResourceId, + ReferenceResourceVersion + FROM (SELECT TOP (@DummyTop) * + FROM @ReferenceSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.ReferenceSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + Code, + CodeOverflow + FROM (SELECT TOP (@DummyTop) * + FROM @TokenSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text + FROM (SELECT TOP (@DummyTop) * + FROM @TokenTexts) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text, + TextOverflow, + IsMin, + IsMax + FROM (SELECT TOP (@DummyTop) * + FROM @StringSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenText AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Uri + FROM (SELECT TOP (@DummyTop) * + FROM @UriSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.UriSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SingleValue, + LowValue, + HighValue + FROM (SELECT TOP (@DummyTop) * + FROM @NumberSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.NumberSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + QuantityCodeId, + SingleValue, + LowValue, + HighValue + FROM (SELECT TOP (@DummyTop) * + FROM @QuantitySearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.QuantitySearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + StartDateTime, + EndDateTime, + IsLongerThanADay, + IsMin, + IsMax + FROM (SELECT TOP (@DummyTop) * + FROM @DateTimeSearchParms) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri1, + ReferenceResourceTypeId1, + ReferenceResourceId1, + ReferenceResourceVersion1, + SystemId2, + Code2, + CodeOverflow2 + FROM (SELECT TOP (@DummyTop) * + FROM @ReferenceTokenCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.DateTimeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SystemId2, + Code2, + CodeOverflow2 + FROM (SELECT TOP (@DummyTop) * + FROM @TokenTokenCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenTokenCompositeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + StartDateTime2, + EndDateTime2, + IsLongerThanADay2 + FROM (SELECT TOP (@DummyTop) * + FROM @TokenDateTimeCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenDateTimeCompositeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + SystemId2, + QuantityCodeId2, + LowValue2, + HighValue2 + FROM (SELECT TOP (@DummyTop) * + FROM @TokenQuantityCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenQuantityCompositeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + Text2, + TextOverflow2 + FROM (SELECT TOP (@DummyTop) * + FROM @TokenStringCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenStringCompositeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + LowValue2, + HighValue2, + SingleValue3, + LowValue3, + HighValue3, + HasRange + FROM (SELECT TOP (@DummyTop) * + FROM @TokenNumberNumberCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenNumberNumberCompositeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + END + IF @IsResourceChangeCaptureEnabled = 1 + EXECUTE dbo.CaptureResourceIdsForChanges @Resources; + IF @TransactionId IS NOT NULL + EXECUTE dbo.MergeResourcesCommitTransaction @TransactionId; + IF @InitialTranCount = 0 + AND @@trancount > 0 + COMMIT TRANSACTION; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @AffectedRows; +END TRY +BEGIN CATCH + IF @InitialTranCount = 0 + AND @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + IF @RaiseExceptionOnConflict = 1 + AND error_number() IN (2601, 2627) + AND error_message() LIKE '%''dbo.Resource''%' + THROW 50409, 'Resource has been recently updated or added, please compare the resource content in code for any duplicate updates', 1; + ELSE + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesAdvanceTransactionVisibility +@AffectedRows INT=0 OUTPUT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = '', @st AS DATETIME = getUTCdate(), @msg AS VARCHAR (1000), @MaxTransactionId AS BIGINT, @MinTransactionId AS BIGINT, @MinNotCompletedTransactionId AS BIGINT, @CurrentTransactionId AS BIGINT; +SET @AffectedRows = 0; +BEGIN TRY + EXECUTE dbo.MergeResourcesGetTransactionVisibility @MinTransactionId OUTPUT; + SET @MinTransactionId += 1; + SET @CurrentTransactionId = (SELECT TOP 1 SurrogateIdRangeFirstValue + FROM dbo.Transactions + ORDER BY SurrogateIdRangeFirstValue DESC); + SET @MinNotCompletedTransactionId = isnull((SELECT TOP 1 SurrogateIdRangeFirstValue + FROM dbo.Transactions + WHERE IsCompleted = 0 + AND SurrogateIdRangeFirstValue BETWEEN @MinTransactionId AND @CurrentTransactionId + ORDER BY SurrogateIdRangeFirstValue), @CurrentTransactionId + 1); + SET @MaxTransactionId = (SELECT TOP 1 SurrogateIdRangeFirstValue + FROM dbo.Transactions + WHERE IsCompleted = 1 + AND SurrogateIdRangeFirstValue BETWEEN @MinTransactionId AND @CurrentTransactionId + AND SurrogateIdRangeFirstValue < @MinNotCompletedTransactionId + ORDER BY SurrogateIdRangeFirstValue DESC); + IF @MaxTransactionId >= @MinTransactionId + BEGIN + UPDATE A + SET IsVisible = 1, + VisibleDate = getUTCdate() + FROM dbo.Transactions AS A WITH (INDEX (1)) + WHERE SurrogateIdRangeFirstValue BETWEEN @MinTransactionId AND @CurrentTransactionId + AND SurrogateIdRangeFirstValue <= @MaxTransactionId; + SET @AffectedRows += @@rowcount; + END + SET @msg = 'Min=' + CONVERT (VARCHAR, @MinTransactionId) + ' C=' + CONVERT (VARCHAR, @CurrentTransactionId) + ' MinNC=' + CONVERT (VARCHAR, @MinNotCompletedTransactionId) + ' Max=' + CONVERT (VARCHAR, @MaxTransactionId); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @AffectedRows, @Text = @msg; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesBeginTransaction +@Count INT, @TransactionId BIGINT OUTPUT, @SequenceRangeFirstValue INT=NULL OUTPUT, @HeartbeatDate DATETIME=NULL +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'MergeResourcesBeginTransaction', @Mode AS VARCHAR (200) = 'Cnt=' + CONVERT (VARCHAR, @Count), @st AS DATETIME = getUTCdate(), @FirstValueVar AS SQL_VARIANT, @LastValueVar AS SQL_VARIANT; +BEGIN TRY + SET @TransactionId = NULL; + IF @@trancount > 0 + RAISERROR ('MergeResourcesBeginTransaction cannot be called inside outer transaction.', 18, 127); + SET @FirstValueVar = NULL; + WHILE @FirstValueVar IS NULL + BEGIN + EXECUTE sys.sp_sequence_get_range @sequence_name = 'dbo.ResourceSurrogateIdUniquifierSequence', @range_size = @Count, @range_first_value = @FirstValueVar OUTPUT, @range_last_value = @LastValueVar OUTPUT; + SET @SequenceRangeFirstValue = CONVERT (INT, @FirstValueVar); + IF @SequenceRangeFirstValue > CONVERT (INT, @LastValueVar) + SET @FirstValueVar = NULL; + END + SET @TransactionId = datediff_big(millisecond, '0001-01-01', sysUTCdatetime()) * 80000 + @SequenceRangeFirstValue; + INSERT INTO dbo.Transactions (SurrogateIdRangeFirstValue, SurrogateIdRangeLastValue, HeartbeatDate) + SELECT @TransactionId, + @TransactionId + @Count - 1, + isnull(@HeartbeatDate, getUTCdate()); +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + IF @@trancount > 0 + ROLLBACK; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesCommitTransaction +@TransactionId BIGINT, @FailureReason VARCHAR (MAX)=NULL, @OverrideIsControlledByClientCheck BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'MergeResourcesCommitTransaction', @st AS DATETIME = getUTCdate(), @InitialTranCount AS INT = @@trancount, @IsCompletedBefore AS BIT, @Rows AS INT, @msg AS VARCHAR (1000); +DECLARE @Mode AS VARCHAR (200) = 'TR=' + CONVERT (VARCHAR, @TransactionId) + ' OC=' + isnull(CONVERT (VARCHAR, @OverrideIsControlledByClientCheck), 'NULL'); +BEGIN TRY + IF @InitialTranCount = 0 + BEGIN TRANSACTION; + UPDATE dbo.Transactions + SET IsCompleted = 1, + @IsCompletedBefore = IsCompleted, + EndDate = getUTCdate(), + IsSuccess = CASE WHEN @FailureReason IS NULL THEN 1 ELSE 0 END, + FailureReason = @FailureReason + WHERE SurrogateIdRangeFirstValue = @TransactionId + AND (IsControlledByClient = 1 + OR @OverrideIsControlledByClientCheck = 1); + SET @Rows = @@rowcount; + IF @Rows = 0 + BEGIN + SET @msg = 'Transaction [' + CONVERT (VARCHAR (20), @TransactionId) + '] is not controlled by client or does not exist.'; + RAISERROR (@msg, 18, 127); + END + IF @IsCompletedBefore = 1 + BEGIN + IF @InitialTranCount = 0 + ROLLBACK; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows, @Target = '@IsCompletedBefore', @Text = '=1'; + RETURN; + END + IF @InitialTranCount = 0 + COMMIT TRANSACTION; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF @InitialTranCount = 0 + AND @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesDeleteInvisibleHistory +@TransactionId BIGINT, @AffectedRows INT=NULL OUTPUT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'T=' + CONVERT (VARCHAR, @TransactionId), @st AS DATETIME = getUTCdate(), @TypeId AS SMALLINT; +SET @AffectedRows = 0; +BEGIN TRY + DECLARE @Types TABLE ( + TypeId SMALLINT PRIMARY KEY, + Name VARCHAR (100)); + INSERT INTO @Types + EXECUTE dbo.GetUsedResourceTypes ; + WHILE EXISTS (SELECT * + FROM @Types) + BEGIN + SET @TypeId = (SELECT TOP 1 TypeId + FROM @Types + ORDER BY TypeId); + DELETE dbo.Resource + WHERE ResourceTypeId = @TypeId + AND HistoryTransactionId = @TransactionId + AND RawResource = 0xF; + SET @AffectedRows += @@rowcount; + DELETE @Types + WHERE TypeId = @TypeId; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @AffectedRows; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesGetTimeoutTransactions +@TimeoutSec INT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'T=' + CONVERT (VARCHAR, @TimeoutSec), @st AS DATETIME = getUTCdate(), @MinTransactionId AS BIGINT; +BEGIN TRY + EXECUTE dbo.MergeResourcesGetTransactionVisibility @MinTransactionId OUTPUT; + SELECT SurrogateIdRangeFirstValue + FROM dbo.Transactions + WHERE SurrogateIdRangeFirstValue > @MinTransactionId + AND IsCompleted = 0 + AND datediff(second, HeartbeatDate, getUTCdate()) > @TimeoutSec + ORDER BY SurrogateIdRangeFirstValue; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesGetTransactionVisibility +@TransactionId BIGINT OUTPUT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = '', @st AS DATETIME = getUTCdate(); +SET @TransactionId = isnull((SELECT TOP 1 SurrogateIdRangeFirstValue + FROM dbo.Transactions + WHERE IsVisible = 1 + ORDER BY SurrogateIdRangeFirstValue DESC), -1); +EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount, @Text = @TransactionId; + +GO +CREATE PROCEDURE dbo.MergeResourcesPutTransactionHeartbeat +@TransactionId BIGINT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'MergeResourcesPutTransactionHeartbeat', @Mode AS VARCHAR (100) = 'TR=' + CONVERT (VARCHAR, @TransactionId); +BEGIN TRY + UPDATE dbo.Transactions + SET HeartbeatDate = getUTCdate() + WHERE SurrogateIdRangeFirstValue = @TransactionId + AND IsControlledByClient = 1; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesPutTransactionInvisibleHistory +@TransactionId BIGINT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'TR=' + CONVERT (VARCHAR, @TransactionId), @st AS DATETIME = getUTCdate(); +BEGIN TRY + UPDATE dbo.Transactions + SET InvisibleHistoryRemovedDate = getUTCdate() + WHERE SurrogateIdRangeFirstValue = @TransactionId + AND InvisibleHistoryRemovedDate IS NULL; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.PutJobCancelation +@QueueType TINYINT, @GroupId BIGINT=NULL, @JobId BIGINT=NULL +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'PutJobCancelation', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' G=' + isnull(CONVERT (VARCHAR, @GroupId), 'NULL') + ' J=' + isnull(CONVERT (VARCHAR, @JobId), 'NULL'), @st AS DATETIME = getUTCdate(), @Rows AS INT, @PartitionId AS TINYINT = @JobId % 16; +BEGIN TRY + IF @JobId IS NULL + AND @GroupId IS NULL + RAISERROR ('@JobId = NULL and @GroupId = NULL', 18, 127); + IF @JobId IS NOT NULL + BEGIN + UPDATE dbo.JobQueue + SET Status = 4, + EndDate = getUTCdate(), + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()) + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Status = 0; + SET @Rows = @@rowcount; + IF @Rows = 0 + BEGIN + UPDATE dbo.JobQueue + SET CancelRequested = 1 + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Status = 1; + SET @Rows = @@rowcount; + END + END + ELSE + BEGIN + UPDATE dbo.JobQueue + SET Status = 4, + EndDate = getUTCdate(), + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()) + WHERE QueueType = @QueueType + AND GroupId = @GroupId + AND Status = 0; + SET @Rows = @@rowcount; + UPDATE dbo.JobQueue + SET CancelRequested = 1 + WHERE QueueType = @QueueType + AND GroupId = @GroupId + AND Status = 1; + SET @Rows += @@rowcount; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.PutJobHeartbeat +@QueueType TINYINT, @JobId BIGINT, @Version BIGINT, @Data BIGINT=NULL, @CurrentResult VARCHAR (MAX)=NULL, @CancelRequested BIT=0 OUTPUT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'PutJobHeartbeat', @Mode AS VARCHAR (100), @st AS DATETIME = getUTCdate(), @Rows AS INT = 0, @PartitionId AS TINYINT = @JobId % 16; +SET @Mode = 'Q=' + CONVERT (VARCHAR, @QueueType) + ' J=' + CONVERT (VARCHAR, @JobId) + ' P=' + CONVERT (VARCHAR, @PartitionId) + ' V=' + CONVERT (VARCHAR, @Version) + ' D=' + isnull(CONVERT (VARCHAR, @Data), 'NULL'); +BEGIN TRY + IF @CurrentResult IS NULL + UPDATE dbo.JobQueue + SET @CancelRequested = CancelRequested, + HeartbeatDate = getUTCdate(), + Data = isnull(@Data, Data) + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Status = 1 + AND Version = @Version; + ELSE + UPDATE dbo.JobQueue + SET @CancelRequested = CancelRequested, + HeartbeatDate = getUTCdate(), + Data = isnull(@Data, Data), + Result = @CurrentResult + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Status = 1 + AND Version = @Version; + SET @Rows = @@rowcount; + IF @Rows = 0 + AND NOT EXISTS (SELECT * + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Version = @Version + AND Status IN (2, 3, 4)) + BEGIN + IF EXISTS (SELECT * + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId) + THROW 50412, 'Precondition failed', 1; + ELSE + THROW 50404, 'Job record not found', 1; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.PutJobStatus +@QueueType TINYINT, @JobId BIGINT, @Version BIGINT, @Failed BIT, @Data BIGINT, @FinalResult VARCHAR (MAX), @RequestCancellationOnFailure BIT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'PutJobStatus', @Mode AS VARCHAR (100), @st AS DATETIME = getUTCdate(), @Rows AS INT = 0, @PartitionId AS TINYINT = @JobId % 16, @GroupId AS BIGINT; +SET @Mode = 'Q=' + CONVERT (VARCHAR, @QueueType) + ' J=' + CONVERT (VARCHAR, @JobId) + ' P=' + CONVERT (VARCHAR, @PartitionId) + ' V=' + CONVERT (VARCHAR, @Version) + ' F=' + CONVERT (VARCHAR, @Failed) + ' R=' + isnull(@FinalResult, 'NULL'); +BEGIN TRY + UPDATE dbo.JobQueue + SET EndDate = getUTCdate(), + Status = CASE WHEN @Failed = 1 THEN 3 WHEN CancelRequested = 1 THEN 4 ELSE 2 END, + Data = @Data, + Result = @FinalResult, + @GroupId = GroupId + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Status = 1 + AND Version = @Version; + SET @Rows = @@rowcount; + IF @Rows = 0 + BEGIN + SET @GroupId = (SELECT GroupId + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Version = @Version + AND Status IN (2, 3, 4)); + IF @GroupId IS NULL + IF EXISTS (SELECT * + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId) + THROW 50412, 'Precondition failed', 1; + ELSE + THROW 50404, 'Job record not found', 1; + END + IF @Failed = 1 + AND @RequestCancellationOnFailure = 1 + EXECUTE dbo.PutJobCancelation @QueueType = @QueueType, @GroupId = @GroupId; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.ReadResource +@resourceTypeId SMALLINT, @resourceId VARCHAR (64), @version INT=NULL +AS +SET NOCOUNT ON; +IF (@version IS NULL) + BEGIN + SELECT ResourceSurrogateId, + Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM dbo.Resource + WHERE ResourceTypeId = @resourceTypeId + AND ResourceId = @resourceId + AND IsHistory = 0; + END +ELSE + BEGIN + SELECT ResourceSurrogateId, + Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM dbo.Resource + WHERE ResourceTypeId = @resourceTypeId + AND ResourceId = @resourceId + AND Version = @version; + END + +GO +CREATE PROCEDURE dbo.ReindexResource_2 +@resourceTypeId SMALLINT, @resourceId VARCHAR (64), @eTag INT=NULL, @searchParamHash VARCHAR (64), @resourceWriteClaims dbo.BulkResourceWriteClaimTableType_1 READONLY, @compartmentAssignments dbo.BulkCompartmentAssignmentTableType_1 READONLY, @referenceSearchParams dbo.BulkReferenceSearchParamTableType_1 READONLY, @tokenSearchParams dbo.BulkTokenSearchParamTableType_2 READONLY, @tokenTextSearchParams dbo.BulkTokenTextTableType_1 READONLY, @stringSearchParams dbo.BulkStringSearchParamTableType_2 READONLY, @numberSearchParams dbo.BulkNumberSearchParamTableType_1 READONLY, @quantitySearchParams dbo.BulkQuantitySearchParamTableType_1 READONLY, @uriSearchParams dbo.BulkUriSearchParamTableType_1 READONLY, @dateTimeSearchParms dbo.BulkDateTimeSearchParamTableType_2 READONLY, @referenceTokenCompositeSearchParams dbo.BulkReferenceTokenCompositeSearchParamTableType_2 READONLY, @tokenTokenCompositeSearchParams dbo.BulkTokenTokenCompositeSearchParamTableType_2 READONLY, @tokenDateTimeCompositeSearchParams dbo.BulkTokenDateTimeCompositeSearchParamTableType_2 READONLY, @tokenQuantityCompositeSearchParams dbo.BulkTokenQuantityCompositeSearchParamTableType_2 READONLY, @tokenStringCompositeSearchParams dbo.BulkTokenStringCompositeSearchParamTableType_2 READONLY, @tokenNumberNumberCompositeSearchParams dbo.BulkTokenNumberNumberCompositeSearchParamTableType_2 READONLY +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @resourceSurrogateId AS BIGINT; +DECLARE @version AS BIGINT; +SELECT @resourceSurrogateId = ResourceSurrogateId, + @version = Version +FROM dbo.Resource WITH (UPDLOCK, HOLDLOCK) +WHERE ResourceTypeId = @resourceTypeId + AND ResourceId = @resourceId + AND IsHistory = 0; +IF (@etag IS NOT NULL + AND @etag <> @version) + BEGIN + THROW 50412, 'Precondition failed', 1; + END +UPDATE dbo.Resource +SET SearchParamHash = @searchParamHash +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.ResourceWriteClaim +WHERE ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.CompartmentAssignment +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.ReferenceSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenText +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.StringSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.UriSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.NumberSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.QuantitySearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.DateTimeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.ReferenceTokenCompositeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenTokenCompositeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenDateTimeCompositeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenQuantityCompositeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenStringCompositeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenNumberNumberCompositeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) +SELECT @resourceSurrogateId, + ClaimTypeId, + ClaimValue +FROM @resourceWriteClaims; +INSERT INTO dbo.CompartmentAssignment (ResourceTypeId, ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + CompartmentTypeId, + ReferenceResourceId, + 0 +FROM @compartmentAssignments; +INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + BaseUri, + ReferenceResourceTypeId, + ReferenceResourceId, + ReferenceResourceVersion, + 0 +FROM @referenceSearchParams; +INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId, + Code, + CodeOverflow, + 0 +FROM @tokenSearchParams; +INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + Text, + 0 +FROM @tokenTextSearchParams; +INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsHistory, IsMin, IsMax) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + Text, + TextOverflow, + 0, + IsMin, + IsMax +FROM @stringSearchParams; +INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + Uri, + 0 +FROM @uriSearchParams; +INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SingleValue, + LowValue, + HighValue, + 0 +FROM @numberSearchParams; +INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId, + QuantityCodeId, + SingleValue, + LowValue, + HighValue, + 0 +FROM @quantitySearchParams; +INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsHistory, IsMin, IsMax) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + StartDateTime, + EndDateTime, + IsLongerThanADay, + 0, + IsMin, + IsMax +FROM @dateTimeSearchParms; +INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + BaseUri1, + ReferenceResourceTypeId1, + ReferenceResourceId1, + ReferenceResourceVersion1, + SystemId2, + Code2, + CodeOverflow2, + 0 +FROM @referenceTokenCompositeSearchParams; +INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SystemId2, + Code2, + CodeOverflow2, + 0 +FROM @tokenTokenCompositeSearchParams; +INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + StartDateTime2, + EndDateTime2, + IsLongerThanADay2, + 0 +FROM @tokenDateTimeCompositeSearchParams; +INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + SystemId2, + QuantityCodeId2, + LowValue2, + HighValue2, + 0 +FROM @tokenQuantityCompositeSearchParams; +INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + Text2, + TextOverflow2, + 0 +FROM @tokenStringCompositeSearchParams; +INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + LowValue2, + HighValue2, + SingleValue3, + LowValue3, + HighValue3, + HasRange, + 0 +FROM @tokenNumberNumberCompositeSearchParams; +COMMIT TRANSACTION; + +GO +CREATE OR ALTER PROCEDURE dbo.RemovePartitionFromResourceChanges_2 +@partitionNumberToSwitchOut INT, @partitionBoundaryToMerge DATETIME2 (7) +AS +BEGIN + TRUNCATE TABLE dbo.ResourceChangeDataStaging; + ALTER TABLE dbo.ResourceChangeData SWITCH PARTITION @partitionNumberToSwitchOut TO dbo.ResourceChangeDataStaging; + ALTER PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp( ) + MERGE RANGE (@partitionBoundaryToMerge); + TRUNCATE TABLE dbo.ResourceChangeDataStaging; +END + +GO +CREATE PROCEDURE dbo.ResetTask_2 +@taskId VARCHAR (64), @runId VARCHAR (50), @result VARCHAR (MAX) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +DECLARE @retryCount AS SMALLINT = NULL; +IF NOT EXISTS (SELECT * + FROM dbo.TaskInfo + WHERE TaskId = @taskId + AND RunId = @runId) + BEGIN + THROW 50404, 'Task not exist or runid not match', 1; + END +UPDATE dbo.TaskInfo +SET Status = 3, + EndDateTime = SYSUTCDATETIME(), + Result = @result, + @retryCount = retryCount +WHERE TaskId = @taskId + AND RunId = @runId + AND (MaxRetryCount <> -1 + AND RetryCount >= MaxRetryCount); +IF @retryCount IS NULL + UPDATE dbo.TaskInfo + SET Status = 1, + Result = @result, + RetryCount = RetryCount + 1, + RestartInfo = ISNULL(RestartInfo, '') + ' Prev: Worker=' + Worker + ' Start=' + CONVERT (VARCHAR, StartDateTime, 121) + WHERE TaskId = @taskId + AND RunId = @runId + AND Status <> 3 + AND (MaxRetryCount = -1 + OR RetryCount < MaxRetryCount); +EXECUTE dbo.GetTaskDetails @TaskId = @taskId; + +GO +CREATE PROCEDURE dbo.SwitchPartitionsIn +@Tbl VARCHAR (100) +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'SwitchPartitionsIn', @Mode AS VARCHAR (200) = 'Tbl=' + isnull(@Tbl, 'NULL'), @st AS DATETIME = getUTCdate(), @ResourceTypeId AS SMALLINT, @Rows AS BIGINT, @Txt AS VARCHAR (1000), @TblInt AS VARCHAR (100), @Ind AS VARCHAR (200), @IndId AS INT, @DataComp AS VARCHAR (100); +DECLARE @Indexes TABLE ( + IndId INT PRIMARY KEY, + name VARCHAR (200)); +DECLARE @ResourceTypes TABLE ( + ResourceTypeId SMALLINT PRIMARY KEY); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + IF @Tbl IS NULL + RAISERROR ('@Tbl IS NULL', 18, 127); + INSERT INTO @Indexes + SELECT index_id, + name + FROM sys.indexes + WHERE object_id = object_id(@Tbl) + AND is_disabled = 1; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Insert', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Indexes) + BEGIN + SELECT TOP 1 @IndId = IndId, + @Ind = name + FROM @Indexes + ORDER BY IndId; + SET @DataComp = CASE WHEN (SELECT PropertyValue + FROM dbo.IndexProperties + WHERE TableName = @Tbl + AND IndexName = @Ind) = 'PAGE' THEN ' PARTITION = ALL WITH (DATA_COMPRESSION = PAGE)' ELSE '' END; + SET @Txt = 'IF EXISTS (SELECT * FROM sys.indexes WHERE object_id = object_id(''' + @Tbl + ''') AND name = ''' + @Ind + ''' AND is_disabled = 1) ALTER INDEX ' + @Ind + ' ON dbo.' + @Tbl + ' REBUILD' + @DataComp; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Ind, @Action = 'Rebuild', @Text = @Txt; + DELETE @Indexes + WHERE IndId = @IndId; + END + INSERT INTO @ResourceTypes + SELECT CONVERT (SMALLINT, substring(name, charindex('_', name) + 1, 6)) AS ResourceTypeId + FROM sys.objects AS O + WHERE name LIKE @Tbl + '[_]%' + AND EXISTS (SELECT * + FROM sysindexes + WHERE id = O.object_id + AND indid IN (0, 1) + AND rows > 0); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '#ResourceTypes', @Action = 'Select Into', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @ResourceTypes) + BEGIN + SET @ResourceTypeId = (SELECT TOP 1 ResourceTypeId + FROM @ResourceTypes); + SET @TblInt = @Tbl + '_' + CONVERT (VARCHAR, @ResourceTypeId); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt; + SET @Txt = 'ALTER TABLE dbo.' + @TblInt + ' SWITCH TO dbo.' + @Tbl + ' PARTITION $partition.PartitionFunction_ResourceTypeId(' + CONVERT (VARCHAR, @ResourceTypeId) + ')'; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Switch in start', @Text = @Txt; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Switch in', @Text = @Txt; + IF EXISTS (SELECT * + FROM sysindexes + WHERE id = object_id(@TblInt) + AND rows > 0) + BEGIN + SET @Txt = @TblInt + ' is not empty after switch'; + RAISERROR (@Txt, 18, 127); + END + EXECUTE ('DROP TABLE dbo.' + @TblInt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Drop'; + DELETE @ResourceTypes + WHERE ResourceTypeId = @ResourceTypeId; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.SwitchPartitionsInAllTables +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'SwitchPartitionsInAllTables', @Mode AS VARCHAR (200) = 'PS=PartitionScheme_ResourceTypeId', @st AS DATETIME = getUTCdate(), @Tbl AS VARCHAR (100); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + DECLARE @Tables TABLE ( + name VARCHAR (100) PRIMARY KEY, + supported BIT ); + INSERT INTO @Tables + EXECUTE dbo.GetPartitionedTables @IncludeNotDisabled = 1, @IncludeNotSupported = 0; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Tables', @Action = 'Insert', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Tables) + BEGIN + SET @Tbl = (SELECT TOP 1 name + FROM @Tables + ORDER BY name); + EXECUTE dbo.SwitchPartitionsIn @Tbl = @Tbl; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = 'SwitchPartitionsIn', @Action = 'Execute', @Text = @Tbl; + DELETE @Tables + WHERE name = @Tbl; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.SwitchPartitionsOut +@Tbl VARCHAR (100), @RebuildClustered BIT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'SwitchPartitionsOut', @Mode AS VARCHAR (200) = 'Tbl=' + isnull(@Tbl, 'NULL') + ' ND=' + isnull(CONVERT (VARCHAR, @RebuildClustered), 'NULL'), @st AS DATETIME = getUTCdate(), @ResourceTypeId AS SMALLINT, @Rows AS BIGINT, @Txt AS VARCHAR (MAX), @TblInt AS VARCHAR (100), @IndId AS INT, @Ind AS VARCHAR (200), @Name AS VARCHAR (100), @checkName AS VARCHAR (200), @definition AS VARCHAR (200); +DECLARE @Indexes TABLE ( + IndId INT PRIMARY KEY, + name VARCHAR (200), + IsDisabled BIT ); +DECLARE @IndexesRT TABLE ( + IndId INT PRIMARY KEY, + name VARCHAR (200), + IsDisabled BIT ); +DECLARE @ResourceTypes TABLE ( + ResourceTypeId SMALLINT PRIMARY KEY, + partition_number_roundtrip INT , + partition_number INT , + row_count BIGINT ); +DECLARE @Names TABLE ( + name VARCHAR (100) PRIMARY KEY); +DECLARE @CheckConstraints TABLE ( + CheckName VARCHAR (200), + CheckDefinition VARCHAR (200)); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + IF @Tbl IS NULL + RAISERROR ('@Tbl IS NULL', 18, 127); + IF @RebuildClustered IS NULL + RAISERROR ('@RebuildClustered IS NULL', 18, 127); + INSERT INTO @Indexes + SELECT index_id, + name, + is_disabled + FROM sys.indexes + WHERE object_id = object_id(@Tbl) + AND (is_disabled = 0 + OR @RebuildClustered = 1); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Insert', @Rows = @@rowcount; + INSERT INTO @ResourceTypes + SELECT partition_number - 1 AS ResourceTypeId, + $PARTITION.PartitionFunction_ResourceTypeId (partition_number - 1) AS partition_number_roundtrip, + partition_number, + row_count + FROM sys.dm_db_partition_stats + WHERE object_id = object_id(@Tbl) + AND index_id = 1 + AND row_count > 0; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@ResourceTypes', @Action = 'Insert', @Rows = @@rowcount, @Text = 'For partition switch'; + IF EXISTS (SELECT * + FROM @ResourceTypes + WHERE partition_number_roundtrip <> partition_number) + RAISERROR ('Partition sanity check failed', 18, 127); + WHILE EXISTS (SELECT * + FROM @ResourceTypes) + BEGIN + SELECT TOP 1 @ResourceTypeId = ResourceTypeId, + @Rows = row_count + FROM @ResourceTypes + ORDER BY ResourceTypeId; + SET @TblInt = @Tbl + '_' + CONVERT (VARCHAR, @ResourceTypeId); + SET @Txt = 'Starting @ResourceTypeId=' + CONVERT (VARCHAR, @ResourceTypeId) + ' row_count=' + CONVERT (VARCHAR, @Rows); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Text = @Txt; + IF NOT EXISTS (SELECT * + FROM sysindexes + WHERE id = object_id(@TblInt) + AND rows > 0) + BEGIN + IF object_id(@TblInt) IS NOT NULL + BEGIN + EXECUTE ('DROP TABLE dbo.' + @TblInt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Drop'; + END + EXECUTE ('SELECT * INTO dbo.' + @TblInt + ' FROM dbo.' + @Tbl + ' WHERE 1 = 2'); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Select Into', @Rows = @@rowcount; + DELETE @CheckConstraints; + INSERT INTO @CheckConstraints + SELECT name, + definition + FROM sys.check_constraints + WHERE parent_object_id = object_id(@Tbl); + WHILE EXISTS (SELECT * + FROM @CheckConstraints) + BEGIN + SELECT TOP 1 @checkName = CheckName, + @definition = CheckDefinition + FROM @CheckConstraints; + SET @Txt = 'ALTER TABLE ' + @TblInt + ' ADD CHECK ' + @definition; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'ALTER', @Text = @Txt; + DELETE @CheckConstraints + WHERE CheckName = @checkName; + END + DELETE @Names; + INSERT INTO @Names + SELECT name + FROM sys.columns + WHERE object_id = object_id(@Tbl) + AND is_sparse = 1; + WHILE EXISTS (SELECT * + FROM @Names) + BEGIN + SET @Name = (SELECT TOP 1 name + FROM @Names + ORDER BY name); + SET @Txt = (SELECT 'ALTER TABLE dbo.' + @TblInt + ' ALTER COLUMN ' + @Name + ' ' + T.name + '(' + CONVERT (VARCHAR, C.precision) + ',' + CONVERT (VARCHAR, C.scale) + ') SPARSE NULL' + FROM sys.types AS T + INNER JOIN + sys.columns AS C + ON C.system_type_id = T.system_type_id + WHERE C.object_id = object_id(@Tbl) + AND C.name = @Name); + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'ALTER', @Text = @Txt; + DELETE @Names + WHERE name = @Name; + END + END + INSERT INTO @IndexesRT + SELECT * + FROM @Indexes + WHERE IsDisabled = 0; + WHILE EXISTS (SELECT * + FROM @IndexesRT) + BEGIN + SELECT TOP 1 @IndId = IndId, + @Ind = name + FROM @IndexesRT + ORDER BY IndId; + IF NOT EXISTS (SELECT * + FROM sys.indexes + WHERE object_id = object_id(@TblInt) + AND name = @Ind) + BEGIN + EXECUTE dbo.GetIndexCommands @Tbl = @Tbl, @Ind = @Ind, @AddPartClause = 0, @IncludeClustered = 1, @Txt = @Txt OUTPUT; + SET @Txt = replace(@Txt, '[' + @Tbl + ']', @TblInt); + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Create Index', @Text = @Txt; + END + DELETE @IndexesRT + WHERE IndId = @IndId; + END + SET @Txt = 'ALTER TABLE dbo.' + @TblInt + ' ADD CHECK (ResourceTypeId >= ' + CONVERT (VARCHAR, @ResourceTypeId) + ' AND ResourceTypeId < ' + CONVERT (VARCHAR, @ResourceTypeId) + ' + 1)'; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Add check', @Text = @Txt; + SET @Txt = 'ALTER TABLE dbo.' + @Tbl + ' SWITCH PARTITION $partition.PartitionFunction_ResourceTypeId(' + CONVERT (VARCHAR, @ResourceTypeId) + ') TO dbo.' + @TblInt; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Switch out start', @Text = @Txt; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Switch out end', @Text = @Txt; + DELETE @ResourceTypes + WHERE ResourceTypeId = @ResourceTypeId; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.SwitchPartitionsOutAllTables +@RebuildClustered BIT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'SwitchPartitionsOutAllTables', @Mode AS VARCHAR (200) = 'PS=PartitionScheme_ResourceTypeId ND=' + isnull(CONVERT (VARCHAR, @RebuildClustered), 'NULL'), @st AS DATETIME = getUTCdate(), @Tbl AS VARCHAR (100); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + DECLARE @Tables TABLE ( + name VARCHAR (100) PRIMARY KEY, + supported BIT ); + INSERT INTO @Tables + EXECUTE dbo.GetPartitionedTables @IncludeNotDisabled = @RebuildClustered, @IncludeNotSupported = 0; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Tables', @Action = 'Insert', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Tables) + BEGIN + SET @Tbl = (SELECT TOP 1 name + FROM @Tables + ORDER BY name); + EXECUTE dbo.SwitchPartitionsOut @Tbl = @Tbl, @RebuildClustered = @RebuildClustered; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = 'SwitchPartitionsOut', @Action = 'Execute', @Text = @Tbl; + DELETE @Tables + WHERE name = @Tbl; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE [dbo].[TaskKeepAlive] +@taskId VARCHAR (64), @runId VARCHAR (50) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +IF NOT EXISTS (SELECT * + FROM [dbo].[TaskInfo] + WHERE TaskId = @taskId + AND RunId = @runId) + BEGIN + THROW 50404, 'Task not exist or runid not match', 1; + END +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +UPDATE dbo.TaskInfo +SET HeartbeatDateTime = @heartbeatDateTime +WHERE TaskId = @taskId; +SELECT TaskId, + QueueId, + Status, + TaskTypeId, + RunId, + IsCanceled, + RetryCount, + MaxRetryCount, + HeartbeatDateTime, + InputData, + TaskContext, + Result +FROM [dbo].[TaskInfo] +WHERE TaskId = @taskId; +COMMIT TRANSACTION; + +GO +CREATE OR ALTER PROCEDURE dbo.UpdateEventAgentCheckpoint +@CheckpointId VARCHAR (64), @LastProcessedDateTime DATETIMEOFFSET (7)=NULL, @LastProcessedIdentifier VARCHAR (64)=NULL +AS +BEGIN + IF EXISTS (SELECT * + FROM dbo.EventAgentCheckpoint + WHERE CheckpointId = @CheckpointId) + UPDATE dbo.EventAgentCheckpoint + SET CheckpointId = @CheckpointId, + LastProcessedDateTime = @LastProcessedDateTime, + LastProcessedIdentifier = @LastProcessedIdentifier, + UpdatedOn = sysutcdatetime() + WHERE CheckpointId = @CheckpointId; + ELSE + INSERT INTO dbo.EventAgentCheckpoint (CheckpointId, LastProcessedDateTime, LastProcessedIdentifier, UpdatedOn) + VALUES (@CheckpointId, @LastProcessedDateTime, @LastProcessedIdentifier, sysutcdatetime()); +END + +GO +CREATE PROCEDURE dbo.UpdateExportJob +@id VARCHAR (64), @status VARCHAR (10), @rawJobRecord VARCHAR (MAX), @jobVersion BINARY (8) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @currentJobVersion AS BINARY (8); +SELECT @currentJobVersion = JobVersion +FROM dbo.ExportJob WITH (UPDLOCK, HOLDLOCK) +WHERE Id = @id; +IF (@currentJobVersion IS NULL) + BEGIN + THROW 50404, 'Export job record not found', 1; + END +IF (@jobVersion <> @currentJobVersion) + BEGIN + THROW 50412, 'Precondition failed', 1; + END +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +UPDATE dbo.ExportJob +SET Status = @status, + HeartbeatDateTime = @heartbeatDateTime, + RawJobRecord = @rawJobRecord +WHERE Id = @id; +SELECT @@DBTS; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.UpdateReindexJob +@id VARCHAR (64), @status VARCHAR (10), @rawJobRecord VARCHAR (MAX), @jobVersion BINARY (8) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @currentJobVersion AS BINARY (8); +SELECT @currentJobVersion = JobVersion +FROM dbo.ReindexJob WITH (UPDLOCK, HOLDLOCK) +WHERE Id = @id; +IF (@currentJobVersion IS NULL) + BEGIN + THROW 50404, 'Reindex job record not found', 1; + END +IF (@jobVersion <> @currentJobVersion) + BEGIN + THROW 50412, 'Precondition failed', 1; + END +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +UPDATE dbo.ReindexJob +SET Status = @status, + HeartbeatDateTime = @heartbeatDateTime, + RawJobRecord = @rawJobRecord +WHERE Id = @id; +SELECT @@DBTS; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.UpdateResourceSearchParams +@FailedResources INT=0 OUTPUT, @Resources dbo.ResourceList READONLY, @ResourceWriteClaims dbo.ResourceWriteClaimList READONLY, @ReferenceSearchParams dbo.ReferenceSearchParamList READONLY, @TokenSearchParams dbo.TokenSearchParamList READONLY, @TokenTexts dbo.TokenTextList READONLY, @StringSearchParams dbo.StringSearchParamList READONLY, @UriSearchParams dbo.UriSearchParamList READONLY, @NumberSearchParams dbo.NumberSearchParamList READONLY, @QuantitySearchParams dbo.QuantitySearchParamList READONLY, @DateTimeSearchParams dbo.DateTimeSearchParamList READONLY, @ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY, @TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY, @TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY, @TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY, @TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY, @TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY +AS +SET NOCOUNT ON; +DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (200) = isnull((SELECT 'RT=[' + CONVERT (VARCHAR, min(ResourceTypeId)) + ',' + CONVERT (VARCHAR, max(ResourceTypeId)) + '] Sur=[' + CONVERT (VARCHAR, min(ResourceSurrogateId)) + ',' + CONVERT (VARCHAR, max(ResourceSurrogateId)) + '] V=' + CONVERT (VARCHAR, max(Version)) + ' Rows=' + CONVERT (VARCHAR, count(*)) + FROM @Resources), 'Input=Empty'), @Rows AS INT; +BEGIN TRY + DECLARE @Ids TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL); + BEGIN TRANSACTION; + UPDATE B + SET SearchParamHash = A.SearchParamHash + OUTPUT deleted.ResourceTypeId, deleted.ResourceSurrogateId INTO @Ids + FROM @Resources AS A + INNER JOIN + dbo.Resource AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + WHERE B.IsHistory = 0; + SET @Rows = @@rowcount; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.ResourceWriteClaim AS B + ON B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.ReferenceSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenText AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.StringSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.UriSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.NumberSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.QuantitySearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.DateTimeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.ReferenceTokenCompositeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenTokenCompositeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenDateTimeCompositeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenQuantityCompositeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenStringCompositeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenNumberNumberCompositeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) + SELECT ResourceSurrogateId, + ClaimTypeId, + ClaimValue + FROM @ResourceWriteClaims; + INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri, + ReferenceResourceTypeId, + ReferenceResourceId, + ReferenceResourceVersion + FROM @ReferenceSearchParams; + INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + Code, + CodeOverflow + FROM @TokenSearchParams; + INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text + FROM @TokenTexts; + INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text, + TextOverflow, + IsMin, + IsMax + FROM @StringSearchParams; + INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Uri + FROM @UriSearchParams; + INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SingleValue, + LowValue, + HighValue + FROM @NumberSearchParams; + INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + QuantityCodeId, + SingleValue, + LowValue, + HighValue + FROM @QuantitySearchParams; + INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + StartDateTime, + EndDateTime, + IsLongerThanADay, + IsMin, + IsMax + FROM @DateTimeSearchParams; + INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri1, + ReferenceResourceTypeId1, + ReferenceResourceId1, + ReferenceResourceVersion1, + SystemId2, + Code2, + CodeOverflow2 + FROM @ReferenceTokenCompositeSearchParams; + INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SystemId2, + Code2, + CodeOverflow2 + FROM @TokenTokenCompositeSearchParams; + INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + StartDateTime2, + EndDateTime2, + IsLongerThanADay2 + FROM @TokenDateTimeCompositeSearchParams; + INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + SystemId2, + QuantityCodeId2, + LowValue2, + HighValue2 + FROM @TokenQuantityCompositeSearchParams; + INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + Text2, + TextOverflow2 + FROM @TokenStringCompositeSearchParams; + INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + LowValue2, + HighValue2, + SingleValue3, + LowValue3, + HighValue3, + HasRange + FROM @TokenNumberNumberCompositeSearchParams; + COMMIT TRANSACTION; + SET @FailedResources = (SELECT count(*) + FROM @Resources) - @Rows; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE [dbo].[UpdateTaskContext] +@taskId VARCHAR (64), @taskContext VARCHAR (MAX), @runId VARCHAR (50) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +IF NOT EXISTS (SELECT * + FROM [dbo].[TaskInfo] + WHERE TaskId = @taskId + AND RunId = @runId) + BEGIN + THROW 50404, 'Task not exist or runid not match', 1; + END +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +UPDATE dbo.TaskInfo +SET HeartbeatDateTime = @heartbeatDateTime, + TaskContext = @taskContext +WHERE TaskId = @taskId; +SELECT TaskId, + QueueId, + Status, + TaskTypeId, + RunId, + IsCanceled, + RetryCount, + MaxRetryCount, + HeartbeatDateTime, + InputData, + TaskContext, + Result +FROM [dbo].[TaskInfo] +WHERE TaskId = @taskId; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.UpsertResource_7 +@baseResourceSurrogateId BIGINT, @resourceTypeId SMALLINT, @resourceId VARCHAR (64), @eTag INT=NULL, @allowCreate BIT, @isDeleted BIT, @keepHistory BIT, @requireETagOnUpdate BIT, @requestMethod VARCHAR (10), @searchParamHash VARCHAR (64), @rawResource VARBINARY (MAX), @resourceWriteClaims dbo.BulkResourceWriteClaimTableType_1 READONLY, @compartmentAssignments dbo.BulkCompartmentAssignmentTableType_1 READONLY, @referenceSearchParams dbo.BulkReferenceSearchParamTableType_1 READONLY, @tokenSearchParams dbo.BulkTokenSearchParamTableType_2 READONLY, @tokenTextSearchParams dbo.BulkTokenTextTableType_1 READONLY, @stringSearchParams dbo.BulkStringSearchParamTableType_2 READONLY, @numberSearchParams dbo.BulkNumberSearchParamTableType_1 READONLY, @quantitySearchParams dbo.BulkQuantitySearchParamTableType_1 READONLY, @uriSearchParams dbo.BulkUriSearchParamTableType_1 READONLY, @dateTimeSearchParms dbo.BulkDateTimeSearchParamTableType_2 READONLY, @referenceTokenCompositeSearchParams dbo.BulkReferenceTokenCompositeSearchParamTableType_2 READONLY, @tokenTokenCompositeSearchParams dbo.BulkTokenTokenCompositeSearchParamTableType_2 READONLY, @tokenDateTimeCompositeSearchParams dbo.BulkTokenDateTimeCompositeSearchParamTableType_2 READONLY, @tokenQuantityCompositeSearchParams dbo.BulkTokenQuantityCompositeSearchParamTableType_2 READONLY, @tokenStringCompositeSearchParams dbo.BulkTokenStringCompositeSearchParamTableType_2 READONLY, @tokenNumberNumberCompositeSearchParams dbo.BulkTokenNumberNumberCompositeSearchParamTableType_2 READONLY, @isResourceChangeCaptureEnabled BIT=0, @comparedVersion INT=NULL +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +DECLARE @previousResourceSurrogateId AS BIGINT, @previousVersion AS BIGINT, @previousIsDeleted AS BIT, @version AS INT, @resourceSurrogateId AS BIGINT, @InitialTranCount AS INT = @@trancount; +IF @InitialTranCount = 0 + BEGIN TRANSACTION; +SELECT @previousResourceSurrogateId = ResourceSurrogateId, + @previousVersion = Version, + @previousIsDeleted = IsDeleted +FROM dbo.Resource WITH (UPDLOCK, HOLDLOCK) +WHERE ResourceTypeId = @resourceTypeId + AND ResourceId = @resourceId + AND IsHistory = 0; +IF @previousResourceSurrogateId IS NULL + SET @version = 1; +ELSE + BEGIN + IF @isDeleted = 0 + BEGIN + IF @comparedVersion IS NULL + OR @comparedVersion <> @previousVersion + BEGIN + THROW 50409, 'Resource has been recently updated or added, please compare the resource content in code for any duplicate updates', 1; + END + END + SET @version = @previousVersion + 1; + IF @keepHistory = 1 + UPDATE dbo.Resource + SET IsHistory = 1 + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + ELSE + DELETE dbo.Resource + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.ResourceWriteClaim + WHERE ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.CompartmentAssignment + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.ReferenceSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenText + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.StringSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.UriSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.NumberSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.QuantitySearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.DateTimeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.ReferenceTokenCompositeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenTokenCompositeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenDateTimeCompositeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenQuantityCompositeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenStringCompositeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenNumberNumberCompositeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + END +SET @resourceSurrogateId = @baseResourceSurrogateId + ( NEXT VALUE FOR ResourceSurrogateIdUniquifierSequence); +INSERT INTO dbo.Resource (ResourceTypeId, ResourceId, Version, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, RawResource, IsRawResourceMetaSet, SearchParamHash) +SELECT @resourceTypeId, + @resourceId, + @version, + 0, + @resourceSurrogateId, + @isDeleted, + @requestMethod, + @rawResource, + CASE WHEN @version = 1 THEN 1 ELSE 0 END, + @searchParamHash; +INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) +SELECT @resourceSurrogateId, + ClaimTypeId, + ClaimValue +FROM @resourceWriteClaims; +INSERT INTO dbo.CompartmentAssignment (ResourceTypeId, ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + CompartmentTypeId, + ReferenceResourceId, + 0 +FROM @compartmentAssignments; +INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + BaseUri, + ReferenceResourceTypeId, + ReferenceResourceId, + ReferenceResourceVersion, + 0 +FROM @referenceSearchParams; +INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId, + Code, + CodeOverflow, + 0 +FROM @tokenSearchParams; +INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + Text, + 0 +FROM @tokenTextSearchParams; +INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsHistory, IsMin, IsMax) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + Text, + TextOverflow, + 0, + IsMin, + IsMax +FROM @stringSearchParams; +INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + Uri, + 0 +FROM @uriSearchParams; +INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SingleValue, + LowValue, + HighValue, + 0 +FROM @numberSearchParams; +INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId, + QuantityCodeId, + SingleValue, + LowValue, + HighValue, + 0 +FROM @quantitySearchParams; +INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsHistory, IsMin, IsMax) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + StartDateTime, + EndDateTime, + IsLongerThanADay, + 0, + IsMin, + IsMax +FROM @dateTimeSearchParms; +INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + BaseUri1, + ReferenceResourceTypeId1, + ReferenceResourceId1, + ReferenceResourceVersion1, + SystemId2, + Code2, + CodeOverflow2, + 0 +FROM @referenceTokenCompositeSearchParams; +INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SystemId2, + Code2, + CodeOverflow2, + 0 +FROM @tokenTokenCompositeSearchParams; +INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + StartDateTime2, + EndDateTime2, + IsLongerThanADay2, + 0 +FROM @tokenDateTimeCompositeSearchParams; +INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + SystemId2, + QuantityCodeId2, + LowValue2, + HighValue2, + 0 +FROM @tokenQuantityCompositeSearchParams; +INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + Text2, + TextOverflow2, + 0 +FROM @tokenStringCompositeSearchParams; +INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + LowValue2, + HighValue2, + SingleValue3, + LowValue3, + HighValue3, + HasRange, + 0 +FROM @tokenNumberNumberCompositeSearchParams; +SELECT @version; +IF @isResourceChangeCaptureEnabled = 1 + EXECUTE dbo.CaptureResourceChanges @isDeleted = @isDeleted, @version = @version, @resourceId = @resourceId, @resourceTypeId = @resourceTypeId; +IF @InitialTranCount = 0 + COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.UpsertSearchParams +@searchParams dbo.SearchParamTableType_2 READONLY +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DECLARE @lastUpdated AS DATETIMEOFFSET (7) = SYSDATETIMEOFFSET(); +DECLARE @summaryOfChanges TABLE ( + Uri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + Action VARCHAR (20) NOT NULL); +MERGE INTO dbo.SearchParam WITH (TABLOCKX) + AS target +USING @searchParams AS source ON target.Uri = source.Uri +WHEN MATCHED THEN UPDATE +SET Status = source.Status, + LastUpdated = @lastUpdated, + IsPartiallySupported = source.IsPartiallySupported +WHEN NOT MATCHED BY TARGET THEN INSERT (Uri, Status, LastUpdated, IsPartiallySupported) VALUES (source.Uri, source.Status, @lastUpdated, source.IsPartiallySupported) +OUTPUT source.Uri, $ACTION INTO @summaryOfChanges; +SELECT SearchParamId, + SearchParam.Uri +FROM dbo.SearchParam AS searchParam + INNER JOIN + @summaryOfChanges AS upsertedSearchParam + ON searchParam.Uri = upsertedSearchParam.Uri +WHERE upsertedSearchParam.Action = 'INSERT'; +COMMIT TRANSACTION; + +GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs index 253a70f2e9..e1d15b6a6f 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs @@ -77,5 +77,6 @@ public enum SchemaVersion V65 = 65, V66 = 66, V67 = 67, + V68 = 68, } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs index a2217cd496..50c480593d 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs @@ -8,7 +8,7 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Schema public static class SchemaVersionConstants { public const int Min = (int)SchemaVersion.V63; - public const int Max = (int)SchemaVersion.V67; + public const int Max = (int)SchemaVersion.V68; public const int MinForUpgrade = (int)SchemaVersion.V63; // this is used for upgrade tests only public const int SearchParameterStatusSchemaVersion = (int)SchemaVersion.V6; public const int SupportForReferencesWithMissingTypeVersion = (int)SchemaVersion.V7; diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/TransactionCheckWithInitialiScript.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/TransactionCheckWithInitialiScript.sql index 4b8c4ba5de..73c1fd8b94 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/TransactionCheckWithInitialiScript.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/TransactionCheckWithInitialiScript.sql @@ -19,6 +19,6 @@ Go INSERT INTO dbo.SchemaVersion VALUES - (67, 'started') + (68, 'started') Go diff --git a/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj b/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj index 9148286c52..6207849b2f 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj +++ b/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj @@ -1,7 +1,7 @@  - 67 + 68 Features\Schema\Migrations\$(LatestSchemaVersion).sql From 17fe2148f2f75cf194e1e18aa6947aae256ca051 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Mon, 6 Nov 2023 14:07:42 -0800 Subject: [PATCH 45/58] decoupled search options from export --- src/Microsoft.Health.Fhir.Api/Resources.resx | 2 +- .../Features/KnownQueryParameterNames.cs | 10 ---- .../Operations/Export/ExportJobTask.cs | 17 +++--- .../Features/Search/ISearchOptionsFactory.cs | 4 +- .../Features/Search/ISearchService.cs | 4 +- .../Features/Search/ResourceVersionType.cs | 31 +++++++++++ .../Features/Search/SearchOptions.cs | 14 ++--- .../Features/Search/SearchService.cs | 13 ++--- .../Search/FhirCosmosSearchService.cs | 15 ------ .../Features/Search/Queries/IQueryBuilder.cs | 2 - .../Features/Search/Queries/QueryBuilder.cs | 50 ++++------------- .../Search/SearchOptionsFactoryTests.cs | 53 ++++++++----------- .../Features/Search/SearchOptionsFactory.cs | 17 +++--- .../Features/Search/SqlSearchOptions.cs | 11 ++-- .../Features/Search/SqlServerSearchService.cs | 8 +-- 15 files changed, 103 insertions(+), 148 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.Core/Features/Search/ResourceVersionType.cs diff --git a/src/Microsoft.Health.Fhir.Api/Resources.resx b/src/Microsoft.Health.Fhir.Api/Resources.resx index 5b9e0f61ec..6885d779a7 100644 --- a/src/Microsoft.Health.Fhir.Api/Resources.resx +++ b/src/Microsoft.Health.Fhir.Api/Resources.resx @@ -412,6 +412,6 @@ The request "_typeFilter" cannot be used with an export request with historical or soft deleted resources. - The export parameter "includeAssociatedData" contains an invalid parameter. + The export parameter "includeAssociatedData" contains an invalid value. Only values "_includeHistory" and "_includeHistory" are supported. \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.Core/Features/KnownQueryParameterNames.cs b/src/Microsoft.Health.Fhir.Core/Features/KnownQueryParameterNames.cs index ceee07c0d0..a34719c317 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/KnownQueryParameterNames.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/KnownQueryParameterNames.cs @@ -99,16 +99,6 @@ public static class KnownQueryParameterNames public const string PurgeHistory = "_purgeHistory"; - /// - /// Used by queries from the $export mediator to signal that history should be included in the export search. - /// - public const string IncludeHistory = "_includeHistory"; - - /// - /// Used by queries from the $export mediator to signal that soft deleted resources should be included in the export search. - /// - public const string IncludeDeleted = "_includeDeleted"; - /// /// Used by $export as a comma-separated list of parameters instructing which initial data should be included. /// diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs index 1e73c7e9c8..a3b17f7bbe 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs @@ -173,8 +173,6 @@ public async Task ExecuteAsync(ExportJobRecord exportJobRecord, WeakETag weakETa { Tuple.Create(KnownQueryParameterNames.Count, _exportJobRecord.MaximumNumberOfResourcesPerQuery.ToString(CultureInfo.InvariantCulture)), Tuple.Create(KnownQueryParameterNames.LastUpdated, $"le{tillTime}"), - Tuple.Create(KnownQueryParameterNames.IncludeHistory, _exportJobRecord.IncludeHistory.ToString(CultureInfo.InvariantCulture)), - Tuple.Create(KnownQueryParameterNames.IncludeDeleted, _exportJobRecord.IncludeDeleted.ToString(CultureInfo.InvariantCulture)), }; if (_exportJobRecord.GlobalEndSurrogateId != null) // no need to check individually as they all should have values if anyone does @@ -190,9 +188,13 @@ public async Task ExecuteAsync(ExportJobRecord exportJobRecord, WeakETag weakETa queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.LastUpdated, $"ge{_exportJobRecord.Since}")); } + var exportResourceVersionTypes = ResourceVersionType.Latest; + exportResourceVersionTypes |= _exportJobRecord.IncludeHistory ? ResourceVersionType.Histoy : 0; + exportResourceVersionTypes |= _exportJobRecord.IncludeDeleted ? ResourceVersionType.SoftDeleted : 0; + ExportJobProgress progress = _exportJobRecord.Progress; - await RunExportSearch(exportJobConfiguration, progress, queryParametersList, cancellationToken); + await RunExportSearch(exportJobConfiguration, progress, queryParametersList, exportResourceVersionTypes, cancellationToken); await CompleteJobAsync(OperationStatus.Completed, cancellationToken); @@ -342,6 +344,7 @@ private async Task RunExportSearch( ExportJobConfiguration exportJobConfiguration, ExportJobProgress progress, List> sharedQueryParametersList, + ResourceVersionType resourceVersionTypes, CancellationToken cancellationToken) { EnsureArg.IsNotNull(exportJobConfiguration, nameof(exportJobConfiguration)); @@ -420,7 +423,7 @@ private async Task RunExportSearch( } } - await SearchWithFilter(exportJobConfiguration, progress, null, queryParametersList, sharedQueryParametersList, anonymizer, cancellationToken); + await SearchWithFilter(exportJobConfiguration, progress, null, queryParametersList, sharedQueryParametersList, resourceVersionTypes, anonymizer, cancellationToken); } } @@ -438,7 +441,7 @@ private async Task ProcessFilter( filterQueryParametersList.Add(param); } - await SearchWithFilter(exportJobConfiguration, exportJobProgress, exportJobProgress.CurrentFilter.ResourceType, filterQueryParametersList, sharedQueryParametersList, anonymizer, cancellationToken); + await SearchWithFilter(exportJobConfiguration, exportJobProgress, exportJobProgress.CurrentFilter.ResourceType, filterQueryParametersList, sharedQueryParametersList, ResourceVersionType.Latest, anonymizer, cancellationToken); exportJobProgress.MarkFilterFinished(); await UpdateJobRecordAsync(cancellationToken); @@ -450,6 +453,7 @@ private async Task SearchWithFilter( string resourceType, List> queryParametersList, List> sharedQueryParametersList, + ResourceVersionType resourceVersionTypes, IAnonymizer anonymizer, CancellationToken cancellationToken) { @@ -471,7 +475,8 @@ private async Task SearchWithFilter( resourceType: resourceType, queryParametersList, cancellationToken, - true); + true, + resourceVersionTypes); } break; diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/ISearchOptionsFactory.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/ISearchOptionsFactory.cs index 232da354cf..c1ae219be3 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/ISearchOptionsFactory.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/ISearchOptionsFactory.cs @@ -10,8 +10,8 @@ namespace Microsoft.Health.Fhir.Core.Features.Search { public interface ISearchOptionsFactory { - SearchOptions Create(string resourceType, IReadOnlyList> queryParameters, bool isAsyncOperation = false); + SearchOptions Create(string resourceType, IReadOnlyList> queryParameters, bool isAsyncOperation = false, ResourceVersionType resourceVersionTypes = ResourceVersionType.Latest); - SearchOptions Create(string compartmentType, string compartmentId, string resourceType, IReadOnlyList> queryParameters, bool isAsyncOperation = false, bool useSmartCompartmentDefinition = false); + SearchOptions Create(string compartmentType, string compartmentId, string resourceType, IReadOnlyList> queryParameters, bool isAsyncOperation = false, bool useSmartCompartmentDefinition = false, ResourceVersionType resourceVersionTypes = ResourceVersionType.Latest); } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/ISearchService.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/ISearchService.cs index 7cf4ed73bb..2a13064e38 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/ISearchService.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/ISearchService.cs @@ -23,12 +23,14 @@ public interface ISearchService /// The search queries. /// The cancellation token. /// Whether the search is part of an async operation. + /// Which version types (latest, soft-deleted, history) to include in search. /// A representing the result. Task SearchAsync( string resourceType, IReadOnlyList> queryParameters, CancellationToken cancellationToken, - bool isAsyncOperation = false); + bool isAsyncOperation = false, + ResourceVersionType resourceVersionTypes = ResourceVersionType.Latest); /// /// Searches the resources using the . diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/ResourceVersionType.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/ResourceVersionType.cs new file mode 100644 index 0000000000..86475ab32e --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/ResourceVersionType.cs @@ -0,0 +1,31 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.Health.Fhir.Core.Features.Search +{ + /// + /// Defines the order of a sort. + /// + [Flags] + public enum ResourceVersionType + { + /// + /// Latest version of the resource. + /// + Latest = 1, + + /// + /// Previous versions of the resource - i.e. historical. + /// + Histoy = 1 << 1, + + /// + /// Resources that have been deleted but are still in the system (soft-delete). + /// + SoftDeleted = 1 << 2, + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SearchOptions.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SearchOptions.cs index c49c137cbe..b58b7daa90 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/SearchOptions.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SearchOptions.cs @@ -49,8 +49,7 @@ internal SearchOptions(SearchOptions other) QueryHints = other.QueryHints; - IncludeHistory = other.IncludeHistory; - IncludeDeleted = other.IncludeDeleted; + ResourceVersionTypes = other.ResourceVersionTypes; } /// @@ -115,16 +114,9 @@ internal set } /// - /// Gets a value indicating whether to include history resources in the search result, - /// as long as they match the other search parameters as well. + /// Which version types (latest, soft-deleted, history) to include in search. /// - public bool IncludeHistory { get; internal set; } = false; - - /// - /// Gets a value indicating whether to include deleted resources in the search result, - /// as long as they match the other search parameters as well. - /// - public bool IncludeDeleted { get; internal set; } = false; + public ResourceVersionType ResourceVersionTypes { get; internal set; } /// /// Gets the search expression. diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SearchService.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SearchService.cs index 8889fa8cfc..e90b346cd1 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/SearchService.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SearchService.cs @@ -44,9 +44,10 @@ public async Task SearchAsync( string resourceType, IReadOnlyList> queryParameters, CancellationToken cancellationToken, - bool isAsyncOperation = false) + bool isAsyncOperation = false, + ResourceVersionType resourceVersionTypes = ResourceVersionType.Latest) { - SearchOptions searchOptions = _searchOptionsFactory.Create(resourceType, queryParameters, isAsyncOperation); + SearchOptions searchOptions = _searchOptionsFactory.Create(resourceType, queryParameters, isAsyncOperation, resourceVersionTypes); // Execute the actual search. return await SearchAsync(searchOptions, cancellationToken); @@ -174,9 +175,9 @@ public async Task SearchHistoryAsync( queryParameters.Add(Tuple.Create(KnownQueryParameterNames.Sort, $"-{KnownQueryParameterNames.LastUpdated}")); } - SearchOptions searchOptions = _searchOptionsFactory.Create(resourceType, queryParameters, isAsyncOperation); + SearchOptions searchOptions = _searchOptionsFactory.Create(resourceType, queryParameters, isAsyncOperation, ResourceVersionType.Histoy); - SearchResult searchResult = await SearchHistoryInternalAsync(searchOptions, cancellationToken); + SearchResult searchResult = await SearchAsync(searchOptions, cancellationToken); // If no results are returned from the _history search // determine if the resource actually exists or if the results were just filtered out. @@ -232,10 +233,6 @@ public abstract Task SearchAsync( SearchOptions searchOptions, CancellationToken cancellationToken); - protected abstract Task SearchHistoryInternalAsync( - SearchOptions searchOptions, - CancellationToken cancellationToken); - protected abstract Task SearchForReindexInternalAsync( SearchOptions searchOptions, string searchParameterHash, diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/FhirCosmosSearchService.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/FhirCosmosSearchService.cs index efdf2e3671..923e1920b2 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/FhirCosmosSearchService.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/FhirCosmosSearchService.cs @@ -343,21 +343,6 @@ private async Task RecurseChainedExpression(ChainedExpression expres Expression.In(FieldName.ReferenceResourceId, null, g.Select(x => x.ResourceId)))).ToList())); } - protected override async Task SearchHistoryInternalAsync( - SearchOptions searchOptions, - CancellationToken cancellationToken) - { - (IReadOnlyList results, string continuationToken, _) = await ExecuteSearchAsync( - _queryBuilder.GenerateHistorySql(searchOptions), - searchOptions, - searchOptions.CountOnly ? null : searchOptions.ContinuationToken, - null, - null, - cancellationToken); - - return CreateSearchResult(searchOptions, results.Select(r => new SearchResultEntry(r)), continuationToken); - } - protected override async Task SearchForReindexInternalAsync( SearchOptions searchOptions, string searchParameterHash, diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/IQueryBuilder.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/IQueryBuilder.cs index b4d36218e9..86ee339da3 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/IQueryBuilder.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/IQueryBuilder.cs @@ -12,8 +12,6 @@ internal interface IQueryBuilder { QueryDefinition BuildSqlQuerySpec(SearchOptions searchOptions, QueryBuilderOptions queryOptions = null); - QueryDefinition GenerateHistorySql(SearchOptions searchOptions); - QueryDefinition GenerateReindexSql(SearchOptions searchOptions, string searchParameterHash); } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/QueryBuilder.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/QueryBuilder.cs index 6593e51817..85142d11ba 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/QueryBuilder.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/QueryBuilder.cs @@ -26,11 +26,6 @@ public QueryDefinition BuildSqlQuerySpec(SearchOptions searchOptions, QueryBuild return new QueryBuilderHelper().BuildSqlQuerySpec(searchOptions, queryOptions ?? new QueryBuilderOptions()); } - public QueryDefinition GenerateHistorySql(SearchOptions searchOptions) - { - return new QueryBuilderHelper().GenerateHistorySql(searchOptions); - } - public QueryDefinition GenerateReindexSql(SearchOptions searchOptions, string searchParameterHash) { return new QueryBuilderHelper().GenerateReindexSql(searchOptions, searchParameterHash); @@ -83,7 +78,15 @@ public QueryDefinition BuildSqlQuerySpec(SearchOptions searchOptions, QueryBuild searchOptions.Expression.AcceptVisitor(expressionQueryBuilder); } - if (searchOptions.IncludeHistory is false) + if (!searchOptions.ResourceVersionTypes.HasFlag(ResourceVersionType.Latest)) + { + AppendFilterCondition( + "AND", + true, + (KnownResourceWrapperProperties.IsHistory, true)); + } + + if (!searchOptions.ResourceVersionTypes.HasFlag(ResourceVersionType.Histoy)) { AppendFilterCondition( "AND", @@ -91,7 +94,7 @@ public QueryDefinition BuildSqlQuerySpec(SearchOptions searchOptions, QueryBuild (KnownResourceWrapperProperties.IsHistory, false)); } - if (searchOptions.IncludeDeleted is false) + if (!searchOptions.ResourceVersionTypes.HasFlag(ResourceVersionType.SoftDeleted)) { AppendFilterCondition( "AND", @@ -132,39 +135,6 @@ public QueryDefinition BuildSqlQuerySpec(SearchOptions searchOptions, QueryBuild return query; } - public QueryDefinition GenerateHistorySql(SearchOptions searchOptions) - { - EnsureArg.IsNotNull(searchOptions, nameof(searchOptions)); - - AppendSelectFromRoot(); - - AppendSystemDataFilter(); - - var expressionQueryBuilder = new ExpressionQueryBuilder( - _queryBuilder, - _queryParameterManager); - - if (searchOptions.Expression != null) - { - _queryBuilder.Append("AND "); - searchOptions.Expression.AcceptVisitor(expressionQueryBuilder); - } - - _queryBuilder.Append("ORDER BY "); - var sortOption = searchOptions.Sort[0]; - -#pragma warning disable CA1834 // Consider using 'StringBuilder.Append(char)' when applicable - _queryBuilder.Append(SearchValueConstants.RootAliasName).Append('.') -#pragma warning restore CA1834 // Consider using 'StringBuilder.Append(char)' when applicable - .Append(KnownResourceWrapperProperties.LastModified).Append(' ') - .AppendLine(sortOption.sortOrder == SortOrder.Ascending ? "ASC" : "DESC"); - - var query = new QueryDefinition(_queryBuilder.ToString()); - _queryParameterManager.AddToQuery(query); - - return query; - } - public QueryDefinition GenerateReindexSql(SearchOptions searchOptions, string searchParameterHash) { EnsureArg.IsNotNull(searchOptions, nameof(searchOptions)); diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs index e32603e74b..d15ab20438 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs @@ -339,8 +339,8 @@ public void GivenAValidCompartmentSearch_WhenCreated_ThenCorrectCompartmentSearc SearchOptions options = CreateSearchOptions( resourceType: resourceType.ToString(), queryParameters: null, - compartmentType.ToString(), - compartmentId); + compartmentType: compartmentType.ToString(), + compartmentId: compartmentId); Assert.NotNull(options); ValidateMultiaryExpression( @@ -363,8 +363,8 @@ public void GivenAValidCompartmentSearchWithNullResourceType_WhenCreated_ThenCor SearchOptions options = CreateSearchOptions( resourceType: null, queryParameters: null, - compartmentType.ToString(), - compartmentId); + compartmentType: compartmentType.ToString(), + compartmentId: compartmentId); Assert.NotNull(options); ValidateCompartmentSearchExpression(options.Expression, compartmentType.ToString(), compartmentId); @@ -382,8 +382,8 @@ public void GivenInvalidCompartmentType_WhenCreated_ThenExceptionShouldBeThrown( InvalidSearchOperationException exception = Assert.Throws(() => CreateSearchOptions( resourceType: null, queryParameters: null, - invalidCompartmentType, - "123")); + compartmentType: invalidCompartmentType, + compartmentId: "123")); Assert.Equal(exception.Message, $"Compartment type {invalidCompartmentType} is invalid."); } @@ -398,8 +398,8 @@ public void GivenInvalidCompartmentId_WhenCreated_ThenExceptionShouldBeThrown(st InvalidSearchOperationException exception = Assert.Throws(() => CreateSearchOptions( resourceType: ResourceType.Claim.ToString(), queryParameters: null, - CompartmentType.Patient.ToString(), - invalidCompartmentId)); + compartmentType: CompartmentType.Patient.ToString(), + compartmentId: invalidCompartmentId)); Assert.Equal("Compartment id is null or empty.", exception.Message); } @@ -488,31 +488,21 @@ public void GivenSearchParameterText_WhenCreated_ThenSearchParameterShouldBeAdde } [Theory] - [InlineData("false", "false")] - [InlineData("true", "false")] - [InlineData("false", "true")] - [InlineData("true", "true")] - [InlineData("TRUE", "TRUE")] - [InlineData("bad", "bad")] - public void GivenIncludeHistoryAndDeletedParameters_WhenCreated_ThenSearchParametersShouldMatchInput(string includeHistory, string includeDeleted) + [InlineData(ResourceVersionType.Latest)] + [InlineData(ResourceVersionType.Histoy)] + [InlineData(ResourceVersionType.SoftDeleted)] + [InlineData(ResourceVersionType.Latest | ResourceVersionType.Histoy)] + [InlineData(ResourceVersionType.Latest | ResourceVersionType.SoftDeleted)] + [InlineData(ResourceVersionType.Histoy | ResourceVersionType.SoftDeleted)] + [InlineData(ResourceVersionType.Latest | ResourceVersionType.Histoy | ResourceVersionType.SoftDeleted)] + public void GivenIncludeHistoryAndDeletedParameters_WhenCreated_ThenSearchParametersShouldMatchInput(ResourceVersionType resourceVersionTypes) { - var queryParameters = new[] - { - Tuple.Create(KnownQueryParameterNames.IncludeHistory, includeHistory), - Tuple.Create(KnownQueryParameterNames.IncludeDeleted, includeDeleted), - }; - - SearchOptions options = CreateSearchOptions(ResourceType.Patient.ToString(), queryParameters); + SearchOptions options = CreateSearchOptions(ResourceType.Patient.ToString(), null, resourceVersionTypes); Assert.NotNull(options); - bool includeHistoryBool = false; - bool includeDeletedBool = false; - - bool.TryParse(includeHistory, out includeHistoryBool); - bool.TryParse(includeDeleted, out includeDeletedBool); - - Assert.Equal(includeHistoryBool, options.IncludeHistory); - Assert.Equal(includeDeletedBool, options.IncludeDeleted); + Assert.Equal(resourceVersionTypes, options.ResourceVersionTypes); + Assert.True(options.ResourceVersionTypes.HasFlag(ResourceVersionType.Histoy)); + Assert.True(options.ResourceVersionTypes.HasFlag(ResourceVersionType.SoftDeleted)); Assert.Empty(options.UnsupportedSearchParams); } @@ -520,10 +510,11 @@ public void GivenIncludeHistoryAndDeletedParameters_WhenCreated_ThenSearchParame private SearchOptions CreateSearchOptions( string resourceType = DefaultResourceType, IReadOnlyList> queryParameters = null, + ResourceVersionType resourceVersionTypes = ResourceVersionType.Latest, string compartmentType = null, string compartmentId = null) { - return _factory.Create(compartmentType, compartmentId, resourceType, queryParameters); + return _factory.Create(compartmentType, compartmentId, resourceType, queryParameters, resourceVersionTypes: resourceVersionTypes); } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SearchOptionsFactory.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SearchOptionsFactory.cs index 42ea055386..7258c5faaf 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SearchOptionsFactory.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SearchOptionsFactory.cs @@ -70,9 +70,9 @@ public SearchOptionsFactory( _resourceTypeSearchParameter = _searchParameterDefinitionManager.GetSearchParameter(ResourceType.Resource.ToString(), SearchParameterNames.ResourceType); } - public SearchOptions Create(string resourceType, IReadOnlyList> queryParameters, bool isAsyncOperation = false) + public SearchOptions Create(string resourceType, IReadOnlyList> queryParameters, bool isAsyncOperation = false, ResourceVersionType resourceVersionTypes = ResourceVersionType.Latest) { - return Create(null, null, resourceType, queryParameters, isAsyncOperation); + return Create(null, null, resourceType, queryParameters, isAsyncOperation, resourceVersionTypes: resourceVersionTypes); } [SuppressMessage("Design", "CA1308", Justification = "ToLower() is required to format parameter output correctly.")] @@ -82,7 +82,8 @@ public SearchOptions Create( string resourceType, IReadOnlyList> queryParameters, bool isAsyncOperation = false, - bool useSmartCompartmentDefinition = false) + bool useSmartCompartmentDefinition = false, + ResourceVersionType resourceVersionTypes = ResourceVersionType.Latest) { var searchOptions = new SearchOptions(); @@ -184,14 +185,6 @@ public SearchOptions Create( throw new BadRequestException(string.Format(Core.Resources.InvalidTotalParameter, query.Item2, SupportedTotalTypes)); } } - else if (string.Equals(query.Item1, KnownQueryParameterNames.IncludeHistory, StringComparison.OrdinalIgnoreCase) && bool.TryParse(query.Item2, out bool includeHistory)) - { - searchOptions.IncludeHistory = includeHistory; - } - else if (string.Equals(query.Item1, KnownQueryParameterNames.IncludeDeleted, StringComparison.OrdinalIgnoreCase) && bool.TryParse(query.Item2, out bool includeDeleted)) - { - searchOptions.IncludeDeleted = includeDeleted; - } else { // Parse the search parameters. @@ -429,6 +422,8 @@ public SearchOptions Create( searchOptions.Sort = Array.Empty<(SearchParameterInfo searchParameterInfo, SortOrder sortOrder)>(); } + searchOptions.ResourceVersionTypes = resourceVersionTypes; + // Processing of parameters is finished. If any of the parameters are unsupported warning is put into the bundle or exception is thrown, // depending on the state of the "Prefer" header. if (unsupportedSearchParameters.Any() || searchSortErrors.Any()) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlSearchOptions.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlSearchOptions.cs index f4dbcfac6d..aa4c43bdb0 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlSearchOptions.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlSearchOptions.cs @@ -37,14 +37,19 @@ public SqlSearchOptions(SearchOptions searchOptions) internal SqlSearchType GetSearchTypeFromOptions() { - var searchType = SqlSearchType.Default; + SqlSearchType searchType = 0; - if (IncludeHistory) + if (ResourceVersionTypes.HasFlag(ResourceVersionType.Latest)) + { + searchType |= SqlSearchType.Default; + } + + if (ResourceVersionTypes.HasFlag(ResourceVersionType.Histoy)) { searchType |= SqlSearchType.IncludeHistory; } - if (IncludeDeleted) + if (ResourceVersionTypes.HasFlag(ResourceVersionType.SoftDeleted)) { searchType |= SqlSearchType.IncludeDeleted; } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs index 94b9a36fd0..f2a7804748 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs @@ -203,12 +203,6 @@ public override async Task SearchAsync(SearchOptions searchOptions return searchResult; } - protected override async Task SearchHistoryInternalAsync(SearchOptions searchOptions, CancellationToken cancellationToken) - { - SqlSearchOptions sqlSearchOptions = new SqlSearchOptions(searchOptions); - return await SearchImpl(sqlSearchOptions, SqlSearchType.IncludeHistory | SqlSearchType.IncludeDeleted, null, cancellationToken); - } - private async Task SearchImpl(SqlSearchOptions sqlSearchOptions, SqlSearchType searchType, string currentSearchParameterHash, CancellationToken cancellationToken) { Expression searchExpression = sqlSearchOptions.Expression; @@ -553,7 +547,7 @@ private void PopulateSqlCommandFromQueryHints(SqlSearchOptions options, SqlComma var globalStartId = long.Parse(hints.First(_ => _.Param == KnownQueryParameterNames.GlobalStartSurrogateId).Value); var globalEndId = long.Parse(hints.First(_ => _.Param == KnownQueryParameterNames.GlobalEndSurrogateId).Value); - PopulateSqlCommandFromQueryHints(command, resourceTypeId, startId, endId, globalStartId, globalEndId, options.IncludeHistory, options.IncludeDeleted); + PopulateSqlCommandFromQueryHints(command, resourceTypeId, startId, endId, globalStartId, globalEndId, options.ResourceVersionTypes.HasFlag(ResourceVersionType.Histoy), options.ResourceVersionTypes.HasFlag(ResourceVersionType.SoftDeleted)); } private static void PopulateSqlCommandFromQueryHints(SqlCommand command, short resourceTypeId, long startId, long endId, long? globalStartId, long? globalEndId, bool? includeHistory, bool? includeDeleted) From f5be1d5fbced7adb72f3fb38d4e61b726bd0dc4a Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Mon, 6 Nov 2023 15:49:59 -0800 Subject: [PATCH 46/58] Fixed test issue --- .../Features/Search/SearchServiceTests.cs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchServiceTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchServiceTests.cs index 4f96890ecd..99eb65be96 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchServiceTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchServiceTests.cs @@ -174,13 +174,6 @@ public override Task SearchAsync( return Task.FromResult(SearchImplementation(searchOptions)); } - protected override Task SearchHistoryInternalAsync( - SearchOptions searchOptions, - CancellationToken cancellationToken) - { - return Task.FromResult(SearchImplementation(searchOptions)); - } - protected override Task SearchForReindexInternalAsync(SearchOptions searchOptions, string searchParameterHash, CancellationToken cancellationToken) { return Task.FromResult(SearchImplementation(searchOptions)); From 24412ebb61289125cf4c8407da6b3f30aaa81652 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Mon, 6 Nov 2023 19:54:34 -0800 Subject: [PATCH 47/58] Removing NA SQL comment --- .../Features/Schema/Migrations/68.diff.sql | 2 +- .../Schema/Sql/Sprocs/GetResourcesByTypeAndSurrogateIdRange.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/68.diff.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/68.diff.sql index 947ba59e22..99c259e06a 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/68.diff.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/68.diff.sql @@ -5,7 +5,7 @@ DECLARE @SP varchar(100) = 'GetResourcesByTypeAndSurrogateIdRange' ,@Mode varchar(100) = 'RT='+isnull(convert(varchar,@ResourceTypeId),'NULL') +' S='+isnull(convert(varchar,@StartId),'NULL') +' E='+isnull(convert(varchar,@EndId),'NULL') - +' GE='+isnull(convert(varchar,@GlobalEndId),'NULL') -- Could this just be a boolean for if historical records should be returned? GlobalEndId should equal EndId in all cases I can think of. + +' GE='+isnull(convert(varchar,@GlobalEndId),'NULL') +' HI='+isnull(convert(varchar,@IncludeHistory),'NULL') +' DE'+isnull(convert(varchar,@IncludeDeleted),'NULL') ,@st datetime = getUTCdate() diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetResourcesByTypeAndSurrogateIdRange.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetResourcesByTypeAndSurrogateIdRange.sql index be7eb142e2..4b8115c946 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetResourcesByTypeAndSurrogateIdRange.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetResourcesByTypeAndSurrogateIdRange.sql @@ -7,7 +7,7 @@ DECLARE @SP varchar(100) = 'GetResourcesByTypeAndSurrogateIdRange' ,@Mode varchar(100) = 'RT='+isnull(convert(varchar,@ResourceTypeId),'NULL') +' S='+isnull(convert(varchar,@StartId),'NULL') +' E='+isnull(convert(varchar,@EndId),'NULL') - +' GE='+isnull(convert(varchar,@GlobalEndId),'NULL') -- Could this just be a boolean for if historical records should be returned? GlobalEndId should equal EndId in all cases I can think of. + +' GE='+isnull(convert(varchar,@GlobalEndId),'NULL') +' HI='+isnull(convert(varchar,@IncludeHistory),'NULL') +' DE'+isnull(convert(varchar,@IncludeDeleted),'NULL') ,@st datetime = getUTCdate() From c6401d46212a1f7cc2d233eaae37a1b2e49e6c34 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Mon, 6 Nov 2023 19:59:36 -0800 Subject: [PATCH 48/58] PR comments on tests fix --- .../Rest/Export/ExportDataTests.cs | 11 ++++------- .../Rest/Export/ExportTestHelper.cs | 1 - 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs index d45703335d..7d8d5fb77c 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs @@ -145,6 +145,7 @@ public async Task GivenFhirServer_WhenAllDataIsExportedToASpecificContainer_Then [Fact] [Trait(Traits.Category, Categories.ExportLongRunning)] + [HttpIntegrationFixtureArgumentSets(dataStores: DataStore.SqlServer)] public async Task GivenFhirServer_WhenDataIsExportedWithHistoryParallel_ThenExportedDataIsSameAsDataInFhirServer() { await ExportAndSoftDeleteTestHelper(parallel: true, history: true, deletes: false); @@ -159,6 +160,7 @@ public async Task GivenFhirServer_WhenDataIsExportedWithHistoryNotParallel_ThenE [Fact] [Trait(Traits.Category, Categories.ExportLongRunning)] + [HttpIntegrationFixtureArgumentSets(dataStores: DataStore.SqlServer)] public async Task GivenFhirServer_WhenDataIsExportedWithSoftDeletesParallel_ThenExportedDataIsSameAsDataInFhirServer() { await ExportAndSoftDeleteTestHelper(parallel: true, history: false, deletes: true); @@ -172,6 +174,7 @@ public async Task GivenFhirServer_WhenDataIsExportedWithSoftDeletesNotParallel_T } [Fact] + [HttpIntegrationFixtureArgumentSets(dataStores: DataStore.SqlServer)] public async Task GivenFhirServer_WhenDataIsExportedWithHistoryAndSoftDeletesParallel_ThenExportedDataIsSameAsDataInFhirServer() { await ExportAndSoftDeleteTestHelper(parallel: true, history: true, deletes: true); @@ -187,12 +190,6 @@ public async Task GivenFhirServer_WhenDataIsExportedWithHistoryAndSoftDeletesNot // _tag filter cannot be used with history or deleted export. Using isParallel to test both SQL code paths. private async Task ExportAndSoftDeleteTestHelper(bool parallel, bool history, bool deletes) { - if (_fixture.DataStore == DataStore.CosmosDb && parallel) - { - // CosmosDB does not have parallel export support yet. - return; - } - string uniqueFixtureResources = string.Join(',', _fixture.TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); string includeAssociatedDataParam = (history ? "_history" : string.Empty) + (deletes ? (history ? "," : string.Empty) + "_deleted" : string.Empty); @@ -219,7 +216,7 @@ private async Task ExportAndSoftDeleteTestHelper(bool parallel, bool history, bo } // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResourcesWithHistoryAndDeletes, dataFromExport, _outputHelper)); + Assert.True(ExportTestHelper.ValidateDataFromBothSources(expectedResources, dataFromExport, _outputHelper)); } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs index f45fe557be..b82f220293 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs @@ -81,7 +81,6 @@ internal static async Task> CheckExportStatus(TestFhirClient testFhir catch (Exception ex) { outputHelper.WriteLine($"Unable to parse response into bundle: {ex}"); - return resourceIdToResourceMapping; } return resourceIdToResourceMapping; From f0fb0dfb26a9d9d62b9ac33f18a541430ba737ea Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Mon, 6 Nov 2023 20:02:36 -0800 Subject: [PATCH 49/58] "un-fancified" ResourceTypeVersion enum --- .../Features/Search/ResourceVersionType.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/ResourceVersionType.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/ResourceVersionType.cs index 86475ab32e..11e0fe4c4c 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/ResourceVersionType.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/ResourceVersionType.cs @@ -21,11 +21,11 @@ public enum ResourceVersionType /// /// Previous versions of the resource - i.e. historical. /// - Histoy = 1 << 1, + Histoy = 2, /// /// Resources that have been deleted but are still in the system (soft-delete). /// - SoftDeleted = 1 << 2, + SoftDeleted = 4, } } From 846660c8954a862def30fbf42f1941498135938e Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Tue, 7 Nov 2023 10:16:21 -0800 Subject: [PATCH 50/58] Fix STU3 build error --- .../Features/Search/SearchOptionsFactoryTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Stu3.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs b/src/Microsoft.Health.Fhir.Stu3.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs index 676497b807..475bf8f78d 100644 --- a/src/Microsoft.Health.Fhir.Stu3.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs +++ b/src/Microsoft.Health.Fhir.Stu3.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs @@ -149,8 +149,8 @@ public void GivenSearchParamsWithValidCompartmentSearch_WhenCreated_ThenCorrectC SearchOptions options = CreateSearchOptions( resourceType: resourceType.ToString(), queryParameters: queryParameters, - compartmentType.ToString(), - compartmentId); + compartmentType: compartmentType.ToString(), + compartmentId: compartmentId); Assert.NotNull(options); Assert.NotNull(options.Expression); From f9709c6244f97130a1f2224110e43f708aa8fb83 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Tue, 7 Nov 2023 12:19:08 -0800 Subject: [PATCH 51/58] Fixed unit tests that no longer use export/history query params --- .../Operations/Export/ExportJobTaskTests.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Export/ExportJobTaskTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Export/ExportJobTaskTests.cs index 2e5bef4790..156afbbfe8 100644 --- a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Export/ExportJobTaskTests.cs +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Export/ExportJobTaskTests.cs @@ -149,7 +149,8 @@ public async Task GivenThereAreTwoPagesOfSearchResults_WhenExecuted_ThenCorrectS null, Arg.Is(CreateQueryParametersExpression(KnownResourceTypes.Patient)), _cancellationToken, - true) + true, + ResourceVersionType.Latest) .Returns(CreateSearchResult(continuationToken: continuationToken)); bool capturedSearch = false; @@ -159,7 +160,8 @@ public async Task GivenThereAreTwoPagesOfSearchResults_WhenExecuted_ThenCorrectS null, Arg.Is(CreateQueryParametersExpressionWithContinuationToken(ContinuationTokenConverter.Encode(continuationToken), KnownResourceTypes.Patient)), _cancellationToken, - true) + true, + ResourceVersionType.Latest) .Returns(x => { capturedSearch = true; @@ -339,7 +341,8 @@ public async Task GivenAnExportJobWithHistoryAndSoftDeletes_WhenExecuted_ThenAll null, Arg.Is(CreateQueryParametersExpression(KnownResourceTypes.Patient, includeHistory: true, includeDeleted: true)), _cancellationToken, - true) + true, + ResourceVersionType.Latest | ResourceVersionType.Histoy | ResourceVersionType.SoftDeleted) .Returns(x => { capturedSearch = true; @@ -357,9 +360,7 @@ private Expression>>> CreateQueryP return arg => arg != null && arg.Any(x => x.Item1 == "_count" && x.Item2 == "1") && arg.Any(x => x.Item1 == "_lastUpdated" && x.Item2 == $"le{_exportJobRecord.Till}") && - arg.Any(x => x.Item1 == "_type" && x.Item2 == resourceType) && - arg.Any(x => x.Item1 == "_includeHistory" && x.Item2 == includeHistory.ToString()) && - arg.Any(x => x.Item1 == "_includeDeleted" && x.Item2 == includeDeleted.ToString()); + arg.Any(x => x.Item1 == "_type" && x.Item2 == resourceType); } private Expression>>> CreateQueryParametersExpression(PartialDateTime since, string resourceType) From de16f084a246875f2787521208476ec52184c28c Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Tue, 7 Nov 2023 13:00:38 -0800 Subject: [PATCH 52/58] fixed test mistake for resourcetypeversion --- .../Features/Search/SearchOptionsFactoryTests.cs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs index d15ab20438..b9de7455a7 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs @@ -497,13 +497,9 @@ public void GivenSearchParameterText_WhenCreated_ThenSearchParameterShouldBeAdde [InlineData(ResourceVersionType.Latest | ResourceVersionType.Histoy | ResourceVersionType.SoftDeleted)] public void GivenIncludeHistoryAndDeletedParameters_WhenCreated_ThenSearchParametersShouldMatchInput(ResourceVersionType resourceVersionTypes) { - SearchOptions options = CreateSearchOptions(ResourceType.Patient.ToString(), null, resourceVersionTypes); + SearchOptions options = CreateSearchOptions(ResourceType.Patient.ToString(), new List>(), resourceVersionTypes); Assert.NotNull(options); - Assert.Equal(resourceVersionTypes, options.ResourceVersionTypes); - Assert.True(options.ResourceVersionTypes.HasFlag(ResourceVersionType.Histoy)); - Assert.True(options.ResourceVersionTypes.HasFlag(ResourceVersionType.SoftDeleted)); - Assert.Empty(options.UnsupportedSearchParams); } From 6adfe70e12b67a2b0776e6109f9a99ecaef38606 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Tue, 7 Nov 2023 17:33:17 -0800 Subject: [PATCH 53/58] Fixed cosmos export search functoin --- src/Microsoft.Health.Fhir.Core/Features/Search/SearchService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SearchService.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SearchService.cs index e90b346cd1..2640232eee 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/SearchService.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SearchService.cs @@ -175,7 +175,7 @@ public async Task SearchHistoryAsync( queryParameters.Add(Tuple.Create(KnownQueryParameterNames.Sort, $"-{KnownQueryParameterNames.LastUpdated}")); } - SearchOptions searchOptions = _searchOptionsFactory.Create(resourceType, queryParameters, isAsyncOperation, ResourceVersionType.Histoy); + SearchOptions searchOptions = _searchOptionsFactory.Create(resourceType, queryParameters, isAsyncOperation, ResourceVersionType.Latest | ResourceVersionType.Histoy); SearchResult searchResult = await SearchAsync(searchOptions, cancellationToken); From 228a42532f19ac5c0f9340dcf23214d4482e9507 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Tue, 7 Nov 2023 20:42:39 -0800 Subject: [PATCH 54/58] Fixed export includeAssociatedData error message --- src/Microsoft.Health.Fhir.Api/Resources.Designer.cs | 6 +++--- src/Microsoft.Health.Fhir.Api/Resources.resx | 4 ++-- .../Controllers/ExportController.cs | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Api/Resources.Designer.cs b/src/Microsoft.Health.Fhir.Api/Resources.Designer.cs index d47d1321ad..b287622c85 100644 --- a/src/Microsoft.Health.Fhir.Api/Resources.Designer.cs +++ b/src/Microsoft.Health.Fhir.Api/Resources.Designer.cs @@ -383,16 +383,16 @@ public static string InvalidElementsParameter { return ResourceManager.GetString("InvalidElementsParameter", resourceCulture); } } - + /// - /// Looks up a localized string similar to invalid parametr given for export associated data. + /// Looks up a localized string similar to The export parameter "includeAssociatedData" contains an invalid value. Supported values are: {0}. . /// public static string InvalidExportAssociatedDataParameter { get { return ResourceManager.GetString("InvalidExportAssociatedDataParameter", resourceCulture); } } - + /// /// Looks up a localized string similar to Invalid launch context parameters.. /// diff --git a/src/Microsoft.Health.Fhir.Api/Resources.resx b/src/Microsoft.Health.Fhir.Api/Resources.resx index 6885d779a7..c53c8f73d4 100644 --- a/src/Microsoft.Health.Fhir.Api/Resources.resx +++ b/src/Microsoft.Health.Fhir.Api/Resources.resx @@ -412,6 +412,6 @@ The request "_typeFilter" cannot be used with an export request with historical or soft deleted resources. - The export parameter "includeAssociatedData" contains an invalid value. Only values "_includeHistory" and "_includeHistory" are supported. + The export parameter "includeAssociatedData" contains an invalid value. Supported values are: {0}. - \ No newline at end of file + diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs b/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs index 551bccf987..319e4fe377 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs @@ -379,7 +379,7 @@ private static (bool includeHistory, bool includeDeleted) ValidateAndParseInclud if ((includeHistory || includeDeleted) && !string.IsNullOrWhiteSpace(typeFilter)) { - throw new RequestNotValidException(Resources.TypeFilterNotSupportedWithHistoryOrDeletedExport); + throw new RequestNotValidException(string.Format(Resources.TypeFilterNotSupportedWithHistoryOrDeletedExport, "_history and _deleted")); } return (includeHistory, includeDeleted); From b295ca567e54e6d98d6ff2cc7fcdabc9251d4540 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Tue, 7 Nov 2023 20:42:52 -0800 Subject: [PATCH 55/58] Default ResourceVersionType to latest --- src/Microsoft.Health.Fhir.Core/Features/Search/SearchOptions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SearchOptions.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SearchOptions.cs index b58b7daa90..0294422500 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/SearchOptions.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SearchOptions.cs @@ -116,7 +116,7 @@ internal set /// /// Which version types (latest, soft-deleted, history) to include in search. /// - public ResourceVersionType ResourceVersionTypes { get; internal set; } + public ResourceVersionType ResourceVersionTypes { get; internal set; } = ResourceVersionType.Latest; /// /// Gets the search expression. From 7731f6fb882086e6b31113b112658faaf77189cc Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Wed, 8 Nov 2023 07:18:38 -0800 Subject: [PATCH 56/58] Fixed history search - added deleted --- .../Features/Search/SearchService.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SearchService.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SearchService.cs index 2640232eee..5903179883 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/SearchService.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SearchService.cs @@ -175,7 +175,9 @@ public async Task SearchHistoryAsync( queryParameters.Add(Tuple.Create(KnownQueryParameterNames.Sort, $"-{KnownQueryParameterNames.LastUpdated}")); } - SearchOptions searchOptions = _searchOptionsFactory.Create(resourceType, queryParameters, isAsyncOperation, ResourceVersionType.Latest | ResourceVersionType.Histoy); + var historyResourceVersionTypes = ResourceVersionType.Latest | ResourceVersionType.Histoy | ResourceVersionType.SoftDeleted; + + SearchOptions searchOptions = _searchOptionsFactory.Create(resourceType, queryParameters, isAsyncOperation, historyResourceVersionTypes); SearchResult searchResult = await SearchAsync(searchOptions, cancellationToken); From 8fb1a5f4715d836254026a1aa12be019967efce8 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 9 Nov 2023 11:49:37 -0800 Subject: [PATCH 57/58] fix export count issue? --- .../Features/Operations/Export/ExportJobTask.cs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs index a3b17f7bbe..773edaf223 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs @@ -169,7 +169,7 @@ public async Task ExecuteAsync(ExportJobRecord exportJobRecord, WeakETag weakETa // from the search result. // As Till is a new property QueuedTime is being used as a backup incase Till doesn't exist in the job record. var tillTime = _exportJobRecord.Till != null ? _exportJobRecord.Till : new PartialDateTime(_exportJobRecord.QueuedTime); - List> queryParametersList = new() + var queryParametersList = new List>() { Tuple.Create(KnownQueryParameterNames.Count, _exportJobRecord.MaximumNumberOfResourcesPerQuery.ToString(CultureInfo.InvariantCulture)), Tuple.Create(KnownQueryParameterNames.LastUpdated, $"le{tillTime}"), @@ -188,9 +188,9 @@ public async Task ExecuteAsync(ExportJobRecord exportJobRecord, WeakETag weakETa queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.LastUpdated, $"ge{_exportJobRecord.Since}")); } - var exportResourceVersionTypes = ResourceVersionType.Latest; - exportResourceVersionTypes |= _exportJobRecord.IncludeHistory ? ResourceVersionType.Histoy : 0; - exportResourceVersionTypes |= _exportJobRecord.IncludeDeleted ? ResourceVersionType.SoftDeleted : 0; + var exportResourceVersionTypes = ResourceVersionType.Latest | + (_exportJobRecord.IncludeHistory ? ResourceVersionType.Histoy : 0) | + (_exportJobRecord.IncludeDeleted ? ResourceVersionType.SoftDeleted : 0); ExportJobProgress progress = _exportJobRecord.Progress; @@ -371,7 +371,7 @@ private async Task RunExportSearch( if (progress.CurrentFilter != null) { - await ProcessFilter(exportJobConfiguration, progress, queryParametersList, sharedQueryParametersList, anonymizer, cancellationToken); + await ProcessFilter(exportJobConfiguration, progress, queryParametersList, sharedQueryParametersList, resourceVersionTypes, anonymizer, cancellationToken); } if (_exportJobRecord.Filters != null && _exportJobRecord.Filters.Any(filter => !progress.CompletedFilters.Contains(filter))) @@ -384,7 +384,7 @@ private async Task RunExportSearch( (_exportJobRecord.ExportType == ExportJobType.All || filter.ResourceType.Equals(KnownResourceTypes.Patient, StringComparison.OrdinalIgnoreCase))) { progress.SetFilter(filter); - await ProcessFilter(exportJobConfiguration, progress, queryParametersList, sharedQueryParametersList, anonymizer, cancellationToken); + await ProcessFilter(exportJobConfiguration, progress, queryParametersList, sharedQueryParametersList, resourceVersionTypes, anonymizer, cancellationToken); } } } @@ -432,6 +432,7 @@ private async Task ProcessFilter( ExportJobProgress exportJobProgress, List> queryParametersList, List> sharedQueryParametersList, + ResourceVersionType resourceVersionTypes, IAnonymizer anonymizer, CancellationToken cancellationToken) { @@ -441,7 +442,7 @@ private async Task ProcessFilter( filterQueryParametersList.Add(param); } - await SearchWithFilter(exportJobConfiguration, exportJobProgress, exportJobProgress.CurrentFilter.ResourceType, filterQueryParametersList, sharedQueryParametersList, ResourceVersionType.Latest, anonymizer, cancellationToken); + await SearchWithFilter(exportJobConfiguration, exportJobProgress, exportJobProgress.CurrentFilter.ResourceType, filterQueryParametersList, sharedQueryParametersList, resourceVersionTypes, anonymizer, cancellationToken); exportJobProgress.MarkFilterFinished(); await UpdateJobRecordAsync(cancellationToken); From c44bdda8ad347c03146e58327c01b169a1ec0ce4 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 9 Nov 2023 12:31:12 -0800 Subject: [PATCH 58/58] small code cleanups --- .../Operations/Export/CosmosExportOrchestratorJob.cs | 10 +++++----- .../Operations/Export/SqlExportOrchestratorJob.cs | 6 +----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Operations/Export/CosmosExportOrchestratorJob.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Operations/Export/CosmosExportOrchestratorJob.cs index 4e4fac74e9..75dfc4b862 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Operations/Export/CosmosExportOrchestratorJob.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Operations/Export/CosmosExportOrchestratorJob.cs @@ -7,6 +7,7 @@ using System.Threading; using System.Threading.Tasks; using EnsureThat; +using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.Core.Features.Operations.Export; using Microsoft.Health.Fhir.Core.Features.Operations.Export.Models; @@ -89,11 +90,10 @@ private static ExportJobRecord CreateExportRecord(ExportJobRecord record, long g record.IncludeDeleted, record.SchemaVersion, (int)JobType.ExportProcessing, - record.SmartRequest) - { - Id = string.Empty, - QueuedTime = record.QueuedTime, // preserve create date of coordinator job in form of queued time for all children, so same time is used on file names. - }; + record.SmartRequest); + + rec.Id = string.Empty; + rec.QueuedTime = record.QueuedTime; // preserve create date of coordinator job in form of queued time for all children, so same time is used on file names. return rec; } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/Export/SqlExportOrchestratorJob.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/Export/SqlExportOrchestratorJob.cs index 3b0b853ec9..6492d454bc 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/Export/SqlExportOrchestratorJob.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/Export/SqlExportOrchestratorJob.cs @@ -41,12 +41,8 @@ public SqlExportOrchestratorJob( _queueClient = queueClient; _searchService = searchService; _exportJobConfiguration = exportJobConfiguration.Value; - - NumberOfSurrogateIdRanges = _exportJobConfiguration.NumberOfParallelRecordRanges; } - internal int NumberOfSurrogateIdRanges { get; set; } - public async Task ExecuteAsync(JobInfo jobInfo, IProgress progress, CancellationToken cancellationToken) { EnsureArg.IsNotNull(jobInfo, nameof(jobInfo)); @@ -91,7 +87,7 @@ public async Task ExecuteAsync(JobInfo jobInfo, IProgress progre while (rows > 0) { var definitions = new List(); - var ranges = await _searchService.GetSurrogateIdRanges(type, startId, globalEndId, surrogateIdRangeSize, NumberOfSurrogateIdRanges, true, cancel); + var ranges = await _searchService.GetSurrogateIdRanges(type, startId, globalEndId, surrogateIdRangeSize, _exportJobConfiguration.NumberOfParallelRecordRanges, true, cancel); foreach (var range in ranges) { if (range.EndId > startId)