diff --git a/docs/rest/ExportRequests.http b/docs/rest/ExportRequests.http index 2be1596599..9beb5e9c73 100644 --- a/docs/rest/ExportRequests.http +++ b/docs/rest/ExportRequests.http @@ -71,6 +71,13 @@ Accept: application/fhir+json Prefer: respond-async Authorization: Bearer {{bearer.response.body.access_token}} +### Export with history and soft deleted records +# @name export +GET https://{{hostname}}/$export?includeAssociatedData=_history,_deleted +Accept: application/fhir+json +Prefer: respond-async +Authorization: Bearer {{bearer.response.body.access_token}} + ### Get Export request GET {{exportLocation}} Authorization: Bearer {{bearer.response.body.access_token}} diff --git a/src/Microsoft.Health.Fhir.Api/Features/Filters/ValidateExportRequestFilterAttribute.cs b/src/Microsoft.Health.Fhir.Api/Features/Filters/ValidateExportRequestFilterAttribute.cs index 4e187b183b..5d5ead79ef 100644 --- a/src/Microsoft.Health.Fhir.Api/Features/Filters/ValidateExportRequestFilterAttribute.cs +++ b/src/Microsoft.Health.Fhir.Api/Features/Filters/ValidateExportRequestFilterAttribute.cs @@ -45,6 +45,7 @@ public ValidateExportRequestFilterAttribute() KnownQueryParameterNames.Format, KnownQueryParameterNames.TypeFilter, KnownQueryParameterNames.IsParallel, + KnownQueryParameterNames.IncludeAssociatedData, KnownQueryParameterNames.MaxCount, KnownQueryParameterNames.AnonymizationConfigurationCollectionReference, KnownQueryParameterNames.AnonymizationConfigurationLocation, diff --git a/src/Microsoft.Health.Fhir.Api/Resources.Designer.cs b/src/Microsoft.Health.Fhir.Api/Resources.Designer.cs index 0e81b2c21e..b287622c85 100644 --- a/src/Microsoft.Health.Fhir.Api/Resources.Designer.cs +++ b/src/Microsoft.Health.Fhir.Api/Resources.Designer.cs @@ -384,6 +384,15 @@ public static string InvalidElementsParameter { } } + /// + /// Looks up a localized string similar to The export parameter "includeAssociatedData" contains an invalid value. Supported values are: {0}. . + /// + public static string InvalidExportAssociatedDataParameter { + get { + return ResourceManager.GetString("InvalidExportAssociatedDataParameter", resourceCulture); + } + } + /// /// Looks up a localized string similar to Invalid launch context parameters.. /// @@ -708,6 +717,15 @@ public static string TransactionFailed { } } + /// + /// Looks up a localized string similar to The request "_typeFilter" cannot be used with an export request with historical or soft deleted resources.. + /// + public static string TypeFilterNotSupportedWithHistoryOrDeletedExport { + get { + return ResourceManager.GetString("TypeFilterNotSupportedWithHistoryOrDeletedExport", resourceCulture); + } + } + /// /// Looks up a localized string similar to The _type parameter must be included when using the _typeFilter parameter. . /// diff --git a/src/Microsoft.Health.Fhir.Api/Resources.resx b/src/Microsoft.Health.Fhir.Api/Resources.resx index 4746b029eb..c53c8f73d4 100644 --- a/src/Microsoft.Health.Fhir.Api/Resources.resx +++ b/src/Microsoft.Health.Fhir.Api/Resources.resx @@ -408,4 +408,10 @@ Invalid combination of processing logic and bundle type: {0} and {1}. Error message when there is a invalid/unknown combination of a bundle type and a processing logic. - \ No newline at end of file + + The request "_typeFilter" cannot be used with an export request with historical or soft deleted resources. + + + The export parameter "includeAssociatedData" contains an invalid value. Supported values are: {0}. + + diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Export/ExportJobTaskTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Export/ExportJobTaskTests.cs index 722d77c1b7..156afbbfe8 100644 --- a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Export/ExportJobTaskTests.cs +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Export/ExportJobTaskTests.cs @@ -6,6 +6,7 @@ using System; using System.Collections.Generic; using System.Globalization; +using System.Linq; using System.Linq.Expressions; using System.Net; using System.Threading; @@ -20,6 +21,7 @@ using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Core.Configs; using Microsoft.Health.Fhir.Core.Exceptions; +using Microsoft.Health.Fhir.Core.Features; using Microsoft.Health.Fhir.Core.Features.Context; using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.Core.Features.Operations.Export; @@ -147,7 +149,8 @@ public async Task GivenThereAreTwoPagesOfSearchResults_WhenExecuted_ThenCorrectS null, Arg.Is(CreateQueryParametersExpression(KnownResourceTypes.Patient)), _cancellationToken, - true) + true, + ResourceVersionType.Latest) .Returns(CreateSearchResult(continuationToken: continuationToken)); bool capturedSearch = false; @@ -157,7 +160,8 @@ public async Task GivenThereAreTwoPagesOfSearchResults_WhenExecuted_ThenCorrectS null, Arg.Is(CreateQueryParametersExpressionWithContinuationToken(ContinuationTokenConverter.Encode(continuationToken), KnownResourceTypes.Patient)), _cancellationToken, - true) + true, + ResourceVersionType.Latest) .Returns(x => { capturedSearch = true; @@ -321,40 +325,70 @@ public async Task GivenThereAreMultiplePagesOfSearchResultsWithSinceParameter_Wh Assert.True(secondCapturedSearch); } - private Expression>>> CreateQueryParametersExpression(string resourceType) + [Fact] + public async Task GivenAnExportJobWithHistoryAndSoftDeletes_WhenExecuted_ThenAllResourcesAreExportedToTheProperLocation() + { + bool capturedSearch = false; + + var exportJobRecordIncludeHistory = CreateExportJobRecord( + exportJobType: ExportJobType.Patient, + includeHistory: true, + includeDeleted: true, + maximumNumberOfResourcesPerQuery: 1); + SetupExportJobRecordAndOperationDataStore(exportJobRecordIncludeHistory); + + _searchService.SearchAsync( + null, + Arg.Is(CreateQueryParametersExpression(KnownResourceTypes.Patient, includeHistory: true, includeDeleted: true)), + _cancellationToken, + true, + ResourceVersionType.Latest | ResourceVersionType.Histoy | ResourceVersionType.SoftDeleted) + .Returns(x => + { + capturedSearch = true; + + return CreateSearchResult(); + }); + + await _exportJobTask.ExecuteAsync(_exportJobRecord, _weakETag, _cancellationToken); + + Assert.True(capturedSearch); + } + + private Expression>>> CreateQueryParametersExpression(string resourceType, bool includeHistory = false, bool includeDeleted = false) { return arg => arg != null && - Tuple.Create("_count", "1").Equals(arg[0]) && - Tuple.Create("_lastUpdated", $"le{_exportJobRecord.Till}").Equals(arg[1]) && - Tuple.Create("_type", resourceType).Equals(arg[2]); + arg.Any(x => x.Item1 == "_count" && x.Item2 == "1") && + arg.Any(x => x.Item1 == "_lastUpdated" && x.Item2 == $"le{_exportJobRecord.Till}") && + arg.Any(x => x.Item1 == "_type" && x.Item2 == resourceType); } private Expression>>> CreateQueryParametersExpression(PartialDateTime since, string resourceType) { return arg => arg != null && - Tuple.Create("_count", "1").Equals(arg[0]) && - Tuple.Create("_lastUpdated", $"le{_exportJobRecord.Till}").Equals(arg[1]) && - Tuple.Create("_lastUpdated", $"ge{since}").Equals(arg[2]) && - Tuple.Create("_type", resourceType).Equals(arg[3]); + arg.Any(x => x.Item1 == "_count" && x.Item2 == "1") && + arg.Any(x => x.Item1 == "_lastUpdated" && x.Item2 == $"le{_exportJobRecord.Till}") && + arg.Any(x => x.Item1 == "_lastUpdated" && x.Item2 == $"ge{since}") && + arg.Any(x => x.Item1 == "_type" && x.Item2 == resourceType); } private Expression>>> CreateQueryParametersExpressionWithContinuationToken(string continuationToken, string resourceType) { return arg => arg != null && - Tuple.Create("_count", "1").Equals(arg[0]) && - Tuple.Create("_lastUpdated", $"le{_exportJobRecord.Till}").Equals(arg[1]) && - Tuple.Create("_type", resourceType).Equals(arg[2]) && - Tuple.Create("ct", continuationToken).Equals(arg[3]); + arg.Any(x => x.Item1 == "_count" && x.Item2 == "1") && + arg.Any(x => x.Item1 == "_lastUpdated" && x.Item2 == $"le{_exportJobRecord.Till}") && + arg.Any(x => x.Item1 == "_type" && x.Item2 == resourceType) && + arg.Any(x => x.Item1 == "ct" && x.Item2 == continuationToken); } private Expression>>> CreateQueryParametersExpressionWithContinuationToken(string continuationToken, PartialDateTime since, string resourceType) { return arg => arg != null && - Tuple.Create("_count", "1").Equals(arg[0]) && - Tuple.Create("_lastUpdated", $"le{_exportJobRecord.Till}").Equals(arg[1]) && - Tuple.Create("_lastUpdated", $"ge{since}").Equals(arg[2]) && - Tuple.Create("_type", resourceType).Equals(arg[3]) && - Tuple.Create("ct", continuationToken).Equals(arg[4]); + arg.Any(x => x.Item1 == "_count" && x.Item2 == "1") && + arg.Any(x => x.Item1 == "_lastUpdated" && x.Item2 == $"le{_exportJobRecord.Till}") && + arg.Any(x => x.Item1 == "_lastUpdated" && x.Item2 == $"ge{since}") && + arg.Any(x => x.Item1 == "_type" && x.Item2 == resourceType) && + arg.Any(x => x.Item1 == "ct" && x.Item2 == continuationToken); } [Fact] @@ -834,7 +868,9 @@ public async Task GivenAnExportJobWithTheTypeParameter_WhenExecuted_ThenOnlyReso true) .Returns(x => { - string[] types = x.ArgAt>>(1)[3].Item2.Split(','); + string[] types = x.ArgAt>>(1) + .Where(x => x.Item1 == KnownQueryParameterNames.Type) + .Select(x => x.Item2).First().Split(','); SearchResultEntry[] entries = new SearchResultEntry[types.Length]; for (int index = 0; index < types.Length; index++) @@ -889,7 +925,9 @@ public async Task GivenAPatientExportJobWithTheTypeParameter_WhenExecuted_ThenOn true) .Returns(x => { - string[] types = x.ArgAt>>(3)[3].Item2.Split(','); + string[] types = x.ArgAt>>(3) + .Where(x => x.Item1 == KnownQueryParameterNames.Type) + .Select(x => x.Item2).First().Split(','); SearchResultEntry[] entries = new SearchResultEntry[types.Length]; for (int index = 0; index < types.Length; index++) @@ -1107,7 +1145,9 @@ public async Task GivenAGroupExportJob_WhenExecuted_ThenAllPatientResourcesInThe true) .Returns(x => { - string[] ids = x.ArgAt>>(1)[2].Item2.Split(','); + string[] ids = x.ArgAt>>(1) + .Where(x => x.Item1 == Core.Features.KnownQueryParameterNames.Id) + .Select(x => x.Item2).First().Split(','); SearchResultEntry[] entries = new SearchResultEntry[ids.Length]; for (int index = 0; index < ids.Length; index++) @@ -1175,7 +1215,9 @@ public async Task GivenAGroupExportJobWithMultiplePagesOfPatients_WhenExecuted_T true) .Returns(x => { - string[] ids = x.ArgAt>>(1)[2].Item2.Split(','); + string[] ids = x.ArgAt>>(1) + .Where(x => x.Item1 == KnownQueryParameterNames.Id) + .Select(x => x.Item2).First().Split(','); countOfSearches++; @@ -1250,7 +1292,9 @@ public async Task GivenAGroupExportJobToResume_WhenExecuted_ThenAllPatientResour if (countOfSearches == 1) { - ids = x.ArgAt>>(1)[2].Item2.Split(','); + ids = x.ArgAt>>(1) + .Where(x => x.Item1 == Core.Features.KnownQueryParameterNames.Id) + .Select(x => x.Item2).First().Split(','); continuationTokenIndex = 0; } else if (countOfSearches == 2) @@ -1261,7 +1305,10 @@ public async Task GivenAGroupExportJobToResume_WhenExecuted_ThenAllPatientResour { // The ids aren't in the query parameters because of the reset ids = new string[] { "1", "2", "3" }; - continuationTokenIndex = int.Parse(ContinuationTokenConverter.Decode(x.ArgAt>>(1)[2].Item2).Substring(2)); + continuationTokenIndex = int.Parse(ContinuationTokenConverter.Decode( + x.ArgAt>>(1) + .Where(x => x.Item1 == Core.Features.KnownQueryParameterNames.ContinuationToken) + .Select(x => x.Item2).First())[2..]); } return CreateSearchResult( @@ -1342,7 +1389,10 @@ public async Task GivenAGroupExportJobWithTheTypeParameter_WhenExecuted_ThenAllP true) .Returns(x => { - string[] ids = x.ArgAt>>(1)[2].Item2.Split(','); + string[] ids = x.ArgAt>>(1) + .Where(x => x.Item1 == KnownQueryParameterNames.Id) + .Select(x => x.Item2).First().Split(','); + SearchResultEntry[] entries = new SearchResultEntry[ids.Length]; for (int index = 0; index < ids.Length; index++) @@ -1363,7 +1413,9 @@ public async Task GivenAGroupExportJobWithTheTypeParameter_WhenExecuted_ThenAllP .Returns(x => { string parentId = x.ArgAt(1); - string[] resourceTypes = x.ArgAt>>(3)[2].Item2.Split(','); + string[] resourceTypes = x.ArgAt>>(3) + .Where(x => x.Item1 == KnownQueryParameterNames.Type) + .Select(x => x.Item2).First().Split(','); SearchResultEntry[] entries = new SearchResultEntry[resourceTypes.Length]; @@ -2076,7 +2128,9 @@ private ExportJobRecord CreateExportJobRecord( uint numberOfPagesPerCommit = 0, string containerName = null, string anonymizationConfigurationLocation = null, - string anonymizationConfigurationFileEtag = null) + string anonymizationConfigurationFileEtag = null, + bool includeHistory = false, + bool includeDeleted = false) { return new ExportJobRecord( new Uri(requestEndpoint), @@ -2094,7 +2148,9 @@ private ExportJobRecord CreateExportJobRecord( numberOfPagesPerCommit: numberOfPagesPerCommit == 0 ? _exportJobConfiguration.NumberOfPagesPerCommit : numberOfPagesPerCommit, storageAccountContainerName: containerName, anonymizationConfigurationLocation: anonymizationConfigurationLocation, - anonymizationConfigurationFileETag: anonymizationConfigurationFileEtag); + anonymizationConfigurationFileETag: anonymizationConfigurationFileEtag, + includeHistory: includeHistory, + includeDeleted: includeDeleted); } private ExportJobTask CreateExportJobTask( diff --git a/src/Microsoft.Health.Fhir.Core/Configs/ExportJobConfiguration.cs b/src/Microsoft.Health.Fhir.Core/Configs/ExportJobConfiguration.cs index aa3a55dcad..b1648525dc 100644 --- a/src/Microsoft.Health.Fhir.Core/Configs/ExportJobConfiguration.cs +++ b/src/Microsoft.Health.Fhir.Core/Configs/ExportJobConfiguration.cs @@ -42,6 +42,16 @@ public class ExportJobConfiguration /// public uint MaximumNumberOfResourcesPerQuery { get; set; } = 10000; + /// + /// For SQL export, controlls the number of parallel id ranges to gather to be used for parallel export. + /// + public int NumberOfParallelRecordRanges { get; set; } = 100; + + /// + /// For SQL export, controlls the DOP (degree of parallelization) used by the coordinator to build sub-jobs. + /// + public int CoordinatorMaxDegreeOfParallelization { get; set; } = 4; + /// /// Number of pages to be iterated before committing the export progress. /// diff --git a/src/Microsoft.Health.Fhir.Core/Extensions/ExportMediatorExtensions.cs b/src/Microsoft.Health.Fhir.Core/Extensions/ExportMediatorExtensions.cs index bc87b2812b..9e7a271334 100644 --- a/src/Microsoft.Health.Fhir.Core/Extensions/ExportMediatorExtensions.cs +++ b/src/Microsoft.Health.Fhir.Core/Extensions/ExportMediatorExtensions.cs @@ -28,6 +28,8 @@ public static async Task ExportAsync( string containerName, string formatName, bool isParallel, + bool includeDeleted, + bool includeHistory, uint maxCount, string anonymizationConfigurationCollectionReference, string anonymizationConfigLocation, @@ -37,7 +39,23 @@ public static async Task ExportAsync( EnsureArg.IsNotNull(mediator, nameof(mediator)); EnsureArg.IsNotNull(requestUri, nameof(requestUri)); - var request = new CreateExportRequest(requestUri, requestType, resourceType, since, till, filters, groupId, containerName, formatName, isParallel, maxCount, anonymizationConfigurationCollectionReference, anonymizationConfigLocation, anonymizationConfigFileETag); + var request = new CreateExportRequest( + requestUri: requestUri, + requestType: requestType, + resourceType: resourceType, + since: since, + till: till, + filters: filters, + groupId: groupId, + containerName: containerName, + formatName: formatName, + isParallel: isParallel, + maxCount: maxCount, + includeDeleted: includeDeleted, + includeHistory: includeHistory, + anonymizationConfigurationCollectionReference: anonymizationConfigurationCollectionReference, + anonymizationConfigurationLocation: anonymizationConfigLocation, + anonymizationConfigurationFileETag: anonymizationConfigFileETag); CreateExportResponse response = await mediator.Send(request, cancellationToken); return response; diff --git a/src/Microsoft.Health.Fhir.Core/Features/KnownQueryParameterNames.cs b/src/Microsoft.Health.Fhir.Core/Features/KnownQueryParameterNames.cs index 34eb0fbc70..a34719c317 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/KnownQueryParameterNames.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/KnownQueryParameterNames.cs @@ -99,6 +99,14 @@ public static class KnownQueryParameterNames public const string PurgeHistory = "_purgeHistory"; + /// + /// Used by $export as a comma-separated list of parameters instructing which initial data should be included. + /// + public const string IncludeAssociatedData = "includeAssociatedData"; + + /// + /// Used by export to specify the number of resources to be processed by the search engine. + /// public const string MaxCount = "_maxCount"; } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/CreateExportRequestHandler.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/CreateExportRequestHandler.cs index cc095e6d42..77f4c506c6 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/CreateExportRequestHandler.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/CreateExportRequestHandler.cs @@ -78,30 +78,28 @@ public async Task Handle(CreateExportRequest request, Canc uint maxCount = request.MaxCount > 0 ? request.MaxCount : _exportJobConfiguration.MaximumNumberOfResourcesPerQuery; var jobRecord = new ExportJobRecord( - request.RequestUri, - request.RequestType, - formatConfiguration.Format, - request.ResourceType, - filters, - "N/A", - _exportJobConfiguration.RollingFileSizeInMB, - requestorClaims, - request.Since, - request.Till, - null, - null, - null, - null, - request.GroupId, - storageAccountConnectionHash, - _exportJobConfiguration.StorageAccountUri, - request.AnonymizationConfigurationCollectionReference, - request.AnonymizationConfigurationLocation, - request.AnonymizationConfigurationFileETag, - maxCount, - _exportJobConfiguration.NumberOfPagesPerCommit, - request.ContainerName, - request.IsParallel, + requestUri: request.RequestUri, + exportType: request.RequestType, + exportFormat: formatConfiguration.Format, + resourceType: request.ResourceType, + filters: filters, + hash: "N/A", + rollingFileSizeInMB: _exportJobConfiguration.RollingFileSizeInMB, + requestorClaims: requestorClaims, + since: request.Since, + till: request.Till, + groupId: request.GroupId, + storageAccountConnectionHash: storageAccountConnectionHash, + storageAccountUri: _exportJobConfiguration.StorageAccountUri, + anonymizationConfigurationCollectionReference: request.AnonymizationConfigurationCollectionReference, + anonymizationConfigurationLocation: request.AnonymizationConfigurationLocation, + anonymizationConfigurationFileETag: request.AnonymizationConfigurationFileETag, + maximumNumberOfResourcesPerQuery: maxCount > 0 ? maxCount : _exportJobConfiguration.MaximumNumberOfResourcesPerQuery, + numberOfPagesPerCommit: _exportJobConfiguration.NumberOfPagesPerCommit, + storageAccountContainerName: request.ContainerName, + isParallel: request.IsParallel, + includeHistory: request.IncludeHistory, + includeDeleted: request.IncludeDeleted, smartRequest: _contextAccessor?.RequestContext?.AccessControlContext?.ApplyFineGrainedAccessControl == true); var outcome = await _fhirOperationDataStore.CreateExportJobAsync(jobRecord, cancellationToken); diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs index 95138ce048..773edaf223 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/ExportJobTask.cs @@ -188,9 +188,13 @@ public async Task ExecuteAsync(ExportJobRecord exportJobRecord, WeakETag weakETa queryParametersList.Add(Tuple.Create(KnownQueryParameterNames.LastUpdated, $"ge{_exportJobRecord.Since}")); } + var exportResourceVersionTypes = ResourceVersionType.Latest | + (_exportJobRecord.IncludeHistory ? ResourceVersionType.Histoy : 0) | + (_exportJobRecord.IncludeDeleted ? ResourceVersionType.SoftDeleted : 0); + ExportJobProgress progress = _exportJobRecord.Progress; - await RunExportSearch(exportJobConfiguration, progress, queryParametersList, cancellationToken); + await RunExportSearch(exportJobConfiguration, progress, queryParametersList, exportResourceVersionTypes, cancellationToken); await CompleteJobAsync(OperationStatus.Completed, cancellationToken); @@ -340,6 +344,7 @@ private async Task RunExportSearch( ExportJobConfiguration exportJobConfiguration, ExportJobProgress progress, List> sharedQueryParametersList, + ResourceVersionType resourceVersionTypes, CancellationToken cancellationToken) { EnsureArg.IsNotNull(exportJobConfiguration, nameof(exportJobConfiguration)); @@ -366,7 +371,7 @@ private async Task RunExportSearch( if (progress.CurrentFilter != null) { - await ProcessFilter(exportJobConfiguration, progress, queryParametersList, sharedQueryParametersList, anonymizer, cancellationToken); + await ProcessFilter(exportJobConfiguration, progress, queryParametersList, sharedQueryParametersList, resourceVersionTypes, anonymizer, cancellationToken); } if (_exportJobRecord.Filters != null && _exportJobRecord.Filters.Any(filter => !progress.CompletedFilters.Contains(filter))) @@ -379,7 +384,7 @@ private async Task RunExportSearch( (_exportJobRecord.ExportType == ExportJobType.All || filter.ResourceType.Equals(KnownResourceTypes.Patient, StringComparison.OrdinalIgnoreCase))) { progress.SetFilter(filter); - await ProcessFilter(exportJobConfiguration, progress, queryParametersList, sharedQueryParametersList, anonymizer, cancellationToken); + await ProcessFilter(exportJobConfiguration, progress, queryParametersList, sharedQueryParametersList, resourceVersionTypes, anonymizer, cancellationToken); } } } @@ -418,7 +423,7 @@ private async Task RunExportSearch( } } - await SearchWithFilter(exportJobConfiguration, progress, null, queryParametersList, sharedQueryParametersList, anonymizer, cancellationToken); + await SearchWithFilter(exportJobConfiguration, progress, null, queryParametersList, sharedQueryParametersList, resourceVersionTypes, anonymizer, cancellationToken); } } @@ -427,6 +432,7 @@ private async Task ProcessFilter( ExportJobProgress exportJobProgress, List> queryParametersList, List> sharedQueryParametersList, + ResourceVersionType resourceVersionTypes, IAnonymizer anonymizer, CancellationToken cancellationToken) { @@ -436,7 +442,7 @@ private async Task ProcessFilter( filterQueryParametersList.Add(param); } - await SearchWithFilter(exportJobConfiguration, exportJobProgress, exportJobProgress.CurrentFilter.ResourceType, filterQueryParametersList, sharedQueryParametersList, anonymizer, cancellationToken); + await SearchWithFilter(exportJobConfiguration, exportJobProgress, exportJobProgress.CurrentFilter.ResourceType, filterQueryParametersList, sharedQueryParametersList, resourceVersionTypes, anonymizer, cancellationToken); exportJobProgress.MarkFilterFinished(); await UpdateJobRecordAsync(cancellationToken); @@ -448,6 +454,7 @@ private async Task SearchWithFilter( string resourceType, List> queryParametersList, List> sharedQueryParametersList, + ResourceVersionType resourceVersionTypes, IAnonymizer anonymizer, CancellationToken cancellationToken) { @@ -469,7 +476,8 @@ private async Task SearchWithFilter( resourceType: resourceType, queryParametersList, cancellationToken, - true); + true, + resourceVersionTypes); } break; @@ -674,31 +682,36 @@ private void ProcessSearchResults(IEnumerable searchResults, foreach (SearchResultEntry result in searchResults) { ResourceWrapper resourceWrapper = result.Resource; - var data = result.Resource.RawResource.Data; + ResourceElement overrideDataElement = null; + var addSoftDeletedExtension = resourceWrapper.IsDeleted && _exportJobRecord.IncludeDeleted; if (anonymizer != null) { - ResourceElement element = _resourceDeserializer.Deserialize(resourceWrapper); + overrideDataElement = _resourceDeserializer.Deserialize(resourceWrapper); try { - element = anonymizer.Anonymize(element); + overrideDataElement = anonymizer.Anonymize(overrideDataElement); } catch (Exception ex) { throw new FailedToAnonymizeResourceException(ex.Message, ex); } - - // Serialize into NDJson and write to the file. - data = _resourceToByteArraySerializer.StringSerialize(element); } - else if (!resourceWrapper.RawResource.IsMetaSet) + else if (!resourceWrapper.RawResource.IsMetaSet || addSoftDeletedExtension) { // For older records in Cosmos the metadata isn't included in the raw resource - ResourceElement element = _resourceDeserializer.Deserialize(resourceWrapper); - data = _resourceToByteArraySerializer.StringSerialize(element); + overrideDataElement = _resourceDeserializer.Deserialize(resourceWrapper); + } + + var outputData = result.Resource.RawResource.Data; + + // If any modifications were made to the resource / are needed, serialize the element instead of using the raw data string. + if (overrideDataElement is not null) + { + outputData = _resourceToByteArraySerializer.StringSerialize(overrideDataElement, addSoftDeletedExtension); } - _fileManager.WriteToFile(resourceWrapper.ResourceTypeName, data); + _fileManager.WriteToFile(resourceWrapper.ResourceTypeName, outputData); } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/IResourceToByteArraySerializer.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/IResourceToByteArraySerializer.cs index 960633d03c..78ef0ad7f1 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/IResourceToByteArraySerializer.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/IResourceToByteArraySerializer.cs @@ -19,6 +19,6 @@ public interface IResourceToByteArraySerializer /// The serialized bytes. byte[] Serialize(ResourceElement resourceElement); - string StringSerialize(ResourceElement resourceElement); + string StringSerialize(ResourceElement resourceElement, bool addSoftDeletedExtension = false); } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/Models/ExportJobRecord.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/Models/ExportJobRecord.cs index bf4b6879e7..18ff1e12f4 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/Models/ExportJobRecord.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Export/Models/ExportJobRecord.cs @@ -43,6 +43,8 @@ public ExportJobRecord( uint numberOfPagesPerCommit = 10, string storageAccountContainerName = null, bool isParallel = true, + bool includeHistory = false, + bool includeDeleted = false, int schemaVersion = 2, int typeId = (int)JobType.ExportOrchestrator, bool smartRequest = false) @@ -69,6 +71,8 @@ public ExportJobRecord( RestartCount = 0; TypeId = typeId; IsParallel = isParallel; + IncludeHistory = includeHistory; + IncludeDeleted = includeDeleted; AnonymizationConfigurationCollectionReference = anonymizationConfigurationCollectionReference; AnonymizationConfigurationLocation = anonymizationConfigurationLocation; @@ -200,6 +204,12 @@ protected ExportJobRecord() [JsonProperty(JobRecordProperties.IsParallel)] public bool IsParallel { get; private set; } + [JsonProperty(JobRecordProperties.IncludeHistory)] + public bool IncludeHistory { get; private set; } + + [JsonProperty(JobRecordProperties.IncludeDeleted)] + public bool IncludeDeleted { get; private set; } + [JsonProperty(JobRecordProperties.SmartRequest)] public bool SmartRequest { get; private set; } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/JobRecordProperties.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/JobRecordProperties.cs index f378c69486..80b68036fd 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/JobRecordProperties.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/JobRecordProperties.cs @@ -157,6 +157,10 @@ public static class JobRecordProperties public const string IsParallel = "isParallel"; + public const string IncludeHistory = "includeHistory"; + + public const string IncludeDeleted = "includeDeleted"; + public const string SmartRequest = "smartRequest"; public const string DeleteOperation = "deleteOperation"; diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/ISearchOptionsFactory.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/ISearchOptionsFactory.cs index 232da354cf..c1ae219be3 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/ISearchOptionsFactory.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/ISearchOptionsFactory.cs @@ -10,8 +10,8 @@ namespace Microsoft.Health.Fhir.Core.Features.Search { public interface ISearchOptionsFactory { - SearchOptions Create(string resourceType, IReadOnlyList> queryParameters, bool isAsyncOperation = false); + SearchOptions Create(string resourceType, IReadOnlyList> queryParameters, bool isAsyncOperation = false, ResourceVersionType resourceVersionTypes = ResourceVersionType.Latest); - SearchOptions Create(string compartmentType, string compartmentId, string resourceType, IReadOnlyList> queryParameters, bool isAsyncOperation = false, bool useSmartCompartmentDefinition = false); + SearchOptions Create(string compartmentType, string compartmentId, string resourceType, IReadOnlyList> queryParameters, bool isAsyncOperation = false, bool useSmartCompartmentDefinition = false, ResourceVersionType resourceVersionTypes = ResourceVersionType.Latest); } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/ISearchService.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/ISearchService.cs index 7cf4ed73bb..2a13064e38 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/ISearchService.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/ISearchService.cs @@ -23,12 +23,14 @@ public interface ISearchService /// The search queries. /// The cancellation token. /// Whether the search is part of an async operation. + /// Which version types (latest, soft-deleted, history) to include in search. /// A representing the result. Task SearchAsync( string resourceType, IReadOnlyList> queryParameters, CancellationToken cancellationToken, - bool isAsyncOperation = false); + bool isAsyncOperation = false, + ResourceVersionType resourceVersionTypes = ResourceVersionType.Latest); /// /// Searches the resources using the . diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/ResourceVersionType.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/ResourceVersionType.cs new file mode 100644 index 0000000000..11e0fe4c4c --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/ResourceVersionType.cs @@ -0,0 +1,31 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; + +namespace Microsoft.Health.Fhir.Core.Features.Search +{ + /// + /// Defines the order of a sort. + /// + [Flags] + public enum ResourceVersionType + { + /// + /// Latest version of the resource. + /// + Latest = 1, + + /// + /// Previous versions of the resource - i.e. historical. + /// + Histoy = 2, + + /// + /// Resources that have been deleted but are still in the system (soft-delete). + /// + SoftDeleted = 4, + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SearchOptions.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SearchOptions.cs index 75bf95548b..0294422500 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/SearchOptions.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SearchOptions.cs @@ -48,6 +48,8 @@ internal SearchOptions(SearchOptions other) } QueryHints = other.QueryHints; + + ResourceVersionTypes = other.ResourceVersionTypes; } /// @@ -111,6 +113,11 @@ internal set } } + /// + /// Which version types (latest, soft-deleted, history) to include in search. + /// + public ResourceVersionType ResourceVersionTypes { get; internal set; } = ResourceVersionType.Latest; + /// /// Gets the search expression. /// diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/SearchService.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/SearchService.cs index 8889fa8cfc..5903179883 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/SearchService.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/SearchService.cs @@ -44,9 +44,10 @@ public async Task SearchAsync( string resourceType, IReadOnlyList> queryParameters, CancellationToken cancellationToken, - bool isAsyncOperation = false) + bool isAsyncOperation = false, + ResourceVersionType resourceVersionTypes = ResourceVersionType.Latest) { - SearchOptions searchOptions = _searchOptionsFactory.Create(resourceType, queryParameters, isAsyncOperation); + SearchOptions searchOptions = _searchOptionsFactory.Create(resourceType, queryParameters, isAsyncOperation, resourceVersionTypes); // Execute the actual search. return await SearchAsync(searchOptions, cancellationToken); @@ -174,9 +175,11 @@ public async Task SearchHistoryAsync( queryParameters.Add(Tuple.Create(KnownQueryParameterNames.Sort, $"-{KnownQueryParameterNames.LastUpdated}")); } - SearchOptions searchOptions = _searchOptionsFactory.Create(resourceType, queryParameters, isAsyncOperation); + var historyResourceVersionTypes = ResourceVersionType.Latest | ResourceVersionType.Histoy | ResourceVersionType.SoftDeleted; + + SearchOptions searchOptions = _searchOptionsFactory.Create(resourceType, queryParameters, isAsyncOperation, historyResourceVersionTypes); - SearchResult searchResult = await SearchHistoryInternalAsync(searchOptions, cancellationToken); + SearchResult searchResult = await SearchAsync(searchOptions, cancellationToken); // If no results are returned from the _history search // determine if the resource actually exists or if the results were just filtered out. @@ -232,10 +235,6 @@ public abstract Task SearchAsync( SearchOptions searchOptions, CancellationToken cancellationToken); - protected abstract Task SearchHistoryInternalAsync( - SearchOptions searchOptions, - CancellationToken cancellationToken); - protected abstract Task SearchForReindexInternalAsync( SearchOptions searchOptions, string searchParameterHash, diff --git a/src/Microsoft.Health.Fhir.Core/Messages/Export/CreateExportRequest.cs b/src/Microsoft.Health.Fhir.Core/Messages/Export/CreateExportRequest.cs index 3363c06078..4e76ff2ebe 100644 --- a/src/Microsoft.Health.Fhir.Core/Messages/Export/CreateExportRequest.cs +++ b/src/Microsoft.Health.Fhir.Core/Messages/Export/CreateExportRequest.cs @@ -24,6 +24,8 @@ public CreateExportRequest( string containerName = null, string formatName = null, bool isParallel = true, + bool includeHistory = false, + bool includeDeleted = false, uint maxCount = 0, string anonymizationConfigurationCollectionReference = null, string anonymizationConfigurationLocation = null, @@ -45,6 +47,8 @@ public CreateExportRequest( ContainerName = containerName; FormatName = formatName; IsParallel = isParallel; + IncludeHistory = includeHistory; + IncludeDeleted = includeDeleted; MaxCount = maxCount; } @@ -74,6 +78,10 @@ public CreateExportRequest( public bool IsParallel { get; } + public bool IncludeHistory { get; } + + public bool IncludeDeleted { get; } + public uint MaxCount { get; } } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Operations/Export/CosmosExportOrchestratorJob.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Operations/Export/CosmosExportOrchestratorJob.cs index b5a8202129..75dfc4b862 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Operations/Export/CosmosExportOrchestratorJob.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Operations/Export/CosmosExportOrchestratorJob.cs @@ -7,6 +7,7 @@ using System.Threading; using System.Threading.Tasks; using EnsureThat; +using Hl7.Fhir.Model; using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.Core.Features.Operations.Export; using Microsoft.Health.Fhir.Core.Features.Operations.Export.Models; @@ -85,12 +86,15 @@ private static ExportJobRecord CreateExportRecord(ExportJobRecord record, long g record.NumberOfPagesPerCommit, container, record.IsParallel, + record.IncludeHistory, + record.IncludeDeleted, record.SchemaVersion, (int)JobType.ExportProcessing, record.SmartRequest); rec.Id = string.Empty; rec.QueuedTime = record.QueuedTime; // preserve create date of coordinator job in form of queued time for all children, so same time is used on file names. + return rec; } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/FhirCosmosSearchService.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/FhirCosmosSearchService.cs index efdf2e3671..923e1920b2 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/FhirCosmosSearchService.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/FhirCosmosSearchService.cs @@ -343,21 +343,6 @@ private async Task RecurseChainedExpression(ChainedExpression expres Expression.In(FieldName.ReferenceResourceId, null, g.Select(x => x.ResourceId)))).ToList())); } - protected override async Task SearchHistoryInternalAsync( - SearchOptions searchOptions, - CancellationToken cancellationToken) - { - (IReadOnlyList results, string continuationToken, _) = await ExecuteSearchAsync( - _queryBuilder.GenerateHistorySql(searchOptions), - searchOptions, - searchOptions.CountOnly ? null : searchOptions.ContinuationToken, - null, - null, - cancellationToken); - - return CreateSearchResult(searchOptions, results.Select(r => new SearchResultEntry(r)), continuationToken); - } - protected override async Task SearchForReindexInternalAsync( SearchOptions searchOptions, string searchParameterHash, diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/IQueryBuilder.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/IQueryBuilder.cs index b4d36218e9..86ee339da3 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/IQueryBuilder.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/IQueryBuilder.cs @@ -12,8 +12,6 @@ internal interface IQueryBuilder { QueryDefinition BuildSqlQuerySpec(SearchOptions searchOptions, QueryBuilderOptions queryOptions = null); - QueryDefinition GenerateHistorySql(SearchOptions searchOptions); - QueryDefinition GenerateReindexSql(SearchOptions searchOptions, string searchParameterHash); } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/QueryBuilder.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/QueryBuilder.cs index d326334184..85142d11ba 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/QueryBuilder.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Search/Queries/QueryBuilder.cs @@ -26,11 +26,6 @@ public QueryDefinition BuildSqlQuerySpec(SearchOptions searchOptions, QueryBuild return new QueryBuilderHelper().BuildSqlQuerySpec(searchOptions, queryOptions ?? new QueryBuilderOptions()); } - public QueryDefinition GenerateHistorySql(SearchOptions searchOptions) - { - return new QueryBuilderHelper().GenerateHistorySql(searchOptions); - } - public QueryDefinition GenerateReindexSql(SearchOptions searchOptions, string searchParameterHash) { return new QueryBuilderHelper().GenerateReindexSql(searchOptions, searchParameterHash); @@ -83,11 +78,29 @@ public QueryDefinition BuildSqlQuerySpec(SearchOptions searchOptions, QueryBuild searchOptions.Expression.AcceptVisitor(expressionQueryBuilder); } - AppendFilterCondition( - "AND", - true, - (KnownResourceWrapperProperties.IsHistory, false), - (KnownResourceWrapperProperties.IsDeleted, false)); + if (!searchOptions.ResourceVersionTypes.HasFlag(ResourceVersionType.Latest)) + { + AppendFilterCondition( + "AND", + true, + (KnownResourceWrapperProperties.IsHistory, true)); + } + + if (!searchOptions.ResourceVersionTypes.HasFlag(ResourceVersionType.Histoy)) + { + AppendFilterCondition( + "AND", + true, + (KnownResourceWrapperProperties.IsHistory, false)); + } + + if (!searchOptions.ResourceVersionTypes.HasFlag(ResourceVersionType.SoftDeleted)) + { + AppendFilterCondition( + "AND", + true, + (KnownResourceWrapperProperties.IsDeleted, false)); + } if (!searchOptions.CountOnly) { @@ -122,39 +135,6 @@ public QueryDefinition BuildSqlQuerySpec(SearchOptions searchOptions, QueryBuild return query; } - public QueryDefinition GenerateHistorySql(SearchOptions searchOptions) - { - EnsureArg.IsNotNull(searchOptions, nameof(searchOptions)); - - AppendSelectFromRoot(); - - AppendSystemDataFilter(); - - var expressionQueryBuilder = new ExpressionQueryBuilder( - _queryBuilder, - _queryParameterManager); - - if (searchOptions.Expression != null) - { - _queryBuilder.Append("AND "); - searchOptions.Expression.AcceptVisitor(expressionQueryBuilder); - } - - _queryBuilder.Append("ORDER BY "); - var sortOption = searchOptions.Sort[0]; - -#pragma warning disable CA1834 // Consider using 'StringBuilder.Append(char)' when applicable - _queryBuilder.Append(SearchValueConstants.RootAliasName).Append('.') -#pragma warning restore CA1834 // Consider using 'StringBuilder.Append(char)' when applicable - .Append(KnownResourceWrapperProperties.LastModified).Append(' ') - .AppendLine(sortOption.sortOrder == SortOrder.Ascending ? "ASC" : "DESC"); - - var query = new QueryDefinition(_queryBuilder.ToString()); - _queryParameterManager.AddToQuery(query); - - return query; - } - public QueryDefinition GenerateReindexSql(SearchOptions searchOptions, string searchParameterHash) { EnsureArg.IsNotNull(searchOptions, nameof(searchOptions)); diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/ExportControllerTests.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/ExportControllerTests.cs index 6543a5816e..d2348c4ef2 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/ExportControllerTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/ExportControllerTests.cs @@ -215,6 +215,41 @@ await Assert.ThrowsAsync(() => exportController.Export typeParameter: ResourceType.Patient.ToString())); } + [Fact] + public async Task GivenAnExportRequestWithHistoryOrDeletedIncluded_WhenHasTypeFilter_ThenRequestNotValidExceptionShouldBeThrown() + { + await Assert.ThrowsAsync(() => _exportEnabledController.Export( + since: null, + till: null, + resourceType: ResourceType.Patient.ToString(), + containerName: null, + formatName: null, + typeFilter: "Patient%3Factive%3Dtrue", + includeAssociatedData: "_history")); + + await Assert.ThrowsAsync(() => _exportEnabledController.Export( + since: null, + till: null, + resourceType: ResourceType.Patient.ToString(), + containerName: null, + formatName: null, + typeFilter: "Patient%3Factive%3Dtrue", + includeAssociatedData: "_deleted")); + } + + [Fact] + public async Task GivenAnExportRequestWithDataIncluded_WhenKeyIsInvalid_ThenRequestNotValidExceptionShouldBeThrown() + { + await Assert.ThrowsAsync(() => _exportEnabledController.Export( + since: null, + till: null, + resourceType: null, + containerName: null, + formatName: null, + typeFilter: null, + includeAssociatedData: "_test")); + } + // We can configure OciArtifacts through three fields: LoginServer, ImageName and Digest // If ImageName and Digest are null, all images under the specified LoginSever are allowed to be used. // Similarly, if LoginSever and ImageName are specified and Digest is empty, all digests under the specified ImageName are allowed to be used. diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs b/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs index 0c4ddde3aa..319e4fe377 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ExportController.cs @@ -4,6 +4,7 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Collections.Generic; using System.Linq; using System.Net; using System.Threading.Tasks; @@ -98,6 +99,7 @@ public async Task Export( [FromQuery(Name = KnownQueryParameterNames.TypeFilter)] string typeFilter, [FromQuery(Name = KnownQueryParameterNames.Format)] string formatName, [FromQuery(Name = KnownQueryParameterNames.IsParallel)] bool isParallel = true, + [FromQuery(Name = KnownQueryParameterNames.IncludeAssociatedData)] string includeAssociatedData = null, [FromQuery(Name = KnownQueryParameterNames.MaxCount)] uint maxCount = 0, [FromQuery(Name = KnownQueryParameterNames.AnonymizationConfigurationCollectionReference)] string anonymizationConfigCollectionReference = null, [FromQuery(Name = KnownQueryParameterNames.AnonymizationConfigurationLocation)] string anonymizationConfigLocation = null, @@ -105,6 +107,7 @@ public async Task Export( { CheckIfExportIsEnabled(); ValidateForAnonymizedExport(containerName, anonymizationConfigCollectionReference, anonymizationConfigLocation, anonymizationConfigFileETag); + (bool includeHistory, bool includeDeleted) = ValidateAndParseIncludeAssociatedData(includeAssociatedData, typeFilter); return await SendExportRequest( exportType: ExportJobType.All, @@ -115,6 +118,8 @@ public async Task Export( containerName: containerName, formatName: formatName, isParallel: isParallel, + includeHistory: includeHistory, + includeDeleted: includeDeleted, maxCount: maxCount, anonymizationConfigCollectionReference: anonymizationConfigCollectionReference, anonymizationConfigLocation: anonymizationConfigLocation, @@ -251,6 +256,8 @@ private async Task SendExportRequest( string containerName = null, string formatName = null, bool isParallel = true, + bool includeHistory = false, + bool includeDeleted = false, uint maxCount = 0, string anonymizationConfigCollectionReference = null, string anonymizationConfigLocation = null, @@ -267,6 +274,8 @@ private async Task SendExportRequest( containerName, formatName, isParallel, + includeDeleted, + includeHistory, maxCount, anonymizationConfigCollectionReference, anonymizationConfigLocation, @@ -301,7 +310,7 @@ private void ValidateForAnonymizedExport(string containerName, string anonymizat { CheckReferenceAndETagParameterConflictForAnonymizedExport(anonymizationConfigCollectionReference, anonymizationConfigFileETag); CheckConfigCollectionReferenceIsValid(anonymizationConfigCollectionReference); - CheckIfConfigCollectionReferencIsConfigured(anonymizationConfigCollectionReference); + CheckIfConfigCollectionReferenceIsConfigured(anonymizationConfigCollectionReference); } } } @@ -335,7 +344,7 @@ private static void CheckReferenceAndETagParameterConflictForAnonymizedExport(st } } - private void CheckIfConfigCollectionReferencIsConfigured(string anonymizationConfigCollectionReference) + private void CheckIfConfigCollectionReferenceIsConfigured(string anonymizationConfigCollectionReference) { var ociImage = ImageInfo.CreateFromImageReference(anonymizationConfigCollectionReference); @@ -353,5 +362,27 @@ private void CheckIfConfigCollectionReferencIsConfigured(string anonymizationCon throw new RequestNotValidException(string.Format(Resources.AnonymizationConfigCollectionNotConfigured, anonymizationConfigCollectionReference)); } } + + private static (bool includeHistory, bool includeDeleted) ValidateAndParseIncludeAssociatedData(string associatedData, string typeFilter) + { + var associatedDataParams = (associatedData ?? string.Empty).Split(',', StringSplitOptions.RemoveEmptyEntries).ToList(); + var possibleParams = new List { "_history", "_deleted" }; + var invalidParams = associatedDataParams.Where(param => !possibleParams.Contains(param)).ToList(); + + if (invalidParams.Any()) + { + throw new RequestNotValidException(string.Format(Resources.InvalidExportAssociatedDataParameter, string.Join(',', invalidParams))); + } + + bool includeHistory = associatedDataParams.Contains("_history"); + bool includeDeleted = associatedDataParams.Contains("_deleted"); + + if ((includeHistory || includeDeleted) && !string.IsNullOrWhiteSpace(typeFilter)) + { + throw new RequestNotValidException(string.Format(Resources.TypeFilterNotSupportedWithHistoryOrDeletedExport, "_history and _deleted")); + } + + return (includeHistory, includeDeleted); + } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs index 41dfd22282..b9de7455a7 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs @@ -339,8 +339,8 @@ public void GivenAValidCompartmentSearch_WhenCreated_ThenCorrectCompartmentSearc SearchOptions options = CreateSearchOptions( resourceType: resourceType.ToString(), queryParameters: null, - compartmentType.ToString(), - compartmentId); + compartmentType: compartmentType.ToString(), + compartmentId: compartmentId); Assert.NotNull(options); ValidateMultiaryExpression( @@ -363,8 +363,8 @@ public void GivenAValidCompartmentSearchWithNullResourceType_WhenCreated_ThenCor SearchOptions options = CreateSearchOptions( resourceType: null, queryParameters: null, - compartmentType.ToString(), - compartmentId); + compartmentType: compartmentType.ToString(), + compartmentId: compartmentId); Assert.NotNull(options); ValidateCompartmentSearchExpression(options.Expression, compartmentType.ToString(), compartmentId); @@ -382,8 +382,8 @@ public void GivenInvalidCompartmentType_WhenCreated_ThenExceptionShouldBeThrown( InvalidSearchOperationException exception = Assert.Throws(() => CreateSearchOptions( resourceType: null, queryParameters: null, - invalidCompartmentType, - "123")); + compartmentType: invalidCompartmentType, + compartmentId: "123")); Assert.Equal(exception.Message, $"Compartment type {invalidCompartmentType} is invalid."); } @@ -398,8 +398,8 @@ public void GivenInvalidCompartmentId_WhenCreated_ThenExceptionShouldBeThrown(st InvalidSearchOperationException exception = Assert.Throws(() => CreateSearchOptions( resourceType: ResourceType.Claim.ToString(), queryParameters: null, - CompartmentType.Patient.ToString(), - invalidCompartmentId)); + compartmentType: CompartmentType.Patient.ToString(), + compartmentId: invalidCompartmentId)); Assert.Equal("Compartment id is null or empty.", exception.Message); } @@ -487,13 +487,30 @@ public void GivenSearchParameterText_WhenCreated_ThenSearchParameterShouldBeAdde Assert.Single(options.UnsupportedSearchParams); } + [Theory] + [InlineData(ResourceVersionType.Latest)] + [InlineData(ResourceVersionType.Histoy)] + [InlineData(ResourceVersionType.SoftDeleted)] + [InlineData(ResourceVersionType.Latest | ResourceVersionType.Histoy)] + [InlineData(ResourceVersionType.Latest | ResourceVersionType.SoftDeleted)] + [InlineData(ResourceVersionType.Histoy | ResourceVersionType.SoftDeleted)] + [InlineData(ResourceVersionType.Latest | ResourceVersionType.Histoy | ResourceVersionType.SoftDeleted)] + public void GivenIncludeHistoryAndDeletedParameters_WhenCreated_ThenSearchParametersShouldMatchInput(ResourceVersionType resourceVersionTypes) + { + SearchOptions options = CreateSearchOptions(ResourceType.Patient.ToString(), new List>(), resourceVersionTypes); + Assert.NotNull(options); + Assert.Equal(resourceVersionTypes, options.ResourceVersionTypes); + Assert.Empty(options.UnsupportedSearchParams); + } + private SearchOptions CreateSearchOptions( string resourceType = DefaultResourceType, IReadOnlyList> queryParameters = null, + ResourceVersionType resourceVersionTypes = ResourceVersionType.Latest, string compartmentType = null, string compartmentId = null) { - return _factory.Create(compartmentType, compartmentId, resourceType, queryParameters); + return _factory.Create(compartmentType, compartmentId, resourceType, queryParameters, resourceVersionTypes: resourceVersionTypes); } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchServiceTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchServiceTests.cs index 4f96890ecd..99eb65be96 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchServiceTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchServiceTests.cs @@ -174,13 +174,6 @@ public override Task SearchAsync( return Task.FromResult(SearchImplementation(searchOptions)); } - protected override Task SearchHistoryInternalAsync( - SearchOptions searchOptions, - CancellationToken cancellationToken) - { - return Task.FromResult(SearchImplementation(searchOptions)); - } - protected override Task SearchForReindexInternalAsync(SearchOptions searchOptions, string searchParameterHash, CancellationToken cancellationToken) { return Task.FromResult(SearchImplementation(searchOptions)); diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Export/ResourceToNdjsonBytesSerializer.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Export/ResourceToNdjsonBytesSerializer.cs index 60b96ae1e5..68c9b0549c 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Export/ResourceToNdjsonBytesSerializer.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Operations/Export/ResourceToNdjsonBytesSerializer.cs @@ -6,6 +6,7 @@ using System.Text; using EnsureThat; using Hl7.Fhir.Serialization; +using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Models; @@ -28,10 +29,15 @@ public byte[] Serialize(ResourceElement resourceElement) return bytesToWrite; } - public string StringSerialize(ResourceElement resourceElement) + public string StringSerialize(ResourceElement resourceElement, bool addSoftDeletedExtension = false) { EnsureArg.IsNotNull(resourceElement, nameof(resourceElement)); + if (addSoftDeletedExtension) + { + resourceElement.TryAddSoftDeletedExtension(); + } + return resourceElement.Instance.ToJson(); } } diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SearchOptionsFactory.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SearchOptionsFactory.cs index ddd4e186d7..7258c5faaf 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SearchOptionsFactory.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/SearchOptionsFactory.cs @@ -70,9 +70,9 @@ public SearchOptionsFactory( _resourceTypeSearchParameter = _searchParameterDefinitionManager.GetSearchParameter(ResourceType.Resource.ToString(), SearchParameterNames.ResourceType); } - public SearchOptions Create(string resourceType, IReadOnlyList> queryParameters, bool isAsyncOperation = false) + public SearchOptions Create(string resourceType, IReadOnlyList> queryParameters, bool isAsyncOperation = false, ResourceVersionType resourceVersionTypes = ResourceVersionType.Latest) { - return Create(null, null, resourceType, queryParameters, isAsyncOperation); + return Create(null, null, resourceType, queryParameters, isAsyncOperation, resourceVersionTypes: resourceVersionTypes); } [SuppressMessage("Design", "CA1308", Justification = "ToLower() is required to format parameter output correctly.")] @@ -82,7 +82,8 @@ public SearchOptions Create( string resourceType, IReadOnlyList> queryParameters, bool isAsyncOperation = false, - bool useSmartCompartmentDefinition = false) + bool useSmartCompartmentDefinition = false, + ResourceVersionType resourceVersionTypes = ResourceVersionType.Latest) { var searchOptions = new SearchOptions(); @@ -421,6 +422,8 @@ public SearchOptions Create( searchOptions.Sort = Array.Empty<(SearchParameterInfo searchParameterInfo, SortOrder sortOrder)>(); } + searchOptions.ResourceVersionTypes = resourceVersionTypes; + // Processing of parameters is finished. If any of the parameters are unsupported warning is put into the bundle or exception is thrown, // depending on the state of the "Prefer" header. if (unsupportedSearchParameters.Any() || searchSortErrors.Any()) diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Operations/Export/ExportOrchestratorJobTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Operations/Export/ExportOrchestratorJobTests.cs index 46edd2c78f..47ca591812 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Operations/Export/ExportOrchestratorJobTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Operations/Export/ExportOrchestratorJobTests.cs @@ -8,7 +8,9 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Options; using Microsoft.Health.Core.Extensions; +using Microsoft.Health.Fhir.Core.Configs; using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.Core.Features.Operations.Export; using Microsoft.Health.Fhir.Core.Features.Operations.Export.Models; @@ -30,6 +32,7 @@ public class ExportOrchestratorJobTests { private ISearchService _mockSearchService = Substitute.For(); private IQueueClient _mockQueueClient = Substitute.For(); + private IOptions _exportJobConfiguration = Options.Create(new ExportJobConfiguration()); [Theory] [InlineData(ExportJobType.Patient)] @@ -42,7 +45,7 @@ public async Task GivenANonSystemLevelExportJob_WhenRun_ThenOneProcessingJobShou SetupMockQueue(numExpectedJobs, orchestratorJobId); var orchestratorJob = GetJobInfoArray(0, orchestratorJobId, false, orchestratorJobId, isParallel: true, exportJobType: exportJobType)[0]; - var exportOrchestratorJob = new SqlExportOrchestratorJob(_mockQueueClient, _mockSearchService); + var exportOrchestratorJob = new SqlExportOrchestratorJob(_mockQueueClient, _mockSearchService, _exportJobConfiguration); var result = await exportOrchestratorJob.ExecuteAsync(orchestratorJob, new Progress((result) => { }), CancellationToken.None); var jobResult = JsonConvert.DeserializeObject(result); @@ -58,7 +61,7 @@ public async Task GivenAnExportJobWithIsParallelSetToFalse_WhenRun_ThenOneProces SetupMockQueue(numExpectedJobs, orchestratorJobId); var orchestratorJob = GetJobInfoArray(0, orchestratorJobId, false, orchestratorJobId, isParallel: false).First(); - var exportOrchestratorJob = new SqlExportOrchestratorJob(_mockQueueClient, _mockSearchService); + var exportOrchestratorJob = new SqlExportOrchestratorJob(_mockQueueClient, _mockSearchService, _exportJobConfiguration); var result = await exportOrchestratorJob.ExecuteAsync(orchestratorJob, new Progress((result) => { }), CancellationToken.None); var jobResult = JsonConvert.DeserializeObject(result); Assert.Equal(OperationStatus.Completed, jobResult.Status); @@ -75,7 +78,7 @@ public async Task GivenAnExportJobWithNoTypeRestriction_WhenRun_ThenMultipleProc SetupMockQueue(numExpectedJobsPerResourceType, orchestratorJobId); var orchestratorJob = GetJobInfoArray(0, orchestratorJobId, false, orchestratorJobId, isParallel: true).First(); - var exportOrchestratorJob = new SqlExportOrchestratorJob(_mockQueueClient, _mockSearchService); + var exportOrchestratorJob = new SqlExportOrchestratorJob(_mockQueueClient, _mockSearchService, _exportJobConfiguration); var result = await exportOrchestratorJob.ExecuteAsync(orchestratorJob, new Progress((result) => { }), CancellationToken.None); var jobResult = JsonConvert.DeserializeObject(result); Assert.Equal(OperationStatus.Completed, jobResult.Status); @@ -92,7 +95,7 @@ public async Task GivenAnExportJobWithTypeRestrictions_WhenRun_ThenProcessingJob SetupMockQueue(numExpectedJobs, orchestratorJobId); JobInfo orchestratorJob = GetJobInfoArray(0, orchestratorJobId, false, orchestratorJobId, isParallel: true, typeFilter: "Patient,Observation").First(); - var exportOrchestratorJob = new SqlExportOrchestratorJob(_mockQueueClient, _mockSearchService); + var exportOrchestratorJob = new SqlExportOrchestratorJob(_mockQueueClient, _mockSearchService, _exportJobConfiguration); string result = await exportOrchestratorJob.ExecuteAsync(orchestratorJob, new Progress(_ => { }), CancellationToken.None); ExportJobRecord jobResult = JsonConvert.DeserializeObject(result); Assert.Equal(OperationStatus.Completed, jobResult.Status); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/Export/SqlExportOrchestratorJob.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/Export/SqlExportOrchestratorJob.cs index 09cd20fd2c..6492d454bc 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/Export/SqlExportOrchestratorJob.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/Export/SqlExportOrchestratorJob.cs @@ -9,6 +9,8 @@ using System.Threading; using System.Threading.Tasks; using EnsureThat; +using Microsoft.Extensions.Options; +using Microsoft.Health.Fhir.Core.Configs; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.Core.Features.Operations.Export; @@ -23,24 +25,24 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Operations.Export [JobTypeId((int)JobType.ExportOrchestrator)] public class SqlExportOrchestratorJob : IJob { - private const int DefaultNumberOfSurrogateIdRanges = 100; - private IQueueClient _queueClient; private ISearchService _searchService; + private readonly ExportJobConfiguration _exportJobConfiguration; public SqlExportOrchestratorJob( IQueueClient queueClient, - ISearchService searchService) + ISearchService searchService, + IOptions exportJobConfiguration) { EnsureArg.IsNotNull(queueClient, nameof(queueClient)); EnsureArg.IsNotNull(searchService, nameof(searchService)); + EnsureArg.IsNotNull(exportJobConfiguration, nameof(exportJobConfiguration)); _queueClient = queueClient; _searchService = searchService; + _exportJobConfiguration = exportJobConfiguration.Value; } - internal int NumberOfSurrogateIdRanges { get; set; } = DefaultNumberOfSurrogateIdRanges; - public async Task ExecuteAsync(JobInfo jobInfo, IProgress progress, CancellationToken cancellationToken) { EnsureArg.IsNotNull(jobInfo, nameof(jobInfo)); @@ -73,7 +75,7 @@ public async Task ExecuteAsync(JobInfo jobInfo, IProgress progre .GroupBy(x => x.ResourceType) .ToDictionary(x => x.Key, x => x.Max(r => long.Parse(r.EndSurrogateId))); - await Parallel.ForEachAsync(resourceTypes, new ParallelOptions { MaxDegreeOfParallelism = 4, CancellationToken = cancellationToken }, async (type, cancel) => + await Parallel.ForEachAsync(resourceTypes, new ParallelOptions { MaxDegreeOfParallelism = _exportJobConfiguration.CoordinatorMaxDegreeOfParallelization, CancellationToken = cancellationToken }, async (type, cancel) => { var startId = globalStartId; if (enqueued.TryGetValue(type, out var max)) @@ -85,7 +87,7 @@ public async Task ExecuteAsync(JobInfo jobInfo, IProgress progre while (rows > 0) { var definitions = new List(); - var ranges = await _searchService.GetSurrogateIdRanges(type, startId, globalEndId, surrogateIdRangeSize, NumberOfSurrogateIdRanges, true, cancel); + var ranges = await _searchService.GetSurrogateIdRanges(type, startId, globalEndId, surrogateIdRangeSize, _exportJobConfiguration.NumberOfParallelRecordRanges, true, cancel); foreach (var range in ranges) { if (range.EndId > startId) @@ -140,33 +142,35 @@ private static ExportJobRecord CreateExportRecord(ExportJobRecord record, long g } var rec = new ExportJobRecord( - record.RequestUri, - record.ExportType, - format, - string.IsNullOrEmpty(resourceType) ? record.ResourceType : resourceType, - record.Filters, - record.Hash, - record.RollingFileSizeInMB, - record.RequestorClaims, - since == null ? record.Since : since, - till == null ? record.Till : till, - startSurrogateId, - endSurrogateId, - globalStartSurrogateId, - globalEndSurrogateId, - record.GroupId, - record.StorageAccountConnectionHash, - record.StorageAccountUri, - record.AnonymizationConfigurationCollectionReference, - record.AnonymizationConfigurationLocation, - record.AnonymizationConfigurationFileETag, - record.MaximumNumberOfResourcesPerQuery, - record.NumberOfPagesPerCommit, - container, - record.IsParallel, - record.SchemaVersion, - (int)JobType.ExportProcessing, - record.SmartRequest); + requestUri: record.RequestUri, + exportType: record.ExportType, + exportFormat: format, + resourceType: string.IsNullOrEmpty(resourceType) ? record.ResourceType : resourceType, + filters: record.Filters, + hash: record.Hash, + rollingFileSizeInMB: record.RollingFileSizeInMB, + requestorClaims: record.RequestorClaims, + since: since == null ? record.Since : since, + till: till == null ? record.Till : till, + startSurrogateId: startSurrogateId, + endSurrogateId: endSurrogateId, + globalStartSurrogateId: globalStartSurrogateId, + globalEndSurrogateId: globalEndSurrogateId, + groupId: record.GroupId, + storageAccountConnectionHash: record.StorageAccountConnectionHash, + storageAccountUri: record.StorageAccountUri, + anonymizationConfigurationCollectionReference: record.AnonymizationConfigurationCollectionReference, + anonymizationConfigurationLocation: record.AnonymizationConfigurationLocation, + anonymizationConfigurationFileETag: record.AnonymizationConfigurationFileETag, + maximumNumberOfResourcesPerQuery: record.MaximumNumberOfResourcesPerQuery, + numberOfPagesPerCommit: record.NumberOfPagesPerCommit, + storageAccountContainerName: container, + isParallel: record.IsParallel, + includeHistory: record.IncludeHistory, + includeDeleted: record.IncludeDeleted, + schemaVersion: record.SchemaVersion, + typeId: (int)JobType.ExportProcessing, + smartRequest: record.SmartRequest); rec.Id = string.Empty; rec.QueuedTime = record.QueuedTime; // preserve create date of coordinator job in form of queued time for all children, so same time is used on file names. return rec; diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/68.diff.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/68.diff.sql new file mode 100644 index 0000000000..99c259e06a --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/68.diff.sql @@ -0,0 +1,81 @@ +CREATE OR ALTER PROCEDURE dbo.GetResourcesByTypeAndSurrogateIdRange @ResourceTypeId smallint, @StartId bigint, @EndId bigint, @GlobalEndId bigint = NULL, @IncludeHistory bit = 0, @IncludeDeleted bit = 0 +AS +set nocount on +DECLARE @SP varchar(100) = 'GetResourcesByTypeAndSurrogateIdRange' + ,@Mode varchar(100) = 'RT='+isnull(convert(varchar,@ResourceTypeId),'NULL') + +' S='+isnull(convert(varchar,@StartId),'NULL') + +' E='+isnull(convert(varchar,@EndId),'NULL') + +' GE='+isnull(convert(varchar,@GlobalEndId),'NULL') + +' HI='+isnull(convert(varchar,@IncludeHistory),'NULL') + +' DE'+isnull(convert(varchar,@IncludeDeleted),'NULL') + ,@st datetime = getUTCdate() + ,@DummyTop bigint = 9223372036854775807 + +BEGIN TRY + DECLARE @ResourceIds TABLE (ResourceId varchar(64) COLLATE Latin1_General_100_CS_AS PRIMARY KEY) + DECLARE @SurrogateIds TABLE (MaxSurrogateId bigint PRIMARY KEY) + + IF @GlobalEndId IS NOT NULL AND @IncludeHistory = 0 -- snapshot view + BEGIN + INSERT INTO @ResourceIds + SELECT DISTINCT ResourceId + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + AND IsHistory = 1 + AND (IsDeleted = 0 OR @IncludeDeleted = 1) + OPTION (MAXDOP 1) + + IF @@rowcount > 0 + INSERT INTO @SurrogateIds + SELECT ResourceSurrogateId + FROM (SELECT ResourceId, ResourceSurrogateId, RowId = row_number() OVER (PARTITION BY ResourceId ORDER BY ResourceSurrogateId DESC) + FROM dbo.Resource WITH (INDEX = IX_Resource_ResourceTypeId_ResourceId_Version) -- w/o hint access to Resource table is inefficient when many versions are present. Hint is ignored if Resource is a view. + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceId IN (SELECT TOP (@DummyTop) ResourceId FROM @ResourceIds) + AND ResourceSurrogateId BETWEEN @StartId AND @GlobalEndId + ) A + WHERE RowId = 1 + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)) + END + + SELECT ResourceTypeId, ResourceId, Version, IsDeleted, ResourceSurrogateId, RequestMethod, IsMatch = convert(bit,1), IsPartial = convert(bit,0), IsRawResourceMetaSet, SearchParamHash, RawResource + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + AND (IsHistory = 0 OR @IncludeHistory = 1) + AND (IsDeleted = 0 OR @IncludeDeleted = 1) + UNION ALL + SELECT ResourceTypeId, ResourceId, Version, IsDeleted, ResourceSurrogateId, RequestMethod, IsMatch = convert(bit,1), IsPartial = convert(bit,0), IsRawResourceMetaSet, SearchParamHash, RawResource + FROM @SurrogateIds + JOIN dbo.Resource ON ResourceTypeId = @ResourceTypeId AND ResourceSurrogateId = MaxSurrogateId + WHERE IsHistory = 1 + AND (IsDeleted = 0 OR @IncludeDeleted = 1) + OPTION (MAXDOP 1) + + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@st,@Rows=@@rowcount +END TRY +BEGIN CATCH + IF error_number() = 1750 THROW -- Real error is before 1750, cannot trap in SQL. + EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='Error'; + THROW +END CATCH +GO +--set nocount on +--DECLARE @Ranges TABLE (UnitId int PRIMARY KEY, MinId bigint, MaxId bigint, Cnt int) +--INSERT INTO @Ranges +-- EXECUTE dbo.GetResourceSurrogateIdRanges 96, 0, 9e18, 90000, 10 +--SELECT count(*) FROM @Ranges +--DECLARE @UnitId int +-- ,@MinId bigint +-- ,@MaxId bigint +--DECLARE @Resources TABLE (RawResource varbinary(max)) +--WHILE EXISTS (SELECT * FROM @Ranges) +--BEGIN +-- SELECT TOP 1 @UnitId = UnitId, @MinId = MinId, @MaxId = MaxId FROM @Ranges ORDER BY UnitId +-- INSERT INTO @Resources +-- EXECUTE dbo.GetResourcesByTypeAndSurrogateIdRange 96, @MinId, @MaxId, NULL, @MaxId -- last is to invoke snapshot logic +-- DELETE FROM @Resources +-- DELETE FROM @Ranges WHERE UnitId = @UnitId +--END diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/68.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/68.sql new file mode 100644 index 0000000000..c17b303de7 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Migrations/68.sql @@ -0,0 +1,6736 @@ + +/************************************************************************************************* + Auto-Generated from Sql build task. Do not manually edit it. +**************************************************************************************************/ +SET XACT_ABORT ON +BEGIN TRAN +IF EXISTS (SELECT * + FROM sys.tables + WHERE name = 'ClaimType') + BEGIN + ROLLBACK; + RETURN; + END + + +GO +INSERT INTO dbo.SchemaVersion +VALUES (68, 'started'); + +CREATE PARTITION FUNCTION PartitionFunction_ResourceTypeId(SMALLINT) + AS RANGE RIGHT + FOR VALUES (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150); + +CREATE PARTITION SCHEME PartitionScheme_ResourceTypeId + AS PARTITION PartitionFunction_ResourceTypeId + ALL TO ([PRIMARY]); + + +GO +CREATE PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp(DATETIME2 (7)) + AS RANGE RIGHT + FOR VALUES (N'1970-01-01T00:00:00.0000000'); + +CREATE PARTITION SCHEME PartitionScheme_ResourceChangeData_Timestamp + AS PARTITION PartitionFunction_ResourceChangeData_Timestamp + ALL TO ([PRIMARY]); + +DECLARE @numberOfHistoryPartitions AS INT = 48; + +DECLARE @numberOfFuturePartitions AS INT = 720; + +DECLARE @rightPartitionBoundary AS DATETIME2 (7); + +DECLARE @currentDateTime AS DATETIME2 (7) = sysutcdatetime(); + +WHILE @numberOfHistoryPartitions >= -@numberOfFuturePartitions + BEGIN + SET @rightPartitionBoundary = DATEADD(hour, DATEDIFF(hour, 0, @currentDateTime) - @numberOfHistoryPartitions, 0); + ALTER PARTITION SCHEME PartitionScheme_ResourceChangeData_Timestamp NEXT USED [Primary]; + ALTER PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp( ) + SPLIT RANGE (@rightPartitionBoundary); + SET @numberOfHistoryPartitions -= 1; + END + +CREATE SEQUENCE dbo.ResourceSurrogateIdUniquifierSequence + AS INT + START WITH 0 + INCREMENT BY 1 + MINVALUE 0 + MAXVALUE 79999 + CYCLE + CACHE 1000000; + +CREATE TYPE dbo.BigintList AS TABLE ( + Id BIGINT NOT NULL PRIMARY KEY); + +CREATE TYPE dbo.CompartmentAssignmentList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + CompartmentTypeId TINYINT NOT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL PRIMARY KEY (ResourceTypeId, ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId)); + +CREATE TYPE dbo.DateTimeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + StartDateTime DATETIMEOFFSET (7) NOT NULL, + EndDateTime DATETIMEOFFSET (7) NOT NULL, + IsLongerThanADay BIT NOT NULL, + IsMin BIT NOT NULL, + IsMax BIT NOT NULL UNIQUE (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax)); + +CREATE TYPE dbo.NumberSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SingleValue DECIMAL (36, 18) NULL, + LowValue DECIMAL (36, 18) NULL, + HighValue DECIMAL (36, 18) NULL UNIQUE (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue)); + +CREATE TYPE dbo.QuantitySearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + QuantityCodeId INT NULL, + SingleValue DECIMAL (36, 18) NULL, + LowValue DECIMAL (36, 18) NULL, + HighValue DECIMAL (36, 18) NULL UNIQUE (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue)); + +CREATE TYPE dbo.ReferenceSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId SMALLINT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion INT NULL UNIQUE (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId)); + +CREATE TYPE dbo.ReferenceTokenCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId1 SMALLINT NULL, + ReferenceResourceId1 VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion1 INT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.ResourceDateKeyList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ResourceSurrogateId BIGINT NOT NULL PRIMARY KEY (ResourceTypeId, ResourceId, ResourceSurrogateId)); + +CREATE TYPE dbo.ResourceKeyList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Version INT NULL UNIQUE (ResourceTypeId, ResourceId, Version)); + +CREATE TYPE dbo.ResourceList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Version INT NOT NULL, + HasVersionToCompare BIT NOT NULL, + IsDeleted BIT NOT NULL, + IsHistory BIT NOT NULL, + KeepHistory BIT NOT NULL, + RawResource VARBINARY (MAX) NOT NULL, + IsRawResourceMetaSet BIT NOT NULL, + RequestMethod VARCHAR (10) NULL, + SearchParamHash VARCHAR (64) NULL PRIMARY KEY (ResourceTypeId, ResourceSurrogateId), + UNIQUE (ResourceTypeId, ResourceId, Version)); + +CREATE TYPE dbo.ResourceWriteClaimList AS TABLE ( + ResourceSurrogateId BIGINT NOT NULL, + ClaimTypeId TINYINT NOT NULL, + ClaimValue NVARCHAR (128) NOT NULL); + +CREATE TYPE dbo.StringList AS TABLE ( + String VARCHAR (MAX)); + +CREATE TYPE dbo.StringSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL, + IsMin BIT NOT NULL, + IsMax BIT NOT NULL); + +CREATE TYPE dbo.TokenDateTimeCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + StartDateTime2 DATETIMEOFFSET (7) NOT NULL, + EndDateTime2 DATETIMEOFFSET (7) NOT NULL, + IsLongerThanADay2 BIT NOT NULL); + +CREATE TYPE dbo.TokenNumberNumberCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SingleValue2 DECIMAL (36, 18) NULL, + LowValue2 DECIMAL (36, 18) NULL, + HighValue2 DECIMAL (36, 18) NULL, + SingleValue3 DECIMAL (36, 18) NULL, + LowValue3 DECIMAL (36, 18) NULL, + HighValue3 DECIMAL (36, 18) NULL, + HasRange BIT NOT NULL); + +CREATE TYPE dbo.TokenQuantityCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SystemId2 INT NULL, + QuantityCodeId2 INT NULL, + SingleValue2 DECIMAL (36, 18) NULL, + LowValue2 DECIMAL (36, 18) NULL, + HighValue2 DECIMAL (36, 18) NULL); + +CREATE TYPE dbo.TokenSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + Code VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.TokenStringCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + Text2 NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow2 NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL); + +CREATE TYPE dbo.TokenTextList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (400) COLLATE Latin1_General_CI_AI NOT NULL); + +CREATE TYPE dbo.TokenTokenCompositeSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.BulkResourceWriteClaimTableType_1 AS TABLE ( + Offset INT NOT NULL, + ClaimTypeId TINYINT NOT NULL, + ClaimValue NVARCHAR (128) NOT NULL); + +CREATE TYPE dbo.BulkCompartmentAssignmentTableType_1 AS TABLE ( + Offset INT NOT NULL, + CompartmentTypeId TINYINT NOT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL); + +CREATE TYPE dbo.BulkReferenceSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId SMALLINT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion INT NULL); + +CREATE TYPE dbo.BulkTokenSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + Code VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL); + +CREATE TYPE dbo.BulkTokenSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + Code VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.BulkTokenTextTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (400) COLLATE Latin1_General_CI_AI NOT NULL); + +CREATE TYPE dbo.BulkStringSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL); + +CREATE TYPE dbo.BulkStringSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL, + IsMin BIT NOT NULL, + IsMax BIT NOT NULL); + +CREATE TYPE dbo.BulkUriSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Uri VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL); + +CREATE TYPE dbo.BulkNumberSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SingleValue DECIMAL (18, 6) NULL, + LowValue DECIMAL (18, 6) NULL, + HighValue DECIMAL (18, 6) NULL); + +CREATE TYPE dbo.BulkQuantitySearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + QuantityCodeId INT NULL, + SingleValue DECIMAL (18, 6) NULL, + LowValue DECIMAL (18, 6) NULL, + HighValue DECIMAL (18, 6) NULL); + +CREATE TYPE dbo.BulkDateTimeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + StartDateTime DATETIMEOFFSET (7) NOT NULL, + EndDateTime DATETIMEOFFSET (7) NOT NULL, + IsLongerThanADay BIT NOT NULL); + +CREATE TYPE dbo.BulkDateTimeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + StartDateTime DATETIMEOFFSET (7) NOT NULL, + EndDateTime DATETIMEOFFSET (7) NOT NULL, + IsLongerThanADay BIT NOT NULL, + IsMin BIT NOT NULL, + IsMax BIT NOT NULL); + +CREATE TYPE dbo.BulkReferenceTokenCompositeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId1 SMALLINT NULL, + ReferenceResourceId1 VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion1 INT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL); + +CREATE TYPE dbo.BulkReferenceTokenCompositeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId1 SMALLINT NULL, + ReferenceResourceId1 VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion1 INT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.BulkTokenTokenCompositeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL); + +CREATE TYPE dbo.BulkTokenTokenCompositeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL); + +CREATE TYPE dbo.BulkTokenDateTimeCompositeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + StartDateTime2 DATETIMEOFFSET (7) NOT NULL, + EndDateTime2 DATETIMEOFFSET (7) NOT NULL, + IsLongerThanADay2 BIT NOT NULL); + +CREATE TYPE dbo.BulkTokenDateTimeCompositeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + StartDateTime2 DATETIMEOFFSET (7) NOT NULL, + EndDateTime2 DATETIMEOFFSET (7) NOT NULL, + IsLongerThanADay2 BIT NOT NULL); + +CREATE TYPE dbo.BulkTokenQuantityCompositeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + SystemId2 INT NULL, + QuantityCodeId2 INT NULL, + SingleValue2 DECIMAL (18, 6) NULL, + LowValue2 DECIMAL (18, 6) NULL, + HighValue2 DECIMAL (18, 6) NULL); + +CREATE TYPE dbo.BulkTokenQuantityCompositeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SystemId2 INT NULL, + QuantityCodeId2 INT NULL, + SingleValue2 DECIMAL (18, 6) NULL, + LowValue2 DECIMAL (18, 6) NULL, + HighValue2 DECIMAL (18, 6) NULL); + +CREATE TYPE dbo.BulkTokenStringCompositeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + Text2 NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow2 NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL); + +CREATE TYPE dbo.BulkTokenStringCompositeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + Text2 NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow2 NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL); + +CREATE TYPE dbo.BulkTokenNumberNumberCompositeSearchParamTableType_1 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + SingleValue2 DECIMAL (18, 6) NULL, + LowValue2 DECIMAL (18, 6) NULL, + HighValue2 DECIMAL (18, 6) NULL, + SingleValue3 DECIMAL (18, 6) NULL, + LowValue3 DECIMAL (18, 6) NULL, + HighValue3 DECIMAL (18, 6) NULL, + HasRange BIT NOT NULL); + +CREATE TYPE dbo.BulkTokenNumberNumberCompositeSearchParamTableType_2 AS TABLE ( + Offset INT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + SingleValue2 DECIMAL (18, 6) NULL, + LowValue2 DECIMAL (18, 6) NULL, + HighValue2 DECIMAL (18, 6) NULL, + SingleValue3 DECIMAL (18, 6) NULL, + LowValue3 DECIMAL (18, 6) NULL, + HighValue3 DECIMAL (18, 6) NULL, + HasRange BIT NOT NULL); + +CREATE TYPE dbo.SearchParamTableType_1 AS TABLE ( + Uri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + Status VARCHAR (10) NOT NULL, + IsPartiallySupported BIT NOT NULL); + +CREATE TYPE dbo.SearchParamTableType_2 AS TABLE ( + Uri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + Status VARCHAR (20) NOT NULL, + IsPartiallySupported BIT NOT NULL); + +CREATE TYPE dbo.BulkReindexResourceTableType_1 AS TABLE ( + Offset INT NOT NULL, + ResourceTypeId SMALLINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ETag INT NULL, + SearchParamHash VARCHAR (64) NOT NULL); + +CREATE TYPE dbo.BulkImportResourceType_1 AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Version INT NOT NULL, + IsHistory BIT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + IsDeleted BIT NOT NULL, + RequestMethod VARCHAR (10) NULL, + RawResource VARBINARY (MAX) NOT NULL, + IsRawResourceMetaSet BIT DEFAULT 0 NOT NULL, + SearchParamHash VARCHAR (64) NULL); + +CREATE TYPE dbo.UriSearchParamList AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Uri VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL PRIMARY KEY (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri)); + +CREATE TABLE dbo.ClaimType ( + ClaimTypeId TINYINT IDENTITY (1, 1) NOT NULL, + Name VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + CONSTRAINT UQ_ClaimType_ClaimTypeId UNIQUE (ClaimTypeId), + CONSTRAINT PKC_ClaimType PRIMARY KEY CLUSTERED (Name) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE dbo.CompartmentAssignment ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + CompartmentTypeId TINYINT NOT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + IsHistory BIT NOT NULL, + CONSTRAINT PKC_CompartmentAssignment PRIMARY KEY CLUSTERED (ResourceTypeId, ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId) WITH (DATA_COMPRESSION = PAGE) ON PartitionScheme_ResourceTypeId (ResourceTypeId) +); + + +GO +ALTER TABLE dbo.CompartmentAssignment + ADD CONSTRAINT DF_CompartmentAssignment_IsHistory DEFAULT 0 FOR IsHistory; + + +GO +ALTER TABLE dbo.CompartmentAssignment SET (LOCK_ESCALATION = AUTO); + + +GO +CREATE NONCLUSTERED INDEX IX_CompartmentAssignment_CompartmentTypeId_ReferenceResourceId + ON dbo.CompartmentAssignment(ResourceTypeId, CompartmentTypeId, ReferenceResourceId, ResourceSurrogateId) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.CompartmentType ( + CompartmentTypeId TINYINT IDENTITY (1, 1) NOT NULL, + Name VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + CONSTRAINT UQ_CompartmentType_CompartmentTypeId UNIQUE (CompartmentTypeId), + CONSTRAINT PKC_CompartmentType PRIMARY KEY CLUSTERED (Name) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE dbo.DateTimeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + StartDateTime DATETIME2 (7) NOT NULL, + EndDateTime DATETIME2 (7) NOT NULL, + IsLongerThanADay BIT NOT NULL, + IsHistory BIT NOT NULL, + IsMin BIT CONSTRAINT date_IsMin_Constraint DEFAULT 0 NOT NULL, + IsMax BIT CONSTRAINT date_IsMax_Constraint DEFAULT 0 NOT NULL +); + +ALTER TABLE dbo.DateTimeSearchParam + ADD CONSTRAINT DF_DateTimeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.DateTimeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_DateTimeSearchParam + ON dbo.DateTimeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_DateTimeSearchParam_SearchParamId_StartDateTime_EndDateTime + ON dbo.DateTimeSearchParam(ResourceTypeId, SearchParamId, StartDateTime, EndDateTime, ResourceSurrogateId) + INCLUDE(IsLongerThanADay, IsMin, IsMax) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_DateTimeSearchParam_SearchParamId_EndDateTime_StartDateTime + ON dbo.DateTimeSearchParam(ResourceTypeId, SearchParamId, EndDateTime, StartDateTime, ResourceSurrogateId) + INCLUDE(IsLongerThanADay, IsMin, IsMax) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_DateTimeSearchParam_SearchParamId_StartDateTime_EndDateTime_Long + ON dbo.DateTimeSearchParam(ResourceTypeId, SearchParamId, StartDateTime, EndDateTime, ResourceSurrogateId) + INCLUDE(IsMin, IsMax) WHERE IsHistory = 0 + AND IsLongerThanADay = 1 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_DateTimeSearchParam_SearchParamId_EndDateTime_StartDateTime_Long + ON dbo.DateTimeSearchParam(ResourceTypeId, SearchParamId, EndDateTime, StartDateTime, ResourceSurrogateId) + INCLUDE(IsMin, IsMax) WHERE IsHistory = 0 + AND IsLongerThanADay = 1 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +IF NOT EXISTS (SELECT 1 + FROM sys.tables + WHERE name = 'EventAgentCheckpoint') + BEGIN + CREATE TABLE dbo.EventAgentCheckpoint ( + CheckpointId VARCHAR (64) NOT NULL, + LastProcessedDateTime DATETIMEOFFSET (7), + LastProcessedIdentifier VARCHAR (64) , + UpdatedOn DATETIME2 (7) DEFAULT sysutcdatetime() NOT NULL, + CONSTRAINT PK_EventAgentCheckpoint PRIMARY KEY CLUSTERED (CheckpointId) + ) ON [PRIMARY]; + END + +CREATE PARTITION FUNCTION EventLogPartitionFunction(TINYINT) + AS RANGE RIGHT + FOR VALUES (0, 1, 2, 3, 4, 5, 6, 7); + + +GO +CREATE PARTITION SCHEME EventLogPartitionScheme + AS PARTITION EventLogPartitionFunction + ALL TO ([PRIMARY]); + + +GO +CREATE TABLE dbo.EventLog ( + PartitionId AS isnull(CONVERT (TINYINT, EventId % 8), 0) PERSISTED, + EventId BIGINT IDENTITY (1, 1) NOT NULL, + EventDate DATETIME NOT NULL, + Process VARCHAR (100) NOT NULL, + Status VARCHAR (10) NOT NULL, + Mode VARCHAR (200) NULL, + Action VARCHAR (20) NULL, + Target VARCHAR (100) NULL, + Rows BIGINT NULL, + Milliseconds INT NULL, + EventText NVARCHAR (3500) NULL, + SPID SMALLINT NOT NULL, + HostName VARCHAR (64) NOT NULL CONSTRAINT PKC_EventLog_EventDate_EventId_PartitionId PRIMARY KEY CLUSTERED (EventDate, EventId, PartitionId) ON EventLogPartitionScheme (PartitionId) +); + +CREATE TABLE dbo.ExportJob ( + Id VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Hash VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Status VARCHAR (10) NOT NULL, + HeartbeatDateTime DATETIME2 (7) NULL, + RawJobRecord VARCHAR (MAX) NOT NULL, + JobVersion ROWVERSION NOT NULL, + CONSTRAINT PKC_ExportJob PRIMARY KEY CLUSTERED (Id) +); + +CREATE UNIQUE NONCLUSTERED INDEX IX_ExportJob_Hash_Status_HeartbeatDateTime + ON dbo.ExportJob(Hash, Status, HeartbeatDateTime); + +CREATE TABLE dbo.IndexProperties ( + TableName VARCHAR (100) NOT NULL, + IndexName VARCHAR (200) NOT NULL, + PropertyName VARCHAR (100) NOT NULL, + PropertyValue VARCHAR (100) NOT NULL, + CreateDate DATETIME CONSTRAINT DF_IndexProperties_CreateDate DEFAULT getUTCdate() NOT NULL CONSTRAINT PKC_IndexProperties_TableName_IndexName_PropertyName PRIMARY KEY CLUSTERED (TableName, IndexName, PropertyName) +); + +CREATE PARTITION FUNCTION TinyintPartitionFunction(TINYINT) + AS RANGE RIGHT + FOR VALUES (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, 255); + + +GO +CREATE PARTITION SCHEME TinyintPartitionScheme + AS PARTITION TinyintPartitionFunction + ALL TO ([PRIMARY]); + + +GO +CREATE TABLE dbo.JobQueue ( + QueueType TINYINT NOT NULL, + GroupId BIGINT NOT NULL, + JobId BIGINT NOT NULL, + PartitionId AS CONVERT (TINYINT, JobId % 16) PERSISTED, + Definition VARCHAR (MAX) NOT NULL, + DefinitionHash VARBINARY (20) NOT NULL, + Version BIGINT CONSTRAINT DF_JobQueue_Version DEFAULT datediff_big(millisecond, '0001-01-01', getUTCdate()) NOT NULL, + Status TINYINT CONSTRAINT DF_JobQueue_Status DEFAULT 0 NOT NULL, + Priority TINYINT CONSTRAINT DF_JobQueue_Priority DEFAULT 100 NOT NULL, + Data BIGINT NULL, + Result VARCHAR (MAX) NULL, + CreateDate DATETIME CONSTRAINT DF_JobQueue_CreateDate DEFAULT getUTCdate() NOT NULL, + StartDate DATETIME NULL, + EndDate DATETIME NULL, + HeartbeatDate DATETIME CONSTRAINT DF_JobQueue_HeartbeatDate DEFAULT getUTCdate() NOT NULL, + Worker VARCHAR (100) NULL, + Info VARCHAR (1000) NULL, + CancelRequested BIT CONSTRAINT DF_JobQueue_CancelRequested DEFAULT 0 NOT NULL CONSTRAINT PKC_JobQueue_QueueType_PartitionId_JobId PRIMARY KEY CLUSTERED (QueueType, PartitionId, JobId) ON TinyintPartitionScheme (QueueType), + CONSTRAINT U_JobQueue_QueueType_JobId UNIQUE (QueueType, JobId) +); + + +GO +CREATE INDEX IX_QueueType_PartitionId_Status_Priority + ON dbo.JobQueue(PartitionId, Status, Priority) + ON TinyintPartitionScheme (QueueType); + + +GO +CREATE INDEX IX_QueueType_GroupId + ON dbo.JobQueue(QueueType, GroupId) + ON TinyintPartitionScheme (QueueType); + + +GO +CREATE INDEX IX_QueueType_DefinitionHash + ON dbo.JobQueue(QueueType, DefinitionHash) + ON TinyintPartitionScheme (QueueType); + +CREATE TABLE dbo.NumberSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SingleValue DECIMAL (36, 18) NULL, + LowValue DECIMAL (36, 18) NOT NULL, + HighValue DECIMAL (36, 18) NOT NULL, + IsHistory BIT NOT NULL +); + +ALTER TABLE dbo.NumberSearchParam + ADD CONSTRAINT DF_NumberSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.NumberSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_NumberSearchParam + ON dbo.NumberSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_NumberSearchParam_SearchParamId_SingleValue + ON dbo.NumberSearchParam(ResourceTypeId, SearchParamId, SingleValue, ResourceSurrogateId) WHERE IsHistory = 0 + AND SingleValue IS NOT NULL + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_NumberSearchParam_SearchParamId_LowValue_HighValue + ON dbo.NumberSearchParam(ResourceTypeId, SearchParamId, LowValue, HighValue, ResourceSurrogateId) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_NumberSearchParam_SearchParamId_HighValue_LowValue + ON dbo.NumberSearchParam(ResourceTypeId, SearchParamId, HighValue, LowValue, ResourceSurrogateId) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.Parameters ( + Id VARCHAR (100) NOT NULL, + Date DATETIME NULL, + Number FLOAT NULL, + Bigint BIGINT NULL, + Char VARCHAR (4000) NULL, + Binary VARBINARY (MAX) NULL, + UpdatedDate DATETIME NULL, + UpdatedBy NVARCHAR (255) NULL CONSTRAINT PKC_Parameters_Id PRIMARY KEY CLUSTERED (Id) WITH (IGNORE_DUP_KEY = ON) +); + + +GO +CREATE TABLE dbo.ParametersHistory ( + ChangeId INT IDENTITY (1, 1) NOT NULL, + Id VARCHAR (100) NOT NULL, + Date DATETIME NULL, + Number FLOAT NULL, + Bigint BIGINT NULL, + Char VARCHAR (4000) NULL, + Binary VARBINARY (MAX) NULL, + UpdatedDate DATETIME NULL, + UpdatedBy NVARCHAR (255) NULL +); + +CREATE TABLE dbo.QuantityCode ( + QuantityCodeId INT IDENTITY (1, 1) NOT NULL, + Value NVARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + CONSTRAINT UQ_QuantityCode_QuantityCodeId UNIQUE (QuantityCodeId), + CONSTRAINT PKC_QuantityCode PRIMARY KEY CLUSTERED (Value) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE dbo.QuantitySearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + QuantityCodeId INT NULL, + SingleValue DECIMAL (36, 18) NULL, + LowValue DECIMAL (36, 18) NOT NULL, + HighValue DECIMAL (36, 18) NOT NULL, + IsHistory BIT NOT NULL +); + +ALTER TABLE dbo.QuantitySearchParam + ADD CONSTRAINT DF_QuantitySearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.QuantitySearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_QuantitySearchParam + ON dbo.QuantitySearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_QuantitySearchParam_SearchParamId_QuantityCodeId_SingleValue + ON dbo.QuantitySearchParam(ResourceTypeId, SearchParamId, QuantityCodeId, SingleValue, ResourceSurrogateId) + INCLUDE(SystemId) WHERE IsHistory = 0 + AND SingleValue IS NOT NULL + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_QuantitySearchParam_SearchParamId_QuantityCodeId_LowValue_HighValue + ON dbo.QuantitySearchParam(ResourceTypeId, SearchParamId, QuantityCodeId, LowValue, HighValue, ResourceSurrogateId) + INCLUDE(SystemId) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_QuantitySearchParam_SearchParamId_QuantityCodeId_HighValue_LowValue + ON dbo.QuantitySearchParam(ResourceTypeId, SearchParamId, QuantityCodeId, HighValue, LowValue, ResourceSurrogateId) + INCLUDE(SystemId) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.ReferenceSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId SMALLINT NULL, + ReferenceResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion INT NULL, + IsHistory BIT NOT NULL +); + +ALTER TABLE dbo.ReferenceSearchParam + ADD CONSTRAINT DF_ReferenceSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.ReferenceSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_ReferenceSearchParam + ON dbo.ReferenceSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_ReferenceSearchParam_SearchParamId_ReferenceResourceTypeId_ReferenceResourceId_BaseUri_ReferenceResourceVersion + ON dbo.ReferenceSearchParam(ResourceTypeId, SearchParamId, ReferenceResourceId, ReferenceResourceTypeId, BaseUri, ResourceSurrogateId) + INCLUDE(ReferenceResourceVersion) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.ReferenceTokenCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + BaseUri1 VARCHAR (128) COLLATE Latin1_General_100_CS_AS NULL, + ReferenceResourceTypeId1 SMALLINT NULL, + ReferenceResourceId1 VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + ReferenceResourceVersion1 INT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + IsHistory BIT NOT NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.ReferenceTokenCompositeSearchParam + ADD CONSTRAINT DF_ReferenceTokenCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.ReferenceTokenCompositeSearchParam + ADD CONSTRAINT CHK_ReferenceTokenCompositeSearchParam_CodeOverflow2 CHECK (LEN(Code2) = 256 + OR CodeOverflow2 IS NULL); + +ALTER TABLE dbo.ReferenceTokenCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_ReferenceTokenCompositeSearchParam + ON dbo.ReferenceTokenCompositeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_ReferenceTokenCompositeSearchParam_ReferenceResourceId1_Code2 + ON dbo.ReferenceTokenCompositeSearchParam(ResourceTypeId, SearchParamId, ReferenceResourceId1, Code2, ResourceSurrogateId) + INCLUDE(ReferenceResourceTypeId1, BaseUri1, SystemId2) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.ReindexJob ( + Id VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Status VARCHAR (10) NOT NULL, + HeartbeatDateTime DATETIME2 (7) NULL, + RawJobRecord VARCHAR (MAX) NOT NULL, + JobVersion ROWVERSION NOT NULL, + CONSTRAINT PKC_ReindexJob PRIMARY KEY CLUSTERED (Id) +); + +CREATE TABLE dbo.Resource ( + ResourceTypeId SMALLINT NOT NULL, + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + Version INT NOT NULL, + IsHistory BIT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + IsDeleted BIT NOT NULL, + RequestMethod VARCHAR (10) NULL, + RawResource VARBINARY (MAX) NOT NULL, + IsRawResourceMetaSet BIT DEFAULT 0 NOT NULL, + SearchParamHash VARCHAR (64) NULL, + TransactionId BIGINT NULL, + HistoryTransactionId BIGINT NULL CONSTRAINT PKC_Resource PRIMARY KEY CLUSTERED (ResourceTypeId, ResourceSurrogateId) WITH (DATA_COMPRESSION = PAGE) ON PartitionScheme_ResourceTypeId (ResourceTypeId), + CONSTRAINT CH_Resource_RawResource_Length CHECK (RawResource > 0x0) +); + +ALTER TABLE dbo.Resource SET (LOCK_ESCALATION = AUTO); + +CREATE INDEX IX_ResourceTypeId_TransactionId + ON dbo.Resource(ResourceTypeId, TransactionId) WHERE TransactionId IS NOT NULL + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE INDEX IX_ResourceTypeId_HistoryTransactionId + ON dbo.Resource(ResourceTypeId, HistoryTransactionId) WHERE HistoryTransactionId IS NOT NULL + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE UNIQUE NONCLUSTERED INDEX IX_Resource_ResourceTypeId_ResourceId_Version + ON dbo.Resource(ResourceTypeId, ResourceId, Version) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE UNIQUE NONCLUSTERED INDEX IX_Resource_ResourceTypeId_ResourceId + ON dbo.Resource(ResourceTypeId, ResourceId) + INCLUDE(Version, IsDeleted) WHERE IsHistory = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE UNIQUE NONCLUSTERED INDEX IX_Resource_ResourceTypeId_ResourceSurrgateId + ON dbo.Resource(ResourceTypeId, ResourceSurrogateId) WHERE IsHistory = 0 + AND IsDeleted = 0 + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.ResourceChangeData ( + Id BIGINT IDENTITY (1, 1) NOT NULL, + Timestamp DATETIME2 (7) CONSTRAINT DF_ResourceChangeData_Timestamp DEFAULT sysutcdatetime() NOT NULL, + ResourceId VARCHAR (64) NOT NULL, + ResourceTypeId SMALLINT NOT NULL, + ResourceVersion INT NOT NULL, + ResourceChangeTypeId TINYINT NOT NULL +) ON PartitionScheme_ResourceChangeData_Timestamp (Timestamp); + +CREATE CLUSTERED INDEX IXC_ResourceChangeData + ON dbo.ResourceChangeData(Id ASC) WITH (ONLINE = ON) + ON PartitionScheme_ResourceChangeData_Timestamp (Timestamp); + +CREATE TABLE dbo.ResourceChangeDataStaging ( + Id BIGINT IDENTITY (1, 1) NOT NULL, + Timestamp DATETIME2 (7) CONSTRAINT DF_ResourceChangeDataStaging_Timestamp DEFAULT sysutcdatetime() NOT NULL, + ResourceId VARCHAR (64) NOT NULL, + ResourceTypeId SMALLINT NOT NULL, + ResourceVersion INT NOT NULL, + ResourceChangeTypeId TINYINT NOT NULL +) ON [PRIMARY]; + +CREATE CLUSTERED INDEX IXC_ResourceChangeDataStaging + ON dbo.ResourceChangeDataStaging(Id ASC, Timestamp ASC) WITH (ONLINE = ON) + ON [PRIMARY]; + +ALTER TABLE dbo.ResourceChangeDataStaging WITH CHECK + ADD CONSTRAINT CHK_ResourceChangeDataStaging_partition CHECK (Timestamp < CONVERT (DATETIME2 (7), N'9999-12-31 23:59:59.9999999')); + +ALTER TABLE dbo.ResourceChangeDataStaging CHECK CONSTRAINT CHK_ResourceChangeDataStaging_partition; + +CREATE TABLE dbo.ResourceChangeType ( + ResourceChangeTypeId TINYINT NOT NULL, + Name NVARCHAR (50) NOT NULL, + CONSTRAINT PK_ResourceChangeType PRIMARY KEY CLUSTERED (ResourceChangeTypeId), + CONSTRAINT UQ_ResourceChangeType_Name UNIQUE NONCLUSTERED (Name) +) ON [PRIMARY]; + + +GO +INSERT dbo.ResourceChangeType (ResourceChangeTypeId, Name) +VALUES (0, N'Creation'); + +INSERT dbo.ResourceChangeType (ResourceChangeTypeId, Name) +VALUES (1, N'Update'); + +INSERT dbo.ResourceChangeType (ResourceChangeTypeId, Name) +VALUES (2, N'Deletion'); + +CREATE TABLE dbo.ResourceType ( + ResourceTypeId SMALLINT IDENTITY (1, 1) NOT NULL, + Name NVARCHAR (50) COLLATE Latin1_General_100_CS_AS NOT NULL, + CONSTRAINT UQ_ResourceType_ResourceTypeId UNIQUE (ResourceTypeId), + CONSTRAINT PKC_ResourceType PRIMARY KEY CLUSTERED (Name) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE dbo.ResourceWriteClaim ( + ResourceSurrogateId BIGINT NOT NULL, + ClaimTypeId TINYINT NOT NULL, + ClaimValue NVARCHAR (128) NOT NULL +) +WITH (DATA_COMPRESSION = PAGE); + +CREATE CLUSTERED INDEX IXC_ResourceWriteClaim + ON dbo.ResourceWriteClaim(ResourceSurrogateId, ClaimTypeId); + +CREATE TABLE dbo.SchemaMigrationProgress ( + Timestamp DATETIME2 (3) DEFAULT CURRENT_TIMESTAMP, + Message NVARCHAR (MAX) +); + +CREATE TABLE dbo.SearchParam ( + SearchParamId SMALLINT IDENTITY (1, 1) NOT NULL, + Uri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + Status VARCHAR (20) NULL, + LastUpdated DATETIMEOFFSET (7) NULL, + IsPartiallySupported BIT NULL, + CONSTRAINT UQ_SearchParam_SearchParamId UNIQUE (SearchParamId), + CONSTRAINT PKC_SearchParam PRIMARY KEY CLUSTERED (Uri) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE dbo.StringSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (256) COLLATE Latin1_General_100_CI_AI_SC NOT NULL, + TextOverflow NVARCHAR (MAX) COLLATE Latin1_General_100_CI_AI_SC NULL, + IsHistory BIT NOT NULL, + IsMin BIT CONSTRAINT string_IsMin_Constraint DEFAULT 0 NOT NULL, + IsMax BIT CONSTRAINT string_IsMax_Constraint DEFAULT 0 NOT NULL +); + +ALTER TABLE dbo.StringSearchParam + ADD CONSTRAINT DF_StringSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.StringSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_StringSearchParam + ON dbo.StringSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_StringSearchParam_SearchParamId_Text + ON dbo.StringSearchParam(ResourceTypeId, SearchParamId, Text, ResourceSurrogateId) + INCLUDE(TextOverflow, IsMin, IsMax) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_StringSearchParam_SearchParamId_TextWithOverflow + ON dbo.StringSearchParam(ResourceTypeId, SearchParamId, Text, ResourceSurrogateId) + INCLUDE(IsMin, IsMax) WHERE IsHistory = 0 + AND TextOverflow IS NOT NULL WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.System ( + SystemId INT IDENTITY (1, 1) NOT NULL, + Value NVARCHAR (256) NOT NULL, + CONSTRAINT UQ_System_SystemId UNIQUE (SystemId), + CONSTRAINT PKC_System PRIMARY KEY CLUSTERED (Value) WITH (DATA_COMPRESSION = PAGE) +); + +CREATE TABLE [dbo].[TaskInfo] ( + [TaskId] VARCHAR (64) NOT NULL, + [QueueId] VARCHAR (64) NOT NULL, + [Status] SMALLINT NOT NULL, + [TaskTypeId] SMALLINT NOT NULL, + [RunId] VARCHAR (50) NULL, + [IsCanceled] BIT NOT NULL, + [RetryCount] SMALLINT NOT NULL, + [MaxRetryCount] SMALLINT NOT NULL, + [HeartbeatDateTime] DATETIME2 (7) NULL, + [InputData] VARCHAR (MAX) NOT NULL, + [TaskContext] VARCHAR (MAX) NULL, + [Result] VARCHAR (MAX) NULL, + [CreateDateTime] DATETIME2 (7) CONSTRAINT DF_TaskInfo_CreateDate DEFAULT SYSUTCDATETIME() NOT NULL, + [StartDateTime] DATETIME2 (7) NULL, + [EndDateTime] DATETIME2 (7) NULL, + [Worker] VARCHAR (100) NULL, + [RestartInfo] VARCHAR (MAX) NULL, + [ParentTaskId] VARCHAR (64) NULL, + CONSTRAINT PKC_TaskInfo PRIMARY KEY CLUSTERED (TaskId) WITH (DATA_COMPRESSION = PAGE) +) ON [PRIMARY] TEXTIMAGE_ON [PRIMARY]; + + +GO +CREATE NONCLUSTERED INDEX IX_QueueId_Status + ON dbo.TaskInfo(QueueId, Status); + + +GO +CREATE NONCLUSTERED INDEX IX_QueueId_ParentTaskId + ON dbo.TaskInfo(QueueId, ParentTaskId); + +CREATE TABLE dbo.TokenDateTimeCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + StartDateTime2 DATETIME2 (7) NOT NULL, + EndDateTime2 DATETIME2 (7) NOT NULL, + IsLongerThanADay2 BIT NOT NULL, + IsHistory BIT NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenDateTimeCompositeSearchParam + ADD CONSTRAINT DF_TokenDateTimeCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenDateTimeCompositeSearchParam + ADD CONSTRAINT CHK_TokenDateTimeCompositeSearchParam_CodeOverflow1 CHECK (LEN(Code1) = 256 + OR CodeOverflow1 IS NULL); + +ALTER TABLE dbo.TokenDateTimeCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenDateTimeCompositeSearchParam + ON dbo.TokenDateTimeCompositeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenDateTimeCompositeSearchParam_Code1_StartDateTime2_EndDateTime2 + ON dbo.TokenDateTimeCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, StartDateTime2, EndDateTime2, ResourceSurrogateId) + INCLUDE(SystemId1, IsLongerThanADay2) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenDateTimeCompositeSearchParam_Code1_EndDateTime2_StartDateTime2 + ON dbo.TokenDateTimeCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, EndDateTime2, StartDateTime2, ResourceSurrogateId) + INCLUDE(SystemId1, IsLongerThanADay2) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenDateTimeCompositeSearchParam_Code1_StartDateTime2_EndDateTime2_Long + ON dbo.TokenDateTimeCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, StartDateTime2, EndDateTime2, ResourceSurrogateId) + INCLUDE(SystemId1) WHERE IsHistory = 0 + AND IsLongerThanADay2 = 1 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenDateTimeCompositeSearchParam_Code1_EndDateTime2_StartDateTime2_Long + ON dbo.TokenDateTimeCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, EndDateTime2, StartDateTime2, ResourceSurrogateId) + INCLUDE(SystemId1) WHERE IsHistory = 0 + AND IsLongerThanADay2 = 1 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenNumberNumberCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + SingleValue2 DECIMAL (36, 18) NULL, + LowValue2 DECIMAL (36, 18) NULL, + HighValue2 DECIMAL (36, 18) NULL, + SingleValue3 DECIMAL (36, 18) NULL, + LowValue3 DECIMAL (36, 18) NULL, + HighValue3 DECIMAL (36, 18) NULL, + HasRange BIT NOT NULL, + IsHistory BIT NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenNumberNumberCompositeSearchParam + ADD CONSTRAINT DF_TokenNumberNumberCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenNumberNumberCompositeSearchParam + ADD CONSTRAINT CHK_TokenNumberNumberCompositeSearchParam_CodeOverflow1 CHECK (LEN(Code1) = 256 + OR CodeOverflow1 IS NULL); + +ALTER TABLE dbo.TokenNumberNumberCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenNumberNumberCompositeSearchParam + ON dbo.TokenNumberNumberCompositeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenNumberNumberCompositeSearchParam_SearchParamId_Code1_Text2 + ON dbo.TokenNumberNumberCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, SingleValue2, SingleValue3, ResourceSurrogateId) + INCLUDE(SystemId1) WHERE IsHistory = 0 + AND HasRange = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenNumberNumberCompositeSearchParam_SearchParamId_Code1_LowValue2_HighValue2_LowValue3_HighValue3 + ON dbo.TokenNumberNumberCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, LowValue2, HighValue2, LowValue3, HighValue3, ResourceSurrogateId) + INCLUDE(SystemId1) WHERE IsHistory = 0 + AND HasRange = 1 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenQuantityCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + SystemId2 INT NULL, + QuantityCodeId2 INT NULL, + SingleValue2 DECIMAL (36, 18) NULL, + LowValue2 DECIMAL (36, 18) NULL, + HighValue2 DECIMAL (36, 18) NULL, + IsHistory BIT NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenQuantityCompositeSearchParam + ADD CONSTRAINT DF_TokenQuantityCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenQuantityCompositeSearchParam + ADD CONSTRAINT CHK_TokenQuantityCompositeSearchParam_CodeOverflow1 CHECK (LEN(Code1) = 256 + OR CodeOverflow1 IS NULL); + +ALTER TABLE dbo.TokenQuantityCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenQuantityCompositeSearchParam + ON dbo.TokenQuantityCompositeSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenQuantityCompositeSearchParam_SearchParamId_Code1_QuantityCodeId2_SingleValue2 + ON dbo.TokenQuantityCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, SingleValue2, ResourceSurrogateId) + INCLUDE(QuantityCodeId2, SystemId1, SystemId2) WHERE IsHistory = 0 + AND SingleValue2 IS NOT NULL WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenQuantityCompositeSearchParam_SearchParamId_Code1_QuantityCodeId2_LowValue2_HighValue2 + ON dbo.TokenQuantityCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, LowValue2, HighValue2, ResourceSurrogateId) + INCLUDE(QuantityCodeId2, SystemId1, SystemId2) WHERE IsHistory = 0 + AND LowValue2 IS NOT NULL WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenQuantityCompositeSearchParam_SearchParamId_Code1_QuantityCodeId2_HighValue2_LowValue2 + ON dbo.TokenQuantityCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, HighValue2, LowValue2, ResourceSurrogateId) + INCLUDE(QuantityCodeId2, SystemId1, SystemId2) WHERE IsHistory = 0 + AND LowValue2 IS NOT NULL WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId INT NULL, + Code VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + IsHistory BIT NOT NULL, + CodeOverflow VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenSearchParam + ADD CONSTRAINT DF_TokenSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenSearchParam + ADD CONSTRAINT CHK_TokenSearchParam_CodeOverflow CHECK (LEN(Code) = 256 + OR CodeOverflow IS NULL); + +ALTER TABLE dbo.TokenSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenSearchParam + ON dbo.TokenSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenSeachParam_SearchParamId_Code_SystemId + ON dbo.TokenSearchParam(ResourceTypeId, SearchParamId, Code, ResourceSurrogateId) + INCLUDE(SystemId) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenStringCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + Text2 NVARCHAR (256) COLLATE Latin1_General_CI_AI NOT NULL, + TextOverflow2 NVARCHAR (MAX) COLLATE Latin1_General_CI_AI NULL, + IsHistory BIT NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenStringCompositeSearchParam + ADD CONSTRAINT DF_TokenStringCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenStringCompositeSearchParam + ADD CONSTRAINT CHK_TokenStringCompositeSearchParam_CodeOverflow1 CHECK (LEN(Code1) = 256 + OR CodeOverflow1 IS NULL); + +ALTER TABLE dbo.TokenStringCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenStringCompositeSearchParam + ON dbo.TokenStringCompositeSearchParam(ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenStringCompositeSearchParam_SearchParamId_Code1_Text2 + ON dbo.TokenStringCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, Text2, ResourceSurrogateId) + INCLUDE(SystemId1, TextOverflow2) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenStringCompositeSearchParam_SearchParamId_Code1_Text2WithOverflow + ON dbo.TokenStringCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, Text2, ResourceSurrogateId) + INCLUDE(SystemId1) WHERE IsHistory = 0 + AND TextOverflow2 IS NOT NULL WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenText ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Text NVARCHAR (400) COLLATE Latin1_General_CI_AI NOT NULL, + IsHistory BIT NOT NULL +); + +ALTER TABLE dbo.TokenText + ADD CONSTRAINT DF_TokenText_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenText SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenText + ON dbo.TokenText(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenText_SearchParamId_Text + ON dbo.TokenText(ResourceTypeId, SearchParamId, Text, ResourceSurrogateId) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.TokenTokenCompositeSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + SystemId1 INT NULL, + Code1 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + SystemId2 INT NULL, + Code2 VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + IsHistory BIT NOT NULL, + CodeOverflow1 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL, + CodeOverflow2 VARCHAR (MAX) COLLATE Latin1_General_100_CS_AS NULL +); + +ALTER TABLE dbo.TokenTokenCompositeSearchParam + ADD CONSTRAINT DF_TokenTokenCompositeSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.TokenTokenCompositeSearchParam + ADD CONSTRAINT CHK_TokenTokenCompositeSearchParam_CodeOverflow1 CHECK (LEN(Code1) = 256 + OR CodeOverflow1 IS NULL); + +ALTER TABLE dbo.TokenTokenCompositeSearchParam + ADD CONSTRAINT CHK_TokenTokenCompositeSearchParam_CodeOverflow2 CHECK (LEN(Code2) = 256 + OR CodeOverflow2 IS NULL); + +ALTER TABLE dbo.TokenTokenCompositeSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_TokenTokenCompositeSearchParam + ON dbo.TokenTokenCompositeSearchParam(ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_TokenTokenCompositeSearchParam_Code1_Code2 + ON dbo.TokenTokenCompositeSearchParam(ResourceTypeId, SearchParamId, Code1, Code2, ResourceSurrogateId) + INCLUDE(SystemId1, SystemId2) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.Transactions ( + SurrogateIdRangeFirstValue BIGINT NOT NULL, + SurrogateIdRangeLastValue BIGINT NOT NULL, + Definition VARCHAR (2000) NULL, + IsCompleted BIT CONSTRAINT DF_Transactions_IsCompleted DEFAULT 0 NOT NULL, + IsSuccess BIT CONSTRAINT DF_Transactions_IsSuccess DEFAULT 0 NOT NULL, + IsVisible BIT CONSTRAINT DF_Transactions_IsVisible DEFAULT 0 NOT NULL, + IsHistoryMoved BIT CONSTRAINT DF_Transactions_IsHistoryMoved DEFAULT 0 NOT NULL, + CreateDate DATETIME CONSTRAINT DF_Transactions_CreateDate DEFAULT getUTCdate() NOT NULL, + EndDate DATETIME NULL, + VisibleDate DATETIME NULL, + HistoryMovedDate DATETIME NULL, + HeartbeatDate DATETIME CONSTRAINT DF_Transactions_HeartbeatDate DEFAULT getUTCdate() NOT NULL, + FailureReason VARCHAR (MAX) NULL, + IsControlledByClient BIT CONSTRAINT DF_Transactions_IsControlledByClient DEFAULT 1 NOT NULL, + InvisibleHistoryRemovedDate DATETIME NULL CONSTRAINT PKC_Transactions_SurrogateIdRangeFirstValue PRIMARY KEY CLUSTERED (SurrogateIdRangeFirstValue) +); + +CREATE INDEX IX_IsVisible + ON dbo.Transactions(IsVisible); + +CREATE TABLE dbo.UriSearchParam ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL, + SearchParamId SMALLINT NOT NULL, + Uri VARCHAR (256) COLLATE Latin1_General_100_CS_AS NOT NULL, + IsHistory BIT NOT NULL +); + +ALTER TABLE dbo.UriSearchParam + ADD CONSTRAINT DF_UriSearchParam_IsHistory DEFAULT 0 FOR IsHistory; + +ALTER TABLE dbo.UriSearchParam SET (LOCK_ESCALATION = AUTO); + +CREATE CLUSTERED INDEX IXC_UriSearchParam + ON dbo.UriSearchParam(ResourceTypeId, ResourceSurrogateId, SearchParamId) WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE NONCLUSTERED INDEX IX_UriSearchParam_SearchParamId_Uri + ON dbo.UriSearchParam(ResourceTypeId, SearchParamId, Uri, ResourceSurrogateId) WHERE IsHistory = 0 WITH (DATA_COMPRESSION = PAGE) + ON PartitionScheme_ResourceTypeId (ResourceTypeId); + +CREATE TABLE dbo.WatchdogLeases ( + Watchdog VARCHAR (100) NOT NULL, + LeaseHolder VARCHAR (100) CONSTRAINT DF_WatchdogLeases_LeaseHolder DEFAULT '' NOT NULL, + LeaseEndTime DATETIME CONSTRAINT DF_WatchdogLeases_LeaseEndTime DEFAULT 0 NOT NULL, + RemainingLeaseTimeSec AS datediff(second, getUTCdate(), LeaseEndTime), + LeaseRequestor VARCHAR (100) CONSTRAINT DF_WatchdogLeases_LeaseRequestor DEFAULT '' NOT NULL, + LeaseRequestTime DATETIME CONSTRAINT DF_WatchdogLeases_LeaseRequestTime DEFAULT 0 NOT NULL CONSTRAINT PKC_WatchdogLeases_Watchdog PRIMARY KEY CLUSTERED (Watchdog) +); + +COMMIT +GO +CREATE PROCEDURE dbo.AcquireExportJobs +@jobHeartbeatTimeoutThresholdInSeconds BIGINT, @maximumNumberOfConcurrentJobsAllowed INT +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DECLARE @expirationDateTime AS DATETIME2 (7); +SELECT @expirationDateTime = DATEADD(second, -@jobHeartbeatTimeoutThresholdInSeconds, SYSUTCDATETIME()); +DECLARE @numberOfRunningJobs AS INT; +SELECT @numberOfRunningJobs = COUNT(*) +FROM dbo.ExportJob WITH (TABLOCKX) +WHERE Status = 'Running' + AND HeartbeatDateTime > @expirationDateTime; +DECLARE @limit AS INT = @maximumNumberOfConcurrentJobsAllowed - @numberOfRunningJobs; +IF (@limit > 0) + BEGIN + DECLARE @availableJobs TABLE ( + Id VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + JobVersion BINARY (8) NOT NULL); + INSERT INTO @availableJobs + SELECT TOP (@limit) Id, + JobVersion + FROM dbo.ExportJob + WHERE (Status = 'Queued' + OR (Status = 'Running' + AND HeartbeatDateTime <= @expirationDateTime)) + ORDER BY HeartbeatDateTime; + DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); + UPDATE dbo.ExportJob + SET Status = 'Running', + HeartbeatDateTime = @heartbeatDateTime, + RawJobRecord = JSON_MODIFY(RawJobRecord, '$.status', 'Running') + OUTPUT inserted.RawJobRecord, inserted.JobVersion + FROM dbo.ExportJob AS job + INNER JOIN + @availableJobs AS availableJob + ON job.Id = availableJob.Id + AND job.JobVersion = availableJob.JobVersion; + END +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.AcquireReindexJobs +@jobHeartbeatTimeoutThresholdInSeconds BIGINT, @maximumNumberOfConcurrentJobsAllowed INT +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DECLARE @expirationDateTime AS DATETIME2 (7); +SELECT @expirationDateTime = DATEADD(second, -@jobHeartbeatTimeoutThresholdInSeconds, SYSUTCDATETIME()); +DECLARE @numberOfRunningJobs AS INT; +SELECT @numberOfRunningJobs = COUNT(*) +FROM dbo.ReindexJob WITH (TABLOCKX) +WHERE Status = 'Running' + AND HeartbeatDateTime > @expirationDateTime; +DECLARE @limit AS INT = @maximumNumberOfConcurrentJobsAllowed - @numberOfRunningJobs; +IF (@limit > 0) + BEGIN + DECLARE @availableJobs TABLE ( + Id VARCHAR (64) COLLATE Latin1_General_100_CS_AS NOT NULL, + JobVersion BINARY (8) NOT NULL); + INSERT INTO @availableJobs + SELECT TOP (@limit) Id, + JobVersion + FROM dbo.ReindexJob + WHERE (Status = 'Queued' + OR (Status = 'Running' + AND HeartbeatDateTime <= @expirationDateTime)) + ORDER BY HeartbeatDateTime; + DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); + UPDATE dbo.ReindexJob + SET Status = 'Running', + HeartbeatDateTime = @heartbeatDateTime, + RawJobRecord = JSON_MODIFY(RawJobRecord, '$.status', 'Running') + OUTPUT inserted.RawJobRecord, inserted.JobVersion + FROM dbo.ReindexJob AS job + INNER JOIN + @availableJobs AS availableJob + ON job.Id = availableJob.Id + AND job.JobVersion = availableJob.JobVersion; + END +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.AcquireWatchdogLease +@Watchdog VARCHAR (100), @Worker VARCHAR (100), @AllowRebalance BIT=1, @ForceAcquire BIT=0, @LeasePeriodSec FLOAT, @WorkerIsRunning BIT=0, @LeaseEndTime DATETIME OUTPUT, @IsAcquired BIT OUTPUT, @CurrentLeaseHolder VARCHAR (100)=NULL OUTPUT +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +DECLARE @SP AS VARCHAR (100) = 'AcquireWatchdogLease', @Mode AS VARCHAR (100), @msg AS VARCHAR (1000), @MyLeasesNumber AS INT, @OtherValidRequestsOrLeasesNumber AS INT, @MyValidRequestsOrLeasesNumber AS INT, @DesiredLeasesNumber AS INT, @NotLeasedWatchdogNumber AS INT, @WatchdogNumber AS INT, @Now AS DATETIME, @MyLastChangeTime AS DATETIME, @PreviousLeaseHolder AS VARCHAR (100), @Rows AS INT = 0, @NumberOfWorkers AS INT, @st AS DATETIME = getUTCdate(), @RowsInt AS INT, @Pattern AS VARCHAR (100); +BEGIN TRY + SET @Mode = 'R=' + isnull(@Watchdog, 'NULL') + ' W=' + isnull(@Worker, 'NULL') + ' F=' + isnull(CONVERT (VARCHAR, @ForceAcquire), 'NULL') + ' LP=' + isnull(CONVERT (VARCHAR, @LeasePeriodSec), 'NULL'); + SET @CurrentLeaseHolder = ''; + SET @IsAcquired = 0; + SET @Now = getUTCdate(); + SET @LeaseEndTime = @Now; + SET @Pattern = NULLIF ((SELECT Char + FROM dbo.Parameters + WHERE Id = 'WatchdogLeaseHolderIncludePatternFor' + @Watchdog), ''); + IF @Pattern IS NULL + SET @Pattern = NULLIF ((SELECT Char + FROM dbo.Parameters + WHERE Id = 'WatchdogLeaseHolderIncludePattern'), ''); + IF @Pattern IS NOT NULL + AND @Worker NOT LIKE @Pattern + BEGIN + SET @msg = 'Worker does not match include pattern=' + @Pattern; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows, @Text = @msg; + SET @CurrentLeaseHolder = isnull((SELECT LeaseHolder + FROM dbo.WatchdogLeases + WHERE Watchdog = @Watchdog), ''); + RETURN; + END + SET @Pattern = NULLIF ((SELECT Char + FROM dbo.Parameters + WHERE Id = 'WatchdogLeaseHolderExcludePatternFor' + @Watchdog), ''); + IF @Pattern IS NULL + SET @Pattern = NULLIF ((SELECT Char + FROM dbo.Parameters + WHERE Id = 'WatchdogLeaseHolderExcludePattern'), ''); + IF @Pattern IS NOT NULL + AND @Worker LIKE @Pattern + BEGIN + SET @msg = 'Worker matches exclude pattern=' + @Pattern; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows, @Text = @msg; + SET @CurrentLeaseHolder = isnull((SELECT LeaseHolder + FROM dbo.WatchdogLeases + WHERE Watchdog = @Watchdog), ''); + RETURN; + END + DECLARE @Watchdogs TABLE ( + Watchdog VARCHAR (100) PRIMARY KEY); + INSERT INTO @Watchdogs + SELECT Watchdog + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE RemainingLeaseTimeSec * (-1) > 10 * @LeasePeriodSec + OR @ForceAcquire = 1 + AND Watchdog = @Watchdog + AND LeaseHolder <> @Worker; + IF @@rowcount > 0 + BEGIN + DELETE dbo.WatchdogLeases + WHERE Watchdog IN (SELECT Watchdog + FROM @Watchdogs); + SET @Rows += @@rowcount; + IF @Rows > 0 + BEGIN + SET @msg = ''; + SELECT @msg = CONVERT (VARCHAR (1000), @msg + CASE WHEN @msg = '' THEN '' ELSE ',' END + Watchdog) + FROM @Watchdogs; + SET @msg = CONVERT (VARCHAR (1000), 'Remove old/forced leases:' + @msg); + EXECUTE dbo.LogEvent @Process = 'AcquireWatchdogLease', @Status = 'Info', @Mode = @Mode, @Target = 'WatchdogLeases', @Action = 'Delete', @Rows = @Rows, @Text = @msg; + END + END + SET @NumberOfWorkers = 1 + (SELECT count(*) + FROM (SELECT LeaseHolder + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseHolder <> @Worker + UNION + SELECT LeaseRequestor + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseRequestor <> @Worker + AND LeaseRequestor <> '') AS A); + SET @Mode = CONVERT (VARCHAR (100), @Mode + ' N=' + CONVERT (VARCHAR (10), @NumberOfWorkers)); + IF NOT EXISTS (SELECT * + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE Watchdog = @Watchdog) + INSERT INTO dbo.WatchdogLeases (Watchdog, LeaseEndTime, LeaseRequestTime) + SELECT @Watchdog, + dateadd(day, -10, @Now), + dateadd(day, -10, @Now) + WHERE NOT EXISTS (SELECT * + FROM dbo.WatchdogLeases WITH (TABLOCKX) + WHERE Watchdog = @Watchdog); + SET @LeaseEndTime = dateadd(second, @LeasePeriodSec, @Now); + SET @WatchdogNumber = (SELECT count(*) + FROM dbo.WatchdogLeases WITH (NOLOCK)); + SET @NotLeasedWatchdogNumber = (SELECT count(*) + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseHolder = '' + OR LeaseEndTime < @Now); + SET @MyLeasesNumber = (SELECT count(*) + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseHolder = @Worker + AND LeaseEndTime > @Now); + SET @OtherValidRequestsOrLeasesNumber = (SELECT count(*) + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseHolder <> @Worker + AND LeaseEndTime > @Now + OR LeaseRequestor <> @Worker + AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec); + SET @MyValidRequestsOrLeasesNumber = (SELECT count(*) + FROM dbo.WatchdogLeases WITH (NOLOCK) + WHERE LeaseHolder = @Worker + AND LeaseEndTime > @Now + OR LeaseRequestor = @Worker + AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec); + SET @DesiredLeasesNumber = ceiling(1.0 * @WatchdogNumber / @NumberOfWorkers); + IF @DesiredLeasesNumber = 0 + SET @DesiredLeasesNumber = 1; + IF @DesiredLeasesNumber = 1 + AND @OtherValidRequestsOrLeasesNumber = 1 + AND @WatchdogNumber = 1 + SET @DesiredLeasesNumber = 0; + IF @MyValidRequestsOrLeasesNumber = floor(1.0 * @WatchdogNumber / @NumberOfWorkers) + AND @OtherValidRequestsOrLeasesNumber + @MyValidRequestsOrLeasesNumber = @WatchdogNumber + SET @DesiredLeasesNumber = @DesiredLeasesNumber - 1; + UPDATE dbo.WatchdogLeases + SET LeaseHolder = @Worker, + LeaseEndTime = @LeaseEndTime, + LeaseRequestor = '', + @PreviousLeaseHolder = LeaseHolder + WHERE Watchdog = @Watchdog + AND NOT (LeaseRequestor <> @Worker + AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec) + AND (LeaseHolder = @Worker + AND (LeaseEndTime > @Now + OR @WorkerIsRunning = 1) + OR LeaseEndTime < @Now + AND (@DesiredLeasesNumber > @MyLeasesNumber + OR @OtherValidRequestsOrLeasesNumber < @WatchdogNumber)); + IF @@rowcount > 0 + BEGIN + SET @IsAcquired = 1; + SET @msg = 'Lease holder changed from [' + isnull(@PreviousLeaseHolder, '') + '] to [' + @Worker + ']'; + IF @PreviousLeaseHolder <> @Worker + EXECUTE dbo.LogEvent @Process = 'AcquireWatchdogLease', @Status = 'Info', @Mode = @Mode, @Text = @msg; + END + ELSE + IF @AllowRebalance = 1 + BEGIN + SET @CurrentLeaseHolder = (SELECT LeaseHolder + FROM dbo.WatchdogLeases + WHERE Watchdog = @Watchdog); + UPDATE dbo.WatchdogLeases + SET LeaseRequestTime = @Now + WHERE Watchdog = @Watchdog + AND LeaseRequestor = @Worker + AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec; + IF @DesiredLeasesNumber > @MyValidRequestsOrLeasesNumber + BEGIN + UPDATE A + SET LeaseRequestor = @Worker, + LeaseRequestTime = @Now + FROM dbo.WatchdogLeases AS A + WHERE Watchdog = @Watchdog + AND NOT (LeaseRequestor <> @Worker + AND datediff(second, LeaseRequestTime, @Now) < @LeasePeriodSec) + AND @NotLeasedWatchdogNumber = 0 + AND (SELECT count(*) + FROM dbo.WatchdogLeases AS B + WHERE B.LeaseHolder = A.LeaseHolder + AND datediff(second, B.LeaseEndTime, @Now) < @LeasePeriodSec) > @DesiredLeasesNumber; + SET @RowsInt = @@rowcount; + SET @msg = '@DesiredLeasesNumber=[' + CONVERT (VARCHAR (10), @DesiredLeasesNumber) + '] > @MyValidRequestsOrLeasesNumber=[' + CONVERT (VARCHAR (10), @MyValidRequestsOrLeasesNumber) + ']'; + EXECUTE dbo.LogEvent @Process = 'AcquireWatchdogLease', @Status = 'Info', @Mode = @Mode, @Rows = @RowsInt, @Text = @msg; + END + END + SET @Mode = CONVERT (VARCHAR (100), @Mode + ' A=' + CONVERT (VARCHAR (1), @IsAcquired)); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = 'AcquireWatchdogLease', @Status = 'Error', @Mode = @Mode; + THROW; +END CATCH + +GO +CREATE OR ALTER PROCEDURE dbo.AddPartitionOnResourceChanges +@partitionBoundary DATETIME2 (7) OUTPUT +AS +BEGIN + SET XACT_ABORT ON; + BEGIN TRANSACTION; + DECLARE @rightPartitionBoundary AS DATETIME2 (7) = CAST ((SELECT TOP (1) value + FROM sys.partition_range_values AS prv + INNER JOIN + sys.partition_functions AS pf + ON pf.function_id = prv.function_id + WHERE pf.name = N'PartitionFunction_ResourceChangeData_Timestamp' + ORDER BY prv.boundary_id DESC) AS DATETIME2 (7)); + DECLARE @timestamp AS DATETIME2 (7) = DATEADD(hour, DATEDIFF(hour, 0, sysutcdatetime()), 0); + IF (@rightPartitionBoundary < @timestamp) + BEGIN + SET @rightPartitionBoundary = @timestamp; + END + SET @rightPartitionBoundary = DATEADD(hour, 1, @rightPartitionBoundary); + ALTER PARTITION SCHEME PartitionScheme_ResourceChangeData_Timestamp NEXT USED [Primary]; + ALTER PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp( ) + SPLIT RANGE (@rightPartitionBoundary); + SET @partitionBoundary = @rightPartitionBoundary; + COMMIT TRANSACTION; +END + +GO +CREATE PROCEDURE dbo.ArchiveJobs +@QueueType TINYINT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'ArchiveJobs', @Mode AS VARCHAR (100) = '', @st AS DATETIME = getUTCdate(), @Rows AS INT = 0, @PartitionId AS TINYINT, @MaxPartitions AS TINYINT = 16, @LookedAtPartitions AS TINYINT = 0, @InflightRows AS INT = 0, @Lock AS VARCHAR (100) = 'DequeueJob_' + CONVERT (VARCHAR, @QueueType); +BEGIN TRY + SET @PartitionId = @MaxPartitions * rand(); + BEGIN TRANSACTION; + EXECUTE sp_getapplock @Lock, 'Exclusive'; + WHILE @LookedAtPartitions <= @MaxPartitions + BEGIN + SET @InflightRows += (SELECT count(*) + FROM dbo.JobQueue + WHERE PartitionId = @PartitionId + AND QueueType = @QueueType + AND Status IN (0, 1)); + SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; + SET @LookedAtPartitions = @LookedAtPartitions + 1; + END + IF @InflightRows = 0 + BEGIN + SET @LookedAtPartitions = 0; + WHILE @LookedAtPartitions <= @MaxPartitions + BEGIN + UPDATE dbo.JobQueue + SET Status = 5 + WHERE PartitionId = @PartitionId + AND QueueType = @QueueType + AND Status IN (2, 3, 4); + SET @Rows += @@rowcount; + SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; + SET @LookedAtPartitions = @LookedAtPartitions + 1; + END + END + COMMIT TRANSACTION; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.BatchDeleteResourceParams +@tableName NVARCHAR (128), @resourceTypeId SMALLINT, @startResourceSurrogateId BIGINT, @endResourceSurrogateId BIGINT, @batchSize INT +AS +SET XACT_ABORT ON; +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DECLARE @Sql AS NVARCHAR (MAX); +DECLARE @ParmDefinition AS NVARCHAR (512); +IF OBJECT_ID(@tableName) IS NOT NULL + BEGIN + SET @sql = N'DELETE TOP(@BatchSizeParam) FROM ' + @tableName + N' WITH (TABLOCK) WHERE ResourceTypeId = @ResourceTypeIdParam AND ResourceSurrogateId >= @StartResourceSurrogateIdParam AND ResourceSurrogateId < @EndResourceSurrogateIdParam'; + SET @parmDefinition = N'@BatchSizeParam int, @ResourceTypeIdParam smallint, @StartResourceSurrogateIdParam bigint, @EndResourceSurrogateIdParam bigint'; + EXECUTE sp_executesql @sql, @parmDefinition, @BatchSizeParam = @batchSize, @ResourceTypeIdParam = @resourceTypeId, @StartResourceSurrogateIdParam = @startResourceSurrogateId, @EndResourceSurrogateIdParam = @endResourceSurrogateId; + END +COMMIT TRANSACTION; +RETURN @@rowcount; + +GO +CREATE PROCEDURE dbo.BatchDeleteResources +@resourceTypeId SMALLINT, @startResourceSurrogateId BIGINT, @endResourceSurrogateId BIGINT, @batchSize INT +AS +SET XACT_ABORT ON; +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DELETE TOP (@batchSize) + dbo.Resource WITH (TABLOCK) +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId >= @startResourceSurrogateId + AND ResourceSurrogateId < @endResourceSurrogateId; +COMMIT TRANSACTION; +RETURN @@rowcount; + +GO +CREATE PROCEDURE dbo.BatchDeleteResourceWriteClaims +@startResourceSurrogateId BIGINT, @endResourceSurrogateId BIGINT, @batchSize INT +AS +SET XACT_ABORT ON; +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DELETE TOP (@batchSize) + dbo.ResourceWriteClaim WITH (TABLOCK) +WHERE ResourceSurrogateId >= @startResourceSurrogateId + AND ResourceSurrogateId < @endResourceSurrogateId; +COMMIT TRANSACTION; +RETURN @@rowcount; + +GO +CREATE PROCEDURE dbo.BulkMergeResource +@resources dbo.BulkImportResourceType_1 READONLY +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +MERGE INTO [dbo].[Resource] WITH (ROWLOCK, INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) + AS target +USING @resources AS source ON source.[ResourceTypeId] = target.[ResourceTypeId] + AND source.[ResourceId] = target.[ResourceId] + AND source.[Version] = target.[Version] +WHEN NOT MATCHED BY TARGET THEN INSERT ([ResourceTypeId], [ResourceId], [Version], [IsHistory], [ResourceSurrogateId], [IsDeleted], [RequestMethod], [RawResource], [IsRawResourceMetaSet], [SearchParamHash]) VALUES ([ResourceTypeId], [ResourceId], [Version], [IsHistory], [ResourceSurrogateId], [IsDeleted], [RequestMethod], [RawResource], [IsRawResourceMetaSet], [SearchParamHash]) OUTPUT Inserted.[ResourceSurrogateId]; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.BulkReindexResources_2 +@resourcesToReindex dbo.BulkReindexResourceTableType_1 READONLY, @resourceWriteClaims dbo.BulkResourceWriteClaimTableType_1 READONLY, @compartmentAssignments dbo.BulkCompartmentAssignmentTableType_1 READONLY, @referenceSearchParams dbo.BulkReferenceSearchParamTableType_1 READONLY, @tokenSearchParams dbo.BulkTokenSearchParamTableType_2 READONLY, @tokenTextSearchParams dbo.BulkTokenTextTableType_1 READONLY, @stringSearchParams dbo.BulkStringSearchParamTableType_2 READONLY, @numberSearchParams dbo.BulkNumberSearchParamTableType_1 READONLY, @quantitySearchParams dbo.BulkQuantitySearchParamTableType_1 READONLY, @uriSearchParams dbo.BulkUriSearchParamTableType_1 READONLY, @dateTimeSearchParms dbo.BulkDateTimeSearchParamTableType_2 READONLY, @referenceTokenCompositeSearchParams dbo.BulkReferenceTokenCompositeSearchParamTableType_2 READONLY, @tokenTokenCompositeSearchParams dbo.BulkTokenTokenCompositeSearchParamTableType_2 READONLY, @tokenDateTimeCompositeSearchParams dbo.BulkTokenDateTimeCompositeSearchParamTableType_2 READONLY, @tokenQuantityCompositeSearchParams dbo.BulkTokenQuantityCompositeSearchParamTableType_2 READONLY, @tokenStringCompositeSearchParams dbo.BulkTokenStringCompositeSearchParamTableType_2 READONLY, @tokenNumberNumberCompositeSearchParams dbo.BulkTokenNumberNumberCompositeSearchParamTableType_2 READONLY +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @computedValues TABLE ( + Offset INT NOT NULL, + ResourceTypeId SMALLINT NOT NULL, + VersionProvided BIGINT NULL, + SearchParamHash VARCHAR (64) NOT NULL, + ResourceSurrogateId BIGINT NULL, + VersionInDatabase BIGINT NULL); +INSERT INTO @computedValues +SELECT resourceToReindex.Offset, + resourceToReindex.ResourceTypeId, + resourceToReindex.ETag, + resourceToReindex.SearchParamHash, + resourceInDB.ResourceSurrogateId, + resourceInDB.Version +FROM @resourcesToReindex AS resourceToReindex + LEFT OUTER JOIN + dbo.Resource AS resourceInDB WITH (UPDLOCK, INDEX (IX_Resource_ResourceTypeId_ResourceId)) + ON resourceInDB.ResourceTypeId = resourceToReindex.ResourceTypeId + AND resourceInDB.ResourceId = resourceToReindex.ResourceId + AND resourceInDB.IsHistory = 0; +DECLARE @versionDiff AS INT; +SET @versionDiff = (SELECT COUNT(*) + FROM @computedValues + WHERE VersionProvided IS NOT NULL + AND VersionProvided <> VersionInDatabase); +IF (@versionDiff > 0) + BEGIN + DELETE @computedValues + WHERE VersionProvided IS NOT NULL + AND VersionProvided <> VersionInDatabase; + END +UPDATE resourceInDB +SET resourceInDB.SearchParamHash = resourceToReindex.SearchParamHash +FROM @computedValues AS resourceToReindex + INNER JOIN + dbo.Resource AS resourceInDB + ON resourceInDB.ResourceTypeId = resourceToReindex.ResourceTypeId + AND resourceInDB.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.ResourceWriteClaim AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.CompartmentAssignment AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.ReferenceSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenText AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.StringSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.UriSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.NumberSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.QuantitySearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.DateTimeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.ReferenceTokenCompositeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenTokenCompositeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenDateTimeCompositeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenQuantityCompositeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenStringCompositeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +DELETE searchIndex +FROM dbo.TokenNumberNumberCompositeSearchParam AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.ResourceTypeId = resourceToReindex.ResourceTypeId + AND searchIndex.ResourceSurrogateId = resourceToReindex.ResourceSurrogateId; +INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) +SELECT DISTINCT resourceToReindex.ResourceSurrogateId, + searchIndex.ClaimTypeId, + searchIndex.ClaimValue +FROM @resourceWriteClaims AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.CompartmentAssignment (ResourceTypeId, ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.CompartmentTypeId, + searchIndex.ReferenceResourceId, + 0 +FROM @compartmentAssignments AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.BaseUri, + searchIndex.ReferenceResourceTypeId, + searchIndex.ReferenceResourceId, + searchIndex.ReferenceResourceVersion, + 0 +FROM @referenceSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId, + searchIndex.Code, + searchIndex.CodeOverflow, + 0 +FROM @tokenSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.Text, + 0 +FROM @tokenTextSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsHistory, IsMin, IsMax) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.Text, + searchIndex.TextOverflow, + 0, + searchIndex.IsMin, + searchIndex.IsMax +FROM @stringSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.Uri, + 0 +FROM @uriSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SingleValue, + searchIndex.LowValue, + searchIndex.HighValue, + 0 +FROM @numberSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId, + searchIndex.QuantityCodeId, + searchIndex.SingleValue, + searchIndex.LowValue, + searchIndex.HighValue, + 0 +FROM @quantitySearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsHistory, IsMin, IsMax) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.StartDateTime, + searchIndex.EndDateTime, + searchIndex.IsLongerThanADay, + 0, + searchIndex.IsMin, + searchIndex.IsMax +FROM @dateTimeSearchParms AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.BaseUri1, + searchIndex.ReferenceResourceTypeId1, + searchIndex.ReferenceResourceId1, + searchIndex.ReferenceResourceVersion1, + searchIndex.SystemId2, + searchIndex.Code2, + searchIndex.CodeOverflow2, + 0 +FROM @referenceTokenCompositeSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId1, + searchIndex.Code1, + searchIndex.CodeOverflow1, + searchIndex.SystemId2, + searchIndex.Code2, + searchIndex.CodeOverflow2, + 0 +FROM @tokenTokenCompositeSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId1, + searchIndex.Code1, + searchIndex.CodeOverflow1, + searchIndex.StartDateTime2, + searchIndex.EndDateTime2, + searchIndex.IsLongerThanADay2, + 0 +FROM @tokenDateTimeCompositeSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId1, + searchIndex.Code1, + searchIndex.CodeOverflow1, + searchIndex.SingleValue2, + searchIndex.SystemId2, + searchIndex.QuantityCodeId2, + searchIndex.LowValue2, + searchIndex.HighValue2, + 0 +FROM @tokenQuantityCompositeSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId1, + searchIndex.Code1, + searchIndex.CodeOverflow1, + searchIndex.Text2, + searchIndex.TextOverflow2, + 0 +FROM @tokenStringCompositeSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange, IsHistory) +SELECT DISTINCT resourceToReindex.ResourceTypeId, + resourceToReindex.ResourceSurrogateId, + searchIndex.SearchParamId, + searchIndex.SystemId1, + searchIndex.Code1, + searchIndex.CodeOverflow1, + searchIndex.SingleValue2, + searchIndex.LowValue2, + searchIndex.HighValue2, + searchIndex.SingleValue3, + searchIndex.LowValue3, + searchIndex.HighValue3, + searchIndex.HasRange, + 0 +FROM @tokenNumberNumberCompositeSearchParams AS searchIndex + INNER JOIN + @computedValues AS resourceToReindex + ON searchIndex.Offset = resourceToReindex.Offset; +SELECT @versionDiff; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE [dbo].[CancelTask] +@taskId VARCHAR (64) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +IF NOT EXISTS (SELECT * + FROM [dbo].[TaskInfo] + WHERE TaskId = @taskId) + BEGIN + THROW 50404, 'Task not exist', 1; + END +UPDATE dbo.TaskInfo +SET IsCanceled = 1, + HeartbeatDateTime = @heartbeatDateTime +WHERE TaskId = @taskId; +SELECT TaskId, + QueueId, + Status, + TaskTypeId, + RunId, + IsCanceled, + RetryCount, + MaxRetryCount, + HeartbeatDateTime, + InputData, + TaskContext, + Result +FROM [dbo].[TaskInfo] +WHERE TaskId = @taskId; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.CaptureResourceChanges +@isDeleted BIT, @version INT, @resourceId VARCHAR (64), @resourceTypeId SMALLINT +AS +BEGIN + DECLARE @changeType AS SMALLINT; + IF (@isDeleted = 1) + BEGIN + SET @changeType = 2; + END + ELSE + BEGIN + IF (@version = 1) + BEGIN + SET @changeType = 0; + END + ELSE + BEGIN + SET @changeType = 1; + END + END + INSERT INTO dbo.ResourceChangeData (ResourceId, ResourceTypeId, ResourceVersion, ResourceChangeTypeId) + VALUES (@resourceId, @resourceTypeId, @version, @changeType); +END + +GO +CREATE PROCEDURE dbo.CaptureResourceIdsForChanges +@Resources dbo.ResourceList READONLY +AS +SET NOCOUNT ON; +INSERT INTO dbo.ResourceChangeData (ResourceId, ResourceTypeId, ResourceVersion, ResourceChangeTypeId) +SELECT ResourceId, + ResourceTypeId, + Version, + CASE WHEN IsDeleted = 1 THEN 2 WHEN Version > 1 THEN 1 ELSE 0 END +FROM @Resources +WHERE IsHistory = 0; + +GO +CREATE PROCEDURE dbo.CheckActiveReindexJobs +AS +SET NOCOUNT ON; +SELECT Id +FROM dbo.ReindexJob +WHERE Status = 'Running' + OR Status = 'Queued' + OR Status = 'Paused'; + +GO +CREATE PROCEDURE dbo.CleanupEventLog +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'CleanupEventLog', @Mode AS VARCHAR (100) = '', @MaxDeleteRows AS INT, @MaxAllowedRows AS BIGINT, @RetentionPeriodSecond AS INT, @DeletedRows AS INT, @TotalDeletedRows AS INT = 0, @TotalRows AS INT, @Now AS DATETIME = getUTCdate(); +EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; +BEGIN TRY + SET @MaxDeleteRows = (SELECT Number + FROM dbo.Parameters + WHERE Id = 'CleanupEventLog.DeleteBatchSize'); + IF @MaxDeleteRows IS NULL + RAISERROR ('Cannot get Parameter.CleanupEventLog.DeleteBatchSize', 18, 127); + SET @MaxAllowedRows = (SELECT Number + FROM dbo.Parameters + WHERE Id = 'CleanupEventLog.AllowedRows'); + IF @MaxAllowedRows IS NULL + RAISERROR ('Cannot get Parameter.CleanupEventLog.AllowedRows', 18, 127); + SET @RetentionPeriodSecond = (SELECT Number * 24 * 60 * 60 + FROM dbo.Parameters + WHERE Id = 'CleanupEventLog.RetentionPeriodDay'); + IF @RetentionPeriodSecond IS NULL + RAISERROR ('Cannot get Parameter.CleanupEventLog.RetentionPeriodDay', 18, 127); + SET @TotalRows = (SELECT sum(row_count) + FROM sys.dm_db_partition_stats + WHERE object_id = object_id('EventLog') + AND index_id IN (0, 1)); + SET @DeletedRows = 1; + WHILE @DeletedRows > 0 + AND EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = 'CleanupEventLog.IsEnabled' + AND Number = 1) + BEGIN + SET @DeletedRows = 0; + IF @TotalRows - @TotalDeletedRows > @MaxAllowedRows + BEGIN + DELETE TOP (@MaxDeleteRows) + dbo.EventLog WITH (PAGLOCK) + WHERE EventDate <= dateadd(second, -@RetentionPeriodSecond, @Now); + SET @DeletedRows = @@rowcount; + SET @TotalDeletedRows += @DeletedRows; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'EventLog', @Action = 'Delete', @Rows = @DeletedRows, @Text = @TotalDeletedRows; + END + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @Now; +END TRY +BEGIN CATCH + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.CompleteTask +@taskId VARCHAR (64), @taskResult VARCHAR (MAX), @runId VARCHAR (50) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +IF NOT EXISTS (SELECT * + FROM [dbo].[TaskInfo] + WHERE TaskId = @taskId + AND RunId = @runId) + BEGIN + THROW 50404, 'Task not exist or runid not match', 1; + END +UPDATE dbo.TaskInfo +SET Status = 3, + EndDateTime = SYSUTCDATETIME(), + Result = @taskResult +WHERE TaskId = @taskId; +COMMIT TRANSACTION; +EXECUTE dbo.GetTaskDetails @TaskId = @taskId; + +GO +CREATE OR ALTER PROCEDURE dbo.ConfigurePartitionOnResourceChanges +@numberOfFuturePartitionsToAdd INT +AS +BEGIN + SET XACT_ABORT ON; + BEGIN TRANSACTION; + DECLARE @partitionBoundary AS DATETIME2 (7) = DATEADD(hour, DATEDIFF(hour, 0, sysutcdatetime()), 0); + DECLARE @startingRightPartitionBoundary AS DATETIME2 (7) = CAST ((SELECT TOP (1) value + FROM sys.partition_range_values AS prv + INNER JOIN + sys.partition_functions AS pf + ON pf.function_id = prv.function_id + WHERE pf.name = N'PartitionFunction_ResourceChangeData_Timestamp' + ORDER BY prv.boundary_id DESC) AS DATETIME2 (7)); + DECLARE @numberOfPartitionsToAdd AS INT = @numberOfFuturePartitionsToAdd + 1; + WHILE @numberOfPartitionsToAdd > 0 + BEGIN + IF (@startingRightPartitionBoundary < @partitionBoundary) + BEGIN + ALTER PARTITION SCHEME PartitionScheme_ResourceChangeData_Timestamp NEXT USED [PRIMARY]; + ALTER PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp( ) + SPLIT RANGE (@partitionBoundary); + END + SET @partitionBoundary = DATEADD(hour, 1, @partitionBoundary); + SET @numberOfPartitionsToAdd -= 1; + END + COMMIT TRANSACTION; +END + +GO +CREATE PROCEDURE dbo.CreateExportJob +@id VARCHAR (64), @hash VARCHAR (64), @status VARCHAR (10), @rawJobRecord VARCHAR (MAX) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +INSERT INTO dbo.ExportJob (Id, Hash, Status, HeartbeatDateTime, RawJobRecord) +VALUES (@id, @hash, @status, @heartbeatDateTime, @rawJobRecord); +SELECT CAST (MIN_ACTIVE_ROWVERSION() AS INT); +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.CreateReindexJob +@id VARCHAR (64), @status VARCHAR (10), @rawJobRecord VARCHAR (MAX) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +INSERT INTO dbo.ReindexJob (Id, Status, HeartbeatDateTime, RawJobRecord) +VALUES (@id, @status, @heartbeatDateTime, @rawJobRecord); +SELECT CAST (MIN_ACTIVE_ROWVERSION() AS INT); +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE [dbo].[CreateTask_3] +@taskId VARCHAR (64), @queueId VARCHAR (64), @taskTypeId SMALLINT, @parentTaskId VARCHAR (64), @maxRetryCount SMALLINT=3, @inputData VARCHAR (MAX), @isUniqueTaskByType BIT +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +DECLARE @status AS SMALLINT = 1; +DECLARE @retryCount AS SMALLINT = 0; +DECLARE @isCanceled AS BIT = 0; +IF (@isUniqueTaskByType = 1) + BEGIN + IF EXISTS (SELECT * + FROM [dbo].[TaskInfo] + WHERE TaskId = @taskId + OR (TaskTypeId = @taskTypeId + AND Status <> 3)) + BEGIN + THROW 50409, 'Task already existed', 1; + END + END +ELSE + BEGIN + IF EXISTS (SELECT * + FROM [dbo].[TaskInfo] + WHERE TaskId = @taskId) + BEGIN + THROW 50409, 'Task already existed', 1; + END + END +INSERT INTO [dbo].[TaskInfo] (TaskId, QueueId, Status, TaskTypeId, IsCanceled, RetryCount, MaxRetryCount, HeartbeatDateTime, InputData, ParentTaskId) +VALUES (@taskId, @queueId, @status, @taskTypeId, @isCanceled, @retryCount, @maxRetryCount, @heartbeatDateTime, @inputData, @parentTaskId); +EXECUTE dbo.GetTaskDetails @TaskId = @taskId; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.Defrag +@TableName VARCHAR (100), @IndexName VARCHAR (200), @PartitionNumber INT, @IsPartitioned BIT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'Defrag', @Mode AS VARCHAR (200) = @TableName + '.' + @IndexName + '.' + CONVERT (VARCHAR, @PartitionNumber) + '.' + CONVERT (VARCHAR, @IsPartitioned), @st AS DATETIME = getUTCdate(), @SQL AS VARCHAR (3500), @msg AS VARCHAR (1000), @SizeBefore AS FLOAT, @SizeAfter AS FLOAT, @IndexId AS INT; +BEGIN TRY + SET @IndexId = (SELECT index_id + FROM sys.indexes + WHERE object_id = object_id(@TableName) + AND name = @IndexName); + SET @SizeBefore = (SELECT sum(reserved_page_count) + FROM sys.dm_db_partition_stats + WHERE object_id = object_id(@TableName) + AND index_id = @IndexId) * 8.0 / 1024 / 1024; + SET @msg = 'Size[GB] before=' + CONVERT (VARCHAR, @SizeBefore); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start', @Text = @msg; + SET @Sql = 'ALTER INDEX ' + quotename(@IndexName) + ' ON dbo.' + quotename(@TableName) + ' REORGANIZE' + CASE WHEN @IsPartitioned = 1 THEN ' PARTITION = ' + CONVERT (VARCHAR, @PartitionNumber) ELSE '' END; + BEGIN TRY + EXECUTE (@Sql); + SET @SizeAfter = (SELECT sum(reserved_page_count) + FROM sys.dm_db_partition_stats + WHERE object_id = object_id(@TableName) + AND index_id = @IndexId) * 8.0 / 1024 / 1024; + SET @msg = 'Size[GB] before=' + CONVERT (VARCHAR, @SizeBefore) + ', after=' + CONVERT (VARCHAR, @SizeAfter) + ', reduced by=' + CONVERT (VARCHAR, @SizeBefore - @SizeAfter); + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'End', @Mode = @Mode, @Action = 'Reorganize', @Start = @st, @Text = @msg; + END TRY + BEGIN CATCH + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'Error', @Mode = @Mode, @Action = 'Reorganize', @Start = @st, @ReRaisError = 0; + END CATCH +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.DefragChangeDatabaseSettings +@IsOn BIT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'DefragChangeDatabaseSettings', @Mode AS VARCHAR (200) = 'On=' + CONVERT (VARCHAR, @IsOn), @st AS DATETIME = getUTCdate(), @SQL AS VARCHAR (3500); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'Start', @Mode = @Mode; + SET @SQL = 'ALTER DATABASE CURRENT SET AUTO_UPDATE_STATISTICS ' + CASE WHEN @IsOn = 1 THEN 'ON' ELSE 'OFF' END; + EXECUTE (@SQL); + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'Run', @Mode = @Mode, @Text = @SQL; + SET @SQL = 'ALTER DATABASE CURRENT SET AUTO_CREATE_STATISTICS ' + CASE WHEN @IsOn = 1 THEN 'ON' ELSE 'OFF' END; + EXECUTE (@SQL); + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'End', @Mode = @Mode, @Start = @st, @Text = @SQL; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.DeleteHistory +@DeleteResources BIT=0, @Reset BIT=0, @DisableLogEvent BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'DeleteHistory', @Mode AS VARCHAR (100) = 'D=' + isnull(CONVERT (VARCHAR, @DeleteResources), 'NULL') + ' R=' + isnull(CONVERT (VARCHAR, @Reset), 'NULL'), @st AS DATETIME = getUTCdate(), @Id AS VARCHAR (100) = 'DeleteHistory.LastProcessed.TypeId.SurrogateId', @ResourceTypeId AS SMALLINT, @SurrogateId AS BIGINT, @RowsToProcess AS INT, @ProcessedResources AS INT = 0, @DeletedResources AS INT = 0, @DeletedSearchParams AS INT = 0, @ReportDate AS DATETIME = getUTCdate(); +BEGIN TRY + IF @DisableLogEvent = 0 + INSERT INTO dbo.Parameters (Id, Char) + SELECT @SP, + 'LogEvent'; + ELSE + DELETE dbo.Parameters + WHERE Id = @SP; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + INSERT INTO dbo.Parameters (Id, Char) + SELECT @Id, + '0.0' + WHERE NOT EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = @Id); + DECLARE @LastProcessed AS VARCHAR (100) = CASE WHEN @Reset = 0 THEN (SELECT Char + FROM dbo.Parameters + WHERE Id = @Id) ELSE '0.0' END; + DECLARE @Types TABLE ( + ResourceTypeId SMALLINT PRIMARY KEY, + Name VARCHAR (100)); + DECLARE @SurrogateIds TABLE ( + ResourceSurrogateId BIGINT PRIMARY KEY, + IsHistory BIT ); + INSERT INTO @Types + EXECUTE dbo.GetUsedResourceTypes ; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = '@Types', @Action = 'Insert', @Rows = @@rowcount; + SET @ResourceTypeId = substring(@LastProcessed, 1, charindex('.', @LastProcessed) - 1); + SET @SurrogateId = substring(@LastProcessed, charindex('.', @LastProcessed) + 1, 255); + DELETE @Types + WHERE ResourceTypeId < @ResourceTypeId; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = '@Types', @Action = 'Delete', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Types) + BEGIN + SET @ResourceTypeId = (SELECT TOP 1 ResourceTypeId + FROM @Types + ORDER BY ResourceTypeId); + SET @ProcessedResources = 0; + SET @DeletedResources = 0; + SET @DeletedSearchParams = 0; + SET @RowsToProcess = 1; + WHILE @RowsToProcess > 0 + BEGIN + DELETE @SurrogateIds; + INSERT INTO @SurrogateIds + SELECT TOP 10000 ResourceSurrogateId, + IsHistory + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId > @SurrogateId + ORDER BY ResourceSurrogateId; + SET @RowsToProcess = @@rowcount; + SET @ProcessedResources += @RowsToProcess; + IF @RowsToProcess > 0 + SET @SurrogateId = (SELECT max(ResourceSurrogateId) + FROM @SurrogateIds); + SET @LastProcessed = CONVERT (VARCHAR, @ResourceTypeId) + '.' + CONVERT (VARCHAR, @SurrogateId); + DELETE @SurrogateIds + WHERE IsHistory = 0; + IF EXISTS (SELECT * + FROM @SurrogateIds) + BEGIN + DELETE dbo.ResourceWriteClaim + WHERE ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.CompartmentAssignment + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.ReferenceSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenText + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.StringSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.UriSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.NumberSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.QuantitySearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.DateTimeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.ReferenceTokenCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenTokenCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenDateTimeCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenQuantityCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenStringCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + DELETE dbo.TokenNumberNumberCompositeSearchParam + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedSearchParams += @@rowcount; + IF @DeleteResources = 1 + BEGIN + DELETE dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId IN (SELECT ResourceSurrogateId + FROM @SurrogateIds); + SET @DeletedResources += @@rowcount; + END + END + UPDATE dbo.Parameters + SET Char = @LastProcessed + WHERE Id = @Id; + IF datediff(second, @ReportDate, getUTCdate()) > 60 + BEGIN + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'Resource', @Action = 'Select', @Rows = @ProcessedResources, @Text = @LastProcessed; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = '*SearchParam', @Action = 'Delete', @Rows = @DeletedSearchParams, @Text = @LastProcessed; + IF @DeleteResources = 1 + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'Resource', @Action = 'Delete', @Rows = @DeletedResources, @Text = @LastProcessed; + SET @ReportDate = getUTCdate(); + SET @ProcessedResources = 0; + SET @DeletedSearchParams = 0; + SET @DeletedResources = 0; + END + END + DELETE @Types + WHERE ResourceTypeId = @ResourceTypeId; + SET @SurrogateId = 0; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'Resource', @Action = 'Select', @Rows = @ProcessedResources, @Text = @LastProcessed; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = '*SearchParam', @Action = 'Delete', @Rows = @DeletedSearchParams, @Text = @LastProcessed; + IF @DeleteResources = 1 + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Run', @Target = 'Resource', @Action = 'Delete', @Rows = @DeletedResources, @Text = @LastProcessed; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.DequeueJob +@QueueType TINYINT, @Worker VARCHAR (100), @HeartbeatTimeoutSec INT, @InputJobId BIGINT=NULL, @CheckTimeoutJobs BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'DequeueJob', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' H=' + isnull(CONVERT (VARCHAR, @HeartbeatTimeoutSec), 'NULL') + ' W=' + isnull(@Worker, 'NULL') + ' IJ=' + isnull(CONVERT (VARCHAR, @InputJobId), 'NULL') + ' T=' + isnull(CONVERT (VARCHAR, @CheckTimeoutJobs), 'NULL'), @Rows AS INT = 0, @st AS DATETIME = getUTCdate(), @JobId AS BIGINT, @msg AS VARCHAR (100), @Lock AS VARCHAR (100), @PartitionId AS TINYINT, @MaxPartitions AS TINYINT = 16, @LookedAtPartitions AS TINYINT = 0; +BEGIN TRY + IF EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = 'DequeueJobStop' + AND Number = 1) + BEGIN + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = 0, @Text = 'Skipped'; + RETURN; + END + IF @InputJobId IS NULL + SET @PartitionId = @MaxPartitions * rand(); + ELSE + SET @PartitionId = @InputJobId % 16; + SET TRANSACTION ISOLATION LEVEL READ COMMITTED; + WHILE @InputJobId IS NULL + AND @JobId IS NULL + AND @LookedAtPartitions < @MaxPartitions + AND @CheckTimeoutJobs = 0 + BEGIN + SET @Lock = 'DequeueJob_' + CONVERT (VARCHAR, @QueueType) + '_' + CONVERT (VARCHAR, @PartitionId); + BEGIN TRANSACTION; + EXECUTE sp_getapplock @Lock, 'Exclusive'; + UPDATE T + SET StartDate = getUTCdate(), + HeartbeatDate = getUTCdate(), + Worker = @Worker, + Status = 1, + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()), + @JobId = T.JobId + FROM dbo.JobQueue AS T WITH (PAGLOCK) + INNER JOIN + (SELECT TOP 1 JobId + FROM dbo.JobQueue WITH (INDEX (IX_QueueType_PartitionId_Status_Priority)) + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND Status = 0 + ORDER BY Priority, JobId) AS S + ON QueueType = @QueueType + AND PartitionId = @PartitionId + AND T.JobId = S.JobId; + SET @Rows += @@rowcount; + COMMIT TRANSACTION; + IF @JobId IS NULL + BEGIN + SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; + SET @LookedAtPartitions = @LookedAtPartitions + 1; + END + END + SET @LookedAtPartitions = 0; + WHILE @InputJobId IS NULL + AND @JobId IS NULL + AND @LookedAtPartitions < @MaxPartitions + BEGIN + SET @Lock = 'DequeueStoreCopyWorkUnit_' + CONVERT (VARCHAR, @PartitionId); + BEGIN TRANSACTION; + EXECUTE sp_getapplock @Lock, 'Exclusive'; + UPDATE T + SET StartDate = getUTCdate(), + HeartbeatDate = getUTCdate(), + Worker = @Worker, + Status = CASE WHEN CancelRequested = 0 THEN 1 ELSE 4 END, + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()), + @JobId = CASE WHEN CancelRequested = 0 THEN T.JobId END, + Info = CONVERT (VARCHAR (1000), isnull(Info, '') + ' Prev: Worker=' + Worker + ' Start=' + CONVERT (VARCHAR, StartDate, 121)) + FROM dbo.JobQueue AS T WITH (PAGLOCK) + INNER JOIN + (SELECT TOP 1 JobId + FROM dbo.JobQueue WITH (INDEX (IX_QueueType_PartitionId_Status_Priority)) + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND Status = 1 + AND datediff(second, HeartbeatDate, getUTCdate()) > @HeartbeatTimeoutSec + ORDER BY Priority, JobId) AS S + ON QueueType = @QueueType + AND PartitionId = @PartitionId + AND T.JobId = S.JobId; + SET @Rows += @@rowcount; + COMMIT TRANSACTION; + IF @JobId IS NULL + BEGIN + SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; + SET @LookedAtPartitions = @LookedAtPartitions + 1; + END + END + IF @InputJobId IS NOT NULL + BEGIN + UPDATE dbo.JobQueue WITH (PAGLOCK) + SET StartDate = getUTCdate(), + HeartbeatDate = getUTCdate(), + Worker = @Worker, + Status = 1, + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()), + @JobId = JobId + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND Status = 0 + AND JobId = @InputJobId; + SET @Rows += @@rowcount; + IF @JobId IS NULL + BEGIN + UPDATE dbo.JobQueue WITH (PAGLOCK) + SET StartDate = getUTCdate(), + HeartbeatDate = getUTCdate(), + Worker = @Worker, + Status = 1, + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()), + @JobId = JobId + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND Status = 1 + AND JobId = @InputJobId + AND datediff(second, HeartbeatDate, getUTCdate()) > @HeartbeatTimeoutSec; + SET @Rows += @@rowcount; + END + END + IF @JobId IS NOT NULL + EXECUTE dbo.GetJobs @QueueType = @QueueType, @JobId = @JobId; + SET @msg = 'J=' + isnull(CONVERT (VARCHAR, @JobId), 'NULL') + ' P=' + CONVERT (VARCHAR, @PartitionId); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows, @Text = @msg; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.DisableIndex +@tableName NVARCHAR (128), @indexName NVARCHAR (128) +WITH EXECUTE AS 'dbo' +AS +DECLARE @errorTxt AS VARCHAR (1000), @sql AS NVARCHAR (1000), @isDisabled AS BIT; +IF object_id(@tableName) IS NULL + BEGIN + SET @errorTxt = @tableName + ' does not exist or you don''t have permissions.'; + RAISERROR (@errorTxt, 18, 127); + END +SET @isDisabled = (SELECT is_disabled + FROM sys.indexes + WHERE object_id = object_id(@tableName) + AND name = @indexName); +IF @isDisabled IS NULL + BEGIN + SET @errorTxt = @indexName + ' does not exist or you don''t have permissions.'; + RAISERROR (@errorTxt, 18, 127); + END +IF @isDisabled = 0 + BEGIN + SET @sql = N'ALTER INDEX ' + QUOTENAME(@indexName) + N' on ' + @tableName + ' Disable'; + EXECUTE sp_executesql @sql; + END + +GO +CREATE PROCEDURE dbo.DisableIndexes +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'DisableIndexes', @Mode AS VARCHAR (200) = '', @st AS DATETIME = getUTCdate(), @Tbl AS VARCHAR (100), @Ind AS VARCHAR (200), @Txt AS VARCHAR (4000); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + DECLARE @Tables TABLE ( + Tbl VARCHAR (100) PRIMARY KEY, + Supported BIT ); + INSERT INTO @Tables + EXECUTE dbo.GetPartitionedTables @IncludeNotDisabled = 1, @IncludeNotSupported = 0; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Tables', @Action = 'Insert', @Rows = @@rowcount; + DECLARE @Indexes TABLE ( + Tbl VARCHAR (100), + Ind VARCHAR (200), + TblId INT , + IndId INT PRIMARY KEY (Tbl, Ind)); + INSERT INTO @Indexes + SELECT Tbl, + I.Name, + TblId, + I.index_id + FROM (SELECT object_id(Tbl) AS TblId, + Tbl + FROM @Tables) AS O + INNER JOIN + sys.indexes AS I + ON I.object_id = TblId; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Insert', @Rows = @@rowcount; + INSERT INTO dbo.IndexProperties (TableName, IndexName, PropertyName, PropertyValue) + SELECT Tbl, + Ind, + 'DATA_COMPRESSION', + data_comp + FROM (SELECT Tbl, + Ind, + isnull((SELECT TOP 1 CASE WHEN data_compression_desc = 'PAGE' THEN 'PAGE' END + FROM sys.partitions + WHERE object_id = TblId + AND index_id = IndId), 'NONE') AS data_comp + FROM @Indexes) AS A + WHERE NOT EXISTS (SELECT * + FROM dbo.IndexProperties + WHERE TableName = Tbl + AND IndexName = Ind); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = 'IndexProperties', @Action = 'Insert', @Rows = @@rowcount; + DELETE @Indexes + WHERE Tbl = 'Resource' + OR IndId = 1; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Delete', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Indexes) + BEGIN + SELECT TOP 1 @Tbl = Tbl, + @Ind = Ind + FROM @Indexes; + SET @Txt = 'ALTER INDEX ' + @Ind + ' ON dbo.' + @Tbl + ' DISABLE'; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Ind, @Action = 'Disable', @Text = @Txt; + DELETE @Indexes + WHERE Tbl = @Tbl + AND Ind = @Ind; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.EnqueueJobs +@QueueType TINYINT, @Definitions StringList READONLY, @GroupId BIGINT=NULL, @ForceOneActiveJobGroup BIT=1, @IsCompleted BIT=NULL, @ReturnJobs BIT=1 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'EnqueueJobs', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' D=' + CONVERT (VARCHAR, (SELECT count(*) + FROM @Definitions)) + ' G=' + isnull(CONVERT (VARCHAR, @GroupId), 'NULL') + ' F=' + isnull(CONVERT (VARCHAR, @ForceOneActiveJobGroup), 'NULL') + ' C=' + isnull(CONVERT (VARCHAR, @IsCompleted), 'NULL'), @st AS DATETIME = getUTCdate(), @Lock AS VARCHAR (100) = 'EnqueueJobs_' + CONVERT (VARCHAR, @QueueType), @MaxJobId AS BIGINT, @Rows AS INT, @msg AS VARCHAR (1000), @JobIds AS BigintList, @InputRows AS INT; +BEGIN TRY + DECLARE @Input TABLE ( + DefinitionHash VARBINARY (20) PRIMARY KEY, + Definition VARCHAR (MAX) ); + INSERT INTO @Input + SELECT hashbytes('SHA1', String) AS DefinitionHash, + String AS Definition + FROM @Definitions; + SET @InputRows = @@rowcount; + INSERT INTO @JobIds + SELECT JobId + FROM @Input AS A + INNER JOIN + dbo.JobQueue AS B + ON B.QueueType = @QueueType + AND B.DefinitionHash = A.DefinitionHash + AND B.Status <> 5; + IF @@rowcount < @InputRows + BEGIN + BEGIN TRANSACTION; + EXECUTE sp_getapplock @Lock, 'Exclusive'; + IF @ForceOneActiveJobGroup = 1 + AND EXISTS (SELECT * + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND Status IN (0, 1) + AND (@GroupId IS NULL + OR GroupId <> @GroupId)) + RAISERROR ('There are other active job groups', 18, 127); + SET @MaxJobId = isnull((SELECT TOP 1 JobId + FROM dbo.JobQueue + WHERE QueueType = @QueueType + ORDER BY JobId DESC), 0); + INSERT INTO dbo.JobQueue (QueueType, GroupId, JobId, Definition, DefinitionHash, Status) + OUTPUT inserted.JobId INTO @JobIds + SELECT @QueueType, + isnull(@GroupId, @MaxJobId + 1) AS GroupId, + JobId, + Definition, + DefinitionHash, + CASE WHEN @IsCompleted = 1 THEN 2 ELSE 0 END AS Status + FROM (SELECT @MaxJobId + row_number() OVER (ORDER BY Dummy) AS JobId, + * + FROM (SELECT *, + 0 AS Dummy + FROM @Input) AS A) AS A + WHERE NOT EXISTS (SELECT * + FROM dbo.JobQueue AS B WITH (INDEX (IX_QueueType_DefinitionHash)) + WHERE B.QueueType = @QueueType + AND B.DefinitionHash = A.DefinitionHash + AND B.Status <> 5); + SET @Rows = @@rowcount; + COMMIT TRANSACTION; + END + IF @ReturnJobs = 1 + EXECUTE dbo.GetJobs @QueueType = @QueueType, @JobIds = @JobIds; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.ExecuteCommandForRebuildIndexes +@Tbl VARCHAR (100), @Ind VARCHAR (1000), @Cmd VARCHAR (MAX) +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'ExecuteCommandForRebuildIndexes', @Mode AS VARCHAR (200) = 'Tbl=' + isnull(@Tbl, 'NULL'), @st AS DATETIME, @Retries AS INT = 0, @Action AS VARCHAR (100), @msg AS VARCHAR (1000); +RetryOnTempdbError: +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start', @Text = @Cmd; + SET @st = getUTCdate(); + IF @Tbl IS NULL + RAISERROR ('@Tbl IS NULL', 18, 127); + IF @Cmd IS NULL + RAISERROR ('@Cmd IS NULL', 18, 127); + SET @Action = CASE WHEN @Cmd LIKE 'UPDATE STAT%' THEN 'Update statistics' WHEN @Cmd LIKE 'CREATE%INDEX%' THEN 'Create Index' WHEN @Cmd LIKE 'ALTER%INDEX%REBUILD%' THEN 'Rebuild Index' WHEN @Cmd LIKE 'ALTER%TABLE%ADD%' THEN 'Add Constraint' END; + IF @Action IS NULL + BEGIN + SET @msg = 'Not supported command = ' + CONVERT (VARCHAR (900), @Cmd); + RAISERROR (@msg, 18, 127); + END + IF @Action = 'Create Index' + WAITFOR DELAY '00:00:05'; + EXECUTE (@Cmd); + SELECT @Ind; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Action = @Action, @Status = 'End', @Start = @st, @Text = @Cmd; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + IF error_number() = 40544 + BEGIN + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st, @Retry = @Retries; + SET @Retries = @Retries + 1; + IF @Tbl = 'TokenText_96' + WAITFOR DELAY '01:00:00'; + ELSE + WAITFOR DELAY '00:10:00'; + GOTO RetryOnTempdbError; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE OR ALTER PROCEDURE dbo.FetchEventAgentCheckpoint +@CheckpointId VARCHAR (64) +AS +BEGIN + SELECT TOP (1) CheckpointId, + LastProcessedDateTime, + LastProcessedIdentifier + FROM dbo.EventAgentCheckpoint + WHERE CheckpointId = @CheckpointId; +END + +GO +CREATE PROCEDURE dbo.FetchResourceChanges_3 +@startId BIGINT, @lastProcessedUtcDateTime DATETIME2 (7), @pageSize SMALLINT +AS +BEGIN + SET NOCOUNT ON; + DECLARE @precedingPartitionBoundary AS DATETIME2 (7) = (SELECT TOP (1) CAST (prv.value AS DATETIME2 (7)) AS value + FROM sys.partition_range_values AS prv WITH (NOLOCK) + INNER JOIN + sys.partition_functions AS pf WITH (NOLOCK) + ON pf.function_id = prv.function_id + WHERE pf.name = N'PartitionFunction_ResourceChangeData_Timestamp' + AND SQL_VARIANT_PROPERTY(prv.Value, 'BaseType') = 'datetime2' + AND CAST (prv.value AS DATETIME2 (7)) < DATEADD(HOUR, DATEDIFF(HOUR, 0, @lastProcessedUtcDateTime), 0) + ORDER BY prv.boundary_id DESC); + IF (@precedingPartitionBoundary IS NULL) + BEGIN + SET @precedingPartitionBoundary = CONVERT (DATETIME2 (7), N'1970-01-01T00:00:00.0000000'); + END + DECLARE @endDateTimeToFilter AS DATETIME2 (7) = DATEADD(HOUR, 1, SYSUTCDATETIME()); + WITH PartitionBoundaries + AS (SELECT CAST (prv.value AS DATETIME2 (7)) AS PartitionBoundary + FROM sys.partition_range_values AS prv WITH (NOLOCK) + INNER JOIN + sys.partition_functions AS pf WITH (NOLOCK) + ON pf.function_id = prv.function_id + WHERE pf.name = N'PartitionFunction_ResourceChangeData_Timestamp' + AND SQL_VARIANT_PROPERTY(prv.Value, 'BaseType') = 'datetime2' + AND CAST (prv.value AS DATETIME2 (7)) BETWEEN @precedingPartitionBoundary AND @endDateTimeToFilter) + SELECT TOP (@pageSize) Id, + Timestamp, + ResourceId, + ResourceTypeId, + ResourceVersion, + ResourceChangeTypeId + FROM PartitionBoundaries AS p CROSS APPLY (SELECT TOP (@pageSize) Id, + Timestamp, + ResourceId, + ResourceTypeId, + ResourceVersion, + ResourceChangeTypeId + FROM dbo.ResourceChangeData WITH (TABLOCK, HOLDLOCK) + WHERE Id >= @startId + AND $PARTITION.PartitionFunction_ResourceChangeData_Timestamp (Timestamp) = $PARTITION.PartitionFunction_ResourceChangeData_Timestamp (p.PartitionBoundary) + ORDER BY Id ASC) AS rcd + ORDER BY rcd.Id ASC; +END + +GO +CREATE PROCEDURE dbo.GetActiveJobs +@QueueType TINYINT, @GroupId BIGINT=NULL +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetActiveJobs', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' G=' + isnull(CONVERT (VARCHAR, @GroupId), 'NULL'), @st AS DATETIME = getUTCdate(), @JobIds AS BigintList, @PartitionId AS TINYINT, @MaxPartitions AS TINYINT = 16, @LookedAtPartitions AS TINYINT = 0, @Rows AS INT = 0; +BEGIN TRY + SET @PartitionId = @MaxPartitions * rand(); + WHILE @LookedAtPartitions < @MaxPartitions + BEGIN + IF @GroupId IS NULL + INSERT INTO @JobIds + SELECT JobId + FROM dbo.JobQueue + WHERE PartitionId = @PartitionId + AND QueueType = @QueueType + AND Status IN (0, 1); + ELSE + INSERT INTO @JobIds + SELECT JobId + FROM dbo.JobQueue + WHERE PartitionId = @PartitionId + AND QueueType = @QueueType + AND GroupId = @GroupId + AND Status IN (0, 1); + SET @Rows += @@rowcount; + SET @PartitionId = CASE WHEN @PartitionId = 15 THEN 0 ELSE @PartitionId + 1 END; + SET @LookedAtPartitions += 1; + END + IF @Rows > 0 + EXECUTE dbo.GetJobs @QueueType = @QueueType, @JobIds = @JobIds; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetCommandsForRebuildIndexes +@RebuildClustered BIT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetCommandsForRebuildIndexes', @Mode AS VARCHAR (200) = 'PS=PartitionScheme_ResourceTypeId RC=' + isnull(CONVERT (VARCHAR, @RebuildClustered), 'NULL'), @st AS DATETIME = getUTCdate(), @Tbl AS VARCHAR (100), @TblInt AS VARCHAR (100), @Ind AS VARCHAR (200), @IndId AS INT, @Supported AS BIT, @Txt AS VARCHAR (MAX), @Rows AS BIGINT, @Pages AS BIGINT, @ResourceTypeId AS SMALLINT, @IndexesCnt AS INT, @DataComp AS VARCHAR (100); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + DECLARE @Commands TABLE ( + Tbl VARCHAR (100), + Ind VARCHAR (200), + Txt VARCHAR (MAX), + Pages BIGINT ); + DECLARE @ResourceTypes TABLE ( + ResourceTypeId SMALLINT PRIMARY KEY); + DECLARE @Indexes TABLE ( + Ind VARCHAR (200) PRIMARY KEY, + IndId INT ); + DECLARE @Tables TABLE ( + name VARCHAR (100) PRIMARY KEY, + Supported BIT ); + INSERT INTO @Tables + EXECUTE dbo.GetPartitionedTables @IncludeNotDisabled = 1, @IncludeNotSupported = 1; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Tables', @Action = 'Insert', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Tables) + BEGIN + SELECT TOP 1 @Tbl = name, + @Supported = Supported + FROM @Tables + ORDER BY name; + IF @Supported = 0 + BEGIN + INSERT INTO @Commands + SELECT @Tbl, + name, + 'ALTER INDEX ' + name + ' ON dbo.' + @Tbl + ' REBUILD' + CASE WHEN (SELECT PropertyValue + FROM dbo.IndexProperties + WHERE TableName = @Tbl + AND IndexName = name) = 'PAGE' THEN ' PARTITION = ALL WITH (DATA_COMPRESSION = PAGE)' ELSE '' END, + CONVERT (BIGINT, 9e18) + FROM sys.indexes + WHERE object_id = object_id(@Tbl) + AND (is_disabled = 1 + AND index_id > 1 + AND @RebuildClustered = 0 + OR index_id = 1 + AND @RebuildClustered = 1); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Commands', @Action = 'Insert', @Rows = @@rowcount, @Text = 'Not supported tables with disabled indexes'; + END + ELSE + BEGIN + DELETE @ResourceTypes; + INSERT INTO @ResourceTypes + SELECT CONVERT (SMALLINT, substring(name, charindex('_', name) + 1, 6)) AS ResourceTypeId + FROM sys.sysobjects + WHERE name LIKE @Tbl + '[_]%'; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@ResourceTypes', @Action = 'Insert', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @ResourceTypes) + BEGIN + SET @ResourceTypeId = (SELECT TOP 1 ResourceTypeId + FROM @ResourceTypes + ORDER BY ResourceTypeId); + SET @TblInt = @Tbl + '_' + CONVERT (VARCHAR, @ResourceTypeId); + SET @Pages = (SELECT dpages + FROM sysindexes + WHERE id = object_id(@TblInt) + AND indid IN (0, 1)); + DELETE @Indexes; + INSERT INTO @Indexes + SELECT name, + index_id + FROM sys.indexes + WHERE object_id = object_id(@Tbl) + AND (index_id > 1 + AND @RebuildClustered = 0 + OR index_id = 1 + AND @RebuildClustered = 1); + SET @IndexesCnt = 0; + WHILE EXISTS (SELECT * + FROM @Indexes) + BEGIN + SELECT TOP 1 @Ind = Ind, + @IndId = IndId + FROM @Indexes + ORDER BY Ind; + IF @IndId = 1 + BEGIN + SET @Txt = 'ALTER INDEX ' + @Ind + ' ON dbo.' + @TblInt + ' REBUILD' + CASE WHEN (SELECT PropertyValue + FROM dbo.IndexProperties + WHERE TableName = @Tbl + AND IndexName = @Ind) = 'PAGE' THEN ' PARTITION = ALL WITH (DATA_COMPRESSION = PAGE)' ELSE '' END; + INSERT INTO @Commands + SELECT @TblInt, + @Ind, + @Txt, + @Pages; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Add command', @Rows = @@rowcount, @Text = @Txt; + END + ELSE + IF NOT EXISTS (SELECT * + FROM sys.indexes + WHERE object_id = object_id(@TblInt) + AND name = @Ind) + BEGIN + EXECUTE dbo.GetIndexCommands @Tbl = @Tbl, @Ind = @Ind, @AddPartClause = 0, @IncludeClustered = 0, @Txt = @Txt OUTPUT; + SET @Txt = replace(@Txt, '[' + @Tbl + ']', @TblInt); + IF @Txt IS NOT NULL + BEGIN + SET @IndexesCnt = @IndexesCnt + 1; + INSERT INTO @Commands + SELECT @TblInt, + @Ind, + @Txt, + @Pages; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Add command', @Rows = @@rowcount, @Text = @Txt; + END + END + DELETE @Indexes + WHERE Ind = @Ind; + END + IF @IndexesCnt > 1 + BEGIN + INSERT INTO @Commands + SELECT @TblInt, + 'UPDATE STAT', + 'UPDATE STATISTICS dbo.' + @TblInt, + @Pages; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Add command', @Rows = @@rowcount, @Text = 'Add stats update'; + END + DELETE @ResourceTypes + WHERE ResourceTypeId = @ResourceTypeId; + END + END + DELETE @Tables + WHERE name = @Tbl; + END + SELECT Tbl, + Ind, + Txt + FROM @Commands + ORDER BY Pages DESC, Tbl, CASE WHEN Txt LIKE 'UPDATE STAT%' THEN 0 ELSE 1 END; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Commands', @Action = 'Select', @Rows = @@rowcount; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetExportJobByHash +@hash VARCHAR (64) +AS +SET NOCOUNT ON; +SELECT TOP (1) RawJobRecord, + JobVersion +FROM dbo.ExportJob +WHERE Hash = @hash + AND (Status = 'Queued' + OR Status = 'Running') +ORDER BY HeartbeatDateTime ASC; + +GO +CREATE PROCEDURE dbo.GetExportJobById +@id VARCHAR (64) +AS +SET NOCOUNT ON; +SELECT RawJobRecord, + JobVersion +FROM dbo.ExportJob +WHERE Id = @id; + +GO +CREATE PROCEDURE [dbo].[GetImportProcessingTaskResult] +@queueId VARCHAR (64), @importTaskId VARCHAR (64) +AS +SET NOCOUNT ON; +SELECT Result +FROM [dbo].[TaskInfo] WITH (INDEX (IX_QueueId_ParentTaskId)) +WHERE ParentTaskId = @importTaskId + AND TaskTypeId = 1 + AND Status = 3; + +GO +CREATE PROCEDURE dbo.GetIndexCommands +@Tbl VARCHAR (100), @Ind VARCHAR (200), @AddPartClause BIT, @IncludeClustered BIT, @Txt VARCHAR (MAX)=NULL OUTPUT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetIndexCommands', @Mode AS VARCHAR (200) = 'Tbl=' + isnull(@Tbl, 'NULL') + ' Ind=' + isnull(@Ind, 'NULL'), @st AS DATETIME = getUTCdate(); +DECLARE @Indexes TABLE ( + Ind VARCHAR (200) PRIMARY KEY, + Txt VARCHAR (MAX)); +BEGIN TRY + IF @Tbl IS NULL + RAISERROR ('@Tbl IS NULL', 18, 127); + INSERT INTO @Indexes + SELECT Ind, + CASE WHEN is_primary_key = 1 THEN 'ALTER TABLE dbo.[' + Tbl + '] ADD PRIMARY KEY ' + CASE WHEN type = 1 THEN ' CLUSTERED' ELSE '' END ELSE 'CREATE' + CASE WHEN is_unique = 1 THEN ' UNIQUE' ELSE '' END + CASE WHEN type = 1 THEN ' CLUSTERED' ELSE '' END + ' INDEX ' + Ind + ' ON dbo.[' + Tbl + ']' END + ' (' + KeyCols + ')' + IncClause + CASE WHEN filter_def IS NOT NULL THEN ' WHERE ' + filter_def ELSE '' END + CASE WHEN data_comp IS NOT NULL THEN ' WITH (DATA_COMPRESSION = ' + data_comp + ')' ELSE '' END + CASE WHEN @AddPartClause = 1 THEN PartClause ELSE '' END + FROM (SELECT O.Name AS Tbl, + I.Name AS Ind, + isnull((SELECT TOP 1 CASE WHEN data_compression_desc = 'PAGE' THEN 'PAGE' END + FROM sys.partitions AS P + WHERE P.object_id = I.object_id + AND I.index_id = P.index_id), (SELECT NULLIF (PropertyValue, 'NONE') + FROM dbo.IndexProperties + WHERE TableName = O.Name + AND IndexName = I.Name + AND PropertyName = 'DATA_COMPRESSION')) AS data_comp, + replace(replace(replace(replace(I.filter_definition, '[', ''), ']', ''), '(', ''), ')', '') AS filter_def, + I.is_unique, + I.is_primary_key, + I.type, + KeyCols, + CASE WHEN IncCols IS NOT NULL THEN ' INCLUDE (' + IncCols + ')' ELSE '' END AS IncClause, + CASE WHEN EXISTS (SELECT * + FROM sys.partition_schemes AS S + WHERE S.data_space_id = I.data_space_id + AND name = 'PartitionScheme_ResourceTypeId') THEN ' ON PartitionScheme_ResourceTypeId (ResourceTypeId)' ELSE '' END AS PartClause + FROM sys.indexes AS I + INNER JOIN + sys.objects AS O + ON O.object_id = I.object_id CROSS APPLY (SELECT string_agg(CASE WHEN IC.key_ordinal > 0 + AND IC.is_included_column = 0 THEN C.name END, ',') WITHIN GROUP (ORDER BY key_ordinal) AS KeyCols, + string_agg(CASE WHEN IC.is_included_column = 1 THEN C.name END, ',') WITHIN GROUP (ORDER BY key_ordinal) AS IncCols + FROM sys.index_columns AS IC + INNER JOIN + sys.columns AS C + ON C.object_id = IC.object_id + AND C.column_id = IC.column_id + WHERE IC.object_id = I.object_id + AND IC.index_id = I.index_id + GROUP BY IC.object_id, IC.index_id) AS IC + WHERE O.name = @Tbl + AND (@Ind IS NULL + OR I.name = @Ind) + AND (@IncludeClustered = 1 + OR index_id > 1)) AS A; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Insert', @Rows = @@rowcount; + IF @Ind IS NULL + SELECT Ind, + Txt + FROM @Indexes; + ELSE + SET @Txt = (SELECT Txt + FROM @Indexes); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Text = @Txt; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetJobs +@QueueType TINYINT, @JobId BIGINT=NULL, @JobIds BigintList READONLY, @GroupId BIGINT=NULL, @ReturnDefinition BIT=1 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetJobs', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' J=' + isnull(CONVERT (VARCHAR, @JobId), 'NULL') + ' G=' + isnull(CONVERT (VARCHAR, @GroupId), 'NULL'), @st AS DATETIME = getUTCdate(), @PartitionId AS TINYINT = @JobId % 16; +BEGIN TRY + IF @JobId IS NULL + AND @GroupId IS NULL + AND NOT EXISTS (SELECT * + FROM @JobIds) + RAISERROR ('@JobId = NULL and @GroupId = NULL and @JobIds is empty', 18, 127); + IF @JobId IS NOT NULL + SELECT GroupId, + JobId, + CASE WHEN @ReturnDefinition = 1 THEN Definition ELSE NULL END AS Definition, + Version, + Status, + Priority, + Data, + Result, + CreateDate, + StartDate, + EndDate, + HeartbeatDate, + CancelRequested + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = isnull(@JobId, -1) + AND Status <> 5; + ELSE + IF @GroupId IS NOT NULL + SELECT GroupId, + JobId, + CASE WHEN @ReturnDefinition = 1 THEN Definition ELSE NULL END AS Definition, + Version, + Status, + Priority, + Data, + Result, + CreateDate, + StartDate, + EndDate, + HeartbeatDate, + CancelRequested + FROM dbo.JobQueue WITH (INDEX (IX_QueueType_GroupId)) + WHERE QueueType = @QueueType + AND GroupId = isnull(@GroupId, -1) + AND Status <> 5; + ELSE + SELECT GroupId, + JobId, + CASE WHEN @ReturnDefinition = 1 THEN Definition ELSE NULL END AS Definition, + Version, + Status, + Priority, + Data, + Result, + CreateDate, + StartDate, + EndDate, + HeartbeatDate, + CancelRequested + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND JobId IN (SELECT Id + FROM @JobIds) + AND PartitionId = JobId % 16 + AND Status <> 5; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetNextTask_3 +@queueId VARCHAR (64), @taskHeartbeatTimeoutThresholdInSeconds INT=600 +AS +SET NOCOUNT ON; +DECLARE @lock AS VARCHAR (200) = 'GetNextTask_Q=' + @queueId, @taskId AS VARCHAR (64) = NULL, @expirationDateTime AS DATETIME2 (7), @startDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +SET @expirationDateTime = DATEADD(second, -@taskHeartbeatTimeoutThresholdInSeconds, @startDateTime); +BEGIN TRY + BEGIN TRANSACTION; + EXECUTE sp_getapplock @lock, 'Exclusive'; + UPDATE T + SET Status = 2, + StartDateTime = @startDateTime, + HeartbeatDateTime = @startDateTime, + Worker = host_name(), + RunId = NEWID(), + @taskId = T.TaskId + FROM dbo.TaskInfo AS T WITH (PAGLOCK) + INNER JOIN + (SELECT TOP 1 TaskId + FROM dbo.TaskInfo WITH (INDEX (IX_QueueId_Status)) + WHERE QueueId = @queueId + AND Status = 1 + ORDER BY TaskId) AS S + ON T.QueueId = @queueId + AND T.TaskId = S.TaskId; + IF @taskId IS NULL + UPDATE T + SET StartDateTime = @startDateTime, + HeartbeatDateTime = @startDateTime, + Worker = host_name(), + RunId = NEWID(), + @taskId = T.TaskId, + RestartInfo = ISNULL(RestartInfo, '') + ' Prev: Worker=' + Worker + ' Start=' + CONVERT (VARCHAR, @startDateTime, 121) + FROM dbo.TaskInfo AS T WITH (PAGLOCK) + INNER JOIN + (SELECT TOP 1 TaskId + FROM dbo.TaskInfo WITH (INDEX (IX_QueueId_Status)) + WHERE QueueId = @queueId + AND Status = 2 + AND HeartbeatDateTime <= @expirationDateTime + ORDER BY TaskId) AS S + ON T.QueueId = @queueId + AND T.TaskId = S.TaskId; + COMMIT TRANSACTION; + EXECUTE dbo.GetTaskDetails @TaskId = @taskId; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK TRANSACTION THROW; +END CATCH + +GO +CREATE OR ALTER PROCEDURE dbo.GetNonCompletedJobCountOfSpecificQueueType +@queueType TINYINT +AS +BEGIN + SET NOCOUNT ON; + SELECT COUNT(*) + FROM dbo.JobQueue + WHERE QueueType = @queueType + AND (Status = 0 + OR Status = 1); +END + +GO +CREATE PROCEDURE dbo.GetPartitionedTables +@IncludeNotDisabled BIT, @IncludeNotSupported BIT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetPartitionedTables', @Mode AS VARCHAR (200) = 'PS=PartitionScheme_ResourceTypeId D=' + isnull(CONVERT (VARCHAR, @IncludeNotDisabled), 'NULL') + ' S=' + isnull(CONVERT (VARCHAR, @IncludeNotSupported), 'NULL'), @st AS DATETIME = getUTCdate(); +DECLARE @NotSupportedTables TABLE ( + id INT PRIMARY KEY); +BEGIN TRY + INSERT INTO @NotSupportedTables + SELECT DISTINCT O.object_id + FROM sys.indexes AS I + INNER JOIN + sys.objects AS O + ON O.object_id = I.object_id + WHERE O.type = 'u' + AND EXISTS (SELECT * + FROM sys.partition_schemes AS PS + WHERE PS.data_space_id = I.data_space_id + AND name = 'PartitionScheme_ResourceTypeId') + AND (NOT EXISTS (SELECT * + FROM sys.index_columns AS IC + INNER JOIN + sys.columns AS C + ON C.object_id = IC.object_id + AND C.column_id = IC.column_id + WHERE IC.object_id = I.object_id + AND IC.index_id = I.index_id + AND IC.key_ordinal > 0 + AND IC.is_included_column = 0 + AND C.name = 'ResourceTypeId') + OR EXISTS (SELECT * + FROM sys.indexes AS NSI + WHERE NSI.object_id = O.object_id + AND NOT EXISTS (SELECT * + FROM sys.partition_schemes AS PS + WHERE PS.data_space_id = NSI.data_space_id + AND name = 'PartitionScheme_ResourceTypeId'))); + SELECT CONVERT (VARCHAR (100), O.name), + CONVERT (BIT, CASE WHEN EXISTS (SELECT * + FROM @NotSupportedTables AS NSI + WHERE NSI.id = O.object_id) THEN 0 ELSE 1 END) + FROM sys.indexes AS I + INNER JOIN + sys.objects AS O + ON O.object_id = I.object_id + WHERE O.type = 'u' + AND I.index_id IN (0, 1) + AND EXISTS (SELECT * + FROM sys.partition_schemes AS PS + WHERE PS.data_space_id = I.data_space_id + AND name = 'PartitionScheme_ResourceTypeId') + AND EXISTS (SELECT * + FROM sys.index_columns AS IC + INNER JOIN + sys.columns AS C + ON C.object_id = I.object_id + AND C.column_id = IC.column_id + AND IC.is_included_column = 0 + AND C.name = 'ResourceTypeId') + AND (@IncludeNotSupported = 1 + OR NOT EXISTS (SELECT * + FROM @NotSupportedTables AS NSI + WHERE NSI.id = O.object_id)) + AND (@IncludeNotDisabled = 1 + OR EXISTS (SELECT * + FROM sys.indexes AS D + WHERE D.object_id = O.object_id + AND D.is_disabled = 1)) + ORDER BY 1; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetReindexJobById +@id VARCHAR (64) +AS +SET NOCOUNT ON; +SELECT RawJobRecord, + JobVersion +FROM dbo.ReindexJob +WHERE Id = @id; + +GO +CREATE PROCEDURE dbo.GetResources +@ResourceKeys dbo.ResourceKeyList READONLY +AS +SET NOCOUNT ON; +DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = 'GetResources', @InputRows AS INT, @DummyTop AS BIGINT = 9223372036854775807, @NotNullVersionExists AS BIT, @NullVersionExists AS BIT, @MinRT AS SMALLINT, @MaxRT AS SMALLINT; +SELECT @MinRT = min(ResourceTypeId), + @MaxRT = max(ResourceTypeId), + @InputRows = count(*), + @NotNullVersionExists = max(CASE WHEN Version IS NOT NULL THEN 1 ELSE 0 END), + @NullVersionExists = max(CASE WHEN Version IS NULL THEN 1 ELSE 0 END) +FROM @ResourceKeys; +DECLARE @Mode AS VARCHAR (100) = 'RT=[' + CONVERT (VARCHAR, @MinRT) + ',' + CONVERT (VARCHAR, @MaxRT) + '] Cnt=' + CONVERT (VARCHAR, @InputRows) + ' NNVE=' + CONVERT (VARCHAR, @NotNullVersionExists) + ' NVE=' + CONVERT (VARCHAR, @NullVersionExists); +BEGIN TRY + IF @NotNullVersionExists = 1 + IF @NullVersionExists = 0 + SELECT B.ResourceTypeId, + B.ResourceId, + ResourceSurrogateId, + B.Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceKeys) AS A + INNER JOIN + dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.Version = A.Version + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + ELSE + SELECT * + FROM (SELECT B.ResourceTypeId, + B.ResourceId, + ResourceSurrogateId, + B.Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceKeys + WHERE Version IS NOT NULL) AS A + INNER JOIN + dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.Version = A.Version + UNION ALL + SELECT B.ResourceTypeId, + B.ResourceId, + ResourceSurrogateId, + B.Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceKeys + WHERE Version IS NULL) AS A + INNER JOIN + dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId)) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + WHERE IsHistory = 0) AS A + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + ELSE + SELECT B.ResourceTypeId, + B.ResourceId, + ResourceSurrogateId, + B.Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceKeys) AS A + INNER JOIN + dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId)) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + WHERE IsHistory = 0 + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetResourcesByTransactionId +@TransactionId BIGINT, @IncludeHistory BIT=0, @ReturnResourceKeysOnly BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'T=' + CONVERT (VARCHAR, @TransactionId) + ' H=' + CONVERT (VARCHAR, @IncludeHistory), @st AS DATETIME = getUTCdate(), @DummyTop AS BIGINT = 9223372036854775807, @TypeId AS SMALLINT; +BEGIN TRY + DECLARE @Types TABLE ( + TypeId SMALLINT PRIMARY KEY, + Name VARCHAR (100)); + INSERT INTO @Types + EXECUTE dbo.GetUsedResourceTypes ; + DECLARE @Keys TABLE ( + TypeId SMALLINT, + SurrogateId BIGINT PRIMARY KEY (TypeId, SurrogateId)); + WHILE EXISTS (SELECT * + FROM @Types) + BEGIN + SET @TypeId = (SELECT TOP 1 TypeId + FROM @Types + ORDER BY TypeId); + INSERT INTO @Keys + SELECT @TypeId, + ResourceSurrogateId + FROM dbo.Resource + WHERE ResourceTypeId = @TypeId + AND TransactionId = @TransactionId; + DELETE @Types + WHERE TypeId = @TypeId; + END + IF @ReturnResourceKeysOnly = 0 + SELECT ResourceTypeId, + ResourceId, + ResourceSurrogateId, + Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash, + RequestMethod + FROM (SELECT TOP (@DummyTop) * + FROM @Keys) AS A + INNER JOIN + dbo.Resource AS B + ON ResourceTypeId = TypeId + AND ResourceSurrogateId = SurrogateId + WHERE IsHistory = 0 + OR @IncludeHistory = 1 + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + ELSE + SELECT ResourceTypeId, + ResourceId, + ResourceSurrogateId, + Version, + IsDeleted + FROM (SELECT TOP (@DummyTop) * + FROM @Keys) AS A + INNER JOIN + dbo.Resource AS B + ON ResourceTypeId = TypeId + AND ResourceSurrogateId = SurrogateId + WHERE IsHistory = 0 + OR @IncludeHistory = 1 + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetResourcesByTypeAndSurrogateIdRange +@ResourceTypeId SMALLINT, @StartId BIGINT, @EndId BIGINT, @GlobalEndId BIGINT=NULL, @IncludeHistory BIT=0, @IncludeDeleted BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetResourcesByTypeAndSurrogateIdRange', @Mode AS VARCHAR (100) = 'RT=' + isnull(CONVERT (VARCHAR, @ResourceTypeId), 'NULL') + ' S=' + isnull(CONVERT (VARCHAR, @StartId), 'NULL') + ' E=' + isnull(CONVERT (VARCHAR, @EndId), 'NULL') + ' GE=' + isnull(CONVERT (VARCHAR, @GlobalEndId), 'NULL') + ' HI=' + isnull(CONVERT (VARCHAR, @IncludeHistory), 'NULL') + ' DE' + isnull(CONVERT (VARCHAR, @IncludeDeleted), 'NULL'), @st AS DATETIME = getUTCdate(), @DummyTop AS BIGINT = 9223372036854775807; +BEGIN TRY + DECLARE @ResourceIds TABLE ( + ResourceId VARCHAR (64) COLLATE Latin1_General_100_CS_AS PRIMARY KEY); + DECLARE @SurrogateIds TABLE ( + MaxSurrogateId BIGINT PRIMARY KEY); + IF @GlobalEndId IS NOT NULL + AND @IncludeHistory = 0 + BEGIN + INSERT INTO @ResourceIds + SELECT DISTINCT ResourceId + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + AND IsHistory = 1 + AND (IsDeleted = 0 + OR @IncludeDeleted = 1) + OPTION (MAXDOP 1); + IF @@rowcount > 0 + INSERT INTO @SurrogateIds + SELECT ResourceSurrogateId + FROM (SELECT ResourceId, + ResourceSurrogateId, + row_number() OVER (PARTITION BY ResourceId ORDER BY ResourceSurrogateId DESC) AS RowId + FROM dbo.Resource WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceId IN (SELECT TOP (@DummyTop) ResourceId + FROM @ResourceIds) + AND ResourceSurrogateId BETWEEN @StartId AND @GlobalEndId) AS A + WHERE RowId = 1 + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + END + SELECT ResourceTypeId, + ResourceId, + Version, + IsDeleted, + ResourceSurrogateId, + RequestMethod, + CONVERT (BIT, 1) AS IsMatch, + CONVERT (BIT, 0) AS IsPartial, + IsRawResourceMetaSet, + SearchParamHash, + RawResource + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + AND (IsHistory = 0 + OR @IncludeHistory = 1) + AND (IsDeleted = 0 + OR @IncludeDeleted = 1) + UNION ALL + SELECT ResourceTypeId, + ResourceId, + Version, + IsDeleted, + ResourceSurrogateId, + RequestMethod, + CONVERT (BIT, 1) AS IsMatch, + CONVERT (BIT, 0) AS IsPartial, + IsRawResourceMetaSet, + SearchParamHash, + RawResource + FROM @SurrogateIds + INNER JOIN + dbo.Resource + ON ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId = MaxSurrogateId + WHERE IsHistory = 1 + AND (IsDeleted = 0 + OR @IncludeDeleted = 1) + OPTION (MAXDOP 1); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetResourceSurrogateIdRanges +@ResourceTypeId SMALLINT, @StartId BIGINT, @EndId BIGINT, @RangeSize INT, @NumberOfRanges INT=100, @Up BIT=1 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetResourceSurrogateIdRanges', @Mode AS VARCHAR (100) = 'RT=' + isnull(CONVERT (VARCHAR, @ResourceTypeId), 'NULL') + ' S=' + isnull(CONVERT (VARCHAR, @StartId), 'NULL') + ' E=' + isnull(CONVERT (VARCHAR, @EndId), 'NULL') + ' R=' + isnull(CONVERT (VARCHAR, @RangeSize), 'NULL') + ' UP=' + isnull(CONVERT (VARCHAR, @Up), 'NULL'), @st AS DATETIME = getUTCdate(); +BEGIN TRY + IF @Up = 1 + SELECT RangeId, + min(ResourceSurrogateId), + max(ResourceSurrogateId), + count(*) + FROM (SELECT isnull(CONVERT (INT, (row_number() OVER (ORDER BY ResourceSurrogateId) - 1) / @RangeSize), 0) AS RangeId, + ResourceSurrogateId + FROM (SELECT TOP (@RangeSize * @NumberOfRanges) ResourceSurrogateId + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId >= @StartId + AND ResourceSurrogateId <= @EndId + ORDER BY ResourceSurrogateId) AS A) AS A + GROUP BY RangeId + OPTION (MAXDOP 1); + ELSE + SELECT RangeId, + min(ResourceSurrogateId), + max(ResourceSurrogateId), + count(*) + FROM (SELECT isnull(CONVERT (INT, (row_number() OVER (ORDER BY ResourceSurrogateId) - 1) / @RangeSize), 0) AS RangeId, + ResourceSurrogateId + FROM (SELECT TOP (@RangeSize * @NumberOfRanges) ResourceSurrogateId + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId >= @StartId + AND ResourceSurrogateId <= @EndId + ORDER BY ResourceSurrogateId DESC) AS A) AS A + GROUP BY RangeId + OPTION (MAXDOP 1); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetResourceVersions +@ResourceDateKeys dbo.ResourceDateKeyList READONLY +AS +SET NOCOUNT ON; +DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = 'GetResourceVersions', @Mode AS VARCHAR (100) = 'Rows=' + CONVERT (VARCHAR, (SELECT count(*) + FROM @ResourceDateKeys)), @DummyTop AS BIGINT = 9223372036854775807; +BEGIN TRY + SELECT A.ResourceTypeId, + A.ResourceId, + A.ResourceSurrogateId, + CASE WHEN EXISTS (SELECT * + FROM dbo.Resource AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId) THEN 0 WHEN isnull(U.Version, 1) - isnull(L.Version, 0) > 1 THEN isnull(U.Version, 1) - 1 ELSE 0 END AS Version + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceDateKeys) AS A OUTER APPLY (SELECT TOP 1 * + FROM dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.ResourceSurrogateId < A.ResourceSurrogateId + ORDER BY B.ResourceSurrogateId DESC) AS L OUTER APPLY (SELECT TOP 1 * + FROM dbo.Resource AS B WITH (INDEX (IX_Resource_ResourceTypeId_ResourceId_Version)) + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.ResourceSurrogateId > A.ResourceSurrogateId + ORDER BY B.ResourceSurrogateId) AS U + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.GetSearchParamStatuses +AS +SET NOCOUNT ON; +SELECT SearchParamId, + Uri, + Status, + LastUpdated, + IsPartiallySupported +FROM dbo.SearchParam; + +GO +CREATE PROCEDURE [dbo].[GetTaskDetails] +@taskId VARCHAR (64) +AS +SET NOCOUNT ON; +SELECT TaskId, + QueueId, + Status, + TaskTypeId, + RunId, + IsCanceled, + RetryCount, + MaxRetryCount, + HeartbeatDateTime, + InputData, + TaskContext, + Result, + ParentTaskId +FROM [dbo].[TaskInfo] +WHERE TaskId = @taskId; + +GO +CREATE PROCEDURE dbo.GetTransactions +@StartNotInclusiveTranId BIGINT, @EndInclusiveTranId BIGINT, @EndDate DATETIME=NULL +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'ST=' + CONVERT (VARCHAR, @StartNotInclusiveTranId) + ' ET=' + CONVERT (VARCHAR, @EndInclusiveTranId) + ' ED=' + isnull(CONVERT (VARCHAR, @EndDate, 121), 'NULL'), @st AS DATETIME = getUTCdate(); +IF @EndDate IS NULL + SET @EndDate = getUTCdate(); +SELECT SurrogateIdRangeFirstValue, + VisibleDate, + InvisibleHistoryRemovedDate +FROM dbo.Transactions +WHERE SurrogateIdRangeFirstValue > @StartNotInclusiveTranId + AND SurrogateIdRangeFirstValue <= @EndInclusiveTranId + AND EndDate <= @EndDate +ORDER BY SurrogateIdRangeFirstValue; +EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; + +GO +CREATE PROCEDURE dbo.GetUsedResourceTypes +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'GetUsedResourceTypes', @Mode AS VARCHAR (100) = '', @st AS DATETIME = getUTCdate(); +BEGIN TRY + SELECT ResourceTypeId, + Name + FROM dbo.ResourceType AS A + WHERE EXISTS (SELECT * + FROM dbo.Resource AS B + WHERE B.ResourceTypeId = A.ResourceTypeId); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.HardDeleteResource +@ResourceTypeId SMALLINT, @ResourceId VARCHAR (64), @KeepCurrentVersion BIT, @IsResourceChangeCaptureEnabled BIT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (200) = 'RT=' + CONVERT (VARCHAR, @ResourceTypeId) + ' R=' + @ResourceId + ' V=' + CONVERT (VARCHAR, @KeepCurrentVersion) + ' CC=' + CONVERT (VARCHAR, @IsResourceChangeCaptureEnabled), @st AS DATETIME = getUTCdate(), @TransactionId AS BIGINT; +BEGIN TRY + IF @IsResourceChangeCaptureEnabled = 1 + EXECUTE dbo.MergeResourcesBeginTransaction @Count = 1, @TransactionId = @TransactionId OUTPUT; + IF @KeepCurrentVersion = 0 + BEGIN TRANSACTION; + DECLARE @SurrogateIds TABLE ( + ResourceSurrogateId BIGINT NOT NULL); + IF @IsResourceChangeCaptureEnabled = 1 + AND NOT EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = 'InvisibleHistory.IsEnabled' + AND Number = 0) + UPDATE dbo.Resource + SET IsDeleted = 1, + RawResource = 0xF, + SearchParamHash = NULL, + HistoryTransactionId = @TransactionId + OUTPUT deleted.ResourceSurrogateId INTO @SurrogateIds + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceId = @ResourceId + AND (@KeepCurrentVersion = 0 + OR IsHistory = 1) + AND RawResource <> 0xF; + ELSE + DELETE dbo.Resource + OUTPUT deleted.ResourceSurrogateId INTO @SurrogateIds + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceId = @ResourceId + AND (@KeepCurrentVersion = 0 + OR IsHistory = 1) + AND RawResource <> 0xF; + IF @KeepCurrentVersion = 0 + BEGIN + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.ResourceWriteClaim AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.ReferenceSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenText AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.StringSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.UriSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.NumberSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.QuantitySearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.DateTimeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.ReferenceTokenCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenTokenCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenDateTimeCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenQuantityCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenStringCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + DELETE B + FROM @SurrogateIds AS A + INNER LOOP JOIN + dbo.TokenNumberNumberCompositeSearchParam AS B WITH (INDEX (1), FORCESEEK, PAGLOCK) + ON B.ResourceTypeId = @ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + OPTION (MAXDOP 1); + END + IF @@trancount > 0 + COMMIT TRANSACTION; + IF @IsResourceChangeCaptureEnabled = 1 + EXECUTE dbo.MergeResourcesCommitTransaction @TransactionId; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.InitDefrag +@QueueType TINYINT, @GroupId BIGINT, @DefragItems INT=NULL OUTPUT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'InitDefrag', @st AS DATETIME = getUTCdate(), @ObjectId AS INT, @msg AS VARCHAR (1000), @Rows AS INT, @MinFragPct AS INT = isnull((SELECT Number + FROM dbo.Parameters + WHERE Id = 'Defrag.MinFragPct'), 10), @MinSizeGB AS FLOAT = isnull((SELECT Number + FROM dbo.Parameters + WHERE Id = 'Defrag.MinSizeGB'), 0.1), @DefinitionsSorted AS StringList; +DECLARE @Mode AS VARCHAR (200) = 'G=' + CONVERT (VARCHAR, @GroupId) + ' MF=' + CONVERT (VARCHAR, @MinFragPct) + ' MS=' + CONVERT (VARCHAR, @MinSizeGB); +DECLARE @Definitions AS TABLE ( + Def VARCHAR (900) PRIMARY KEY, + FragGB FLOAT ); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + SELECT * + INTO #filter + FROM (SELECT object_id, + sum(reserved_page_count * 8.0 / 1024 / 1024) AS ReservedGB + FROM sys.dm_db_partition_stats AS A + WHERE object_id IN (SELECT object_id + FROM sys.objects + WHERE type = 'U' + AND name NOT IN ('EventLog')) + GROUP BY object_id) AS A + WHERE ReservedGB > @MinSizeGB; + WHILE EXISTS (SELECT * + FROM #filter) + BEGIN + SET @ObjectId = (SELECT TOP 1 object_id + FROM #filter + ORDER BY ReservedGB DESC); + INSERT INTO @Definitions + SELECT object_name(@ObjectId) + ';' + I.name + ';' + CONVERT (VARCHAR, partition_number) + ';' + CONVERT (VARCHAR, CASE WHEN EXISTS (SELECT * + FROM sys.partition_schemes AS PS + WHERE PS.data_space_id = I.data_space_id) THEN 1 ELSE 0 END) + ';' + CONVERT (VARCHAR, (SELECT sum(reserved_page_count) + FROM sys.dm_db_partition_stats AS S + WHERE S.object_id = A.object_id + AND S.index_id = A.index_id + AND S.partition_number = A.partition_number) * 8.0 / 1024 / 1024), + FragGB + FROM (SELECT object_id, + index_id, + partition_number, + A.avg_fragmentation_in_percent * A.page_count * 8.0 / 1024 / 1024 / 100 AS FragGB + FROM sys.dm_db_index_physical_stats(db_id(), @ObjectId, NULL, NULL, 'LIMITED') AS A + WHERE index_id > 0 + AND avg_fragmentation_in_percent >= @MinFragPct + AND A.page_count > 500) AS A + INNER JOIN + sys.indexes AS I + ON I.object_id = A.object_id + AND I.index_id = A.index_id; + SET @Rows = @@rowcount; + SET @msg = object_name(@ObjectId); + EXECUTE dbo.LogEvent @Process = @SP, @Status = 'Run', @Mode = @Mode, @Target = '@Definitions', @Action = 'Insert', @Rows = @Rows, @Text = @msg; + DELETE #filter + WHERE object_id = @ObjectId; + END + INSERT INTO @DefinitionsSorted + SELECT Def + ';' + CONVERT (VARCHAR, FragGB) + FROM @Definitions + ORDER BY FragGB DESC; + SET @DefragItems = @@rowcount; + IF @DefragItems > 0 + EXECUTE dbo.EnqueueJobs @QueueType = @QueueType, @Definitions = @DefinitionsSorted, @GroupId = @GroupId, @ForceOneActiveJobGroup = 1; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.InitializeIndexProperties +AS +SET NOCOUNT ON; +INSERT INTO dbo.IndexProperties (TableName, IndexName, PropertyName, PropertyValue) +SELECT Tbl, + Ind, + 'DATA_COMPRESSION', + isnull(data_comp, 'NONE') +FROM (SELECT O.Name AS Tbl, + I.Name AS Ind, + (SELECT TOP 1 CASE WHEN data_compression_desc = 'PAGE' THEN 'PAGE' END + FROM sys.partitions AS P + WHERE P.object_id = I.object_id + AND I.index_id = P.index_id) AS data_comp + FROM sys.indexes AS I + INNER JOIN + sys.objects AS O + ON O.object_id = I.object_id + WHERE O.type = 'u' + AND EXISTS (SELECT * + FROM sys.partition_schemes AS PS + WHERE PS.data_space_id = I.data_space_id + AND name = 'PartitionScheme_ResourceTypeId')) AS A +WHERE NOT EXISTS (SELECT * + FROM dbo.IndexProperties + WHERE TableName = Tbl + AND IndexName = Ind); + +GO +CREATE PROCEDURE dbo.LogEvent +@Process VARCHAR (100), @Status VARCHAR (10), @Mode VARCHAR (200)=NULL, @Action VARCHAR (20)=NULL, @Target VARCHAR (100)=NULL, @Rows BIGINT=NULL, @Start DATETIME=NULL, @Text NVARCHAR (3500)=NULL, @EventId BIGINT=NULL OUTPUT, @Retry INT=NULL +AS +SET NOCOUNT ON; +DECLARE @ErrorNumber AS INT = error_number(), @ErrorMessage AS VARCHAR (1000) = '', @TranCount AS INT = @@trancount, @DoWork AS BIT = 0, @NumberAdded AS BIT; +IF @ErrorNumber IS NOT NULL + OR @Status IN ('Warn', 'Error') + SET @DoWork = 1; +IF @DoWork = 0 + SET @DoWork = CASE WHEN EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = isnull(@Process, '') + AND Char = 'LogEvent') THEN 1 ELSE 0 END; +IF @DoWork = 0 + RETURN; +IF @ErrorNumber IS NOT NULL + SET @ErrorMessage = CASE WHEN @Retry IS NOT NULL THEN 'Retry ' + CONVERT (VARCHAR, @Retry) + ', ' ELSE '' END + 'Error ' + CONVERT (VARCHAR, error_number()) + ': ' + CONVERT (VARCHAR (1000), error_message()) + ', Level ' + CONVERT (VARCHAR, error_severity()) + ', State ' + CONVERT (VARCHAR, error_state()) + CASE WHEN error_procedure() IS NOT NULL THEN ', Procedure ' + error_procedure() ELSE '' END + ', Line ' + CONVERT (VARCHAR, error_line()); +IF @TranCount > 0 + AND @ErrorNumber IS NOT NULL + ROLLBACK; +IF databasepropertyex(db_name(), 'UpdateAbility') = 'READ_WRITE' + BEGIN + INSERT INTO dbo.EventLog (Process, Status, Mode, Action, Target, Rows, Milliseconds, EventDate, EventText, SPID, HostName) + SELECT @Process, + @Status, + @Mode, + @Action, + @Target, + @Rows, + datediff(millisecond, @Start, getUTCdate()), + getUTCdate() AS EventDate, + CASE WHEN @ErrorNumber IS NULL THEN @Text ELSE @ErrorMessage + CASE WHEN isnull(@Text, '') <> '' THEN '. ' + @Text ELSE '' END END AS Text, + @@SPID, + host_name() AS HostName; + SET @EventId = scope_identity(); + END +IF @TranCount > 0 + AND @ErrorNumber IS NOT NULL + BEGIN TRANSACTION; + +GO +CREATE PROCEDURE dbo.LogSchemaMigrationProgress +@message VARCHAR (MAX) +AS +INSERT INTO dbo.SchemaMigrationProgress (Message) +VALUES (@message); + +GO +CREATE PROCEDURE dbo.MergeResources +@AffectedRows INT=0 OUTPUT, @RaiseExceptionOnConflict BIT=1, @IsResourceChangeCaptureEnabled BIT=0, @TransactionId BIGINT=NULL, @SingleTransaction BIT=1, @Resources dbo.ResourceList READONLY, @ResourceWriteClaims dbo.ResourceWriteClaimList READONLY, @ReferenceSearchParams dbo.ReferenceSearchParamList READONLY, @TokenSearchParams dbo.TokenSearchParamList READONLY, @TokenTexts dbo.TokenTextList READONLY, @StringSearchParams dbo.StringSearchParamList READONLY, @UriSearchParams dbo.UriSearchParamList READONLY, @NumberSearchParams dbo.NumberSearchParamList READONLY, @QuantitySearchParams dbo.QuantitySearchParamList READONLY, @DateTimeSearchParms dbo.DateTimeSearchParamList READONLY, @ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY, @TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY, @TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY, @TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY, @TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY, @TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY +AS +SET NOCOUNT ON; +DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = object_name(@@procid), @DummyTop AS BIGINT = 9223372036854775807, @InitialTranCount AS INT = @@trancount, @IsRetry AS BIT = 0; +DECLARE @Mode AS VARCHAR (200) = isnull((SELECT 'RT=[' + CONVERT (VARCHAR, min(ResourceTypeId)) + ',' + CONVERT (VARCHAR, max(ResourceTypeId)) + '] Sur=[' + CONVERT (VARCHAR, min(ResourceSurrogateId)) + ',' + CONVERT (VARCHAR, max(ResourceSurrogateId)) + '] V=' + CONVERT (VARCHAR, max(Version)) + ' Rows=' + CONVERT (VARCHAR, count(*)) + FROM @Resources), 'Input=Empty'); +SET @Mode += ' E=' + CONVERT (VARCHAR, @RaiseExceptionOnConflict) + ' CC=' + CONVERT (VARCHAR, @IsResourceChangeCaptureEnabled) + ' IT=' + CONVERT (VARCHAR, @InitialTranCount) + ' T=' + isnull(CONVERT (VARCHAR, @TransactionId), 'NULL'); +SET @AffectedRows = 0; +BEGIN TRY + DECLARE @Existing AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + SurrogateId BIGINT NOT NULL PRIMARY KEY (ResourceTypeId, SurrogateId)); + DECLARE @ResourceInfos AS TABLE ( + ResourceTypeId SMALLINT NOT NULL, + SurrogateId BIGINT NOT NULL, + Version INT NOT NULL, + KeepHistory BIT NOT NULL, + PreviousVersion INT NULL, + PreviousSurrogateId BIGINT NULL PRIMARY KEY (ResourceTypeId, SurrogateId)); + DECLARE @PreviousSurrogateIds AS TABLE ( + TypeId SMALLINT NOT NULL, + SurrogateId BIGINT NOT NULL PRIMARY KEY (TypeId, SurrogateId), + KeepHistory BIT ); + IF @SingleTransaction = 0 + AND isnull((SELECT Number + FROM dbo.Parameters + WHERE Id = 'MergeResources.NoTransaction.IsEnabled'), 0) = 0 + SET @SingleTransaction = 1; + SET @Mode += ' ST=' + CONVERT (VARCHAR, @SingleTransaction); + IF @InitialTranCount = 0 + BEGIN + IF EXISTS (SELECT * + FROM @Resources AS A + INNER JOIN + dbo.Resource AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId) + BEGIN + BEGIN TRANSACTION; + INSERT INTO @Existing (ResourceTypeId, SurrogateId) + SELECT B.ResourceTypeId, + B.ResourceSurrogateId + FROM (SELECT TOP (@DummyTop) * + FROM @Resources) AS A + INNER JOIN + dbo.Resource AS B WITH (ROWLOCK, HOLDLOCK) + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + WHERE B.IsHistory = 0 + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + IF @@rowcount > 0 + SET @IsRetry = 1; + IF @IsRetry = 0 + COMMIT TRANSACTION; + END + END + SET @Mode += ' R=' + CONVERT (VARCHAR, @IsRetry); + IF @SingleTransaction = 1 + AND @@trancount = 0 + BEGIN TRANSACTION; + IF @IsRetry = 0 + BEGIN + INSERT INTO @ResourceInfos (ResourceTypeId, SurrogateId, Version, KeepHistory, PreviousVersion, PreviousSurrogateId) + SELECT A.ResourceTypeId, + A.ResourceSurrogateId, + A.Version, + A.KeepHistory, + B.Version, + B.ResourceSurrogateId + FROM (SELECT TOP (@DummyTop) * + FROM @Resources + WHERE HasVersionToCompare = 1) AS A + LEFT OUTER JOIN + dbo.Resource AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceId = A.ResourceId + AND B.IsHistory = 0 + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + IF @RaiseExceptionOnConflict = 1 + AND EXISTS (SELECT * + FROM @ResourceInfos + WHERE PreviousVersion IS NOT NULL + AND Version <> PreviousVersion + 1) + THROW 50409, 'Resource has been recently updated or added, please compare the resource content in code for any duplicate updates', 1; + INSERT INTO @PreviousSurrogateIds + SELECT ResourceTypeId, + PreviousSurrogateId, + KeepHistory + FROM @ResourceInfos + WHERE PreviousSurrogateId IS NOT NULL; + IF @@rowcount > 0 + BEGIN + UPDATE dbo.Resource + SET IsHistory = 1 + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId + AND KeepHistory = 1); + SET @AffectedRows += @@rowcount; + IF @IsResourceChangeCaptureEnabled = 1 + AND NOT EXISTS (SELECT * + FROM dbo.Parameters + WHERE Id = 'InvisibleHistory.IsEnabled' + AND Number = 0) + UPDATE dbo.Resource + SET IsHistory = 1, + RawResource = 0xF, + SearchParamHash = NULL, + HistoryTransactionId = @TransactionId + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId + AND KeepHistory = 0); + ELSE + DELETE dbo.Resource + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId + AND KeepHistory = 0); + SET @AffectedRows += @@rowcount; + DELETE dbo.ResourceWriteClaim + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.ReferenceSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenText + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.StringSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.UriSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.NumberSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.QuantitySearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.DateTimeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.ReferenceTokenCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenTokenCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenDateTimeCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenQuantityCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenStringCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + DELETE dbo.TokenNumberNumberCompositeSearchParam + WHERE EXISTS (SELECT * + FROM @PreviousSurrogateIds + WHERE TypeId = ResourceTypeId + AND SurrogateId = ResourceSurrogateId); + SET @AffectedRows += @@rowcount; + END + INSERT INTO dbo.Resource (ResourceTypeId, ResourceId, Version, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, RawResource, IsRawResourceMetaSet, SearchParamHash, TransactionId) + SELECT ResourceTypeId, + ResourceId, + Version, + IsHistory, + ResourceSurrogateId, + IsDeleted, + RequestMethod, + RawResource, + IsRawResourceMetaSet, + SearchParamHash, + @TransactionId + FROM @Resources; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) + SELECT ResourceSurrogateId, + ClaimTypeId, + ClaimValue + FROM @ResourceWriteClaims; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri, + ReferenceResourceTypeId, + ReferenceResourceId, + ReferenceResourceVersion + FROM @ReferenceSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + Code, + CodeOverflow + FROM @TokenSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text + FROM @TokenTexts; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text, + TextOverflow, + IsMin, + IsMax + FROM @StringSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Uri + FROM @UriSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SingleValue, + LowValue, + HighValue + FROM @NumberSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + QuantityCodeId, + SingleValue, + LowValue, + HighValue + FROM @QuantitySearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + StartDateTime, + EndDateTime, + IsLongerThanADay, + IsMin, + IsMax + FROM @DateTimeSearchParms; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri1, + ReferenceResourceTypeId1, + ReferenceResourceId1, + ReferenceResourceVersion1, + SystemId2, + Code2, + CodeOverflow2 + FROM @ReferenceTokenCompositeSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SystemId2, + Code2, + CodeOverflow2 + FROM @TokenTokenCompositeSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + StartDateTime2, + EndDateTime2, + IsLongerThanADay2 + FROM @TokenDateTimeCompositeSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + SystemId2, + QuantityCodeId2, + LowValue2, + HighValue2 + FROM @TokenQuantityCompositeSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + Text2, + TextOverflow2 + FROM @TokenStringCompositeSearchParams; + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + LowValue2, + HighValue2, + SingleValue3, + LowValue3, + HighValue3, + HasRange + FROM @TokenNumberNumberCompositeSearchParams; + SET @AffectedRows += @@rowcount; + END + ELSE + BEGIN + INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) + SELECT ResourceSurrogateId, + ClaimTypeId, + ClaimValue + FROM (SELECT TOP (@DummyTop) * + FROM @ResourceWriteClaims) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.ResourceWriteClaim AS C + WHERE C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri, + ReferenceResourceTypeId, + ReferenceResourceId, + ReferenceResourceVersion + FROM (SELECT TOP (@DummyTop) * + FROM @ReferenceSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.ReferenceSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + Code, + CodeOverflow + FROM (SELECT TOP (@DummyTop) * + FROM @TokenSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text + FROM (SELECT TOP (@DummyTop) * + FROM @TokenTexts) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text, + TextOverflow, + IsMin, + IsMax + FROM (SELECT TOP (@DummyTop) * + FROM @StringSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenText AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Uri + FROM (SELECT TOP (@DummyTop) * + FROM @UriSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.UriSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SingleValue, + LowValue, + HighValue + FROM (SELECT TOP (@DummyTop) * + FROM @NumberSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.NumberSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + QuantityCodeId, + SingleValue, + LowValue, + HighValue + FROM (SELECT TOP (@DummyTop) * + FROM @QuantitySearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.QuantitySearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + StartDateTime, + EndDateTime, + IsLongerThanADay, + IsMin, + IsMax + FROM (SELECT TOP (@DummyTop) * + FROM @DateTimeSearchParms) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri1, + ReferenceResourceTypeId1, + ReferenceResourceId1, + ReferenceResourceVersion1, + SystemId2, + Code2, + CodeOverflow2 + FROM (SELECT TOP (@DummyTop) * + FROM @ReferenceTokenCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.DateTimeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SystemId2, + Code2, + CodeOverflow2 + FROM (SELECT TOP (@DummyTop) * + FROM @TokenTokenCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenTokenCompositeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + StartDateTime2, + EndDateTime2, + IsLongerThanADay2 + FROM (SELECT TOP (@DummyTop) * + FROM @TokenDateTimeCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenDateTimeCompositeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + SystemId2, + QuantityCodeId2, + LowValue2, + HighValue2 + FROM (SELECT TOP (@DummyTop) * + FROM @TokenQuantityCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenQuantityCompositeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + Text2, + TextOverflow2 + FROM (SELECT TOP (@DummyTop) * + FROM @TokenStringCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenStringCompositeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + LowValue2, + HighValue2, + SingleValue3, + LowValue3, + HighValue3, + HasRange + FROM (SELECT TOP (@DummyTop) * + FROM @TokenNumberNumberCompositeSearchParams) AS A + WHERE EXISTS (SELECT * + FROM @Existing AS B + WHERE B.ResourceTypeId = A.ResourceTypeId + AND B.SurrogateId = A.ResourceSurrogateId) + AND NOT EXISTS (SELECT * + FROM dbo.TokenNumberNumberCompositeSearchParam AS C + WHERE C.ResourceTypeId = A.ResourceTypeId + AND C.ResourceSurrogateId = A.ResourceSurrogateId) + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)); + SET @AffectedRows += @@rowcount; + END + IF @IsResourceChangeCaptureEnabled = 1 + EXECUTE dbo.CaptureResourceIdsForChanges @Resources; + IF @TransactionId IS NOT NULL + EXECUTE dbo.MergeResourcesCommitTransaction @TransactionId; + IF @InitialTranCount = 0 + AND @@trancount > 0 + COMMIT TRANSACTION; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @AffectedRows; +END TRY +BEGIN CATCH + IF @InitialTranCount = 0 + AND @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + IF @RaiseExceptionOnConflict = 1 + AND error_number() IN (2601, 2627) + AND error_message() LIKE '%''dbo.Resource''%' + THROW 50409, 'Resource has been recently updated or added, please compare the resource content in code for any duplicate updates', 1; + ELSE + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesAdvanceTransactionVisibility +@AffectedRows INT=0 OUTPUT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = '', @st AS DATETIME = getUTCdate(), @msg AS VARCHAR (1000), @MaxTransactionId AS BIGINT, @MinTransactionId AS BIGINT, @MinNotCompletedTransactionId AS BIGINT, @CurrentTransactionId AS BIGINT; +SET @AffectedRows = 0; +BEGIN TRY + EXECUTE dbo.MergeResourcesGetTransactionVisibility @MinTransactionId OUTPUT; + SET @MinTransactionId += 1; + SET @CurrentTransactionId = (SELECT TOP 1 SurrogateIdRangeFirstValue + FROM dbo.Transactions + ORDER BY SurrogateIdRangeFirstValue DESC); + SET @MinNotCompletedTransactionId = isnull((SELECT TOP 1 SurrogateIdRangeFirstValue + FROM dbo.Transactions + WHERE IsCompleted = 0 + AND SurrogateIdRangeFirstValue BETWEEN @MinTransactionId AND @CurrentTransactionId + ORDER BY SurrogateIdRangeFirstValue), @CurrentTransactionId + 1); + SET @MaxTransactionId = (SELECT TOP 1 SurrogateIdRangeFirstValue + FROM dbo.Transactions + WHERE IsCompleted = 1 + AND SurrogateIdRangeFirstValue BETWEEN @MinTransactionId AND @CurrentTransactionId + AND SurrogateIdRangeFirstValue < @MinNotCompletedTransactionId + ORDER BY SurrogateIdRangeFirstValue DESC); + IF @MaxTransactionId >= @MinTransactionId + BEGIN + UPDATE A + SET IsVisible = 1, + VisibleDate = getUTCdate() + FROM dbo.Transactions AS A WITH (INDEX (1)) + WHERE SurrogateIdRangeFirstValue BETWEEN @MinTransactionId AND @CurrentTransactionId + AND SurrogateIdRangeFirstValue <= @MaxTransactionId; + SET @AffectedRows += @@rowcount; + END + SET @msg = 'Min=' + CONVERT (VARCHAR, @MinTransactionId) + ' C=' + CONVERT (VARCHAR, @CurrentTransactionId) + ' MinNC=' + CONVERT (VARCHAR, @MinNotCompletedTransactionId) + ' Max=' + CONVERT (VARCHAR, @MaxTransactionId); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @AffectedRows, @Text = @msg; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesBeginTransaction +@Count INT, @TransactionId BIGINT OUTPUT, @SequenceRangeFirstValue INT=NULL OUTPUT, @HeartbeatDate DATETIME=NULL +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'MergeResourcesBeginTransaction', @Mode AS VARCHAR (200) = 'Cnt=' + CONVERT (VARCHAR, @Count), @st AS DATETIME = getUTCdate(), @FirstValueVar AS SQL_VARIANT, @LastValueVar AS SQL_VARIANT; +BEGIN TRY + SET @TransactionId = NULL; + IF @@trancount > 0 + RAISERROR ('MergeResourcesBeginTransaction cannot be called inside outer transaction.', 18, 127); + SET @FirstValueVar = NULL; + WHILE @FirstValueVar IS NULL + BEGIN + EXECUTE sys.sp_sequence_get_range @sequence_name = 'dbo.ResourceSurrogateIdUniquifierSequence', @range_size = @Count, @range_first_value = @FirstValueVar OUTPUT, @range_last_value = @LastValueVar OUTPUT; + SET @SequenceRangeFirstValue = CONVERT (INT, @FirstValueVar); + IF @SequenceRangeFirstValue > CONVERT (INT, @LastValueVar) + SET @FirstValueVar = NULL; + END + SET @TransactionId = datediff_big(millisecond, '0001-01-01', sysUTCdatetime()) * 80000 + @SequenceRangeFirstValue; + INSERT INTO dbo.Transactions (SurrogateIdRangeFirstValue, SurrogateIdRangeLastValue, HeartbeatDate) + SELECT @TransactionId, + @TransactionId + @Count - 1, + isnull(@HeartbeatDate, getUTCdate()); +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + IF @@trancount > 0 + ROLLBACK; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesCommitTransaction +@TransactionId BIGINT, @FailureReason VARCHAR (MAX)=NULL, @OverrideIsControlledByClientCheck BIT=0 +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'MergeResourcesCommitTransaction', @st AS DATETIME = getUTCdate(), @InitialTranCount AS INT = @@trancount, @IsCompletedBefore AS BIT, @Rows AS INT, @msg AS VARCHAR (1000); +DECLARE @Mode AS VARCHAR (200) = 'TR=' + CONVERT (VARCHAR, @TransactionId) + ' OC=' + isnull(CONVERT (VARCHAR, @OverrideIsControlledByClientCheck), 'NULL'); +BEGIN TRY + IF @InitialTranCount = 0 + BEGIN TRANSACTION; + UPDATE dbo.Transactions + SET IsCompleted = 1, + @IsCompletedBefore = IsCompleted, + EndDate = getUTCdate(), + IsSuccess = CASE WHEN @FailureReason IS NULL THEN 1 ELSE 0 END, + FailureReason = @FailureReason + WHERE SurrogateIdRangeFirstValue = @TransactionId + AND (IsControlledByClient = 1 + OR @OverrideIsControlledByClientCheck = 1); + SET @Rows = @@rowcount; + IF @Rows = 0 + BEGIN + SET @msg = 'Transaction [' + CONVERT (VARCHAR (20), @TransactionId) + '] is not controlled by client or does not exist.'; + RAISERROR (@msg, 18, 127); + END + IF @IsCompletedBefore = 1 + BEGIN + IF @InitialTranCount = 0 + ROLLBACK; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows, @Target = '@IsCompletedBefore', @Text = '=1'; + RETURN; + END + IF @InitialTranCount = 0 + COMMIT TRANSACTION; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF @InitialTranCount = 0 + AND @@trancount > 0 + ROLLBACK; + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesDeleteInvisibleHistory +@TransactionId BIGINT, @AffectedRows INT=NULL OUTPUT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'T=' + CONVERT (VARCHAR, @TransactionId), @st AS DATETIME = getUTCdate(), @TypeId AS SMALLINT; +SET @AffectedRows = 0; +BEGIN TRY + DECLARE @Types TABLE ( + TypeId SMALLINT PRIMARY KEY, + Name VARCHAR (100)); + INSERT INTO @Types + EXECUTE dbo.GetUsedResourceTypes ; + WHILE EXISTS (SELECT * + FROM @Types) + BEGIN + SET @TypeId = (SELECT TOP 1 TypeId + FROM @Types + ORDER BY TypeId); + DELETE dbo.Resource + WHERE ResourceTypeId = @TypeId + AND HistoryTransactionId = @TransactionId + AND RawResource = 0xF; + SET @AffectedRows += @@rowcount; + DELETE @Types + WHERE TypeId = @TypeId; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @AffectedRows; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesGetTimeoutTransactions +@TimeoutSec INT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'T=' + CONVERT (VARCHAR, @TimeoutSec), @st AS DATETIME = getUTCdate(), @MinTransactionId AS BIGINT; +BEGIN TRY + EXECUTE dbo.MergeResourcesGetTransactionVisibility @MinTransactionId OUTPUT; + SELECT SurrogateIdRangeFirstValue + FROM dbo.Transactions + WHERE SurrogateIdRangeFirstValue > @MinTransactionId + AND IsCompleted = 0 + AND datediff(second, HeartbeatDate, getUTCdate()) > @TimeoutSec + ORDER BY SurrogateIdRangeFirstValue; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesGetTransactionVisibility +@TransactionId BIGINT OUTPUT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = '', @st AS DATETIME = getUTCdate(); +SET @TransactionId = isnull((SELECT TOP 1 SurrogateIdRangeFirstValue + FROM dbo.Transactions + WHERE IsVisible = 1 + ORDER BY SurrogateIdRangeFirstValue DESC), -1); +EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount, @Text = @TransactionId; + +GO +CREATE PROCEDURE dbo.MergeResourcesPutTransactionHeartbeat +@TransactionId BIGINT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'MergeResourcesPutTransactionHeartbeat', @Mode AS VARCHAR (100) = 'TR=' + CONVERT (VARCHAR, @TransactionId); +BEGIN TRY + UPDATE dbo.Transactions + SET HeartbeatDate = getUTCdate() + WHERE SurrogateIdRangeFirstValue = @TransactionId + AND IsControlledByClient = 1; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.MergeResourcesPutTransactionInvisibleHistory +@TransactionId BIGINT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (100) = 'TR=' + CONVERT (VARCHAR, @TransactionId), @st AS DATETIME = getUTCdate(); +BEGIN TRY + UPDATE dbo.Transactions + SET InvisibleHistoryRemovedDate = getUTCdate() + WHERE SurrogateIdRangeFirstValue = @TransactionId + AND InvisibleHistoryRemovedDate IS NULL; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @@rowcount; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.PutJobCancelation +@QueueType TINYINT, @GroupId BIGINT=NULL, @JobId BIGINT=NULL +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'PutJobCancelation', @Mode AS VARCHAR (100) = 'Q=' + isnull(CONVERT (VARCHAR, @QueueType), 'NULL') + ' G=' + isnull(CONVERT (VARCHAR, @GroupId), 'NULL') + ' J=' + isnull(CONVERT (VARCHAR, @JobId), 'NULL'), @st AS DATETIME = getUTCdate(), @Rows AS INT, @PartitionId AS TINYINT = @JobId % 16; +BEGIN TRY + IF @JobId IS NULL + AND @GroupId IS NULL + RAISERROR ('@JobId = NULL and @GroupId = NULL', 18, 127); + IF @JobId IS NOT NULL + BEGIN + UPDATE dbo.JobQueue + SET Status = 4, + EndDate = getUTCdate(), + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()) + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Status = 0; + SET @Rows = @@rowcount; + IF @Rows = 0 + BEGIN + UPDATE dbo.JobQueue + SET CancelRequested = 1 + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Status = 1; + SET @Rows = @@rowcount; + END + END + ELSE + BEGIN + UPDATE dbo.JobQueue + SET Status = 4, + EndDate = getUTCdate(), + Version = datediff_big(millisecond, '0001-01-01', getUTCdate()) + WHERE QueueType = @QueueType + AND GroupId = @GroupId + AND Status = 0; + SET @Rows = @@rowcount; + UPDATE dbo.JobQueue + SET CancelRequested = 1 + WHERE QueueType = @QueueType + AND GroupId = @GroupId + AND Status = 1; + SET @Rows += @@rowcount; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.PutJobHeartbeat +@QueueType TINYINT, @JobId BIGINT, @Version BIGINT, @Data BIGINT=NULL, @CurrentResult VARCHAR (MAX)=NULL, @CancelRequested BIT=0 OUTPUT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'PutJobHeartbeat', @Mode AS VARCHAR (100), @st AS DATETIME = getUTCdate(), @Rows AS INT = 0, @PartitionId AS TINYINT = @JobId % 16; +SET @Mode = 'Q=' + CONVERT (VARCHAR, @QueueType) + ' J=' + CONVERT (VARCHAR, @JobId) + ' P=' + CONVERT (VARCHAR, @PartitionId) + ' V=' + CONVERT (VARCHAR, @Version) + ' D=' + isnull(CONVERT (VARCHAR, @Data), 'NULL'); +BEGIN TRY + IF @CurrentResult IS NULL + UPDATE dbo.JobQueue + SET @CancelRequested = CancelRequested, + HeartbeatDate = getUTCdate(), + Data = isnull(@Data, Data) + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Status = 1 + AND Version = @Version; + ELSE + UPDATE dbo.JobQueue + SET @CancelRequested = CancelRequested, + HeartbeatDate = getUTCdate(), + Data = isnull(@Data, Data), + Result = @CurrentResult + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Status = 1 + AND Version = @Version; + SET @Rows = @@rowcount; + IF @Rows = 0 + AND NOT EXISTS (SELECT * + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Version = @Version + AND Status IN (2, 3, 4)) + BEGIN + IF EXISTS (SELECT * + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId) + THROW 50412, 'Precondition failed', 1; + ELSE + THROW 50404, 'Job record not found', 1; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.PutJobStatus +@QueueType TINYINT, @JobId BIGINT, @Version BIGINT, @Failed BIT, @Data BIGINT, @FinalResult VARCHAR (MAX), @RequestCancellationOnFailure BIT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'PutJobStatus', @Mode AS VARCHAR (100), @st AS DATETIME = getUTCdate(), @Rows AS INT = 0, @PartitionId AS TINYINT = @JobId % 16, @GroupId AS BIGINT; +SET @Mode = 'Q=' + CONVERT (VARCHAR, @QueueType) + ' J=' + CONVERT (VARCHAR, @JobId) + ' P=' + CONVERT (VARCHAR, @PartitionId) + ' V=' + CONVERT (VARCHAR, @Version) + ' F=' + CONVERT (VARCHAR, @Failed) + ' R=' + isnull(@FinalResult, 'NULL'); +BEGIN TRY + UPDATE dbo.JobQueue + SET EndDate = getUTCdate(), + Status = CASE WHEN @Failed = 1 THEN 3 WHEN CancelRequested = 1 THEN 4 ELSE 2 END, + Data = @Data, + Result = @FinalResult, + @GroupId = GroupId + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Status = 1 + AND Version = @Version; + SET @Rows = @@rowcount; + IF @Rows = 0 + BEGIN + SET @GroupId = (SELECT GroupId + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId + AND Version = @Version + AND Status IN (2, 3, 4)); + IF @GroupId IS NULL + IF EXISTS (SELECT * + FROM dbo.JobQueue + WHERE QueueType = @QueueType + AND PartitionId = @PartitionId + AND JobId = @JobId) + THROW 50412, 'Precondition failed', 1; + ELSE + THROW 50404, 'Job record not found', 1; + END + IF @Failed = 1 + AND @RequestCancellationOnFailure = 1 + EXECUTE dbo.PutJobCancelation @QueueType = @QueueType, @GroupId = @GroupId; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error'; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.ReadResource +@resourceTypeId SMALLINT, @resourceId VARCHAR (64), @version INT=NULL +AS +SET NOCOUNT ON; +IF (@version IS NULL) + BEGIN + SELECT ResourceSurrogateId, + Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM dbo.Resource + WHERE ResourceTypeId = @resourceTypeId + AND ResourceId = @resourceId + AND IsHistory = 0; + END +ELSE + BEGIN + SELECT ResourceSurrogateId, + Version, + IsDeleted, + IsHistory, + RawResource, + IsRawResourceMetaSet, + SearchParamHash + FROM dbo.Resource + WHERE ResourceTypeId = @resourceTypeId + AND ResourceId = @resourceId + AND Version = @version; + END + +GO +CREATE PROCEDURE dbo.ReindexResource_2 +@resourceTypeId SMALLINT, @resourceId VARCHAR (64), @eTag INT=NULL, @searchParamHash VARCHAR (64), @resourceWriteClaims dbo.BulkResourceWriteClaimTableType_1 READONLY, @compartmentAssignments dbo.BulkCompartmentAssignmentTableType_1 READONLY, @referenceSearchParams dbo.BulkReferenceSearchParamTableType_1 READONLY, @tokenSearchParams dbo.BulkTokenSearchParamTableType_2 READONLY, @tokenTextSearchParams dbo.BulkTokenTextTableType_1 READONLY, @stringSearchParams dbo.BulkStringSearchParamTableType_2 READONLY, @numberSearchParams dbo.BulkNumberSearchParamTableType_1 READONLY, @quantitySearchParams dbo.BulkQuantitySearchParamTableType_1 READONLY, @uriSearchParams dbo.BulkUriSearchParamTableType_1 READONLY, @dateTimeSearchParms dbo.BulkDateTimeSearchParamTableType_2 READONLY, @referenceTokenCompositeSearchParams dbo.BulkReferenceTokenCompositeSearchParamTableType_2 READONLY, @tokenTokenCompositeSearchParams dbo.BulkTokenTokenCompositeSearchParamTableType_2 READONLY, @tokenDateTimeCompositeSearchParams dbo.BulkTokenDateTimeCompositeSearchParamTableType_2 READONLY, @tokenQuantityCompositeSearchParams dbo.BulkTokenQuantityCompositeSearchParamTableType_2 READONLY, @tokenStringCompositeSearchParams dbo.BulkTokenStringCompositeSearchParamTableType_2 READONLY, @tokenNumberNumberCompositeSearchParams dbo.BulkTokenNumberNumberCompositeSearchParamTableType_2 READONLY +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @resourceSurrogateId AS BIGINT; +DECLARE @version AS BIGINT; +SELECT @resourceSurrogateId = ResourceSurrogateId, + @version = Version +FROM dbo.Resource WITH (UPDLOCK, HOLDLOCK) +WHERE ResourceTypeId = @resourceTypeId + AND ResourceId = @resourceId + AND IsHistory = 0; +IF (@etag IS NOT NULL + AND @etag <> @version) + BEGIN + THROW 50412, 'Precondition failed', 1; + END +UPDATE dbo.Resource +SET SearchParamHash = @searchParamHash +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.ResourceWriteClaim +WHERE ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.CompartmentAssignment +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.ReferenceSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenText +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.StringSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.UriSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.NumberSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.QuantitySearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.DateTimeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.ReferenceTokenCompositeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenTokenCompositeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenDateTimeCompositeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenQuantityCompositeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenStringCompositeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +DELETE dbo.TokenNumberNumberCompositeSearchParam +WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @resourceSurrogateId; +INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) +SELECT @resourceSurrogateId, + ClaimTypeId, + ClaimValue +FROM @resourceWriteClaims; +INSERT INTO dbo.CompartmentAssignment (ResourceTypeId, ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + CompartmentTypeId, + ReferenceResourceId, + 0 +FROM @compartmentAssignments; +INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + BaseUri, + ReferenceResourceTypeId, + ReferenceResourceId, + ReferenceResourceVersion, + 0 +FROM @referenceSearchParams; +INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId, + Code, + CodeOverflow, + 0 +FROM @tokenSearchParams; +INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + Text, + 0 +FROM @tokenTextSearchParams; +INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsHistory, IsMin, IsMax) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + Text, + TextOverflow, + 0, + IsMin, + IsMax +FROM @stringSearchParams; +INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + Uri, + 0 +FROM @uriSearchParams; +INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SingleValue, + LowValue, + HighValue, + 0 +FROM @numberSearchParams; +INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId, + QuantityCodeId, + SingleValue, + LowValue, + HighValue, + 0 +FROM @quantitySearchParams; +INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsHistory, IsMin, IsMax) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + StartDateTime, + EndDateTime, + IsLongerThanADay, + 0, + IsMin, + IsMax +FROM @dateTimeSearchParms; +INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + BaseUri1, + ReferenceResourceTypeId1, + ReferenceResourceId1, + ReferenceResourceVersion1, + SystemId2, + Code2, + CodeOverflow2, + 0 +FROM @referenceTokenCompositeSearchParams; +INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SystemId2, + Code2, + CodeOverflow2, + 0 +FROM @tokenTokenCompositeSearchParams; +INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + StartDateTime2, + EndDateTime2, + IsLongerThanADay2, + 0 +FROM @tokenDateTimeCompositeSearchParams; +INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + SystemId2, + QuantityCodeId2, + LowValue2, + HighValue2, + 0 +FROM @tokenQuantityCompositeSearchParams; +INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + Text2, + TextOverflow2, + 0 +FROM @tokenStringCompositeSearchParams; +INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + LowValue2, + HighValue2, + SingleValue3, + LowValue3, + HighValue3, + HasRange, + 0 +FROM @tokenNumberNumberCompositeSearchParams; +COMMIT TRANSACTION; + +GO +CREATE OR ALTER PROCEDURE dbo.RemovePartitionFromResourceChanges_2 +@partitionNumberToSwitchOut INT, @partitionBoundaryToMerge DATETIME2 (7) +AS +BEGIN + TRUNCATE TABLE dbo.ResourceChangeDataStaging; + ALTER TABLE dbo.ResourceChangeData SWITCH PARTITION @partitionNumberToSwitchOut TO dbo.ResourceChangeDataStaging; + ALTER PARTITION FUNCTION PartitionFunction_ResourceChangeData_Timestamp( ) + MERGE RANGE (@partitionBoundaryToMerge); + TRUNCATE TABLE dbo.ResourceChangeDataStaging; +END + +GO +CREATE PROCEDURE dbo.ResetTask_2 +@taskId VARCHAR (64), @runId VARCHAR (50), @result VARCHAR (MAX) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +DECLARE @retryCount AS SMALLINT = NULL; +IF NOT EXISTS (SELECT * + FROM dbo.TaskInfo + WHERE TaskId = @taskId + AND RunId = @runId) + BEGIN + THROW 50404, 'Task not exist or runid not match', 1; + END +UPDATE dbo.TaskInfo +SET Status = 3, + EndDateTime = SYSUTCDATETIME(), + Result = @result, + @retryCount = retryCount +WHERE TaskId = @taskId + AND RunId = @runId + AND (MaxRetryCount <> -1 + AND RetryCount >= MaxRetryCount); +IF @retryCount IS NULL + UPDATE dbo.TaskInfo + SET Status = 1, + Result = @result, + RetryCount = RetryCount + 1, + RestartInfo = ISNULL(RestartInfo, '') + ' Prev: Worker=' + Worker + ' Start=' + CONVERT (VARCHAR, StartDateTime, 121) + WHERE TaskId = @taskId + AND RunId = @runId + AND Status <> 3 + AND (MaxRetryCount = -1 + OR RetryCount < MaxRetryCount); +EXECUTE dbo.GetTaskDetails @TaskId = @taskId; + +GO +CREATE PROCEDURE dbo.SwitchPartitionsIn +@Tbl VARCHAR (100) +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'SwitchPartitionsIn', @Mode AS VARCHAR (200) = 'Tbl=' + isnull(@Tbl, 'NULL'), @st AS DATETIME = getUTCdate(), @ResourceTypeId AS SMALLINT, @Rows AS BIGINT, @Txt AS VARCHAR (1000), @TblInt AS VARCHAR (100), @Ind AS VARCHAR (200), @IndId AS INT, @DataComp AS VARCHAR (100); +DECLARE @Indexes TABLE ( + IndId INT PRIMARY KEY, + name VARCHAR (200)); +DECLARE @ResourceTypes TABLE ( + ResourceTypeId SMALLINT PRIMARY KEY); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + IF @Tbl IS NULL + RAISERROR ('@Tbl IS NULL', 18, 127); + INSERT INTO @Indexes + SELECT index_id, + name + FROM sys.indexes + WHERE object_id = object_id(@Tbl) + AND is_disabled = 1; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Insert', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Indexes) + BEGIN + SELECT TOP 1 @IndId = IndId, + @Ind = name + FROM @Indexes + ORDER BY IndId; + SET @DataComp = CASE WHEN (SELECT PropertyValue + FROM dbo.IndexProperties + WHERE TableName = @Tbl + AND IndexName = @Ind) = 'PAGE' THEN ' PARTITION = ALL WITH (DATA_COMPRESSION = PAGE)' ELSE '' END; + SET @Txt = 'IF EXISTS (SELECT * FROM sys.indexes WHERE object_id = object_id(''' + @Tbl + ''') AND name = ''' + @Ind + ''' AND is_disabled = 1) ALTER INDEX ' + @Ind + ' ON dbo.' + @Tbl + ' REBUILD' + @DataComp; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Ind, @Action = 'Rebuild', @Text = @Txt; + DELETE @Indexes + WHERE IndId = @IndId; + END + INSERT INTO @ResourceTypes + SELECT CONVERT (SMALLINT, substring(name, charindex('_', name) + 1, 6)) AS ResourceTypeId + FROM sys.objects AS O + WHERE name LIKE @Tbl + '[_]%' + AND EXISTS (SELECT * + FROM sysindexes + WHERE id = O.object_id + AND indid IN (0, 1) + AND rows > 0); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '#ResourceTypes', @Action = 'Select Into', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @ResourceTypes) + BEGIN + SET @ResourceTypeId = (SELECT TOP 1 ResourceTypeId + FROM @ResourceTypes); + SET @TblInt = @Tbl + '_' + CONVERT (VARCHAR, @ResourceTypeId); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt; + SET @Txt = 'ALTER TABLE dbo.' + @TblInt + ' SWITCH TO dbo.' + @Tbl + ' PARTITION $partition.PartitionFunction_ResourceTypeId(' + CONVERT (VARCHAR, @ResourceTypeId) + ')'; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Switch in start', @Text = @Txt; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Switch in', @Text = @Txt; + IF EXISTS (SELECT * + FROM sysindexes + WHERE id = object_id(@TblInt) + AND rows > 0) + BEGIN + SET @Txt = @TblInt + ' is not empty after switch'; + RAISERROR (@Txt, 18, 127); + END + EXECUTE ('DROP TABLE dbo.' + @TblInt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Drop'; + DELETE @ResourceTypes + WHERE ResourceTypeId = @ResourceTypeId; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.SwitchPartitionsInAllTables +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'SwitchPartitionsInAllTables', @Mode AS VARCHAR (200) = 'PS=PartitionScheme_ResourceTypeId', @st AS DATETIME = getUTCdate(), @Tbl AS VARCHAR (100); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + DECLARE @Tables TABLE ( + name VARCHAR (100) PRIMARY KEY, + supported BIT ); + INSERT INTO @Tables + EXECUTE dbo.GetPartitionedTables @IncludeNotDisabled = 1, @IncludeNotSupported = 0; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Tables', @Action = 'Insert', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Tables) + BEGIN + SET @Tbl = (SELECT TOP 1 name + FROM @Tables + ORDER BY name); + EXECUTE dbo.SwitchPartitionsIn @Tbl = @Tbl; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = 'SwitchPartitionsIn', @Action = 'Execute', @Text = @Tbl; + DELETE @Tables + WHERE name = @Tbl; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.SwitchPartitionsOut +@Tbl VARCHAR (100), @RebuildClustered BIT +WITH EXECUTE AS 'dbo' +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'SwitchPartitionsOut', @Mode AS VARCHAR (200) = 'Tbl=' + isnull(@Tbl, 'NULL') + ' ND=' + isnull(CONVERT (VARCHAR, @RebuildClustered), 'NULL'), @st AS DATETIME = getUTCdate(), @ResourceTypeId AS SMALLINT, @Rows AS BIGINT, @Txt AS VARCHAR (MAX), @TblInt AS VARCHAR (100), @IndId AS INT, @Ind AS VARCHAR (200), @Name AS VARCHAR (100), @checkName AS VARCHAR (200), @definition AS VARCHAR (200); +DECLARE @Indexes TABLE ( + IndId INT PRIMARY KEY, + name VARCHAR (200), + IsDisabled BIT ); +DECLARE @IndexesRT TABLE ( + IndId INT PRIMARY KEY, + name VARCHAR (200), + IsDisabled BIT ); +DECLARE @ResourceTypes TABLE ( + ResourceTypeId SMALLINT PRIMARY KEY, + partition_number_roundtrip INT , + partition_number INT , + row_count BIGINT ); +DECLARE @Names TABLE ( + name VARCHAR (100) PRIMARY KEY); +DECLARE @CheckConstraints TABLE ( + CheckName VARCHAR (200), + CheckDefinition VARCHAR (200)); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + IF @Tbl IS NULL + RAISERROR ('@Tbl IS NULL', 18, 127); + IF @RebuildClustered IS NULL + RAISERROR ('@RebuildClustered IS NULL', 18, 127); + INSERT INTO @Indexes + SELECT index_id, + name, + is_disabled + FROM sys.indexes + WHERE object_id = object_id(@Tbl) + AND (is_disabled = 0 + OR @RebuildClustered = 1); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Indexes', @Action = 'Insert', @Rows = @@rowcount; + INSERT INTO @ResourceTypes + SELECT partition_number - 1 AS ResourceTypeId, + $PARTITION.PartitionFunction_ResourceTypeId (partition_number - 1) AS partition_number_roundtrip, + partition_number, + row_count + FROM sys.dm_db_partition_stats + WHERE object_id = object_id(@Tbl) + AND index_id = 1 + AND row_count > 0; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@ResourceTypes', @Action = 'Insert', @Rows = @@rowcount, @Text = 'For partition switch'; + IF EXISTS (SELECT * + FROM @ResourceTypes + WHERE partition_number_roundtrip <> partition_number) + RAISERROR ('Partition sanity check failed', 18, 127); + WHILE EXISTS (SELECT * + FROM @ResourceTypes) + BEGIN + SELECT TOP 1 @ResourceTypeId = ResourceTypeId, + @Rows = row_count + FROM @ResourceTypes + ORDER BY ResourceTypeId; + SET @TblInt = @Tbl + '_' + CONVERT (VARCHAR, @ResourceTypeId); + SET @Txt = 'Starting @ResourceTypeId=' + CONVERT (VARCHAR, @ResourceTypeId) + ' row_count=' + CONVERT (VARCHAR, @Rows); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Text = @Txt; + IF NOT EXISTS (SELECT * + FROM sysindexes + WHERE id = object_id(@TblInt) + AND rows > 0) + BEGIN + IF object_id(@TblInt) IS NOT NULL + BEGIN + EXECUTE ('DROP TABLE dbo.' + @TblInt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Drop'; + END + EXECUTE ('SELECT * INTO dbo.' + @TblInt + ' FROM dbo.' + @Tbl + ' WHERE 1 = 2'); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Select Into', @Rows = @@rowcount; + DELETE @CheckConstraints; + INSERT INTO @CheckConstraints + SELECT name, + definition + FROM sys.check_constraints + WHERE parent_object_id = object_id(@Tbl); + WHILE EXISTS (SELECT * + FROM @CheckConstraints) + BEGIN + SELECT TOP 1 @checkName = CheckName, + @definition = CheckDefinition + FROM @CheckConstraints; + SET @Txt = 'ALTER TABLE ' + @TblInt + ' ADD CHECK ' + @definition; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'ALTER', @Text = @Txt; + DELETE @CheckConstraints + WHERE CheckName = @checkName; + END + DELETE @Names; + INSERT INTO @Names + SELECT name + FROM sys.columns + WHERE object_id = object_id(@Tbl) + AND is_sparse = 1; + WHILE EXISTS (SELECT * + FROM @Names) + BEGIN + SET @Name = (SELECT TOP 1 name + FROM @Names + ORDER BY name); + SET @Txt = (SELECT 'ALTER TABLE dbo.' + @TblInt + ' ALTER COLUMN ' + @Name + ' ' + T.name + '(' + CONVERT (VARCHAR, C.precision) + ',' + CONVERT (VARCHAR, C.scale) + ') SPARSE NULL' + FROM sys.types AS T + INNER JOIN + sys.columns AS C + ON C.system_type_id = T.system_type_id + WHERE C.object_id = object_id(@Tbl) + AND C.name = @Name); + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'ALTER', @Text = @Txt; + DELETE @Names + WHERE name = @Name; + END + END + INSERT INTO @IndexesRT + SELECT * + FROM @Indexes + WHERE IsDisabled = 0; + WHILE EXISTS (SELECT * + FROM @IndexesRT) + BEGIN + SELECT TOP 1 @IndId = IndId, + @Ind = name + FROM @IndexesRT + ORDER BY IndId; + IF NOT EXISTS (SELECT * + FROM sys.indexes + WHERE object_id = object_id(@TblInt) + AND name = @Ind) + BEGIN + EXECUTE dbo.GetIndexCommands @Tbl = @Tbl, @Ind = @Ind, @AddPartClause = 0, @IncludeClustered = 1, @Txt = @Txt OUTPUT; + SET @Txt = replace(@Txt, '[' + @Tbl + ']', @TblInt); + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @TblInt, @Action = 'Create Index', @Text = @Txt; + END + DELETE @IndexesRT + WHERE IndId = @IndId; + END + SET @Txt = 'ALTER TABLE dbo.' + @TblInt + ' ADD CHECK (ResourceTypeId >= ' + CONVERT (VARCHAR, @ResourceTypeId) + ' AND ResourceTypeId < ' + CONVERT (VARCHAR, @ResourceTypeId) + ' + 1)'; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Add check', @Text = @Txt; + SET @Txt = 'ALTER TABLE dbo.' + @Tbl + ' SWITCH PARTITION $partition.PartitionFunction_ResourceTypeId(' + CONVERT (VARCHAR, @ResourceTypeId) + ') TO dbo.' + @TblInt; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Switch out start', @Text = @Txt; + EXECUTE (@Txt); + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = @Tbl, @Action = 'Switch out end', @Text = @Txt; + DELETE @ResourceTypes + WHERE ResourceTypeId = @ResourceTypeId; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE dbo.SwitchPartitionsOutAllTables +@RebuildClustered BIT +AS +SET NOCOUNT ON; +DECLARE @SP AS VARCHAR (100) = 'SwitchPartitionsOutAllTables', @Mode AS VARCHAR (200) = 'PS=PartitionScheme_ResourceTypeId ND=' + isnull(CONVERT (VARCHAR, @RebuildClustered), 'NULL'), @st AS DATETIME = getUTCdate(), @Tbl AS VARCHAR (100); +BEGIN TRY + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Start'; + DECLARE @Tables TABLE ( + name VARCHAR (100) PRIMARY KEY, + supported BIT ); + INSERT INTO @Tables + EXECUTE dbo.GetPartitionedTables @IncludeNotDisabled = @RebuildClustered, @IncludeNotSupported = 0; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = '@Tables', @Action = 'Insert', @Rows = @@rowcount; + WHILE EXISTS (SELECT * + FROM @Tables) + BEGIN + SET @Tbl = (SELECT TOP 1 name + FROM @Tables + ORDER BY name); + EXECUTE dbo.SwitchPartitionsOut @Tbl = @Tbl, @RebuildClustered = @RebuildClustered; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Info', @Target = 'SwitchPartitionsOut', @Action = 'Execute', @Text = @Tbl; + DELETE @Tables + WHERE name = @Tbl; + END + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st; +END TRY +BEGIN CATCH + IF error_number() = 1750 + THROW; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE [dbo].[TaskKeepAlive] +@taskId VARCHAR (64), @runId VARCHAR (50) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +IF NOT EXISTS (SELECT * + FROM [dbo].[TaskInfo] + WHERE TaskId = @taskId + AND RunId = @runId) + BEGIN + THROW 50404, 'Task not exist or runid not match', 1; + END +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +UPDATE dbo.TaskInfo +SET HeartbeatDateTime = @heartbeatDateTime +WHERE TaskId = @taskId; +SELECT TaskId, + QueueId, + Status, + TaskTypeId, + RunId, + IsCanceled, + RetryCount, + MaxRetryCount, + HeartbeatDateTime, + InputData, + TaskContext, + Result +FROM [dbo].[TaskInfo] +WHERE TaskId = @taskId; +COMMIT TRANSACTION; + +GO +CREATE OR ALTER PROCEDURE dbo.UpdateEventAgentCheckpoint +@CheckpointId VARCHAR (64), @LastProcessedDateTime DATETIMEOFFSET (7)=NULL, @LastProcessedIdentifier VARCHAR (64)=NULL +AS +BEGIN + IF EXISTS (SELECT * + FROM dbo.EventAgentCheckpoint + WHERE CheckpointId = @CheckpointId) + UPDATE dbo.EventAgentCheckpoint + SET CheckpointId = @CheckpointId, + LastProcessedDateTime = @LastProcessedDateTime, + LastProcessedIdentifier = @LastProcessedIdentifier, + UpdatedOn = sysutcdatetime() + WHERE CheckpointId = @CheckpointId; + ELSE + INSERT INTO dbo.EventAgentCheckpoint (CheckpointId, LastProcessedDateTime, LastProcessedIdentifier, UpdatedOn) + VALUES (@CheckpointId, @LastProcessedDateTime, @LastProcessedIdentifier, sysutcdatetime()); +END + +GO +CREATE PROCEDURE dbo.UpdateExportJob +@id VARCHAR (64), @status VARCHAR (10), @rawJobRecord VARCHAR (MAX), @jobVersion BINARY (8) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @currentJobVersion AS BINARY (8); +SELECT @currentJobVersion = JobVersion +FROM dbo.ExportJob WITH (UPDLOCK, HOLDLOCK) +WHERE Id = @id; +IF (@currentJobVersion IS NULL) + BEGIN + THROW 50404, 'Export job record not found', 1; + END +IF (@jobVersion <> @currentJobVersion) + BEGIN + THROW 50412, 'Precondition failed', 1; + END +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +UPDATE dbo.ExportJob +SET Status = @status, + HeartbeatDateTime = @heartbeatDateTime, + RawJobRecord = @rawJobRecord +WHERE Id = @id; +SELECT @@DBTS; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.UpdateReindexJob +@id VARCHAR (64), @status VARCHAR (10), @rawJobRecord VARCHAR (MAX), @jobVersion BINARY (8) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +DECLARE @currentJobVersion AS BINARY (8); +SELECT @currentJobVersion = JobVersion +FROM dbo.ReindexJob WITH (UPDLOCK, HOLDLOCK) +WHERE Id = @id; +IF (@currentJobVersion IS NULL) + BEGIN + THROW 50404, 'Reindex job record not found', 1; + END +IF (@jobVersion <> @currentJobVersion) + BEGIN + THROW 50412, 'Precondition failed', 1; + END +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +UPDATE dbo.ReindexJob +SET Status = @status, + HeartbeatDateTime = @heartbeatDateTime, + RawJobRecord = @rawJobRecord +WHERE Id = @id; +SELECT @@DBTS; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.UpdateResourceSearchParams +@FailedResources INT=0 OUTPUT, @Resources dbo.ResourceList READONLY, @ResourceWriteClaims dbo.ResourceWriteClaimList READONLY, @ReferenceSearchParams dbo.ReferenceSearchParamList READONLY, @TokenSearchParams dbo.TokenSearchParamList READONLY, @TokenTexts dbo.TokenTextList READONLY, @StringSearchParams dbo.StringSearchParamList READONLY, @UriSearchParams dbo.UriSearchParamList READONLY, @NumberSearchParams dbo.NumberSearchParamList READONLY, @QuantitySearchParams dbo.QuantitySearchParamList READONLY, @DateTimeSearchParams dbo.DateTimeSearchParamList READONLY, @ReferenceTokenCompositeSearchParams dbo.ReferenceTokenCompositeSearchParamList READONLY, @TokenTokenCompositeSearchParams dbo.TokenTokenCompositeSearchParamList READONLY, @TokenDateTimeCompositeSearchParams dbo.TokenDateTimeCompositeSearchParamList READONLY, @TokenQuantityCompositeSearchParams dbo.TokenQuantityCompositeSearchParamList READONLY, @TokenStringCompositeSearchParams dbo.TokenStringCompositeSearchParamList READONLY, @TokenNumberNumberCompositeSearchParams dbo.TokenNumberNumberCompositeSearchParamList READONLY +AS +SET NOCOUNT ON; +DECLARE @st AS DATETIME = getUTCdate(), @SP AS VARCHAR (100) = object_name(@@procid), @Mode AS VARCHAR (200) = isnull((SELECT 'RT=[' + CONVERT (VARCHAR, min(ResourceTypeId)) + ',' + CONVERT (VARCHAR, max(ResourceTypeId)) + '] Sur=[' + CONVERT (VARCHAR, min(ResourceSurrogateId)) + ',' + CONVERT (VARCHAR, max(ResourceSurrogateId)) + '] V=' + CONVERT (VARCHAR, max(Version)) + ' Rows=' + CONVERT (VARCHAR, count(*)) + FROM @Resources), 'Input=Empty'), @Rows AS INT; +BEGIN TRY + DECLARE @Ids TABLE ( + ResourceTypeId SMALLINT NOT NULL, + ResourceSurrogateId BIGINT NOT NULL); + BEGIN TRANSACTION; + UPDATE B + SET SearchParamHash = A.SearchParamHash + OUTPUT deleted.ResourceTypeId, deleted.ResourceSurrogateId INTO @Ids + FROM @Resources AS A + INNER JOIN + dbo.Resource AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId + WHERE B.IsHistory = 0; + SET @Rows = @@rowcount; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.ResourceWriteClaim AS B + ON B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.ReferenceSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenText AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.StringSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.UriSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.NumberSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.QuantitySearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.DateTimeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.ReferenceTokenCompositeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenTokenCompositeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenDateTimeCompositeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenQuantityCompositeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenStringCompositeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + DELETE B + FROM @Ids AS A + INNER JOIN + dbo.TokenNumberNumberCompositeSearchParam AS B + ON B.ResourceTypeId = A.ResourceTypeId + AND B.ResourceSurrogateId = A.ResourceSurrogateId; + INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) + SELECT ResourceSurrogateId, + ClaimTypeId, + ClaimValue + FROM @ResourceWriteClaims; + INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri, + ReferenceResourceTypeId, + ReferenceResourceId, + ReferenceResourceVersion + FROM @ReferenceSearchParams; + INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + Code, + CodeOverflow + FROM @TokenSearchParams; + INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text + FROM @TokenTexts; + INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Text, + TextOverflow, + IsMin, + IsMax + FROM @StringSearchParams; + INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + Uri + FROM @UriSearchParams; + INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SingleValue, + LowValue, + HighValue + FROM @NumberSearchParams; + INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId, + QuantityCodeId, + SingleValue, + LowValue, + HighValue + FROM @QuantitySearchParams; + INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsMin, IsMax) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + StartDateTime, + EndDateTime, + IsLongerThanADay, + IsMin, + IsMax + FROM @DateTimeSearchParams; + INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + BaseUri1, + ReferenceResourceTypeId1, + ReferenceResourceId1, + ReferenceResourceVersion1, + SystemId2, + Code2, + CodeOverflow2 + FROM @ReferenceTokenCompositeSearchParams; + INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SystemId2, + Code2, + CodeOverflow2 + FROM @TokenTokenCompositeSearchParams; + INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + StartDateTime2, + EndDateTime2, + IsLongerThanADay2 + FROM @TokenDateTimeCompositeSearchParams; + INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + SystemId2, + QuantityCodeId2, + LowValue2, + HighValue2 + FROM @TokenQuantityCompositeSearchParams; + INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + Text2, + TextOverflow2 + FROM @TokenStringCompositeSearchParams; + INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange) + SELECT ResourceTypeId, + ResourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + LowValue2, + HighValue2, + SingleValue3, + LowValue3, + HighValue3, + HasRange + FROM @TokenNumberNumberCompositeSearchParams; + COMMIT TRANSACTION; + SET @FailedResources = (SELECT count(*) + FROM @Resources) - @Rows; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'End', @Start = @st, @Rows = @Rows; +END TRY +BEGIN CATCH + IF @@trancount > 0 + ROLLBACK; + EXECUTE dbo.LogEvent @Process = @SP, @Mode = @Mode, @Status = 'Error', @Start = @st; + THROW; +END CATCH + +GO +CREATE PROCEDURE [dbo].[UpdateTaskContext] +@taskId VARCHAR (64), @taskContext VARCHAR (MAX), @runId VARCHAR (50) +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +BEGIN TRANSACTION; +IF NOT EXISTS (SELECT * + FROM [dbo].[TaskInfo] + WHERE TaskId = @taskId + AND RunId = @runId) + BEGIN + THROW 50404, 'Task not exist or runid not match', 1; + END +DECLARE @heartbeatDateTime AS DATETIME2 (7) = SYSUTCDATETIME(); +UPDATE dbo.TaskInfo +SET HeartbeatDateTime = @heartbeatDateTime, + TaskContext = @taskContext +WHERE TaskId = @taskId; +SELECT TaskId, + QueueId, + Status, + TaskTypeId, + RunId, + IsCanceled, + RetryCount, + MaxRetryCount, + HeartbeatDateTime, + InputData, + TaskContext, + Result +FROM [dbo].[TaskInfo] +WHERE TaskId = @taskId; +COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.UpsertResource_7 +@baseResourceSurrogateId BIGINT, @resourceTypeId SMALLINT, @resourceId VARCHAR (64), @eTag INT=NULL, @allowCreate BIT, @isDeleted BIT, @keepHistory BIT, @requireETagOnUpdate BIT, @requestMethod VARCHAR (10), @searchParamHash VARCHAR (64), @rawResource VARBINARY (MAX), @resourceWriteClaims dbo.BulkResourceWriteClaimTableType_1 READONLY, @compartmentAssignments dbo.BulkCompartmentAssignmentTableType_1 READONLY, @referenceSearchParams dbo.BulkReferenceSearchParamTableType_1 READONLY, @tokenSearchParams dbo.BulkTokenSearchParamTableType_2 READONLY, @tokenTextSearchParams dbo.BulkTokenTextTableType_1 READONLY, @stringSearchParams dbo.BulkStringSearchParamTableType_2 READONLY, @numberSearchParams dbo.BulkNumberSearchParamTableType_1 READONLY, @quantitySearchParams dbo.BulkQuantitySearchParamTableType_1 READONLY, @uriSearchParams dbo.BulkUriSearchParamTableType_1 READONLY, @dateTimeSearchParms dbo.BulkDateTimeSearchParamTableType_2 READONLY, @referenceTokenCompositeSearchParams dbo.BulkReferenceTokenCompositeSearchParamTableType_2 READONLY, @tokenTokenCompositeSearchParams dbo.BulkTokenTokenCompositeSearchParamTableType_2 READONLY, @tokenDateTimeCompositeSearchParams dbo.BulkTokenDateTimeCompositeSearchParamTableType_2 READONLY, @tokenQuantityCompositeSearchParams dbo.BulkTokenQuantityCompositeSearchParamTableType_2 READONLY, @tokenStringCompositeSearchParams dbo.BulkTokenStringCompositeSearchParamTableType_2 READONLY, @tokenNumberNumberCompositeSearchParams dbo.BulkTokenNumberNumberCompositeSearchParamTableType_2 READONLY, @isResourceChangeCaptureEnabled BIT=0, @comparedVersion INT=NULL +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +DECLARE @previousResourceSurrogateId AS BIGINT, @previousVersion AS BIGINT, @previousIsDeleted AS BIT, @version AS INT, @resourceSurrogateId AS BIGINT, @InitialTranCount AS INT = @@trancount; +IF @InitialTranCount = 0 + BEGIN TRANSACTION; +SELECT @previousResourceSurrogateId = ResourceSurrogateId, + @previousVersion = Version, + @previousIsDeleted = IsDeleted +FROM dbo.Resource WITH (UPDLOCK, HOLDLOCK) +WHERE ResourceTypeId = @resourceTypeId + AND ResourceId = @resourceId + AND IsHistory = 0; +IF @previousResourceSurrogateId IS NULL + SET @version = 1; +ELSE + BEGIN + IF @isDeleted = 0 + BEGIN + IF @comparedVersion IS NULL + OR @comparedVersion <> @previousVersion + BEGIN + THROW 50409, 'Resource has been recently updated or added, please compare the resource content in code for any duplicate updates', 1; + END + END + SET @version = @previousVersion + 1; + IF @keepHistory = 1 + UPDATE dbo.Resource + SET IsHistory = 1 + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + ELSE + DELETE dbo.Resource + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.ResourceWriteClaim + WHERE ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.CompartmentAssignment + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.ReferenceSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenText + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.StringSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.UriSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.NumberSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.QuantitySearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.DateTimeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.ReferenceTokenCompositeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenTokenCompositeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenDateTimeCompositeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenQuantityCompositeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenStringCompositeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + DELETE dbo.TokenNumberNumberCompositeSearchParam + WHERE ResourceTypeId = @resourceTypeId + AND ResourceSurrogateId = @previousResourceSurrogateId; + END +SET @resourceSurrogateId = @baseResourceSurrogateId + ( NEXT VALUE FOR ResourceSurrogateIdUniquifierSequence); +INSERT INTO dbo.Resource (ResourceTypeId, ResourceId, Version, IsHistory, ResourceSurrogateId, IsDeleted, RequestMethod, RawResource, IsRawResourceMetaSet, SearchParamHash) +SELECT @resourceTypeId, + @resourceId, + @version, + 0, + @resourceSurrogateId, + @isDeleted, + @requestMethod, + @rawResource, + CASE WHEN @version = 1 THEN 1 ELSE 0 END, + @searchParamHash; +INSERT INTO dbo.ResourceWriteClaim (ResourceSurrogateId, ClaimTypeId, ClaimValue) +SELECT @resourceSurrogateId, + ClaimTypeId, + ClaimValue +FROM @resourceWriteClaims; +INSERT INTO dbo.CompartmentAssignment (ResourceTypeId, ResourceSurrogateId, CompartmentTypeId, ReferenceResourceId, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + CompartmentTypeId, + ReferenceResourceId, + 0 +FROM @compartmentAssignments; +INSERT INTO dbo.ReferenceSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri, ReferenceResourceTypeId, ReferenceResourceId, ReferenceResourceVersion, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + BaseUri, + ReferenceResourceTypeId, + ReferenceResourceId, + ReferenceResourceVersion, + 0 +FROM @referenceSearchParams; +INSERT INTO dbo.TokenSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, Code, CodeOverflow, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId, + Code, + CodeOverflow, + 0 +FROM @tokenSearchParams; +INSERT INTO dbo.TokenText (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + Text, + 0 +FROM @tokenTextSearchParams; +INSERT INTO dbo.StringSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Text, TextOverflow, IsHistory, IsMin, IsMax) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + Text, + TextOverflow, + 0, + IsMin, + IsMax +FROM @stringSearchParams; +INSERT INTO dbo.UriSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, Uri, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + Uri, + 0 +FROM @uriSearchParams; +INSERT INTO dbo.NumberSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SingleValue, LowValue, HighValue, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SingleValue, + LowValue, + HighValue, + 0 +FROM @numberSearchParams; +INSERT INTO dbo.QuantitySearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId, QuantityCodeId, SingleValue, LowValue, HighValue, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId, + QuantityCodeId, + SingleValue, + LowValue, + HighValue, + 0 +FROM @quantitySearchParams; +INSERT INTO dbo.DateTimeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, StartDateTime, EndDateTime, IsLongerThanADay, IsHistory, IsMin, IsMax) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + StartDateTime, + EndDateTime, + IsLongerThanADay, + 0, + IsMin, + IsMax +FROM @dateTimeSearchParms; +INSERT INTO dbo.ReferenceTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, BaseUri1, ReferenceResourceTypeId1, ReferenceResourceId1, ReferenceResourceVersion1, SystemId2, Code2, CodeOverflow2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + BaseUri1, + ReferenceResourceTypeId1, + ReferenceResourceId1, + ReferenceResourceVersion1, + SystemId2, + Code2, + CodeOverflow2, + 0 +FROM @referenceTokenCompositeSearchParams; +INSERT INTO dbo.TokenTokenCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SystemId2, Code2, CodeOverflow2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SystemId2, + Code2, + CodeOverflow2, + 0 +FROM @tokenTokenCompositeSearchParams; +INSERT INTO dbo.TokenDateTimeCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, StartDateTime2, EndDateTime2, IsLongerThanADay2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + StartDateTime2, + EndDateTime2, + IsLongerThanADay2, + 0 +FROM @tokenDateTimeCompositeSearchParams; +INSERT INTO dbo.TokenQuantityCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, SystemId2, QuantityCodeId2, LowValue2, HighValue2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + SystemId2, + QuantityCodeId2, + LowValue2, + HighValue2, + 0 +FROM @tokenQuantityCompositeSearchParams; +INSERT INTO dbo.TokenStringCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, Text2, TextOverflow2, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + Text2, + TextOverflow2, + 0 +FROM @tokenStringCompositeSearchParams; +INSERT INTO dbo.TokenNumberNumberCompositeSearchParam (ResourceTypeId, ResourceSurrogateId, SearchParamId, SystemId1, Code1, CodeOverflow1, SingleValue2, LowValue2, HighValue2, SingleValue3, LowValue3, HighValue3, HasRange, IsHistory) +SELECT DISTINCT @resourceTypeId, + @resourceSurrogateId, + SearchParamId, + SystemId1, + Code1, + CodeOverflow1, + SingleValue2, + LowValue2, + HighValue2, + SingleValue3, + LowValue3, + HighValue3, + HasRange, + 0 +FROM @tokenNumberNumberCompositeSearchParams; +SELECT @version; +IF @isResourceChangeCaptureEnabled = 1 + EXECUTE dbo.CaptureResourceChanges @isDeleted = @isDeleted, @version = @version, @resourceId = @resourceId, @resourceTypeId = @resourceTypeId; +IF @InitialTranCount = 0 + COMMIT TRANSACTION; + +GO +CREATE PROCEDURE dbo.UpsertSearchParams +@searchParams dbo.SearchParamTableType_2 READONLY +AS +SET NOCOUNT ON; +SET XACT_ABORT ON; +SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; +BEGIN TRANSACTION; +DECLARE @lastUpdated AS DATETIMEOFFSET (7) = SYSDATETIMEOFFSET(); +DECLARE @summaryOfChanges TABLE ( + Uri VARCHAR (128) COLLATE Latin1_General_100_CS_AS NOT NULL, + Action VARCHAR (20) NOT NULL); +MERGE INTO dbo.SearchParam WITH (TABLOCKX) + AS target +USING @searchParams AS source ON target.Uri = source.Uri +WHEN MATCHED THEN UPDATE +SET Status = source.Status, + LastUpdated = @lastUpdated, + IsPartiallySupported = source.IsPartiallySupported +WHEN NOT MATCHED BY TARGET THEN INSERT (Uri, Status, LastUpdated, IsPartiallySupported) VALUES (source.Uri, source.Status, @lastUpdated, source.IsPartiallySupported) +OUTPUT source.Uri, $ACTION INTO @summaryOfChanges; +SELECT SearchParamId, + SearchParam.Uri +FROM dbo.SearchParam AS searchParam + INNER JOIN + @summaryOfChanges AS upsertedSearchParam + ON searchParam.Uri = upsertedSearchParam.Uri +WHERE upsertedSearchParam.Action = 'INSERT'; +COMMIT TRANSACTION; + +GO diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs index 253a70f2e9..e1d15b6a6f 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersion.cs @@ -77,5 +77,6 @@ public enum SchemaVersion V65 = 65, V66 = 66, V67 = 67, + V68 = 68, } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs index a2217cd496..50c480593d 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs @@ -8,7 +8,7 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Schema public static class SchemaVersionConstants { public const int Min = (int)SchemaVersion.V63; - public const int Max = (int)SchemaVersion.V67; + public const int Max = (int)SchemaVersion.V68; public const int MinForUpgrade = (int)SchemaVersion.V63; // this is used for upgrade tests only public const int SearchParameterStatusSchemaVersion = (int)SchemaVersion.V6; public const int SupportForReferencesWithMissingTypeVersion = (int)SchemaVersion.V7; diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/TransactionCheckWithInitialiScript.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/TransactionCheckWithInitialiScript.sql index 4b8c4ba5de..73c1fd8b94 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/TransactionCheckWithInitialiScript.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Scripts/TransactionCheckWithInitialiScript.sql @@ -19,6 +19,6 @@ Go INSERT INTO dbo.SchemaVersion VALUES - (67, 'started') + (68, 'started') Go diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetResourcesByTypeAndSurrogateIdRange.sql b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetResourcesByTypeAndSurrogateIdRange.sql index be6b269fa6..4b8115c946 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetResourcesByTypeAndSurrogateIdRange.sql +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/Sql/Sprocs/GetResourcesByTypeAndSurrogateIdRange.sql @@ -1,69 +1,60 @@ --DROP PROCEDURE dbo.GetResourcesByTypeAndSurrogateIdRange GO -CREATE PROCEDURE dbo.GetResourcesByTypeAndSurrogateIdRange @ResourceTypeId smallint, @StartId bigint, @EndId bigint, @GlobalStartId bigint = NULL, @GlobalEndId bigint = NULL +CREATE PROCEDURE dbo.GetResourcesByTypeAndSurrogateIdRange @ResourceTypeId smallint, @StartId bigint, @EndId bigint, @GlobalEndId bigint = NULL, @IncludeHistory bit = 0, @IncludeDeleted bit = 0 AS set nocount on DECLARE @SP varchar(100) = 'GetResourcesByTypeAndSurrogateIdRange' ,@Mode varchar(100) = 'RT='+isnull(convert(varchar,@ResourceTypeId),'NULL') +' S='+isnull(convert(varchar,@StartId),'NULL') +' E='+isnull(convert(varchar,@EndId),'NULL') - +' GS='+isnull(convert(varchar,@GlobalStartId),'NULL') -- Is global start id needed? I'm not seeing a usecase for setting it. - +' GE='+isnull(convert(varchar,@GlobalEndId),'NULL') -- Could this just be a boolean for if historical records should be returned? GlobalEndId should equal EndId in all cases I can think of. + +' GE='+isnull(convert(varchar,@GlobalEndId),'NULL') + +' HI='+isnull(convert(varchar,@IncludeHistory),'NULL') + +' DE'+isnull(convert(varchar,@IncludeDeleted),'NULL') ,@st datetime = getUTCdate() + ,@DummyTop bigint = 9223372036854775807 BEGIN TRY - DECLARE @ResourceIds TABLE (ResourceId varchar(64) COLLATE Latin1_General_100_CS_AS, ResourceSurrogateId bigint, RowId int, PRIMARY KEY (ResourceId, RowId)) + DECLARE @ResourceIds TABLE (ResourceId varchar(64) COLLATE Latin1_General_100_CS_AS PRIMARY KEY) + DECLARE @SurrogateIds TABLE (MaxSurrogateId bigint PRIMARY KEY) - IF @GlobalStartId IS NULL -- export from time zero (no lower boundary) - SET @GlobalStartId = 0 - - IF @GlobalEndId IS NOT NULL -- snapshot view + IF @GlobalEndId IS NOT NULL AND @IncludeHistory = 0 -- snapshot view + BEGIN INSERT INTO @ResourceIds - SELECT ResourceId, ResourceSurrogateId, RowId = row_number() OVER (PARTITION BY ResourceId ORDER BY ResourceSurrogateId DESC) + SELECT DISTINCT ResourceId FROM dbo.Resource - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceId IN (SELECT DISTINCT ResourceId - FROM dbo.Resource - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId BETWEEN @StartId AND @EndId - AND IsHistory = 1 - AND IsDeleted = 0 - ) - AND ResourceSurrogateId BETWEEN @GlobalStartId AND @GlobalEndId - - IF EXISTS (SELECT * FROM @ResourceIds) - BEGIN - DECLARE @SurrogateIdMap TABLE (MaxSurrogateId bigint PRIMARY KEY) - INSERT INTO @SurrogateIdMap - SELECT MaxSurrogateId = A.ResourceSurrogateId - FROM (SELECT * FROM @ResourceIds WHERE RowId = 1 AND ResourceSurrogateId BETWEEN @StartId AND @EndId) A + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + AND IsHistory = 1 + AND (IsDeleted = 0 OR @IncludeDeleted = 1) + OPTION (MAXDOP 1) - SELECT @ResourceTypeId - ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.ResourceId ELSE A.ResourceId END - ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.Version ELSE A.Version END - ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.IsDeleted ELSE A.IsDeleted END - ,isnull(C.ResourceSurrogateId, A.ResourceSurrogateId) - ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.RequestMethod ELSE A.RequestMethod END - ,IsMatch = convert(bit,1) - ,IsPartial = convert(bit,0) - ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.IsRawResourceMetaSet ELSE A.IsRawResourceMetaSet END - ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.SearchParamHash ELSE A.SearchParamHash END - ,CASE WHEN C.ResourceSurrogateId IS NOT NULL THEN C.RawResource ELSE A.RawResource END - FROM dbo.Resource A - LEFT OUTER JOIN @SurrogateIdMap B ON B.MaxSurrogateId = A.ResourceSurrogateId - LEFT OUTER JOIN dbo.Resource C ON C.ResourceTypeId = @ResourceTypeId AND C.ResourceSurrogateId = MaxSurrogateId - WHERE A.ResourceTypeId = @ResourceTypeId - AND A.ResourceSurrogateId BETWEEN @StartId AND @EndId - AND (A.IsHistory = 0 OR MaxSurrogateId IS NOT NULL) - AND A.IsDeleted = 0 + IF @@rowcount > 0 + INSERT INTO @SurrogateIds + SELECT ResourceSurrogateId + FROM (SELECT ResourceId, ResourceSurrogateId, RowId = row_number() OVER (PARTITION BY ResourceId ORDER BY ResourceSurrogateId DESC) + FROM dbo.Resource WITH (INDEX = IX_Resource_ResourceTypeId_ResourceId_Version) -- w/o hint access to Resource table is inefficient when many versions are present. Hint is ignored if Resource is a view. + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceId IN (SELECT TOP (@DummyTop) ResourceId FROM @ResourceIds) + AND ResourceSurrogateId BETWEEN @StartId AND @GlobalEndId + ) A + WHERE RowId = 1 + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + OPTION (MAXDOP 1, OPTIMIZE FOR (@DummyTop = 1)) END - ELSE - SELECT ResourceTypeId, ResourceId, Version, IsDeleted, ResourceSurrogateId, RequestMethod, IsMatch = convert(bit,1), IsPartial = convert(bit,0), IsRawResourceMetaSet, SearchParamHash, RawResource - FROM dbo.Resource - WHERE ResourceTypeId = @ResourceTypeId - AND ResourceSurrogateId BETWEEN @StartId AND @EndId - AND IsHistory = 0 - AND IsDeleted = 0 + + SELECT ResourceTypeId, ResourceId, Version, IsDeleted, ResourceSurrogateId, RequestMethod, IsMatch = convert(bit,1), IsPartial = convert(bit,0), IsRawResourceMetaSet, SearchParamHash, RawResource + FROM dbo.Resource + WHERE ResourceTypeId = @ResourceTypeId + AND ResourceSurrogateId BETWEEN @StartId AND @EndId + AND (IsHistory = 0 OR @IncludeHistory = 1) + AND (IsDeleted = 0 OR @IncludeDeleted = 1) + UNION ALL + SELECT ResourceTypeId, ResourceId, Version, IsDeleted, ResourceSurrogateId, RequestMethod, IsMatch = convert(bit,1), IsPartial = convert(bit,0), IsRawResourceMetaSet, SearchParamHash, RawResource + FROM @SurrogateIds + JOIN dbo.Resource ON ResourceTypeId = @ResourceTypeId AND ResourceSurrogateId = MaxSurrogateId + WHERE IsHistory = 1 + AND (IsDeleted = 0 OR @IncludeDeleted = 1) + OPTION (MAXDOP 1) EXECUTE dbo.LogEvent @Process=@SP,@Mode=@Mode,@Status='End',@Start=@st,@Rows=@@rowcount END TRY diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs index 97e08c3b10..4a5ec4e5c6 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs @@ -191,7 +191,8 @@ public override object VisitSqlRoot(SqlRootExpression expression, SearchOptions StringBuilder.Append("FROM ").Append(VLatest.Resource).Append(" ").Append(resourceTableAlias); if (expression.SearchParamTableExpressions.Count == 0 && - !_searchType.HasFlag(SqlSearchType.History) && + !_searchType.HasFlag(SqlSearchType.IncludeHistory) && + !_searchType.HasFlag(SqlSearchType.IncludeDeleted) && expression.ResourceTableExpressions.Any(e => e.AcceptVisitor(ExpressionContainsParameterVisitor.Instance, SearchParameterNames.ResourceType)) && !expression.ResourceTableExpressions.Any(e => e.AcceptVisitor(ExpressionContainsParameterVisitor.Instance, SearchParameterNames.Id))) { @@ -1314,19 +1315,21 @@ int FindImpl(int currentIndex) private void AppendDeletedClause(in IndentedStringBuilder.DelimitedScope delimited, string tableAlias = null) { - if (!_searchType.HasFlag(SqlSearchType.History)) + if (!_searchType.HasFlag(SqlSearchType.IncludeDeleted)) { - delimited.BeginDelimitedElement().Append(VLatest.Resource.IsDeleted, tableAlias).Append(" = 0"); + delimited.BeginDelimitedElement(); + + StringBuilder.Append(VLatest.Resource.IsDeleted, tableAlias).Append(" = 0 "); } } private void AppendHistoryClause(in IndentedStringBuilder.DelimitedScope delimited, string tableAlias = null) { - if (!_searchType.HasFlag(SqlSearchType.History)) + if (!_searchType.HasFlag(SqlSearchType.IncludeHistory)) { delimited.BeginDelimitedElement(); - StringBuilder.Append(VLatest.Resource.IsHistory, tableAlias).Append(" = 0"); + StringBuilder.Append(VLatest.Resource.IsHistory, tableAlias).Append(" = 0 "); } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlSearchOptions.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlSearchOptions.cs index 0dae4ca022..aa4c43bdb0 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlSearchOptions.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlSearchOptions.cs @@ -34,5 +34,27 @@ public SqlSearchOptions(SearchOptions searchOptions) /// Performs a shallow clone of this instance /// public SqlSearchOptions CloneSqlSearchOptions() => (SqlSearchOptions)MemberwiseClone(); + + internal SqlSearchType GetSearchTypeFromOptions() + { + SqlSearchType searchType = 0; + + if (ResourceVersionTypes.HasFlag(ResourceVersionType.Latest)) + { + searchType |= SqlSearchType.Default; + } + + if (ResourceVersionTypes.HasFlag(ResourceVersionType.Histoy)) + { + searchType |= SqlSearchType.IncludeHistory; + } + + if (ResourceVersionTypes.HasFlag(ResourceVersionType.SoftDeleted)) + { + searchType |= SqlSearchType.IncludeDeleted; + } + + return searchType; + } } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlSearchType.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlSearchType.cs index 6760be6329..dacbc31368 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlSearchType.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlSearchType.cs @@ -16,13 +16,16 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Search [Flags] internal enum SqlSearchType { - // Set if we do not need to consider history or reindexing + // Set if we do not need to consider history, deleted resources, or re-indexing Default = 0, - // Set if we are including previous resource versions or deleted resources - History = 1 << 0, + // Set if we are including previous resource versions without deleted resources + IncludeHistory = 1 << 0, // Set if the search parameter hash value needs to be considered in a search Reindex = 1 << 1, + + // Set if we are including deleted resource versions + IncludeDeleted = 1 << 2, } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs index 6aac0c4f0e..f2a7804748 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs @@ -122,8 +122,11 @@ public SqlServerSearchService( public override async Task SearchAsync(SearchOptions searchOptions, CancellationToken cancellationToken) { SqlSearchOptions sqlSearchOptions = new SqlSearchOptions(searchOptions); - SearchResult searchResult = await SearchImpl(sqlSearchOptions, SqlSearchType.Default, null, cancellationToken); + SqlSearchType searchType = sqlSearchOptions.GetSearchTypeFromOptions(); + + SearchResult searchResult = await SearchImpl(sqlSearchOptions, searchType, null, cancellationToken); int resultCount = searchResult.Results.Count(); + if (!sqlSearchOptions.IsSortWithFilter && searchResult.ContinuationToken == null && resultCount <= sqlSearchOptions.MaxItemCount && @@ -200,12 +203,6 @@ public override async Task SearchAsync(SearchOptions searchOptions return searchResult; } - protected override async Task SearchHistoryInternalAsync(SearchOptions searchOptions, CancellationToken cancellationToken) - { - SqlSearchOptions sqlSearchOptions = new SqlSearchOptions(searchOptions); - return await SearchImpl(sqlSearchOptions, SqlSearchType.History, null, cancellationToken); - } - private async Task SearchImpl(SqlSearchOptions sqlSearchOptions, SqlSearchType searchType, string currentSearchParameterHash, CancellationToken cancellationToken) { Expression searchExpression = sqlSearchOptions.Expression; @@ -282,6 +279,7 @@ private async Task SearchImpl(SqlSearchOptions sqlSearchOptions, S searchExpression = searchExpression?.AcceptVisitor(RemoveIncludesRewriter.Instance); } + // ! - Trace SqlRootExpression expression = (SqlRootExpression)searchExpression ?.AcceptVisitor(LastUpdatedToResourceSurrogateIdRewriter.Instance) .AcceptVisitor(_compartmentSearchRewriter) @@ -549,18 +547,19 @@ private void PopulateSqlCommandFromQueryHints(SqlSearchOptions options, SqlComma var globalStartId = long.Parse(hints.First(_ => _.Param == KnownQueryParameterNames.GlobalStartSurrogateId).Value); var globalEndId = long.Parse(hints.First(_ => _.Param == KnownQueryParameterNames.GlobalEndSurrogateId).Value); - PopulateSqlCommandFromQueryHints(command, resourceTypeId, startId, endId, globalStartId, globalEndId); + PopulateSqlCommandFromQueryHints(command, resourceTypeId, startId, endId, globalStartId, globalEndId, options.ResourceVersionTypes.HasFlag(ResourceVersionType.Histoy), options.ResourceVersionTypes.HasFlag(ResourceVersionType.SoftDeleted)); } - private static void PopulateSqlCommandFromQueryHints(SqlCommand command, short resourceTypeId, long startId, long endId, long? globalStartId, long? globalEndId) + private static void PopulateSqlCommandFromQueryHints(SqlCommand command, short resourceTypeId, long startId, long endId, long? globalStartId, long? globalEndId, bool? includeHistory, bool? includeDeleted) { command.CommandType = CommandType.StoredProcedure; command.CommandText = "dbo.GetResourcesByTypeAndSurrogateIdRange"; command.Parameters.AddWithValue("@ResourceTypeId", resourceTypeId); command.Parameters.AddWithValue("@StartId", startId); command.Parameters.AddWithValue("@EndId", endId); - command.Parameters.AddWithValue("@GlobalStartId", globalStartId); command.Parameters.AddWithValue("@GlobalEndId", globalEndId); + command.Parameters.AddWithValue("@IncludeHistory", includeHistory); + command.Parameters.AddWithValue("@IncludeDeleted", includeDeleted); } /// @@ -573,13 +572,15 @@ private static void PopulateSqlCommandFromQueryHints(SqlCommand command, short r /// The upper bound for the window of time to consider for historical records /// Cancellation token /// When not null then we filter using the searchParameterHash + /// Return historical records that match the other parameters. + /// Return deleted records that match the other parameters. /// All resources with surrogate ids greater than or equal to startId and less than or equal to endId. If windowEndId is set it will return the most recent version of a resource that was created before windowEndId that is within the range of startId to endId. - public async Task SearchBySurrogateIdRange(string resourceType, long startId, long endId, long? windowStartId, long? windowEndId, CancellationToken cancellationToken, string searchParamHashFilter = null) + public async Task SearchBySurrogateIdRange(string resourceType, long startId, long endId, long? windowStartId, long? windowEndId, CancellationToken cancellationToken, string searchParamHashFilter = null, bool includeHistory = false, bool includeDeleted = false) { var resourceTypeId = _model.GetResourceTypeId(resourceType); using var sqlCommand = new SqlCommand(); sqlCommand.CommandTimeout = GetReindexCommandTimeout(); - PopulateSqlCommandFromQueryHints(sqlCommand, resourceTypeId, startId, endId, windowStartId, windowEndId); + PopulateSqlCommandFromQueryHints(sqlCommand, resourceTypeId, startId, endId, windowStartId, windowEndId, includeHistory, includeDeleted); LogSqlCommand(sqlCommand); List resources = null; await _sqlRetryService.ExecuteSql( @@ -694,9 +695,9 @@ public override async Task> GetUsedResourceTypes(Cancellat private SqlSearchOptions UpdateSort(SqlSearchOptions searchOptions, Expression searchExpression, SqlSearchType sqlSearchType) { SqlSearchOptions newSearchOptions = searchOptions; - if (sqlSearchType == SqlSearchType.History) + if (sqlSearchType.HasFlag(SqlSearchType.IncludeHistory) && searchOptions.Sort.Any()) { - // history is always sorted by _lastUpdated. + // history is always sorted by _lastUpdated (except for export). newSearchOptions = searchOptions.CloneSqlSearchOptions(); return newSearchOptions; diff --git a/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj b/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj index 9148286c52..035bb9db4b 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj +++ b/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj @@ -1,7 +1,7 @@  - + - 67 + 68 Features\Schema\Migrations\$(LatestSchemaVersion).sql diff --git a/src/Microsoft.Health.Fhir.Stu3.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs b/src/Microsoft.Health.Fhir.Stu3.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs index 676497b807..475bf8f78d 100644 --- a/src/Microsoft.Health.Fhir.Stu3.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs +++ b/src/Microsoft.Health.Fhir.Stu3.Core.UnitTests/Features/Search/SearchOptionsFactoryTests.cs @@ -149,8 +149,8 @@ public void GivenSearchParamsWithValidCompartmentSearch_WhenCreated_ThenCorrectC SearchOptions options = CreateSearchOptions( resourceType: resourceType.ToString(), queryParameters: queryParameters, - compartmentType.ToString(), - compartmentId); + compartmentType: compartmentType.ToString(), + compartmentId: compartmentId); Assert.NotNull(options); Assert.NotNull(options.Expression); diff --git a/test/Microsoft.Health.Fhir.R4.Tests.E2E/Microsoft.Health.Fhir.R4.Tests.E2E.csproj b/test/Microsoft.Health.Fhir.R4.Tests.E2E/Microsoft.Health.Fhir.R4.Tests.E2E.csproj index a78523efaa..0abc057fbf 100644 --- a/test/Microsoft.Health.Fhir.R4.Tests.E2E/Microsoft.Health.Fhir.R4.Tests.E2E.csproj +++ b/test/Microsoft.Health.Fhir.R4.Tests.E2E/Microsoft.Health.Fhir.R4.Tests.E2E.csproj @@ -15,8 +15,8 @@ - - + + diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.projitems b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.projitems index 8c2d0228a8..6d360d6ca6 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.projitems +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.projitems @@ -32,10 +32,10 @@ - - - - + + + + @@ -68,7 +68,7 @@ - + @@ -116,7 +116,7 @@ - + diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.shproj b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.shproj index e97728cced..174ed2ed52 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.shproj +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Microsoft.Health.Fhir.Shared.Tests.E2E.shproj @@ -9,8 +9,8 @@ - - + + diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/AnonymizedExportTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/AnonymizedExportTests.cs similarity index 99% rename from test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/AnonymizedExportTests.cs rename to test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/AnonymizedExportTests.cs index 4aadd3e39a..de197526da 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/AnonymizedExportTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/AnonymizedExportTests.cs @@ -31,12 +31,12 @@ using FhirGroup = Hl7.Fhir.Model.Group; using Task = System.Threading.Tasks.Task; -namespace Microsoft.Health.Fhir.Tests.E2E.Rest +namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Export { [Trait(Traits.OwningTeam, OwningTeam.FhirImport)] [Trait(Traits.Category, Categories.AnonymizedExport)] [HttpIntegrationFixtureArgumentSets(DataStore.All, Format.Json)] - public class AnonymizedExportTests : IClassFixture + public class AnonymizedExportTests : IClassFixture { private const string TestExportStoreUriEnvironmentVariableName = "TestExportStoreUri"; private const string TestExportStoreKeyEnvironmentVariableName = "TestExportStoreKey"; @@ -52,7 +52,7 @@ public class AnonymizedExportTests : IClassFixture ] }"; - public AnonymizedExportTests(ExportTestFixture fixture) + public AnonymizedExportTests(ExportDataTestFixture fixture) { _isUsingInProcTestServer = fixture.IsUsingInProcTestServer; _testFhirClient = fixture.TestFhirClient; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/AnonymizedExportUsingAcrTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/AnonymizedExportUsingAcrTests.cs similarity index 99% rename from test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/AnonymizedExportUsingAcrTests.cs rename to test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/AnonymizedExportUsingAcrTests.cs index c4b1252c93..08360f9087 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/AnonymizedExportUsingAcrTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/AnonymizedExportUsingAcrTests.cs @@ -35,12 +35,12 @@ using FhirGroup = Hl7.Fhir.Model.Group; using Task = System.Threading.Tasks.Task; -namespace Microsoft.Health.Fhir.Tests.E2E.Rest +namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Export { [Trait(Traits.OwningTeam, OwningTeam.FhirImport)] [Trait(Traits.Category, Categories.AnonymizedExport)] [HttpIntegrationFixtureArgumentSets(DataStore.All, Format.Json)] - public class AnonymizedExportUsingAcrTests : IClassFixture + public class AnonymizedExportUsingAcrTests : IClassFixture { private const string TestRepositoryName = "testanonymizationconfigs"; private const string TestConfigName = "testconfigname.json"; @@ -60,7 +60,7 @@ public class AnonymizedExportUsingAcrTests : IClassFixture ] }"; - public AnonymizedExportUsingAcrTests(ExportTestFixture fixture) + public AnonymizedExportUsingAcrTests(ExportDataTestFixture fixture) { _isUsingInProcTestServer = fixture.IsUsingInProcTestServer; _testFhirClient = fixture.TestFhirClient; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs new file mode 100644 index 0000000000..b288aafd19 --- /dev/null +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTestFixture.cs @@ -0,0 +1,260 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using Antlr4.Runtime.Atn; +using Hl7.Fhir.Model; +using MediatR; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Health.Fhir.Client; +using Microsoft.Health.Fhir.Core.Features.Operations.Export; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using Microsoft.Health.Fhir.Tests.E2E.Rest.Metric; +using Task = System.Threading.Tasks.Task; + +namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Export +{ + public class ExportDataTestFixture : HttpIntegrationTestFixture + { + private MetricHandler _metricHandler; + + public ExportDataTestFixture(DataStore dataStore, Format format, TestFhirServerFactory testFhirServerFactory) + : base(dataStore, format, testFhirServerFactory) + { + DataStore = dataStore; + } + + public MetricHandler MetricHandler + { + get => _metricHandler ?? (_metricHandler = (MetricHandler)(TestFhirServer as InProcTestFhirServer)?.Server.Host.Services.GetRequiredService>()); + } + + internal DataStore DataStore { get; private set; } + + internal Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestResourcesWithHistoryAndDeletes { get; } = new(); + + internal Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestResourcesWithHistory => TestResourcesWithHistoryAndDeletes + .Where(entry => !entry.Value.Meta.Extension.Any(extension => + extension.Url == "http://azurehealthcareapis.com/data-extensions/deleted-state" + && ((FhirString)extension.Value).Value == "soft-deleted")) + .ToDictionary(entry => entry.Key, entry => entry.Value); + + internal Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestResourcesWithDeletes => TestResourcesWithHistoryAndDeletes + .GroupBy(entry => entry.Key.resourceId) + .Select(group => group.OrderByDescending(entry => entry.Value.Meta.LastUpdated).First()) + .ToDictionary(entry => entry.Key, entry => entry.Value); + + internal Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestResources => + TestResourcesWithHistory.Where(pair => TestResourcesWithDeletes.ContainsKey(pair.Key)).ToDictionary(pair => pair.Key, pair => pair.Value); + + // If the patient is deleted but the child resources are not, they should not be returned in patient centric exports. + internal Dictionary<(string resourceType, string resourceId, string versionId), Resource> TestPatientCompartmentResources => TestResources + .Where(x => x.Key.resourceType != "Encounter" || TestResources.Keys.Any(pat => pat.resourceType == "Patient" && pat.resourceId == (x.Value as Encounter).Subject.Reference.Split("/")[1])) + .Where(x => x.Key.resourceType != "Observation" || TestResources.Keys.Any(pat => pat.resourceType == "Patient" && pat.resourceId == (x.Value as Observation).Subject.Reference.Split("/")[1])) + .ToDictionary(pair => pair.Key, pair => pair.Value); + + internal string FixtureTag { get; } = Guid.NewGuid().ToString(); + + internal DateTime TestDataInsertionTime { get; } = DateTime.UtcNow; + + internal string ExportTestFilterQueryParameters(params string[] uniqueResourceTypes) + { + if (uniqueResourceTypes.Length == 0) + { + uniqueResourceTypes = TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct().ToArray(); + } + + var typeFilterPart = string.Join(',', uniqueResourceTypes.Select(rt => $"{rt}%3F_tag%3D{FixtureTag}")); + + return $"_type={string.Join(',', uniqueResourceTypes)}&_typeFilter={typeFilterPart}"; + } + + protected override async Task OnInitializedAsync() + { + await SaveTestResourcesToServer(); + } + + private async Task SaveTestResourcesToServer() + { + void AddResourceToTestResources(Resource resource) => + TestResourcesWithHistoryAndDeletes[(resource.TypeName, resource.Id, resource.VersionId)] = resource; + + void AddResourcesToTestResources(List resources) => resources.ForEach(AddResourceToTestResources); + + var testResourcesInfo = GenerateTestResources().Select(x => (x, false)).ToList(); + + while (testResourcesInfo.Count > 0) + { + var testResourceResponse = await SaveResourceListToServer(testResourcesInfo); + + AddResourcesToTestResources(testResourceResponse); + + testResourcesInfo = new(); + + for (int i = 0; i < testResourceResponse.Count; i++) + { + var resource = testResourceResponse[i]; + + if (resource.Meta.Extension.Any(x => x.Url == KnownFhirPaths.AzureSoftDeletedExtensionUrl)) + { + // Skip already deleted resources for now. + // TODO - add un-deletes in here. + continue; + } + else if (i % 10 == 0) + { + testResourcesInfo.Add((resource.DeepCopy() as Resource, true)); + } + else if (i % 4 == 1) + { + Resource updatedResource = resource.DeepCopy() as Resource; + + if (updatedResource is Patient) + { + Patient updatedPatient = updatedResource as Patient; + updatedPatient.Name.Add(new() + { + Given = updatedPatient.Name.First().Given, + Family = $"UpdatedFromVersion{updatedResource.Meta.VersionId}", + }); + testResourcesInfo.Add((updatedPatient, false)); + } + + if (updatedResource is Encounter) + { + Encounter updatedEncounter = updatedResource as Encounter; + updatedEncounter.Type.Add(new CodeableConcept("http://e2e-test", $"UpdatedFromVersion{updatedResource.Meta.VersionId}")); + testResourcesInfo.Add((updatedEncounter, false)); + } + + if (updatedResource is Observation) + { + Observation updatedObservation = updatedResource as Observation; + updatedObservation.Category.Add(new CodeableConcept("http://e2e-test", $"UpdatedFromVersion{updatedResource.Meta.VersionId}")); + testResourcesInfo.Add((updatedObservation, false)); + } + } + } + } + } + + private async System.Threading.Tasks.Task> SaveResourceListToServer(List<(Resource resource, bool delete)> entries) + { + if (entries.Count > 500) + { + throw new ArgumentException("The number of resources to save must be less than or equal to 500."); + } + + var bundle = new Bundle + { + Type = Bundle.BundleType.Batch, + Entry = new List(), + }; + + foreach (var entry in entries) + { + bundle.Entry.Add(new Bundle.EntryComponent + { + Resource = entry.resource, + Request = new Bundle.RequestComponent + { + Method = entry.delete ? Bundle.HTTPVerb.DELETE : Bundle.HTTPVerb.PUT, + Url = $"{entry.resource.TypeName}/{entry.resource.Id}", + }, + FullUrl = $"{TestFhirClient.HttpClient.BaseAddress}{entry.resource.TypeName}/{entry.resource.Id}", + }); + } + + FhirResponse response = await TestFhirClient.PostBundleAsync(bundle); + + response.Response.EnsureSuccessStatusCode(); + + List rtn = new(); + + for (int i = 0; i < response.Resource.Entry.Count; i++) + { + var inputResource = entries[i].resource; + var responseEntry = response.Resource.Entry[i]; + + if (responseEntry.Resource is not null) + { + rtn.Add(responseEntry.Resource); + } + else + { + var allResourcesWithDeleted = await TestFhirClient.SearchAsync($"{inputResource.TypeName}/{inputResource.Id}/_history"); + var deletedResource = allResourcesWithDeleted.Resource.Entry.OrderByDescending(x => x.Resource.Meta.LastUpdated).First().Resource; + deletedResource.Meta.Extension.Add(new Extension(KnownFhirPaths.AzureSoftDeletedExtensionUrl, new FhirString("soft-deleted"))); + + // The history endpoint does not return the version id in the resource, so we need to parse it from the etag. + var etagVersionMatch = Regex.Match(responseEntry.Response.Etag, @"\d+"); + + if (deletedResource.Meta.VersionId is null && etagVersionMatch.Success) + { + deletedResource.Meta.VersionId = etagVersionMatch.Value; + } + + rtn.Add(deletedResource); + } + } + + return rtn; + } + + // 27 patients, 54 encounters, and, 108 observations. + private List GenerateTestResources(int numberOfPatients = 27, int numberOfEncountersPerPatient = 2, int numberOfObservationsPerEncounter = 2) + { + var resources = new List(); + + for (int i = 0; i < numberOfPatients; i++) + { + var patient = new Patient() + { + Id = Guid.NewGuid().ToString("N"), + Meta = new() { Tag = new List() { new Coding("http://e2e-test", FixtureTag) }}, + Active = true, + Name = new List() { new HumanName() { Family = $"Test{i}", Given = new List { "Export", "History", "SoftDelete" } } }, + }; + resources.Add(patient); + + for (int j = 0; j < numberOfEncountersPerPatient; j++) + { + Encounter encounter = new() + { + Id = Guid.NewGuid().ToString("N"), + Meta = new() { Tag = new List() { new Coding("http://e2e-test", FixtureTag) } }, + Status = Encounter.EncounterStatus.Planned, + Type = new() { new CodeableConcept("http://e2e-test", $"Test{i}") }, + Class = new Coding("http://e2e-test", $"Test{i}"), + Subject = new ResourceReference($"Patient/{patient.Id}"), + }; + resources.Add(encounter); + + for (int k = 0; k < numberOfObservationsPerEncounter; k++) + { + Observation observation = new() + { + Id = Guid.NewGuid().ToString("N"), + Meta = new() { Tag = new List() { new Coding("http://e2e-test", FixtureTag) } }, + Status = ObservationStatus.Preliminary, + Category = new() { new CodeableConcept("http://e2e-test", $"Test{i}") }, + Code = new CodeableConcept("http://e2e-test", $"Test{i}"), + Subject = new ResourceReference($"Patient/{patient.Id}"), + + // Encounter = new ResourceReference($"Encounter/{encounter.Id}"), + }; + resources.Add(observation); + } + } + } + + return resources; + } + } +} diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs new file mode 100644 index 0000000000..7d8d5fb77c --- /dev/null +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataTests.cs @@ -0,0 +1,222 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using Hl7.Fhir.Model; +using Hl7.Fhir.Serialization; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using Microsoft.Health.Fhir.Tests.E2E.Common; +using Microsoft.Health.Test.Utilities; +using Xunit; +using Xunit.Abstractions; +using Task = System.Threading.Tasks.Task; + +namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Export +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Export)] + [HttpIntegrationFixtureArgumentSets(DataStore.All, Format.Json)] + public class ExportDataTests : IClassFixture + { + private readonly TestFhirClient _testFhirClient; + private readonly ITestOutputHelper _outputHelper; + private readonly FhirJsonParser _fhirJsonParser; + private readonly ExportDataTestFixture _fixture; + + public ExportDataTests(ExportDataTestFixture fixture, ITestOutputHelper testOutputHelper) + { + _testFhirClient = fixture.TestFhirClient; + _outputHelper = testOutputHelper; + _fhirJsonParser = new FhirJsonParser(); + _fixture = fixture; + } + + [Fact] + public async Task GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() + { + // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally.\ + string parameters = _fixture.ExportTestFilterQueryParameters(); + + // Trigger export request and check for export status + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: parameters); + + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); + + // Download exported data from storage account + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = + await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); + + // Assert both data are equal + Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResources, dataFromExport, _outputHelper)); + } + + [Fact] + public async Task GivenFhirServer_WhenPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() + { + // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. + string parameters = _fixture.ExportTestFilterQueryParameters(); + + // Trigger export request and check for export status + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(path: "Patient/", parameters: parameters); + + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); + + // Download exported data from storage account + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = + await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); + + // Assert both data are equal + Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestPatientCompartmentResources, dataFromExport, _outputHelper)); + } + + [Fact] + public async Task GivenFhirServer_WhenAllObservationAndPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() + { + // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. + string[] testResorceTypes = { "Observation", "Patient" }; + var parameters = _fixture.ExportTestFilterQueryParameters(testResorceTypes); + + // Trigger export request and check for export status + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: parameters); + + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); + + // Download exported data from storage account + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = + await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); + + var expectedResources = _fixture.TestResources + .Where(r => testResorceTypes.Contains(r.Key.resourceType)) + .ToDictionary(x => x.Key, x => x.Value); + + // Assert both data are equal + Assert.True(ExportTestHelper.ValidateDataFromBothSources(expectedResources, dataFromExport, _outputHelper)); + } + + [Fact] + public async Task GivenFhirServer_WhenPatientObservationDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() + { + // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. + var parameters = _fixture.ExportTestFilterQueryParameters("Observation"); + + // Trigger export request and check for export status + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(path: "Patient/", parameters: parameters); + + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); + + // Download exported data from storage account + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = + await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); + + var expectedResources = _fixture.TestPatientCompartmentResources + .Where(r => r.Key.resourceType == "Observation") + .ToDictionary(x => x.Key, x => x.Value); + + // Assert both data are equal. Only Observation data is expected due to the type query parameter. + Assert.True(ExportTestHelper.ValidateDataFromBothSources(expectedResources, dataFromExport, _outputHelper)); + } + + // No need to test both code paths for testing container is written to. + [Fact] + public async Task GivenFhirServer_WhenAllDataIsExportedToASpecificContainer_ThenExportedDataIsInTheSpecifiedContianer() + { + // NOTE: Azurite or Azure Storage Explorer is required to run these tests locally. + string testContainer = "test-container"; + + // Trigger export request and check for export status + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: $"_container={testContainer}&{_fixture.ExportTestFilterQueryParameters()}"); + + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); + + // Download exported data from storage account + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = + await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); + + // Assert both data are equal + Assert.True(ExportTestHelper.ValidateDataFromBothSources(_fixture.TestResources, dataFromExport, _outputHelper)); + Assert.True(blobUris.All((url) => url.OriginalString.Contains(testContainer))); + } + + [Fact] + [Trait(Traits.Category, Categories.ExportLongRunning)] + [HttpIntegrationFixtureArgumentSets(dataStores: DataStore.SqlServer)] + public async Task GivenFhirServer_WhenDataIsExportedWithHistoryParallel_ThenExportedDataIsSameAsDataInFhirServer() + { + await ExportAndSoftDeleteTestHelper(parallel: true, history: true, deletes: false); + } + + [Fact] + [Trait(Traits.Category, Categories.ExportLongRunning)] + public async Task GivenFhirServer_WhenDataIsExportedWithHistoryNotParallel_ThenExportedDataIsSameAsDataInFhirServer() + { + await ExportAndSoftDeleteTestHelper(parallel: false, history: true, deletes: false); + } + + [Fact] + [Trait(Traits.Category, Categories.ExportLongRunning)] + [HttpIntegrationFixtureArgumentSets(dataStores: DataStore.SqlServer)] + public async Task GivenFhirServer_WhenDataIsExportedWithSoftDeletesParallel_ThenExportedDataIsSameAsDataInFhirServer() + { + await ExportAndSoftDeleteTestHelper(parallel: true, history: false, deletes: true); + } + + [Fact] + [Trait(Traits.Category, Categories.ExportLongRunning)] + public async Task GivenFhirServer_WhenDataIsExportedWithSoftDeletesNotParallel_ThenExportedDataIsSameAsDataInFhirServer() + { + await ExportAndSoftDeleteTestHelper(parallel: false, history: false, deletes: true); + } + + [Fact] + [HttpIntegrationFixtureArgumentSets(dataStores: DataStore.SqlServer)] + public async Task GivenFhirServer_WhenDataIsExportedWithHistoryAndSoftDeletesParallel_ThenExportedDataIsSameAsDataInFhirServer() + { + await ExportAndSoftDeleteTestHelper(parallel: true, history: true, deletes: true); + } + + [Fact] + [Trait(Traits.Category, Categories.ExportLongRunning)] + public async Task GivenFhirServer_WhenDataIsExportedWithHistoryAndSoftDeletesNotParallel_ThenExportedDataIsSameAsDataInFhirServer() + { + await ExportAndSoftDeleteTestHelper(parallel: false, history: true, deletes: true); + } + + // _tag filter cannot be used with history or deleted export. Using isParallel to test both SQL code paths. + private async Task ExportAndSoftDeleteTestHelper(bool parallel, bool history, bool deletes) + { + string uniqueFixtureResources = string.Join(',', _fixture.TestResourcesWithHistoryAndDeletes.Keys.Select(x => x.resourceType).Distinct()); + string includeAssociatedDataParam = (history ? "_history" : string.Empty) + (deletes ? (history ? "," : string.Empty) + "_deleted" : string.Empty); + + // Trigger export request and check for export status. _typeFilter and history/soft delete parameters cannot be used together. + string parallelQueryParam = $"_isParallel={parallel}"; + Uri contentLocation = await _fixture.TestFhirClient.ExportAsync(parameters: $"_since={_fixture.TestDataInsertionTime:O}&_type={uniqueFixtureResources}&includeAssociatedData={includeAssociatedDataParam}&{parallelQueryParam}"); + + IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); + + // Download exported data from storage account + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = + await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); + + var expectedResources = _fixture.TestResourcesWithHistoryAndDeletes; + + if (!history) + { + expectedResources = _fixture.TestResourcesWithDeletes; + } + + if (!deletes) + { + expectedResources = _fixture.TestResourcesWithHistory; + } + + // Assert both data are equal + Assert.True(ExportTestHelper.ValidateDataFromBothSources(expectedResources, dataFromExport, _outputHelper)); + } + } +} diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportDataValidationTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataValidationTests.cs similarity index 83% rename from test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportDataValidationTests.cs rename to test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataValidationTests.cs index c9ddbac0c2..468220bccc 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportDataValidationTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportDataValidationTests.cs @@ -19,20 +19,20 @@ using FhirGroup = Hl7.Fhir.Model.Group; using Task = System.Threading.Tasks.Task; -namespace Microsoft.Health.Fhir.Tests.E2E.Rest +namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Export { [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.Export)] [Trait(Traits.Category, Categories.ExportDataValidation)] [HttpIntegrationFixtureArgumentSets(DataStore.All, Format.Json)] - public class ExportDataValidationTests : IClassFixture + public class ExportDataValidationTests : IClassFixture { private readonly TestFhirClient _testFhirClient; private readonly ITestOutputHelper _outputHelper; private readonly FhirJsonParser _fhirJsonParser; - private readonly ExportTestFixture _fixture; + private readonly ExportDataTestFixture _fixture; - public ExportDataValidationTests(ExportTestFixture fixture, ITestOutputHelper testOutputHelper) + public ExportDataValidationTests(ExportDataTestFixture fixture, ITestOutputHelper testOutputHelper) { _testFhirClient = fixture.TestFhirClient; _outputHelper = testOutputHelper; @@ -43,7 +43,7 @@ public ExportDataValidationTests(ExportTestFixture fixture, ITestOutputHelper te [Fact] public async Task GivenFhirServer_WhenGroupDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() { - // NOTE: Azure Storage Emulator is required to run these tests locally. + // NOTE: Azurite is required to run these tests locally. // Add data for test var (dataInFhirServer, groupId) = await CreateGroupWithPatient(true); @@ -53,7 +53,7 @@ public async Task GivenFhirServer_WhenGroupDataIsExported_ThenExportedDataIsSame IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); // Download exported data from storage account - Dictionary<(string resourceType, string resourceId), Resource> dataFromExport = + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); // Assert both sets of data are equal @@ -63,7 +63,7 @@ public async Task GivenFhirServer_WhenGroupDataIsExported_ThenExportedDataIsSame [Fact] public async Task GivenFhirServer_WhenGroupDataIsExportedWithTypeParameter_ThenExportedDataIsSameAsDataInFhirServer() { - // NOTE: Azure Storage Emulator is required to run these tests locally. + // NOTE: Azurite is required to run these tests locally. // Add data for test var (dataInFhirServer, groupId) = await CreateGroupWithPatient(false); @@ -73,7 +73,7 @@ public async Task GivenFhirServer_WhenGroupDataIsExportedWithTypeParameter_ThenE IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation); // Download exported data from storage account - Dictionary<(string resourceType, string resourceId), Resource> dataFromExport = + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromExport = await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); // Assert both sets of data are equal @@ -83,7 +83,7 @@ public async Task GivenFhirServer_WhenGroupDataIsExportedWithTypeParameter_ThenE [Fact] public async Task GivenFhirServer_WhenGroupDataWithNoMemberPatientIdIsExported_ThenNoDataIsExported() { - // NOTE: Azure Storage Emulator is required to run these tests locally. + // NOTE: Azurite is required to run these tests locally. // Add data for test string groupId = await CreateGroupWithoutPatientIds(); @@ -110,7 +110,7 @@ async Task CreateGroupWithoutPatientIds() [Fact] public async Task GivenFhirServer_WhenDataIsExported_ThenExportTaskMetricsNotificationShouldBePosted() { - // NOTE: Azure Storage Emulator is required to run these tests locally. + // NOTE: Azurite is required to run these tests locally. if (!_fixture.IsUsingInProcTestServer) { @@ -132,12 +132,13 @@ public async Task GivenFhirServer_WhenDataIsExported_ThenExportTaskMetricsNotifi Assert.Single(_fixture.MetricHandler.NotificationMapping[typeof(ExportTaskMetricsNotification)]); } - private async Task<(Dictionary<(string resourceType, string resourceId), Resource> serverData, string groupId)> CreateGroupWithPatient(bool includeAllResources) + private async Task<(Dictionary<(string resourceType, string resourceId, string versionId), Resource> serverData, string groupId)> CreateGroupWithPatient(bool includeAllResources) { // Add data for test var patient = new Patient(); var patientResponse = await _testFhirClient.CreateAsync(patient); var patientId = patientResponse.Resource.Id; + var patientVersionId = patientResponse.Resource.VersionId; var relative = new RelatedPerson() { @@ -146,6 +147,7 @@ public async Task GivenFhirServer_WhenDataIsExported_ThenExportTaskMetricsNotifi var relativeResponse = await _testFhirClient.CreateAsync(relative); var relativeId = relativeResponse.Resource.Id; + var relativeVersionId = relativeResponse.Resource.VersionId; var encounter = new Encounter() { @@ -159,6 +161,7 @@ public async Task GivenFhirServer_WhenDataIsExported_ThenExportTaskMetricsNotifi var encounterResponse = await _testFhirClient.CreateAsync(encounter); var encounterId = encounterResponse.Resource.Id; + var encounterVersionId = encounterResponse.Resource.VersionId; var observation = new Observation() { @@ -178,6 +181,7 @@ public async Task GivenFhirServer_WhenDataIsExported_ThenExportTaskMetricsNotifi var observationResponse = await _testFhirClient.CreateAsync(observation); var observationId = observationResponse.Resource.Id; + var observationVersionId = observationResponse.Resource.VersionId; var group = new FhirGroup() { @@ -194,16 +198,17 @@ public async Task GivenFhirServer_WhenDataIsExported_ThenExportTaskMetricsNotifi var groupResponse = await _testFhirClient.CreateAsync(group); var groupId = groupResponse.Resource.Id; + var groupVersionId = groupResponse.Resource.VersionId; - var resourceDictionary = new Dictionary<(string resourceType, string resourceId), Resource>(); - resourceDictionary.Add((KnownResourceTypes.RelatedPerson, relativeId), relativeResponse.Resource); - resourceDictionary.Add((KnownResourceTypes.Encounter, encounterId), encounterResponse.Resource); + var resourceDictionary = new Dictionary<(string resourceType, string resourceId, string versionId), Resource>(); + resourceDictionary.Add((KnownResourceTypes.RelatedPerson, relativeId, relativeVersionId), relativeResponse.Resource); + resourceDictionary.Add((KnownResourceTypes.Encounter, encounterId, encounterVersionId), encounterResponse.Resource); if (includeAllResources) { - resourceDictionary.Add((KnownResourceTypes.Patient, patientId), patientResponse.Resource); - resourceDictionary.Add((KnownResourceTypes.Observation, observationId), observationResponse.Resource); - resourceDictionary.Add((KnownResourceTypes.Group, groupId), groupResponse.Resource); + resourceDictionary.Add((KnownResourceTypes.Patient, patientId, patientVersionId), patientResponse.Resource); + resourceDictionary.Add((KnownResourceTypes.Observation, observationId, observationVersionId), observationResponse.Resource); + resourceDictionary.Add((KnownResourceTypes.Group, groupId, groupVersionId), groupResponse.Resource); } return (resourceDictionary, groupId); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestHelper.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs similarity index 73% rename from test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestHelper.cs rename to test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs index 4dd79eb599..b82f220293 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestHelper.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTestHelper.cs @@ -19,7 +19,7 @@ using Xunit.Abstractions; using Task = System.Threading.Tasks.Task; -namespace Microsoft.Health.Fhir.Tests.E2E.Rest +namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Export { internal static class ExportTestHelper { @@ -63,14 +63,68 @@ internal static async Task> CheckExportStatus(TestFhirClient testFhir return exportJobResult.Output.Select(x => x.FileUri).ToList(); } - internal static async Task> GetResourcesFromFhirServer( + internal static async Task> GetResourcesFromFhirServer( TestFhirClient testFhirClient, Uri requestUri, FhirJsonParser fhirJsonParser, ITestOutputHelper outputHelper) { - var resourceIdToResourceMapping = new Dictionary<(string resourceType, string resourceId), Resource>(); + var resourceIdToResourceMapping = new Dictionary<(string resourceType, string resourceId, string versionId), Resource>(); + try + { + await foreach (Resource resource in GetResourceListFromFhirServer(testFhirClient, requestUri, fhirJsonParser)) + { + resourceIdToResourceMapping.TryAdd((resource.TypeName, resource.Id, resource.VersionId), resource); + } + } + catch (Exception ex) + { + outputHelper.WriteLine($"Unable to parse response into bundle: {ex}"); + } + + return resourceIdToResourceMapping; + } + + internal static async Task> GetResourcesWithHistoryFromFhirServer( + TestFhirClient testFhirClient, + Uri requestUri, + FhirJsonParser fhirJsonParser, + ITestOutputHelper outputHelper) + { + var resourceIdToResourceMapping = new Dictionary<(string resourceType, string resourceId, string versionId), Resource>(); + + try + { + await foreach (Resource resource in GetResourceListFromFhirServer(testFhirClient, requestUri, fhirJsonParser)) + { + string resourceWithHistoryUriString = $"{testFhirClient.HttpClient.BaseAddress}/{resource.TypeName}/{resource.Id}/_history"; + + if (requestUri.Query is not null) + { + resourceWithHistoryUriString += requestUri.Query; + } + + await foreach (Resource historyResource in GetResourceListFromFhirServer(testFhirClient, new Uri(resourceWithHistoryUriString), fhirJsonParser)) + { + resourceIdToResourceMapping.TryAdd((historyResource.TypeName, historyResource.Id, historyResource.VersionId), historyResource); + } + } + } + catch (Exception ex) + { + outputHelper.WriteLine($"Unable to parse response into bundle: {ex}"); + return resourceIdToResourceMapping; + } + + return resourceIdToResourceMapping; + } + + private static async IAsyncEnumerable GetResourceListFromFhirServer( + TestFhirClient testFhirClient, + Uri requestUri, + FhirJsonParser fhirJsonParser) + { while (requestUri != null) { HttpRequestMessage request = new HttpRequestMessage() @@ -82,28 +136,17 @@ internal static async Task> CheckExportStatus(TestFhirClient testFhir using HttpResponseMessage response = await testFhirClient.HttpClient.SendAsync(request); var responseString = await response.Content.ReadAsStringAsync(); - Bundle searchResults; - try - { - searchResults = fhirJsonParser.Parse(responseString); - } - catch (Exception ex) - { - outputHelper.WriteLine($"Unable to parse response into bundle: {ex}"); - return resourceIdToResourceMapping; - } - - foreach (Bundle.EntryComponent entry in searchResults.Entry) - { - resourceIdToResourceMapping.TryAdd((entry.Resource.TypeName, entry.Resource.Id), entry.Resource); - } + Bundle searchResults = fhirJsonParser.Parse(responseString); // Look at whether a continuation token has been returned. string nextLink = searchResults.NextLink?.ToString(); requestUri = nextLink == null ? null : new Uri(nextLink); - } - return resourceIdToResourceMapping; + foreach (Bundle.EntryComponent entry in searchResults.Entry) + { + yield return entry.Resource; + } + } } internal static (StorageSharedKeyCredential credential, string connectionString) GetStorageCredentialOrConnectionString(Uri storageServiceUri) @@ -152,14 +195,14 @@ internal static (StorageSharedKeyCredential credential, string connectionString) return (storageCredential, null); } - internal static async Task> DownloadBlobAndParse( + internal static async Task> DownloadBlobAndParse( IList blobUri, FhirJsonParser fhirJsonParser, ITestOutputHelper outputHelper) { if (blobUri == null || blobUri.Count == 0) { - return new Dictionary<(string resourceType, string resourceId), Resource>(); + return new Dictionary<(string resourceType, string resourceId, string versionId), Resource>(); } // Extract storage account name from blob uri in order to get corresponding access token. @@ -167,7 +210,7 @@ internal static (StorageSharedKeyCredential credential, string connectionString) Uri storageServiceUri = new UriBuilder(sampleUri.Scheme, sampleUri.Host).Uri; (StorageSharedKeyCredential credential, string connectionString) = GetStorageCredentialOrConnectionString(storageServiceUri); - var resourceIdToResourceMapping = new Dictionary<(string resourceType, string resourceId), Resource>(); + var resourceIdToResourceMapping = new Dictionary<(string resourceType, string resourceId, string versionId), Resource>(); var localRun = credential == null; foreach (Uri uri in blobUri) @@ -197,7 +240,7 @@ internal static (StorageSharedKeyCredential credential, string connectionString) // Ideally this should just be Add, but until we prevent duplicates from being added to the server // there is a chance the same resource being added multiple times resulting in a key conflict. - resourceIdToResourceMapping.TryAdd((resource.TypeName, resource.Id), resource); + resourceIdToResourceMapping.TryAdd((resource.TypeName, resource.Id, resource.VersionId), resource); } } @@ -205,18 +248,19 @@ internal static (StorageSharedKeyCredential credential, string connectionString) } internal static bool ValidateDataFromBothSources( - Dictionary<(string resourceType, string resourceId), Resource> dataFromServer, - Dictionary<(string resourceType, string resourceId), Resource> dataFromStorageAccount, - ITestOutputHelper outputHelper) + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromServer, + Dictionary<(string resourceType, string resourceId, string versionId), Resource> dataFromStorageAccount, + ITestOutputHelper outputHelper, + bool allowDataFromServerToBeSubsetOfExportData = false) { bool result = true; - if (dataFromStorageAccount.Count != dataFromServer.Count) + if (dataFromStorageAccount.Count != dataFromServer.Count && !(allowDataFromServerToBeSubsetOfExportData && dataFromServer.Count < dataFromStorageAccount.Count)) { outputHelper.WriteLine($"Count differs. Exported data count: {dataFromStorageAccount.Count} Fhir Server Count: {dataFromServer.Count}"); result = false; - foreach (KeyValuePair<(string resourceType, string resourceId), Resource> kvp in dataFromStorageAccount) + foreach (KeyValuePair<(string resourceType, string resourceId, string versionId), Resource> kvp in dataFromStorageAccount) { if (!dataFromServer.ContainsKey(kvp.Key)) { @@ -226,16 +270,14 @@ internal static bool ValidateDataFromBothSources( } // Enable this check when creating/updating data validation tests to ensure there is data to export - /* - if (dataFromStorageAccount.Count == 0) - { - _outputHelper.WriteLine("No data exported. This test expects data to be present."); - return false; - } - */ + // if (dataFromStorageAccount.Count == 0) + // { + // outputHelper.WriteLine("No data exported. This test expects data to be present."); + // return false; + // } int wrongCount = 0; - foreach (KeyValuePair<(string resourceType, string resourceId), Resource> kvp in dataFromServer) + foreach (KeyValuePair<(string resourceType, string resourceId, string versionId), Resource> kvp in dataFromServer) { if (!dataFromStorageAccount.ContainsKey(kvp.Key)) { @@ -256,7 +298,6 @@ internal static bool ValidateDataFromBothSources( } } - outputHelper.WriteLine($"Missing or wrong match count: {wrongCount}"); return result; } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTests.cs similarity index 99% rename from test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTests.cs rename to test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTests.cs index c59d5ff679..6fdd1382bb 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/ExportTests.cs @@ -18,7 +18,7 @@ using Microsoft.Net.Http.Headers; using Xunit; -namespace Microsoft.Health.Fhir.Tests.E2E.Rest +namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Export { [Trait(Traits.OwningTeam, OwningTeam.Fhir)] [Trait(Traits.Category, Categories.Export)] diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/StartupForExportTestProvider.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/StartupForExportTestProvider.cs similarity index 55% rename from test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/StartupForExportTestProvider.cs rename to test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/StartupForExportTestProvider.cs index bb9be374ec..295535f474 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/StartupForExportTestProvider.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Export/StartupForExportTestProvider.cs @@ -10,29 +10,43 @@ using Microsoft.Extensions.Options; using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Api.Configs; +using Microsoft.Health.Fhir.Core.Configs; using Microsoft.Health.Fhir.Core.Features.Operations.Export; using Microsoft.Health.Fhir.Tests.E2E.Rest.Metric; -namespace Microsoft.Health.Fhir.Tests.E2E.Rest +namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Export { [RequiresIsolatedDatabase] public class StartupForExportTestProvider : StartupBaseForCustomProviders { + private IConfiguration _configuration; + public StartupForExportTestProvider(IConfiguration configuration) : base(configuration) { + _configuration = configuration; } public override void ConfigureServices(IServiceCollection services) { base.ConfigureServices(services); - FeatureConfiguration configuration = new FeatureConfiguration() + FeatureConfiguration featureConfiguration = new() { SupportsAnonymizedExport = true, }; - IOptions options = Options.Create(configuration); - services.Replace(new ServiceDescriptor(typeof(IOptions), options)); + IOptions featureOptions = Options.Create(featureConfiguration); + services.Replace(new ServiceDescriptor(typeof(IOptions), featureOptions)); + + ExportJobConfiguration existingExportOptions = new(); + _configuration.GetSection("FhirServer:Operations:Export").Bind(existingExportOptions); + + // ExportDataTestFixture generates 27 patients, 54 encounters, and, 108 observations with history / deletes. + // We want to test the splitting of jobs and orchestration continuation. Hence this config. + existingExportOptions.MaximumNumberOfResourcesPerQuery = 20; + existingExportOptions.NumberOfParallelRecordRanges = 4; + IOptions exportOptions = Options.Create(existingExportOptions); + services.Replace(new ServiceDescriptor(typeof(IOptions), exportOptions)); services.Add() .Singleton() diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportLongRunningTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportLongRunningTests.cs deleted file mode 100644 index 5fad2be130..0000000000 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportLongRunningTests.cs +++ /dev/null @@ -1,178 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System; -using System.Collections.Generic; -using System.Linq; -using Hl7.Fhir.Model; -using Hl7.Fhir.Serialization; -using Microsoft.Health.Fhir.Tests.Common; -using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; -using Microsoft.Health.Fhir.Tests.E2E.Common; -using Microsoft.Health.Test.Utilities; -using Xunit; -using Xunit.Abstractions; -using Task = System.Threading.Tasks.Task; - -namespace Microsoft.Health.Fhir.Tests.E2E.Rest -{ - [Trait(Traits.OwningTeam, OwningTeam.Fhir)] - [Trait(Traits.Category, Categories.Export)] - [Trait(Traits.Category, Categories.ExportLongRunning)] - [HttpIntegrationFixtureArgumentSets(DataStore.All, Format.Json)] - public class ExportLongRunningTests : IClassFixture - { - private readonly TestFhirClient _testFhirClient; - private readonly ITestOutputHelper _outputHelper; - private readonly FhirJsonParser _fhirJsonParser; - private readonly ExportTestFixture _fixture; - - public ExportLongRunningTests(ExportTestFixture fixture, ITestOutputHelper testOutputHelper) - { - _testFhirClient = fixture.TestFhirClient; - _outputHelper = testOutputHelper; - _fhirJsonParser = new FhirJsonParser(); - _fixture = fixture; - } - - [Fact] - public async Task GivenFhirServer_WhenAllDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() - { - // NOTE: Azure Storage Emulator is required to run these tests locally. - - // Trigger export request and check for export status - Uri contentLocation = await _testFhirClient.ExportAsync(); - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); - - // Download exported data from storage account - Dictionary<(string resourceType, string resourceId), Resource> dataFromExport = - await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); - - // Download all resources from fhir server - Dictionary<(string resourceType, string resourceId), Resource> dataFromFhirServer = - await ExportTestHelper.GetResourcesFromFhirServer(_testFhirClient, _testFhirClient.HttpClient.BaseAddress, _fhirJsonParser, _outputHelper); - - // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(dataFromFhirServer, dataFromExport, _outputHelper)); - } - - [Fact] - public async Task GivenFhirServer_WhenPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() - { - // NOTE: Azure Storage Emulator is required to run these tests locally. - - // Trigger export request and check for export status - Uri contentLocation = await _testFhirClient.ExportAsync("Patient/"); - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); - - // Download exported data from storage account - Dictionary<(string resourceType, string resourceId), Resource> dataFromExport = - await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); - - // Download resources from fhir server - Uri address = new Uri(_testFhirClient.HttpClient.BaseAddress, "Patient/"); - Dictionary<(string resourceType, string resourceId), Resource> dataFromFhirServer = await ExportTestHelper.GetResourcesFromFhirServer(_testFhirClient, address, _fhirJsonParser, _outputHelper); - - Dictionary<(string resourceType, string resourceId), Resource> compartmentData = new Dictionary<(string resourceType, string resourceId), Resource>(); - foreach ((string resourceType, string resourceId) key in dataFromFhirServer.Keys) - { - address = new Uri(_testFhirClient.HttpClient.BaseAddress, "Patient/" + key.resourceId + "/*"); - - // copies all the new values into the compartment data dictionary - (await ExportTestHelper.GetResourcesFromFhirServer(_testFhirClient, address, _fhirJsonParser, _outputHelper)) - .ToList() - .ForEach(x => compartmentData.TryAdd(x.Key, x.Value)); - } - - compartmentData.ToList().ForEach(x => dataFromFhirServer.TryAdd(x.Key, x.Value)); - dataFromFhirServer.Union(compartmentData); - - // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(dataFromFhirServer, dataFromExport, _outputHelper)); - } - - [Fact] - public async Task GivenFhirServer_WhenAllObservationAndPatientDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() - { - // NOTE: Azure Storage Emulator is required to run these tests locally. - - // Trigger export request and check for export status - Uri contentLocation = await _testFhirClient.ExportAsync(string.Empty, "_type=Observation,Patient"); - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); - - // Download exported data from storage account - Dictionary<(string resourceType, string resourceId), Resource> dataFromExport = - await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); - - // Download resources from fhir server - Uri address = new Uri(_testFhirClient.HttpClient.BaseAddress, "?_type=Observation,Patient"); - Dictionary<(string resourceType, string resourceId), Resource> dataFromFhirServer = - await ExportTestHelper.GetResourcesFromFhirServer(_testFhirClient, address, _fhirJsonParser, _outputHelper); - - // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(dataFromFhirServer, dataFromExport, _outputHelper)); - } - - [Fact] - public async Task GivenFhirServer_WhenPatientObservationDataIsExported_ThenExportedDataIsSameAsDataInFhirServer() - { - // NOTE: Azure Storage Emulator is required to run these tests locally. - - // Trigger export request and check for export status - Uri contentLocation = await _testFhirClient.ExportAsync("Patient/", "_type=Observation"); - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); - - // Download exported data from storage account - Dictionary<(string resourceType, string resourceId), Resource> dataFromExport = - await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); - - // Download resources from fhir server - Uri address = new Uri(_testFhirClient.HttpClient.BaseAddress, "Patient/"); - Dictionary<(string resourceType, string resourceId), Resource> patientData = - await ExportTestHelper.GetResourcesFromFhirServer(_testFhirClient, address, _fhirJsonParser, _outputHelper); - - Dictionary<(string resourceType, string resourceId), Resource> compartmentData = new Dictionary<(string resourceType, string resourceId), Resource>(); - foreach ((string resourceType, string resourceId) key in patientData.Keys) - { - address = new Uri(_testFhirClient.HttpClient.BaseAddress, "Patient/" + key.resourceId + "/Observation"); - - // copies all the new values into the compartment data dictionary - (await ExportTestHelper.GetResourcesFromFhirServer(_testFhirClient, address, _fhirJsonParser, _outputHelper)) - .ToList() - .ForEach(x => compartmentData.TryAdd(x.Key, x.Value)); - } - - compartmentData.ToList().ForEach(x => patientData.TryAdd(x.Key, x.Value)); - patientData.Union(compartmentData); - - // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(compartmentData, dataFromExport, _outputHelper)); - } - - [Fact] - public async Task GivenFhirServer_WhenAllDataIsExportedToASpecificContainer_ThenExportedDataIsInTheSpecifiedContianer() - { - // NOTE: Azure Storage Emulator is required to run these tests locally. - - string testContainer = "test-container"; - - // Trigger export request and check for export status - Uri contentLocation = await _testFhirClient.ExportAsync(parameters: $"_container={testContainer}"); - IList blobUris = await ExportTestHelper.CheckExportStatus(_testFhirClient, contentLocation, timeToWaitInMinutes: 15); - - // Download exported data from storage account - Dictionary<(string resourceType, string resourceId), Resource> dataFromExport = - await ExportTestHelper.DownloadBlobAndParse(blobUris, _fhirJsonParser, _outputHelper); - - // Download all resources from fhir server - Dictionary<(string resourceType, string resourceId), Resource> dataFromFhirServer = - await ExportTestHelper.GetResourcesFromFhirServer(_testFhirClient, _testFhirClient.HttpClient.BaseAddress, _fhirJsonParser, _outputHelper); - - // Assert both data are equal - Assert.True(ExportTestHelper.ValidateDataFromBothSources(dataFromFhirServer, dataFromExport, _outputHelper)); - Assert.True(blobUris.All((url) => url.OriginalString.Contains(testContainer))); - } - } -} diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestFixture.cs deleted file mode 100644 index 70fa434a8a..0000000000 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/ExportTestFixture.cs +++ /dev/null @@ -1,28 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using MediatR; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Health.Fhir.Core.Features.Operations.Export; -using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; -using Microsoft.Health.Fhir.Tests.E2E.Rest.Metric; - -namespace Microsoft.Health.Fhir.Tests.E2E.Rest -{ - public class ExportTestFixture : HttpIntegrationTestFixture - { - private MetricHandler _metricHandler; - - public ExportTestFixture(DataStore dataStore, Format format, TestFhirServerFactory testFhirServerFactory) - : base(dataStore, format, testFhirServerFactory) - { - } - - public MetricHandler MetricHandler - { - get => _metricHandler ?? (_metricHandler = (MetricHandler)(TestFhirServer as InProcTestFhirServer)?.Server.Host.Services.GetRequiredService>()); - } - } -} diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/SqlServerExportTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/SqlServerExportTests.cs index 4cfb725478..a6aee70ae9 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/SqlServerExportTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Export/SqlServerExportTests.cs @@ -9,6 +9,8 @@ using System.Threading.Tasks; using Microsoft.Data.SqlClient; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using Microsoft.Health.Fhir.Core.Configs; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.Core.Features.Operations.Export; @@ -36,6 +38,10 @@ public class SqlServerExportTests : IClassFixture _exportJobConfiguration = Options.Create(new ExportJobConfiguration() { NumberOfParallelRecordRanges = 5 }); + public SqlServerExportTests(SqlServerFhirStorageTestsFixture fixture, ITestOutputHelper testOutputHelper) { _fixture = fixture; @@ -52,9 +58,7 @@ public async Task ExportWorkRegistration() { PrepareData(); // 1000 patients + 1000 observations + 1000 claims. !!! RawResource is invalid. - var coordJob = new SqlExportOrchestratorJob(_queueClient, _searchService); - //// surrogate id range size is set via max number of resources per query on coord record - coordJob.NumberOfSurrogateIdRanges = 5; // 100*5=500 is 50% of 1000, so there are 2 insert transactions in JobQueue per each resource type + var coordJob = new SqlExportOrchestratorJob(_queueClient, _searchService, _exportJobConfiguration); await RunExport(null, coordJob, 31, 6); // 31=coord+3*1000/SurrogateIdRangeSize 6=coord+100*5/SurrogateIdRangeSize diff --git a/test/Microsoft.Health.Fhir.Stu3.Tests.E2E/Microsoft.Health.Fhir.Stu3.Tests.E2E.csproj b/test/Microsoft.Health.Fhir.Stu3.Tests.E2E/Microsoft.Health.Fhir.Stu3.Tests.E2E.csproj index 3143ad0c2a..45a4767080 100644 --- a/test/Microsoft.Health.Fhir.Stu3.Tests.E2E/Microsoft.Health.Fhir.Stu3.Tests.E2E.csproj +++ b/test/Microsoft.Health.Fhir.Stu3.Tests.E2E/Microsoft.Health.Fhir.Stu3.Tests.E2E.csproj @@ -14,8 +14,8 @@ - - + +