From 9c3462d0cf0ffa38281053617d86bcf31ae29a95 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Thu, 4 Jun 2026 13:30:47 -0700 Subject: [PATCH 01/17] Reuse SMART search integration fixture (#5588) * Add SMART search integration timing diagnostics Log SMART integration fixture setup, resource upsert, and search-service timings so the PR pipeline can identify whether the slow SmartSearch shard is dominated by setup or specific query patterns. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Reuse SMART search integration fixture Seed SMART integration data once per version and data store so SmartOnFhir tests avoid rebuilding the expensive storage fixture for every test method. Use unique IDs for mutation tests while preserving update semantics, and dispose cached fixtures when the test process exits. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Simplify SMART search shared fixture Remove process-lifetime fixture caching and diagnostic timing wrappers while keeping shared SMART test data setup under xUnit fixture lifetime. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Address SMART fixture review comments Move test setup out of the no-op async lifecycle and inline the one-line upsert wrapper. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Potential fix for pull request finding 'CodeQL / Missed 'readonly' opportunity' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> * Move SmartSearchSharedFixture into its own file Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Potential fix for pull request finding 'CodeQL / Missed opportunity to use Where' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .../Smart/SmartSearchSharedFixture.cs | 140 +++++++++++++ .../Features/Smart/SmartSearchTests.cs | 188 ++++-------------- ...th.Fhir.Shared.Tests.Integration.projitems | 1 + 3 files changed, 180 insertions(+), 149 deletions(-) create mode 100644 test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchSharedFixture.cs diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchSharedFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchSharedFixture.cs new file mode 100644 index 0000000000..47e49d3b2e --- /dev/null +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchSharedFixture.cs @@ -0,0 +1,140 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Hl7.Fhir.Model; +using Hl7.Fhir.Serialization; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Health.Extensions.DependencyInjection; +using Microsoft.Health.Fhir.Core.Extensions; +using Microsoft.Health.Fhir.Core.Features.Context; +using Microsoft.Health.Fhir.Core.Features.Definition; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Features.Search.Converters; +using Microsoft.Health.Fhir.Core.Features.Search.Registry; +using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.Core.UnitTests.Extensions; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using Microsoft.Health.Fhir.Tests.Integration.Persistence; +using Microsoft.Health.Test.Utilities; +using NSubstitute; +using Xunit; +using Task = System.Threading.Tasks.Task; + +namespace Microsoft.Health.Fhir.Tests.Integration.Features.Smart +{ + public sealed class SmartSearchSharedFixture : IAsyncLifetime + { + private readonly DataStore _dataStore; + private FhirStorageTestsFixture _fixture; + private IScoped _scopedDataStore; + private SearchParameterDefinitionManager _searchParameterDefinitionManager; + private ISearchIndexer _searchIndexer; + + public SmartSearchSharedFixture(DataStore dataStore) + { + _dataStore = dataStore; + } + + public FhirStorageTestsFixture Fixture => _fixture; + + public async Task InitializeAsync() + { + if (ModelInfoProvider.Instance.Version != FhirSpecification.R4 && + ModelInfoProvider.Instance.Version != FhirSpecification.R4B) + { + return; + } + + _fixture = new FhirStorageTestsFixture(_dataStore); + await _fixture.InitializeAsync(); + + var typedElementToSearchValueConverterManager = await CreateFhirTypedElementToSearchValueConverterManagerAsync(); + _searchIndexer = new TypedElementSearchIndexer( + _fixture.SupportedSearchParameterDefinitionManager, + typedElementToSearchValueConverterManager, + Substitute.For(), + ModelInfoProvider.Instance, + NullLogger.Instance); + _searchParameterDefinitionManager = _fixture.SearchParameterDefinitionManager; + _scopedDataStore = _fixture.DataStore.CreateMockScope(); + + await LoadBundleAsync("SmartPatientA"); + await LoadBundleAsync("SmartPatientB"); + await LoadBundleAsync("SmartPatientC"); + await LoadBundleAsync("SmartPatientD"); + await LoadBundleAsync("SmartCommon"); + + await UpsertResource(Samples.GetJsonSample("Medication")); + await UpsertResource(Samples.GetJsonSample("Organization")); + await UpsertResource(Samples.GetJsonSample("Location-example-hq")); + } + + public async Task DisposeAsync() + { + if (_fixture != null) + { + await _fixture.DisposeAsync(); + } + } + + public async Task UpsertResource(Resource resource, string httpMethod = "PUT") + { + resource.Meta ??= new Meta(); + resource.Meta.LastUpdated = DateTimeOffset.UtcNow; + + ResourceElement resourceElement = resource.ToResourceElement(); + + var rawResource = new RawResource(resource.ToJson(), FhirResourceFormat.Json, isMetaSet: false); + var resourceRequest = new ResourceRequest(httpMethod); + var compartmentIndices = Substitute.For(); + var searchIndices = _searchIndexer.Extract(resourceElement); + var wrapper = new ResourceWrapper(resourceElement, rawResource, resourceRequest, false, searchIndices, compartmentIndices, new List>(), _searchParameterDefinitionManager.GetSearchParameterHashForResourceType("Patient")); + wrapper.SearchParameterHash = "hash"; + + return await _scopedDataStore.Value.UpsertAsync(new ResourceWrapperOperation(wrapper, true, true, null, false, false, bundleResourceContext: null), CancellationToken.None); + } + + private static async Task CreateFhirTypedElementToSearchValueConverterManagerAsync() + { + var types = typeof(ITypedElementToSearchValueConverter) + .Assembly + .GetTypes() + .Where(x => typeof(ITypedElementToSearchValueConverter).IsAssignableFrom(x) && !x.IsAbstract && !x.IsInterface); + + var referenceSearchValueParser = new ReferenceSearchValueParser(new FhirRequestContextAccessor(), new FhirServerInstanceConfiguration()); + var codeSystemResolver = new CodeSystemResolver(ModelInfoProvider.Instance); + await codeSystemResolver.StartAsync(CancellationToken.None); + + var fhirElementToSearchValueConverters = new List(); + + foreach (Type type in types.Where(type => type.Name != nameof(FhirTypedElementToSearchValueConverterManager.ExtensionConverter))) + { + // Filter out the extension converter because it will be added to the converter dictionary in the converter manager's constructor + var x = (ITypedElementToSearchValueConverter)Mock.TypeWithArguments(type, referenceSearchValueParser, codeSystemResolver); + fhirElementToSearchValueConverters.Add(x); + } + + return new FhirTypedElementToSearchValueConverterManager(fhirElementToSearchValueConverters); + } + + private async Task LoadBundleAsync(string sampleName) + { + var smartBundle = Samples.GetJsonSample(sampleName); + + foreach (var entry in smartBundle.Entry) + { + await UpsertResource(entry.Resource); + } + } + } +} diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchTests.cs index fe7e4eff8d..bba5a798f3 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchTests.cs @@ -39,7 +39,6 @@ using Microsoft.Health.Fhir.Tests.Integration.Persistence; using Microsoft.Health.Test.Utilities; using NSubstitute; -using NSubstitute.Core; using Xunit; using Task = System.Threading.Tasks.Task; @@ -49,116 +48,28 @@ namespace Microsoft.Health.Fhir.Tests.Integration.Features.Smart [Trait(Traits.Category, Categories.SmartOnFhir)] [FhirStorageTestsFixtureArgumentSets(DataStore.All)] - public class SmartSearchTests : IClassFixture, IAsyncLifetime + public class SmartSearchTests : IClassFixture { + private readonly SmartSearchSharedFixture _smartFixture; private readonly FhirStorageTestsFixture _fixture; - private readonly IFhirStorageTestHelper _testHelper; - private IFhirOperationDataStore _fhirOperationDataStore; - private IScoped _scopedDataStore; - private IFhirStorageTestHelper _fhirStorageTestHelper; - private SearchParameterDefinitionManager _searchParameterDefinitionManager; - private ITypedElementToSearchValueConverterManager _typedElementToSearchValueConverterManager; - private ISearchIndexer _searchIndexer; - private readonly ISearchParameterSupportResolver _searchParameterSupportResolver = Substitute.For(); - private ISupportedSearchParameterDefinitionManager _supportedSearchParameterDefinitionManager; - private SearchParameterStatusManager _searchParameterStatusManager; private IScoped _searchService; private RequestContextAccessor _contextAccessor; - private readonly IDataStoreSearchParameterValidator _dataStoreSearchParameterValidator = Substitute.For(); - public SmartSearchTests(FhirStorageTestsFixture fixture) + public SmartSearchTests(SmartSearchSharedFixture smartFixture) { - _fixture = fixture; - _testHelper = _fixture.TestHelper; - } + _smartFixture = smartFixture; - public async Task InitializeAsync() - { if (ModelInfoProvider.Instance.Version == FhirSpecification.R4 || ModelInfoProvider.Instance.Version == FhirSpecification.R4B) { - _dataStoreSearchParameterValidator.ValidateSearchParameter(default, out Arg.Any()).ReturnsForAnyArgs(x => - { - x[1] = null; - return true; - }); - - _searchParameterSupportResolver.IsSearchParameterSupported(Arg.Any()).Returns((true, false)); - + _fixture = _smartFixture.Fixture; _contextAccessor = _fixture.FhirRequestContextAccessor; - - _fhirOperationDataStore = _fixture.OperationDataStore; - _fhirStorageTestHelper = _fixture.TestHelper; - _scopedDataStore = _fixture.DataStore.CreateMockScope(); - - _searchParameterDefinitionManager = _fixture.SearchParameterDefinitionManager; - _supportedSearchParameterDefinitionManager = _fixture.SupportedSearchParameterDefinitionManager; - - _typedElementToSearchValueConverterManager = await CreateFhirTypedElementToSearchValueConverterManagerAsync(); - - _searchIndexer = new TypedElementSearchIndexer( - _supportedSearchParameterDefinitionManager, - _typedElementToSearchValueConverterManager, - Substitute.For(), - ModelInfoProvider.Instance, - NullLogger.Instance); - - ResourceWrapperFactory wrapperFactory = Mock.TypeWithArguments( - new RawResourceFactory(new FhirJsonSerializer()), - new FhirRequestContextAccessor(), - _searchIndexer, - _searchParameterDefinitionManager, - Deserializers.ResourceDeserializer); - - _searchParameterStatusManager = _fixture.SearchParameterStatusManager; - _searchService = _fixture.SearchService.CreateMockScope(); - - _contextAccessor = _fixture.FhirRequestContextAccessor; - - var smartBundle = Samples.GetJsonSample("SmartPatientA"); - foreach (var entry in smartBundle.Entry) - { - await UpsertResource(entry.Resource); - } - - smartBundle = Samples.GetJsonSample("SmartPatientB"); - foreach (var entry in smartBundle.Entry) - { - await UpsertResource(entry.Resource); - } - - smartBundle = Samples.GetJsonSample("SmartPatientC"); - foreach (var entry in smartBundle.Entry) - { - await UpsertResource(entry.Resource); - } - - smartBundle = Samples.GetJsonSample("SmartPatientD"); - foreach (var entry in smartBundle.Entry) - { - await UpsertResource(entry.Resource); - } - - smartBundle = Samples.GetJsonSample("SmartCommon"); - foreach (var entry in smartBundle.Entry) - { - await UpsertResource(entry.Resource); - } - - await UpsertResource(Samples.GetJsonSample("Medication")); - await UpsertResource(Samples.GetJsonSample("Organization")); - await UpsertResource(Samples.GetJsonSample("Location-example-hq")); } } - public Task DisposeAsync() - { - return Task.CompletedTask; - } - [SkippableFact] public async Task GivenScopesWithReadForAllResources_WhenRevIncludeObservations_PatientAndObservationReturned() { @@ -1119,47 +1030,9 @@ public async Task GivenReadScopeOnOnlyEncountersInACompartment_OnRevincludeWithW Assert.DoesNotContain(results.Results, r => r.Resource.ResourceTypeName == KnownResourceTypes.Observation); } - private async Task UpsertResource(Resource resource, string httpMethod = "PUT") - { - resource.Meta ??= new Meta(); - resource.Meta.LastUpdated = DateTimeOffset.UtcNow; - - ResourceElement resourceElement = resource.ToResourceElement(); - - var rawResource = new RawResource(resource.ToJson(), FhirResourceFormat.Json, isMetaSet: false); - var resourceRequest = new ResourceRequest(httpMethod); - var compartmentIndices = Substitute.For(); - var searchIndices = _searchIndexer.Extract(resourceElement); - var wrapper = new ResourceWrapper(resourceElement, rawResource, resourceRequest, false, searchIndices, compartmentIndices, new List>(), _searchParameterDefinitionManager.GetSearchParameterHashForResourceType("Patient")); - wrapper.SearchParameterHash = "hash"; - - return await _scopedDataStore.Value.UpsertAsync(new ResourceWrapperOperation(wrapper, true, true, null, false, false, bundleResourceContext: null), CancellationToken.None); - } - - private static async Task CreateFhirTypedElementToSearchValueConverterManagerAsync() + private static string CreateSmartV2TestResourceId(string scenario) { - var types = typeof(ITypedElementToSearchValueConverter) - .Assembly - .GetTypes() - .Where(x => typeof(ITypedElementToSearchValueConverter).IsAssignableFrom(x) && !x.IsAbstract && !x.IsInterface); - - var referenceSearchValueParser = new ReferenceSearchValueParser(new FhirRequestContextAccessor(), new FhirServerInstanceConfiguration()); - var codeSystemResolver = new CodeSystemResolver(ModelInfoProvider.Instance); - await codeSystemResolver.StartAsync(CancellationToken.None); - - var fhirElementToSearchValueConverters = new List(); - - foreach (Type type in types) - { - // Filter out the extension converter because it will be added to the converter dictionary in the converter manager's constructor - if (type.Name != nameof(FhirTypedElementToSearchValueConverterManager.ExtensionConverter)) - { - var x = (ITypedElementToSearchValueConverter)Mock.TypeWithArguments(type, referenceSearchValueParser, codeSystemResolver); - fhirElementToSearchValueConverters.Add(x); - } - } - - return new FhirTypedElementToSearchValueConverterManager(fhirElementToSearchValueConverters); + return $"smart-v2-{scenario}-{Guid.NewGuid():N}"; } private void ConfigureFhirRequestContext( @@ -1192,20 +1065,21 @@ public async Task GivenSmartV2CreateScope_WhenCreatingPatient_ThenPatientIsCreat "This test is only valid for R4 and R4B"); var scopeRestriction = new ScopeRestriction("Patient", Core.Features.Security.DataActions.Create, "patient"); + var patientId = CreateSmartV2TestResourceId("create"); ConfigureFhirRequestContext(_contextAccessor, new List() { scopeRestriction }); - _contextAccessor.RequestContext.AccessControlContext.CompartmentId = "smart-patient-test"; + _contextAccessor.RequestContext.AccessControlContext.CompartmentId = patientId; _contextAccessor.RequestContext.AccessControlContext.CompartmentResourceType = "Patient"; var newPatient = new Patient { - Id = "smart-v2-create-test", + Id = patientId, Name = new List { new HumanName().WithGiven("TestCreate").AndFamily("SmartV2") }, }; - var result = await UpsertResource(newPatient); + var result = await _smartFixture.UpsertResource(newPatient); Assert.NotNull(result); - Assert.Equal("smart-v2-create-test", result.Wrapper.ResourceId); + Assert.Equal(patientId, result.Wrapper.ResourceId); } [SkippableFact] @@ -1262,20 +1136,28 @@ public async Task GivenSmartV2UpdateScope_WhenUpdatingPatient_ThenPatientIsUpdat "This test is only valid for R4 and R4B"); var scopeRestriction = new ScopeRestriction("Patient", Core.Features.Security.DataActions.Update, "patient"); + var patientId = CreateSmartV2TestResourceId("update"); + + await _smartFixture.UpsertResource(new Patient + { + Id = patientId, + Name = new List { new HumanName().WithGiven("InitialName").AndFamily("SmartV2") }, + }); ConfigureFhirRequestContext(_contextAccessor, new List() { scopeRestriction }); - _contextAccessor.RequestContext.AccessControlContext.CompartmentId = "smart-patient-A"; + _contextAccessor.RequestContext.AccessControlContext.CompartmentId = patientId; _contextAccessor.RequestContext.AccessControlContext.CompartmentResourceType = "Patient"; // Create an updated patient resource var updatedPatient = new Patient { - Id = "smart-patient-A", + Id = patientId, Name = new List { new HumanName().WithGiven("UpdatedName").AndFamily("Updated") }, }; - var result = await UpsertResource(updatedPatient); + var result = await _smartFixture.UpsertResource(updatedPatient); Assert.NotNull(result); + Assert.Equal(patientId, result.Wrapper.ResourceId); } [SkippableFact] @@ -1288,9 +1170,10 @@ public async Task GivenSmartV2SearchAndCreateScopes_WhenSearchingWithCreate_Then var scopeRestriction1 = new ScopeRestriction("Patient", Core.Features.Security.DataActions.Search, "patient"); var scopeRestriction2 = new ScopeRestriction("Patient", Core.Features.Security.DataActions.Create, "patient"); + var patientId = CreateSmartV2TestResourceId("search-create"); ConfigureFhirRequestContext(_contextAccessor, new List() { scopeRestriction1, scopeRestriction2 }); - _contextAccessor.RequestContext.AccessControlContext.CompartmentId = "smart-patient-test"; + _contextAccessor.RequestContext.AccessControlContext.CompartmentId = patientId; _contextAccessor.RequestContext.AccessControlContext.CompartmentResourceType = "Patient"; // Test search capability @@ -1303,11 +1186,11 @@ public async Task GivenSmartV2SearchAndCreateScopes_WhenSearchingWithCreate_Then // Test create capability var newPatient = new Patient { - Id = "smart-v2-search-create-test", + Id = patientId, Name = new List { new HumanName().WithGiven("SearchCreate").AndFamily("SmartV2") }, }; - var createResult = await UpsertResource(newPatient); + var createResult = await _smartFixture.UpsertResource(newPatient); Assert.NotNull(createResult); } @@ -1321,27 +1204,34 @@ public async Task GivenSmartV2SearchAndUpdateScopes_WhenSearchingWithUpdate_Then var scopeRestriction1 = new ScopeRestriction("Patient", Core.Features.Security.DataActions.Search, "patient"); var scopeRestriction2 = new ScopeRestriction("Patient", Core.Features.Security.DataActions.Update, "patient"); + var patientId = CreateSmartV2TestResourceId("search-update"); + + await _smartFixture.UpsertResource(new Patient + { + Id = patientId, + Name = new List { new HumanName().WithGiven("InitialSearchUpdate").AndFamily("SmartV2") }, + }); ConfigureFhirRequestContext(_contextAccessor, new List() { scopeRestriction1, scopeRestriction2 }); - _contextAccessor.RequestContext.AccessControlContext.CompartmentId = "smart-patient-A"; + _contextAccessor.RequestContext.AccessControlContext.CompartmentId = patientId; _contextAccessor.RequestContext.AccessControlContext.CompartmentResourceType = "Patient"; // Test search capability var query = new List>(); - query.Add(new Tuple("_id", "smart-patient-A")); + query.Add(new Tuple("_id", patientId)); var searchResults = await _searchService.Value.SearchAsync("Patient", query, CancellationToken.None); Assert.NotEmpty(searchResults.Results); // Test update capability var updatedPatient = new Patient { - Id = "smart-patient-A", + Id = patientId, Name = new List { new HumanName().WithGiven("SearchUpdate").AndFamily("SmartV2") }, }; - var updateResult = await UpsertResource(updatedPatient); + var updateResult = await _smartFixture.UpsertResource(updatedPatient); Assert.NotNull(updateResult); - Assert.Equal("smart-patient-A", updateResult.Wrapper.ResourceId); + Assert.Equal(patientId, updateResult.Wrapper.ResourceId); } // SMART v2 Granular Scope with Search parameters Tests diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems index 1fe1372be9..58ef46d699 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems @@ -24,6 +24,7 @@ + From 3e191c549b9c19e94f42a092649e982c67322883 Mon Sep 17 00:00:00 2001 From: Richa Bansal <57157506+rbans96@users.noreply.github.com> Date: Mon, 8 Jun 2026 11:30:05 -0700 Subject: [PATCH 02/17] Fixes the $expand to return 404 when ValueSet not found (#5580) * Downgrade unknown ValueSet expand failures to LogWarning (#192757) * Potential fix for pull request finding 'CodeQL / Dereferenced variable may be null' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> * Add test * Return HTTP 404 for unknown ValueSet during $expand Throw ResourceNotFoundException instead of returning OperationOutcome when Firely surfaces an unknown ValueSet (HttpStatusCode.NotFound). The existing OperationOutcomeExceptionFilter maps it to 404. Added catch (ResourceNotFoundException) { throw; } in the handler to prevent double Error-level logging. Only Warning is now logged. FHIR spec only requires error OperationOutcome with non-2xx status; 404 is consistent with Firely NotFound semantics. * Add E2E test verifying 404 for unknown ValueSet expand * Add E2E test for expand by ID returning 404 --------- Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .../Controllers/TerminologyControllerTests.cs | 16 ++++ .../FirelyTerminologyServiceProxyTests.cs | 81 ++++++++++++++++++- .../FirelyTerminologyServiceProxy.cs | 6 ++ .../Conformance/TerminologyRequestHandler.cs | 4 + .../Rest/Conformance/ExpandOperationTests.cs | 38 +++++++++ 5 files changed, 144 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/TerminologyControllerTests.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/TerminologyControllerTests.cs index 75bda5733a..e35a79b75c 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/TerminologyControllerTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/TerminologyControllerTests.cs @@ -416,5 +416,21 @@ public static IEnumerable GetExpandByPostTestData() yield return d; } } + + [Fact] + public async Task GivenUnknownValueSet_WhenExpanding_ThenThrowsResourceNotFoundException() + { + // When the terminology service throws ResourceNotFoundException for an unknown ValueSet, + // the exception propagates and the OperationOutcomeExceptionFilter maps it to HTTP 404. + _mediator.Send( + Arg.Any(), + Arg.Any()) + .Returns(x => throw new ResourceNotFoundException( + "ValueSet 'http://example.org/fhir/ValueSet/unknown' is unknown")); + + _controller.HttpContext.Request.QueryString = new QueryString("?url=http://example.org/fhir/ValueSet/unknown"); + + await Assert.ThrowsAsync(() => _controller.Expand()); + } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Conformance/FirelyTerminologyServiceProxyTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Conformance/FirelyTerminologyServiceProxyTests.cs index bb93f7d06b..98fd8ed9d6 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Conformance/FirelyTerminologyServiceProxyTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Conformance/FirelyTerminologyServiceProxyTests.cs @@ -18,6 +18,7 @@ using Hl7.Fhir.Specification.Source; using Hl7.Fhir.Specification.Terminology; using Microsoft.Extensions.Logging; +using Microsoft.Health.Fhir.Core.Exceptions; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Conformance; using Microsoft.Health.Fhir.Shared.Core.Features.Conformance; @@ -37,6 +38,7 @@ public class FirelyTerminologyServiceProxyTests private readonly FirelyTerminologyServiceProxy _proxy; private readonly ITerminologyService _terminologyService; private readonly IAsyncResourceResolver _resourceResolver; + private readonly ILogger _logger; public FirelyTerminologyServiceProxyTests() { @@ -51,10 +53,11 @@ public FirelyTerminologyServiceProxyTests() _resourceResolver.ResolveByCanonicalUriAsync( Arg.Any()) .Returns(Task.FromResult(null)); + _logger = Substitute.For>(); _proxy = new FirelyTerminologyServiceProxy( _terminologyService, _resourceResolver, - Substitute.For>()); + _logger); } [Theory] @@ -342,6 +345,82 @@ private static ValueSet CreateValueSet(string id = default) }; } + [Fact] + public async Task GivenUnknownValueSet_WhenExpanding_ThenLogsWarningAndThrowsResourceNotFoundException() + { + var exception = new FhirOperationException( + "ValueSet 'http://terminology.medigent.ca/fhir/ValueSet/123' is unknown", + HttpStatusCode.NotFound); + + _terminologyService.Expand( + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Throws(exception); + + var parameters = new List> + { + Tuple.Create(TerminologyOperationParameterNames.Expand.Url, "http://terminology.medigent.ca/fhir/ValueSet/123"), + }; + + var ex = await Assert.ThrowsAsync( + () => _proxy.ExpandAsync(parameters, null, CancellationToken.None)); + Assert.Contains("is unknown", ex.Message, StringComparison.OrdinalIgnoreCase); + + _logger.Received(1).Log( + LogLevel.Warning, + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any>()); + + _logger.DidNotReceive().Log( + LogLevel.Error, + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any>()); + } + + [Fact] + public async Task GivenUnexpectedError_WhenExpanding_ThenLogsErrorAndReturnsOperationOutcome() + { + var exception = new FhirOperationException( + "An unexpected server error occurred.", + HttpStatusCode.InternalServerError); + + _terminologyService.Expand( + Arg.Any(), + Arg.Any(), + Arg.Any()) + .Throws(exception); + + var parameters = new List> + { + Tuple.Create(TerminologyOperationParameterNames.Expand.Url, "http://acme.com/fhir/ValueSet/23"), + }; + + var resourceElement = await _proxy.ExpandAsync(parameters, null, CancellationToken.None); + Assert.NotNull(resourceElement); + + var resource = resourceElement.ToPoco() as OperationOutcome; + Assert.NotNull(resource); + + _logger.Received(1).Log( + LogLevel.Error, + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any>()); + + _logger.DidNotReceive().Log( + LogLevel.Warning, + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any>()); + } + public static IEnumerable GetExpandTestData() { var data = new[] diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Conformance/FirelyTerminologyServiceProxy.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Conformance/FirelyTerminologyServiceProxy.cs index 66c645ac4a..cfeac62bc6 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Conformance/FirelyTerminologyServiceProxy.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Conformance/FirelyTerminologyServiceProxy.cs @@ -17,6 +17,7 @@ using Hl7.Fhir.Specification.Terminology; using Microsoft.Extensions.Logging; using Microsoft.Health.Fhir.Core; +using Microsoft.Health.Fhir.Core.Exceptions; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Conformance; using Microsoft.Health.Fhir.Core.Features.Persistence; @@ -96,6 +97,11 @@ public async Task ExpandAsync( #endif return resource.ToResourceElement(); } + catch (FhirOperationException ex) when (ex.Message.Contains("is unknown", StringComparison.OrdinalIgnoreCase)) + { + _logger.LogWarning(ex, "Failed to expand: ValueSet not found."); + throw new ResourceNotFoundException(ex.Message); + } catch (Exception ex) { _logger.LogError(ex, "Failed to expand."); diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Conformance/TerminologyRequestHandler.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Conformance/TerminologyRequestHandler.cs index daf211a15a..0b83537d54 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Conformance/TerminologyRequestHandler.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Conformance/TerminologyRequestHandler.cs @@ -79,6 +79,10 @@ public async Task Handle( cancellationToken); return new ExpandResponse(resource); } + catch (ResourceNotFoundException) + { + throw; + } catch (Exception ex) { _logger.LogError(ex, "Failed to handle the request."); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/ExpandOperationTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/ExpandOperationTests.cs index 5fe210cc4c..42cf7ba86e 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/ExpandOperationTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/ExpandOperationTests.cs @@ -358,5 +358,43 @@ private void Validate( && !string.IsNullOrEmpty(x.Diagnostics) && invalid.Any(y => x.Diagnostics.Contains(y))); } + + [SkippableFact] + public async Task GivenUnknownValueSet_WhenExpanding_ThenReturns404WithOperationOutcome() + { + var expandEnabled = Server.Metadata.SupportsOperation(OperationsConstants.ValueSetExpand); + Skip.IfNot(expandEnabled, "The $expand operation is disabled"); + + var unknownUrl = "http://example.org/fhir/ValueSet/nonexistent-" + Guid.NewGuid().ToString("N"); + var url = $"{KnownResourceTypes.ValueSet}/{KnownRoutes.Expand}?url={unknownUrl}"; + + var ex = await Assert.ThrowsAsync(() => Client.ReadAsync(url)); + Assert.Equal(HttpStatusCode.NotFound, ex.StatusCode); + Assert.NotNull(ex.OperationOutcome); + Assert.NotEmpty(ex.OperationOutcome.Issue); + Assert.Contains( + ex.OperationOutcome.Issue, + x => x.Severity == OperationOutcome.IssueSeverity.Error + && x.Code == OperationOutcome.IssueType.NotFound); + } + + [SkippableFact] + public async Task GivenUnknownValueSetId_WhenExpandingById_ThenReturns404WithOperationOutcome() + { + var expandEnabled = Server.Metadata.SupportsOperation(OperationsConstants.ValueSetExpand); + Skip.IfNot(expandEnabled, "The $expand operation is disabled"); + + var invalidId = Guid.NewGuid().ToString("N"); + var url = $"{KnownResourceTypes.ValueSet}/{invalidId}/{KnownRoutes.Expand}"; + + var ex = await Assert.ThrowsAsync(() => Client.ReadAsync(url)); + Assert.Equal(HttpStatusCode.NotFound, ex.StatusCode); + Assert.NotNull(ex.OperationOutcome); + Assert.NotEmpty(ex.OperationOutcome.Issue); + Assert.Contains( + ex.OperationOutcome.Issue, + x => x.Severity == OperationOutcome.IssueSeverity.Error + && x.Code == OperationOutcome.IssueType.NotFound); + } } } From 42d748994f5b7ad2f7496b6c29e624cb6d745ba3 Mon Sep 17 00:00:00 2001 From: apurvabhaleMS <86023331+apurvabhaleMS@users.noreply.github.com> Date: Tue, 9 Jun 2026 15:08:25 -0700 Subject: [PATCH 03/17] Optimize slow query logger to reduce CPU usage (#5601) * Optimize slow query logger: hash-based Query Store lookup, circuit breaker, timeout cap * Remove circuit breaker from slow query logger (moved to separate PR); clean up tests and harden ExtractParameterHash * Use consistent 'StoredProc' lookup tag for stored procedure Query Store path * Restore fire-and-forget query logging behavior tests * Narrow fire-and-forget Query Store catch to specific exception types (CodeQL) * Catch all exceptions in fire-and-forget Query Store guard * Consolidate ExtractParameterHash tests into a single theory * Address PR review: enrich fire-and-forget warning, Ordinal hash match, sync-comment SQL/test rationale - #1 Include exception type/message in the Warning log so operators can diagnose Query Store lookup failures without Debug logging. - #2 Add comment cross-referencing SqlQueryGenerator.ParametersHashStart for the hardcoded '/* HASH ' SQL literal. - #3 Use StringComparison.Ordinal for the hash markers (always emitted uppercase). - #4 Expand QueryStoreLookupTimeoutSeconds test rationale for the upper bound. - #6 Note the three lookup SQL consts share structure and must be kept in sync. * Address review: use class logger, drop redundant exception text, inline effectiveQuery, XML-doc timeout const - Revert AI suggestion: exception is passed to LogWarning (queryable via env_ex_*), so its type/message is no longer duplicated in the message string. - Remove the ILogger parameter from FireAndForgetQueryStoreLookup and LogQueryStoreByTextAsync; use the class _logger field. - Inline the redundant effectiveQuery alias. - Convert the QueryStoreLookupTimeoutSeconds comment to an XML doc comment. --- .../SqlServerSearchServiceQueryStoreTests.cs | 85 ++++ .../Features/Search/SqlServerSearchService.cs | 372 +++++++++++++----- 2 files changed, 366 insertions(+), 91 deletions(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlServerSearchServiceQueryStoreTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlServerSearchServiceQueryStoreTests.cs index ec384cd308..f55e67d3c0 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlServerSearchServiceQueryStoreTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlServerSearchServiceQueryStoreTests.cs @@ -1057,5 +1057,90 @@ public void StripDboSchemaPrefix_NullInput_ReturnsNull() // Assert Assert.Null(result); } + + // ----------------------------------------------------------------------- + // ExtractParameterHash + // ----------------------------------------------------------------------- + + [Theory] + + // No hash comment -> null + [InlineData(null, null)] + [InlineData("", null)] + [InlineData("SELECT * FROM dbo.Resource", null)] + [InlineData("WITH cte0 AS (SELECT 1) SELECT * FROM cte0", null)] + + // Hash comment with params= suffix -> returns just the base64 hash + [InlineData( + "WITH cte0 AS (SELECT 1)\r\n/* HASH LKzCYkd+9LKATJ3BQebo9R3aYgN9ZXVU/42C2WntCUo= params=@p0,@p1,@p2,@p3,@p4,@p5,@p6,@p7 */\r\nSELECT * FROM cte0", + "LKzCYkd+9LKATJ3BQebo9R3aYgN9ZXVU/42C2WntCUo=")] + [InlineData( + "SELECT 1 /* HASH Ek+CaDdNQVS2c/2D7Gw5XA0Dts4v0KOssOrfn2bNYm0= params=@p0 */", + "Ek+CaDdNQVS2c/2D7Gw5XA0Dts4v0KOssOrfn2bNYm0=")] + + // Hash comment without params= suffix -> returns the hash + [InlineData("SELECT 1 /* HASH abc123def456= */", "abc123def456=")] + + // Realistic multi-CTE chained search query -> returns the embedded hash + [InlineData( + @" + SET STATISTICS IO ON; + SET STATISTICS TIME ON; + + DECLARE @p0 varchar(256) = 'active' + DECLARE @p1 varchar(256) = 'Smith%' + DECLARE @p8 int = 11 + ;WITH + cte0 AS + ( + SELECT ResourceTypeId AS T1, ResourceSurrogateId AS Sid1 + FROM dbo.TokenSearchParam + WHERE SearchParamId = 1260 AND Code = @p0 AND ResourceTypeId = 124 + ) + ,cte7 AS + ( + SELECT DISTINCT TOP (@p8) T1, Sid1, 1 AS IsMatch, 0 AS IsPartial + FROM cte0 + ORDER BY T1 DESC, Sid1 DESC + ) + /* HASH LKzCYkd+9LKATJ3BQebo9R3aYgN9ZXVU/42C2WntCUo= params=@p0,@p1,@p2,@p3,@p4,@p5,@p6,@p7 */ + SELECT * FROM (SELECT DISTINCT r.ResourceTypeId, r.ResourceId FROM dbo.Resource r + JOIN cte7 ON r.ResourceTypeId = cte7.T1 AND r.ResourceSurrogateId = cte7.Sid1 + WHERE IsHistory = 0 AND IsDeleted = 0 + ) AS t ORDER BY t.ResourceTypeId DESC, t.ResourceSurrogateId DESC + option (recompile)", + "LKzCYkd+9LKATJ3BQebo9R3aYgN9ZXVU/42C2WntCUo=")] + + // Malformed: hash start present but no closing */ -> null + [InlineData("SELECT 1 /* HASH abc123=", null)] + + // Empty / whitespace-only hash -> null (otherwise the downstream LIKE filter + // would match every hash-bearing Query Store row) + [InlineData("SELECT 1 /* HASH params=@p0 */", null)] // empty hash before params= + [InlineData("SELECT 1 /* HASH */", null)] // whitespace-only hash, no params + public void ExtractParameterHash_ReturnsExpectedHashOrNull(string input, string expectedHash) + { + // Act + string result = SqlServerSearchService.ExtractParameterHash(input); + + // Assert + Assert.Equal(expectedHash, result); + } + + // ----------------------------------------------------------------------- + // Query Store throttling constants + // ----------------------------------------------------------------------- + + [Fact] + public void QueryStoreLookupTimeoutSeconds_IsCappedReasonably() + { + // The diagnostic query timeout must stay short so it never inherits the (much longer) + // main query timeout. It also acts only as a backup to the 2s CancellationToken, so it + // must be at least 1s and comfortably above the CTS deadline. The 10s upper bound is a + // sanity guard, not a hard requirement: if a future change intentionally raises the + // constant past 10s, update both this bound and the rationale comment on + // SqlServerSearchService.QueryStoreLookupTimeoutSeconds. + Assert.InRange(SqlServerSearchService.QueryStoreLookupTimeoutSeconds, 1, 10); + } } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs index cd6ee8be33..1baeb4acac 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs @@ -86,6 +86,14 @@ internal class SqlServerSearchService : SearchService private static readonly Regex WhitespacePattern = new Regex(@"\s+", RegexOptions.Compiled); private static ResourceSearchParamStats _resourceSearchParamStats; private static object _locker = new object(); + + /// + /// Hard cap for the diagnostic query command timeout (seconds). The CancellationToken + /// timeout (2s) is the first line of defense; this is a backup in case cancellation + /// doesn't terminate the SQL command promptly. Set on a NEW connection — does not + /// affect search query connections. + /// + internal const int QueryStoreLookupTimeoutSeconds = 5; private static CachedParameter _longRunningQueryDetails; private static CachedParameter _longRunningThreshold; @@ -830,31 +838,8 @@ await _sqlRetryService.ExecuteSql( string queryTextSnapshot = sqlCommand.CommandText; bool isStoredProcSnapshot = sqlCommand.CommandType == CommandType.StoredProcedure; long executionTimeSnapshot = executionStopwatch.ElapsedMilliseconds; - int timeoutSnapshot = (int)_sqlServerDataStoreConfiguration.CommandTimeout.TotalSeconds; - // Fire-and-forget: Log query details without blocking the response - _ = Task.Run(async () => - { - using var loggingCts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); - try - { - await LogQueryStoreByTextAsync( - queryTextSnapshot, - isStoredProcSnapshot, - _logger, - timeoutSnapshot, - executionTimeSnapshot, - loggingCts.Token); - } - catch (Exception ex) - { - _logger.LogWarning( - "Long-running SQL ({ElapsedMilliseconds}ms). Query: {QueryText}. Query Store lookup failed for long-running query.", - executionTimeSnapshot, - queryTextSnapshot); - _logger.LogDebug(ex, "Query Store lookup failed for long-running query."); - } - }); + FireAndForgetQueryStoreLookup(queryTextSnapshot, isStoredProcSnapshot, executionTimeSnapshot); } } } @@ -1140,10 +1125,82 @@ internal static string StripAllWhitespace(string text) internal static string StripDboSchemaPrefix(string procName) => procName?.Replace("dbo.", string.Empty, StringComparison.OrdinalIgnoreCase); + /// + /// Extracts the parameter hash value from a query text that contains a + /// /* HASH {base64hash} params=... */ comment embedded by . + /// Returns null if no hash comment is found. + /// + internal static string ExtractParameterHash(string queryText) + { + if (string.IsNullOrEmpty(queryText)) + { + return null; + } + + // ParametersHashStart/End are always emitted in fixed uppercase by SqlQueryGenerator, + // so use Ordinal (not OrdinalIgnoreCase) to avoid matching arbitrary user-authored + // lowercase comments such as "/* hash ... */". + int hashStart = queryText.IndexOf(Expressions.Visitors.QueryGenerators.SqlQueryGenerator.ParametersHashStart, StringComparison.Ordinal); + if (hashStart < 0) + { + return null; + } + + int valueStart = hashStart + Expressions.Visitors.QueryGenerators.SqlQueryGenerator.ParametersHashStart.Length; + int hashEnd = queryText.IndexOf(Expressions.Visitors.QueryGenerators.SqlQueryGenerator.ParametersHashEnd, valueStart, StringComparison.Ordinal); + if (hashEnd < 0) + { + return null; + } + + // Extract just the base64 hash, stopping at the space before "params=" + string hashAndParams = queryText[valueStart..hashEnd]; + int spaceIndex = hashAndParams.IndexOf(' ', StringComparison.Ordinal); + string hash = spaceIndex >= 0 ? hashAndParams[..spaceIndex] : hashAndParams; + + // Guard against an empty/whitespace-only hash, which would make the downstream + // LIKE '%/* HASH {hash}%' filter match every hash-bearing row. + return string.IsNullOrWhiteSpace(hash) ? null : hash; + } + + /// + /// Runs as a fire-and-forget background task so the + /// diagnostic Query Store lookup never blocks or fails the originating search request. + /// + private void FireAndForgetQueryStoreLookup(string queryText, bool isStoredProcedure, long executionTime) + { + _ = Task.Run(async () => + { + try + { + // CancellationToken fires at 2s; CommandTimeout at 5s is backup. + using var loggingCts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + + await LogQueryStoreByTextAsync( + queryText, + isStoredProcedure, + QueryStoreLookupTimeoutSeconds, + executionTime, + loggingCts.Token); + } + catch (Exception ex) + { + // The Query Store lookup is best-effort diagnostics. Swallow any failure so the + // fire-and-forget task never surfaces an unobserved exception. The exception is + // passed to the logger (queryable via env_ex_* columns), so it isn't repeated in the message. + _logger.LogWarning( + ex, + "Long-running SQL ({ElapsedMilliseconds}ms). Query={Query} QueryStoreStats={QueryStoreStats}", + executionTime, + queryText, + "Query Store lookup failed."); + } + }); + } + private async Task LogQueryStoreByTextAsync( string queryText, bool isStoredProcedure, - ILogger logger, int timeoutSeconds, long executionTime, CancellationToken ct) @@ -1158,12 +1215,84 @@ await _sqlRetryService.ExecuteSql( var sb = new StringBuilder(); - // Query Store records only the bare procedure name without the schema prefix. - string effectiveQuery = isStoredProcedure ? StripDboSchemaPrefix(queryText) : queryText; - var normalizedText = StripQueryPreambleLines(effectiveQuery); - var searchFragments = SplitIntoSearchFragments(normalizedText); + if (isStoredProcedure) + { + // For stored procedures, use OBJECT_ID to filter directly by the procedure's + // hash/identity in Query Store. This avoids the expensive LIKE scan on + // query_sql_text entirely, since Query Store records object_id for every + // statement executed inside a stored procedure. + string procName = StripDboSchemaPrefix(queryText); - cmd.CommandText = @" + cmd.CommandText = @" + DECLARE @CutoffTime datetimeoffset = DATEADD(HOUR, -1, SYSUTCDATETIME()); + + SELECT TOP (5) + rs.count_executions, + rs.avg_duration / 1000.0 AS avg_duration_ms, + rs.avg_cpu_time / 1000.0 AS avg_cpu_ms, + rs.avg_logical_io_reads, + rs.avg_physical_io_reads, + rs.avg_logical_io_writes, + rs.avg_rowcount, + rs.max_duration / 1000.0 AS max_duration_ms, + rs.last_execution_time, + p.plan_id, + q.query_id + FROM sys.query_store_query q + JOIN sys.query_store_plan p ON p.query_id = q.query_id + JOIN sys.query_store_runtime_stats rs ON rs.plan_id = p.plan_id + WHERE q.object_id = OBJECT_ID(@ProcName) + AND rs.last_execution_time >= @CutoffTime + ORDER BY rs.last_execution_time DESC;"; + + cmd.Parameters.AddWithValue("@ProcName", procName); + + using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false); + await AppendQueryStoreResults(reader, sb, 0, 1, "StoredProc", ct); + } + else + { + // For ad-hoc queries, split into fragments (include queries have 2 statements + // split at INSERT INTO @FilteredData). For each fragment individually: + // - If it contains a parameter hash comment: use the hash for a fast LIKE lookup + // - If hash lookup returns nothing: fall back to the expensive REPLACE+LIKE + // - If it has no hash: filter OUT hash-bearing rows to reduce the LIKE scan set + var normalizedText = StripQueryPreambleLines(queryText); + var searchFragments = SplitIntoSearchFragments(normalizedText); + + // NOTE: The three lookup SQL strings below (HashLookupSql, + // TextLookupWithHashExclusionSql, TextLookupSql) share an identical + // SELECT / FROM / JOIN / ORDER BY structure and only differ in their WHERE + // clause. Any column, cutoff-window, or index-hint change must be applied to + // all three to avoid drift. + // + // SQL for the fast hash-based lookup (no REPLACE chain needed). + const string HashLookupSql = @" + DECLARE @CutoffTime datetimeoffset = DATEADD(HOUR, -1, SYSUTCDATETIME()); + + SELECT TOP (5) + rs.count_executions, + rs.avg_duration / 1000.0 AS avg_duration_ms, + rs.avg_cpu_time / 1000.0 AS avg_cpu_ms, + rs.avg_logical_io_reads, + rs.avg_physical_io_reads, + rs.avg_logical_io_writes, + rs.avg_rowcount, + rs.max_duration / 1000.0 AS max_duration_ms, + rs.last_execution_time, + p.plan_id, + q.query_id + FROM sys.query_store_query_text qt + JOIN sys.query_store_query q ON q.query_text_id = qt.query_text_id + JOIN sys.query_store_plan p ON p.query_id = q.query_id + JOIN sys.query_store_runtime_stats rs ON rs.plan_id = p.plan_id + WHERE qt.query_sql_text LIKE '%' + @HashFilter + '%' + AND rs.last_execution_time >= @CutoffTime + ORDER BY rs.last_execution_time DESC;"; + + // SQL for the expensive REPLACE+LIKE fallback. + // For fragments without a hash, also filter OUT hash-bearing rows to reduce scan set. + const string TextLookupWithHashExclusionSql = @" DECLARE @CutoffTime datetimeoffset = DATEADD(HOUR, -1, SYSUTCDATETIME()); SELECT TOP (5) @@ -1183,58 +1312,95 @@ FROM sys.query_store_query_text qt JOIN sys.query_store_plan p ON p.query_id = q.query_id JOIN sys.query_store_runtime_stats rs ON rs.plan_id = p.plan_id WHERE @NormalizedText <> '' + -- The '/* HASH ' literal below must stay in sync with + -- SqlQueryGenerator.ParametersHashStart (SQL const strings cannot reference the C# constant). + AND qt.query_sql_text NOT LIKE '%/* HASH %' AND replace(replace(replace(replace(replace(replace(qt.query_sql_text, char(9), ''), char(10), ''), char(11), ''), char(12), ''), char(13), ''), char(32), '') LIKE '%' + @NormalizedText + '%' AND rs.last_execution_time >= @CutoffTime ORDER BY rs.last_execution_time DESC;"; - for (int segmentIndex = 0; segmentIndex < searchFragments.Count; segmentIndex++) - { - string searchFragment = searchFragments[segmentIndex]; + // SQL for the expensive REPLACE+LIKE fallback (no hash exclusion, + // used when hash lookup found nothing for a hash-bearing fragment). + const string TextLookupSql = @" + DECLARE @CutoffTime datetimeoffset = DATEADD(HOUR, -1, SYSUTCDATETIME()); - // Strip whitespace first so the 4000-char limit applies to stripped content, - // maximising the amount of meaningful text sent to the LIKE comparison. - string strippedFragment = StripAllWhitespace(searchFragment); + SELECT TOP (5) + rs.count_executions, + rs.avg_duration / 1000.0 AS avg_duration_ms, + rs.avg_cpu_time / 1000.0 AS avg_cpu_ms, + rs.avg_logical_io_reads, + rs.avg_physical_io_reads, + rs.avg_logical_io_writes, + rs.avg_rowcount, + rs.max_duration / 1000.0 AS max_duration_ms, + rs.last_execution_time, + p.plan_id, + q.query_id + FROM sys.query_store_query_text qt + JOIN sys.query_store_query q ON q.query_text_id = qt.query_text_id + JOIN sys.query_store_plan p ON p.query_id = q.query_id + JOIN sys.query_store_runtime_stats rs ON rs.plan_id = p.plan_id + WHERE @NormalizedText <> '' + AND replace(replace(replace(replace(replace(replace(qt.query_sql_text, char(9), ''), char(10), ''), char(11), ''), char(12), ''), char(13), ''), char(32), '') LIKE '%' + @NormalizedText + '%' + AND rs.last_execution_time >= @CutoffTime + ORDER BY rs.last_execution_time DESC;"; - if (strippedFragment.Length > 4000) + for (int segmentIndex = 0; segmentIndex < searchFragments.Count; segmentIndex++) { - strippedFragment = strippedFragment[..4000]; - } + string searchFragment = searchFragments[segmentIndex]; - cmd.Parameters.Clear(); - cmd.Parameters.AddWithValue("@NormalizedText", strippedFragment); + // Check each fragment individually for an embedded parameter hash. + // Include queries split into 2 fragments: fragment 1 (before INSERT INTO @FilteredData) + // typically has no hash, fragment 2 (after) has the hash comment. + string fragmentHash = ExtractParameterHash(searchFragment); + bool fragmentHasHash = fragmentHash != null; + int matchCount = 0; - using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false); - int matchIndex = 0; - while (await reader.ReadAsync(ct).ConfigureAwait(false)) - { - if (await reader.IsDBNullAsync(0, ct).ConfigureAwait(false)) + if (fragmentHasHash) { - continue; + // Fast path: search by the embedded parameter hash string. + cmd.CommandText = HashLookupSql; + cmd.Parameters.Clear(); + string hashFilter = Expressions.Visitors.QueryGenerators.SqlQueryGenerator.ParametersHashStart + fragmentHash; + cmd.Parameters.AddWithValue("@HashFilter", hashFilter); + + using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false); + matchCount = await AppendQueryStoreResults(reader, sb, segmentIndex, searchFragments.Count, "Hash", ct); } - matchIndex++; - long planId = reader.GetInt64(9); - long queryId = reader.GetInt64(10); - - sb.AppendLine() - .Append($" batch[{segmentIndex + 1}] match[{matchIndex}]") - .Append($" execs={reader.GetInt64(0)}") - .Append($" avgDurMs={Convert.ToDouble(reader.GetValue(1)):F1}") - .Append($" avgCpuMs={Convert.ToDouble(reader.GetValue(2)):F1}") - .Append($" avgLReads={Convert.ToDouble(reader.GetValue(3)):F0}") - .Append($" avgPReads={Convert.ToDouble(reader.GetValue(4)):F0}") - .Append($" avgLWrites={Convert.ToDouble(reader.GetValue(5)):F0}") - .Append($" avgRows={Convert.ToDouble(reader.GetValue(6)):F0}") - .Append($" maxDurMs={Convert.ToDouble(reader.GetValue(7)):F1}") - .Append($" lastExec={reader.GetDateTimeOffset(8):o}") - .Append($" queryId={queryId}") - .Append($" planId={planId}"); + // Fall back to REPLACE+LIKE if hash lookup found nothing or fragment has no hash. + if (matchCount == 0) + { + string strippedFragment = StripAllWhitespace(searchFragment); + + if (strippedFragment.Length > 4000) + { + strippedFragment = strippedFragment[..4000]; + } + + // Fragments without a hash: exclude hash-bearing query store rows. + // Fragments with a hash that had no hash match: search all rows as fallback. + if (fragmentHasHash) + { + cmd.CommandText = TextLookupSql; + } + else + { + cmd.CommandText = TextLookupWithHashExclusionSql; + } + + cmd.Parameters.Clear(); + cmd.Parameters.AddWithValue("@NormalizedText", strippedFragment); + + using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false); + await AppendQueryStoreResults(reader, sb, segmentIndex, searchFragments.Count, fragmentHasHash ? "TextFallback" : "TextNoHash", ct); + } } } if (sb.Length > 0) { - logger.LogWarning( + _logger.LogWarning( "Long-running SQL ({ElapsedMilliseconds}ms). Query={Query} QueryStoreStats={QueryStoreStats}", executionTime, queryText, @@ -1242,18 +1408,65 @@ AND replace(replace(replace(replace(replace(replace(qt.query_sql_text, char(9), } else { - logger.LogWarning( + _logger.LogWarning( "Long-running SQL ({ElapsedMilliseconds}ms). Query={Query} QueryStoreStats={QueryStoreStats}", executionTime, queryText, "No Query Store matches found."); } }, - logger, + _logger, ct, isReadOnly: true); } + /// + /// Reads Query Store results from a and appends formatted + /// stats to the . Returns the number of matches read. + /// + private static async Task AppendQueryStoreResults( + SqlDataReader reader, + StringBuilder sb, + int segmentIndex, + int totalSegments, + string lookupMethod, + CancellationToken ct) + { + int matchIndex = 0; + while (await reader.ReadAsync(ct).ConfigureAwait(false)) + { + if (await reader.IsDBNullAsync(0, ct).ConfigureAwait(false)) + { + continue; + } + + matchIndex++; + long planId = reader.GetInt64(9); + long queryId = reader.GetInt64(10); + + string prefix = totalSegments > 1 + ? $" batch[{segmentIndex + 1}] match[{matchIndex}]" + : $" match[{matchIndex}]"; + + sb.AppendLine() + .Append(prefix) + .Append($" lookup={lookupMethod}") + .Append($" execs={reader.GetInt64(0)}") + .Append($" avgDurMs={Convert.ToDouble(reader.GetValue(1)):F1}") + .Append($" avgCpuMs={Convert.ToDouble(reader.GetValue(2)):F1}") + .Append($" avgLReads={Convert.ToDouble(reader.GetValue(3)):F0}") + .Append($" avgPReads={Convert.ToDouble(reader.GetValue(4)):F0}") + .Append($" avgLWrites={Convert.ToDouble(reader.GetValue(5)):F0}") + .Append($" avgRows={Convert.ToDouble(reader.GetValue(6)):F0}") + .Append($" maxDurMs={Convert.ToDouble(reader.GetValue(7)):F1}") + .Append($" lastExec={reader.GetDateTimeOffset(8):o}") + .Append($" queryId={queryId}") + .Append($" planId={planId}"); + } + + return matchIndex; + } + private static (long StartId, long EndId, int Count) ReaderToSurrogateIdRange(SqlDataReader sqlDataReader) { return (sqlDataReader.GetInt64(1), sqlDataReader.GetInt64(2), sqlDataReader.GetInt32(3)); @@ -2017,31 +2230,8 @@ await _sqlRetryService.ExecuteSql( string queryTextSnapshot = sqlCommand.CommandText; bool isStoredProcSnapshot = sqlCommand.CommandType == CommandType.StoredProcedure; long executionTimeSnapshot = executionStopwatch.ElapsedMilliseconds; - int timeoutSnapshot = (int)_sqlServerDataStoreConfiguration.CommandTimeout.TotalSeconds; - // Fire-and-forget: Log query details without blocking the response - _ = Task.Run(async () => - { - using var loggingCts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); - try - { - await LogQueryStoreByTextAsync( - queryTextSnapshot, - isStoredProcSnapshot, - _logger, - timeoutSnapshot, - executionTimeSnapshot, - loggingCts.Token); - } - catch (Exception ex) - { - _logger.LogWarning( - "Long-running SQL ({ElapsedMilliseconds}ms). Query: {QueryText}. Query Store lookup failed for long-running query.", - executionTimeSnapshot, - queryTextSnapshot); - _logger.LogDebug(ex, "Query Store lookup failed for long-running query."); - } - }); + FireAndForgetQueryStoreLookup(queryTextSnapshot, isStoredProcSnapshot, executionTimeSnapshot); } } } From 4605738fa44137bf09dac80fdb7958c6e9179eb2 Mon Sep 17 00:00:00 2001 From: SergeyGaluzo <95932081+SergeyGaluzo@users.noreply.github.com> Date: Tue, 9 Jun 2026 21:52:25 -0700 Subject: [PATCH 04/17] Search param concurrency based on max last updated (#5576) * added max last updated to behavior * Removed incorrect "cycle conclusive" logic. Exclude deleted and pending delete from in cache logic * Added mx last updated logic. * log only for background * Added logic to invoke cache refresh * Fixed delete test * Removed in cache checks * Changed pipreline to process all via mediator * Delete search params on-by-one * Added last updated to status update * Removed concurrency manager * last * deleted * Fixing search param tests * fix context * Fixed reindex job test * Fixed test * cmd using * context in smart search tests * Restored filtering on urls * 500 and comment out theories that show bugs * skip cache refresh * split urls * Revert "Restored filtering on urls" This reverts commit 385a0be1894820d5e5b691019ad6424789792e5a. * comments * obsolete * Bad request * comment * common ex logic in single place * Corrected exception check in the test * better error message * Partial implementation of max before validate * Skip 500 for Cosmos * skip cosmos * Test fixes * Second stage * fix unit test * customer message * Remove ==true * min 112 * make date not null * Fix name * Fixing pre-existing exception bug * Use set extension * Comments and removed foreach for included search params * serializing search param deletes * shorter name * removed to list * search result entry * Fix * Retries * Fix to retries * rename * Header * more renames * Add concurrent bundles test * Latest AI suggestion * Simple retries on controller level * Removed obsolete test * fixed deletion serrvice tests * using * using * extra * s * Added conditional * added bug link * added fail * Potential fix for pull request finding 'CodeQL / Useless assignment to local variable' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --------- Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- ...-searchparameter-concurrency-management.md | 5 +- .../adr-2603-search-params-concurrency.md | 5 + .../SearchParameterConcurrencyManagerTests.cs | 492 ------------------ .../Context/FhirRequestContextExtensions.cs | 36 ++ .../Reindex/ReindexOrchestratorJob.cs | 6 +- .../Reindex/ReindexProcessingJob.cs | 2 +- .../CreateOrUpdateSearchParameterBehavior.cs | 43 +- .../DeleteSearchParameterBehavior.cs | 17 +- .../Parameters/ISearchParameterOperations.cs | 4 +- .../Parameters/SearchParameterOperations.cs | 248 ++++----- ...rchParameterRequestContextPropertyNames.cs | 1 + .../Search/Parameters/SearchParameterRetry.cs | 57 ++ .../Registry/ISearchParameterStatusManager.cs | 2 +- .../SearchParameterConcurrencyException.cs | 40 -- .../SearchParameterConcurrencyManager.cs | 148 ------ .../Registry/SearchParameterStatusManager.cs | 4 +- .../Resources.Designer.cs | 11 +- src/Microsoft.Health.Fhir.Core/Resources.resx | 5 +- .../Controllers/FhirControllerTests.cs | 209 +++++++- .../SearchParameterFilterAttributeTests.cs | 42 +- .../Controllers/FhirController.cs | 96 +++- .../Filters/SearchParameterFilterAttribute.cs | 27 +- .../Resources/Bundle/BundleHandler.cs | 36 +- .../Resources/Delete/DeletionServiceTests.cs | 116 +++++ .../SearchParameterBehaviorTests.cs | 15 +- .../SearchParameterRetryTests.cs | 94 ++++ .../SearchParameterValidatorTests.cs | 1 + ...ealth.Fhir.Shared.Core.UnitTests.projitems | 1 + .../Resources/Delete/DeletionService.cs | 72 ++- .../Parameters/ISearchParameterValidator.cs | 4 +- .../Parameters/SearchParameterValidator.cs | 13 +- .../Features/ExceptionExtension.cs | 10 + .../Features/Schema/SchemaVersionConstants.cs | 4 +- ...SqlServerSearchParameterStatusDataStore.cs | 65 +-- .../Storage/SqlServerFhirDataStore.cs | 8 +- .../Rest/Reindex/ReindexTests.cs | 268 +++++++++- .../FailingSearchParameterStatusManager.cs | 2 +- .../Operations/Reindex/ReindexJobTests.cs | 39 +- .../Features/Smart/SmartSearchTests.cs | 5 +- ...th.Fhir.Shared.Tests.Integration.projitems | 2 +- .../Persistence/FhirStorageTests.cs | 12 +- .../Persistence/FhirStorageTestsFixture.cs | 59 ++- ...erOptimisticConcurrencyIntegrationTests.cs | 454 ---------------- ...archParameterOptimisticConcurrencyTests.cs | 176 +++++++ .../SearchParameterStatusDataStoreTests.cs | 46 +- .../SqlServerCreateStatsForSmartTests.cs | 5 +- .../SqlServerFhirStorageTestHelper.cs | 13 +- ...rverSearchParameterStatusDataStoreTests.cs | 117 +++-- 48 files changed, 1544 insertions(+), 1593 deletions(-) delete mode 100644 src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/Registry/SearchParameterConcurrencyManagerTests.cs create mode 100644 src/Microsoft.Health.Fhir.Core/Features/Context/FhirRequestContextExtensions.cs create mode 100644 src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterRetry.cs delete mode 100644 src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterConcurrencyException.cs delete mode 100644 src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterConcurrencyManager.cs create mode 100644 src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterRetryTests.cs delete mode 100644 test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SearchParameterOptimisticConcurrencyIntegrationTests.cs create mode 100644 test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SearchParameterOptimisticConcurrencyTests.cs diff --git a/docs/arch/adr-2512-searchparameter-concurrency-management.md b/docs/arch/adr-2512-searchparameter-concurrency-management.md index a5223aa748..59148abd28 100644 --- a/docs/arch/adr-2512-searchparameter-concurrency-management.md +++ b/docs/arch/adr-2512-searchparameter-concurrency-management.md @@ -1,3 +1,6 @@ +## Status +**Obsolete** - Code supporting this ADR removed per https://github.com/microsoft/fhir-server/blob/main/docs/arch/adr-2603-search-param-concurrency.md + # ADR 2512: SearchParameter Concurrency Management - Application Level Locking and Database Optimistic Concurrency *Labels*: [SQL](https://github.com/microsoft/fhir-server/labels/Area-SQL) | [Core](https://github.com/microsoft/fhir-server/labels/Area-Core) | [SearchParameter](https://github.com/microsoft/fhir-server/labels/Area-SearchParameter) @@ -99,7 +102,7 @@ HTTP Request ? CreateOrUpdateSearchParameterBehavior ? SearchParameterOperations - Proper LastUpdated handling in GET and UPSERT operations ## Status -**Accepted** - Implemented and deployed +**Obsolete** - Code supporting this ADR removed per https://github.com/microsoft/fhir-server/blob/main/docs/arch/adr-2603-search-param-concurrency.md ## Consequences diff --git a/docs/arch/adr-2603-search-params-concurrency.md b/docs/arch/adr-2603-search-params-concurrency.md index c77bbf71b1..423533fa6e 100644 --- a/docs/arch/adr-2603-search-params-concurrency.md +++ b/docs/arch/adr-2603-search-params-concurrency.md @@ -38,3 +38,8 @@ We will implement optimistic concurrency across all search params based on max(L ## Notes This ADR superceeds previous implementation https://github.com/microsoft/fhir-server/blob/main/docs/arch/adr-2512-searchparameter-concurrency-management.md. Previous implementation should to be removed. + +## Message to customers +To guarantee store data integrity, FHIR server implemented strict concurrency control for search parameter writes. +When requests to write search parameters are sent in parallel, depending on timing, some requests might fail with concurrency conflict errors. This is because each write operation requires validation against the reference set, and concurrent modifications might lead to data integrity issues, therefore they are restricted. +When writing search parameters, avoid sending multiple parallel requests. If you need to process multiple search parameters, send requests one after another, or use a single bundle call. diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/Registry/SearchParameterConcurrencyManagerTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/Registry/SearchParameterConcurrencyManagerTests.cs deleted file mode 100644 index d5a9e08a64..0000000000 --- a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/Registry/SearchParameterConcurrencyManagerTests.cs +++ /dev/null @@ -1,492 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Health.Fhir.Core.Features.Search.Registry; -using Microsoft.Health.Fhir.Tests.Common; -using Microsoft.Health.Test.Utilities; -using Xunit; - -namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Search.Registry -{ - [Trait(Traits.OwningTeam, OwningTeam.Fhir)] - [Trait(Traits.Category, Categories.Search)] - public class SearchParameterConcurrencyManagerTests - { - private const string TestUri1 = "http://test.com/searchparam1"; - private const string TestUri2 = "http://test.com/searchparam2"; - - [Fact] - public async Task GivenSingleSearchParameterUri_WhenExecutingWithLock_ThenOperationCompletes() - { - // Arrange - const string expectedResult = "test result"; - var executionCount = 0; - - // Act - var result = await SearchParameterConcurrencyManager.ExecuteWithLockAsync(TestUri1, () => - { - executionCount++; - return Task.FromResult(expectedResult); - }); - - // Assert - Assert.Equal(expectedResult, result); - Assert.Equal(1, executionCount); - } - - [Fact] - public async Task GivenSingleSearchParameterUri_WhenExecutingWithLockVoidOperation_ThenOperationCompletes() - { - // Arrange - var executionCount = 0; - - // Act - await SearchParameterConcurrencyManager.ExecuteWithLockAsync(TestUri1, () => - { - executionCount++; - return Task.CompletedTask; - }); - - // Assert - Assert.Equal(1, executionCount); - } - - [Fact] - public async Task GivenSameSearchParameterUri_WhenExecutingConcurrently_ThenOperationsExecuteSequentially() - { - // Arrange - const int concurrentOperations = 5; - var executionOrder = new List(); - var lockObject = new object(); - var entryBarriers = new List>(); - var continueSignals = new List>(); - - // Create synchronization primitives for each operation - for (int i = 0; i < concurrentOperations; i++) - { - entryBarriers.Add(new TaskCompletionSource()); - continueSignals.Add(new TaskCompletionSource()); - } - - // Act - var tasks = new List>(); - for (int i = 0; i < concurrentOperations; i++) - { - var operationId = i; - var entryBarrier = entryBarriers[i]; - var continueSignal = continueSignals[i]; - - tasks.Add(SearchParameterConcurrencyManager.ExecuteWithLockAsync(TestUri1, async () => - { - lock (lockObject) - { - executionOrder.Add(operationId); - } - - // Signal that this operation has entered the critical section - entryBarrier.SetResult(true); - - // Wait for permission to continue (controlled by test) - await continueSignal.Task; - - return operationId; - })); - } - - // Verify operations execute sequentially by controlling their execution - for (int i = 0; i < concurrentOperations; i++) - { - // Wait for the next operation to enter the critical section - await entryBarriers[i].Task; - - // Verify only this operation has executed so far - lock (lockObject) - { - Assert.Equal(i + 1, executionOrder.Count); - Assert.Equal(i, executionOrder[i]); - } - - // Allow this operation to complete - continueSignals[i].SetResult(true); - } - - var results = await Task.WhenAll(tasks); - - // Assert - Assert.Equal(concurrentOperations, results.Length); - Assert.Equal(concurrentOperations, executionOrder.Count); - - // Verify all operations completed in the correct order - for (int i = 0; i < concurrentOperations; i++) - { - Assert.Equal(i, executionOrder[i]); - Assert.Equal(i, results[i]); - } - } - - [Fact] - public async Task GivenDifferentSearchParameterUris_WhenExecutingConcurrently_ThenOperationsExecuteInParallel() - { - // Arrange - const int operationsPerUri = 2; - var uri1StartedCount = 0; - var uri2StartedCount = 0; - var bothUrisStarted = new TaskCompletionSource(); - var canContinue = new TaskCompletionSource(); - var lockObject = new object(); - - // Act - var tasks = new List(); - - // Add operations for first URI - for (int i = 0; i < operationsPerUri; i++) - { - tasks.Add(SearchParameterConcurrencyManager.ExecuteWithLockAsync(TestUri1, async () => - { - bool shouldSignal = false; - lock (lockObject) - { - uri1StartedCount++; - - // Signal when both URIs have at least one operation started - if (uri1StartedCount > 0 && uri2StartedCount > 0) - { - shouldSignal = true; - } - } - - if (shouldSignal) - { - bothUrisStarted.TrySetResult(true); - } - - await canContinue.Task; - })); - } - - // Add operations for second URI - for (int i = 0; i < operationsPerUri; i++) - { - tasks.Add(SearchParameterConcurrencyManager.ExecuteWithLockAsync(TestUri2, async () => - { - bool shouldSignal = false; - lock (lockObject) - { - uri2StartedCount++; - - // Signal when both URIs have at least one operation started - if (uri1StartedCount > 0 && uri2StartedCount > 0) - { - shouldSignal = true; - } - } - - if (shouldSignal) - { - bothUrisStarted.TrySetResult(true); - } - - await canContinue.Task; - })); - } - - // Wait for operations on both URIs to start concurrently - await bothUrisStarted.Task; - - // Verify both URIs have operations running concurrently - lock (lockObject) - { - Assert.True(uri1StartedCount > 0, "At least one operation on URI1 should have started"); - Assert.True(uri2StartedCount > 0, "At least one operation on URI2 should have started"); - } - - // Allow all operations to complete - canContinue.SetResult(true); - await Task.WhenAll(tasks); - - // Assert final counts - Assert.Equal(operationsPerUri, uri1StartedCount); - Assert.Equal(operationsPerUri, uri2StartedCount); - } - - [Fact] - public void GivenInitialState_WhenCheckingActiveLockCount_ThenReturnsZero() - { - // Act & Assert - Assert.Equal(0, SearchParameterConcurrencyManager.ActiveLockCount); - } - - [Fact] - public async Task GivenOperationsInProgress_WhenCheckingActiveLockCount_ThenReturnsCorrectCount() - { - // Arrange - var task1Started = new TaskCompletionSource(); - var task1CanContinue = new TaskCompletionSource(); - var task2Started = new TaskCompletionSource(); - var task2CanContinue = new TaskCompletionSource(); - - // Act - Start two operations on different URIs - var task1 = SearchParameterConcurrencyManager.ExecuteWithLockAsync(TestUri1, async () => - { - task1Started.SetResult(true); - await task1CanContinue.Task; - }); - - var task2 = SearchParameterConcurrencyManager.ExecuteWithLockAsync(TestUri2, async () => - { - task2Started.SetResult(true); - await task2CanContinue.Task; - }); - - // Wait for both operations to start - await task1Started.Task; - await task2Started.Task; - - // Assert - Should have 2 active locks - Assert.Equal(2, SearchParameterConcurrencyManager.ActiveLockCount); - - // Complete first operation - task1CanContinue.SetResult(true); - await task1; - - // Wait for cleanup with exponential backoff instead of fixed delay - var activeLockCount = SearchParameterConcurrencyManager.ActiveLockCount; - var attempts = 0; - while (activeLockCount != 1 && attempts < 10) - { - await Task.Delay(10 * (int)Math.Pow(2, attempts)); - activeLockCount = SearchParameterConcurrencyManager.ActiveLockCount; - attempts++; - } - - // Assert - Should have 1 active lock - Assert.Equal(1, activeLockCount); - - // Complete second operation - task2CanContinue.SetResult(true); - await task2; - - // Wait for final cleanup with exponential backoff - activeLockCount = SearchParameterConcurrencyManager.ActiveLockCount; - attempts = 0; - while (activeLockCount != 0 && attempts < 10) - { - await Task.Delay(10 * (int)Math.Pow(2, attempts)); - activeLockCount = SearchParameterConcurrencyManager.ActiveLockCount; - attempts++; - } - - // Assert - Should have 0 active locks - Assert.Equal(0, activeLockCount); - } - - [Fact] - public async Task GivenExceptionInOperation_WhenExecutingWithLock_ThenExceptionIsPropagatedAndLockIsReleased() - { - // Arrange - var expectedException = new InvalidOperationException("Test exception"); - - // Act & Assert - var actualException = await Assert.ThrowsAsync(() => - SearchParameterConcurrencyManager.ExecuteWithLockAsync(TestUri1, () => - { - throw expectedException; - })); - - Assert.Same(expectedException, actualException); - - // Wait for cleanup with exponential backoff instead of fixed delay - var activeLockCount = SearchParameterConcurrencyManager.ActiveLockCount; - var attempts = 0; - while (activeLockCount != 0 && attempts < 10) - { - await Task.Delay(10 * (int)Math.Pow(2, attempts)); - activeLockCount = SearchParameterConcurrencyManager.ActiveLockCount; - attempts++; - } - - // Verify lock is released by ensuring a subsequent operation can execute - var subsequentExecuted = false; - await SearchParameterConcurrencyManager.ExecuteWithLockAsync(TestUri1, () => - { - subsequentExecuted = true; - return Task.CompletedTask; - }); - - Assert.True(subsequentExecuted); - Assert.Equal(0, SearchParameterConcurrencyManager.ActiveLockCount); - } - - [Fact] - public async Task GivenCancellationToken_WhenOperationIsCancelled_ThenOperationCancelsAndLockIsReleased() - { - // Arrange - var operationStarted = new TaskCompletionSource(); - using var cts = new CancellationTokenSource(); - - // Act - var operationTask = SearchParameterConcurrencyManager.ExecuteWithLockAsync(TestUri1, async () => - { - operationStarted.SetResult(true); - await Task.Delay(10000, cts.Token); // Long delay that will be cancelled - }); - - // Wait for operation to start, then cancel it - await operationStarted.Task; - cts.Cancel(); - - // Assert - await Assert.ThrowsAsync(() => operationTask); - - // Wait for cleanup with exponential backoff instead of fixed delay - var activeLockCount = SearchParameterConcurrencyManager.ActiveLockCount; - var attempts = 0; - while (activeLockCount != 0 && attempts < 10) - { - await Task.Delay(10 * (int)Math.Pow(2, attempts)); - activeLockCount = SearchParameterConcurrencyManager.ActiveLockCount; - attempts++; - } - - Assert.Equal(0, activeLockCount); - } - - [Fact] - public async Task GivenLongRunningOperations_WhenExecutingManySequentiallyOnSameUri_ThenMemoryIsReclaimed() - { - // Arrange - const int iterations = 100; - - // Act - Execute many operations sequentially - for (int i = 0; i < iterations; i++) - { - await SearchParameterConcurrencyManager.ExecuteWithLockAsync($"{TestUri1}_{i}", async () => - { - await Task.Delay(1); // Minimal work - }); - } - - // Wait for cleanup with exponential backoff instead of fixed delay - var activeLockCount = SearchParameterConcurrencyManager.ActiveLockCount; - var attempts = 0; - while (activeLockCount >= 10 && attempts < 10) - { - await Task.Delay(10 * (int)Math.Pow(2, attempts)); - activeLockCount = SearchParameterConcurrencyManager.ActiveLockCount; - attempts++; - } - - // Assert - Should not have accumulated locks - Assert.True( - activeLockCount < 10, - $"Expected less than 10 active locks but found {activeLockCount}"); - } - - [Fact] - public async Task GivenMultipleThreads_WhenExecutingSameOperation_ThenOnlyOneExecutes() - { - // Arrange - var executionCount = 0; - using var barrier = new Barrier(3); // 3 threads will hit this barrier - const int threadCount = 3; - - // Act - var tasks = new Task[threadCount]; - for (int i = 0; i < threadCount; i++) - { - tasks[i] = Task.Run(async () => - { - barrier.SignalAndWait(); // Ensure all threads start simultaneously - - await SearchParameterConcurrencyManager.ExecuteWithLockAsync(TestUri1, async () => - { - var currentCount = Interlocked.Increment(ref executionCount); - await Task.Delay(10); // Simulate work - return currentCount; - }); - }); - } - - await Task.WhenAll(tasks); - - // Assert - Assert.Equal(threadCount, executionCount); - } - - [Theory] - [InlineData(null)] - [InlineData("")] - [InlineData(" ")] - public async Task GivenInvalidUri_WhenExecutingWithLock_ThenThrowsArgumentException(string invalidUri) - { - // Act & Assert - await Assert.ThrowsAsync(() => - SearchParameterConcurrencyManager.ExecuteWithLockAsync(invalidUri, () => Task.FromResult(1))); - } - - [Fact] - public async Task GivenNullOperation_WhenExecutingWithLock_ThenThrowsArgumentNullException() - { - // Act & Assert - await Assert.ThrowsAsync(() => - SearchParameterConcurrencyManager.ExecuteWithLockAsync(TestUri1, null)); - - await Assert.ThrowsAsync(() => - SearchParameterConcurrencyManager.ExecuteWithLockAsync(TestUri1, null)); - } - - [Fact] - public void GivenSearchParameterConcurrencyException_WhenCreatedWithUris_ThenPropertiesAreSet() - { - // Arrange - var uris = new[] { "http://test.com/param1", "http://test.com/param2" }; - - // Act - var exception = new SearchParameterConcurrencyException(uris); - - // Assert - Assert.NotNull(exception.Message); - Assert.Contains("param1", exception.Message); - Assert.Contains("param2", exception.Message); - Assert.Equal(2, exception.ConflictedUris.Count); - Assert.Contains("http://test.com/param1", exception.ConflictedUris); - Assert.Contains("http://test.com/param2", exception.ConflictedUris); - } - - [Fact] - public void GivenSearchParameterConcurrencyException_WhenCreatedWithMessage_ThenMessageIsSet() - { - // Arrange - const string expectedMessage = "Test concurrency error"; - - // Act - var exception = new SearchParameterConcurrencyException(expectedMessage); - - // Assert - Assert.Equal(expectedMessage, exception.Message); - Assert.Empty(exception.ConflictedUris); - } - - [Fact] - public void GivenSearchParameterConcurrencyException_WhenCreatedWithMessageAndInnerException_ThenBothAreSet() - { - // Arrange - const string expectedMessage = "Test concurrency error with inner exception"; - var innerException = new InvalidOperationException("Inner exception"); - - // Act - var exception = new SearchParameterConcurrencyException(expectedMessage, innerException); - - // Assert - Assert.Equal(expectedMessage, exception.Message); - Assert.Same(innerException, exception.InnerException); - Assert.Empty(exception.ConflictedUris); - } - } -} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Context/FhirRequestContextExtensions.cs b/src/Microsoft.Health.Fhir.Core/Features/Context/FhirRequestContextExtensions.cs new file mode 100644 index 0000000000..e702958071 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Context/FhirRequestContextExtensions.cs @@ -0,0 +1,36 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using Microsoft.Health.Fhir.Core.Features.Search.Parameters; + +namespace Microsoft.Health.Fhir.Core.Features.Context +{ + public static class FhirRequestContextExtensions + { + public static DateTimeOffset? GetSearchParameterLastUpdated(this IFhirRequestContext context) + { + if (context?.Properties.TryGetValue(SearchParameterRequestContextPropertyNames.LastUpdated, out var value) == true) + { + return (DateTimeOffset)value; + } + + return null; + } + + public static void SetSearchParameterLastUpdated(this IFhirRequestContext context, DateTimeOffset? lastUpdated) + { + if (lastUpdated.HasValue && context != null) + { + context.Properties[SearchParameterRequestContextPropertyNames.LastUpdated] = lastUpdated.Value; + } + } + + public static void ClearSearchParameterLastUpdated(this IFhirRequestContext context) + { + context?.Properties.Remove(SearchParameterRequestContextPropertyNames.LastUpdated); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexOrchestratorJob.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexOrchestratorJob.cs index 7cadc99afc..aa2d1fa6c4 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexOrchestratorJob.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexOrchestratorJob.cs @@ -221,8 +221,7 @@ private async Task RefreshSearchParameterCache(bool isReindexStart) await Task.Delay(delayMs, _cancellationToken); } - var currentDate = _searchParameterOperations.SearchParamLastUpdated.HasValue ? _searchParameterOperations.SearchParamLastUpdated.Value : DateTimeOffset.MinValue; - _searchParamLastUpdated = currentDate; + _searchParamLastUpdated = _searchParameterOperations.SearchParamLastUpdated; _logger.LogJobInformation(_jobInfo, $"Reindex orchestrator job completed cache refresh at the {suffix}: SearchParamLastUpdated {_searchParamLastUpdated}"); await TryLogEvent($"ReindexOrchestratorJob={_jobInfo.Id}.ExecuteAsync.{suffix}", "Warn", $"SearchParamLastUpdated={_searchParamLastUpdated.ToString("yyyy-MM-dd HH:mm:ss.fff")}", null, _cancellationToken); @@ -240,8 +239,7 @@ async Task WaitForAllInstancesCacheSyncAsync(DateTime updateEventsSince, C if (result.IsConsistent) { - var logDate = _searchParameterOperations.SearchParamLastUpdated.HasValue ? _searchParameterOperations.SearchParamLastUpdated.Value : DateTimeOffset.MinValue; - _logger.LogJobInformation(_jobInfo, $"Cache sync check: All {result.ActiveHosts} active host(s) have converged to SearchParamLastUpdated={logDate.ToString("yyyy-MM-dd HH:mm:ss.fff")}."); + _logger.LogJobInformation(_jobInfo, $"Cache sync check: All {result.ActiveHosts} active host(s) have converged to SearchParamLastUpdated={_searchParameterOperations.SearchParamLastUpdated.ToString("yyyy-MM-dd HH:mm:ss.fff")}."); break; } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexProcessingJob.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexProcessingJob.cs index 3d9caf33ea..e0ce134a50 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexProcessingJob.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexProcessingJob.cs @@ -143,7 +143,7 @@ private async Task CheckDiscrepancies(CancellationToken cancellationToken) // use the same value as used in resource writes _searchParameterHash = searchParameterHash; - var currentDate = _searchParameterOperations.SearchParamLastUpdated.HasValue ? _searchParameterOperations.SearchParamLastUpdated.Value : DateTimeOffset.MinValue; + var currentDate = _searchParameterOperations.SearchParamLastUpdated; var current = currentDate.ToString("yyyy-MM-dd HH:mm:ss.fff"); var requested = _reindexProcessingJobDefinition.SearchParamLastUpdated.ToString("yyyy-MM-dd HH:mm:ss.fff"); isBad = _reindexProcessingJobDefinition.SearchParamLastUpdated > currentDate; diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/CreateOrUpdateSearchParameterBehavior.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/CreateOrUpdateSearchParameterBehavior.cs index e5ed9aa545..bbed409f18 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/CreateOrUpdateSearchParameterBehavior.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/CreateOrUpdateSearchParameterBehavior.cs @@ -15,6 +15,7 @@ using Microsoft.Health.Fhir.Core.Exceptions; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Context; +using Microsoft.Health.Fhir.Core.Features.Definition; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Search.Registry; using Microsoft.Health.Fhir.Core.Messages.Create; @@ -28,26 +29,26 @@ public class CreateOrUpdateSearchParameterBehavior _requestContextAccessor; private readonly IModelInfoProvider _modelInfoProvider; + private readonly ISearchParameterDefinitionManager _searchParameterDefinitionManager; public CreateOrUpdateSearchParameterBehavior( ISearchParameterOperations searchParameterOperations, IFhirDataStore fhirDataStore, - ISearchParameterStatusManager searchParameterStatusManager, + ISearchParameterDefinitionManager searchParameterDefinitionManager, RequestContextAccessor requestContextAccessor, IModelInfoProvider modelInfoProvider) { EnsureArg.IsNotNull(searchParameterOperations, nameof(searchParameterOperations)); EnsureArg.IsNotNull(fhirDataStore, nameof(fhirDataStore)); - EnsureArg.IsNotNull(searchParameterStatusManager, nameof(searchParameterStatusManager)); + EnsureArg.IsNotNull(searchParameterDefinitionManager, nameof(searchParameterDefinitionManager)); EnsureArg.IsNotNull(requestContextAccessor, nameof(requestContextAccessor)); EnsureArg.IsNotNull(modelInfoProvider, nameof(modelInfoProvider)); _searchParameterOperations = searchParameterOperations; _fhirDataStore = fhirDataStore; - _searchParameterStatusManager = searchParameterStatusManager; + _searchParameterDefinitionManager = searchParameterDefinitionManager; _requestContextAccessor = requestContextAccessor; _modelInfoProvider = modelInfoProvider; } @@ -56,13 +57,13 @@ public async Task Handle(CreateResourceRequest request, { if (request.Resource.InstanceType.Equals(KnownResourceTypes.SearchParameter, StringComparison.Ordinal)) { - var refreshCache = request.BundleResourceContext == null || !request.BundleResourceContext.IsParallelBundle; - // Before committing the SearchParameter resource to the data store, validate the parameter type - await _searchParameterOperations.ValidateSearchParameterAsync(request.Resource.Instance, cancellationToken, refreshCache); + var lastUpdated = await _searchParameterOperations.ValidateSearchParameterAsync(request.Resource.Instance, cancellationToken, _requestContextAccessor.RequestContext.GetSearchParameterLastUpdated()); + + QueueStatus(request.Resource.Instance.GetStringScalar("url"), SearchParameterStatus.Supported, lastUpdated); - var url = request.Resource.Instance.GetStringScalar("url"); - await QueueStatusAsync(url, SearchParameterStatus.Supported, cancellationToken); + // Allow the resource to be updated with the normal handler + return await next(cancellationToken); } // Allow the resource to be updated with the normal handler @@ -90,38 +91,35 @@ public async Task Handle(UpsertResourceRequest request, prevSearchParamResource = null; } - var refreshCache = request.BundleResourceContext == null || !request.BundleResourceContext.IsParallelBundle; + var lastUpdated = await _searchParameterOperations.ValidateSearchParameterAsync(request.Resource.Instance, cancellationToken, _requestContextAccessor.RequestContext.GetSearchParameterLastUpdated()); if (prevSearchParamResource != null && prevSearchParamResource.IsDeleted == false) { - // Validate any changes to the fhirpath or the datatype - await _searchParameterOperations.ValidateSearchParameterAsync(request.Resource.Instance, cancellationToken, refreshCache); - var previousUrl = _modelInfoProvider.ToTypedElement(prevSearchParamResource.RawResource).GetStringScalar("url"); var newUrl = request.Resource.Instance.GetStringScalar("url"); if (!string.IsNullOrWhiteSpace(previousUrl) && !previousUrl.Equals(newUrl, StringComparison.Ordinal)) { - await QueueStatusAsync(previousUrl, SearchParameterStatus.Deleted, cancellationToken); + QueueStatus(previousUrl, SearchParameterStatus.Deleted, lastUpdated); } - await QueueStatusAsync(newUrl, SearchParameterStatus.Supported, cancellationToken); + QueueStatus(newUrl, SearchParameterStatus.Supported, lastUpdated); } else { // No previous version exists or it was deleted, so add it as a new SearchParameter - await _searchParameterOperations.ValidateSearchParameterAsync(request.Resource.Instance, cancellationToken, refreshCache); - - var url = request.Resource.Instance.GetStringScalar("url"); - await QueueStatusAsync(url, SearchParameterStatus.Supported, cancellationToken); + QueueStatus(request.Resource.Instance.GetStringScalar("url"), SearchParameterStatus.Supported, lastUpdated); } + + // Now allow the resource to updated per the normal behavior + return await next(cancellationToken); } // Now allow the resource to updated per the normal behavior return await next(cancellationToken); } - private async Task QueueStatusAsync(string url, SearchParameterStatus status, CancellationToken cancellationToken) + private void QueueStatus(string url, SearchParameterStatus status, DateTimeOffset lastUpdated) { if (string.IsNullOrWhiteSpace(url)) { @@ -141,14 +139,13 @@ private async Task QueueStatusAsync(string url, SearchParameterStatus status, Ca context.Properties[SearchParameterRequestContextPropertyNames.PendingStatusUpdates] = pendingStatuses; } - var currentStatuses = await _searchParameterStatusManager.GetAllSearchParameterStatus(cancellationToken); - var existing = currentStatuses.FirstOrDefault(s => string.Equals(s.Uri?.OriginalString, url, StringComparison.Ordinal)); + _searchParameterDefinitionManager.TryGetSearchParameter(url, out var existing); var update = new ResourceSearchParameterStatus { Uri = new Uri(url), Status = status, - LastUpdated = existing?.LastUpdated ?? DateTimeOffset.UtcNow, + LastUpdated = lastUpdated, IsPartiallySupported = existing?.IsPartiallySupported ?? false, SortStatus = existing?.SortStatus ?? SortParameterStatus.Disabled, }; diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/DeleteSearchParameterBehavior.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/DeleteSearchParameterBehavior.cs index 7be8c387dc..a002bf7318 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/DeleteSearchParameterBehavior.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/DeleteSearchParameterBehavior.cs @@ -60,7 +60,6 @@ public DeleteSearchParameterBehavior( public async Task Handle(TDeleteResourceRequest request, RequestHandlerDelegate next, CancellationToken cancellationToken) { var deleteRequest = request as DeleteResourceRequest; - ResourceWrapper searchParamResource = null; if (deleteRequest.ResourceKey.ResourceType.Equals(KnownResourceTypes.SearchParameter, StringComparison.Ordinal)) { @@ -76,7 +75,7 @@ public async Task Handle(TDeleteResourceRequest request } // Now try to get the custom search parameter from the data store - searchParamResource = await _fhirDataStore.GetAsync(deleteRequest.ResourceKey, cancellationToken); + var searchParamResource = await _fhirDataStore.GetAsync(deleteRequest.ResourceKey, cancellationToken); if (searchParamResource == null) { @@ -90,6 +89,8 @@ public async Task Handle(TDeleteResourceRequest request var url = typed.GetStringScalar("url"); await QueuePendingDeleteStatusAsync(url, cancellationToken); } + + return await next(cancellationToken); } return await next(cancellationToken); @@ -108,6 +109,13 @@ private async Task QueuePendingDeleteStatusAsync(string url, CancellationToken c return; } + var lastUpdated = _requestContextAccessor.RequestContext.GetSearchParameterLastUpdated(); + if (!lastUpdated.HasValue) + { + await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken); + lastUpdated = _searchParameterOperations.SearchParamLastUpdated; + } + if (!context.Properties.TryGetValue(SearchParameterRequestContextPropertyNames.PendingStatusUpdates, out var value) || value is not List pendingStatuses) { @@ -115,14 +123,13 @@ private async Task QueuePendingDeleteStatusAsync(string url, CancellationToken c context.Properties[SearchParameterRequestContextPropertyNames.PendingStatusUpdates] = pendingStatuses; } - var currentStatuses = await _searchParameterStatusManager.GetAllSearchParameterStatus(cancellationToken); - var existing = currentStatuses.FirstOrDefault(s => string.Equals(s.Uri?.OriginalString, url, StringComparison.Ordinal)); + _searchParameterDefinitionManager.TryGetSearchParameter(url, out var existing); var update = new ResourceSearchParameterStatus { Uri = new Uri(url), Status = SearchParameterStatus.PendingDelete, - LastUpdated = existing?.LastUpdated ?? DateTimeOffset.UtcNow, + LastUpdated = lastUpdated.Value, IsPartiallySupported = existing?.IsPartiallySupported ?? false, SortStatus = existing?.SortStatus ?? SortParameterStatus.Disabled, }; diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/ISearchParameterOperations.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/ISearchParameterOperations.cs index f95d23eee0..87e2b50779 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/ISearchParameterOperations.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/ISearchParameterOperations.cs @@ -15,11 +15,11 @@ namespace Microsoft.Health.Fhir.Core.Features.Search.Parameters { public interface ISearchParameterOperations { - DateTimeOffset? SearchParamLastUpdated { get; } + DateTimeOffset SearchParamLastUpdated { get; } Task DeleteSearchParameterAsync(RawResource searchParamResource, CancellationToken cancellationToken, bool ignoreSearchParameterNotSupportedException = false); - Task ValidateSearchParameterAsync(ITypedElement searchParam, CancellationToken cancellationToken, bool refreshCache = true); + Task ValidateSearchParameterAsync(ITypedElement searchParam, CancellationToken cancellationToken, DateTimeOffset? lastUpdated = null); Task UpdateSearchParameterStatusAsync(IReadOnlyCollection searchParameterUris, SearchParameterStatus status, CancellationToken cancellationToken, bool ignoreSearchParameterNotSupportedException = false); diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs index 44c0aa78c8..fc12d5f58a 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs @@ -5,11 +5,9 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; -using DotLiquid.Tags.Html; using EnsureThat; using Hl7.Fhir.ElementModel; using Microsoft.Extensions.Logging; @@ -71,7 +69,18 @@ public SearchParameterOperations( _refreshSemaphore = new SemaphoreSlim(1, 1); } - public DateTimeOffset? SearchParamLastUpdated => _searchParamLastUpdated; + public DateTimeOffset SearchParamLastUpdated + { + get + { + if (!_searchParamLastUpdated.HasValue) + { + throw new InvalidOperationException("Search param cache has not been updated yet."); + } + + return _searchParamLastUpdated.Value; + } + } public string GetSearchParameterHash(string resourceType) { @@ -94,82 +103,79 @@ public async Task EnsureNoActiveReindexJobAsync(CancellationToken cancellationTo if (activeReindexJob.found) { - throw new JobConflictException(Core.Resources.ChangesToSearchParametersNotAllowedWhileReindexing); + throw new JobConflictException(string.Format(Core.Resources.ChangesToSearchParametersNotAllowedWhileReindexing, activeReindexJob.id)); } } - public async Task ValidateSearchParameterAsync(ITypedElement searchParam, CancellationToken cancellationToken, bool refreshCache = true) + public async Task ValidateSearchParameterAsync(ITypedElement searchParam, CancellationToken cancellationToken, DateTimeOffset? lastUpdated = null) { var searchParameterWrapper = new SearchParameterWrapper(searchParam); var searchParameterUrl = searchParameterWrapper.Url; + try + { + // We need to make sure we have the latest search parameters before trying to add + // a search parameter. This is to avoid creating a duplicate search parameter that + // was recently added and that hasn't propogated to all fhir-server instances. + // if last updated is provided, it means that updates were applied by pipeline. In this case do not update and keep the input. + if (!lastUpdated.HasValue) + { + await GetAndApplySearchParameterUpdates(cancellationToken); + lastUpdated = SearchParamLastUpdated; + } + + // verify the parameter is supported before continuing + var searchParameterInfo = new SearchParameterInfo(searchParameterWrapper); - await SearchParameterConcurrencyManager.ExecuteWithLockAsync( - searchParameterUrl, - async () => + if (searchParameterInfo.Component?.Any() == true) { - try + foreach (SearchParameterComponentInfo c in searchParameterInfo.Component) { - // We need to make sure we have the latest search parameters before trying to add - // a search parameter. This is to avoid creating a duplicate search parameter that - // was recently added and that hasn't propogated to all fhir-server instances. - if (refreshCache) - { - await GetAndApplySearchParameterUpdates(cancellationToken); - } + c.ResolvedSearchParameter = _searchParameterDefinitionManager.GetSearchParameter(c.DefinitionUrl.OriginalString); + } + } - // verify the parameter is supported before continuing - var searchParameterInfo = new SearchParameterInfo(searchParameterWrapper); + (bool Supported, bool IsPartiallySupported) supportedResult = _searchParameterSupportResolver.IsSearchParameterSupported(searchParameterInfo); - if (searchParameterInfo.Component?.Any() == true) - { - foreach (SearchParameterComponentInfo c in searchParameterInfo.Component) - { - c.ResolvedSearchParameter = _searchParameterDefinitionManager.GetSearchParameter(c.DefinitionUrl.OriginalString); - } - } - - (bool Supported, bool IsPartiallySupported) supportedResult = _searchParameterSupportResolver.IsSearchParameterSupported(searchParameterInfo); + if (!supportedResult.Supported) + { + throw new SearchParameterNotSupportedException(string.Format(Core.Resources.NoConverterForSearchParamType, searchParameterInfo.Type, searchParameterInfo.Expression)); + } - if (!supportedResult.Supported) - { - throw new SearchParameterNotSupportedException(string.Format(Core.Resources.NoConverterForSearchParamType, searchParameterInfo.Type, searchParameterInfo.Expression)); - } + // check data store specific support for SearchParameter + if (!_dataStoreSearchParameterValidator.ValidateSearchParameter(searchParameterInfo, out var errorMessage)) + { + throw new SearchParameterNotSupportedException(errorMessage); + } + } + catch (FhirException fex) + { + _logger.LogError(fex, "Error adding search parameter."); + fex.Issues.Add(new OperationOutcomeIssue( + OperationOutcomeConstants.IssueSeverity.Error, + OperationOutcomeConstants.IssueType.Exception, + Core.Resources.CustomSearchCreateError)); - // check data store specific support for SearchParameter - if (!_dataStoreSearchParameterValidator.ValidateSearchParameter(searchParameterInfo, out var errorMessage)) - { - throw new SearchParameterNotSupportedException(errorMessage); - } - } - catch (FhirException fex) - { - _logger.LogError(fex, "Error adding search parameter."); - fex.Issues.Add(new OperationOutcomeIssue( - OperationOutcomeConstants.IssueSeverity.Error, - OperationOutcomeConstants.IssueType.Exception, - Core.Resources.CustomSearchCreateError)); + throw; + } + catch (Exception ex) + { + _logger.LogError(ex, "Unexpected error adding search parameter."); + var customSearchException = new ConfigureCustomSearchException(Core.Resources.CustomSearchCreateError); + customSearchException.Issues.Add(new OperationOutcomeIssue( + OperationOutcomeConstants.IssueSeverity.Error, + OperationOutcomeConstants.IssueType.Exception, + ex.Message)); + + throw customSearchException; + } - throw; - } - catch (Exception ex) - { - _logger.LogError(ex, "Unexpected error adding search parameter."); - var customSearchException = new ConfigureCustomSearchException(Core.Resources.CustomSearchCreateError); - customSearchException.Issues.Add(new OperationOutcomeIssue( - OperationOutcomeConstants.IssueSeverity.Error, - OperationOutcomeConstants.IssueType.Exception, - ex.Message)); - - throw customSearchException; - } - }, - _logger, - cancellationToken); + return lastUpdated.Value; } /// /// Marks the Search Parameter as PendingDelete. This is only used by DeletionService.cs and will be removed when refactoring is done /// to allow deletion service to properly handle Hard deletions for Search Parameters (e.g. allow reindex prior to removing resource from DB). + /// !!! This method has incorrect name. It does not delete search parameter, it just updates its status. /// /// Search Parameter to update to Pending Delete status. /// Cancellation Token @@ -179,41 +185,36 @@ public async Task DeleteSearchParameterAsync(RawResource searchParamResource, Ca var searchParam = _modelInfoProvider.ToTypedElement(searchParamResource); var searchParameterUrl = searchParam.GetStringScalar("url"); - await SearchParameterConcurrencyManager.ExecuteWithLockAsync( - searchParameterUrl, - async () => - { - try - { - await EnsureNoActiveReindexJobAsync(cancellationToken); + try + { + await EnsureNoActiveReindexJobAsync(cancellationToken); - _logger.LogInformation("Deleting the search parameter '{Url}'", searchParameterUrl); - await _searchParameterStatusManager.UpdateSearchParameterStatusAsync(new[] { searchParameterUrl }, SearchParameterStatus.PendingDelete, cancellationToken); - } - catch (FhirException fex) - { - _logger.LogError(fex, "Error deleting search parameter."); - fex.Issues.Add(new OperationOutcomeIssue( - OperationOutcomeConstants.IssueSeverity.Error, - OperationOutcomeConstants.IssueType.Exception, - Core.Resources.CustomSearchDeleteError)); + _logger.LogInformation("DeleteSearchParameterAsync: Refreshing cache"); + await GetAndApplySearchParameterUpdates(cancellationToken); + _logger.LogInformation("DeleteSearchParameterAsync: Deleting the search parameter '{Url}'", searchParameterUrl); + await _searchParameterStatusManager.UpdateSearchParameterStatusAsync(new[] { searchParameterUrl }, SearchParameterStatus.PendingDelete, cancellationToken, lastUpdated: SearchParamLastUpdated); + } + catch (FhirException fex) + { + _logger.LogError(fex, "Error deleting search parameter."); + fex.Issues.Add(new OperationOutcomeIssue( + OperationOutcomeConstants.IssueSeverity.Error, + OperationOutcomeConstants.IssueType.Exception, + Core.Resources.CustomSearchDeleteError)); - throw; - } - catch (Exception ex) when (!(ex is FhirException)) - { - _logger.LogError(ex, "Unexpected error deleting search parameter."); - var customSearchException = new ConfigureCustomSearchException(Core.Resources.CustomSearchDeleteError); - customSearchException.Issues.Add(new OperationOutcomeIssue( - OperationOutcomeConstants.IssueSeverity.Error, - OperationOutcomeConstants.IssueType.Exception, - ex.Message)); - - throw customSearchException; - } - }, - _logger, - cancellationToken); + throw; + } + catch (Exception ex) when (!(ex is FhirException)) + { + _logger.LogError(ex, "Unexpected error deleting search parameter."); + var customSearchException = new ConfigureCustomSearchException(Core.Resources.CustomSearchDeleteError); + customSearchException.Issues.Add(new OperationOutcomeIssue( + OperationOutcomeConstants.IssueSeverity.Error, + OperationOutcomeConstants.IssueType.Exception, + ex.Message)); + + throw customSearchException; + } } public async Task UpdateSearchParameterStatusAsync(IReadOnlyCollection searchParameterUris, SearchParameterStatus status, CancellationToken cancellationToken, bool ignoreSearchParameterNotSupportedException = false) @@ -275,14 +276,9 @@ public async Task GetAndApplySearchParameterUpdates(CancellationToken canc .Where(p => !systemDefinedSearchParameterUris.Contains(p.Uri.OriginalString)).ToList(); // Batch fetch all SearchParameter resources in one call - var searchParamResources = await GetSearchParametersByUrls( - statusesToFetch - .Select(p => p.Uri.OriginalString) - .ToList(), - cancellationToken); + var searchParamResources = await GetSearchParametersByUrls(statusesToFetch.Select(p => p.Uri.OriginalString).ToList(), cancellationToken); var paramsToAdd = new List(); - var allHaveResources = true; foreach (var searchParam in statusesToFetch) { if (!searchParamResources.TryGetValue(searchParam.Uri.OriginalString, out var searchParamResource)) @@ -290,12 +286,6 @@ public async Task GetAndApplySearchParameterUpdates(CancellationToken canc _logger.LogInformation( "Updated SearchParameter status found for SearchParameter: {Url}, but did not find any SearchParameter resources when querying for this url.", searchParam.Uri); - - if (searchParam.LastUpdated > DateTimeOffset.UtcNow.AddMinutes(-10)) // same as for in cache - { - allHaveResources = false; - } - continue; } @@ -325,19 +315,14 @@ public async Task GetAndApplySearchParameterUpdates(CancellationToken canc // Once added to the definition manager we can update their status await _searchParameterStatusManager.ApplySearchParameterStatus(statuses, cancellationToken); - var inCache = ParametersAreInCache(statusesToFetch, cancellationToken); - var cycleConclusive = statuses.Count == 0 || (inCache && allHaveResources); - - // If cache is updated directly and not from the database not all will have corresponding resources. - // Do not advance or log the timestamp unless the cache contents are conclusive for this cycle. - if (inCache && allHaveResources && results.LastUpdated.HasValue) + if (results.LastUpdated.HasValue) { _searchParamLastUpdated = results.LastUpdated.Value; // this should be the only place in the code to assign last updated } - if (cycleConclusive && _searchParamLastUpdated.HasValue) + if (zeroWaitForSemaphore && _searchParamLastUpdated.HasValue) // log only for background { - // Log to EventLog for cross-instance convergence tracking (SQL only; Cosmos/File are no-ops). + // log for cross-instance cache refresh tracking (SQL only; Cosmos/File are no-ops). var lastUpdatedText = _searchParamLastUpdated.Value.ToString("yyyy-MM-dd HH:mm:ss.fffffff"); await _searchParameterStatusManager.TryLogEvent(_searchParameterStatusManager.SearchParamCacheUpdateProcessName, "Warn", lastUpdatedText, null, cancellationToken); } @@ -357,32 +342,6 @@ public async Task GetAndApplySearchParameterUpdates(CancellationToken canc return true; } - // This should handle racing condition between saving new parameter on one VM and refreshing cache on the other, - // when refresh is invoked between saving status and saving resource. - // This will not be needed when order of saves is reversed (resource first, then status) - private bool ParametersAreInCache(IReadOnlyCollection statuses, CancellationToken cancellationToken) - { - var inCache = true; - foreach (var status in statuses) - { - _searchParameterDefinitionManager.TryGetSearchParameter(status.Uri.OriginalString, out var existingSearchParam); - if (existingSearchParam == null) - { - var msg = $"Did not find in cache uri={status.Uri.OriginalString} status={status.Status}"; - _logger.LogInformation(msg); - - // if the parameter was updated in the last 10 minutes it's possible we hit race condition - // where status was updated but resource is not yet saved, so we should not consider this as cache miss - if (status.LastUpdated > DateTimeOffset.UtcNow.AddMinutes(-10)) - { - inCache = false; - } - } - } - - return inCache; - } - private void DeleteSearchParameter(string url) { try @@ -414,6 +373,7 @@ private async Task> GetSearchParametersByUrls( { cancellationToken.ThrowIfCancellationRequested(); + // search is not by url because it should work for deleted resources. this can be fixed only when resource deletes are delayed. var queryParams = new List> { Tuple.Create(KnownQueryParameterNames.Count, chunkSize.ToString()), @@ -421,10 +381,7 @@ private async Task> GetSearchParametersByUrls( if (!string.IsNullOrEmpty(continuationToken)) { - queryParams.Add( - Tuple.Create( - KnownQueryParameterNames.ContinuationToken, - ContinuationTokenEncoder.Encode(continuationToken))); + queryParams.Add(Tuple.Create(KnownQueryParameterNames.ContinuationToken, ContinuationTokenEncoder.Encode(continuationToken))); } var result = await search.Value.SearchAsync(KnownResourceTypes.SearchParameter, queryParams, cancellationToken); @@ -457,10 +414,7 @@ private async Task> GetSearchParametersByUrls( if (unresolvedUrls.Count > 0) { - _logger.LogWarning( - "Could not resolve {Count} SearchParameter URL(s). Samples: {Urls}", - unresolvedUrls.Count, - string.Join(", ", unresolvedUrls.Take(10))); + _logger.LogWarning("Could not resolve {Count} SearchParameter URL(s). Samples: {Urls}", unresolvedUrls.Count, string.Join(", ", unresolvedUrls.Take(10))); } return searchParametersByUrl; diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterRequestContextPropertyNames.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterRequestContextPropertyNames.cs index 6c725a699a..efc3eeac9a 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterRequestContextPropertyNames.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterRequestContextPropertyNames.cs @@ -8,5 +8,6 @@ namespace Microsoft.Health.Fhir.Core.Features.Search.Parameters public static class SearchParameterRequestContextPropertyNames { public const string PendingStatusUpdates = "SearchParameter.PendingStatusUpdates"; + public const string LastUpdated = "SearchParameter.LastUpdated"; } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterRetry.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterRetry.cs new file mode 100644 index 0000000000..ceabeda66a --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterRetry.cs @@ -0,0 +1,57 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Security.Cryptography; +using System.Threading.Tasks; +using Microsoft.Health.Fhir.Core.Exceptions; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Polly; + +namespace Microsoft.Health.Fhir.Core.Features.Search.Parameters +{ + public static class SearchParameterRetry + { + private const int MaxRetryCount = 3; + + /// + /// Executes the provided function with retry logic. + /// + /// The type of result returned by the action. + /// The action to execute with optional retry. + /// Additional context information to append to exception messages. + public static async Task ExecuteAsync(Func> action, string info = null) + { + var retryPolicy = Policy + .Handle(ex => ex.Message == Core.Resources.SearchParameterConcurrencyConflict) + .WaitAndRetryAsync( + retryCount: MaxRetryCount, + sleepDurationProvider: retryAttempt => TimeSpan.FromSeconds(RandomNumberGenerator.GetInt32(1, 5) * 0.1)); + + try + { + return await retryPolicy.ExecuteAsync(action); + } + catch (BadRequestException ex) when (ex.Message == Core.Resources.SearchParameterConcurrencyConflict) + { + throw new BadRequestException($"{ex.Message} {info}.{MaxRetryCount}"); + } + } + + /// + /// Convenience overload for actions with no return value. + /// + public static async Task ExecuteAsync(Func action, string info = null) + { + await ExecuteAsync( + async () => + { + await action(); + return 0; + }, + info); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/ISearchParameterStatusManager.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/ISearchParameterStatusManager.cs index e13f5ce72f..ea62c6c956 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/ISearchParameterStatusManager.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/ISearchParameterStatusManager.cs @@ -21,7 +21,7 @@ public interface ISearchParameterStatusManager Task Handle(SearchParameterDefinitionManagerInitialized notification, CancellationToken cancellationToken); - Task UpdateSearchParameterStatusAsync(IReadOnlyCollection searchParameterUris, SearchParameterStatus status, CancellationToken cancellationToken, bool ignoreSearchParameterNotSupportedException = false, long? reindexId = null); + Task UpdateSearchParameterStatusAsync(IReadOnlyCollection searchParameterUris, SearchParameterStatus status, CancellationToken cancellationToken, bool ignoreSearchParameterNotSupportedException = false, long? reindexId = null, DateTimeOffset? lastUpdated = null); Task CheckCacheConsistencyAsync(DateTime updateEventsSince, DateTime activeHostsSince, CancellationToken cancellationToken); } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterConcurrencyException.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterConcurrencyException.cs deleted file mode 100644 index a7ce4350e7..0000000000 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterConcurrencyException.cs +++ /dev/null @@ -1,40 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System; -using System.Collections.Generic; -using System.Linq; - -namespace Microsoft.Health.Fhir.Core.Features.Search.Registry -{ - /// - /// Exception thrown when an optimistic concurrency conflict occurs during search parameter updates. - /// - public class SearchParameterConcurrencyException : Exception - { - public SearchParameterConcurrencyException(IEnumerable conflictedUris) - : base($"Optimistic concurrency conflict detected for search parameters: {string.Join(", ", conflictedUris)}") - { - ConflictedUris = conflictedUris?.ToList() ?? new List(); - } - - public SearchParameterConcurrencyException(string message) - : base(message) - { - ConflictedUris = new List(); - } - - public SearchParameterConcurrencyException(string message, Exception innerException) - : base(message, innerException) - { - ConflictedUris = new List(); - } - - /// - /// Gets the URIs of search parameters that had concurrency conflicts. - /// - public IReadOnlyList ConflictedUris { get; } - } -} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterConcurrencyManager.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterConcurrencyManager.cs deleted file mode 100644 index a2262be511..0000000000 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterConcurrencyManager.cs +++ /dev/null @@ -1,148 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System; -using System.Collections.Concurrent; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; - -namespace Microsoft.Health.Fhir.Core.Features.Search.Registry -{ - /// - /// Static manager for search parameter concurrency to prevent race conditions - /// when multiple requests try to update the same search parameter simultaneously. - /// - public static class SearchParameterConcurrencyManager - { - private static readonly ConcurrentDictionary _semaphores = new(); - private static readonly object _cleanupLock = new object(); - - /// - /// Gets the current number of active locks for debugging/monitoring purposes. - /// - public static int ActiveLockCount => _semaphores.Count; - - /// - /// Executes the given function with exclusive access for the specified search parameter URI. - /// This prevents concurrent modifications to the same search parameter. - /// - /// The return type of the function - /// The URI of the search parameter to lock on - /// The function to execute with exclusive access - /// Optional logger for debug information - /// The cancellation token - /// The result of the function execution - public static async Task ExecuteWithLockAsync( - string searchParameterUri, - Func> function, - ILogger logger = null, - CancellationToken cancellationToken = default) - { - if (string.IsNullOrWhiteSpace(searchParameterUri)) - { - throw new ArgumentException("Search parameter URI cannot be null or empty", nameof(searchParameterUri)); - } - - ArgumentNullException.ThrowIfNull(function); - - var semaphore = _semaphores.GetOrAdd(searchParameterUri, _ => new SemaphoreSlim(1, 1)); - - logger?.LogDebug("Acquiring lock for search parameter: {SearchParameterUri}", searchParameterUri); - - await semaphore.WaitAsync(cancellationToken); - - try - { - logger?.LogDebug("Lock acquired for search parameter: {SearchParameterUri}", searchParameterUri); - return await function(); - } - finally - { - semaphore.Release(); - logger?.LogDebug("Lock released for search parameter: {SearchParameterUri}", searchParameterUri); - - // Clean up semaphore if no one is waiting and it's available for immediate use - // Use a lock to prevent race conditions during cleanup - lock (_cleanupLock) - { - // Only clean up if the semaphore is available (CurrentCount == 1) and we can successfully remove it - if (semaphore.CurrentCount == 1 && - _semaphores.TryRemove(searchParameterUri, out var removedSemaphore)) - { - bool shouldPutBack = false; - - try - { - if (ReferenceEquals(removedSemaphore, semaphore)) - { - // Double-check that no other thread acquired the semaphore between our check and removal - if (semaphore.CurrentCount == 1) - { - logger?.LogDebug("Cleaned up semaphore for search parameter: {SearchParameterUri}", searchParameterUri); - - // Don't put back - will dispose in finally - } - else - { - // Put it back if someone acquired it in the meantime - shouldPutBack = true; - } - } - else - { - // Put it back if it's a different semaphore instance - shouldPutBack = true; - } - - if (shouldPutBack) - { - _semaphores.TryAdd(searchParameterUri, removedSemaphore); - removedSemaphore = null; // Prevent dispose in finally since we put it back - } - } - finally - { - // Only dispose if we didn't put it back into the dictionary - removedSemaphore?.Dispose(); - } - } - } - } - } - - /// - /// Executes the given action with exclusive access for the specified search parameter URI. - /// This prevents concurrent modifications to the same search parameter. - /// - /// The URI of the search parameter to lock on - /// The action to execute with exclusive access - /// Optional logger for debug information - /// The cancellation token - public static async Task ExecuteWithLockAsync( - string searchParameterUri, - Func action, - ILogger logger = null, - CancellationToken cancellationToken = default) - { - if (string.IsNullOrWhiteSpace(searchParameterUri)) - { - throw new ArgumentException("Search parameter URI cannot be null or empty", nameof(searchParameterUri)); - } - - ArgumentNullException.ThrowIfNull(action); - - await ExecuteWithLockAsync( - searchParameterUri, - async () => - { - await action(); - return 0; // Return dummy value for action overload - }, - logger, - cancellationToken); - } - } -} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterStatusManager.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterStatusManager.cs index fffd1cbc0b..8773f2ecb6 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterStatusManager.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterStatusManager.cs @@ -136,7 +136,7 @@ public async Task Handle(SearchParameterDefinitionManagerInitialized notificatio await EnsureInitializedAsync(cancellationToken); } - public async Task UpdateSearchParameterStatusAsync(IReadOnlyCollection searchParameterUris, SearchParameterStatus status, CancellationToken cancellationToken, bool ignoreSearchParameterNotSupportedException = false, long? reindexId = null) + public async Task UpdateSearchParameterStatusAsync(IReadOnlyCollection searchParameterUris, SearchParameterStatus status, CancellationToken cancellationToken, bool ignoreSearchParameterNotSupportedException = false, long? reindexId = null, DateTimeOffset? lastUpdated = null) { EnsureArg.IsNotNull(searchParameterUris); @@ -160,6 +160,7 @@ public async Task UpdateSearchParameterStatusAsync(IReadOnlyCollection s { existingStatus.Status = status; searchParameterStatusList.Add(existingStatus); + existingStatus.LastUpdated = lastUpdated ?? DateTimeOffset.UtcNow; } else { @@ -167,6 +168,7 @@ public async Task UpdateSearchParameterStatusAsync(IReadOnlyCollection s { Status = status, Uri = new Uri(uri), + LastUpdated = lastUpdated ?? DateTimeOffset.UtcNow, }); } } diff --git a/src/Microsoft.Health.Fhir.Core/Resources.Designer.cs b/src/Microsoft.Health.Fhir.Core/Resources.Designer.cs index 8964664284..f5196ed484 100644 --- a/src/Microsoft.Health.Fhir.Core/Resources.Designer.cs +++ b/src/Microsoft.Health.Fhir.Core/Resources.Designer.cs @@ -19,7 +19,7 @@ namespace Microsoft.Health.Fhir.Core { // class via a tool like ResGen or Visual Studio. // To add or remove a member, edit your .ResX file then rerun ResGen // with the /str option, or rebuild your VS project. - [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "18.0.0.0")] + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "17.0.0.0")] [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] internal class Resources { @@ -1592,6 +1592,15 @@ internal static string SearchParameterByDefinitionUriNotSupported { } } + /// + /// Looks up a localized string similar to Optimistic concurrency conflict detected while writing custom search parameter(s). Make sure that custom search parameters are not written in parallel. Consider sequential writes or a bundle.. + /// + internal static string SearchParameterConcurrencyConflict { + get { + return ResourceManager.GetString("SearchParameterConcurrencyConflict", resourceCulture); + } + } + /// /// Looks up a localized string similar to SearchParameter[{0}].resource.base is not defined.. /// diff --git a/src/Microsoft.Health.Fhir.Core/Resources.resx b/src/Microsoft.Health.Fhir.Core/Resources.resx index 5f5f456b12..1f9a8dc2b5 100644 --- a/src/Microsoft.Health.Fhir.Core/Resources.resx +++ b/src/Microsoft.Health.Fhir.Core/Resources.resx @@ -876,4 +876,7 @@ Search Parameter URL {0} exceeds the maximum length limit of {1} - \ No newline at end of file + + Optimistic concurrency conflict detected while writing custom search parameter(s). Make sure that custom search parameters are not written in parallel. Consider sequential writes or a bundle. + + diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/FhirControllerTests.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/FhirControllerTests.cs index 7e577aab6c..a3691a357c 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/FhirControllerTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/FhirControllerTests.cs @@ -36,6 +36,7 @@ using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Persistence.Orchestration; using Microsoft.Health.Fhir.Core.Features.Routing; +using Microsoft.Health.Fhir.Core.Features.Search.Parameters; using Microsoft.Health.Fhir.Core.Messages.Bundle; using Microsoft.Health.Fhir.Core.Messages.Create; using Microsoft.Health.Fhir.Core.Messages.Delete; @@ -67,6 +68,7 @@ public sealed class FhirControllerTests private readonly IUrlResolver _urlResolver; private readonly IOptions _configuration; private readonly IAuthorizationService _authorizationService; + private readonly ISearchParameterOperations _searchParameterOperations; public FhirControllerTests() { @@ -76,6 +78,7 @@ public FhirControllerTests() _configuration = Substitute.For>(); _configuration.Value.Returns(new FeatureConfiguration()); _authorizationService = Substitute.For(); + _searchParameterOperations = Substitute.For(); _mediator.Send( Arg.Any(), @@ -90,7 +93,8 @@ public FhirControllerTests() _requestContextAccessor, _urlResolver, _configuration, - _authorizationService); + _authorizationService, + _searchParameterOperations); _fhirController.ControllerContext = new ControllerContext( new ActionContext( Substitute.For(), @@ -1452,5 +1456,208 @@ private static void TestIfTargetMethodContainsCustomAttribute(Type expectedCusto Assert.True(latencyFilter != null, $"The expected filter '{expectedCustomAttributeType.Name}' was not found in the method '{methodName}' from '{targetClassType.Name}'."); } + + [Fact] + public async Task GivenSearchParameterCreate_WhenConcurrencyConflictOccurs_ThenRetriesAndSucceeds() + { + var searchParameter = new SearchParameter { Id = "test", Url = "http://test.com/param", VersionId = Guid.NewGuid().ToString() }; + var wrapper = CreateMockResourceWrapper(searchParameter); + + var attemptCount = 0; + _searchParameterOperations + .GetAndApplySearchParameterUpdates(Arg.Any()) + .Returns(Task.FromResult(true)); + _searchParameterOperations.SearchParamLastUpdated.Returns(DateTimeOffset.UtcNow); + + _mediator.Send( + Arg.Any(), + Arg.Any()) + .Returns(callInfo => + { + attemptCount++; + if (attemptCount < 3) + { + throw new BadRequestException(Core.Resources.SearchParameterConcurrencyConflict); + } + + return new UpsertResourceResponse(new SaveOutcome(new RawResourceElement(wrapper), SaveOutcomeType.Created)); + }); + + var response = await _fhirController.Create(searchParameter); + + Assert.Equal(3, attemptCount); + Assert.IsType(response); + } + + [Fact] + public async Task GivenSearchParameterUpdate_WhenConcurrencyConflictOccurs_ThenRetriesAndSucceeds() + { + var searchParameter = new SearchParameter { Id = "test", Url = "http://test.com/param", VersionId = Guid.NewGuid().ToString() }; + var wrapper = CreateMockResourceWrapper(searchParameter); + + var attemptCount = 0; + _searchParameterOperations + .GetAndApplySearchParameterUpdates(Arg.Any()) + .Returns(Task.FromResult(true)); + _searchParameterOperations.SearchParamLastUpdated.Returns(DateTimeOffset.UtcNow); + + _mediator.Send( + Arg.Any(), + Arg.Any()) + .Returns(callInfo => + { + attemptCount++; + if (attemptCount < 2) + { + throw new BadRequestException(Core.Resources.SearchParameterConcurrencyConflict); + } + + return new UpsertResourceResponse(new SaveOutcome(new RawResourceElement(wrapper), SaveOutcomeType.Updated)); + }); + + var response = await _fhirController.Update(searchParameter, null, true); + + Assert.Equal(2, attemptCount); + Assert.IsType(response); + } + + [Fact] + public async Task GivenSearchParameterDelete_WhenConcurrencyConflictOccurs_ThenRetriesAndSucceeds() + { + var key = new ResourceKey("SearchParameter", "test"); + + var attemptCount = 0; + _searchParameterOperations + .GetAndApplySearchParameterUpdates(Arg.Any()) + .Returns(Task.FromResult(true)); + _searchParameterOperations.SearchParamLastUpdated.Returns(DateTimeOffset.UtcNow); + + _mediator.Send( + Arg.Any(), + Arg.Any()) + .Returns(callInfo => + { + attemptCount++; + if (attemptCount < 2) + { + throw new BadRequestException(Core.Resources.SearchParameterConcurrencyConflict); + } + + return new DeleteResourceResponse(key); + }); + + var response = await _fhirController.Delete("SearchParameter", "test", new HardDeleteModel { HardDelete = false }, false); + + Assert.Equal(2, attemptCount); + Assert.IsType(response); + } + + [Fact] + public async Task GivenSearchParameterConditionalCreate_WhenConcurrencyConflictOccurs_ThenRetriesAndSucceeds() + { + var searchParameter = new SearchParameter { Id = "test", Url = "http://test.com/param", VersionId = Guid.NewGuid().ToString() }; + var wrapper = CreateMockResourceWrapper(searchParameter); + + var httpContext = new DefaultHttpContext(); + httpContext.Request.Headers[KnownHeaders.IfNoneExist] = "url=http://test.com/param"; + _fhirController.ControllerContext.HttpContext = httpContext; + + var attemptCount = 0; + _searchParameterOperations + .GetAndApplySearchParameterUpdates(Arg.Any()) + .Returns(Task.FromResult(true)); + _searchParameterOperations.SearchParamLastUpdated.Returns(DateTimeOffset.UtcNow); + + _mediator.Send( + Arg.Any(), + Arg.Any()) + .Returns(callInfo => + { + attemptCount++; + if (attemptCount < 3) + { + throw new BadRequestException(Core.Resources.SearchParameterConcurrencyConflict); + } + + return new UpsertResourceResponse(new SaveOutcome(new RawResourceElement(wrapper), SaveOutcomeType.Created)); + }); + + var response = await _fhirController.ConditionalCreate(searchParameter); + + Assert.Equal(3, attemptCount); + Assert.IsType(response); + } + + [Fact] + public async Task GivenSearchParameterConditionalUpdate_WhenConcurrencyConflictOccurs_ThenRetriesAndSucceeds() + { + var searchParameter = new SearchParameter { Id = "test", Url = "http://test.com/param", VersionId = Guid.NewGuid().ToString() }; + var wrapper = CreateMockResourceWrapper(searchParameter); + + var httpContext = new DefaultHttpContext(); + httpContext.Request.QueryString = new QueryString("?url=http://test.com/param"); + _fhirController.ControllerContext.HttpContext = httpContext; + + var attemptCount = 0; + _searchParameterOperations + .GetAndApplySearchParameterUpdates(Arg.Any()) + .Returns(Task.FromResult(true)); + _searchParameterOperations.SearchParamLastUpdated.Returns(DateTimeOffset.UtcNow); + + _mediator.Send( + Arg.Any(), + Arg.Any()) + .Returns(callInfo => + { + attemptCount++; + if (attemptCount < 2) + { + throw new BadRequestException(Core.Resources.SearchParameterConcurrencyConflict); + } + + return new UpsertResourceResponse(new SaveOutcome(new RawResourceElement(wrapper), SaveOutcomeType.Updated)); + }); + + var response = await _fhirController.ConditionalUpdate(searchParameter); + + Assert.Equal(2, attemptCount); + Assert.IsType(response); + } + + [Fact] + public async Task GivenNonSearchParameterResource_WhenOperationExecuted_ThenNoRetry() + { + var patient = new Patient { Id = "test", VersionId = Guid.NewGuid().ToString() }; + var wrapper = CreateMockResourceWrapper(patient); + + var attemptCount = 0; + _mediator.Send( + Arg.Any(), + Arg.Any()) + .Returns(callInfo => + { + attemptCount++; + return new UpsertResourceResponse(new SaveOutcome(new RawResourceElement(wrapper), SaveOutcomeType.Created)); + }); + + var response = await _fhirController.Create(patient); + + Assert.Equal(1, attemptCount); + Assert.IsType(response); + await _searchParameterOperations.DidNotReceive().GetAndApplySearchParameterUpdates(Arg.Any()); + } + + private ResourceWrapper CreateMockResourceWrapper(Resource resource) + { + var rawJson = new FhirJsonSerializer().SerializeToString(resource); + return new ResourceWrapper( + resource.ToResourceElement(), + new RawResource(rawJson, FhirResourceFormat.Json, isMetaSet: false), + null, + false, + null, + null, + null); + } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Filters/SearchParameterFilterAttributeTests.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Filters/SearchParameterFilterAttributeTests.cs index bb739f3442..08558c8a3a 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Filters/SearchParameterFilterAttributeTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Filters/SearchParameterFilterAttributeTests.cs @@ -3,6 +3,7 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System; using System.Collections.Generic; using System.Threading; using Hl7.Fhir.Model; @@ -11,8 +12,11 @@ using Microsoft.AspNetCore.Mvc.Abstractions; using Microsoft.AspNetCore.Mvc.Filters; using Microsoft.AspNetCore.Routing; +using Microsoft.Health.Core.Features.Context; using Microsoft.Health.Fhir.Api.Features.Filters; +using Microsoft.Health.Fhir.Core.Features.Context; using Microsoft.Health.Fhir.Core.Features.Routing; +using Microsoft.Health.Fhir.Core.UnitTests.Features.Context; using Microsoft.Health.Fhir.Shared.Core.Features.Search.Parameters; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Test.Utilities; @@ -28,11 +32,18 @@ namespace Microsoft.Health.Fhir.Api.UnitTests.Features.Filters public class SearchParameterFilterAttributeTests { private readonly ISearchParameterValidator _searchParameterValidator = Substitute.For(); + private readonly RequestContextAccessor _fhirRequestContextAccessor = Substitute.For>(); + private readonly DefaultFhirRequestContext _fhirRequestContext = new DefaultFhirRequestContext(); + + public SearchParameterFilterAttributeTests() + { + _fhirRequestContextAccessor.RequestContext.Returns(_fhirRequestContext); + } [Fact] public async Task GivenAnAction_WhenPostingAnObservationObject_ThenNoSearchParameterActionTaken() { - var filter = new SearchParameterFilterAttribute(_searchParameterValidator); + var filter = new SearchParameterFilterAttribute(_searchParameterValidator, _fhirRequestContextAccessor); var context = CreateContext(new Observation()); var actionExecutedContext = new ActionExecutedContext(context, new List(), null); @@ -40,13 +51,36 @@ public async Task GivenAnAction_WhenPostingAnObservationObject_ThenNoSearchParam await filter.OnActionExecutionAsync(context, actionExecutionDelegate); - await _searchParameterValidator.DidNotReceive().ValidateSearchParameterInput(Arg.Any(), Arg.Any(), Arg.Any()); + await _searchParameterValidator.DidNotReceive().ValidateSearchParameterInput(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()); } [Fact] public async Task GivenAnAction_WhenPostingASearchParameterObject_ThenSearchParameterActionsTaken() { - var filter = new SearchParameterFilterAttribute(_searchParameterValidator); + _searchParameterValidator.ValidateSearchParameterInput(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(DateTimeOffset.UtcNow); + + var filter = new SearchParameterFilterAttribute(_searchParameterValidator, _fhirRequestContextAccessor); + + var context = CreateContext(new SearchParameter()); + var actionExecutedContext = new ActionExecutedContext(context, new List(), null); + ActionExecutionDelegate actionExecutionDelegate = () => Task.Run(() => actionExecutedContext); + + await filter.OnActionExecutionAsync(context, actionExecutionDelegate); + + await _searchParameterValidator.Received().ValidateSearchParameterInput(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()); + } + + [Fact] + public async Task GivenAnAction_WhenPostingASearchParameterObjectWithExistingLastUpdated_ThenExistingLastUpdatedIsPassedToValidator() + { + var existingLastUpdated = DateTimeOffset.UtcNow.AddHours(-1); + _fhirRequestContext.SetSearchParameterLastUpdated(existingLastUpdated); + + _searchParameterValidator.ValidateSearchParameterInput(Arg.Any(), Arg.Any(), Arg.Any(), existingLastUpdated) + .Returns(existingLastUpdated); + + var filter = new SearchParameterFilterAttribute(_searchParameterValidator, _fhirRequestContextAccessor); var context = CreateContext(new SearchParameter()); var actionExecutedContext = new ActionExecutedContext(context, new List(), null); @@ -54,7 +88,7 @@ public async Task GivenAnAction_WhenPostingASearchParameterObject_ThenSearchPara await filter.OnActionExecutionAsync(context, actionExecutionDelegate); - await _searchParameterValidator.Received().ValidateSearchParameterInput(Arg.Any(), Arg.Any(), Arg.Any()); + await _searchParameterValidator.Received().ValidateSearchParameterInput(Arg.Any(), Arg.Any(), Arg.Any(), existingLastUpdated); } private static ActionExecutingContext CreateContext(Base type) diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Controllers/FhirController.cs b/src/Microsoft.Health.Fhir.Shared.Api/Controllers/FhirController.cs index 47b4b91cee..b0ee52d763 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Controllers/FhirController.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api/Controllers/FhirController.cs @@ -42,6 +42,7 @@ using Microsoft.Health.Fhir.Core.Features.Persistence.Orchestration; using Microsoft.Health.Fhir.Core.Features.Resources.Patch; using Microsoft.Health.Fhir.Core.Features.Routing; +using Microsoft.Health.Fhir.Core.Features.Search.Parameters; using Microsoft.Health.Fhir.Core.Messages.Create; using Microsoft.Health.Fhir.Core.Messages.Delete; using Microsoft.Health.Fhir.Core.Messages.Get; @@ -68,6 +69,7 @@ public class FhirController : Controller private readonly IMediator _mediator; private readonly RequestContextAccessor _fhirRequestContextAccessor; private readonly IUrlResolver _urlResolver; + private readonly ISearchParameterOperations _searchParameterOperations; /// /// Initializes a new instance of the class. @@ -77,12 +79,14 @@ public class FhirController : Controller /// The urlResolver. /// The UI configuration. /// The authorization service. + /// The search parameter operations. public FhirController( IMediator mediator, RequestContextAccessor fhirRequestContextAccessor, IUrlResolver urlResolver, IOptions uiConfiguration, - IAuthorizationService authorizationService) + IAuthorizationService authorizationService, + ISearchParameterOperations searchParameterOperations) { EnsureArg.IsNotNull(mediator, nameof(mediator)); EnsureArg.IsNotNull(fhirRequestContextAccessor, nameof(fhirRequestContextAccessor)); @@ -90,10 +94,12 @@ public FhirController( EnsureArg.IsNotNull(uiConfiguration, nameof(uiConfiguration)); EnsureArg.IsNotNull(uiConfiguration.Value, nameof(uiConfiguration)); EnsureArg.IsNotNull(authorizationService, nameof(authorizationService)); + EnsureArg.IsNotNull(searchParameterOperations, nameof(searchParameterOperations)); _mediator = mediator; _fhirRequestContextAccessor = fhirRequestContextAccessor; _urlResolver = urlResolver; + _searchParameterOperations = searchParameterOperations; } [ApiExplorerSettings(IgnoreApi = true)] @@ -162,9 +168,12 @@ public IActionResult CustomError(int? statusCode = null) [TypeFilter(typeof(CrudEndpointMetricEmitterAttribute))] public async Task Create([FromBody] Resource resource) { - RawResourceElement response = await _mediator.CreateResourceAsync( - new CreateResourceRequest(resource.ToResourceElement(), GetBundleResourceContext()), - HttpContext.RequestAborted); + var response = await ExecuteWithSearchParameterRetryAsync( + resource.TypeName, + () => _mediator.CreateResourceAsync( + new CreateResourceRequest(resource.ToResourceElement(), GetBundleResourceContext()), + HttpContext.RequestAborted), + "Create"); return FhirResult.Create(response, HttpStatusCode.Created) .SetETagHeader() @@ -191,25 +200,28 @@ public async Task ConditionalCreate([FromBody] Resource resource) Tuple[] conditionalParameters = QueryHelpers.ParseQuery(conditionalCreateHeader) .SelectMany(query => query.Value, (query, value) => Tuple.Create(query.Key, value)).ToArray(); - UpsertResourceResponse createResponse = await _mediator.Send( + var response = await ExecuteWithSearchParameterRetryAsync( + resource.TypeName, + () => _mediator.Send( new ConditionalCreateResourceRequest(resource.ToResourceElement(), conditionalParameters, GetBundleResourceContext()), - HttpContext.RequestAborted); + HttpContext.RequestAborted), + "ConditionalCreate"); - if (createResponse?.Outcome == null) + if (response?.Outcome == null) { return Ok(); } var statusCode = HttpStatusCode.Created; var message = Resources.ConditionalCreateResourceCreated; - if (createResponse.Outcome.Outcome != SaveOutcomeType.Created) + if (response.Outcome.Outcome != SaveOutcomeType.Created) { statusCode = HttpStatusCode.OK; message = Resources.ConditionalCreateResourceAlreadyExists; } return FhirResult.Create( - createResponse.Outcome.RawResourceElement, + response.Outcome.RawResourceElement, statusCode, true, true, @@ -233,9 +245,12 @@ public async Task ConditionalCreate([FromBody] Resource resource) [TypeFilter(typeof(CrudEndpointMetricEmitterAttribute))] public async Task Update([FromBody] Resource resource, [ModelBinder(typeof(WeakETagBinder))] WeakETag ifMatchHeader, [FromQuery(Name = KnownQueryParameterNames.MetaHistory)] bool metaHistory = true) { - SaveOutcome response = await _mediator.UpsertResourceAsync( - new UpsertResourceRequest(resource.ToResourceElement(), GetBundleResourceContext(), ifMatchHeader, metaHistory), - HttpContext.RequestAborted); + var response = await ExecuteWithSearchParameterRetryAsync( + resource.TypeName, + () => _mediator.UpsertResourceAsync( + new UpsertResourceRequest(resource.ToResourceElement(), GetBundleResourceContext(), ifMatchHeader, metaHistory), + HttpContext.RequestAborted), + "Update"); return ToSaveOutcomeResult(response); } @@ -254,9 +269,12 @@ public async Task ConditionalUpdate([FromBody] Resource resource) IReadOnlyList> conditionalParameters = GetQueriesForSearch(); - UpsertResourceResponse response = await _mediator.Send( + var response = await ExecuteWithSearchParameterRetryAsync( + resource.TypeName, + () => _mediator.Send( new ConditionalUpsertResourceRequest(resource.ToResourceElement(), conditionalParameters, GetBundleResourceContext()), - HttpContext.RequestAborted); + HttpContext.RequestAborted), + "ConditionalUpdate"); SaveOutcome saveOutcome = response.Outcome; @@ -419,13 +437,16 @@ public async Task VRead(string typeParameter, string idParameter, [TypeFilter(typeof(CrudEndpointMetricEmitterAttribute))] public async Task Delete(string typeParameter, string idParameter, HardDeleteModel hardDeleteModel, [FromQuery] bool allowPartialSuccess) { - DeleteResourceResponse response = await _mediator.DeleteResourceAsync( - new DeleteResourceRequest( - new ResourceKey(typeParameter, idParameter), - hardDeleteModel.IsHardDelete ? DeleteOperation.HardDelete : DeleteOperation.SoftDelete, - GetBundleResourceContext(), - allowPartialSuccess), - HttpContext.RequestAborted); + var response = await ExecuteWithSearchParameterRetryAsync( + typeParameter, + () => _mediator.DeleteResourceAsync( + new DeleteResourceRequest( + new ResourceKey(typeParameter, idParameter), + hardDeleteModel.IsHardDelete ? DeleteOperation.HardDelete : DeleteOperation.SoftDelete, + GetBundleResourceContext(), + allowPartialSuccess), + HttpContext.RequestAborted), + "Delete"); return FhirResult.NoContent().SetETagHeader(response.WeakETag); } @@ -469,14 +490,17 @@ public async Task ConditionalDelete(string typeParameter, HardDel SetupConditionalRequestWithQueryOptimizeConcurrency(); - DeleteResourceResponse response = await _mediator.Send( + var response = await ExecuteWithSearchParameterRetryAsync( + typeParameter, + () => _mediator.Send( new ConditionalDeleteResourceRequest( typeParameter, conditionalParameters, hardDeleteModel.IsHardDelete ? DeleteOperation.HardDelete : DeleteOperation.SoftDelete, maxDeleteCount.GetValueOrDefault(1), GetBundleResourceContext()), - HttpContext.RequestAborted); + HttpContext.RequestAborted), + "ConditionalDelete"); if (maxDeleteCount.HasValue) { @@ -705,6 +729,32 @@ public async Task BatchAndTransactions([FromBody] Resource bundle return FhirResult.Create(bundleResponse); } + /// + /// Executes an action with retry logic if the resource type is SearchParameter, and it is not a part of parallel bundle. + /// + private async Task ExecuteWithSearchParameterRetryAsync(string resourceType, Func> action, string info) + { + if (resourceType == KnownResourceTypes.SearchParameter) + { + var context = GetBundleResourceContext(); + if (context != null && context.IsParallelBundle) + { + return await action(); + } + + return await SearchParameterRetry.ExecuteAsync( + async () => + { + await _searchParameterOperations.GetAndApplySearchParameterUpdates(HttpContext.RequestAborted); + _fhirRequestContextAccessor.RequestContext.SetSearchParameterLastUpdated(_searchParameterOperations.SearchParamLastUpdated); + return await action(); + }, + info); + } + + return await action(); + } + /// /// Returns an instance of with bundle related information, if a resource if part of a bundle. /// diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Features/Filters/SearchParameterFilterAttribute.cs b/src/Microsoft.Health.Fhir.Shared.Api/Features/Filters/SearchParameterFilterAttribute.cs index bc7b78aee3..6bf5fe93d5 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Features/Filters/SearchParameterFilterAttribute.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api/Features/Filters/SearchParameterFilterAttribute.cs @@ -4,10 +4,15 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Linq; using EnsureThat; using Hl7.Fhir.Model; using Microsoft.AspNetCore.Mvc.Filters; +using Microsoft.Health.Core.Features.Context; +using Microsoft.Health.Fhir.Core.Features.Context; +using Microsoft.Health.Fhir.Core.Features.Persistence.Orchestration; using Microsoft.Health.Fhir.Core.Features.Routing; +using Microsoft.Health.Fhir.Core.Features.Search.Parameters; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Shared.Core.Features.Search.Parameters; using Task = System.Threading.Tasks.Task; @@ -17,13 +22,16 @@ namespace Microsoft.Health.Fhir.Api.Features.Filters [AttributeUsage(AttributeTargets.Method)] internal sealed class SearchParameterFilterAttribute : ActionFilterAttribute { - private ISearchParameterValidator _searchParameterValidator; + private readonly ISearchParameterValidator _searchParameterValidator; + private readonly RequestContextAccessor _fhirRequestContextAccessor; - public SearchParameterFilterAttribute(ISearchParameterValidator searchParamValidator) + public SearchParameterFilterAttribute(ISearchParameterValidator searchParamValidator, RequestContextAccessor fhirRequestContextAccessor) { EnsureArg.IsNotNull(searchParamValidator, nameof(searchParamValidator)); + EnsureArg.IsNotNull(fhirRequestContextAccessor, nameof(fhirRequestContextAccessor)); _searchParameterValidator = searchParamValidator; + _fhirRequestContextAccessor = fhirRequestContextAccessor; } public override async Task OnActionExecutionAsync(ActionExecutingContext context, ActionExecutionDelegate next) @@ -33,11 +41,20 @@ public override async Task OnActionExecutionAsync(ActionExecutingContext context var searchParameter = ExtractSearchParameter(context); if (searchParameter != null) { - // wait for the validation checks to pass before allowing the FHIRController action to continue - await _searchParameterValidator.ValidateSearchParameterInput( + var fhirRequestContext = _fhirRequestContextAccessor.RequestContext; + var lastUpdated = fhirRequestContext.GetSearchParameterLastUpdated(); + var hasLastUpdated = lastUpdated.HasValue; + + lastUpdated = await _searchParameterValidator.ValidateSearchParameterInput( searchParameter, context.HttpContext.Request.Method, - context.HttpContext.RequestAborted); + context.HttpContext.RequestAborted, + lastUpdated); + + if (!hasLastUpdated) + { + fhirRequestContext.SetSearchParameterLastUpdated(lastUpdated); + } } await next(); diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Features/Resources/Bundle/BundleHandler.cs b/src/Microsoft.Health.Fhir.Shared.Api/Features/Resources/Bundle/BundleHandler.cs index de747eb90c..745dcf9c00 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Features/Resources/Bundle/BundleHandler.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api/Features/Resources/Bundle/BundleHandler.cs @@ -114,12 +114,12 @@ public partial class BundleHandler : IRequestHandler /// Headers to propagate from the inner actions to the outer HTTP request. /// - private static readonly string[] HeadersToAccumulate = new[] { KnownHeaders.RetryAfter, KnownHeaders.RetryAfterMilliseconds, "x-ms-session-token", "x-ms-request-charge" }; + private static readonly string[] HeadersToAccumulate = [KnownHeaders.RetryAfter, KnownHeaders.RetryAfterMilliseconds, "x-ms-session-token", "x-ms-request-charge"]; /// /// Properties to propagate from the outer HTTP requests to the inner actions. /// - private static readonly string[] PropertiesToAccumulate = new[] { KnownQueryParameterNames.OptimizeConcurrency }; + private static readonly string[] PropertiesToAccumulate = [KnownQueryParameterNames.OptimizeConcurrency, SearchParameterRequestContextPropertyNames.LastUpdated]; private static readonly Uri LocalHost = new("http://localhost/"); @@ -219,10 +219,7 @@ public async Task Handle(BundleRequest request, CancellationToke { await FillRequestLists(bundleResource.Entry, cancellationToken); - var responseBundle = new Hl7.Fhir.Model.Bundle - { - Type = BundleType.BatchResponse, - }; + var responseBundle = new Hl7.Fhir.Model.Bundle { Type = BundleType.BatchResponse }; if (bundleProcessingLogic == BundleProcessingLogic.Parallel) { @@ -264,12 +261,9 @@ public async Task Handle(BundleRequest request, CancellationToke } } - await CheckSearchParamInputConflictsAndUpdateCache(bundleResource, cancellationToken); + var responseBundle = new Hl7.Fhir.Model.Bundle { Type = BundleType.TransactionResponse }; - var responseBundle = new Hl7.Fhir.Model.Bundle - { - Type = BundleType.TransactionResponse, - }; + await CheckSearchParamInputConflictsAndUpdateCache(bundleResource, cancellationToken); await ExecuteTransactionForAllRequestsAsync(responseBundle, bundleProcessingLogic, cancellationToken); @@ -332,9 +326,16 @@ private async Task CheckSearchParamInputConflictsAndUpdateCache(Hl7.Fhir.Model.B throw new RequestNotValidException(string.Format(Api.Resources.DuplicateSearchParamCodesAndUrlsInBundle, string.Join(", ", dupCodes), string.Join(", ", dupUrls))); } + // for deletes Entry.Resource is null. need to check in other way + if (!searchParamsInBundle && bundle.Entry.Any(e => e.Request.Method == HTTPVerb.DELETE && e.Request.Url.StartsWith(KnownResourceTypes.SearchParameter, StringComparison.OrdinalIgnoreCase))) + { + searchParamsInBundle = true; + } + if (searchParamsInBundle) { await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken); // refresh search param cache + _fhirRequestContextAccessor.RequestContext.SetSearchParameterLastUpdated(_searchParameterOperations.SearchParamLastUpdated); // capture last updated } } @@ -1111,19 +1112,6 @@ private static OperationOutcome CreateOperationOutcome(OperationOutcome.IssueSev }; } - private static string SanitizeString(string input) - { - if (string.IsNullOrWhiteSpace(input)) - { - return string.Empty; - } - - return input - .Replace(Environment.NewLine, string.Empty, StringComparison.OrdinalIgnoreCase) - .Replace("\r", " ", StringComparison.OrdinalIgnoreCase) - .Replace("\n", " ", StringComparison.OrdinalIgnoreCase); - } - private BundleHandlerStatistics CreateNewBundleHandlerStatistics(BundleProcessingLogic processingLogic) { BundleHandlerStatistics statistics = new BundleHandlerStatistics( diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Resources/Delete/DeletionServiceTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Resources/Delete/DeletionServiceTests.cs index 84e01f8f27..2ddbe4eee5 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Resources/Delete/DeletionServiceTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Resources/Delete/DeletionServiceTests.cs @@ -16,6 +16,7 @@ using Microsoft.Health.Core.Features.Audit; using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Core.Exceptions; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Audit; using Microsoft.Health.Fhir.Core.Features.Conformance; @@ -155,5 +156,120 @@ public async Task GivenBulkHardDelete_WhenResourcesAreDeleted_ThenAuditLoggerIsC Arg.Any(), Arg.Is>(d => d.ContainsKey("Affected Items"))); } + + [Fact] + public async Task GivenSearchParameterDelete_WhenConcurrencyConflictOccurs_ThenRetries() + { + var resourceType = "SearchParameter"; + var parameters = new List>() + { + Tuple.Create("url", "http://test.com/param"), + }; + + var request = new ConditionalDeleteResourceRequest( + resourceType, + parameters, + DeleteOperation.HardDelete, + maxDeleteCount: 10, + deleteAll: false); + + var searchService = Substitute.For(); + var scopedSearchService = Substitute.For>(); + scopedSearchService.Value.Returns(searchService); + _searchServiceFactory.Invoke().Returns(scopedSearchService); + + var searchParameter = new SearchParameter { Id = "test", Url = "http://test.com/param" }; + var resource = searchParameter.ToResourceElement(); + var rawResource = new RawResource(searchParameter.ToJson(), FhirResourceFormat.Json, isMetaSet: false); + var resourceRequest = Substitute.For(); + var compartmentIndices = Substitute.For(); + var wrapper = new ResourceWrapper(resource, rawResource, resourceRequest, false, null, compartmentIndices, new List>(), "hash"); + var entries = new List { new SearchResultEntry(wrapper, SearchEntryMode.Match) }; + + searchService.SearchAsync( + Arg.Any(), + Arg.Any>>(), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()).Returns( + Task.FromResult(new SearchResult(entries, null, null, Array.Empty>()))); + + var fhirDataStore = Substitute.For(); + var scopedDataStore = new DeletionServiceScopedDataStore(fhirDataStore); + _dataStoreFactory.GetScopedDataStore().Returns(scopedDataStore); + + var attemptCount = 0; + _searchParameterOperations + .DeleteSearchParameterAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(callInfo => + { + attemptCount++; + if (attemptCount < 3) + { + throw new BadRequestException(Core.Resources.SearchParameterConcurrencyConflict); + } + + return Task.CompletedTask; + }); + + await _service.DeleteMultipleAsync(request, CancellationToken.None); + + Assert.Equal(3, attemptCount); + } + + [Fact] + public async Task GivenSearchParameterDelete_WhenConcurrencyConflictExhaustsRetries_ThenThrowsWithRetryCount() + { + var resourceType = "SearchParameter"; + var parameters = new List>() + { + Tuple.Create("url", "http://test.com/param"), + }; + + var request = new ConditionalDeleteResourceRequest( + resourceType, + parameters, + DeleteOperation.HardDelete, + maxDeleteCount: 10, + deleteAll: false); + + var searchService = Substitute.For(); + var scopedSearchService = Substitute.For>(); + scopedSearchService.Value.Returns(searchService); + _searchServiceFactory.Invoke().Returns(scopedSearchService); + + var searchParameter = new SearchParameter { Id = "test", Url = "http://test.com/param" }; + var resource = searchParameter.ToResourceElement(); + var rawResource = new RawResource(searchParameter.ToJson(), FhirResourceFormat.Json, isMetaSet: false); + var resourceRequest = Substitute.For(); + var compartmentIndices = Substitute.For(); + var wrapper = new ResourceWrapper(resource, rawResource, resourceRequest, false, null, compartmentIndices, new List>(), "hash"); + var entries = new List { new SearchResultEntry(wrapper, SearchEntryMode.Match) }; + + searchService.SearchAsync( + Arg.Any(), + Arg.Any>>(), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any(), + Arg.Any()).Returns( + Task.FromResult(new SearchResult(entries, null, null, Array.Empty>()))); + + var fhirDataStore = Substitute.For(); + var scopedDataStore = new DeletionServiceScopedDataStore(fhirDataStore); + _dataStoreFactory.GetScopedDataStore().Returns(scopedDataStore); + + _searchParameterOperations + .DeleteSearchParameterAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(_ => throw new BadRequestException(Core.Resources.SearchParameterConcurrencyConflict)); + + var exception = await Assert.ThrowsAsync>>(async () => + await _service.DeleteMultipleAsync(request, CancellationToken.None)); + + Assert.Contains(" Deletion.3", exception.InnerException.Message); + } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterBehaviorTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterBehaviorTests.cs index ce29c555ca..160f218dcd 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterBehaviorTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterBehaviorTests.cs @@ -66,6 +66,8 @@ public SearchParameterBehaviorTests() .Returns(x => CreateResourceWrapper(x.ArgAt(0), x.ArgAt(1))); _fhirDataStore = Substitute.For(); + + _searchParameterOperations.SearchParamLastUpdated.Returns(System.DateTimeOffset.UtcNow); } [Fact] @@ -78,7 +80,7 @@ public async Task GivenACreateResourceRequest_WhenCreatingAResourceOtherThanSear var response = new UpsertResourceResponse(new SaveOutcome(new RawResourceElement(wrapper), SaveOutcomeType.Created)); - var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterStatusManager, _requestContextAccessor, _modelInfoProvider); + var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterDefinitionManager, _requestContextAccessor, _modelInfoProvider); await behavior.Handle(request, async (ct) => await Task.Run(() => response), CancellationToken.None); await _searchParameterOperations.DidNotReceive().ValidateSearchParameterAsync(Arg.Any(), Arg.Any()); @@ -95,7 +97,7 @@ public async Task GivenACreateResourceRequest_WhenCreatingASearchParameterResour var response = new UpsertResourceResponse(new SaveOutcome(new RawResourceElement(wrapper), SaveOutcomeType.Created)); - var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterStatusManager, _requestContextAccessor, _modelInfoProvider); + var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterDefinitionManager, _requestContextAccessor, _modelInfoProvider); await behavior.Handle(request, async (ct) => await Task.Run(() => response), CancellationToken.None); await _searchParameterOperations.Received().ValidateSearchParameterAsync(Arg.Any(), Arg.Any()); @@ -132,8 +134,6 @@ public async Task GivenADeleteResourceRequest_WhenDeletingASearchParameterResour var wrapper = CreateResourceWrapper(resource, false); _fhirDataStore.GetAsync(key, Arg.Any()).Returns(wrapper); - _searchParameterStatusManager.GetAllSearchParameterStatus(Arg.Any()) - .Returns(new List()); var contextProperties = new Dictionary(); var fhirContext = Substitute.For(); @@ -145,7 +145,6 @@ public async Task GivenADeleteResourceRequest_WhenDeletingASearchParameterResour var behavior = new DeleteSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterDefinitionManager, _searchParameterStatusManager, _requestContextAccessor, _modelInfoProvider); await behavior.Handle(request, async (ct) => await Task.Run(() => response), CancellationToken.None); - await _searchParameterStatusManager.Received().GetAllSearchParameterStatus(Arg.Any()); Assert.True(contextProperties.ContainsKey(SearchParameterRequestContextPropertyNames.PendingStatusUpdates)); var pendingStatuses = contextProperties[SearchParameterRequestContextPropertyNames.PendingStatusUpdates] as List; Assert.Single(pendingStatuses); @@ -227,7 +226,7 @@ public async Task GivenAnUpsertResourceRequest_WhenSearchParameterDoesNotExist_T var response = new UpsertResourceResponse(new SaveOutcome(new RawResourceElement(wrapper), SaveOutcomeType.Created)); - var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterStatusManager, _requestContextAccessor, _modelInfoProvider); + var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterDefinitionManager, _requestContextAccessor, _modelInfoProvider); await behavior.Handle(request, async (ct) => await Task.Run(() => response), CancellationToken.None); await _searchParameterOperations.Received().ValidateSearchParameterAsync(Arg.Any(), Arg.Any()); @@ -251,7 +250,7 @@ public async Task GivenAnUpsertResourceRequest_WhenSearchParameterExists_ThenVal var response = new UpsertResourceResponse(new SaveOutcome(new RawResourceElement(newWrapper), SaveOutcomeType.Updated)); - var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterStatusManager, _requestContextAccessor, _modelInfoProvider); + var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterDefinitionManager, _requestContextAccessor, _modelInfoProvider); await behavior.Handle(request, async (ct) => await Task.Run(() => response), CancellationToken.None); await _searchParameterOperations.Received().ValidateSearchParameterAsync(Arg.Any(), Arg.Any()); @@ -282,7 +281,7 @@ public async Task GivenAnUpsertResourceRequest_WhenSearchParameterUrlChanges_The var response = new UpsertResourceResponse(new SaveOutcome(new RawResourceElement(newWrapper), SaveOutcomeType.Updated)); - var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterStatusManager, _requestContextAccessor, _modelInfoProvider); + var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterDefinitionManager, _requestContextAccessor, _modelInfoProvider); await behavior.Handle(request, async (ct) => await Task.Run(() => response), CancellationToken.None); var pendingStatuses = contextProperties[SearchParameterRequestContextPropertyNames.PendingStatusUpdates] as List; diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterRetryTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterRetryTests.cs new file mode 100644 index 0000000000..fe7f372957 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterRetryTests.cs @@ -0,0 +1,94 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading.Tasks; +using Microsoft.Health.Fhir.Core.Exceptions; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Search.Parameters; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using Xunit; +using Task = System.Threading.Tasks.Task; + +namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Search.SearchParameters +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Search)] + public class SearchParameterRetryTests + { + [Fact] + public async Task GivenConcurrencyConflict_WhenRetriesExhausted_ThenThrowsWithRetryCount() + { + var attemptCount = 0; + var maxAttempts = 4; // 1 initial + 3 retries + + var exception = await Assert.ThrowsAsync(async () => + { + await SearchParameterRetry.ExecuteAsync( + async () => + { + attemptCount++; + await Task.CompletedTask; + throw new BadRequestException(Core.Resources.SearchParameterConcurrencyConflict); + }); + }); + + Assert.Equal(maxAttempts, attemptCount); + Assert.Contains(" .3", exception.Message); + } + + [Fact] + public async Task GivenSuccessfulOperation_WhenExecuted_ThenNoRetry() + { + var attemptCount = 0; + + var result = await SearchParameterRetry.ExecuteAsync( + async () => + { + attemptCount++; + await Task.CompletedTask; + return "success"; + }); + + Assert.Equal(1, attemptCount); + Assert.Equal("success", result); + } + + [Fact] + public async Task GivenNonGenericOverload_WhenOperationSucceeds_ThenCompletes() + { + var executed = false; + + await SearchParameterRetry.ExecuteAsync( + async () => + { + await Task.CompletedTask; + executed = true; + }); + + Assert.True(executed); + } + + [Fact] + public async Task GivenNonConcurrencyException_WhenThrown_ThenNoRetry() + { + var attemptCount = 0; + + var exception = await Assert.ThrowsAsync(async () => + { + await SearchParameterRetry.ExecuteAsync( + async () => + { + attemptCount++; + await Task.CompletedTask; + throw new BadRequestException("some other error"); + }); + }); + + Assert.Equal(1, attemptCount); + Assert.DoesNotContain(" .", exception.Message); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterValidatorTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterValidatorTests.cs index c1912af0cd..7ae8ef6fd1 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterValidatorTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterValidatorTests.cs @@ -69,6 +69,7 @@ public SearchParameterValidatorTests() return true; }); _searchParameterOperations.EnsureNoActiveReindexJobAsync(CancellationToken.None).Returns(Task.CompletedTask); + _searchParameterOperations.SearchParamLastUpdated.Returns(System.DateTimeOffset.UtcNow); } [Theory] diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems index a3518aa589..54d313edb1 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems @@ -63,6 +63,7 @@ + diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Resources/Delete/DeletionService.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Resources/Delete/DeletionService.cs index e7e9796c6d..8c34c154ec 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Resources/Delete/DeletionService.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Resources/Delete/DeletionService.cs @@ -39,7 +39,7 @@ namespace Microsoft.Health.Fhir.Core.Features.Persistence { - public class DeletionService : IDeletionService + public class DeletionService : IDeletionService, IDisposable { private readonly IResourceWrapperFactory _resourceWrapperFactory; private readonly Lazy _conformanceProvider; @@ -54,6 +54,8 @@ public class DeletionService : IDeletionService private readonly ISearchParameterOperations _searchParameterOperations; private readonly IResourceDeserializer _resourceDeserializer; private readonly ILogger _logger; + private readonly SemaphoreSlim _searchParamDeleteSemaphore; + private bool _disposed; internal const string DefaultCallerAgent = "Microsoft.Health.Fhir.Server"; private const int MaxParallelThreads = 64; @@ -83,6 +85,7 @@ public DeletionService( _fhirRuntimeConfiguration = EnsureArg.IsNotNull(fhirRuntimeConfiguration, nameof(fhirRuntimeConfiguration)); _searchParameterOperations = EnsureArg.IsNotNull(searchParameterOperations, nameof(searchParameterOperations)); _resourceDeserializer = EnsureArg.IsNotNull(resourceDeserializer, nameof(resourceDeserializer)); + _searchParamDeleteSemaphore = new SemaphoreSlim(1, 1); _retryPolicy = Policy .Handle() @@ -362,7 +365,7 @@ await CreateAuditLog( false, resourcesToDelete.Select((item) => (item.Resource.ResourceTypeName, item.Resource.ResourceId, item.SearchEntryMode == ValueSets.SearchEntryMode.Include))); - ResourceWrapperOperation[] softDeleteIncludes = await Task.WhenAll(resourcesToDelete.Where(resource => resource.SearchEntryMode == ValueSets.SearchEntryMode.Include).Select(async item => + var softDeleteIncludes = await Task.WhenAll(resourcesToDelete.Where(resource => resource.SearchEntryMode == ValueSets.SearchEntryMode.Include).Select(async item => { // If there isn't a cached capability statement (IE this is the first request made after a service starts up) then performance on this request will be terrible as the capability statement needs to be rebuilt for every resource. // This is because the capability statement can't be made correctly in a background job, so it doesn't cache the result. @@ -372,7 +375,7 @@ await CreateAuditLog( return new ResourceWrapperOperation(deletedWrapper, true, keepHistory, null, false, false, bundleResourceContext: request.BundleResourceContext); })); - ResourceWrapperOperation[] softDeleteMatches = await Task.WhenAll(resourcesToDelete.Where(resource => resource.SearchEntryMode == ValueSets.SearchEntryMode.Match).Select(async item => + var softDeleteMatches = await Task.WhenAll(resourcesToDelete.Where(resource => resource.SearchEntryMode == ValueSets.SearchEntryMode.Match).Select(async item => { bool keepHistory = await _conformanceProvider.Value.CanKeepHistory(item.Resource.ResourceTypeName, cancellationToken); ResourceWrapper deletedWrapper = CreateSoftDeletedWrapper(item.Resource.ResourceTypeName, item.Resource.ResourceId); @@ -397,9 +400,7 @@ await CreateAuditLog( .FirstOrDefault().SearchEntryMode == ValueSets.SearchEntryMode.Include))); } - await DeleteSearchParametersAsync( - resourcesToDelete.Where(resource => resource.SearchEntryMode == ValueSets.SearchEntryMode.Match).Select(x => x.Resource).ToList(), - cancellationToken); + await DeleteSearchParametersAsync(resourcesToDelete.Where(resource => resource.SearchEntryMode == ValueSets.SearchEntryMode.Match), cancellationToken); await fhirDataStore.MergeAsync(softDeleteMatches, cancellationToken); } @@ -458,20 +459,27 @@ await CreateAuditLog( var matchedResources = resourcesToDelete.Where(resource => resource.SearchEntryMode == ValueSets.SearchEntryMode.Match).ToList(); // Delete includes first so that if there is a failure, the match resources are not deleted. This allows the job to restart. - // This throws AggrigateExceptions + // This throws AggregateExceptions. + // Note: includedResources cannot have search params await Parallel.ForEachAsync(includedResources, cancellationToken, async (item, innerCt) => { - await DeleteSearchParameterAsync(item.Resource, cancellationToken); await _retryPolicy.ExecuteAsync(async () => await fhirDataStore.HardDeleteAsync(new ResourceKey(item.Resource.ResourceTypeName, item.Resource.ResourceId), request.DeleteOperation == DeleteOperation.PurgeHistory, request.AllowPartialSuccess, innerCt)); parallelBag.Add((item.Resource.ResourceTypeName, item.Resource.ResourceId, item.SearchEntryMode == ValueSets.SearchEntryMode.Include)); }); - await Parallel.ForEachAsync(matchedResources, cancellationToken, async (item, innerCt) => + await Parallel.ForEachAsync(matchedResources.Where(_ => _.Resource.ResourceTypeName != KnownResourceTypes.SearchParameter), cancellationToken, async (item, innerCt) => { - await DeleteSearchParameterAsync(item.Resource, cancellationToken); await _retryPolicy.ExecuteAsync(async () => await fhirDataStore.HardDeleteAsync(new ResourceKey(item.Resource.ResourceTypeName, item.Resource.ResourceId), request.DeleteOperation == DeleteOperation.PurgeHistory, request.AllowPartialSuccess, innerCt)); parallelBag.Add((item.Resource.ResourceTypeName, item.Resource.ResourceId, item.SearchEntryMode == ValueSets.SearchEntryMode.Include)); }); + + // With concurrency based on max last updated search params must be deleted one-by-one. + foreach (var item in matchedResources.Where(_ => _.Resource.ResourceTypeName == KnownResourceTypes.SearchParameter)) + { + await DeleteSearchParameterWithLockAsync(item, cancellationToken); + await _retryPolicy.ExecuteAsync(async () => await fhirDataStore.HardDeleteAsync(new ResourceKey(item.Resource.ResourceTypeName, item.Resource.ResourceId), request.DeleteOperation == DeleteOperation.PurgeHistory, request.AllowPartialSuccess, cancellationToken)); + parallelBag.Add((item.Resource.ResourceTypeName, item.Resource.ResourceId, item.SearchEntryMode == ValueSets.SearchEntryMode.Include)); + } } catch (Exception ex) { @@ -638,22 +646,48 @@ private bool IsIncludeEnabled() return _configuration.SupportsIncludes && (_fhirRuntimeConfiguration.DataStore?.Equals(KnownDataStores.SqlServer, StringComparison.OrdinalIgnoreCase) ?? false); } - private async Task DeleteSearchParametersAsync(IEnumerable resources, CancellationToken cancellationToken) + private async Task DeleteSearchParametersAsync(IEnumerable entries, CancellationToken cancellationToken) { - if (resources?.Any() ?? false) + foreach (var entry in entries.Where(_ => _.Resource.ResourceTypeName == KnownResourceTypes.SearchParameter)) { - foreach (var resource in resources.Where(x => string.Equals(x?.ResourceTypeName, KnownResourceTypes.SearchParameter, StringComparison.OrdinalIgnoreCase))) - { - await _searchParameterOperations.DeleteSearchParameterAsync(resource.RawResource, cancellationToken, true); - } + await DeleteSearchParameterWithLockAsync(entry, cancellationToken); } } - private async Task DeleteSearchParameterAsync(ResourceWrapper resource, CancellationToken cancellationToken) + private async Task DeleteSearchParameterWithLockAsync(SearchResultEntry item, CancellationToken cancellationToken) { - if (string.Equals(resource?.ResourceTypeName, KnownResourceTypes.SearchParameter, StringComparison.OrdinalIgnoreCase)) + await _searchParamDeleteSemaphore.WaitAsync(cancellationToken); + try { - await _searchParameterOperations.DeleteSearchParameterAsync(resource.RawResource, cancellationToken, true); + await SearchParameterRetry.ExecuteAsync( + async () => + { + await _searchParameterOperations.DeleteSearchParameterAsync(item.Resource.RawResource, cancellationToken, true); + }, + "Deletion"); + } + finally + { + _searchParamDeleteSemaphore.Release(); + } + } + + public void Dispose() + { + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (!_disposed) + { + if (disposing) + { + _searchParamDeleteSemaphore?.Dispose(); + } + + _disposed = true; } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/ISearchParameterValidator.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/ISearchParameterValidator.cs index 1980e3e004..d2db9d2525 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/ISearchParameterValidator.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/ISearchParameterValidator.cs @@ -3,13 +3,15 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System; using System.Threading; +using System.Threading.Tasks; using Hl7.Fhir.Model; namespace Microsoft.Health.Fhir.Shared.Core.Features.Search.Parameters { public interface ISearchParameterValidator { - System.Threading.Tasks.Task ValidateSearchParameterInput(SearchParameter searchParam, string method, CancellationToken cancellationToken); + Task ValidateSearchParameterInput(SearchParameter searchParam, string method, CancellationToken cancellationToken, DateTimeOffset? lastUpdated = null); } } diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterValidator.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterValidator.cs index 4b668833e5..c656183392 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterValidator.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterValidator.cs @@ -8,6 +8,7 @@ using System.Linq; using System.Reflection.Metadata; using System.Threading; +using System.Threading.Tasks; using EnsureThat; using FluentValidation.Results; using Hl7.Fhir.Model; @@ -68,7 +69,7 @@ public SearchParameterValidator( _logger = EnsureArg.IsNotNull(logger, nameof(logger)); } - public async Task ValidateSearchParameterInput(SearchParameter searchParam, string method, CancellationToken cancellationToken) + public async Task ValidateSearchParameterInput(SearchParameter searchParam, string method, CancellationToken cancellationToken, DateTimeOffset? lastUpdated = null) { await _authorizationService.CheckAccess(DataActions.Reindex, true, cancellationToken); @@ -77,7 +78,7 @@ public async Task ValidateSearchParameterInput(SearchParameter searchParam, stri if (string.IsNullOrEmpty(searchParam.Url) && (method.Equals(HttpDeleteName, StringComparison.Ordinal) || method.Equals(HttpPatchName, StringComparison.Ordinal))) { // Return out if this is delete OR patch call and no Url so FHIRController can move to next action - return; + return null; } var validationFailures = new List(); @@ -108,7 +109,11 @@ public async Task ValidateSearchParameterInput(SearchParameter searchParam, stri else { // Refresh the search parameter cache in the search parameter definition manager before starting the validation. - await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken); + if (!lastUpdated.HasValue) + { + await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken); + lastUpdated = _searchParameterOperations.SearchParamLastUpdated; + } // If a search parameter with the same uri exists already if (_searchParameterDefinitionManager.TryGetSearchParameter(searchParam.Url, out var searchParameterInfo)) @@ -159,6 +164,8 @@ public async Task ValidateSearchParameterInput(SearchParameter searchParam, stri { throw new ResourceNotValidException(validationFailures); } + + return lastUpdated.Value; // value should not be null here. } private void CheckForConflictingCodeValue(SearchParameter searchParam, List validationFailures) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/ExceptionExtension.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/ExceptionExtension.cs index ed80757b3d..d907d13e2d 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/ExceptionExtension.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/ExceptionExtension.cs @@ -3,6 +3,7 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- using System; +using Microsoft.Data.SqlClient; namespace Microsoft.Health.Fhir.SqlServer.Features { @@ -25,6 +26,15 @@ public static bool IsExecutionTimeout(this Exception e) return str.Contains("execution timeout expired", StringComparison.OrdinalIgnoreCase); } + public static bool IsSearchParameterConcurrencyConflict(this Exception e) + { + var sqlEx = e as SqlException; + return sqlEx != null + && sqlEx.Number == 50001 + && e.Message.StartsWith("optimistic concurrency conflict", StringComparison.OrdinalIgnoreCase) + && e.Message.Contains("expected last updated", StringComparison.OrdinalIgnoreCase); + } + private static bool HasDeadlockErrorPattern(string str) { return str.Contains("deadlock", StringComparison.OrdinalIgnoreCase); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs index 6f74747256..712b985e4d 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs @@ -7,9 +7,9 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Schema { public static class SchemaVersionConstants { - public const int Min = (int)SchemaVersion.V103; + public const int Min = (int)SchemaVersion.V112; public const int Max = (int)SchemaVersion.V113; - public const int MinForUpgrade = (int)SchemaVersion.V103; // this is used for upgrade tests only + public const int MinForUpgrade = (int)SchemaVersion.V110; // this is used for upgrade tests only public const int SearchParameterStatusSchemaVersion = (int)SchemaVersion.V6; public const int SupportForReferencesWithMissingTypeVersion = (int)SchemaVersion.V7; public const int SearchParameterHashSchemaVersion = (int)SchemaVersion.V8; diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/Registry/SqlServerSearchParameterStatusDataStore.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/Registry/SqlServerSearchParameterStatusDataStore.cs index 3c0640c3f9..3026b2a430 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/Registry/SqlServerSearchParameterStatusDataStore.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/Registry/SqlServerSearchParameterStatusDataStore.cs @@ -12,8 +12,10 @@ using EnsureThat; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; +using Microsoft.Health.Fhir.Core.Exceptions; using Microsoft.Health.Fhir.Core.Features.Definition; using Microsoft.Health.Fhir.Core.Features.Operations; +using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Search; using Microsoft.Health.Fhir.Core.Features.Search.Registry; using Microsoft.Health.Fhir.Core.Models; @@ -139,64 +141,21 @@ public async Task UpsertStatuses(IReadOnlyCollection statuses, int maxRetries, CancellationToken cancellationToken, long? reindexId = null) - { - var currentStatuses = statuses.ToList(); - int retryCount = 0; - - while (retryCount <= maxRetries) - { - try - { - await UpsertStatusesInternal(currentStatuses, cancellationToken, reindexId); - return; // Success - } - catch (SqlException sqlEx) when (sqlEx.Number == 50001 && retryCount < maxRetries) // Our custom concurrency error - { - // Optimistic concurrency conflict detected - refresh and retry - retryCount++; - _logger.LogWarning("Optimistic concurrency conflict detected on attempt {RetryCount}. Retrying...", retryCount); - - // Refresh the statuses with current LastUpdated values - var refreshedStatuses = await GetSearchParameterStatuses(cancellationToken); - var refreshedDict = refreshedStatuses.ToDictionary(s => s.Uri.OriginalString, s => s); - - // Update our statuses with fresh LastUpdated values - foreach (var status in currentStatuses) - { - if (refreshedDict.TryGetValue(status.Uri.OriginalString, out var refreshed)) - { - status.LastUpdated = refreshed.LastUpdated; - } - } - - // Wait before retry to reduce contention - await Task.Delay(TimeSpan.FromMilliseconds(100.0 * retryCount), cancellationToken); - } - catch (SqlException sqlEx) when (sqlEx.Number == 50001) - { - // Max retries exceeded - throw new SearchParameterConcurrencyException("Maximum retry attempts exceeded due to concurrency conflicts", sqlEx); - } - } - } - - private async Task UpsertStatusesInternal(IReadOnlyCollection statuses, CancellationToken cancellationToken, long? reindexId = null) - { using var cmd = new SqlCommand(); cmd.CommandType = CommandType.StoredProcedure; cmd.CommandText = "dbo.MergeSearchParams"; - if (_schemaInformation.Current >= 112 && reindexId.HasValue) // remove value check to invoke new max(LastUpdated) logic - { - cmd.Parameters.AddWithValue("@ReindexId", reindexId ?? 0); - } - + cmd.Parameters.AddWithValue("@ReindexId", reindexId ?? 0); new SearchParamListTableValuedParameterDefinition("@SearchParams").AddParameter(cmd.Parameters, new SearchParamListRowGenerator().GenerateRows(statuses.ToList())); - await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, cancellationToken); + try + { + await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, cancellationToken); + } + catch (SqlException ex) when (ex.IsSearchParameterConcurrencyConflict()) + { + _logger.LogWarning(ex, $"Optimistic concurrency conflict occurred while calling dbo.MergeSearchParams. ReindexId={reindexId ?? 0}"); + throw new BadRequestException(Core.Resources.SearchParameterConcurrencyConflict); + } } // Synchronize the FHIR model dictionary with the data in SQL search parameter status table diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs index f16fe88d87..a912672151 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs @@ -212,6 +212,11 @@ public async Task MergeAsync(IReadOnlyList 0; - if (hasPendingStatuses && _schemaInformation.Current >= 109) + if (hasPendingStatuses) { cmd.CommandText = "dbo.MergeResourcesAndSearchParams"; new SearchParamListTableValuedParameterDefinition("@SearchParams").AddParameter(cmd.Parameters, new SearchParamListRowGenerator().GenerateRows(pendingStatuses.ToList())); + cmd.Parameters.AddWithValue("@ReindexId", 0); } else { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs index a6eaa5435e..4afa4b0d80 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs @@ -13,9 +13,12 @@ using System.Threading; using System.Threading.Tasks; using Hl7.Fhir.Model; +using Microsoft.AspNetCore.Server.HttpSys; +using Microsoft.CodeAnalysis; using Microsoft.Health.Extensions.Xunit; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Core.Features.Operations; +using Microsoft.Health.Fhir.Core.Features.Search.Parameters; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Microsoft.Health.Test.Utilities; @@ -24,6 +27,7 @@ using Xunit; using Xunit.Abstractions; using static Hl7.Fhir.Model.Bundle; +using static Microsoft.EntityFrameworkCore.DbLoggerCategory; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Reindex @@ -69,10 +73,185 @@ public Task DisposeAsync() return Task.CompletedTask; } + [Fact] + public async Task GivenSequentialBundleSearchParamCreates_ShouldSuceed() + { + const int numberOfSearchParams = 10; + const string urlPrefix = "http://my.org/"; + var codes = new List(); + try + { + for (var i = 0; i < numberOfSearchParams; i++) + { + var code = $"c-id-{i}"; + codes.Add(code); + } + + var bundle = await CreatePersonSearchParamsAsync(); + Assert.Equal(numberOfSearchParams, bundle.Entry.Count); + foreach (var entry in bundle.Entry) + { + Assert.True(entry.Resource as SearchParameter != null, $"actual={JsonConvert.SerializeObject(entry)}"); + } + } + finally + { + await DeleteSearchParamsAsync(codes); + } + + async Task CreatePersonSearchParamsAsync() + { + var bundle = new Bundle { Type = Bundle.BundleType.Batch, Entry = new List() }; + +#if R5 + var resourceTypes = new List([Enum.Parse("Person")]); +#else + var resourceTypes = new List([Enum.Parse("Person")]); +#endif + + foreach (var code in codes) + { + var searchParam = new SearchParameter + { + Id = code, + Url = $"{urlPrefix}{code}", + Name = code, + Code = code, + Status = PublicationStatus.Active, + Type = SearchParamType.Token, + Expression = "Person.id", + Description = "any", + Base = resourceTypes, + }; + + bundle.Entry.Add(new EntryComponent { Request = new RequestComponent { Method = Bundle.HTTPVerb.PUT, Url = $"SearchParameter/{code}" }, Resource = searchParam }); + } + + var result = await _fixture.TestFhirClient.PostBundleAsync(bundle, new FhirBundleOptions { BundleProcessingLogic = FhirBundleProcessingLogic.Sequential }); + return result; + } + } + + [Theory] + [InlineData(false, false)] // single creats + [InlineData(true, false)] // batch bundle + [InlineData(true, true)] // parallel batch bundle + public async Task GivenConcurrentSearchParamCreates_SomeShouldFail(bool isBundle, bool isParallel) + { + const string urlPrefix = "http://my.org/"; + var codes = new List(); + try + { + var threw = false; + await Parallel.ForAsync(0, 20, new ParallelOptions { MaxDegreeOfParallelism = 4 }, async (i, ct) => + { + var code = $"c-id-{i}"; + lock (codes) + { + codes.Add(code); + } + + try + { + var result = await CreatePersonSearchParamAsync(code, isBundle, isParallel); + if (isBundle && result is FhirResponse bundleResponse) + { + Assert.Single(bundleResponse.Resource.Entry); + + var entry = bundleResponse.Resource.Entry[0]; + if (entry.Response != null && entry.Response.Status.StartsWith("4")) + { + if (entry.Response.Outcome is OperationOutcome outcome) + { + var diagnostics = outcome.Issue?.FirstOrDefault()?.Diagnostics; + _output.WriteLine($"Param={code}. Diagnostics={diagnostics}"); + var expected = $"{Core.Resources.SearchParameterConcurrencyConflict}{(isParallel ? string.Empty : " Update.3")}"; + Assert.True(diagnostics == expected, $"Expected={expected} Actual={diagnostics}"); + threw = true; + + lock (codes) + { + codes.Remove(code); + } + + return; + } + } + } + + _output.WriteLine($"Created search param = {code}"); + } + catch (FhirClientException ex) + { + _output.WriteLine($"Param={code}. StatusCode={ex.StatusCode}, Error={ex.Message}"); + + if (ex.StatusCode != HttpStatusCode.InternalServerError) // this can happen because of short wait limit to accquire "lock" in "get and apply" code. testing only. + { + Assert.Equal(HttpStatusCode.BadRequest, ex.StatusCode); + var expected = $"BadRequest: {Core.Resources.SearchParameterConcurrencyConflict} {(isBundle ? "Update.3" : "Create.3")}"; + Assert.True(ex.Message.StartsWith(expected), $"Expected={expected} Actual={ex.Message}"); + threw = true; + } + + lock (codes) + { + codes.Remove(code); + } + } + }); + + Assert.True(threw || !_isSql, "Expected at least one create to fail due to concurrency for SQL only."); + } + finally + { + await DeleteSearchParamsAsync(codes); + } + + async Task CreatePersonSearchParamAsync(string code, bool isBundle, bool isParal) + { + var bundle = new Bundle { Type = Bundle.BundleType.Batch, Entry = new List() }; +#if R5 + var resourceTypes = new List([Enum.Parse("Person")]); +#else + var resourceTypes = new List([Enum.Parse("Person")]); +#endif + + var searchParam = new SearchParameter + { + Id = code, + Url = $"{urlPrefix}{code}", + Name = code, + Code = code, + Status = PublicationStatus.Active, + Type = SearchParamType.Token, + Expression = "Person.id", + Description = "any", + Base = resourceTypes, + }; + + if (isBundle) + { + bundle.Entry.Add(new EntryComponent { Request = new RequestComponent { Method = Bundle.HTTPVerb.PUT, Url = $"SearchParameter/{code}" }, Resource = searchParam }); + var result = await _fixture.TestFhirClient.PostBundleAsync(bundle, new FhirBundleOptions { BundleProcessingLogic = isParal ? FhirBundleProcessingLogic.Parallel : FhirBundleProcessingLogic.Sequential }); + return result; + } + else + { + var result = await _fixture.TestFhirClient.CreateAsync(searchParam); + return result; + } + } + } + [Fact] public async Task Given500SearchParams_WhenReindexCompletes_ThenSearchParamsAreEnabled() { - const int numberOfSearchParams = 10; // increase to 500 when cache is not updated by API calls and status is saved with resources in a single SQL transaction + if (!_isSql) // max(lastUpdated) works only for SQL. For Cosmos - NOOP. + { + return; + } + + const int numberOfSearchParams = 500; // increase to 500 when cache is not updated by API calls and status is saved with resources in a single SQL transaction const string urlPrefix = "http://my.org/"; var codes = new List(); try @@ -91,8 +270,17 @@ public async Task Given500SearchParams_WhenReindexCompletes_ThenSearchParamsAreE } // check by urls - var search = await _fixture.TestFhirClient.SearchAsync($"SearchParameter?_summary=count&url={string.Join(",", codes.Select(_ => $"{urlPrefix}{_}"))}"); - Assert.True(search.Resource.Total == numberOfSearchParams, $"Urls expected={numberOfSearchParams} actual={search.Resource.Total}"); + // code works locally for all 500, but in PR it throws - FhirClientException : RequestUriTooLong (NO_FHIR_ACTIVITY_ID_FOR_THIS_TRANSACTION) + var total = 0; + var chunk = numberOfSearchParams / 10; // assumes there is no remainder. + for (var i = 0; i < 10; i++) + { + var urls = string.Join(",", codes.Skip(i * chunk).Take(chunk).Select(_ => $"{urlPrefix}{_}")); + var search = await _fixture.TestFhirClient.SearchAsync($"SearchParameter?_summary=count&url={urls}"); + total += search.Resource.Total.Value; + } + + Assert.True(total == numberOfSearchParams, $"Urls: expected={numberOfSearchParams} actual={total}"); var reindex = await _fixture.TestFhirClient.PostReindexJobAsync(new Parameters { Parameter = [] }); Assert.Equal(HttpStatusCode.Created, reindex.reponse.Response.StatusCode); @@ -114,11 +302,9 @@ async Task CreatePersonSearchParamsAsync() var bundle = new Bundle { Type = Bundle.BundleType.Batch, Entry = new List() }; #if R5 - var resourceTypes = new List(); - resourceTypes.Add(Enum.Parse("Person")); + var resourceTypes = new List([Enum.Parse("Person")]); #else - var resourceTypes = new List(); - resourceTypes.Add(Enum.Parse("Person")); + var resourceTypes = new List([Enum.Parse("Person")]); #endif foreach (var code in codes) @@ -215,6 +401,64 @@ public async Task GivenTwoSearchParamsWithCodeConflictOnDerived_ThenBadRequestIs } } + [Fact] + public async Task GivenTwoSearchParamsInSequentialBatchBundle_ThenBothCreated() + { +#if R5 + var personTypes = new List() { VersionIndependentResourceTypesAll.Person }; + var supplyDeliveryTypes = new List() { VersionIndependentResourceTypesAll.SupplyDelivery }; +#else + var personTypes = new List() { ResourceType.Person }; + var supplyDeliveryTypes = new List() { ResourceType.SupplyDelivery }; +#endif + const string urlPrefix = "http://my.org/"; + var ids = new List { "c-id-1", "c-id-2" }; + try + { + var bundle = new Bundle { Type = Bundle.BundleType.Batch, Entry = [] }; + + var id = ids[0]; + var searchParam = new SearchParameter + { + Id = id, + Url = $"{urlPrefix}c-1", + Name = id, + Code = id, + Status = PublicationStatus.Active, + Type = SearchParamType.Token, + Expression = "Person.id", + Description = "any", + Base = personTypes, + }; + + bundle.Entry.Add(new EntryComponent { Request = new RequestComponent { Method = Bundle.HTTPVerb.PUT, Url = $"SearchParameter/{id}" }, Resource = searchParam }); + + id = ids[1]; + searchParam = new SearchParameter + { + Id = id, + Url = $"{urlPrefix}c-2", + Name = id, + Code = id, + Status = PublicationStatus.Active, + Type = SearchParamType.Token, + Expression = "SupplyDelivery.id", + Description = "any", + Base = supplyDeliveryTypes, + }; + + bundle.Entry.Add(new EntryComponent { Request = new RequestComponent { Method = Bundle.HTTPVerb.PUT, Url = $"SearchParameter/{id}" }, Resource = searchParam }); + + var response = await _fixture.TestFhirClient.PostBundleAsync(bundle, new FhirBundleOptions { BundleProcessingLogic = FhirBundleProcessingLogic.Sequential }); + Assert.Equal(2, response.Resource.Entry.Count); + Assert.All(response.Resource.Entry, _ => Assert.NotNull(_.Resource as SearchParameter)); + } + finally + { + await DeleteSearchParamsAsync(ids); + } + } + [Fact] public async Task GivenTwoSearchParamsForDifferentResourceTypesUsingSameCode_ThenBothCreated() { @@ -282,8 +526,9 @@ public async Task GivenTwoSearchParamsForDifferentResourceTypesUsingSameCode_The [InlineData(true, false, true, true)] [InlineData(true, false, false, true)] [InlineData(false, true, true, true)] - [InlineData(false, true, false, true)] - [InlineData(true, true, false, true)] + //// https://microsofthealth.visualstudio.com/Health/_workitems/edit/187119 + ////[InlineData(false, true, false, true)] // this creates 2 resources for the same url. after fixing this bug - uncomment. + ////[InlineData(true, true, false, true)] // this creates 2 resources for the same url. after fixing this bug - uncomment. [InlineData(true, true, false, false)] [InlineData(true, true, true, true)] [InlineData(true, true, true, false)] @@ -388,12 +633,13 @@ async Task CreatePersonSearchParamsAsync() private async Task DeleteSearchParamsAsync(List ids) { + var bundle = new Bundle { Type = Bundle.BundleType.Batch, Entry = new List() }; foreach (var id in ids) { - var bundle = new Bundle { Type = Bundle.BundleType.Batch, Entry = new List() }; bundle.Entry.Add(new EntryComponent { Request = new RequestComponent { Method = Bundle.HTTPVerb.DELETE, Url = $"SearchParameter/{id}" } }); - await _fixture.TestFhirClient.PostBundleAsync(bundle, new FhirBundleOptions { BundleProcessingLogic = FhirBundleProcessingLogic.Parallel }); } + + await _fixture.TestFhirClient.PostBundleAsync(bundle, new FhirBundleOptions { BundleProcessingLogic = FhirBundleProcessingLogic.Parallel }); } [Fact] diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/FailingSearchParameterStatusManager.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/FailingSearchParameterStatusManager.cs index 99143e9d1d..819f5fad86 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/FailingSearchParameterStatusManager.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/FailingSearchParameterStatusManager.cs @@ -31,7 +31,7 @@ public Task Handle(SearchParameterDefinitionManagerInitialized notification, Can public Task> GetAllSearchParameterStatus(CancellationToken cancellationToken) => Task.FromResult>(Array.Empty()); - public Task UpdateSearchParameterStatusAsync(IReadOnlyCollection searchParameterUris, SearchParameterStatus status, CancellationToken cancellationToken, bool ignoreSearchParameterNotSupportedException = false, long? reindexId = null) => Task.CompletedTask; + public Task UpdateSearchParameterStatusAsync(IReadOnlyCollection searchParameterUris, SearchParameterStatus status, CancellationToken cancellationToken, bool ignoreSearchParameterNotSupportedException = false, long? reindexId = null, DateTimeOffset? lastUpdated = null) => Task.CompletedTask; public Task CheckCacheConsistencyAsync(DateTime updateEventsSince, DateTime activeHostsSince, CancellationToken cancellationToken) => Task.FromResult(new CacheConsistencyResult()); } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs index bfe360f156..316b314dfa 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs @@ -328,7 +328,7 @@ private void StartCacheUpdateTask(CancellationToken cancellationToken) { try { - await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken); + await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken, true); } catch (Exception) { @@ -542,7 +542,9 @@ public async Task GivenNoMatchingResources_WhenRunningReindexJob_ThenJobIsComple #else var searchParam = _supportedSearchParameterDefinitionManager.GetSearchParameter("http://hl7.org/fhir/SearchParameter/CanonicalResource-name"); #endif - await _searchParameterStatusManager.UpdateSearchParameterStatusAsync(new List() { searchParam.Url.ToString() }, SearchParameterStatus.Supported, default); + await _searchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); + + await _searchParameterStatusManager.UpdateSearchParameterStatusAsync(new List() { searchParam.Url.ToString() }, SearchParameterStatus.Supported, default, lastUpdated: _searchParameterOperations.SearchParamLastUpdated); var request = new CreateReindexRequest(new List(), new List()); CreateReindexResponse response = await SetUpForReindexing(request); @@ -1240,6 +1242,8 @@ public async Task GivenSurrogateRangeFetchOom_WhenProcessingJobRuns_ThenSplitUse _searchParameterStatusManager, NullLogger.Instance); + await _searchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); + string resultJson = await processingJob.ExecuteAsync(jobInfo, CancellationToken.None); var result = JsonConvert.DeserializeObject(resultJson); @@ -1372,42 +1376,11 @@ private async Task CreateSearchParam(string searchParamName, Se await _fixture.Mediator.UpsertResourceAsync(searchParam.ToResourceElement()); - if (!_searchParameterDefinitionManager.TryGetSearchParameter(searchParam.Url, out _)) - { - _searchParameterDefinitionManager.AddNewSearchParameters(new List { searchParam.ToTypedElement() }); - } - - await _searchParameterStatusManager.UpdateSearchParameterStatusAsync( - new List { searchParam.Url }, - SearchParameterStatus.Supported, - CancellationToken.None); - - // These tests start a background cache refresh loop, so a zero-wait refresh can - // skip the one apply that populates the SQL search-parameter URI->ID mapping. await _searchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); return searchParam; } - private async Task SearchForSearchParameterByUrlAsync(string searchParamUrl, CancellationToken cancellationToken, int maxAttempts = 10, int delayMilliseconds = 200) - { - SearchResult result = null; - var queryParams = new List> { new("url", searchParamUrl) }; - - for (int attempt = 0; attempt < maxAttempts; attempt++) - { - result = await _searchService.Value.SearchAsync(KnownResourceTypes.SearchParameter, queryParams, cancellationToken); - if (result.Results.Any()) - { - return result; - } - - await Task.Delay(delayMilliseconds, cancellationToken); - } - - return result; - } - private ResourceWrapper CreatePatientResourceWrapper(string patientName, string patientId) { Patient patientResource = Samples.GetDefaultPatient().ToPoco(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchTests.cs index bba5a798f3..b7b0fc03ce 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchTests.cs @@ -1051,7 +1051,10 @@ private void ConfigureFhirRequestContext( accessControlContext.AllowedResourceActions.Add(scope); } - contextAccessor.RequestContext.AccessControlContext.Returns(accessControlContext); + var fhirRequestContext = Substitute.For(); + fhirRequestContext.AccessControlContext.Returns(accessControlContext); + + contextAccessor.RequestContext.Returns(fhirRequestContext); } // SMART v2 Granular Scope Tests diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems index 58ef46d699..46544a2f53 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems @@ -28,7 +28,7 @@ - + diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTests.cs index 6f71076443..b791d5e5c5 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTests.cs @@ -1201,10 +1201,12 @@ private async Task CreatePatientSearchParam(string searchParamN Code = searchParamName, }; - _searchParameterDefinitionManager.AddNewSearchParameters(new List { searchParam.ToTypedElement() }); + _searchParameterDefinitionManager.AddNewSearchParameters([searchParam.ToTypedElement()]); + + await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(); // Add the search parameter to the datastore - await _fixture.SearchParameterStatusManager.UpdateSearchParameterStatusAsync(new List { searchParam.Url }, SearchParameterStatus.Supported, CancellationToken.None); + await _fixture.SearchParameterStatusManager.UpdateSearchParameterStatusAsync([searchParam.Url], SearchParameterStatus.Supported, CancellationToken.None, lastUpdated: _fixture.SearchParameterOperations.SearchParamLastUpdated); await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(); @@ -1220,12 +1222,6 @@ private ResourceElement CreatePatientResourceElement(string patientName, string return Deserializers.ResourceDeserializer.DeserializeRaw(rawResource, "v1", DateTimeOffset.UtcNow); } - private async Task ExecuteAndVerifyException(Func action) - where TException : Exception - { - await Assert.ThrowsAsync(action); - } - private async Task SetAllowCreateForOperation(bool allowCreate, Func operation) { var observation = _capabilityStatement.Rest[0].Resource.Find(r => ResourceType.Observation.EqualsString(r.Type.ToString())); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs index 64b79666a2..68738bbcde 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs @@ -20,6 +20,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; +using Microsoft.Extensions.Primitives; using Microsoft.Health.Abstractions.Features.Transactions; using Microsoft.Health.Core.Features.Context; using Microsoft.Health.Extensions.DependencyInjection; @@ -52,6 +53,7 @@ using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Core.Registration; using Microsoft.Health.Fhir.Core.UnitTests.Extensions; +using Microsoft.Health.Fhir.Core.UnitTests.Features.Context; using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Operations; using Microsoft.Health.Fhir.SqlServer.Features.Storage; using Microsoft.Health.Fhir.Tests.Common; @@ -195,6 +197,16 @@ public async Task InitializeAsync() await asyncLifetime.InitializeAsync(); } + // Initialize FhirRequestContext to ensure pending status updates are captured + // This needs to be here (like ResourceIdProvider) because it uses AsyncLocal + FhirRequestContextAccessor.RequestContext = new DefaultFhirRequestContext + { + BaseUri = new Uri("http://localhost/"), + CorrelationId = Guid.NewGuid().ToString(), + RequestHeaders = new Dictionary(), + ResponseHeaders = new Dictionary(), + }; + CapabilityStatement = CapabilityStatementMock.GetMockedCapabilityStatement(); IDeletionServiceDataStoreFactory deletionServiceDataStoreFactory = Substitute.For(); @@ -267,6 +279,7 @@ public async Task InitializeAsync() var collection = new ServiceCollection(); + // Register request handlers collection.AddSingleton(typeof(IRequestHandler), new CreateResourceHandler(DataStore, new Lazy(() => ConformanceProvider), resourceWrapperFactory, _resourceIdProvider, new ResourceReferenceResolver(SearchService, new TestQueryStringParser(), Substitute.For>()), DisabledFhirAuthorizationService.Instance)); collection.AddSingleton(typeof(IRequestHandler), new UpsertResourceHandler(DataStore, new Lazy(() => ConformanceProvider), resourceWrapperFactory, _resourceIdProvider, new ResourceReferenceResolver(SearchService, new TestQueryStringParser(), Substitute.For>()), FhirRequestContextAccessor, DisabledFhirAuthorizationService.Instance, ModelInfoProvider.Instance)); collection.AddSingleton(typeof(IRequestHandler), GetResourceHandler); @@ -274,18 +287,54 @@ public async Task InitializeAsync() collection.AddSingleton(typeof(IRequestHandler), new SearchResourceHistoryHandler(SearchService, bundleFactory, DisabledFhirAuthorizationService.Instance, new DataResourceFilter(MissingDataFilterCriteria.Default))); collection.AddSingleton(typeof(IRequestHandler), new SearchResourceHandler(SearchService, bundleFactory, DisabledFhirAuthorizationService.Instance, new DataResourceFilter(MissingDataFilterCriteria.Default))); - ServiceProvider services = collection.BuildServiceProvider(); + var searchParameterSupportResolver = Substitute.For(); + searchParameterSupportResolver.IsSearchParameterSupported(Arg.Any()).Returns((true, false)); + + var dataStoreSearchParameterValidator = Substitute.For(); + dataStoreSearchParameterValidator.ValidateSearchParameter(Arg.Any(), out Arg.Any()).Returns(x => + { + x[1] = null; // out parameter for errorMessage + return true; + }); _searchParameterOperations = new SearchParameterOperations( SearchParameterStatusManager, SearchParameterDefinitionManager, ModelInfoProvider.Instance, - Substitute.For(), - Substitute.For(), - () => Substitute.For>(), - () => Substitute.For>(), + searchParameterSupportResolver, + dataStoreSearchParameterValidator, + () => OperationDataStore.CreateMockScope(), + () => SearchService.CreateMockScope(), NullLogger.Instance); + // Register pipeline behaviors for search parameter handling + collection.AddTransient>( + sp => new CreateOrUpdateSearchParameterBehavior( + _searchParameterOperations, + DataStore, + SearchParameterDefinitionManager, + FhirRequestContextAccessor, + ModelInfoProvider.Instance)); + + collection.AddTransient>( + sp => new CreateOrUpdateSearchParameterBehavior( + _searchParameterOperations, + DataStore, + SearchParameterDefinitionManager, + FhirRequestContextAccessor, + ModelInfoProvider.Instance)); + + collection.AddTransient>( + sp => new DeleteSearchParameterBehavior( + _searchParameterOperations, + DataStore, + SearchParameterDefinitionManager, + SearchParameterStatusManager, + FhirRequestContextAccessor, + ModelInfoProvider.Instance)); + + ServiceProvider services = collection.BuildServiceProvider(); + Mediator = new Mediator(services); } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SearchParameterOptimisticConcurrencyIntegrationTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SearchParameterOptimisticConcurrencyIntegrationTests.cs deleted file mode 100644 index fe573ac987..0000000000 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SearchParameterOptimisticConcurrencyIntegrationTests.cs +++ /dev/null @@ -1,454 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Health.Extensions.Xunit; -using Microsoft.Health.Fhir.Core.Features.Search.Registry; -using Microsoft.Health.Fhir.SqlServer.Features.Schema; -using Microsoft.Health.Fhir.SqlServer.Features.Storage.Registry; -using Microsoft.Health.Fhir.Tests.Common; -using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; -using Microsoft.Health.Test.Utilities; -using Xunit; - -namespace Microsoft.Health.Fhir.Tests.Integration.Persistence -{ - [Trait(Traits.OwningTeam, OwningTeam.Fhir)] - [Trait(Traits.Category, Categories.Search)] - [FhirStorageTestsFixtureArgumentSets(DataStore.SqlServer)] - public class SearchParameterOptimisticConcurrencyIntegrationTests : IClassFixture - { - private readonly FhirStorageTestsFixture _fixture; - private readonly IFhirStorageTestHelper _testHelper; - - public SearchParameterOptimisticConcurrencyIntegrationTests(FhirStorageTestsFixture fixture) - { - _fixture = fixture; - _testHelper = fixture.TestHelper; - } - - [Fact] - public async Task GivenSchemaVersion94OrHigher_WhenGettingSearchParameterStatuses_ThenLastUpdatedIsReturned() - { - // Act - var statuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); - - // Assert - Assert.NotEmpty(statuses); - - // Verify that all statuses have LastUpdated (all parameters should have LastUpdated values) - foreach (var status in statuses) - { - Assert.True(status.LastUpdated != default(DateTimeOffset), "All search parameter statuses should have valid LastUpdated values"); - } - } - - [Fact] - public async Task GivenNewSearchParameterStatus_WhenUpserting_ThenLastUpdatedIsReturned() - { - // Arrange - var testUri = $"http://test.com/SearchParameter/ConcurrencyTest_{Guid.NewGuid()}"; - var newStatus = new ResourceSearchParameterStatus - { - Uri = new Uri(testUri), - Status = SearchParameterStatus.Disabled, - IsPartiallySupported = false, - LastUpdated = default(DateTimeOffset), // New parameter, no previous LastUpdated - }; - - try - { - // Act - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { newStatus }, CancellationToken.None); - - // Get the upserted status to check LastUpdated was assigned - var allStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); - var upsertedStatus = allStatuses.FirstOrDefault(s => s.Uri.ToString() == testUri); - - // Assert - Assert.NotNull(upsertedStatus); - Assert.True(upsertedStatus.LastUpdated != default(DateTimeOffset), "LastUpdated should be assigned for new parameters"); - } - finally - { - // Cleanup - await _testHelper.DeleteSearchParameterStatusAsync(testUri); - } - } - - [Fact] - public async Task GivenExistingSearchParameterStatus_WhenUpdatingWithCorrectLastUpdated_ThenSucceeds() - { - // Arrange - var testUri = $"http://test.com/SearchParameter/ConcurrencyTest_{Guid.NewGuid()}"; - var initialStatus = new ResourceSearchParameterStatus - { - Uri = new Uri(testUri), - Status = SearchParameterStatus.Disabled, - IsPartiallySupported = false, - }; - - try - { - // Create initial status - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { initialStatus }, CancellationToken.None); - - // Get the created status with its LastUpdated - var allStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); - var createdStatus = allStatuses.First(s => s.Uri.ToString() == testUri); - - // Modify and update with correct LastUpdated - var updatedStatus = new ResourceSearchParameterStatus - { - Uri = createdStatus.Uri, - Status = SearchParameterStatus.Enabled, // Changed status - IsPartiallySupported = true, // Changed partially supported - LastUpdated = createdStatus.LastUpdated, // Use the current LastUpdated - }; - - // Act - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { updatedStatus }, CancellationToken.None); - - // Verify the update succeeded - var updatedAllStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); - var result = updatedAllStatuses.First(s => s.Uri.ToString() == testUri); - - // Assert - Assert.Equal(SearchParameterStatus.Enabled, result.Status); - Assert.True(result.IsPartiallySupported); - Assert.True(result.LastUpdated > createdStatus.LastUpdated, "LastUpdated should change after update"); - } - finally - { - // Cleanup - await _testHelper.DeleteSearchParameterStatusAsync(testUri); - } - } - - [Fact] - public async Task GivenExistingSearchParameterStatus_WhenUpdatingWithIncorrectLastUpdated_ThenEventuallySucceedsWithRetry() - { - // Arrange - var testUri = $"http://test.com/SearchParameter/ConcurrencyTest_{Guid.NewGuid()}"; - var initialStatus = new ResourceSearchParameterStatus - { - Uri = new Uri(testUri), - Status = SearchParameterStatus.Disabled, - IsPartiallySupported = false, - }; - - try - { - // Create initial status - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { initialStatus }, CancellationToken.None); - - // Get the created status with its LastUpdated - var allStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); - var createdStatus = allStatuses.First(s => s.Uri.ToString() == testUri); - - // Make an intermediate update to change the LastUpdated - var intermediateUpdate = new ResourceSearchParameterStatus - { - Uri = createdStatus.Uri, - Status = SearchParameterStatus.Enabled, - IsPartiallySupported = false, - LastUpdated = createdStatus.LastUpdated, - }; - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { intermediateUpdate }, CancellationToken.None); - - // Now try to update with the stale LastUpdated - // The retry mechanism should detect the conflict, refresh the LastUpdated, and succeed - var staleUpdate = new ResourceSearchParameterStatus - { - Uri = createdStatus.Uri, - Status = SearchParameterStatus.Supported, - IsPartiallySupported = true, - LastUpdated = createdStatus.LastUpdated, // This is now stale - }; - - // Act - This should succeed due to retry mechanism - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { staleUpdate }, CancellationToken.None); - - // Assert - Verify the final status shows the update succeeded - var finalStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); - var finalStatus = finalStatuses.First(s => s.Uri.ToString() == testUri); - - // The retry mechanism should have allowed the update to succeed - Assert.Equal(SearchParameterStatus.Supported, finalStatus.Status); - Assert.True(finalStatus.IsPartiallySupported); - } - finally - { - // Cleanup - await _testHelper.DeleteSearchParameterStatusAsync(testUri); - } - } - - [Fact] - public async Task GivenMultipleConsecutiveStaleUpdates_WhenUpdatingSearchParameter_ThenRetryMechanismIsTriggeredMultipleTimes() - { - // Arrange - var testUri = $"http://test.com/SearchParameter/RetryTest_{Guid.NewGuid()}"; - var initialStatus = new ResourceSearchParameterStatus - { - Uri = new Uri(testUri), - Status = SearchParameterStatus.Disabled, - IsPartiallySupported = false, - }; - - try - { - // Create initial status - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { initialStatus }, CancellationToken.None); - - // Get the created status with its LastUpdated - var allStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); - var createdStatus = allStatuses.First(s => s.Uri.ToString() == testUri); - var originalLastUpdated = createdStatus.LastUpdated; - - // Force multiple intermediate updates to create staleness - for (int i = 0; i < 3; i++) - { - var intermediateUpdate = new ResourceSearchParameterStatus - { - Uri = createdStatus.Uri, - Status = SearchParameterStatus.Enabled, - IsPartiallySupported = false, - LastUpdated = createdStatus.LastUpdated, - }; - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { intermediateUpdate }, CancellationToken.None); - - // Refresh status for next iteration - allStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); - createdStatus = allStatuses.First(s => s.Uri.ToString() == testUri); - } - - // Now attempt update with very stale LastUpdated - this should trigger retry - var veryStaleUpdate = new ResourceSearchParameterStatus - { - Uri = createdStatus.Uri, - Status = SearchParameterStatus.Supported, - IsPartiallySupported = true, - LastUpdated = originalLastUpdated, // This is very stale (3 updates behind) - }; - - // Act - This should succeed due to retry mechanism - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { veryStaleUpdate }, CancellationToken.None); - - // Assert - Verify the final status shows the update succeeded despite staleness - var finalStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); - var finalStatus = finalStatuses.First(s => s.Uri.ToString() == testUri); - - Assert.Equal(SearchParameterStatus.Supported, finalStatus.Status); - Assert.True(finalStatus.IsPartiallySupported); - Assert.True(finalStatus.LastUpdated > originalLastUpdated, "LastUpdated should have changed significantly"); - } - finally - { - // Cleanup - await _testHelper.DeleteSearchParameterStatusAsync(testUri); - } - } - - [RetryFact] - public async Task GivenRapidConcurrentUpdates_WhenUsingStaleLastUpdated_ThenRetryMechanismHandlesHighContentionScenario() - { - // Arrange - var testUri = $"http://test.com/SearchParameter/HighContentionTest_{Guid.NewGuid()}"; - var initialStatus = new ResourceSearchParameterStatus - { - Uri = new Uri(testUri), - Status = SearchParameterStatus.Disabled, - IsPartiallySupported = false, - }; - - try - { - // Create initial status - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { initialStatus }, CancellationToken.None); - - // Get the created status with its LastUpdated - var allStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); - var createdStatus = allStatuses.First(s => s.Uri.ToString() == testUri); - var originalLastUpdated = createdStatus.LastUpdated; - - // Create high contention by launching multiple rapid updates - var rapidUpdateTasks = new List(); - for (int i = 0; i < 5; i++) - { - var updateIndex = i; - rapidUpdateTasks.Add(Task.Run(async () => - { - var rapidUpdate = new ResourceSearchParameterStatus - { - Uri = createdStatus.Uri, - Status = SearchParameterStatus.Enabled, - IsPartiallySupported = updateIndex % 2 == 0, - LastUpdated = createdStatus.LastUpdated, // Using same (potentially stale) LastUpdated - }; - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { rapidUpdate }, CancellationToken.None); - })); - } - - // Act - Wait for all rapid updates to complete (some may trigger retries due to contention) - await Task.WhenAll(rapidUpdateTasks); - - // Now attempt one final update with the original (definitely stale) LastUpdated - var finalStaleUpdate = new ResourceSearchParameterStatus - { - Uri = createdStatus.Uri, - Status = SearchParameterStatus.Supported, - IsPartiallySupported = true, - LastUpdated = originalLastUpdated, // This is definitely stale after all the rapid updates - }; - - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { finalStaleUpdate }, CancellationToken.None); - - // Assert - Verify the final update succeeded despite high contention - var finalStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); - var finalStatus = finalStatuses.First(s => s.Uri.ToString() == testUri); - - Assert.Equal(SearchParameterStatus.Supported, finalStatus.Status); - Assert.True(finalStatus.IsPartiallySupported); - Assert.True(finalStatus.LastUpdated > originalLastUpdated, "LastUpdated should have changed after high contention scenario"); - } - finally - { - // Cleanup - await _testHelper.DeleteSearchParameterStatusAsync(testUri); - } - } - - [Fact] - public async Task GivenOptimisticConcurrencyDetection_WhenLastUpdatedChangesBeforeUpdate_ThenRetryMechanismHandlesConflict() - { - // Arrange - var testUri = $"http://test.com/SearchParameter/ConcurrencyTest_{Guid.NewGuid()}"; - var initialStatus = new ResourceSearchParameterStatus - { - Uri = new Uri(testUri), - Status = SearchParameterStatus.Disabled, - IsPartiallySupported = false, - }; - - try - { - // Create initial status - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { initialStatus }, CancellationToken.None); - - // Get the created status with its LastUpdated - var allStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); - var createdStatus = allStatuses.First(s => s.Uri.ToString() == testUri); - var originalLastUpdated = createdStatus.LastUpdated; - - // First, update with current LastUpdated to change it - var intermediateUpdate = new ResourceSearchParameterStatus - { - Uri = createdStatus.Uri, - Status = SearchParameterStatus.Enabled, - IsPartiallySupported = false, - LastUpdated = originalLastUpdated, - }; - - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { intermediateUpdate }, CancellationToken.None); - - // Now try to update with the stale LastUpdated - this should trigger retry mechanism - var staleUpdate = new ResourceSearchParameterStatus - { - Uri = createdStatus.Uri, - Status = SearchParameterStatus.Supported, - IsPartiallySupported = true, - LastUpdated = originalLastUpdated, // This is now stale - }; - - // Act - This should succeed due to retry mechanism - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { staleUpdate }, CancellationToken.None); - - // Verify the final status - var finalStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); - var finalStatus = finalStatuses.First(s => s.Uri.ToString() == testUri); - - // Assert - The update should have succeeded (with retry) - Assert.Equal(SearchParameterStatus.Supported, finalStatus.Status); - Assert.True(finalStatus.IsPartiallySupported); - Assert.True(finalStatus.LastUpdated > originalLastUpdated, "LastUpdated should have changed"); - } - finally - { - // Cleanup - await _testHelper.DeleteSearchParameterStatusAsync(testUri); - } - } - - [Fact] - public async Task GivenMixedUpdatesWithAndWithoutRowVersion_WhenUpserting_ThenBothSucceed() - { - // Arrange - var existingTestUri = $"http://test.com/SearchParameter/Existing_{Guid.NewGuid()}"; - var newTestUri = $"http://test.com/SearchParameter/New_{Guid.NewGuid()}_"; - - var existingStatus = new ResourceSearchParameterStatus - { - Uri = new Uri(existingTestUri), - Status = SearchParameterStatus.Disabled, - IsPartiallySupported = false, - }; - - try - { - // Create existing parameter - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { existingStatus }, CancellationToken.None); - - // Get the created status with its LastUpdated - var allStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); - var createdStatus = allStatuses.First(s => s.Uri.ToString() == existingTestUri); - - // Prepare mixed updates - var updateExisting = new ResourceSearchParameterStatus - { - Uri = createdStatus.Uri, - Status = SearchParameterStatus.Enabled, - IsPartiallySupported = true, - LastUpdated = createdStatus.LastUpdated, // With LastUpdated - }; - - var createNew = new ResourceSearchParameterStatus - { - Uri = new Uri(newTestUri), - Status = SearchParameterStatus.Supported, - IsPartiallySupported = false, - LastUpdated = default(DateTimeOffset), // Without LastUpdated (new parameter) - }; - - // Act - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { updateExisting, createNew }, CancellationToken.None); - - // Verify both operations succeeded - var updatedAllStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); - var updatedExisting = updatedAllStatuses.First(r => r.Uri.ToString() == existingTestUri); - var createdNew = updatedAllStatuses.First(r => r.Uri.ToString() == newTestUri); - - // Assert - Assert.Equal(SearchParameterStatus.Enabled, updatedExisting.Status); - Assert.True(updatedExisting.IsPartiallySupported); - Assert.True(updatedExisting.LastUpdated > createdStatus.LastUpdated, "Updated parameter should have newer LastUpdated"); - - Assert.Equal(SearchParameterStatus.Supported, createdNew.Status); - Assert.False(createdNew.IsPartiallySupported); - Assert.True(createdNew.LastUpdated != default(DateTimeOffset), "New parameter should have valid LastUpdated"); - } - finally - { - // Cleanup - await _testHelper.DeleteSearchParameterStatusAsync(existingTestUri); - await _testHelper.DeleteSearchParameterStatusAsync(newTestUri); - } - } - } -} diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SearchParameterOptimisticConcurrencyTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SearchParameterOptimisticConcurrencyTests.cs new file mode 100644 index 0000000000..5e78194ce6 --- /dev/null +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SearchParameterOptimisticConcurrencyTests.cs @@ -0,0 +1,176 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Data.SqlClient; +using Microsoft.Health.Extensions.Xunit; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Search.Registry; +using Microsoft.Health.Fhir.SqlServer.Features.Schema; +using Microsoft.Health.Fhir.SqlServer.Features.Storage.Registry; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using Microsoft.Health.Test.Utilities; +using Xunit; + +namespace Microsoft.Health.Fhir.Tests.Integration.Persistence +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Search)] + [FhirStorageTestsFixtureArgumentSets(DataStore.SqlServer)] + public class SearchParameterOptimisticConcurrencyTests : IClassFixture + { + private readonly FhirStorageTestsFixture _fixture; + private readonly IFhirStorageTestHelper _testHelper; + + public SearchParameterOptimisticConcurrencyTests(FhirStorageTestsFixture fixture) + { + _fixture = fixture; + _testHelper = fixture.TestHelper; + } + + [Fact] + public async Task GivenNewSearchParameterStatus_WhenUpserting_ThenLastUpdatedIsReturned() + { + await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); + + var testUri = $"http://test.com/SearchParameter/ConcurrencyTest_{Guid.NewGuid()}"; + var newStatus = new ResourceSearchParameterStatus + { + Uri = new Uri(testUri), + Status = SearchParameterStatus.Disabled, + IsPartiallySupported = false, + LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, + }; + + try + { + await _fixture.SearchParameterStatusDataStore.UpsertStatuses([newStatus], CancellationToken.None); + + // Get the upserted status to check LastUpdated was assigned + var allStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); + var upsertedStatus = allStatuses.FirstOrDefault(s => s.Uri.ToString() == testUri); + + Assert.NotNull(upsertedStatus); + Assert.True(upsertedStatus.LastUpdated != default(DateTimeOffset), "LastUpdated should be assigned for new parameters"); + } + finally + { + await _testHelper.DeleteSearchParameterStatusAsync(testUri); + } + } + + [Fact] + public async Task GivenExistingSearchParameterStatus_WhenUpdatingWithCorrectLastUpdated_ThenSucceeds() + { + await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); + + var testUri = $"http://test.com/SearchParameter/ConcurrencyTest_{Guid.NewGuid()}"; + var initialStatus = new ResourceSearchParameterStatus + { + Uri = new Uri(testUri), + Status = SearchParameterStatus.Disabled, + IsPartiallySupported = false, + LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, + }; + + try + { + await _fixture.SearchParameterStatusDataStore.UpsertStatuses([initialStatus], CancellationToken.None); + + // Get the created status with its LastUpdated + var allStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); + var createdStatus = allStatuses.First(s => s.Uri.ToString() == testUri); + + await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); + + // Modify and update with correct LastUpdated + var updatedStatus = new ResourceSearchParameterStatus + { + Uri = createdStatus.Uri, + Status = SearchParameterStatus.Enabled, // Changed status + IsPartiallySupported = true, // Changed partially supported + LastUpdated = createdStatus.LastUpdated, // Use the current LastUpdated, it should match max one. + }; + + await _fixture.SearchParameterStatusDataStore.UpsertStatuses([updatedStatus], CancellationToken.None); + + // Verify the update succeeded + var updatedAllStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); + var result = updatedAllStatuses.First(s => s.Uri.ToString() == testUri); + + Assert.Equal(SearchParameterStatus.Enabled, result.Status); + Assert.True(result.IsPartiallySupported); + Assert.True(result.LastUpdated > createdStatus.LastUpdated, "LastUpdated should change after update"); + } + finally + { + await _testHelper.DeleteSearchParameterStatusAsync(testUri); + } + } + + [Fact] + public async Task GivenExistingSearchParameterStatus_WhenUpdatingWithIncorrectLastUpdated_ThenShouldFail() + { + await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); + + var testUri = $"http://test.com/SearchParameter/ConcurrencyTest_{Guid.NewGuid()}"; + var initialStatus = new ResourceSearchParameterStatus + { + Uri = new Uri(testUri), + Status = SearchParameterStatus.Disabled, + IsPartiallySupported = false, + LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, + }; + + try + { + // Create initial status + await _fixture.SearchParameterStatusDataStore.UpsertStatuses([initialStatus], CancellationToken.None); + + // Get the created status with its LastUpdated + var allStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); + var createdStatus = allStatuses.First(s => s.Uri.ToString() == testUri); + + // Make an intermediate update to change the LastUpdated + var intermediateUpdate = new ResourceSearchParameterStatus + { + Uri = createdStatus.Uri, + Status = SearchParameterStatus.Enabled, + IsPartiallySupported = false, + LastUpdated = createdStatus.LastUpdated, + }; + await _fixture.SearchParameterStatusDataStore.UpsertStatuses([intermediateUpdate], CancellationToken.None); + + // Now try to update with the stale LastUpdated, should fail + var staleUpdate = new ResourceSearchParameterStatus + { + Uri = createdStatus.Uri, + Status = SearchParameterStatus.Supported, + IsPartiallySupported = true, + LastUpdated = createdStatus.LastUpdated, // This is now stale + }; + + try + { + await _fixture.SearchParameterStatusDataStore.UpsertStatuses([staleUpdate], CancellationToken.None); + Assert.Fail("This point should not be reached"); + } + catch (BadRequestException ex) + { + Assert.True(ex.Message.StartsWith(Core.Resources.SearchParameterConcurrencyConflict), $"expected={Core.Resources.SearchParameterConcurrencyConflict}, actual={ex.Message}"); + } + } + finally + { + await _testHelper.DeleteSearchParameterStatusAsync(testUri); + } + } + } +} diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SearchParameterStatusDataStoreTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SearchParameterStatusDataStoreTests.cs index 08ce3968ea..1ab33effb3 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SearchParameterStatusDataStoreTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SearchParameterStatusDataStoreTests.cs @@ -36,7 +36,7 @@ public async Task GivenAStatusRegistry_WhenGettingStatuses_ThenTheStatusesAreRet IReadOnlyCollection expectedStatuses = await _fixture.FilebasedSearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); IReadOnlyCollection actualStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); - ValidateSearchParameterStatuses(expectedStatuses, actualStatuses); + ValidateSearchParameterStatuses(expectedStatuses, actualStatuses, true); } [Fact] @@ -45,14 +45,16 @@ public async Task GivenAStatusRegistry_WhenUpsertingNewStatuses_ThenTheStatusesA string statusName1 = "http://hl7.org/fhir/SearchParameter/Test-1"; string statusName2 = "http://hl7.org/fhir/SearchParameter/Test-2"; + await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); + var status1 = new ResourceSearchParameterStatus { - Uri = new Uri(statusName1), Status = SearchParameterStatus.Disabled, IsPartiallySupported = false, + Uri = new Uri(statusName1), Status = SearchParameterStatus.Disabled, IsPartiallySupported = false, LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, }; var status2 = new ResourceSearchParameterStatus { - Uri = new Uri(statusName2), Status = SearchParameterStatus.Disabled, IsPartiallySupported = false, + Uri = new Uri(statusName2), Status = SearchParameterStatus.Disabled, IsPartiallySupported = false, LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, }; IReadOnlyCollection readonlyStatusesBeforeUpsert = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); @@ -80,16 +82,21 @@ public async Task GivenAStatusRegistry_WhenUpsertingNewStatuses_ThenTheStatusesA [Fact] public async Task GivenAStatusRegistry_WhenUpsertingExistingStatuses_ThenTheExistingStatusesAreUpdated() { - IReadOnlyCollection statusesBeforeUpdate = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); + await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); + + var statusesBeforeUpdate = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); // Get two existing statuses. - ResourceSearchParameterStatus expectedStatus1 = statusesBeforeUpdate.First(); - ResourceSearchParameterStatus expectedStatus2 = statusesBeforeUpdate.Last(); + var expectedStatus1 = statusesBeforeUpdate.First(); + var expectedStatus2 = statusesBeforeUpdate.Last(); // Modify them in some way. expectedStatus1.IsPartiallySupported = !expectedStatus1.IsPartiallySupported; expectedStatus2.IsPartiallySupported = !expectedStatus2.IsPartiallySupported; + // set last updated on at least one so it will set correct max + expectedStatus1.LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated; + var statusesToUpsert = new List { expectedStatus1, expectedStatus2 }; try @@ -97,8 +104,7 @@ public async Task GivenAStatusRegistry_WhenUpsertingExistingStatuses_ThenTheExis // Upsert the two existing, modified statuses. await _fixture.SearchParameterStatusDataStore.UpsertStatuses(statusesToUpsert, CancellationToken.None); - IReadOnlyCollection statusesAfterUpdate = - await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); + var statusesAfterUpdate = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); Assert.Equal(statusesBeforeUpdate.Count, statusesAfterUpdate.Count); @@ -120,20 +126,40 @@ public async Task GivenAStatusRegistry_WhenUpsertingExistingStatuses_ThenTheExis expectedStatus1.IsPartiallySupported = !expectedStatus1.IsPartiallySupported; expectedStatus2.IsPartiallySupported = !expectedStatus2.IsPartiallySupported; + await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); + + expectedStatus2.LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated; + statusesToUpsert = new List { expectedStatus1, expectedStatus2 }; await _fixture.SearchParameterStatusDataStore.UpsertStatuses(statusesToUpsert, CancellationToken.None); } } - private static void ValidateSearchParameterStatuses(IReadOnlyCollection expectedStatuses, IReadOnlyCollection actualStatuses) + private static void ValidateSearchParameterStatuses(IReadOnlyCollection expectedStatuses, IReadOnlyCollection actualStatuses, bool allowMore = false) { Assert.NotEmpty(expectedStatuses); var sortedExpected = expectedStatuses.OrderBy(status => status.Uri.ToString()).ToList(); var sortedActual = actualStatuses.OrderBy(status => status.Uri.ToString()).ToList(); - Assert.Equal(sortedExpected.Count, sortedActual.Count); + if (allowMore) + { + Assert.True(sortedExpected.Count <= sortedActual.Count); // we are not deleting so main store can accumulate more items than in file based + + // remove extra + foreach (var status in sortedActual.ToList()) + { + if (!sortedExpected.Any(_ => _.Uri == status.Uri)) + { + sortedActual.Remove(status); + } + } + } + else + { + Assert.Equal(sortedExpected.Count, sortedActual.Count); + } for (int i = 0; i < sortedExpected.Count; i++) { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerCreateStatsForSmartTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerCreateStatsForSmartTests.cs index 6ac26899ea..1afcdeabf2 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerCreateStatsForSmartTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerCreateStatsForSmartTests.cs @@ -343,7 +343,10 @@ private void ConfigureFhirRequestContext( accessControlContext.AllowedResourceActions.Add(scope); } - contextAccessor.RequestContext.AccessControlContext.Returns(accessControlContext); + var mockFhirRequestContext = Substitute.For(); + mockFhirRequestContext.AccessControlContext.Returns(accessControlContext); + + contextAccessor.RequestContext.Returns(mockFhirRequestContext); } private static SearchParams CreateSearchParams(params (string key, string value)[] items) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestHelper.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestHelper.cs index 0ecb5acd5e..2e5e0f02c6 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestHelper.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestHelper.cs @@ -270,14 +270,17 @@ public Task DeleteExportJobRecordAsync(string id, CancellationToken cancellation public async Task DeleteSearchParameterStatusAsync(string uri, CancellationToken cancellationToken = default) { - await using SqlConnection connection = await _sqlConnectionBuilder.GetSqlConnectionAsync(cancellationToken: cancellationToken); - var command = new SqlCommand("DELETE FROM dbo.SearchParam WHERE Uri = @uri", connection); + await using var connection = await _sqlConnectionBuilder.GetSqlConnectionAsync(cancellationToken: cancellationToken); + using var command = new SqlCommand( + @" +UPDATE dbo.SearchParam + SET Status = 'PendingDelete', LastUpdated = convert(datetimeoffset(7), sysUTCdatetime()) + WHERE Uri = @uri", + connection); command.Parameters.AddWithValue("@uri", uri); - await command.Connection.OpenAsync(cancellationToken); + await connection.OpenAsync(cancellationToken); await command.ExecuteNonQueryAsync(cancellationToken); - await connection.CloseAsync(); - _sqlServerFhirModel.RemoveSearchParamIdToUriMapping(uri); } public async Task DeleteAllReindexJobRecordsAsync(CancellationToken cancellationToken = default) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSearchParameterStatusDataStoreTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSearchParameterStatusDataStoreTests.cs index acfdc58aa8..871a41af47 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSearchParameterStatusDataStoreTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSearchParameterStatusDataStoreTests.cs @@ -42,6 +42,8 @@ public SqlServerSearchParameterStatusDataStoreTests(FhirStorageTestsFixture fixt [Fact] public async Task GivenUpsertStatuses_WhenUpsertingWithSameUri_ThenLastUpdatedIsRefreshed() { + await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); + // Arrange var testUri = "http://hl7.org/fhir/SearchParameter/Test-Upsert-" + Guid.NewGuid(); var status = new ResourceSearchParameterStatus @@ -49,13 +51,13 @@ public async Task GivenUpsertStatuses_WhenUpsertingWithSameUri_ThenLastUpdatedIs Uri = new Uri(testUri), Status = SearchParameterStatus.Disabled, IsPartiallySupported = false, - LastUpdated = DateTimeOffset.UtcNow, + LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, }; try { // Act - First upsert - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { status }, CancellationToken.None); + await _fixture.SearchParameterStatusDataStore.UpsertStatuses([status], CancellationToken.None); // Get the result var allStatuses1 = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); @@ -66,12 +68,14 @@ public async Task GivenUpsertStatuses_WhenUpsertingWithSameUri_ThenLastUpdatedIs // Small delay to ensure different timestamp await Task.Delay(100); + await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); + // Modify and upsert again status.Status = SearchParameterStatus.Enabled; status.IsPartiallySupported = true; - status.LastUpdated = createdStatus.LastUpdated; // Use the LastUpdated from DB + status.LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated; - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { status }, CancellationToken.None); + await _fixture.SearchParameterStatusDataStore.UpsertStatuses([status], CancellationToken.None); // Get the updated result var allStatuses2 = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); @@ -109,7 +113,7 @@ public void GivenSyncStatuses_WhenCalledWithStatuses_ThenFhirModelIsSynchronized }; // Act - Call SyncStatuses (this should not throw) - var exception = Record.Exception(() => dataStore!.SyncStatuses(new[] { status })); + var exception = Record.Exception(() => dataStore!.SyncStatuses([status])); // Assert - Method completes without exception Assert.Null(exception); @@ -118,6 +122,8 @@ public void GivenSyncStatuses_WhenCalledWithStatuses_ThenFhirModelIsSynchronized [Fact] public async Task GivenUpsertStatuses_WhenUpsertingMultipleStatuses_ThenAllAreCreated() { + await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); + // Arrange var testUri1 = "http://hl7.org/fhir/SearchParameter/Test-Batch1-" + Guid.NewGuid(); var testUri2 = "http://hl7.org/fhir/SearchParameter/Test-Batch2-" + Guid.NewGuid(); @@ -130,21 +136,21 @@ public async Task GivenUpsertStatuses_WhenUpsertingMultipleStatuses_ThenAllAreCr Uri = new Uri(testUri1), Status = SearchParameterStatus.Disabled, IsPartiallySupported = false, - LastUpdated = DateTimeOffset.UtcNow, + LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, }, new ResourceSearchParameterStatus { Uri = new Uri(testUri2), Status = SearchParameterStatus.Enabled, IsPartiallySupported = true, - LastUpdated = DateTimeOffset.UtcNow, + LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, }, new ResourceSearchParameterStatus { Uri = new Uri(testUri3), Status = SearchParameterStatus.Supported, IsPartiallySupported = false, - LastUpdated = DateTimeOffset.UtcNow, + LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, }, }; @@ -227,6 +233,8 @@ public async Task GivenGetSearchParameterStatuses_WhenStatusHasSortableType_Then [Fact] public async Task GivenUpsertStatuses_WhenUpdatingExistingStatus_ThenPreservesOtherStatuses() { + await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); + // Arrange var testUri = "http://hl7.org/fhir/SearchParameter/Test-Preserve-" + Guid.NewGuid(); var status = new ResourceSearchParameterStatus @@ -234,20 +242,22 @@ public async Task GivenUpsertStatuses_WhenUpdatingExistingStatus_ThenPreservesOt Uri = new Uri(testUri), Status = SearchParameterStatus.Disabled, IsPartiallySupported = false, - LastUpdated = DateTimeOffset.UtcNow, + LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, }; try { // Create initial status - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { status }, CancellationToken.None); + await _fixture.SearchParameterStatusDataStore.UpsertStatuses([status], CancellationToken.None); var countBefore = (await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None)).Count; + await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); + // Update the status status.Status = SearchParameterStatus.Enabled; - status.LastUpdated = DateTimeOffset.UtcNow; - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { status }, CancellationToken.None); + status.LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated; + await _fixture.SearchParameterStatusDataStore.UpsertStatuses([status], CancellationToken.None); var countAfter = (await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None)).Count; @@ -261,38 +271,26 @@ public async Task GivenUpsertStatuses_WhenUpdatingExistingStatus_ThenPreservesOt } [Fact] - public async Task GivenUpsertStatuses_WhenLastUpdatedIsPropagated_ThenInputCollectionIsUpdated() + public async Task GivenUpsertStatuses_WhenCollectionIsUpdated_ThenReturnedLastUpdatedIsGreaterThanOriginal() { - // Arrange + await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); + var testUri = "http://hl7.org/fhir/SearchParameter/Test-Propagate-" + Guid.NewGuid(); var status = new ResourceSearchParameterStatus { Uri = new Uri(testUri), Status = SearchParameterStatus.Disabled, IsPartiallySupported = false, - LastUpdated = DateTimeOffset.UtcNow, + LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, }; try { - // Act - Upsert and verify LastUpdated is propagated back - var originalLastUpdated = status.LastUpdated; - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { status }, CancellationToken.None); - - // Assert - The status object should have an updated LastUpdated value from the database - // Note: The database may return timestamps in a different timezone, so we compare using UtcDateTime - Assert.True( - status.LastUpdated.UtcDateTime >= originalLastUpdated.UtcDateTime, - $"Expected LastUpdated ({status.LastUpdated}) to be >= original ({originalLastUpdated})"); + await _fixture.SearchParameterStatusDataStore.UpsertStatuses([status], CancellationToken.None); - // Verify the value in the database matches what was propagated to the input collection - var dbStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); - var dbStatus = dbStatuses.FirstOrDefault(s => s.Uri.OriginalString == testUri); + var dbStatus = (await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None)).FirstOrDefault(s => s.Uri.OriginalString == testUri); - Assert.NotNull(dbStatus); - Assert.True( - Math.Abs((dbStatus.LastUpdated - status.LastUpdated).TotalSeconds) < 1, - $"Expected propagated LastUpdated ({status.LastUpdated}) to match database ({dbStatus.LastUpdated}) within 1 second"); + Assert.True(dbStatus.LastUpdated > status.LastUpdated, $"Expected {status.LastUpdated.ToString("yyyy-MM-ddTHH:mm:ss.fff")} < {dbStatus.LastUpdated.ToString("yyyy-MM-ddTHH:mm:ss.fff")}"); } finally { @@ -303,6 +301,8 @@ public async Task GivenUpsertStatuses_WhenLastUpdatedIsPropagated_ThenInputColle [Fact] public async Task GivenSqlServerResourceSearchParameterStatus_WhenIdIsAssigned_ThenIdIsPersisted() { + await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); + // Arrange var testUri = "http://hl7.org/fhir/SearchParameter/Test-Id-" + Guid.NewGuid(); var status = new SqlServerResourceSearchParameterStatus @@ -310,13 +310,13 @@ public async Task GivenSqlServerResourceSearchParameterStatus_WhenIdIsAssigned_T Uri = new Uri(testUri), Status = SearchParameterStatus.Disabled, IsPartiallySupported = false, - LastUpdated = DateTimeOffset.UtcNow, + LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, }; try { // Act - Upsert and retrieve - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { status }, CancellationToken.None); + await _fixture.SearchParameterStatusDataStore.UpsertStatuses([status], CancellationToken.None); var dbStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); var dbStatus = dbStatuses.FirstOrDefault(s => s.Uri.OriginalString == testUri); @@ -337,20 +337,21 @@ public async Task GivenSqlServerResourceSearchParameterStatus_WhenIdIsAssigned_T [Fact] public async Task GivenGetSearchParameterStatuses_WhenIsPartiallySupported_ThenValueIsPreserved() { - // Arrange + await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); + var testUri = "http://hl7.org/fhir/SearchParameter/Test-PartialSupport-" + Guid.NewGuid(); var status = new ResourceSearchParameterStatus { Uri = new Uri(testUri), Status = SearchParameterStatus.Enabled, IsPartiallySupported = true, - LastUpdated = DateTimeOffset.UtcNow, + LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, }; try { // Act - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { status }, CancellationToken.None); + await _fixture.SearchParameterStatusDataStore.UpsertStatuses([status], CancellationToken.None); var dbStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); var dbStatus = dbStatuses.FirstOrDefault(s => s.Uri.OriginalString == testUri); @@ -369,6 +370,8 @@ public async Task GivenGetSearchParameterStatuses_WhenIsPartiallySupported_ThenV [Fact] public async Task GivenUpsertStatuses_WhenStatusIsUnsupported_ThenStatusIsPersisted() { + await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); + // Arrange - Test that Unsupported status is handled correctly // Note: In older schemas (< V52), Unsupported is converted to Disabled // In newer schemas (>= V52), Unsupported is preserved @@ -378,13 +381,13 @@ public async Task GivenUpsertStatuses_WhenStatusIsUnsupported_ThenStatusIsPersis Uri = new Uri(testUri), Status = SearchParameterStatus.Unsupported, IsPartiallySupported = false, - LastUpdated = DateTimeOffset.UtcNow, + LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, }; try { // Act - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { status }, CancellationToken.None); + await _fixture.SearchParameterStatusDataStore.UpsertStatuses([status], CancellationToken.None); var dbStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); var dbStatus = dbStatuses.FirstOrDefault(s => s.Uri.OriginalString == testUri); @@ -404,6 +407,8 @@ public async Task GivenUpsertStatuses_WhenStatusIsUnsupported_ThenStatusIsPersis [Fact] public async Task GivenUpsertStatuses_WhenMixedNewAndExistingStatuses_ThenBothAreHandledCorrectly() { + await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); + // Arrange var existingUri = "http://hl7.org/fhir/SearchParameter/Test-MixedExisting-" + Guid.NewGuid(); var newUri = "http://hl7.org/fhir/SearchParameter/Test-MixedNew-" + Guid.NewGuid(); @@ -413,7 +418,7 @@ public async Task GivenUpsertStatuses_WhenMixedNewAndExistingStatuses_ThenBothAr Uri = new Uri(existingUri), Status = SearchParameterStatus.Disabled, IsPartiallySupported = false, - LastUpdated = DateTimeOffset.UtcNow, + LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, }; try @@ -425,13 +430,15 @@ public async Task GivenUpsertStatuses_WhenMixedNewAndExistingStatuses_ThenBothAr var allStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); var createdStatus = allStatuses.First(s => s.Uri.OriginalString == existingUri); + await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); + // Prepare mixed batch: update existing + create new var updateExisting = new ResourceSearchParameterStatus { Uri = createdStatus.Uri, Status = SearchParameterStatus.Enabled, IsPartiallySupported = true, - LastUpdated = createdStatus.LastUpdated, + LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, }; var createNew = new ResourceSearchParameterStatus @@ -439,13 +446,11 @@ public async Task GivenUpsertStatuses_WhenMixedNewAndExistingStatuses_ThenBothAr Uri = new Uri(newUri), Status = SearchParameterStatus.Supported, IsPartiallySupported = false, - LastUpdated = DateTimeOffset.UtcNow, + LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, }; // Act - await _fixture.SearchParameterStatusDataStore.UpsertStatuses( - new[] { updateExisting, createNew }, - CancellationToken.None); + await _fixture.SearchParameterStatusDataStore.UpsertStatuses([updateExisting, createNew], CancellationToken.None); // Assert var finalStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); @@ -469,6 +474,8 @@ await _fixture.SearchParameterStatusDataStore.UpsertStatuses( [Fact] public async Task GivenUpsertStatuses_WhenStatusValueChanges_ThenChangeIsReflectedInDatabase() { + await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); + // Comprehensive test for all status transition scenarios // Consolidates multiple transition tests into one comprehensive test @@ -479,13 +486,13 @@ public async Task GivenUpsertStatuses_WhenStatusValueChanges_ThenChangeIsReflect Uri = new Uri(testUri), Status = SearchParameterStatus.Disabled, IsPartiallySupported = false, - LastUpdated = DateTimeOffset.UtcNow, + LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, }; try { // Create initial status (Disabled) - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { status }, CancellationToken.None); + await _fixture.SearchParameterStatusDataStore.UpsertStatuses([status], CancellationToken.None); var statuses1 = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); var dbStatus1 = statuses1.First(s => s.Uri.OriginalString == testUri); @@ -494,11 +501,13 @@ public async Task GivenUpsertStatuses_WhenStatusValueChanges_ThenChangeIsReflect await Task.Delay(100); + await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); + // Transition to Enabled status.Status = SearchParameterStatus.Enabled; status.IsPartiallySupported = true; - status.LastUpdated = dbStatus1.LastUpdated; - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { status }, CancellationToken.None); + status.LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated; + await _fixture.SearchParameterStatusDataStore.UpsertStatuses([status], CancellationToken.None); var statuses2 = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); var dbStatus2 = statuses2.First(s => s.Uri.OriginalString == testUri); @@ -509,11 +518,13 @@ public async Task GivenUpsertStatuses_WhenStatusValueChanges_ThenChangeIsReflect await Task.Delay(100); + await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); + // Transition to Supported status.Status = SearchParameterStatus.Supported; status.IsPartiallySupported = false; - status.LastUpdated = dbStatus2.LastUpdated; - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { status }, CancellationToken.None); + status.LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated; + await _fixture.SearchParameterStatusDataStore.UpsertStatuses([status], CancellationToken.None); var statuses3 = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); var dbStatus3 = statuses3.First(s => s.Uri.OriginalString == testUri); @@ -530,6 +541,8 @@ public async Task GivenUpsertStatuses_WhenStatusValueChanges_ThenChangeIsReflect [Fact] public async Task GivenUpsertStatuses_WhenCancellationRequested_ThenOperationIsCancelled() { + await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); + // Arrange var testUri = "http://hl7.org/fhir/SearchParameter/Test-Cancellation-" + Guid.NewGuid(); var status = new ResourceSearchParameterStatus @@ -537,7 +550,7 @@ public async Task GivenUpsertStatuses_WhenCancellationRequested_ThenOperationIsC Uri = new Uri(testUri), Status = SearchParameterStatus.Disabled, IsPartiallySupported = false, - LastUpdated = DateTimeOffset.UtcNow, + LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, }; using var cts = new CancellationTokenSource(); @@ -546,7 +559,7 @@ public async Task GivenUpsertStatuses_WhenCancellationRequested_ThenOperationIsC // Act & Assert await Assert.ThrowsAnyAsync(async () => { - await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { status }, cts.Token); + await _fixture.SearchParameterStatusDataStore.UpsertStatuses([status], cts.Token); }); } From 777005d66dc9044c1b04541e3f4b4f2945c86699 Mon Sep 17 00:00:00 2001 From: SergeyGaluzo <95932081+SergeyGaluzo@users.noreply.github.com> Date: Wed, 10 Jun 2026 11:30:03 -0700 Subject: [PATCH 05/17] Correct search param pending status model (#5605) * correct pending statuses * corrections * moved imside if * single method --- .../Persistence/ResourceWrapperOperation.cs | 3 +- .../CreateOrUpdateSearchParameterBehavior.cs | 13 +--- .../DeleteSearchParameterBehavior.cs | 13 +--- ...rchParameterRequestContextPropertyNames.cs | 2 +- .../Features/Storage/CosmosFhirDataStore.cs | 47 ++------------ .../SearchParameterBehaviorTests.cs | 19 +++--- .../Storage/SqlServerFhirDataStore.cs | 65 ++++--------------- .../SearchParamListRowGenerator.cs | 4 +- 8 files changed, 34 insertions(+), 132 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceWrapperOperation.cs b/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceWrapperOperation.cs index eb361489f8..94bcbd567c 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceWrapperOperation.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceWrapperOperation.cs @@ -3,7 +3,6 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -using System.Collections.Generic; using EnsureThat; using Microsoft.Health.Fhir.Core.Features.Search.Registry; using Microsoft.Health.Fhir.Core.Models; @@ -48,7 +47,7 @@ public ResourceWrapperOperation( public BundleResourceContext BundleResourceContext { get; } - public IReadOnlyList PendingSearchParameterStatuses { get; internal set; } + public ResourceSearchParameterStatus PendingSearchParameterStatus { get; internal set; } #pragma warning disable CA1024 // Use properties where appropriate public DataStoreOperationIdentifier GetIdentifier() diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/CreateOrUpdateSearchParameterBehavior.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/CreateOrUpdateSearchParameterBehavior.cs index bbed409f18..b4d8194a50 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/CreateOrUpdateSearchParameterBehavior.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/CreateOrUpdateSearchParameterBehavior.cs @@ -132,13 +132,6 @@ private void QueueStatus(string url, SearchParameterStatus status, DateTimeOffse return; } - if (!context.Properties.TryGetValue(SearchParameterRequestContextPropertyNames.PendingStatusUpdates, out var value) || - value is not List pendingStatuses) - { - pendingStatuses = new List(); - context.Properties[SearchParameterRequestContextPropertyNames.PendingStatusUpdates] = pendingStatuses; - } - _searchParameterDefinitionManager.TryGetSearchParameter(url, out var existing); var update = new ResourceSearchParameterStatus @@ -150,11 +143,7 @@ private void QueueStatus(string url, SearchParameterStatus status, DateTimeOffse SortStatus = existing?.SortStatus ?? SortParameterStatus.Disabled, }; - lock (pendingStatuses) - { - pendingStatuses.RemoveAll(s => string.Equals(s.Uri?.OriginalString, url, StringComparison.Ordinal)); - pendingStatuses.Add(update); - } + context.Properties[SearchParameterRequestContextPropertyNames.PendingStatus] = update; } } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/DeleteSearchParameterBehavior.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/DeleteSearchParameterBehavior.cs index a002bf7318..7d36ecd364 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/DeleteSearchParameterBehavior.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/DeleteSearchParameterBehavior.cs @@ -116,13 +116,6 @@ private async Task QueuePendingDeleteStatusAsync(string url, CancellationToken c lastUpdated = _searchParameterOperations.SearchParamLastUpdated; } - if (!context.Properties.TryGetValue(SearchParameterRequestContextPropertyNames.PendingStatusUpdates, out var value) || - value is not List pendingStatuses) - { - pendingStatuses = new List(); - context.Properties[SearchParameterRequestContextPropertyNames.PendingStatusUpdates] = pendingStatuses; - } - _searchParameterDefinitionManager.TryGetSearchParameter(url, out var existing); var update = new ResourceSearchParameterStatus @@ -134,11 +127,7 @@ private async Task QueuePendingDeleteStatusAsync(string url, CancellationToken c SortStatus = existing?.SortStatus ?? SortParameterStatus.Disabled, }; - lock (pendingStatuses) - { - pendingStatuses.RemoveAll(s => string.Equals(s.Uri?.OriginalString, url, StringComparison.Ordinal)); - pendingStatuses.Add(update); - } + context.Properties[SearchParameterRequestContextPropertyNames.PendingStatus] = update; } } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterRequestContextPropertyNames.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterRequestContextPropertyNames.cs index efc3eeac9a..6a0b7dcf55 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterRequestContextPropertyNames.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterRequestContextPropertyNames.cs @@ -7,7 +7,7 @@ namespace Microsoft.Health.Fhir.Core.Features.Search.Parameters { public static class SearchParameterRequestContextPropertyNames { - public const string PendingStatusUpdates = "SearchParameter.PendingStatusUpdates"; + public const string PendingStatus = "SearchParameter.PendingStatus"; public const string LastUpdated = "SearchParameter.LastUpdated"; } } diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosFhirDataStore.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosFhirDataStore.cs index 7cffade697..31b1ff5e4c 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosFhirDataStore.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosFhirDataStore.cs @@ -262,7 +262,7 @@ public async Task UpsertAsync(ResourceWrapperOperation resource, UpsertOutcome result = await operation.AppendResourceAsync(resource, this, cancellationToken).ConfigureAwait(false); if (string.Equals(resource.Wrapper.ResourceTypeName, KnownResourceTypes.SearchParameter, StringComparison.Ordinal)) { - await PersistPendingSearchParameterStatusUpdatesAsync(cancellationToken); + await PersistPendingSearchParameterStatusUpdateAsync(cancellationToken); } return result; @@ -281,7 +281,7 @@ public async Task UpsertAsync(ResourceWrapperOperation resource, if (string.Equals(resource.Wrapper.ResourceTypeName, KnownResourceTypes.SearchParameter, StringComparison.Ordinal)) { - await PersistPendingSearchParameterStatusUpdatesAsync(cancellationToken); + await PersistPendingSearchParameterStatusUpdateAsync(cancellationToken); } return upsertOutcome; @@ -502,7 +502,7 @@ await retryPolicy.ExecuteAsync( } } - private async Task PersistPendingSearchParameterStatusUpdatesAsync(CancellationToken cancellationToken) + private async Task PersistPendingSearchParameterStatusUpdateAsync(CancellationToken cancellationToken) { var context = _requestContextAccessor.RequestContext; if (context?.Properties == null) @@ -510,47 +510,14 @@ private async Task PersistPendingSearchParameterStatusUpdatesAsync(CancellationT return; } - if (!context.Properties.TryGetValue(SearchParameterRequestContextPropertyNames.PendingStatusUpdates, out var value) || - value is not List pendingStatuses || - pendingStatuses.Count == 0) + if (!context.Properties.TryGetValue(SearchParameterRequestContextPropertyNames.PendingStatus, out var value) || + value is not ResourceSearchParameterStatus status) { return; } - List snapshot; - lock (pendingStatuses) - { - if (pendingStatuses.Count == 0) - { - return; - } - - snapshot = pendingStatuses - .Where(status => status?.Uri != null) - .GroupBy(status => status.Uri.OriginalString, StringComparer.Ordinal) - .Select(group => group.Last()) - .ToList(); - } - - if (snapshot.Count == 0) - { - return; - } - - await _searchParameterStatusDataStore.UpsertStatuses(snapshot, cancellationToken); - - lock (pendingStatuses) - { - foreach (var item in snapshot) - { - pendingStatuses.Remove(item); - } - - if (pendingStatuses.Count == 0) - { - context.Properties.Remove(SearchParameterRequestContextPropertyNames.PendingStatusUpdates); - } - } + await _searchParameterStatusDataStore.UpsertStatuses([status], cancellationToken); + context.Properties.Remove(SearchParameterRequestContextPropertyNames.PendingStatus); } public async Task GetAsync(ResourceKey key, CancellationToken cancellationToken) diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterBehaviorTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterBehaviorTests.cs index 160f218dcd..112663ad09 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterBehaviorTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterBehaviorTests.cs @@ -145,11 +145,11 @@ public async Task GivenADeleteResourceRequest_WhenDeletingASearchParameterResour var behavior = new DeleteSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterDefinitionManager, _searchParameterStatusManager, _requestContextAccessor, _modelInfoProvider); await behavior.Handle(request, async (ct) => await Task.Run(() => response), CancellationToken.None); - Assert.True(contextProperties.ContainsKey(SearchParameterRequestContextPropertyNames.PendingStatusUpdates)); - var pendingStatuses = contextProperties[SearchParameterRequestContextPropertyNames.PendingStatusUpdates] as List; - Assert.Single(pendingStatuses); - Assert.Equal(SearchParameterStatus.PendingDelete, pendingStatuses[0].Status); - Assert.Equal("http://example.com/Id", pendingStatuses[0].Uri.OriginalString); + Assert.True(contextProperties.ContainsKey(SearchParameterRequestContextPropertyNames.PendingStatus)); + var pendingStatus = contextProperties[SearchParameterRequestContextPropertyNames.PendingStatus] as ResourceSearchParameterStatus; + Assert.NotNull(pendingStatus); + Assert.Equal(SearchParameterStatus.PendingDelete, pendingStatus.Status); + Assert.Equal("http://example.com/Id", pendingStatus.Uri.OriginalString); } [Fact] @@ -284,11 +284,10 @@ public async Task GivenAnUpsertResourceRequest_WhenSearchParameterUrlChanges_The var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterDefinitionManager, _requestContextAccessor, _modelInfoProvider); await behavior.Handle(request, async (ct) => await Task.Run(() => response), CancellationToken.None); - var pendingStatuses = contextProperties[SearchParameterRequestContextPropertyNames.PendingStatusUpdates] as List; - Assert.NotNull(pendingStatuses); - Assert.Equal(2, pendingStatuses.Count); - Assert.Contains(pendingStatuses, s => s.Uri.OriginalString == "http://example.com/old-url" && s.Status == SearchParameterStatus.Deleted); - Assert.Contains(pendingStatuses, s => s.Uri.OriginalString == "http://example.com/new-url" && s.Status == SearchParameterStatus.Supported); + var pendingStatus = contextProperties[SearchParameterRequestContextPropertyNames.PendingStatus] as ResourceSearchParameterStatus; + Assert.NotNull(pendingStatus); + Assert.Equal("http://example.com/new-url", pendingStatus.Uri.OriginalString); + Assert.Equal(SearchParameterStatus.Supported, pendingStatus.Status); } private ResourceWrapper CreateResourceWrapper(ResourceElement resource, bool isDeleted) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs index a912672151..5c0c25f0fa 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs @@ -441,13 +441,10 @@ private async Task MergeInternalAsync(IReadOnlyList r.PendingSearchParameterStatuses?.Count > 0) - .SelectMany(r => r.PendingSearchParameterStatuses) - .ToList(); - if (mergeWrappersWithVersions.Count > 0) // Do not call DB with empty input { + var pendingStatuses = resources.Where(_ => _.PendingSearchParameterStatus != null).Select(_ => _.PendingSearchParameterStatus).ToList(); + await using (new Timer(async _ => await _sqlStoreClient.MergeResourcesPutTransactionHeartbeatAsync(transactionId, MergeResourcesTransactionHeartbeatPeriod, cancellationToken), null, TimeSpan.FromSeconds(RandomNumberGenerator.GetInt32(100) / 100.0 * MergeResourcesTransactionHeartbeatPeriod.TotalSeconds), MergeResourcesTransactionHeartbeatPeriod)) { var retries = 0; @@ -456,7 +453,7 @@ private async Task MergeInternalAsync(IReadOnlyList _.Wrapper).ToList(), enlistInTransaction, timeoutRetries, allSearchParameterStatuses, cancellationToken); + await MergeResourcesWrapperAsync(transactionId, singleTransaction, mergeWrappersWithVersions.Select(_ => _.Wrapper).ToList(), enlistInTransaction, timeoutRetries, pendingStatuses, cancellationToken); break; } catch (Exception e) @@ -810,12 +807,11 @@ internal async Task MergeResourcesWrapperAsync(long transactionId, bool singleTr using var cmd = new SqlCommand(); //// Do not use auto generated tvp generator as it does not allow to skip compartment tvp and paramters with default values cmd.CommandType = CommandType.StoredProcedure; - bool hasPendingStatuses = pendingStatuses?.Count > 0; - if (hasPendingStatuses) + if (pendingStatuses?.Count > 0) { cmd.CommandText = "dbo.MergeResourcesAndSearchParams"; - new SearchParamListTableValuedParameterDefinition("@SearchParams").AddParameter(cmd.Parameters, new SearchParamListRowGenerator().GenerateRows(pendingStatuses.ToList())); + new SearchParamListTableValuedParameterDefinition("@SearchParams").AddParameter(cmd.Parameters, new SearchParamListRowGenerator().GenerateRows(pendingStatuses)); cmd.Parameters.AddWithValue("@ReindexId", 0); } else @@ -858,43 +854,16 @@ internal async Task MergeResourcesWrapperAsync(long transactionId, bool singleTr await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, cancellationToken, disableRetries: true, applicationName: MergeApplicationName); } - _logger.LogInformation($"MergeResourcesWrapperAsync: resources={mergeWrappers.Count}, searchParams={(hasPendingStatuses ? pendingStatuses.Count : 0)} transactionId={transactionId}, singleTransaction={singleTransaction}, enlistInTran={enlistInTransaction}, commandTimeout={commandTimeout}, elapsed={sw.Elapsed.TotalMilliseconds} ms."); + _logger.LogInformation($"MergeResourcesWrapperAsync: resources={mergeWrappers.Count}, searchParams={pendingStatuses?.Count ?? 0} transactionId={transactionId}, singleTransaction={singleTransaction}, enlistInTran={enlistInTransaction}, commandTimeout={commandTimeout}, elapsed={sw.Elapsed.TotalMilliseconds} ms."); } - private bool TryGetPendingSearchParameterStatusUpdates(out List pendingStatuses) + private void SetAndClearPendingSearchParameterStatus(ResourceWrapperOperation resource) { - pendingStatuses = null; - - var context = _requestContextAccessor?.RequestContext; - if (context?.Properties == null) - { - return false; - } - - if (!context.Properties.TryGetValue(SearchParameterRequestContextPropertyNames.PendingStatusUpdates, out object value) || - value is not List statuses || - statuses.Count == 0) - { - return false; - } - - lock (statuses) + if (_requestContextAccessor?.RequestContext?.Properties?.TryGetValue(SearchParameterRequestContextPropertyNames.PendingStatus, out object value) == true) { - if (statuses.Count == 0) - { - return false; - } - - pendingStatuses = statuses.ToList(); + resource.PendingSearchParameterStatus = (ResourceSearchParameterStatus)value; + _requestContextAccessor.RequestContext.Properties.Remove(SearchParameterRequestContextPropertyNames.PendingStatus); } - - return true; - } - - private void ClearPendingSearchParameterStatusUpdates() - { - var context = _requestContextAccessor?.RequestContext; - context?.Properties?.Remove(SearchParameterRequestContextPropertyNames.PendingStatusUpdates); } public async Task UpsertAsync(ResourceWrapperOperation resource, CancellationToken cancellationToken) @@ -910,12 +879,7 @@ public async Task UpsertAsync(ResourceWrapperOperation resource, if (isBundleParallelOperation) { IBundleOrchestratorOperation bundleOperation = _bundleOrchestrator.GetOperation(resource.BundleResourceContext.BundleOperationId); - TryGetPendingSearchParameterStatusUpdates(out var pendingStatuses); - if (pendingStatuses?.Count > 0) - { - resource.PendingSearchParameterStatuses = pendingStatuses; - ClearPendingSearchParameterStatusUpdates(); - } + SetAndClearPendingSearchParameterStatus(resource); return await bundleOperation.AppendResourceAsync(resource, this, cancellationToken).ConfigureAwait(false); } @@ -927,12 +891,7 @@ public async Task UpsertAsync(ResourceWrapperOperation resource, // statuses ride along with the resource through dbo.MergeResourcesAndSearchParams. if (!isBundleTransaction) { - TryGetPendingSearchParameterStatusUpdates(out var pendingStatuses); - if (pendingStatuses?.Count > 0) - { - resource.PendingSearchParameterStatuses = pendingStatuses; - ClearPendingSearchParameterStatusUpdates(); - } + SetAndClearPendingSearchParameterStatus(resource); } // For regular upserts and sequential bundle operations, enlistTransaction is set to true. diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/SearchParamListRowGenerator.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/SearchParamListRowGenerator.cs index 2d8f546d04..b28edeac2a 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/SearchParamListRowGenerator.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/SearchParamListRowGenerator.cs @@ -11,9 +11,9 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Storage.TvpRowGeneration { - internal class SearchParamListRowGenerator : ITableValuedParameterRowGenerator, SearchParamListRow> + internal class SearchParamListRowGenerator : ITableValuedParameterRowGenerator, SearchParamListRow> { - public IEnumerable GenerateRows(List searchParameterStatuses) + public IEnumerable GenerateRows(IReadOnlyList searchParameterStatuses) { return searchParameterStatuses.Select(searchParameterStatus => new SearchParamListRow( searchParameterStatus.Uri.OriginalString, From ff8a81dfe76614c2f6ca26ede6843e4e004201e0 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Fri, 12 Jun 2026 07:05:24 -0700 Subject: [PATCH 06/17] Disable scalar temporal rewrite for chained searches (#5607) Prevent ScalarTemporalEqualityRewriter from producing day-split UNION expressions while visiting chained search predicates, while preserving the direct birthdate rewrite path. Add SQL Server chained birthdate E2E coverage and a unit guard for chained visitor context. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../ScalarTemporalEqualityRewriterTests.cs | 50 +++++++++++++++++-- .../ScalarTemporalEqualityRewriter.cs | 20 +++++++- .../Rest/Search/ChainingSearchTests.cs | 17 ++++++- 3 files changed, 79 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/ScalarTemporalEqualityRewriterTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/ScalarTemporalEqualityRewriterTests.cs index f6a7784f15..31d8e9eeb6 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/ScalarTemporalEqualityRewriterTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/ScalarTemporalEqualityRewriterTests.cs @@ -4,6 +4,8 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Reflection; +using System.Runtime.CompilerServices; using Microsoft.Health.Fhir.Core.Features.Search.Expressions; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions; @@ -67,6 +69,35 @@ private static SearchParameterInfo BuildBirthdateParam(Uri url = null, SearchPar baseResourceTypes: new[] { "Patient" }); } + private static SearchParameterInfo BuildReferenceParam() + { + return new SearchParameterInfo( + "Observation-patient", + "patient", + SearchParamType.Reference, + new Uri("http://hl7.org/fhir/SearchParameter/Observation-patient"), + expression: "Observation.subject", + baseResourceTypes: new[] { "Observation" }, + targetResourceTypes: new[] { "Patient" }); + } + + private static ChainedExpression BuildChainedExpression(Expression inner) + { + var expression = (ChainedExpression)RuntimeHelpers.GetUninitializedObject(typeof(ChainedExpression)); + SetBackingField(expression, nameof(ChainedExpression.ResourceTypes), new[] { "Observation" }); + SetBackingField(expression, nameof(ChainedExpression.ReferenceSearchParameter), BuildReferenceParam()); + SetBackingField(expression, nameof(ChainedExpression.TargetResourceTypes), new[] { "Patient" }); + SetBackingField(expression, nameof(ChainedExpression.Reversed), false); + SetBackingField(expression, nameof(ChainedExpression.Expression), inner); + return expression; + } + + private static void SetBackingField(ChainedExpression expression, string propertyName, T value) + { + FieldInfo field = typeof(ChainedExpression).GetField($"<{propertyName}>k__BackingField", BindingFlags.Instance | BindingFlags.NonPublic); + field.SetValue(expression, value); + } + private static MultiaryExpression EqualityPattern(DateTimeOffset start, DateTimeOffset end) => Expression.And( Expression.GreaterThanOrEqual(FieldName.DateTimeStart, null, start), @@ -78,7 +109,7 @@ public void GivenAllowListedBirthdateExactDay_WhenRewritten_ThenEmitsDaySplitUni { var expr = new SearchParameterExpression(BuildBirthdateParam(), EqualityPattern(start, end)); - var result = expr.AcceptVisitor(ScalarTemporalEqualityRewriter.Instance, null); + var result = expr.AcceptVisitor(ScalarTemporalEqualityRewriter.Instance); AssertDaySplitUnion(result, start, end); } @@ -91,18 +122,29 @@ public void GivenAllowListedBirthdateExactDayReversedOperandOrder_WhenRewritten_ Expression.GreaterThanOrEqual(FieldName.DateTimeStart, null, StartOfDay)); var expr = new SearchParameterExpression(BuildBirthdateParam(), reversedPattern); - var result = expr.AcceptVisitor(ScalarTemporalEqualityRewriter.Instance, null); + var result = expr.AcceptVisitor(ScalarTemporalEqualityRewriter.Instance); AssertDaySplitUnion(result, StartOfDay, EndOfDay); } + [Fact] + public void GivenAllowListedBirthdateExactDayInChainedExpression_WhenRewritten_ThenPassThrough() + { + var inner = new SearchParameterExpression(BuildBirthdateParam(), EqualityPattern(StartOfDay, EndOfDay)); + var expr = BuildChainedExpression(inner); + + var result = Assert.IsType(expr.AcceptVisitor(ScalarTemporalEqualityRewriter.Instance)); + + Assert.Same(inner, result.Expression); + } + [Theory] [MemberData(nameof(NonRewritableExpressions))] public void GivenAllowListedBirthdateWithNonExactDayExpression_WhenRewritten_ThenPassThrough(Expression inner) { var expr = new SearchParameterExpression(BuildBirthdateParam(), inner); - var result = Assert.IsType(expr.AcceptVisitor(ScalarTemporalEqualityRewriter.Instance, null)); + var result = Assert.IsType(expr.AcceptVisitor(ScalarTemporalEqualityRewriter.Instance)); Assert.Same(expr, result); } @@ -113,7 +155,7 @@ public void GivenNonAllowListedParameter_WhenEqualityPatternMatched_ThenPassThro { var expr = new SearchParameterExpression(param, EqualityPattern(StartOfDay, EndOfDay)); - var result = Assert.IsType(expr.AcceptVisitor(ScalarTemporalEqualityRewriter.Instance, null)); + var result = Assert.IsType(expr.AcceptVisitor(ScalarTemporalEqualityRewriter.Instance)); Assert.Same(expr, result); } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs index a7ce061de5..8297ad6aa1 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs @@ -33,7 +33,7 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions.Visitors /// This rewriter must run BEFORE so the input pattern still has only two /// predicates. Composite parameters and range operators are out of scope and pass through unchanged. /// - internal class ScalarTemporalEqualityRewriter : SqlExpressionRewriterWithInitialContext + internal class ScalarTemporalEqualityRewriter : SqlExpressionRewriterWithInitialContext { internal static readonly ScalarTemporalEqualityRewriter Instance = new ScalarTemporalEqualityRewriter(); @@ -50,8 +50,24 @@ private enum Precision ExactDay, } - public override Expression VisitSearchParameter(SearchParameterExpression expression, object context) + public override Expression VisitChained(ChainedExpression expression, bool context) { + Expression visitedExpression = expression.Expression.AcceptVisitor(this, context: true); + if (ReferenceEquals(visitedExpression, expression.Expression)) + { + return expression; + } + + return new ChainedExpression(expression.ResourceTypes, expression.ReferenceSearchParameter, expression.TargetResourceTypes, expression.Reversed, visitedExpression); + } + + public override Expression VisitSearchParameter(SearchParameterExpression expression, bool context) + { + if (context) + { + return expression; + } + // 1. Only allow-listed scalar date parameters are eligible. if (!IsActivatedScalarTemporalParameter(expression)) { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs index 5a9e37a36f..29133b4005 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs @@ -85,6 +85,17 @@ public async Task GivenAChainedSearchExpressionOverASimpleParameter_WhenSearched ValidateBundle(bundle, Fixture.SmithSnomedDiagnosticReport, Fixture.SmithLoincDiagnosticReport, Fixture.TrumanSnomedDiagnosticReport, Fixture.TrumanLoincDiagnosticReport); } + [HttpIntegrationFixtureArgumentSets(DataStore.SqlServer, Format.Json)] + [Fact] + public async Task GivenAChainedSearchExpressionOverBirthdate_WhenSearched_ThenCorrectBundleShouldBeReturned() + { + string query = $"_tag={Fixture.Tag}&subject:Patient.birthdate={Fixture.SmithPatientBirthDate}"; + + Bundle bundle = await Client.SearchAsync(ResourceType.DiagnosticReport, query); + + ValidateBundle(bundle, Fixture.SmithSnomedDiagnosticReport, Fixture.SmithLoincDiagnosticReport); + } + [Fact] public async Task GivenAChainedSearchExpressionOverASimpleParameter_WhenSearchedWithPaging_ThenCorrectBundleShouldBeReturned() { @@ -379,6 +390,8 @@ public ClassFixture(DataStore dataStore, Format format, TestFhirServerFactory te public string SmithPatientGivenName { get; } = Guid.NewGuid().ToString(); + public string SmithPatientBirthDate { get; } = "1990-05-15"; + public string TrumanPatientGivenName { get; } = Guid.NewGuid().ToString(); public string SnomedCode { get; } = Guid.NewGuid().ToString(); @@ -426,8 +439,8 @@ protected override async Task OnInitializedAsync() #endif AdamsPatient = (await TestFhirClient.CreateAsync(new Patient { Meta = meta, Gender = AdministrativeGender.Female, Name = new List { new HumanName { Family = "Adams" } } })).Resource; - SmithPatient = (await TestFhirClient.CreateAsync(new Patient { Meta = meta, Gender = AdministrativeGender.Male, Name = new List { new HumanName { Given = new[] { SmithPatientGivenName }, Family = "Smith" } }, ManagingOrganization = new ResourceReference($"Organization/{organization.Id}") })).Resource; - TrumanPatient = (await TestFhirClient.CreateAsync(new Patient { Meta = meta, Gender = AdministrativeGender.Male, Name = new List { new HumanName { Given = new[] { TrumanPatientGivenName }, Family = "Truman" } } })).Resource; + SmithPatient = (await TestFhirClient.CreateAsync(new Patient { Meta = meta, Gender = AdministrativeGender.Male, BirthDate = SmithPatientBirthDate, Name = new List { new HumanName { Given = new[] { SmithPatientGivenName }, Family = "Smith" } }, ManagingOrganization = new ResourceReference($"Organization/{organization.Id}") })).Resource; + TrumanPatient = (await TestFhirClient.CreateAsync(new Patient { Meta = meta, Gender = AdministrativeGender.Male, BirthDate = "1990-05-16", Name = new List { new HumanName { Given = new[] { TrumanPatientGivenName }, Family = "Truman" } } })).Resource; DeviceLoincSubject = (await TestFhirClient.CreateAsync(new Device { Meta = meta })).Resource; DeviceSnomedSubject = (await TestFhirClient.CreateAsync(new Device { Meta = meta })).Resource; From 7769ecc5eb170920f384f309afa6e852e7fdee78 Mon Sep 17 00:00:00 2001 From: Robert Johnson Date: Fri, 12 Jun 2026 09:19:00 -0500 Subject: [PATCH 07/17] Fix Bulk Job Creation (#5609) --- .../CreateBulkDeleteHandlerTests.cs | 32 +++++++++++++++++++ .../CreateBulkUpdateHandlerTests.cs | 4 +++ .../Handlers/CreateBulkDeleteHandler.cs | 6 +++- .../Handlers/CreateBulkUpdateHandler.cs | 8 ++++- 4 files changed, 48 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/BulkDelete/CreateBulkDeleteHandlerTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/BulkDelete/CreateBulkDeleteHandlerTests.cs index 42717db532..e9ba04d7ed 100644 --- a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/BulkDelete/CreateBulkDeleteHandlerTests.cs +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/BulkDelete/CreateBulkDeleteHandlerTests.cs @@ -100,6 +100,38 @@ public async Task GivenBulkDeleteRequest_WhenJobCreationRequested_ThenJobIsCreat await _queueClient.ReceivedWithAnyArgs(1).EnqueueAsync((byte)QueueType.BulkDelete, Arg.Any(), Arg.Any(), false, Arg.Any()); } + [Fact] + public async Task GivenBulkDeleteRequestWithNoParameters_WhenJobCreationRequested_ThenJobIsCreated() + { + var searchParams = new List>(); + + _authorizationService.CheckAccess(Arg.Any(), Arg.Any()).Returns(DataActions.HardDelete | DataActions.Delete); + _contextAccessor.RequestContext.BundleIssues.Clear(); + _queueClient.EnqueueAsync((byte)QueueType.BulkDelete, Arg.Any(), Arg.Any(), false, Arg.Any()).Returns(args => + { + var definition = JsonConvert.DeserializeObject(args.ArgAt(1)[0]); + Assert.Equal(_testUrl, definition.Url); + Assert.Equal(_testUrl, definition.BaseUrl); + Assert.Equal(DeleteOperation.HardDelete, definition.DeleteOperation); + Assert.Equal(searchParams.Count, definition.SearchParameters.Count); + + return new List() + { + new JobInfo() + { + Id = 1, + }, + }; + }); + + var request = new CreateBulkDeleteRequest(DeleteOperation.HardDelete, KnownResourceTypes.Patient, searchParams, false, null, false); + + var response = await _handler.Handle(request, CancellationToken.None); + Assert.NotNull(response); + Assert.Equal(1, response.Id); + await _queueClient.ReceivedWithAnyArgs(1).EnqueueAsync((byte)QueueType.BulkDelete, Arg.Any(), Arg.Any(), false, Arg.Any()); + } + [Fact] public async Task GivenBulkDeleteRequestWithInvalidSearchParameter_WhenJobCreationRequested_ThenBadRequestIsReturned() { diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/BulkUpdate/CreateBulkUpdateHandlerTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/BulkUpdate/CreateBulkUpdateHandlerTests.cs index 7c7a5623e4..2787411e25 100644 --- a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/BulkUpdate/CreateBulkUpdateHandlerTests.cs +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/BulkUpdate/CreateBulkUpdateHandlerTests.cs @@ -348,6 +348,10 @@ public static IEnumerable GetSearchParamsForJobCreation() new Tuple("_lastUpdated", "value3"), }, }; + yield return new object[] + { + new List>(), + }; } } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/BulkDelete/Handlers/CreateBulkDeleteHandler.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/BulkDelete/Handlers/CreateBulkDeleteHandler.cs index b7a580a37a..20144ef874 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/BulkDelete/Handlers/CreateBulkDeleteHandler.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/BulkDelete/Handlers/CreateBulkDeleteHandler.cs @@ -64,6 +64,10 @@ public async Task Handle(CreateBulkDeleteRequest reque var searchParameters = new List>(request.ConditionalParameters); + // Temporarily add _lastUpdated to the search parameters to mimic the behavior of the processing job. Conditional search will also fail if there are no search criteria. + var dateCurrent = new PartialDateTime(Clock.UtcNow); + searchParameters.Add(Tuple.Create("_lastUpdated", $"lt{dateCurrent}")); + // Should not run bulk delete if any of the search parameters are invalid as it can lead to unpredicatable results await _searchService.ConditionalSearchAsync(request.ResourceType, searchParameters, cancellationToken, count: 1, logger: _logger); if (_contextAccessor.RequestContext?.BundleIssues?.Count > 0 && _contextAccessor.RequestContext.BundleIssues.Any(x => !string.Equals(x.Diagnostics, Core.Resources.TruncatedIncludeMessageForIncludes, StringComparison.OrdinalIgnoreCase))) @@ -75,7 +79,7 @@ public async Task Handle(CreateBulkDeleteRequest reque JobType.BulkDeleteOrchestrator, request.DeleteOperation, request.ResourceType, - searchParameters, + request.ConditionalParameters, request.ExcludedResourceTypes, _contextAccessor.RequestContext.Uri.ToString(), _contextAccessor.RequestContext.BaseUri.ToString(), diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/BulkUpdate/Handlers/CreateBulkUpdateHandler.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/BulkUpdate/Handlers/CreateBulkUpdateHandler.cs index 968b739654..6000be7757 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/BulkUpdate/Handlers/CreateBulkUpdateHandler.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/BulkUpdate/Handlers/CreateBulkUpdateHandler.cs @@ -86,6 +86,12 @@ public async Task Handle(CreateBulkUpdateRequest reque // Remove bulk update specific parameters from search parameters searchParameters.RemoveAll(t => BulkUpdateQueryParameters.Any(param => param.Equals(t.Item1, StringComparison.OrdinalIgnoreCase))); + var searchParameterCopy = searchParameters.Select(t => Tuple.Create(t.Item1, t.Item2)).ToList(); + + // Temporarily add _lastUpdated to the search parameters to mimic the behavior of the processing job. Conditional search will also fail if there are no search criteria. + var dateCurrent = new PartialDateTime(Clock.UtcNow); + searchParameters.Add(Tuple.Create("_lastUpdated", $"lt{dateCurrent}")); + // Should not run bulk Update if any of the search parameters are invalid as it can lead to unpredicatable results await _searchService.ConditionalSearchAsync(request.ResourceType, searchParameters, cancellationToken, count: 1, logger: _logger); if (_contextAccessor.RequestContext?.BundleIssues?.Count > 0 && _contextAccessor.RequestContext.BundleIssues.Any(x => !string.Equals(x.Diagnostics, Core.Resources.TruncatedIncludeMessageForIncludes, StringComparison.OrdinalIgnoreCase))) @@ -110,7 +116,7 @@ public async Task Handle(CreateBulkUpdateRequest reque var processingDefinition = new BulkUpdateDefinition( JobType.BulkUpdateOrchestrator, request.ResourceType, - searchParameters, + searchParameterCopy, _contextAccessor.RequestContext.Uri.ToString(), _contextAccessor.RequestContext.BaseUri.ToString(), _contextAccessor.RequestContext.CorrelationId, From 5f350b693b897b50e999074512ba05e7f0c06a9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Henrique=20Inoc=C3=AAncio=20Borba=20Ferreira?= Date: Mon, 15 Jun 2026 15:31:34 -0400 Subject: [PATCH 08/17] [Bundles] Handling invalid bundle types (#5610) * Handling invalid bundle types * Refactoring code to make it simpler. * Upgrating health shared components. * Typo. * Remove unused using statements. * Undo version updated. --- .../Resources.Designer.cs | 11 ++++- src/Microsoft.Health.Fhir.Core/Resources.resx | 3 ++ ...ntextRouteDataPopulatingFilterAttribute.cs | 46 ++++++++++++++++--- .../Microsoft.Health.Fhir.Tests.Common.csproj | 2 + .../Normative/Bundle-InvalidBundleType.json | 29 ++++++++++++ .../Rest/BundleEdgeCaseTests.cs | 10 ++++ 6 files changed, 93 insertions(+), 8 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.Tests.Common/TestFiles/Normative/Bundle-InvalidBundleType.json diff --git a/src/Microsoft.Health.Fhir.Core/Resources.Designer.cs b/src/Microsoft.Health.Fhir.Core/Resources.Designer.cs index f5196ed484..498c91475d 100644 --- a/src/Microsoft.Health.Fhir.Core/Resources.Designer.cs +++ b/src/Microsoft.Health.Fhir.Core/Resources.Designer.cs @@ -19,7 +19,7 @@ namespace Microsoft.Health.Fhir.Core { // class via a tool like ResGen or Visual Studio. // To add or remove a member, edit your .ResX file then rerun ResGen // with the /str option, or rebuild your VS project. - [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "17.0.0.0")] + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "18.0.0.0")] [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] internal class Resources { @@ -2024,6 +2024,15 @@ internal static string UnsupportedBulkUpdateOperation { } } + /// + /// Looks up a localized string similar to The provided Bundle type is not supported. Supported values are: batch and transaction.. + /// + internal static string UnsupportedBundleType { + get { + return ResourceManager.GetString("UnsupportedBundleType", resourceCulture); + } + } + /// /// Looks up a localized string similar to Unsupported configuration found.. /// diff --git a/src/Microsoft.Health.Fhir.Core/Resources.resx b/src/Microsoft.Health.Fhir.Core/Resources.resx index 1f9a8dc2b5..1663775162 100644 --- a/src/Microsoft.Health.Fhir.Core/Resources.resx +++ b/src/Microsoft.Health.Fhir.Core/Resources.resx @@ -444,6 +444,9 @@ Bundle type is not present. Possible values are: transaction or batch + + The provided Bundle type is not supported. Supported values are: batch and transaction. + Unable to delete secret from SecretStore diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Features/Filters/FhirRequestContextRouteDataPopulatingFilterAttribute.cs b/src/Microsoft.Health.Fhir.Shared.Api/Features/Filters/FhirRequestContextRouteDataPopulatingFilterAttribute.cs index 399c81fda0..e3944c3bff 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Features/Filters/FhirRequestContextRouteDataPopulatingFilterAttribute.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api/Features/Filters/FhirRequestContextRouteDataPopulatingFilterAttribute.cs @@ -4,12 +4,16 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Diagnostics; +using System.Net; using EnsureThat; +using Hl7.Fhir.Model; using Microsoft.AspNetCore.Mvc.Controllers; using Microsoft.AspNetCore.Mvc.Filters; using Microsoft.AspNetCore.Routing; using Microsoft.Health.Api.Features.Audit; using Microsoft.Health.Core.Features.Context; +using Microsoft.Health.Fhir.Api.Features.ActionResults; using Microsoft.Health.Fhir.Core.Features; using Microsoft.Health.Fhir.Core.Features.Context; using Microsoft.Health.Fhir.Core.Features.Routing; @@ -69,14 +73,42 @@ public override void OnActionExecuting(ActionExecutingContext context) return; } - switch (bundle.Type) + try { - case Hl7.Fhir.Model.Bundle.BundleType.Batch: - fhirRequestContext.AuditEventType = AuditEventSubType.Batch; - break; - case Hl7.Fhir.Model.Bundle.BundleType.Transaction: - fhirRequestContext.AuditEventType = AuditEventSubType.Transaction; - break; + // bundle.Type can raise InvalidCastException if the incoming value is not a valid BundleType. + switch (bundle.Type) + { + case Hl7.Fhir.Model.Bundle.BundleType.Batch: + fhirRequestContext.AuditEventType = AuditEventSubType.Batch; + break; + case Hl7.Fhir.Model.Bundle.BundleType.Transaction: + fhirRequestContext.AuditEventType = AuditEventSubType.Transaction; + break; + } + } + catch (InvalidCastException) + { + // I had to add the 'timer' to the Http context, as the filter Microsoft.Health.Api.Features.Audit.AuditLoggingFilterAttribute expected it to be there. + // This change avoid a null reference exception to happen. + // The correct implementation would be handling the absence of the 'timer' in AuditLoggingFilterAttribute. + context.HttpContext.Items["timer"] = Stopwatch.StartNew(); + + context.Result = new OperationOutcomeResult( + new OperationOutcome + { + Id = fhirRequestContext.CorrelationId, + Issue = + { + new OperationOutcome.IssueComponent + { + Severity = OperationOutcome.IssueSeverity.Error, + Code = OperationOutcome.IssueType.Invalid, + Diagnostics = Core.Resources.UnsupportedBundleType, + }, + }, + }, + HttpStatusCode.BadRequest); + return; } } } diff --git a/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj b/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj index 19263e861d..debbb9cfd7 100644 --- a/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj +++ b/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj @@ -14,6 +14,7 @@ + @@ -167,6 +168,7 @@ + diff --git a/src/Microsoft.Health.Fhir.Tests.Common/TestFiles/Normative/Bundle-InvalidBundleType.json b/src/Microsoft.Health.Fhir.Tests.Common/TestFiles/Normative/Bundle-InvalidBundleType.json new file mode 100644 index 0000000000..d3110f156f --- /dev/null +++ b/src/Microsoft.Health.Fhir.Tests.Common/TestFiles/Normative/Bundle-InvalidBundleType.json @@ -0,0 +1,29 @@ +{ + "resourceType": "Bundle", + "type": "invalid-type", + "entry": [ + { + "resource": { + "resourceType": "Substance", + "id": "c8972ea6-aff8-421f-9be1-6baf2b49ffed", + "text": { + "status": "generated", + "div": "
Test Substance
" + }, + "code": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "test-substance", + "display": "Test Substance" + } + ] + } + }, + "request": { + "method": "POST", + "url": "Substance" + } + } + ] +} diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/BundleEdgeCaseTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/BundleEdgeCaseTests.cs index 17a0b2a0ae..2c2d96f3ec 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/BundleEdgeCaseTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/BundleEdgeCaseTests.cs @@ -35,6 +35,16 @@ public BundleEdgeCaseTests(HttpIntegrationTestFixture fixture) _client = fixture.TestFhirClient; } + [Fact] + public async Task GivenABundle_WhenAnInvalidBundleTypeIsUsed_ThenHttp400IsReturned() + { + var bundleAsString = Samples.GetJson("Bundle-InvalidBundleType"); + + using var fhirException = await Assert.ThrowsAsync(async () => await _client.PostAsync(string.Empty, bundleAsString)); + Assert.Equal(HttpStatusCode.BadRequest, fhirException.StatusCode); + Assert.True(fhirException.Message.Contains("The provided Bundle type is not supported.", StringComparison.OrdinalIgnoreCase)); + } + [Fact] [Trait(Traits.Priority, Priority.One)] public async Task GivenABundleWithConditionalUpdateByReference_WhenExecutedWithMaximizedConditionalQueryParallelism_RunsTheQueryInParallel() From abca6189e53b3e8f95996eca013d4d791074cfb6 Mon Sep 17 00:00:00 2001 From: Robert Johnson Date: Tue, 16 Jun 2026 10:39:11 -0500 Subject: [PATCH 09/17] Fix copy (#5612) --- .../Models/ListedCapabilityStatementTests.cs | 2 ++ .../Conformance/Models/ListedCapabilityStatement.cs | 12 ++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Conformance/Models/ListedCapabilityStatementTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Conformance/Models/ListedCapabilityStatementTests.cs index fc7b4d5b1a..680af81142 100644 --- a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Conformance/Models/ListedCapabilityStatementTests.cs +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Conformance/Models/ListedCapabilityStatementTests.cs @@ -373,6 +373,8 @@ public async Task GivenAStatement_WhenClonedWhileCollectionsAreModified_ThenNoEx await Task.WhenAll(cloneTask, modifyTask); // Assert + // If this step fails the excpetions will be logged to the test output + // This can look like this step threw an exception, but it is just reporting an exception that happened in the clone or modify tasks Assert.Empty(exceptions); } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Conformance/Models/ListedCapabilityStatement.cs b/src/Microsoft.Health.Fhir.Core/Features/Conformance/Models/ListedCapabilityStatement.cs index e871605e4b..d575b4364d 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Conformance/Models/ListedCapabilityStatement.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Conformance/Models/ListedCapabilityStatement.cs @@ -6,6 +6,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Linq; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Models; using Newtonsoft.Json; @@ -109,21 +110,20 @@ public ListedCapabilityStatement Clone() /// /// Generic data type being copied. /// Origin. - /// Destiny. - private static void SafeCopyTo(ICollection origin, ICollection destiny) + /// Destination. + private static void SafeCopyTo(ICollection origin, ICollection destination) { if (origin == null) { - destiny = null; + destination = null; return; } - T[] temp = new T[origin.Count]; - origin.CopyTo(temp, 0); + T[] temp = origin.ToArray(); foreach (T item in temp) { - destiny.Add(item); + destination.Add(item); } } } From 2d43d4362f5b4698ba4d2d58eeff23467e0fda74 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Wed, 17 Jun 2026 06:39:37 -0700 Subject: [PATCH 10/17] Skip scalar temporal union rewrite for chained queries ## What\n- Skip ScalarTemporalEqualityRewriter day-split union rewrite when the root expression contains any chain expression (forward or reverse).\n- Add regression unit tests covering both reverse-chain and forward-chain root query shapes to ensure birthdate equality stays pass-through.\n- Add an E2E forward-chain ImagingStudy started + patient birthdate + tag POST search test that mirrors the reverse-chain scenario.\n\n## Why\nUnion-expanded CTEs can shift predecessor selection in SqlQueryGenerator for complex chain queries. Skipping the scalar union rewrite for chain-containing roots preserves expected CTE ordering and prevents joins from binding to the wrong CTE branch.\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../ScalarTemporalEqualityRewriterTests.cs | 30 +++++++- .../ScalarTemporalEqualityRewriter.cs | 48 +++++++++++++ .../Rest/Search/ChainingSearchTests.cs | 71 +++++++++++++++++++ 3 files changed, 147 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/ScalarTemporalEqualityRewriterTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/ScalarTemporalEqualityRewriterTests.cs index 31d8e9eeb6..493c3c4651 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/ScalarTemporalEqualityRewriterTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/ScalarTemporalEqualityRewriterTests.cs @@ -81,13 +81,13 @@ private static SearchParameterInfo BuildReferenceParam() targetResourceTypes: new[] { "Patient" }); } - private static ChainedExpression BuildChainedExpression(Expression inner) + private static ChainedExpression BuildChainedExpression(Expression inner, bool reversed = false) { var expression = (ChainedExpression)RuntimeHelpers.GetUninitializedObject(typeof(ChainedExpression)); SetBackingField(expression, nameof(ChainedExpression.ResourceTypes), new[] { "Observation" }); SetBackingField(expression, nameof(ChainedExpression.ReferenceSearchParameter), BuildReferenceParam()); SetBackingField(expression, nameof(ChainedExpression.TargetResourceTypes), new[] { "Patient" }); - SetBackingField(expression, nameof(ChainedExpression.Reversed), false); + SetBackingField(expression, nameof(ChainedExpression.Reversed), reversed); SetBackingField(expression, nameof(ChainedExpression.Expression), inner); return expression; } @@ -138,6 +138,32 @@ public void GivenAllowListedBirthdateExactDayInChainedExpression_WhenRewritten_T Assert.Same(inner, result.Expression); } + [Fact] + public void GivenRootExpressionContainsReverseChainAndAllowListedBirthdateExactDay_WhenRewritten_ThenPassThroughBirthdate() + { + var birthdate = new SearchParameterExpression(BuildBirthdateParam(), EqualityPattern(StartOfDay, EndOfDay)); + var reverseChain = BuildChainedExpression(Expression.GreaterThanOrEqual(FieldName.DateTimeStart, null, StartOfDay), reversed: true); + var root = Expression.And(reverseChain, birthdate); + + var result = Assert.IsType(root.AcceptVisitor(ScalarTemporalEqualityRewriter.Instance)); + var rewrittenBirthdate = Assert.IsType(result.Expressions[1]); + + Assert.Same(birthdate, rewrittenBirthdate); + } + + [Fact] + public void GivenRootExpressionContainsForwardChainAndAllowListedBirthdateExactDay_WhenRewritten_ThenPassThroughBirthdate() + { + var birthdate = new SearchParameterExpression(BuildBirthdateParam(), EqualityPattern(StartOfDay, EndOfDay)); + var forwardChain = BuildChainedExpression(Expression.GreaterThanOrEqual(FieldName.DateTimeStart, null, StartOfDay), reversed: false); + var root = Expression.And(forwardChain, birthdate); + + var result = Assert.IsType(root.AcceptVisitor(ScalarTemporalEqualityRewriter.Instance)); + var rewrittenBirthdate = Assert.IsType(result.Expressions[1]); + + Assert.Same(birthdate, rewrittenBirthdate); + } + [Theory] [MemberData(nameof(NonRewritableExpressions))] public void GivenAllowListedBirthdateWithNonExactDayExpression_WhenRewritten_ThenPassThrough(Expression inner) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs index 8297ad6aa1..f65aad20ba 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs @@ -50,6 +50,20 @@ private enum Precision ExactDay, } + public override Expression VisitMultiary(MultiaryExpression expression, bool context) + { + // Top-level chain queries can introduce chain CTEs that are generated after a scalar temporal UNION. + // In that layout, predecessor resolution may bind joins to a UNION branch CTE instead of the aggregate CTE. + // Until UNION predecessor selection is chain-aware, skip this rewrite when the root query contains any + // chain so the downstream CTE ordering remains the expected single-CTE-per-term shape. + if (!context && ContainsChain(expression)) + { + return expression; + } + + return base.VisitMultiary(expression, context); + } + public override Expression VisitChained(ChainedExpression expression, bool context) { Expression visitedExpression = expression.Expression.AcceptVisitor(this, context: true); @@ -208,5 +222,39 @@ private static bool IsEndLe(BinaryExpression be) => private static bool IsUtc(DateTimeOffset value) => value.Offset == TimeSpan.Zero; private static bool IsUtcMidnight(DateTimeOffset value) => IsUtc(value) && value.TimeOfDay == TimeSpan.Zero; + + private static bool ContainsChain(Expression expression) + { + if (expression is ChainedExpression chained) + { + return true; + } + + if (expression is MultiaryExpression multiary) + { + foreach (Expression child in multiary.Expressions) + { + if (ContainsChain(child)) + { + return true; + } + } + + return false; + } + + if (expression is UnionExpression union) + { + foreach (Expression child in union.Expressions) + { + if (ContainsChain(child)) + { + return true; + } + } + } + + return false; + } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs index 29133b4005..05413f123f 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs @@ -144,6 +144,61 @@ public async Task GivenAReverseChainSearchExpressionOverASimpleParameter_WhenSea ValidateBundle(bundle, Fixture.TrumanPatient); } + [HttpIntegrationFixtureArgumentSets(DataStore.SqlServer, Format.Json)] + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task GivenAReverseChainSearchExpressionCombinedWithAnExactDayBirthdate_WhenSearched_ThenCorrectBundleShouldBeReturned(bool includeAccurateTotal) + { + string query = $"_tag={Fixture.Tag}&birthdate={Fixture.SmithPatientBirthDate}&_has:Observation:patient:code={Fixture.SnomedCode}"; + if (includeAccurateTotal) + { + query += "&_total=accurate"; + } + + Bundle bundle = await Client.SearchAsync(ResourceType.Patient, query); + + if (includeAccurateTotal) + { + Assert.Equal(1, bundle.Total.GetValueOrDefault()); + } + + ValidateBundle(bundle, Fixture.SmithPatient); + } + +#if !Stu3 + [HttpIntegrationFixtureArgumentSets(DataStore.SqlServer, Format.Json)] + [Fact] + public async Task GivenAReverseChainSearchExpressionOverImagingStudyStartedCombinedWithAnExactDayBirthdateAndTag_WhenSearchedWithPost_ThenCorrectBundleShouldBeReturned() + { + Bundle bundle = await Client.SearchPostAsync( + ResourceType.Patient.ToString(), + null, + default, + ("_has:ImagingStudy:patient:started", Fixture.ImagingStudyStarted), + ("birthdate", Fixture.ImagingStudyPatientBirthDate), + ("_tag", $"{Fixture.TenantTagSystem}|{Fixture.TenantTagCode}")); + + ValidateBundle(bundle, Fixture.ImagingStudyExactDayPatient, Fixture.ImagingStudyMonthPrecisionPatient); + } + + [HttpIntegrationFixtureArgumentSets(DataStore.SqlServer, Format.Json)] + [Fact] + public async Task GivenAForwardChainSearchExpressionOverPatientBirthdateCombinedWithImagingStudyStartedAndTag_WhenSearchedWithPost_ThenCorrectBundleShouldBeReturned() + { + Bundle bundle = await Client.SearchPostAsync( + ResourceType.ImagingStudy.ToString(), + null, + default, + ("started", Fixture.ImagingStudyStarted), + ("patient:Patient.birthdate", Fixture.ImagingStudyPatientBirthDate), + ("_tag", $"{Fixture.TenantTagSystem}|{Fixture.TenantTagCode}")); + + Assert.Equal(2, bundle.Entry.Count); + Assert.All(bundle.Entry, entry => Assert.IsType(entry.Resource)); + } +#endif + [Fact] public async Task GivenAReverseChainSearchExpressionWithMultipleTargetTypes_WhenSearched_ThenCorrectBundleShouldBeReturned() { @@ -400,6 +455,22 @@ public ClassFixture(DataStore dataStore, Format format, TestFhirServerFactory te public string OrganizationIdentifier { get; } = Guid.NewGuid().ToString(); +#if !Stu3 + public string TenantTagSystem { get; } = "urn:tenantId"; + + public string TenantTagCode { get; } = "1016"; + + public string ImagingStudyPatientBirthDate { get; } = "2018-06-06"; + + public string ImagingStudyMonthPrecisionPatientBirthDate { get; } = "2018-06"; + + public string ImagingStudyStarted { get; } = "2018-02-02T05:00:00.000"; + + public Patient ImagingStudyExactDayPatient { get; private set; } + + public Patient ImagingStudyMonthPrecisionPatient { get; private set; } +#endif + public Patient SmithPatient { get; private set; } public DiagnosticReport SmithSnomedDiagnosticReport { get; private set; } From 5787553eae3026f75585e7b754d6be6292fdf43c Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Wed, 17 Jun 2026 08:58:31 -0700 Subject: [PATCH 11/17] Clarify UNION handling, update chaining search tests - Improved comments in SqlQueryGenerator and ScalarTemporalEqualityRewriter to clarify UNION ALL expansion, SMART scope unions, and DOB day-split handling, documenting current limitations and guardrails. - Refactored chaining search E2E test to use unique tenant tags and dynamic patient creation; removed redundant forward chain test. - Cleaned up Fixture class by removing static tenant tag and patient fields now handled within tests. --- .../QueryGenerators/SqlQueryGenerator.cs | 11 ++- .../ScalarTemporalEqualityRewriter.cs | 2 + .../Rest/Search/ChainingSearchTests.cs | 67 ++++++++++++------- 3 files changed, 54 insertions(+), 26 deletions(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs index e2ad628365..27a37b9baa 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs @@ -1682,8 +1682,15 @@ private int FindRestrictingPredecessorTableExpressionIndex() { int FindImpl(int currentIndex) { - // Due to the UnionAll expressions, the number of the current index used to create new CTEs can be greater than - // the number of expressions in '_rootExpression.SearchParamTableExpressions'. + // Due to UNION ALL expansion (including SMART scope unions), the number of generated CTEs can be greater + // than the number of table expressions in '_rootExpression.SearchParamTableExpressions'. + // + // NOTE: + // Returning currentIndex - 1 here is a best-effort fallback that assumes the immediately previous CTE + // is the correct restricting predecessor. That assumption is known to be fragile when a query shape + // introduces additional UNION fan-out CTEs (for example, DOB scalar-temporal day-split UNION or SMART + // union aggregation). For now, DOB rewrite is guarded for chain-containing roots; this predecessor + // resolver still needs a union-aware mapping for the general SMART/union case. if (currentIndex >= _rootExpression.SearchParamTableExpressions.Count) { return currentIndex - 1; diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs index f65aad20ba..7c9dd02dda 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs @@ -56,6 +56,8 @@ public override Expression VisitMultiary(MultiaryExpression expression, bool con // In that layout, predecessor resolution may bind joins to a UNION branch CTE instead of the aggregate CTE. // Until UNION predecessor selection is chain-aware, skip this rewrite when the root query contains any // chain so the downstream CTE ordering remains the expected single-CTE-per-term shape. + // This is intentionally scoped to DOB rewrite safety; SMART-scope union paths are documented in + // SqlQueryGenerator.FindRestrictingPredecessorTableExpressionIndex and remain a broader follow-up. if (!context && ContainsChain(expression)) { return expression; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs index 05413f123f..f9e7fb4fd0 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs @@ -171,31 +171,56 @@ public async Task GivenAReverseChainSearchExpressionCombinedWithAnExactDayBirthd [Fact] public async Task GivenAReverseChainSearchExpressionOverImagingStudyStartedCombinedWithAnExactDayBirthdateAndTag_WhenSearchedWithPost_ThenCorrectBundleShouldBeReturned() { + string tenantTagCode = Guid.NewGuid().ToString("N"); + var tenantMeta = new Meta + { + Tag = new List + { + new Coding(Fixture.TenantTagSystem, tenantTagCode), + }, + }; + + Patient imagingStudyExactDayPatient = (await Client.CreateAsync( + new Patient + { + Meta = tenantMeta, + BirthDate = Fixture.ImagingStudyPatientBirthDate, + })).Resource; + + Patient imagingStudyMonthPrecisionPatient = (await Client.CreateAsync( + new Patient + { + Meta = tenantMeta, + BirthDate = Fixture.ImagingStudyMonthPrecisionPatientBirthDate, + })).Resource; + + await Client.CreateAsync( + new ImagingStudy + { + Meta = tenantMeta, + Status = ImagingStudy.ImagingStudyStatus.Available, + Subject = new ResourceReference($"Patient/{imagingStudyExactDayPatient.Id}"), + Started = Fixture.ImagingStudyStarted, + }); + + await Client.CreateAsync( + new ImagingStudy + { + Meta = tenantMeta, + Status = ImagingStudy.ImagingStudyStatus.Available, + Subject = new ResourceReference($"Patient/{imagingStudyMonthPrecisionPatient.Id}"), + Started = Fixture.ImagingStudyStarted, + }); + Bundle bundle = await Client.SearchPostAsync( ResourceType.Patient.ToString(), null, default, ("_has:ImagingStudy:patient:started", Fixture.ImagingStudyStarted), ("birthdate", Fixture.ImagingStudyPatientBirthDate), - ("_tag", $"{Fixture.TenantTagSystem}|{Fixture.TenantTagCode}")); + ("_tag", $"{Fixture.TenantTagSystem}|{tenantTagCode}")); - ValidateBundle(bundle, Fixture.ImagingStudyExactDayPatient, Fixture.ImagingStudyMonthPrecisionPatient); - } - - [HttpIntegrationFixtureArgumentSets(DataStore.SqlServer, Format.Json)] - [Fact] - public async Task GivenAForwardChainSearchExpressionOverPatientBirthdateCombinedWithImagingStudyStartedAndTag_WhenSearchedWithPost_ThenCorrectBundleShouldBeReturned() - { - Bundle bundle = await Client.SearchPostAsync( - ResourceType.ImagingStudy.ToString(), - null, - default, - ("started", Fixture.ImagingStudyStarted), - ("patient:Patient.birthdate", Fixture.ImagingStudyPatientBirthDate), - ("_tag", $"{Fixture.TenantTagSystem}|{Fixture.TenantTagCode}")); - - Assert.Equal(2, bundle.Entry.Count); - Assert.All(bundle.Entry, entry => Assert.IsType(entry.Resource)); + ValidateBundle(bundle, imagingStudyExactDayPatient, imagingStudyMonthPrecisionPatient); } #endif @@ -458,17 +483,11 @@ public ClassFixture(DataStore dataStore, Format format, TestFhirServerFactory te #if !Stu3 public string TenantTagSystem { get; } = "urn:tenantId"; - public string TenantTagCode { get; } = "1016"; - public string ImagingStudyPatientBirthDate { get; } = "2018-06-06"; public string ImagingStudyMonthPrecisionPatientBirthDate { get; } = "2018-06"; public string ImagingStudyStarted { get; } = "2018-02-02T05:00:00.000"; - - public Patient ImagingStudyExactDayPatient { get; private set; } - - public Patient ImagingStudyMonthPrecisionPatient { get; private set; } #endif public Patient SmithPatient { get; private set; } From c00c7a569b488ce5faa1ac5a09123957e9914217 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Wed, 17 Jun 2026 10:29:41 -0700 Subject: [PATCH 12/17] cleanup for PR --- .../Visitors/QueryGenerators/SqlQueryGenerator.cs | 11 ++--------- .../Visitors/ScalarTemporalEqualityRewriter.cs | 11 +++++------ 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs index 27a37b9baa..e2ad628365 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/QueryGenerators/SqlQueryGenerator.cs @@ -1682,15 +1682,8 @@ private int FindRestrictingPredecessorTableExpressionIndex() { int FindImpl(int currentIndex) { - // Due to UNION ALL expansion (including SMART scope unions), the number of generated CTEs can be greater - // than the number of table expressions in '_rootExpression.SearchParamTableExpressions'. - // - // NOTE: - // Returning currentIndex - 1 here is a best-effort fallback that assumes the immediately previous CTE - // is the correct restricting predecessor. That assumption is known to be fragile when a query shape - // introduces additional UNION fan-out CTEs (for example, DOB scalar-temporal day-split UNION or SMART - // union aggregation). For now, DOB rewrite is guarded for chain-containing roots; this predecessor - // resolver still needs a union-aware mapping for the general SMART/union case. + // Due to the UnionAll expressions, the number of the current index used to create new CTEs can be greater than + // the number of expressions in '_rootExpression.SearchParamTableExpressions'. if (currentIndex >= _rootExpression.SearchParamTableExpressions.Count) { return currentIndex - 1; diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs index 7c9dd02dda..b8c1494c5f 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs @@ -52,12 +52,10 @@ private enum Precision public override Expression VisitMultiary(MultiaryExpression expression, bool context) { - // Top-level chain queries can introduce chain CTEs that are generated after a scalar temporal UNION. - // In that layout, predecessor resolution may bind joins to a UNION branch CTE instead of the aggregate CTE. - // Until UNION predecessor selection is chain-aware, skip this rewrite when the root query contains any - // chain so the downstream CTE ordering remains the expected single-CTE-per-term shape. - // This is intentionally scoped to DOB rewrite safety; SMART-scope union paths are documented in - // SqlQueryGenerator.FindRestrictingPredecessorTableExpressionIndex and remain a broader follow-up. + // Don't run rewriter for chained or reverse chained expression. Once we remove the union in this rewriter below and + // adhere to the FHIR spec for dates, we can remove this method. Our current SQL generator has difficulty with unions + // emitted by this rewriter due to FindRestrictingPredecessorTableExpressionIndex() has a catch all that returns + // currentIndex - 1 for unions. if (!context && ContainsChain(expression)) { return expression; @@ -68,6 +66,7 @@ public override Expression VisitMultiary(MultiaryExpression expression, bool con public override Expression VisitChained(ChainedExpression expression, bool context) { + // Logic here is the same as VisitMultiary. Expression visitedExpression = expression.Expression.AcceptVisitor(this, context: true); if (ReferenceEquals(visitedExpression, expression.Expression)) { From 6d9df0ee87fb27deed4c76868af336386f3c28e2 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Wed, 17 Jun 2026 10:42:23 -0700 Subject: [PATCH 13/17] Add SQL-only DOB E2E coverage Add SQL-only E2E tests for partial birthdate OR/ne/total/sort and leap-day cases in DateSearchTests, plus chained birthdate month/day scenarios in ChainingSearchTests. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Rest/Search/ChainingSearchTests.cs | 22 +++ .../Rest/Search/DateSearchTests.cs | 151 ++++++++++++++++++ 2 files changed, 173 insertions(+) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs index f9e7fb4fd0..dd16fe0bf3 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs @@ -96,6 +96,28 @@ public async Task GivenAChainedSearchExpressionOverBirthdate_WhenSearched_ThenCo ValidateBundle(bundle, Fixture.SmithSnomedDiagnosticReport, Fixture.SmithLoincDiagnosticReport); } + [HttpIntegrationFixtureArgumentSets(DataStore.SqlServer, Format.Json)] + [Fact] + public async Task GivenAChainedSearchExpressionOverBirthdateMonthPrecision_WhenSearched_ThenCorrectBundleShouldBeReturned() + { + string query = $"_tag={Fixture.Tag}&subject:Patient.birthdate=1990-05"; + + Bundle bundle = await Client.SearchAsync(ResourceType.DiagnosticReport, query); + + ValidateBundle(bundle, Fixture.SmithSnomedDiagnosticReport, Fixture.SmithLoincDiagnosticReport, Fixture.TrumanSnomedDiagnosticReport, Fixture.TrumanLoincDiagnosticReport); + } + + [HttpIntegrationFixtureArgumentSets(DataStore.SqlServer, Format.Json)] + [Fact] + public async Task GivenAChainedSearchExpressionOverBirthdateDayWithAdditionalCriteriaOnTheSameTarget_WhenSearched_ThenCorrectBundleShouldBeReturned() + { + string query = $"_tag={Fixture.Tag}&subject:Patient.birthdate={Fixture.SmithPatientBirthDate}&subject:Patient.family=Smith"; + + Bundle bundle = await Client.SearchAsync(ResourceType.DiagnosticReport, query); + + ValidateBundle(bundle, Fixture.SmithSnomedDiagnosticReport, Fixture.SmithLoincDiagnosticReport); + } + [Fact] public async Task GivenAChainedSearchExpressionOverASimpleParameter_WhenSearchedWithPaging_ThenCorrectBundleShouldBeReturned() { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/DateSearchTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/DateSearchTests.cs index f4237aad2c..ddba09e3e2 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/DateSearchTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/DateSearchTests.cs @@ -298,10 +298,161 @@ public async Task GivenPatientsWithPartialBirthdates_WhenSearchedByEquality_Then } } + [HttpIntegrationFixtureArgumentSets(DataStore.SqlServer, Format.Json)] + [Fact] + [Trait(Traits.Priority, Priority.One)] + public async Task GivenPatientsWithPartialBirthdates_WhenSearchedByMultiValueOr_ThenUnionOfPerValueOverlapSetsIsReturned() + { + string tag = Guid.NewGuid().ToString(); + PartialBirthdateMatrix matrix = await CreatePartialBirthdateMatrixAsync(tag); + + Bundle bundle = await Client.SearchAsync(ResourceType.Patient, $"birthdate=2000-03-03,2001-12-31&_tag={tag}"); + + ValidateBundle(bundle, matrix.Year2000, matrix.March2000, matrix.March03, matrix.Year2001, matrix.December2001, matrix.December31_2001); + AssertNoDuplicateEntries(bundle); + } + + [HttpIntegrationFixtureArgumentSets(DataStore.SqlServer, Format.Json)] + [Fact] + [Trait(Traits.Priority, Priority.One)] + public async Task GivenPatientsWithPartialBirthdates_WhenSearchedByNotEquals_ThenOnlyTheExactDayValueIsExcluded() + { + string tag = Guid.NewGuid().ToString(); + PartialBirthdateMatrix matrix = await CreatePartialBirthdateMatrixAsync(tag); + + Bundle bundle = await Client.SearchAsync(ResourceType.Patient, $"birthdate=ne2000-03-03&_tag={tag}&_total=accurate&_count=100"); + + Assert.Equal(11, bundle.Total.GetValueOrDefault()); + ValidateBundle( + bundle, + matrix.Year2000, + matrix.March2000, + matrix.December31_1999, + matrix.April01_2000, + matrix.December2000, + matrix.March31_2000, + matrix.Year2001, + matrix.November30_2001, + matrix.December2001, + matrix.December31_2001, + matrix.January01_2002); + AssertNoDuplicateEntries(bundle); + } + + [HttpIntegrationFixtureArgumentSets(DataStore.SqlServer, Format.Json)] + [Fact] + [Trait(Traits.Priority, Priority.One)] + public async Task GivenPatientsWithPartialBirthdates_WhenDayEqualitySearchedWithTotal_ThenCountIsAccurateAndEntriesAreNotDuplicated() + { + string tag = Guid.NewGuid().ToString(); + PartialBirthdateMatrix matrix = await CreatePartialBirthdateMatrixAsync(tag); + + Bundle bundle = await Client.SearchAsync(ResourceType.Patient, $"birthdate=2000-03-03&_tag={tag}&_total=accurate"); + + Assert.Equal(3, bundle.Total.GetValueOrDefault()); + ValidateBundle(bundle, matrix.Year2000, matrix.March2000, matrix.March03); + AssertNoDuplicateEntries(bundle); + } + + [HttpIntegrationFixtureArgumentSets(DataStore.SqlServer, Format.Json)] + [Fact] + [Trait(Traits.Priority, Priority.One)] + public async Task GivenPatientsWithPartialBirthdates_WhenDayEqualitySearchedWithSort_ThenResultsAreOrderedByBirthdateWithoutDuplicates() + { + string tag = Guid.NewGuid().ToString(); + PartialBirthdateMatrix matrix = await CreatePartialBirthdateMatrixAsync(tag); + + Bundle bundle = await Client.SearchAsync(ResourceType.Patient, $"birthdate=2000-03-03&_sort=birthdate&_tag={tag}"); + + AssertNoDuplicateEntries(bundle); + Assert.Equal( + new[] { matrix.Year2000.Id, matrix.March2000.Id, matrix.March03.Id }, + bundle.Entry.Select(e => e.Resource.Id).ToArray()); + } + + [HttpIntegrationFixtureArgumentSets(DataStore.SqlServer, Format.Json)] + [Fact] + [Trait(Traits.Priority, Priority.One)] + public async Task GivenPatientsWithLeapDayBirthdates_WhenSearchedByDay_ThenCorrectBundleShouldBeReturned() + { + string tag = Guid.NewGuid().ToString(); + Patient[] patients = await Client.CreateResourcesAsync( + p => SetPatientBirthDate(p, "2020", tag), + p => SetPatientBirthDate(p, "2020-02", tag), + p => SetPatientBirthDate(p, "2020-02-28", tag), + p => SetPatientBirthDate(p, "2020-02-29", tag), + p => SetPatientBirthDate(p, "2020-03-01", tag)); + Patient year2020 = patients[0]; + Patient february2020 = patients[1]; + Patient february28 = patients[2]; + Patient february29 = patients[3]; + Patient march01 = patients[4]; + + Bundle leapDayBundle = await Client.SearchAsync(ResourceType.Patient, $"birthdate=2020-02-29&_tag={tag}"); + ValidateBundle(leapDayBundle, year2020, february2020, february29); + + Bundle feb28Bundle = await Client.SearchAsync(ResourceType.Patient, $"birthdate=2020-02-28&_tag={tag}"); + ValidateBundle(feb28Bundle, year2020, february2020, february28); + + Bundle march01Bundle = await Client.SearchAsync(ResourceType.Patient, $"birthdate=2020-03-01&_tag={tag}"); + ValidateBundle(march01Bundle, year2020, march01); + } + + private async System.Threading.Tasks.Task CreatePartialBirthdateMatrixAsync(string tag) + { + Patient[] patients = await Client.CreateResourcesAsync( + p => SetPatientBirthDate(p, "2000", tag), + p => SetPatientBirthDate(p, "2000-03", tag), + p => SetPatientBirthDate(p, "2000-03-03", tag), + p => SetPatientBirthDate(p, "1999-12-31", tag), + p => SetPatientBirthDate(p, "2000-04-01", tag), + p => SetPatientBirthDate(p, "2000-12", tag), + p => SetPatientBirthDate(p, "2000-03-31", tag), + p => SetPatientBirthDate(p, "2001", tag), + p => SetPatientBirthDate(p, "2001-11-30", tag), + p => SetPatientBirthDate(p, "2001-12", tag), + p => SetPatientBirthDate(p, "2001-12-31", tag), + p => SetPatientBirthDate(p, "2002-01-01", tag)); + + return new PartialBirthdateMatrix( + Year2000: patients[0], + March2000: patients[1], + March03: patients[2], + December31_1999: patients[3], + April01_2000: patients[4], + December2000: patients[5], + March31_2000: patients[6], + Year2001: patients[7], + November30_2001: patients[8], + December2001: patients[9], + December31_2001: patients[10], + January01_2002: patients[11]); + } + + private static void AssertNoDuplicateEntries(Bundle bundle) + { + List ids = bundle.Entry.Select(e => e.Resource.Id).ToList(); + Assert.Equal(ids.Count, ids.Distinct().Count()); + } + private static void SetPatientBirthDate(Patient patient, string birthDate, string tag) { patient.Meta = new Meta { Tag = new List { new Coding(null, tag) } }; patient.BirthDate = birthDate; } + + private sealed record PartialBirthdateMatrix( + Patient Year2000, + Patient March2000, + Patient March03, + Patient December31_1999, + Patient April01_2000, + Patient December2000, + Patient March31_2000, + Patient Year2001, + Patient November30_2001, + Patient December2001, + Patient December31_2001, + Patient January01_2002); } } From 7fe1340bffaeceb6f820ac63006fc354f14cfb18 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Wed, 17 Jun 2026 10:58:08 -0700 Subject: [PATCH 14/17] Potential fix for pull request finding 'CodeQL / Useless assignment to local variable' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .../Expressions/Visitors/ScalarTemporalEqualityRewriter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs index b8c1494c5f..bf0bbc0707 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs @@ -226,7 +226,7 @@ private static bool IsEndLe(BinaryExpression be) => private static bool ContainsChain(Expression expression) { - if (expression is ChainedExpression chained) + if (expression is ChainedExpression) { return true; } From 26e8ea4ddcba68d67ac69f217d65a1e3096b29c4 Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Wed, 17 Jun 2026 10:58:21 -0700 Subject: [PATCH 15/17] Potential fix for pull request finding 'CodeQL / Missed opportunity to use Where' Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .../ScalarTemporalEqualityRewriter.cs | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs index bf0bbc0707..40a595e488 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; +using System.Linq; using Microsoft.Health.Fhir.Core.Features.Search.Expressions; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions; @@ -233,26 +234,12 @@ private static bool ContainsChain(Expression expression) if (expression is MultiaryExpression multiary) { - foreach (Expression child in multiary.Expressions) - { - if (ContainsChain(child)) - { - return true; - } - } - - return false; + return multiary.Expressions.Any(ContainsChain); } if (expression is UnionExpression union) { - foreach (Expression child in union.Expressions) - { - if (ContainsChain(child)) - { - return true; - } - } + return union.Expressions.Any(ContainsChain); } return false; From 9484916bcd433e9c257416948312d7080d4b8d0b Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Wed, 17 Jun 2026 11:07:20 -0700 Subject: [PATCH 16/17] Revert "Merge origin/main and resolve conflicts" This reverts commit de597131e07c3ad2bf49367a9ecdd717216773be, reversing changes made to 26e8ea4ddcba68d67ac69f217d65a1e3096b29c4. --- ...-searchparameter-concurrency-management.md | 5 +- .../adr-2603-search-params-concurrency.md | 5 - .../Models/ListedCapabilityStatementTests.cs | 2 - .../CreateBulkDeleteHandlerTests.cs | 32 -- .../CreateBulkUpdateHandlerTests.cs | 4 - .../SearchParameterConcurrencyManagerTests.cs | 492 ++++++++++++++++++ .../Models/ListedCapabilityStatement.cs | 12 +- .../Context/FhirRequestContextExtensions.cs | 36 -- .../Handlers/CreateBulkDeleteHandler.cs | 6 +- .../Handlers/CreateBulkUpdateHandler.cs | 8 +- .../Reindex/ReindexOrchestratorJob.cs | 6 +- .../Reindex/ReindexProcessingJob.cs | 2 +- .../Persistence/ResourceWrapperOperation.cs | 3 +- .../CreateOrUpdateSearchParameterBehavior.cs | 56 +- .../DeleteSearchParameterBehavior.cs | 24 +- .../Parameters/ISearchParameterOperations.cs | 4 +- .../Parameters/SearchParameterOperations.cs | 248 +++++---- ...rchParameterRequestContextPropertyNames.cs | 3 +- .../Search/Parameters/SearchParameterRetry.cs | 57 -- .../Registry/ISearchParameterStatusManager.cs | 2 +- .../SearchParameterConcurrencyException.cs | 40 ++ .../SearchParameterConcurrencyManager.cs | 148 ++++++ .../Registry/SearchParameterStatusManager.cs | 4 +- .../Resources.Designer.cs | 18 - src/Microsoft.Health.Fhir.Core/Resources.resx | 8 +- .../Features/Storage/CosmosFhirDataStore.cs | 47 +- .../Controllers/FhirControllerTests.cs | 209 +------- .../Controllers/TerminologyControllerTests.cs | 16 - .../SearchParameterFilterAttributeTests.cs | 42 +- .../Controllers/FhirController.cs | 96 +--- ...ntextRouteDataPopulatingFilterAttribute.cs | 46 +- .../Filters/SearchParameterFilterAttribute.cs | 27 +- .../Resources/Bundle/BundleHandler.cs | 36 +- .../FirelyTerminologyServiceProxyTests.cs | 81 +-- .../Resources/Delete/DeletionServiceTests.cs | 116 ----- .../SearchParameterBehaviorTests.cs | 34 +- .../SearchParameterRetryTests.cs | 94 ---- .../SearchParameterValidatorTests.cs | 1 - ...ealth.Fhir.Shared.Core.UnitTests.projitems | 1 - .../FirelyTerminologyServiceProxy.cs | 6 - .../Conformance/TerminologyRequestHandler.cs | 4 - .../Resources/Delete/DeletionService.cs | 72 +-- .../Parameters/ISearchParameterValidator.cs | 4 +- .../Parameters/SearchParameterValidator.cs | 13 +- .../ScalarTemporalEqualityRewriterTests.cs | 1 + .../SqlServerSearchServiceQueryStoreTests.cs | 85 --- .../Features/ExceptionExtension.cs | 10 - .../Features/Schema/SchemaVersionConstants.cs | 4 +- .../Features/Search/SqlServerSearchService.cs | 372 ++++--------- ...SqlServerSearchParameterStatusDataStore.cs | 65 ++- .../Storage/SqlServerFhirDataStore.cs | 71 ++- .../SearchParamListRowGenerator.cs | 4 +- .../Microsoft.Health.Fhir.Tests.Common.csproj | 2 - .../Normative/Bundle-InvalidBundleType.json | 29 -- .../Rest/BundleEdgeCaseTests.cs | 10 - .../Rest/Conformance/ExpandOperationTests.cs | 38 -- .../Rest/Reindex/ReindexTests.cs | 268 +--------- .../Rest/Search/ChainingSearchTests.cs | 1 + .../FailingSearchParameterStatusManager.cs | 2 +- .../Operations/Reindex/ReindexJobTests.cs | 39 +- .../Smart/SmartSearchSharedFixture.cs | 140 ----- .../Features/Smart/SmartSearchTests.cs | 193 +++++-- ...th.Fhir.Shared.Tests.Integration.projitems | 3 +- .../Persistence/FhirStorageTests.cs | 12 +- .../Persistence/FhirStorageTestsFixture.cs | 59 +-- ...erOptimisticConcurrencyIntegrationTests.cs | 454 ++++++++++++++++ ...archParameterOptimisticConcurrencyTests.cs | 176 ------- .../SearchParameterStatusDataStoreTests.cs | 46 +- .../SqlServerCreateStatsForSmartTests.cs | 5 +- .../SqlServerFhirStorageTestHelper.cs | 13 +- ...rverSearchParameterStatusDataStoreTests.cs | 117 ++--- 71 files changed, 1978 insertions(+), 2411 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/Registry/SearchParameterConcurrencyManagerTests.cs delete mode 100644 src/Microsoft.Health.Fhir.Core/Features/Context/FhirRequestContextExtensions.cs delete mode 100644 src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterRetry.cs create mode 100644 src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterConcurrencyException.cs create mode 100644 src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterConcurrencyManager.cs delete mode 100644 src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterRetryTests.cs delete mode 100644 src/Microsoft.Health.Fhir.Tests.Common/TestFiles/Normative/Bundle-InvalidBundleType.json delete mode 100644 test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchSharedFixture.cs create mode 100644 test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SearchParameterOptimisticConcurrencyIntegrationTests.cs delete mode 100644 test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SearchParameterOptimisticConcurrencyTests.cs diff --git a/docs/arch/adr-2512-searchparameter-concurrency-management.md b/docs/arch/adr-2512-searchparameter-concurrency-management.md index 59148abd28..a5223aa748 100644 --- a/docs/arch/adr-2512-searchparameter-concurrency-management.md +++ b/docs/arch/adr-2512-searchparameter-concurrency-management.md @@ -1,6 +1,3 @@ -## Status -**Obsolete** - Code supporting this ADR removed per https://github.com/microsoft/fhir-server/blob/main/docs/arch/adr-2603-search-param-concurrency.md - # ADR 2512: SearchParameter Concurrency Management - Application Level Locking and Database Optimistic Concurrency *Labels*: [SQL](https://github.com/microsoft/fhir-server/labels/Area-SQL) | [Core](https://github.com/microsoft/fhir-server/labels/Area-Core) | [SearchParameter](https://github.com/microsoft/fhir-server/labels/Area-SearchParameter) @@ -102,7 +99,7 @@ HTTP Request ? CreateOrUpdateSearchParameterBehavior ? SearchParameterOperations - Proper LastUpdated handling in GET and UPSERT operations ## Status -**Obsolete** - Code supporting this ADR removed per https://github.com/microsoft/fhir-server/blob/main/docs/arch/adr-2603-search-param-concurrency.md +**Accepted** - Implemented and deployed ## Consequences diff --git a/docs/arch/adr-2603-search-params-concurrency.md b/docs/arch/adr-2603-search-params-concurrency.md index 423533fa6e..c77bbf71b1 100644 --- a/docs/arch/adr-2603-search-params-concurrency.md +++ b/docs/arch/adr-2603-search-params-concurrency.md @@ -38,8 +38,3 @@ We will implement optimistic concurrency across all search params based on max(L ## Notes This ADR superceeds previous implementation https://github.com/microsoft/fhir-server/blob/main/docs/arch/adr-2512-searchparameter-concurrency-management.md. Previous implementation should to be removed. - -## Message to customers -To guarantee store data integrity, FHIR server implemented strict concurrency control for search parameter writes. -When requests to write search parameters are sent in parallel, depending on timing, some requests might fail with concurrency conflict errors. This is because each write operation requires validation against the reference set, and concurrent modifications might lead to data integrity issues, therefore they are restricted. -When writing search parameters, avoid sending multiple parallel requests. If you need to process multiple search parameters, send requests one after another, or use a single bundle call. diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Conformance/Models/ListedCapabilityStatementTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Conformance/Models/ListedCapabilityStatementTests.cs index 680af81142..fc7b4d5b1a 100644 --- a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Conformance/Models/ListedCapabilityStatementTests.cs +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Conformance/Models/ListedCapabilityStatementTests.cs @@ -373,8 +373,6 @@ public async Task GivenAStatement_WhenClonedWhileCollectionsAreModified_ThenNoEx await Task.WhenAll(cloneTask, modifyTask); // Assert - // If this step fails the excpetions will be logged to the test output - // This can look like this step threw an exception, but it is just reporting an exception that happened in the clone or modify tasks Assert.Empty(exceptions); } } diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/BulkDelete/CreateBulkDeleteHandlerTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/BulkDelete/CreateBulkDeleteHandlerTests.cs index e9ba04d7ed..42717db532 100644 --- a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/BulkDelete/CreateBulkDeleteHandlerTests.cs +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/BulkDelete/CreateBulkDeleteHandlerTests.cs @@ -100,38 +100,6 @@ public async Task GivenBulkDeleteRequest_WhenJobCreationRequested_ThenJobIsCreat await _queueClient.ReceivedWithAnyArgs(1).EnqueueAsync((byte)QueueType.BulkDelete, Arg.Any(), Arg.Any(), false, Arg.Any()); } - [Fact] - public async Task GivenBulkDeleteRequestWithNoParameters_WhenJobCreationRequested_ThenJobIsCreated() - { - var searchParams = new List>(); - - _authorizationService.CheckAccess(Arg.Any(), Arg.Any()).Returns(DataActions.HardDelete | DataActions.Delete); - _contextAccessor.RequestContext.BundleIssues.Clear(); - _queueClient.EnqueueAsync((byte)QueueType.BulkDelete, Arg.Any(), Arg.Any(), false, Arg.Any()).Returns(args => - { - var definition = JsonConvert.DeserializeObject(args.ArgAt(1)[0]); - Assert.Equal(_testUrl, definition.Url); - Assert.Equal(_testUrl, definition.BaseUrl); - Assert.Equal(DeleteOperation.HardDelete, definition.DeleteOperation); - Assert.Equal(searchParams.Count, definition.SearchParameters.Count); - - return new List() - { - new JobInfo() - { - Id = 1, - }, - }; - }); - - var request = new CreateBulkDeleteRequest(DeleteOperation.HardDelete, KnownResourceTypes.Patient, searchParams, false, null, false); - - var response = await _handler.Handle(request, CancellationToken.None); - Assert.NotNull(response); - Assert.Equal(1, response.Id); - await _queueClient.ReceivedWithAnyArgs(1).EnqueueAsync((byte)QueueType.BulkDelete, Arg.Any(), Arg.Any(), false, Arg.Any()); - } - [Fact] public async Task GivenBulkDeleteRequestWithInvalidSearchParameter_WhenJobCreationRequested_ThenBadRequestIsReturned() { diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/BulkUpdate/CreateBulkUpdateHandlerTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/BulkUpdate/CreateBulkUpdateHandlerTests.cs index 2787411e25..7c7a5623e4 100644 --- a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/BulkUpdate/CreateBulkUpdateHandlerTests.cs +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/BulkUpdate/CreateBulkUpdateHandlerTests.cs @@ -348,10 +348,6 @@ public static IEnumerable GetSearchParamsForJobCreation() new Tuple("_lastUpdated", "value3"), }, }; - yield return new object[] - { - new List>(), - }; } } } diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/Registry/SearchParameterConcurrencyManagerTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/Registry/SearchParameterConcurrencyManagerTests.cs new file mode 100644 index 0000000000..d5a9e08a64 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Search/Registry/SearchParameterConcurrencyManagerTests.cs @@ -0,0 +1,492 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Health.Fhir.Core.Features.Search.Registry; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using Xunit; + +namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Search.Registry +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Search)] + public class SearchParameterConcurrencyManagerTests + { + private const string TestUri1 = "http://test.com/searchparam1"; + private const string TestUri2 = "http://test.com/searchparam2"; + + [Fact] + public async Task GivenSingleSearchParameterUri_WhenExecutingWithLock_ThenOperationCompletes() + { + // Arrange + const string expectedResult = "test result"; + var executionCount = 0; + + // Act + var result = await SearchParameterConcurrencyManager.ExecuteWithLockAsync(TestUri1, () => + { + executionCount++; + return Task.FromResult(expectedResult); + }); + + // Assert + Assert.Equal(expectedResult, result); + Assert.Equal(1, executionCount); + } + + [Fact] + public async Task GivenSingleSearchParameterUri_WhenExecutingWithLockVoidOperation_ThenOperationCompletes() + { + // Arrange + var executionCount = 0; + + // Act + await SearchParameterConcurrencyManager.ExecuteWithLockAsync(TestUri1, () => + { + executionCount++; + return Task.CompletedTask; + }); + + // Assert + Assert.Equal(1, executionCount); + } + + [Fact] + public async Task GivenSameSearchParameterUri_WhenExecutingConcurrently_ThenOperationsExecuteSequentially() + { + // Arrange + const int concurrentOperations = 5; + var executionOrder = new List(); + var lockObject = new object(); + var entryBarriers = new List>(); + var continueSignals = new List>(); + + // Create synchronization primitives for each operation + for (int i = 0; i < concurrentOperations; i++) + { + entryBarriers.Add(new TaskCompletionSource()); + continueSignals.Add(new TaskCompletionSource()); + } + + // Act + var tasks = new List>(); + for (int i = 0; i < concurrentOperations; i++) + { + var operationId = i; + var entryBarrier = entryBarriers[i]; + var continueSignal = continueSignals[i]; + + tasks.Add(SearchParameterConcurrencyManager.ExecuteWithLockAsync(TestUri1, async () => + { + lock (lockObject) + { + executionOrder.Add(operationId); + } + + // Signal that this operation has entered the critical section + entryBarrier.SetResult(true); + + // Wait for permission to continue (controlled by test) + await continueSignal.Task; + + return operationId; + })); + } + + // Verify operations execute sequentially by controlling their execution + for (int i = 0; i < concurrentOperations; i++) + { + // Wait for the next operation to enter the critical section + await entryBarriers[i].Task; + + // Verify only this operation has executed so far + lock (lockObject) + { + Assert.Equal(i + 1, executionOrder.Count); + Assert.Equal(i, executionOrder[i]); + } + + // Allow this operation to complete + continueSignals[i].SetResult(true); + } + + var results = await Task.WhenAll(tasks); + + // Assert + Assert.Equal(concurrentOperations, results.Length); + Assert.Equal(concurrentOperations, executionOrder.Count); + + // Verify all operations completed in the correct order + for (int i = 0; i < concurrentOperations; i++) + { + Assert.Equal(i, executionOrder[i]); + Assert.Equal(i, results[i]); + } + } + + [Fact] + public async Task GivenDifferentSearchParameterUris_WhenExecutingConcurrently_ThenOperationsExecuteInParallel() + { + // Arrange + const int operationsPerUri = 2; + var uri1StartedCount = 0; + var uri2StartedCount = 0; + var bothUrisStarted = new TaskCompletionSource(); + var canContinue = new TaskCompletionSource(); + var lockObject = new object(); + + // Act + var tasks = new List(); + + // Add operations for first URI + for (int i = 0; i < operationsPerUri; i++) + { + tasks.Add(SearchParameterConcurrencyManager.ExecuteWithLockAsync(TestUri1, async () => + { + bool shouldSignal = false; + lock (lockObject) + { + uri1StartedCount++; + + // Signal when both URIs have at least one operation started + if (uri1StartedCount > 0 && uri2StartedCount > 0) + { + shouldSignal = true; + } + } + + if (shouldSignal) + { + bothUrisStarted.TrySetResult(true); + } + + await canContinue.Task; + })); + } + + // Add operations for second URI + for (int i = 0; i < operationsPerUri; i++) + { + tasks.Add(SearchParameterConcurrencyManager.ExecuteWithLockAsync(TestUri2, async () => + { + bool shouldSignal = false; + lock (lockObject) + { + uri2StartedCount++; + + // Signal when both URIs have at least one operation started + if (uri1StartedCount > 0 && uri2StartedCount > 0) + { + shouldSignal = true; + } + } + + if (shouldSignal) + { + bothUrisStarted.TrySetResult(true); + } + + await canContinue.Task; + })); + } + + // Wait for operations on both URIs to start concurrently + await bothUrisStarted.Task; + + // Verify both URIs have operations running concurrently + lock (lockObject) + { + Assert.True(uri1StartedCount > 0, "At least one operation on URI1 should have started"); + Assert.True(uri2StartedCount > 0, "At least one operation on URI2 should have started"); + } + + // Allow all operations to complete + canContinue.SetResult(true); + await Task.WhenAll(tasks); + + // Assert final counts + Assert.Equal(operationsPerUri, uri1StartedCount); + Assert.Equal(operationsPerUri, uri2StartedCount); + } + + [Fact] + public void GivenInitialState_WhenCheckingActiveLockCount_ThenReturnsZero() + { + // Act & Assert + Assert.Equal(0, SearchParameterConcurrencyManager.ActiveLockCount); + } + + [Fact] + public async Task GivenOperationsInProgress_WhenCheckingActiveLockCount_ThenReturnsCorrectCount() + { + // Arrange + var task1Started = new TaskCompletionSource(); + var task1CanContinue = new TaskCompletionSource(); + var task2Started = new TaskCompletionSource(); + var task2CanContinue = new TaskCompletionSource(); + + // Act - Start two operations on different URIs + var task1 = SearchParameterConcurrencyManager.ExecuteWithLockAsync(TestUri1, async () => + { + task1Started.SetResult(true); + await task1CanContinue.Task; + }); + + var task2 = SearchParameterConcurrencyManager.ExecuteWithLockAsync(TestUri2, async () => + { + task2Started.SetResult(true); + await task2CanContinue.Task; + }); + + // Wait for both operations to start + await task1Started.Task; + await task2Started.Task; + + // Assert - Should have 2 active locks + Assert.Equal(2, SearchParameterConcurrencyManager.ActiveLockCount); + + // Complete first operation + task1CanContinue.SetResult(true); + await task1; + + // Wait for cleanup with exponential backoff instead of fixed delay + var activeLockCount = SearchParameterConcurrencyManager.ActiveLockCount; + var attempts = 0; + while (activeLockCount != 1 && attempts < 10) + { + await Task.Delay(10 * (int)Math.Pow(2, attempts)); + activeLockCount = SearchParameterConcurrencyManager.ActiveLockCount; + attempts++; + } + + // Assert - Should have 1 active lock + Assert.Equal(1, activeLockCount); + + // Complete second operation + task2CanContinue.SetResult(true); + await task2; + + // Wait for final cleanup with exponential backoff + activeLockCount = SearchParameterConcurrencyManager.ActiveLockCount; + attempts = 0; + while (activeLockCount != 0 && attempts < 10) + { + await Task.Delay(10 * (int)Math.Pow(2, attempts)); + activeLockCount = SearchParameterConcurrencyManager.ActiveLockCount; + attempts++; + } + + // Assert - Should have 0 active locks + Assert.Equal(0, activeLockCount); + } + + [Fact] + public async Task GivenExceptionInOperation_WhenExecutingWithLock_ThenExceptionIsPropagatedAndLockIsReleased() + { + // Arrange + var expectedException = new InvalidOperationException("Test exception"); + + // Act & Assert + var actualException = await Assert.ThrowsAsync(() => + SearchParameterConcurrencyManager.ExecuteWithLockAsync(TestUri1, () => + { + throw expectedException; + })); + + Assert.Same(expectedException, actualException); + + // Wait for cleanup with exponential backoff instead of fixed delay + var activeLockCount = SearchParameterConcurrencyManager.ActiveLockCount; + var attempts = 0; + while (activeLockCount != 0 && attempts < 10) + { + await Task.Delay(10 * (int)Math.Pow(2, attempts)); + activeLockCount = SearchParameterConcurrencyManager.ActiveLockCount; + attempts++; + } + + // Verify lock is released by ensuring a subsequent operation can execute + var subsequentExecuted = false; + await SearchParameterConcurrencyManager.ExecuteWithLockAsync(TestUri1, () => + { + subsequentExecuted = true; + return Task.CompletedTask; + }); + + Assert.True(subsequentExecuted); + Assert.Equal(0, SearchParameterConcurrencyManager.ActiveLockCount); + } + + [Fact] + public async Task GivenCancellationToken_WhenOperationIsCancelled_ThenOperationCancelsAndLockIsReleased() + { + // Arrange + var operationStarted = new TaskCompletionSource(); + using var cts = new CancellationTokenSource(); + + // Act + var operationTask = SearchParameterConcurrencyManager.ExecuteWithLockAsync(TestUri1, async () => + { + operationStarted.SetResult(true); + await Task.Delay(10000, cts.Token); // Long delay that will be cancelled + }); + + // Wait for operation to start, then cancel it + await operationStarted.Task; + cts.Cancel(); + + // Assert + await Assert.ThrowsAsync(() => operationTask); + + // Wait for cleanup with exponential backoff instead of fixed delay + var activeLockCount = SearchParameterConcurrencyManager.ActiveLockCount; + var attempts = 0; + while (activeLockCount != 0 && attempts < 10) + { + await Task.Delay(10 * (int)Math.Pow(2, attempts)); + activeLockCount = SearchParameterConcurrencyManager.ActiveLockCount; + attempts++; + } + + Assert.Equal(0, activeLockCount); + } + + [Fact] + public async Task GivenLongRunningOperations_WhenExecutingManySequentiallyOnSameUri_ThenMemoryIsReclaimed() + { + // Arrange + const int iterations = 100; + + // Act - Execute many operations sequentially + for (int i = 0; i < iterations; i++) + { + await SearchParameterConcurrencyManager.ExecuteWithLockAsync($"{TestUri1}_{i}", async () => + { + await Task.Delay(1); // Minimal work + }); + } + + // Wait for cleanup with exponential backoff instead of fixed delay + var activeLockCount = SearchParameterConcurrencyManager.ActiveLockCount; + var attempts = 0; + while (activeLockCount >= 10 && attempts < 10) + { + await Task.Delay(10 * (int)Math.Pow(2, attempts)); + activeLockCount = SearchParameterConcurrencyManager.ActiveLockCount; + attempts++; + } + + // Assert - Should not have accumulated locks + Assert.True( + activeLockCount < 10, + $"Expected less than 10 active locks but found {activeLockCount}"); + } + + [Fact] + public async Task GivenMultipleThreads_WhenExecutingSameOperation_ThenOnlyOneExecutes() + { + // Arrange + var executionCount = 0; + using var barrier = new Barrier(3); // 3 threads will hit this barrier + const int threadCount = 3; + + // Act + var tasks = new Task[threadCount]; + for (int i = 0; i < threadCount; i++) + { + tasks[i] = Task.Run(async () => + { + barrier.SignalAndWait(); // Ensure all threads start simultaneously + + await SearchParameterConcurrencyManager.ExecuteWithLockAsync(TestUri1, async () => + { + var currentCount = Interlocked.Increment(ref executionCount); + await Task.Delay(10); // Simulate work + return currentCount; + }); + }); + } + + await Task.WhenAll(tasks); + + // Assert + Assert.Equal(threadCount, executionCount); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task GivenInvalidUri_WhenExecutingWithLock_ThenThrowsArgumentException(string invalidUri) + { + // Act & Assert + await Assert.ThrowsAsync(() => + SearchParameterConcurrencyManager.ExecuteWithLockAsync(invalidUri, () => Task.FromResult(1))); + } + + [Fact] + public async Task GivenNullOperation_WhenExecutingWithLock_ThenThrowsArgumentNullException() + { + // Act & Assert + await Assert.ThrowsAsync(() => + SearchParameterConcurrencyManager.ExecuteWithLockAsync(TestUri1, null)); + + await Assert.ThrowsAsync(() => + SearchParameterConcurrencyManager.ExecuteWithLockAsync(TestUri1, null)); + } + + [Fact] + public void GivenSearchParameterConcurrencyException_WhenCreatedWithUris_ThenPropertiesAreSet() + { + // Arrange + var uris = new[] { "http://test.com/param1", "http://test.com/param2" }; + + // Act + var exception = new SearchParameterConcurrencyException(uris); + + // Assert + Assert.NotNull(exception.Message); + Assert.Contains("param1", exception.Message); + Assert.Contains("param2", exception.Message); + Assert.Equal(2, exception.ConflictedUris.Count); + Assert.Contains("http://test.com/param1", exception.ConflictedUris); + Assert.Contains("http://test.com/param2", exception.ConflictedUris); + } + + [Fact] + public void GivenSearchParameterConcurrencyException_WhenCreatedWithMessage_ThenMessageIsSet() + { + // Arrange + const string expectedMessage = "Test concurrency error"; + + // Act + var exception = new SearchParameterConcurrencyException(expectedMessage); + + // Assert + Assert.Equal(expectedMessage, exception.Message); + Assert.Empty(exception.ConflictedUris); + } + + [Fact] + public void GivenSearchParameterConcurrencyException_WhenCreatedWithMessageAndInnerException_ThenBothAreSet() + { + // Arrange + const string expectedMessage = "Test concurrency error with inner exception"; + var innerException = new InvalidOperationException("Inner exception"); + + // Act + var exception = new SearchParameterConcurrencyException(expectedMessage, innerException); + + // Assert + Assert.Equal(expectedMessage, exception.Message); + Assert.Same(innerException, exception.InnerException); + Assert.Empty(exception.ConflictedUris); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Conformance/Models/ListedCapabilityStatement.cs b/src/Microsoft.Health.Fhir.Core/Features/Conformance/Models/ListedCapabilityStatement.cs index d575b4364d..e871605e4b 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Conformance/Models/ListedCapabilityStatement.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Conformance/Models/ListedCapabilityStatement.cs @@ -6,7 +6,6 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; -using System.Linq; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Models; using Newtonsoft.Json; @@ -110,20 +109,21 @@ public ListedCapabilityStatement Clone() /// /// Generic data type being copied. /// Origin. - /// Destination. - private static void SafeCopyTo(ICollection origin, ICollection destination) + /// Destiny. + private static void SafeCopyTo(ICollection origin, ICollection destiny) { if (origin == null) { - destination = null; + destiny = null; return; } - T[] temp = origin.ToArray(); + T[] temp = new T[origin.Count]; + origin.CopyTo(temp, 0); foreach (T item in temp) { - destination.Add(item); + destiny.Add(item); } } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Context/FhirRequestContextExtensions.cs b/src/Microsoft.Health.Fhir.Core/Features/Context/FhirRequestContextExtensions.cs deleted file mode 100644 index e702958071..0000000000 --- a/src/Microsoft.Health.Fhir.Core/Features/Context/FhirRequestContextExtensions.cs +++ /dev/null @@ -1,36 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System; -using Microsoft.Health.Fhir.Core.Features.Search.Parameters; - -namespace Microsoft.Health.Fhir.Core.Features.Context -{ - public static class FhirRequestContextExtensions - { - public static DateTimeOffset? GetSearchParameterLastUpdated(this IFhirRequestContext context) - { - if (context?.Properties.TryGetValue(SearchParameterRequestContextPropertyNames.LastUpdated, out var value) == true) - { - return (DateTimeOffset)value; - } - - return null; - } - - public static void SetSearchParameterLastUpdated(this IFhirRequestContext context, DateTimeOffset? lastUpdated) - { - if (lastUpdated.HasValue && context != null) - { - context.Properties[SearchParameterRequestContextPropertyNames.LastUpdated] = lastUpdated.Value; - } - } - - public static void ClearSearchParameterLastUpdated(this IFhirRequestContext context) - { - context?.Properties.Remove(SearchParameterRequestContextPropertyNames.LastUpdated); - } - } -} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/BulkDelete/Handlers/CreateBulkDeleteHandler.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/BulkDelete/Handlers/CreateBulkDeleteHandler.cs index 20144ef874..b7a580a37a 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/BulkDelete/Handlers/CreateBulkDeleteHandler.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/BulkDelete/Handlers/CreateBulkDeleteHandler.cs @@ -64,10 +64,6 @@ public async Task Handle(CreateBulkDeleteRequest reque var searchParameters = new List>(request.ConditionalParameters); - // Temporarily add _lastUpdated to the search parameters to mimic the behavior of the processing job. Conditional search will also fail if there are no search criteria. - var dateCurrent = new PartialDateTime(Clock.UtcNow); - searchParameters.Add(Tuple.Create("_lastUpdated", $"lt{dateCurrent}")); - // Should not run bulk delete if any of the search parameters are invalid as it can lead to unpredicatable results await _searchService.ConditionalSearchAsync(request.ResourceType, searchParameters, cancellationToken, count: 1, logger: _logger); if (_contextAccessor.RequestContext?.BundleIssues?.Count > 0 && _contextAccessor.RequestContext.BundleIssues.Any(x => !string.Equals(x.Diagnostics, Core.Resources.TruncatedIncludeMessageForIncludes, StringComparison.OrdinalIgnoreCase))) @@ -79,7 +75,7 @@ public async Task Handle(CreateBulkDeleteRequest reque JobType.BulkDeleteOrchestrator, request.DeleteOperation, request.ResourceType, - request.ConditionalParameters, + searchParameters, request.ExcludedResourceTypes, _contextAccessor.RequestContext.Uri.ToString(), _contextAccessor.RequestContext.BaseUri.ToString(), diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/BulkUpdate/Handlers/CreateBulkUpdateHandler.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/BulkUpdate/Handlers/CreateBulkUpdateHandler.cs index 6000be7757..968b739654 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/BulkUpdate/Handlers/CreateBulkUpdateHandler.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/BulkUpdate/Handlers/CreateBulkUpdateHandler.cs @@ -86,12 +86,6 @@ public async Task Handle(CreateBulkUpdateRequest reque // Remove bulk update specific parameters from search parameters searchParameters.RemoveAll(t => BulkUpdateQueryParameters.Any(param => param.Equals(t.Item1, StringComparison.OrdinalIgnoreCase))); - var searchParameterCopy = searchParameters.Select(t => Tuple.Create(t.Item1, t.Item2)).ToList(); - - // Temporarily add _lastUpdated to the search parameters to mimic the behavior of the processing job. Conditional search will also fail if there are no search criteria. - var dateCurrent = new PartialDateTime(Clock.UtcNow); - searchParameters.Add(Tuple.Create("_lastUpdated", $"lt{dateCurrent}")); - // Should not run bulk Update if any of the search parameters are invalid as it can lead to unpredicatable results await _searchService.ConditionalSearchAsync(request.ResourceType, searchParameters, cancellationToken, count: 1, logger: _logger); if (_contextAccessor.RequestContext?.BundleIssues?.Count > 0 && _contextAccessor.RequestContext.BundleIssues.Any(x => !string.Equals(x.Diagnostics, Core.Resources.TruncatedIncludeMessageForIncludes, StringComparison.OrdinalIgnoreCase))) @@ -116,7 +110,7 @@ public async Task Handle(CreateBulkUpdateRequest reque var processingDefinition = new BulkUpdateDefinition( JobType.BulkUpdateOrchestrator, request.ResourceType, - searchParameterCopy, + searchParameters, _contextAccessor.RequestContext.Uri.ToString(), _contextAccessor.RequestContext.BaseUri.ToString(), _contextAccessor.RequestContext.CorrelationId, diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexOrchestratorJob.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexOrchestratorJob.cs index aa2d1fa6c4..7cadc99afc 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexOrchestratorJob.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexOrchestratorJob.cs @@ -221,7 +221,8 @@ private async Task RefreshSearchParameterCache(bool isReindexStart) await Task.Delay(delayMs, _cancellationToken); } - _searchParamLastUpdated = _searchParameterOperations.SearchParamLastUpdated; + var currentDate = _searchParameterOperations.SearchParamLastUpdated.HasValue ? _searchParameterOperations.SearchParamLastUpdated.Value : DateTimeOffset.MinValue; + _searchParamLastUpdated = currentDate; _logger.LogJobInformation(_jobInfo, $"Reindex orchestrator job completed cache refresh at the {suffix}: SearchParamLastUpdated {_searchParamLastUpdated}"); await TryLogEvent($"ReindexOrchestratorJob={_jobInfo.Id}.ExecuteAsync.{suffix}", "Warn", $"SearchParamLastUpdated={_searchParamLastUpdated.ToString("yyyy-MM-dd HH:mm:ss.fff")}", null, _cancellationToken); @@ -239,7 +240,8 @@ async Task WaitForAllInstancesCacheSyncAsync(DateTime updateEventsSince, C if (result.IsConsistent) { - _logger.LogJobInformation(_jobInfo, $"Cache sync check: All {result.ActiveHosts} active host(s) have converged to SearchParamLastUpdated={_searchParameterOperations.SearchParamLastUpdated.ToString("yyyy-MM-dd HH:mm:ss.fff")}."); + var logDate = _searchParameterOperations.SearchParamLastUpdated.HasValue ? _searchParameterOperations.SearchParamLastUpdated.Value : DateTimeOffset.MinValue; + _logger.LogJobInformation(_jobInfo, $"Cache sync check: All {result.ActiveHosts} active host(s) have converged to SearchParamLastUpdated={logDate.ToString("yyyy-MM-dd HH:mm:ss.fff")}."); break; } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexProcessingJob.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexProcessingJob.cs index e0ce134a50..3d9caf33ea 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexProcessingJob.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexProcessingJob.cs @@ -143,7 +143,7 @@ private async Task CheckDiscrepancies(CancellationToken cancellationToken) // use the same value as used in resource writes _searchParameterHash = searchParameterHash; - var currentDate = _searchParameterOperations.SearchParamLastUpdated; + var currentDate = _searchParameterOperations.SearchParamLastUpdated.HasValue ? _searchParameterOperations.SearchParamLastUpdated.Value : DateTimeOffset.MinValue; var current = currentDate.ToString("yyyy-MM-dd HH:mm:ss.fff"); var requested = _reindexProcessingJobDefinition.SearchParamLastUpdated.ToString("yyyy-MM-dd HH:mm:ss.fff"); isBad = _reindexProcessingJobDefinition.SearchParamLastUpdated > currentDate; diff --git a/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceWrapperOperation.cs b/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceWrapperOperation.cs index 94bcbd567c..eb361489f8 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceWrapperOperation.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceWrapperOperation.cs @@ -3,6 +3,7 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System.Collections.Generic; using EnsureThat; using Microsoft.Health.Fhir.Core.Features.Search.Registry; using Microsoft.Health.Fhir.Core.Models; @@ -47,7 +48,7 @@ public ResourceWrapperOperation( public BundleResourceContext BundleResourceContext { get; } - public ResourceSearchParameterStatus PendingSearchParameterStatus { get; internal set; } + public IReadOnlyList PendingSearchParameterStatuses { get; internal set; } #pragma warning disable CA1024 // Use properties where appropriate public DataStoreOperationIdentifier GetIdentifier() diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/CreateOrUpdateSearchParameterBehavior.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/CreateOrUpdateSearchParameterBehavior.cs index b4d8194a50..e5ed9aa545 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/CreateOrUpdateSearchParameterBehavior.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/CreateOrUpdateSearchParameterBehavior.cs @@ -15,7 +15,6 @@ using Microsoft.Health.Fhir.Core.Exceptions; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Context; -using Microsoft.Health.Fhir.Core.Features.Definition; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Search.Registry; using Microsoft.Health.Fhir.Core.Messages.Create; @@ -29,26 +28,26 @@ public class CreateOrUpdateSearchParameterBehavior _requestContextAccessor; private readonly IModelInfoProvider _modelInfoProvider; - private readonly ISearchParameterDefinitionManager _searchParameterDefinitionManager; public CreateOrUpdateSearchParameterBehavior( ISearchParameterOperations searchParameterOperations, IFhirDataStore fhirDataStore, - ISearchParameterDefinitionManager searchParameterDefinitionManager, + ISearchParameterStatusManager searchParameterStatusManager, RequestContextAccessor requestContextAccessor, IModelInfoProvider modelInfoProvider) { EnsureArg.IsNotNull(searchParameterOperations, nameof(searchParameterOperations)); EnsureArg.IsNotNull(fhirDataStore, nameof(fhirDataStore)); - EnsureArg.IsNotNull(searchParameterDefinitionManager, nameof(searchParameterDefinitionManager)); + EnsureArg.IsNotNull(searchParameterStatusManager, nameof(searchParameterStatusManager)); EnsureArg.IsNotNull(requestContextAccessor, nameof(requestContextAccessor)); EnsureArg.IsNotNull(modelInfoProvider, nameof(modelInfoProvider)); _searchParameterOperations = searchParameterOperations; _fhirDataStore = fhirDataStore; - _searchParameterDefinitionManager = searchParameterDefinitionManager; + _searchParameterStatusManager = searchParameterStatusManager; _requestContextAccessor = requestContextAccessor; _modelInfoProvider = modelInfoProvider; } @@ -57,13 +56,13 @@ public async Task Handle(CreateResourceRequest request, { if (request.Resource.InstanceType.Equals(KnownResourceTypes.SearchParameter, StringComparison.Ordinal)) { - // Before committing the SearchParameter resource to the data store, validate the parameter type - var lastUpdated = await _searchParameterOperations.ValidateSearchParameterAsync(request.Resource.Instance, cancellationToken, _requestContextAccessor.RequestContext.GetSearchParameterLastUpdated()); + var refreshCache = request.BundleResourceContext == null || !request.BundleResourceContext.IsParallelBundle; - QueueStatus(request.Resource.Instance.GetStringScalar("url"), SearchParameterStatus.Supported, lastUpdated); + // Before committing the SearchParameter resource to the data store, validate the parameter type + await _searchParameterOperations.ValidateSearchParameterAsync(request.Resource.Instance, cancellationToken, refreshCache); - // Allow the resource to be updated with the normal handler - return await next(cancellationToken); + var url = request.Resource.Instance.GetStringScalar("url"); + await QueueStatusAsync(url, SearchParameterStatus.Supported, cancellationToken); } // Allow the resource to be updated with the normal handler @@ -91,35 +90,38 @@ public async Task Handle(UpsertResourceRequest request, prevSearchParamResource = null; } - var lastUpdated = await _searchParameterOperations.ValidateSearchParameterAsync(request.Resource.Instance, cancellationToken, _requestContextAccessor.RequestContext.GetSearchParameterLastUpdated()); + var refreshCache = request.BundleResourceContext == null || !request.BundleResourceContext.IsParallelBundle; if (prevSearchParamResource != null && prevSearchParamResource.IsDeleted == false) { + // Validate any changes to the fhirpath or the datatype + await _searchParameterOperations.ValidateSearchParameterAsync(request.Resource.Instance, cancellationToken, refreshCache); + var previousUrl = _modelInfoProvider.ToTypedElement(prevSearchParamResource.RawResource).GetStringScalar("url"); var newUrl = request.Resource.Instance.GetStringScalar("url"); if (!string.IsNullOrWhiteSpace(previousUrl) && !previousUrl.Equals(newUrl, StringComparison.Ordinal)) { - QueueStatus(previousUrl, SearchParameterStatus.Deleted, lastUpdated); + await QueueStatusAsync(previousUrl, SearchParameterStatus.Deleted, cancellationToken); } - QueueStatus(newUrl, SearchParameterStatus.Supported, lastUpdated); + await QueueStatusAsync(newUrl, SearchParameterStatus.Supported, cancellationToken); } else { // No previous version exists or it was deleted, so add it as a new SearchParameter - QueueStatus(request.Resource.Instance.GetStringScalar("url"), SearchParameterStatus.Supported, lastUpdated); - } + await _searchParameterOperations.ValidateSearchParameterAsync(request.Resource.Instance, cancellationToken, refreshCache); - // Now allow the resource to updated per the normal behavior - return await next(cancellationToken); + var url = request.Resource.Instance.GetStringScalar("url"); + await QueueStatusAsync(url, SearchParameterStatus.Supported, cancellationToken); + } } // Now allow the resource to updated per the normal behavior return await next(cancellationToken); } - private void QueueStatus(string url, SearchParameterStatus status, DateTimeOffset lastUpdated) + private async Task QueueStatusAsync(string url, SearchParameterStatus status, CancellationToken cancellationToken) { if (string.IsNullOrWhiteSpace(url)) { @@ -132,18 +134,30 @@ private void QueueStatus(string url, SearchParameterStatus status, DateTimeOffse return; } - _searchParameterDefinitionManager.TryGetSearchParameter(url, out var existing); + if (!context.Properties.TryGetValue(SearchParameterRequestContextPropertyNames.PendingStatusUpdates, out var value) || + value is not List pendingStatuses) + { + pendingStatuses = new List(); + context.Properties[SearchParameterRequestContextPropertyNames.PendingStatusUpdates] = pendingStatuses; + } + + var currentStatuses = await _searchParameterStatusManager.GetAllSearchParameterStatus(cancellationToken); + var existing = currentStatuses.FirstOrDefault(s => string.Equals(s.Uri?.OriginalString, url, StringComparison.Ordinal)); var update = new ResourceSearchParameterStatus { Uri = new Uri(url), Status = status, - LastUpdated = lastUpdated, + LastUpdated = existing?.LastUpdated ?? DateTimeOffset.UtcNow, IsPartiallySupported = existing?.IsPartiallySupported ?? false, SortStatus = existing?.SortStatus ?? SortParameterStatus.Disabled, }; - context.Properties[SearchParameterRequestContextPropertyNames.PendingStatus] = update; + lock (pendingStatuses) + { + pendingStatuses.RemoveAll(s => string.Equals(s.Uri?.OriginalString, url, StringComparison.Ordinal)); + pendingStatuses.Add(update); + } } } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/DeleteSearchParameterBehavior.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/DeleteSearchParameterBehavior.cs index 7d36ecd364..7be8c387dc 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/DeleteSearchParameterBehavior.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/DeleteSearchParameterBehavior.cs @@ -60,6 +60,7 @@ public DeleteSearchParameterBehavior( public async Task Handle(TDeleteResourceRequest request, RequestHandlerDelegate next, CancellationToken cancellationToken) { var deleteRequest = request as DeleteResourceRequest; + ResourceWrapper searchParamResource = null; if (deleteRequest.ResourceKey.ResourceType.Equals(KnownResourceTypes.SearchParameter, StringComparison.Ordinal)) { @@ -75,7 +76,7 @@ public async Task Handle(TDeleteResourceRequest request } // Now try to get the custom search parameter from the data store - var searchParamResource = await _fhirDataStore.GetAsync(deleteRequest.ResourceKey, cancellationToken); + searchParamResource = await _fhirDataStore.GetAsync(deleteRequest.ResourceKey, cancellationToken); if (searchParamResource == null) { @@ -89,8 +90,6 @@ public async Task Handle(TDeleteResourceRequest request var url = typed.GetStringScalar("url"); await QueuePendingDeleteStatusAsync(url, cancellationToken); } - - return await next(cancellationToken); } return await next(cancellationToken); @@ -109,25 +108,30 @@ private async Task QueuePendingDeleteStatusAsync(string url, CancellationToken c return; } - var lastUpdated = _requestContextAccessor.RequestContext.GetSearchParameterLastUpdated(); - if (!lastUpdated.HasValue) + if (!context.Properties.TryGetValue(SearchParameterRequestContextPropertyNames.PendingStatusUpdates, out var value) || + value is not List pendingStatuses) { - await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken); - lastUpdated = _searchParameterOperations.SearchParamLastUpdated; + pendingStatuses = new List(); + context.Properties[SearchParameterRequestContextPropertyNames.PendingStatusUpdates] = pendingStatuses; } - _searchParameterDefinitionManager.TryGetSearchParameter(url, out var existing); + var currentStatuses = await _searchParameterStatusManager.GetAllSearchParameterStatus(cancellationToken); + var existing = currentStatuses.FirstOrDefault(s => string.Equals(s.Uri?.OriginalString, url, StringComparison.Ordinal)); var update = new ResourceSearchParameterStatus { Uri = new Uri(url), Status = SearchParameterStatus.PendingDelete, - LastUpdated = lastUpdated.Value, + LastUpdated = existing?.LastUpdated ?? DateTimeOffset.UtcNow, IsPartiallySupported = existing?.IsPartiallySupported ?? false, SortStatus = existing?.SortStatus ?? SortParameterStatus.Disabled, }; - context.Properties[SearchParameterRequestContextPropertyNames.PendingStatus] = update; + lock (pendingStatuses) + { + pendingStatuses.RemoveAll(s => string.Equals(s.Uri?.OriginalString, url, StringComparison.Ordinal)); + pendingStatuses.Add(update); + } } } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/ISearchParameterOperations.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/ISearchParameterOperations.cs index 87e2b50779..f95d23eee0 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/ISearchParameterOperations.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/ISearchParameterOperations.cs @@ -15,11 +15,11 @@ namespace Microsoft.Health.Fhir.Core.Features.Search.Parameters { public interface ISearchParameterOperations { - DateTimeOffset SearchParamLastUpdated { get; } + DateTimeOffset? SearchParamLastUpdated { get; } Task DeleteSearchParameterAsync(RawResource searchParamResource, CancellationToken cancellationToken, bool ignoreSearchParameterNotSupportedException = false); - Task ValidateSearchParameterAsync(ITypedElement searchParam, CancellationToken cancellationToken, DateTimeOffset? lastUpdated = null); + Task ValidateSearchParameterAsync(ITypedElement searchParam, CancellationToken cancellationToken, bool refreshCache = true); Task UpdateSearchParameterStatusAsync(IReadOnlyCollection searchParameterUris, SearchParameterStatus status, CancellationToken cancellationToken, bool ignoreSearchParameterNotSupportedException = false); diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs index fc12d5f58a..44c0aa78c8 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterOperations.cs @@ -5,9 +5,11 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; +using DotLiquid.Tags.Html; using EnsureThat; using Hl7.Fhir.ElementModel; using Microsoft.Extensions.Logging; @@ -69,18 +71,7 @@ public SearchParameterOperations( _refreshSemaphore = new SemaphoreSlim(1, 1); } - public DateTimeOffset SearchParamLastUpdated - { - get - { - if (!_searchParamLastUpdated.HasValue) - { - throw new InvalidOperationException("Search param cache has not been updated yet."); - } - - return _searchParamLastUpdated.Value; - } - } + public DateTimeOffset? SearchParamLastUpdated => _searchParamLastUpdated; public string GetSearchParameterHash(string resourceType) { @@ -103,79 +94,82 @@ public async Task EnsureNoActiveReindexJobAsync(CancellationToken cancellationTo if (activeReindexJob.found) { - throw new JobConflictException(string.Format(Core.Resources.ChangesToSearchParametersNotAllowedWhileReindexing, activeReindexJob.id)); + throw new JobConflictException(Core.Resources.ChangesToSearchParametersNotAllowedWhileReindexing); } } - public async Task ValidateSearchParameterAsync(ITypedElement searchParam, CancellationToken cancellationToken, DateTimeOffset? lastUpdated = null) + public async Task ValidateSearchParameterAsync(ITypedElement searchParam, CancellationToken cancellationToken, bool refreshCache = true) { var searchParameterWrapper = new SearchParameterWrapper(searchParam); var searchParameterUrl = searchParameterWrapper.Url; - try - { - // We need to make sure we have the latest search parameters before trying to add - // a search parameter. This is to avoid creating a duplicate search parameter that - // was recently added and that hasn't propogated to all fhir-server instances. - // if last updated is provided, it means that updates were applied by pipeline. In this case do not update and keep the input. - if (!lastUpdated.HasValue) - { - await GetAndApplySearchParameterUpdates(cancellationToken); - lastUpdated = SearchParamLastUpdated; - } - - // verify the parameter is supported before continuing - var searchParameterInfo = new SearchParameterInfo(searchParameterWrapper); - if (searchParameterInfo.Component?.Any() == true) + await SearchParameterConcurrencyManager.ExecuteWithLockAsync( + searchParameterUrl, + async () => { - foreach (SearchParameterComponentInfo c in searchParameterInfo.Component) + try { - c.ResolvedSearchParameter = _searchParameterDefinitionManager.GetSearchParameter(c.DefinitionUrl.OriginalString); - } - } + // We need to make sure we have the latest search parameters before trying to add + // a search parameter. This is to avoid creating a duplicate search parameter that + // was recently added and that hasn't propogated to all fhir-server instances. + if (refreshCache) + { + await GetAndApplySearchParameterUpdates(cancellationToken); + } - (bool Supported, bool IsPartiallySupported) supportedResult = _searchParameterSupportResolver.IsSearchParameterSupported(searchParameterInfo); + // verify the parameter is supported before continuing + var searchParameterInfo = new SearchParameterInfo(searchParameterWrapper); - if (!supportedResult.Supported) - { - throw new SearchParameterNotSupportedException(string.Format(Core.Resources.NoConverterForSearchParamType, searchParameterInfo.Type, searchParameterInfo.Expression)); - } + if (searchParameterInfo.Component?.Any() == true) + { + foreach (SearchParameterComponentInfo c in searchParameterInfo.Component) + { + c.ResolvedSearchParameter = _searchParameterDefinitionManager.GetSearchParameter(c.DefinitionUrl.OriginalString); + } + } - // check data store specific support for SearchParameter - if (!_dataStoreSearchParameterValidator.ValidateSearchParameter(searchParameterInfo, out var errorMessage)) - { - throw new SearchParameterNotSupportedException(errorMessage); - } - } - catch (FhirException fex) - { - _logger.LogError(fex, "Error adding search parameter."); - fex.Issues.Add(new OperationOutcomeIssue( - OperationOutcomeConstants.IssueSeverity.Error, - OperationOutcomeConstants.IssueType.Exception, - Core.Resources.CustomSearchCreateError)); + (bool Supported, bool IsPartiallySupported) supportedResult = _searchParameterSupportResolver.IsSearchParameterSupported(searchParameterInfo); - throw; - } - catch (Exception ex) - { - _logger.LogError(ex, "Unexpected error adding search parameter."); - var customSearchException = new ConfigureCustomSearchException(Core.Resources.CustomSearchCreateError); - customSearchException.Issues.Add(new OperationOutcomeIssue( - OperationOutcomeConstants.IssueSeverity.Error, - OperationOutcomeConstants.IssueType.Exception, - ex.Message)); - - throw customSearchException; - } + if (!supportedResult.Supported) + { + throw new SearchParameterNotSupportedException(string.Format(Core.Resources.NoConverterForSearchParamType, searchParameterInfo.Type, searchParameterInfo.Expression)); + } + + // check data store specific support for SearchParameter + if (!_dataStoreSearchParameterValidator.ValidateSearchParameter(searchParameterInfo, out var errorMessage)) + { + throw new SearchParameterNotSupportedException(errorMessage); + } + } + catch (FhirException fex) + { + _logger.LogError(fex, "Error adding search parameter."); + fex.Issues.Add(new OperationOutcomeIssue( + OperationOutcomeConstants.IssueSeverity.Error, + OperationOutcomeConstants.IssueType.Exception, + Core.Resources.CustomSearchCreateError)); - return lastUpdated.Value; + throw; + } + catch (Exception ex) + { + _logger.LogError(ex, "Unexpected error adding search parameter."); + var customSearchException = new ConfigureCustomSearchException(Core.Resources.CustomSearchCreateError); + customSearchException.Issues.Add(new OperationOutcomeIssue( + OperationOutcomeConstants.IssueSeverity.Error, + OperationOutcomeConstants.IssueType.Exception, + ex.Message)); + + throw customSearchException; + } + }, + _logger, + cancellationToken); } /// /// Marks the Search Parameter as PendingDelete. This is only used by DeletionService.cs and will be removed when refactoring is done /// to allow deletion service to properly handle Hard deletions for Search Parameters (e.g. allow reindex prior to removing resource from DB). - /// !!! This method has incorrect name. It does not delete search parameter, it just updates its status. /// /// Search Parameter to update to Pending Delete status. /// Cancellation Token @@ -185,36 +179,41 @@ public async Task DeleteSearchParameterAsync(RawResource searchParamResource, Ca var searchParam = _modelInfoProvider.ToTypedElement(searchParamResource); var searchParameterUrl = searchParam.GetStringScalar("url"); - try - { - await EnsureNoActiveReindexJobAsync(cancellationToken); + await SearchParameterConcurrencyManager.ExecuteWithLockAsync( + searchParameterUrl, + async () => + { + try + { + await EnsureNoActiveReindexJobAsync(cancellationToken); - _logger.LogInformation("DeleteSearchParameterAsync: Refreshing cache"); - await GetAndApplySearchParameterUpdates(cancellationToken); - _logger.LogInformation("DeleteSearchParameterAsync: Deleting the search parameter '{Url}'", searchParameterUrl); - await _searchParameterStatusManager.UpdateSearchParameterStatusAsync(new[] { searchParameterUrl }, SearchParameterStatus.PendingDelete, cancellationToken, lastUpdated: SearchParamLastUpdated); - } - catch (FhirException fex) - { - _logger.LogError(fex, "Error deleting search parameter."); - fex.Issues.Add(new OperationOutcomeIssue( - OperationOutcomeConstants.IssueSeverity.Error, - OperationOutcomeConstants.IssueType.Exception, - Core.Resources.CustomSearchDeleteError)); + _logger.LogInformation("Deleting the search parameter '{Url}'", searchParameterUrl); + await _searchParameterStatusManager.UpdateSearchParameterStatusAsync(new[] { searchParameterUrl }, SearchParameterStatus.PendingDelete, cancellationToken); + } + catch (FhirException fex) + { + _logger.LogError(fex, "Error deleting search parameter."); + fex.Issues.Add(new OperationOutcomeIssue( + OperationOutcomeConstants.IssueSeverity.Error, + OperationOutcomeConstants.IssueType.Exception, + Core.Resources.CustomSearchDeleteError)); - throw; - } - catch (Exception ex) when (!(ex is FhirException)) - { - _logger.LogError(ex, "Unexpected error deleting search parameter."); - var customSearchException = new ConfigureCustomSearchException(Core.Resources.CustomSearchDeleteError); - customSearchException.Issues.Add(new OperationOutcomeIssue( - OperationOutcomeConstants.IssueSeverity.Error, - OperationOutcomeConstants.IssueType.Exception, - ex.Message)); - - throw customSearchException; - } + throw; + } + catch (Exception ex) when (!(ex is FhirException)) + { + _logger.LogError(ex, "Unexpected error deleting search parameter."); + var customSearchException = new ConfigureCustomSearchException(Core.Resources.CustomSearchDeleteError); + customSearchException.Issues.Add(new OperationOutcomeIssue( + OperationOutcomeConstants.IssueSeverity.Error, + OperationOutcomeConstants.IssueType.Exception, + ex.Message)); + + throw customSearchException; + } + }, + _logger, + cancellationToken); } public async Task UpdateSearchParameterStatusAsync(IReadOnlyCollection searchParameterUris, SearchParameterStatus status, CancellationToken cancellationToken, bool ignoreSearchParameterNotSupportedException = false) @@ -276,9 +275,14 @@ public async Task GetAndApplySearchParameterUpdates(CancellationToken canc .Where(p => !systemDefinedSearchParameterUris.Contains(p.Uri.OriginalString)).ToList(); // Batch fetch all SearchParameter resources in one call - var searchParamResources = await GetSearchParametersByUrls(statusesToFetch.Select(p => p.Uri.OriginalString).ToList(), cancellationToken); + var searchParamResources = await GetSearchParametersByUrls( + statusesToFetch + .Select(p => p.Uri.OriginalString) + .ToList(), + cancellationToken); var paramsToAdd = new List(); + var allHaveResources = true; foreach (var searchParam in statusesToFetch) { if (!searchParamResources.TryGetValue(searchParam.Uri.OriginalString, out var searchParamResource)) @@ -286,6 +290,12 @@ public async Task GetAndApplySearchParameterUpdates(CancellationToken canc _logger.LogInformation( "Updated SearchParameter status found for SearchParameter: {Url}, but did not find any SearchParameter resources when querying for this url.", searchParam.Uri); + + if (searchParam.LastUpdated > DateTimeOffset.UtcNow.AddMinutes(-10)) // same as for in cache + { + allHaveResources = false; + } + continue; } @@ -315,14 +325,19 @@ public async Task GetAndApplySearchParameterUpdates(CancellationToken canc // Once added to the definition manager we can update their status await _searchParameterStatusManager.ApplySearchParameterStatus(statuses, cancellationToken); - if (results.LastUpdated.HasValue) + var inCache = ParametersAreInCache(statusesToFetch, cancellationToken); + var cycleConclusive = statuses.Count == 0 || (inCache && allHaveResources); + + // If cache is updated directly and not from the database not all will have corresponding resources. + // Do not advance or log the timestamp unless the cache contents are conclusive for this cycle. + if (inCache && allHaveResources && results.LastUpdated.HasValue) { _searchParamLastUpdated = results.LastUpdated.Value; // this should be the only place in the code to assign last updated } - if (zeroWaitForSemaphore && _searchParamLastUpdated.HasValue) // log only for background + if (cycleConclusive && _searchParamLastUpdated.HasValue) { - // log for cross-instance cache refresh tracking (SQL only; Cosmos/File are no-ops). + // Log to EventLog for cross-instance convergence tracking (SQL only; Cosmos/File are no-ops). var lastUpdatedText = _searchParamLastUpdated.Value.ToString("yyyy-MM-dd HH:mm:ss.fffffff"); await _searchParameterStatusManager.TryLogEvent(_searchParameterStatusManager.SearchParamCacheUpdateProcessName, "Warn", lastUpdatedText, null, cancellationToken); } @@ -342,6 +357,32 @@ public async Task GetAndApplySearchParameterUpdates(CancellationToken canc return true; } + // This should handle racing condition between saving new parameter on one VM and refreshing cache on the other, + // when refresh is invoked between saving status and saving resource. + // This will not be needed when order of saves is reversed (resource first, then status) + private bool ParametersAreInCache(IReadOnlyCollection statuses, CancellationToken cancellationToken) + { + var inCache = true; + foreach (var status in statuses) + { + _searchParameterDefinitionManager.TryGetSearchParameter(status.Uri.OriginalString, out var existingSearchParam); + if (existingSearchParam == null) + { + var msg = $"Did not find in cache uri={status.Uri.OriginalString} status={status.Status}"; + _logger.LogInformation(msg); + + // if the parameter was updated in the last 10 minutes it's possible we hit race condition + // where status was updated but resource is not yet saved, so we should not consider this as cache miss + if (status.LastUpdated > DateTimeOffset.UtcNow.AddMinutes(-10)) + { + inCache = false; + } + } + } + + return inCache; + } + private void DeleteSearchParameter(string url) { try @@ -373,7 +414,6 @@ private async Task> GetSearchParametersByUrls( { cancellationToken.ThrowIfCancellationRequested(); - // search is not by url because it should work for deleted resources. this can be fixed only when resource deletes are delayed. var queryParams = new List> { Tuple.Create(KnownQueryParameterNames.Count, chunkSize.ToString()), @@ -381,7 +421,10 @@ private async Task> GetSearchParametersByUrls( if (!string.IsNullOrEmpty(continuationToken)) { - queryParams.Add(Tuple.Create(KnownQueryParameterNames.ContinuationToken, ContinuationTokenEncoder.Encode(continuationToken))); + queryParams.Add( + Tuple.Create( + KnownQueryParameterNames.ContinuationToken, + ContinuationTokenEncoder.Encode(continuationToken))); } var result = await search.Value.SearchAsync(KnownResourceTypes.SearchParameter, queryParams, cancellationToken); @@ -414,7 +457,10 @@ private async Task> GetSearchParametersByUrls( if (unresolvedUrls.Count > 0) { - _logger.LogWarning("Could not resolve {Count} SearchParameter URL(s). Samples: {Urls}", unresolvedUrls.Count, string.Join(", ", unresolvedUrls.Take(10))); + _logger.LogWarning( + "Could not resolve {Count} SearchParameter URL(s). Samples: {Urls}", + unresolvedUrls.Count, + string.Join(", ", unresolvedUrls.Take(10))); } return searchParametersByUrl; diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterRequestContextPropertyNames.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterRequestContextPropertyNames.cs index 6a0b7dcf55..6c725a699a 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterRequestContextPropertyNames.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterRequestContextPropertyNames.cs @@ -7,7 +7,6 @@ namespace Microsoft.Health.Fhir.Core.Features.Search.Parameters { public static class SearchParameterRequestContextPropertyNames { - public const string PendingStatus = "SearchParameter.PendingStatus"; - public const string LastUpdated = "SearchParameter.LastUpdated"; + public const string PendingStatusUpdates = "SearchParameter.PendingStatusUpdates"; } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterRetry.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterRetry.cs deleted file mode 100644 index ceabeda66a..0000000000 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/SearchParameterRetry.cs +++ /dev/null @@ -1,57 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System; -using System.Security.Cryptography; -using System.Threading.Tasks; -using Microsoft.Health.Fhir.Core.Exceptions; -using Microsoft.Health.Fhir.Core.Features.Persistence; -using Polly; - -namespace Microsoft.Health.Fhir.Core.Features.Search.Parameters -{ - public static class SearchParameterRetry - { - private const int MaxRetryCount = 3; - - /// - /// Executes the provided function with retry logic. - /// - /// The type of result returned by the action. - /// The action to execute with optional retry. - /// Additional context information to append to exception messages. - public static async Task ExecuteAsync(Func> action, string info = null) - { - var retryPolicy = Policy - .Handle(ex => ex.Message == Core.Resources.SearchParameterConcurrencyConflict) - .WaitAndRetryAsync( - retryCount: MaxRetryCount, - sleepDurationProvider: retryAttempt => TimeSpan.FromSeconds(RandomNumberGenerator.GetInt32(1, 5) * 0.1)); - - try - { - return await retryPolicy.ExecuteAsync(action); - } - catch (BadRequestException ex) when (ex.Message == Core.Resources.SearchParameterConcurrencyConflict) - { - throw new BadRequestException($"{ex.Message} {info}.{MaxRetryCount}"); - } - } - - /// - /// Convenience overload for actions with no return value. - /// - public static async Task ExecuteAsync(Func action, string info = null) - { - await ExecuteAsync( - async () => - { - await action(); - return 0; - }, - info); - } - } -} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/ISearchParameterStatusManager.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/ISearchParameterStatusManager.cs index ea62c6c956..e13f5ce72f 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/ISearchParameterStatusManager.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/ISearchParameterStatusManager.cs @@ -21,7 +21,7 @@ public interface ISearchParameterStatusManager Task Handle(SearchParameterDefinitionManagerInitialized notification, CancellationToken cancellationToken); - Task UpdateSearchParameterStatusAsync(IReadOnlyCollection searchParameterUris, SearchParameterStatus status, CancellationToken cancellationToken, bool ignoreSearchParameterNotSupportedException = false, long? reindexId = null, DateTimeOffset? lastUpdated = null); + Task UpdateSearchParameterStatusAsync(IReadOnlyCollection searchParameterUris, SearchParameterStatus status, CancellationToken cancellationToken, bool ignoreSearchParameterNotSupportedException = false, long? reindexId = null); Task CheckCacheConsistencyAsync(DateTime updateEventsSince, DateTime activeHostsSince, CancellationToken cancellationToken); } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterConcurrencyException.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterConcurrencyException.cs new file mode 100644 index 0000000000..a7ce4350e7 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterConcurrencyException.cs @@ -0,0 +1,40 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.Health.Fhir.Core.Features.Search.Registry +{ + /// + /// Exception thrown when an optimistic concurrency conflict occurs during search parameter updates. + /// + public class SearchParameterConcurrencyException : Exception + { + public SearchParameterConcurrencyException(IEnumerable conflictedUris) + : base($"Optimistic concurrency conflict detected for search parameters: {string.Join(", ", conflictedUris)}") + { + ConflictedUris = conflictedUris?.ToList() ?? new List(); + } + + public SearchParameterConcurrencyException(string message) + : base(message) + { + ConflictedUris = new List(); + } + + public SearchParameterConcurrencyException(string message, Exception innerException) + : base(message, innerException) + { + ConflictedUris = new List(); + } + + /// + /// Gets the URIs of search parameters that had concurrency conflicts. + /// + public IReadOnlyList ConflictedUris { get; } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterConcurrencyManager.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterConcurrencyManager.cs new file mode 100644 index 0000000000..a2262be511 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterConcurrencyManager.cs @@ -0,0 +1,148 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Concurrent; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Health.Fhir.Core.Features.Search.Registry +{ + /// + /// Static manager for search parameter concurrency to prevent race conditions + /// when multiple requests try to update the same search parameter simultaneously. + /// + public static class SearchParameterConcurrencyManager + { + private static readonly ConcurrentDictionary _semaphores = new(); + private static readonly object _cleanupLock = new object(); + + /// + /// Gets the current number of active locks for debugging/monitoring purposes. + /// + public static int ActiveLockCount => _semaphores.Count; + + /// + /// Executes the given function with exclusive access for the specified search parameter URI. + /// This prevents concurrent modifications to the same search parameter. + /// + /// The return type of the function + /// The URI of the search parameter to lock on + /// The function to execute with exclusive access + /// Optional logger for debug information + /// The cancellation token + /// The result of the function execution + public static async Task ExecuteWithLockAsync( + string searchParameterUri, + Func> function, + ILogger logger = null, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(searchParameterUri)) + { + throw new ArgumentException("Search parameter URI cannot be null or empty", nameof(searchParameterUri)); + } + + ArgumentNullException.ThrowIfNull(function); + + var semaphore = _semaphores.GetOrAdd(searchParameterUri, _ => new SemaphoreSlim(1, 1)); + + logger?.LogDebug("Acquiring lock for search parameter: {SearchParameterUri}", searchParameterUri); + + await semaphore.WaitAsync(cancellationToken); + + try + { + logger?.LogDebug("Lock acquired for search parameter: {SearchParameterUri}", searchParameterUri); + return await function(); + } + finally + { + semaphore.Release(); + logger?.LogDebug("Lock released for search parameter: {SearchParameterUri}", searchParameterUri); + + // Clean up semaphore if no one is waiting and it's available for immediate use + // Use a lock to prevent race conditions during cleanup + lock (_cleanupLock) + { + // Only clean up if the semaphore is available (CurrentCount == 1) and we can successfully remove it + if (semaphore.CurrentCount == 1 && + _semaphores.TryRemove(searchParameterUri, out var removedSemaphore)) + { + bool shouldPutBack = false; + + try + { + if (ReferenceEquals(removedSemaphore, semaphore)) + { + // Double-check that no other thread acquired the semaphore between our check and removal + if (semaphore.CurrentCount == 1) + { + logger?.LogDebug("Cleaned up semaphore for search parameter: {SearchParameterUri}", searchParameterUri); + + // Don't put back - will dispose in finally + } + else + { + // Put it back if someone acquired it in the meantime + shouldPutBack = true; + } + } + else + { + // Put it back if it's a different semaphore instance + shouldPutBack = true; + } + + if (shouldPutBack) + { + _semaphores.TryAdd(searchParameterUri, removedSemaphore); + removedSemaphore = null; // Prevent dispose in finally since we put it back + } + } + finally + { + // Only dispose if we didn't put it back into the dictionary + removedSemaphore?.Dispose(); + } + } + } + } + } + + /// + /// Executes the given action with exclusive access for the specified search parameter URI. + /// This prevents concurrent modifications to the same search parameter. + /// + /// The URI of the search parameter to lock on + /// The action to execute with exclusive access + /// Optional logger for debug information + /// The cancellation token + public static async Task ExecuteWithLockAsync( + string searchParameterUri, + Func action, + ILogger logger = null, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(searchParameterUri)) + { + throw new ArgumentException("Search parameter URI cannot be null or empty", nameof(searchParameterUri)); + } + + ArgumentNullException.ThrowIfNull(action); + + await ExecuteWithLockAsync( + searchParameterUri, + async () => + { + await action(); + return 0; // Return dummy value for action overload + }, + logger, + cancellationToken); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterStatusManager.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterStatusManager.cs index 8773f2ecb6..fffd1cbc0b 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterStatusManager.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Registry/SearchParameterStatusManager.cs @@ -136,7 +136,7 @@ public async Task Handle(SearchParameterDefinitionManagerInitialized notificatio await EnsureInitializedAsync(cancellationToken); } - public async Task UpdateSearchParameterStatusAsync(IReadOnlyCollection searchParameterUris, SearchParameterStatus status, CancellationToken cancellationToken, bool ignoreSearchParameterNotSupportedException = false, long? reindexId = null, DateTimeOffset? lastUpdated = null) + public async Task UpdateSearchParameterStatusAsync(IReadOnlyCollection searchParameterUris, SearchParameterStatus status, CancellationToken cancellationToken, bool ignoreSearchParameterNotSupportedException = false, long? reindexId = null) { EnsureArg.IsNotNull(searchParameterUris); @@ -160,7 +160,6 @@ public async Task UpdateSearchParameterStatusAsync(IReadOnlyCollection s { existingStatus.Status = status; searchParameterStatusList.Add(existingStatus); - existingStatus.LastUpdated = lastUpdated ?? DateTimeOffset.UtcNow; } else { @@ -168,7 +167,6 @@ public async Task UpdateSearchParameterStatusAsync(IReadOnlyCollection s { Status = status, Uri = new Uri(uri), - LastUpdated = lastUpdated ?? DateTimeOffset.UtcNow, }); } } diff --git a/src/Microsoft.Health.Fhir.Core/Resources.Designer.cs b/src/Microsoft.Health.Fhir.Core/Resources.Designer.cs index 498c91475d..8964664284 100644 --- a/src/Microsoft.Health.Fhir.Core/Resources.Designer.cs +++ b/src/Microsoft.Health.Fhir.Core/Resources.Designer.cs @@ -1592,15 +1592,6 @@ internal static string SearchParameterByDefinitionUriNotSupported { } } - /// - /// Looks up a localized string similar to Optimistic concurrency conflict detected while writing custom search parameter(s). Make sure that custom search parameters are not written in parallel. Consider sequential writes or a bundle.. - /// - internal static string SearchParameterConcurrencyConflict { - get { - return ResourceManager.GetString("SearchParameterConcurrencyConflict", resourceCulture); - } - } - /// /// Looks up a localized string similar to SearchParameter[{0}].resource.base is not defined.. /// @@ -2024,15 +2015,6 @@ internal static string UnsupportedBulkUpdateOperation { } } - /// - /// Looks up a localized string similar to The provided Bundle type is not supported. Supported values are: batch and transaction.. - /// - internal static string UnsupportedBundleType { - get { - return ResourceManager.GetString("UnsupportedBundleType", resourceCulture); - } - } - /// /// Looks up a localized string similar to Unsupported configuration found.. /// diff --git a/src/Microsoft.Health.Fhir.Core/Resources.resx b/src/Microsoft.Health.Fhir.Core/Resources.resx index 1663775162..5f5f456b12 100644 --- a/src/Microsoft.Health.Fhir.Core/Resources.resx +++ b/src/Microsoft.Health.Fhir.Core/Resources.resx @@ -444,9 +444,6 @@ Bundle type is not present. Possible values are: transaction or batch - - The provided Bundle type is not supported. Supported values are: batch and transaction. - Unable to delete secret from SecretStore @@ -879,7 +876,4 @@ Search Parameter URL {0} exceeds the maximum length limit of {1} - - Optimistic concurrency conflict detected while writing custom search parameter(s). Make sure that custom search parameters are not written in parallel. Consider sequential writes or a bundle. - - + \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosFhirDataStore.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosFhirDataStore.cs index 31b1ff5e4c..7cffade697 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosFhirDataStore.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/CosmosFhirDataStore.cs @@ -262,7 +262,7 @@ public async Task UpsertAsync(ResourceWrapperOperation resource, UpsertOutcome result = await operation.AppendResourceAsync(resource, this, cancellationToken).ConfigureAwait(false); if (string.Equals(resource.Wrapper.ResourceTypeName, KnownResourceTypes.SearchParameter, StringComparison.Ordinal)) { - await PersistPendingSearchParameterStatusUpdateAsync(cancellationToken); + await PersistPendingSearchParameterStatusUpdatesAsync(cancellationToken); } return result; @@ -281,7 +281,7 @@ public async Task UpsertAsync(ResourceWrapperOperation resource, if (string.Equals(resource.Wrapper.ResourceTypeName, KnownResourceTypes.SearchParameter, StringComparison.Ordinal)) { - await PersistPendingSearchParameterStatusUpdateAsync(cancellationToken); + await PersistPendingSearchParameterStatusUpdatesAsync(cancellationToken); } return upsertOutcome; @@ -502,7 +502,7 @@ await retryPolicy.ExecuteAsync( } } - private async Task PersistPendingSearchParameterStatusUpdateAsync(CancellationToken cancellationToken) + private async Task PersistPendingSearchParameterStatusUpdatesAsync(CancellationToken cancellationToken) { var context = _requestContextAccessor.RequestContext; if (context?.Properties == null) @@ -510,14 +510,47 @@ private async Task PersistPendingSearchParameterStatusUpdateAsync(CancellationTo return; } - if (!context.Properties.TryGetValue(SearchParameterRequestContextPropertyNames.PendingStatus, out var value) || - value is not ResourceSearchParameterStatus status) + if (!context.Properties.TryGetValue(SearchParameterRequestContextPropertyNames.PendingStatusUpdates, out var value) || + value is not List pendingStatuses || + pendingStatuses.Count == 0) { return; } - await _searchParameterStatusDataStore.UpsertStatuses([status], cancellationToken); - context.Properties.Remove(SearchParameterRequestContextPropertyNames.PendingStatus); + List snapshot; + lock (pendingStatuses) + { + if (pendingStatuses.Count == 0) + { + return; + } + + snapshot = pendingStatuses + .Where(status => status?.Uri != null) + .GroupBy(status => status.Uri.OriginalString, StringComparer.Ordinal) + .Select(group => group.Last()) + .ToList(); + } + + if (snapshot.Count == 0) + { + return; + } + + await _searchParameterStatusDataStore.UpsertStatuses(snapshot, cancellationToken); + + lock (pendingStatuses) + { + foreach (var item in snapshot) + { + pendingStatuses.Remove(item); + } + + if (pendingStatuses.Count == 0) + { + context.Properties.Remove(SearchParameterRequestContextPropertyNames.PendingStatusUpdates); + } + } } public async Task GetAsync(ResourceKey key, CancellationToken cancellationToken) diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/FhirControllerTests.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/FhirControllerTests.cs index a3691a357c..7e577aab6c 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/FhirControllerTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/FhirControllerTests.cs @@ -36,7 +36,6 @@ using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Persistence.Orchestration; using Microsoft.Health.Fhir.Core.Features.Routing; -using Microsoft.Health.Fhir.Core.Features.Search.Parameters; using Microsoft.Health.Fhir.Core.Messages.Bundle; using Microsoft.Health.Fhir.Core.Messages.Create; using Microsoft.Health.Fhir.Core.Messages.Delete; @@ -68,7 +67,6 @@ public sealed class FhirControllerTests private readonly IUrlResolver _urlResolver; private readonly IOptions _configuration; private readonly IAuthorizationService _authorizationService; - private readonly ISearchParameterOperations _searchParameterOperations; public FhirControllerTests() { @@ -78,7 +76,6 @@ public FhirControllerTests() _configuration = Substitute.For>(); _configuration.Value.Returns(new FeatureConfiguration()); _authorizationService = Substitute.For(); - _searchParameterOperations = Substitute.For(); _mediator.Send( Arg.Any(), @@ -93,8 +90,7 @@ public FhirControllerTests() _requestContextAccessor, _urlResolver, _configuration, - _authorizationService, - _searchParameterOperations); + _authorizationService); _fhirController.ControllerContext = new ControllerContext( new ActionContext( Substitute.For(), @@ -1456,208 +1452,5 @@ private static void TestIfTargetMethodContainsCustomAttribute(Type expectedCusto Assert.True(latencyFilter != null, $"The expected filter '{expectedCustomAttributeType.Name}' was not found in the method '{methodName}' from '{targetClassType.Name}'."); } - - [Fact] - public async Task GivenSearchParameterCreate_WhenConcurrencyConflictOccurs_ThenRetriesAndSucceeds() - { - var searchParameter = new SearchParameter { Id = "test", Url = "http://test.com/param", VersionId = Guid.NewGuid().ToString() }; - var wrapper = CreateMockResourceWrapper(searchParameter); - - var attemptCount = 0; - _searchParameterOperations - .GetAndApplySearchParameterUpdates(Arg.Any()) - .Returns(Task.FromResult(true)); - _searchParameterOperations.SearchParamLastUpdated.Returns(DateTimeOffset.UtcNow); - - _mediator.Send( - Arg.Any(), - Arg.Any()) - .Returns(callInfo => - { - attemptCount++; - if (attemptCount < 3) - { - throw new BadRequestException(Core.Resources.SearchParameterConcurrencyConflict); - } - - return new UpsertResourceResponse(new SaveOutcome(new RawResourceElement(wrapper), SaveOutcomeType.Created)); - }); - - var response = await _fhirController.Create(searchParameter); - - Assert.Equal(3, attemptCount); - Assert.IsType(response); - } - - [Fact] - public async Task GivenSearchParameterUpdate_WhenConcurrencyConflictOccurs_ThenRetriesAndSucceeds() - { - var searchParameter = new SearchParameter { Id = "test", Url = "http://test.com/param", VersionId = Guid.NewGuid().ToString() }; - var wrapper = CreateMockResourceWrapper(searchParameter); - - var attemptCount = 0; - _searchParameterOperations - .GetAndApplySearchParameterUpdates(Arg.Any()) - .Returns(Task.FromResult(true)); - _searchParameterOperations.SearchParamLastUpdated.Returns(DateTimeOffset.UtcNow); - - _mediator.Send( - Arg.Any(), - Arg.Any()) - .Returns(callInfo => - { - attemptCount++; - if (attemptCount < 2) - { - throw new BadRequestException(Core.Resources.SearchParameterConcurrencyConflict); - } - - return new UpsertResourceResponse(new SaveOutcome(new RawResourceElement(wrapper), SaveOutcomeType.Updated)); - }); - - var response = await _fhirController.Update(searchParameter, null, true); - - Assert.Equal(2, attemptCount); - Assert.IsType(response); - } - - [Fact] - public async Task GivenSearchParameterDelete_WhenConcurrencyConflictOccurs_ThenRetriesAndSucceeds() - { - var key = new ResourceKey("SearchParameter", "test"); - - var attemptCount = 0; - _searchParameterOperations - .GetAndApplySearchParameterUpdates(Arg.Any()) - .Returns(Task.FromResult(true)); - _searchParameterOperations.SearchParamLastUpdated.Returns(DateTimeOffset.UtcNow); - - _mediator.Send( - Arg.Any(), - Arg.Any()) - .Returns(callInfo => - { - attemptCount++; - if (attemptCount < 2) - { - throw new BadRequestException(Core.Resources.SearchParameterConcurrencyConflict); - } - - return new DeleteResourceResponse(key); - }); - - var response = await _fhirController.Delete("SearchParameter", "test", new HardDeleteModel { HardDelete = false }, false); - - Assert.Equal(2, attemptCount); - Assert.IsType(response); - } - - [Fact] - public async Task GivenSearchParameterConditionalCreate_WhenConcurrencyConflictOccurs_ThenRetriesAndSucceeds() - { - var searchParameter = new SearchParameter { Id = "test", Url = "http://test.com/param", VersionId = Guid.NewGuid().ToString() }; - var wrapper = CreateMockResourceWrapper(searchParameter); - - var httpContext = new DefaultHttpContext(); - httpContext.Request.Headers[KnownHeaders.IfNoneExist] = "url=http://test.com/param"; - _fhirController.ControllerContext.HttpContext = httpContext; - - var attemptCount = 0; - _searchParameterOperations - .GetAndApplySearchParameterUpdates(Arg.Any()) - .Returns(Task.FromResult(true)); - _searchParameterOperations.SearchParamLastUpdated.Returns(DateTimeOffset.UtcNow); - - _mediator.Send( - Arg.Any(), - Arg.Any()) - .Returns(callInfo => - { - attemptCount++; - if (attemptCount < 3) - { - throw new BadRequestException(Core.Resources.SearchParameterConcurrencyConflict); - } - - return new UpsertResourceResponse(new SaveOutcome(new RawResourceElement(wrapper), SaveOutcomeType.Created)); - }); - - var response = await _fhirController.ConditionalCreate(searchParameter); - - Assert.Equal(3, attemptCount); - Assert.IsType(response); - } - - [Fact] - public async Task GivenSearchParameterConditionalUpdate_WhenConcurrencyConflictOccurs_ThenRetriesAndSucceeds() - { - var searchParameter = new SearchParameter { Id = "test", Url = "http://test.com/param", VersionId = Guid.NewGuid().ToString() }; - var wrapper = CreateMockResourceWrapper(searchParameter); - - var httpContext = new DefaultHttpContext(); - httpContext.Request.QueryString = new QueryString("?url=http://test.com/param"); - _fhirController.ControllerContext.HttpContext = httpContext; - - var attemptCount = 0; - _searchParameterOperations - .GetAndApplySearchParameterUpdates(Arg.Any()) - .Returns(Task.FromResult(true)); - _searchParameterOperations.SearchParamLastUpdated.Returns(DateTimeOffset.UtcNow); - - _mediator.Send( - Arg.Any(), - Arg.Any()) - .Returns(callInfo => - { - attemptCount++; - if (attemptCount < 2) - { - throw new BadRequestException(Core.Resources.SearchParameterConcurrencyConflict); - } - - return new UpsertResourceResponse(new SaveOutcome(new RawResourceElement(wrapper), SaveOutcomeType.Updated)); - }); - - var response = await _fhirController.ConditionalUpdate(searchParameter); - - Assert.Equal(2, attemptCount); - Assert.IsType(response); - } - - [Fact] - public async Task GivenNonSearchParameterResource_WhenOperationExecuted_ThenNoRetry() - { - var patient = new Patient { Id = "test", VersionId = Guid.NewGuid().ToString() }; - var wrapper = CreateMockResourceWrapper(patient); - - var attemptCount = 0; - _mediator.Send( - Arg.Any(), - Arg.Any()) - .Returns(callInfo => - { - attemptCount++; - return new UpsertResourceResponse(new SaveOutcome(new RawResourceElement(wrapper), SaveOutcomeType.Created)); - }); - - var response = await _fhirController.Create(patient); - - Assert.Equal(1, attemptCount); - Assert.IsType(response); - await _searchParameterOperations.DidNotReceive().GetAndApplySearchParameterUpdates(Arg.Any()); - } - - private ResourceWrapper CreateMockResourceWrapper(Resource resource) - { - var rawJson = new FhirJsonSerializer().SerializeToString(resource); - return new ResourceWrapper( - resource.ToResourceElement(), - new RawResource(rawJson, FhirResourceFormat.Json, isMetaSet: false), - null, - false, - null, - null, - null); - } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/TerminologyControllerTests.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/TerminologyControllerTests.cs index e35a79b75c..75bda5733a 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/TerminologyControllerTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/TerminologyControllerTests.cs @@ -416,21 +416,5 @@ public static IEnumerable GetExpandByPostTestData() yield return d; } } - - [Fact] - public async Task GivenUnknownValueSet_WhenExpanding_ThenThrowsResourceNotFoundException() - { - // When the terminology service throws ResourceNotFoundException for an unknown ValueSet, - // the exception propagates and the OperationOutcomeExceptionFilter maps it to HTTP 404. - _mediator.Send( - Arg.Any(), - Arg.Any()) - .Returns(x => throw new ResourceNotFoundException( - "ValueSet 'http://example.org/fhir/ValueSet/unknown' is unknown")); - - _controller.HttpContext.Request.QueryString = new QueryString("?url=http://example.org/fhir/ValueSet/unknown"); - - await Assert.ThrowsAsync(() => _controller.Expand()); - } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Filters/SearchParameterFilterAttributeTests.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Filters/SearchParameterFilterAttributeTests.cs index 08558c8a3a..bb739f3442 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Filters/SearchParameterFilterAttributeTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Filters/SearchParameterFilterAttributeTests.cs @@ -3,7 +3,6 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -using System; using System.Collections.Generic; using System.Threading; using Hl7.Fhir.Model; @@ -12,11 +11,8 @@ using Microsoft.AspNetCore.Mvc.Abstractions; using Microsoft.AspNetCore.Mvc.Filters; using Microsoft.AspNetCore.Routing; -using Microsoft.Health.Core.Features.Context; using Microsoft.Health.Fhir.Api.Features.Filters; -using Microsoft.Health.Fhir.Core.Features.Context; using Microsoft.Health.Fhir.Core.Features.Routing; -using Microsoft.Health.Fhir.Core.UnitTests.Features.Context; using Microsoft.Health.Fhir.Shared.Core.Features.Search.Parameters; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Test.Utilities; @@ -32,18 +28,11 @@ namespace Microsoft.Health.Fhir.Api.UnitTests.Features.Filters public class SearchParameterFilterAttributeTests { private readonly ISearchParameterValidator _searchParameterValidator = Substitute.For(); - private readonly RequestContextAccessor _fhirRequestContextAccessor = Substitute.For>(); - private readonly DefaultFhirRequestContext _fhirRequestContext = new DefaultFhirRequestContext(); - - public SearchParameterFilterAttributeTests() - { - _fhirRequestContextAccessor.RequestContext.Returns(_fhirRequestContext); - } [Fact] public async Task GivenAnAction_WhenPostingAnObservationObject_ThenNoSearchParameterActionTaken() { - var filter = new SearchParameterFilterAttribute(_searchParameterValidator, _fhirRequestContextAccessor); + var filter = new SearchParameterFilterAttribute(_searchParameterValidator); var context = CreateContext(new Observation()); var actionExecutedContext = new ActionExecutedContext(context, new List(), null); @@ -51,36 +40,13 @@ public async Task GivenAnAction_WhenPostingAnObservationObject_ThenNoSearchParam await filter.OnActionExecutionAsync(context, actionExecutionDelegate); - await _searchParameterValidator.DidNotReceive().ValidateSearchParameterInput(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()); + await _searchParameterValidator.DidNotReceive().ValidateSearchParameterInput(Arg.Any(), Arg.Any(), Arg.Any()); } [Fact] public async Task GivenAnAction_WhenPostingASearchParameterObject_ThenSearchParameterActionsTaken() { - _searchParameterValidator.ValidateSearchParameterInput(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(DateTimeOffset.UtcNow); - - var filter = new SearchParameterFilterAttribute(_searchParameterValidator, _fhirRequestContextAccessor); - - var context = CreateContext(new SearchParameter()); - var actionExecutedContext = new ActionExecutedContext(context, new List(), null); - ActionExecutionDelegate actionExecutionDelegate = () => Task.Run(() => actionExecutedContext); - - await filter.OnActionExecutionAsync(context, actionExecutionDelegate); - - await _searchParameterValidator.Received().ValidateSearchParameterInput(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()); - } - - [Fact] - public async Task GivenAnAction_WhenPostingASearchParameterObjectWithExistingLastUpdated_ThenExistingLastUpdatedIsPassedToValidator() - { - var existingLastUpdated = DateTimeOffset.UtcNow.AddHours(-1); - _fhirRequestContext.SetSearchParameterLastUpdated(existingLastUpdated); - - _searchParameterValidator.ValidateSearchParameterInput(Arg.Any(), Arg.Any(), Arg.Any(), existingLastUpdated) - .Returns(existingLastUpdated); - - var filter = new SearchParameterFilterAttribute(_searchParameterValidator, _fhirRequestContextAccessor); + var filter = new SearchParameterFilterAttribute(_searchParameterValidator); var context = CreateContext(new SearchParameter()); var actionExecutedContext = new ActionExecutedContext(context, new List(), null); @@ -88,7 +54,7 @@ public async Task GivenAnAction_WhenPostingASearchParameterObjectWithExistingLas await filter.OnActionExecutionAsync(context, actionExecutionDelegate); - await _searchParameterValidator.Received().ValidateSearchParameterInput(Arg.Any(), Arg.Any(), Arg.Any(), existingLastUpdated); + await _searchParameterValidator.Received().ValidateSearchParameterInput(Arg.Any(), Arg.Any(), Arg.Any()); } private static ActionExecutingContext CreateContext(Base type) diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Controllers/FhirController.cs b/src/Microsoft.Health.Fhir.Shared.Api/Controllers/FhirController.cs index b0ee52d763..47b4b91cee 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Controllers/FhirController.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api/Controllers/FhirController.cs @@ -42,7 +42,6 @@ using Microsoft.Health.Fhir.Core.Features.Persistence.Orchestration; using Microsoft.Health.Fhir.Core.Features.Resources.Patch; using Microsoft.Health.Fhir.Core.Features.Routing; -using Microsoft.Health.Fhir.Core.Features.Search.Parameters; using Microsoft.Health.Fhir.Core.Messages.Create; using Microsoft.Health.Fhir.Core.Messages.Delete; using Microsoft.Health.Fhir.Core.Messages.Get; @@ -69,7 +68,6 @@ public class FhirController : Controller private readonly IMediator _mediator; private readonly RequestContextAccessor _fhirRequestContextAccessor; private readonly IUrlResolver _urlResolver; - private readonly ISearchParameterOperations _searchParameterOperations; /// /// Initializes a new instance of the class. @@ -79,14 +77,12 @@ public class FhirController : Controller /// The urlResolver. /// The UI configuration. /// The authorization service. - /// The search parameter operations. public FhirController( IMediator mediator, RequestContextAccessor fhirRequestContextAccessor, IUrlResolver urlResolver, IOptions uiConfiguration, - IAuthorizationService authorizationService, - ISearchParameterOperations searchParameterOperations) + IAuthorizationService authorizationService) { EnsureArg.IsNotNull(mediator, nameof(mediator)); EnsureArg.IsNotNull(fhirRequestContextAccessor, nameof(fhirRequestContextAccessor)); @@ -94,12 +90,10 @@ public FhirController( EnsureArg.IsNotNull(uiConfiguration, nameof(uiConfiguration)); EnsureArg.IsNotNull(uiConfiguration.Value, nameof(uiConfiguration)); EnsureArg.IsNotNull(authorizationService, nameof(authorizationService)); - EnsureArg.IsNotNull(searchParameterOperations, nameof(searchParameterOperations)); _mediator = mediator; _fhirRequestContextAccessor = fhirRequestContextAccessor; _urlResolver = urlResolver; - _searchParameterOperations = searchParameterOperations; } [ApiExplorerSettings(IgnoreApi = true)] @@ -168,12 +162,9 @@ public IActionResult CustomError(int? statusCode = null) [TypeFilter(typeof(CrudEndpointMetricEmitterAttribute))] public async Task Create([FromBody] Resource resource) { - var response = await ExecuteWithSearchParameterRetryAsync( - resource.TypeName, - () => _mediator.CreateResourceAsync( - new CreateResourceRequest(resource.ToResourceElement(), GetBundleResourceContext()), - HttpContext.RequestAborted), - "Create"); + RawResourceElement response = await _mediator.CreateResourceAsync( + new CreateResourceRequest(resource.ToResourceElement(), GetBundleResourceContext()), + HttpContext.RequestAborted); return FhirResult.Create(response, HttpStatusCode.Created) .SetETagHeader() @@ -200,28 +191,25 @@ public async Task ConditionalCreate([FromBody] Resource resource) Tuple[] conditionalParameters = QueryHelpers.ParseQuery(conditionalCreateHeader) .SelectMany(query => query.Value, (query, value) => Tuple.Create(query.Key, value)).ToArray(); - var response = await ExecuteWithSearchParameterRetryAsync( - resource.TypeName, - () => _mediator.Send( + UpsertResourceResponse createResponse = await _mediator.Send( new ConditionalCreateResourceRequest(resource.ToResourceElement(), conditionalParameters, GetBundleResourceContext()), - HttpContext.RequestAborted), - "ConditionalCreate"); + HttpContext.RequestAborted); - if (response?.Outcome == null) + if (createResponse?.Outcome == null) { return Ok(); } var statusCode = HttpStatusCode.Created; var message = Resources.ConditionalCreateResourceCreated; - if (response.Outcome.Outcome != SaveOutcomeType.Created) + if (createResponse.Outcome.Outcome != SaveOutcomeType.Created) { statusCode = HttpStatusCode.OK; message = Resources.ConditionalCreateResourceAlreadyExists; } return FhirResult.Create( - response.Outcome.RawResourceElement, + createResponse.Outcome.RawResourceElement, statusCode, true, true, @@ -245,12 +233,9 @@ public async Task ConditionalCreate([FromBody] Resource resource) [TypeFilter(typeof(CrudEndpointMetricEmitterAttribute))] public async Task Update([FromBody] Resource resource, [ModelBinder(typeof(WeakETagBinder))] WeakETag ifMatchHeader, [FromQuery(Name = KnownQueryParameterNames.MetaHistory)] bool metaHistory = true) { - var response = await ExecuteWithSearchParameterRetryAsync( - resource.TypeName, - () => _mediator.UpsertResourceAsync( - new UpsertResourceRequest(resource.ToResourceElement(), GetBundleResourceContext(), ifMatchHeader, metaHistory), - HttpContext.RequestAborted), - "Update"); + SaveOutcome response = await _mediator.UpsertResourceAsync( + new UpsertResourceRequest(resource.ToResourceElement(), GetBundleResourceContext(), ifMatchHeader, metaHistory), + HttpContext.RequestAborted); return ToSaveOutcomeResult(response); } @@ -269,12 +254,9 @@ public async Task ConditionalUpdate([FromBody] Resource resource) IReadOnlyList> conditionalParameters = GetQueriesForSearch(); - var response = await ExecuteWithSearchParameterRetryAsync( - resource.TypeName, - () => _mediator.Send( + UpsertResourceResponse response = await _mediator.Send( new ConditionalUpsertResourceRequest(resource.ToResourceElement(), conditionalParameters, GetBundleResourceContext()), - HttpContext.RequestAborted), - "ConditionalUpdate"); + HttpContext.RequestAborted); SaveOutcome saveOutcome = response.Outcome; @@ -437,16 +419,13 @@ public async Task VRead(string typeParameter, string idParameter, [TypeFilter(typeof(CrudEndpointMetricEmitterAttribute))] public async Task Delete(string typeParameter, string idParameter, HardDeleteModel hardDeleteModel, [FromQuery] bool allowPartialSuccess) { - var response = await ExecuteWithSearchParameterRetryAsync( - typeParameter, - () => _mediator.DeleteResourceAsync( - new DeleteResourceRequest( - new ResourceKey(typeParameter, idParameter), - hardDeleteModel.IsHardDelete ? DeleteOperation.HardDelete : DeleteOperation.SoftDelete, - GetBundleResourceContext(), - allowPartialSuccess), - HttpContext.RequestAborted), - "Delete"); + DeleteResourceResponse response = await _mediator.DeleteResourceAsync( + new DeleteResourceRequest( + new ResourceKey(typeParameter, idParameter), + hardDeleteModel.IsHardDelete ? DeleteOperation.HardDelete : DeleteOperation.SoftDelete, + GetBundleResourceContext(), + allowPartialSuccess), + HttpContext.RequestAborted); return FhirResult.NoContent().SetETagHeader(response.WeakETag); } @@ -490,17 +469,14 @@ public async Task ConditionalDelete(string typeParameter, HardDel SetupConditionalRequestWithQueryOptimizeConcurrency(); - var response = await ExecuteWithSearchParameterRetryAsync( - typeParameter, - () => _mediator.Send( + DeleteResourceResponse response = await _mediator.Send( new ConditionalDeleteResourceRequest( typeParameter, conditionalParameters, hardDeleteModel.IsHardDelete ? DeleteOperation.HardDelete : DeleteOperation.SoftDelete, maxDeleteCount.GetValueOrDefault(1), GetBundleResourceContext()), - HttpContext.RequestAborted), - "ConditionalDelete"); + HttpContext.RequestAborted); if (maxDeleteCount.HasValue) { @@ -729,32 +705,6 @@ public async Task BatchAndTransactions([FromBody] Resource bundle return FhirResult.Create(bundleResponse); } - /// - /// Executes an action with retry logic if the resource type is SearchParameter, and it is not a part of parallel bundle. - /// - private async Task ExecuteWithSearchParameterRetryAsync(string resourceType, Func> action, string info) - { - if (resourceType == KnownResourceTypes.SearchParameter) - { - var context = GetBundleResourceContext(); - if (context != null && context.IsParallelBundle) - { - return await action(); - } - - return await SearchParameterRetry.ExecuteAsync( - async () => - { - await _searchParameterOperations.GetAndApplySearchParameterUpdates(HttpContext.RequestAborted); - _fhirRequestContextAccessor.RequestContext.SetSearchParameterLastUpdated(_searchParameterOperations.SearchParamLastUpdated); - return await action(); - }, - info); - } - - return await action(); - } - /// /// Returns an instance of with bundle related information, if a resource if part of a bundle. /// diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Features/Filters/FhirRequestContextRouteDataPopulatingFilterAttribute.cs b/src/Microsoft.Health.Fhir.Shared.Api/Features/Filters/FhirRequestContextRouteDataPopulatingFilterAttribute.cs index e3944c3bff..399c81fda0 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Features/Filters/FhirRequestContextRouteDataPopulatingFilterAttribute.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api/Features/Filters/FhirRequestContextRouteDataPopulatingFilterAttribute.cs @@ -4,16 +4,12 @@ // ------------------------------------------------------------------------------------------------- using System; -using System.Diagnostics; -using System.Net; using EnsureThat; -using Hl7.Fhir.Model; using Microsoft.AspNetCore.Mvc.Controllers; using Microsoft.AspNetCore.Mvc.Filters; using Microsoft.AspNetCore.Routing; using Microsoft.Health.Api.Features.Audit; using Microsoft.Health.Core.Features.Context; -using Microsoft.Health.Fhir.Api.Features.ActionResults; using Microsoft.Health.Fhir.Core.Features; using Microsoft.Health.Fhir.Core.Features.Context; using Microsoft.Health.Fhir.Core.Features.Routing; @@ -73,42 +69,14 @@ public override void OnActionExecuting(ActionExecutingContext context) return; } - try + switch (bundle.Type) { - // bundle.Type can raise InvalidCastException if the incoming value is not a valid BundleType. - switch (bundle.Type) - { - case Hl7.Fhir.Model.Bundle.BundleType.Batch: - fhirRequestContext.AuditEventType = AuditEventSubType.Batch; - break; - case Hl7.Fhir.Model.Bundle.BundleType.Transaction: - fhirRequestContext.AuditEventType = AuditEventSubType.Transaction; - break; - } - } - catch (InvalidCastException) - { - // I had to add the 'timer' to the Http context, as the filter Microsoft.Health.Api.Features.Audit.AuditLoggingFilterAttribute expected it to be there. - // This change avoid a null reference exception to happen. - // The correct implementation would be handling the absence of the 'timer' in AuditLoggingFilterAttribute. - context.HttpContext.Items["timer"] = Stopwatch.StartNew(); - - context.Result = new OperationOutcomeResult( - new OperationOutcome - { - Id = fhirRequestContext.CorrelationId, - Issue = - { - new OperationOutcome.IssueComponent - { - Severity = OperationOutcome.IssueSeverity.Error, - Code = OperationOutcome.IssueType.Invalid, - Diagnostics = Core.Resources.UnsupportedBundleType, - }, - }, - }, - HttpStatusCode.BadRequest); - return; + case Hl7.Fhir.Model.Bundle.BundleType.Batch: + fhirRequestContext.AuditEventType = AuditEventSubType.Batch; + break; + case Hl7.Fhir.Model.Bundle.BundleType.Transaction: + fhirRequestContext.AuditEventType = AuditEventSubType.Transaction; + break; } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Features/Filters/SearchParameterFilterAttribute.cs b/src/Microsoft.Health.Fhir.Shared.Api/Features/Filters/SearchParameterFilterAttribute.cs index 6bf5fe93d5..bc7b78aee3 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Features/Filters/SearchParameterFilterAttribute.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api/Features/Filters/SearchParameterFilterAttribute.cs @@ -4,15 +4,10 @@ // ------------------------------------------------------------------------------------------------- using System; -using System.Linq; using EnsureThat; using Hl7.Fhir.Model; using Microsoft.AspNetCore.Mvc.Filters; -using Microsoft.Health.Core.Features.Context; -using Microsoft.Health.Fhir.Core.Features.Context; -using Microsoft.Health.Fhir.Core.Features.Persistence.Orchestration; using Microsoft.Health.Fhir.Core.Features.Routing; -using Microsoft.Health.Fhir.Core.Features.Search.Parameters; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Shared.Core.Features.Search.Parameters; using Task = System.Threading.Tasks.Task; @@ -22,16 +17,13 @@ namespace Microsoft.Health.Fhir.Api.Features.Filters [AttributeUsage(AttributeTargets.Method)] internal sealed class SearchParameterFilterAttribute : ActionFilterAttribute { - private readonly ISearchParameterValidator _searchParameterValidator; - private readonly RequestContextAccessor _fhirRequestContextAccessor; + private ISearchParameterValidator _searchParameterValidator; - public SearchParameterFilterAttribute(ISearchParameterValidator searchParamValidator, RequestContextAccessor fhirRequestContextAccessor) + public SearchParameterFilterAttribute(ISearchParameterValidator searchParamValidator) { EnsureArg.IsNotNull(searchParamValidator, nameof(searchParamValidator)); - EnsureArg.IsNotNull(fhirRequestContextAccessor, nameof(fhirRequestContextAccessor)); _searchParameterValidator = searchParamValidator; - _fhirRequestContextAccessor = fhirRequestContextAccessor; } public override async Task OnActionExecutionAsync(ActionExecutingContext context, ActionExecutionDelegate next) @@ -41,20 +33,11 @@ public override async Task OnActionExecutionAsync(ActionExecutingContext context var searchParameter = ExtractSearchParameter(context); if (searchParameter != null) { - var fhirRequestContext = _fhirRequestContextAccessor.RequestContext; - var lastUpdated = fhirRequestContext.GetSearchParameterLastUpdated(); - var hasLastUpdated = lastUpdated.HasValue; - - lastUpdated = await _searchParameterValidator.ValidateSearchParameterInput( + // wait for the validation checks to pass before allowing the FHIRController action to continue + await _searchParameterValidator.ValidateSearchParameterInput( searchParameter, context.HttpContext.Request.Method, - context.HttpContext.RequestAborted, - lastUpdated); - - if (!hasLastUpdated) - { - fhirRequestContext.SetSearchParameterLastUpdated(lastUpdated); - } + context.HttpContext.RequestAborted); } await next(); diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Features/Resources/Bundle/BundleHandler.cs b/src/Microsoft.Health.Fhir.Shared.Api/Features/Resources/Bundle/BundleHandler.cs index 745dcf9c00..de747eb90c 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Features/Resources/Bundle/BundleHandler.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api/Features/Resources/Bundle/BundleHandler.cs @@ -114,12 +114,12 @@ public partial class BundleHandler : IRequestHandler /// Headers to propagate from the inner actions to the outer HTTP request. /// - private static readonly string[] HeadersToAccumulate = [KnownHeaders.RetryAfter, KnownHeaders.RetryAfterMilliseconds, "x-ms-session-token", "x-ms-request-charge"]; + private static readonly string[] HeadersToAccumulate = new[] { KnownHeaders.RetryAfter, KnownHeaders.RetryAfterMilliseconds, "x-ms-session-token", "x-ms-request-charge" }; /// /// Properties to propagate from the outer HTTP requests to the inner actions. /// - private static readonly string[] PropertiesToAccumulate = [KnownQueryParameterNames.OptimizeConcurrency, SearchParameterRequestContextPropertyNames.LastUpdated]; + private static readonly string[] PropertiesToAccumulate = new[] { KnownQueryParameterNames.OptimizeConcurrency }; private static readonly Uri LocalHost = new("http://localhost/"); @@ -219,7 +219,10 @@ public async Task Handle(BundleRequest request, CancellationToke { await FillRequestLists(bundleResource.Entry, cancellationToken); - var responseBundle = new Hl7.Fhir.Model.Bundle { Type = BundleType.BatchResponse }; + var responseBundle = new Hl7.Fhir.Model.Bundle + { + Type = BundleType.BatchResponse, + }; if (bundleProcessingLogic == BundleProcessingLogic.Parallel) { @@ -261,10 +264,13 @@ public async Task Handle(BundleRequest request, CancellationToke } } - var responseBundle = new Hl7.Fhir.Model.Bundle { Type = BundleType.TransactionResponse }; - await CheckSearchParamInputConflictsAndUpdateCache(bundleResource, cancellationToken); + var responseBundle = new Hl7.Fhir.Model.Bundle + { + Type = BundleType.TransactionResponse, + }; + await ExecuteTransactionForAllRequestsAsync(responseBundle, bundleProcessingLogic, cancellationToken); var response = new BundleResponse( @@ -326,16 +332,9 @@ private async Task CheckSearchParamInputConflictsAndUpdateCache(Hl7.Fhir.Model.B throw new RequestNotValidException(string.Format(Api.Resources.DuplicateSearchParamCodesAndUrlsInBundle, string.Join(", ", dupCodes), string.Join(", ", dupUrls))); } - // for deletes Entry.Resource is null. need to check in other way - if (!searchParamsInBundle && bundle.Entry.Any(e => e.Request.Method == HTTPVerb.DELETE && e.Request.Url.StartsWith(KnownResourceTypes.SearchParameter, StringComparison.OrdinalIgnoreCase))) - { - searchParamsInBundle = true; - } - if (searchParamsInBundle) { await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken); // refresh search param cache - _fhirRequestContextAccessor.RequestContext.SetSearchParameterLastUpdated(_searchParameterOperations.SearchParamLastUpdated); // capture last updated } } @@ -1112,6 +1111,19 @@ private static OperationOutcome CreateOperationOutcome(OperationOutcome.IssueSev }; } + private static string SanitizeString(string input) + { + if (string.IsNullOrWhiteSpace(input)) + { + return string.Empty; + } + + return input + .Replace(Environment.NewLine, string.Empty, StringComparison.OrdinalIgnoreCase) + .Replace("\r", " ", StringComparison.OrdinalIgnoreCase) + .Replace("\n", " ", StringComparison.OrdinalIgnoreCase); + } + private BundleHandlerStatistics CreateNewBundleHandlerStatistics(BundleProcessingLogic processingLogic) { BundleHandlerStatistics statistics = new BundleHandlerStatistics( diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Conformance/FirelyTerminologyServiceProxyTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Conformance/FirelyTerminologyServiceProxyTests.cs index 98fd8ed9d6..bb93f7d06b 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Conformance/FirelyTerminologyServiceProxyTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Conformance/FirelyTerminologyServiceProxyTests.cs @@ -18,7 +18,6 @@ using Hl7.Fhir.Specification.Source; using Hl7.Fhir.Specification.Terminology; using Microsoft.Extensions.Logging; -using Microsoft.Health.Fhir.Core.Exceptions; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Conformance; using Microsoft.Health.Fhir.Shared.Core.Features.Conformance; @@ -38,7 +37,6 @@ public class FirelyTerminologyServiceProxyTests private readonly FirelyTerminologyServiceProxy _proxy; private readonly ITerminologyService _terminologyService; private readonly IAsyncResourceResolver _resourceResolver; - private readonly ILogger _logger; public FirelyTerminologyServiceProxyTests() { @@ -53,11 +51,10 @@ public FirelyTerminologyServiceProxyTests() _resourceResolver.ResolveByCanonicalUriAsync( Arg.Any()) .Returns(Task.FromResult(null)); - _logger = Substitute.For>(); _proxy = new FirelyTerminologyServiceProxy( _terminologyService, _resourceResolver, - _logger); + Substitute.For>()); } [Theory] @@ -345,82 +342,6 @@ private static ValueSet CreateValueSet(string id = default) }; } - [Fact] - public async Task GivenUnknownValueSet_WhenExpanding_ThenLogsWarningAndThrowsResourceNotFoundException() - { - var exception = new FhirOperationException( - "ValueSet 'http://terminology.medigent.ca/fhir/ValueSet/123' is unknown", - HttpStatusCode.NotFound); - - _terminologyService.Expand( - Arg.Any(), - Arg.Any(), - Arg.Any()) - .Throws(exception); - - var parameters = new List> - { - Tuple.Create(TerminologyOperationParameterNames.Expand.Url, "http://terminology.medigent.ca/fhir/ValueSet/123"), - }; - - var ex = await Assert.ThrowsAsync( - () => _proxy.ExpandAsync(parameters, null, CancellationToken.None)); - Assert.Contains("is unknown", ex.Message, StringComparison.OrdinalIgnoreCase); - - _logger.Received(1).Log( - LogLevel.Warning, - Arg.Any(), - Arg.Any(), - Arg.Any(), - Arg.Any>()); - - _logger.DidNotReceive().Log( - LogLevel.Error, - Arg.Any(), - Arg.Any(), - Arg.Any(), - Arg.Any>()); - } - - [Fact] - public async Task GivenUnexpectedError_WhenExpanding_ThenLogsErrorAndReturnsOperationOutcome() - { - var exception = new FhirOperationException( - "An unexpected server error occurred.", - HttpStatusCode.InternalServerError); - - _terminologyService.Expand( - Arg.Any(), - Arg.Any(), - Arg.Any()) - .Throws(exception); - - var parameters = new List> - { - Tuple.Create(TerminologyOperationParameterNames.Expand.Url, "http://acme.com/fhir/ValueSet/23"), - }; - - var resourceElement = await _proxy.ExpandAsync(parameters, null, CancellationToken.None); - Assert.NotNull(resourceElement); - - var resource = resourceElement.ToPoco() as OperationOutcome; - Assert.NotNull(resource); - - _logger.Received(1).Log( - LogLevel.Error, - Arg.Any(), - Arg.Any(), - Arg.Any(), - Arg.Any>()); - - _logger.DidNotReceive().Log( - LogLevel.Warning, - Arg.Any(), - Arg.Any(), - Arg.Any(), - Arg.Any>()); - } - public static IEnumerable GetExpandTestData() { var data = new[] diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Resources/Delete/DeletionServiceTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Resources/Delete/DeletionServiceTests.cs index 2ddbe4eee5..84e01f8f27 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Resources/Delete/DeletionServiceTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Resources/Delete/DeletionServiceTests.cs @@ -16,7 +16,6 @@ using Microsoft.Health.Core.Features.Audit; using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Core.Configs; -using Microsoft.Health.Fhir.Core.Exceptions; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Audit; using Microsoft.Health.Fhir.Core.Features.Conformance; @@ -156,120 +155,5 @@ public async Task GivenBulkHardDelete_WhenResourcesAreDeleted_ThenAuditLoggerIsC Arg.Any(), Arg.Is>(d => d.ContainsKey("Affected Items"))); } - - [Fact] - public async Task GivenSearchParameterDelete_WhenConcurrencyConflictOccurs_ThenRetries() - { - var resourceType = "SearchParameter"; - var parameters = new List>() - { - Tuple.Create("url", "http://test.com/param"), - }; - - var request = new ConditionalDeleteResourceRequest( - resourceType, - parameters, - DeleteOperation.HardDelete, - maxDeleteCount: 10, - deleteAll: false); - - var searchService = Substitute.For(); - var scopedSearchService = Substitute.For>(); - scopedSearchService.Value.Returns(searchService); - _searchServiceFactory.Invoke().Returns(scopedSearchService); - - var searchParameter = new SearchParameter { Id = "test", Url = "http://test.com/param" }; - var resource = searchParameter.ToResourceElement(); - var rawResource = new RawResource(searchParameter.ToJson(), FhirResourceFormat.Json, isMetaSet: false); - var resourceRequest = Substitute.For(); - var compartmentIndices = Substitute.For(); - var wrapper = new ResourceWrapper(resource, rawResource, resourceRequest, false, null, compartmentIndices, new List>(), "hash"); - var entries = new List { new SearchResultEntry(wrapper, SearchEntryMode.Match) }; - - searchService.SearchAsync( - Arg.Any(), - Arg.Any>>(), - Arg.Any(), - Arg.Any(), - Arg.Any(), - Arg.Any(), - Arg.Any()).Returns( - Task.FromResult(new SearchResult(entries, null, null, Array.Empty>()))); - - var fhirDataStore = Substitute.For(); - var scopedDataStore = new DeletionServiceScopedDataStore(fhirDataStore); - _dataStoreFactory.GetScopedDataStore().Returns(scopedDataStore); - - var attemptCount = 0; - _searchParameterOperations - .DeleteSearchParameterAsync(Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(callInfo => - { - attemptCount++; - if (attemptCount < 3) - { - throw new BadRequestException(Core.Resources.SearchParameterConcurrencyConflict); - } - - return Task.CompletedTask; - }); - - await _service.DeleteMultipleAsync(request, CancellationToken.None); - - Assert.Equal(3, attemptCount); - } - - [Fact] - public async Task GivenSearchParameterDelete_WhenConcurrencyConflictExhaustsRetries_ThenThrowsWithRetryCount() - { - var resourceType = "SearchParameter"; - var parameters = new List>() - { - Tuple.Create("url", "http://test.com/param"), - }; - - var request = new ConditionalDeleteResourceRequest( - resourceType, - parameters, - DeleteOperation.HardDelete, - maxDeleteCount: 10, - deleteAll: false); - - var searchService = Substitute.For(); - var scopedSearchService = Substitute.For>(); - scopedSearchService.Value.Returns(searchService); - _searchServiceFactory.Invoke().Returns(scopedSearchService); - - var searchParameter = new SearchParameter { Id = "test", Url = "http://test.com/param" }; - var resource = searchParameter.ToResourceElement(); - var rawResource = new RawResource(searchParameter.ToJson(), FhirResourceFormat.Json, isMetaSet: false); - var resourceRequest = Substitute.For(); - var compartmentIndices = Substitute.For(); - var wrapper = new ResourceWrapper(resource, rawResource, resourceRequest, false, null, compartmentIndices, new List>(), "hash"); - var entries = new List { new SearchResultEntry(wrapper, SearchEntryMode.Match) }; - - searchService.SearchAsync( - Arg.Any(), - Arg.Any>>(), - Arg.Any(), - Arg.Any(), - Arg.Any(), - Arg.Any(), - Arg.Any()).Returns( - Task.FromResult(new SearchResult(entries, null, null, Array.Empty>()))); - - var fhirDataStore = Substitute.For(); - var scopedDataStore = new DeletionServiceScopedDataStore(fhirDataStore); - _dataStoreFactory.GetScopedDataStore().Returns(scopedDataStore); - - _searchParameterOperations - .DeleteSearchParameterAsync(Arg.Any(), Arg.Any(), Arg.Any()) - .Returns(_ => throw new BadRequestException(Core.Resources.SearchParameterConcurrencyConflict)); - - var exception = await Assert.ThrowsAsync>>(async () => - await _service.DeleteMultipleAsync(request, CancellationToken.None)); - - Assert.Contains(" Deletion.3", exception.InnerException.Message); - } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterBehaviorTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterBehaviorTests.cs index 112663ad09..ce29c555ca 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterBehaviorTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterBehaviorTests.cs @@ -66,8 +66,6 @@ public SearchParameterBehaviorTests() .Returns(x => CreateResourceWrapper(x.ArgAt(0), x.ArgAt(1))); _fhirDataStore = Substitute.For(); - - _searchParameterOperations.SearchParamLastUpdated.Returns(System.DateTimeOffset.UtcNow); } [Fact] @@ -80,7 +78,7 @@ public async Task GivenACreateResourceRequest_WhenCreatingAResourceOtherThanSear var response = new UpsertResourceResponse(new SaveOutcome(new RawResourceElement(wrapper), SaveOutcomeType.Created)); - var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterDefinitionManager, _requestContextAccessor, _modelInfoProvider); + var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterStatusManager, _requestContextAccessor, _modelInfoProvider); await behavior.Handle(request, async (ct) => await Task.Run(() => response), CancellationToken.None); await _searchParameterOperations.DidNotReceive().ValidateSearchParameterAsync(Arg.Any(), Arg.Any()); @@ -97,7 +95,7 @@ public async Task GivenACreateResourceRequest_WhenCreatingASearchParameterResour var response = new UpsertResourceResponse(new SaveOutcome(new RawResourceElement(wrapper), SaveOutcomeType.Created)); - var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterDefinitionManager, _requestContextAccessor, _modelInfoProvider); + var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterStatusManager, _requestContextAccessor, _modelInfoProvider); await behavior.Handle(request, async (ct) => await Task.Run(() => response), CancellationToken.None); await _searchParameterOperations.Received().ValidateSearchParameterAsync(Arg.Any(), Arg.Any()); @@ -134,6 +132,8 @@ public async Task GivenADeleteResourceRequest_WhenDeletingASearchParameterResour var wrapper = CreateResourceWrapper(resource, false); _fhirDataStore.GetAsync(key, Arg.Any()).Returns(wrapper); + _searchParameterStatusManager.GetAllSearchParameterStatus(Arg.Any()) + .Returns(new List()); var contextProperties = new Dictionary(); var fhirContext = Substitute.For(); @@ -145,11 +145,12 @@ public async Task GivenADeleteResourceRequest_WhenDeletingASearchParameterResour var behavior = new DeleteSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterDefinitionManager, _searchParameterStatusManager, _requestContextAccessor, _modelInfoProvider); await behavior.Handle(request, async (ct) => await Task.Run(() => response), CancellationToken.None); - Assert.True(contextProperties.ContainsKey(SearchParameterRequestContextPropertyNames.PendingStatus)); - var pendingStatus = contextProperties[SearchParameterRequestContextPropertyNames.PendingStatus] as ResourceSearchParameterStatus; - Assert.NotNull(pendingStatus); - Assert.Equal(SearchParameterStatus.PendingDelete, pendingStatus.Status); - Assert.Equal("http://example.com/Id", pendingStatus.Uri.OriginalString); + await _searchParameterStatusManager.Received().GetAllSearchParameterStatus(Arg.Any()); + Assert.True(contextProperties.ContainsKey(SearchParameterRequestContextPropertyNames.PendingStatusUpdates)); + var pendingStatuses = contextProperties[SearchParameterRequestContextPropertyNames.PendingStatusUpdates] as List; + Assert.Single(pendingStatuses); + Assert.Equal(SearchParameterStatus.PendingDelete, pendingStatuses[0].Status); + Assert.Equal("http://example.com/Id", pendingStatuses[0].Uri.OriginalString); } [Fact] @@ -226,7 +227,7 @@ public async Task GivenAnUpsertResourceRequest_WhenSearchParameterDoesNotExist_T var response = new UpsertResourceResponse(new SaveOutcome(new RawResourceElement(wrapper), SaveOutcomeType.Created)); - var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterDefinitionManager, _requestContextAccessor, _modelInfoProvider); + var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterStatusManager, _requestContextAccessor, _modelInfoProvider); await behavior.Handle(request, async (ct) => await Task.Run(() => response), CancellationToken.None); await _searchParameterOperations.Received().ValidateSearchParameterAsync(Arg.Any(), Arg.Any()); @@ -250,7 +251,7 @@ public async Task GivenAnUpsertResourceRequest_WhenSearchParameterExists_ThenVal var response = new UpsertResourceResponse(new SaveOutcome(new RawResourceElement(newWrapper), SaveOutcomeType.Updated)); - var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterDefinitionManager, _requestContextAccessor, _modelInfoProvider); + var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterStatusManager, _requestContextAccessor, _modelInfoProvider); await behavior.Handle(request, async (ct) => await Task.Run(() => response), CancellationToken.None); await _searchParameterOperations.Received().ValidateSearchParameterAsync(Arg.Any(), Arg.Any()); @@ -281,13 +282,14 @@ public async Task GivenAnUpsertResourceRequest_WhenSearchParameterUrlChanges_The var response = new UpsertResourceResponse(new SaveOutcome(new RawResourceElement(newWrapper), SaveOutcomeType.Updated)); - var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterDefinitionManager, _requestContextAccessor, _modelInfoProvider); + var behavior = new CreateOrUpdateSearchParameterBehavior(_searchParameterOperations, _fhirDataStore, _searchParameterStatusManager, _requestContextAccessor, _modelInfoProvider); await behavior.Handle(request, async (ct) => await Task.Run(() => response), CancellationToken.None); - var pendingStatus = contextProperties[SearchParameterRequestContextPropertyNames.PendingStatus] as ResourceSearchParameterStatus; - Assert.NotNull(pendingStatus); - Assert.Equal("http://example.com/new-url", pendingStatus.Uri.OriginalString); - Assert.Equal(SearchParameterStatus.Supported, pendingStatus.Status); + var pendingStatuses = contextProperties[SearchParameterRequestContextPropertyNames.PendingStatusUpdates] as List; + Assert.NotNull(pendingStatuses); + Assert.Equal(2, pendingStatuses.Count); + Assert.Contains(pendingStatuses, s => s.Uri.OriginalString == "http://example.com/old-url" && s.Status == SearchParameterStatus.Deleted); + Assert.Contains(pendingStatuses, s => s.Uri.OriginalString == "http://example.com/new-url" && s.Status == SearchParameterStatus.Supported); } private ResourceWrapper CreateResourceWrapper(ResourceElement resource, bool isDeleted) diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterRetryTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterRetryTests.cs deleted file mode 100644 index fe7f372957..0000000000 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterRetryTests.cs +++ /dev/null @@ -1,94 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System.Threading.Tasks; -using Microsoft.Health.Fhir.Core.Exceptions; -using Microsoft.Health.Fhir.Core.Features.Persistence; -using Microsoft.Health.Fhir.Core.Features.Search.Parameters; -using Microsoft.Health.Fhir.Tests.Common; -using Microsoft.Health.Test.Utilities; -using Xunit; -using Task = System.Threading.Tasks.Task; - -namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Search.SearchParameters -{ - [Trait(Traits.OwningTeam, OwningTeam.Fhir)] - [Trait(Traits.Category, Categories.Search)] - public class SearchParameterRetryTests - { - [Fact] - public async Task GivenConcurrencyConflict_WhenRetriesExhausted_ThenThrowsWithRetryCount() - { - var attemptCount = 0; - var maxAttempts = 4; // 1 initial + 3 retries - - var exception = await Assert.ThrowsAsync(async () => - { - await SearchParameterRetry.ExecuteAsync( - async () => - { - attemptCount++; - await Task.CompletedTask; - throw new BadRequestException(Core.Resources.SearchParameterConcurrencyConflict); - }); - }); - - Assert.Equal(maxAttempts, attemptCount); - Assert.Contains(" .3", exception.Message); - } - - [Fact] - public async Task GivenSuccessfulOperation_WhenExecuted_ThenNoRetry() - { - var attemptCount = 0; - - var result = await SearchParameterRetry.ExecuteAsync( - async () => - { - attemptCount++; - await Task.CompletedTask; - return "success"; - }); - - Assert.Equal(1, attemptCount); - Assert.Equal("success", result); - } - - [Fact] - public async Task GivenNonGenericOverload_WhenOperationSucceeds_ThenCompletes() - { - var executed = false; - - await SearchParameterRetry.ExecuteAsync( - async () => - { - await Task.CompletedTask; - executed = true; - }); - - Assert.True(executed); - } - - [Fact] - public async Task GivenNonConcurrencyException_WhenThrown_ThenNoRetry() - { - var attemptCount = 0; - - var exception = await Assert.ThrowsAsync(async () => - { - await SearchParameterRetry.ExecuteAsync( - async () => - { - attemptCount++; - await Task.CompletedTask; - throw new BadRequestException("some other error"); - }); - }); - - Assert.Equal(1, attemptCount); - Assert.DoesNotContain(" .", exception.Message); - } - } -} diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterValidatorTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterValidatorTests.cs index 7ae8ef6fd1..c1912af0cd 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterValidatorTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/SearchParameters/SearchParameterValidatorTests.cs @@ -69,7 +69,6 @@ public SearchParameterValidatorTests() return true; }); _searchParameterOperations.EnsureNoActiveReindexJobAsync(CancellationToken.None).Returns(Task.CompletedTask); - _searchParameterOperations.SearchParamLastUpdated.Returns(System.DateTimeOffset.UtcNow); } [Theory] diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems index 54d313edb1..a3518aa589 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems @@ -63,7 +63,6 @@ - diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Conformance/FirelyTerminologyServiceProxy.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Conformance/FirelyTerminologyServiceProxy.cs index cfeac62bc6..66c645ac4a 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Conformance/FirelyTerminologyServiceProxy.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Conformance/FirelyTerminologyServiceProxy.cs @@ -17,7 +17,6 @@ using Hl7.Fhir.Specification.Terminology; using Microsoft.Extensions.Logging; using Microsoft.Health.Fhir.Core; -using Microsoft.Health.Fhir.Core.Exceptions; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Conformance; using Microsoft.Health.Fhir.Core.Features.Persistence; @@ -97,11 +96,6 @@ public async Task ExpandAsync( #endif return resource.ToResourceElement(); } - catch (FhirOperationException ex) when (ex.Message.Contains("is unknown", StringComparison.OrdinalIgnoreCase)) - { - _logger.LogWarning(ex, "Failed to expand: ValueSet not found."); - throw new ResourceNotFoundException(ex.Message); - } catch (Exception ex) { _logger.LogError(ex, "Failed to expand."); diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Conformance/TerminologyRequestHandler.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Conformance/TerminologyRequestHandler.cs index 0b83537d54..daf211a15a 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Conformance/TerminologyRequestHandler.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Conformance/TerminologyRequestHandler.cs @@ -79,10 +79,6 @@ public async Task Handle( cancellationToken); return new ExpandResponse(resource); } - catch (ResourceNotFoundException) - { - throw; - } catch (Exception ex) { _logger.LogError(ex, "Failed to handle the request."); diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Resources/Delete/DeletionService.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Resources/Delete/DeletionService.cs index 8c34c154ec..e7e9796c6d 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Resources/Delete/DeletionService.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Resources/Delete/DeletionService.cs @@ -39,7 +39,7 @@ namespace Microsoft.Health.Fhir.Core.Features.Persistence { - public class DeletionService : IDeletionService, IDisposable + public class DeletionService : IDeletionService { private readonly IResourceWrapperFactory _resourceWrapperFactory; private readonly Lazy _conformanceProvider; @@ -54,8 +54,6 @@ public class DeletionService : IDeletionService, IDisposable private readonly ISearchParameterOperations _searchParameterOperations; private readonly IResourceDeserializer _resourceDeserializer; private readonly ILogger _logger; - private readonly SemaphoreSlim _searchParamDeleteSemaphore; - private bool _disposed; internal const string DefaultCallerAgent = "Microsoft.Health.Fhir.Server"; private const int MaxParallelThreads = 64; @@ -85,7 +83,6 @@ public DeletionService( _fhirRuntimeConfiguration = EnsureArg.IsNotNull(fhirRuntimeConfiguration, nameof(fhirRuntimeConfiguration)); _searchParameterOperations = EnsureArg.IsNotNull(searchParameterOperations, nameof(searchParameterOperations)); _resourceDeserializer = EnsureArg.IsNotNull(resourceDeserializer, nameof(resourceDeserializer)); - _searchParamDeleteSemaphore = new SemaphoreSlim(1, 1); _retryPolicy = Policy .Handle() @@ -365,7 +362,7 @@ await CreateAuditLog( false, resourcesToDelete.Select((item) => (item.Resource.ResourceTypeName, item.Resource.ResourceId, item.SearchEntryMode == ValueSets.SearchEntryMode.Include))); - var softDeleteIncludes = await Task.WhenAll(resourcesToDelete.Where(resource => resource.SearchEntryMode == ValueSets.SearchEntryMode.Include).Select(async item => + ResourceWrapperOperation[] softDeleteIncludes = await Task.WhenAll(resourcesToDelete.Where(resource => resource.SearchEntryMode == ValueSets.SearchEntryMode.Include).Select(async item => { // If there isn't a cached capability statement (IE this is the first request made after a service starts up) then performance on this request will be terrible as the capability statement needs to be rebuilt for every resource. // This is because the capability statement can't be made correctly in a background job, so it doesn't cache the result. @@ -375,7 +372,7 @@ await CreateAuditLog( return new ResourceWrapperOperation(deletedWrapper, true, keepHistory, null, false, false, bundleResourceContext: request.BundleResourceContext); })); - var softDeleteMatches = await Task.WhenAll(resourcesToDelete.Where(resource => resource.SearchEntryMode == ValueSets.SearchEntryMode.Match).Select(async item => + ResourceWrapperOperation[] softDeleteMatches = await Task.WhenAll(resourcesToDelete.Where(resource => resource.SearchEntryMode == ValueSets.SearchEntryMode.Match).Select(async item => { bool keepHistory = await _conformanceProvider.Value.CanKeepHistory(item.Resource.ResourceTypeName, cancellationToken); ResourceWrapper deletedWrapper = CreateSoftDeletedWrapper(item.Resource.ResourceTypeName, item.Resource.ResourceId); @@ -400,7 +397,9 @@ await CreateAuditLog( .FirstOrDefault().SearchEntryMode == ValueSets.SearchEntryMode.Include))); } - await DeleteSearchParametersAsync(resourcesToDelete.Where(resource => resource.SearchEntryMode == ValueSets.SearchEntryMode.Match), cancellationToken); + await DeleteSearchParametersAsync( + resourcesToDelete.Where(resource => resource.SearchEntryMode == ValueSets.SearchEntryMode.Match).Select(x => x.Resource).ToList(), + cancellationToken); await fhirDataStore.MergeAsync(softDeleteMatches, cancellationToken); } @@ -459,27 +458,20 @@ await CreateAuditLog( var matchedResources = resourcesToDelete.Where(resource => resource.SearchEntryMode == ValueSets.SearchEntryMode.Match).ToList(); // Delete includes first so that if there is a failure, the match resources are not deleted. This allows the job to restart. - // This throws AggregateExceptions. - // Note: includedResources cannot have search params + // This throws AggrigateExceptions await Parallel.ForEachAsync(includedResources, cancellationToken, async (item, innerCt) => { + await DeleteSearchParameterAsync(item.Resource, cancellationToken); await _retryPolicy.ExecuteAsync(async () => await fhirDataStore.HardDeleteAsync(new ResourceKey(item.Resource.ResourceTypeName, item.Resource.ResourceId), request.DeleteOperation == DeleteOperation.PurgeHistory, request.AllowPartialSuccess, innerCt)); parallelBag.Add((item.Resource.ResourceTypeName, item.Resource.ResourceId, item.SearchEntryMode == ValueSets.SearchEntryMode.Include)); }); - await Parallel.ForEachAsync(matchedResources.Where(_ => _.Resource.ResourceTypeName != KnownResourceTypes.SearchParameter), cancellationToken, async (item, innerCt) => + await Parallel.ForEachAsync(matchedResources, cancellationToken, async (item, innerCt) => { + await DeleteSearchParameterAsync(item.Resource, cancellationToken); await _retryPolicy.ExecuteAsync(async () => await fhirDataStore.HardDeleteAsync(new ResourceKey(item.Resource.ResourceTypeName, item.Resource.ResourceId), request.DeleteOperation == DeleteOperation.PurgeHistory, request.AllowPartialSuccess, innerCt)); parallelBag.Add((item.Resource.ResourceTypeName, item.Resource.ResourceId, item.SearchEntryMode == ValueSets.SearchEntryMode.Include)); }); - - // With concurrency based on max last updated search params must be deleted one-by-one. - foreach (var item in matchedResources.Where(_ => _.Resource.ResourceTypeName == KnownResourceTypes.SearchParameter)) - { - await DeleteSearchParameterWithLockAsync(item, cancellationToken); - await _retryPolicy.ExecuteAsync(async () => await fhirDataStore.HardDeleteAsync(new ResourceKey(item.Resource.ResourceTypeName, item.Resource.ResourceId), request.DeleteOperation == DeleteOperation.PurgeHistory, request.AllowPartialSuccess, cancellationToken)); - parallelBag.Add((item.Resource.ResourceTypeName, item.Resource.ResourceId, item.SearchEntryMode == ValueSets.SearchEntryMode.Include)); - } } catch (Exception ex) { @@ -646,48 +638,22 @@ private bool IsIncludeEnabled() return _configuration.SupportsIncludes && (_fhirRuntimeConfiguration.DataStore?.Equals(KnownDataStores.SqlServer, StringComparison.OrdinalIgnoreCase) ?? false); } - private async Task DeleteSearchParametersAsync(IEnumerable entries, CancellationToken cancellationToken) + private async Task DeleteSearchParametersAsync(IEnumerable resources, CancellationToken cancellationToken) { - foreach (var entry in entries.Where(_ => _.Resource.ResourceTypeName == KnownResourceTypes.SearchParameter)) + if (resources?.Any() ?? false) { - await DeleteSearchParameterWithLockAsync(entry, cancellationToken); - } - } - - private async Task DeleteSearchParameterWithLockAsync(SearchResultEntry item, CancellationToken cancellationToken) - { - await _searchParamDeleteSemaphore.WaitAsync(cancellationToken); - try - { - await SearchParameterRetry.ExecuteAsync( - async () => - { - await _searchParameterOperations.DeleteSearchParameterAsync(item.Resource.RawResource, cancellationToken, true); - }, - "Deletion"); - } - finally - { - _searchParamDeleteSemaphore.Release(); + foreach (var resource in resources.Where(x => string.Equals(x?.ResourceTypeName, KnownResourceTypes.SearchParameter, StringComparison.OrdinalIgnoreCase))) + { + await _searchParameterOperations.DeleteSearchParameterAsync(resource.RawResource, cancellationToken, true); + } } } - public void Dispose() - { - Dispose(disposing: true); - GC.SuppressFinalize(this); - } - - protected virtual void Dispose(bool disposing) + private async Task DeleteSearchParameterAsync(ResourceWrapper resource, CancellationToken cancellationToken) { - if (!_disposed) + if (string.Equals(resource?.ResourceTypeName, KnownResourceTypes.SearchParameter, StringComparison.OrdinalIgnoreCase)) { - if (disposing) - { - _searchParamDeleteSemaphore?.Dispose(); - } - - _disposed = true; + await _searchParameterOperations.DeleteSearchParameterAsync(resource.RawResource, cancellationToken, true); } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/ISearchParameterValidator.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/ISearchParameterValidator.cs index d2db9d2525..1980e3e004 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/ISearchParameterValidator.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/ISearchParameterValidator.cs @@ -3,15 +3,13 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -using System; using System.Threading; -using System.Threading.Tasks; using Hl7.Fhir.Model; namespace Microsoft.Health.Fhir.Shared.Core.Features.Search.Parameters { public interface ISearchParameterValidator { - Task ValidateSearchParameterInput(SearchParameter searchParam, string method, CancellationToken cancellationToken, DateTimeOffset? lastUpdated = null); + System.Threading.Tasks.Task ValidateSearchParameterInput(SearchParameter searchParam, string method, CancellationToken cancellationToken); } } diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterValidator.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterValidator.cs index c656183392..4b668833e5 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterValidator.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/Parameters/SearchParameterValidator.cs @@ -8,7 +8,6 @@ using System.Linq; using System.Reflection.Metadata; using System.Threading; -using System.Threading.Tasks; using EnsureThat; using FluentValidation.Results; using Hl7.Fhir.Model; @@ -69,7 +68,7 @@ public SearchParameterValidator( _logger = EnsureArg.IsNotNull(logger, nameof(logger)); } - public async Task ValidateSearchParameterInput(SearchParameter searchParam, string method, CancellationToken cancellationToken, DateTimeOffset? lastUpdated = null) + public async Task ValidateSearchParameterInput(SearchParameter searchParam, string method, CancellationToken cancellationToken) { await _authorizationService.CheckAccess(DataActions.Reindex, true, cancellationToken); @@ -78,7 +77,7 @@ public SearchParameterValidator( if (string.IsNullOrEmpty(searchParam.Url) && (method.Equals(HttpDeleteName, StringComparison.Ordinal) || method.Equals(HttpPatchName, StringComparison.Ordinal))) { // Return out if this is delete OR patch call and no Url so FHIRController can move to next action - return null; + return; } var validationFailures = new List(); @@ -109,11 +108,7 @@ public SearchParameterValidator( else { // Refresh the search parameter cache in the search parameter definition manager before starting the validation. - if (!lastUpdated.HasValue) - { - await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken); - lastUpdated = _searchParameterOperations.SearchParamLastUpdated; - } + await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken); // If a search parameter with the same uri exists already if (_searchParameterDefinitionManager.TryGetSearchParameter(searchParam.Url, out var searchParameterInfo)) @@ -164,8 +159,6 @@ public SearchParameterValidator( { throw new ResourceNotValidException(validationFailures); } - - return lastUpdated.Value; // value should not be null here. } private void CheckForConflictingCodeValue(SearchParameter searchParam, List validationFailures) diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/ScalarTemporalEqualityRewriterTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/ScalarTemporalEqualityRewriterTests.cs index ebfcfe68fc..493c3c4651 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/ScalarTemporalEqualityRewriterTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/ScalarTemporalEqualityRewriterTests.cs @@ -163,6 +163,7 @@ public void GivenRootExpressionContainsForwardChainAndAllowListedBirthdateExactD Assert.Same(birthdate, rewrittenBirthdate); } + [Theory] [MemberData(nameof(NonRewritableExpressions))] public void GivenAllowListedBirthdateWithNonExactDayExpression_WhenRewritten_ThenPassThrough(Expression inner) diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlServerSearchServiceQueryStoreTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlServerSearchServiceQueryStoreTests.cs index f55e67d3c0..ec384cd308 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlServerSearchServiceQueryStoreTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/SqlServerSearchServiceQueryStoreTests.cs @@ -1057,90 +1057,5 @@ public void StripDboSchemaPrefix_NullInput_ReturnsNull() // Assert Assert.Null(result); } - - // ----------------------------------------------------------------------- - // ExtractParameterHash - // ----------------------------------------------------------------------- - - [Theory] - - // No hash comment -> null - [InlineData(null, null)] - [InlineData("", null)] - [InlineData("SELECT * FROM dbo.Resource", null)] - [InlineData("WITH cte0 AS (SELECT 1) SELECT * FROM cte0", null)] - - // Hash comment with params= suffix -> returns just the base64 hash - [InlineData( - "WITH cte0 AS (SELECT 1)\r\n/* HASH LKzCYkd+9LKATJ3BQebo9R3aYgN9ZXVU/42C2WntCUo= params=@p0,@p1,@p2,@p3,@p4,@p5,@p6,@p7 */\r\nSELECT * FROM cte0", - "LKzCYkd+9LKATJ3BQebo9R3aYgN9ZXVU/42C2WntCUo=")] - [InlineData( - "SELECT 1 /* HASH Ek+CaDdNQVS2c/2D7Gw5XA0Dts4v0KOssOrfn2bNYm0= params=@p0 */", - "Ek+CaDdNQVS2c/2D7Gw5XA0Dts4v0KOssOrfn2bNYm0=")] - - // Hash comment without params= suffix -> returns the hash - [InlineData("SELECT 1 /* HASH abc123def456= */", "abc123def456=")] - - // Realistic multi-CTE chained search query -> returns the embedded hash - [InlineData( - @" - SET STATISTICS IO ON; - SET STATISTICS TIME ON; - - DECLARE @p0 varchar(256) = 'active' - DECLARE @p1 varchar(256) = 'Smith%' - DECLARE @p8 int = 11 - ;WITH - cte0 AS - ( - SELECT ResourceTypeId AS T1, ResourceSurrogateId AS Sid1 - FROM dbo.TokenSearchParam - WHERE SearchParamId = 1260 AND Code = @p0 AND ResourceTypeId = 124 - ) - ,cte7 AS - ( - SELECT DISTINCT TOP (@p8) T1, Sid1, 1 AS IsMatch, 0 AS IsPartial - FROM cte0 - ORDER BY T1 DESC, Sid1 DESC - ) - /* HASH LKzCYkd+9LKATJ3BQebo9R3aYgN9ZXVU/42C2WntCUo= params=@p0,@p1,@p2,@p3,@p4,@p5,@p6,@p7 */ - SELECT * FROM (SELECT DISTINCT r.ResourceTypeId, r.ResourceId FROM dbo.Resource r - JOIN cte7 ON r.ResourceTypeId = cte7.T1 AND r.ResourceSurrogateId = cte7.Sid1 - WHERE IsHistory = 0 AND IsDeleted = 0 - ) AS t ORDER BY t.ResourceTypeId DESC, t.ResourceSurrogateId DESC - option (recompile)", - "LKzCYkd+9LKATJ3BQebo9R3aYgN9ZXVU/42C2WntCUo=")] - - // Malformed: hash start present but no closing */ -> null - [InlineData("SELECT 1 /* HASH abc123=", null)] - - // Empty / whitespace-only hash -> null (otherwise the downstream LIKE filter - // would match every hash-bearing Query Store row) - [InlineData("SELECT 1 /* HASH params=@p0 */", null)] // empty hash before params= - [InlineData("SELECT 1 /* HASH */", null)] // whitespace-only hash, no params - public void ExtractParameterHash_ReturnsExpectedHashOrNull(string input, string expectedHash) - { - // Act - string result = SqlServerSearchService.ExtractParameterHash(input); - - // Assert - Assert.Equal(expectedHash, result); - } - - // ----------------------------------------------------------------------- - // Query Store throttling constants - // ----------------------------------------------------------------------- - - [Fact] - public void QueryStoreLookupTimeoutSeconds_IsCappedReasonably() - { - // The diagnostic query timeout must stay short so it never inherits the (much longer) - // main query timeout. It also acts only as a backup to the 2s CancellationToken, so it - // must be at least 1s and comfortably above the CTS deadline. The 10s upper bound is a - // sanity guard, not a hard requirement: if a future change intentionally raises the - // constant past 10s, update both this bound and the rationale comment on - // SqlServerSearchService.QueryStoreLookupTimeoutSeconds. - Assert.InRange(SqlServerSearchService.QueryStoreLookupTimeoutSeconds, 1, 10); - } } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/ExceptionExtension.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/ExceptionExtension.cs index d907d13e2d..ed80757b3d 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/ExceptionExtension.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/ExceptionExtension.cs @@ -3,7 +3,6 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- using System; -using Microsoft.Data.SqlClient; namespace Microsoft.Health.Fhir.SqlServer.Features { @@ -26,15 +25,6 @@ public static bool IsExecutionTimeout(this Exception e) return str.Contains("execution timeout expired", StringComparison.OrdinalIgnoreCase); } - public static bool IsSearchParameterConcurrencyConflict(this Exception e) - { - var sqlEx = e as SqlException; - return sqlEx != null - && sqlEx.Number == 50001 - && e.Message.StartsWith("optimistic concurrency conflict", StringComparison.OrdinalIgnoreCase) - && e.Message.Contains("expected last updated", StringComparison.OrdinalIgnoreCase); - } - private static bool HasDeadlockErrorPattern(string str) { return str.Contains("deadlock", StringComparison.OrdinalIgnoreCase); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs index 712b985e4d..6f74747256 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Schema/SchemaVersionConstants.cs @@ -7,9 +7,9 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Schema { public static class SchemaVersionConstants { - public const int Min = (int)SchemaVersion.V112; + public const int Min = (int)SchemaVersion.V103; public const int Max = (int)SchemaVersion.V113; - public const int MinForUpgrade = (int)SchemaVersion.V110; // this is used for upgrade tests only + public const int MinForUpgrade = (int)SchemaVersion.V103; // this is used for upgrade tests only public const int SearchParameterStatusSchemaVersion = (int)SchemaVersion.V6; public const int SupportForReferencesWithMissingTypeVersion = (int)SchemaVersion.V7; public const int SearchParameterHashSchemaVersion = (int)SchemaVersion.V8; diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs index 1baeb4acac..cd6ee8be33 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/SqlServerSearchService.cs @@ -86,14 +86,6 @@ internal class SqlServerSearchService : SearchService private static readonly Regex WhitespacePattern = new Regex(@"\s+", RegexOptions.Compiled); private static ResourceSearchParamStats _resourceSearchParamStats; private static object _locker = new object(); - - /// - /// Hard cap for the diagnostic query command timeout (seconds). The CancellationToken - /// timeout (2s) is the first line of defense; this is a backup in case cancellation - /// doesn't terminate the SQL command promptly. Set on a NEW connection — does not - /// affect search query connections. - /// - internal const int QueryStoreLookupTimeoutSeconds = 5; private static CachedParameter _longRunningQueryDetails; private static CachedParameter _longRunningThreshold; @@ -838,8 +830,31 @@ await _sqlRetryService.ExecuteSql( string queryTextSnapshot = sqlCommand.CommandText; bool isStoredProcSnapshot = sqlCommand.CommandType == CommandType.StoredProcedure; long executionTimeSnapshot = executionStopwatch.ElapsedMilliseconds; + int timeoutSnapshot = (int)_sqlServerDataStoreConfiguration.CommandTimeout.TotalSeconds; - FireAndForgetQueryStoreLookup(queryTextSnapshot, isStoredProcSnapshot, executionTimeSnapshot); + // Fire-and-forget: Log query details without blocking the response + _ = Task.Run(async () => + { + using var loggingCts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + try + { + await LogQueryStoreByTextAsync( + queryTextSnapshot, + isStoredProcSnapshot, + _logger, + timeoutSnapshot, + executionTimeSnapshot, + loggingCts.Token); + } + catch (Exception ex) + { + _logger.LogWarning( + "Long-running SQL ({ElapsedMilliseconds}ms). Query: {QueryText}. Query Store lookup failed for long-running query.", + executionTimeSnapshot, + queryTextSnapshot); + _logger.LogDebug(ex, "Query Store lookup failed for long-running query."); + } + }); } } } @@ -1125,82 +1140,10 @@ internal static string StripAllWhitespace(string text) internal static string StripDboSchemaPrefix(string procName) => procName?.Replace("dbo.", string.Empty, StringComparison.OrdinalIgnoreCase); - /// - /// Extracts the parameter hash value from a query text that contains a - /// /* HASH {base64hash} params=... */ comment embedded by . - /// Returns null if no hash comment is found. - /// - internal static string ExtractParameterHash(string queryText) - { - if (string.IsNullOrEmpty(queryText)) - { - return null; - } - - // ParametersHashStart/End are always emitted in fixed uppercase by SqlQueryGenerator, - // so use Ordinal (not OrdinalIgnoreCase) to avoid matching arbitrary user-authored - // lowercase comments such as "/* hash ... */". - int hashStart = queryText.IndexOf(Expressions.Visitors.QueryGenerators.SqlQueryGenerator.ParametersHashStart, StringComparison.Ordinal); - if (hashStart < 0) - { - return null; - } - - int valueStart = hashStart + Expressions.Visitors.QueryGenerators.SqlQueryGenerator.ParametersHashStart.Length; - int hashEnd = queryText.IndexOf(Expressions.Visitors.QueryGenerators.SqlQueryGenerator.ParametersHashEnd, valueStart, StringComparison.Ordinal); - if (hashEnd < 0) - { - return null; - } - - // Extract just the base64 hash, stopping at the space before "params=" - string hashAndParams = queryText[valueStart..hashEnd]; - int spaceIndex = hashAndParams.IndexOf(' ', StringComparison.Ordinal); - string hash = spaceIndex >= 0 ? hashAndParams[..spaceIndex] : hashAndParams; - - // Guard against an empty/whitespace-only hash, which would make the downstream - // LIKE '%/* HASH {hash}%' filter match every hash-bearing row. - return string.IsNullOrWhiteSpace(hash) ? null : hash; - } - - /// - /// Runs as a fire-and-forget background task so the - /// diagnostic Query Store lookup never blocks or fails the originating search request. - /// - private void FireAndForgetQueryStoreLookup(string queryText, bool isStoredProcedure, long executionTime) - { - _ = Task.Run(async () => - { - try - { - // CancellationToken fires at 2s; CommandTimeout at 5s is backup. - using var loggingCts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); - - await LogQueryStoreByTextAsync( - queryText, - isStoredProcedure, - QueryStoreLookupTimeoutSeconds, - executionTime, - loggingCts.Token); - } - catch (Exception ex) - { - // The Query Store lookup is best-effort diagnostics. Swallow any failure so the - // fire-and-forget task never surfaces an unobserved exception. The exception is - // passed to the logger (queryable via env_ex_* columns), so it isn't repeated in the message. - _logger.LogWarning( - ex, - "Long-running SQL ({ElapsedMilliseconds}ms). Query={Query} QueryStoreStats={QueryStoreStats}", - executionTime, - queryText, - "Query Store lookup failed."); - } - }); - } - private async Task LogQueryStoreByTextAsync( string queryText, bool isStoredProcedure, + ILogger logger, int timeoutSeconds, long executionTime, CancellationToken ct) @@ -1215,84 +1158,12 @@ await _sqlRetryService.ExecuteSql( var sb = new StringBuilder(); - if (isStoredProcedure) - { - // For stored procedures, use OBJECT_ID to filter directly by the procedure's - // hash/identity in Query Store. This avoids the expensive LIKE scan on - // query_sql_text entirely, since Query Store records object_id for every - // statement executed inside a stored procedure. - string procName = StripDboSchemaPrefix(queryText); + // Query Store records only the bare procedure name without the schema prefix. + string effectiveQuery = isStoredProcedure ? StripDboSchemaPrefix(queryText) : queryText; + var normalizedText = StripQueryPreambleLines(effectiveQuery); + var searchFragments = SplitIntoSearchFragments(normalizedText); - cmd.CommandText = @" - DECLARE @CutoffTime datetimeoffset = DATEADD(HOUR, -1, SYSUTCDATETIME()); - - SELECT TOP (5) - rs.count_executions, - rs.avg_duration / 1000.0 AS avg_duration_ms, - rs.avg_cpu_time / 1000.0 AS avg_cpu_ms, - rs.avg_logical_io_reads, - rs.avg_physical_io_reads, - rs.avg_logical_io_writes, - rs.avg_rowcount, - rs.max_duration / 1000.0 AS max_duration_ms, - rs.last_execution_time, - p.plan_id, - q.query_id - FROM sys.query_store_query q - JOIN sys.query_store_plan p ON p.query_id = q.query_id - JOIN sys.query_store_runtime_stats rs ON rs.plan_id = p.plan_id - WHERE q.object_id = OBJECT_ID(@ProcName) - AND rs.last_execution_time >= @CutoffTime - ORDER BY rs.last_execution_time DESC;"; - - cmd.Parameters.AddWithValue("@ProcName", procName); - - using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false); - await AppendQueryStoreResults(reader, sb, 0, 1, "StoredProc", ct); - } - else - { - // For ad-hoc queries, split into fragments (include queries have 2 statements - // split at INSERT INTO @FilteredData). For each fragment individually: - // - If it contains a parameter hash comment: use the hash for a fast LIKE lookup - // - If hash lookup returns nothing: fall back to the expensive REPLACE+LIKE - // - If it has no hash: filter OUT hash-bearing rows to reduce the LIKE scan set - var normalizedText = StripQueryPreambleLines(queryText); - var searchFragments = SplitIntoSearchFragments(normalizedText); - - // NOTE: The three lookup SQL strings below (HashLookupSql, - // TextLookupWithHashExclusionSql, TextLookupSql) share an identical - // SELECT / FROM / JOIN / ORDER BY structure and only differ in their WHERE - // clause. Any column, cutoff-window, or index-hint change must be applied to - // all three to avoid drift. - // - // SQL for the fast hash-based lookup (no REPLACE chain needed). - const string HashLookupSql = @" - DECLARE @CutoffTime datetimeoffset = DATEADD(HOUR, -1, SYSUTCDATETIME()); - - SELECT TOP (5) - rs.count_executions, - rs.avg_duration / 1000.0 AS avg_duration_ms, - rs.avg_cpu_time / 1000.0 AS avg_cpu_ms, - rs.avg_logical_io_reads, - rs.avg_physical_io_reads, - rs.avg_logical_io_writes, - rs.avg_rowcount, - rs.max_duration / 1000.0 AS max_duration_ms, - rs.last_execution_time, - p.plan_id, - q.query_id - FROM sys.query_store_query_text qt - JOIN sys.query_store_query q ON q.query_text_id = qt.query_text_id - JOIN sys.query_store_plan p ON p.query_id = q.query_id - JOIN sys.query_store_runtime_stats rs ON rs.plan_id = p.plan_id - WHERE qt.query_sql_text LIKE '%' + @HashFilter + '%' - AND rs.last_execution_time >= @CutoffTime - ORDER BY rs.last_execution_time DESC;"; - - // SQL for the expensive REPLACE+LIKE fallback. - // For fragments without a hash, also filter OUT hash-bearing rows to reduce scan set. - const string TextLookupWithHashExclusionSql = @" + cmd.CommandText = @" DECLARE @CutoffTime datetimeoffset = DATEADD(HOUR, -1, SYSUTCDATETIME()); SELECT TOP (5) @@ -1312,95 +1183,58 @@ FROM sys.query_store_query_text qt JOIN sys.query_store_plan p ON p.query_id = q.query_id JOIN sys.query_store_runtime_stats rs ON rs.plan_id = p.plan_id WHERE @NormalizedText <> '' - -- The '/* HASH ' literal below must stay in sync with - -- SqlQueryGenerator.ParametersHashStart (SQL const strings cannot reference the C# constant). - AND qt.query_sql_text NOT LIKE '%/* HASH %' AND replace(replace(replace(replace(replace(replace(qt.query_sql_text, char(9), ''), char(10), ''), char(11), ''), char(12), ''), char(13), ''), char(32), '') LIKE '%' + @NormalizedText + '%' AND rs.last_execution_time >= @CutoffTime ORDER BY rs.last_execution_time DESC;"; - // SQL for the expensive REPLACE+LIKE fallback (no hash exclusion, - // used when hash lookup found nothing for a hash-bearing fragment). - const string TextLookupSql = @" - DECLARE @CutoffTime datetimeoffset = DATEADD(HOUR, -1, SYSUTCDATETIME()); + for (int segmentIndex = 0; segmentIndex < searchFragments.Count; segmentIndex++) + { + string searchFragment = searchFragments[segmentIndex]; - SELECT TOP (5) - rs.count_executions, - rs.avg_duration / 1000.0 AS avg_duration_ms, - rs.avg_cpu_time / 1000.0 AS avg_cpu_ms, - rs.avg_logical_io_reads, - rs.avg_physical_io_reads, - rs.avg_logical_io_writes, - rs.avg_rowcount, - rs.max_duration / 1000.0 AS max_duration_ms, - rs.last_execution_time, - p.plan_id, - q.query_id - FROM sys.query_store_query_text qt - JOIN sys.query_store_query q ON q.query_text_id = qt.query_text_id - JOIN sys.query_store_plan p ON p.query_id = q.query_id - JOIN sys.query_store_runtime_stats rs ON rs.plan_id = p.plan_id - WHERE @NormalizedText <> '' - AND replace(replace(replace(replace(replace(replace(qt.query_sql_text, char(9), ''), char(10), ''), char(11), ''), char(12), ''), char(13), ''), char(32), '') LIKE '%' + @NormalizedText + '%' - AND rs.last_execution_time >= @CutoffTime - ORDER BY rs.last_execution_time DESC;"; + // Strip whitespace first so the 4000-char limit applies to stripped content, + // maximising the amount of meaningful text sent to the LIKE comparison. + string strippedFragment = StripAllWhitespace(searchFragment); - for (int segmentIndex = 0; segmentIndex < searchFragments.Count; segmentIndex++) + if (strippedFragment.Length > 4000) { - string searchFragment = searchFragments[segmentIndex]; + strippedFragment = strippedFragment[..4000]; + } - // Check each fragment individually for an embedded parameter hash. - // Include queries split into 2 fragments: fragment 1 (before INSERT INTO @FilteredData) - // typically has no hash, fragment 2 (after) has the hash comment. - string fragmentHash = ExtractParameterHash(searchFragment); - bool fragmentHasHash = fragmentHash != null; - int matchCount = 0; + cmd.Parameters.Clear(); + cmd.Parameters.AddWithValue("@NormalizedText", strippedFragment); - if (fragmentHasHash) + using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false); + int matchIndex = 0; + while (await reader.ReadAsync(ct).ConfigureAwait(false)) + { + if (await reader.IsDBNullAsync(0, ct).ConfigureAwait(false)) { - // Fast path: search by the embedded parameter hash string. - cmd.CommandText = HashLookupSql; - cmd.Parameters.Clear(); - string hashFilter = Expressions.Visitors.QueryGenerators.SqlQueryGenerator.ParametersHashStart + fragmentHash; - cmd.Parameters.AddWithValue("@HashFilter", hashFilter); - - using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false); - matchCount = await AppendQueryStoreResults(reader, sb, segmentIndex, searchFragments.Count, "Hash", ct); + continue; } - // Fall back to REPLACE+LIKE if hash lookup found nothing or fragment has no hash. - if (matchCount == 0) - { - string strippedFragment = StripAllWhitespace(searchFragment); - - if (strippedFragment.Length > 4000) - { - strippedFragment = strippedFragment[..4000]; - } - - // Fragments without a hash: exclude hash-bearing query store rows. - // Fragments with a hash that had no hash match: search all rows as fallback. - if (fragmentHasHash) - { - cmd.CommandText = TextLookupSql; - } - else - { - cmd.CommandText = TextLookupWithHashExclusionSql; - } - - cmd.Parameters.Clear(); - cmd.Parameters.AddWithValue("@NormalizedText", strippedFragment); - - using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false); - await AppendQueryStoreResults(reader, sb, segmentIndex, searchFragments.Count, fragmentHasHash ? "TextFallback" : "TextNoHash", ct); - } + matchIndex++; + long planId = reader.GetInt64(9); + long queryId = reader.GetInt64(10); + + sb.AppendLine() + .Append($" batch[{segmentIndex + 1}] match[{matchIndex}]") + .Append($" execs={reader.GetInt64(0)}") + .Append($" avgDurMs={Convert.ToDouble(reader.GetValue(1)):F1}") + .Append($" avgCpuMs={Convert.ToDouble(reader.GetValue(2)):F1}") + .Append($" avgLReads={Convert.ToDouble(reader.GetValue(3)):F0}") + .Append($" avgPReads={Convert.ToDouble(reader.GetValue(4)):F0}") + .Append($" avgLWrites={Convert.ToDouble(reader.GetValue(5)):F0}") + .Append($" avgRows={Convert.ToDouble(reader.GetValue(6)):F0}") + .Append($" maxDurMs={Convert.ToDouble(reader.GetValue(7)):F1}") + .Append($" lastExec={reader.GetDateTimeOffset(8):o}") + .Append($" queryId={queryId}") + .Append($" planId={planId}"); } } if (sb.Length > 0) { - _logger.LogWarning( + logger.LogWarning( "Long-running SQL ({ElapsedMilliseconds}ms). Query={Query} QueryStoreStats={QueryStoreStats}", executionTime, queryText, @@ -1408,65 +1242,18 @@ AND replace(replace(replace(replace(replace(replace(qt.query_sql_text, char(9), } else { - _logger.LogWarning( + logger.LogWarning( "Long-running SQL ({ElapsedMilliseconds}ms). Query={Query} QueryStoreStats={QueryStoreStats}", executionTime, queryText, "No Query Store matches found."); } }, - _logger, + logger, ct, isReadOnly: true); } - /// - /// Reads Query Store results from a and appends formatted - /// stats to the . Returns the number of matches read. - /// - private static async Task AppendQueryStoreResults( - SqlDataReader reader, - StringBuilder sb, - int segmentIndex, - int totalSegments, - string lookupMethod, - CancellationToken ct) - { - int matchIndex = 0; - while (await reader.ReadAsync(ct).ConfigureAwait(false)) - { - if (await reader.IsDBNullAsync(0, ct).ConfigureAwait(false)) - { - continue; - } - - matchIndex++; - long planId = reader.GetInt64(9); - long queryId = reader.GetInt64(10); - - string prefix = totalSegments > 1 - ? $" batch[{segmentIndex + 1}] match[{matchIndex}]" - : $" match[{matchIndex}]"; - - sb.AppendLine() - .Append(prefix) - .Append($" lookup={lookupMethod}") - .Append($" execs={reader.GetInt64(0)}") - .Append($" avgDurMs={Convert.ToDouble(reader.GetValue(1)):F1}") - .Append($" avgCpuMs={Convert.ToDouble(reader.GetValue(2)):F1}") - .Append($" avgLReads={Convert.ToDouble(reader.GetValue(3)):F0}") - .Append($" avgPReads={Convert.ToDouble(reader.GetValue(4)):F0}") - .Append($" avgLWrites={Convert.ToDouble(reader.GetValue(5)):F0}") - .Append($" avgRows={Convert.ToDouble(reader.GetValue(6)):F0}") - .Append($" maxDurMs={Convert.ToDouble(reader.GetValue(7)):F1}") - .Append($" lastExec={reader.GetDateTimeOffset(8):o}") - .Append($" queryId={queryId}") - .Append($" planId={planId}"); - } - - return matchIndex; - } - private static (long StartId, long EndId, int Count) ReaderToSurrogateIdRange(SqlDataReader sqlDataReader) { return (sqlDataReader.GetInt64(1), sqlDataReader.GetInt64(2), sqlDataReader.GetInt32(3)); @@ -2230,8 +2017,31 @@ await _sqlRetryService.ExecuteSql( string queryTextSnapshot = sqlCommand.CommandText; bool isStoredProcSnapshot = sqlCommand.CommandType == CommandType.StoredProcedure; long executionTimeSnapshot = executionStopwatch.ElapsedMilliseconds; + int timeoutSnapshot = (int)_sqlServerDataStoreConfiguration.CommandTimeout.TotalSeconds; - FireAndForgetQueryStoreLookup(queryTextSnapshot, isStoredProcSnapshot, executionTimeSnapshot); + // Fire-and-forget: Log query details without blocking the response + _ = Task.Run(async () => + { + using var loggingCts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + try + { + await LogQueryStoreByTextAsync( + queryTextSnapshot, + isStoredProcSnapshot, + _logger, + timeoutSnapshot, + executionTimeSnapshot, + loggingCts.Token); + } + catch (Exception ex) + { + _logger.LogWarning( + "Long-running SQL ({ElapsedMilliseconds}ms). Query: {QueryText}. Query Store lookup failed for long-running query.", + executionTimeSnapshot, + queryTextSnapshot); + _logger.LogDebug(ex, "Query Store lookup failed for long-running query."); + } + }); } } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/Registry/SqlServerSearchParameterStatusDataStore.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/Registry/SqlServerSearchParameterStatusDataStore.cs index 3026b2a430..3c0640c3f9 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/Registry/SqlServerSearchParameterStatusDataStore.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/Registry/SqlServerSearchParameterStatusDataStore.cs @@ -12,10 +12,8 @@ using EnsureThat; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; -using Microsoft.Health.Fhir.Core.Exceptions; using Microsoft.Health.Fhir.Core.Features.Definition; using Microsoft.Health.Fhir.Core.Features.Operations; -using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Search; using Microsoft.Health.Fhir.Core.Features.Search.Registry; using Microsoft.Health.Fhir.Core.Models; @@ -141,21 +139,64 @@ public async Task UpsertStatuses(IReadOnlyCollection statuses, int maxRetries, CancellationToken cancellationToken, long? reindexId = null) + { + var currentStatuses = statuses.ToList(); + int retryCount = 0; - try + while (retryCount <= maxRetries) { - await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, cancellationToken); + try + { + await UpsertStatusesInternal(currentStatuses, cancellationToken, reindexId); + return; // Success + } + catch (SqlException sqlEx) when (sqlEx.Number == 50001 && retryCount < maxRetries) // Our custom concurrency error + { + // Optimistic concurrency conflict detected - refresh and retry + retryCount++; + _logger.LogWarning("Optimistic concurrency conflict detected on attempt {RetryCount}. Retrying...", retryCount); + + // Refresh the statuses with current LastUpdated values + var refreshedStatuses = await GetSearchParameterStatuses(cancellationToken); + var refreshedDict = refreshedStatuses.ToDictionary(s => s.Uri.OriginalString, s => s); + + // Update our statuses with fresh LastUpdated values + foreach (var status in currentStatuses) + { + if (refreshedDict.TryGetValue(status.Uri.OriginalString, out var refreshed)) + { + status.LastUpdated = refreshed.LastUpdated; + } + } + + // Wait before retry to reduce contention + await Task.Delay(TimeSpan.FromMilliseconds(100.0 * retryCount), cancellationToken); + } + catch (SqlException sqlEx) when (sqlEx.Number == 50001) + { + // Max retries exceeded + throw new SearchParameterConcurrencyException("Maximum retry attempts exceeded due to concurrency conflicts", sqlEx); + } } - catch (SqlException ex) when (ex.IsSearchParameterConcurrencyConflict()) + } + + private async Task UpsertStatusesInternal(IReadOnlyCollection statuses, CancellationToken cancellationToken, long? reindexId = null) + { + using var cmd = new SqlCommand(); + cmd.CommandType = CommandType.StoredProcedure; + cmd.CommandText = "dbo.MergeSearchParams"; + if (_schemaInformation.Current >= 112 && reindexId.HasValue) // remove value check to invoke new max(LastUpdated) logic { - _logger.LogWarning(ex, $"Optimistic concurrency conflict occurred while calling dbo.MergeSearchParams. ReindexId={reindexId ?? 0}"); - throw new BadRequestException(Core.Resources.SearchParameterConcurrencyConflict); + cmd.Parameters.AddWithValue("@ReindexId", reindexId ?? 0); } + + new SearchParamListTableValuedParameterDefinition("@SearchParams").AddParameter(cmd.Parameters, new SearchParamListRowGenerator().GenerateRows(statuses.ToList())); + + await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, cancellationToken); } // Synchronize the FHIR model dictionary with the data in SQL search parameter status table diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs index 5c0c25f0fa..f16fe88d87 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs @@ -212,11 +212,6 @@ public async Task MergeAsync(IReadOnlyList MergeInternalAsync(IReadOnlyList r.PendingSearchParameterStatuses?.Count > 0) + .SelectMany(r => r.PendingSearchParameterStatuses) + .ToList(); + if (mergeWrappersWithVersions.Count > 0) // Do not call DB with empty input { - var pendingStatuses = resources.Where(_ => _.PendingSearchParameterStatus != null).Select(_ => _.PendingSearchParameterStatus).ToList(); - await using (new Timer(async _ => await _sqlStoreClient.MergeResourcesPutTransactionHeartbeatAsync(transactionId, MergeResourcesTransactionHeartbeatPeriod, cancellationToken), null, TimeSpan.FromSeconds(RandomNumberGenerator.GetInt32(100) / 100.0 * MergeResourcesTransactionHeartbeatPeriod.TotalSeconds), MergeResourcesTransactionHeartbeatPeriod)) { var retries = 0; @@ -453,7 +451,7 @@ private async Task MergeInternalAsync(IReadOnlyList _.Wrapper).ToList(), enlistInTransaction, timeoutRetries, pendingStatuses, cancellationToken); + await MergeResourcesWrapperAsync(transactionId, singleTransaction, mergeWrappersWithVersions.Select(_ => _.Wrapper).ToList(), enlistInTransaction, timeoutRetries, allSearchParameterStatuses, cancellationToken); break; } catch (Exception e) @@ -807,12 +805,12 @@ internal async Task MergeResourcesWrapperAsync(long transactionId, bool singleTr using var cmd = new SqlCommand(); //// Do not use auto generated tvp generator as it does not allow to skip compartment tvp and paramters with default values cmd.CommandType = CommandType.StoredProcedure; + bool hasPendingStatuses = pendingStatuses?.Count > 0; - if (pendingStatuses?.Count > 0) + if (hasPendingStatuses && _schemaInformation.Current >= 109) { cmd.CommandText = "dbo.MergeResourcesAndSearchParams"; - new SearchParamListTableValuedParameterDefinition("@SearchParams").AddParameter(cmd.Parameters, new SearchParamListRowGenerator().GenerateRows(pendingStatuses)); - cmd.Parameters.AddWithValue("@ReindexId", 0); + new SearchParamListTableValuedParameterDefinition("@SearchParams").AddParameter(cmd.Parameters, new SearchParamListRowGenerator().GenerateRows(pendingStatuses.ToList())); } else { @@ -854,16 +852,43 @@ internal async Task MergeResourcesWrapperAsync(long transactionId, bool singleTr await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, cancellationToken, disableRetries: true, applicationName: MergeApplicationName); } - _logger.LogInformation($"MergeResourcesWrapperAsync: resources={mergeWrappers.Count}, searchParams={pendingStatuses?.Count ?? 0} transactionId={transactionId}, singleTransaction={singleTransaction}, enlistInTran={enlistInTransaction}, commandTimeout={commandTimeout}, elapsed={sw.Elapsed.TotalMilliseconds} ms."); + _logger.LogInformation($"MergeResourcesWrapperAsync: resources={mergeWrappers.Count}, searchParams={(hasPendingStatuses ? pendingStatuses.Count : 0)} transactionId={transactionId}, singleTransaction={singleTransaction}, enlistInTran={enlistInTransaction}, commandTimeout={commandTimeout}, elapsed={sw.Elapsed.TotalMilliseconds} ms."); } - private void SetAndClearPendingSearchParameterStatus(ResourceWrapperOperation resource) + private bool TryGetPendingSearchParameterStatusUpdates(out List pendingStatuses) { - if (_requestContextAccessor?.RequestContext?.Properties?.TryGetValue(SearchParameterRequestContextPropertyNames.PendingStatus, out object value) == true) + pendingStatuses = null; + + var context = _requestContextAccessor?.RequestContext; + if (context?.Properties == null) + { + return false; + } + + if (!context.Properties.TryGetValue(SearchParameterRequestContextPropertyNames.PendingStatusUpdates, out object value) || + value is not List statuses || + statuses.Count == 0) + { + return false; + } + + lock (statuses) { - resource.PendingSearchParameterStatus = (ResourceSearchParameterStatus)value; - _requestContextAccessor.RequestContext.Properties.Remove(SearchParameterRequestContextPropertyNames.PendingStatus); + if (statuses.Count == 0) + { + return false; + } + + pendingStatuses = statuses.ToList(); } + + return true; + } + + private void ClearPendingSearchParameterStatusUpdates() + { + var context = _requestContextAccessor?.RequestContext; + context?.Properties?.Remove(SearchParameterRequestContextPropertyNames.PendingStatusUpdates); } public async Task UpsertAsync(ResourceWrapperOperation resource, CancellationToken cancellationToken) @@ -879,7 +904,12 @@ public async Task UpsertAsync(ResourceWrapperOperation resource, if (isBundleParallelOperation) { IBundleOrchestratorOperation bundleOperation = _bundleOrchestrator.GetOperation(resource.BundleResourceContext.BundleOperationId); - SetAndClearPendingSearchParameterStatus(resource); + TryGetPendingSearchParameterStatusUpdates(out var pendingStatuses); + if (pendingStatuses?.Count > 0) + { + resource.PendingSearchParameterStatuses = pendingStatuses; + ClearPendingSearchParameterStatusUpdates(); + } return await bundleOperation.AppendResourceAsync(resource, this, cancellationToken).ConfigureAwait(false); } @@ -891,7 +921,12 @@ public async Task UpsertAsync(ResourceWrapperOperation resource, // statuses ride along with the resource through dbo.MergeResourcesAndSearchParams. if (!isBundleTransaction) { - SetAndClearPendingSearchParameterStatus(resource); + TryGetPendingSearchParameterStatusUpdates(out var pendingStatuses); + if (pendingStatuses?.Count > 0) + { + resource.PendingSearchParameterStatuses = pendingStatuses; + ClearPendingSearchParameterStatusUpdates(); + } } // For regular upserts and sequential bundle operations, enlistTransaction is set to true. diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/SearchParamListRowGenerator.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/SearchParamListRowGenerator.cs index b28edeac2a..2d8f546d04 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/SearchParamListRowGenerator.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/TvpRowGeneration/SearchParamListRowGenerator.cs @@ -11,9 +11,9 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Storage.TvpRowGeneration { - internal class SearchParamListRowGenerator : ITableValuedParameterRowGenerator, SearchParamListRow> + internal class SearchParamListRowGenerator : ITableValuedParameterRowGenerator, SearchParamListRow> { - public IEnumerable GenerateRows(IReadOnlyList searchParameterStatuses) + public IEnumerable GenerateRows(List searchParameterStatuses) { return searchParameterStatuses.Select(searchParameterStatus => new SearchParamListRow( searchParameterStatus.Uri.OriginalString, diff --git a/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj b/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj index debbb9cfd7..19263e861d 100644 --- a/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj +++ b/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj @@ -14,7 +14,6 @@ - @@ -168,7 +167,6 @@ - diff --git a/src/Microsoft.Health.Fhir.Tests.Common/TestFiles/Normative/Bundle-InvalidBundleType.json b/src/Microsoft.Health.Fhir.Tests.Common/TestFiles/Normative/Bundle-InvalidBundleType.json deleted file mode 100644 index d3110f156f..0000000000 --- a/src/Microsoft.Health.Fhir.Tests.Common/TestFiles/Normative/Bundle-InvalidBundleType.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "resourceType": "Bundle", - "type": "invalid-type", - "entry": [ - { - "resource": { - "resourceType": "Substance", - "id": "c8972ea6-aff8-421f-9be1-6baf2b49ffed", - "text": { - "status": "generated", - "div": "
Test Substance
" - }, - "code": { - "coding": [ - { - "system": "http://snomed.info/sct", - "code": "test-substance", - "display": "Test Substance" - } - ] - } - }, - "request": { - "method": "POST", - "url": "Substance" - } - } - ] -} diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/BundleEdgeCaseTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/BundleEdgeCaseTests.cs index 2c2d96f3ec..17a0b2a0ae 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/BundleEdgeCaseTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/BundleEdgeCaseTests.cs @@ -35,16 +35,6 @@ public BundleEdgeCaseTests(HttpIntegrationTestFixture fixture) _client = fixture.TestFhirClient; } - [Fact] - public async Task GivenABundle_WhenAnInvalidBundleTypeIsUsed_ThenHttp400IsReturned() - { - var bundleAsString = Samples.GetJson("Bundle-InvalidBundleType"); - - using var fhirException = await Assert.ThrowsAsync(async () => await _client.PostAsync(string.Empty, bundleAsString)); - Assert.Equal(HttpStatusCode.BadRequest, fhirException.StatusCode); - Assert.True(fhirException.Message.Contains("The provided Bundle type is not supported.", StringComparison.OrdinalIgnoreCase)); - } - [Fact] [Trait(Traits.Priority, Priority.One)] public async Task GivenABundleWithConditionalUpdateByReference_WhenExecutedWithMaximizedConditionalQueryParallelism_RunsTheQueryInParallel() diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/ExpandOperationTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/ExpandOperationTests.cs index 42cf7ba86e..5fe210cc4c 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/ExpandOperationTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Conformance/ExpandOperationTests.cs @@ -358,43 +358,5 @@ private void Validate( && !string.IsNullOrEmpty(x.Diagnostics) && invalid.Any(y => x.Diagnostics.Contains(y))); } - - [SkippableFact] - public async Task GivenUnknownValueSet_WhenExpanding_ThenReturns404WithOperationOutcome() - { - var expandEnabled = Server.Metadata.SupportsOperation(OperationsConstants.ValueSetExpand); - Skip.IfNot(expandEnabled, "The $expand operation is disabled"); - - var unknownUrl = "http://example.org/fhir/ValueSet/nonexistent-" + Guid.NewGuid().ToString("N"); - var url = $"{KnownResourceTypes.ValueSet}/{KnownRoutes.Expand}?url={unknownUrl}"; - - var ex = await Assert.ThrowsAsync(() => Client.ReadAsync(url)); - Assert.Equal(HttpStatusCode.NotFound, ex.StatusCode); - Assert.NotNull(ex.OperationOutcome); - Assert.NotEmpty(ex.OperationOutcome.Issue); - Assert.Contains( - ex.OperationOutcome.Issue, - x => x.Severity == OperationOutcome.IssueSeverity.Error - && x.Code == OperationOutcome.IssueType.NotFound); - } - - [SkippableFact] - public async Task GivenUnknownValueSetId_WhenExpandingById_ThenReturns404WithOperationOutcome() - { - var expandEnabled = Server.Metadata.SupportsOperation(OperationsConstants.ValueSetExpand); - Skip.IfNot(expandEnabled, "The $expand operation is disabled"); - - var invalidId = Guid.NewGuid().ToString("N"); - var url = $"{KnownResourceTypes.ValueSet}/{invalidId}/{KnownRoutes.Expand}"; - - var ex = await Assert.ThrowsAsync(() => Client.ReadAsync(url)); - Assert.Equal(HttpStatusCode.NotFound, ex.StatusCode); - Assert.NotNull(ex.OperationOutcome); - Assert.NotEmpty(ex.OperationOutcome.Issue); - Assert.Contains( - ex.OperationOutcome.Issue, - x => x.Severity == OperationOutcome.IssueSeverity.Error - && x.Code == OperationOutcome.IssueType.NotFound); - } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs index 4afa4b0d80..a6eaa5435e 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Reindex/ReindexTests.cs @@ -13,12 +13,9 @@ using System.Threading; using System.Threading.Tasks; using Hl7.Fhir.Model; -using Microsoft.AspNetCore.Server.HttpSys; -using Microsoft.CodeAnalysis; using Microsoft.Health.Extensions.Xunit; using Microsoft.Health.Fhir.Client; using Microsoft.Health.Fhir.Core.Features.Operations; -using Microsoft.Health.Fhir.Core.Features.Search.Parameters; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; using Microsoft.Health.Test.Utilities; @@ -27,7 +24,6 @@ using Xunit; using Xunit.Abstractions; using static Hl7.Fhir.Model.Bundle; -using static Microsoft.EntityFrameworkCore.DbLoggerCategory; using Task = System.Threading.Tasks.Task; namespace Microsoft.Health.Fhir.Tests.E2E.Rest.Reindex @@ -73,185 +69,10 @@ public Task DisposeAsync() return Task.CompletedTask; } - [Fact] - public async Task GivenSequentialBundleSearchParamCreates_ShouldSuceed() - { - const int numberOfSearchParams = 10; - const string urlPrefix = "http://my.org/"; - var codes = new List(); - try - { - for (var i = 0; i < numberOfSearchParams; i++) - { - var code = $"c-id-{i}"; - codes.Add(code); - } - - var bundle = await CreatePersonSearchParamsAsync(); - Assert.Equal(numberOfSearchParams, bundle.Entry.Count); - foreach (var entry in bundle.Entry) - { - Assert.True(entry.Resource as SearchParameter != null, $"actual={JsonConvert.SerializeObject(entry)}"); - } - } - finally - { - await DeleteSearchParamsAsync(codes); - } - - async Task CreatePersonSearchParamsAsync() - { - var bundle = new Bundle { Type = Bundle.BundleType.Batch, Entry = new List() }; - -#if R5 - var resourceTypes = new List([Enum.Parse("Person")]); -#else - var resourceTypes = new List([Enum.Parse("Person")]); -#endif - - foreach (var code in codes) - { - var searchParam = new SearchParameter - { - Id = code, - Url = $"{urlPrefix}{code}", - Name = code, - Code = code, - Status = PublicationStatus.Active, - Type = SearchParamType.Token, - Expression = "Person.id", - Description = "any", - Base = resourceTypes, - }; - - bundle.Entry.Add(new EntryComponent { Request = new RequestComponent { Method = Bundle.HTTPVerb.PUT, Url = $"SearchParameter/{code}" }, Resource = searchParam }); - } - - var result = await _fixture.TestFhirClient.PostBundleAsync(bundle, new FhirBundleOptions { BundleProcessingLogic = FhirBundleProcessingLogic.Sequential }); - return result; - } - } - - [Theory] - [InlineData(false, false)] // single creats - [InlineData(true, false)] // batch bundle - [InlineData(true, true)] // parallel batch bundle - public async Task GivenConcurrentSearchParamCreates_SomeShouldFail(bool isBundle, bool isParallel) - { - const string urlPrefix = "http://my.org/"; - var codes = new List(); - try - { - var threw = false; - await Parallel.ForAsync(0, 20, new ParallelOptions { MaxDegreeOfParallelism = 4 }, async (i, ct) => - { - var code = $"c-id-{i}"; - lock (codes) - { - codes.Add(code); - } - - try - { - var result = await CreatePersonSearchParamAsync(code, isBundle, isParallel); - if (isBundle && result is FhirResponse bundleResponse) - { - Assert.Single(bundleResponse.Resource.Entry); - - var entry = bundleResponse.Resource.Entry[0]; - if (entry.Response != null && entry.Response.Status.StartsWith("4")) - { - if (entry.Response.Outcome is OperationOutcome outcome) - { - var diagnostics = outcome.Issue?.FirstOrDefault()?.Diagnostics; - _output.WriteLine($"Param={code}. Diagnostics={diagnostics}"); - var expected = $"{Core.Resources.SearchParameterConcurrencyConflict}{(isParallel ? string.Empty : " Update.3")}"; - Assert.True(diagnostics == expected, $"Expected={expected} Actual={diagnostics}"); - threw = true; - - lock (codes) - { - codes.Remove(code); - } - - return; - } - } - } - - _output.WriteLine($"Created search param = {code}"); - } - catch (FhirClientException ex) - { - _output.WriteLine($"Param={code}. StatusCode={ex.StatusCode}, Error={ex.Message}"); - - if (ex.StatusCode != HttpStatusCode.InternalServerError) // this can happen because of short wait limit to accquire "lock" in "get and apply" code. testing only. - { - Assert.Equal(HttpStatusCode.BadRequest, ex.StatusCode); - var expected = $"BadRequest: {Core.Resources.SearchParameterConcurrencyConflict} {(isBundle ? "Update.3" : "Create.3")}"; - Assert.True(ex.Message.StartsWith(expected), $"Expected={expected} Actual={ex.Message}"); - threw = true; - } - - lock (codes) - { - codes.Remove(code); - } - } - }); - - Assert.True(threw || !_isSql, "Expected at least one create to fail due to concurrency for SQL only."); - } - finally - { - await DeleteSearchParamsAsync(codes); - } - - async Task CreatePersonSearchParamAsync(string code, bool isBundle, bool isParal) - { - var bundle = new Bundle { Type = Bundle.BundleType.Batch, Entry = new List() }; -#if R5 - var resourceTypes = new List([Enum.Parse("Person")]); -#else - var resourceTypes = new List([Enum.Parse("Person")]); -#endif - - var searchParam = new SearchParameter - { - Id = code, - Url = $"{urlPrefix}{code}", - Name = code, - Code = code, - Status = PublicationStatus.Active, - Type = SearchParamType.Token, - Expression = "Person.id", - Description = "any", - Base = resourceTypes, - }; - - if (isBundle) - { - bundle.Entry.Add(new EntryComponent { Request = new RequestComponent { Method = Bundle.HTTPVerb.PUT, Url = $"SearchParameter/{code}" }, Resource = searchParam }); - var result = await _fixture.TestFhirClient.PostBundleAsync(bundle, new FhirBundleOptions { BundleProcessingLogic = isParal ? FhirBundleProcessingLogic.Parallel : FhirBundleProcessingLogic.Sequential }); - return result; - } - else - { - var result = await _fixture.TestFhirClient.CreateAsync(searchParam); - return result; - } - } - } - [Fact] public async Task Given500SearchParams_WhenReindexCompletes_ThenSearchParamsAreEnabled() { - if (!_isSql) // max(lastUpdated) works only for SQL. For Cosmos - NOOP. - { - return; - } - - const int numberOfSearchParams = 500; // increase to 500 when cache is not updated by API calls and status is saved with resources in a single SQL transaction + const int numberOfSearchParams = 10; // increase to 500 when cache is not updated by API calls and status is saved with resources in a single SQL transaction const string urlPrefix = "http://my.org/"; var codes = new List(); try @@ -270,17 +91,8 @@ public async Task Given500SearchParams_WhenReindexCompletes_ThenSearchParamsAreE } // check by urls - // code works locally for all 500, but in PR it throws - FhirClientException : RequestUriTooLong (NO_FHIR_ACTIVITY_ID_FOR_THIS_TRANSACTION) - var total = 0; - var chunk = numberOfSearchParams / 10; // assumes there is no remainder. - for (var i = 0; i < 10; i++) - { - var urls = string.Join(",", codes.Skip(i * chunk).Take(chunk).Select(_ => $"{urlPrefix}{_}")); - var search = await _fixture.TestFhirClient.SearchAsync($"SearchParameter?_summary=count&url={urls}"); - total += search.Resource.Total.Value; - } - - Assert.True(total == numberOfSearchParams, $"Urls: expected={numberOfSearchParams} actual={total}"); + var search = await _fixture.TestFhirClient.SearchAsync($"SearchParameter?_summary=count&url={string.Join(",", codes.Select(_ => $"{urlPrefix}{_}"))}"); + Assert.True(search.Resource.Total == numberOfSearchParams, $"Urls expected={numberOfSearchParams} actual={search.Resource.Total}"); var reindex = await _fixture.TestFhirClient.PostReindexJobAsync(new Parameters { Parameter = [] }); Assert.Equal(HttpStatusCode.Created, reindex.reponse.Response.StatusCode); @@ -302,9 +114,11 @@ async Task CreatePersonSearchParamsAsync() var bundle = new Bundle { Type = Bundle.BundleType.Batch, Entry = new List() }; #if R5 - var resourceTypes = new List([Enum.Parse("Person")]); + var resourceTypes = new List(); + resourceTypes.Add(Enum.Parse("Person")); #else - var resourceTypes = new List([Enum.Parse("Person")]); + var resourceTypes = new List(); + resourceTypes.Add(Enum.Parse("Person")); #endif foreach (var code in codes) @@ -401,64 +215,6 @@ public async Task GivenTwoSearchParamsWithCodeConflictOnDerived_ThenBadRequestIs } } - [Fact] - public async Task GivenTwoSearchParamsInSequentialBatchBundle_ThenBothCreated() - { -#if R5 - var personTypes = new List() { VersionIndependentResourceTypesAll.Person }; - var supplyDeliveryTypes = new List() { VersionIndependentResourceTypesAll.SupplyDelivery }; -#else - var personTypes = new List() { ResourceType.Person }; - var supplyDeliveryTypes = new List() { ResourceType.SupplyDelivery }; -#endif - const string urlPrefix = "http://my.org/"; - var ids = new List { "c-id-1", "c-id-2" }; - try - { - var bundle = new Bundle { Type = Bundle.BundleType.Batch, Entry = [] }; - - var id = ids[0]; - var searchParam = new SearchParameter - { - Id = id, - Url = $"{urlPrefix}c-1", - Name = id, - Code = id, - Status = PublicationStatus.Active, - Type = SearchParamType.Token, - Expression = "Person.id", - Description = "any", - Base = personTypes, - }; - - bundle.Entry.Add(new EntryComponent { Request = new RequestComponent { Method = Bundle.HTTPVerb.PUT, Url = $"SearchParameter/{id}" }, Resource = searchParam }); - - id = ids[1]; - searchParam = new SearchParameter - { - Id = id, - Url = $"{urlPrefix}c-2", - Name = id, - Code = id, - Status = PublicationStatus.Active, - Type = SearchParamType.Token, - Expression = "SupplyDelivery.id", - Description = "any", - Base = supplyDeliveryTypes, - }; - - bundle.Entry.Add(new EntryComponent { Request = new RequestComponent { Method = Bundle.HTTPVerb.PUT, Url = $"SearchParameter/{id}" }, Resource = searchParam }); - - var response = await _fixture.TestFhirClient.PostBundleAsync(bundle, new FhirBundleOptions { BundleProcessingLogic = FhirBundleProcessingLogic.Sequential }); - Assert.Equal(2, response.Resource.Entry.Count); - Assert.All(response.Resource.Entry, _ => Assert.NotNull(_.Resource as SearchParameter)); - } - finally - { - await DeleteSearchParamsAsync(ids); - } - } - [Fact] public async Task GivenTwoSearchParamsForDifferentResourceTypesUsingSameCode_ThenBothCreated() { @@ -526,9 +282,8 @@ public async Task GivenTwoSearchParamsForDifferentResourceTypesUsingSameCode_The [InlineData(true, false, true, true)] [InlineData(true, false, false, true)] [InlineData(false, true, true, true)] - //// https://microsofthealth.visualstudio.com/Health/_workitems/edit/187119 - ////[InlineData(false, true, false, true)] // this creates 2 resources for the same url. after fixing this bug - uncomment. - ////[InlineData(true, true, false, true)] // this creates 2 resources for the same url. after fixing this bug - uncomment. + [InlineData(false, true, false, true)] + [InlineData(true, true, false, true)] [InlineData(true, true, false, false)] [InlineData(true, true, true, true)] [InlineData(true, true, true, false)] @@ -633,13 +388,12 @@ async Task CreatePersonSearchParamsAsync() private async Task DeleteSearchParamsAsync(List ids) { - var bundle = new Bundle { Type = Bundle.BundleType.Batch, Entry = new List() }; foreach (var id in ids) { + var bundle = new Bundle { Type = Bundle.BundleType.Batch, Entry = new List() }; bundle.Entry.Add(new EntryComponent { Request = new RequestComponent { Method = Bundle.HTTPVerb.DELETE, Url = $"SearchParameter/{id}" } }); + await _fixture.TestFhirClient.PostBundleAsync(bundle, new FhirBundleOptions { BundleProcessingLogic = FhirBundleProcessingLogic.Parallel }); } - - await _fixture.TestFhirClient.PostBundleAsync(bundle, new FhirBundleOptions { BundleProcessingLogic = FhirBundleProcessingLogic.Parallel }); } [Fact] diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs index ef89339f68..dd16fe0bf3 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs @@ -117,6 +117,7 @@ public async Task GivenAChainedSearchExpressionOverBirthdateDayWithAdditionalCri ValidateBundle(bundle, Fixture.SmithSnomedDiagnosticReport, Fixture.SmithLoincDiagnosticReport); } + [Fact] public async Task GivenAChainedSearchExpressionOverASimpleParameter_WhenSearchedWithPaging_ThenCorrectBundleShouldBeReturned() { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/FailingSearchParameterStatusManager.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/FailingSearchParameterStatusManager.cs index 819f5fad86..99143e9d1d 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/FailingSearchParameterStatusManager.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/FailingSearchParameterStatusManager.cs @@ -31,7 +31,7 @@ public Task Handle(SearchParameterDefinitionManagerInitialized notification, Can public Task> GetAllSearchParameterStatus(CancellationToken cancellationToken) => Task.FromResult>(Array.Empty()); - public Task UpdateSearchParameterStatusAsync(IReadOnlyCollection searchParameterUris, SearchParameterStatus status, CancellationToken cancellationToken, bool ignoreSearchParameterNotSupportedException = false, long? reindexId = null, DateTimeOffset? lastUpdated = null) => Task.CompletedTask; + public Task UpdateSearchParameterStatusAsync(IReadOnlyCollection searchParameterUris, SearchParameterStatus status, CancellationToken cancellationToken, bool ignoreSearchParameterNotSupportedException = false, long? reindexId = null) => Task.CompletedTask; public Task CheckCacheConsistencyAsync(DateTime updateEventsSince, DateTime activeHostsSince, CancellationToken cancellationToken) => Task.FromResult(new CacheConsistencyResult()); } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs index 316b314dfa..bfe360f156 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs @@ -328,7 +328,7 @@ private void StartCacheUpdateTask(CancellationToken cancellationToken) { try { - await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken, true); + await _searchParameterOperations.GetAndApplySearchParameterUpdates(cancellationToken); } catch (Exception) { @@ -542,9 +542,7 @@ public async Task GivenNoMatchingResources_WhenRunningReindexJob_ThenJobIsComple #else var searchParam = _supportedSearchParameterDefinitionManager.GetSearchParameter("http://hl7.org/fhir/SearchParameter/CanonicalResource-name"); #endif - await _searchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); - - await _searchParameterStatusManager.UpdateSearchParameterStatusAsync(new List() { searchParam.Url.ToString() }, SearchParameterStatus.Supported, default, lastUpdated: _searchParameterOperations.SearchParamLastUpdated); + await _searchParameterStatusManager.UpdateSearchParameterStatusAsync(new List() { searchParam.Url.ToString() }, SearchParameterStatus.Supported, default); var request = new CreateReindexRequest(new List(), new List()); CreateReindexResponse response = await SetUpForReindexing(request); @@ -1242,8 +1240,6 @@ public async Task GivenSurrogateRangeFetchOom_WhenProcessingJobRuns_ThenSplitUse _searchParameterStatusManager, NullLogger.Instance); - await _searchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); - string resultJson = await processingJob.ExecuteAsync(jobInfo, CancellationToken.None); var result = JsonConvert.DeserializeObject(resultJson); @@ -1376,11 +1372,42 @@ private async Task CreateSearchParam(string searchParamName, Se await _fixture.Mediator.UpsertResourceAsync(searchParam.ToResourceElement()); + if (!_searchParameterDefinitionManager.TryGetSearchParameter(searchParam.Url, out _)) + { + _searchParameterDefinitionManager.AddNewSearchParameters(new List { searchParam.ToTypedElement() }); + } + + await _searchParameterStatusManager.UpdateSearchParameterStatusAsync( + new List { searchParam.Url }, + SearchParameterStatus.Supported, + CancellationToken.None); + + // These tests start a background cache refresh loop, so a zero-wait refresh can + // skip the one apply that populates the SQL search-parameter URI->ID mapping. await _searchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); return searchParam; } + private async Task SearchForSearchParameterByUrlAsync(string searchParamUrl, CancellationToken cancellationToken, int maxAttempts = 10, int delayMilliseconds = 200) + { + SearchResult result = null; + var queryParams = new List> { new("url", searchParamUrl) }; + + for (int attempt = 0; attempt < maxAttempts; attempt++) + { + result = await _searchService.Value.SearchAsync(KnownResourceTypes.SearchParameter, queryParams, cancellationToken); + if (result.Results.Any()) + { + return result; + } + + await Task.Delay(delayMilliseconds, cancellationToken); + } + + return result; + } + private ResourceWrapper CreatePatientResourceWrapper(string patientName, string patientId) { Patient patientResource = Samples.GetDefaultPatient().ToPoco(); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchSharedFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchSharedFixture.cs deleted file mode 100644 index 47e49d3b2e..0000000000 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchSharedFixture.cs +++ /dev/null @@ -1,140 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Hl7.Fhir.Model; -using Hl7.Fhir.Serialization; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Health.Extensions.DependencyInjection; -using Microsoft.Health.Fhir.Core.Extensions; -using Microsoft.Health.Fhir.Core.Features.Context; -using Microsoft.Health.Fhir.Core.Features.Definition; -using Microsoft.Health.Fhir.Core.Features.Persistence; -using Microsoft.Health.Fhir.Core.Features.Search; -using Microsoft.Health.Fhir.Core.Features.Search.Converters; -using Microsoft.Health.Fhir.Core.Features.Search.Registry; -using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; -using Microsoft.Health.Fhir.Core.Models; -using Microsoft.Health.Fhir.Core.UnitTests.Extensions; -using Microsoft.Health.Fhir.Tests.Common; -using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; -using Microsoft.Health.Fhir.Tests.Integration.Persistence; -using Microsoft.Health.Test.Utilities; -using NSubstitute; -using Xunit; -using Task = System.Threading.Tasks.Task; - -namespace Microsoft.Health.Fhir.Tests.Integration.Features.Smart -{ - public sealed class SmartSearchSharedFixture : IAsyncLifetime - { - private readonly DataStore _dataStore; - private FhirStorageTestsFixture _fixture; - private IScoped _scopedDataStore; - private SearchParameterDefinitionManager _searchParameterDefinitionManager; - private ISearchIndexer _searchIndexer; - - public SmartSearchSharedFixture(DataStore dataStore) - { - _dataStore = dataStore; - } - - public FhirStorageTestsFixture Fixture => _fixture; - - public async Task InitializeAsync() - { - if (ModelInfoProvider.Instance.Version != FhirSpecification.R4 && - ModelInfoProvider.Instance.Version != FhirSpecification.R4B) - { - return; - } - - _fixture = new FhirStorageTestsFixture(_dataStore); - await _fixture.InitializeAsync(); - - var typedElementToSearchValueConverterManager = await CreateFhirTypedElementToSearchValueConverterManagerAsync(); - _searchIndexer = new TypedElementSearchIndexer( - _fixture.SupportedSearchParameterDefinitionManager, - typedElementToSearchValueConverterManager, - Substitute.For(), - ModelInfoProvider.Instance, - NullLogger.Instance); - _searchParameterDefinitionManager = _fixture.SearchParameterDefinitionManager; - _scopedDataStore = _fixture.DataStore.CreateMockScope(); - - await LoadBundleAsync("SmartPatientA"); - await LoadBundleAsync("SmartPatientB"); - await LoadBundleAsync("SmartPatientC"); - await LoadBundleAsync("SmartPatientD"); - await LoadBundleAsync("SmartCommon"); - - await UpsertResource(Samples.GetJsonSample("Medication")); - await UpsertResource(Samples.GetJsonSample("Organization")); - await UpsertResource(Samples.GetJsonSample("Location-example-hq")); - } - - public async Task DisposeAsync() - { - if (_fixture != null) - { - await _fixture.DisposeAsync(); - } - } - - public async Task UpsertResource(Resource resource, string httpMethod = "PUT") - { - resource.Meta ??= new Meta(); - resource.Meta.LastUpdated = DateTimeOffset.UtcNow; - - ResourceElement resourceElement = resource.ToResourceElement(); - - var rawResource = new RawResource(resource.ToJson(), FhirResourceFormat.Json, isMetaSet: false); - var resourceRequest = new ResourceRequest(httpMethod); - var compartmentIndices = Substitute.For(); - var searchIndices = _searchIndexer.Extract(resourceElement); - var wrapper = new ResourceWrapper(resourceElement, rawResource, resourceRequest, false, searchIndices, compartmentIndices, new List>(), _searchParameterDefinitionManager.GetSearchParameterHashForResourceType("Patient")); - wrapper.SearchParameterHash = "hash"; - - return await _scopedDataStore.Value.UpsertAsync(new ResourceWrapperOperation(wrapper, true, true, null, false, false, bundleResourceContext: null), CancellationToken.None); - } - - private static async Task CreateFhirTypedElementToSearchValueConverterManagerAsync() - { - var types = typeof(ITypedElementToSearchValueConverter) - .Assembly - .GetTypes() - .Where(x => typeof(ITypedElementToSearchValueConverter).IsAssignableFrom(x) && !x.IsAbstract && !x.IsInterface); - - var referenceSearchValueParser = new ReferenceSearchValueParser(new FhirRequestContextAccessor(), new FhirServerInstanceConfiguration()); - var codeSystemResolver = new CodeSystemResolver(ModelInfoProvider.Instance); - await codeSystemResolver.StartAsync(CancellationToken.None); - - var fhirElementToSearchValueConverters = new List(); - - foreach (Type type in types.Where(type => type.Name != nameof(FhirTypedElementToSearchValueConverterManager.ExtensionConverter))) - { - // Filter out the extension converter because it will be added to the converter dictionary in the converter manager's constructor - var x = (ITypedElementToSearchValueConverter)Mock.TypeWithArguments(type, referenceSearchValueParser, codeSystemResolver); - fhirElementToSearchValueConverters.Add(x); - } - - return new FhirTypedElementToSearchValueConverterManager(fhirElementToSearchValueConverters); - } - - private async Task LoadBundleAsync(string sampleName) - { - var smartBundle = Samples.GetJsonSample(sampleName); - - foreach (var entry in smartBundle.Entry) - { - await UpsertResource(entry.Resource); - } - } - } -} diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchTests.cs index b7b0fc03ce..fe7e4eff8d 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Smart/SmartSearchTests.cs @@ -39,6 +39,7 @@ using Microsoft.Health.Fhir.Tests.Integration.Persistence; using Microsoft.Health.Test.Utilities; using NSubstitute; +using NSubstitute.Core; using Xunit; using Task = System.Threading.Tasks.Task; @@ -48,28 +49,116 @@ namespace Microsoft.Health.Fhir.Tests.Integration.Features.Smart [Trait(Traits.Category, Categories.SmartOnFhir)] [FhirStorageTestsFixtureArgumentSets(DataStore.All)] - public class SmartSearchTests : IClassFixture + public class SmartSearchTests : IClassFixture, IAsyncLifetime { - private readonly SmartSearchSharedFixture _smartFixture; private readonly FhirStorageTestsFixture _fixture; + private readonly IFhirStorageTestHelper _testHelper; + private IFhirOperationDataStore _fhirOperationDataStore; + private IScoped _scopedDataStore; + private IFhirStorageTestHelper _fhirStorageTestHelper; + private SearchParameterDefinitionManager _searchParameterDefinitionManager; + private ITypedElementToSearchValueConverterManager _typedElementToSearchValueConverterManager; + private ISearchIndexer _searchIndexer; + private readonly ISearchParameterSupportResolver _searchParameterSupportResolver = Substitute.For(); + private ISupportedSearchParameterDefinitionManager _supportedSearchParameterDefinitionManager; + private SearchParameterStatusManager _searchParameterStatusManager; private IScoped _searchService; private RequestContextAccessor _contextAccessor; + private readonly IDataStoreSearchParameterValidator _dataStoreSearchParameterValidator = Substitute.For(); - public SmartSearchTests(SmartSearchSharedFixture smartFixture) + public SmartSearchTests(FhirStorageTestsFixture fixture) { - _smartFixture = smartFixture; + _fixture = fixture; + _testHelper = _fixture.TestHelper; + } + public async Task InitializeAsync() + { if (ModelInfoProvider.Instance.Version == FhirSpecification.R4 || ModelInfoProvider.Instance.Version == FhirSpecification.R4B) { - _fixture = _smartFixture.Fixture; + _dataStoreSearchParameterValidator.ValidateSearchParameter(default, out Arg.Any()).ReturnsForAnyArgs(x => + { + x[1] = null; + return true; + }); + + _searchParameterSupportResolver.IsSearchParameterSupported(Arg.Any()).Returns((true, false)); + _contextAccessor = _fixture.FhirRequestContextAccessor; + + _fhirOperationDataStore = _fixture.OperationDataStore; + _fhirStorageTestHelper = _fixture.TestHelper; + _scopedDataStore = _fixture.DataStore.CreateMockScope(); + + _searchParameterDefinitionManager = _fixture.SearchParameterDefinitionManager; + _supportedSearchParameterDefinitionManager = _fixture.SupportedSearchParameterDefinitionManager; + + _typedElementToSearchValueConverterManager = await CreateFhirTypedElementToSearchValueConverterManagerAsync(); + + _searchIndexer = new TypedElementSearchIndexer( + _supportedSearchParameterDefinitionManager, + _typedElementToSearchValueConverterManager, + Substitute.For(), + ModelInfoProvider.Instance, + NullLogger.Instance); + + ResourceWrapperFactory wrapperFactory = Mock.TypeWithArguments( + new RawResourceFactory(new FhirJsonSerializer()), + new FhirRequestContextAccessor(), + _searchIndexer, + _searchParameterDefinitionManager, + Deserializers.ResourceDeserializer); + + _searchParameterStatusManager = _fixture.SearchParameterStatusManager; + _searchService = _fixture.SearchService.CreateMockScope(); + + _contextAccessor = _fixture.FhirRequestContextAccessor; + + var smartBundle = Samples.GetJsonSample("SmartPatientA"); + foreach (var entry in smartBundle.Entry) + { + await UpsertResource(entry.Resource); + } + + smartBundle = Samples.GetJsonSample("SmartPatientB"); + foreach (var entry in smartBundle.Entry) + { + await UpsertResource(entry.Resource); + } + + smartBundle = Samples.GetJsonSample("SmartPatientC"); + foreach (var entry in smartBundle.Entry) + { + await UpsertResource(entry.Resource); + } + + smartBundle = Samples.GetJsonSample("SmartPatientD"); + foreach (var entry in smartBundle.Entry) + { + await UpsertResource(entry.Resource); + } + + smartBundle = Samples.GetJsonSample("SmartCommon"); + foreach (var entry in smartBundle.Entry) + { + await UpsertResource(entry.Resource); + } + + await UpsertResource(Samples.GetJsonSample("Medication")); + await UpsertResource(Samples.GetJsonSample("Organization")); + await UpsertResource(Samples.GetJsonSample("Location-example-hq")); } } + public Task DisposeAsync() + { + return Task.CompletedTask; + } + [SkippableFact] public async Task GivenScopesWithReadForAllResources_WhenRevIncludeObservations_PatientAndObservationReturned() { @@ -1030,9 +1119,47 @@ public async Task GivenReadScopeOnOnlyEncountersInACompartment_OnRevincludeWithW Assert.DoesNotContain(results.Results, r => r.Resource.ResourceTypeName == KnownResourceTypes.Observation); } - private static string CreateSmartV2TestResourceId(string scenario) + private async Task UpsertResource(Resource resource, string httpMethod = "PUT") { - return $"smart-v2-{scenario}-{Guid.NewGuid():N}"; + resource.Meta ??= new Meta(); + resource.Meta.LastUpdated = DateTimeOffset.UtcNow; + + ResourceElement resourceElement = resource.ToResourceElement(); + + var rawResource = new RawResource(resource.ToJson(), FhirResourceFormat.Json, isMetaSet: false); + var resourceRequest = new ResourceRequest(httpMethod); + var compartmentIndices = Substitute.For(); + var searchIndices = _searchIndexer.Extract(resourceElement); + var wrapper = new ResourceWrapper(resourceElement, rawResource, resourceRequest, false, searchIndices, compartmentIndices, new List>(), _searchParameterDefinitionManager.GetSearchParameterHashForResourceType("Patient")); + wrapper.SearchParameterHash = "hash"; + + return await _scopedDataStore.Value.UpsertAsync(new ResourceWrapperOperation(wrapper, true, true, null, false, false, bundleResourceContext: null), CancellationToken.None); + } + + private static async Task CreateFhirTypedElementToSearchValueConverterManagerAsync() + { + var types = typeof(ITypedElementToSearchValueConverter) + .Assembly + .GetTypes() + .Where(x => typeof(ITypedElementToSearchValueConverter).IsAssignableFrom(x) && !x.IsAbstract && !x.IsInterface); + + var referenceSearchValueParser = new ReferenceSearchValueParser(new FhirRequestContextAccessor(), new FhirServerInstanceConfiguration()); + var codeSystemResolver = new CodeSystemResolver(ModelInfoProvider.Instance); + await codeSystemResolver.StartAsync(CancellationToken.None); + + var fhirElementToSearchValueConverters = new List(); + + foreach (Type type in types) + { + // Filter out the extension converter because it will be added to the converter dictionary in the converter manager's constructor + if (type.Name != nameof(FhirTypedElementToSearchValueConverterManager.ExtensionConverter)) + { + var x = (ITypedElementToSearchValueConverter)Mock.TypeWithArguments(type, referenceSearchValueParser, codeSystemResolver); + fhirElementToSearchValueConverters.Add(x); + } + } + + return new FhirTypedElementToSearchValueConverterManager(fhirElementToSearchValueConverters); } private void ConfigureFhirRequestContext( @@ -1051,10 +1178,7 @@ private void ConfigureFhirRequestContext( accessControlContext.AllowedResourceActions.Add(scope); } - var fhirRequestContext = Substitute.For(); - fhirRequestContext.AccessControlContext.Returns(accessControlContext); - - contextAccessor.RequestContext.Returns(fhirRequestContext); + contextAccessor.RequestContext.AccessControlContext.Returns(accessControlContext); } // SMART v2 Granular Scope Tests @@ -1068,21 +1192,20 @@ public async Task GivenSmartV2CreateScope_WhenCreatingPatient_ThenPatientIsCreat "This test is only valid for R4 and R4B"); var scopeRestriction = new ScopeRestriction("Patient", Core.Features.Security.DataActions.Create, "patient"); - var patientId = CreateSmartV2TestResourceId("create"); ConfigureFhirRequestContext(_contextAccessor, new List() { scopeRestriction }); - _contextAccessor.RequestContext.AccessControlContext.CompartmentId = patientId; + _contextAccessor.RequestContext.AccessControlContext.CompartmentId = "smart-patient-test"; _contextAccessor.RequestContext.AccessControlContext.CompartmentResourceType = "Patient"; var newPatient = new Patient { - Id = patientId, + Id = "smart-v2-create-test", Name = new List { new HumanName().WithGiven("TestCreate").AndFamily("SmartV2") }, }; - var result = await _smartFixture.UpsertResource(newPatient); + var result = await UpsertResource(newPatient); Assert.NotNull(result); - Assert.Equal(patientId, result.Wrapper.ResourceId); + Assert.Equal("smart-v2-create-test", result.Wrapper.ResourceId); } [SkippableFact] @@ -1139,28 +1262,20 @@ public async Task GivenSmartV2UpdateScope_WhenUpdatingPatient_ThenPatientIsUpdat "This test is only valid for R4 and R4B"); var scopeRestriction = new ScopeRestriction("Patient", Core.Features.Security.DataActions.Update, "patient"); - var patientId = CreateSmartV2TestResourceId("update"); - - await _smartFixture.UpsertResource(new Patient - { - Id = patientId, - Name = new List { new HumanName().WithGiven("InitialName").AndFamily("SmartV2") }, - }); ConfigureFhirRequestContext(_contextAccessor, new List() { scopeRestriction }); - _contextAccessor.RequestContext.AccessControlContext.CompartmentId = patientId; + _contextAccessor.RequestContext.AccessControlContext.CompartmentId = "smart-patient-A"; _contextAccessor.RequestContext.AccessControlContext.CompartmentResourceType = "Patient"; // Create an updated patient resource var updatedPatient = new Patient { - Id = patientId, + Id = "smart-patient-A", Name = new List { new HumanName().WithGiven("UpdatedName").AndFamily("Updated") }, }; - var result = await _smartFixture.UpsertResource(updatedPatient); + var result = await UpsertResource(updatedPatient); Assert.NotNull(result); - Assert.Equal(patientId, result.Wrapper.ResourceId); } [SkippableFact] @@ -1173,10 +1288,9 @@ public async Task GivenSmartV2SearchAndCreateScopes_WhenSearchingWithCreate_Then var scopeRestriction1 = new ScopeRestriction("Patient", Core.Features.Security.DataActions.Search, "patient"); var scopeRestriction2 = new ScopeRestriction("Patient", Core.Features.Security.DataActions.Create, "patient"); - var patientId = CreateSmartV2TestResourceId("search-create"); ConfigureFhirRequestContext(_contextAccessor, new List() { scopeRestriction1, scopeRestriction2 }); - _contextAccessor.RequestContext.AccessControlContext.CompartmentId = patientId; + _contextAccessor.RequestContext.AccessControlContext.CompartmentId = "smart-patient-test"; _contextAccessor.RequestContext.AccessControlContext.CompartmentResourceType = "Patient"; // Test search capability @@ -1189,11 +1303,11 @@ public async Task GivenSmartV2SearchAndCreateScopes_WhenSearchingWithCreate_Then // Test create capability var newPatient = new Patient { - Id = patientId, + Id = "smart-v2-search-create-test", Name = new List { new HumanName().WithGiven("SearchCreate").AndFamily("SmartV2") }, }; - var createResult = await _smartFixture.UpsertResource(newPatient); + var createResult = await UpsertResource(newPatient); Assert.NotNull(createResult); } @@ -1207,34 +1321,27 @@ public async Task GivenSmartV2SearchAndUpdateScopes_WhenSearchingWithUpdate_Then var scopeRestriction1 = new ScopeRestriction("Patient", Core.Features.Security.DataActions.Search, "patient"); var scopeRestriction2 = new ScopeRestriction("Patient", Core.Features.Security.DataActions.Update, "patient"); - var patientId = CreateSmartV2TestResourceId("search-update"); - - await _smartFixture.UpsertResource(new Patient - { - Id = patientId, - Name = new List { new HumanName().WithGiven("InitialSearchUpdate").AndFamily("SmartV2") }, - }); ConfigureFhirRequestContext(_contextAccessor, new List() { scopeRestriction1, scopeRestriction2 }); - _contextAccessor.RequestContext.AccessControlContext.CompartmentId = patientId; + _contextAccessor.RequestContext.AccessControlContext.CompartmentId = "smart-patient-A"; _contextAccessor.RequestContext.AccessControlContext.CompartmentResourceType = "Patient"; // Test search capability var query = new List>(); - query.Add(new Tuple("_id", patientId)); + query.Add(new Tuple("_id", "smart-patient-A")); var searchResults = await _searchService.Value.SearchAsync("Patient", query, CancellationToken.None); Assert.NotEmpty(searchResults.Results); // Test update capability var updatedPatient = new Patient { - Id = patientId, + Id = "smart-patient-A", Name = new List { new HumanName().WithGiven("SearchUpdate").AndFamily("SmartV2") }, }; - var updateResult = await _smartFixture.UpsertResource(updatedPatient); + var updateResult = await UpsertResource(updatedPatient); Assert.NotNull(updateResult); - Assert.Equal(patientId, updateResult.Wrapper.ResourceId); + Assert.Equal("smart-patient-A", updateResult.Wrapper.ResourceId); } // SMART v2 Granular Scope with Search parameters Tests diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems index 46544a2f53..1fe1372be9 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Microsoft.Health.Fhir.Shared.Tests.Integration.projitems @@ -24,11 +24,10 @@ - - + diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTests.cs index b791d5e5c5..6f71076443 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTests.cs @@ -1201,12 +1201,10 @@ private async Task CreatePatientSearchParam(string searchParamN Code = searchParamName, }; - _searchParameterDefinitionManager.AddNewSearchParameters([searchParam.ToTypedElement()]); - - await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(); + _searchParameterDefinitionManager.AddNewSearchParameters(new List { searchParam.ToTypedElement() }); // Add the search parameter to the datastore - await _fixture.SearchParameterStatusManager.UpdateSearchParameterStatusAsync([searchParam.Url], SearchParameterStatus.Supported, CancellationToken.None, lastUpdated: _fixture.SearchParameterOperations.SearchParamLastUpdated); + await _fixture.SearchParameterStatusManager.UpdateSearchParameterStatusAsync(new List { searchParam.Url }, SearchParameterStatus.Supported, CancellationToken.None); await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(); @@ -1222,6 +1220,12 @@ private ResourceElement CreatePatientResourceElement(string patientName, string return Deserializers.ResourceDeserializer.DeserializeRaw(rawResource, "v1", DateTimeOffset.UtcNow); } + private async Task ExecuteAndVerifyException(Func action) + where TException : Exception + { + await Assert.ThrowsAsync(action); + } + private async Task SetAllowCreateForOperation(bool allowCreate, Func operation) { var observation = _capabilityStatement.Rest[0].Resource.Find(r => ResourceType.Observation.EqualsString(r.Type.ToString())); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs index 68738bbcde..64b79666a2 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs @@ -20,7 +20,6 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; -using Microsoft.Extensions.Primitives; using Microsoft.Health.Abstractions.Features.Transactions; using Microsoft.Health.Core.Features.Context; using Microsoft.Health.Extensions.DependencyInjection; @@ -53,7 +52,6 @@ using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Core.Registration; using Microsoft.Health.Fhir.Core.UnitTests.Extensions; -using Microsoft.Health.Fhir.Core.UnitTests.Features.Context; using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Operations; using Microsoft.Health.Fhir.SqlServer.Features.Storage; using Microsoft.Health.Fhir.Tests.Common; @@ -197,16 +195,6 @@ public async Task InitializeAsync() await asyncLifetime.InitializeAsync(); } - // Initialize FhirRequestContext to ensure pending status updates are captured - // This needs to be here (like ResourceIdProvider) because it uses AsyncLocal - FhirRequestContextAccessor.RequestContext = new DefaultFhirRequestContext - { - BaseUri = new Uri("http://localhost/"), - CorrelationId = Guid.NewGuid().ToString(), - RequestHeaders = new Dictionary(), - ResponseHeaders = new Dictionary(), - }; - CapabilityStatement = CapabilityStatementMock.GetMockedCapabilityStatement(); IDeletionServiceDataStoreFactory deletionServiceDataStoreFactory = Substitute.For(); @@ -279,7 +267,6 @@ public async Task InitializeAsync() var collection = new ServiceCollection(); - // Register request handlers collection.AddSingleton(typeof(IRequestHandler), new CreateResourceHandler(DataStore, new Lazy(() => ConformanceProvider), resourceWrapperFactory, _resourceIdProvider, new ResourceReferenceResolver(SearchService, new TestQueryStringParser(), Substitute.For>()), DisabledFhirAuthorizationService.Instance)); collection.AddSingleton(typeof(IRequestHandler), new UpsertResourceHandler(DataStore, new Lazy(() => ConformanceProvider), resourceWrapperFactory, _resourceIdProvider, new ResourceReferenceResolver(SearchService, new TestQueryStringParser(), Substitute.For>()), FhirRequestContextAccessor, DisabledFhirAuthorizationService.Instance, ModelInfoProvider.Instance)); collection.AddSingleton(typeof(IRequestHandler), GetResourceHandler); @@ -287,54 +274,18 @@ public async Task InitializeAsync() collection.AddSingleton(typeof(IRequestHandler), new SearchResourceHistoryHandler(SearchService, bundleFactory, DisabledFhirAuthorizationService.Instance, new DataResourceFilter(MissingDataFilterCriteria.Default))); collection.AddSingleton(typeof(IRequestHandler), new SearchResourceHandler(SearchService, bundleFactory, DisabledFhirAuthorizationService.Instance, new DataResourceFilter(MissingDataFilterCriteria.Default))); - var searchParameterSupportResolver = Substitute.For(); - searchParameterSupportResolver.IsSearchParameterSupported(Arg.Any()).Returns((true, false)); - - var dataStoreSearchParameterValidator = Substitute.For(); - dataStoreSearchParameterValidator.ValidateSearchParameter(Arg.Any(), out Arg.Any()).Returns(x => - { - x[1] = null; // out parameter for errorMessage - return true; - }); + ServiceProvider services = collection.BuildServiceProvider(); _searchParameterOperations = new SearchParameterOperations( SearchParameterStatusManager, SearchParameterDefinitionManager, ModelInfoProvider.Instance, - searchParameterSupportResolver, - dataStoreSearchParameterValidator, - () => OperationDataStore.CreateMockScope(), - () => SearchService.CreateMockScope(), + Substitute.For(), + Substitute.For(), + () => Substitute.For>(), + () => Substitute.For>(), NullLogger.Instance); - // Register pipeline behaviors for search parameter handling - collection.AddTransient>( - sp => new CreateOrUpdateSearchParameterBehavior( - _searchParameterOperations, - DataStore, - SearchParameterDefinitionManager, - FhirRequestContextAccessor, - ModelInfoProvider.Instance)); - - collection.AddTransient>( - sp => new CreateOrUpdateSearchParameterBehavior( - _searchParameterOperations, - DataStore, - SearchParameterDefinitionManager, - FhirRequestContextAccessor, - ModelInfoProvider.Instance)); - - collection.AddTransient>( - sp => new DeleteSearchParameterBehavior( - _searchParameterOperations, - DataStore, - SearchParameterDefinitionManager, - SearchParameterStatusManager, - FhirRequestContextAccessor, - ModelInfoProvider.Instance)); - - ServiceProvider services = collection.BuildServiceProvider(); - Mediator = new Mediator(services); } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SearchParameterOptimisticConcurrencyIntegrationTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SearchParameterOptimisticConcurrencyIntegrationTests.cs new file mode 100644 index 0000000000..fe573ac987 --- /dev/null +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SearchParameterOptimisticConcurrencyIntegrationTests.cs @@ -0,0 +1,454 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Health.Extensions.Xunit; +using Microsoft.Health.Fhir.Core.Features.Search.Registry; +using Microsoft.Health.Fhir.SqlServer.Features.Schema; +using Microsoft.Health.Fhir.SqlServer.Features.Storage.Registry; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; +using Microsoft.Health.Test.Utilities; +using Xunit; + +namespace Microsoft.Health.Fhir.Tests.Integration.Persistence +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Search)] + [FhirStorageTestsFixtureArgumentSets(DataStore.SqlServer)] + public class SearchParameterOptimisticConcurrencyIntegrationTests : IClassFixture + { + private readonly FhirStorageTestsFixture _fixture; + private readonly IFhirStorageTestHelper _testHelper; + + public SearchParameterOptimisticConcurrencyIntegrationTests(FhirStorageTestsFixture fixture) + { + _fixture = fixture; + _testHelper = fixture.TestHelper; + } + + [Fact] + public async Task GivenSchemaVersion94OrHigher_WhenGettingSearchParameterStatuses_ThenLastUpdatedIsReturned() + { + // Act + var statuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); + + // Assert + Assert.NotEmpty(statuses); + + // Verify that all statuses have LastUpdated (all parameters should have LastUpdated values) + foreach (var status in statuses) + { + Assert.True(status.LastUpdated != default(DateTimeOffset), "All search parameter statuses should have valid LastUpdated values"); + } + } + + [Fact] + public async Task GivenNewSearchParameterStatus_WhenUpserting_ThenLastUpdatedIsReturned() + { + // Arrange + var testUri = $"http://test.com/SearchParameter/ConcurrencyTest_{Guid.NewGuid()}"; + var newStatus = new ResourceSearchParameterStatus + { + Uri = new Uri(testUri), + Status = SearchParameterStatus.Disabled, + IsPartiallySupported = false, + LastUpdated = default(DateTimeOffset), // New parameter, no previous LastUpdated + }; + + try + { + // Act + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { newStatus }, CancellationToken.None); + + // Get the upserted status to check LastUpdated was assigned + var allStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); + var upsertedStatus = allStatuses.FirstOrDefault(s => s.Uri.ToString() == testUri); + + // Assert + Assert.NotNull(upsertedStatus); + Assert.True(upsertedStatus.LastUpdated != default(DateTimeOffset), "LastUpdated should be assigned for new parameters"); + } + finally + { + // Cleanup + await _testHelper.DeleteSearchParameterStatusAsync(testUri); + } + } + + [Fact] + public async Task GivenExistingSearchParameterStatus_WhenUpdatingWithCorrectLastUpdated_ThenSucceeds() + { + // Arrange + var testUri = $"http://test.com/SearchParameter/ConcurrencyTest_{Guid.NewGuid()}"; + var initialStatus = new ResourceSearchParameterStatus + { + Uri = new Uri(testUri), + Status = SearchParameterStatus.Disabled, + IsPartiallySupported = false, + }; + + try + { + // Create initial status + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { initialStatus }, CancellationToken.None); + + // Get the created status with its LastUpdated + var allStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); + var createdStatus = allStatuses.First(s => s.Uri.ToString() == testUri); + + // Modify and update with correct LastUpdated + var updatedStatus = new ResourceSearchParameterStatus + { + Uri = createdStatus.Uri, + Status = SearchParameterStatus.Enabled, // Changed status + IsPartiallySupported = true, // Changed partially supported + LastUpdated = createdStatus.LastUpdated, // Use the current LastUpdated + }; + + // Act + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { updatedStatus }, CancellationToken.None); + + // Verify the update succeeded + var updatedAllStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); + var result = updatedAllStatuses.First(s => s.Uri.ToString() == testUri); + + // Assert + Assert.Equal(SearchParameterStatus.Enabled, result.Status); + Assert.True(result.IsPartiallySupported); + Assert.True(result.LastUpdated > createdStatus.LastUpdated, "LastUpdated should change after update"); + } + finally + { + // Cleanup + await _testHelper.DeleteSearchParameterStatusAsync(testUri); + } + } + + [Fact] + public async Task GivenExistingSearchParameterStatus_WhenUpdatingWithIncorrectLastUpdated_ThenEventuallySucceedsWithRetry() + { + // Arrange + var testUri = $"http://test.com/SearchParameter/ConcurrencyTest_{Guid.NewGuid()}"; + var initialStatus = new ResourceSearchParameterStatus + { + Uri = new Uri(testUri), + Status = SearchParameterStatus.Disabled, + IsPartiallySupported = false, + }; + + try + { + // Create initial status + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { initialStatus }, CancellationToken.None); + + // Get the created status with its LastUpdated + var allStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); + var createdStatus = allStatuses.First(s => s.Uri.ToString() == testUri); + + // Make an intermediate update to change the LastUpdated + var intermediateUpdate = new ResourceSearchParameterStatus + { + Uri = createdStatus.Uri, + Status = SearchParameterStatus.Enabled, + IsPartiallySupported = false, + LastUpdated = createdStatus.LastUpdated, + }; + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { intermediateUpdate }, CancellationToken.None); + + // Now try to update with the stale LastUpdated + // The retry mechanism should detect the conflict, refresh the LastUpdated, and succeed + var staleUpdate = new ResourceSearchParameterStatus + { + Uri = createdStatus.Uri, + Status = SearchParameterStatus.Supported, + IsPartiallySupported = true, + LastUpdated = createdStatus.LastUpdated, // This is now stale + }; + + // Act - This should succeed due to retry mechanism + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { staleUpdate }, CancellationToken.None); + + // Assert - Verify the final status shows the update succeeded + var finalStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); + var finalStatus = finalStatuses.First(s => s.Uri.ToString() == testUri); + + // The retry mechanism should have allowed the update to succeed + Assert.Equal(SearchParameterStatus.Supported, finalStatus.Status); + Assert.True(finalStatus.IsPartiallySupported); + } + finally + { + // Cleanup + await _testHelper.DeleteSearchParameterStatusAsync(testUri); + } + } + + [Fact] + public async Task GivenMultipleConsecutiveStaleUpdates_WhenUpdatingSearchParameter_ThenRetryMechanismIsTriggeredMultipleTimes() + { + // Arrange + var testUri = $"http://test.com/SearchParameter/RetryTest_{Guid.NewGuid()}"; + var initialStatus = new ResourceSearchParameterStatus + { + Uri = new Uri(testUri), + Status = SearchParameterStatus.Disabled, + IsPartiallySupported = false, + }; + + try + { + // Create initial status + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { initialStatus }, CancellationToken.None); + + // Get the created status with its LastUpdated + var allStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); + var createdStatus = allStatuses.First(s => s.Uri.ToString() == testUri); + var originalLastUpdated = createdStatus.LastUpdated; + + // Force multiple intermediate updates to create staleness + for (int i = 0; i < 3; i++) + { + var intermediateUpdate = new ResourceSearchParameterStatus + { + Uri = createdStatus.Uri, + Status = SearchParameterStatus.Enabled, + IsPartiallySupported = false, + LastUpdated = createdStatus.LastUpdated, + }; + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { intermediateUpdate }, CancellationToken.None); + + // Refresh status for next iteration + allStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); + createdStatus = allStatuses.First(s => s.Uri.ToString() == testUri); + } + + // Now attempt update with very stale LastUpdated - this should trigger retry + var veryStaleUpdate = new ResourceSearchParameterStatus + { + Uri = createdStatus.Uri, + Status = SearchParameterStatus.Supported, + IsPartiallySupported = true, + LastUpdated = originalLastUpdated, // This is very stale (3 updates behind) + }; + + // Act - This should succeed due to retry mechanism + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { veryStaleUpdate }, CancellationToken.None); + + // Assert - Verify the final status shows the update succeeded despite staleness + var finalStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); + var finalStatus = finalStatuses.First(s => s.Uri.ToString() == testUri); + + Assert.Equal(SearchParameterStatus.Supported, finalStatus.Status); + Assert.True(finalStatus.IsPartiallySupported); + Assert.True(finalStatus.LastUpdated > originalLastUpdated, "LastUpdated should have changed significantly"); + } + finally + { + // Cleanup + await _testHelper.DeleteSearchParameterStatusAsync(testUri); + } + } + + [RetryFact] + public async Task GivenRapidConcurrentUpdates_WhenUsingStaleLastUpdated_ThenRetryMechanismHandlesHighContentionScenario() + { + // Arrange + var testUri = $"http://test.com/SearchParameter/HighContentionTest_{Guid.NewGuid()}"; + var initialStatus = new ResourceSearchParameterStatus + { + Uri = new Uri(testUri), + Status = SearchParameterStatus.Disabled, + IsPartiallySupported = false, + }; + + try + { + // Create initial status + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { initialStatus }, CancellationToken.None); + + // Get the created status with its LastUpdated + var allStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); + var createdStatus = allStatuses.First(s => s.Uri.ToString() == testUri); + var originalLastUpdated = createdStatus.LastUpdated; + + // Create high contention by launching multiple rapid updates + var rapidUpdateTasks = new List(); + for (int i = 0; i < 5; i++) + { + var updateIndex = i; + rapidUpdateTasks.Add(Task.Run(async () => + { + var rapidUpdate = new ResourceSearchParameterStatus + { + Uri = createdStatus.Uri, + Status = SearchParameterStatus.Enabled, + IsPartiallySupported = updateIndex % 2 == 0, + LastUpdated = createdStatus.LastUpdated, // Using same (potentially stale) LastUpdated + }; + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { rapidUpdate }, CancellationToken.None); + })); + } + + // Act - Wait for all rapid updates to complete (some may trigger retries due to contention) + await Task.WhenAll(rapidUpdateTasks); + + // Now attempt one final update with the original (definitely stale) LastUpdated + var finalStaleUpdate = new ResourceSearchParameterStatus + { + Uri = createdStatus.Uri, + Status = SearchParameterStatus.Supported, + IsPartiallySupported = true, + LastUpdated = originalLastUpdated, // This is definitely stale after all the rapid updates + }; + + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { finalStaleUpdate }, CancellationToken.None); + + // Assert - Verify the final update succeeded despite high contention + var finalStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); + var finalStatus = finalStatuses.First(s => s.Uri.ToString() == testUri); + + Assert.Equal(SearchParameterStatus.Supported, finalStatus.Status); + Assert.True(finalStatus.IsPartiallySupported); + Assert.True(finalStatus.LastUpdated > originalLastUpdated, "LastUpdated should have changed after high contention scenario"); + } + finally + { + // Cleanup + await _testHelper.DeleteSearchParameterStatusAsync(testUri); + } + } + + [Fact] + public async Task GivenOptimisticConcurrencyDetection_WhenLastUpdatedChangesBeforeUpdate_ThenRetryMechanismHandlesConflict() + { + // Arrange + var testUri = $"http://test.com/SearchParameter/ConcurrencyTest_{Guid.NewGuid()}"; + var initialStatus = new ResourceSearchParameterStatus + { + Uri = new Uri(testUri), + Status = SearchParameterStatus.Disabled, + IsPartiallySupported = false, + }; + + try + { + // Create initial status + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { initialStatus }, CancellationToken.None); + + // Get the created status with its LastUpdated + var allStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); + var createdStatus = allStatuses.First(s => s.Uri.ToString() == testUri); + var originalLastUpdated = createdStatus.LastUpdated; + + // First, update with current LastUpdated to change it + var intermediateUpdate = new ResourceSearchParameterStatus + { + Uri = createdStatus.Uri, + Status = SearchParameterStatus.Enabled, + IsPartiallySupported = false, + LastUpdated = originalLastUpdated, + }; + + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { intermediateUpdate }, CancellationToken.None); + + // Now try to update with the stale LastUpdated - this should trigger retry mechanism + var staleUpdate = new ResourceSearchParameterStatus + { + Uri = createdStatus.Uri, + Status = SearchParameterStatus.Supported, + IsPartiallySupported = true, + LastUpdated = originalLastUpdated, // This is now stale + }; + + // Act - This should succeed due to retry mechanism + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { staleUpdate }, CancellationToken.None); + + // Verify the final status + var finalStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); + var finalStatus = finalStatuses.First(s => s.Uri.ToString() == testUri); + + // Assert - The update should have succeeded (with retry) + Assert.Equal(SearchParameterStatus.Supported, finalStatus.Status); + Assert.True(finalStatus.IsPartiallySupported); + Assert.True(finalStatus.LastUpdated > originalLastUpdated, "LastUpdated should have changed"); + } + finally + { + // Cleanup + await _testHelper.DeleteSearchParameterStatusAsync(testUri); + } + } + + [Fact] + public async Task GivenMixedUpdatesWithAndWithoutRowVersion_WhenUpserting_ThenBothSucceed() + { + // Arrange + var existingTestUri = $"http://test.com/SearchParameter/Existing_{Guid.NewGuid()}"; + var newTestUri = $"http://test.com/SearchParameter/New_{Guid.NewGuid()}_"; + + var existingStatus = new ResourceSearchParameterStatus + { + Uri = new Uri(existingTestUri), + Status = SearchParameterStatus.Disabled, + IsPartiallySupported = false, + }; + + try + { + // Create existing parameter + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { existingStatus }, CancellationToken.None); + + // Get the created status with its LastUpdated + var allStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); + var createdStatus = allStatuses.First(s => s.Uri.ToString() == existingTestUri); + + // Prepare mixed updates + var updateExisting = new ResourceSearchParameterStatus + { + Uri = createdStatus.Uri, + Status = SearchParameterStatus.Enabled, + IsPartiallySupported = true, + LastUpdated = createdStatus.LastUpdated, // With LastUpdated + }; + + var createNew = new ResourceSearchParameterStatus + { + Uri = new Uri(newTestUri), + Status = SearchParameterStatus.Supported, + IsPartiallySupported = false, + LastUpdated = default(DateTimeOffset), // Without LastUpdated (new parameter) + }; + + // Act + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { updateExisting, createNew }, CancellationToken.None); + + // Verify both operations succeeded + var updatedAllStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); + var updatedExisting = updatedAllStatuses.First(r => r.Uri.ToString() == existingTestUri); + var createdNew = updatedAllStatuses.First(r => r.Uri.ToString() == newTestUri); + + // Assert + Assert.Equal(SearchParameterStatus.Enabled, updatedExisting.Status); + Assert.True(updatedExisting.IsPartiallySupported); + Assert.True(updatedExisting.LastUpdated > createdStatus.LastUpdated, "Updated parameter should have newer LastUpdated"); + + Assert.Equal(SearchParameterStatus.Supported, createdNew.Status); + Assert.False(createdNew.IsPartiallySupported); + Assert.True(createdNew.LastUpdated != default(DateTimeOffset), "New parameter should have valid LastUpdated"); + } + finally + { + // Cleanup + await _testHelper.DeleteSearchParameterStatusAsync(existingTestUri); + await _testHelper.DeleteSearchParameterStatusAsync(newTestUri); + } + } + } +} diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SearchParameterOptimisticConcurrencyTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SearchParameterOptimisticConcurrencyTests.cs deleted file mode 100644 index 5e78194ce6..0000000000 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SearchParameterOptimisticConcurrencyTests.cs +++ /dev/null @@ -1,176 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Data.SqlClient; -using Microsoft.Health.Extensions.Xunit; -using Microsoft.Health.Fhir.Core.Features.Persistence; -using Microsoft.Health.Fhir.Core.Features.Search.Registry; -using Microsoft.Health.Fhir.SqlServer.Features.Schema; -using Microsoft.Health.Fhir.SqlServer.Features.Storage.Registry; -using Microsoft.Health.Fhir.Tests.Common; -using Microsoft.Health.Fhir.Tests.Common.FixtureParameters; -using Microsoft.Health.Test.Utilities; -using Xunit; - -namespace Microsoft.Health.Fhir.Tests.Integration.Persistence -{ - [Trait(Traits.OwningTeam, OwningTeam.Fhir)] - [Trait(Traits.Category, Categories.Search)] - [FhirStorageTestsFixtureArgumentSets(DataStore.SqlServer)] - public class SearchParameterOptimisticConcurrencyTests : IClassFixture - { - private readonly FhirStorageTestsFixture _fixture; - private readonly IFhirStorageTestHelper _testHelper; - - public SearchParameterOptimisticConcurrencyTests(FhirStorageTestsFixture fixture) - { - _fixture = fixture; - _testHelper = fixture.TestHelper; - } - - [Fact] - public async Task GivenNewSearchParameterStatus_WhenUpserting_ThenLastUpdatedIsReturned() - { - await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); - - var testUri = $"http://test.com/SearchParameter/ConcurrencyTest_{Guid.NewGuid()}"; - var newStatus = new ResourceSearchParameterStatus - { - Uri = new Uri(testUri), - Status = SearchParameterStatus.Disabled, - IsPartiallySupported = false, - LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, - }; - - try - { - await _fixture.SearchParameterStatusDataStore.UpsertStatuses([newStatus], CancellationToken.None); - - // Get the upserted status to check LastUpdated was assigned - var allStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); - var upsertedStatus = allStatuses.FirstOrDefault(s => s.Uri.ToString() == testUri); - - Assert.NotNull(upsertedStatus); - Assert.True(upsertedStatus.LastUpdated != default(DateTimeOffset), "LastUpdated should be assigned for new parameters"); - } - finally - { - await _testHelper.DeleteSearchParameterStatusAsync(testUri); - } - } - - [Fact] - public async Task GivenExistingSearchParameterStatus_WhenUpdatingWithCorrectLastUpdated_ThenSucceeds() - { - await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); - - var testUri = $"http://test.com/SearchParameter/ConcurrencyTest_{Guid.NewGuid()}"; - var initialStatus = new ResourceSearchParameterStatus - { - Uri = new Uri(testUri), - Status = SearchParameterStatus.Disabled, - IsPartiallySupported = false, - LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, - }; - - try - { - await _fixture.SearchParameterStatusDataStore.UpsertStatuses([initialStatus], CancellationToken.None); - - // Get the created status with its LastUpdated - var allStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); - var createdStatus = allStatuses.First(s => s.Uri.ToString() == testUri); - - await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); - - // Modify and update with correct LastUpdated - var updatedStatus = new ResourceSearchParameterStatus - { - Uri = createdStatus.Uri, - Status = SearchParameterStatus.Enabled, // Changed status - IsPartiallySupported = true, // Changed partially supported - LastUpdated = createdStatus.LastUpdated, // Use the current LastUpdated, it should match max one. - }; - - await _fixture.SearchParameterStatusDataStore.UpsertStatuses([updatedStatus], CancellationToken.None); - - // Verify the update succeeded - var updatedAllStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); - var result = updatedAllStatuses.First(s => s.Uri.ToString() == testUri); - - Assert.Equal(SearchParameterStatus.Enabled, result.Status); - Assert.True(result.IsPartiallySupported); - Assert.True(result.LastUpdated > createdStatus.LastUpdated, "LastUpdated should change after update"); - } - finally - { - await _testHelper.DeleteSearchParameterStatusAsync(testUri); - } - } - - [Fact] - public async Task GivenExistingSearchParameterStatus_WhenUpdatingWithIncorrectLastUpdated_ThenShouldFail() - { - await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); - - var testUri = $"http://test.com/SearchParameter/ConcurrencyTest_{Guid.NewGuid()}"; - var initialStatus = new ResourceSearchParameterStatus - { - Uri = new Uri(testUri), - Status = SearchParameterStatus.Disabled, - IsPartiallySupported = false, - LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, - }; - - try - { - // Create initial status - await _fixture.SearchParameterStatusDataStore.UpsertStatuses([initialStatus], CancellationToken.None); - - // Get the created status with its LastUpdated - var allStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); - var createdStatus = allStatuses.First(s => s.Uri.ToString() == testUri); - - // Make an intermediate update to change the LastUpdated - var intermediateUpdate = new ResourceSearchParameterStatus - { - Uri = createdStatus.Uri, - Status = SearchParameterStatus.Enabled, - IsPartiallySupported = false, - LastUpdated = createdStatus.LastUpdated, - }; - await _fixture.SearchParameterStatusDataStore.UpsertStatuses([intermediateUpdate], CancellationToken.None); - - // Now try to update with the stale LastUpdated, should fail - var staleUpdate = new ResourceSearchParameterStatus - { - Uri = createdStatus.Uri, - Status = SearchParameterStatus.Supported, - IsPartiallySupported = true, - LastUpdated = createdStatus.LastUpdated, // This is now stale - }; - - try - { - await _fixture.SearchParameterStatusDataStore.UpsertStatuses([staleUpdate], CancellationToken.None); - Assert.Fail("This point should not be reached"); - } - catch (BadRequestException ex) - { - Assert.True(ex.Message.StartsWith(Core.Resources.SearchParameterConcurrencyConflict), $"expected={Core.Resources.SearchParameterConcurrencyConflict}, actual={ex.Message}"); - } - } - finally - { - await _testHelper.DeleteSearchParameterStatusAsync(testUri); - } - } - } -} diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SearchParameterStatusDataStoreTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SearchParameterStatusDataStoreTests.cs index 1ab33effb3..08ce3968ea 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SearchParameterStatusDataStoreTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SearchParameterStatusDataStoreTests.cs @@ -36,7 +36,7 @@ public async Task GivenAStatusRegistry_WhenGettingStatuses_ThenTheStatusesAreRet IReadOnlyCollection expectedStatuses = await _fixture.FilebasedSearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); IReadOnlyCollection actualStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); - ValidateSearchParameterStatuses(expectedStatuses, actualStatuses, true); + ValidateSearchParameterStatuses(expectedStatuses, actualStatuses); } [Fact] @@ -45,16 +45,14 @@ public async Task GivenAStatusRegistry_WhenUpsertingNewStatuses_ThenTheStatusesA string statusName1 = "http://hl7.org/fhir/SearchParameter/Test-1"; string statusName2 = "http://hl7.org/fhir/SearchParameter/Test-2"; - await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); - var status1 = new ResourceSearchParameterStatus { - Uri = new Uri(statusName1), Status = SearchParameterStatus.Disabled, IsPartiallySupported = false, LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, + Uri = new Uri(statusName1), Status = SearchParameterStatus.Disabled, IsPartiallySupported = false, }; var status2 = new ResourceSearchParameterStatus { - Uri = new Uri(statusName2), Status = SearchParameterStatus.Disabled, IsPartiallySupported = false, LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, + Uri = new Uri(statusName2), Status = SearchParameterStatus.Disabled, IsPartiallySupported = false, }; IReadOnlyCollection readonlyStatusesBeforeUpsert = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); @@ -82,21 +80,16 @@ public async Task GivenAStatusRegistry_WhenUpsertingNewStatuses_ThenTheStatusesA [Fact] public async Task GivenAStatusRegistry_WhenUpsertingExistingStatuses_ThenTheExistingStatusesAreUpdated() { - await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); - - var statusesBeforeUpdate = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); + IReadOnlyCollection statusesBeforeUpdate = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); // Get two existing statuses. - var expectedStatus1 = statusesBeforeUpdate.First(); - var expectedStatus2 = statusesBeforeUpdate.Last(); + ResourceSearchParameterStatus expectedStatus1 = statusesBeforeUpdate.First(); + ResourceSearchParameterStatus expectedStatus2 = statusesBeforeUpdate.Last(); // Modify them in some way. expectedStatus1.IsPartiallySupported = !expectedStatus1.IsPartiallySupported; expectedStatus2.IsPartiallySupported = !expectedStatus2.IsPartiallySupported; - // set last updated on at least one so it will set correct max - expectedStatus1.LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated; - var statusesToUpsert = new List { expectedStatus1, expectedStatus2 }; try @@ -104,7 +97,8 @@ public async Task GivenAStatusRegistry_WhenUpsertingExistingStatuses_ThenTheExis // Upsert the two existing, modified statuses. await _fixture.SearchParameterStatusDataStore.UpsertStatuses(statusesToUpsert, CancellationToken.None); - var statusesAfterUpdate = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); + IReadOnlyCollection statusesAfterUpdate = + await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); Assert.Equal(statusesBeforeUpdate.Count, statusesAfterUpdate.Count); @@ -126,40 +120,20 @@ public async Task GivenAStatusRegistry_WhenUpsertingExistingStatuses_ThenTheExis expectedStatus1.IsPartiallySupported = !expectedStatus1.IsPartiallySupported; expectedStatus2.IsPartiallySupported = !expectedStatus2.IsPartiallySupported; - await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); - - expectedStatus2.LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated; - statusesToUpsert = new List { expectedStatus1, expectedStatus2 }; await _fixture.SearchParameterStatusDataStore.UpsertStatuses(statusesToUpsert, CancellationToken.None); } } - private static void ValidateSearchParameterStatuses(IReadOnlyCollection expectedStatuses, IReadOnlyCollection actualStatuses, bool allowMore = false) + private static void ValidateSearchParameterStatuses(IReadOnlyCollection expectedStatuses, IReadOnlyCollection actualStatuses) { Assert.NotEmpty(expectedStatuses); var sortedExpected = expectedStatuses.OrderBy(status => status.Uri.ToString()).ToList(); var sortedActual = actualStatuses.OrderBy(status => status.Uri.ToString()).ToList(); - if (allowMore) - { - Assert.True(sortedExpected.Count <= sortedActual.Count); // we are not deleting so main store can accumulate more items than in file based - - // remove extra - foreach (var status in sortedActual.ToList()) - { - if (!sortedExpected.Any(_ => _.Uri == status.Uri)) - { - sortedActual.Remove(status); - } - } - } - else - { - Assert.Equal(sortedExpected.Count, sortedActual.Count); - } + Assert.Equal(sortedExpected.Count, sortedActual.Count); for (int i = 0; i < sortedExpected.Count; i++) { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerCreateStatsForSmartTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerCreateStatsForSmartTests.cs index 1afcdeabf2..6ac26899ea 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerCreateStatsForSmartTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerCreateStatsForSmartTests.cs @@ -343,10 +343,7 @@ private void ConfigureFhirRequestContext( accessControlContext.AllowedResourceActions.Add(scope); } - var mockFhirRequestContext = Substitute.For(); - mockFhirRequestContext.AccessControlContext.Returns(accessControlContext); - - contextAccessor.RequestContext.Returns(mockFhirRequestContext); + contextAccessor.RequestContext.AccessControlContext.Returns(accessControlContext); } private static SearchParams CreateSearchParams(params (string key, string value)[] items) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestHelper.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestHelper.cs index 2e5e0f02c6..0ecb5acd5e 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestHelper.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestHelper.cs @@ -270,17 +270,14 @@ public Task DeleteExportJobRecordAsync(string id, CancellationToken cancellation public async Task DeleteSearchParameterStatusAsync(string uri, CancellationToken cancellationToken = default) { - await using var connection = await _sqlConnectionBuilder.GetSqlConnectionAsync(cancellationToken: cancellationToken); - using var command = new SqlCommand( - @" -UPDATE dbo.SearchParam - SET Status = 'PendingDelete', LastUpdated = convert(datetimeoffset(7), sysUTCdatetime()) - WHERE Uri = @uri", - connection); + await using SqlConnection connection = await _sqlConnectionBuilder.GetSqlConnectionAsync(cancellationToken: cancellationToken); + var command = new SqlCommand("DELETE FROM dbo.SearchParam WHERE Uri = @uri", connection); command.Parameters.AddWithValue("@uri", uri); - await connection.OpenAsync(cancellationToken); + await command.Connection.OpenAsync(cancellationToken); await command.ExecuteNonQueryAsync(cancellationToken); + await connection.CloseAsync(); + _sqlServerFhirModel.RemoveSearchParamIdToUriMapping(uri); } public async Task DeleteAllReindexJobRecordsAsync(CancellationToken cancellationToken = default) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSearchParameterStatusDataStoreTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSearchParameterStatusDataStoreTests.cs index 871a41af47..acfdc58aa8 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSearchParameterStatusDataStoreTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerSearchParameterStatusDataStoreTests.cs @@ -42,8 +42,6 @@ public SqlServerSearchParameterStatusDataStoreTests(FhirStorageTestsFixture fixt [Fact] public async Task GivenUpsertStatuses_WhenUpsertingWithSameUri_ThenLastUpdatedIsRefreshed() { - await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); - // Arrange var testUri = "http://hl7.org/fhir/SearchParameter/Test-Upsert-" + Guid.NewGuid(); var status = new ResourceSearchParameterStatus @@ -51,13 +49,13 @@ public async Task GivenUpsertStatuses_WhenUpsertingWithSameUri_ThenLastUpdatedIs Uri = new Uri(testUri), Status = SearchParameterStatus.Disabled, IsPartiallySupported = false, - LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, + LastUpdated = DateTimeOffset.UtcNow, }; try { // Act - First upsert - await _fixture.SearchParameterStatusDataStore.UpsertStatuses([status], CancellationToken.None); + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { status }, CancellationToken.None); // Get the result var allStatuses1 = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); @@ -68,14 +66,12 @@ public async Task GivenUpsertStatuses_WhenUpsertingWithSameUri_ThenLastUpdatedIs // Small delay to ensure different timestamp await Task.Delay(100); - await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); - // Modify and upsert again status.Status = SearchParameterStatus.Enabled; status.IsPartiallySupported = true; - status.LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated; + status.LastUpdated = createdStatus.LastUpdated; // Use the LastUpdated from DB - await _fixture.SearchParameterStatusDataStore.UpsertStatuses([status], CancellationToken.None); + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { status }, CancellationToken.None); // Get the updated result var allStatuses2 = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); @@ -113,7 +109,7 @@ public void GivenSyncStatuses_WhenCalledWithStatuses_ThenFhirModelIsSynchronized }; // Act - Call SyncStatuses (this should not throw) - var exception = Record.Exception(() => dataStore!.SyncStatuses([status])); + var exception = Record.Exception(() => dataStore!.SyncStatuses(new[] { status })); // Assert - Method completes without exception Assert.Null(exception); @@ -122,8 +118,6 @@ public void GivenSyncStatuses_WhenCalledWithStatuses_ThenFhirModelIsSynchronized [Fact] public async Task GivenUpsertStatuses_WhenUpsertingMultipleStatuses_ThenAllAreCreated() { - await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); - // Arrange var testUri1 = "http://hl7.org/fhir/SearchParameter/Test-Batch1-" + Guid.NewGuid(); var testUri2 = "http://hl7.org/fhir/SearchParameter/Test-Batch2-" + Guid.NewGuid(); @@ -136,21 +130,21 @@ public async Task GivenUpsertStatuses_WhenUpsertingMultipleStatuses_ThenAllAreCr Uri = new Uri(testUri1), Status = SearchParameterStatus.Disabled, IsPartiallySupported = false, - LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, + LastUpdated = DateTimeOffset.UtcNow, }, new ResourceSearchParameterStatus { Uri = new Uri(testUri2), Status = SearchParameterStatus.Enabled, IsPartiallySupported = true, - LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, + LastUpdated = DateTimeOffset.UtcNow, }, new ResourceSearchParameterStatus { Uri = new Uri(testUri3), Status = SearchParameterStatus.Supported, IsPartiallySupported = false, - LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, + LastUpdated = DateTimeOffset.UtcNow, }, }; @@ -233,8 +227,6 @@ public async Task GivenGetSearchParameterStatuses_WhenStatusHasSortableType_Then [Fact] public async Task GivenUpsertStatuses_WhenUpdatingExistingStatus_ThenPreservesOtherStatuses() { - await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); - // Arrange var testUri = "http://hl7.org/fhir/SearchParameter/Test-Preserve-" + Guid.NewGuid(); var status = new ResourceSearchParameterStatus @@ -242,22 +234,20 @@ public async Task GivenUpsertStatuses_WhenUpdatingExistingStatus_ThenPreservesOt Uri = new Uri(testUri), Status = SearchParameterStatus.Disabled, IsPartiallySupported = false, - LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, + LastUpdated = DateTimeOffset.UtcNow, }; try { // Create initial status - await _fixture.SearchParameterStatusDataStore.UpsertStatuses([status], CancellationToken.None); + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { status }, CancellationToken.None); var countBefore = (await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None)).Count; - await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); - // Update the status status.Status = SearchParameterStatus.Enabled; - status.LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated; - await _fixture.SearchParameterStatusDataStore.UpsertStatuses([status], CancellationToken.None); + status.LastUpdated = DateTimeOffset.UtcNow; + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { status }, CancellationToken.None); var countAfter = (await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None)).Count; @@ -271,26 +261,38 @@ public async Task GivenUpsertStatuses_WhenUpdatingExistingStatus_ThenPreservesOt } [Fact] - public async Task GivenUpsertStatuses_WhenCollectionIsUpdated_ThenReturnedLastUpdatedIsGreaterThanOriginal() + public async Task GivenUpsertStatuses_WhenLastUpdatedIsPropagated_ThenInputCollectionIsUpdated() { - await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); - + // Arrange var testUri = "http://hl7.org/fhir/SearchParameter/Test-Propagate-" + Guid.NewGuid(); var status = new ResourceSearchParameterStatus { Uri = new Uri(testUri), Status = SearchParameterStatus.Disabled, IsPartiallySupported = false, - LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, + LastUpdated = DateTimeOffset.UtcNow, }; try { - await _fixture.SearchParameterStatusDataStore.UpsertStatuses([status], CancellationToken.None); + // Act - Upsert and verify LastUpdated is propagated back + var originalLastUpdated = status.LastUpdated; + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { status }, CancellationToken.None); + + // Assert - The status object should have an updated LastUpdated value from the database + // Note: The database may return timestamps in a different timezone, so we compare using UtcDateTime + Assert.True( + status.LastUpdated.UtcDateTime >= originalLastUpdated.UtcDateTime, + $"Expected LastUpdated ({status.LastUpdated}) to be >= original ({originalLastUpdated})"); - var dbStatus = (await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None)).FirstOrDefault(s => s.Uri.OriginalString == testUri); + // Verify the value in the database matches what was propagated to the input collection + var dbStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); + var dbStatus = dbStatuses.FirstOrDefault(s => s.Uri.OriginalString == testUri); - Assert.True(dbStatus.LastUpdated > status.LastUpdated, $"Expected {status.LastUpdated.ToString("yyyy-MM-ddTHH:mm:ss.fff")} < {dbStatus.LastUpdated.ToString("yyyy-MM-ddTHH:mm:ss.fff")}"); + Assert.NotNull(dbStatus); + Assert.True( + Math.Abs((dbStatus.LastUpdated - status.LastUpdated).TotalSeconds) < 1, + $"Expected propagated LastUpdated ({status.LastUpdated}) to match database ({dbStatus.LastUpdated}) within 1 second"); } finally { @@ -301,8 +303,6 @@ public async Task GivenUpsertStatuses_WhenCollectionIsUpdated_ThenReturnedLastUp [Fact] public async Task GivenSqlServerResourceSearchParameterStatus_WhenIdIsAssigned_ThenIdIsPersisted() { - await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); - // Arrange var testUri = "http://hl7.org/fhir/SearchParameter/Test-Id-" + Guid.NewGuid(); var status = new SqlServerResourceSearchParameterStatus @@ -310,13 +310,13 @@ public async Task GivenSqlServerResourceSearchParameterStatus_WhenIdIsAssigned_T Uri = new Uri(testUri), Status = SearchParameterStatus.Disabled, IsPartiallySupported = false, - LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, + LastUpdated = DateTimeOffset.UtcNow, }; try { // Act - Upsert and retrieve - await _fixture.SearchParameterStatusDataStore.UpsertStatuses([status], CancellationToken.None); + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { status }, CancellationToken.None); var dbStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); var dbStatus = dbStatuses.FirstOrDefault(s => s.Uri.OriginalString == testUri); @@ -337,21 +337,20 @@ public async Task GivenSqlServerResourceSearchParameterStatus_WhenIdIsAssigned_T [Fact] public async Task GivenGetSearchParameterStatuses_WhenIsPartiallySupported_ThenValueIsPreserved() { - await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); - + // Arrange var testUri = "http://hl7.org/fhir/SearchParameter/Test-PartialSupport-" + Guid.NewGuid(); var status = new ResourceSearchParameterStatus { Uri = new Uri(testUri), Status = SearchParameterStatus.Enabled, IsPartiallySupported = true, - LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, + LastUpdated = DateTimeOffset.UtcNow, }; try { // Act - await _fixture.SearchParameterStatusDataStore.UpsertStatuses([status], CancellationToken.None); + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { status }, CancellationToken.None); var dbStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); var dbStatus = dbStatuses.FirstOrDefault(s => s.Uri.OriginalString == testUri); @@ -370,8 +369,6 @@ public async Task GivenGetSearchParameterStatuses_WhenIsPartiallySupported_ThenV [Fact] public async Task GivenUpsertStatuses_WhenStatusIsUnsupported_ThenStatusIsPersisted() { - await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); - // Arrange - Test that Unsupported status is handled correctly // Note: In older schemas (< V52), Unsupported is converted to Disabled // In newer schemas (>= V52), Unsupported is preserved @@ -381,13 +378,13 @@ public async Task GivenUpsertStatuses_WhenStatusIsUnsupported_ThenStatusIsPersis Uri = new Uri(testUri), Status = SearchParameterStatus.Unsupported, IsPartiallySupported = false, - LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, + LastUpdated = DateTimeOffset.UtcNow, }; try { // Act - await _fixture.SearchParameterStatusDataStore.UpsertStatuses([status], CancellationToken.None); + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { status }, CancellationToken.None); var dbStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); var dbStatus = dbStatuses.FirstOrDefault(s => s.Uri.OriginalString == testUri); @@ -407,8 +404,6 @@ public async Task GivenUpsertStatuses_WhenStatusIsUnsupported_ThenStatusIsPersis [Fact] public async Task GivenUpsertStatuses_WhenMixedNewAndExistingStatuses_ThenBothAreHandledCorrectly() { - await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); - // Arrange var existingUri = "http://hl7.org/fhir/SearchParameter/Test-MixedExisting-" + Guid.NewGuid(); var newUri = "http://hl7.org/fhir/SearchParameter/Test-MixedNew-" + Guid.NewGuid(); @@ -418,7 +413,7 @@ public async Task GivenUpsertStatuses_WhenMixedNewAndExistingStatuses_ThenBothAr Uri = new Uri(existingUri), Status = SearchParameterStatus.Disabled, IsPartiallySupported = false, - LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, + LastUpdated = DateTimeOffset.UtcNow, }; try @@ -430,15 +425,13 @@ public async Task GivenUpsertStatuses_WhenMixedNewAndExistingStatuses_ThenBothAr var allStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); var createdStatus = allStatuses.First(s => s.Uri.OriginalString == existingUri); - await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); - // Prepare mixed batch: update existing + create new var updateExisting = new ResourceSearchParameterStatus { Uri = createdStatus.Uri, Status = SearchParameterStatus.Enabled, IsPartiallySupported = true, - LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, + LastUpdated = createdStatus.LastUpdated, }; var createNew = new ResourceSearchParameterStatus @@ -446,11 +439,13 @@ public async Task GivenUpsertStatuses_WhenMixedNewAndExistingStatuses_ThenBothAr Uri = new Uri(newUri), Status = SearchParameterStatus.Supported, IsPartiallySupported = false, - LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, + LastUpdated = DateTimeOffset.UtcNow, }; // Act - await _fixture.SearchParameterStatusDataStore.UpsertStatuses([updateExisting, createNew], CancellationToken.None); + await _fixture.SearchParameterStatusDataStore.UpsertStatuses( + new[] { updateExisting, createNew }, + CancellationToken.None); // Assert var finalStatuses = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); @@ -474,8 +469,6 @@ public async Task GivenUpsertStatuses_WhenMixedNewAndExistingStatuses_ThenBothAr [Fact] public async Task GivenUpsertStatuses_WhenStatusValueChanges_ThenChangeIsReflectedInDatabase() { - await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); - // Comprehensive test for all status transition scenarios // Consolidates multiple transition tests into one comprehensive test @@ -486,13 +479,13 @@ public async Task GivenUpsertStatuses_WhenStatusValueChanges_ThenChangeIsReflect Uri = new Uri(testUri), Status = SearchParameterStatus.Disabled, IsPartiallySupported = false, - LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, + LastUpdated = DateTimeOffset.UtcNow, }; try { // Create initial status (Disabled) - await _fixture.SearchParameterStatusDataStore.UpsertStatuses([status], CancellationToken.None); + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { status }, CancellationToken.None); var statuses1 = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); var dbStatus1 = statuses1.First(s => s.Uri.OriginalString == testUri); @@ -501,13 +494,11 @@ public async Task GivenUpsertStatuses_WhenStatusValueChanges_ThenChangeIsReflect await Task.Delay(100); - await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); - // Transition to Enabled status.Status = SearchParameterStatus.Enabled; status.IsPartiallySupported = true; - status.LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated; - await _fixture.SearchParameterStatusDataStore.UpsertStatuses([status], CancellationToken.None); + status.LastUpdated = dbStatus1.LastUpdated; + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { status }, CancellationToken.None); var statuses2 = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); var dbStatus2 = statuses2.First(s => s.Uri.OriginalString == testUri); @@ -518,13 +509,11 @@ public async Task GivenUpsertStatuses_WhenStatusValueChanges_ThenChangeIsReflect await Task.Delay(100); - await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); - // Transition to Supported status.Status = SearchParameterStatus.Supported; status.IsPartiallySupported = false; - status.LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated; - await _fixture.SearchParameterStatusDataStore.UpsertStatuses([status], CancellationToken.None); + status.LastUpdated = dbStatus2.LastUpdated; + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { status }, CancellationToken.None); var statuses3 = await _fixture.SearchParameterStatusDataStore.GetSearchParameterStatuses(CancellationToken.None); var dbStatus3 = statuses3.First(s => s.Uri.OriginalString == testUri); @@ -541,8 +530,6 @@ public async Task GivenUpsertStatuses_WhenStatusValueChanges_ThenChangeIsReflect [Fact] public async Task GivenUpsertStatuses_WhenCancellationRequested_ThenOperationIsCancelled() { - await _fixture.SearchParameterOperations.GetAndApplySearchParameterUpdates(CancellationToken.None); - // Arrange var testUri = "http://hl7.org/fhir/SearchParameter/Test-Cancellation-" + Guid.NewGuid(); var status = new ResourceSearchParameterStatus @@ -550,7 +537,7 @@ public async Task GivenUpsertStatuses_WhenCancellationRequested_ThenOperationIsC Uri = new Uri(testUri), Status = SearchParameterStatus.Disabled, IsPartiallySupported = false, - LastUpdated = _fixture.SearchParameterOperations.SearchParamLastUpdated, + LastUpdated = DateTimeOffset.UtcNow, }; using var cts = new CancellationTokenSource(); @@ -559,7 +546,7 @@ public async Task GivenUpsertStatuses_WhenCancellationRequested_ThenOperationIsC // Act & Assert await Assert.ThrowsAnyAsync(async () => { - await _fixture.SearchParameterStatusDataStore.UpsertStatuses([status], cts.Token); + await _fixture.SearchParameterStatusDataStore.UpsertStatuses(new[] { status }, cts.Token); }); } From 7ac0d3e5f7e3e2308a212fe767abd96b523487bb Mon Sep 17 00:00:00 2001 From: Mikael Weaver Date: Wed, 17 Jun 2026 11:28:20 -0700 Subject: [PATCH 17/17] fix whitespace --- .../Expressions/Visitors/ScalarTemporalEqualityRewriter.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs index 40a595e488..eaa74bfed5 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs @@ -53,9 +53,9 @@ private enum Precision public override Expression VisitMultiary(MultiaryExpression expression, bool context) { - // Don't run rewriter for chained or reverse chained expression. Once we remove the union in this rewriter below and - // adhere to the FHIR spec for dates, we can remove this method. Our current SQL generator has difficulty with unions - // emitted by this rewriter due to FindRestrictingPredecessorTableExpressionIndex() has a catch all that returns + // Don't run rewriter for chained or reverse chained expression. Once we remove the union in this rewriter below and + // adhere to the FHIR spec for dates, we can remove this method. Our current SQL generator has difficulty with unions + // emitted by this rewriter due to FindRestrictingPredecessorTableExpressionIndex() has a catch all that returns // currentIndex - 1 for unions. if (!context && ContainsChain(expression)) {