diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/ScalarTemporalEqualityRewriterTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/ScalarTemporalEqualityRewriterTests.cs index 31d8e9eeb6..493c3c4651 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/ScalarTemporalEqualityRewriterTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Search/Expressions/ScalarTemporalEqualityRewriterTests.cs @@ -81,13 +81,13 @@ private static SearchParameterInfo BuildReferenceParam() targetResourceTypes: new[] { "Patient" }); } - private static ChainedExpression BuildChainedExpression(Expression inner) + private static ChainedExpression BuildChainedExpression(Expression inner, bool reversed = false) { var expression = (ChainedExpression)RuntimeHelpers.GetUninitializedObject(typeof(ChainedExpression)); SetBackingField(expression, nameof(ChainedExpression.ResourceTypes), new[] { "Observation" }); SetBackingField(expression, nameof(ChainedExpression.ReferenceSearchParameter), BuildReferenceParam()); SetBackingField(expression, nameof(ChainedExpression.TargetResourceTypes), new[] { "Patient" }); - SetBackingField(expression, nameof(ChainedExpression.Reversed), false); + SetBackingField(expression, nameof(ChainedExpression.Reversed), reversed); SetBackingField(expression, nameof(ChainedExpression.Expression), inner); return expression; } @@ -138,6 +138,32 @@ public void GivenAllowListedBirthdateExactDayInChainedExpression_WhenRewritten_T Assert.Same(inner, result.Expression); } + [Fact] + public void GivenRootExpressionContainsReverseChainAndAllowListedBirthdateExactDay_WhenRewritten_ThenPassThroughBirthdate() + { + var birthdate = new SearchParameterExpression(BuildBirthdateParam(), EqualityPattern(StartOfDay, EndOfDay)); + var reverseChain = BuildChainedExpression(Expression.GreaterThanOrEqual(FieldName.DateTimeStart, null, StartOfDay), reversed: true); + var root = Expression.And(reverseChain, birthdate); + + var result = Assert.IsType(root.AcceptVisitor(ScalarTemporalEqualityRewriter.Instance)); + var rewrittenBirthdate = Assert.IsType(result.Expressions[1]); + + Assert.Same(birthdate, rewrittenBirthdate); + } + + [Fact] + public void GivenRootExpressionContainsForwardChainAndAllowListedBirthdateExactDay_WhenRewritten_ThenPassThroughBirthdate() + { + var birthdate = new SearchParameterExpression(BuildBirthdateParam(), EqualityPattern(StartOfDay, EndOfDay)); + var forwardChain = BuildChainedExpression(Expression.GreaterThanOrEqual(FieldName.DateTimeStart, null, StartOfDay), reversed: false); + var root = Expression.And(forwardChain, birthdate); + + var result = Assert.IsType(root.AcceptVisitor(ScalarTemporalEqualityRewriter.Instance)); + var rewrittenBirthdate = Assert.IsType(result.Expressions[1]); + + Assert.Same(birthdate, rewrittenBirthdate); + } + [Theory] [MemberData(nameof(NonRewritableExpressions))] public void GivenAllowListedBirthdateWithNonExactDayExpression_WhenRewritten_ThenPassThrough(Expression inner) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs index 8297ad6aa1..eaa74bfed5 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Search/Expressions/Visitors/ScalarTemporalEqualityRewriter.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; +using System.Linq; using Microsoft.Health.Fhir.Core.Features.Search.Expressions; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.SqlServer.Features.Search.Expressions; @@ -50,8 +51,23 @@ private enum Precision ExactDay, } + public override Expression VisitMultiary(MultiaryExpression expression, bool context) + { + // Don't run rewriter for chained or reverse chained expression. Once we remove the union in this rewriter below and + // adhere to the FHIR spec for dates, we can remove this method. Our current SQL generator has difficulty with unions + // emitted by this rewriter due to FindRestrictingPredecessorTableExpressionIndex() has a catch all that returns + // currentIndex - 1 for unions. + if (!context && ContainsChain(expression)) + { + return expression; + } + + return base.VisitMultiary(expression, context); + } + public override Expression VisitChained(ChainedExpression expression, bool context) { + // Logic here is the same as VisitMultiary. Expression visitedExpression = expression.Expression.AcceptVisitor(this, context: true); if (ReferenceEquals(visitedExpression, expression.Expression)) { @@ -208,5 +224,25 @@ private static bool IsEndLe(BinaryExpression be) => private static bool IsUtc(DateTimeOffset value) => value.Offset == TimeSpan.Zero; private static bool IsUtcMidnight(DateTimeOffset value) => IsUtc(value) && value.TimeOfDay == TimeSpan.Zero; + + private static bool ContainsChain(Expression expression) + { + if (expression is ChainedExpression) + { + return true; + } + + if (expression is MultiaryExpression multiary) + { + return multiary.Expressions.Any(ContainsChain); + } + + if (expression is UnionExpression union) + { + return union.Expressions.Any(ContainsChain); + } + + return false; + } } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs index 29133b4005..dd16fe0bf3 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/ChainingSearchTests.cs @@ -96,6 +96,28 @@ public async Task GivenAChainedSearchExpressionOverBirthdate_WhenSearched_ThenCo ValidateBundle(bundle, Fixture.SmithSnomedDiagnosticReport, Fixture.SmithLoincDiagnosticReport); } + [HttpIntegrationFixtureArgumentSets(DataStore.SqlServer, Format.Json)] + [Fact] + public async Task GivenAChainedSearchExpressionOverBirthdateMonthPrecision_WhenSearched_ThenCorrectBundleShouldBeReturned() + { + string query = $"_tag={Fixture.Tag}&subject:Patient.birthdate=1990-05"; + + Bundle bundle = await Client.SearchAsync(ResourceType.DiagnosticReport, query); + + ValidateBundle(bundle, Fixture.SmithSnomedDiagnosticReport, Fixture.SmithLoincDiagnosticReport, Fixture.TrumanSnomedDiagnosticReport, Fixture.TrumanLoincDiagnosticReport); + } + + [HttpIntegrationFixtureArgumentSets(DataStore.SqlServer, Format.Json)] + [Fact] + public async Task GivenAChainedSearchExpressionOverBirthdateDayWithAdditionalCriteriaOnTheSameTarget_WhenSearched_ThenCorrectBundleShouldBeReturned() + { + string query = $"_tag={Fixture.Tag}&subject:Patient.birthdate={Fixture.SmithPatientBirthDate}&subject:Patient.family=Smith"; + + Bundle bundle = await Client.SearchAsync(ResourceType.DiagnosticReport, query); + + ValidateBundle(bundle, Fixture.SmithSnomedDiagnosticReport, Fixture.SmithLoincDiagnosticReport); + } + [Fact] public async Task GivenAChainedSearchExpressionOverASimpleParameter_WhenSearchedWithPaging_ThenCorrectBundleShouldBeReturned() { @@ -144,6 +166,86 @@ public async Task GivenAReverseChainSearchExpressionOverASimpleParameter_WhenSea ValidateBundle(bundle, Fixture.TrumanPatient); } + [HttpIntegrationFixtureArgumentSets(DataStore.SqlServer, Format.Json)] + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task GivenAReverseChainSearchExpressionCombinedWithAnExactDayBirthdate_WhenSearched_ThenCorrectBundleShouldBeReturned(bool includeAccurateTotal) + { + string query = $"_tag={Fixture.Tag}&birthdate={Fixture.SmithPatientBirthDate}&_has:Observation:patient:code={Fixture.SnomedCode}"; + if (includeAccurateTotal) + { + query += "&_total=accurate"; + } + + Bundle bundle = await Client.SearchAsync(ResourceType.Patient, query); + + if (includeAccurateTotal) + { + Assert.Equal(1, bundle.Total.GetValueOrDefault()); + } + + ValidateBundle(bundle, Fixture.SmithPatient); + } + +#if !Stu3 + [HttpIntegrationFixtureArgumentSets(DataStore.SqlServer, Format.Json)] + [Fact] + public async Task GivenAReverseChainSearchExpressionOverImagingStudyStartedCombinedWithAnExactDayBirthdateAndTag_WhenSearchedWithPost_ThenCorrectBundleShouldBeReturned() + { + string tenantTagCode = Guid.NewGuid().ToString("N"); + var tenantMeta = new Meta + { + Tag = new List + { + new Coding(Fixture.TenantTagSystem, tenantTagCode), + }, + }; + + Patient imagingStudyExactDayPatient = (await Client.CreateAsync( + new Patient + { + Meta = tenantMeta, + BirthDate = Fixture.ImagingStudyPatientBirthDate, + })).Resource; + + Patient imagingStudyMonthPrecisionPatient = (await Client.CreateAsync( + new Patient + { + Meta = tenantMeta, + BirthDate = Fixture.ImagingStudyMonthPrecisionPatientBirthDate, + })).Resource; + + await Client.CreateAsync( + new ImagingStudy + { + Meta = tenantMeta, + Status = ImagingStudy.ImagingStudyStatus.Available, + Subject = new ResourceReference($"Patient/{imagingStudyExactDayPatient.Id}"), + Started = Fixture.ImagingStudyStarted, + }); + + await Client.CreateAsync( + new ImagingStudy + { + Meta = tenantMeta, + Status = ImagingStudy.ImagingStudyStatus.Available, + Subject = new ResourceReference($"Patient/{imagingStudyMonthPrecisionPatient.Id}"), + Started = Fixture.ImagingStudyStarted, + }); + + Bundle bundle = await Client.SearchPostAsync( + ResourceType.Patient.ToString(), + null, + default, + ("_has:ImagingStudy:patient:started", Fixture.ImagingStudyStarted), + ("birthdate", Fixture.ImagingStudyPatientBirthDate), + ("_tag", $"{Fixture.TenantTagSystem}|{tenantTagCode}")); + + ValidateBundle(bundle, imagingStudyExactDayPatient, imagingStudyMonthPrecisionPatient); + } +#endif + [Fact] public async Task GivenAReverseChainSearchExpressionWithMultipleTargetTypes_WhenSearched_ThenCorrectBundleShouldBeReturned() { @@ -400,6 +502,16 @@ public ClassFixture(DataStore dataStore, Format format, TestFhirServerFactory te public string OrganizationIdentifier { get; } = Guid.NewGuid().ToString(); +#if !Stu3 + public string TenantTagSystem { get; } = "urn:tenantId"; + + public string ImagingStudyPatientBirthDate { get; } = "2018-06-06"; + + public string ImagingStudyMonthPrecisionPatientBirthDate { get; } = "2018-06"; + + public string ImagingStudyStarted { get; } = "2018-02-02T05:00:00.000"; +#endif + public Patient SmithPatient { get; private set; } public DiagnosticReport SmithSnomedDiagnosticReport { get; private set; } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/DateSearchTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/DateSearchTests.cs index f4237aad2c..ddba09e3e2 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/DateSearchTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Search/DateSearchTests.cs @@ -298,10 +298,161 @@ public async Task GivenPatientsWithPartialBirthdates_WhenSearchedByEquality_Then } } + [HttpIntegrationFixtureArgumentSets(DataStore.SqlServer, Format.Json)] + [Fact] + [Trait(Traits.Priority, Priority.One)] + public async Task GivenPatientsWithPartialBirthdates_WhenSearchedByMultiValueOr_ThenUnionOfPerValueOverlapSetsIsReturned() + { + string tag = Guid.NewGuid().ToString(); + PartialBirthdateMatrix matrix = await CreatePartialBirthdateMatrixAsync(tag); + + Bundle bundle = await Client.SearchAsync(ResourceType.Patient, $"birthdate=2000-03-03,2001-12-31&_tag={tag}"); + + ValidateBundle(bundle, matrix.Year2000, matrix.March2000, matrix.March03, matrix.Year2001, matrix.December2001, matrix.December31_2001); + AssertNoDuplicateEntries(bundle); + } + + [HttpIntegrationFixtureArgumentSets(DataStore.SqlServer, Format.Json)] + [Fact] + [Trait(Traits.Priority, Priority.One)] + public async Task GivenPatientsWithPartialBirthdates_WhenSearchedByNotEquals_ThenOnlyTheExactDayValueIsExcluded() + { + string tag = Guid.NewGuid().ToString(); + PartialBirthdateMatrix matrix = await CreatePartialBirthdateMatrixAsync(tag); + + Bundle bundle = await Client.SearchAsync(ResourceType.Patient, $"birthdate=ne2000-03-03&_tag={tag}&_total=accurate&_count=100"); + + Assert.Equal(11, bundle.Total.GetValueOrDefault()); + ValidateBundle( + bundle, + matrix.Year2000, + matrix.March2000, + matrix.December31_1999, + matrix.April01_2000, + matrix.December2000, + matrix.March31_2000, + matrix.Year2001, + matrix.November30_2001, + matrix.December2001, + matrix.December31_2001, + matrix.January01_2002); + AssertNoDuplicateEntries(bundle); + } + + [HttpIntegrationFixtureArgumentSets(DataStore.SqlServer, Format.Json)] + [Fact] + [Trait(Traits.Priority, Priority.One)] + public async Task GivenPatientsWithPartialBirthdates_WhenDayEqualitySearchedWithTotal_ThenCountIsAccurateAndEntriesAreNotDuplicated() + { + string tag = Guid.NewGuid().ToString(); + PartialBirthdateMatrix matrix = await CreatePartialBirthdateMatrixAsync(tag); + + Bundle bundle = await Client.SearchAsync(ResourceType.Patient, $"birthdate=2000-03-03&_tag={tag}&_total=accurate"); + + Assert.Equal(3, bundle.Total.GetValueOrDefault()); + ValidateBundle(bundle, matrix.Year2000, matrix.March2000, matrix.March03); + AssertNoDuplicateEntries(bundle); + } + + [HttpIntegrationFixtureArgumentSets(DataStore.SqlServer, Format.Json)] + [Fact] + [Trait(Traits.Priority, Priority.One)] + public async Task GivenPatientsWithPartialBirthdates_WhenDayEqualitySearchedWithSort_ThenResultsAreOrderedByBirthdateWithoutDuplicates() + { + string tag = Guid.NewGuid().ToString(); + PartialBirthdateMatrix matrix = await CreatePartialBirthdateMatrixAsync(tag); + + Bundle bundle = await Client.SearchAsync(ResourceType.Patient, $"birthdate=2000-03-03&_sort=birthdate&_tag={tag}"); + + AssertNoDuplicateEntries(bundle); + Assert.Equal( + new[] { matrix.Year2000.Id, matrix.March2000.Id, matrix.March03.Id }, + bundle.Entry.Select(e => e.Resource.Id).ToArray()); + } + + [HttpIntegrationFixtureArgumentSets(DataStore.SqlServer, Format.Json)] + [Fact] + [Trait(Traits.Priority, Priority.One)] + public async Task GivenPatientsWithLeapDayBirthdates_WhenSearchedByDay_ThenCorrectBundleShouldBeReturned() + { + string tag = Guid.NewGuid().ToString(); + Patient[] patients = await Client.CreateResourcesAsync( + p => SetPatientBirthDate(p, "2020", tag), + p => SetPatientBirthDate(p, "2020-02", tag), + p => SetPatientBirthDate(p, "2020-02-28", tag), + p => SetPatientBirthDate(p, "2020-02-29", tag), + p => SetPatientBirthDate(p, "2020-03-01", tag)); + Patient year2020 = patients[0]; + Patient february2020 = patients[1]; + Patient february28 = patients[2]; + Patient february29 = patients[3]; + Patient march01 = patients[4]; + + Bundle leapDayBundle = await Client.SearchAsync(ResourceType.Patient, $"birthdate=2020-02-29&_tag={tag}"); + ValidateBundle(leapDayBundle, year2020, february2020, february29); + + Bundle feb28Bundle = await Client.SearchAsync(ResourceType.Patient, $"birthdate=2020-02-28&_tag={tag}"); + ValidateBundle(feb28Bundle, year2020, february2020, february28); + + Bundle march01Bundle = await Client.SearchAsync(ResourceType.Patient, $"birthdate=2020-03-01&_tag={tag}"); + ValidateBundle(march01Bundle, year2020, march01); + } + + private async System.Threading.Tasks.Task CreatePartialBirthdateMatrixAsync(string tag) + { + Patient[] patients = await Client.CreateResourcesAsync( + p => SetPatientBirthDate(p, "2000", tag), + p => SetPatientBirthDate(p, "2000-03", tag), + p => SetPatientBirthDate(p, "2000-03-03", tag), + p => SetPatientBirthDate(p, "1999-12-31", tag), + p => SetPatientBirthDate(p, "2000-04-01", tag), + p => SetPatientBirthDate(p, "2000-12", tag), + p => SetPatientBirthDate(p, "2000-03-31", tag), + p => SetPatientBirthDate(p, "2001", tag), + p => SetPatientBirthDate(p, "2001-11-30", tag), + p => SetPatientBirthDate(p, "2001-12", tag), + p => SetPatientBirthDate(p, "2001-12-31", tag), + p => SetPatientBirthDate(p, "2002-01-01", tag)); + + return new PartialBirthdateMatrix( + Year2000: patients[0], + March2000: patients[1], + March03: patients[2], + December31_1999: patients[3], + April01_2000: patients[4], + December2000: patients[5], + March31_2000: patients[6], + Year2001: patients[7], + November30_2001: patients[8], + December2001: patients[9], + December31_2001: patients[10], + January01_2002: patients[11]); + } + + private static void AssertNoDuplicateEntries(Bundle bundle) + { + List ids = bundle.Entry.Select(e => e.Resource.Id).ToList(); + Assert.Equal(ids.Count, ids.Distinct().Count()); + } + private static void SetPatientBirthDate(Patient patient, string birthDate, string tag) { patient.Meta = new Meta { Tag = new List { new Coding(null, tag) } }; patient.BirthDate = birthDate; } + + private sealed record PartialBirthdateMatrix( + Patient Year2000, + Patient March2000, + Patient March03, + Patient December31_1999, + Patient April01_2000, + Patient December2000, + Patient March31_2000, + Patient Year2001, + Patient November30_2001, + Patient December2001, + Patient December31_2001, + Patient January01_2002); } }