From 35cc359627168c001ae3f3867d0ad8b40f018934 Mon Sep 17 00:00:00 2001 From: Jason Loux Date: Thu, 16 Jul 2026 18:14:15 -0400 Subject: [PATCH 1/2] First pass at external reference swapping --- .../brapi/v2/dao/BrAPIObservationUnitDAO.java | 93 +++++++++++++------ .../brapi/v2/services/BrAPITrialService.java | 20 +--- .../sample/SampleSubmissionImport.java | 4 +- .../processors/SampleSubmissionProcessor.java | 5 +- .../entity/PendingObservationUnit.java | 5 +- .../process/ImportTableProcess.java | 14 +-- .../service/ObservationUnitService.java | 25 ----- .../utilities/BrAPIDAOUtil.java | 8 ++ .../importer/ExperimentFileImportTest.java | 36 ++----- .../SampleSubmissionFileImportTest.java | 6 +- 10 files changed, 96 insertions(+), 120 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java index ca025c076..d56b21dfe 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java @@ -27,19 +27,20 @@ import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.JSON; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.client.v2.model.queryParams.phenotype.ObservationQueryParams; import org.brapi.client.v2.model.queryParams.phenotype.ObservationUnitQueryParams; import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi; -import org.brapi.client.v2.modules.phenotype.ObservationsApi; import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPIProgram; +import org.brapi.v2.model.core.request.BrAPITrialSearchRequest; import org.brapi.v2.model.germ.BrAPIGermplasm; -import org.brapi.v2.model.pheno.BrAPIObservation; import org.brapi.v2.model.pheno.BrAPIObservationTreatment; import org.brapi.v2.model.pheno.BrAPIObservationUnit; import org.brapi.v2.model.pheno.BrAPIObservationUnitLevelRelationship; import org.brapi.v2.model.pheno.request.BrAPIObservationUnitSearchRequest; +import org.brapi.v2.model.pheno.response.BrAPIObservationUnitListResponse; +import org.breedinginsight.brapi.v1.model.request.query.BrapiQuery; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; +import org.breedinginsight.brapi.v2.model.request.query.ExperimentQuery; import org.breedinginsight.brapi.v2.services.BrAPIGermplasmService; import org.breedinginsight.brapps.importer.daos.ImportDAO; import org.breedinginsight.brapps.importer.model.ImportUpload; @@ -167,12 +168,7 @@ public List getProgramObservationUnits(UUID programId) thr .findFirst() .orElseThrow(); - return getBrAPIObservationUnitsUsingBrAPIProgramId(program); - } - - private List getBrAPIObservationUnitsUsingBrAPIProgramId(Program program) throws ApiException { - - if (program == null || program.getId() == null) { + if (program.getId() == null) { throw new InternalServerException("BI-API Program or Program ID is null"); } @@ -185,12 +181,23 @@ private List getBrAPIObservationUnitsUsingBrAPIProgramId(P brapiProgramDbId = programDAO.getProgramBrAPI(program).getProgramDbId(); } - ObservationUnitQueryParams observationUnitQueryParams = - ObservationUnitQueryParams.builder() - .programDbId(brapiProgramDbId) - .pageSize(brapiMaxPageSize) - .page(0) - .build(); + ObservationUnitQueryParams observationUnitQueryParams = ObservationUnitQueryParams.builder() + .programDbId(brapiProgramDbId) + .build(); + + return getBrAPIObservationUnitsUsingQueryParams(observationUnitQueryParams, program); + } + + private List getBrAPIObservationUnitsUsingQueryParams(ObservationUnitQueryParams observationUnitQueryParams, + Program program) throws ApiException { + + if (observationUnitQueryParams.page() == null) { + observationUnitQueryParams.setPage(0); + } + + if (observationUnitQueryParams.pageSize() == null) { + observationUnitQueryParams.setPageSize(brapiMaxPageSize); + } ObservationUnitsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationUnitsApi.class); @@ -201,6 +208,22 @@ private List getBrAPIObservationUnitsUsingBrAPIProgramId(P return result; } + private List searchBrapiObservationUnits(BrAPIObservationUnitSearchRequest observationUnitSearchRequest, + Program program) throws ApiException { + ObservationUnitsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationUnitsApi.class); + + BrAPIObservationUnitListResponse brAPIResponse = + brAPIDAOUtil.simpleSearch( + api::searchObservationunitsPost, + observationUnitSearchRequest + ); + + List observationUnits = brAPIDAOUtil.getListResult(brAPIResponse); + processObservationUnits(program, observationUnits, false); + + return observationUnits; + } + /** * Create observation units with import progress. * Mutates brAPIObservationUnitList. @@ -243,22 +266,15 @@ public List createBrAPIObservationUnits(List getObservationUnitsById(Collection observationUnitExternalIds, Program program) throws ApiException { - if(observationUnitExternalIds.isEmpty()) { + public List getObservationUnitsById(Collection observationUnitDbIds, Program program) throws ApiException { + if (observationUnitDbIds.isEmpty()) { return Collections.emptyList(); } - return getProgramObservationUnits(program.getId()).stream() - .filter(ou -> { - var ouExRef = Utilities.getExternalReference(ou.getExternalReferences(), referenceSource, ExternalReferenceSource.OBSERVATION_UNITS).orElse(null); - if (ouExRef == null) { - return false; - } else { - return observationUnitExternalIds.contains(ouExRef.getReferenceId()); - } - - }).collect(Collectors.toList()); + // TODO: Optimize and change to search/get on observationUnitDbId. This will cause sample submission tests to fail until we have a better solution for exists checks for tabular errors. [BI-2987] + return getProgramObservationUnits(program.getId()).stream() + .filter(ou -> observationUnitDbIds.contains(ou.getObservationUnitDbId())) + .collect(Collectors.toList()); } public List getObservationUnitsForStudyDbId(@NotNull String studyDbId, Program program) throws ApiException { @@ -424,6 +440,27 @@ private List searchObservationUnitsAndProcess(BrAPIObserva return brapiObservationUnits; } + private BrAPIObservationUnitSearchRequest buildSearchRequest(Program program, List brapiOUDbIds) { + BrAPIProgram brAPIProgram = programDAO.getProgramBrAPI(program); + + if (brAPIProgram == null || brAPIProgram.getProgramDbId() == null) { + throw new InternalServerException(String.format("BI program with id [%s] not found in BrAPI db", program.getId())); + } + + BrAPIObservationUnitSearchRequest searchRequest = new BrAPIObservationUnitSearchRequest(); + + searchRequest.programDbIds(List.of(brAPIProgram.getProgramDbId())); + + if (brapiOUDbIds != null && !brapiOUDbIds.isEmpty()) { + searchRequest.setObservationUnitDbIds(brapiOUDbIds); + } + + // TODO: Utilize a search query for filtering/pagination on ous/datasets [BI-2961] + brAPIDAOUtil.setGenericSearchParameters(searchRequest, null); + + return searchRequest; + } + private void processObservationUnits(Program program, List brapiObservationUnits, boolean withGID) throws ApiException { HashMap germplasmByDbId = new HashMap<>(); diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index 4d1544940..e6f7c4bb0 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -198,6 +198,7 @@ public DownloadFile exportObservations( // add columns for requested dataset obsvars and timestamps log.debug(logHash + ": fetching experiment for export"); BrAPITrial experiment = getExperiment(program, experimentId); + // TODO: Replace this lookup and usages others with the brapiTrialDbId [BI-2933] String expExRefId = Utilities.getExternalReference(experiment.getExternalReferences(), this.referenceSource, ExternalReferenceSource.TRIALS) .map(BrAPIExternalReference::getReferenceId) .orElse(null); @@ -293,7 +294,7 @@ public DownloadFile exportObservations( // make export rows for OUs without observations if (rowByOUId.size() < ous.size()) { for (BrAPIObservationUnit ou: ous) { - String ouId = getOUId(ou); + String ouId = ou.getObservationUnitDbId(); // Map Observation Unit to the Study it belongs to. studyDbIdByOUId.put(ouId, ou.getStudyDbId()); if (!rowByOUId.containsKey(ouId)) { @@ -712,7 +713,7 @@ private void addBrAPIObsToRecords( // get observation unit for observation BrAPIObservationUnit ou = ouByOUDbId.get(obs.getObservationUnitDbId()); - String ouId = getOUId(ou); + String ouId = ou.getObservationUnitDbId(); // get observation variable for BrAPI observation Trait var = varByDbId.get(obs.getObservationVariableDbId()); @@ -733,14 +734,6 @@ private void addBrAPIObsToRecords( } } - private String getOUId(BrAPIObservationUnit ou) { - BrAPIExternalReference ouXref = Utilities.getExternalReference( - ou.getExternalReferences(), - String.format("%s/%s", referenceSource, ExternalReferenceSource.OBSERVATION_UNITS.getName())) - .orElseThrow(() -> new RuntimeException("observation unit id not found")); - return ouXref.getReferenceID(); - } - private String getStudyId(BrAPIStudy study) { // HACK: avoid null reference exceptions. if (study == null) return null; @@ -865,12 +858,7 @@ private Map createExportRow( boolean isSubEntity) throws ApiException, DoesNotExistException { HashMap row = new HashMap<>(); - // get OU id, germplasm, and study - BrAPIExternalReference ouXref = Utilities.getExternalReference( - ou.getExternalReferences(), - String.format("%s/%s", referenceSource, ExternalReferenceSource.OBSERVATION_UNITS.getName())) - .orElseThrow(() -> new RuntimeException("observation unit id not found")); - String ouId = ouXref.getReferenceID(); + String ouId = ou.getObservationUnitDbId(); BrAPIGermplasm germplasm = Optional.ofNullable(programGermplasmByDbId.get(ou.getGermplasmDbId())) .orElseThrow(() -> new DoesNotExistException("Germplasm not returned from BrAPI service")); BrAPIStudy study = Optional.ofNullable(studyByDbId.get(ou.getStudyDbId())) diff --git a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java index 6ae9fc2ff..8c18bfba5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java +++ b/src/main/java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java @@ -151,9 +151,7 @@ public BrAPISample constructBrAPISample(boolean commit, Program program, User us if (ou != null) { brAPISample .putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBS_UNIT_ID, - Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS)) - .get() - .getReferenceId()) + ou.getObservationUnitDbId()) .observationUnitDbId(ou.getObservationUnitDbId()); } diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java index 8ded93d89..d946d9175 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java @@ -107,11 +107,8 @@ public void getExistingBrapiData(List importRows, Program program) List observationUnits = observationUnitDAO.getObservationUnitsById(obsUnitIds, program); Set germDbIds = new HashSet<>(); - String ouRefSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS); observationUnits.forEach(ou -> { - observationUnitsById.put(Utilities.getExternalReference(ou.getExternalReferences(), ouRefSource) - .get() - .getReferenceId(), ou); + observationUnitsById.put(ou.getObservationUnitDbId(), ou); germDbIds.add(ou.getGermplasmDbId()); }); germplasm.stream() diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java index f9ec45a2b..589fc5f59 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/factory/entity/PendingObservationUnit.java @@ -213,8 +213,9 @@ public void initializeWorkflow(List members) { // Construct pending import objects from the units List> pendingUnits = members.stream().map(u -> (BrAPIObservationUnit) u).map(observationUnitService::constructPIOFromBrapiUnit).collect(Collectors.toList()); - // Construct a hashmap to look up the pending unit by ID - Map> pendingUnitById = observationUnitService.mapPendingUnitById(new ArrayList<>(pendingUnits)); + // Construct a hashmap to look up the pending unit by brapiOUDbId + Map> pendingUnitById = pendingUnits.stream() + .collect(Collectors.toMap(pio -> pio.getBrAPIObject().getObservationUnitDbId(), pio -> pio)); // Construct a hashmap to look up the pending unit by Study+Unit names with program keys removed Map> pendingUnitByNameNoScope = observationUnitService.mapPendingUnitByNameNoScope(new ArrayList<>(pendingUnits), importContext.getProgram()); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java index 9aa92fca7..9c0f304c5 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/appendoverwrite/middleware/process/ImportTableProcess.java @@ -26,7 +26,6 @@ import org.apache.commons.collections4.map.CaseInsensitiveMap; import org.apache.commons.lang3.StringUtils; import org.brapi.client.v2.model.exceptions.ApiException; -import org.brapi.v2.model.BrAPIExternalReference; import org.brapi.v2.model.core.BrAPIStudy; import org.brapi.v2.model.core.BrAPITrial; import org.brapi.v2.model.core.response.BrAPIListDetails; @@ -38,7 +37,6 @@ import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.response.ImportObjectState; import org.breedinginsight.brapps.importer.model.response.PendingImportObject; -import org.breedinginsight.brapps.importer.services.ExternalReferenceSource; import org.breedinginsight.brapps.importer.services.processors.experiment.ExperimentUtilities; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data.ProcessedDataFactory; import org.breedinginsight.brapps.importer.services.processors.experiment.appendoverwrite.factory.data.VisitedObservationData; @@ -158,14 +156,8 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext Set varBrapiDbIds = sortedTraits.stream().map(t->t.getObservationVariableDbId()).collect(Collectors.toSet()); List observations = brAPIObservationDAO.getObservationsByObservationUnitsAndVariables(ouBrapiDbIds, varBrapiDbIds, program); - // OU Exref Ids are what are displayed to users in the Obs Unit Id columns in experiments in DeltaBreed. - Map OuExRefIdByObsBrAPIDbId = new HashMap<>(); - - for (BrAPIObservation observation : observations) { - Optional ouExref = Utilities.getExternalReference(observation.getExternalReferences(), brapiReferenceSource, ExternalReferenceSource.OBSERVATION_UNITS); - ouExref.ifPresent(exRef -> OuExRefIdByObsBrAPIDbId.put(observation.getObservationDbId(), exRef.getReferenceId().toString())); - } - + // OU BrAPI DB Ids are what are displayed to users in the Obs Unit Id columns in experiments in DeltaBreed. + Map OuBrAPIDbIdByObsBrAPIDbId = observations.stream().collect(Collectors.toMap(BrAPIObservation::getObservationDbId, BrAPIObservation::getObservationUnitDbId)); // Construct helper lookup tables to use for hashing stored observation data Map variableNameByDbId = sortedTraits.stream().collect(Collectors.toMap(Trait::getObservationVariableDbId, Trait::getObservationVariableName)); Map studyNameByDbId = context.getAppendOverwriteWorkflowContext().getStudyByNameNoScope().values().stream() @@ -175,7 +167,7 @@ public AppendOverwriteMiddlewareContext process(AppendOverwriteMiddlewareContext // Hash stored observation data using a signature of unit, variable, and study names Map observationByObsHash = observations.stream().collect(Collectors.toMap(o->{ - return observationService.getObservationHash(OuExRefIdByObsBrAPIDbId.get(o.getObservationDbId()), + return observationService.getObservationHash(OuBrAPIDbIdByObsBrAPIDbId.get(o.getObservationDbId()), variableNameByDbId.get(o.getObservationVariableDbId()), studyNameByDbId.get(o.getStudyDbId())); }, o->o)); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java index 3c387f061..44267969e 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java @@ -114,31 +114,6 @@ public PendingImportObject constructPIOFromBrapiUnit(BrAPI return pio[0]; } - /** - * Maps pending observation units by their reference IDs. - * This function takes a list of pending import objects representing BrAPI observation units - * and constructs a map where the key is the external reference ID of the observation unit - * and the value is the pending import object itself. - * - * @param pios List of pending import objects for BrAPI observation units - * @return A map of pending observation units keyed by their external reference ID - */ - public Map> mapPendingUnitById(List> pios) { - Map> pendingUnitById = new HashMap<>(); - - // Construct the DeltaBreed observation unit source for external references - String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); - - for (PendingImportObject pio : pios) { - - // Get external reference for the Observation Unit - Optional xref = Utilities.getExternalReference(pio.getBrAPIObject().getExternalReferences(), deltaBreedOUSource); - pendingUnitById.put(xref.get().getReferenceId(),pio); - } - - return pendingUnitById; - } - /** * This method takes a list of PendingImportObject objects and a Program object as input * and maps the PendingImportObject objects by their observation unit name without the program scope. diff --git a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java index 0917998fb..0209d0afb 100644 --- a/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java +++ b/src/main/java/org/breedinginsight/utilities/BrAPIDAOUtil.java @@ -577,6 +577,14 @@ private BrAPISortBy constructSortBy(String sortOn, String sortOrder) { * - Pagination */ public void setGenericSearchParameters(T brapiSearchRequest, U biSearchQuery) { + + if (biSearchQuery == null) { + // If the search query is not available, assume maximum size fetch is required and break out of this method. + brapiSearchRequest.setPage(0); + brapiSearchRequest.setPageSize(brapiFetchPageSize); + return; + } + // Set SortBy List brAPISortBy = new ArrayList<>(); diff --git a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java index 07f5f583c..c8028150e 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/ExperimentFileImportTest.java @@ -929,8 +929,6 @@ public void importNewObsVarExistingOu() { BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByBrAPITrialExRefId(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); - Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); - assertTrue(ouIdXref.isPresent()); Map newObsVar = new HashMap<>(); newObsVar.put(Columns.GERMPLASM_GID, "1"); @@ -946,7 +944,7 @@ public void importNewObsVarExistingOu() { newObsVar.put(Columns.BLOCK_NUM, "1"); newObsVar.put(Columns.ROW, "1"); newObsVar.put(Columns.COLUMN, "1"); - newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); + newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ou.getObservationUnitDbId()); newObsVar.put(traits.get(1).getObservationVariableName(), null); JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits, true, false, null), null, true, client, program, mappingId, appendOverwriteWorkflowId); @@ -995,11 +993,9 @@ public void importNewObsVarByObsUnitId() { BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByBrAPITrialExRefId(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); - Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); - assertTrue(ouIdXref.isPresent()); Map newObsVar = new HashMap<>(); - newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); + newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ou.getObservationUnitDbId()); newObsVar.put(traits.get(1).getObservationVariableName(), null); JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits, true, false, null), null, true, client, program, mappingId, appendOverwriteWorkflowId); @@ -1049,8 +1045,6 @@ public void importNewObservationDataByObsUnitId(boolean commit) { BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByBrAPITrialExRefId(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); - Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); - assertTrue(ouIdXref.isPresent()); Map newObsVar = new HashMap<>(); newObsVar.put(Columns.GERMPLASM_GID, "1"); @@ -1066,7 +1060,7 @@ public void importNewObservationDataByObsUnitId(boolean commit) { newObsVar.put(Columns.BLOCK_NUM, "1"); newObsVar.put(Columns.ROW, "1"); newObsVar.put(Columns.COLUMN, "1"); - newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); + newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ou.getObservationUnitDbId()); newObsVar.put(traits.get(0).getObservationVariableName(), "1"); JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObsVar), traits, true, false, null), null, commit, client, program, mappingId, appendOverwriteWorkflowId); @@ -1125,8 +1119,6 @@ public void verifyBlankObsInOverwriteIsNoOp(boolean commit) { assertTrue(trialIdXref.isPresent()); BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByBrAPITrialExRefId(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); - Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); - assertTrue(ouIdXref.isPresent()); assertRowSaved(newExp, program, traits); @@ -1144,7 +1136,7 @@ public void verifyBlankObsInOverwriteIsNoOp(boolean commit) { newObsVar.put(Columns.BLOCK_NUM, "1"); newObsVar.put(Columns.ROW, "1"); newObsVar.put(Columns.COLUMN, "1"); - newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); // Indicates this is an overwrite. + newObsVar.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ou.getObservationUnitDbId()); // Indicates this is an overwrite. newObsVar.put(traits.get(0).getObservationVariableName(), ""); // Empty string should be no op. Map requestBody = new HashMap<>(); @@ -1195,8 +1187,6 @@ public void importNewObsExistingOu(boolean commit) { BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByBrAPITrialExRefId(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); - Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); - assertTrue(ouIdXref.isPresent()); Map newObservation = new HashMap<>(); newObservation.put(Columns.GERMPLASM_GID, "1"); @@ -1212,7 +1202,7 @@ public void importNewObsExistingOu(boolean commit) { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); + newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ou.getObservationUnitDbId()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); JsonObject result = importTestUtils.uploadAndFetchWorkflow(importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits, true, false, null), null, commit, client, program, mappingId, appendOverwriteWorkflowId); @@ -1263,8 +1253,6 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByBrAPITrialExRefId(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); - Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); - assertTrue(ouIdXref.isPresent()); Map newObservation = new HashMap<>(); newObservation.put(Columns.GERMPLASM_GID, "1"); @@ -1280,7 +1268,7 @@ public void verifyFailureImportNewObsExistingOuWithExistingObs(boolean commit) { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); + newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ou.getObservationUnitDbId()); newObservation.put(traits.get(0).getObservationVariableName(), "2"); uploadAndVerifyWorkflowFailureNonTabular(program, importTestUtils.writeExperimentDataToFile(List.of(newObservation), traits, true, false, null), traits.get(0).getObservationVariableName(), commit, newExperimentWorkflowId); @@ -1392,8 +1380,6 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByBrAPITrialExRefId(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); - Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); - assertTrue(ouIdXref.isPresent()); Map newObservation = new HashMap<>(); newObservation.put(Columns.GERMPLASM_GID, "1"); @@ -1409,7 +1395,7 @@ public void importNewObsAfterFirstExpWithObs(boolean commit) { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); + newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ou.getObservationUnitDbId()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); newObservation.put(traits.get(1).getObservationVariableName(), "2"); @@ -1469,8 +1455,6 @@ public void importNewObsAfterFirstExpWithObsAndTimestamps() { BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByBrAPITrialExRefId(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); - Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); - assertTrue(ouIdXref.isPresent()); Map newObservation = new HashMap<>(); newObservation.put(Columns.GERMPLASM_GID, "1"); @@ -1486,7 +1470,7 @@ public void importNewObsAfterFirstExpWithObsAndTimestamps() { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); + newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ou.getObservationUnitDbId()); newObservation.put(traits.get(0).getObservationVariableName(), "1"); newObservation.put(traits.get(1).getObservationVariableName(), "1"); @@ -1545,8 +1529,6 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { BrAPIStudy brAPIStudy = brAPIStudyDAO.getStudiesByBrAPITrialExRefId(UUID.fromString(trialIdXref.get().getReferenceId()), program).get(0); BrAPIObservationUnit ou = ouDAO.getObservationUnitsForStudyDbId(brAPIStudy.getStudyDbId(), program).get(0); - Optional ouIdXref = Utilities.getExternalReference(ou.getExternalReferences(), String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName())); - assertTrue(ouIdXref.isPresent()); assertRowSaved(newExp, program, traits); @@ -1564,7 +1546,7 @@ public void importNewObsAfterFirstExpWithObs_blank(boolean commit) { newObservation.put(Columns.BLOCK_NUM, "1"); newObservation.put(Columns.ROW, "1"); newObservation.put(Columns.COLUMN, "1"); - newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ouIdXref.get().getReferenceId()); + newObservation.put("Plot "+OBSERVATION_UNIT_ID_SUFFIX, ou.getObservationUnitDbId()); newObservation.put(traits.get(0).getObservationVariableName(), ""); // This blank value should not overwrite. newObservation.put(traits.get(1).getObservationVariableName(), "3"); // This valid value should overwrite. newObservation.put(traits.get(2).getObservationVariableName(), "4"); // This valid new observation should be appended. diff --git a/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java b/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java index 1705360f3..0f9feafb9 100644 --- a/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java +++ b/src/test/java/org/breedinginsight/brapps/importer/SampleSubmissionFileImportTest.java @@ -45,7 +45,6 @@ import org.breedinginsight.api.model.v1.request.SpeciesRequest; import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields; import org.breedinginsight.brapi.v2.dao.*; -import org.breedinginsight.brapi.v2.model.request.query.ExperimentQuery; import org.breedinginsight.brapi.v2.services.BrAPITrialService; import org.breedinginsight.brapps.importer.model.imports.experimentObservation.ExperimentObservation; import org.breedinginsight.brapps.importer.model.imports.sample.SampleSubmissionImport.Columns; @@ -231,8 +230,7 @@ public void importObsUnitIdSuccess() { BrAPITrial trial = brAPITrialDAO.getTrialById(program.getId(), UUID.fromString(experimentId)).get(); - List ous = ouDAO.getObservationUnitsForTrialDbId(program.getId(), trial.getTrialDbId()); - BrAPIExternalReference obsUnitId = Utilities.getExternalReference(ous.get(0).getExternalReferences(), Utilities.generateReferenceSource(BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS)).get(); + BrAPIObservationUnit ou = ouDAO.getObservationUnitsForTrialDbId(program.getId(), trial.getTrialDbId()).get(0); List> validFile = new ArrayList<>(); @@ -244,7 +242,7 @@ public void importObsUnitIdSuccess() { validRow.put(Columns.SPECIES, "TEST"); validRow.put(Columns.GERMPLASM_NAME, ""); validRow.put(Columns.GERMPLASM_GID, ""); - validRow.put(Columns.OBS_UNIT_ID, obsUnitId.getReferenceId()); + validRow.put(Columns.OBS_UNIT_ID, ou.getObservationUnitDbId()); validRow.put(Columns.TISSUE, "TEST"); validRow.put(Columns.COMMENTS, "Test sample"); validFile.add(validRow); From bdc4abd06ae167a410e93c3176a85536d1ee8a45 Mon Sep 17 00:00:00 2001 From: Jason Loux Date: Thu, 23 Jul 2026 14:18:59 -0400 Subject: [PATCH 2/2] Swap remaining essential ou exref usages to brapi ouDbId --- .../v2/BrAPIObservationUnitController.java | 6 ---- .../brapi/v2/dao/BrAPIObservationDAO.java | 9 ----- .../brapi/v2/dao/BrAPIObservationUnitDAO.java | 11 ++----- .../brapi/v2/services/BrAPITrialService.java | 3 +- .../service/ObservationUnitService.java | 33 +------------------ 5 files changed, 5 insertions(+), 57 deletions(-) diff --git a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitController.java b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitController.java index ca2b470f8..5df3da421 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitController.java +++ b/src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitController.java @@ -220,15 +220,9 @@ public HttpResponse observationunitsTableGet(@PathVariable("programId") UUID } private void setDbIds(BrAPIObservationUnit ou) { - ou.observationUnitDbId(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS)) - .orElseThrow(() -> new IllegalStateException("No BI external reference found")) - .getReferenceID()); ou.studyDbId(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.STUDIES)) .orElseThrow(() -> new IllegalStateException("No BI external reference found")) .getReferenceID()); - ou.trialDbId(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS)) - .orElseThrow(() -> new IllegalStateException("No BI external reference found")) - .getReferenceID()); ou.programDbId(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS)) .orElseThrow(() -> new IllegalStateException("No BI external reference found")) .getReferenceID()); diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java index dfd1fddc1..098cd2089 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java @@ -189,8 +189,6 @@ public List getObservationsByObservationUnits(Collection getObservationsByFilters(Program program, String studyDbId) throws ApiException, DoesNotExistException { String studySource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.STUDIES); - String observationUnitSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS); - String observationSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATIONS); // Get all observations for the program. Collection observations = getProgramObservations(program.getId()); @@ -205,16 +203,9 @@ public List getObservationsByFilters(Program program, String s Optional xref = Utilities.getExternalReference(o.getExternalReferences(), studySource); return xref.filter(brAPIExternalReference -> studyDbId.equals(brAPIExternalReference.getReferenceId())).isPresent(); }) - // Try to figure out why/how this translation is used. .peek(o -> { // Translate ObservationVariableDbId. o.setObservationVariableDbId(traitIdsByObservationVariableDbId.get(o.getObservationVariableDbId())); - // Translate ObservationUnitDbId. - o.setObservationUnitDbId(Utilities.getExternalReference(o.getExternalReferences(), observationUnitSource) - .orElseThrow(() -> new RuntimeException("observationUnit xref not found on observation")).getReferenceId()); - // Translate ObservationId. - o.setObservationDbId(Utilities.getExternalReference(o.getExternalReferences(), observationSource) - .orElseThrow(() -> new RuntimeException("observation xref not found on observation")).getReferenceId()); // Translate StudyDbId. o.setStudyDbId(Utilities.getExternalReference(o.getExternalReferences(), studySource) .orElseThrow(() -> new RuntimeException("study xref not found on observation")).getReferenceId()); diff --git a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java index d56b21dfe..fa50fc081 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java +++ b/src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java @@ -364,8 +364,7 @@ public List getObservationUnits(Program program, BrAPIObservationUnitLevelRelationship relationship = new BrAPIObservationUnitLevelRelationship(); AtomicBoolean relationshipFilter = new AtomicBoolean(false); - // TODO: Use observationUnitSearchRequest.setObservationUnitDbIds() instead of xrefs [BI-2914] - observationUnitId.ifPresent(ouId -> addXRefFilter(ouId, ExternalReferenceSource.OBSERVATION_UNITS, xrefIds, xrefSources)); + observationUnitId.ifPresent(dbid -> observationUnitSearchRequest.setObservationUnitDbIds(List.of(dbid))); observationUnitName.ifPresent(name -> observationUnitSearchRequest.setObservationUnitNames(List.of(Utilities.appendProgramKey(name, program.getKey())))); locationDbId.ifPresent(dbid -> observationUnitSearchRequest.setLocationDbIds(List.of(dbid))); seasonDbId.ifPresent(dbid -> observationUnitSearchRequest.setSeasonDbIds(List.of(dbid))); @@ -388,12 +387,8 @@ public List getObservationUnits(Program program, } return searchObservationUnitsAndProcess(observationUnitSearchRequest, program, true).stream().filter(ou -> { - //xref search does an OR, so we need to convert the searching for ouId/expId/envId to be an AND - boolean matches = observationUnitId.map(id -> id.equals(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS)) - .get() - .getReferenceId())) - .orElse(true); - matches = matches && environmentId.map(id -> id.equals(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.STUDIES)) + //xref search does an OR, so we need to convert the searching for expId/envId to be an AND + boolean matches = environmentId.map(id -> id.equals(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.STUDIES)) .get() .getReferenceId())) .orElse(true); diff --git a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java index e6f7c4bb0..ea2d19f9f 100644 --- a/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java +++ b/src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java @@ -681,8 +681,7 @@ public BrAPIObservationUnit createSubObservationUnit( // ObservationLevelRelationships for top-level Exp Unit linking. BrAPIObservationUnitLevelRelationship expUnitLevel = new BrAPIObservationUnitLevelRelationship(); expUnitLevel.setLevelNameDbId(expUnit.getObservationUnitPosition().getObservationLevel().getLevelNameDbId()); - String expUnitUUID = Utilities.getExternalReference(expUnit.getExternalReferences(), referenceSource, ExternalReferenceSource.OBSERVATION_UNITS).orElseThrow().getReferenceId(); - expUnitLevel.setLevelCode(Utilities.appendProgramKey(expUnitUUID, program.getKey(), seqVal)); + expUnitLevel.setLevelCode(Utilities.appendProgramKey(expUnit.getObservationUnitDbId(), program.getKey(), seqVal)); levelRelationships.add(expUnitLevel); position.setObservationLevelRelationships(levelRelationships); diff --git a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java index 44267969e..b219164f3 100644 --- a/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java +++ b/src/main/java/org/breedinginsight/brapps/importer/services/processors/experiment/service/ObservationUnitService.java @@ -70,12 +70,7 @@ public List getObservationUnitsById(Set obsUnitIds Set missingIds = new HashSet<>(obsUnitIds); // Calculate missing IDs based on retrieved BrAPI units - //missingIds.removeAll(brapiUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toSet())); - missingIds.removeAll(brapiUnits.stream() - .map(unit -> Utilities.getExternalReference(unit.getExternalReferences(), BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS)) - .filter(Optional::isPresent) - .map(Optional::get) - .map(BrAPIExternalReference::getReferenceId).collect(Collectors.toSet())); + missingIds.removeAll(brapiUnits.stream().map(BrAPIObservationUnit::getObservationUnitDbId).collect(Collectors.toSet())); // Throw exception with missing IDs information throw new EntityNotFoundException(missingIds); @@ -142,30 +137,4 @@ public Map> mapPendingUnitByNa return pendingUnitByNameNoScope; } - - /** - * Collects missing Observation Unit IDs from a set of reference IDs and a list of existing Observation Units. - * - * This function takes a Set of reference IDs and a List of existing Observation Units, filters out the Observation Units - * that have external references matching a specific source, and returns a List of missing Observation Unit IDs that are - * present in the reference IDs but not found in the existing Observation Units. - * - * @param referenceIds The Set of reference IDs representing all possible Observation Unit IDs to match against. - * @param existingUnits The List of existing Observation Units to compare against the reference IDs. - * @return A List of Observation Unit IDs that are missing from the existing Observation Units but present in the reference IDs. - */ - public List collectMissingOUIds(Set referenceIds, List existingUnits) { - List missingIds = new ArrayList<>(referenceIds); - - // Construct the DeltaBreed observation unit source for external references - String deltaBreedOUSource = String.format("%s/%s", BRAPI_REFERENCE_SOURCE, ExternalReferenceSource.OBSERVATION_UNITS.getName()); - - Set fetchedIds = existingUnits.stream() - .filter(unit ->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).isPresent()) - .map(unit->Utilities.getExternalReference(unit.getExternalReferences(), deltaBreedOUSource).get().getReferenceId()) - .collect(Collectors.toSet()); - missingIds.removeAll(fetchedIds); - - return missingIds; - } }