Skip to content

[BI-2914] Swap OU exref usages with brapi OU dbId #536

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jloux-brapi wants to merge 2 commits into
base: feature/BI-2913
Choose a base branch
from
Open

[BI-2914] Swap OU exref usages with brapi OU dbId #536

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
35cc359
First pass at external reference swapping
jloux-brapi Jul 16, 2026
bdc4abd
Swap remaining essential ou exref usages to brapi ouDbId
jloux-brapi Jul 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions src/main/java/org/breedinginsight/brapi/v2/BrAPIObservationUnitController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,15 +220,9 @@ public HttpResponse<?> observationunitsTableGet(@PathVariable("programId") UUID
}

private void setDbIds(BrAPIObservationUnit ou) {
ou.observationUnitDbId(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS))
.orElseThrow(() -> new IllegalStateException("No BI external reference found"))
.getReferenceID());
ou.studyDbId(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.STUDIES))
.orElseThrow(() -> new IllegalStateException("No BI external reference found"))
.getReferenceID());
ou.trialDbId(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.TRIALS))
.orElseThrow(() -> new IllegalStateException("No BI external reference found"))
.getReferenceID());
ou.programDbId(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS))
.orElseThrow(() -> new IllegalStateException("No BI external reference found"))
.getReferenceID());
Expand Down
9 changes: 0 additions & 9 deletions src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationDAO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,8 +189,6 @@ public List<BrAPIObservation> getObservationsByObservationUnits(Collection<Strin
public List<BrAPIObservation> getObservationsByFilters(Program program, String studyDbId) throws ApiException, DoesNotExistException {

String studySource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.STUDIES);
String observationUnitSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS);
String observationSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATIONS);

// Get all observations for the program.
Collection<BrAPIObservation> observations = getProgramObservations(program.getId());
Expand All @@ -205,16 +203,9 @@ public List<BrAPIObservation> getObservationsByFilters(Program program, String s
Optional<BrAPIExternalReference> xref = Utilities.getExternalReference(o.getExternalReferences(), studySource);
return xref.filter(brAPIExternalReference -> studyDbId.equals(brAPIExternalReference.getReferenceId())).isPresent();
})
// Try to figure out why/how this translation is used.
.peek(o -> {
// Translate ObservationVariableDbId.
o.setObservationVariableDbId(traitIdsByObservationVariableDbId.get(o.getObservationVariableDbId()));
// Translate ObservationUnitDbId.
o.setObservationUnitDbId(Utilities.getExternalReference(o.getExternalReferences(), observationUnitSource)
.orElseThrow(() -> new RuntimeException("observationUnit xref not found on observation")).getReferenceId());
// Translate ObservationId.
o.setObservationDbId(Utilities.getExternalReference(o.getExternalReferences(), observationSource)
.orElseThrow(() -> new RuntimeException("observation xref not found on observation")).getReferenceId());
// Translate StudyDbId.
o.setStudyDbId(Utilities.getExternalReference(o.getExternalReferences(), studySource)
.orElseThrow(() -> new RuntimeException("study xref not found on observation")).getReferenceId());
Expand Down
104 changes: 68 additions & 36 deletions src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,19 +27,20 @@
import org.apache.commons.lang3.StringUtils;
import org.brapi.client.v2.JSON;
import org.brapi.client.v2.model.exceptions.ApiException;
import org.brapi.client.v2.model.queryParams.phenotype.ObservationQueryParams;
import org.brapi.client.v2.model.queryParams.phenotype.ObservationUnitQueryParams;
import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi;
import org.brapi.client.v2.modules.phenotype.ObservationsApi;
import org.brapi.v2.model.BrAPIExternalReference;
import org.brapi.v2.model.core.BrAPIProgram;
import org.brapi.v2.model.core.request.BrAPITrialSearchRequest;
import org.brapi.v2.model.germ.BrAPIGermplasm;
import org.brapi.v2.model.pheno.BrAPIObservation;
import org.brapi.v2.model.pheno.BrAPIObservationTreatment;
import org.brapi.v2.model.pheno.BrAPIObservationUnit;
import org.brapi.v2.model.pheno.BrAPIObservationUnitLevelRelationship;
import org.brapi.v2.model.pheno.request.BrAPIObservationUnitSearchRequest;
import org.brapi.v2.model.pheno.response.BrAPIObservationUnitListResponse;
import org.breedinginsight.brapi.v1.model.request.query.BrapiQuery;
import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields;
import org.breedinginsight.brapi.v2.model.request.query.ExperimentQuery;
import org.breedinginsight.brapi.v2.services.BrAPIGermplasmService;
import org.breedinginsight.brapps.importer.daos.ImportDAO;
import org.breedinginsight.brapps.importer.model.ImportUpload;
Expand Down Expand Up @@ -167,12 +168,7 @@ public List<BrAPIObservationUnit> getProgramObservationUnits(UUID programId) thr
.findFirst()
.orElseThrow();

return getBrAPIObservationUnitsUsingBrAPIProgramId(program);
}

private List<BrAPIObservationUnit> getBrAPIObservationUnitsUsingBrAPIProgramId(Program program) throws ApiException {

if (program == null || program.getId() == null) {
if (program.getId() == null) {
throw new InternalServerException("BI-API Program or Program ID is null");
}

Expand All @@ -185,12 +181,23 @@ private List<BrAPIObservationUnit> getBrAPIObservationUnitsUsingBrAPIProgramId(P
brapiProgramDbId = programDAO.getProgramBrAPI(program).getProgramDbId();
}

ObservationUnitQueryParams observationUnitQueryParams =
ObservationUnitQueryParams.builder()
.programDbId(brapiProgramDbId)
.pageSize(brapiMaxPageSize)
.page(0)
.build();
ObservationUnitQueryParams observationUnitQueryParams = ObservationUnitQueryParams.builder()
.programDbId(brapiProgramDbId)
.build();

return getBrAPIObservationUnitsUsingQueryParams(observationUnitQueryParams, program);
}

private List<BrAPIObservationUnit> getBrAPIObservationUnitsUsingQueryParams(ObservationUnitQueryParams observationUnitQueryParams,
Program program) throws ApiException {

if (observationUnitQueryParams.page() == null) {
observationUnitQueryParams.setPage(0);
}

if (observationUnitQueryParams.pageSize() == null) {
observationUnitQueryParams.setPageSize(brapiMaxPageSize);
}

ObservationUnitsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationUnitsApi.class);

Expand All @@ -201,6 +208,22 @@ private List<BrAPIObservationUnit> getBrAPIObservationUnitsUsingBrAPIProgramId(P
return result;
}

private List<BrAPIObservationUnit> searchBrapiObservationUnits(BrAPIObservationUnitSearchRequest observationUnitSearchRequest,
Program program) throws ApiException {
ObservationUnitsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(program.getId()), ObservationUnitsApi.class);

BrAPIObservationUnitListResponse brAPIResponse =
brAPIDAOUtil.simpleSearch(
api::searchObservationunitsPost,
observationUnitSearchRequest
);

List<BrAPIObservationUnit> observationUnits = brAPIDAOUtil.getListResult(brAPIResponse);
processObservationUnits(program, observationUnits, false);

return observationUnits;
}

/**
* Create observation units with import progress.
* Mutates brAPIObservationUnitList.
Expand Down Expand Up @@ -243,22 +266,15 @@ public List<BrAPIObservationUnit> createBrAPIObservationUnits(List<BrAPIObservat
}
}

// TODO: Swap exrefs for dbIds [BI-2914]
public List<BrAPIObservationUnit> getObservationUnitsById(Collection<String> observationUnitExternalIds, Program program) throws ApiException {
if(observationUnitExternalIds.isEmpty()) {
public List<BrAPIObservationUnit> getObservationUnitsById(Collection<String> observationUnitDbIds, Program program) throws ApiException {
if (observationUnitDbIds.isEmpty()) {
return Collections.emptyList();
}
return getProgramObservationUnits(program.getId()).stream()
.filter(ou -> {
var ouExRef = Utilities.getExternalReference(ou.getExternalReferences(), referenceSource, ExternalReferenceSource.OBSERVATION_UNITS).orElse(null);

if (ouExRef == null) {
return false;
} else {
return observationUnitExternalIds.contains(ouExRef.getReferenceId());
}

}).collect(Collectors.toList());
// TODO: Optimize and change to search/get on observationUnitDbId. This will cause sample submission tests to fail until we have a better solution for exists checks for tabular errors. [BI-2987]
return getProgramObservationUnits(program.getId()).stream()
.filter(ou -> observationUnitDbIds.contains(ou.getObservationUnitDbId()))
.collect(Collectors.toList());
}

public List<BrAPIObservationUnit> getObservationUnitsForStudyDbId(@NotNull String studyDbId, Program program) throws ApiException {
Expand Down Expand Up @@ -348,8 +364,7 @@ public List<BrAPIObservationUnit> getObservationUnits(Program program,
BrAPIObservationUnitLevelRelationship relationship = new BrAPIObservationUnitLevelRelationship();
AtomicBoolean relationshipFilter = new AtomicBoolean(false);

// TODO: Use observationUnitSearchRequest.setObservationUnitDbIds() instead of xrefs [BI-2914]
observationUnitId.ifPresent(ouId -> addXRefFilter(ouId, ExternalReferenceSource.OBSERVATION_UNITS, xrefIds, xrefSources));
observationUnitId.ifPresent(dbid -> observationUnitSearchRequest.setObservationUnitDbIds(List.of(dbid)));
observationUnitName.ifPresent(name -> observationUnitSearchRequest.setObservationUnitNames(List.of(Utilities.appendProgramKey(name, program.getKey()))));
locationDbId.ifPresent(dbid -> observationUnitSearchRequest.setLocationDbIds(List.of(dbid)));
seasonDbId.ifPresent(dbid -> observationUnitSearchRequest.setSeasonDbIds(List.of(dbid)));
Expand All @@ -372,12 +387,8 @@ public List<BrAPIObservationUnit> getObservationUnits(Program program,
}

return searchObservationUnitsAndProcess(observationUnitSearchRequest, program, true).stream().filter(ou -> {
//xref search does an OR, so we need to convert the searching for ouId/expId/envId to be an AND
boolean matches = observationUnitId.map(id -> id.equals(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS))
.get()
.getReferenceId()))
.orElse(true);
matches = matches && environmentId.map(id -> id.equals(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.STUDIES))
//xref search does an OR, so we need to convert the searching for expId/envId to be an AND
boolean matches = environmentId.map(id -> id.equals(Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.STUDIES))
.get()
.getReferenceId()))
.orElse(true);
Expand Down Expand Up @@ -424,6 +435,27 @@ private List<BrAPIObservationUnit> searchObservationUnitsAndProcess(BrAPIObserva
return brapiObservationUnits;
}

private BrAPIObservationUnitSearchRequest buildSearchRequest(Program program, List<String> brapiOUDbIds) {
BrAPIProgram brAPIProgram = programDAO.getProgramBrAPI(program);

if (brAPIProgram == null || brAPIProgram.getProgramDbId() == null) {
throw new InternalServerException(String.format("BI program with id [%s] not found in BrAPI db", program.getId()));
}

BrAPIObservationUnitSearchRequest searchRequest = new BrAPIObservationUnitSearchRequest();

searchRequest.programDbIds(List.of(brAPIProgram.getProgramDbId()));

if (brapiOUDbIds != null && !brapiOUDbIds.isEmpty()) {
searchRequest.setObservationUnitDbIds(brapiOUDbIds);
}

// TODO: Utilize a search query for filtering/pagination on ous/datasets [BI-2961]
brAPIDAOUtil.setGenericSearchParameters(searchRequest, null);

return searchRequest;
}

private void processObservationUnits(Program program, List<BrAPIObservationUnit> brapiObservationUnits, boolean withGID) throws ApiException {

HashMap<String, BrAPIGermplasm> germplasmByDbId = new HashMap<>();
Expand Down
23 changes: 5 additions & 18 deletions src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,7 @@ public DownloadFile exportObservations(
// add columns for requested dataset obsvars and timestamps
log.debug(logHash + ": fetching experiment for export");
BrAPITrial experiment = getExperiment(program, experimentId);
// TODO: Replace this lookup and usages others with the brapiTrialDbId [BI-2933]
String expExRefId = Utilities.getExternalReference(experiment.getExternalReferences(), this.referenceSource, ExternalReferenceSource.TRIALS)
.map(BrAPIExternalReference::getReferenceId)
.orElse(null);
Expand Down Expand Up @@ -293,7 +294,7 @@ public DownloadFile exportObservations(
// make export rows for OUs without observations
if (rowByOUId.size() < ous.size()) {
for (BrAPIObservationUnit ou: ous) {
String ouId = getOUId(ou);
String ouId = ou.getObservationUnitDbId();
// Map Observation Unit to the Study it belongs to.
studyDbIdByOUId.put(ouId, ou.getStudyDbId());
if (!rowByOUId.containsKey(ouId)) {
Expand Down Expand Up @@ -680,8 +681,7 @@ public BrAPIObservationUnit createSubObservationUnit(
// ObservationLevelRelationships for top-level Exp Unit linking.
BrAPIObservationUnitLevelRelationship expUnitLevel = new BrAPIObservationUnitLevelRelationship();
expUnitLevel.setLevelNameDbId(expUnit.getObservationUnitPosition().getObservationLevel().getLevelNameDbId());
String expUnitUUID = Utilities.getExternalReference(expUnit.getExternalReferences(), referenceSource, ExternalReferenceSource.OBSERVATION_UNITS).orElseThrow().getReferenceId();
expUnitLevel.setLevelCode(Utilities.appendProgramKey(expUnitUUID, program.getKey(), seqVal));
expUnitLevel.setLevelCode(Utilities.appendProgramKey(expUnit.getObservationUnitDbId(), program.getKey(), seqVal));
levelRelationships.add(expUnitLevel);
position.setObservationLevelRelationships(levelRelationships);

Expand Down Expand Up @@ -712,7 +712,7 @@ private void addBrAPIObsToRecords(

// get observation unit for observation
BrAPIObservationUnit ou = ouByOUDbId.get(obs.getObservationUnitDbId());
String ouId = getOUId(ou);
String ouId = ou.getObservationUnitDbId();

// get observation variable for BrAPI observation
Trait var = varByDbId.get(obs.getObservationVariableDbId());
Expand All @@ -733,14 +733,6 @@ private void addBrAPIObsToRecords(
}
}

private String getOUId(BrAPIObservationUnit ou) {
BrAPIExternalReference ouXref = Utilities.getExternalReference(
ou.getExternalReferences(),
String.format("%s/%s", referenceSource, ExternalReferenceSource.OBSERVATION_UNITS.getName()))
.orElseThrow(() -> new RuntimeException("observation unit id not found"));
return ouXref.getReferenceID();
}

private String getStudyId(BrAPIStudy study) {
// HACK: avoid null reference exceptions.
if (study == null) return null;
Expand Down Expand Up @@ -865,12 +857,7 @@ private Map<String, Object> createExportRow(
boolean isSubEntity) throws ApiException, DoesNotExistException {
HashMap<String, Object> row = new HashMap<>();

// get OU id, germplasm, and study
BrAPIExternalReference ouXref = Utilities.getExternalReference(
ou.getExternalReferences(),
String.format("%s/%s", referenceSource, ExternalReferenceSource.OBSERVATION_UNITS.getName()))
.orElseThrow(() -> new RuntimeException("observation unit id not found"));
String ouId = ouXref.getReferenceID();
String ouId = ou.getObservationUnitDbId();
BrAPIGermplasm germplasm = Optional.ofNullable(programGermplasmByDbId.get(ou.getGermplasmDbId()))
.orElseThrow(() -> new DoesNotExistException("Germplasm not returned from BrAPI service"));
BrAPIStudy study = Optional.ofNullable(studyByDbId.get(ou.getStudyDbId()))
Expand Down
4 changes: 1 addition & 3 deletions ...java/org/breedinginsight/brapps/importer/model/imports/sample/SampleSubmissionImport.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -151,9 +151,7 @@ public BrAPISample constructBrAPISample(boolean commit, Program program, User us
if (ou != null) {
brAPISample
.putAdditionalInfoItem(BrAPIAdditionalInfoFields.OBS_UNIT_ID,
Utilities.getExternalReference(ou.getExternalReferences(), Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS))
.get()
.getReferenceId())
ou.getObservationUnitDbId())
.observationUnitDbId(ou.getObservationUnitDbId());
}

Expand Down
5 changes: 1 addition & 4 deletions ...va/org/breedinginsight/brapps/importer/services/processors/SampleSubmissionProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,11 +107,8 @@ public void getExistingBrapiData(List<BrAPIImport> importRows, Program program)

List<BrAPIObservationUnit> observationUnits = observationUnitDAO.getObservationUnitsById(obsUnitIds, program);
Set<String> germDbIds = new HashSet<>();
String ouRefSource = Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.OBSERVATION_UNITS);
observationUnits.forEach(ou -> {
observationUnitsById.put(Utilities.getExternalReference(ou.getExternalReferences(), ouRefSource)
.get()
.getReferenceId(), ou);
observationUnitsById.put(ou.getObservationUnitDbId(), ou);
germDbIds.add(ou.getGermplasmDbId());
});
germplasm.stream()
Expand Down
Loading
Loading