Skip to content

[BI-2916] Remove remaining CRUD operations and references to OU programCache #537

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jloux-brapi wants to merge 1 commit into
base: feature/BI-2914
Choose a base branch
from
+7 −86
Open

[BI-2916] Remove remaining CRUD operations and references to OU programCache #537

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
92 changes: 7 additions & 85 deletions src/main/java/org/breedinginsight/brapi/v2/dao/BrAPIObservationUnitDAO.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,36 +17,27 @@

package org.breedinginsight.brapi.v2.dao;

import com.google.gson.Gson;
import com.google.gson.JsonObject;
import com.google.gson.reflect.TypeToken;
import io.micronaut.context.annotation.Property;
import io.micronaut.http.server.exceptions.InternalServerException;
import io.micronaut.scheduling.annotation.Scheduled;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.brapi.client.v2.JSON;
import org.brapi.client.v2.model.exceptions.ApiException;
import org.brapi.client.v2.model.queryParams.phenotype.ObservationUnitQueryParams;
import org.brapi.client.v2.modules.phenotype.ObservationUnitsApi;
import org.brapi.v2.model.BrAPIExternalReference;
import org.brapi.v2.model.core.BrAPIProgram;
import org.brapi.v2.model.core.request.BrAPITrialSearchRequest;
import org.brapi.v2.model.germ.BrAPIGermplasm;
import org.brapi.v2.model.pheno.BrAPIObservationTreatment;
import org.brapi.v2.model.pheno.BrAPIObservationUnit;
import org.brapi.v2.model.pheno.BrAPIObservationUnitLevelRelationship;
import org.brapi.v2.model.pheno.request.BrAPIObservationUnitSearchRequest;
import org.brapi.v2.model.pheno.response.BrAPIObservationUnitListResponse;
import org.breedinginsight.brapi.v1.model.request.query.BrapiQuery;
import org.breedinginsight.brapi.v2.constants.BrAPIAdditionalInfoFields;
import org.breedinginsight.brapi.v2.model.request.query.ExperimentQuery;
import org.breedinginsight.brapi.v2.services.BrAPIGermplasmService;
import org.breedinginsight.brapps.importer.daos.ImportDAO;
import org.breedinginsight.brapps.importer.model.ImportUpload;
import org.breedinginsight.brapps.importer.services.ExternalReferenceSource;
import org.breedinginsight.daos.ProgramDAO;
import org.breedinginsight.daos.cache.ProgramCacheProvider;
import org.breedinginsight.model.Program;
import org.breedinginsight.services.ProgramService;
import org.breedinginsight.services.brapi.BrAPIEndpointProvider;
Expand All @@ -57,15 +48,13 @@
import javax.inject.Inject;
import javax.inject.Singleton;
import javax.validation.constraints.NotNull;
import java.lang.reflect.Type;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;

@Slf4j
@Singleton
public class BrAPIObservationUnitDAO extends BrAPICachedDAO<BrAPIObservationUnit> {
public class BrAPIObservationUnitDAO {
private final ProgramDAO programDAO;
private final ImportDAO importDAO;
private final BrAPIDAOUtil brAPIDAOUtil;
Expand All @@ -74,10 +63,6 @@ public class BrAPIObservationUnitDAO extends BrAPICachedDAO<BrAPIObservationUnit
private final BrAPIGermplasmService germplasmService;

private final String referenceSource;
private final boolean runScheduledTasks;

private final Gson gson = new JSON().getGson();
private final Type treatmentlistType = new TypeToken<ArrayList<BrAPIObservationTreatment>>(){}.getType();

private final int brapiMaxPageSize;

Expand All @@ -89,76 +74,17 @@ public BrAPIObservationUnitDAO(ProgramDAO programDAO,
BrAPIGermplasmService germplasmService,
ProgramService programService,
@Property(name = "brapi.server.reference-source") String referenceSource,
@Property(name = "micronaut.bi.api.run-scheduled-tasks") boolean runScheduledTasks,
ProgramCacheProvider programCacheProvider,
@Property(name = "brapi.cache.fetch-page-size") int brapiFetchPageSize) {
this.programDAO = programDAO;
this.importDAO = importDAO;
this.brAPIDAOUtil = brAPIDAOUtil;
this.brAPIEndpointProvider = brAPIEndpointProvider;
this.referenceSource = referenceSource;
this.runScheduledTasks = runScheduledTasks;
this.programService = programService;
this.germplasmService = germplasmService;
this.programCache = programCacheProvider.getProgramCache(this::fetchProgramObservationUnits, BrAPIObservationUnit.class);
this.brapiMaxPageSize = brapiFetchPageSize;
}

@Scheduled(initialDelay = "${startup.delay.observation_unit}")
public void setup() {
if(!runScheduledTasks) {
return;
}
// Populate observation unit cache for all programs on startup.
log.debug("populating observation unit cache");
List<Program> programs = programDAO.getActive();
if(programs != null) {
programCache.populate(programs.stream().map(Program::getId).collect(Collectors.toList()));
}
}

/**
* Fetch formatted observation units for this program.
*/
private Map<String, BrAPIObservationUnit> fetchProgramObservationUnits(UUID programId) throws ApiException {
ObservationUnitsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationUnitsApi.class);
// Get the program.
List<Program> programs = programDAO.get(programId);
if (programs.size() != 1) {
throw new InternalServerException("Program was not found for given id");
}
Program program = programs.get(0);

// Set query params and make call.
BrAPIObservationUnitSearchRequest observationUnitSearch = new BrAPIObservationUnitSearchRequest();
observationUnitSearch.externalReferenceIds(List.of(programId.toString()));
observationUnitSearch.externalReferenceSources(List.of(Utilities.generateReferenceSource(referenceSource, ExternalReferenceSource.PROGRAMS)));
return processObservationUnitsForCache(brAPIDAOUtil.search(
api::searchObservationunitsPost,
api::searchObservationunitsSearchResultsDbIdGet,
observationUnitSearch
), program, true);
}

/**
* Process a list of observation units for insertion into the cache.
*/
private Map<String, BrAPIObservationUnit> processObservationUnitsForCache(List<BrAPIObservationUnit> programObservationUnits, Program program, boolean withGID) throws ApiException {
// Process programObservationUnits in place (strip program key, etc.).
processObservationUnits(program, programObservationUnits, withGID);
// Build map.
Map<String, BrAPIObservationUnit> programObservationUnitsMap = new HashMap<>();
for (BrAPIObservationUnit observationUnit: programObservationUnits) {
BrAPIExternalReference xref = observationUnit
.getExternalReferences()
.stream()
.filter(reference -> String.format("%s/%s", referenceSource, ExternalReferenceSource.OBSERVATION_UNITS.getName()).equals(reference.getReferenceSource()))
.findFirst().orElseThrow(() -> new IllegalStateException("No BI external reference found"));
programObservationUnitsMap.put(xref.getReferenceId(), observationUnit);
}
return programObservationUnitsMap;
}

/**
* Get all observation units for a program from the cache.
*/
Expand Down Expand Up @@ -233,11 +159,9 @@ public List<BrAPIObservationUnit> createBrAPIObservationUnits(List<BrAPIObservat
ObservationUnitsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationUnitsApi.class);
try {
if (!brAPIObservationUnitList.isEmpty()) {
Callable<Map<String, BrAPIObservationUnit>> postFunction = () -> {
List<BrAPIObservationUnit> ous = brAPIDAOUtil.post(brAPIObservationUnitList, upload, api::observationunitsPost, importDAO::update);
return processObservationUnitsForCache(ous, program, false);
};
return programCache.post(programId, postFunction);
List<BrAPIObservationUnit> ous = brAPIDAOUtil.post(brAPIObservationUnitList, upload, api::observationunitsPost, importDAO::update);
processObservationUnits(program, ous, false);
return ous;
}
return new ArrayList<>();
} catch (Exception e) {
Expand All @@ -254,11 +178,9 @@ public List<BrAPIObservationUnit> createBrAPIObservationUnits(List<BrAPIObservat
ObservationUnitsApi api = brAPIEndpointProvider.get(programDAO.getCoreClient(programId), ObservationUnitsApi.class);
try {
if (!brAPIObservationUnitList.isEmpty()) {
Callable<Map<String, BrAPIObservationUnit>> postFunction = () -> {
List<BrAPIObservationUnit> ous = brAPIDAOUtil.post(brAPIObservationUnitList, api::observationunitsPost);
return processObservationUnitsForCache(ous, program, false);
};
return programCache.post(programId, postFunction);
List<BrAPIObservationUnit> ous = brAPIDAOUtil.post(brAPIObservationUnitList, api::observationunitsPost);
processObservationUnits(program, ous, false);
return ous;
}
return new ArrayList<>();
} catch (Exception e) {
Expand Down
1 change: 0 additions & 1 deletion src/main/java/org/breedinginsight/brapi/v2/services/BrAPITrialService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -828,7 +828,6 @@ public int deleteExperiment(Program program, UUID experimentId, boolean hard) th
}
// TODO: if performance is poor, implement more precise invalidation, possibly using hierarchical cache keys.
studyDAO.repopulateCache(program.getId());
observationUnitDAO.repopulateCache(program.getId());
}

// Successful or not, return the number of observations in this experiment.
Expand Down
Loading