diff --git a/bgee-core/src/main/java/org/bgee/model/CommonService.java b/bgee-core/src/main/java/org/bgee/model/CommonService.java index d8653de69..8ebcbba32 100644 --- a/bgee-core/src/main/java/org/bgee/model/CommonService.java +++ b/bgee-core/src/main/java/org/bgee/model/CommonService.java @@ -317,10 +317,6 @@ protected static GeneBioType mapGeneBioTypeTOToGeneBioType(GeneBioTypeTO geneBio protected static DataType convertDaoDataTypeToDataType(DAODataType dt) { log.traceEntry("{}", dt); switch(dt) { - case AFFYMETRIX: - return log.traceExit(DataType.AFFYMETRIX); - case EST: - return log.traceExit(DataType.EST); case IN_SITU: return log.traceExit(DataType.IN_SITU); case RNA_SEQ: @@ -342,10 +338,6 @@ public static DAODataType convertDataTypeToDAODataType(DataType dt) return log.traceExit((DAODataType) null); } switch(dt) { - case AFFYMETRIX: - return log.traceExit(DAODataType.AFFYMETRIX); - case EST: - return log.traceExit(DAODataType.EST); case IN_SITU: return log.traceExit(DAODataType.IN_SITU); case RNA_SEQ: @@ -402,6 +394,11 @@ protected static RawDataSex mapDAORawDataSexToRawDataSex(DAORawDataSex daoRawDat throw log.throwing(new IllegalStateException("Unrecognized DAORawDataSex: " + daoRawDataSex)); } } + public static DAOSex convertDAORawDataSexToDAOSex(DAORawDataSex daoRawDataSex) { + log.traceEntry("{}", daoRawDataSex); + return log.traceExit(convertSexToDAOSex(mapRawDataSexToSex( + mapDAORawDataSexToRawDataSex(daoRawDataSex)))); + } protected static Sex mapRawDataSexToSex(RawDataSex daoRawDataSex) { log.traceEntry("{}", daoRawDataSex); if (daoRawDataSex == null) { @@ -1054,7 +1051,7 @@ protected static DAOConditionFilter generateDAOConditionFilter(ConditionFilter c return log.traceExit(daoCondFilter); } - protected static Strain mapRawDataStrainToStrain(String strain) { + public static Strain mapRawDataStrainToStrain(String strain) { log.traceEntry("{}", strain); if (StringUtils.isBlank(strain)) { log.traceExit(); return null; diff --git a/bgee-core/src/main/java/org/bgee/model/ServiceFactory.java b/bgee-core/src/main/java/org/bgee/model/ServiceFactory.java index 0e22cd807..07d0a6a5f 100644 --- a/bgee-core/src/main/java/org/bgee/model/ServiceFactory.java +++ b/bgee-core/src/main/java/org/bgee/model/ServiceFactory.java @@ -13,6 +13,7 @@ import org.bgee.model.anatdev.multispemapping.DevStageSimilarityService; import org.bgee.model.dao.api.DAOManager; import org.bgee.model.expressiondata.call.CallService; +import org.bgee.model.expressiondata.call.ConditionGraphCacheService; import org.bgee.model.expressiondata.call.ConditionGraphService; import org.bgee.model.expressiondata.call.ConditionService; import org.bgee.model.expressiondata.call.ExpressionCallService; @@ -304,6 +305,14 @@ public ConditionService getConditionService() { return log.traceExit(new ConditionService(this)); } + /** + * @return A newly instantiated {@code ConditionGraphCacheService} + */ + public ConditionGraphCacheService getConditionGraphCacheService() { + log.traceEntry(); + return log.traceExit(new ConditionGraphCacheService(this)); + } + /** * @return A newly instantiated {@code ConditionGraphService} */ diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/baseelements/CallType.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/baseelements/CallType.java index 20004198f..5ad721040 100644 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/baseelements/CallType.java +++ b/bgee-core/src/main/java/org/bgee/model/expressiondata/baseelements/CallType.java @@ -45,7 +45,7 @@ public interface CallType { public static enum Expression implements CallType, BgeeEnumField { EXPRESSED(Collections.unmodifiableSet(EnumSet.allOf(DataType.class))), NOT_EXPRESSED(Collections.unmodifiableSet( - EnumSet.of(DataType.AFFYMETRIX, DataType.IN_SITU, DataType.RNA_SEQ))); + EnumSet.of(DataType.IN_SITU, DataType.RNA_SEQ))); private final static Logger log = LogManager.getLogger(Expression.class.getName()); /** @@ -167,7 +167,7 @@ public static enum DiffExpression implements CallType, BgeeEnumField { * @see #getAllowedDataTypes() */ private static final Set DIFF_EXPR_DATA_TYPES = - Collections.unmodifiableSet(EnumSet.of(DataType.AFFYMETRIX, DataType.RNA_SEQ)); + Collections.unmodifiableSet(EnumSet.of(DataType.RNA_SEQ)); @Override public Set getAllowedDataTypes() { diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/baseelements/DataType.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/baseelements/DataType.java index ccc122df5..c63a8e308 100644 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/baseelements/DataType.java +++ b/bgee-core/src/main/java/org/bgee/model/expressiondata/baseelements/DataType.java @@ -37,7 +37,6 @@ //TODO: why don't we have a "ALL" data type?? This would be much cleaner than having to provide "null" //everywhere... public enum DataType implements BgeeEnumField { - AFFYMETRIX("Affymetrix", true, null, true), EST("EST", false, null, false), IN_SITU("in situ hybridization", true, null, true), //Note: It is important to keep SC_RNA_SEQ before RNA_SEQ until we fix the issue of retrieving // experiment information consisting in different datatype (e.g both RNA_SEQ and SC_RNA_SEQ) as diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/call/CallMapping.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/call/CallMapping.java index 503865909..3594b3009 100644 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/call/CallMapping.java +++ b/bgee-core/src/main/java/org/bgee/model/expressiondata/call/CallMapping.java @@ -320,10 +320,6 @@ private static Set mapDAODataTypeToDataType(Set dts, mappedDataTypes = dts.stream() .map(dt -> { switch(dt) { - case AFFYMETRIX: - return log.traceExit(DataType.AFFYMETRIX); - case EST: - return log.traceExit(DataType.EST); case IN_SITU: return log.traceExit(DataType.IN_SITU); case RNA_SEQ: diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/call/CallService.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/call/CallService.java index 1880f5d62..bc5470f72 100644 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/call/CallService.java +++ b/bgee-core/src/main/java/org/bgee/model/expressiondata/call/CallService.java @@ -1880,10 +1880,6 @@ private static Set mapDAODataTypeToDataType(Set dts, mappedDataTypes = dts.stream() .map(dt -> { switch(dt) { - case AFFYMETRIX: - return log.traceExit(DataType.AFFYMETRIX); - case EST: - return log.traceExit(DataType.EST); case IN_SITU: return log.traceExit(DataType.IN_SITU); case RNA_SEQ: diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/call/CallServiceUtils.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/call/CallServiceUtils.java index c5b7de2e1..b438635bc 100644 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/call/CallServiceUtils.java +++ b/bgee-core/src/main/java/org/bgee/model/expressiondata/call/CallServiceUtils.java @@ -30,11 +30,11 @@ import org.bgee.model.dao.api.expressiondata.DAODataType; import org.bgee.model.dao.api.expressiondata.call.CallObservedDataDAOFilter2; import org.bgee.model.dao.api.expressiondata.call.ConditionDAO; +import org.bgee.model.dao.api.expressiondata.call.ConditionDAO.ConditionTO; +import org.bgee.model.dao.api.expressiondata.call.ConditionDAO.ConditionTOResultSet; import org.bgee.model.dao.api.expressiondata.call.DAOConditionFilter2; import org.bgee.model.dao.api.expressiondata.call.DAOFDRPValueFilter2; import org.bgee.model.dao.api.expressiondata.call.DAOPropagationState; -import org.bgee.model.dao.api.expressiondata.call.ConditionDAO.ConditionTO; -import org.bgee.model.dao.api.expressiondata.call.ConditionDAO.ConditionTOResultSet; import org.bgee.model.expressiondata.BaseConditionFilter2.FilterIds; import org.bgee.model.expressiondata.baseelements.ConditionParameter; import org.bgee.model.expressiondata.baseelements.DataType; @@ -301,7 +301,10 @@ public Set convertConditionFiltersToDAOConditionFilters( AnatEntityService anatEntityService, Set consideredSpeciesIds) { log.traceEntry("{}, {}, {}, {}", condFilters, ontService, anatEntityService, consideredSpeciesIds); if (condFilters == null || condFilters.isEmpty()) { - return log.traceExit(new HashSet<>()); + if(consideredSpeciesIds == null || consideredSpeciesIds.isEmpty()) { + return log.traceExit(new HashSet<>()); + } + return log.traceExit(Set.of(new DAOConditionFilter2(consideredSpeciesIds, null, null, null, null, null, null, null))); } //First, in order to load appropriately the ontologies, @@ -318,12 +321,17 @@ public Set convertConditionFiltersToDAOConditionFilters( ConditionParameter.ANAT_ENTITY_CELL_TYPE).getFilterIds(0); FilterIds cellTypeFilterIds = filter.getComposedFilterIds( ConditionParameter.ANAT_ENTITY_CELL_TYPE).getFilterIds(1); - if (anatEntityFilterIds != null && anatEntityFilterIds.isIncludeChildTerms()) { + //XXX: we used to consider isIncludeChildTerms == true detect terms to retrieve for anat. enttiy, + // Cell type and dev. stage. + // Since Bgee 16.0 we generate the calls on the fly. All descendant condition of a requested + // one have to be processed. Then we always need to retrieve all child terms of a requested term. + // The subseting of condition is done later once the propagation has been done. + if (anatEntityFilterIds != null) { anatEntityAndCellTypeIdsWithChildrenRequested.addAll(anatEntityFilterIds.getIds()); anatEntityAndCellTypeIdsWithChildrenRequested.addAll(anatEntityFilterIds.getExcludeTermsAndChildrenIds()); speciesIdsWithAnatCellChildrenRequested.add(filter.getSpeciesId()); } - if (cellTypeFilterIds != null && cellTypeFilterIds.isIncludeChildTerms()) { + if (cellTypeFilterIds != null) { anatEntityAndCellTypeIdsWithChildrenRequested.addAll(cellTypeFilterIds.getIds()); anatEntityAndCellTypeIdsWithChildrenRequested.addAll(cellTypeFilterIds.getExcludeTermsAndChildrenIds()); speciesIdsWithAnatCellChildrenRequested.add(filter.getSpeciesId()); @@ -332,7 +340,7 @@ public Set convertConditionFiltersToDAOConditionFilters( assert !filter.getComposedFilterIds(ConditionParameter.DEV_STAGE).isComposed(); FilterIds devStageFilterIds = filter.getComposedFilterIds( ConditionParameter.DEV_STAGE).getFilterIds(0); - if (devStageFilterIds != null && devStageFilterIds.isIncludeChildTerms()) { + if (devStageFilterIds != null) { devStageIdsWithChildrenRequested.addAll(devStageFilterIds.getIds()); devStageIdsWithChildrenRequested.addAll(devStageFilterIds.getExcludeTermsAndChildrenIds()); speciesIdsWithDevStageChildrenRequested.add(filter.getSpeciesId()); @@ -340,16 +348,28 @@ public Set convertConditionFiltersToDAOConditionFilters( } //Now we load the ontologies if needed + long t0 = System.currentTimeMillis(); MultiSpeciesOntology anatOntology = anatEntityAndCellTypeIdsWithChildrenRequested.isEmpty()? null: ontService.getAnatEntityOntology( speciesIdsWithAnatCellChildrenRequested, anatEntityAndCellTypeIdsWithChildrenRequested, EnumSet.of(RelationType.ISA_PARTOF), false, true); + log.debug("getAnatEntityOntology() completed in {} ms (requested {} terms in {} species)", + System.currentTimeMillis() - t0, + anatEntityAndCellTypeIdsWithChildrenRequested.size(), + speciesIdsWithAnatCellChildrenRequested.size()); + t0 = System.currentTimeMillis(); MultiSpeciesOntology stageOntology = devStageIdsWithChildrenRequested.isEmpty()? null: ontService.getDevStageOntology( speciesIdsWithDevStageChildrenRequested, devStageIdsWithChildrenRequested, false, true); + log.debug("getDevStageOntology() completed in {} ms (requested {} terms in {} species)", + System.currentTimeMillis() - t0, + devStageIdsWithChildrenRequested.size(), + speciesIdsWithDevStageChildrenRequested.size()); //There is no ontology for RawDataSex and RawDataStrain (String), really it's simply one root //with all other terms at the first level. + t0 = System.currentTimeMillis(); + t0 = System.currentTimeMillis(); Map> nonInformativePerSpeciesId = condFilters.stream() .filter(f -> f.isExcludeNonInformative()) .map(f -> f.getSpeciesId()).distinct() @@ -363,8 +383,11 @@ public Set convertConditionFiltersToDAOConditionFilters( .filter(aeid -> !aeid.equals(ConditionDAO.ANAT_ENTITY_ROOT_ID) && !aeid.equals(ConditionDAO.CELL_TYPE_ROOT_ID)) .collect(Collectors.toSet()))); + log.debug("loadNonInformativeAnatEntities() completed in {} ms ({} species with exclusion)", + System.currentTimeMillis() - t0, nonInformativePerSpeciesId.size()); //Now we have everything we need to create the DAO filters + t0 = System.currentTimeMillis(); Set daoCondFilters = new HashSet<>(); for (ConditionFilter2 filter: condFilters) { Set anatEntityIds = new HashSet<>(); @@ -380,15 +403,13 @@ public Set convertConditionFiltersToDAOConditionFilters( ConditionParameter.ANAT_ENTITY_CELL_TYPE).getFilterIds(1); if (anatEntityFilterIds != null) { anatEntityIds.addAll(anatEntityFilterIds.getIds()); - if (anatEntityFilterIds.isIncludeChildTerms()) { - anatEntityIds.addAll( - anatEntityFilterIds.getIds().stream() - .flatMap(id -> anatOntology.getDescendantIds( - id, false, Collections.singleton(filter.getSpeciesId())) - .stream()) - .collect(Collectors.toSet()) - ); - } + anatEntityIds.addAll( + anatEntityFilterIds.getIds().stream() + .flatMap(id -> anatOntology.getDescendantIds( + id, false, Collections.singleton(filter.getSpeciesId())) + .stream()) + .collect(Collectors.toSet()) + ); if (!anatEntityFilterIds.getExcludeTermsAndChildrenIds().isEmpty()) { Set anatEntityIdsToExclude = new HashSet<>(); anatEntityIdsToExclude.addAll(anatEntityFilterIds.getExcludeTermsAndChildrenIds()); @@ -398,7 +419,7 @@ public Set convertConditionFiltersToDAOConditionFilters( id, false, Collections.singleton(filter.getSpeciesId())) .stream()) .collect(Collectors.toSet()) - ); + ); anatEntityIdsToExclude.removeAll(anatEntityFilterIds.getNotToExcludeIds()); if (anatEntityIds.removeAll(anatEntityIdsToExclude) && anatEntityIds.isEmpty()) { throw log.throwing(new IllegalArgumentException( @@ -408,15 +429,13 @@ public Set convertConditionFiltersToDAOConditionFilters( } if (cellTypeFilterIds != null) { cellTypeIds.addAll(cellTypeFilterIds.getIds()); - if (cellTypeFilterIds.isIncludeChildTerms()) { - cellTypeIds.addAll( - cellTypeFilterIds.getIds().stream() - .flatMap(id -> anatOntology.getDescendantIds( - id, false, Collections.singleton(filter.getSpeciesId())) - .stream()) - .collect(Collectors.toSet()) - ); - } + cellTypeIds.addAll( + cellTypeFilterIds.getIds().stream() + .flatMap(id -> anatOntology.getDescendantIds( + id, false, Collections.singleton(filter.getSpeciesId())) + .stream()) + .collect(Collectors.toSet()) + ); if (!cellTypeFilterIds.getExcludeTermsAndChildrenIds().isEmpty()) { Set cellTypeIdsToExclude = new HashSet<>(); cellTypeIdsToExclude.addAll(cellTypeFilterIds.getExcludeTermsAndChildrenIds()); @@ -426,7 +445,7 @@ public Set convertConditionFiltersToDAOConditionFilters( id, false, Collections.singleton(filter.getSpeciesId())) .stream()) .collect(Collectors.toSet()) - ); + ); //we don't want to exclude the selected terms themselves cellTypeIdsToExclude.removeAll(cellTypeFilterIds.getNotToExcludeIds()); if (cellTypeIds.removeAll(cellTypeIdsToExclude) && cellTypeIds.isEmpty()) { @@ -442,7 +461,6 @@ public Set convertConditionFiltersToDAOConditionFilters( ConditionParameter.DEV_STAGE).getFilterIds(0); if (devStageFilterIds != null) { devStageIds.addAll(devStageFilterIds.getIds()); - if (devStageFilterIds.isIncludeChildTerms()) { devStageIds.addAll( devStageFilterIds.getIds().stream() .flatMap(id -> stageOntology.getDescendantIds( @@ -450,24 +468,6 @@ public Set convertConditionFiltersToDAOConditionFilters( .stream()) .collect(Collectors.toSet()) ); - } - if (!devStageFilterIds.getExcludeTermsAndChildrenIds().isEmpty()) { - Set devStageIdsToExclude = new HashSet<>(); - devStageIdsToExclude.addAll(devStageFilterIds.getExcludeTermsAndChildrenIds()); - devStageIdsToExclude.addAll( - devStageFilterIds.getExcludeTermsAndChildrenIds().stream() - .flatMap(id -> stageOntology.getDescendantIds( - id, false, Collections.singleton(filter.getSpeciesId())) - .stream()) - .collect(Collectors.toSet()) - ); - //we don't want to exclude the selected terms themselves - devStageIdsToExclude.removeAll(devStageFilterIds.getNotToExcludeIds()); - if (devStageIds.removeAll(devStageIdsToExclude) && devStageIds.isEmpty()) { - throw log.throwing(new IllegalArgumentException( - "No result should be retrieved because of dev. stage exclusion")); - } - } } //For now we consider there is no composition for sexes and strains @@ -504,6 +504,8 @@ public Set convertConditionFiltersToDAOConditionFilters( filter, condParamComb, daoCondFilter); daoCondFilters.add(daoCondFilter); } + log.debug("DAOConditionFilter2 construction loop completed in {} ms ({} filters built)", + System.currentTimeMillis() - t0, daoCondFilters.size()); //Now we filter the daoCondFilters: if one of them target a species with no additional parameters, //then we discard any other filter targeting the same species diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/call/ConditionGraph.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/call/ConditionGraph.java index fbf8ec020..ef2358a74 100644 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/call/ConditionGraph.java +++ b/bgee-core/src/main/java/org/bgee/model/expressiondata/call/ConditionGraph.java @@ -2,8 +2,12 @@ import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -69,6 +73,17 @@ public class ConditionGraph { * @see #isInferredDescendantConditions() */ private final boolean inferDescendantConditions; + /** + * Inverted indices mapping each parameter value to the {@code Condition}s that contain it. + * Built once at construction to replace the O(n) full-scan in {@link #getRelativeConditions} + * with O(|ancestor_values|) index lookups. Stores only references to already-existing + * {@code Condition} objects, so memory cost is ~5 * n * 8 bytes. + */ + private final Map> devStageIndex; + private final Map> anatEntityIndex; + private final Map> cellTypeIndex; + private final Map> sexIndex; + private final Map> strainIndex; /** * Constructor accepting all parameters. * @@ -186,6 +201,26 @@ public ConditionGraph(Collection conditions, this.sexOnt = sexOnt; this.inferAncestralConditions = inferAncestralConds; this.inferDescendantConditions = inferDescendantConds; + + // Build inverted indices: paramValue -> Set. + // Allows getRelativeConditions to avoid scanning all conditions for every query. + Map> devStageIdx = new HashMap<>(); + Map> anatEntityIdx = new HashMap<>(); + Map> cellTypeIdx = new HashMap<>(); + Map> sexIdx = new HashMap<>(); + Map> strainIdx = new HashMap<>(); + for (Condition c : this.conditions) { + devStageIdx.computeIfAbsent(c.getDevStage(), k -> new HashSet<>()).add(c); + anatEntityIdx.computeIfAbsent(c.getAnatEntity(), k -> new HashSet<>()).add(c); + cellTypeIdx.computeIfAbsent(c.getCellType(), k -> new HashSet<>()).add(c); + sexIdx.computeIfAbsent(c.getSex(), k -> new HashSet<>()).add(c); + strainIdx.computeIfAbsent(c.getStrain(), k -> new HashSet<>()).add(c); + } + this.devStageIndex = Collections.unmodifiableMap(devStageIdx); + this.anatEntityIndex = Collections.unmodifiableMap(anatEntityIdx); + this.cellTypeIndex = Collections.unmodifiableMap(cellTypeIdx); + this.sexIndex = Collections.unmodifiableMap(sexIdx); + this.strainIndex = Collections.unmodifiableMap(strainIdx); log.traceExit(); } @@ -354,6 +389,29 @@ public Set getAncestorConditions(Condition cond, boolean directRelOnl return log.traceExit(this.getRelativeConditions(cond, true, directRelOnly)); } + /** + * Same as {@link #getAncestorConditions(Condition, boolean)} but uses caller-supplied + * memoization caches to avoid recomputing the same ancestor sets repeatedly. + * This is useful when querying many conditions in parallel (e.g., from a + * {@code parallelStream}): the cache is created once by the caller, passed here, + * and shared across all calls so that each unique condition's ancestor set + * is computed at most once. + *

+ * The cache must be a {@code ConcurrentHashMap} since it is accessed concurrently. + * + * @param cond A {@code Condition} for which we want to retrieve ancestor {@code Condition}s. + * @param directRelOnly A {@code boolean} defining whether only direct parents should be returned. + * @param ancestorCache A {@code ConcurrentHashMap} used to memoize all-ancestor results + * (i.e., results for {@code directRelOnly=false}). + * @return A {@code Set} of ancestor {@code Condition}s. + */ + public Set getAncestorConditions(Condition cond, boolean directRelOnly, + ConcurrentHashMap> ancestorCache) + throws IllegalArgumentException { + log.traceEntry("{}, {}", cond, directRelOnly); + return log.traceExit(this.getRelativeConditions(cond, true, directRelOnly, ancestorCache)); + } + /** * Notes on implementation of this method: because we do not insert all possible conditions * in the database, but only those that have some parameters observed in annotations, @@ -396,48 +454,142 @@ private Set getRelativeConditions(Condition cond, boolean ancestors, log.trace("Sexes retrieved: {}", sexes); log.trace("Strains retrieved: {}", strains); - Set relativeConds = this.conditions.stream() - .filter(e -> !e.equals(cond) && - devStages.contains(e.getDevStage()) && - anatEntities.contains(e.getAnatEntity()) && - cellTypes.contains(e.getCellType()) && - sexes.contains(e.getSex()) && - strains.contains(e.getStrain())) - .collect(Collectors.toSet()); + // Use inverted indices to avoid a full O(n) scan over all conditions. + // Start with the union of conditions matching any ancestor dev. stage, then + // progressively narrow down by intersecting with the other dimensions. + Set relativeConds = unionFromIndex(devStages, this.devStageIndex); + intersectWithIndex(relativeConds, anatEntities, this.anatEntityIndex); + intersectWithIndex(relativeConds, cellTypes, this.cellTypeIndex); + intersectWithIndex(relativeConds, sexes, this.sexIndex); + intersectWithIndex(relativeConds, strains, this.strainIndex); + relativeConds.remove(cond); if (directRelOnly) { - Set directDevStages = getRelativeElements(this.devStageOnt, cond.getDevStage(), - ancestors, true); - Set directAnatEntities = getRelativeElements(this.anatEntityOnt, cond.getAnatEntity(), - ancestors, true); - Set directCellTypes = getRelativeElements(this.cellTypeOnt, cond.getCellType(), - ancestors, true); - Set directSexes = getRelativeElements(this.sexOnt, cond.getSex(), - ancestors, true); - Set directStrains = getRelativeElements(this.strainOnt, cond.getStrain(), - ancestors, true); - Set relativesOfRelatives = relativeConds.stream() - .flatMap(c -> this.getRelativeConditions(c, ancestors, false).stream()) + // Keep only the maximal elements of relativeConds (transitive reduction). + // Ancestor sets are precomputed once per unique dimension value — O(R) calls to + // ont.getAncestors() instead of O(R²) inside the pairwise dominance check. + Map> anatAncMap = buildDimAncestorMap( + relativeConds, Condition::getAnatEntity, this.anatEntityOnt); + Map> cellTypeAncMap = buildDimAncestorMap( + relativeConds, Condition::getCellType, this.cellTypeOnt); + Map> devStageAncMap = buildDimAncestorMap( + relativeConds, Condition::getDevStage, this.devStageOnt); + Map> sexAncMap = buildDimAncestorMap( + relativeConds, Condition::getSex, this.sexOnt); + Map> strainAncMap = buildDimAncestorMap( + relativeConds, Condition::getStrain, this.strainOnt); + final Set allRelatives = relativeConds; + relativeConds = relativeConds.stream() + .filter(e -> !isDominated(e, allRelatives, ancestors, + anatAncMap, cellTypeAncMap, devStageAncMap, sexAncMap, strainAncMap)) .collect(Collectors.toSet()); + } + log.trace("Done retrieving relative conditions for {}: {}", cond, relativeConds.size()); + return log.traceExit(relativeConds); + } + + /** + * Cache-aware overload of {@link #getRelativeConditions(Condition, boolean, boolean)}. + * When {@code directRelOnly=false}, checks {@code ancestorCache} before computing and stores + * the result afterwards. The recursive calls inside the {@code directRelOnly=true} path + * also use the cache, so each unique condition's full ancestor set is computed + * at most once across all calls sharing the same cache. + * + * @param ancestorCache caller-supplied cache for all-ancestor results ({@code directRelOnly=false}); + * may be {@code null} to skip memoization. + */ + private Set getRelativeConditions(Condition cond, boolean ancestors, boolean directRelOnly, + ConcurrentHashMap> ancestorCache) + throws IllegalArgumentException { + log.traceEntry("{}, {}, {}", cond, ancestors, directRelOnly); + // Fast path: return memoized result for the all-ancestors case. + if (!directRelOnly && ancestorCache != null) { + Set cached = ancestorCache.get(cond); + if (cached != null) { + return log.traceExit(cached); + } + } + log.trace("Start retrieving relative conditions for {}", cond); + if (!this.getConditions().contains(cond)) { + throw log.throwing(new IllegalArgumentException("The provided condition " + + "is not registered to this ConditionGraph: " + cond)); + } + + Set devStages = getRelativeElements(this.devStageOnt, cond.getDevStage(), ancestors, false); + Set anatEntities = getRelativeElements(this.anatEntityOnt, cond.getAnatEntity(), ancestors, false); + Set cellTypes = getRelativeElements(this.cellTypeOnt, cond.getCellType(), ancestors, false); + Set sexes = getRelativeElements(this.sexOnt, cond.getSex(), ancestors, false); + Set strains = getRelativeElements(this.strainOnt, cond.getStrain(), ancestors, false); + + log.trace("Stages retrieved: {}", devStages); + log.trace("Anat. entities retrieved: {}", anatEntities); + log.trace("Cell types retrieved: {}", cellTypes); + log.trace("Sexes retrieved: {}", sexes); + log.trace("Strains retrieved: {}", strains); + + Set relativeConds = unionFromIndex(devStages, this.devStageIndex); + intersectWithIndex(relativeConds, anatEntities, this.anatEntityIndex); + intersectWithIndex(relativeConds, cellTypes, this.cellTypeIndex); + intersectWithIndex(relativeConds, sexes, this.sexIndex); + intersectWithIndex(relativeConds, strains, this.strainIndex); + relativeConds.remove(cond); + if (directRelOnly) { + Map> anatAncMap = buildDimAncestorMap( + relativeConds, Condition::getAnatEntity, this.anatEntityOnt); + Map> cellTypeAncMap = buildDimAncestorMap( + relativeConds, Condition::getCellType, this.cellTypeOnt); + Map> devStageAncMap = buildDimAncestorMap( + relativeConds, Condition::getDevStage, this.devStageOnt); + Map> sexAncMap = buildDimAncestorMap( + relativeConds, Condition::getSex, this.sexOnt); + Map> strainAncMap = buildDimAncestorMap( + relativeConds, Condition::getStrain, this.strainOnt); + final Set allRelatives = relativeConds; relativeConds = relativeConds.stream() - .filter(e -> - //Either the relative conditions is really a direct relative - //by the relations in the ontologies - directDevStages.contains(e.getDevStage()) && - directAnatEntities.contains(e.getAnatEntity()) && - directCellTypes.contains(e.getCellType()) && - directSexes.contains(e.getSex()) && - directStrains.contains(e.getStrain()) || - //Or it is a disconnected relative (because of condition filtering), - //not reachable by any other relatives, so we consider it as "direct". - !relativesOfRelatives.contains(e)) + .filter(e -> !isDominated(e, allRelatives, ancestors, + anatAncMap, cellTypeAncMap, devStageAncMap, sexAncMap, strainAncMap)) .collect(Collectors.toSet()); } log.trace("Done retrieving relative conditions for {}: {}", cond, relativeConds.size()); + if (!directRelOnly && ancestorCache != null) { + Set result = Collections.unmodifiableSet(relativeConds); + ancestorCache.putIfAbsent(cond, result); + return log.traceExit(result); + } return log.traceExit(relativeConds); } + /** + * Returns the union of all {@code Condition}s in {@code index} whose key is in {@code params}. + */ + private static Set unionFromIndex(Set params, Map> index) { + Set result = new HashSet<>(); + for (T param : params) { + Set conds = index.get(param); + if (conds != null) { + result.addAll(conds); + } + } + return result; + } + + /** + * Retains in {@code candidates} only those {@code Condition}s whose parameter value + * (of the dimension represented by {@code index}) is in {@code params}. + */ + private static void intersectWithIndex(Set candidates, Set params, + Map> index) { + Set matching = new HashSet<>(); + for (T param : params) { + Set conds = index.get(param); + if (conds != null) { + matching.addAll(conds); + } + } + candidates.retainAll(matching); + } + private static & OntologyElement> Set getRelativeElements(Ontology ont, T startElement, boolean ancestors, boolean directRelsOnly) { log.traceEntry("{}, {}, {}", ont, startElement, directRelsOnly); @@ -456,6 +608,96 @@ private Set getRelativeConditions(Condition cond, boolean ancestors, ).collect(Collectors.toSet())); } + /** + * Precomputes a map from each unique dimension value found in {@code conds} to the full + * set of its ancestors in {@code ont}. Used to reduce {@code ont.getAncestors()} calls + * from O(R²) to O(R) in the pairwise dominance check, where R = {@code |conds|}. + */ + private static & OntologyElement> Map> + buildDimAncestorMap(Set conds, Function extractor, Ontology ont) { + Map> map = new HashMap<>(); + for (Condition c : conds) { + T val = extractor.apply(c); + if (val != null) { + map.computeIfAbsent(val, v -> ont == null ? Collections.emptySet() + : Collections.unmodifiableSet(ont.getAncestors(v, false))); + } + } + return map; + } + + /** + * Returns {@code true} if some condition {@code c} in {@code candidates} (other than + * {@code e} itself) lies strictly between the source condition and {@code e} in the + * condition partial order, making {@code e} non-maximal (dominated). + *

+ * Ancestor sets are supplied as precomputed maps (one per dimension) to avoid repeated + * {@code ont.getAncestors()} calls in the inner loop. + */ + private static boolean isDominated(Condition e, Set candidates, boolean ancestors, + Map> anatAncMap, + Map> cellTypeAncMap, + Map> devStageAncMap, + Map> sexAncMap, + Map> strainAncMap) { + for (Condition c : candidates) { + if (c.equals(e)) continue; + if (isCloserToCondInAllDimensions(c, e, ancestors, + anatAncMap, cellTypeAncMap, devStageAncMap, sexAncMap, strainAncMap)) return true; + } + return false; + } + + /** + * Returns {@code true} if {@code c} is strictly closer to the source condition than + * {@code e} in every dimension simultaneously. + */ + private static boolean isCloserToCondInAllDimensions(Condition c, Condition e, boolean ancestors, + Map> anatAncMap, + Map> cellTypeAncMap, + Map> devStageAncMap, + Map> sexAncMap, + Map> strainAncMap) { + if (c.equals(e)) return false; + // ancestors=true: c is closer iff e.dim ∈ ancestors(c.dim) → look up c's ancestor set + // ancestors=false: c is closer iff c.dim ∈ ancestors(e.dim) → look up e's ancestor set + return isDimCloserToCond(c.getAnatEntity(), e.getAnatEntity(), + ancestors ? anatAncMap.get(c.getAnatEntity()) : anatAncMap.get(e.getAnatEntity()), ancestors) + && isDimCloserToCond(c.getCellType(), e.getCellType(), + ancestors ? cellTypeAncMap.get(c.getCellType()) : cellTypeAncMap.get(e.getCellType()), ancestors) + && isDimCloserToCond(c.getDevStage(), e.getDevStage(), + ancestors ? devStageAncMap.get(c.getDevStage()) : devStageAncMap.get(e.getDevStage()), ancestors) + && isDimCloserToCond(c.getSex(), e.getSex(), + ancestors ? sexAncMap.get(c.getSex()) : sexAncMap.get(e.getSex()), ancestors) + && isDimCloserToCond(c.getStrain(), e.getStrain(), + ancestors ? strainAncMap.get(c.getStrain()) : strainAncMap.get(e.getStrain()), ancestors); + } + + /** + * Returns {@code true} if {@code cVal} is at least as close to the source condition as + * {@code eVal} in a single ontology dimension, using a precomputed ancestor set. + *

+ * Ancestor direction ({@code ancestors=true}): {@code precomputedAncestors} = ancestors of + * {@code cVal}; check that {@code eVal} is among them.
+ * Descendant direction ({@code ancestors=false}): {@code precomputedAncestors} = ancestors of + * {@code eVal}; check that {@code cVal} is among them. + */ + private static > boolean isDimCloserToCond( + T cVal, T eVal, Set precomputedAncestors, boolean ancestors) { + if (eVal == null && cVal == null) return true; + if (ancestors) { + if (eVal == null) return true; // null (most general) is ancestor of everything + if (cVal == null) return false; // null cannot be more specific than a real eVal + if (cVal.equals(eVal)) return true; + return precomputedAncestors != null && precomputedAncestors.contains(eVal); + } else { + if (cVal == null) return true; // null (most general) is always an ancestor + if (eVal == null) return false; + if (cVal.equals(eVal)) return true; + return precomputedAncestors != null && precomputedAncestors.contains(cVal); + } + } + /** * Get all the {@code Condition}s that are more precise than {@code cond}, * among the {@code Condition}s provided at instantiation. diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/call/ConditionGraphCacheService.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/call/ConditionGraphCacheService.java new file mode 100644 index 000000000..83cc5652d --- /dev/null +++ b/bgee-core/src/main/java/org/bgee/model/expressiondata/call/ConditionGraphCacheService.java @@ -0,0 +1,179 @@ +package org.bgee.model.expressiondata.call; + + + +import org.bgee.model.CommonService; +import org.bgee.model.ServiceFactory; +import org.bgee.model.dao.api.DAOManager; +import org.bgee.model.dao.api.expressiondata.call.ConditionDAO; +import org.bgee.model.dao.api.expressiondata.call.ConditionDAO.GlobalConditionToDirectAncestorTO; +import org.bgee.model.dao.api.expressiondata.call.ConditionDAO.GlobalConditionToDirectAncestorTOResultSet; + +import java.util.*; +import java.util.Map.Entry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.logging.Logger; + + +public final class ConditionGraphCacheService extends CommonService{ + private static final Logger log = Logger.getLogger(ConditionGraphCacheService.class.getName()); + + // Main cache: speciesId → (graph data) + private static final Map speciesGraphs = new ConcurrentHashMap<>(); + + + private static final int[] EMPTY_ARRAY = new int[0]; + + public ConditionGraphCacheService(ServiceFactory serviceFactory) { + super(serviceFactory); + } + + /** + * Load all species graphs at startup. + */ + public void loadAllSpeciesGraphs(List speciesIds) { + for (Integer speciesId : speciesIds) { + log.info("Loading condition graph for species " + speciesId); + ConditionGraphCache graph = buildConditionGraph(speciesId); + speciesGraphs.put(speciesId, graph); + log.info("Species " + speciesId + " graph loaded."); + } + } + + /** + * Retrieve cached graph for a species. + */ + public static ConditionGraphCache getGraph(int speciesId) { + return speciesGraphs.get(speciesId); + } + + /** + * Retrieve cached graph for a species, building and caching it on-demand if not yet loaded. + */ + public ConditionGraphCache getOrLoadGraph(int speciesId) { + return speciesGraphs.computeIfAbsent(speciesId, id -> { + log.warning("ConditionGraphCache not pre-loaded for species " + id + + " — building on-demand."); + return buildConditionGraph(id); + }); + } + + /** + * Build and cache the graph for one species. + */ + private ConditionGraphCache buildConditionGraph(Integer speciesId) { + DAOManager manager = this.getDaoManager(); + ConditionDAO conditionDAO = manager.getConditionDAO(); + + // Load all parent-child relations + log.info("start to retrieve relations"); + GlobalConditionToDirectAncestorTOResultSet relations = + conditionDAO.getGlobalConditionToDirectAncestor(speciesId); + + // Build adjacency maps (child → parents) and (parent -> children) + Map> directAncestorsSet = new HashMap<>(); + Map> directDescendantsSet = new HashMap<>(); + for (GlobalConditionToDirectAncestorTO rel : relations.getAllTOs()) { + directAncestorsSet.computeIfAbsent(rel.getSourceConditionId(), k -> new HashSet<>()) + .add(rel.getTargetConditionId()); + directDescendantsSet.computeIfAbsent(rel.getTargetConditionId(), k -> new HashSet<>()) + .add(rel.getSourceConditionId()); + directAncestorsSet.computeIfAbsent(rel.getTargetConditionId(), k -> new HashSet<>()); + directDescendantsSet.computeIfAbsent(rel.getSourceConditionId(), k -> new HashSet<>()); + } + + // Convert to primitive arrays for compact storage + Map directAncestorsMap = new HashMap<>(directAncestorsSet.size()); + for (Entry> e : directAncestorsSet.entrySet()) { + int[] arr = e.getValue().stream() + .mapToInt(Integer::intValue) + .toArray(); + directAncestorsMap.put(e.getKey(), arr); + } + Map directDescendantsMap = new HashMap<>(directDescendantsSet.size()); + for (Entry> e : directDescendantsSet.entrySet()) { + int[] arr = e.getValue().stream() + .mapToInt(Integer::intValue) + .toArray(); + directDescendantsMap.put(e.getKey(), arr); + } + + // Compute topological order (children before parents) + int[] topoOrder = computeTopologicalOrder(directAncestorsMap); + + return new ConditionGraphCache( + Collections.unmodifiableMap(directAncestorsMap), + Collections.unmodifiableMap(directDescendantsMap), + topoOrder + ); + } + + /** + * Compute topological order (children first). + */ + private static int[] computeTopologicalOrder(Map parentMap) { + Map indegree = new HashMap<>(); + for (Map.Entry e : parentMap.entrySet()) { + indegree.putIfAbsent(e.getKey(), 0); + for (int parent : e.getValue()) { + indegree.merge(parent, 1, Integer::sum); + } + } + + Deque queue = new ArrayDeque<>(); + for (Map.Entry e : indegree.entrySet()) { + if (e.getValue() == 0) queue.add(e.getKey()); + } + + int[] order = new int[parentMap.size()]; + int orderPos = 0; + while (!queue.isEmpty()) { + int node = queue.removeFirst(); + order[orderPos++] = node; + for (int parent : parentMap.getOrDefault(node, EMPTY_ARRAY)) { + int deg = indegree.get(parent) - 1; + indegree.put(parent, deg); + if (deg == 0) queue.add(parent); + } + } + + if (orderPos != parentMap.size()) { + throw new IllegalStateException("Cycle detected in condition graph!"); + } + + return order; + } + + /** + * Simple immutable holder for graph data. + */ + public static final class ConditionGraphCache { + + private final Map globalCondToDirectAncestors; + //XXX: Not useful for on-the-fly propagation but allows to easily filter calls + // as in the gene page. May be removed if the memory footprint is too high as it could + // be generated from globalCondToParents + private final Map globalCondToDirectDescendants; + private final int[] topoOrder; + + public ConditionGraphCache(Map globalCondToDirectAncestors, + Map globalCondToDirectDescendants, int[] topoOrder) { + this.globalCondToDirectAncestors = globalCondToDirectAncestors; + this.globalCondToDirectDescendants = globalCondToDirectDescendants; + this.topoOrder = topoOrder; + } + + public Map getGlobalCondToDirectAncestors() { + return globalCondToDirectAncestors; + } + + public Map getGlobalCondToDirectDescendants() { + return globalCondToDirectDescendants; + } + + public int[] getTopoOrder() { + return topoOrder; + } + + } +} diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/call/ConditionGraphService.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/call/ConditionGraphService.java index f507a0595..6014e1c92 100644 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/call/ConditionGraphService.java +++ b/bgee-core/src/main/java/org/bgee/model/expressiondata/call/ConditionGraphService.java @@ -414,7 +414,9 @@ private ConditionGraph loadConditionGraphFromMultipleArgs(Collection propStages.retainAll(devStagesUsedInAnnots); } } - + //to make sure we don't exclude the annotated term, we add it afterwards + propStages.add(cond.getDevStage()); + Set propAnatEntities = new HashSet<>(); if (anatEntityOntToUse != null && cond.getAnatEntityId() != null) { if (inferAncestralConds) { diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/call/ExpressionCallLoader.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/call/ExpressionCallLoader.java index c056f2485..79f5dccfb 100644 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/call/ExpressionCallLoader.java +++ b/bgee-core/src/main/java/org/bgee/model/expressiondata/call/ExpressionCallLoader.java @@ -1,8 +1,13 @@ package org.bgee.model.expressiondata.call; +import java.math.BigDecimal; +import java.math.MathContext; +import java.math.RoundingMode; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; @@ -12,6 +17,7 @@ import java.util.Optional; import java.util.Set; import java.util.function.Function; +import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -29,15 +35,24 @@ import org.bgee.model.anatdev.StrainService; import org.bgee.model.dao.api.DAO; import org.bgee.model.dao.api.expressiondata.DAODataType; +import org.bgee.model.dao.api.expressiondata.DAOObservedExpressionFilter; +import org.bgee.model.dao.api.expressiondata.ObservedExpressionDAO; +import org.bgee.model.dao.api.expressiondata.ObservedExpressionDAO.ObservedExpressionTO; import org.bgee.model.dao.api.expressiondata.call.ConditionDAO; import org.bgee.model.dao.api.expressiondata.call.ConditionDAO.ConditionTOResultSet; +import org.bgee.model.dao.api.expressiondata.call.ConditionDAO.RawConditionToSelfGlobalConditionTO; import org.bgee.model.dao.api.expressiondata.call.GlobalExpressionCallDAO; import org.bgee.model.dao.api.expressiondata.call.GlobalExpressionCallDAO.GlobalExpressionCallTO; import org.bgee.model.dao.api.expressiondata.call.GlobalExpressionCallDAO.GlobalExpressionCallTOResultSet; import org.bgee.model.dao.api.gene.GeneDAO; import org.bgee.model.expressiondata.baseelements.ConditionParameter; +import org.bgee.model.expressiondata.baseelements.DataType; +import org.bgee.model.expressiondata.baseelements.PropagationState; +import org.bgee.model.expressiondata.baseelements.SummaryCallType.ExpressionSummary; +import org.bgee.model.expressiondata.baseelements.SummaryQuality; import org.bgee.model.expressiondata.call.Call.ExpressionCall2; import org.bgee.model.expressiondata.call.CallFilter.ExpressionCallFilter2; +import org.bgee.model.expressiondata.call.ConditionGraphCacheService.ConditionGraphCache; import org.bgee.model.gene.Gene; import org.bgee.model.gene.GeneBioType; import org.bgee.model.species.Species; @@ -51,6 +66,10 @@ public class ExpressionCallLoader extends CommonService { * Value: 10,000. */ public static int LIMIT_MAX = 10000; + public final static BigDecimal EXPRESSION_SCORE_MAX_VALUE = new BigDecimal("100"); + private final static BigDecimal ZERO_BIGDECIMAL = new BigDecimal("0"); + private final static BigDecimal ABOVE_ZERO_BIGDECIMAL = new BigDecimal("0.000000000000000000000000000001"); + private final static BigDecimal MIN_FDR_BIGDECIMAL = new BigDecimal("0.00000000000001"); /** * An {@code int} that is the maximum number of elements * in {@link #conditionMap} and {@link #geneMap} before starting @@ -223,9 +242,546 @@ public List loadData(Long offset, Integer limit) { this.geneMap, this.conditionMap, callFilter, this.processedFilter.getMaxRankPerSpecies(), attrs)) .collect(Collectors.toList())); - } + //right now + public Map> loadDataOnTheFly() { + //If the DAOCallFilters are null (different from: not-null and empty) + //it means there was no matching conds and thus no result for sure + if (this.processedFilter.getDaoFilters() == null) { + return log.traceExit(new HashMap<>()); + } + + EnumSet daoCondParams = + this.utils.convertCondParamsToDAOCondParams( + this.processedFilter.getSourceFilter().getCondParamCombination()); + + EnumSet queriedDataTypes = this.processedFilter.getSourceFilter().getDataTypeFilters(); + EnumSet queriedDaoDataTypes = queriedDataTypes + .stream() + .map(dt -> convertDataTypeToDAODataType(dt)).collect(() -> + EnumSet.noneOf(DAODataType.class), + EnumSet::add, + EnumSet::addAll); + + //FIXME: at this point we assume a single species per request (validated by processExprCallPage) + int speciesId = this.processedFilter.getGeneSpeciesPart() + .getSpeciesMap().keySet().iterator().next(); + long startTimeCondGraph = System.currentTimeMillis(); + ConditionGraphCache condGraphCache = new ConditionGraphCacheService(this.getServiceFactory()) + .getOrLoadGraph(speciesId); + log.debug("Condition graph retrieved for species {} in {} ms", + speciesId, System.currentTimeMillis() - startTimeCondGraph); + + //2. retrieve rawconditionIds from the globalCond and the condition parameters. + // If conditionMap is empty (no condition filter provided), use all global conditions + // from the graph so that observed expressions are not missed. + // Snapshot filter-matching condition IDs before any ancestor expansion so that + // propagateCalls() can stop propagating upward at the filter boundary. + final Set filterConditionIds = conditionMap.isEmpty()? + Collections.emptySet(): new HashSet<>(conditionMap.keySet()); + Set globalCondIdsToQuery = conditionMap.isEmpty()? + condGraphCache.getGlobalCondToDirectAncestors().keySet(): + conditionMap.keySet(); + long startTimeRawConds = System.currentTimeMillis(); + List rawCondToSeflGlobalCondTOs = this.condDAO + .getRawConditionToSelfGlobalConditionFromGlobalConditionIds(globalCondIdsToQuery, + daoCondParams).getAllTOs(); + Map rawCondIdToGlobalCondIds = rawCondToSeflGlobalCondTOs.stream() + .collect(Collectors.toMap( + RawConditionToSelfGlobalConditionTO::getRawConditionId, + RawConditionToSelfGlobalConditionTO::getGlobalConditionId)); + log.debug("Raw condition IDs retrieved ({} entries) in {} ms", + rawCondIdToGlobalCondIds.size(), System.currentTimeMillis() - startTimeRawConds); + + if (rawCondIdToGlobalCondIds.isEmpty()) { + log.debug("No raw conditions matched the requested global conditions; returning empty result"); + return log.traceExit(new HashMap<>()); + } + + //3. retrieve the rawExpressionCalls filtering on rawConditionIds and datatypes + ObservedExpressionDAO obsExprDAO = this.getDaoManager().getObservedExpressionDAO(); + // generate the filter from all info we already have + //XXX: Could be created directly when instantiating the ExpressionCallLoader, Didn't want to touch the Loader while testing the new approach + DAOObservedExpressionFilter obsExprFilter = new DAOObservedExpressionFilter(this.geneMap.keySet(), + queriedDaoDataTypes, rawCondIdToGlobalCondIds.keySet()); + + // first key -> bgeeGeneId, 2nd key globalConditionId + long startTimeObsExpr = System.currentTimeMillis(); + List observedExpressionTOs = + obsExprDAO.getObservedExpression(obsExprFilter, null).stream().toList(); + Set unmatchedRawCondIds = observedExpressionTOs.stream() + .map(ObservedExpressionTO::getConditionId) + .filter(id -> !rawCondIdToGlobalCondIds.containsKey(id)) + .collect(Collectors.toSet()); + if (!unmatchedRawCondIds.isEmpty()) { + throw log.throwing(new IllegalStateException( + "Observed expression rows reference raw condition IDs missing from " + + "raw-to-global mapping: " + unmatchedRawCondIds)); + } + Map>> geneToGlobalCondIdToRawExpressionCall = + observedExpressionTOs.stream() + .collect(Collectors.groupingBy( + ObservedExpressionTO::getBgeeGeneId, + Collectors.groupingBy( + to -> rawCondIdToGlobalCondIds.get(to.getConditionId()), + Collectors.toSet() + ) + )); + log.debug("Observed expression calls retrieved ({} genes) in {} ms", + geneToGlobalCondIdToRawExpressionCall.size(), System.currentTimeMillis() - startTimeObsExpr); + + //5. use the topological order and the map> to propagate the calls. + // filterConditionIds restricts score computation to the queried conditions; + // propagation stops at the filter boundary so no wasteful scores are computed + // for ancestor conditions (e.g. "nervous system" when only "brain" was requested). + long startTimePropagation = System.currentTimeMillis(); + Map> propagatedExpressionCalls = propagateCalls( + geneToGlobalCondIdToRawExpressionCall, condGraphCache, filterConditionIds); + log.debug("Calls propagated ({} genes) in {} ms", + propagatedExpressionCalls.size(), System.currentTimeMillis() - startTimePropagation); + // filter condition needed for on-the-fly propagation but not requested by the condition filters + // happens when a condition parameter value is provided for anat. entity, cell type of dev. stage + // but child terms are not expected. + // ALSO filter on the requested summary call type (present/absent) if any. + //TODO: benchmark advantage of doing these steps during propagation. It would probably be harder to debug + // but would be faster + Predicate filter = + OTFExpressionCallFilterEngine.compile(this.processedFilter.getSourceFilter().getConditionFilters()); + Map> filtered = + propagatedExpressionCalls.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> e.getValue().stream() + .filter(filter) + .filter(this::matchesRequestedSummaryCallType) + .collect(Collectors.toSet()) + )); + //order result and filter present/absent if required. + Map> sortedCalls = + filtered.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + e -> e.getValue().stream() + .sorted(Comparator.comparing( + OTFExpressionCall::getExpressionScore, + Comparator.nullsLast(Comparator.reverseOrder()) + )) + .toList() + )); + + return log.traceExit(sortedCalls); + } + + private boolean matchesRequestedSummaryCallType(OTFExpressionCall call) { + log.traceEntry("{}", call); + + Map requestedSummaryCallTypeQualityFilter = + this.processedFilter.getSourceFilter().getSummaryCallTypeQualityFilter(); + if (requestedSummaryCallTypeQualityFilter == null || + requestedSummaryCallTypeQualityFilter.isEmpty() || + requestedSummaryCallTypeQualityFilter.equals(ExpressionCallFilter2.ALL_CALLS)) { + return log.traceExit(true); + } + + BigDecimal allPValue = call.getAllDataTypePValue(); + if (allPValue == null) { + return log.traceExit(false); + } + + boolean match = requestedSummaryCallTypeQualityFilter.entrySet().stream() + .anyMatch(e -> { + ExpressionSummary summary = e.getKey(); + SummaryQuality quality = e.getValue(); + + if (ExpressionSummary.EXPRESSED.equals(summary)) { + if (SummaryQuality.GOLD.equals(quality)) { + return allPValue.compareTo(this.processedFilter.getPresentHighThreshold()) <= 0; + } + // SILVER and BRONZE: present in self+descendant. + if (allPValue.compareTo(this.processedFilter.getPresentLowThreshold()) <= 0) { + return true; + } + // BRONZE also accepts calls present in at least one descendant condition. + return SummaryQuality.BRONZE.equals(quality) + && call.getBestDirectDescendantAllDataTypePValue() != null + && call.getBestDirectDescendantAllDataTypePValue() + .compareTo(this.processedFilter.getPresentLowThreshold()) <= 0; + } + if (ExpressionSummary.NOT_EXPRESSED.equals(summary)) { + BigDecimal absentThreshold = SummaryQuality.GOLD.equals(quality)? + this.processedFilter.getAbsentHighThreshold(): + this.processedFilter.getAbsentLowThreshold(); + + // Must be absent in self+descendant considering all requested data types. + if (allPValue.compareTo(absentThreshold) <= 0) { + return false; + } + // Must have observed data in self condition. + if (!Boolean.TRUE.equals(call.getDataPropagation().isIncludingObservedData())) { + return false; + } + // Must have no PRESENT evidence in descendants (all requested data types). + if (call.getBestDirectDescendantAllDataTypePValue() != null && + call.getBestDirectDescendantAllDataTypePValue() + .compareTo(this.processedFilter.getPresentLowThreshold()) <= 0) { + return false; + } + + if (SummaryQuality.BRONZE.equals(quality)) { + return true; + } + + // SILVER/GOLD: same constraints must hold on trusted data types. + BigDecimal trustedPValue = call.getTrustedDataTypePValue(); + if (trustedPValue == null || trustedPValue.compareTo(absentThreshold) <= 0) { + return false; + } + return call.getBestDirectDescendantTrustedDataTypePValue() == null || + call.getBestDirectDescendantTrustedDataTypePValue() + .compareTo(this.processedFilter.getPresentLowThreshold()) > 0; + } + return false; + }); + + return log.traceExit(match); + } + + private Map> propagateCalls( + Map>> geneToGlobalCondIdToRawExpressionCall, + ConditionGraphCache condGraphCache, Set filterConditionIds) { + log.traceEntry("{}, {}, {}", geneToGlobalCondIdToRawExpressionCall, condGraphCache, filterConditionIds); + + Map parentCondIds = condGraphCache.getGlobalCondToDirectAncestors(); + Map descendantCondIds = condGraphCache.getGlobalCondToDirectDescendants(); + int[] topoOrder = condGraphCache.getTopoOrder(); + + Map> geneToExpressionCall = new HashMap<>(); + + // For each gene independently + for (Map.Entry>> geneEntry : + geneToGlobalCondIdToRawExpressionCall.entrySet()) { + long startTimeGene = System.currentTimeMillis(); + + //key globalCondId value ExpressionCall to retrieve at the end + Map globalCondIdToExpressionCall = new HashMap<>(); + + //the Id of the gene for which we propagate calls + Integer geneId = geneEntry.getKey(); + //Map that contains all self observed expression from the database per condition + Map> globalCondIdToObservedExpressionTOs = geneEntry.getValue(); + + //Init the Set of conditions to parse. Once empty, propagation is over. + Set conditionToParse = new HashSet<>(globalCondIdToObservedExpressionTOs.keySet()); +// // Pre-load the initial leaf conditions into conditionMap so that +// // generateOTFExpressionCall can look them up even when conditionMap was +// // seeded from an empty condition filter. +// updateConditionMap(conditionToParse); + + Set parsedConditions = new HashSet<>(); + // Collected during propagation; removed after the loop so that every ancestor + // can still read a child's entry while computing its own call. + // Transitive redundancy (A==B score/pval, B==C) is handled correctly: B stays + // in the map when C is processed, so C is also detected and collected. + Set redundantCondIds = new HashSet<>(); + + // Topological propagation (child → parents) + redundancy detection + for (int condId : topoOrder) { + if (conditionToParse.isEmpty()) { + break; + } + if (conditionToParse.contains(condId)) { + conditionToParse.remove(condId); + // Guard against re-processing (indicates a cycle) and lazily load + // Condition2 objects for parent conditions not yet in conditionMap. + // Both are done per-parent to avoid a separate full-set scan. + // Only propagate to parents that are within the filter. When a filter + // is active, parents outside it (e.g. "nervous system" when brain was + // queried) are skipped: their scores are never computed and they are + // never added to conditionToParse, so the loop terminates early. + Set parentIdSet = new HashSet<>(); + for (int parentId : parentCondIds.get(condId)) { + if (filterConditionIds.isEmpty() || filterConditionIds.contains(parentId)) { + if (parsedConditions.contains(parentId)) { + throw log.throwing(new IllegalStateException( + "Condition " + parentId + " already parsed — cycle or propagation bug")); + } + parentIdSet.add(parentId); + conditionToParse.add(parentId); + } + } +// updateConditionMap(parentIdSet); + + // retrieve self expression + Set selfExpressionTOs = globalCondIdToObservedExpressionTOs.get(condId); + int[] children = descendantCondIds.get(condId); + List descendantCalls = new ArrayList<>(children == null ? 0 : children.length); + if (children != null) { + for (int childId: children) { + if (!parsedConditions.contains(childId)) { + continue; + } + OTFExpressionCall childCall = globalCondIdToExpressionCall.get(childId); + if (childCall != null) { + descendantCalls.add(childCall); + } + } + } + OTFExpressionCall expressionCall = generateOTFExpressionCall( + geneMap.get(geneId), conditionMap.get(condId), + selfExpressionTOs, descendantCalls); + globalCondIdToExpressionCall.put(condId, expressionCall); + + // Check immediately whether condId is redundant by comparing it to the best + // descendant score/p-value already computed in expressionCall. + // We only collect here; actual removal is deferred to after the loop. + if (expressionCall.getExpressionScore() != null + && expressionCall.getBestDirectDescendantExpressionScore() != null + && expressionCall.getExpressionScore().compareTo( + expressionCall.getBestDirectDescendantExpressionScore()) == 0) { + redundantCondIds.add(condId); + } + } + parsedConditions.add(condId); + } + + if (!redundantCondIds.isEmpty()) { + log.debug("Pruning {} redundant ancestor condition(s) for gene {} " + + "(same score and p-value as a descendant)", redundantCondIds.size(), geneId); + globalCondIdToExpressionCall.keySet().removeAll(redundantCondIds); + } + + geneToExpressionCall.put(geneMap.get(geneId), + globalCondIdToExpressionCall.values().stream().collect(Collectors.toSet())); + log.debug("Propagation for gene {} completed in {} ms, {} calls generated", + geneId, System.currentTimeMillis() - startTimeGene, + globalCondIdToExpressionCall.size()); + } + + return log.traceExit(geneToExpressionCall); + } + + /** + * + * @param unsortedCalls A {@code Set} of {@code ExpressionCallOTF} that contains all calls to filter and/or order + * @param keepOnlyParentsMoreExpressed A boolean used to filter (true) or not filter (false) calls that have a descendant call with + * higher or equal expression score. It allows to avoid showing lots of generic terms + * @param orderingAttribute + * @return + */ + //TODO: investigate why keepOnlyParentsMoreExpressed is useful and choosing the summary quality is not enough. SummaryQuality.SILVER allows to remove all + // condition for which a gene does not have delf observation. The only calls this filtering removes compared to SummaryQuality.BRONZE are the calls + // that have self expression lower than the descendant condition. Isn't it an interesting info to provide? +// public List filterAndOrderExpressionCalls(Set unsortedCalls, boolean keepOnlyParentsMoreExpressed, +// EnumSet orderingAttribute) { +// log.traceEntry("{}, {}, {}", unsortedCalls, keepOnlyParentsMoreExpressed, orderingAttribute); +// +// return null; +// } + + private OTFExpressionCall generateOTFExpressionCall(Gene gene, Condition2 cond, + Collection selfObservations, + Collection descExpressionCalls) { + log.traceEntry("{}, {}, {}, {}", gene, cond, selfObservations, descExpressionCalls); + Collection usedSelfObservations = selfObservations == null ? + Collections.emptySet() : selfObservations; + Collection usedDescExprCalls = descExpressionCalls == null ? + Collections.emptyList() : descExpressionCalls; + + if (usedSelfObservations.isEmpty() && usedDescExprCalls.isEmpty()) { + throw log.throwing(new IllegalArgumentException("Raw data and child calls cannot be both empty")); + } + + //rawData can be empty if no raw data in the condition itself. + //first, compute information from data in the condition itself. + //We use Lists not to loose equals PValues + List allDataTypePValues = new ArrayList<>(); + List trustedDataTypePValues = new ArrayList<>(); + BigDecimal scoreByWeightSum = BigDecimal.ZERO; + BigDecimal weightSum = BigDecimal.ZERO; + //XXX: The number of observation is propbably useful to calculate the pvalue. Right now we add the pvalue as many time as number of observation. + //TODO: To discuss with Fred + EnumSet supportingDataTypes = EnumSet.noneOf(DataType.class); + PropagationState dataPropagation = usedSelfObservations.isEmpty()? + null: PropagationState.SELF; + + // retrieve self info of the expression call + for (ObservedExpressionTO obsExpression : usedSelfObservations) { + if (obsExpression.getBulkNumberObs() != null && obsExpression.getBulkNumberObs() != 0) { + allDataTypePValues.addAll(Collections.nCopies(obsExpression.getBulkNumberObs(), obsExpression.getBulkPValue())); + trustedDataTypePValues.addAll(Collections.nCopies(obsExpression.getBulkNumberObs(), obsExpression.getBulkPValue())); + scoreByWeightSum = scoreByWeightSum + .add((obsExpression.getBulkScore() + .multiply(obsExpression.getBulkWeight()) + .multiply(BigDecimal.valueOf(obsExpression.getBulkNumberObs())))); + weightSum = weightSum. + add(obsExpression.getBulkWeight() + .multiply(BigDecimal.valueOf(obsExpression.getBulkNumberObs()))); + supportingDataTypes.add(DataType.RNA_SEQ); + } + if (obsExpression.getInSituNumberObs() != null && obsExpression.getInSituNumberObs() != 0) { + allDataTypePValues.addAll(Collections.nCopies(obsExpression.getInSituNumberObs(), obsExpression.getInSituPValue())); + trustedDataTypePValues.addAll(Collections.nCopies(obsExpression.getInSituNumberObs(), obsExpression.getInSituPValue())); + scoreByWeightSum = scoreByWeightSum + .add((obsExpression.getInSituScore() + .multiply(obsExpression.getInSituWeight()) + .multiply(BigDecimal.valueOf(obsExpression.getInSituNumberObs())))); + weightSum = weightSum. + add(obsExpression.getInSituWeight() + .multiply(BigDecimal.valueOf(obsExpression.getInSituNumberObs()))); + supportingDataTypes.add(DataType.IN_SITU); + } + if (obsExpression.getFullLengthNumberObs() != null && obsExpression.getFullLengthNumberObs() != 0) { + allDataTypePValues.addAll(Collections.nCopies(obsExpression.getFullLengthNumberObs(), obsExpression.getFullLengthPValue())); + trustedDataTypePValues.addAll(Collections.nCopies(obsExpression.getFullLengthNumberObs(), obsExpression.getFullLengthPValue())); + scoreByWeightSum = scoreByWeightSum + .add((obsExpression.getFullLengthScore() + .multiply(obsExpression.getFullLengthWeight()) + .multiply(BigDecimal.valueOf(obsExpression.getFullLengthNumberObs())))); + weightSum = weightSum. + add(obsExpression.getFullLengthWeight() + .multiply(BigDecimal.valueOf(obsExpression.getFullLengthNumberObs()))); + supportingDataTypes.add(DataType.SC_RNA_SEQ); + } + if (obsExpression.getDropletNumberObs() != null && obsExpression.getDropletNumberObs() != 0) { + allDataTypePValues.addAll(Collections.nCopies(obsExpression.getDropletNumberObs(), obsExpression.getDropletPValue())); + trustedDataTypePValues.addAll(Collections.nCopies(obsExpression.getDropletNumberObs(), obsExpression.getDropletPValue())); + scoreByWeightSum = scoreByWeightSum + .add((obsExpression.getDropletScore() + .multiply(obsExpression.getDropletWeight()) + .multiply(BigDecimal.valueOf(obsExpression.getDropletNumberObs())))); + weightSum = weightSum. + add(obsExpression.getDropletWeight() + .multiply(BigDecimal.valueOf(obsExpression.getDropletNumberObs()))); + supportingDataTypes.add(DataType.SC_RNA_SEQ); + } + } + + BigDecimal bestDescendantAllDataTypePValue = null; + BigDecimal bestDescendantTrustedDataTypePValue = null; + BigDecimal bestDescendantExpressionScore = null; + BigDecimal bestDescendantExpressionScoreWeight = null; + if (!usedDescExprCalls.isEmpty()) { + + dataPropagation = dataPropagation == null? PropagationState.DESCENDANT: PropagationState.SELF_AND_DESCENDANT; + List descAllDataTypePValues = new ArrayList<>(usedDescExprCalls.size()); + List descTrustedDataTypePValues = new ArrayList<>(usedDescExprCalls.size()); + for (OTFExpressionCall childCall: usedDescExprCalls) { + supportingDataTypes.addAll(childCall.getSupportingDataTypes()); + descAllDataTypePValues.add(childCall.getAllDataTypePValue()); + if (childCall.getTrustedDataTypePValue() != null) { + descTrustedDataTypePValues.add(childCall.getTrustedDataTypePValue()); + } + BigDecimal scoreByWeight = childCall.getExpressionScoreWeight().multiply(childCall.getExpressionScore()); + scoreByWeightSum = scoreByWeightSum.add(scoreByWeight); + weightSum = weightSum.add(childCall.getExpressionScoreWeight()); + + bestDescendantAllDataTypePValue = getBestDescendantValue(bestDescendantAllDataTypePValue, + childCall.getAllDataTypePValue(), childCall.getBestDirectDescendantAllDataTypePValue()); + bestDescendantTrustedDataTypePValue = getBestDescendantValue(bestDescendantTrustedDataTypePValue, + childCall.getTrustedDataTypePValue(), childCall.getBestDirectDescendantTrustedDataTypePValue()); + if (bestDescendantExpressionScore == null || + childCall.getExpressionScore().compareTo(bestDescendantExpressionScore) > 0) { + bestDescendantExpressionScore = childCall.getExpressionScore(); + bestDescendantExpressionScoreWeight = childCall.getExpressionScoreWeight(); + } + if (childCall.getBestDirectDescendantExpressionScore() != null && + childCall.getBestDirectDescendantExpressionScore().compareTo(bestDescendantExpressionScore) > 0) { + bestDescendantExpressionScore = childCall.getBestDirectDescendantExpressionScore(); + bestDescendantExpressionScoreWeight = childCall.getBestDirectDescendantExpressionScoreWeight(); + } + } + allDataTypePValues.add(computeFDRCorrectedPValue(descAllDataTypePValues)); + if (!descTrustedDataTypePValues.isEmpty()) { + trustedDataTypePValues.add(computeFDRCorrectedPValue(descTrustedDataTypePValues)); + } + } + BigDecimal ultimateAllDataTypePValue = computeMean(allDataTypePValues); + BigDecimal ultimateTrustedDataTypePValue = computeMean(trustedDataTypePValues); +// log.debug("weightSum: {}, scoreByWeightSum: {}", weightSum, scoreByWeightSum); + if (BigDecimal.ZERO.compareTo(weightSum) == 0) { + log.warn("weightSum is zero for gene {} in condition {} - all observation counts are null/0. Defaulting score to 0.", gene, cond); + } + BigDecimal weightedAverageExpressionScore = BigDecimal.ZERO.compareTo(weightSum) == 0 ? + BigDecimal.ZERO : + scoreByWeightSum.divide(weightSum, 2, RoundingMode.HALF_UP); + + OTFExpressionCall resultingCall = new OTFExpressionCall(gene, cond, supportingDataTypes, + ultimateAllDataTypePValue, ultimateTrustedDataTypePValue, + bestDescendantAllDataTypePValue, bestDescendantTrustedDataTypePValue, + weightSum, weightedAverageExpressionScore, + bestDescendantExpressionScoreWeight, bestDescendantExpressionScore, + dataPropagation); + + return log.traceExit(resultingCall); + } + + private static BigDecimal getBestDescendantValue(BigDecimal currentBestDescendantValue, + BigDecimal descendantValue, BigDecimal descendantBestDescendantValue) { + log.traceEntry("{}, {}, {}", currentBestDescendantValue, descendantValue, descendantBestDescendantValue); + + if (descendantValue != null && (currentBestDescendantValue == null || + descendantValue.compareTo(currentBestDescendantValue) < 0)) { + currentBestDescendantValue = descendantValue; + } + if (descendantBestDescendantValue != null && (currentBestDescendantValue == null || + descendantBestDescendantValue.compareTo(currentBestDescendantValue) < 0)) { + currentBestDescendantValue = descendantBestDescendantValue; + } + return log.traceExit(currentBestDescendantValue); + } + + protected BigDecimal computeFDRCorrectedPValue(List pValues) { + log.traceEntry("{}", pValues); + + int m = pValues.size(); + Double[] pValuesDouble = + pValues.stream() + .map(p -> p.compareTo(ZERO_BIGDECIMAL) == 0 ? ABOVE_ZERO_BIGDECIMAL : p) + .map(p -> p.doubleValue()) + .toArray(length -> new Double[length]); + double[] adjustedPValues = new double[m]; + + Arrays.sort(pValuesDouble); + // iterate through all p-values: largest to smallest + for (int i = m - 1; i >= 0; i--) { + if (i == m - 1) { + adjustedPValues[i] = pValuesDouble[i]; + } else { + double unadjustedPvalue = pValuesDouble[i]; + int divideByM = i + 1; + double left = adjustedPValues[i + 1]; + double right = (m / (double) divideByM) * unadjustedPvalue; + adjustedPValues[i] = Math.min(left, right); + } + } + //Find the smallest corrected p-value + BigDecimal fdr = BigDecimal.valueOf(Arrays.stream(adjustedPValues).min().getAsDouble()); + //If the FDR is less than MIN_FDR_BIGDECIMAL, change it to MIN_FDR_BIGDECIMAL + //(in order to avoid having fields in the globalExpression table with too much precision) + if (fdr.compareTo(MIN_FDR_BIGDECIMAL) < 0) { + fdr = MIN_FDR_BIGDECIMAL; + } + return log.traceExit(fdr); + } + + protected BigDecimal computeMean(List pValues) { + log.traceEntry("{}", pValues); + if (pValues == null || pValues.isEmpty()) { + return null; + } + + BigDecimal sum = pValues.stream() + .reduce(BigDecimal.ZERO, BigDecimal::add); + + return sum.divide( + BigDecimal.valueOf(pValues.size()), + //34 significant digits and RoundingMode.HALF_EVEN + MathContext.DECIMAL128 + ); + } + + public long loadDataCount() { log.traceEntry(); @@ -310,16 +866,6 @@ public ExpressionCallPostFilter loadPostFilter() { condParamEntities.put(ConditionParameter.SEX, sexes); } - //Species are unnecessary, we allow filtering only when one species is selected -// Set speciesIds = condRequestFun.apply( -// Set.of(ConditionDAO.Attribute.SPECIES_ID)) -// .stream().map(c -> c.getSpeciesId()).collect(Collectors.toSet()); -// Set species = speciesIds.isEmpty()? -// new HashSet<>() : this.getProcessedFilter().getSpeciesMap().values() -// .stream().filter(s -> speciesIds.contains(s.getId())) -// .collect(Collectors.toSet()); -// assert speciesIds.size() == species.size(); - return log.traceExit(new ExpressionCallPostFilter(condParamEntities)); } @@ -327,67 +873,14 @@ public ExpressionCallProcessedFilter getProcessedFilter() { return processedFilter; } - //TODO to continue here -// private ExpressionCallPostFilter loadConditionPostFilter(BiFunction, -// Collection, ConditionTOResultSet> condRequest) { -// log.traceEntry("{}", condRequest); -// -// //If the DaoRawDataFilters are null it means there was no matching conds -// //and thus no result for sure -// if (this.processedFilter.getDaoFilters() == null) { -// return log.traceExit(new ExpressionCallPostFilter(null)); -// } -// -// // retrieve anatEntities -// Set anatEntityIds = condRequest.apply(this.processedFilter -// .getDaoFilters(), Set.of(ConditionDAO.Attribute.ANAT_ENTITY_ID)).stream() -// .map(a -> a.getAnatEntityId()).collect(Collectors.toSet()); -// Set anatEntities = anatEntityIds.isEmpty()? -// new HashSet<>() : anatEntityService.loadAnatEntities(anatEntityIds, false) -// .collect(Collectors.toSet()); -// -// // retrieve cellTypes -// Set cellTypeIds = condRequest.apply(this.getRawDataProcessedFilter() -// .getDaoFilters(), Set.of(RawDataConditionDAO.Attribute.CELL_TYPE_ID)) -// .stream() -// .map(c -> c.getCellTypeId()) -// //cell type is the only condition param that can be NULL, -// //we end up requesting an anat. entity with ID "NULL" -// .filter(s -> s != null) -// .collect(Collectors.toSet()); -// Set cellTypes = cellTypeIds.isEmpty()? -// new HashSet<>() : anatEntityService.loadAnatEntities(cellTypeIds, false) -// .collect(Collectors.toSet()); -// -// //retrieve dev. stages -// Set stageIds = condRequest.apply(this.getRawDataProcessedFilter() -// .getDaoFilters(), Set.of(RawDataConditionDAO.Attribute.STAGE_ID)) -// .stream().map(c -> c.getStageId()).collect(Collectors.toSet()); -// Set stages = stageIds.isEmpty()? -// new HashSet<>() : devStageService.loadDevStages(null, null, stageIds, false) -// .collect(Collectors.toSet()); -// -// // retrieve strains -// Set strains = condRequest.apply(this.getRawDataProcessedFilter() -// .getDaoFilters(), Set.of(RawDataConditionDAO.Attribute.STRAIN)) -// .stream().map(c -> c.getStrainId()).collect(Collectors.toSet()); -// -// //retrieve sexes -// Set sexes = condRequest.apply(this.getRawDataProcessedFilter() -// .getDaoFilters(), Set.of(RawDataConditionDAO.Attribute.SEX)).stream() -// .map(c -> mapDAORawDataSexToRawDataSex(c.getSex())).collect(Collectors.toSet()); -// -// return log.traceExit(new RawDataPostFilter(anatEntities, stages, cellTypes, -// sexes, strains, dataType)); -// } - private void updateConditionMap(Set condIds) { log.traceEntry("{}", condIds); Set missingCondIds = new HashSet<>(condIds); missingCondIds.removeAll(this.conditionMap.keySet()); if (missingCondIds.isEmpty()) { - log.traceExit(); return; + log.traceExit(); + return; } Map speciesMap = this.processedFilter.getSpeciesMap(); Map missingCondMap = this.utils.loadConditionMapFromResultSet( @@ -403,7 +896,8 @@ private void updateConditionMap(Set condIds) { } this.conditionMap.putAll(missingCondMap); - log.traceExit(); return; + log.traceExit(); + return; } private void updateGeneMap(Set bgeeGeneIds) { log.traceEntry("{}", bgeeGeneIds); @@ -587,5 +1081,5 @@ private Set convertServiceAttrToGlobalExp return log.traceExit(orderAttrs); } - + } diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/call/ExpressionCallProcessedFilter.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/call/ExpressionCallProcessedFilter.java index b1d0b46d0..1892a20cc 100644 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/call/ExpressionCallProcessedFilter.java +++ b/bgee-core/src/main/java/org/bgee/model/expressiondata/call/ExpressionCallProcessedFilter.java @@ -86,6 +86,7 @@ public static class ExpressionCallProcessedFilterInvariablePart extends Processe private final Map maxRankPerSpecies; + //TODO: remove maxRankPerSpecies from that processed filter. Scores are now calculated by the pipeline and stored in the database. OTF propagation does not require maxRanksPerSpecies anymore. ExpressionCallProcessedFilterInvariablePart(Map geneBioTypeMap, Map sourceMap, Map maxRankPerSpecies) { super(geneBioTypeMap, sourceMap); diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/call/ExpressionCallService.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/call/ExpressionCallService.java index bd401162d..01de16f25 100644 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/call/ExpressionCallService.java +++ b/bgee-core/src/main/java/org/bgee/model/expressiondata/call/ExpressionCallService.java @@ -160,8 +160,11 @@ public ExpressionCallProcessedFilter processExpressionCallFilter(ExpressionCallF throw log.throwing(new IllegalArgumentException("The ExpressionCallProcessedFilterConditionPart " + "does not correspond to the ExpressionCallFilter2")); } + long startTimeInvariable = System.currentTimeMillis(); final ExpressionCallProcessedFilterInvariablePart procInvariablePart = invariablePart != null? invariablePart: loadIfNecessaryAndGetInvariablePart(); + log.debug("loadIfNecessaryAndGetInvariablePart() completed in {} ms", + System.currentTimeMillis() - startTimeInvariable); final ExpressionCallProcessedFilterGeneSpeciesPart procGeneSpeciesPart = geneSpeciesPart != null? geneSpeciesPart: @@ -170,6 +173,9 @@ public ExpressionCallProcessedFilter processExpressionCallFilter(ExpressionCallF loadGeneSpeciesPart(filter, procInvariablePart.getGeneBioTypeMap())); //It's OK that the filter is null or empty if we want to retrieve any raw data + //TODO: In which context do we allow null or empty filter? + // Filter should never be null and at least one gene/species should be provided + // => should be removed as OTF propagation does not work in that context if (filter == null || filter.isEmptyFilter()) { return log.traceExit(new ExpressionCallProcessedFilter(filter, //Important to provide a HashSet here, a null value means @@ -189,8 +195,12 @@ public ExpressionCallProcessedFilter processExpressionCallFilter(ExpressionCallF //At this point, the filter cannot be null and we can use the method loadConditionPart assert filter != null; + long startTimeCondPart = System.currentTimeMillis(); final ExpressionCallProcessedFilterConditionPart procConditionPart = conditionPart != null? conditionPart: loadConditionPart(filter, procGeneSpeciesPart.getSpeciesMap()); + log.debug("loadConditionPart() completed in {} ms, {} conditions found", + System.currentTimeMillis() - startTimeCondPart, + procConditionPart.getRequestedConditionMap().size()); //At this point, there should always be at least a GeneFilter, it is mandatory @@ -277,6 +287,7 @@ public ExpressionCallProcessedFilter processExpressionCallFilter(ExpressionCallF ABSENT_HIGH_GREATER_THAN)); } + //XXX As of Bgee 16.0 the max rank is not required anymore. private ExpressionCallProcessedFilterInvariablePart loadIfNecessaryAndGetInvariablePart() { //We don't fear a race condition here, because this information is cheap to compute //and does not change, so no problem to retrieve and set it multiple times. @@ -289,15 +300,17 @@ private ExpressionCallProcessedFilterInvariablePart loadIfNecessaryAndGetInvaria //Retrieve max rank for the requested species if EXPRESSION_SCORE requested //(the max rank is required to convert mean ranks into expression scores) //TODO: in a future version with Attributes, to retrieve only if necessary - Map maxRankPerSpecies = conditionDAO - .getMaxRanks(null, - //We always request the max rank over all data types, - //independently of the data types requested in the query, - //because ranks are all normalized based on the max rank over all data types - null); +// Map maxRankPerSpecies = conditionDAO +// .getMaxRanks(null, +// //We always request the max rank over all data types, +// //independently of the data types requested in the query, +// //because ranks are all normalized based on the max rank over all data types +// null); PROCESSED_FILTER_INVARIABLE_PART = new ExpressionCallProcessedFilterInvariablePart(geneBioTypeMap, sourceMap, - maxRankPerSpecies); + null); + } else { + log.debug("loadIfNecessaryAndGetInvariablePart: cache hit, reusing invariable part"); } return log.traceExit(PROCESSED_FILTER_INVARIABLE_PART); } @@ -335,9 +348,17 @@ private ExpressionCallProcessedFilterConditionPart loadConditionPart(ExpressionC //Now, we load specific conditions that can be queried. Again, we need to retrieve //all of them to configure the DAOCallFilter, even if there is a large number. + long t0 = System.currentTimeMillis(); + //FIXME: creation of daoCondFilters does not require to use the ontologies. + // Everything is available in the database to create a DAO that would return values + // of condition parameters with there descendants. It would be way way faster. Set daoCondFilters = this.utils.convertConditionFiltersToDAOConditionFilters(filter.getConditionFilters(), this.ontService, this.anatEntityService, filter.getSpeciesIdsConsidered()); + log.debug("convertConditionFiltersToDAOConditionFilters() completed in {} ms ({} DAO filters)", + System.currentTimeMillis() - t0, daoCondFilters.size()); + t0 = System.currentTimeMillis(); + //TODO: check that we really want to allow empty daoCondFilters Map requestedCondMap = daoCondFilters.isEmpty()? new HashMap<>(): this.utils.loadGlobalConditionMap(speciesMap.values(), @@ -345,8 +366,11 @@ private ExpressionCallProcessedFilterConditionPart loadConditionPart(ExpressionC this.utils.convertCondParamsToDAOCondAttributes(filter.getCondParamCombination()), this.conditionDAO, this.anatEntityService, this.devStageService, this.sexService, this.strainService); + log.debug("loadGlobalConditionMap() completed in {} ms ({} conditions)", + System.currentTimeMillis() - t0, requestedCondMap.size()); return log.traceExit(new ExpressionCallProcessedFilterConditionPart( filter.getConditionFilters(), requestedCondMap)); } + } \ No newline at end of file diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/call/OTFExpressionCall.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/call/OTFExpressionCall.java new file mode 100644 index 000000000..8b7a4c731 --- /dev/null +++ b/bgee-core/src/main/java/org/bgee/model/expressiondata/call/OTFExpressionCall.java @@ -0,0 +1,183 @@ +package org.bgee.model.expressiondata.call; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.text.DecimalFormat; +import java.text.NumberFormat; +import java.util.Comparator; +import java.util.EnumSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bgee.model.expressiondata.baseelements.DataType; +import org.bgee.model.expressiondata.baseelements.PropagationState; +import org.bgee.model.gene.Gene; + +public class OTFExpressionCall { + + private final static Logger log = LogManager.getLogger(OTFExpressionCall.class.getName()); + + //TODO: implement method allowing advanced ordering options using ordering attributes + /** + * Sorts all calls from the provided map into a single list ordered by decreasing + * {@code expressionScore}. Calls with a {@code null} expression score are placed last. + * + * @param propagatedExpressionCalls a map of genes to their propagated calls + * @return a flat list of all calls sorted by decreasing expression score + */ + public static List sortByDecreasingExpressionScore( + Map> propagatedExpressionCalls) { + return propagatedExpressionCalls.values().stream() + .flatMap(Set::stream) + .sorted(Comparator.comparing( + OTFExpressionCall::getExpressionScore, + Comparator.nullsLast(Comparator.reverseOrder()))) + .collect(Collectors.toList()); + } + + private final Gene gene; + private final Condition2 condition; + private final EnumSet supportingDataTypes; + private final BigDecimal allDataTypePValue; + private final BigDecimal trustedDataTypePValue; + private final BigDecimal bestDirectDescendantAllDataTypePValue; + private final BigDecimal bestDirectDescendantTrustedDataTypePValue; + private final BigDecimal expressionScoreWeight; + private final BigDecimal expressionScore; + private final BigDecimal bestDirectDescendantExpressionScoreWeight; + private final BigDecimal bestDirectDescendantExpressionScore; + private final PropagationState dataPropagation; + + public OTFExpressionCall(Gene gene, Condition2 condition, EnumSet supportingDataTypes, + BigDecimal allDataTypePValue, BigDecimal trustedDataTypePValue, + BigDecimal bestDirectDescendantAllDataTypePValue, BigDecimal bestDirectDescendantTrustedDataTypePValue, + BigDecimal expressionScoreWeight, BigDecimal expressionScore, + BigDecimal bestDirectDescendantExpressionScoreWeight, BigDecimal bestDirectDescendantExpressionScore, + PropagationState dataPropagation) { + this.gene = gene; + this.condition = condition; + this.supportingDataTypes = supportingDataTypes; + this.allDataTypePValue = allDataTypePValue; + this.trustedDataTypePValue = trustedDataTypePValue; + this.bestDirectDescendantAllDataTypePValue = bestDirectDescendantAllDataTypePValue; + this.bestDirectDescendantTrustedDataTypePValue = bestDirectDescendantTrustedDataTypePValue; + this.expressionScoreWeight = expressionScoreWeight; + this.expressionScore = expressionScore; + this.bestDirectDescendantExpressionScoreWeight = bestDirectDescendantExpressionScoreWeight; + this.bestDirectDescendantExpressionScore = bestDirectDescendantExpressionScore; + this.dataPropagation = dataPropagation; + } + + public Gene getGene() { + return gene; + } + public Condition2 getCondition() { + return condition; + } + public EnumSet getSupportingDataTypes() { + return supportingDataTypes; + } + public BigDecimal getAllDataTypePValue() { + return allDataTypePValue; + } + public BigDecimal getTrustedDataTypePValue() { + return trustedDataTypePValue; + } + public BigDecimal getBestDirectDescendantAllDataTypePValue() { + return bestDirectDescendantAllDataTypePValue; + } + public BigDecimal getBestDirectDescendantTrustedDataTypePValue() { + return bestDirectDescendantTrustedDataTypePValue; + } + public BigDecimal getExpressionScoreWeight() { + return expressionScoreWeight; + } + public BigDecimal getExpressionScore() { + return expressionScore; + } + public BigDecimal getBestDirectDescendantExpressionScoreWeight() { + return bestDirectDescendantExpressionScoreWeight; + } + public BigDecimal getBestDirectDescendantExpressionScore() { + return bestDirectDescendantExpressionScore; + } + public PropagationState getDataPropagation() { + return dataPropagation; + } + + public String getFormattedAllDatatypePValue() { + log.traceEntry(); + NumberFormat formatter = NumberFormat.getInstance(Locale.US); + formatter.setRoundingMode(RoundingMode.HALF_UP); + // do not use scientific notation when FDR pValue is bigger than 0.001 or equal + // to 0 + if(allDataTypePValue.compareTo(new BigDecimal(0.001)) >= 0 || + allDataTypePValue.compareTo(new BigDecimal(0)) == 0) { + formatter.setMaximumFractionDigits(3); + formatter.setMinimumFractionDigits(0); + } else if (formatter instanceof DecimalFormat) { + ((DecimalFormat) formatter).applyPattern("0.00E0"); + } else { + throw log.throwing(new IllegalStateException("No formatter could be defined " + + "for " + allDataTypePValue)); + } + //In Bgee 16 we limited the precision to 30 digits + return log.traceExit((allDataTypePValue.compareTo(new BigDecimal("0")) != 0 && + allDataTypePValue.compareTo(new BigDecimal("1E-30")) <= 0 ? "<= ": "") + + formatter.format(allDataTypePValue).toLowerCase(Locale.US)); + } + + @Override + public int hashCode() { + return Objects.hash(allDataTypePValue, bestDirectDescendantAllDataTypePValue, + bestDirectDescendantExpressionScore, bestDirectDescendantExpressionScoreWeight, + bestDirectDescendantTrustedDataTypePValue, condition, dataPropagation, expressionScore, + expressionScoreWeight, gene, supportingDataTypes, trustedDataTypePValue); + } + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + OTFExpressionCall other = (OTFExpressionCall) obj; + return Objects.equals(allDataTypePValue, other.allDataTypePValue) + && Objects.equals(bestDirectDescendantAllDataTypePValue, other.bestDirectDescendantAllDataTypePValue) + && Objects.equals(bestDirectDescendantExpressionScore, other.bestDirectDescendantExpressionScore) + && Objects.equals(bestDirectDescendantExpressionScoreWeight, other.bestDirectDescendantExpressionScoreWeight) + && Objects.equals(bestDirectDescendantTrustedDataTypePValue, other.bestDirectDescendantTrustedDataTypePValue) + && Objects.equals(condition, other.condition) && Objects.equals(dataPropagation, other.dataPropagation) + && Objects.equals(expressionScore, other.expressionScore) + && Objects.equals(expressionScoreWeight, other.expressionScoreWeight) + && Objects.equals(gene, other.gene) && Objects.equals(supportingDataTypes, other.supportingDataTypes) + && Objects.equals(trustedDataTypePValue, other.trustedDataTypePValue); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("OTFExpressionCall [") + .append("gene=").append(gene) + .append(", condition=").append(condition) + .append(", supportingDataTypes=").append(supportingDataTypes) + .append(", allDataTypePValue=").append(allDataTypePValue) + .append(", trustedDataTypePValue=").append(trustedDataTypePValue) + .append(", bestDirectDescendantAllDataTypePValue=").append(bestDirectDescendantAllDataTypePValue) + .append(", bestDirectDescendantTrustedDataTypePValue=").append(bestDirectDescendantTrustedDataTypePValue) + .append(", expressionScoreWeight=").append(expressionScoreWeight) + .append(", expressionScore=").append(expressionScore) + .append(", bestDirectDescendantExpressionScoreWeight=").append(bestDirectDescendantExpressionScoreWeight) + .append(", bestDirectDescendantExpressionScore=").append(bestDirectDescendantExpressionScore) + .append(", dataPropagation=").append(dataPropagation) + .append("]"); + return builder.toString(); + } +} diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/call/OTFExpressionCallFilterEngine.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/call/OTFExpressionCallFilterEngine.java new file mode 100644 index 000000000..8f7200db9 --- /dev/null +++ b/bgee-core/src/main/java/org/bgee/model/expressiondata/call/OTFExpressionCallFilterEngine.java @@ -0,0 +1,179 @@ +package org.bgee.model.expressiondata.call; + +import java.util.Set; +import java.util.function.Predicate; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bgee.model.expressiondata.BaseConditionFilter2.FilterIds; +import org.bgee.model.expressiondata.baseelements.ConditionParameter; + +public class OTFExpressionCallFilterEngine { + private static final Logger log = LogManager.getLogger(OTFExpressionCallFilterEngine.class.getName()); + + public static Predicate compile( + Set filters) { + + return filters.stream() + .map(OTFExpressionCallFilterEngine::compile) + .reduce(x -> true, Predicate::and); + } + + private static Predicate compile(ConditionFilter2 filter) { + + Predicate p = call -> true; + + // ========================================================= + // ANAT ENTITY (index 0 = anat entity) + // ========================================================= + { + FilterIds anat = + filter.getComposedFilterIds(ConditionParameter.ANAT_ENTITY_CELL_TYPE) + .getFilterIds(0); + + if (anat != null && !anat.isEmpty()) { + + Set allowed = anat.getIds(); + Set excluded = anat.getExcludeTermsAndChildrenIds(); + + p = p.and(call -> { + + String anatId = call.getCondition() + .getConditionParameterValue(ConditionParameter.ANAT_ENTITY_CELL_TYPE) + //Here index 1 = anat. entity :-/ + .getEntity(1) + .getId(); + + boolean allowedOk = true; + boolean excludedOk = true; + + // include constraint + if (!anat.isIncludeChildTerms()) { + allowedOk = allowed.contains(anatId); + } + + // exclude constraint + if (excluded != null && !excluded.isEmpty()) { + excludedOk = !excluded.contains(anatId); + } + + boolean accepted = allowedOk && excludedOk; + + log.debug( + "Gene={} Condition={} ANAT={} allowedOk={} excludedOk={} -> {}", + call.getGene().getGeneId(), + call.getCondition(), + anatId, + allowedOk, + excludedOk, + accepted ? "KEEP" : "REJECT" + ); + + return accepted; + }); + } + } + + // ========================================================= + // CELL TYPE (index 1 = cell type) + // ========================================================= + { + FilterIds cell = + filter.getComposedFilterIds(ConditionParameter.ANAT_ENTITY_CELL_TYPE) + .getFilterIds(1); + + if (cell != null && !cell.isEmpty()) { + + Set allowed = cell.getIds(); + Set excluded = cell.getExcludeTermsAndChildrenIds(); + + p = p.and(call -> { + + String cellId = call.getCondition() + .getConditionParameterValue(ConditionParameter.ANAT_ENTITY_CELL_TYPE) + //Here index 0 = anat. entity :-/ + .getEntity(0) + .getId(); + + boolean allowedOk = true; + boolean excludedOk = true; + + // include constraint + if (!cell.isIncludeChildTerms()) { + allowedOk = allowed.contains(cellId); + } + + // exclude constraint + if (excluded != null && !excluded.isEmpty()) { + excludedOk = !excluded.contains(cellId); + } + + boolean accepted = allowedOk && excludedOk; + + log.debug( + "Gene={} Condition={} CELL={} allowedOk={} excludedOk={} -> {}", + call.getGene().getGeneId(), + call.getCondition(), + cellId, + allowedOk, + excludedOk, + accepted ? "KEEP" : "REJECT" + ); + + return accepted; + }); + } + } + + // ========================================================= + // DEV STAGE + // ========================================================= + { + FilterIds stage = + filter.getComposedFilterIds(ConditionParameter.DEV_STAGE) + .getFilterIds(0); + + if (stage != null && !stage.isEmpty()) { + + Set allowed = stage.getIds(); + Set excluded = stage.getExcludeTermsAndChildrenIds(); + + p = p.and(call -> { + + String stageId = call.getCondition() + .getConditionParameterId(ConditionParameter.DEV_STAGE); + + boolean allowedOk = true; + boolean excludedOk = true; + + // include constraint + if (!stage.isIncludeChildTerms()) { + allowedOk = allowed.contains(stageId); + } + + // exclude constraint + if (excluded != null && !excluded.isEmpty()) { + excludedOk = !excluded.contains(stageId); + } + + boolean accepted = allowedOk && excludedOk; + + log.debug( + "Gene={} Condition={} STAGE={} allowedOk={} excludedOk={} -> {}", + call.getGene().getGeneId(), + call.getCondition(), + stageId, + allowedOk, + excludedOk, + accepted ? "KEEP" : "REJECT" + ); + + return accepted; + }); + } + } + + return p; + } + +} diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/RawDataContainer.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/RawDataContainer.java index 8c4ea30c6..9c1346c26 100644 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/RawDataContainer.java +++ b/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/RawDataContainer.java @@ -11,14 +11,9 @@ import org.apache.logging.log4j.Logger; import org.bgee.model.expressiondata.baseelements.DataType; import org.bgee.model.expressiondata.rawdata.baseelements.DataContainer; -import org.bgee.model.expressiondata.rawdata.est.EST; -import org.bgee.model.expressiondata.rawdata.est.ESTLibrary; import org.bgee.model.expressiondata.rawdata.insitu.InSituEvidence; import org.bgee.model.expressiondata.rawdata.insitu.InSituExperiment; import org.bgee.model.expressiondata.rawdata.insitu.InSituSpot; -import org.bgee.model.expressiondata.rawdata.microarray.AffymetrixChip; -import org.bgee.model.expressiondata.rawdata.microarray.AffymetrixExperiment; -import org.bgee.model.expressiondata.rawdata.microarray.AffymetrixProbeset; import org.bgee.model.expressiondata.rawdata.rnaseq.RnaSeqExperiment; import org.bgee.model.expressiondata.rawdata.rnaseq.RnaSeqLibrary; import org.bgee.model.expressiondata.rawdata.rnaseq.RnaSeqLibraryAnnotatedSample; @@ -37,26 +32,16 @@ public class RawDataContainer extends DataContainer { private final static Logger log = LogManager.getLogger(RawDataContainer.class.getName()); private static EnumSet computeRequestedDataTypes( - Collection affymetrixExperiments, - Collection affymetrixAssays, Collection affymetrixCalls, Collection rnaSeqExperiments, Collection rnaSeqLibraries, Collection rnaSeqAssays, Collection rnaSeqCalls, Collection inSituExperiments, - Collection inSituAssays, Collection inSituCalls, - Collection estAssays, Collection estCalls) { - log.traceEntry("{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}", - affymetrixExperiments, affymetrixAssays, affymetrixCalls, + Collection inSituAssays, Collection inSituCalls) { + log.traceEntry("{}, {}, {}, {}, {}, {}, {}", rnaSeqExperiments, rnaSeqLibraries, rnaSeqAssays, rnaSeqCalls, - inSituExperiments, inSituAssays, inSituCalls, - estAssays, estCalls); + inSituExperiments, inSituAssays, inSituCalls); EnumSet requestedDataTypes = EnumSet.noneOf(DataType.class); - if (affymetrixExperiments != null || - affymetrixAssays != null || - affymetrixCalls != null) { - requestedDataTypes.add(DataType.AFFYMETRIX); - } if (rnaSeqExperiments != null || rnaSeqLibraries != null || rnaSeqAssays != null || @@ -68,34 +53,20 @@ private static EnumSet computeRequestedDataTypes( inSituCalls != null) { requestedDataTypes.add(DataType.IN_SITU); } - if (estAssays != null || - estCalls != null) { - requestedDataTypes.add(DataType.EST); - } return log.traceExit(requestedDataTypes); } private static EnumSet computeDataTypesWithResults( - Collection affymetrixExperiments, - Collection affymetrixAssays, Collection affymetrixCalls, Collection rnaSeqExperiments, Collection rnaSeqLibraries, Collection rnaSeqAssays, Collection rnaSeqCalls, Collection inSituExperiments, - Collection inSituAssays, Collection inSituCalls, - Collection estAssays, Collection estCalls) { - log.traceEntry("{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}", - affymetrixExperiments, affymetrixAssays, affymetrixCalls, + Collection inSituAssays, Collection inSituCalls) { + log.traceEntry("{}, {}, {}, {}, {}, {}, {}", rnaSeqExperiments, rnaSeqLibraries, rnaSeqAssays, rnaSeqCalls, - inSituExperiments, inSituAssays, inSituCalls, - estAssays, estCalls); + inSituExperiments, inSituAssays, inSituCalls); EnumSet dataTypesWithResults = EnumSet.noneOf(DataType.class); - if (affymetrixExperiments != null && !affymetrixExperiments.isEmpty() || - affymetrixAssays != null && !affymetrixAssays.isEmpty() || - affymetrixCalls != null && !affymetrixCalls.isEmpty()) { - dataTypesWithResults.add(DataType.AFFYMETRIX); - } if (rnaSeqExperiments != null && !rnaSeqExperiments.isEmpty() || rnaSeqLibraries != null && !rnaSeqLibraries.isEmpty() || rnaSeqAssays != null && !rnaSeqAssays.isEmpty() || @@ -107,18 +78,10 @@ private static EnumSet computeDataTypesWithResults( inSituCalls != null && !inSituCalls.isEmpty()) { dataTypesWithResults.add(DataType.IN_SITU); } - if (estAssays != null && !estAssays.isEmpty() || - estCalls != null && !estCalls.isEmpty()) { - dataTypesWithResults.add(DataType.EST); - } return log.traceExit(dataTypesWithResults); } - private final Set affymetrixExperiments; - private final Set affymetrixAssays; - private final Set affymetrixCalls; - private final Set rnaSeqExperiments; private final Set rnaSeqLibraries; private final Set rnaSeqAssays; @@ -128,33 +91,18 @@ private static EnumSet computeDataTypesWithResults( private final Set inSituAssays; private final Set inSituCalls; - private final Set estAssays; - private final Set estCalls; - - public RawDataContainer(Collection affymetrixExperiments, - Collection affymetrixAssays, Collection affymetrixCalls, + public RawDataContainer( Collection rnaSeqExperiments, Collection rnaSeqLibraries, Collection rnaSeqAssays, Collection rnaSeqCalls, Collection inSituExperiments, - Collection inSituAssays, Collection inSituCalls, - Collection estAssays, Collection estCalls) { + Collection inSituAssays, Collection inSituCalls) { super(computeRequestedDataTypes( - affymetrixExperiments, affymetrixAssays, affymetrixCalls, rnaSeqExperiments, rnaSeqLibraries, rnaSeqAssays, rnaSeqCalls, - inSituExperiments, inSituAssays, - inSituCalls, estAssays, estCalls), + inSituExperiments, inSituAssays, inSituCalls), computeDataTypesWithResults( - affymetrixExperiments, affymetrixAssays, affymetrixCalls, rnaSeqExperiments, rnaSeqLibraries, rnaSeqAssays, rnaSeqCalls, inSituExperiments, inSituAssays, - inSituCalls, estAssays, estCalls)); - - this.affymetrixExperiments = affymetrixExperiments == null? null: - Collections.unmodifiableSet(new LinkedHashSet<>(affymetrixExperiments)); - this.affymetrixAssays = affymetrixAssays == null? null: - Collections.unmodifiableSet(new LinkedHashSet<>(affymetrixAssays)); - this.affymetrixCalls = affymetrixCalls == null? null: - Collections.unmodifiableSet(new LinkedHashSet<>(affymetrixCalls)); + inSituCalls)); this.rnaSeqExperiments = rnaSeqExperiments == null? null: Collections.unmodifiableSet(new LinkedHashSet<>(rnaSeqExperiments)); @@ -171,42 +119,6 @@ public RawDataContainer(Collection affymetrixExperiments, Collections.unmodifiableSet(new LinkedHashSet<>(inSituAssays)); this.inSituCalls = inSituCalls == null? null: Collections.unmodifiableSet(new LinkedHashSet<>(inSituCalls)); - - this.estAssays = estAssays == null? null: - Collections.unmodifiableSet(new LinkedHashSet<>(estAssays)); - this.estCalls = estCalls == null? null: - Collections.unmodifiableSet(new LinkedHashSet<>(estCalls)); - } - - /** - * @return A {@code Set} of {@code AffymetrixExperiment}s that were requested. - * If {@code null}, it means that this information was not requested. - * If empty, it means that there was no result based on query parameters. - * When non-null, the underlying instance is a {@code LinkedHashSet}, - * but returned as a {@code Set} to be unmodifiable. - */ - public Set getAffymetrixExperiments() { - return affymetrixExperiments; - } - /** - * @return A {@code Set} of {@code AffymetrixChip}s that were requested. - * If {@code null}, it means that this information was not requested. - * If empty, it means that there was no result based on query parameters. - * When non-null, the underlying instance is a {@code LinkedHashSet}, - * but returned as a {@code Set} to be unmodifiable. - */ - public Set getAffymetrixAssays() { - return affymetrixAssays; - } - /** - * @return A {@code Set} of {@code AffymetrixProbeset}s that were requested. - * If {@code null}, it means that this information was not requested. - * If empty, it means that there was no result based on query parameters. - * When non-null, the underlying instance is a {@code LinkedHashSet}, - * but returned as a {@code Set} to be unmodifiable. - */ - public Set getAffymetrixCalls() { - return affymetrixCalls; } /** @@ -281,35 +193,13 @@ public Set getInSituCalls() { return inSituCalls; } - /** - * @return A {@code Set} of {@code ESTLibrary}s that were requested. - * If {@code null}, it means that this information was not requested. - * If empty, it means that there was no result based on query parameters. - * When non-null, the underlying instance is a {@code LinkedHashSet}, - * but returned as a {@code Set} to be unmodifiable. - */ - public Set getEstAssays() { - return estAssays; - } - /** - * @return A {@code Set} of {@code EST}s that were requested. - * If {@code null}, it means that this information was not requested. - * If empty, it means that there was no result based on query parameters. - * When non-null, the underlying instance is a {@code LinkedHashSet}, - * but returned as a {@code Set} to be unmodifiable. - */ - public Set getEstCalls() { - return estCalls; - } - @Override public int hashCode() { final int prime = 31; int result = super.hashCode(); result = prime * result + Objects.hash( - affymetrixAssays, affymetrixCalls, affymetrixExperiments, estAssays, - estCalls, inSituAssays, inSituCalls, inSituExperiments, rnaSeqAssays, + inSituAssays, inSituCalls, inSituExperiments, rnaSeqAssays, rnaSeqCalls, rnaSeqExperiments, rnaSeqLibraries); return result; } @@ -322,12 +212,7 @@ public boolean equals(Object obj) { if (getClass() != obj.getClass()) return false; RawDataContainer other = (RawDataContainer) obj; - return Objects.equals(affymetrixAssays, other.affymetrixAssays) - && Objects.equals(affymetrixCalls, other.affymetrixCalls) - && Objects.equals(affymetrixExperiments, other.affymetrixExperiments) - && Objects.equals(estAssays, other.estAssays) - && Objects.equals(estCalls, other.estCalls) - && Objects.equals(inSituAssays, other.inSituAssays) + return Objects.equals(inSituAssays, other.inSituAssays) && Objects.equals(inSituCalls, other.inSituCalls) && Objects.equals(inSituExperiments, other.inSituExperiments) && Objects.equals(rnaSeqAssays, other.rnaSeqAssays) @@ -341,18 +226,13 @@ public String toString() { StringBuilder builder = new StringBuilder(); builder.append("RawDataContainer [requestedDataTypes=").append(this.getRequestedDataTypes()) .append(", dataTypesWithResults=").append(this.getDataTypesWithResults()) - .append(", affymetrixExperiments=").append(affymetrixExperiments) - .append(", affymetrixAssays=").append(affymetrixAssays) - .append(", affymetrixCalls=").append(affymetrixCalls) .append(", rnaSeqExperiments=").append(rnaSeqExperiments) .append(", rnaSeqLibraries=").append(rnaSeqLibraries) .append(", rnaSeqAssays=").append(rnaSeqAssays) .append(", rnaSeqCalls=").append(rnaSeqCalls) .append(", inSituExperiments=").append(inSituExperiments) .append(", inSituAssays=").append(inSituAssays) - .append(", inSituCalls=").append(inSituCalls) - .append(", estAssays=").append(estAssays) - .append(", estCalls=").append(estCalls).append("]"); + .append(", inSituCalls=").append(inSituCalls).append("]"); return builder.toString(); } } \ No newline at end of file diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/RawDataCountContainer.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/RawDataCountContainer.java index 306ddf05e..d1126fa76 100644 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/RawDataCountContainer.java +++ b/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/RawDataCountContainer.java @@ -21,37 +21,24 @@ public class RawDataCountContainer extends DataContainer { private final static Logger log = LogManager.getLogger(RawDataCountContainer.class.getName()); private static EnumSet computeRequestedDataTypes( - Integer affymetrixExperimentCount, Integer affymetrixAssayCount, - Integer affymetrixCallCount, Integer insituExperimentCount, Integer insituAssayCount, - Integer insituCallCount, Integer estAssayCount, Integer estCallCount, + Integer insituExperimentCount, Integer insituAssayCount, Integer insituCallCount, Integer bulkRnaSeqExperimentCount, Integer bulkRnaSeqAssayCount, Integer bulkRnaSeqLibraryCount, Integer bulkRnaSeqCallCount, Integer singleCellRnaSeqExperimentCount, Integer singleCellRnaSeqAssayCount, Integer singleCellRnaSeqLibraryCount, Integer singleCellRnaSeqCallCount) { - log.traceEntry("{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}", - affymetrixExperimentCount, affymetrixAssayCount, affymetrixCallCount, + log.traceEntry("{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}", insituExperimentCount, insituAssayCount, insituCallCount, - estAssayCount, estCallCount, bulkRnaSeqExperimentCount, bulkRnaSeqAssayCount, bulkRnaSeqLibraryCount, bulkRnaSeqCallCount, singleCellRnaSeqExperimentCount, singleCellRnaSeqAssayCount, singleCellRnaSeqLibraryCount, singleCellRnaSeqCallCount); EnumSet requestedDataTypes = EnumSet.noneOf(DataType.class); - if (affymetrixExperimentCount != null || - affymetrixAssayCount != null || - affymetrixCallCount != null) { - requestedDataTypes.add(DataType.AFFYMETRIX); - } if (insituExperimentCount != null || insituAssayCount != null || insituCallCount != null) { requestedDataTypes.add(DataType.IN_SITU); } - if (estAssayCount != null || - estCallCount != null) { - requestedDataTypes.add(DataType.EST); - } if (bulkRnaSeqExperimentCount != null || bulkRnaSeqAssayCount != null || bulkRnaSeqLibraryCount != null || @@ -68,37 +55,24 @@ private static EnumSet computeRequestedDataTypes( return log.traceExit(requestedDataTypes); } private static EnumSet computeDataTypesWithResults( - Integer affymetrixExperimentCount, Integer affymetrixAssayCount, - Integer affymetrixCallCount, Integer insituExperimentCount, Integer insituAssayCount, - Integer insituCallCount, Integer estAssayCount, Integer estCallCount, + Integer insituExperimentCount, Integer insituAssayCount, Integer insituCallCount, Integer bulkRnaSeqExperimentCount, Integer bulkRnaSeqAssayCount, Integer bulkRnaSeqLibraryCount, Integer bulkRnaSeqCallCount, Integer singleCellRnaSeqExperimentCount, Integer singleCellRnaSeqAssayCount, Integer singleCellRnaSeqLibraryCount, Integer singleCellRnaSeqCallCount) { - log.traceEntry("{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}", - affymetrixExperimentCount, affymetrixAssayCount, affymetrixCallCount, + log.traceEntry("{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}", insituExperimentCount, insituAssayCount, insituCallCount, - estAssayCount, estCallCount, bulkRnaSeqExperimentCount, bulkRnaSeqAssayCount, bulkRnaSeqLibraryCount, bulkRnaSeqCallCount, singleCellRnaSeqExperimentCount, singleCellRnaSeqAssayCount, singleCellRnaSeqLibraryCount, singleCellRnaSeqCallCount); EnumSet dataTypesWithResults = EnumSet.noneOf(DataType.class); - if (affymetrixExperimentCount != null && affymetrixExperimentCount != 0 || - affymetrixAssayCount != null && affymetrixAssayCount != 0 || - affymetrixCallCount != null && affymetrixCallCount != 0) { - dataTypesWithResults.add(DataType.AFFYMETRIX); - } if (insituExperimentCount != null && insituExperimentCount != 0 || insituAssayCount != null && insituAssayCount != 0 || insituCallCount != null && insituCallCount != 0) { dataTypesWithResults.add(DataType.IN_SITU); } - if (estAssayCount != null && estAssayCount != 0 || - estCallCount != null && estCallCount != 0) { - dataTypesWithResults.add(DataType.EST); - } if (bulkRnaSeqExperimentCount != null && bulkRnaSeqExperimentCount != 0 || bulkRnaSeqAssayCount != null && bulkRnaSeqAssayCount != 0 || bulkRnaSeqLibraryCount != null && bulkRnaSeqLibraryCount != 0 || @@ -115,14 +89,9 @@ private static EnumSet computeDataTypesWithResults( return log.traceExit(dataTypesWithResults); } - private final Integer affymetrixExperimentCount; - private final Integer affymetrixAssayCount; - private final Integer affymetrixCallCount; private final Integer insituExperimentCount; private final Integer insituAssayCount; private final Integer insituCallCount; - private final Integer estAssayCount; - private final Integer estCallCount; private final Integer bulkRnaSeqExperimentCount; private final Integer bulkRnaSeqAssayCount; private final Integer bulkRnaSeqLibraryCount; @@ -133,41 +102,29 @@ private static EnumSet computeDataTypesWithResults( private final Integer singleCellRnaSeqCallCount; // null if not queried and 0 if no results - public RawDataCountContainer(Integer affymetrixExperimentCount, Integer affymetrixAssayCount, - Integer affymetrixCallCount, Integer insituExperimentCount, Integer insituAssayCount, - Integer insituCallCount, Integer estAssayCount, Integer estCallCount, + public RawDataCountContainer(Integer insituExperimentCount, Integer insituAssayCount, + Integer insituCallCount, Integer bulkRnaSeqExperimentCount, Integer bulkRnaSeqAssayCount, Integer bulkRnaSeqLibraryCount, Integer bulkRnaSeqCallCount, Integer singleCellRnaSeqExperimentCount, Integer singleCellRnaSeqAssayCount, Integer singleCellRnaSeqLibraryCount, Integer singleCellRnaSeqCallCount) { super(computeRequestedDataTypes( - affymetrixExperimentCount, affymetrixAssayCount, affymetrixCallCount, insituExperimentCount, insituAssayCount, insituCallCount, - estAssayCount, estCallCount, bulkRnaSeqExperimentCount, bulkRnaSeqAssayCount, bulkRnaSeqLibraryCount, bulkRnaSeqCallCount, singleCellRnaSeqExperimentCount, singleCellRnaSeqAssayCount, singleCellRnaSeqLibraryCount, singleCellRnaSeqCallCount), computeDataTypesWithResults( - affymetrixExperimentCount, affymetrixAssayCount, affymetrixCallCount, insituExperimentCount, insituAssayCount, insituCallCount, - estAssayCount, estCallCount, bulkRnaSeqExperimentCount, bulkRnaSeqAssayCount, bulkRnaSeqLibraryCount, bulkRnaSeqCallCount, singleCellRnaSeqExperimentCount, singleCellRnaSeqAssayCount, singleCellRnaSeqLibraryCount, singleCellRnaSeqCallCount)); - this.affymetrixExperimentCount = affymetrixExperimentCount; - this.affymetrixAssayCount = affymetrixAssayCount; - this.affymetrixCallCount = affymetrixCallCount; - this.insituExperimentCount = insituExperimentCount; this.insituAssayCount = insituAssayCount; this.insituCallCount = insituCallCount; - this.estAssayCount = estAssayCount; - this.estCallCount = estCallCount; - this.bulkRnaSeqExperimentCount = bulkRnaSeqExperimentCount; this.bulkRnaSeqAssayCount = bulkRnaSeqAssayCount; this.bulkRnaSeqLibraryCount = bulkRnaSeqLibraryCount; @@ -179,15 +136,6 @@ public RawDataCountContainer(Integer affymetrixExperimentCount, Integer affymetr this.singleCellRnaSeqCallCount = singleCellRnaSeqCallCount; } - public Integer getAffymetrixExperimentCount() { - return affymetrixExperimentCount; - } - public Integer getAffymetrixAssayCount() { - return affymetrixAssayCount; - } - public Integer getAffymetrixCallCount() { - return affymetrixCallCount; - } public Integer getInsituExperimentCount() { return insituExperimentCount; } @@ -197,13 +145,7 @@ public Integer getInsituAssayCount() { public Integer getInsituCallCount() { return insituCallCount; } - public Integer getEstAssayCount() { - return estAssayCount; - } - public Integer getEstCallCount() { - return estCallCount; - } - public Integer getBulkRnaSeqExperimentCount() { + public Integer getBulkRnaSeqExperimentCount() { return bulkRnaSeqExperimentCount; } public Integer getBulkRnaSeqAssayCount() { @@ -233,9 +175,8 @@ public int hashCode() { final int prime = 31; int result = super.hashCode(); result = prime * result + Objects.hash( - affymetrixAssayCount, affymetrixCallCount, affymetrixExperimentCount, bulkRnaSeqAssayCount, bulkRnaSeqCallCount, bulkRnaSeqExperimentCount, bulkRnaSeqLibraryCount, - estAssayCount, estCallCount, insituAssayCount, insituCallCount, insituExperimentCount, + insituAssayCount, insituCallCount, insituExperimentCount, singleCellRnaSeqAssayCount, singleCellRnaSeqCallCount, singleCellRnaSeqExperimentCount, singleCellRnaSeqLibraryCount); return result; @@ -249,15 +190,10 @@ public boolean equals(Object obj) { if (getClass() != obj.getClass()) return false; RawDataCountContainer other = (RawDataCountContainer) obj; - return Objects.equals(affymetrixAssayCount, other.affymetrixAssayCount) - && Objects.equals(affymetrixCallCount, other.affymetrixCallCount) - && Objects.equals(affymetrixExperimentCount, other.affymetrixExperimentCount) - && Objects.equals(bulkRnaSeqAssayCount, other.bulkRnaSeqAssayCount) + return Objects.equals(bulkRnaSeqAssayCount, other.bulkRnaSeqAssayCount) && Objects.equals(bulkRnaSeqCallCount, other.bulkRnaSeqCallCount) && Objects.equals(bulkRnaSeqExperimentCount, other.bulkRnaSeqExperimentCount) && Objects.equals(bulkRnaSeqLibraryCount, other.bulkRnaSeqLibraryCount) - && Objects.equals(estAssayCount, other.estAssayCount) - && Objects.equals(estCallCount, other.estCallCount) && Objects.equals(insituAssayCount, other.insituAssayCount) && Objects.equals(insituCallCount, other.insituCallCount) && Objects.equals(insituExperimentCount, other.insituExperimentCount) @@ -271,14 +207,9 @@ public boolean equals(Object obj) { public String toString() { StringBuilder builder = new StringBuilder(); builder.append("RawDataCountContainer [") - .append("affymetrixExperimentCount=").append(affymetrixExperimentCount) - .append(", affymetrixAssayCount=").append(affymetrixAssayCount) - .append(", affymetrixCallCount=").append(affymetrixCallCount) .append(", insituExperimentCount=").append(insituExperimentCount) .append(", insituAssayCount=").append(insituAssayCount) .append(", insituCallCount=").append(insituCallCount) - .append(", estAssayCount=").append(estAssayCount) - .append(", estCallCount=").append(estCallCount) .append(", bulkRnaSeqExperimentCount=").append(bulkRnaSeqExperimentCount) .append(", bulkRnaSeqAssayCount=").append(bulkRnaSeqAssayCount) .append(", bulkRnaSeqCallCount=").append(bulkRnaSeqCallCount) diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/RawDataLoader.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/RawDataLoader.java index 4734142ea..0b00370d1 100644 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/RawDataLoader.java +++ b/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/RawDataLoader.java @@ -27,15 +27,9 @@ import org.bgee.model.dao.api.expressiondata.DAODataType; import org.bgee.model.dao.api.expressiondata.rawdata.DAORawDataFilter; import org.bgee.model.dao.api.expressiondata.rawdata.RawDataConditionDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.RawDataCountDAO; import org.bgee.model.dao.api.expressiondata.rawdata.RawDataConditionDAO.RawDataConditionTOResultSet; +import org.bgee.model.dao.api.expressiondata.rawdata.RawDataCountDAO; import org.bgee.model.dao.api.expressiondata.rawdata.RawDataCountDAO.RawDataCountContainerTO; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTDAO.ESTTO; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTDAO.ESTTOResultSet; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTLibraryDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTLibraryDAO.ESTLibraryTO; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTLibraryDAO.ESTLibraryTOResultSet; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituEvidenceDAO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituEvidenceDAO.InSituEvidenceTO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituEvidenceDAO.InSituEvidenceTOResultSet; @@ -44,14 +38,6 @@ import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituSpotDAO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituSpotDAO.InSituSpotTO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituSpotDAO.InSituSpotTOResultSet; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipDAO.AffymetrixChipTO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipDAO.AffymetrixChipTOResultSet; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixProbesetDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixProbesetDAO.AffymetrixProbesetTO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixProbesetDAO.AffymetrixProbesetTOResultSet; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.MicroarrayExperimentDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.MicroarrayExperimentDAO.MicroarrayExperimentTOResultSet; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqExperimentDAO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqExperimentDAO.RNASeqExperimentTOResultSet; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqLibraryAnnotatedSampleDAO; @@ -69,32 +55,23 @@ import org.bgee.model.expressiondata.rawdata.baseelements.CellCompartment; import org.bgee.model.expressiondata.rawdata.baseelements.Experiment; import org.bgee.model.expressiondata.rawdata.baseelements.RawCall; +import org.bgee.model.expressiondata.rawdata.baseelements.RawCall.ExclusionReason; import org.bgee.model.expressiondata.rawdata.baseelements.RawDataAnnotation; import org.bgee.model.expressiondata.rawdata.baseelements.RawDataAuthorAnnotation; import org.bgee.model.expressiondata.rawdata.baseelements.RawDataCondition; +import org.bgee.model.expressiondata.rawdata.baseelements.RawDataCondition.RawDataSex; import org.bgee.model.expressiondata.rawdata.baseelements.RawDataContainer; import org.bgee.model.expressiondata.rawdata.baseelements.RawDataContainerWithExperiment; import org.bgee.model.expressiondata.rawdata.baseelements.RawDataCountContainer; -import org.bgee.model.expressiondata.rawdata.baseelements.RawCall.ExclusionReason; -import org.bgee.model.expressiondata.rawdata.baseelements.RawDataCondition.RawDataSex; import org.bgee.model.expressiondata.rawdata.baseelements.RawDataDataType; import org.bgee.model.expressiondata.rawdata.baseelements.SequencedTranscriptPart; import org.bgee.model.expressiondata.rawdata.baseelements.Strand; -import org.bgee.model.expressiondata.rawdata.est.EST; -import org.bgee.model.expressiondata.rawdata.est.ESTContainer; -import org.bgee.model.expressiondata.rawdata.est.ESTCountContainer; -import org.bgee.model.expressiondata.rawdata.est.ESTDataType; -import org.bgee.model.expressiondata.rawdata.est.ESTLibrary; import org.bgee.model.expressiondata.rawdata.insitu.InSituContainer; import org.bgee.model.expressiondata.rawdata.insitu.InSituCountContainer; import org.bgee.model.expressiondata.rawdata.insitu.InSituDataType; import org.bgee.model.expressiondata.rawdata.insitu.InSituEvidence; import org.bgee.model.expressiondata.rawdata.insitu.InSituExperiment; import org.bgee.model.expressiondata.rawdata.insitu.InSituSpot; -import org.bgee.model.expressiondata.rawdata.microarray.AffymetrixChip; -import org.bgee.model.expressiondata.rawdata.microarray.AffymetrixChipPipelineSummary; -import org.bgee.model.expressiondata.rawdata.microarray.AffymetrixExperiment; -import org.bgee.model.expressiondata.rawdata.microarray.AffymetrixProbeset; import org.bgee.model.expressiondata.rawdata.rnaseq.RnaSeqContainer; import org.bgee.model.expressiondata.rawdata.rnaseq.RnaSeqCountContainer; import org.bgee.model.expressiondata.rawdata.rnaseq.RnaSeqDataType; @@ -104,12 +81,9 @@ import org.bgee.model.expressiondata.rawdata.rnaseq.RnaSeqLibraryAnnotatedSamplePipelineSummary; import org.bgee.model.expressiondata.rawdata.rnaseq.RnaSeqLibraryPipelineSummary; import org.bgee.model.expressiondata.rawdata.rnaseq.RnaSeqResultAnnotatedSample; +import org.bgee.model.expressiondata.rawdata.rnaseq.RnaSeqResultAnnotatedSample.AbundanceUnit; import org.bgee.model.expressiondata.rawdata.rnaseq.RnaSeqTechnology; import org.bgee.model.file.DownloadFileService; -import org.bgee.model.expressiondata.rawdata.rnaseq.RnaSeqResultAnnotatedSample.AbundanceUnit; -import org.bgee.model.expressiondata.rawdata.microarray.AffymetrixContainer; -import org.bgee.model.expressiondata.rawdata.microarray.AffymetrixCountContainer; -import org.bgee.model.expressiondata.rawdata.microarray.AffymetrixDataType; import org.bgee.model.gene.Gene; import org.bgee.model.gene.GeneBioType; import org.bgee.model.source.Source; @@ -226,15 +200,10 @@ public static final EnumSet convertToDataTypeSet(Collection convertToDataTypeSet(Collection convertToDataTypeSet(Collection rawDataContainerClass = rawDataDataType.getRawDataContainerClass(); T rawDataContainer = null; switch (requestedDataType) { - case AFFYMETRIX: - rawDataContainer = rawDataContainerClass.cast( - this.loadAffymetrixData(infoType, offset, limit, partialInfo)); - break; case RNA_SEQ: case SC_RNA_SEQ: rawDataContainer = rawDataContainerClass.cast( @@ -399,10 +359,6 @@ public static final EnumSet convertToDataTypeSet(Collection T loadDataCount(Collection T loadDataCount(Collection rawDataDataType, return log.traceExit(RawDataPostFilter.merge(expAssayFilter, condFilter)); } -//***************************************************************************************** -// METHODS LOADING AFFYMETRIX RAW DATA -//***************************************************************************************** - - //Long and Integer instead of long and int because used internally to retrieve all results for filtering - private AffymetrixContainer loadAffymetrixData(InformationType infoType, Long offset, Integer limit, - boolean partialInfo) { - log.traceEntry("{}, {}, {}, {}", infoType, offset, limit, partialInfo); - - //If the DaoRawDataFilters are null it means there was no matching conds - //and thus no result for sure - if (this.getRawDataProcessedFilter().getDaoFilters() == null) { - return log.traceExit(this.getNoResultAffymetrixContainer(infoType)); - } - - //************************************************************ - // First, we retrieve all necessary TransferObjects - //************************************************************ - LinkedHashSet affyProbesetTOs = new LinkedHashSet<>(); - LinkedHashSet affyChipTOs = new LinkedHashSet<>(); - Set bgeeChipIds = new HashSet<>(); - Set bgeeGeneIds = new HashSet<>(); - Set daoRawDataFilters = this.getRawDataProcessedFilter() - .getDaoFilters(); - - //*********** Calls *********** - if (infoType == InformationType.CALL) { - AffymetrixProbesetTOResultSet probesetTORS = this.affymetrixProbesetDAO.getAffymetrixProbesets( - daoRawDataFilters, offset, limit, null); - while (probesetTORS.next()) { - AffymetrixProbesetTO probesetTO = probesetTORS.getTO(); - bgeeChipIds.add(probesetTO.getAssayId()); - bgeeGeneIds.add(probesetTO.getBgeeGeneId()); - affyProbesetTOs.add(probesetTO); - } - } - - //*********** Assays *********** - AffymetrixChipTOResultSet chipTORS = null; - Set affyExpIds = new HashSet<>(); - Set rawDataCondIds = new HashSet<>(); - //We need to write the test in this way, in case CALLs were requested, but there was - //no result retrieved - if (!bgeeChipIds.isEmpty()) { - assert !partialInfo; - chipTORS = this.affymetrixChipDAO.getAffymetrixChipsFromBgeeChipIds(bgeeChipIds, - null); - } else if (infoType == InformationType.ASSAY) { - chipTORS = this.affymetrixChipDAO.getAffymetrixChips(daoRawDataFilters, offset, limit, - !partialInfo? null: - Set.of(AffymetrixChipDAO.Attribute.EXPERIMENT_ID, - AffymetrixChipDAO.Attribute.AFFYMETRIX_CHIP_ID, - AffymetrixChipDAO.Attribute.BGEE_AFFYMETRIX_CHIP_ID)); - } - //We need to identify the species for providing download links for experiments. - //If no gene and no conditions were retrieved, we have to make a special query for it. - else if (infoType == InformationType.EXPERIMENT && !partialInfo) { - chipTORS = this.affymetrixChipDAO.getAffymetrixChips(daoRawDataFilters, 0L, 1, - Set.of(AffymetrixChipDAO.Attribute.CONDITION_ID)); - } - if (chipTORS != null) { - while (chipTORS.next()) { - AffymetrixChipTO chipTO = chipTORS.getTO(); - if (chipTO.getExperimentId() != null) { - affyExpIds.add(chipTO.getExperimentId()); - } - if (chipTO.getConditionId() != null) { - rawDataCondIds.add(chipTO.getConditionId()); - } - if (infoType != InformationType.EXPERIMENT) { - affyChipTOs.add(chipTO); - } - } - } - - - //************************************************************ - // Now, we load missing Genes and RawDataConditions - //************************************************************ - this.updateRawDataConditionMap(rawDataCondIds); - this.updateGeneMap(bgeeGeneIds); - - - //*********** Experiments *********** - MicroarrayExperimentTOResultSet expTORS = null; - //Experiments should always be retrieved at this point if there is any result, - //but we need this check in case there was no result returned when requesting - //CALLs or ASSAYs. - if (!affyExpIds.isEmpty()) { - //we can use a new DAORawDataFilter to retrieve the requested experiments - expTORS = this.microarrayExperimentDAO.getExperiments( - Set.of(new DAORawDataFilter(affyExpIds, null, null, null)), null, null, - !partialInfo? null: - Set.of(MicroarrayExperimentDAO.Attribute.ID, - MicroarrayExperimentDAO.Attribute.NAME)); - } else if (infoType == InformationType.EXPERIMENT) { - //otherwise, it was the information requested originally - expTORS = this.microarrayExperimentDAO.getExperiments(daoRawDataFilters, offset, limit, - !partialInfo? null: - Set.of(MicroarrayExperimentDAO.Attribute.ID, - MicroarrayExperimentDAO.Attribute.NAME)); - } - LinkedHashMap expIdToAffyExp = - expTORS == null? new LinkedHashMap<>(): - expTORS.stream() - .collect(Collectors.toMap( - to -> to.getId(), - to -> new AffymetrixExperiment(to.getId(), to.getName(), - to.getDescription(), - to.getDataSourceId() == null? null: getSourceById(to.getDataSourceId()), - this.downloadFileService.loadExperimentDownloadFiles(to.getId(), DataType.AFFYMETRIX), - 0), - (v1, v2) -> {throw new IllegalStateException("No key collision possible");}, - LinkedHashMap::new)); - - - //************************************************************ - // Finally, we instantiate all bgee-core objects necessary - //************************************************************ - //Experiments are always needed - LinkedHashSet affymetrixExperiments = - new LinkedHashSet<>(expIdToAffyExp.values()); - - //Now we load the LinkedHashSets only if needed, to distinguish between - //null value = info not requested, and empty Collection = no result - LinkedHashSet affymetrixAssays = null; - LinkedHashSet affymetrixCalls = null; - if (infoType == InformationType.ASSAY || infoType == InformationType.CALL) { - //We create a Map bgeeChipId -> AffymetrixChip for easier instantiation - //of AffymetrixProbesets - LinkedHashMap affyChipMap = affyChipTOs - .stream() - .collect(Collectors.toMap( - to -> to.getId(), - - to -> new AffymetrixChip( - to.getAffymetrixChipId(), - Optional.ofNullable(expIdToAffyExp.get(to.getExperimentId())) - .orElseThrow(() -> new IllegalStateException( - "Missing experiment ID " + to.getExperimentId() - + " for chip ID " + to.getAffymetrixChipId())), - to.getConditionId() == null? null: new RawDataAnnotation( - Optional.ofNullable( - this.rawDataConditionMap.get(to.getConditionId())) - .orElseThrow(() -> new IllegalStateException( - "Missing RawDataCondition ID " - + to.getConditionId() - + " for chip ID " + to.getAffymetrixChipId())), - null, null, null, null, null), - null, - to.getDistinctRankCount() == null? null: new AffymetrixChipPipelineSummary( - to.getDistinctRankCount(), to.getMaxRank(), to.getScanDate(), - to.getNormalizationType().getStringRepresentation(), - to.getQualityScore(), to.getPercentPresent())), - - (v1, v2) -> {throw new IllegalStateException("No key collision possible");}, - LinkedHashMap::new)); - affymetrixAssays = new LinkedHashSet<>(affyChipMap.values()); - - if (infoType == InformationType.CALL) { - affymetrixCalls = affyProbesetTOs - .stream() - .map(to -> new AffymetrixProbeset( - to.getId(), - Optional.ofNullable(affyChipMap.get(to.getAssayId())) - .orElseThrow(() -> new IllegalStateException( - "Missing chip ID " + to.getAssayId() - + " for probeset ID " + to.getId())), - new RawCall( - Optional.ofNullable(geneMap.get(to.getBgeeGeneId())) - .orElseThrow(() -> new IllegalStateException( - "Missing gene ID " + to.getBgeeGeneId() - + " for probeset ID " + to.getId())), - to.getPValue(), - to.getExpressionConfidence(), - ExclusionReason.convertToExclusionReason( - to.getExclusionReason().name())), - to.getNormalizedSignalIntensity(), to.getqValue(), to.getRank())) - .collect(Collectors.toCollection(LinkedHashSet::new)); - } - } - - return log.traceExit(new AffymetrixContainer( - affymetrixExperiments, affymetrixAssays, affymetrixCalls)); - } - private AffymetrixContainer getNoResultAffymetrixContainer(InformationType infoType) { - log.traceEntry("{}", infoType); - - return log.traceExit(new AffymetrixContainer( - //Experiments always end up being requested - Set.of(), - //We also get the chips when we request the probesets - infoType == InformationType.CALL || infoType == InformationType.ASSAY? Set.of(): null, - infoType == InformationType.CALL? Set.of(): null)); - } - - private AffymetrixCountContainer loadAffymetrixCount(boolean withExperiment, - boolean withAssay, boolean withCall) { - log.traceEntry("{}, {}, {}", withExperiment, withAssay, withCall); - - //If the DaoRawDataFilters are null it means there was no matching conds - //and thus no result for sure - if (this.getRawDataProcessedFilter().getDaoFilters() == null) { - return log.traceExit(new AffymetrixCountContainer( - withExperiment? 0: null, - withAssay? 0: null, - withCall? 0: null)); - } - - RawDataCountContainerTO countTO = this.rawDataCountDAO.getAffymetrixCount( - this.getRawDataProcessedFilter().getDaoFilters(), - withExperiment, withAssay, withCall); - - return log.traceExit(new AffymetrixCountContainer( - countTO.getExperimentCount(), - countTO.getAssayCount(), - countTO.getCallCount())); - } - //***************************************************************************************** // METHODS LOADING RNA-SEQ RAW DATA //***************************************************************************************** @@ -1053,162 +782,6 @@ private RnaSeqCountContainer loadRnaSeqCount(boolean isSingleCell, boolean withE countTO.getCallCount())); } -//***************************************************************************************** -// METHODS LOADING EST RAW DATA -//***************************************************************************************** - - //Long and Integer instead of long and int because used internally to retrieve all results for filtering - private ESTContainer loadESTData(InformationType infoType, Long offset, Integer limit, - boolean partialInfo) { - log.traceEntry("{}, {}, {}, {}", infoType, offset, limit, partialInfo); - - //If the DaoRawDataFilters are null it means there was no matching conds - //and thus no result for sure - if (this.getRawDataProcessedFilter().getDaoFilters() == null) { - return log.traceExit(this.getNoResultESTContainer(infoType)); - } - - //************************************************************ - // First, we retrieve all necessary TransferObjects - //************************************************************ - LinkedHashSet callTOs = new LinkedHashSet<>(); - LinkedHashSet assayTOs = new LinkedHashSet<>(); - Set daoRawDataFilters = this.getRawDataProcessedFilter() - .getDaoFilters(); - - //*********** Calls *********** - Set estLibraryIds = new HashSet<>(); - Set bgeeGeneIds = new HashSet<>(); - if (infoType == InformationType.CALL) { - ESTTOResultSet callTORS = this.estDAO.getESTs(daoRawDataFilters, offset, limit, null); - while (callTORS.next()) { - ESTTO callTO = callTORS.getTO(); - estLibraryIds.add(callTO.getAssayId()); - bgeeGeneIds.add(callTO.getBgeeGeneId()); - callTOs.add(callTO); - } - } - - //*********** Assays *********** - ESTLibraryTOResultSet assayTORS = null; - //We need to write the test in this way, in case CALLs were requested, but there was - //no result retrieved - if (!estLibraryIds.isEmpty()) { - assert !partialInfo; - //Create a new DAORawDataFilter for retrieving libraries based on their ID - DAORawDataFilter daoFilter = new DAORawDataFilter(null, estLibraryIds, null, null); - assayTORS = this.estLibraryDAO.getESTLibraries(Set.of(daoFilter), null, null, - null); - - // For EST, it is equivalent to request for assays or for experiments, - //since there are no experiments - } else if (infoType == InformationType.ASSAY || infoType == InformationType.EXPERIMENT) { - assayTORS = this.estLibraryDAO.getESTLibraries(daoRawDataFilters, offset, limit, - !partialInfo? null: Set.of( - ESTLibraryDAO.Attribute.ID, - ESTLibraryDAO.Attribute.NAME)); - } - Set rawDataCondIds = new HashSet<>(); - if (assayTORS != null) { - while (assayTORS.next()) { - ESTLibraryTO assayTO = assayTORS.getTO(); - if (assayTO.getConditionId() != null) { - rawDataCondIds.add(assayTO.getConditionId()); - } - assayTOs.add(assayTO); - } - } - - //************************************************************ - // Now, we load missing Genes and RawDataConditions - //************************************************************ - this.updateRawDataConditionMap(rawDataCondIds); - this.updateGeneMap(bgeeGeneIds); - - //************************************************************ - // Finally, we instantiate all bgee-core objects necessary - //************************************************************ - - LinkedHashMap libIdToLib = assayTOs.stream() - .collect(Collectors.toMap( - to -> to.getId(), - - to -> new ESTLibrary( - to.getId(), to.getName(), to.getDescription(), - to.getConditionId() == null? null: new RawDataAnnotation( - Optional.ofNullable( - this.rawDataConditionMap.get( - to.getConditionId())) - .orElseThrow(() -> new IllegalStateException( - "Missing RawDataCondition ID " - + to.getConditionId() - + " for annotated sample ID " + to.getId())), - null, null, null, null, null), - to.getDataSourceId() == null? null: getSourceById(to.getDataSourceId())), - - (v1, v2) -> {throw new IllegalStateException("No key collision possible");}, - LinkedHashMap::new)); - - - //Libraries are always needed - LinkedHashSet estLibraries = - new LinkedHashSet<>(libIdToLib.values()); - - //Now we load the LinkedHashSets only if needed, to distinguish between - //null value = info not requested, and empty Collection = no result - LinkedHashSet calls = null; - if (infoType == InformationType.CALL) { - calls = callTOs.stream() - .map(to -> new EST( - to.getId(), - Optional.ofNullable(libIdToLib.get(to.getAssayId())) - .orElseThrow(() -> new IllegalStateException( - "Missing assay ID " + to.getAssayId() - + " for Bgee gene ID " + to.getBgeeGeneId())), - new RawCall( - Optional.ofNullable(this.geneMap.get(to.getBgeeGeneId())) - .orElseThrow(() -> new IllegalStateException( - "Missing gene ID " + to.getBgeeGeneId() - + " for assay ID " + to.getAssayId())), - to.getPValue(), - to.getExpressionConfidence(), - ExclusionReason.convertToExclusionReason( - to.getExclusionReason().name())))) - .collect(Collectors.toCollection(LinkedHashSet::new)); - } - - return log.traceExit(new ESTContainer(estLibraries, calls)); - } - private ESTContainer getNoResultESTContainer(InformationType infoType) { - log.traceEntry("{}", infoType); - - return log.traceExit(new ESTContainer( - //Libraries always end up being requested - Set.of(), - infoType == InformationType.CALL? Set.of(): null)); - } - - private ESTCountContainer loadESTCount(boolean withExperiment, boolean withAssay, - boolean withCall) { - log.traceEntry("{}, {}, {}", withExperiment, withAssay, withCall); - - //If the DaoRawDataFilters are null it means there was no matching conds - //and thus no result for sure - if (this.getRawDataProcessedFilter().getDaoFilters() == null) { - return log.traceExit(new ESTCountContainer( - withExperiment || withAssay? 0: null, - withCall? 0: null)); - } - - RawDataCountContainerTO countTO = rawDataCountDAO.getESTCount( - this.getRawDataProcessedFilter().getDaoFilters(), - withExperiment || withAssay, withCall); - - return log.traceExit(new ESTCountContainer( - countTO.getAssayCount(), - countTO.getCallCount())); - } - //***************************************************************************************** // METHODS LOADING IN SITU RAW DATA //***************************************************************************************** @@ -1604,12 +1177,8 @@ private DAODataType convertRawDataDataTypeToDAODataType(RawDataDataType dt if (dt == null) { return log.traceExit((DAODataType) null); } - if (dt instanceof AffymetrixDataType) { - return log.traceExit(DAODataType.AFFYMETRIX); - } else if (dt instanceof RnaSeqDataType) { + if (dt instanceof RnaSeqDataType) { return log.traceExit(DAODataType.RNA_SEQ); - } else if (dt instanceof ESTDataType) { - return log.traceExit(DAODataType.EST); } else if (dt instanceof InSituDataType) { return log.traceExit(DAODataType.IN_SITU); } diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/baseelements/RawCall.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/baseelements/RawCall.java index 06b50ec10..2acb0560b 100644 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/baseelements/RawCall.java +++ b/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/baseelements/RawCall.java @@ -35,7 +35,7 @@ public class RawCall { public enum ExclusionReason implements BgeeEnumField { NOT_EXCLUDED("not excluded"), PRE_FILTERING("pre-filtering"), UNDEFINED("undefined"), NO_EXPRESSION_CONFLICT("noExpression conflict"), - ABSENT_NOT_RELIABLE("absent call not reliable"); + BIOTYPE_NOT_TARGETED("biotype not targeted"); /** * Convert the {@code String} representation of a exclusion reason (for instance, diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/baseelements/RawDataDataType.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/baseelements/RawDataDataType.java index d0376ca2b..512cb09d5 100644 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/baseelements/RawDataDataType.java +++ b/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/baseelements/RawDataDataType.java @@ -9,9 +9,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bgee.model.expressiondata.baseelements.DataType; -import org.bgee.model.expressiondata.rawdata.est.ESTDataType; import org.bgee.model.expressiondata.rawdata.insitu.InSituDataType; -import org.bgee.model.expressiondata.rawdata.microarray.AffymetrixDataType; import org.bgee.model.expressiondata.rawdata.rnaseq.RnaSeqDataType; /** @@ -40,33 +38,26 @@ public abstract class RawDataDataType, U extends RawDataCountContainer> { private final static Logger log = LogManager.getLogger(RawDataDataType.class.getName()); - public final static AffymetrixDataType AFFYMETRIX = new AffymetrixDataType(); public final static RnaSeqDataType BULK_RNA_SEQ = new RnaSeqDataType(DataType.RNA_SEQ); public final static RnaSeqDataType SC_RNA_SEQ = new RnaSeqDataType(DataType.SC_RNA_SEQ); - public final static ESTDataType EST = new ESTDataType(); public final static InSituDataType IN_SITU = new InSituDataType(); private final static LinkedHashSet> ALL_OF = new LinkedHashSet<>(Arrays.asList( - AFFYMETRIX, BULK_RNA_SEQ, SC_RNA_SEQ, - EST, IN_SITU)); private final static Map> DATA_TYPE_TO_RAW_DATA_DATA_TYPE = Collections.unmodifiableMap(Map.ofEntries( - Map.entry(AFFYMETRIX.getDataType(), AFFYMETRIX), Map.entry(BULK_RNA_SEQ.getDataType(), BULK_RNA_SEQ), Map.entry(SC_RNA_SEQ.getDataType(), SC_RNA_SEQ), - Map.entry(EST.getDataType(), EST), Map.entry(IN_SITU.getDataType(), IN_SITU))); private final static Map, ? extends RawDataCountContainerWithExperiment>> DATA_TYPE_TO_RAW_DATA_DATA_TYPE_WITH_EXPERIMENT = Collections.unmodifiableMap(Map.ofEntries( - Map.entry(AFFYMETRIX.getDataType(), AFFYMETRIX), Map.entry(BULK_RNA_SEQ.getDataType(), BULK_RNA_SEQ), Map.entry(SC_RNA_SEQ.getDataType(), SC_RNA_SEQ), Map.entry(IN_SITU.getDataType(), IN_SITU))); diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/est/EST.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/est/EST.java deleted file mode 100644 index 8bc1120d2..000000000 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/est/EST.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.bgee.model.expressiondata.rawdata.est; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bgee.model.Entity; -import org.bgee.model.expressiondata.rawdata.baseelements.RawCall; -import org.bgee.model.expressiondata.rawdata.baseelements.RawCallSource; - -public class EST extends Entity implements RawCallSource{ - - private final static Logger log = LogManager.getLogger(EST.class); - private final ESTLibrary assay; - private final RawCall rawCall; - - @Override - public ESTLibrary getAssay() { - return this.assay; - } - - @Override - public RawCall getRawCall() { - return this.rawCall; - } - - public EST(String id, ESTLibrary assay, RawCall rawCall) throws IllegalArgumentException { - super(id); - if (assay == null) { - throw log.throwing(new IllegalArgumentException("ESTLibrary cannot be null")); - } - this.assay = assay; - if (rawCall == null) { - throw log.throwing(new IllegalArgumentException("RawCall cannot be null")); - } - this.rawCall = rawCall; - } - -} diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/est/ESTContainer.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/est/ESTContainer.java deleted file mode 100644 index 7fc47b529..000000000 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/est/ESTContainer.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.bgee.model.expressiondata.rawdata.est; - -import java.util.Collection; - -import org.bgee.model.expressiondata.rawdata.baseelements.RawDataContainer; - -/** - * A {@code RawDataContainer} for EST. - * - * @author Frederic Bastian - * @version Bgee 15.0, Dec. 2022 - * @since Bgee 15.0, Dec. 2022 - */ -public class ESTContainer extends RawDataContainer { - - public ESTContainer(Collection assays, Collection calls) { - super(assays, calls); - } -} diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/est/ESTCountContainer.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/est/ESTCountContainer.java deleted file mode 100644 index 913d6ff9e..000000000 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/est/ESTCountContainer.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.bgee.model.expressiondata.rawdata.est; - -import org.bgee.model.expressiondata.rawdata.baseelements.RawDataCountContainer; - -/** - * A {@code RawDataCountContainer} for EST. - * - * @author Frederic Bastian - * @version Bgee 15.0, Dec. 2022 - * @since Bgee 15.0, Dec. 2022 - */ -public class ESTCountContainer extends RawDataCountContainer { - - public ESTCountContainer(Integer assayCount, Integer callCount) { - super(assayCount, callCount); - } -} diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/est/ESTDataType.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/est/ESTDataType.java deleted file mode 100644 index 8a0912d7e..000000000 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/est/ESTDataType.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.bgee.model.expressiondata.rawdata.est; - -import org.bgee.model.expressiondata.baseelements.DataType; -import org.bgee.model.expressiondata.rawdata.baseelements.Assay; -import org.bgee.model.expressiondata.rawdata.baseelements.RawDataDataType; - -/** - * A {@code RawDataDataType} specific to EST. - *

- * The typical way to obtain an object from this class is to use the public static attribute - * {@link org.bgee.model.expressiondata.rawdata.baseelements.RawDataDataType#EST - * RawDataDataType#EST}, in order to mimic an {@code enum}. - * - * @author Frederic Bastian - * @version Bgee 15.0, Dec. 2022 - * @since Bgee 15.0, Dec. 2022 - * @see org.bgee.model.expressiondata.rawdata.baseelements.RawDataDataType RawDataDataType - */ -public class ESTDataType extends RawDataDataType { - - public ESTDataType() { - super(DataType.EST, ESTContainer.class, ESTCountContainer.class); - } - - @Override - public String getAssayId(Assay a) throws IllegalArgumentException { - if (!(a instanceof ESTLibrary)) { - throw new IllegalArgumentException("Assay is not an ESTLibrary"); - } - return ((ESTLibrary) a).getId(); - } - @Override - public String getAssayName(Assay a) throws IllegalArgumentException { - if (!(a instanceof ESTLibrary)) { - throw new IllegalArgumentException("Assay is not an ESTLibrary"); - } - return ((ESTLibrary) a).getName(); - } - - @Override - public boolean isInformativeAssayId() { - return true; - } - @Override - public boolean isInformativeAssayName() { - return true; - } - @Override - public boolean isInformativeExperimentName() { - return true; - } -} diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/est/ESTLibrary.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/est/ESTLibrary.java deleted file mode 100644 index 47105d102..000000000 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/est/ESTLibrary.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.bgee.model.expressiondata.rawdata.est; - -import org.bgee.model.NamedEntity; -import org.bgee.model.XRef; -import org.bgee.model.expressiondata.rawdata.baseelements.Assay; -import org.bgee.model.expressiondata.rawdata.baseelements.RawDataAnnotated; -import org.bgee.model.expressiondata.rawdata.baseelements.RawDataAnnotation; -import org.bgee.model.expressiondata.rawdata.baseelements.RawDataWithDataSource; -import org.bgee.model.source.Source; - -public class ESTLibrary extends NamedEntity implements Assay, RawDataWithDataSource, - RawDataAnnotated{ - - private final RawDataAnnotation annotation; - private final Source dataSource; - private final XRef xRef; - - public ESTLibrary(String id, String name, String description, RawDataAnnotation annotation, - Source dataSource) throws IllegalArgumentException { - super(id, name, description); - this.annotation = annotation; - this.dataSource = dataSource; - if (dataSource != null) { - this.xRef = new XRef(id.toString(), name, dataSource, dataSource.getExperimentUrl()); - } else { - this.xRef = null; - } - } - - @Override - public RawDataAnnotation getAnnotation() { - return this.annotation; - } - @Override - public Source getDataSource() { - return this.dataSource; - } - @Override - public XRef getXRef() { - return this.xRef; - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("ESTLibrary [id=").append(getId()) - .append(", name=").append(getName()) - .append(", description=").append(getDescription()) - .append(", annotation=").append(annotation) - .append(", dataSource=").append(this.dataSource) - .append(", xRef=").append(this.xRef) - .append("]"); - return builder.toString(); - } -} diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/microarray/AffymetrixChip.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/microarray/AffymetrixChip.java deleted file mode 100644 index a176a0171..000000000 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/microarray/AffymetrixChip.java +++ /dev/null @@ -1,93 +0,0 @@ -package org.bgee.model.expressiondata.rawdata.microarray; - -import java.util.Objects; - -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bgee.model.expressiondata.rawdata.baseelements.AssayPartOfExp; -import org.bgee.model.expressiondata.rawdata.baseelements.RawDataAnnotated; -import org.bgee.model.expressiondata.rawdata.baseelements.RawDataAnnotation; - -//AffymetrixChip IDs are not unique, they are unique inside a given experiment. -//This is why this class does not extend Entity. -public class AffymetrixChip implements AssayPartOfExp, RawDataAnnotated { - private final static Logger log = LogManager.getLogger(AffymetrixChip.class.getName()); - - private final String id; - private final AffymetrixExperiment experiment; - private final RawDataAnnotation annotation; - private final ChipType chipType; - private final AffymetrixChipPipelineSummary pipelineSummary; - - /** - * @param id A {@code String} that is the ID of the {@code AffymetrixChip} - * @throws IllegalArgumentException If {@code id} is blank, or {@code experiment} is {@code null}. - */ - public AffymetrixChip(String id, AffymetrixExperiment experiment, RawDataAnnotation annotation, - ChipType chipType, AffymetrixChipPipelineSummary pipelineSummary) - throws IllegalArgumentException { - if (StringUtils.isBlank(id)) { - throw log.throwing(new IllegalArgumentException("ID cannot be blank")); - } - this.id = id; - if (experiment == null) { - throw log.throwing(new IllegalArgumentException("Experiment cannot be null")); - } - this.experiment = experiment; - this.annotation = annotation; - this.chipType = chipType; - this.pipelineSummary = pipelineSummary; - } - - public String getId() { - return this.id; - } - @Override - public AffymetrixExperiment getExperiment() { - return this.experiment; - } - @Override - public RawDataAnnotation getAnnotation() { - return this.annotation; - } - - public ChipType getChipType() { - return chipType; - } - - public AffymetrixChipPipelineSummary getPipelineSummary() { - return pipelineSummary; - } - - //AffymetrixChip IDs are not unique, they are unique inside a given experiment. - //We use the ID and the experiment as primary key - @Override - public int hashCode() { - return Objects.hash(experiment, id); - } - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - AffymetrixChip other = (AffymetrixChip) obj; - return Objects.equals(experiment, other.experiment) && Objects.equals(id, other.id); - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("AffymetrixChip [") - .append("id=").append(id) - .append(", experiment=").append(experiment) - .append(", annotation=").append(annotation) - .append(", chipType=").append(chipType) - .append(", pipelineSummary=").append(pipelineSummary) - .append("]"); - return builder.toString(); - } -} \ No newline at end of file diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/microarray/AffymetrixChipPipelineSummary.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/microarray/AffymetrixChipPipelineSummary.java deleted file mode 100644 index cfd4046a6..000000000 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/microarray/AffymetrixChipPipelineSummary.java +++ /dev/null @@ -1,72 +0,0 @@ -package org.bgee.model.expressiondata.rawdata.microarray; - -import java.math.BigDecimal; -import java.util.Objects; - -import org.bgee.model.expressiondata.rawdata.baseelements.RawDataPipelineSummary; - -/** - * Class describing pipeline summary for affymetrix chips - * - * @author Julien Wollbrett - * @version Bgee 15, Nov. 2022 - */ -public class AffymetrixChipPipelineSummary extends RawDataPipelineSummary{ - - private final String scanDate; - private final String normalizationType; - private final BigDecimal qualityScore; - private final BigDecimal percentPresent; - - public AffymetrixChipPipelineSummary(Integer distinctRankCount, BigDecimal maxRank, String scanDate, - String normalizationType, BigDecimal qualityScore, BigDecimal percentPresent) { - super(distinctRankCount, maxRank); - this.scanDate = scanDate; - this.normalizationType = normalizationType; - this.qualityScore = qualityScore; - this.percentPresent = percentPresent; - } - public String getScanDate() { - return scanDate; - } - public String getNormalizationType() { - return normalizationType; - } - public BigDecimal getQualityScore() { - return qualityScore; - } - public BigDecimal getPercentPresent() { - return percentPresent; - } - @Override - public int hashCode() { - final int prime = 31; - int result = super.hashCode(); - result = prime * result + Objects.hash(normalizationType, percentPresent, qualityScore, scanDate); - return result; - } - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (!super.equals(obj)) - return false; - if (getClass() != obj.getClass()) - return false; - AffymetrixChipPipelineSummary other = (AffymetrixChipPipelineSummary) obj; - return Objects.equals(normalizationType, other.normalizationType) - && Objects.equals(percentPresent, other.percentPresent) - && Objects.equals(qualityScore, other.qualityScore) && - Objects.equals(scanDate, other.scanDate); - } - @Override - public String toString() { - return "AffymetrixChipPipelineSummary [scanDate=" + scanDate + ", normalizationType=" - + normalizationType + ", qualityScore=" + qualityScore + ", percentPresent=" - + percentPresent + ", distinctRankCount=" + getDistinctRankCount() + ", maxRank=" - + getMaxRank() + "]"; - } - - - -} diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/microarray/AffymetrixContainer.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/microarray/AffymetrixContainer.java deleted file mode 100644 index bde9839e4..000000000 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/microarray/AffymetrixContainer.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.bgee.model.expressiondata.rawdata.microarray; - -import java.util.Collection; - -import org.bgee.model.expressiondata.rawdata.baseelements.RawDataContainerWithExperiment; - -/** - * A {@code RawDataContainerWithExperiment} for Affymetrix. - * - * @author Frederic Bastian - * @version Bgee 15.0, Nov. 2022 - * @since Bgee 15.0, Nov. 2022 - */ -public class AffymetrixContainer -extends RawDataContainerWithExperiment { - - public AffymetrixContainer(Collection experiments, - Collection assays, Collection calls) { - super(experiments, assays, calls); - } -} \ No newline at end of file diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/microarray/AffymetrixCountContainer.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/microarray/AffymetrixCountContainer.java deleted file mode 100644 index 4db44630c..000000000 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/microarray/AffymetrixCountContainer.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.bgee.model.expressiondata.rawdata.microarray; - -import org.bgee.model.expressiondata.rawdata.baseelements.RawDataCountContainerWithExperiment; - -/** - * A {@code RawDataCountContainerWithExperiment} for Affymetrix. - * - * @author Frederic Bastian - * @version Bgee 15.0, Nov. 2022 - * @since Bgee 15.0, Nov. 2022 - */ -public class AffymetrixCountContainer extends RawDataCountContainerWithExperiment { - - public AffymetrixCountContainer(Integer experimentCount, - Integer assayCount, Integer callCount) { - super(experimentCount, assayCount, callCount); - } -} \ No newline at end of file diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/microarray/AffymetrixDataType.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/microarray/AffymetrixDataType.java deleted file mode 100644 index e0a016788..000000000 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/microarray/AffymetrixDataType.java +++ /dev/null @@ -1,54 +0,0 @@ -package org.bgee.model.expressiondata.rawdata.microarray; - -import org.bgee.model.expressiondata.baseelements.DataType; -import org.bgee.model.expressiondata.rawdata.baseelements.Assay; -import org.bgee.model.expressiondata.rawdata.baseelements.RawDataDataType; - -/** - * A {@code RawDataDataType} specific to Affymetrix. - *

- * The typical way to obtain an object from this class is to use the public static attribute - * {@link org.bgee.model.expressiondata.rawdata.baseelements.RawDataDataType#AFFYMETRIX - * RawDataDataType#AFFYMETRIX}, in order to mimic an {@code enum}. - * - * @author Frederic Bastian - * @version Bgee 15.0, Nov. 2022 - * @since Bgee 15.0, Nov. 2022 - * @see org.bgee.model.expressiondata.rawdata.baseelements.RawDataDataType RawDataDataType - */ -public class AffymetrixDataType extends RawDataDataType { - - public AffymetrixDataType() { - super(DataType.AFFYMETRIX, AffymetrixContainer.class, - AffymetrixCountContainer.class); - } - - @Override - public String getAssayId(Assay a) throws IllegalArgumentException { - if (!(a instanceof AffymetrixChip)) { - throw new IllegalArgumentException("Assay is not an AffymetrixChip"); - } - return ((AffymetrixChip) a).getId(); - } - @Override - public String getAssayName(Assay a) throws IllegalArgumentException { - if (!(a instanceof AffymetrixChip)) { - throw new IllegalArgumentException("Assay is not an AffymetrixChip"); - } - return ((AffymetrixChip) a).getId(); - } - - @Override - public boolean isInformativeAssayId() { - return true; - } - @Override - public boolean isInformativeAssayName() { - return false; - } - @Override - public boolean isInformativeExperimentName() { - return true; - } -} diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/microarray/AffymetrixExperiment.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/microarray/AffymetrixExperiment.java deleted file mode 100644 index 02e892b72..000000000 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/microarray/AffymetrixExperiment.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.bgee.model.expressiondata.rawdata.microarray; - -import java.util.List; - -import org.bgee.model.expressiondata.rawdata.baseelements.ExperimentWithDataDownload; -import org.bgee.model.file.ExperimentDownloadFile; -import org.bgee.model.source.Source; - -public class AffymetrixExperiment extends ExperimentWithDataDownload { - - public AffymetrixExperiment(String id, String name, String description, Source dataSource, - List downloadFiles, int assayCount) - throws IllegalArgumentException { - //DOI is set to null as it is not yet provided for in affymetrix data - super(id, name, description, null, dataSource, downloadFiles, assayCount); - } - - //we do not reimplement hashCode/equals but use the 'NamedEntity' implementation from 'Experiment' inheritance -} diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/microarray/AffymetrixProbeset.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/microarray/AffymetrixProbeset.java deleted file mode 100644 index fc1139fb1..000000000 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/microarray/AffymetrixProbeset.java +++ /dev/null @@ -1,98 +0,0 @@ -package org.bgee.model.expressiondata.rawdata.microarray; - -import java.math.BigDecimal; - -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bgee.model.expressiondata.rawdata.baseelements.RawCall; -import org.bgee.model.expressiondata.rawdata.baseelements.RawCallSource; - -public class AffymetrixProbeset implements RawCallSource { - private final static Logger log = LogManager.getLogger(AffymetrixProbeset.class.getName()); - - private final String id; - private final AffymetrixChip assay; - private final RawCall rawCall; - private final BigDecimal normalizedSignalIntensity; - private final BigDecimal qValue; - private final BigDecimal rank; - - public AffymetrixProbeset(String id, AffymetrixChip assay, RawCall rawCall, - BigDecimal normalizedSignalIntensity, BigDecimal qValue, BigDecimal rank) { - if (StringUtils.isBlank(id)) { - throw log.throwing(new IllegalArgumentException("ID cannot be blank")); - } - this.id = id; - if (rawCall == null) { - throw log.throwing(new IllegalArgumentException("RawCall cannot be null")); - } - this.rawCall = rawCall; - this.assay = assay; - this.normalizedSignalIntensity = normalizedSignalIntensity; - this.qValue = qValue; - this.rank = rank; - } - - public String getId() { - return this.id; - } - @Override - public AffymetrixChip getAssay() { - return this.assay; - } - @Override - public RawCall getRawCall() { - return this.rawCall; - } - public BigDecimal getNormalizedSignalIntensity() { - return normalizedSignalIntensity; - } - public BigDecimal getqValue() { - return qValue; - } - public BigDecimal getRank() { - return rank; - } - - //AffymetrixProbeset IDs are not unique, they are unique inside a given AffymetrixChip. - //This is why we reimplement hashCode/equals rather than using the 'Entity' implementation. - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((assay == null) ? 0 : assay.hashCode()); - result = prime * result + ((id == null) ? 0 : id.hashCode()); - return result; - } - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - AffymetrixProbeset other = (AffymetrixProbeset) obj; - if (assay == null) { - if (other.assay != null) - return false; - } else if (!assay.equals(other.assay)) - return false; - if (id == null) { - if (other.id != null) - return false; - } else if (!id.equals(other.id)) - return false; - return true; - } - - @Override - public String toString() { - return "AffymetrixProbeset [id=" + id + ", assay=" + assay + ", rawCall=" + rawCall - + ", normalizedSignalIntensity=" + normalizedSignalIntensity + ", qValue=" + qValue + ", rank=" + rank - + "]"; - } - - -} diff --git a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/microarray/ChipType.java b/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/microarray/ChipType.java deleted file mode 100644 index b003477c4..000000000 --- a/bgee-core/src/main/java/org/bgee/model/expressiondata/rawdata/microarray/ChipType.java +++ /dev/null @@ -1,61 +0,0 @@ -package org.bgee.model.expressiondata.rawdata.microarray; - -import java.math.BigDecimal; -import java.util.Objects; - -import org.bgee.model.Entity; - -public class ChipType extends Entity{ - - private final String name; - private final String cdfName; - private final boolean isCompatible; - private final BigDecimal qualityScoreThreshold; - private final BigDecimal percentPresentThreshold; - private final BigDecimal maxRank; - - public ChipType(String id, String name, String cdfName, boolean isCompatible, BigDecimal qualityScoreThreshold, - BigDecimal percentPresentThreshold, BigDecimal maxRank) throws IllegalArgumentException { - super(id); - this.name = name; - this.cdfName = cdfName; - this.isCompatible = isCompatible; - this.qualityScoreThreshold = qualityScoreThreshold; - this.percentPresentThreshold = percentPresentThreshold; - this.maxRank = maxRank; - } - - - public String getName() { - return name; - } - - public String getCdfName() { - return cdfName; - } - - public boolean isCompatible() { - return isCompatible; - } - - public BigDecimal getQualityScoreThreshold() { - return qualityScoreThreshold; - } - - public BigDecimal getPercentPresentThreshold() { - return percentPresentThreshold; - } - - public BigDecimal getMaxRank() { - return maxRank; - } - - //ChiType IDs are unique. Reuse hashCode/equals from 'Entity'. - @Override - public String toString() { - return "ChipType [name=" + name + ", cdfName=" + cdfName + ", isCompatible=" + isCompatible - + ", qualityScoreThreshold=" + qualityScoreThreshold + ", percentPresentThreshold=" - + percentPresentThreshold + ", maxRank=" + maxRank + "]"; - } - -} diff --git a/bgee-core/src/main/java/org/bgee/model/file/DownloadFileService.java b/bgee-core/src/main/java/org/bgee/model/file/DownloadFileService.java index f94488184..eb1e5325c 100644 --- a/bgee-core/src/main/java/org/bgee/model/file/DownloadFileService.java +++ b/bgee-core/src/main/java/org/bgee/model/file/DownloadFileService.java @@ -78,7 +78,7 @@ public List loadExperimentDownloadFiles(String experimen } //We have no experiment download files for data types other than AFFYMETRIX, RNA_SEQ, SC_RNA_SEQ, //so we return an empty list. - if (!dataType.equals(DataType.AFFYMETRIX) && !dataType.equals(DataType.RNA_SEQ) + if (!dataType.equals(DataType.RNA_SEQ) && !dataType.equals(DataType.SC_RNA_SEQ)) { return log.traceExit(List.of()); } @@ -96,31 +96,13 @@ public List loadExperimentDownloadFiles(String experimen Set speciesIds = condDAO.getRawDataConditionsLinkedToDataType( Set.of(filter), daoDataType, - dataType.equals(DataType.AFFYMETRIX)? null: (dataType.equals(DataType.SC_RNA_SEQ)? true: false), + dataType.equals(DataType.SC_RNA_SEQ)? true: false, EnumSet.of(RawDataConditionDAO.Attribute.SPECIES_ID) ).stream() .map(condTO -> condTO.getSpeciesId()) .collect(Collectors.toSet()); Set species = this.getServiceFactory().getSpeciesService().loadSpeciesByIds(speciesIds, false); - if (dataType.equals(DataType.AFFYMETRIX)) { - return log.traceExit(species.stream() - .map(s -> new ExperimentDownloadFile( - this.getServiceFactory().getBgeeProperties().getDownloadAffyProcExprValueFilesRootDirectory() + - s.getSpeciesFullNameWithoutSpace() + "/", - s.getSpeciesFullNameWithoutSpace() + - "_Affymetrix_probesets_" + experimentId + ".tar.gz", - "Affymetrix processed expression values in " + s.getScientificName()/* + - " in experiment " + experimentId*/, - //Since we don't store yet this info to database, we don't have access to the file size - 0L, - ExperimentDownloadFile.Category.ANNOTATED_SAMPLES, - dataType, - false, - s) - ).collect(Collectors.toList())); - } - assert dataType.equals(DataType.RNA_SEQ) || dataType.equals(DataType.SC_RNA_SEQ); //Bulk or single-cell RNA-Seq experiments can contain multiple species and multiple "data types" @@ -251,10 +233,6 @@ private static SpeciesDownloadFile.Category mapDAOCategoryToServiceCategory( return log.traceExit(SpeciesDownloadFile.Category.DIFF_EXPR_DEV_SIMPLE); case ORTHOLOG: return log.traceExit(SpeciesDownloadFile.Category.ORTHOLOG); - case AFFY_ANNOT: - return log.traceExit(SpeciesDownloadFile.Category.AFFY_ANNOT); - case AFFY_DATA: - return log.traceExit(SpeciesDownloadFile.Category.AFFY_DATA); case RNASEQ_ANNOT: return log.traceExit(SpeciesDownloadFile.Category.RNASEQ_ANNOT); case RNASEQ_DATA: @@ -294,10 +272,6 @@ private static DownloadFileTO.CategoryEnum convertServiceCategoryToDAOCategory( return log.traceExit(DownloadFileTO.CategoryEnum.DIFF_EXPR_DEV_SIMPLE); case ORTHOLOG: return log.traceExit(DownloadFileTO.CategoryEnum.ORTHOLOG); - case AFFY_ANNOT: - return log.traceExit(DownloadFileTO.CategoryEnum.AFFY_ANNOT); - case AFFY_DATA: - return log.traceExit(DownloadFileTO.CategoryEnum.AFFY_DATA); case RNASEQ_ANNOT: return log.traceExit(DownloadFileTO.CategoryEnum.RNASEQ_ANNOT); case RNASEQ_DATA: diff --git a/bgee-core/src/main/java/org/bgee/model/file/SpeciesDataGroupService.java b/bgee-core/src/main/java/org/bgee/model/file/SpeciesDataGroupService.java index 9ef8113e4..a0365f0d0 100644 --- a/bgee-core/src/main/java/org/bgee/model/file/SpeciesDataGroupService.java +++ b/bgee-core/src/main/java/org/bgee/model/file/SpeciesDataGroupService.java @@ -191,8 +191,6 @@ private static SpeciesDataGroup newSpeciesDataGroup(SpeciesDataGroupDAO.SpeciesD private static SpeciesDataGroup newSpeciesDataGroup(Integer id, String name, String description, List species, Set files) { log.traceEntry("{}, {}, {}, {}, {}", id, name, description, species, files); -// files = new HashSet(); -// files.add(new DownloadFile("path", "my_name", DownloadFile.Category.AFFY_ANNOT, 100L, id)); return log.traceExit(new SpeciesDataGroup(id, name, description, species, files)); } diff --git a/bgee-core/src/main/java/org/bgee/model/file/SpeciesDownloadFile.java b/bgee-core/src/main/java/org/bgee/model/file/SpeciesDownloadFile.java index 623a1e5c3..af5149f4d 100644 --- a/bgee-core/src/main/java/org/bgee/model/file/SpeciesDownloadFile.java +++ b/bgee-core/src/main/java/org/bgee/model/file/SpeciesDownloadFile.java @@ -29,8 +29,6 @@ public class SpeciesDownloadFile extends DownloadFile { *

  • {@code DIFF_EXPR_DEV_COMPLETE} a complete differential expression across developmental stages file
    • *
  • {@code DIFF_EXPR_DEV_SIMPLE}a simple differential expression across developmental stages file
    • *
  • {@code ORTHOLOG} corresponds to an orthologies file
    • - *
  • {@code AFFY_ANNOT} corresponds to an Affymetrix annoations file
    • - *
  • {@code AFFY_DATA} corresponds to an Affymetrix signal intensities file
    • *
  • {@code RNASEQ_ANNOT} corresponds to RNA-Seq annotations file
    • *
  • {@code RNASEQ_DATA} corresponds to RNA-Seq data file
    • *
  • {@code FULL_LENGTH_ANNOT} corresponds to full length single cell RNA-Seq annotations file
    • @@ -55,8 +53,6 @@ public enum Category implements BgeeEnumField { DIFF_EXPR_DEV_COMPLETE("diff_expr_dev_complete", true, false), DIFF_EXPR_DEV_SIMPLE("diff_expr_dev_simple", true, false), ORTHOLOG("ortholog", false, true), - AFFY_ANNOT("affy_annot", false, true), - AFFY_DATA("affy_data", false, true), RNASEQ_ANNOT("rnaseq_annot", false, true), RNASEQ_DATA("rnaseq_data", false, true), FULL_LENGTH_ANNOT("full_length_annot", false, true), diff --git a/bgee-core/src/main/java/org/bgee/model/source/SourceService.java b/bgee-core/src/main/java/org/bgee/model/source/SourceService.java index 77910dcf3..1bae87e5f 100644 --- a/bgee-core/src/main/java/org/bgee/model/source/SourceService.java +++ b/bgee-core/src/main/java/org/bgee/model/source/SourceService.java @@ -184,10 +184,6 @@ private static SourceCategory convertSourceCategoryTOToSourceCategory(SourceTO.S return log.traceExit(SourceCategory.PROTEOMICS); case IN_SITU: return log.traceExit(SourceCategory.IN_SITU); - case AFFYMETRIX: - return log.traceExit(SourceCategory.AFFYMETRIX); - case EST: - return log.traceExit(SourceCategory.EST); case RNA_SEQ: return log.traceExit(SourceCategory.RNA_SEQ); case SC_RNA_SEQ: diff --git a/bgee-core/src/test/java/org/bgee/model/TestAncestor.java b/bgee-core/src/test/java/org/bgee/model/TestAncestor.java index 5a3b26df2..21daa8229 100644 --- a/bgee-core/src/test/java/org/bgee/model/TestAncestor.java +++ b/bgee-core/src/test/java/org/bgee/model/TestAncestor.java @@ -149,12 +149,12 @@ private static Date parseDate(String date) { //************************* protected static final Map SPECIES_TOS = unmodifiableLinkedHashMap(List.of( Map.entry(1, new SpeciesTO(1, "spe1", "spe", "1", 1, 100, "genomeFilePath1", - "genomeVersion1", "genomeAssemblyXRef1", 1, 1)), + "genomeVersion1", "genomeAssemblyXRef1", 1, 1, null)), Map.entry(2, new SpeciesTO(2, "spe2", "spe", "2", 2, 100, "genomeFilePath2", - "genomeVersion2", "genomeAssemblyXRef2", 2, 2)), + "genomeVersion2", "genomeAssemblyXRef2", 2, 2, null)), Map.entry(3, new SpeciesTO(3, "spe3", "spe", "3", 3, 200, "genomeFilePath3", "genomeVersion2", "genomeAssemblyXRef2", 2, - 2 //use the same genome as species 2 + 2, null //use the same genome as species 2 )))); protected static Map loadSpeciesMap(boolean withSpeciesSourceInfo) { return unmodifiableLinkedHashMap(SPECIES_TOS.values().stream().map(speciesTO -> @@ -197,8 +197,8 @@ protected static Map loadSpeciesMap(boolean withSpeciesSourceI "geneDescription1", //description 1, //speciesId 1, //geneBioTypeId - 100, //OMAParentNodeId true, //From Ensembl? + "reg1", //seqRegionName 1, //Number of genes with same public ID null )), @@ -206,21 +206,23 @@ protected static Map loadSpeciesMap(boolean withSpeciesSourceI 2, "geneId2", "geneName2", "geneDescription2", 1, //same species as geneId1 2, //alternative geneBioType - 100, true, 1, "expression summary")), + true, "reg1", 1, "expression summary")), Map.entry(3, new GeneTO(3, "geneId3_4", //two different genes with same public ID in species 2 and species 3 "geneName3", "geneDescription3", 2, //species 2 - 1, 100, + 1, true, //species 2 and 3 has a genome from a different database than Ensembl + "reg1", 2, //two different genes with same public ID in species 2 and species 3 null)), Map.entry(4, new GeneTO(4, "geneId3_4", //two different genes with same public ID in species 2 and species 3 "geneName4", "geneDescription4", 2, //species 3 - 1, 100, + 1, false, //species 2 and 3 has a genome from a different database than Ensembl + "reg1", 2, //two different genes with same public ID in species 2 and species 3 null)))); protected static final Map GENE_X_REF_TOS = unmodifiableLinkedHashMap(List.of( diff --git a/bgee-core/src/test/java/org/bgee/model/expressiondata/call/OTFExpressionCallLoaderTest.java b/bgee-core/src/test/java/org/bgee/model/expressiondata/call/OTFExpressionCallLoaderTest.java new file mode 100644 index 000000000..3c0bbab39 --- /dev/null +++ b/bgee-core/src/test/java/org/bgee/model/expressiondata/call/OTFExpressionCallLoaderTest.java @@ -0,0 +1,285 @@ +// tests implemented during the biohackathon +//TODO: adapt them to the new implementation +// +//package org.bgee.model.expressiondata.call; +// +//import static org.junit.Assert.assertEquals; +//import static org.junit.Assert.assertTrue; +// +//import java.math.BigDecimal; +//import java.math.RoundingMode; +//import java.util.Arrays; +//import java.util.Collections; +//import java.util.EnumSet; +//import java.util.List; +//import java.util.Map; +//import java.util.Set; +// +//import org.apache.logging.log4j.LogManager; +//import org.apache.logging.log4j.Logger; +//import org.bgee.model.anatdev.AnatEntity; +//import org.bgee.model.anatdev.DevStage; +//import org.bgee.model.dao.api.expressiondata.call.CallDAO.CallTO.DataState; +//import org.bgee.model.expressiondata.baseelements.DataType; +//import org.bgee.model.expressiondata.baseelements.PropagationState; +//import org.bgee.model.expressiondata.rawdata.baseelements.RawCall; +//import org.bgee.model.expressiondata.rawdata.baseelements.RawDataAnnotation; +//import org.bgee.model.expressiondata.rawdata.baseelements.RawDataCondition; +//import org.bgee.model.expressiondata.rawdata.baseelements.RawDataContainer; +//import org.bgee.model.expressiondata.rawdata.baseelements.RawDataDataType; +//import org.bgee.model.expressiondata.rawdata.baseelements.RawCall.ExclusionReason; +//import org.bgee.model.expressiondata.rawdata.baseelements.RawCallSource; +//import org.bgee.model.expressiondata.rawdata.baseelements.RawDataCondition.RawDataSex; +//import org.bgee.model.expressiondata.rawdata.rnaseq.RnaSeqContainer; +//import org.bgee.model.expressiondata.rawdata.rnaseq.RnaSeqExperiment; +//import org.bgee.model.expressiondata.rawdata.rnaseq.RnaSeqLibrary; +//import org.bgee.model.expressiondata.rawdata.rnaseq.RnaSeqLibraryAnnotatedSample; +//import org.bgee.model.expressiondata.rawdata.rnaseq.RnaSeqLibraryAnnotatedSamplePipelineSummary; +//import org.bgee.model.expressiondata.rawdata.rnaseq.RnaSeqResultAnnotatedSample; +//import org.bgee.model.gene.Gene; +//import org.bgee.model.gene.GeneBioType; +//import org.bgee.model.species.Species; +//import org.junit.Test; +// +//public class OTFExpressionCallLoaderTest { +// private static final Logger log = LogManager.getLogger(OTFExpressionCallLoaderTest.class.getName()); +// +// @Test +// public void testComputeExpressionScore() { +// // Define max rank +// BigDecimal maxRank = new BigDecimal(50); +// +// // Test cases for edge ranks and mid-ranks +// // Rank = 1 (should return 100) +// OTFExpressionCallLoader loader = new OTFExpressionCallLoader(); +// BigDecimal result = loader.computeExpressionScore(BigDecimal.ONE, maxRank); +// assertEquals("Rank 1 should return a score of 100", new BigDecimal("100.00000"), result); +// +// // Rank = maxRank (should return 1) +// result = loader.computeExpressionScore(maxRank, maxRank); +// assertEquals("Max rank should return a score of 1", new BigDecimal("1.00000"), result); +// +// // Rank = 25 (middle rank in this case, should return around 50) +// result = loader.computeExpressionScore(new BigDecimal(25), maxRank); +// assertEquals("Middle rank should return a score of around 50", new BigDecimal("51.51020"), result); +// +// // Rank = 10 (lower part of the range) +// result = loader.computeExpressionScore(new BigDecimal(10), maxRank); +// assertEquals("Rank 10 should return a score of around 82", new BigDecimal("81.81633"), result); +// +// // Rank = 40 (upper part of the range) +// result = loader.computeExpressionScore(new BigDecimal(40), maxRank); +// assertEquals("Rank 40 should return a score of around 20", new BigDecimal("21.20408"), result); +// } +// +// @Test +// public void testTransformToRawDataPerCondition() { +// AnatEntity anatEntity1 = new AnatEntity("anatEntity1"); +// AnatEntity anatEntity2 = new AnatEntity("anatEntity2"); +// AnatEntity cellType = new AnatEntity("cellType"); +// DevStage devStage1 = new DevStage("devStage1"); +// DevStage devStage2 = new DevStage("devStage2"); +// RawDataSex sex = RawDataSex.FEMALE; +// String strain = "wild-type"; +// Species species = new Species(9606); +// +// RawDataCondition rawDataCond1 = new RawDataCondition(anatEntity1, devStage1, cellType, +// sex, strain, species); +// RawDataCondition rawDataCond2 = new RawDataCondition(anatEntity2, devStage1, cellType, +// sex, strain, species); +// RawDataCondition rawDataCond3 = new RawDataCondition(anatEntity1, devStage2, cellType, +// sex, strain, species); +// +// Condition cond1 = new Condition(anatEntity1, null, cellType, null, null, species); +// Condition cond2 = new Condition(anatEntity2, null, cellType, null, null, species); +// +// Map rawDataCondToCond = Map.of( +// rawDataCond1, cond1, +// rawDataCond2, cond2, +// rawDataCond3, cond1); +// +// RawDataAnnotation annot1 = new RawDataAnnotation(rawDataCond1, null, null, null, null, null); +// RawDataAnnotation annot2 = new RawDataAnnotation(rawDataCond2, null, null, null, null, null); +// RawDataAnnotation annot3 = new RawDataAnnotation(rawDataCond3, null, null, null, null, null); +// +// GeneBioType geneBioType = new GeneBioType("geneBioType"); +// Gene gene1 = new Gene("gene1", species, geneBioType); +// RawCall call1 = new RawCall(gene1, new BigDecimal(0.01), DataState.HIGHQUALITY, +// ExclusionReason.NOT_EXCLUDED, new BigDecimal(1)); +// RawCall call2 = new RawCall(gene1, new BigDecimal(0.05), DataState.HIGHQUALITY, +// ExclusionReason.NOT_EXCLUDED, new BigDecimal(2)); +// RawCall call3 = new RawCall(gene1, new BigDecimal(0.1), DataState.HIGHQUALITY, +// ExclusionReason.NOT_EXCLUDED, new BigDecimal(3)); +// +// RnaSeqExperiment rnaSeqExp = new RnaSeqExperiment("rnaSeqExp", null, null, null, null, +// null, 1, false); +// RnaSeqLibrary rnaSeqLib = new RnaSeqLibrary("rnaSeqLib", null, null, rnaSeqExp); +// RnaSeqLibraryAnnotatedSample sample = new RnaSeqLibraryAnnotatedSample(rnaSeqLib, +// annot1, null, null); +// RnaSeqResultAnnotatedSample rnaSeqCall = new RnaSeqResultAnnotatedSample(sample, call1, +// null, null, null, null, null); +// RnaSeqContainer rnaSeqContainer = new RnaSeqContainer(Set.of(rnaSeqExp), +// Set.of(rnaSeqLib), Set.of(sample), Set.of(rnaSeqCall)); +// +// Map, RawDataContainer> rawDataContainers = Map.of( +// RawDataDataType.BULK_RNA_SEQ, rnaSeqContainer); +// +// //Cannot just do an assertEquals on this expected Map because we don't care about the order +// //of the RawCalls in the Lists, we use a List to not loose in a Set the RawCalls that are equal +//// Map, List>> expectedMap = Map.of( +//// cond1, Map.of( +//// RawDataDataType.AFFYMETRIX, List.of(probeset1, probeset1bis, probeset2, probeset4), +//// RawDataDataType.BULK_RNA_SEQ, List.of(rnaSeqCall)), +//// cond2, Map.of(RawDataDataType.AFFYMETRIX, List.of(probeset3))); +// Map>>> transformedMap = OTFExpressionCallLoader +// .transformToRawDataPerCondition(rawDataCondToCond, rawDataContainers); +// assertTrue(transformedMap.keySet().equals(Set.of(cond1, cond2))); +// Map>> dataCond1 = transformedMap.get(cond1); +// List> rnaSeqCond1 = dataCond1.get(DataType.RNA_SEQ); +// assertTrue(rnaSeqCond1.equals(List.of(rnaSeqCall))); +// log.info("TransformedMap: {}", transformedMap); +// } +// +// @Test +// public void testcomputeMedian() { +// OTFExpressionCallLoader loader = new OTFExpressionCallLoader(); +// // Test with an odd-sized list +// List oddList = Arrays.asList( +// new BigDecimal("0.6"), new BigDecimal("0.5"), new BigDecimal("0.4"), new BigDecimal("0.3"), new BigDecimal("0.2") +// ); +// BigDecimal result = loader.computeMedian(oddList); +// assertEquals("The median multiplied by 2 for odd-sized list should be 0.8", new BigDecimal("0.8"), result); +// +// // Test with an even-sized list +// List evenList = Arrays.asList( +// new BigDecimal("0.01"), new BigDecimal("0.07"), new BigDecimal("0.002"), new BigDecimal("0.01") +// ); +// result = loader.computeMedian(evenList); +// assertEquals("The median multiplied by 2 for even-sized list should be 0.02", new BigDecimal("0.02"), result); +// +// // Test with an odd-sized list +// List bigPvalueList = Arrays.asList( +// new BigDecimal("0.9"), new BigDecimal("0.9"), new BigDecimal("0.8"), new BigDecimal("0.9"), new BigDecimal("0.7") +// ); +// result = loader.computeMedian(bigPvalueList); +// assertEquals("The median calculation should return 1 and not above when pValues are too large", new BigDecimal("1"), result); +// +// // Test with a single-element list +// List singleElementList = Collections.singletonList(new BigDecimal("0.05")); +// result = loader.computeMedian(singleElementList); +// assertEquals("The median calculation on a single pValue list should return the same pValue", new BigDecimal("0.05"), result); +// } +// +// @Test +// public void testLoadOTFExpressionCall() { +// AnatEntity anatEntity1 = new AnatEntity("anatEntity1"); +// AnatEntity anatEntity2 = new AnatEntity("anatEntity2"); +// AnatEntity cellType1 = new AnatEntity("cellType1"); +// AnatEntity cellType2 = new AnatEntity("cellType2"); +// DevStage devStage1 = new DevStage("devStage1"); +// RawDataSex sex = RawDataSex.FEMALE; +// String strain = "wild-type"; +// Species species = new Species(9606); +// +// RawDataCondition rawDataCond1 = new RawDataCondition(anatEntity1, devStage1, cellType1, +// sex, strain, species); +// +// GeneBioType geneBioType = new GeneBioType("geneBioType"); +// Gene gene1 = new Gene("gene1", species, geneBioType); +// +// +// Condition cond1 = new Condition(anatEntity1, null, cellType1, null, null, species); +// Condition cond2 = new Condition(anatEntity2, null, cellType1, null, null, species); +// Condition cond3 = new Condition(anatEntity1, null, cellType2, null, null, species); +// +// RawDataAnnotation annot1 = new RawDataAnnotation(rawDataCond1, null, null, null, null, null); +// +// // Initialize raw calls +// RawCall RCaffymetrix = new RawCall(gene1, new BigDecimal("0.20"), DataState.HIGHQUALITY, ExclusionReason.NOT_EXCLUDED, new BigDecimal("200")); +// RawCall RCrnaseq1 = new RawCall(gene1, new BigDecimal("0.25"), DataState.HIGHQUALITY, ExclusionReason.NOT_EXCLUDED, new BigDecimal("1000")); +// RawCall RCrnaseq2 = new RawCall(gene1, new BigDecimal("0.40"), DataState.HIGHQUALITY, ExclusionReason.NOT_EXCLUDED, new BigDecimal("30")); +// +// +// // Initialize experiments and samples +// RnaSeqLibraryAnnotatedSamplePipelineSummary rnaseqsummary1 = new RnaSeqLibraryAnnotatedSamplePipelineSummary(new BigDecimal("0.0003"),new BigDecimal("0.01"), new BigDecimal("0.001"), 100000, 95000, new BigDecimal("50000"), 4500); +// RnaSeqLibraryAnnotatedSamplePipelineSummary rnaseqsummary2 = new RnaSeqLibraryAnnotatedSamplePipelineSummary(new BigDecimal("0.0003"),new BigDecimal("0.01"), new BigDecimal("0.001"), 100000, 95000, new BigDecimal("30000"), 17500); +// +// RnaSeqExperiment rnaSeqExp1 = new RnaSeqExperiment("rnaSeqExp1", null, null, null, null, null, 1, false); +// RnaSeqLibrary rnaSeqLib1 = new RnaSeqLibrary("rnaSeqLib1", null, null, rnaSeqExp1); +// RnaSeqLibraryAnnotatedSample sample1 = new RnaSeqLibraryAnnotatedSample(rnaSeqLib1, annot1, rnaseqsummary1, null); +// +// RnaSeqExperiment rnaSeqExp2 = new RnaSeqExperiment("rnaSeqExp2", null, null, null, null, null, 1, false); +// RnaSeqLibrary rnaSeqLib2 = new RnaSeqLibrary("rnaSeqLib2", null, null, rnaSeqExp2); +// RnaSeqLibraryAnnotatedSample sample2 = new RnaSeqLibraryAnnotatedSample(rnaSeqLib2, annot1, rnaseqsummary2, null); +// +// // Initialize raw data map +// Map>> rawData = Map.of( +// DataType.RNA_SEQ, List.of( +// new RnaSeqResultAnnotatedSample(sample1, RCrnaseq1, null, null, null, null, null), +// new RnaSeqResultAnnotatedSample(sample2, RCrnaseq2, null, null, null, null, null) +// ) +// ); +// +// +// OTFExpressionCall childCall1 = new OTFExpressionCall(gene1, cond2, +// EnumSet.of(DataType.RNA_SEQ), +// new BigDecimal("0.10"), new BigDecimal("0.15"), +// new BigDecimal("0.001"), new BigDecimal("0.003"), +// new BigDecimal("20000"), new BigDecimal("12"), +// new BigDecimal("10000"), new BigDecimal("90"), +// PropagationState.SELF_AND_DESCENDANT); +// +// OTFExpressionCall childCall2 = new OTFExpressionCall(gene1, cond3, +// EnumSet.of(DataType.RNA_SEQ), +// new BigDecimal("0.50"), new BigDecimal("0.70"), +// new BigDecimal("0.25"), new BigDecimal("0.35"), +// new BigDecimal("10000"), new BigDecimal("15"), +// new BigDecimal("50000"), new BigDecimal("40"), +// PropagationState.SELF_AND_DESCENDANT); +// +// OTFExpressionCall expectedCall = new OTFExpressionCall(gene1, cond1, +// EnumSet.of(DataType.RNA_SEQ), +// new BigDecimal("0.45").setScale(50, RoundingMode.HALF_UP), new BigDecimal("0.55").setScale(50, RoundingMode.HALF_UP), +// new BigDecimal("0.001"), new BigDecimal("0.003"), +// new BigDecimal("52100"), new BigDecimal("49.51"), +// new BigDecimal("10000"), new BigDecimal("90"), +// PropagationState.SELF_AND_DESCENDANT); +// +// OTFExpressionCallLoader loader = new OTFExpressionCallLoader(); +// OTFExpressionCall testCall = loader.loadOTFExpressionCall(gene1, cond1, rawData, Set.of(childCall1, childCall2)); +// +// // Logging attributes for debugging +// log.debug("Expected Call Attributes: Gene = {}, Condition = {}, Supporting Data Types = {}, Trusted DataType P-Value = {}, All DataType P-Value = {}, Best Descendant Trusted DataType P-Value = {}, Best Descendant All DataType P-Value = {}, Expression Score Weight = {}, Expression Score = {}, Best Descendant Expression Score Weight = {}, Best Descendant Expression Score = {}, Propagation State = {}", +// expectedCall.getGene(), +// expectedCall.getCondition(), +// expectedCall.getSupportingDataTypes(), +// expectedCall.getTrustedDataTypePValue(), +// expectedCall.getAllDataTypePValue(), +// expectedCall.getBestDescendantTrustedDataTypePValue(), +// expectedCall.getBestDescendantAllDataTypePValue(), +// expectedCall.getExpressionScoreWeight(), +// expectedCall.getExpressionScore(), +// expectedCall.getBestDescendantExpressionScoreWeight(), +// expectedCall.getBestDescendantExpressionScore(), +// expectedCall.getDataPropagation()); +// +// log.debug("Test Call Attributes: Gene = {}, Condition = {}, Supporting Data Types = {}, Trusted DataType P-Value = {}, All DataType P-Value = {}, Best Descendant Trusted DataType P-Value = {}, Best Descendant All DataType P-Value = {}, Expression Score Weight = {}, Expression Score = {}, Best Descendant Expression Score Weight = {}, Best Descendant Expression Score = {}, Propagation State = {}", +// testCall.getGene(), +// testCall.getCondition(), +// testCall.getSupportingDataTypes(), +// testCall.getTrustedDataTypePValue(), +// testCall.getAllDataTypePValue(), +// testCall.getBestDescendantTrustedDataTypePValue(), +// testCall.getBestDescendantAllDataTypePValue(), +// testCall.getExpressionScoreWeight(), +// testCall.getExpressionScore(), +// testCall.getBestDescendantExpressionScoreWeight(), +// testCall.getBestDescendantExpressionScore(), +// testCall.getDataPropagation()); +// +// assertEquals(expectedCall, testCall); +// +// +// } +//} +// diff --git a/bgee-core/src/test/java/org/bgee/model/file/DownloadFileServiceTest.java b/bgee-core/src/test/java/org/bgee/model/file/DownloadFileServiceTest.java index 20a81cd3e..6074aa541 100644 --- a/bgee-core/src/test/java/org/bgee/model/file/DownloadFileServiceTest.java +++ b/bgee-core/src/test/java/org/bgee/model/file/DownloadFileServiceTest.java @@ -38,7 +38,7 @@ public void testGetAllDownlaodFiles() { DownloadFileDAO downloadFileDaoMock = mock(DownloadFileDAO.class); DownloadFileTOResultSet resultSetMock = getMockResultSet(DownloadFileTOResultSet.class, Arrays.asList(new DownloadFileTO(1, "NAME", "DESC", "/tmp/foo", Long.valueOf(42), - DownloadFileTO.CategoryEnum.AFFY_ANNOT, 22, + DownloadFileTO.CategoryEnum.RNASEQ_ANNOT, 22, Arrays.asList(ConditionDAO.Attribute.ANAT_ENTITY_ID)), new DownloadFileTO(2, "NAME2", "DESC2", "/tmp/foo", Long.valueOf(1337), DownloadFileTO.CategoryEnum.DIFF_EXPR_ANAT_COMPLETE, 22, @@ -51,7 +51,7 @@ public void testGetAllDownlaodFiles() { //expected values List expected = Arrays.asList( - new SpeciesDownloadFile("/tmp/foo", "NAME", null, 42L, Category.AFFY_ANNOT, 22, + new SpeciesDownloadFile("/tmp/foo", "NAME", null, 42L, Category.DROPLET_BASED_ANNOT, 22, Arrays.asList(CallService.Attribute.ANAT_ENTITY_ID)), new SpeciesDownloadFile("/tmp/foo", "NAME2", null, 1337L, Category.DIFF_EXPR_ANAT_COMPLETE, 22, Arrays.asList(CallService.Attribute.ANAT_ENTITY_ID, diff --git a/bgee-core/src/test/java/org/bgee/model/file/SpeciesDataGroupServiceTest.java b/bgee-core/src/test/java/org/bgee/model/file/SpeciesDataGroupServiceTest.java index 9d464becc..96996b333 100644 --- a/bgee-core/src/test/java/org/bgee/model/file/SpeciesDataGroupServiceTest.java +++ b/bgee-core/src/test/java/org/bgee/model/file/SpeciesDataGroupServiceTest.java @@ -56,9 +56,9 @@ public void testGetAllDatagroups() { Set species = new HashSet<>(); species.add(v1); species.add(v2); - SpeciesDownloadFile df1 = new SpeciesDownloadFile("/tmp/foo", "NAME", null, 42L, Category.AFFY_ANNOT, 22); - SpeciesDownloadFile df2 = new SpeciesDownloadFile("/tmp/foo2", "NAME2", null, 1337L, Category.DIFF_EXPR_ANAT_COMPLETE, 22); - SpeciesDownloadFile df3 = new SpeciesDownloadFile("/tmp/foo3", "NAME", null, 42L, Category.AFFY_ANNOT, 42); + SpeciesDownloadFile df1 = new SpeciesDownloadFile("/tmp/foo", "NAME", null, 42L, Category.RNASEQ_ANNOT, 22); + SpeciesDownloadFile df2 = new SpeciesDownloadFile("/tmp/foo2", "NAME2", null, 1337L, Category.DROPLET_BASED_H5AD, 22); + SpeciesDownloadFile df3 = new SpeciesDownloadFile("/tmp/foo3", "NAME", null, 42L, Category.RNASEQ_DATA, 42); List downloadFiles = Arrays.asList(df1,df2,df3); diff --git a/bgee-core/src/test/java/org/bgee/model/gene/GeneHomologsServiceTest.java b/bgee-core/src/test/java/org/bgee/model/gene/GeneHomologsServiceTest.java index 2be9d7a8b..6a9d0e1ee 100644 --- a/bgee-core/src/test/java/org/bgee/model/gene/GeneHomologsServiceTest.java +++ b/bgee-core/src/test/java/org/bgee/model/gene/GeneHomologsServiceTest.java @@ -60,9 +60,9 @@ public void testGetOrthologies() { GeneDAO geneDao = mock(GeneDAO.class); when(managerMock.getGeneDAO()).thenReturn(geneDao); GeneTOResultSet mockGeneRs = getMockResultSet(GeneTOResultSet.class, - Arrays.asList(new GeneTO(123, "ID1", "Name1", "Desc1", 11, 1, 1, true, 1, null), - new GeneTO(124, "ID2", "Name2", "Desc2", 22, 1, 1, true, 1, null), - new GeneTO(223, "ID4", "Name4", "Desc4", 44, 2, 1, true, 1, null))); + Arrays.asList(new GeneTO(123, "ID1", "Name1", "Desc1", 11, 1, true, "reg1", 1, null), + new GeneTO(124, "ID2", "Name2", "Desc2", 22, 1, true, "reg1", 1, null), + new GeneTO(223, "ID4", "Name4", "Desc4", 44, 2, true, "reg1", 1, null))); when(geneDao.getGenesBySpeciesIds(null)).thenReturn(mockGeneRs); GeneBioTypeTOResultSet mockBioTypeRs = getMockResultSet(GeneBioTypeTOResultSet.class, Arrays.asList(new GeneBioTypeTO(1, "type1"), new GeneBioTypeTO(2, "type2"))); diff --git a/bgee-core/src/test/java/org/bgee/model/gene/GeneServiceTest.java b/bgee-core/src/test/java/org/bgee/model/gene/GeneServiceTest.java index d3938b8cb..9037e284a 100644 --- a/bgee-core/src/test/java/org/bgee/model/gene/GeneServiceTest.java +++ b/bgee-core/src/test/java/org/bgee/model/gene/GeneServiceTest.java @@ -69,18 +69,18 @@ public void shouldLoadGenes() { when(managerMock.getSpeciesDAO()).thenReturn(speciesDAO); SpeciesTOResultSet mockSpeciesRs = getMockResultSet(SpeciesTOResultSet.class, Arrays.asList( - new SpeciesTO(11, null, null, null, null, null, null, null, null, 1, null), - new SpeciesTO(22, null, null, null, null, null, null, null, null, 1, null), - new SpeciesTO(44, null, null, null, null, null, null, null, null, 1, null))); + new SpeciesTO(11, null, null, null, null, null, null, null, null, 1, null, null), + new SpeciesTO(22, null, null, null, null, null, null, null, null, 1, null, null), + new SpeciesTO(44, null, null, null, null, null, null, null, null, 1, null, null))); when(speciesDAO.getSpeciesByIds(filtersToMap.keySet(), null)).thenReturn(mockSpeciesRs); // Mock GeneDAO GeneDAO dao = mock(GeneDAO.class); when(managerMock.getGeneDAO()).thenReturn(dao); GeneTOResultSet mockGeneRs = getMockResultSet(GeneTOResultSet.class, - Arrays.asList(new GeneTO(1, "ID1", "Name1", "Desc1", 11, 1, 1, true, 1, null), - new GeneTO(2, "ID2", "Name2", "Desc2", 22, 1, 1, true, 1, null), - new GeneTO(4, "ID4", "Name4", "Desc4", 44, 2, 1, true, 1, null))); + Arrays.asList(new GeneTO(1, "ID1", "Name1", "Desc1", 11, 1, true, "reg1", 1, null), + new GeneTO(2, "ID2", "Name2", "Desc2", 22, 1, true, "reg1", 1, null), + new GeneTO(4, "ID4", "Name4", "Desc4", 44, 2, true, "reg1", 1, null))); when(dao.getGenesBySpeciesAndGeneIds(filtersToMap, false)).thenReturn(mockGeneRs); GeneBioTypeTOResultSet mockBioTypeRs = getMockResultSet(GeneBioTypeTOResultSet.class, Arrays.asList(new GeneBioTypeTO(1, "type1"), new GeneBioTypeTO(2, "type2"))); @@ -228,7 +228,7 @@ public void shouldLoadGeneById() { // Mock gene DAO GeneTOResultSet mockGeneRs = getMockResultSet(GeneTOResultSet.class, - Arrays.asList(new GeneTO(bgeeGeneId, geneId, "Name1", "", 10090, 1, 1, true, 1, null))); + Arrays.asList(new GeneTO(bgeeGeneId, geneId, "Name1", "", 10090, 1, true, "reg1", 1, null))); when(geneDao.getGenesByGeneIds(new HashSet(Arrays.asList(geneId)))).thenReturn(mockGeneRs); // Mock species service diff --git a/bgee-core/src/test/java/org/bgee/model/source/SourceServiceTest.java b/bgee-core/src/test/java/org/bgee/model/source/SourceServiceTest.java index b3c205e46..1ce25bc14 100644 --- a/bgee-core/src/test/java/org/bgee/model/source/SourceServiceTest.java +++ b/bgee-core/src/test/java/org/bgee/model/source/SourceServiceTest.java @@ -71,9 +71,7 @@ public void shouldLoadSources() { Arrays.asList( new SourceToSpeciesTO(2, 11, DAODataType.IN_SITU, InfoType.ANNOTATION), new SourceToSpeciesTO(2, 11, DAODataType.RNA_SEQ, InfoType.ANNOTATION), - new SourceToSpeciesTO(2, 11, DAODataType.IN_SITU, InfoType.DATA), - new SourceToSpeciesTO(2, 21, DAODataType.EST, InfoType.DATA), - new SourceToSpeciesTO(4, 11, DAODataType.AFFYMETRIX, InfoType.DATA))); + new SourceToSpeciesTO(2, 11, DAODataType.IN_SITU, InfoType.DATA))); when(sourceToSpeciesDao.getAllSourceToSpecies(null)).thenReturn(mockSourceToSpeciesRs); List expectedSources = new ArrayList(); @@ -93,14 +91,12 @@ public void shouldLoadSources() { expectedSources.clear(); Map> forData2 = new HashMap<>(); forData2.put(11, new HashSet(Arrays.asList(DataType.IN_SITU))); - forData2.put(21, new HashSet(Arrays.asList(DataType.EST))); Map> forAnnot2 = new HashMap<>(); forAnnot2.put(11, new HashSet(Arrays.asList(DataType.IN_SITU, DataType.RNA_SEQ))); expectedSources.add(new Source(2, "NCBI Taxonomy", "Source taxonomy used in Bgee", "", "", "", "https://www.ncbi.nlm.nih.gov/taxonomy", date1, "v13", false, org.bgee.model.source.SourceCategory.NONE, 3, forData2, forAnnot2)); Map> forData4 = new HashMap<>(); - forData4.put(11, new HashSet(Arrays.asList(DataType.AFFYMETRIX))); expectedSources.add(new Source(4, "ZFIN", "ZFIN desc", "https://zfin.org/[xref_id]", "https://zfin.org/[experiment_id]", @@ -136,8 +132,8 @@ public void shouldLoadDisplayableSources() { when(managerMock.getSourceToSpeciesDAO()).thenReturn(sourceToSpeciesDao); SourceToSpeciesTOResultSet mockSourceToSpeciesRs = getMockResultSet(SourceToSpeciesTOResultSet.class, Arrays.asList( - new SourceToSpeciesTO(2, 21, DAODataType.EST, InfoType.DATA), - new SourceToSpeciesTO(4, 11, DAODataType.AFFYMETRIX, InfoType.ANNOTATION))); + new SourceToSpeciesTO(2, 21, DAODataType.IN_SITU, InfoType.DATA), + new SourceToSpeciesTO(4, 11, DAODataType.RNA_SEQ, InfoType.ANNOTATION))); when(sourceToSpeciesDao.getAllSourceToSpecies(null)).thenReturn(mockSourceToSpeciesRs); List expectedSources = new ArrayList(); @@ -151,7 +147,7 @@ public void shouldLoadDisplayableSources() { assertEquals("Incorrect sources", expectedSources, service.loadDisplayableSources(false)); Map> forAnnot4 = new HashMap<>(); - forAnnot4.put(11, new HashSet(Arrays.asList(DataType.AFFYMETRIX))); + forAnnot4.put(11, new HashSet(Arrays.asList(DataType.RNA_SEQ))); expectedSources.clear(); expectedSources.add(new Source(4, "ZFIN", "ZFIN desc", "https://zfin.org/[xref_id]", diff --git a/bgee-core/src/test/java/org/bgee/model/species/SpeciesServiceTest.java b/bgee-core/src/test/java/org/bgee/model/species/SpeciesServiceTest.java index fc95e7b73..99975805f 100644 --- a/bgee-core/src/test/java/org/bgee/model/species/SpeciesServiceTest.java +++ b/bgee-core/src/test/java/org/bgee/model/species/SpeciesServiceTest.java @@ -64,9 +64,9 @@ public void testLoadSpeciesInDataGroups() { // mock behavior List speciesTos = Arrays.asList( new SpeciesTO(9606, "human", "Homo", "sapiens", 1, 4312, "3241", - "version1", "assemblyXRef1", 1, 321), + "version1", "assemblyXRef1", 1, 321, null), new SpeciesTO(1234, "name", "genus", "someSpecies", 2, 1123, "3432241", - "versionA", "assemblyXRefA", 1, 1321)); + "versionA", "assemblyXRefA", 1, 1321, null)); // ResultSet cannot be reused. As we have 2 tests, we need 2 ResultSet SpeciesTOResultSet speciesRS = getMockResultSet(SpeciesTOResultSet.class, speciesTos); SpeciesTOResultSet speciesRS2 = getMockResultSet(SpeciesTOResultSet.class, speciesTos); @@ -74,9 +74,7 @@ public void testLoadSpeciesInDataGroups() { SourceToSpeciesTOResultSet sToSpRS = getMockResultSet(SourceToSpeciesTOResultSet.class, Arrays.asList( - new SourceToSpeciesTO(1, 9606, DAODataType.EST, InfoType.DATA), new SourceToSpeciesTO(1, 9606, DAODataType.IN_SITU, InfoType.DATA), - new SourceToSpeciesTO(2, 9606, DAODataType.AFFYMETRIX, InfoType.ANNOTATION), new SourceToSpeciesTO(3, 9606, DAODataType.RNA_SEQ, InfoType.DATA), new SourceToSpeciesTO(2, 1234, DAODataType.IN_SITU, InfoType.ANNOTATION))); when(sourceToSpeciesDAOMock.getSourceToSpecies(null, @@ -92,10 +90,10 @@ public void testLoadSpeciesInDataGroups() { assertEquals(expectedSpecies, speciesService.loadSpeciesInDataGroups(false)); Map> forData9606 = new HashMap<>(); - forData9606.put(new Source(1), new HashSet(Arrays.asList(DataType.EST, DataType.IN_SITU))); + forData9606.put(new Source(1), new HashSet(Arrays.asList(DataType.IN_SITU))); forData9606.put(new Source(3), new HashSet(Arrays.asList(DataType.RNA_SEQ))); Map> forAnnot9606 = new HashMap<>(); - forAnnot9606.put(new Source(2), new HashSet(Arrays.asList(DataType.AFFYMETRIX))); + forAnnot9606.put(new Source(2), new HashSet(Arrays.asList(DataType.RNA_SEQ))); Map> forAnnot1234 = new HashMap<>(); forAnnot1234.put(new Source(2), new HashSet(Arrays.asList(DataType.IN_SITU))); expectedSpecies.clear(); @@ -114,9 +112,9 @@ public void testLoadSpeciesInDataGroups() { public void prepareMockObjects() { List speciesTos = Arrays.asList( new SpeciesTO(9606, "human", "Homo", "sapiens", 1, 4312, "3241", - "version1", "assemblyXRef1", 1, 321), + "version1", "assemblyXRef1", 1, 321, null), new SpeciesTO(1234, "name", "genus", "someSpecies", 2, 1123, "3432241", - "versionA", "assemblyXRefA", 1, 1321)); + "versionA", "assemblyXRefA", 1, 1321, null)); // ResultSet cannot be reused. As we have 2 tests, we need 2 ResultSet SpeciesTOResultSet speciesRS = getMockResultSet(SpeciesTOResultSet.class, speciesTos); SpeciesTOResultSet speciesRS2 = getMockResultSet(SpeciesTOResultSet.class, speciesTos); @@ -129,9 +127,7 @@ public void prepareMockObjects() { SourceToSpeciesTOResultSet sToSpRS = getMockResultSet(SourceToSpeciesTOResultSet.class, Arrays.asList( - new SourceToSpeciesTO(1, 9606, DAODataType.EST, InfoType.DATA), new SourceToSpeciesTO(1, 9606, DAODataType.IN_SITU, InfoType.DATA), - new SourceToSpeciesTO(2, 9606, DAODataType.AFFYMETRIX, InfoType.ANNOTATION), new SourceToSpeciesTO(3, 9606, DAODataType.RNA_SEQ, InfoType.DATA), new SourceToSpeciesTO(2, 1234, DAODataType.IN_SITU, InfoType.ANNOTATION))); when(sourceToSpeciesDAO.getSourceToSpecies(null, @@ -151,10 +147,10 @@ public void testLoadSpeciesByIds() { assertEquals(expected, service.loadSpeciesByIds(SPECIES_IDS, false)); Map> forData9606 = new HashMap<>(); - forData9606.put(new Source(1), new HashSet(Arrays.asList(DataType.EST, DataType.IN_SITU))); + forData9606.put(new Source(1), new HashSet(Arrays.asList(DataType.IN_SITU))); forData9606.put(new Source(3), new HashSet(Arrays.asList(DataType.RNA_SEQ))); Map> forAnnot9606 = new HashMap<>(); - forAnnot9606.put(new Source(2), new HashSet(Arrays.asList(DataType.AFFYMETRIX))); + forAnnot9606.put(new Source(2), new HashSet(Arrays.asList(DataType.RNA_SEQ))); Map> forAnnot1234 = new HashMap<>(); forAnnot1234.put(new Source(2), new HashSet(Arrays.asList(DataType.IN_SITU))); expected.clear(); @@ -179,10 +175,10 @@ public void testLoadSpeciesByTaxonIds() { assertEquals(expected, service.loadSpeciesByTaxonIds(TAXON_IDS, false)); Map> forData9606 = new HashMap<>(); - forData9606.put(new Source(1), new HashSet(Arrays.asList(DataType.EST, DataType.IN_SITU))); + forData9606.put(new Source(1), new HashSet(Arrays.asList(DataType.IN_SITU))); forData9606.put(new Source(3), new HashSet(Arrays.asList(DataType.RNA_SEQ))); Map> forAnnot9606 = new HashMap<>(); - forAnnot9606.put(new Source(2), new HashSet(Arrays.asList(DataType.AFFYMETRIX))); + forAnnot9606.put(new Source(2), new HashSet(Arrays.asList(DataType.RNA_SEQ))); Map> forAnnot1234 = new HashMap<>(); forAnnot1234.put(new Source(2), new HashSet(Arrays.asList(DataType.IN_SITU))); expected.clear(); diff --git a/bgee-core/src/test/java/org/bgee/model/topanat/TopAnatControllerTest.java b/bgee-core/src/test/java/org/bgee/model/topanat/TopAnatControllerTest.java index 4798caab0..8cc0a22d1 100644 --- a/bgee-core/src/test/java/org/bgee/model/topanat/TopAnatControllerTest.java +++ b/bgee-core/src/test/java/org/bgee/model/topanat/TopAnatControllerTest.java @@ -138,7 +138,7 @@ public void initTest() { AnatEntityService mockAnatEntityService = mock(AnatEntityService.class); AnatEntity mockEntity = mock(AnatEntity.class); HashMap> relations = new HashMap>(); - Set dataTypes = new HashSet(Arrays.asList(DataType.AFFYMETRIX)); + Set dataTypes = new HashSet(Arrays.asList(DataType.RNA_SEQ)); relations.put("A", new HashSet(Arrays.asList("B","C"))); ExpressionCall mockCall = mock(ExpressionCall.class); Gene myGene = new Gene("ENSG001", new Species(9606), new GeneBioType("type1")); diff --git a/bgee-core/src/test/java/org/bgee/model/topanat/TopAnatParamsTest.java b/bgee-core/src/test/java/org/bgee/model/topanat/TopAnatParamsTest.java index 5ae2cfae1..64109bb07 100644 --- a/bgee-core/src/test/java/org/bgee/model/topanat/TopAnatParamsTest.java +++ b/bgee-core/src/test/java/org/bgee/model/topanat/TopAnatParamsTest.java @@ -55,7 +55,7 @@ public void initTest() throws MissingParameterException{ new HashSet(Arrays.asList("G1","G2","G3","G4")), 999, SummaryCallType.ExpressionSummary.EXPRESSED); topAnatParamsBuilder.summaryQuality(SummaryQuality.GOLD); - topAnatParamsBuilder.dataTypes(new HashSet(Arrays.asList(DataType.AFFYMETRIX))); + topAnatParamsBuilder.dataTypes(new HashSet(Arrays.asList(DataType.RNA_SEQ))); topAnatParamsBuilder.decorrelationType(DecorrelationType.ELIM); topAnatParamsBuilder.devStageId("a"); topAnatParamsBuilder.fdrThreshold(1d); diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/DAOManager.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/DAOManager.java index 6b4d9afb3..a9f41970d 100644 --- a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/DAOManager.java +++ b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/DAOManager.java @@ -27,22 +27,16 @@ import org.bgee.model.dao.api.anatdev.mapping.StageGroupingDAO; import org.bgee.model.dao.api.anatdev.mapping.SummarySimilarityAnnotationDAO; import org.bgee.model.dao.api.exception.DAOException; -import org.bgee.model.dao.api.expressiondata.rawdata.RawExpressionCallDAO; import org.bgee.model.dao.api.expressiondata.rawdata.SamplePValueDAO; +import org.bgee.model.dao.api.expressiondata.ObservedExpressionDAO; import org.bgee.model.dao.api.expressiondata.call.ConditionDAO; import org.bgee.model.dao.api.expressiondata.call.DiffExpressionCallDAO; import org.bgee.model.dao.api.expressiondata.call.GlobalExpressionCallDAO; import org.bgee.model.dao.api.expressiondata.rawdata.RawDataConditionDAO; import org.bgee.model.dao.api.expressiondata.rawdata.RawDataCountDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTLibraryDAO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituEvidenceDAO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituExperimentDAO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituSpotDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipTypeDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixProbesetDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.MicroarrayExperimentDAO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqExperimentDAO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqLibraryAnnotatedSampleDAO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqLibraryDAO; @@ -1068,18 +1062,24 @@ public SexDAO getSexDAO() { this.checkClosed(); return log.traceExit(this.getNewSexDAO()); } - /** - * Get a new {@link org.org.bgee.model.dao.api.expressiondata.rawdata.RawExpressionCallDAO RawExpressionCallDAO}, - * unless this {@code DAOManager} is already closed. - * - * @return a new {@code RawExpressionCallDAO}. - * @throws IllegalStateException If this {@code DAOManager} is already closed. - * @see org.org.bgee.model.dao.api.expressiondata.rawdata.RawExpressionCallDAO RawExpressionCallDAO - */ - public RawExpressionCallDAO getRawExpressionCallDAO() { +// /** +// * Get a new {@link org.org.bgee.model.dao.api.expressiondata.rawdata.RawExpressionCallDAO RawExpressionCallDAO}, +// * unless this {@code DAOManager} is already closed. +// * +// * @return a new {@code RawExpressionCallDAO}. +// * @throws IllegalStateException If this {@code DAOManager} is already closed. +// * @see org.org.bgee.model.dao.api.expressiondata.rawdata.RawExpressionCallDAO RawExpressionCallDAO +// */ +// public RawExpressionCallDAO getRawExpressionCallDAO() { +// log.traceEntry(); +// this.checkClosed(); +// return log.traceExit(this.getNewRawExpressionCallDAO()); +// } + + public ObservedExpressionDAO getObservedExpressionDAO() { log.traceEntry(); - this.checkClosed(); - return log.traceExit(this.getNewRawExpressionCallDAO()); + this.checkClosed(); + return log.traceExit(this.getNewObservedExpressionDAO()); } /** * Get a new {@link org.bgee.model.dao.api.expressiondata.call.GlobalExpressionCallDAO GlobalExpressionCallDAO}, @@ -1135,61 +1135,6 @@ public AnatEntityDAO getAnatEntityDAO() { this.checkClosed(); return log.traceExit(this.getNewAnatEntityDAO()); } - - /** - * Get a new {@link org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixProbesetDAO - * AffymetrixProbesetDAO}, unless this {@code DAOManager} is already closed. - * - * @return a new {@code AffymetrixProbesetDAO}. - * @throws IllegalStateException If this {@code DAOManager} is already closed. - * @see org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixProbesetDAO - * AffymetrixProbesetDAO - */ - public AffymetrixProbesetDAO getAffymetrixProbesetDAO() { - log.traceEntry(); - this.checkClosed(); - return log.traceExit(this.getNewAffymetrixProbesetDAO()); - } - - /** - * Get a new {@link org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipDAO}, - * unless this {@code DAOManager} is already closed. - * - * @return a new {@code AffymetrixChipDAO}. - * @throws IllegalStateException If this {@code DAOManager} is already closed. - * @see org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipDAO - */ - public AffymetrixChipDAO getAffymetrixChipDAO() { - log.traceEntry(); - this.checkClosed(); - return log.traceExit(this.getNewAffymetrixChipDAO()); - } - /** - * Get a new {@link org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipTypeDAO}, - * unless this {@code DAOManager} is already closed. - * - * @return a new {@code AffymetrixChipTypeDAO}. - * @throws IllegalStateException If this {@code DAOManager} is already closed. - * @see org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipTypeDAO - */ - public AffymetrixChipTypeDAO getAffymetrixChipTypeDAO() { - log.traceEntry(); - this.checkClosed(); - return log.traceExit(this.getNewAffymetrixChipTypeDAO()); - } - /** - * Get a new {@link org.bgee.model.dao.api.expressiondata.rawdata.microarray.MicroarrayExperimentDAO}, - * unless this {@code DAOManager} is already closed. - * - * @return a new {@code MicroarrayExperimentDAO}. - * @throws IllegalStateException If this {@code DAOManager} is already closed. - * @see org.bgee.model.dao.api.expressiondata.rawdata.microarray.MicroarrayExperimentDAO - */ - public MicroarrayExperimentDAO getMicroarrayExperimentDAO() { - log.traceEntry(); - this.checkClosed(); - return log.traceExit(this.getNewMicroarrayExperimentDAO()); - } /** * Get a new {@link org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqExperimentDAO}, @@ -1246,33 +1191,6 @@ public RNASeqLibraryDAO getRnaSeqLibraryDAO() { return log.traceExit(this.getNewRnaSeqLibraryDAO()); } - /** - * Get a new {@link org.bgee.model.dao.api.expressiondata.rawdata.est.ESTLibraryDAO}, - * unless this {@code DAOManager} is already closed. - * - * @return a new {@code ESTLibraryDAO}. - * @throws IllegalStateException If this {@code DAOManager} is already closed. - * @see org.bgee.model.dao.api.expressiondata.rawdata.est.ESTLibraryDAO - */ - public ESTLibraryDAO getESTLibraryDAO() { - log.traceEntry(); - this.checkClosed(); - return log.traceExit(this.getNewESTLibraryDAO()); - } - /** - * Get a new {@link org.bgee.model.dao.api.expressiondata.rawdata.est.ESTDAO}, - * unless this {@code DAOManager} is already closed. - * - * @return a new {@code ESTDAO}. - * @throws IllegalStateException If this {@code DAOManager} is already closed. - * @see org.bgee.model.dao.api.expressiondata.rawdata.est.ESTDAO - */ - public ESTDAO getESTDAO() { - log.traceEntry(); - this.checkClosed(); - return log.traceExit(this.getNewESTDAO()); - } - /** * Get a new {@link org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituExperimentDAO * InSituExperimentDAO}, unless this {@code DAOManager} is already closed. @@ -1683,14 +1601,22 @@ public Properties getParameters() { * @return A new {@code SexDAO} */ protected abstract SexDAO getNewSexDAO(); +// /** +// * Service provider must return a new +// * {@link org.org.bgee.model.dao.api.expressiondata.rawdata.RawExpressionCallDAO RawExpressionCallDAO} +// * instance when this method is called. +// * +// * @return A new {@code RawExpressionCallDAO} +// */ +// protected abstract RawExpressionCallDAO getNewRawExpressionCallDAO(); /** * Service provider must return a new - * {@link org.org.bgee.model.dao.api.expressiondata.rawdata.RawExpressionCallDAO RawExpressionCallDAO} - * instance when this method is called. - * - * @return A new {@code RawExpressionCallDAO} - */ - protected abstract RawExpressionCallDAO getNewRawExpressionCallDAO(); + * {@link org.org.bgee.model.dao.api.expressiondata.ObservedExpressionDAO ObservedExpressionDAO} + * instance when this method is called. + * + * @return A new {@code ObservedExpressionDAO} + */ + protected abstract ObservedExpressionDAO getNewObservedExpressionDAO(); /** * Service provider must return a new * {@link org.bgee.model.dao.api.expressiondata.call.GlobalExpressionCallDAO GlobalExpressionCallDAO} @@ -1723,38 +1649,6 @@ public Properties getParameters() { * @return A new {@code AnatEntityDAO} */ protected abstract AnatEntityDAO getNewAnatEntityDAO(); - /** - * Service provider must return a new - * {@link org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixProbesetDAO - * AffymetrixProbesetDAO} instance when this method is called. - * - * @return A new {@code AffymetrixProbesetDAO} - */ - protected abstract AffymetrixProbesetDAO getNewAffymetrixProbesetDAO(); - /** - * Service provider must return a new - * {@link org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipDAO - * AffymetrixChipDAO} instance when this method is called. - * - * @return A new {@code AffymetrixChipDAO} - */ - protected abstract AffymetrixChipDAO getNewAffymetrixChipDAO(); - /** - * Service provider must return a new - * {@link org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipTypeDAO - * AffymetrixChipTypeDAO} instance when this method is called. - * - * @return A new {@code AffymetrixChipTypeDAO} - */ - protected abstract AffymetrixChipTypeDAO getNewAffymetrixChipTypeDAO(); - /** - * Service provider must return a new - * {@link org.bgee.model.dao.api.expressiondata.rawdata.microarray.MicroarrayExperimentDAO - * MicroarrayExperimentDAO} instance when this method is called. - * - * @return A new {@code MicroarrayExperimentDAO} - */ - protected abstract MicroarrayExperimentDAO getNewMicroarrayExperimentDAO(); /** * Service provider must return a new * {@link org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RnaSeqExperimentDAO @@ -1787,22 +1681,6 @@ public Properties getParameters() { * @return A new {@code RNASeqResultDAO} */ protected abstract RNASeqResultAnnotatedSampleDAO getNewRNASeqResultAnnotatedSampleDAO(); - /** - * Service provider must return a new - * {@link org.bgee.model.dao.api.expressiondata.rawdata.est.ESTLibraryDAO - * ESTLibraryDAO} instance when this method is called. - * - * @return A new {@code ESTLibraryDAO} - */ - protected abstract ESTLibraryDAO getNewESTLibraryDAO(); - /** - * Service provider must return a new - * {@link org.bgee.model.dao.api.expressiondata.rawdata.est.ESTDAO - * ESTDAO} instance when this method is called. - * - * @return A new {@code ESTDAO} - */ - protected abstract ESTDAO getNewESTDAO(); /** * Service provider must return a new diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/anatdev/AnatEntityDAO.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/anatdev/AnatEntityDAO.java index e456c8d90..0b4099f84 100644 --- a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/anatdev/AnatEntityDAO.java +++ b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/anatdev/AnatEntityDAO.java @@ -35,7 +35,7 @@ public interface AnatEntityDAO extends DAO { * @see org.bgee.model.dao.api.DAO#clearAttributes() */ public enum Attribute implements DAO.Attribute { - ID("id"), NAME("name"), DESCRIPTION("description"), + ID("anatEntityId"), NAME("anatEntityName"), DESCRIPTION("anatEntityDescription"), START_STAGE_ID("startStageId"), END_STAGE_ID("endStageId"), NON_INFORMATIVE("nonInformative")/*, CELL_TYPE("cellType")*/; diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/DAODataType.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/DAODataType.java index a3e8502e2..18cdb596a 100644 --- a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/DAODataType.java +++ b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/DAODataType.java @@ -14,8 +14,6 @@ * {@code Enum} listing the data types used in Bgee: * *
      - *
    • {@code AFFYMETRIX} - *
    • {@code EST} *
    • {@code IN_SITU} *
    • {@code RNA_SEQ} *
    • {@code SC_RNA_SEQ} @@ -25,25 +23,20 @@ * @version Bgee 14 Mar. 2017 * @since Bgee 14 Mar. 2017 */ +//FIXME: review these fields as some are probably not required anymore with OTF propagation public enum DAODataType implements EnumDAOField { //The order of these Enum elements is important and is used to generate field names - AFFYMETRIX("affymetrix", "affymetrix", "Affy", "affymetrixMeanRank", "affymetrixGlobalMeanRank", - "affymetrixMeanRankNorm", "affymetrixGlobalMeanRankNorm", "affymetrixDistinctRankSum", - "affymetrixGlobalDistinctRankSum", "affymetrixMaxRank", "affymetrixGlobalMaxRank", false, - true, true), - EST("est", "est", "Est", "estRank", "estGlobalRank", "estRankNorm", "estGlobalRankNorm", - "estMaxRank", "estGlobalMaxRank", "estMaxRank", "estGlobalMaxRank", true, true, true), IN_SITU("in situ", "inSitu", "InSitu", "inSituRank", "inSituGlobalRank", "inSituRankNorm", "inSituGlobalRankNorm", "inSituMaxRank", "inSituGlobalMaxRank", "inSituMaxRank", "inSituGlobalMaxRank", true, false, true), RNA_SEQ("rna-seq", "rnaSeq", "RnaSeq", "rnaSeqMeanRank", "rnaSeqGlobalMeanRank", "rnaSeqMeanRankNorm", "rnaSeqGlobalMeanRankNorm", "rnaSeqDistinctRankSum", - "rnaSeqGlobalDistinctRankSum", "rnaSeqMaxRank", "rnaSeqGlobalMaxRank", false, true, false), + "rnaSeqGlobalDistinctRankSum", "bulkMaxRank", "bulkGlobalMaxRank", false, true, false), SC_RNA_SEQ("single-cell RNA-Seq", "scRnaSeqFullLength", "ScRnaSeqFL", "scRnaSeqFullLengthMeanRank", "scRnaSeqFullLengthGlobalMeanRank", "scRnaSeqFullLengthMeanRankNorm", "scRnaSeqFullLengthGlobalMeanRankNorm", "scRnaSeqFullLengthDistinctRankSum", "scRnaSeqFullLengthGlobalDistinctRankSum", - "scRnaSeqFullLengthMaxRank", "scRnaSeqFullLengthGlobalMaxRank", false, true, false); + "singleCellMaxRank", "singleCellGlobalMaxRank", false, true, false); private final static Logger log = LogManager.getLogger(DAODataType.class.getName()); diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/DAOObservedExpressionFilter.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/DAOObservedExpressionFilter.java new file mode 100644 index 000000000..9855cb43d --- /dev/null +++ b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/DAOObservedExpressionFilter.java @@ -0,0 +1,113 @@ +package org.bgee.model.dao.api.expressiondata; + +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bgee.model.dao.api.expressiondata.rawdata.DAORawDataConditionFilter; + +public class DAOObservedExpressionFilter { + + //XXX: the limitation of this approach is that we can not combine conditionIds and speciesIds as they are part + //of rawDataConditionFilter and we force to have either conditionIds or rawDataConditionFilter. The solution would be + //for DAOObservedExpressionFilter to extends DAOBaseConditionFilter and remove DAORawDataConditionFilter as an attribute. + private final Set bgeeGeneIds; + private final EnumSet datatypes; + private final DAORawDataConditionFilter rawDataConditionFilter; + private final Set conditionIds; + private final static Logger log = LogManager.getLogger(DAOObservedExpressionFilter.class.getName()); + + + public DAOObservedExpressionFilter(Collection bgeeGeneIds, EnumSet datatypes, + DAORawDataConditionFilter rawDataConditionFilter) { + this(bgeeGeneIds, datatypes, rawDataConditionFilter, null); + } + + public DAOObservedExpressionFilter(Collection bgeeGeneIds, EnumSet datatypes, + Collection conditionIds) { + this(bgeeGeneIds, datatypes, null, conditionIds); + } + + private DAOObservedExpressionFilter(Collection bgeeGeneIds, EnumSet datatypes, + DAORawDataConditionFilter rawDataConditionFilter, Collection conditionIds) { + log.traceEntry("{}, {}, {}, {}", bgeeGeneIds, datatypes, rawDataConditionFilter, conditionIds); + this.bgeeGeneIds = Collections.unmodifiableSet(bgeeGeneIds == null ? new HashSet<>() : + bgeeGeneIds.stream().filter(id -> {return (id != null && id >= 1);}).collect(Collectors.toSet())); + this.datatypes = datatypes == null || datatypes.isEmpty() ? EnumSet.allOf(DAODataType.class) : + EnumSet.copyOf(datatypes); + this.rawDataConditionFilter = rawDataConditionFilter; + this.conditionIds = Collections.unmodifiableSet(conditionIds == null ? new HashSet<>() : new HashSet<>(conditionIds)); + //For now we force at least one filter other than datatype to be provided. It avoids retrieving the full expression table + if (allFiltersExceptDatatypesAreEmpty()) { + throw log.throwing(new IllegalArgumentException("At least one expression filter other than datatypes" + + "should be provided")); + } + } + + public Set getBgeeGeneIds() { + return bgeeGeneIds; + } + + public EnumSet getDatatypes() { + return datatypes; + } + + public DAORawDataConditionFilter getRawDataConditionFilter() { + return rawDataConditionFilter; + } + + public Set getConditionIds() { + return conditionIds; + } + + @Override + public int hashCode() { + return Objects.hash(bgeeGeneIds, datatypes, rawDataConditionFilter); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + DAOObservedExpressionFilter other = (DAOObservedExpressionFilter) obj; + return Objects.equals(bgeeGeneIds, other.bgeeGeneIds) && Objects.equals(datatypes, other.datatypes) + && Objects.equals(rawDataConditionFilter, other.rawDataConditionFilter); + } + + + + @Override + public String toString() { + return "DAOObservedExpressionFilter [bgeeGeneIds=" + bgeeGeneIds + ", datatypes=" + datatypes + + ", rawDataConditionFilter=" + rawDataConditionFilter + ", conditionIds=" + conditionIds + "]"; + } + + private boolean allFiltersExceptDatatypesAreEmpty() { + if (this.rawDataConditionFilter != null) { + if (this.rawDataConditionFilter.getSpeciesIds() != null && ! this.rawDataConditionFilter.getSpeciesIds().isEmpty()) { + return false; + } + if (! this.rawDataConditionFilter.areAllCondParamFiltersEmpty()) { + return false; + } + } + if (! this.bgeeGeneIds.isEmpty()) { + return false; + } + if (! this.conditionIds.isEmpty()) { + return false; + } + + return true; + } +} diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/ObservedExpressionDAO.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/ObservedExpressionDAO.java new file mode 100644 index 000000000..e0679dc11 --- /dev/null +++ b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/ObservedExpressionDAO.java @@ -0,0 +1,215 @@ +package org.bgee.model.dao.api.expressiondata; + +import java.math.BigDecimal; +import java.util.Collection; + +import org.bgee.model.dao.api.DAO; +import org.bgee.model.dao.api.DAOResultSet; +import org.bgee.model.dao.api.EntityTO; + + +public interface ObservedExpressionDAO extends DAO { + + /** + * {@code Enum} used to define the attributes to populate in the {@code ObservedExpressionTO}s + * obtained from this {@code ObservedExpressionDAO}. + *
        + // columns from Expression table + *
      • {@code EXPRESSION_ID}: corresponds to {@link ObservedExpressionTO#getId()}. + *
      • {@code CONDITION_ID}: corresponds to {@link ObservedExpressionTO#getConditionId()}. + *
      • {@code BGEE_GENE_ID}: corresponds to {@link ObservedExpressionTO#getBgeeGeneId()}. + *
      • {@code BULK_SCORE}: corresponds to {@link ObservedExpressionTO#getBulkScore()}. + *
      • {@code BULK_PVALUE}: corresponds to {@link ObservedExpressionTO#getBulkPValue()}. + *
      • {@code BULK_WEIGHT}: corresponds to {@link ObservedExpressionTO#getBulkWeight()}. + *
      • {@code BULK_NUM_OBS}: corresponds to {@link ObservedExpressionTO#getBulkNumberObs()}. + *
      • {@code FULL_LENGTH_SCORE}: corresponds to {@link ObservedExpressionTO#getFullLengthScore()}. + *
      • {@code FULL_LENGTH_PVALUE}: corresponds to {@link ObservedExpressionTO#getFullLengthPValue()}. + *
      • {@code FULL_LENGTH_WEIGHT}: corresponds to {@link ObservedExpressionTO#getFullLengthWeight()}. + *
      • {@code FULL_LENGTH_NUM_OBS}: corresponds to {@link ObservedExpressionTO#getFullLengthNumberObs()}. + *
      • {@code DROPLET_SCORE}: corresponds to {@link ObservedExpressionTO#getDropletScore()}. + *
      • {@code DROPLET_PVALUE}: corresponds to {@link ObservedExpressionTO#getDropletPValue()}. + *
      • {@code DROPLET_WEIGHT}: corresponds to {@link ObservedExpressionTO#getDropletWeight()}. + *
      • {@code DROPLET_NUM_OBS}: corresponds to {@link ObservedExpressionTO#getDropletNumberObs()}. + *
      • {@code IN_SITU_SCORE}: corresponds to {@link ObservedExpressionTO#getInSituScore()}. + *
      • {@code IN_SITU_PVALUE}: corresponds to {@link ObservedExpressionTO#getInSituPValue()}. + *
      • {@code IN_SITU_WEIGHT}: corresponds to {@link ObservedExpressionTO#getInSituWeight()}. + *
      • {@code IN_SITU_NUM_OBS}: corresponds to {@link ObservedExpressionTO#getInSituNumberObs()}. + **/ + + public enum Attribute implements DAO.Attribute { + EXPRESSION_ID("expressionId"), CONDITION_ID("conditionId"), BGEE_GENE_ID("bgeeGeneId"), + BULK_SCORE("bulkScore"), BULK_PVALUE("bulkPValue"), BULK_WEIGHT("bulkWeight"), BULK_NUM_OBS("bulkNumberObs"), + FULL_LENGTH_SCORE("fullLengthScore"), FULL_LENGTH_PVALUE("fullLengthPValue"), + FULL_LENGTH_WEIGHT("fullLengthWeight"), FULL_LENGTH_NUM_OBS("fullLengthNumberObs"), + DROPLET_SCORE("dropletScore"), DROPLET_PVALUE("dropletPValue"), + DROPLET_WEIGHT("dropletWeight"), DROPLET_NUM_OBS("dropletNumberObs"), + IN_SITU_SCORE("inSituScore"), IN_SITU_PVALUE("inSituPValue"), IN_SITU_WEIGHT("inSituWeight"), + IN_SITU_NUM_OBS("inSituNumberObs"); + + /** + * A {@code String} that is the corresponding field name in {@code ExpressionTO} class. + * @see {@link Attribute#getTOFieldName()} + */ + private final String fieldName; + + private Attribute(String fieldName) { + this.fieldName = fieldName; + } + + @Override + public String getTOFieldName() { + return this.fieldName; + } + } + + //XXX: Should maybe add orderingAttributes to sort by gene, conditionId, etc. + public ObservedExpressionTOResultSet getObservedExpression(DAOObservedExpressionFilter dataFilter, + Collection attributes); + + /** + * {@code DAOResultSet} for {@code ObservedExpressionTO}s + * + * @author Julien Wollbrett + * @version Bgee 16, Oct. 2025 + * @since Bgee 16, Oct. 2025 + */ + public interface ObservedExpressionTOResultSet extends DAOResultSet { + } + + public final class ObservedExpressionTO extends EntityTO { + + private static final long serialVersionUID = -7154753240891071477L; + + private final Integer conditionId; + private final Integer bgeeGeneId; + private final BigDecimal bulkScore; + private final BigDecimal bulkPValue; + private final BigDecimal bulkWeight; + private final Integer bulkNumberObs; + private final BigDecimal fullLengthScore; + private final BigDecimal fullLengthPValue; + private final BigDecimal fullLengthWeight; + private final Integer fullLengthNumberObs; + private final BigDecimal dropletScore; + private final BigDecimal dropletPValue; + private final BigDecimal dropletWeight; + private final Integer dropletNumberObs; + private final BigDecimal inSituScore; + private final BigDecimal inSituPValue; + private final BigDecimal inSituWeight; + private final Integer inSituNumberObs; + + public ObservedExpressionTO(Integer expressionId, Integer conditionId, Integer bgeeGeneId, + BigDecimal bulkScore, BigDecimal bulkPValue, BigDecimal bulkWeight, Integer bulkNumberObs, + BigDecimal fullLengthScore, BigDecimal fullLengthPValue, BigDecimal fullLengthWeight, + Integer fullLengthNumberObs, BigDecimal dropletScore, BigDecimal dropletPValue, + BigDecimal dropletWeight, Integer dropletNumberObs, BigDecimal inSituScore, + BigDecimal inSituPValue, BigDecimal inSituWeight, Integer inSituNumberObs) { + super(expressionId); + this.conditionId = conditionId; + this.bgeeGeneId = bgeeGeneId; + this.bulkScore = bulkScore; + this.bulkPValue = bulkPValue; + this.bulkWeight = bulkWeight; + this.bulkNumberObs = bulkNumberObs; + this.fullLengthScore = fullLengthScore; + this.fullLengthPValue = fullLengthPValue; + this.fullLengthWeight = fullLengthWeight; + this.fullLengthNumberObs = fullLengthNumberObs; + this.dropletScore = dropletScore; + this.dropletPValue = dropletPValue; + this.dropletWeight = dropletWeight; + this.dropletNumberObs = dropletNumberObs; + this.inSituScore = inSituScore; + this.inSituPValue = inSituPValue; + this.inSituWeight = inSituWeight; + this.inSituNumberObs = inSituNumberObs; + } + + public Integer getConditionId() { + return conditionId; + } + + public Integer getBgeeGeneId() { + return bgeeGeneId; + } + + public BigDecimal getBulkPValue() { + return bulkPValue; + } + + public BigDecimal getBulkWeight() { + return bulkWeight; + } + + public BigDecimal getFullLengthPValue() { + return fullLengthPValue; + } + + public BigDecimal getFullLengthWeight() { + return fullLengthWeight; + } + + public BigDecimal getDropletPValue() { + return dropletPValue; + } + + public BigDecimal getDropletWeight() { + return dropletWeight; + } + + public BigDecimal getInSituPValue() { + return inSituPValue; + } + + public BigDecimal getInSituWeight() { + return inSituWeight; + } + + public Integer getBulkNumberObs() { + return bulkNumberObs; + } + + public Integer getFullLengthNumberObs() { + return fullLengthNumberObs; + } + + public Integer getDropletNumberObs() { + return dropletNumberObs; + } + + public Integer getInSituNumberObs() { + return inSituNumberObs; + } + + public BigDecimal getBulkScore() { + return bulkScore; + } + + public BigDecimal getFullLengthScore() { + return fullLengthScore; + } + + public BigDecimal getDropletScore() { + return dropletScore; + } + + public BigDecimal getInSituScore() { + return inSituScore; + } + + @Override + public String toString() { + return "ObservedExpressionTO [conditionId=" + conditionId + ", bgeeGeneId=" + bgeeGeneId + ", bulkScore=" + + bulkScore + ", bulkPValue=" + bulkPValue + ", bulkWeight=" + bulkWeight + ", bulkNumberObs=" + + bulkNumberObs + ", fullLengthScore=" + fullLengthScore + ", fullLengthPValue=" + fullLengthPValue + + ", fullLengthWeight=" + fullLengthWeight + ", fullLengthNumberObs=" + fullLengthNumberObs + + ", dropletScore=" + dropletScore + ", dropletPValue=" + dropletPValue + ", dropletWeight=" + + dropletWeight + ", dropletNumberObs=" + dropletNumberObs + ", inSituScore=" + inSituScore + + ", inSituPValue=" + inSituPValue + ", inSituWeight=" + inSituWeight + ", inSituNumberObs=" + + inSituNumberObs + "]"; + } + + + } +} diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/CallDAO.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/CallDAO.java index 30821f417..e9afb0069 100644 --- a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/CallDAO.java +++ b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/CallDAO.java @@ -87,7 +87,7 @@ public static abstract class CallTO & CallDAO.Attribute> exten /** * An {@code enum} used to define, for each data type that allowed to generate - * a call (Affymetrix, RNA-Seq, ...), its contribution to the generation + * a call (InSitu, RNA-Seq, ...), its contribution to the generation * of the call. *
          *
        • {@code NODATA}: no data from the associated data type allowed to produce @@ -163,16 +163,6 @@ public String toString() { private Integer conditionId; //-----------DataState for each data type--------------- - /** - * The {@code DataState} defining the contribution of Affymetrix data - * to the generation of this call. - */ - private DataState affymetrixData; - /** - * The {@code DataState} defining the contribution of EST data - * to the generation of this call. - */ - private DataState estData; /** * The {@code DataState} defining the contribution of in situ data * to the generation of this call. @@ -197,13 +187,12 @@ public String toString() { * Default constructor. */ protected CallTO() { - this(null, null, null, DataState.NODATA, DataState.NODATA, - DataState.NODATA, DataState.NODATA, DataState.NODATA); + this(null, null, null, DataState.NODATA, DataState.NODATA, DataState.NODATA); } /** * Constructor providing the gene ID, the anatomical entity ID, the developmental stage ID, - * the contribution of Affymetrix, EST, in situ, "relaxed" in situ and, + * the contribution of in situ, "relaxed" in situ and, * RNA-Seq data to the generation of this call. *

          * All of these parameters are optional, so they can be {@code null} when not used. @@ -213,10 +202,6 @@ protected CallTO() { * this call. * @param conditionId An {@code Integer} that is the ID of the condition * associated to this call. - * @param affymetrixData A {@code DataSate} that is the contribution of Affymetrix - * data to the generation of this call. - * @param estData A {@code DataSate} that is the contribution of EST data - * to the generation of this call. * @param inSituData A {@code DataSate} that is the contribution of * in situ data to the generation of this call. * @param relaxedInSituData A {@code DataSate} that is the contribution of "relaxed" @@ -225,13 +210,10 @@ protected CallTO() { * to the generation of this call. */ protected CallTO(Integer id, Integer bgeeGeneId, Integer conditionId, - DataState affymetrixData, DataState estData, DataState inSituData, - DataState relaxedInSituData, DataState rnaSeqData) { + DataState inSituData, DataState relaxedInSituData, DataState rnaSeqData) { super(id); this.bgeeGeneId = bgeeGeneId; this.conditionId = conditionId; - this.affymetrixData = affymetrixData; - this.estData = estData; this.inSituData = inSituData; this.relaxedInSituData = relaxedInSituData; this.rnaSeqData = rnaSeqData; @@ -256,7 +238,7 @@ protected CallTO(Integer id, Integer bgeeGeneId, Integer conditionId, /** * Retrieve from this {@code CallTO} the data types with a filtering requested, * allowing to parameterize queries to the data source. For instance, to only retrieve - * calls with an Affymetrix data state equal to {@code HIGHQUALITY}, or with some RNA-Seq data + * calls with some RNA-Seq data * of any quality (minimal data state {@code LOWQUALITY}). *

          * The data types are represented as {@code Attribute}s allowing to request a data type parameter @@ -327,38 +309,6 @@ public Integer getConditionId() { } //-----------DataState for each data type--------------- - /** - * @return the {@code DataState} defining the contribution of Affymetrix data - * to the generation of this call. - */ - public DataState getAffymetrixData() { - return affymetrixData; - } - /** - * @param affymetrixData the {@code DataState} defining the contribution - * of Affymetrix data to the generation of this call. - */ - //deprecated because all TOs should now be immutable. - @Deprecated - void setAffymetrixData(DataState affymetrixData) { - this.affymetrixData = affymetrixData; - } - /** - * @return the {@code DataState} defining the contribution of EST data - * to the generation of this call. - */ - public DataState getESTData() { - return estData; - } - /** - * @param estData the {@code DataState} defining the contribution - * of EST data to the generation of this call. - */ - //deprecated because all TOs should now be immutable. - @Deprecated - void setESTData(DataState estData) { - this.estData = estData; - } /** * @return the {@code DataState} defining the contribution of in situ data * to the generation of this call. @@ -424,8 +374,6 @@ void setRNASeqData(DataState rnaSeqData) { public String toString() { return "ID: " + this.getId() + " - Bgee Gene ID: " + this.getBgeeGeneId() + " - Condition ID: " + this.getConditionId() + - " - Affymetrix data: " + this.getAffymetrixData() + - " - EST data: " + this.getESTData() + " - in situ data: " + this.getInSituData() + " - relaxed in situ data: " + this.getRelaxedInSituData() + " - RNA-Seq data: " + this.getRNASeqData(); diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/CallObservedDataDAOFilter.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/CallObservedDataDAOFilter.java index 95bc8fd3e..a602cc925 100644 --- a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/CallObservedDataDAOFilter.java +++ b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/CallObservedDataDAOFilter.java @@ -10,7 +10,7 @@ * A filter to request expression calls based on their observation status in a condition: * whether the call has been directly observed in a condition, or propagated from some descendant * conditions of the considered condition. This filter accepts the data types to consider - * (for instance, whether the call was observed from Affymetrix and/or RNA-Seq data) + * (for instance, whether the call was observed from RNA-Seq data) * and the condition parameters to consider (for instance, whether the call was observed * in an anat. entity, while accepting that it might have been propagated along the dev. stage * ontology). diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/ConditionDAO.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/ConditionDAO.java index fd3634fed..583c11a19 100644 --- a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/ConditionDAO.java +++ b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/ConditionDAO.java @@ -60,6 +60,8 @@ public interface ConditionDAO extends DAO { */ public final static String STRAIN_ROOT_ID = "wild-type"; + public static final int MAX_MASK = (1 << ConditionParameter.values().length) - 1; + public enum ConditionParameter { ANAT_ENTITY(), CELL_TYPE(), @@ -197,6 +199,20 @@ public ConditionTOResultSet getGlobalConditions(Collection speciesIds, Collection conditionFilters, Collection attributes) throws DAOException, IllegalArgumentException; + /** + * Retrieves all relations between global condition Ids and its direct parents for the provided {@code speciesId}. + * The relations are retrieved as a {@code GlobalConditionToDirectAncestorTOResultSet}. + * + * @param speciesId An {@code Integer} that is the ID of species allowing to filter the relations to retrieve. + * Throws an error if {@code null} or empty. + * @return A {@code GlobalConditionToDirectAncestorTOResultSet} containing the direct relations between + * global conditions retrieved from the data source for the provided speciesId. + * @throws DAOException + * @throws IllegalArgumentException + */ + public GlobalConditionToDirectAncestorTOResultSet getGlobalConditionToDirectAncestor(Integer speciesId) + throws DAOException, IllegalArgumentException; + /** * Retrieves global conditions belonging to the provided {@code speciesIds} with parameters defined * as specified by {@code DAOConditionFilter2}. @@ -242,30 +258,45 @@ public ConditionTOResultSet getGlobalConditionsFromCallFilters(Collection condIds, Collection attributes) throws DAOException, IllegalArgumentException; + //XXX: could create a filter containing both the globalCondition IDs and the condition parameters /** - * Retrieve the correspondence between raw condition and global conditions, represented as - * {@code GlobalConditionToRawConditionTO}s. - *

          - * The results are retrieved and returned as a {@code GlobalConditionToRawConditionTOResultSet}. - * It is the responsibility of the caller to close this {@code DAOResultSet} - * once results are retrieved. - * - * @param speciesIds A {@code Collection} of {@code Integer}s that are the IDs - * of species allowing to filter the results to retrieve. - * If {@code null} or empty, results for all species are retrieved. - * @param conditionParameters A {@code Collection} of {@code ConditionDAO.Attribute}s - * defining the condition parameters of the global conditions - * to retrieve mappings for. - * (see {@link Attribute#isConditionParameter()}). - * @return A {@code GlobalConditionToRawConditionTOResultSet} - * allowing to retrieve the requested - * {@code GlobalConditionToRawConditionTO}s. - * @throws DAOException - * @throws IllegalArgumentException + * Retrieve the mapping between global conditions and raw conditions from global condition IDs and + * condition parameters + * @param globalConditionIds A {@code Collection} of {@code Integer} + * @param condParams An {@code EnumSet} of {@code ConditionParameter} corresponding to the + * condition parameters for which we want to retrieve the mapping between + * global conditions and raw conditions + * + * @return A {@code RawConditionToSelfGlobalConditionTOResultSet} containing all mapping + * between raw condition IDs and global condition IDs for the requested condition + * parameters */ - public GlobalConditionToRawConditionTOResultSet getGlobalCondToRawCondBySpeciesIds( - Collection speciesIds, Collection conditionParameters) - throws DAOException, IllegalArgumentException; + public RawConditionToSelfGlobalConditionTOResultSet getRawConditionToSelfGlobalConditionFromGlobalConditionIds( + Collection globalConditionIds, EnumSet condParams) throws DAOException; +// /** +// * Retrieve the correspondence between raw condition and global conditions, represented as +// * {@code GlobalConditionToRawConditionTO}s. +// *

          +// * The results are retrieved and returned as a {@code GlobalConditionToRawConditionTOResultSet}. +// * It is the responsibility of the caller to close this {@code DAOResultSet} +// * once results are retrieved. +// * +// * @param speciesIds A {@code Collection} of {@code Integer}s that are the IDs +// * of species allowing to filter the results to retrieve. +// * If {@code null} or empty, results for all species are retrieved. +// * @param conditionParameters A {@code Collection} of {@code ConditionDAO.Attribute}s +// * defining the condition parameters of the global conditions +// * to retrieve mappings for. +// * (see {@link Attribute#isConditionParameter()}). +// * @return A {@code GlobalConditionToRawConditionTOResultSet} +// * allowing to retrieve the requested +// * {@code GlobalConditionToRawConditionTO}s. +// * @throws DAOException +// * @throws IllegalArgumentException +// */ +// public GlobalConditionToRawConditionTOResultSet getGlobalCondToRawCondBySpeciesIds( +// Collection speciesIds, Collection conditionParameters) +// throws DAOException, IllegalArgumentException; /** * Retrieve the maximum of global condition IDs, used in the global expression data, @@ -322,14 +353,27 @@ public Map getMaxRanks(Collection species * Inserts the provided correspondence between raw condition and global conditions * into the data source, represented as a {@code Collection} of {@code GlobalConditionToRawConditionTO}s. * - * @param globalCondToRawCondTOs A {@code Collection} of {@code GlobalConditionToRawConditionTO}s - * to be inserted into the data source. - * @return An {@code int} that is the number of inserted TOs. - * @throws DAOException If an error occurred while trying to insert data. - * @throws IllegalArgumentException If {@code globalCondToRawCondTOs} is {@code null} or empty. + * @param RawConditionToSelfGlobalCondition A {@code Collection} of {@code GlobalConditionToRawConditionTO}s + * to be inserted into the data source. + * @return An {@code int} that is the number of inserted TOs. + * @throws DAOException If an error occurred while trying to insert data. + * @throws IllegalArgumentException If {@code rawConditionToSelfGlobalCondition} is {@code null} or empty. */ - public int insertGlobalConditionToRawCondition( - Collection globalCondToRawCondTOs) + public int insertRawConditionToSelfGlobalCondition( + Collection rawConditionToSelfGlobalCondition) + throws DAOException, IllegalArgumentException; + + /** + * Inserts the direct relation between a source conditionId and its target direct parents conditionIds. + * + * @param conditionIdToParentConditionIds A {@code Collection} of {@code GlobalConditionToRawConditionTO}s + * to be inserted into the data source. + * @return An {@code int} that is the number of inserted TOs. + * @throws DAOException If an error occurred while trying to insert data. + * @throws IllegalArgumentException If {@code condIdToDirectAncestorId} is {@code null} or empty. + */ + public int insertcondIdToDirectAncestorId( + Collection condIdToDirectAncestorId) throws DAOException, IllegalArgumentException; /** @@ -527,77 +571,47 @@ public String toString() { } /** - * {@code DAOResultSet} specifics to {@code GlobalConditionToRawConditionTO}s + * {@code DAOResultSet} specifics to {@code RawConditionToSelfGlobalConditionTO}s * - * @author Frederic Bastian - * @version Bgee 14 Feb. 2017 - * @since Bgee 14 Feb. 2017 + * @author Julien Wollbrett + * @version Bgee 16 Oct. 2025 + * @since Bgee 16 Oct. 2025 */ - public interface GlobalConditionToRawConditionTOResultSet - extends DAOResultSet { + public interface RawConditionToSelfGlobalConditionTOResultSet + extends DAOResultSet { } /** - * A {@code TransferObject} representing a relation between a globalCondition and - * one of the raw conditions considered when aggregating the data in the related globalCondition. + * A {@code TransferObject} representing a relation between a raw condition and + * its self condition for a given combination of condition parameters *

          - * This class defines a raw condition ID (see {@link #getConditionId()} + * This class defines a raw condition ID (see {@link #getRawConditionId()} * and a global condition ID (see {@link #getGlobalConditionId()}), and also stores - * the origin of the relations (association from sub-conditions or parent conditions - * or from the same condition, see {@link #getCondtionRelationOrigin()}). + * the combination of condition parameters as a bitwise. * - * @author Frederic Bastian - * @version Bgee 14 Mar. 2017 - * @since Bgee 14 Mar. 2017 + * @author Julien Wollbrett + * @version Bgee 16 Oct. 2025 + * @since Bgee 16 Oct. 2025 */ //TODO: add related method in TOComparator - public static class GlobalConditionToRawConditionTO extends TransferObject { - private final static Logger log = LogManager.getLogger(GlobalConditionToRawConditionTO.class.getName()); - private static final long serialVersionUID = -553628358149907274L; - - public enum ConditionRelationOrigin implements TransferObject.EnumDAOField { - SELF("self"), DESCENDANT("descendant"), PARENT("parent"); - - /** - * The {@code String} representation of the enum. - */ - private String stringRepresentation; - /** - * Constructor - * @param stringRepresentation the {@code String} representation of the enum. - */ - ConditionRelationOrigin(String stringRepresentation) { - this.stringRepresentation = stringRepresentation; - } - @Override - public String getStringRepresentation() { - return stringRepresentation; - } - /** - * Return the mapped {@link ConditionRelationOrigin} from a string representation. - * @param stringRepresentation A string representation - * @return The corresponding {@code ConditionRelationOrigin} - * @see org.bgee.model.dao.api.TransferObject.EnumDAOField#convert(Class, String) - */ - public static ConditionRelationOrigin convertToCondRelOrigin(String stringRepresentation){ - log.traceEntry("{}", stringRepresentation); - return log.traceExit(GlobalConditionToRawConditionTO.convert(ConditionRelationOrigin.class, - stringRepresentation)); - } - } - - /** + public static class RawConditionToSelfGlobalConditionTO extends TransferObject { + private final static Logger log = LogManager.getLogger(RawConditionToSelfGlobalConditionTO.class.getName()); + private static final long serialVersionUID = 7293350787610176970L; + /** * A {@code Integer} representing the ID of the raw condition. */ - private final Integer rawConditionId; + private final int rawConditionId; /** * A {@code Integer} representing the ID of the global condition. */ - private final Integer globalConditionId; + private final int globalConditionId; /** - * A {@code ConditionRelationOrigin} representing the origin of the association. + * A {@code Integer} representing the combination of condition parameters that allows to map + * the raw condition to the global condition. */ - private final ConditionRelationOrigin conditionRelationOrigin; + private final int condParamBitwise; + +// private final EnumSet conditionParameters; /** * Constructor providing the condition ID (see {@link #getRawConditionId()}) and @@ -605,43 +619,119 @@ public static ConditionRelationOrigin convertToCondRelOrigin(String stringRepres * * @param rawExpressionId An {@code Integer} that is the ID of the raw condition. * @param globalExpressionId An {@code Integer} that is the ID of the global condition. - * @param conditionRelationOrigin An {@code ConditionRelationOrigin} representing - * the origin of the association. + * @param conditionParameters An {@code EnumSet} of {@code ConditionParameter} representing + * the combination of condition parameters. **/ - public GlobalConditionToRawConditionTO(Integer rawConditionId, Integer globalConditionId, - ConditionRelationOrigin conditionRelationOrigin) { - super(); + public RawConditionToSelfGlobalConditionTO(int rawConditionId, int globalConditionId, + EnumSet conditionParameters) { this.rawConditionId = rawConditionId; this.globalConditionId = globalConditionId; - this.conditionRelationOrigin = conditionRelationOrigin; + this.condParamBitwise = fromCondParamToSubsetMask(conditionParameters); +// this.conditionParameters = conditionParameters; + } + + /** + * Constructor providing the raw condition ID (see {@link #getRawConditionId()}) and + * the global condition ID (see {@link #getGlobalConditionId()}). + * + * @param rawExpressionId An {@code Integer} that is the ID of the raw condition. + * @param globalExpressionId An {@code Integer} that is the ID of the global condition. + * @param condParamBitwise An {@code Integer} that is the bitwise representation of + * the combination of condition parameters. + **/ + public RawConditionToSelfGlobalConditionTO(int rawConditionId, int globalConditionId, + int condParamBitwise) { + this.rawConditionId = rawConditionId; + this.globalConditionId = globalConditionId; + this.condParamBitwise = condParamBitwise; +// this.conditionParameters = fromSubsetMaskToCondParam(condParamBitwise); + } + + //TODO: Should be moved Somewhere else + public final static int fromCondParamToSubsetMask (EnumSet conditionParameters) { + if (conditionParameters == null || conditionParameters.isEmpty()) { + throw log.throwing(new IllegalArgumentException("conditionParameters can not be null or empty")); + } + int mask = 0; + ConditionParameter[] values = ConditionParameter.values(); + for (int i = 0; i < values.length; i++) { + if (conditionParameters.contains(values[i])) { + mask |= (1 << i); + } + } + return mask; + } + //TODO: Should be moved Somewhere else + public final static EnumSet fromSubsetMaskToCondParam(int subsetMask) { + if (subsetMask < 1 || subsetMask > MAX_MASK) { + throw log.throwing(new IllegalArgumentException("Invalid subsetMask: " + subsetMask + + ". Expected a value between 1 and 31 inclusive.")); + } + EnumSet params = EnumSet.noneOf(ConditionParameter.class); + ConditionParameter[] values = ConditionParameter.values(); + + for (int i = 0; i < values.length; i++) { + if ((subsetMask & (1 << i)) != 0) { + params.add(values[i]); + } + } + return params; } /** * @return the {@code Integer} representing the ID of the raw condition. */ - public Integer getRawConditionId() { + public int getRawConditionId() { return rawConditionId; } /** * @return the {@code Integer} representing the ID of the global condition. */ - public Integer getGlobalConditionId() { + public int getGlobalConditionId() { return globalConditionId; } /** - * @return {@code ConditionRelationOrigin} representing the origin of the association. + * @return the {@code Integer} representing the bitwise combination of condition parameters. */ - public ConditionRelationOrigin getConditionRelationOrigin() { - return conditionRelationOrigin; + public int getCondParamBitwise() { + return condParamBitwise; } +// public EnumSet getConditionParameters() { +// return conditionParameters; +// } + @Override public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("GlobalConditionToRawConditionTO [rawConditionId=").append(rawConditionId) - .append(", globalConditionId=").append(globalConditionId) - .append(", conditionRelationOrigin=").append(conditionRelationOrigin).append("]"); - return builder.toString(); + return "RawConditionToSelfCondition [rawConditionId=" + rawConditionId + ", globalConditionId=" + + globalConditionId + ", condParamBitwise=" + condParamBitwise + "]"; + } + + } + + public interface GlobalConditionToDirectAncestorTOResultSet extends DAOResultSet{ + } + + public static class GlobalConditionToDirectAncestorTO extends TransferObject{ + + private static final long serialVersionUID = -8774630317282011145L; + + private final Integer sourceConditionId; + private final Integer targetConditionId; + + public GlobalConditionToDirectAncestorTO(Integer sourceConditionId, Integer targetConditionId) { + this.sourceConditionId = sourceConditionId; + this.targetConditionId = targetConditionId; + } + + public Integer getSourceConditionId() { + return sourceConditionId; } + + public Integer getTargetConditionId() { + return targetConditionId; + } + } + } diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/DAOConditionFilter.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/DAOConditionFilter.java index 08fd14c70..69b6e6c86 100644 --- a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/DAOConditionFilter.java +++ b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/DAOConditionFilter.java @@ -2,6 +2,9 @@ import java.util.Collection; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + /** * A filter to parameterize queries using expression data conditions. * @@ -10,7 +13,9 @@ * @since Bgee 13 Oct. 2015 */ public class DAOConditionFilter extends DAOConditionFilterBase { - + + private final static Logger log = LogManager.getLogger(DAOConditionFilter.class.getName()); + /** * @param anatEntityIds A {@code Collection} of {@code String}s that are the IDs * of the anatomical entities that this {@code DAOConditionFilter} @@ -49,6 +54,16 @@ public DAOConditionFilter(Collection anatEntitieIds, Collection Collection observedCondForParams) throws IllegalArgumentException { super(anatEntitieIds, devStageIds, cellTypeIds, sexIds, strainIds, observedCondForParams, ConditionDAO.Attribute.class, null); + if ((anatEntitieIds == null || anatEntitieIds.isEmpty()) && + (devStageIds == null || devStageIds.isEmpty()) && + (cellTypeIds == null || cellTypeIds.isEmpty()) && + (sexIds == null || sexIds.isEmpty()) && + (strainIds == null || strainIds.isEmpty()) && + (observedCondForParams == null || observedCondForParams.isEmpty())) { + throw log.throwing(new IllegalArgumentException("Some anatatomical entity IDs, " + + "developmental stage IDs, cell type IDs, sex IDs, strain IDs or observed " + + "data status must be provided.")); + } if (this.getObservedCondForParams().stream().anyMatch(a -> !a.isConditionParameter())) { throw new IllegalArgumentException( "A ConditionDAO.Attribute that is not a condition parameter was provided"); diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/DAOConditionFilter2.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/DAOConditionFilter2.java index 60d2666a3..aa0048711 100644 --- a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/DAOConditionFilter2.java +++ b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/DAOConditionFilter2.java @@ -4,8 +4,12 @@ import java.util.Objects; import java.util.Set; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + public class DAOConditionFilter2 extends DAOConditionFilterBase { + private final static Logger log = LogManager.getLogger(DAOConditionFilter2.class.getName()); private final Set speciesIds; /** * @param speciesIds A {@code Collection} of {@code Integer}s that are the IDs @@ -50,6 +54,17 @@ public DAOConditionFilter2(Collection speciesIds, Collection an throws IllegalArgumentException { super(anatEntitieIds, devStageIds, cellTypeIds, sexIds, strainIds, observedCondForParams, ConditionDAO.ConditionParameter.class, excludedAnatEntityCellTypeIds); + if ((anatEntitieIds == null || anatEntitieIds.isEmpty()) && + (devStageIds == null || devStageIds.isEmpty()) && + (cellTypeIds == null || cellTypeIds.isEmpty()) && + (sexIds == null || sexIds.isEmpty()) && + (strainIds == null || strainIds.isEmpty()) && + (observedCondForParams == null || observedCondForParams.isEmpty()) && + (speciesIds == null || speciesIds.isEmpty())) { + throw log.throwing(new IllegalArgumentException("Some anatatomical entity IDs, " + + "developmental stage IDs, cell type IDs, sex IDs, strain IDs, species IDs or observed " + + "data status must be provided.")); + } if (speciesIds != null && speciesIds.stream().anyMatch(id -> id == null || id < 1)) { throw new IllegalArgumentException("No speciesId can be null or less than 1"); } diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/DAOConditionFilterBase.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/DAOConditionFilterBase.java index d5c150432..7e7838bcd 100644 --- a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/DAOConditionFilterBase.java +++ b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/DAOConditionFilterBase.java @@ -55,16 +55,7 @@ public DAOConditionFilterBase(Collection anatEntitieIds, Collection observedCondForParams, Class condParamType, Collection excludedAnatEntityCellTypeIds) throws IllegalArgumentException { super(anatEntitieIds, devStageIds, cellTypeIds, sexIds, strainIds, excludedAnatEntityCellTypeIds); - if ((anatEntitieIds == null || anatEntitieIds.isEmpty()) && - (devStageIds == null || devStageIds.isEmpty()) && - (cellTypeIds == null || cellTypeIds.isEmpty()) && - (sexIds == null || sexIds.isEmpty()) && - (strainIds == null || strainIds.isEmpty()) && - (observedCondForParams == null || observedCondForParams.isEmpty())) { - throw log.throwing(new IllegalArgumentException("Some anatatomical entity IDs, " - + "developmental stage IDs, cell type IDs, sex IDs, strain IDs or observed " - + "data status must be provided.")); - } + this.observedCondForParams = observedCondForParams == null || observedCondForParams.isEmpty()? EnumSet.noneOf(condParamType): EnumSet.copyOf(observedCondForParams); } diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/DiffExpressionCallDAO.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/DiffExpressionCallDAO.java index 981565bdc..3996e5082 100644 --- a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/DiffExpressionCallDAO.java +++ b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/call/DiffExpressionCallDAO.java @@ -1,15 +1,16 @@ package org.bgee.model.dao.api.expressiondata.call; +import java.util.Collection; import java.util.EnumMap; +import java.util.LinkedHashMap; import java.util.Map; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bgee.model.dao.api.DAO; -import org.bgee.model.dao.api.OrderingDAO; import org.bgee.model.dao.api.DAOResultSet; +import org.bgee.model.dao.api.OrderingDAO; import org.bgee.model.dao.api.TransferObject; -import org.bgee.model.dao.api.exception.DAOException; import org.bgee.model.dao.api.expressiondata.call.CallDAO.CallTO; /** @@ -32,16 +33,6 @@ public interface DiffExpressionCallDAO *

        • {@code CONDITION_ID}: corresponds to {@link DiffExpressionCallTO#getConditionId()}. *
        • {@code COMPARISON_FACTOR}: corresponds to * {@link DiffExpressionCallTO#getComparisonFactor()}. - *
        • {@code DIFF_EXPR_CALL_AFFYMETRIX}: corresponds to - * {@link DiffExpressionCallTO#getDiffExprCallTypeAffymetrix()}. - *
        • {@code DIFF_EXPR_AFFYMETRIX_DATA}: corresponds to - * {@link DiffExpressionCallTO#getAffymetrixData()}. - *
        • {@code BEST_P_VALUE_AFFYMETRIX}: corresponds to - * {@link DiffExpressionCallTO#getBestPValueAffymetrix()}. - *
        • {@code CONSISTENT_DEA_COUNT_AFFYMETRIX}: corresponds to - * {@link DiffExpressionCallTO#getConsistentDEACountAffymetrix()}. - *
        • {@code INCONSISTENT_DEA_COUNT_AFFYMETRIX_FOUND}: corresponds to - * {@link DiffExpressionCallTO#getInconsistentDEACountAffymetrix()}. *
        • {@code DIFF_EXPR_CALL_RNA_SEQ}: corresponds to * {@link DiffExpressionCallTO#getDiffExprCallTypeRNASeq()}. *
        • {@code DIFF_EXPR_RNA_SEQ_DATA}: corresponds to @@ -59,8 +50,6 @@ public interface DiffExpressionCallDAO */ public enum Attribute implements CallDAO.Attribute { ID(false), GENE_ID(false), CONDITION_ID(false), COMPARISON_FACTOR(false), - DIFF_EXPR_CALL_AFFYMETRIX(false), DIFF_EXPR_AFFYMETRIX_DATA(true), BEST_P_VALUE_AFFYMETRIX(false), - CONSISTENT_DEA_COUNT_AFFYMETRIX(false), INCONSISTENT_DEA_COUNT_AFFYMETRIX(false), DIFF_EXPR_CALL_RNA_SEQ(false), DIFF_EXPR_RNA_SEQ_DATA(true), BEST_P_VALUE_RNA_SEQ(false), CONSISTENT_DEA_COUNT_RNA_SEQ(false), INCONSISTENT_DEA_COUNT_RNA_SEQ(false); @@ -257,31 +246,7 @@ public String toString() { * the differential expression analyzes generating this call. */ private ComparisonFactor comparisonFactor; - - /** - * A {@code DiffExprCallType} that is the type of differential expression of this call - * generated by Affymetrix. - */ - private DiffExprCallType diffExprCallTypeAffymetrix; - - /** - * A {@code Float} that is best p-value associated to this call among all the analysis - * using Affymetrix comparing this condition. - */ - private Float bestPValueAffymetrix; - - /** - * An {@code Integer} that is the number of analysis using Affymetrix data - * where the same call is found. - */ - private Integer consistentDEACountAffymetrix; - - /** - * An {@code Integer} that is the number of analysis using Affymetrix data where - * a different call is found. - */ - private Integer inconsistentDEACountAffymetrix; - + /** * A {@code DiffExprCallType} that is the type of differential expression of this call * generated by RNA-seq. @@ -311,7 +276,7 @@ public String toString() { */ DiffExpressionCallTO() { this(null, null, null, null, null, null, null, - null, null, null, null, null, null, null); + null, null); } /** @@ -329,18 +294,6 @@ public String toString() { * the comparison factor used during the * differential expression analyzes generating * this call. - * @param diffExprCallTypeAffymetrix A {@code DiffExprCallType} that is the type of - * differential expression of this call generated - * by Affymetrix. - * @param bestPValueAffymetrix A {@code Float} that is best p-value associated - * to this call among all the analysis using - * Affymetrix comparing this condition. - * @param consistentDEACountAffymetrix An {@code Integer} that is the number of - * analysis using Affymetrix data where the same - * call is found. - * @param inconsistentDEACountAffymetrix An {@code Integer} that is the number of - * analysis using Affymetrix data where - * a different call is found. * @param diffExprCallTypeRNASeq A {@code DiffExprCallType} that is the type of * differential expression of this call generated * by RNA-seq. @@ -355,19 +308,13 @@ public String toString() { * call is found. */ public DiffExpressionCallTO(Integer id, Integer bgeeGeneId, Integer conditionId, - ComparisonFactor comparisonFactor, DiffExprCallType diffExprCallTypeAffymetrix, - DataState diffExprAffymetrixData, Float bestPValueAffymetrix, - Integer consistentDEACountAffymetrix, Integer inconsistentDEACountAffymetrix, + ComparisonFactor comparisonFactor, DiffExprCallType diffExprCallTypeRNASeq, DataState diffExprRNASeqData, Float bestPValueRNASeq, Integer consistentDEACountRNASeq, Integer inconsistentDEACountRNASeq) { - super(id, bgeeGeneId, conditionId, diffExprAffymetrixData, null, null, + super(id, bgeeGeneId, conditionId, null, null, diffExprRNASeqData); this.comparisonFactor = comparisonFactor; - this.diffExprCallTypeAffymetrix = diffExprCallTypeAffymetrix; - this.bestPValueAffymetrix = bestPValueAffymetrix; - this.consistentDEACountAffymetrix = consistentDEACountAffymetrix; - this.inconsistentDEACountAffymetrix = inconsistentDEACountAffymetrix; this.diffExprCallTypeRNASeq = diffExprCallTypeRNASeq; this.bestPValueRNASeq = bestPValueRNASeq; this.consistentDEACountRNASeq = consistentDEACountRNASeq; @@ -380,7 +327,6 @@ public Map extractDataTypesToDataStates() { Map typesToStates = new EnumMap<>(Attribute.class); - typesToStates.put(Attribute.DIFF_EXPR_AFFYMETRIX_DATA, this.getAffymetrixData()); typesToStates.put(Attribute.DIFF_EXPR_RNA_SEQ_DATA, this.getRNASeqData()); return log.traceExit(typesToStates); @@ -388,7 +334,7 @@ public Map extractDataTypesToDataStates() { /** * Retrieve from this {@code CallTO} the data types with a filtering requested, * allowing to parameterize queries to the data source. For instance, to only retrieve - * calls with an Affymetrix data state equal to {@code HIGHQUALITY}, or with some RNA-Seq data + * calls with some RNA-Seq data * of any quality (minimal data state {@code LOWQUALITY}). *

          * The data types are represented as {@code Attribute}s allowing to request a data type parameter @@ -400,9 +346,8 @@ public Map extractDataTypesToDataStates() { * by this method will be empty. *

          * Each quality associated to a data type in a same {@code CallTO} is considered - * as an AND condition (for instance, "affymetrixData >= HIGH_QUALITY AND - * rnaSeqData >= HIGH_QUALITY"). To configure OR conditions, (for instance, - * "affymetrixData >= HIGH_QUALITY OR rnaSeqData >= HIGH_QUALITY"), several {@code CallTO}s + * as an AND condition (for instance, rnaSeqData >= HIGH_QUALITY"). To configure OR conditions, (for instance, + * "rnaSeqData >= HIGH_QUALITY"), several {@code CallTO}s * must be provided to this {@code CallDAOFilter}. So for instance, if the quality * of all data types of {@code callTO} are set to {@code LOW_QUALITY}, it will only allow * to retrieve calls with data in all data types. @@ -439,81 +384,6 @@ void setComparisonFactor(ComparisonFactor comparisonFactor) { this.comparisonFactor = comparisonFactor; } - /** - * @return the {@code DiffExprCallType} that is the type of differential expression - * of this call generated by Affymetrix. - */ - public DiffExprCallType getDiffExprCallTypeAffymetrix() { - return this.diffExprCallTypeAffymetrix; - } - - /** - * @param diffExprCallTypeAffymetrix A {@code DiffExprCallType} that is the type of - * differential expression of this call generated by - * Affymetrix. - */ - //deprecated because all TOs should now be immutable. - @Deprecated - void setDiffExprCallTypeAffymetrix(DiffExprCallType diffExprCallTypeAffymetrix) { - this.diffExprCallTypeAffymetrix = diffExprCallTypeAffymetrix; - } - - /** - * @return the {@code Float} that is best p-value associated to this call among all the - * analysis using Affymetrix comparing this condition. - */ - public Float getBestPValueAffymetrix() { - return this.bestPValueAffymetrix; - } - - /** - * @param bestPValueAffymetrix A {@code Float} that is best p-value associated to this - * call among all the analysis using Affymetrix comparing - * this condition. - */ - //deprecated because all TOs should now be immutable. - @Deprecated - void setBestPValueAffymetrix(Float bestPValueAffymetrix) { - this.bestPValueAffymetrix = bestPValueAffymetrix; - } - - /** - * @return the {@code Integer} that is the number of analysis using Affymetrix data where - * the same call is found. - */ - public Integer getConsistentDEACountAffymetrix() { - return this.consistentDEACountAffymetrix; - } - - /** - * @param consistentDEACountAffymetrix An {@code Integer} that is the number of analysis - * using Affymetrix data where the same call is found. - */ - //deprecated because all TOs should now be immutable. - @Deprecated - void setConsistentDEACountAffymetrix(Integer consistentDEACountAffymetrix) { - this.consistentDEACountAffymetrix = consistentDEACountAffymetrix; - } - - /** - * @return the {@code Integer} that is the number of analysis using Affymetrix data where - * a different call is found - */ - public Integer getInconsistentDEACountAffymetrix() { - return this.inconsistentDEACountAffymetrix; - } - - /** - * @param inconsistentDEACountAffymetrix An {@code Integer} that is the number of - * analysis using Affymetrix data where - * a different call is found. - */ - //deprecated because all TOs should now be immutable. - @Deprecated - void setInconsistentDEACountAffymetrix(Integer inconsistentDEACountAffymetrix) { - this.inconsistentDEACountAffymetrix = inconsistentDEACountAffymetrix; - } - /** * @return the {@code DiffExprCallType} that is the type of differential expression * of this call generated by RNA-seq. @@ -594,10 +464,6 @@ void setInconsistentDEACountRNASeq(Integer inconsistentDEACountRNASeq) { @Override public String toString() { return super.toString() + " - Comparison factor: " + this.getComparisonFactor() + - " - Differential expression call by Affymetrix: " + this.getDiffExprCallTypeAffymetrix() + - " - Best p-value with Affymetrix: " + this.getBestPValueAffymetrix() + - " - Consistent DEA Count with Affymetrix: " + this.getConsistentDEACountAffymetrix() + - " - Inconsistent DEA Count with Affymetrix: " + this.getInconsistentDEACountAffymetrix() + " - Differential expression call by RNA-seq: " + this.getDiffExprCallTypeRNASeq() + " - Best p-value with RNA-seq: " + this.getBestPValueRNASeq() + " - Consistent DEA Count with RNA-seq: " + this.getConsistentDEACountRNASeq() + diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/RawDataCallSourceDAO.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/RawDataCallSourceDAO.java index e5aa389ed..3e2d0d9d1 100644 --- a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/RawDataCallSourceDAO.java +++ b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/RawDataCallSourceDAO.java @@ -36,7 +36,7 @@ public static interface CallSourceWithRankTO { } /** * An interface describing elements allowing to produce {@link CallSourceDataTO}, for instance: - * an Affymetrix probeset, an RNA-Seq result, an EST, an in situ hybridization spot. + * an RNA-Seq result, an in situ hybridization spot. *

          * Implementation note: this an interface rather than a class, because some {@code CallSourceTO} * have an ID, some do not, so we want these elements to freely extend {@code EntityTO}, or not, @@ -53,7 +53,7 @@ public static interface CallSourceWithRankTO { public static interface CallSourceTO> extends Serializable { /** * @return the ID this {@code CallSourceTO} is part of, for instance, the ID - * of an Affymetrix chip, or of an RNA-Seq library. + * of an RNA-Seq library. */ public T getAssayId(); /** @@ -115,9 +115,8 @@ public default Long getExpressionId() { /** * A {@code TransferObject} carrying information specifically about the aspect of the raw data * allowing to generate a call of presence/absence of expression. For information about the element - * producing this call (for instance, an Affymetrix probeset, having a specific ID), - * or the assay which this element belongs to (for instance, the Affymetrix chip which the probeset - * belongs to), see {@link CallSourceTO}. + * producing this call, + * or the assay which this element belongs to, see {@link CallSourceTO}. * * @author Frederic Bastian * @author Valentine Rech de Laval @@ -152,7 +151,7 @@ public static class CallSourceDataTO extends TransferObject { public enum ExclusionReason implements EnumDAOField { NOT_EXCLUDED("not excluded"), PRE_FILTERING("pre-filtering"), UNDEFINED("undefined"), NO_EXPRESSION_CONFLICT("noExpression conflict"), - ABSENT_NOT_RELIABLE("absent call not reliable"); + BIOTYPE_NOT_TARGETED("biotype not targeted"); /** * Convert the {@code String} representation of a exclusion reason (for instance, diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/RawDataCountDAO.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/RawDataCountDAO.java index 6ac794262..706b6373e 100644 --- a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/RawDataCountDAO.java +++ b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/RawDataCountDAO.java @@ -39,33 +39,6 @@ public String getTOFieldName() { } } - /** - * Retrieve affymetrix count of experiment, assay and calls based on a {@code Collection} of - * {@code DAORawDataFilter}. - * - * @param rawDataFilters A {@code Collection} of {@code DAORawDataFilter} used to filter - * affymetrix data for which count are queried. - * @param experimentCount A boolean defining rather experiment count has to be retrieved. - * @param assayCount A boolean defining rather assay count has to be retrieved - * @param callCount A boolean defining rather calls count has to be retrieved - * @return A {@code RawDataConditionTO} containing requested counts. - */ - public RawDataCountContainerTO getAffymetrixCount(Collection rawDataFilters, - boolean experimentCount, boolean assayCount, boolean callCount); - - /** - * Retrieve EST count of assay and calls based on a {@code Collection} of - * {@code DAORawDataFilter}. - * - * @param rawDataFilters A {@code Collection} of {@code DAORawDataFilter} used to filter - * EST data for which count are queried. - * @param assayCount A boolean defining rather assay count has to be retrieved - * @param callCount A boolean defining rather calls count has to be retrieved - * @return A {@code RawDataConditionTO} containing requested counts. - */ - public RawDataCountContainerTO getESTCount(Collection rawDataFilters, - boolean assayCount, boolean callCount); - /** * Retrieve insitu count of experiment, assay and calls based on a {@code Collection} of * {@code DAORawDataFilter}. diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/SamplePValueDAO.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/SamplePValueDAO.java index 2850b71c7..cfdeb7740 100644 --- a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/SamplePValueDAO.java +++ b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/SamplePValueDAO.java @@ -29,27 +29,6 @@ enum Attribute implements DAO.Attribute { EXPRESSION_ID, EXPERIMENT_ID, SAMPLE_ID, P_VALUE; } - /** - * Retrieve affymetrix p-values from the data source, linked to expression IDs - * of the raw expression table, and bgee internal affymetrix chip ID, - * and ordered by gene ID and expression ID of raw expression table. - *

          - * The data are retrieved and returned as an {@code SamplePValueTOResultSet}. - * It is the responsibility of the caller to close this {@code DAOResultSet} once results are retrieved. - * - * @param geneIds A {@code Collection of {@code Integer}s that are the Bgee IDs - * of the genes to retrieve Affymetrix p-values for. - * @return An {@code SamplePValueTOResultSet} allowing to obtain - * the requested {@code SamplePValueTO}s. - * @throws DAOException If an error occurred while accessing the data source. - * @throws IllegalArgumentException If {@code geneIds} is {@code null} or empty. - */ - //If we retrieve the experiment ID, the public chip ID is a String and is unique in an experiment. - //If we don't retrieve the experiment ID, the Bgee internal chip ID is an int and is unique - //over the whole Bgee database, it's why we define the sample ID type as Integer. - public SamplePValueTOResultSet getAffymetrixPValuesOrderedByGeneIdAndExprId( - Collection geneIds) throws DAOException, IllegalArgumentException; - /** * Retrieve RNA-Seq p-values from the data source, linked to expression IDs * of the raw expression table, and RNA-Seq experiment and library IDs, @@ -86,26 +65,6 @@ public SamplePValueTOResultSet getRNASeqPValuesOrderedByGeneIdAn public SamplePValueTOResultSet getInSituPValuesOrderedByGeneIdAndExprId( Collection geneIds) throws DAOException, IllegalArgumentException; - /** - * Retrieve EST p-values from the data source, linked to expression IDs - * of the raw expression table, and EST library IDs, - * and ordered by gene ID and expression ID of raw expression table. - *

          - * The data are retrieved and returned as an {@code SamplePValueTOResultSet}. - * It is the responsibility of the caller to close this {@code DAOResultSet} once results are retrieved. - * - * @param geneIds A {@code Collection of {@code Integer}s that are the Bgee IDs - * of the genes to retrieve EST p-values for. - * @return An {@code SamplePValueTOResultSet} allowing to obtain - * the requested {@code SamplePValueTO}s. - * @throws DAOException If an error occurred while accessing the data source. - * @throws IllegalArgumentException If {@code geneIds} is {@code null} or empty. - */ - //There is no experiment ID for EST data, only library IDs, that will be populated - //in the returned SamplePValueTOs as 'sampleId'. - public SamplePValueTOResultSet getESTPValuesOrderedByGeneIdAndExprId( - Collection geneIds) throws DAOException, IllegalArgumentException; - /** * Retrieve single-cell RNA-Seq full lenth p-values from the data source, linked to expression IDs * of the raw expression table, and experiment and library IDs, diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/call/DAORawCallFilter.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/call/DAORawCallFilter.java new file mode 100644 index 000000000..678667cec --- /dev/null +++ b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/call/DAORawCallFilter.java @@ -0,0 +1,59 @@ +package org.bgee.model.dao.api.expressiondata.rawdata.call; + +import java.util.Collection; +import java.util.EnumSet; +import java.util.Objects; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bgee.model.dao.api.expressiondata.DAODataFilter; +import org.bgee.model.dao.api.expressiondata.DAODataType; +import org.bgee.model.dao.api.expressiondata.rawdata.DAORawDataConditionFilter; + +public class DAORawCallFilter extends DAODataFilter{ + + private final static Logger log = LogManager.getLogger(DAORawCallFilter.class.getName()); + private final EnumSet dataTypes; + public DAORawCallFilter(Collection geneIds, Collection speciesIds, + Collection conditionFilters, EnumSet dataTypes) { + super(geneIds, speciesIds, conditionFilters); + if ((geneIds == null || geneIds.isEmpty()) && (speciesIds == null || speciesIds.isEmpty()) && + (conditionFilters == null || conditionFilters.isEmpty())) { + throw log.throwing(new IllegalArgumentException("at least one geneId, speciesId or condition filter" + + " should be provided")); + } + this.dataTypes = dataTypes == null ? EnumSet.allOf(DAODataType.class) : dataTypes; + } + + public EnumSet getDataTypes() { + return dataTypes; + } + + @Override + public String toString() { + return "DAORawCallFilter [dataTypes=" + dataTypes + ", getGeneIds()=" + getGeneIds() + ", getSpeciesIds()=" + + getSpeciesIds() + ", getConditionFilters()=" + getConditionFilters() + "]"; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + Objects.hash(dataTypes); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (!super.equals(obj)) + return false; + if (getClass() != obj.getClass()) + return false; + DAORawCallFilter other = (DAORawCallFilter) obj; + return Objects.equals(dataTypes, other.dataTypes); + } + + +} diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/call/DAORawCallValues.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/call/DAORawCallValues.java new file mode 100644 index 000000000..875537d69 --- /dev/null +++ b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/call/DAORawCallValues.java @@ -0,0 +1,52 @@ +package org.bgee.model.dao.api.expressiondata.rawdata.call; + +import java.math.BigDecimal; +import java.util.Objects; + +public class DAORawCallValues { + private final BigDecimal score; + private final BigDecimal pValue; + private final BigDecimal weight; + + public DAORawCallValues(BigDecimal score, BigDecimal pValue, BigDecimal weight) { + this.score = score; + this.pValue = pValue; + this.weight = weight; + } + + public BigDecimal getScore() { + return score; + } + + public BigDecimal getpValue() { + return pValue; + } + + public BigDecimal getWeight() { + return weight; + } + + @Override + public String toString() { + return "DAODRawCallValues [score=" + score + ", pValue=" + pValue + ", weight=" + weight + "]"; + } + + @Override + public int hashCode() { + return Objects.hash(pValue, score, weight); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + DAORawCallValues other = (DAORawCallValues) obj; + return Objects.equals(pValue, other.pValue) && Objects.equals(score, other.score) + && Objects.equals(weight, other.weight); + } + +} diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/call/RawExpressionCallDAO.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/call/RawExpressionCallDAO.java new file mode 100644 index 000000000..4d0bebc83 --- /dev/null +++ b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/call/RawExpressionCallDAO.java @@ -0,0 +1,186 @@ +package org.bgee.model.dao.api.expressiondata.rawdata.call; + +import java.util.Collection; +import java.util.EnumSet; +import java.util.Map; +import java.util.stream.Collectors; + +import org.bgee.model.dao.api.DAO; +import org.bgee.model.dao.api.DAOResultSet; +import org.bgee.model.dao.api.TransferObject; +import org.bgee.model.dao.api.exception.DAOException; +import org.bgee.model.dao.api.expressiondata.DAODataType; + +/** + * DAO defining queries using or retrieving {@link RawExpressionCallTO}s. + * + * @author Valentine Rech de Laval + * @author Frederic Bastian + * @version Bgee 15.0, Apr. 2021 + * @since Bgee 14, Feb. 2017 + * @see RawExpressionCallTO + */ +public interface RawExpressionCallDAO extends DAO { + + /** + * {@code Enum} used to define the attributes to populate in the {@code RawExpressionCallTO}s + * obtained from this {@code RawExpressionCallDAO}. + *

            + *
          • {@code ID}: corresponds to {@link RawExpressionCallTO#getId()}. + *
          • {@code BGEE_GENE_ID}: corresponds to {@link RawExpressionCallTO#getBgeeGeneId()}. + *
          • {@code CONDITION_ID}: corresponds to {@link RawExpressionCallTO#getConditionId()}. + *
          • {@code SCORE}: corresponds to {@link RawExpressionCallTO#getScore()}. + *
          • {@code CONDITION_ID}: corresponds to {@link RawExpressionCallTO#getConditionId()}. + *
          • {@code CONDITION_ID}: corresponds to {@link RawExpressionCallTO#getConditionId()}. + *
          + */ + public enum Attribute implements DAO.Attribute { + EXPRESSION_ID("expressionId", false), BGEE_GENE_ID("bgeeGeneId", false), CONDITION_ID("conditionId", false), + SCORE("exprScore", true), PVALUE("pValue", true), WEIGHT("weight", true); + + /** + * A {@code String} that is the corresponding field name in {@code RelationTO} class. + * @see {@link Attribute#getTOFieldName()} + */ + private final String fieldName; + /** + * A {@code boolean} defining if the {@code Attribute} is data type dependant. + * If true then the {@code RawExpressionCallTO} will have a different attribute for each + * data type. + */ + private final boolean dataTypeDependant; + + private Attribute(String fieldName, boolean dataTypeDependant) { + this.fieldName = fieldName; + this.dataTypeDependant = dataTypeDependant; + } + @Override + public String getTOFieldName() { + return this.fieldName; + } + public boolean isDataTypeDependant() { + return dataTypeDependant; + } + + public static EnumSet getDataTypeDependentAttributes () { + return EnumSet.allOf(Attribute.class).stream().filter(a -> a.isDataTypeDependant()) + .collect(Collectors.toCollection(() -> EnumSet.noneOf(Attribute.class))); + } + + } + + /** + * Allows to retrieve {@code RawExpressionCallTO}s according to the provided filters. + *

          + * The {@code RawExpressionCallOTFTO}s are retrieved and returned as a + * {@code RawExpressionCallOTFTOResultSet}. It is the responsibility of the caller to close this + * {@code DAOResultSet} once results are retrieved. + * + * @param rawCallFilter A {@code DAORawCallFilter} allowing to filter raw expression + * calls to retrieve + * @return A {@code RawExpressionCallTOResultSet} allowing to retrieve the + * targeted {@code RawExpressionCallTO}s. + * @throws DAOException If an error occurred while accessing the data source. + */ + public RawExpressionCallTOResultSet getRawExpressionCalls(DAORawCallFilter rawCallFilter) throws DAOException; + + /** + * {@code DAOResultSet} specifics to {@code RawExpressionCallTO}s + * + * @author Valentine Rech de Laval + * @version Bgee 14, Feb. 2017 + * @since Bgee 14, Feb. 2017 + */ + public interface RawExpressionCallTOResultSet extends DAOResultSet { + } + + /** + * {@code EntityTO} representing a raw expression call in the Bgee database. + * + * @author Julien Wollbrett + * @version Bgee 16.0, Mar. 2025 + * @since Bgee 16.0, Mar. 2025 + */ + public class RawExpressionCallTO extends ExpressionCallTO { + + private static final long serialVersionUID = -6659443741328805241L; + + private final Map rawCallValuesPerDataType; + + public RawExpressionCallTO(Long id, Integer bgeeGeneId, Integer conditionId, + Map rawCallValuesPerDataType) { + super(id, bgeeGeneId, conditionId); + this.rawCallValuesPerDataType = rawCallValuesPerDataType; + } + + public Map getRawCallValuesPerDataType() { + return rawCallValuesPerDataType; + } + + @Override + public String toString() { + return "RawExpressionCallTO [rawCallValuesPerDataType=" + rawCallValuesPerDataType + ", getId()=" + getId() + + ", getBgeeGeneId()=" + getBgeeGeneId() + ", getConditionId()=" + getConditionId() + "]"; + } + + } + + /** + * Retrieve raw expression calls for a requested collection of gene IDs, ordered by gene IDs + * and expression IDs. + * + * @param geneIds A {@code Collection of {@code Integer}s that are the Bgee IDs + * of the genes to retrieve calls for. + * @return A {@code RawExpressionCallTOResultSet} allowing to obtain + * the requested {@code RawExpressionCallTO}s. + * @throws DAOException If an error occurred while accessing the data source. + * @throws IllegalArgumentException If {@code geneIds} is {@code null} or empty. + */ + public RawExpressionCallTOResultSet getExpressionCallsOrderedByGeneIdAndExprId( + Collection geneIds) throws DAOException, IllegalArgumentException; + + /** + * {@code EntityTO} representing an expression call in the Bgee database. + * + * @author Valentine Rech de Laval + * @author Frederic Bastian + * @version Bgee 15.0, Apr. 2021 + * @since Bgee 14, Feb. 2017 + */ + public abstract class ExpressionCallTO extends TransferObject { + + private static final long serialVersionUID = -1057540315343857464L; + + private final Long id; + /** + * An {@code Integer} representing the ID of the gene associated to this call. + */ + private final Integer bgeeGeneId; + /** + * An {@code Integer} representing the ID of the condition associated to this call. + */ + private final Integer conditionId; + + public ExpressionCallTO(Long id, Integer bgeeGeneId, Integer conditionId) { + this.id = id; + this.bgeeGeneId = bgeeGeneId; + this.conditionId = conditionId; + } + + public Long getId() { + return this.id; + } + public Integer getBgeeGeneId() { + return this.bgeeGeneId; + } + public Integer getConditionId() { + return this.conditionId; + } + + @Override + public String toString() { + return "RawExpressionCallTO [id=" + this.getId() + ", bgeeGeneId=" + bgeeGeneId + + ", conditionId=" + conditionId + "]"; + } + } +} diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/est/ESTDAO.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/est/ESTDAO.java deleted file mode 100644 index d8515af87..000000000 --- a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/est/ESTDAO.java +++ /dev/null @@ -1,157 +0,0 @@ -package org.bgee.model.dao.api.expressiondata.rawdata.est; - -import java.math.BigDecimal; -import java.util.Collection; - -import org.bgee.model.dao.api.DAO; -import org.bgee.model.dao.api.DAOResultSet; -import org.bgee.model.dao.api.EntityTO; -import org.bgee.model.dao.api.exception.DAOException; -import org.bgee.model.dao.api.expressiondata.call.CallDAO.CallTO.DataState; -import org.bgee.model.dao.api.expressiondata.rawdata.DAORawDataFilter; -import org.bgee.model.dao.api.expressiondata.rawdata.RawDataCallSourceDAO.CallSourceDataTO; -import org.bgee.model.dao.api.expressiondata.rawdata.RawDataCallSourceDAO.CallSourceDataTO.ExclusionReason; -import org.bgee.model.dao.api.expressiondata.rawdata.RawDataCallSourceDAO.CallSourceTO; - -/** - * DAO defining queries using or retrieving {@link ESTTO}s. - * - * @author Frederic Bastian - * @author Valentine Rech de Laval - * @version Bgee 14 - * @since Bgee 01 - */ -public interface ESTDAO extends DAO { - - /** - * {@code Enum} used to define the attributes to populate in the {@code ESTTO}s obtained from - * this {@code ESTDAO}. - *

            - *
          • {@code EST_ID}: corresponds to {@link ESTTO#getId()}. - *
          • {@code EST_ID2}: corresponds to {@link ESTTO#getEstId2()}. - *
          • {@code EST_LIBRARY_ID}: corresponds to {@link ESTTO#getAssayId()}. - *
          • {@code BGEE_GENE_ID}: corresponds to {@link ESTTO#getBgeeGeneId()}. - *
          • {@code UNIGENE_CLUSTER_ID}: corresponds to {@link ESTTO#getUniGeneClusterId()}. - *
          • {@code EXPRESSION_ID}: corresponds to {@link ESTTO#getExpressionId()}. - *
          • {@code PVALUE}: corresponds to {@link ESTTO#getPValue()}. - *
          • {@code EST_DATA}: corresponds to {@link ESTTO#getExpressionConfidence()}. - *
          - */ - public enum Attribute implements DAO.Attribute { - EST_ID("estId"), EST_ID2("estId2"), EST_LIBRARY_ID("estLibraryId"), - BGEE_GENE_ID("bgeeGeneId"), UNIGENE_CLUSTER_ID("UniGeneClusterId"), - EXPRESSION_ID("expressionId"), PVALUE("pValue"), EST_DATA("estData"); - - /** - * A {@code String} that is the corresponding field name in {@code ESTTO} class. - * @see {@link Attribute#getTOFieldName()} - */ - private final String fieldName; - - private Attribute(String fieldName) { - this.fieldName = fieldName; - } - - @Override - public String getTOFieldName() { - return this.fieldName; - } - } - - /** - * Allows to retrieve {@code ESTTO}s according to the provided filters. - *

          - * The {@code ESTTO}s are retrieved and returned as a {@code ESTTOResultSet}. It is the - * responsibility of the caller to close this {@code DAOResultSet} once results are retrieved. - * - * @param rawDataFilters A {@code Collection} of {@code DAORawDataFilter} allowing to specify - * how to filter ESTs to retrieve. The query uses AND between elements - * of a same filter and uses OR between filters. - * @param offset An {@code Integer} used to specify which row to start from retrieving data - * in the result of a query. If null, retrieve data from the first row. - * @param limit A {@code Long} used to limit the number of rows returned in a query - * result. If null, all results are returned. - * {@code Long} because sometimes the number of potential results - * can be very large. - * @param attributes A {@code Collection} of {@code Attribute}s to specify the information - * to retrieve from the data source. - * @return A {@code ESTTOResultSet} allowing to retrieve the - * targeted {@code ESTTOResultSet}s. - * @throws DAOException If an error occurred while accessing the data source. - */ - public ESTTOResultSet getESTs(Collection rawDataFilters, - Long offset, Integer limit, Collection attributes) throws DAOException; - - public interface ESTTOResultSet extends DAOResultSet {} - - /** - * An {@code EntityTO} representing an EST, as stored in the Bgee database. - * - * @author Frederic Bastian - * @author Valentine Rech de Laval - * @version Bgee 14 - * @since Bgee 11 - */ - public final class ESTTO extends EntityTO implements CallSourceTO { - private static final long serialVersionUID = -6130411930176920545L; - - /** - * A {@code String} that is the ID of the assay this EST is part of. - */ - private final String estLibraryId; - /** - * A {@code String} representing the secondary ID of the EST (ESTs have two IDs in Unigene). - */ - private final String estId2; - /** - * A {@code String} representing the ID of UniGene Cluster associated to this EST. - */ - private final String uniGeneClusterId; - /** - * The {@code CallSourceDataTO} carrying the information about - * the produced call of presence/absence of expression. - */ - private final CallSourceDataTO callSourceDataTO; - - public ESTTO(String estId, String estId2, String estLibraryId, String uniGeneClusterId, Integer bgeeGeneId, - DataState expressionConfidence, BigDecimal pValue, Long expressionId) { - super(estId); - this.estId2 = estId2; - this.uniGeneClusterId = uniGeneClusterId; - this.estLibraryId = estLibraryId; - this.callSourceDataTO = new CallSourceDataTO(bgeeGeneId, pValue, - expressionConfidence, ExclusionReason.NOT_EXCLUDED, expressionId); - } - - @Override - public String getAssayId() { - return this.estLibraryId; - } - @Override - public CallSourceDataTO getCallSourceDataTO() { - return this.callSourceDataTO; - } - /** - * @return the {@code String} representing the secondary ID of the EST (ESTs have two IDs in Unigene). - */ - public String getEstId2() { - return this.estId2; - } - /** - * @return the {@code String} representing the ID of UniGene Cluster associated to this EST. - */ - public String getUniGeneClusterId() { - return this.uniGeneClusterId; - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("ESTTO [id=").append(this.getId()).append(", estId2=").append(estId2) - .append(", estLibraryId=").append(estLibraryId) - .append(", uniGeneClusterId=").append(uniGeneClusterId) - .append(", callSourceDataTO=").append(this.callSourceDataTO).append("]"); - return builder.toString(); - } - } -} \ No newline at end of file diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/est/ESTLibraryDAO.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/est/ESTLibraryDAO.java deleted file mode 100644 index 773cafee5..000000000 --- a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/est/ESTLibraryDAO.java +++ /dev/null @@ -1,129 +0,0 @@ -package org.bgee.model.dao.api.expressiondata.rawdata.est; - -import java.util.Collection; - -import org.bgee.model.dao.api.DAO; -import org.bgee.model.dao.api.DAOResultSet; -import org.bgee.model.dao.api.NamedEntityTO; -import org.bgee.model.dao.api.exception.DAOException; -import org.bgee.model.dao.api.expressiondata.rawdata.DAORawDataFilter; -import org.bgee.model.dao.api.expressiondata.rawdata.RawDataAnnotatedTO; -import org.bgee.model.dao.api.expressiondata.rawdata.RawDataAssayDAO.AssayTO; -import org.bgee.model.dao.api.expressiondata.rawdata.RawDataWithDataSourceTO; - -/** - * DAO for {@link ESTLibraryTO}s. - * - * @author Frederic Bastian - * @author Valentine Rech de Laval - * @version Bgee 14 - * @see ESTLibraryTO - * @since Bgee 01 - */ -public interface ESTLibraryDAO extends DAO { - - /** - * {@code Enum} used to define the attributes to populate in the {@code ESTLibraryTO}s - * obtained from this {@code ESTLibraryDAO}. - *

            - *
          • {@code ID}: corresponds to {@link ESTLibraryTO#getId()}. - *
          • {@code NAME}: corresponds to {@link ESTLibraryTO#getName()}. - *
          • {@code DESCRIPTION}: corresponds to {@link ESTLibraryTO#getDescription()}. - *
          • {@code DATA_SOURCE_ID}: corresponds to {@link ESTLibraryTO#getDataSourceId()}. - *
          • {@code CONDITION_ID}: corresponds to {@link ESTLibraryTO#getConditionId()}. - *
          - */ - public enum Attribute implements DAO.Attribute { - ID("estLibraryId"), NAME("estLibraryName"), DESCRIPTION("estLibraryDescription"), - DATA_SOURCE_ID("dataSourceId"), CONDITION_ID("conditionId"); - - /** - * A {@code String} that is the corresponding field name in {@code ESTLibraryTO} class. - * @see {@link Attribute#getTOFieldName()} - */ - private final String fieldName; - - private Attribute(String fieldName) { - this.fieldName = fieldName; - } - - @Override - public String getTOFieldName() { - return this.fieldName; - } - } - - /** - * Allows to retrieve {@code ESTLibraryTO}s according to the provided filters. - *

          - * The {@code ESTLibraryTO}s are retrieved and returned as a {@code ESTLibraryTOResultSet}. It is the - * responsibility of the caller to close this {@code DAOResultSet} once results are retrieved. - * - * @param rawDataFilters A {@code Collection} of {@code DAORawDataFilter} allowing to specify - * how to filter EST libraries to retrieve. The query uses AND between elements - * of a same filter and uses OR between filters. - * @param offset A {@code Long} used to specify which row to start from retrieving data - * in the result of a query. If null, retrieve data from the first row. - * {@code Long} because sometimes the number of potential results - * can be very large. - * @param limit An {@code Integer} used to limit the number of rows returned in a query - * result. If null, all results are returned. - * @param attributes A {@code Collection} of {@code Attribute}s to specify the information - * to retrieve from the data source. - * @return A {@code ESTLibraryTOResultSet} allowing to retrieve the - * targeted {@code ESTLibraryTOResultSet}s. - * @throws DAOException If an error occurred while accessing the data source. - */ - public ESTLibraryTOResultSet getESTLibraries(Collection rawDataFilters, - Long offset, Integer limit, Collection attributes) throws DAOException; - - /** - * {@code DAOResultSet} specifics to {@code ESTLibraryTO}s - * - * @author Julien Wollbrett - * @version Bgee 15 - * @since Bgee 15 - */ - public interface ESTLibraryTOResultSet extends DAOResultSet {} - - /** - * {@code TransferObject} for EST libraries. - * - * @author Frederic Bastian - * @author Valentine Rech de Laval - * @version Bgee 14 - * @since Bgee 11 - */ - public final class ESTLibraryTO extends NamedEntityTO - implements AssayTO, RawDataAnnotatedTO, RawDataWithDataSourceTO { - private static final long serialVersionUID = 6500670452213931420L; - - private final Integer dataSourceId; - private final Integer conditionId; - - public ESTLibraryTO(String id, String name, String description, Integer dataSourceId, - Integer conditionId) { - super(id, name, description); - this.dataSourceId = dataSourceId; - this.conditionId = conditionId; - } - - @Override - public Integer getDataSourceId() { - return this.dataSourceId; - } - @Override - public Integer getConditionId() { - return this.conditionId; - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("ESTLibraryTO [id=").append(getId()).append(", name=").append(getName()) - .append(", description=").append(getDescription()).append(", dataSourceId=").append(dataSourceId) - .append(", conditionId=").append(conditionId).append("]"); - return builder.toString(); - } - } -} \ No newline at end of file diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/microarray/AffymetrixChipDAO.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/microarray/AffymetrixChipDAO.java deleted file mode 100644 index 12db0ed31..000000000 --- a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/microarray/AffymetrixChipDAO.java +++ /dev/null @@ -1,308 +0,0 @@ -package org.bgee.model.dao.api.expressiondata.rawdata.microarray; - -import java.math.BigDecimal; -import java.util.Collection; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bgee.model.dao.api.DAO; -import org.bgee.model.dao.api.DAOResultSet; -import org.bgee.model.dao.api.EntityTO; -import org.bgee.model.dao.api.TransferObject; -import org.bgee.model.dao.api.exception.DAOException; -import org.bgee.model.dao.api.expressiondata.rawdata.DAORawDataFilter; -import org.bgee.model.dao.api.expressiondata.rawdata.RawDataAnnotatedTO; -import org.bgee.model.dao.api.expressiondata.rawdata.RawDataAssayDAO.AssayPartOfExpTO; - -/** - * DAO defining queries using or retrieving {@link AffymetrixChipTO}s. - * - * @author Frederic Bastian - * @author Valentine Rech de Laval - * @author Julien Wollbrett - * @version Bgee 15 Oct. 2022 - * @since Bgee 01 - */ -public interface AffymetrixChipDAO extends DAO { - - /** - * {@code Enum} used to define the attributes to populate in the {@code AffymetrixChipTO}s - * obtained from this {@code AffymetrixChipDAO}. - *

            - *
          • {@code BGEE_AFFYMETRIX_CHIP_ID}: corresponds to {@link AffymetrixChipTO#getId()}. - *
          • {@code AFFYMETRIX_CHIP_ID}: corresponds to {@link AffymetrixChipTO#getAffymetrixChipId()}. - *
          • {@code EXPERIMENT_ID}: corresponds to {@link AffymetrixChipTO#getExperimentId()}. - *
          • {@code CONDITION_ID}: corresponds to {@link AffymetrixChipTO#getConditionId()}. - *
          • {@code SCAN_DATE}: corresponds to {@link AffymetrixChipTO#getScanDate()}. - *
          • {@code CHIP_TYPE_ID}: corresponds to {@link AffymetrixChipTO#getChipTypeId()}. - *
          • {@code NORMALIZATION_TYPE}: corresponds to {@link AffymetrixChipTO#getNormalizationType()}. - *
          • {@code DETECTION_TYPE}: corresponds to {@link AffymetrixChipTO#getDetectionType()}. - *
          • {@code QUALITY_SCORE}: corresponds to {@link AffymetrixChipTO#getQualityScore()}. - *
          • {@code PERCENT_PRESENT}: corresponds to {@link AffymetrixChipTO#getPercentPresent()}. - *
          • {@code MAX_RANK}: corresponds to {@link AffymetrixChipTO#getMaxRank()}. - *
          • {@code DISTINCT_RANK_COUNT}: corresponds to {@link AffymetrixChipTO#getDistinctRankCount()}. - *
          - */ - public enum Attribute implements DAO.Attribute { - BGEE_AFFYMETRIX_CHIP_ID("bgeeAffymetrixChipId"), AFFYMETRIX_CHIP_ID("affymetrixChipId"), - EXPERIMENT_ID("microarrayExperimentId"), CHIP_TYPE_ID("chipTypeId"), SCAN_DATE("scanDate"), - NORMALIZATION_TYPE("normalizationType"), DETECTION_TYPE("detectionType"), - CONDITION_ID("conditionId"), QUALITY_SCORE("qualityScore"), - PERCENT_PRESENT("percentPresent"), MAX_RANK("chipMaxRank"), - DISTINCT_RANK_COUNT("chipDistinctRankCount"); - - /** - * A {@code String} that is the corresponding field name in {@code AffymetrixChipTO} class. - * @see {@link Attribute#getTOFieldName()} - */ - private final String fieldName; - - private Attribute(String fieldName) { - this.fieldName = fieldName; - } - - @Override - public String getTOFieldName() { - return this.fieldName; - } - } - public AffymetrixChipTOResultSet getAffymetrixChipsFromBgeeChipIds(Collection bgeeChipIds, - Collection attrs) throws DAOException; - - /** - * Allows to retrieve {@code AffymetrixChipTO}s according to the provided filters, - * ordered by microarray experiment IDs and bgee Affymetrix chip IDs. - *

          - * The {@code AffymetrixChipTO}s are retrieved and returned as a - * {@code AffymetrixChipTOResultSet}. It is the responsibility of the caller to close this - * {@code DAOResultSet} once results are retrieved. - * - * @param rawDatafilters A {@code Collection} of {@code DAORawDataFilter} allowing to filter which - * chips to retrieve. The query uses AND between elements of a same filter and - * uses OR between filters. - * @param offset A {@code Long} used to specify which row to start from retrieving data - * in the result of a query. If null, retrieve data from the first row. - * {@code Long} because sometimes the number of potential results - * can be very large. - * @param limit An {@code Integer} used to limit the number of rows returned in a query - * result. If null, all results are returned. - * @param attributes A {@code Collection} of {@code Attribute}s to specify the information - * to retrieve from the data source. - * @return A {@code AffymetrixChipTOResultSet} allowing to retrieve the targeted - * {@code AffymetrixChipTO}s. - * @throws DAOException If an error occurred while accessing the data source. - */ - public AffymetrixChipTOResultSet getAffymetrixChips(Collection rawDatafilters, - Long offset, Integer limit, Collection attributes) throws DAOException; - - /** - * {@code DAOResultSet} for {@code AffymetrixChipTO}s - * - * @author Frederic Bastian - * @version Bgee 14, Sept. 2018 - * @since Bgee 14, Sept. 2018 - */ - public interface AffymetrixChipTOResultSet extends DAOResultSet { - } - - /** - * {@code TransferObject} for Affymetrix chips. - * - * @author Frederic Bastian - * @version Bgee 14 Sept. 2018 - * @since Bgee 11 - */ - public final class AffymetrixChipTO extends EntityTO - implements AssayPartOfExpTO, RawDataAnnotatedTO { - - private static final long serialVersionUID = 7479060565564264352L; - private final static Logger log = LogManager.getLogger(AffymetrixChipTO.class.getName()); - - /** - * {@code Enum} representing the different types of normalization that can be applied in Bgee to Affymetrix data. - * - * @author Frederic Bastian - * @version Bgee 14 - * @since Bgee 14 - */ - public enum NormalizationType implements EnumDAOField { - MAS5("MAS5"), RMA("RMA"), GC_RMA("gcRMA"); - - /** - * See {@link #getStringRepresentation()} - */ - private final String stringRepresentation; - /** - * Constructor providing the {@code String} representation of this {@code NormalizationType}. - * - * @param stringRepresentation A {@code String} corresponding to this {@code NormalizationType}. - */ - private NormalizationType(String stringRepresentation) { - this.stringRepresentation = stringRepresentation; - } - - /** - * Convert the {@code String} representation of a normalization type (for instance, - * retrieved from a database) into a {@code NormalizationType}. This method compares - * {@code representation} to the value returned by {@link #getStringRepresentation()}, - * as well as to the value returned by {@link Enum#name()}, for each {@code NormalizationType}. - * - * @param representation A {@code String} representing a normalization type. - * @return A {@code NormalizationType} corresponding to {@code representation}. - * @throws IllegalArgumentException If {@code representation} does not correspond to - * any {@code NormalizationType}. - */ - public static final NormalizationType convertToNormalizationType(String representation) { - log.traceEntry("{}", representation); - return log.traceExit(TransferObject.convert(NormalizationType.class, representation)); - } - - @Override - public String getStringRepresentation() { - return this.stringRepresentation; - } - @Override - public String toString() { - return this.getStringRepresentation(); - } - } - /** - * {@code Enum} representing the different methods used to detect signal of active expression - * from Affymetrix chips. - *

            - *
          • {@code MAS5}: present/marginal/absent calls produced from the MAS5 software, - * when the raw data are not available, but only the MAS5 processed data. - *
          • {@code SCHUSTER}: method from Schuster et al. using a subset of lowly expressed probeset - * to define background transcriptional noise, when raw data are available. - *
          - * - * @author Frederic Bastian - * @version Bgee 14 - * @since Bgee 14 - */ - public enum DetectionType implements EnumDAOField { - MAS5("MAS5"), SCHUSTER("Schuster"); - - /** - * See {@link #getStringRepresentation()} - */ - private final String stringRepresentation; - /** - * Constructor providing the {@code String} representation of this {@code DetectionType}. - * - * @param stringRepresentation A {@code String} corresponding to this {@code DetectionType}. - */ - private DetectionType(String stringRepresentation) { - this.stringRepresentation = stringRepresentation; - } - - /** - * Convert the {@code String} representation of a detection type (for instance, - * retrieved from a database) into a {@code DetectionType}. This method compares - * {@code representation} to the value returned by {@link #getStringRepresentation()}, - * as well as to the value returned by {@link Enum#name()}, for each {@code DetectionType}. - * - * @param representation A {@code String} representing a detection type. - * @return A {@code DetectionType} corresponding to {@code representation}. - * @throws IllegalArgumentException If {@code representation} does not correspond to any {@code DetectionType}. - */ - public static final DetectionType convertToDetectionType(String representation) { - log.traceEntry("{}", representation); - return log.traceExit(TransferObject.convert(DetectionType.class, representation)); - } - - @Override - public String getStringRepresentation() { - return this.stringRepresentation; - } - @Override - public String toString() { - return this.getStringRepresentation(); - } - } - - private final String microarrayExperimentId; - private final String affymetrixChipId; - private final Integer conditionId; - - private final String scanDate; - private final String chipTypeId; - private final NormalizationType normalizationType; - private final DetectionType detectionType; - private final BigDecimal qualityScore; - private final BigDecimal percentPresent; - private final BigDecimal maxRank; - private final Integer distinctRankCount; - - /** - * Default constructor. - */ - public AffymetrixChipTO(Integer bgeeAffymetrixChipId, String affymetrixChipId, String microarrayExperimentId, - String chipTypeId, String scanDate, NormalizationType normalizationType, - DetectionType detectionType, Integer conditionId, BigDecimal qualityScore, BigDecimal percentPresent, - BigDecimal maxRank, Integer distinctRankCount) { - super(bgeeAffymetrixChipId); - this.microarrayExperimentId = microarrayExperimentId; - this.affymetrixChipId = affymetrixChipId; - this.conditionId = conditionId; - this.scanDate = scanDate; - this.chipTypeId = chipTypeId; - this.normalizationType = normalizationType; - this.detectionType = detectionType; - this.qualityScore = qualityScore; - this.percentPresent = percentPresent; - this.maxRank = maxRank; - this.distinctRankCount = distinctRankCount; - } - - @Override - public String getExperimentId() { - return this.microarrayExperimentId; - } - @Override - public Integer getConditionId() { - return this.conditionId; - } - public String getAffymetrixChipId() { - return affymetrixChipId; - } - public String getScanDate() { - return scanDate; - } - public String getChipTypeId() { - return chipTypeId; - } - public NormalizationType getNormalizationType() { - return normalizationType; - } - public DetectionType getDetectionType() { - return detectionType; - } - public BigDecimal getQualityScore() { - return qualityScore; - } - public BigDecimal getPercentPresent() { - return percentPresent; - } - public BigDecimal getMaxRank() { - return maxRank; - } - public Integer getDistinctRankCount() { - return distinctRankCount; - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("AffymetrixChipTO [bgeeAffymetrixChipId=").append(getId()) - .append(", microarrayExperimentId=").append(microarrayExperimentId) - .append(", affymetrixChipId=").append(affymetrixChipId).append(", conditionId=").append(conditionId) - .append(", scanDate=").append(scanDate).append(", chipTypeId=").append(chipTypeId) - .append(", normalizationType=").append(normalizationType).append(", detectionType=") - .append(detectionType).append(", qualityScore=").append(qualityScore).append(", percentPresent=") - .append(percentPresent).append(", maxRank=").append(maxRank).append(", distinctRankCount=") - .append(distinctRankCount).append("]"); - return builder.toString(); - } - } -} diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/microarray/AffymetrixChipTypeDAO.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/microarray/AffymetrixChipTypeDAO.java deleted file mode 100644 index f385ecfc0..000000000 --- a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/microarray/AffymetrixChipTypeDAO.java +++ /dev/null @@ -1,141 +0,0 @@ -package org.bgee.model.dao.api.expressiondata.rawdata.microarray; - -import java.math.BigDecimal; -import java.util.Collection; - -import org.bgee.model.dao.api.DAO; -import org.bgee.model.dao.api.DAOResultSet; -import org.bgee.model.dao.api.EntityTO; -import org.bgee.model.dao.api.exception.DAOException; - -/** - * DAO defining queries using or retrieving {@link AffymetrixChipTypeTO}s. - * - * @author Julien Wollbrett - * @version Bgee 15 Nov. 2022 - */ -public interface AffymetrixChipTypeDAO extends DAO { - - /** - * {@code Enum} used to define the attributes to populate in the {@code AffymetrixChipTypeTO}s - * obtained from this {@code AffymetrixChipTypeDAO}. - *
            - *
          • {@code CHIP_TYPE_ID}: corresponds to {@link AffymetrixChipTypeTO#getId()}. - *
          • {@code CHIP_TYPE_NAME}: corresponds to {@link AffymetrixChipTypeTO#getAffymetrixChipTypeName()}. - *
          • {@code CDF_NAME}: corresponds to {@link AffymetrixChipTypeTO#getCdfName()}. - *
          • {@code IS_COMPATIBLE}: corresponds to {@link AffymetrixChipTypeTO#isCompatible()}. - *
          • {@code QUALITY_SCORE_THRESHOLD}: corresponds to {@link AffymetrixChipTypeTO#getQualityScoreThreshold()}. - *
          • {@code PERCENT_PRESENT_THRESHOLD}: corresponds to {@link AffymetrixChipTypeTO#getPercentPresentThreshold()}. - *
          • {@code CHIP_TYPE_MAX_RANK}: corresponds to {@link AffymetrixChipTypeTO#getChipTypeMaxRank()}. - *
          - */ - public enum Attribute implements DAO.Attribute { - CHIP_TYPE_ID("chipTypeId"), CHIP_TYPE_NAME("chipTypeName"), - CDF_NAME("cdfName"), IS_COMPATIBLE("isCompatible"), QUALITY_SCORE_THRESHOLD("qualityScoreThreshold"), - PERCENT_PRESENT_THRESHOLD("percentPresentThreshold"), CHIP_TYPE_MAX_RANK("chipTypeMaxRank"); - - /** - * A {@code String} that is the corresponding field name in {@code AffymetrixChipTypeTO} class. - * @see {@link Attribute#getTOFieldName()} - */ - private final String fieldName; - - private Attribute(String fieldName) { - this.fieldName = fieldName; - } - - @Override - public String getTOFieldName() { - return this.fieldName; - } - } - - /** - * Allows to retrieve {@code AffymetrixChipTypeTO}s according to the provided chip type IDs - *

          - * The {@code AffymetrixChipTypeTO}s are retrieved and returned as a - * {@code AffymetrixChipTypeTOResultSet}. It is the responsibility of the caller to close this - * {@code DAOResultSet} once results are retrieved. - * - * @param chipTypeIds A {@code Collection} of {@code String} allowing to filter which - * chip types to retrieve. If null or empty retrieve all chip types - * @param attributes A {@code Collection} of {@code Attribute}s to specify the information - * to retrieve from the data source. - * @return A {@code AffymetrixChipTypeTOResultSet} allowing to retrieve the targeted - * {@code AffymetrixChipTypeTO}s. - * @throws DAOException If an error occurred while accessing the data source. - */ - public AffymetrixChipTypeTOResultSet getAffymetrixChipTypes(Collection chipTypeIds, - Collection attributes) throws DAOException; - - /** - * {@code DAOResultSet} for {@code AffymetrixChipTypeTO}s - * - * @author Julien Wollbrett - * @version Bgee 15, Nov. 2022 - */ - public interface AffymetrixChipTypeTOResultSet extends DAOResultSet { - } - - /** - * {@code TransferObject} for Affymetrix chip types. - * - * @author Julien Wollbrett - * @version Bgee 15 Nov. 2022 - */ - public final class AffymetrixChipTypeTO extends EntityTO { - - private static final long serialVersionUID = -885779088447205595L; - - private final String affymetrixChipTypeName; - private final String cdfName; - - private final Boolean isCompatible; - private final BigDecimal qualityScoreThreshold; - private final BigDecimal percentPresentThreshold; - private final BigDecimal chipTypeMaxRank; - - /** - * Default constructor. - */ - public AffymetrixChipTypeTO(String affymetrixChipTypeId, String affymetrixChipTypeName, - String cdfName, boolean isCompatible, BigDecimal qualityScoreThreshold, - BigDecimal percentPresentThreshold, - BigDecimal chipTypeMaxRank) { - super(affymetrixChipTypeId); - this.affymetrixChipTypeName = affymetrixChipTypeName; - this.cdfName = cdfName; - this.isCompatible = isCompatible; - this.qualityScoreThreshold = qualityScoreThreshold; - this.percentPresentThreshold = percentPresentThreshold; - this.chipTypeMaxRank = chipTypeMaxRank; - } - - public String getAffymetrixChipTypeName() { - return affymetrixChipTypeName; - } - public String getCdfName() { - return cdfName; - } - public Boolean getIsCompatible() { - return isCompatible; - } - public BigDecimal getQualityScoreThreshold() { - return qualityScoreThreshold; - } - public BigDecimal getPercentPresentThreshold() { - return percentPresentThreshold; - } - public BigDecimal getChipTypeMaxRank() { - return chipTypeMaxRank; - } - - @Override - public String toString() { - return "AffymetrixChipTypeTO [affymetrixChipTypeName=" + affymetrixChipTypeName + ", cdfName=" + cdfName - + ", isCompatible=" + isCompatible + ", qualityScoreThreshold=" + qualityScoreThreshold - + ", percentPresentThreshold=" + percentPresentThreshold + ", chipTypeMaxRank=" + chipTypeMaxRank - + "]"; - } - } -} diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/microarray/AffymetrixProbesetDAO.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/microarray/AffymetrixProbesetDAO.java deleted file mode 100644 index 8cc661f92..000000000 --- a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/microarray/AffymetrixProbesetDAO.java +++ /dev/null @@ -1,199 +0,0 @@ -package org.bgee.model.dao.api.expressiondata.rawdata.microarray; - -import java.math.BigDecimal; -import java.util.Collection; - -import org.bgee.model.dao.api.DAO; -import org.bgee.model.dao.api.DAOResultSet; -import org.bgee.model.dao.api.EntityTO; -import org.bgee.model.dao.api.exception.DAOException; -import org.bgee.model.dao.api.expressiondata.call.CallDAO.CallTO.DataState; -import org.bgee.model.dao.api.expressiondata.rawdata.DAORawDataFilter; -import org.bgee.model.dao.api.expressiondata.rawdata.RawDataCallSourceDAO.CallSourceDataTO; -import org.bgee.model.dao.api.expressiondata.rawdata.RawDataCallSourceDAO.CallSourceDataTO.ExclusionReason; -import org.bgee.model.dao.api.expressiondata.rawdata.RawDataCallSourceDAO.CallSourceTO; -import org.bgee.model.dao.api.expressiondata.rawdata.RawDataCallSourceDAO.CallSourceWithRankTO; - -/** - * DAO defining queries using or retrieving {@link AffymetrixProbesetTO}s. - * - * @author Frederic Bastian - * @author Valentine Rech de Laval - * @version Bgee 14 Sept. 2018 - * @see AffymetrixProbesetTO - * @since Bgee 01 - */ -public interface AffymetrixProbesetDAO extends DAO { - - /** - * {@code Enum} used to define the attributes to populate in the {@code AffymetrixProbesetTO}s - * obtained from this {@code AffymetrixProbesetDAO}. - *

            - *
          • {@code ID}: corresponds to {@link AffymetrixProbesetTO#getId()}. - *
          • {@code BGEE_AFFYMETRIX_CHIP_ID}: corresponds to {@link AffymetrixProbesetTO#getAssayId()}. - *
          • {@code BGEE_GENE_ID}: corresponds to {@link AffymetrixProbesetTO#getBgeeGeneId()}. - *
          • {@code NORMALIZED_SIGNAL_INTENSITY}: corresponds to {@link AffymetrixProbesetTO#getNormalizedSignalIntensity()}. - *
          • {@code PVALUE}: corresponds to {@link AffymetrixProbesetTO#getPValue()}. - *
          • {@code QVALUE}: corresponds to {@link AffymetrixProbesetTO#getQValue()}. - *
          • {@code EXPRESSION_ID}: corresponds to {@link AffymetrixProbesetTO#getExpressionId()}. - *
          • {@code RANK}: corresponds to {@link AffymetrixProbesetTO#getRank()}. - *
          • {@code AFFYMETRIX_DATA}: corresponds to {@link AffymetrixProbesetTO#getExpressionConfidence()}. - *
          • {@code REASON_FOR_EXCLUSION}: corresponds to {@link AffymetrixProbesetTO#getExclusionReason()}. - *
          - */ - public enum Attribute implements DAO.Attribute { - ID("affymetrixProbesetId"), BGEE_AFFYMETRIX_CHIP_ID("bgeeAffymetrixChipId"), RAW_DETECTION_FLAG("rawDetectionFlag"), - BGEE_GENE_ID("bgeeGeneId"), NORMALIZED_SIGNAL_INTENSITY("normalizedSignalIntensity"), - PVALUE("pValue"), QVALUE("qValue"), EXPRESSION_ID("expressionId"), - RANK("rawRank"),AFFYMETRIX_DATA("affymetrixData"), - REASON_FOR_EXCLUSION("reasonForExclusion"); - - /** - * A {@code String} that is the corresponding field name in {@code AffymetrixChipTO} class. - * @see {@link Attribute#getTOFieldName()} - */ - private final String fieldName; - - private Attribute(String fieldName) { - this.fieldName = fieldName; - } - - @Override - public String getTOFieldName() { - return this.fieldName; - } - } - - /** - * Allows to retrieve {@code AffymetrixProbesetTO}s according to the provided filters. - *

          - * The {@code AffymetrixProbesetTO}s are retrieved and returned as a - * {@code AffymetrixProbesetTOResultSet}. It is the responsibility of the caller to close this - * {@code DAOResultSet} once results are retrieved. - * - * @param rawDataFilters A {@code Collection} of {@code DAORawDataFilter} allowing to specify - * how to filter probesets to retrieve. The query uses AND between elements - * of a same filter and uses OR between filters. - * @param offset A {@code Long} used to specify which row to start from retrieving data - * in the result of a query. If null, retrieve data from the first row. - * {@code Long} because sometimes the number of potential results - * can be very large. - * @param limit An {@code Integer} used to limit the number of rows returned in a query - * result. If null, all results are returned. - * @param attributes A {@code Collection} of {@code Attribute}s to specify the information - * to retrieve from the data source. - * @return A {@code AffymetrixProbesetTOResultSet} allowing to retrieve the - * targeted {@code AffymetrixProbesetTO}s. - * @throws DAOException If an error occurred while accessing the data source. - */ - public AffymetrixProbesetTOResultSet getAffymetrixProbesets(Collection rawDataFilters, - Long offset, Integer limit, Collection attributes) throws DAOException; - - /** - * {@code DAOResultSet} for {@code AffymetrixProbesetTO}s - * - * @author Frederic Bastian - * @version Bgee 14, Sept. 2018 - * @since Bgee 14, Sept. 2018 - */ - public interface AffymetrixProbesetTOResultSet extends DAOResultSet { - } - - /** - * A {@code TransferObject} representing an Affymetrix probeset, as stored in the Bgee database. - * - * @author Frederic Bastian - * @author Valentine Rech de Laval - * @version Bgee 14 - * @see org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixProbesetDAO - * @since Bgee 11 - */ - public final class AffymetrixProbesetTO extends EntityTO - implements CallSourceTO, CallSourceWithRankTO { - private static final long serialVersionUID = 1081576994949088868L; - - private final Integer bgeeAffymetrixChipId; - /** - * A {@code BigDecimal} defining the normalized signal intensity of this probeset. - */ - private final BigDecimal normalizedSignalIntensity; - /** - * A {@code BigDecimal} that is the rank of this call source raw data. - */ - private final BigDecimal rank; - /** - * The {@code CallSourceDataTO} carrying the information about - * the produced call of presence/absence of expression. - */ - private final CallSourceDataTO callSourceDataTO; - private final BigDecimal qValue; - - /** - * All of these parameters are optional, so they can be {@code null} when not used. - * - * @param affymetrixProbesetId A {@code String} that is the ID of this probeset. - * @param bgeeAffymetrixChipId An {@code Integer} that is the internal Bgee Affymetrix chip ID - * associated to this probeset. - * @param bgeeGeneId An {@code Integer} that is the internal Bgee gene ID of the gene associated - * to this probeset. - * @param normalizedSignalIntensity A {@code BigDecimal} defining the normalized signal intensity - * of this probeset. - * @param pValue A {@code BigDecimal} representing the pValue used to define presence/absence - * of expression - * @param qValue A {@code BigDecimal} representing the qValue of the call - * @param expressionId A {@code String} that is the ID of the expression - * associated to this probeset. - * @param rank A {@code BigDecimal} that is the rank associated to this probeset on this chip. - * @param expressionConfidence A {@code DataState} that is the expression confidence - * of this probeset. - * @param reasonForExclusion An {@code ExclusionReason} that is the reason of - * exclusion of this probeset. - */ - public AffymetrixProbesetTO(String affymetrixProbesetId, Integer bgeeAffymetrixChipId, Integer bgeeGeneId, - BigDecimal normalizedSignalIntensity, BigDecimal pValue, BigDecimal qValue, - Long expressionId, BigDecimal rank, DataState expressionConfidence, ExclusionReason exclusionReason) { - super(affymetrixProbesetId); - this.bgeeAffymetrixChipId = bgeeAffymetrixChipId; - this.normalizedSignalIntensity = normalizedSignalIntensity; - this.rank = rank; - this.qValue = qValue; - this.callSourceDataTO = new CallSourceDataTO(bgeeGeneId, pValue, - expressionConfidence, exclusionReason, expressionId); - } - - @Override - public Integer getAssayId() { - return this.bgeeAffymetrixChipId; - } - @Override - public CallSourceDataTO getCallSourceDataTO() { - return this.callSourceDataTO; - } - /** - * @return the {@code BigDecimal} defining the normalized signal intensity of this probeset. - */ - public BigDecimal getNormalizedSignalIntensity() { - return this.normalizedSignalIntensity; - } - /** - * @return A {@code BigDecimal} that is the rank of this call source raw data. - */ - public BigDecimal getRank() { - return this.rank; - } - /** - * @return A {@code BigDecimal} that is the qvalue of this call source raw data. - */ - public BigDecimal getqValue() { - return qValue; - } - - @Override - public String toString() { - return "AffymetrixProbesetTO [bgeeAffymetrixChipId=" + bgeeAffymetrixChipId + ", normalizedSignalIntensity=" - + normalizedSignalIntensity + ", rank=" + rank + ", callSourceDataTO=" + callSourceDataTO - + ", qValue=" + qValue + "]"; - } - - } -} diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/microarray/MicroarrayExperimentDAO.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/microarray/MicroarrayExperimentDAO.java deleted file mode 100644 index e8487dfd3..000000000 --- a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/expressiondata/rawdata/microarray/MicroarrayExperimentDAO.java +++ /dev/null @@ -1,107 +0,0 @@ -package org.bgee.model.dao.api.expressiondata.rawdata.microarray; - -import java.util.Collection; - -import org.bgee.model.dao.api.DAO; -import org.bgee.model.dao.api.DAOResultSet; -import org.bgee.model.dao.api.exception.DAOException; -import org.bgee.model.dao.api.expressiondata.rawdata.DAORawDataFilter; -import org.bgee.model.dao.api.expressiondata.rawdata.RawDataExperimentDAO.ExperimentTO; - -/** - * DAO defining queries using or retrieving {@link MicroarrayExperimentTO}s. - * - * @author Frederic Bastian - * @author Valentine Rech de Laval - * @version Bgee 14 Sept. 2018 - * @since Bgee 01 - */ -public interface MicroarrayExperimentDAO extends DAO { - - /** - * {@code Enum} used to define the attributes to populate in the {@code MicroarrayExperimentTO}s - * obtained from this {@code MicroarrayExperimentDAO}. - *

            - *
          • {@code ID}: corresponds to {@link MicroarrayExperimentTO#getId()}. - *
          • {@code NAME}: corresponds to {@link MicroarrayExperimentTO#getName()}. - *
          • {@code DESCRIPTION}: corresponds to {@link MicroarrayExperimentTO#getDescription()}. - *
          • {@code DATA_SOURCE_ID}: corresponds to {@link MicroarrayExperimentTO#getDataSourceId()}. - *
          - */ - public enum Attribute implements DAO.Attribute { - ID("microarrayExperimentId"), NAME("microarrayExperimentName"), - DESCRIPTION("microarrayExperimentDescription"), DATA_SOURCE_ID("dataSourceId"); - - /** - * A {@code String} that is the corresponding field name in {@code MicroarrayExperimentTO} - * class. - * @see {@link Attribute#getTOFieldName()} - */ - private final String fieldName; - - private Attribute(String fieldName) { - this.fieldName = fieldName; - } - - @Override - public String getTOFieldName() { - return this.fieldName; - } - } - - /** - * Allows to retrieve {@code MicroarrayExperimentTO}s according to the provided filters. - *

          - * The {@code MicroarrayExperimentTO}s are retrieved and returned as a - * {@code MicroarrayExperimentTOResultSet}. It is the responsibility of the caller to close this - * {@code DAOResultSet} once results are retrieved. - * - * @param rawDataFilters A {@code Collection} of {@code DAORawDataFilter} allowing to specify - * how to filter experiments to retrieve. The query uses AND between - * elements of a same filter and uses OR between filters. - * @param offset A {@code Long} used to specify which row to start from retrieving data - * in the result of a query. If null, retrieve data from the first row. If - * not null, a limit should be also provided. - * {@code Long} because sometimes the number of potential results - * can be very large. - * @param limit An {@code Integer} used to limit the number of rows returned in a query - * result. If null, all results are returned. - * @param attributes A {@code Collection} of {@code Attribute}s to specify the information - * to retrieve from the data source. - * @return A {@code AffymetrixProbesetTOResultSet} allowing to retrieve the - * targeted {@code AffymetrixProbesetTO}s. - * @throws DAOException If an error occurred while accessing the data source. - */ - public MicroarrayExperimentTOResultSet getExperiments(Collection rawDataFilters, - Long offset, Integer limit, Collection attributes) throws DAOException; - - /** - * {@code DAOResultSet} for {@code MicroarrayExperimentTO}s - * - * @author Frederic Bastian - * @version Bgee 14, Sept. 2018 - * @since Bgee 14, Sept. 2018 - */ - public interface MicroarrayExperimentTOResultSet extends DAOResultSet { - } - - /** - * {@code TransferObject} for Affymetrix {@coe ExperimentTO}. - * - * @author Frederic Bastian - * @author Valentine Rech de Laval - * @version Bgee 14 Sept. 2018 - * @since Bgee 11 - */ - public final class MicroarrayExperimentTO extends ExperimentTO { - private static final long serialVersionUID = 5255742948654816580L; - - /** - * Default constructor. - */ - public MicroarrayExperimentTO(String id, String name, String description, - Integer dataSourceId) { - super(id, name, description, dataSourceId); - } - } -} diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/file/DownloadFileDAO.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/file/DownloadFileDAO.java index b30a3048d..7b5f48d2e 100644 --- a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/file/DownloadFileDAO.java +++ b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/file/DownloadFileDAO.java @@ -85,8 +85,6 @@ public final class DownloadFileTO extends NamedEntityTO { *

        • {@code DIFF_EXPR_DEV_COMPLETE} a complete differential expression across developmental stages file
          • *
        • {@code DIFF_EXPR_DEV_SIMPLE}a simple differential expression across developmental stages file
          • *
        • {@code ORTHOLOG} corresponds to an orthologies file
          • - *
        • {@code AFFY_ANNOT} corresponds to an Affymetrix annoations file
          • - *
        • {@code AFFY_DATA} corresponds to an Affymetrix signal intensities file
          • *
        • {@code RNASEQ_ANNOT} corresponds to RNA-Seq annotations file
          • *
        • {@code RNASEQ_DATA} corresponds toRNA-Seq data file
          • *
        @@ -103,8 +101,6 @@ public enum CategoryEnum implements TransferObject.EnumDAOField { DIFF_EXPR_DEV_COMPLETE("diff_expr_dev_complete"), DIFF_EXPR_DEV_SIMPLE("diff_expr_dev_simple"), ORTHOLOG("ortholog"), - AFFY_ANNOT("affy_annot"), - AFFY_DATA("affy_data"), RNASEQ_ANNOT("rnaseq_annot"), RNASEQ_DATA("rnaseq_data"), FULL_LENGTH_ANNOT("full_length_annot"), diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/gene/GeneDAO.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/gene/GeneDAO.java index 6a4eb3c99..4a6752ea7 100644 --- a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/gene/GeneDAO.java +++ b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/gene/GeneDAO.java @@ -33,6 +33,7 @@ public interface GeneDAO extends DAO { *
      • {@code SPECIES_ID}: corresponds to {@link GeneTO#getSpeciesId()}. *
      • {@code GENE_BIO_TYPE_ID}: corresponds to {@link GeneTO#getGeneBioTypeId()}. *
      • {@code ENSEMBL_GENE}: corresponds to {@link GeneTO#isEnsemblGene()}. + *
      • {@code SEQ_REGION_NAME}: corresponds to {@link GeneTO#getSeqRegionName()} *
      • {@code GENE_MAPPED_TO_SAME_GENE_ID_COUNT}: corresponds to {@link GeneTO#getGeneMappedToGeneIdCount()}. *
      • {@code EXPRESSION_SUMMARY}: corresponds to {@link GeneTO#getExpressionSummary()}. *
      @@ -42,8 +43,8 @@ public interface GeneDAO extends DAO { */ public enum Attribute implements DAO.Attribute { ID("bgeeGeneId"), GENE_ID("geneId"), NAME("geneName"), DESCRIPTION("geneDescription"), - SPECIES_ID("speciesId"), GENE_BIO_TYPE_ID("geneBiotypeId"),ENSEMBL_GENE("ensemblGene"), - GENE_MAPPED_TO_SAME_GENE_ID_COUNT("geneMappedToGeneIdCount"), + SPECIES_ID("speciesId"), GENE_BIO_TYPE_ID("geneBioTypeId"),ENSEMBL_GENE("ensemblGene"), + SEQ_REGION_NAME("seqRegionName"), GENE_MAPPED_TO_SAME_GENE_ID_COUNT("geneMappedToGeneIdCount"), EXPRESSION_SUMMARY("expressionSummary"); /** @@ -232,18 +233,16 @@ public class GeneTO extends NamedEntityTO { */ private final Integer geneBioTypeId; - /** - * An {@code Integer} that is unique ID for each node inside an OMA Hierarchical Orthologous - * Group. It can be {@code null} if the gene does not belong to a hierarchical group. A gene - * can belong to one and only one group. - */ - private final Integer OMAParentNodeId; - /** * A {@code Boolean} defining whether this gene is present in Ensembl. For some species, * they are not (for instance, we generate our own custom IDs for some species) */ private final Boolean ensemblGene; + + /** + * A {@code String} that is the region where this gene comes from. + */ + private final String seqRegionName; /** * @see #getGeneMappedToGeneIdCount() @@ -296,14 +295,14 @@ public GeneTO(Integer bgeeGeneId, String geneId, String geneName, Integer specie * for anat. entities and celltypes. */ public GeneTO(Integer bgeeGeneId, String geneId, String geneName, String geneDescription, - Integer speciesId, Integer geneBioTypeId, Integer OMAParentNodeId, Boolean ensemblGene, + Integer speciesId, Integer geneBioTypeId, Boolean ensemblGene, String seqRegionName, Integer geneMappedToGeneIdCount, String expressionSummary) { super(bgeeGeneId, geneName, geneDescription); this.geneId = geneId; this.speciesId = speciesId; this.geneBioTypeId = geneBioTypeId; - this.OMAParentNodeId = OMAParentNodeId; this.ensemblGene = ensemblGene; + this.seqRegionName = seqRegionName; this.geneMappedToGeneIdCount = geneMappedToGeneIdCount; this.expressionSummary = expressionSummary; } @@ -326,12 +325,11 @@ public Integer getSpeciesId() { public Integer getGeneBioTypeId() { return this.geneBioTypeId; } - /** - * @return The OMA Hierarchical Orthologous Group ID that this gene belongs to. - */ - public Integer getOMAParentNodeId() { - return this.OMAParentNodeId; + + public String getSeqRegionName() { + return seqRegionName; } + /** * @return The {@code Boolean} defining whether this gene is present in Ensembl. */ @@ -357,9 +355,8 @@ public String getExpressionSummary() { @Override public String toString() { return "GeneTO [geneId=" + geneId + ", speciesId=" + speciesId + ", geneBioTypeId=" + geneBioTypeId - + ", OMAParentNodeId=" + OMAParentNodeId + ", ensemblGene=" + ensemblGene - + ", geneMappedToGeneIdCount=" + geneMappedToGeneIdCount + ", expressionSummary=" - + expressionSummary + "]"; + + ", ensemblGene=" + ensemblGene + ", seqRegionName=" + seqRegionName + ", geneMappedToGeneIdCount=" + + geneMappedToGeneIdCount + ", expressionSummary=" + expressionSummary + "]"; } } diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/source/SourceDAO.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/source/SourceDAO.java index cc858848a..8b9c37bd6 100644 --- a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/source/SourceDAO.java +++ b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/source/SourceDAO.java @@ -129,16 +129,13 @@ public final class SourceTO extends NamedEntityTO { *
    • {@code GENOMICS}: the category is a genomics database. *
    • {@code PROTEOMICS}: the category is a proteomics database. *
    • {@code IN_SITU}: the category is a in situ data source. - *
    • {@code AFFYMETRIX}: the category is an Affymetrix data source. - *
    • {@code EST}: the category is an EST data source. *
    • {@code RNA_SEQ}: the category is a RNA-Seq data source. *
    • {@code ONTOLOGY}: the category is an ontology. *
    */ public enum SourceCategory implements EnumDAOField { NONE(""), GENOMICS("Genomics database"), PROTEOMICS("Proteomics database"), - IN_SITU("In situ data source"), AFFYMETRIX("Affymetrix data source"), - EST("EST data source"), RNA_SEQ("RNA-Seq data source"), + IN_SITU("In situ data source"), RNA_SEQ("RNA-Seq data source"), SC_RNA_SEQ("Single-cell RNA-Seq data source"), ONTOLOGY("Ontology"); /** @@ -194,7 +191,7 @@ public String toString() { /** * A {@code String} that is the URL to evidence if it is expression data source (in situ - * evidence for in situ databases or Affymetrix chips for affymetrix data). + * evidence for in situ databases). *

    * The parameter evidence ID is defined by the syntax [evidence_id]. */ diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/source/SourceToSpeciesDAO.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/source/SourceToSpeciesDAO.java index 953731313..9cd243bc0 100644 --- a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/source/SourceToSpeciesDAO.java +++ b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/source/SourceToSpeciesDAO.java @@ -170,7 +170,7 @@ public String toString() { private Integer speciesId; /** - * A {@code DataType} that is the data type (for instance, affymetrix). + * A {@code DataType} that is the data type (for instance, rna-seq). */ private DAODataType dataType; diff --git a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/species/SpeciesDAO.java b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/species/SpeciesDAO.java index 44822f30a..ab2aa04c4 100644 --- a/bgee-dao-api/src/main/java/org/bgee/model/dao/api/species/SpeciesDAO.java +++ b/bgee-dao-api/src/main/java/org/bgee/model/dao/api/species/SpeciesDAO.java @@ -32,17 +32,18 @@ public interface SpeciesDAO extends DAO { *

  • {@code DATA_SOURCE_ID}: corresponds to {@link SpeciesTO#getDataSourceId()}. *
  • {@code GENOME_SPECIES_ID}: corresponds to {@link SpeciesTO#getGenomeSpeciesId()}. *
  • {@code DISPLAY_ORDER}: corresponds to {@link SpeciesTO#getDisplayOrder()}. + *
  • {@code DEV_ONTOLOGY_XREF}: corresponds to {@link SpeciesTO#getdevOntologyXRef()}. * * @see org.bgee.model.dao.api.DAO#setAttributes(Collection) * @see org.bgee.model.dao.api.DAO#setAttributes(Enum[]) * @see org.bgee.model.dao.api.DAO#clearAttributes() */ public enum Attribute implements DAO.Attribute { - ID("speciesId"), COMMON_NAME("name"), GENUS("genus"), SPECIES_NAME("speciesName"), - PARENT_TAXON_ID("parentTaxonId"), GENOME_FILE_PATH("genomeFilePath"), - GENOME_VERSION("genomeVersion"), GENOME_ASSEMBLY_XREF("getGenomeAssemblyXRef"), - DATA_SOURCE_ID("dataSourceId"), GENOME_SPECIES_ID("genomeSpeciesId"), - DISPLAY_ORDER("speciesDisplayOrder"); + ID("speciesId"), GENUS("genus"), SPECIES_NAME("species"), COMMON_NAME("speciesCommonName"), + DISPLAY_ORDER("speciesDisplayOrder"), PARENT_TAXON_ID("taxonId"), + GENOME_FILE_PATH("genomeFilePath"), GENOME_VERSION("genomeVersion"), + GENOME_ASSEMBLY_XREF("genomeAssemblyXRef"), DATA_SOURCE_ID("dataSourceId"), + GENOME_SPECIES_ID("genomeSpeciesId"), DEV_ONTOLOGY_XREF("devOntologyXRef"); /** * A {@code String} that is the corresponding field name in {@code RelationTO} class. @@ -201,6 +202,11 @@ public final class SpeciesTO extends NamedEntityTO { * (ID 9598), because bonobo is not in Ensembl. */ private final Integer genomeSpeciesId; + + /** + * A {@code String} that is the URL to the potential species specific dev. ontology. + */ + private final String devOntologyXRef; /** * Constructor providing the ID, the common name, the genus, the species, and the ID @@ -231,7 +237,8 @@ public final class SpeciesTO extends NamedEntityTO { */ public SpeciesTO(Integer id, String commonName, String genus, String speciesName, Integer displayOrder, Integer parentTaxonId, String genomeFilePath, String genomeVersion, - String genomeAssemblyXRef, Integer dataSourceId, Integer genomeSpeciesId) { + String genomeAssemblyXRef, Integer dataSourceId, Integer genomeSpeciesId, + String devOntologyXRef) { super(id, commonName); this.genus = genus; @@ -243,6 +250,7 @@ public SpeciesTO(Integer id, String commonName, String genus, String speciesName this.genomeAssemblyXRef = genomeAssemblyXRef; this.dataSourceId = dataSourceId; this.genomeSpeciesId = genomeSpeciesId; + this.devOntologyXRef = devOntologyXRef; } /** @@ -329,16 +337,18 @@ public Integer getGenomeSpeciesId() { return genomeSpeciesId; } + public String getDevOntologyXRef() { + return devOntologyXRef; + } + @Override public String toString() { - return "ID: " + this.getId() + " - Common name: " + this.getName() + - " - Genus: " + this.getGenus() + " - Species name: " + this.getSpeciesName() + - " - Parent taxon ID: " + this.getParentTaxonId() + " - Description: " + - this.getDescription() + " - Genome file path: " + this.getGenomeFilePath() + - " - Genome version: " + this.getGenomeFilePath() + - " - Genome assembly XRef: " + this.getGenomeAssemblyXRef() + - " - Data source ID: " + this.getDataSourceId() + " - Genome species ID: " + - this.getGenomeSpeciesId(); + return "SpeciesTO [genus=" + genus + ", speciesName=" + speciesName + ", displayOrder=" + displayOrder + + ", parentTaxonId=" + parentTaxonId + ", genomeFilePath=" + genomeFilePath + ", genomeVersion=" + + genomeVersion + ", genomeAssemblyXRef=" + genomeAssemblyXRef + + ", dataSourceId=" + dataSourceId + ", genomeSpeciesId=" + genomeSpeciesId + ", devOntologyXRef=" + + devOntologyXRef + "]"; } + } } diff --git a/bgee-dao-api/src/test/java/org/bgee/model/dao/api/MockDAOManager.java b/bgee-dao-api/src/test/java/org/bgee/model/dao/api/MockDAOManager.java index 3ab814f6e..68c3a0501 100644 --- a/bgee-dao-api/src/test/java/org/bgee/model/dao/api/MockDAOManager.java +++ b/bgee-dao-api/src/test/java/org/bgee/model/dao/api/MockDAOManager.java @@ -1,5 +1,7 @@ package org.bgee.model.dao.api; +import static org.mockito.Mockito.mock; + import java.util.Properties; import org.bgee.model.dao.api.anatdev.AnatEntityDAO; @@ -10,22 +12,16 @@ import org.bgee.model.dao.api.anatdev.mapping.StageGroupingDAO; import org.bgee.model.dao.api.anatdev.mapping.SummarySimilarityAnnotationDAO; import org.bgee.model.dao.api.exception.DAOException; -import org.bgee.model.dao.api.expressiondata.rawdata.RawExpressionCallDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.SamplePValueDAO; +import org.bgee.model.dao.api.expressiondata.ObservedExpressionDAO; import org.bgee.model.dao.api.expressiondata.call.ConditionDAO; import org.bgee.model.dao.api.expressiondata.call.DiffExpressionCallDAO; import org.bgee.model.dao.api.expressiondata.call.GlobalExpressionCallDAO; import org.bgee.model.dao.api.expressiondata.rawdata.RawDataConditionDAO; import org.bgee.model.dao.api.expressiondata.rawdata.RawDataCountDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTLibraryDAO; +import org.bgee.model.dao.api.expressiondata.rawdata.SamplePValueDAO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituEvidenceDAO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituExperimentDAO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituSpotDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipTypeDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixProbesetDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.MicroarrayExperimentDAO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqExperimentDAO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqLibraryAnnotatedSampleDAO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqLibraryDAO; @@ -47,8 +43,6 @@ import org.bgee.model.dao.api.species.SpeciesDAO; import org.bgee.model.dao.api.species.TaxonDAO; -import static org.mockito.Mockito.mock; - /** * A class to simulate an implementation of {@link DAOManager}, that is discovered * by the {@code Service Loader} thanks to the test file @@ -157,9 +151,13 @@ protected ConditionDAO getNewConditionDAO() { protected RawDataConditionDAO getNewRawDataConditionDAO() { return this.instanceMockManager.getNewRawDataConditionDAO(); } +// @Override +// protected RawExpressionCallDAO getNewRawExpressionCallDAO() { +// return this.instanceMockManager.getNewRawExpressionCallDAO(); +// } @Override - protected RawExpressionCallDAO getNewRawExpressionCallDAO() { - return this.instanceMockManager.getNewRawExpressionCallDAO(); + protected ObservedExpressionDAO getNewObservedExpressionDAO() { + return this.instanceMockManager.getNewObservedExpressionDAO(); } @Override protected GlobalExpressionCallDAO getNewGlobalExpressionCallDAO() { @@ -174,10 +172,6 @@ protected AnatEntityDAO getNewAnatEntityDAO() { return this.instanceMockManager.getNewAnatEntityDAO(); } @Override - protected AffymetrixProbesetDAO getNewAffymetrixProbesetDAO() { - return this.instanceMockManager.getNewAffymetrixProbesetDAO(); - } - @Override protected InSituSpotDAO getNewInSituSpotDAO() { return this.instanceMockManager.getInSituSpotDAO(); } @@ -246,16 +240,6 @@ protected SamplePValueDAO getNewSamplePValueDAO() { return this.instanceMockManager.getNewSamplePValueDAO(); } - @Override - protected AffymetrixChipDAO getNewAffymetrixChipDAO() { - return this.instanceMockManager.getNewAffymetrixChipDAO(); - } - - @Override - protected MicroarrayExperimentDAO getNewMicroarrayExperimentDAO() { - return this.instanceMockManager.getNewMicroarrayExperimentDAO(); - } - @Override protected RNASeqExperimentDAO getNewRnaSeqExperimentDAO() { return this.instanceMockManager.getNewRnaSeqExperimentDAO(); @@ -281,21 +265,6 @@ protected RawDataCountDAO getNewRawDataCountDAO() { return this.instanceMockManager.getRawDataCountDAO(); } - @Override - protected AffymetrixChipTypeDAO getNewAffymetrixChipTypeDAO() { - return this.instanceMockManager.getAffymetrixChipTypeDAO(); - } - - @Override - protected ESTLibraryDAO getNewESTLibraryDAO() { - return this.instanceMockManager.getESTLibraryDAO(); - } - - @Override - protected ESTDAO getNewESTDAO() { - return this.instanceMockManager.getESTDAO(); - } - @Override protected InSituExperimentDAO getNewInSituExperimentDAO() { return this.instanceMockManager.getInSituExperimentDAO(); diff --git a/bgee-dao-api/src/test/java/org/bgee/model/dao/api/MockDAOManager2.java b/bgee-dao-api/src/test/java/org/bgee/model/dao/api/MockDAOManager2.java index 57ab7a0c7..e3238aa07 100644 --- a/bgee-dao-api/src/test/java/org/bgee/model/dao/api/MockDAOManager2.java +++ b/bgee-dao-api/src/test/java/org/bgee/model/dao/api/MockDAOManager2.java @@ -10,22 +10,16 @@ import org.bgee.model.dao.api.anatdev.mapping.StageGroupingDAO; import org.bgee.model.dao.api.anatdev.mapping.SummarySimilarityAnnotationDAO; import org.bgee.model.dao.api.exception.DAOException; -import org.bgee.model.dao.api.expressiondata.rawdata.RawExpressionCallDAO; import org.bgee.model.dao.api.expressiondata.rawdata.SamplePValueDAO; +import org.bgee.model.dao.api.expressiondata.ObservedExpressionDAO; import org.bgee.model.dao.api.expressiondata.call.ConditionDAO; import org.bgee.model.dao.api.expressiondata.call.DiffExpressionCallDAO; import org.bgee.model.dao.api.expressiondata.call.GlobalExpressionCallDAO; import org.bgee.model.dao.api.expressiondata.rawdata.RawDataConditionDAO; import org.bgee.model.dao.api.expressiondata.rawdata.RawDataCountDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTLibraryDAO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituEvidenceDAO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituExperimentDAO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituSpotDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipTypeDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixProbesetDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.MicroarrayExperimentDAO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqExperimentDAO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqLibraryAnnotatedSampleDAO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqLibraryDAO; @@ -166,9 +160,13 @@ protected ConditionDAO getNewConditionDAO() { protected RawDataConditionDAO getNewRawDataConditionDAO() { return this.instanceMockManager.getNewRawDataConditionDAO(); } +// @Override +// protected RawExpressionCallDAO getNewRawExpressionCallDAO() { +// return this.instanceMockManager.getNewRawExpressionCallDAO(); +// } @Override - protected RawExpressionCallDAO getNewRawExpressionCallDAO() { - return this.instanceMockManager.getNewRawExpressionCallDAO(); + protected ObservedExpressionDAO getNewObservedExpressionDAO() { + return this.instanceMockManager.getNewObservedExpressionDAO(); } @Override protected GlobalExpressionCallDAO getNewGlobalExpressionCallDAO() { @@ -183,10 +181,6 @@ protected AnatEntityDAO getNewAnatEntityDAO() { return this.instanceMockManager.getNewAnatEntityDAO(); } @Override - protected AffymetrixProbesetDAO getNewAffymetrixProbesetDAO() { - return this.instanceMockManager.getNewAffymetrixProbesetDAO(); - } - @Override protected InSituSpotDAO getNewInSituSpotDAO() { return this.instanceMockManager.getNewInSituSpotDAO(); } @@ -255,16 +249,6 @@ protected SamplePValueDAO getNewSamplePValueDAO() { return instanceMockManager.getSamplePValueDAO(); } - @Override - protected AffymetrixChipDAO getNewAffymetrixChipDAO() { - return instanceMockManager.getNewAffymetrixChipDAO(); - } - - @Override - protected MicroarrayExperimentDAO getNewMicroarrayExperimentDAO() { - return instanceMockManager.getNewMicroarrayExperimentDAO(); - } - @Override protected RNASeqExperimentDAO getNewRnaSeqExperimentDAO() { return instanceMockManager.getNewRnaSeqExperimentDAO(); @@ -290,21 +274,6 @@ protected RawDataCountDAO getNewRawDataCountDAO() { return instanceMockManager.getNewRawDataCountDAO(); } - @Override - protected AffymetrixChipTypeDAO getNewAffymetrixChipTypeDAO() { - return instanceMockManager.getNewAffymetrixChipTypeDAO(); - } - - @Override - protected ESTLibraryDAO getNewESTLibraryDAO() { - return instanceMockManager.getNewESTLibraryDAO(); - } - - @Override - protected ESTDAO getNewESTDAO() { - return instanceMockManager.getNewESTDAO(); - } - @Override protected InSituExperimentDAO getNewInSituExperimentDAO() { return instanceMockManager.getNewInSituExperimentDAO(); diff --git a/bgee-dao-api/src/test/java/org/bgee/model/dao/api/TOComparator.java b/bgee-dao-api/src/test/java/org/bgee/model/dao/api/TOComparator.java index 3a9bfd8ca..02483a217 100644 --- a/bgee-dao-api/src/test/java/org/bgee/model/dao/api/TOComparator.java +++ b/bgee-dao-api/src/test/java/org/bgee/model/dao/api/TOComparator.java @@ -29,14 +29,9 @@ import org.bgee.model.dao.api.expressiondata.rawdata.RawDataCallSourceDAO.CallSourceTO; import org.bgee.model.dao.api.expressiondata.rawdata.RawDataConditionDAO.RawDataConditionTO; import org.bgee.model.dao.api.expressiondata.rawdata.RawDataExperimentDAO.ExperimentTO; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTDAO.ESTTO; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTLibraryDAO.ESTLibraryTO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituEvidenceDAO.InSituEvidenceTO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituExperimentDAO.InSituExperimentTO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituSpotDAO.InSituSpotTO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipDAO.AffymetrixChipTO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixProbesetDAO.AffymetrixProbesetTO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.MicroarrayExperimentDAO.MicroarrayExperimentTO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqExperimentDAO.RNASeqExperimentTO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqLibraryDAO.RNASeqLibraryTO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqResultAnnotatedSampleDAO; @@ -203,22 +198,12 @@ public static boolean areTOsEqual(T to1, T to2, boole return log.traceExit(areTOsEqual((SourceTO) to1, (SourceTO) to2, compareId)); } else if (to2 instanceof SourceToSpeciesTO) { return log.traceExit(areTOsEqual((SourceToSpeciesTO) to1, (SourceToSpeciesTO) to2)); - } else if (to2 instanceof AffymetrixProbesetTO) { - return log.traceExit(areTOsEqual((AffymetrixProbesetTO) to1, (AffymetrixProbesetTO) to2, compareId)); - } else if (to2 instanceof AffymetrixChipTO) { - return log.traceExit(areTOsEqual((AffymetrixChipTO) to1, (AffymetrixChipTO) to2, compareId)); - } else if (to2 instanceof MicroarrayExperimentTO) { - return log.traceExit(areTOsEqual((MicroarrayExperimentTO) to1, (MicroarrayExperimentTO) to2, compareId)); } else if (to2 instanceof RNASeqResultAnnotatedSampleTO) { return log.traceExit(areTOsEqual((RNASeqResultAnnotatedSampleTO) to1, (RNASeqResultAnnotatedSampleTO) to2)); } else if (to2 instanceof RNASeqLibraryTO) { return log.traceExit(areTOsEqual((RNASeqLibraryTO) to1, (RNASeqLibraryTO) to2, compareId)); } else if (to2 instanceof RNASeqExperimentTO) { return log.traceExit(areTOsEqual((RNASeqExperimentTO) to1, (RNASeqExperimentTO) to2, compareId)); - } else if (to2 instanceof ESTLibraryTO) { - return log.traceExit(areTOsEqual((ESTLibraryTO) to1, (ESTLibraryTO) to2, compareId)); - } else if (to2 instanceof ESTTO) { - return log.traceExit(areTOsEqual((ESTTO) to1, (ESTTO) to2, compareId)); } else if (to2 instanceof InSituExperimentTO) { return log.traceExit(areTOsEqual((InSituExperimentTO) to1, (InSituExperimentTO) to2, compareId)); } else if (to2 instanceof InSituEvidenceTO) { @@ -486,7 +471,7 @@ private static boolean areTOsEqual(GeneTO geneTO1, GeneTO geneTO2, Objects.equals(geneTO1.getGeneId(), geneTO2.getGeneId()) && Objects.equals(geneTO1.getSpeciesId(), geneTO2.getSpeciesId()) && Objects.equals(geneTO1.getGeneBioTypeId(), geneTO2.getGeneBioTypeId()) && - Objects.equals(geneTO1.getOMAParentNodeId(), geneTO2.getOMAParentNodeId()) && + Objects.equals(geneTO1.getSeqRegionName(), geneTO2.getSeqRegionName()) && Objects.equals(geneTO1.isEnsemblGene(), geneTO2.isEnsemblGene()) && Objects.equals(geneTO1.getGeneMappedToGeneIdCount(), geneTO2.getGeneMappedToGeneIdCount())) { return log.traceExit(true); @@ -843,8 +828,6 @@ private static boolean areCallTOsEqual(CallTO to1, CallTO to2, if (areEntityTOsEqual(to1, to2, compareId) && Objects.equals(to1.getBgeeGeneId(), to2.getBgeeGeneId()) && Objects.equals(to1.getConditionId(), to2.getConditionId()) && - Objects.equals(to1.getAffymetrixData(), to2.getAffymetrixData()) && - Objects.equals(to1.getESTData(), to2.getESTData()) && Objects.equals(to1.getInSituData(), to2.getInSituData()) && Objects.equals(to1.getRelaxedInSituData(), to2.getRelaxedInSituData()) && Objects.equals(to1.getRNASeqData(), to2.getRNASeqData())) { @@ -982,11 +965,6 @@ private static boolean areTOsEqual(DiffExpressionCallTO to1, DiffExpressionCallT log.entry(to1, to2); if (TOComparator.areCallTOsEqual(to1, to2, compareId) && Objects.equals(to1.getComparisonFactor(), to2.getComparisonFactor()) && - Objects.equals(to1.getDiffExprCallTypeAffymetrix(), to2.getDiffExprCallTypeAffymetrix()) && - TOComparator.areNearlyEqualFloat( - to1.getBestPValueAffymetrix(), to2.getBestPValueAffymetrix()) && - Objects.equals(to1.getConsistentDEACountAffymetrix(), to2.getConsistentDEACountAffymetrix()) && - Objects.equals(to1.getInconsistentDEACountAffymetrix(), to2.getInconsistentDEACountAffymetrix()) && Objects.equals(to1.getDiffExprCallTypeRNASeq(), to2.getDiffExprCallTypeRNASeq()) && TOComparator.areNearlyEqualFloat( to1.getBestPValueRNASeq(), to2.getBestPValueRNASeq()) && @@ -1200,24 +1178,6 @@ private static boolean areTOsEqual(SourceToSpeciesTO to1, SourceToSpeciesTO to2) return log.traceExit(false); } - /** - * Method to compare two {@code AffymetrixProbesetTO}s, to check for complete - * equality of each attribute. - * - * @param to1 A {@code AffymetrixProbesetTO} to be compared to {@code to2}. - * @param to2 A {@code AffymetrixProbesetTO} to be compared to {@code to1}. - * @return {@code true} if {@code to1} and {@code to2} have all attributes equal. - */ - private static boolean areTOsEqual(AffymetrixProbesetTO to1, AffymetrixProbesetTO to2, boolean compareId) { - log.entry(to1, to2, compareId); - if (areEntityTOsEqual(to1, to2, compareId) && - areBigDecimalEquals(to1.getNormalizedSignalIntensity(), to2.getNormalizedSignalIntensity()) && - areBigDecimalEquals(to1.getRank(), to2.getRank()) && - areCallSourceTOsEqual(to1, to2)) { - return log.traceExit(true); - } - return log.traceExit(false); - } /** * Method to compare two {@code RNASeqResultTO}s, to check for complete * equality of each attribute. @@ -1258,24 +1218,6 @@ private static boolean areTOsEqual(InSituSpotTO to1, InSituSpotTO to2, boolean c } return log.traceExit(false); } - /** - * Method to compare two {@code ESTTO}s, to check for complete - * equality of each attribute. - * - * @param to1 A {@code ESTTO} to be compared to {@code to2}. - * @param to2 A {@code ESTTO} to be compared to {@code to1}. - * @return {@code true} if {@code to1} and {@code to2} have all attributes equal. - */ - private static boolean areTOsEqual(ESTTO to1, ESTTO to2, boolean compareId) { - log.entry(to1, to2); - if (areEntityTOsEqual(to1, to2, compareId) && - areCallSourceTOsEqual(to1, to2) && - Objects.equals(to1.getEstId2(), to2.getEstId2()) && - Objects.equals(to1.getUniGeneClusterId(), to2.getUniGeneClusterId())) { - return log.traceExit(true); - } - return log.traceExit(false); - } /** * Method to compare two {@code CallSourceTO}s, to check for complete @@ -1313,34 +1255,6 @@ private static boolean areTOsEqual(CallSourceDataTO to1, CallSourceDataTO to2) { return log.traceExit(false); } - /** - * Method to compare two {@code AffymetrixChipTO}s, to check for complete - * equality of each attribute. - * - * @param to1 An {@code AffymetrixChipTO} to be compared to {@code to2}. - * @param to2 An {@code AffymetrixChipTO} to be compared to {@code to1}. - * @param compareId A {@code boolean} defining whether IDs of {@code EntityTO}s should be - * used for comparisons. - * @return {@code true} if {@code to1} and {@code to2} have all attributes equal. - */ - private static boolean areTOsEqual(AffymetrixChipTO to1, AffymetrixChipTO to2, boolean compareId) { - log.entry(to1, to2); - if (TOComparator.areEntityTOsEqual(to1, to2, compareId) && - Objects.equals(to1.getExperimentId(), to2.getExperimentId()) && - Objects.equals(to1.getConditionId(), to2.getConditionId()) && - Objects.equals(to1.getAffymetrixChipId(), to2.getAffymetrixChipId()) && - Objects.equals(to1.getScanDate(), to2.getScanDate()) && - Objects.equals(to1.getChipTypeId(), to2.getChipTypeId()) && - Objects.equals(to1.getNormalizationType(), to2.getNormalizationType()) && - Objects.equals(to1.getDetectionType(), to2.getDetectionType()) && - Objects.equals(to1.getDistinctRankCount(), to2.getDistinctRankCount()) && - areBigDecimalEquals(to1.getQualityScore(), to2.getQualityScore()) && - areBigDecimalEquals(to1.getPercentPresent(), to2.getPercentPresent()) && - areBigDecimalEquals(to1.getMaxRank(), to2.getMaxRank())) { - return log.traceExit(true); - } - return log.traceExit(false); - } // /** // * Method to compare two {@code RNASeqLibraryTO}s, to check for complete // * equality of each attribute. @@ -1396,43 +1310,6 @@ private static boolean areTOsEqual(InSituEvidenceTO to1, InSituEvidenceTO to2, b } return log.traceExit(false); } - /** - * Method to compare two {@code ESTLibraryTO}s, to check for complete - * equality of each attribute. - * - * @param to1 A {@code ESTLibraryTO} to be compared to {@code to2}. - * @param to2 A {@code ESTLibraryTO} to be compared to {@code to1}. - * @param compareId A {@code boolean} defining whether IDs of {@code EntityTO}s should be - * used for comparisons. - * @return {@code true} if {@code to1} and {@code to2} have all attributes equal. - */ - private static boolean areTOsEqual(ESTLibraryTO to1, ESTLibraryTO to2, boolean compareId) { - log.entry(to1, to2); - if (TOComparator.areEntityTOsEqual(to1, to2, compareId) && - Objects.equals(to1.getConditionId(), to2.getConditionId()) && - Objects.equals(to1.getDataSourceId(), to2.getDataSourceId())) { - return log.traceExit(true); - } - return log.traceExit(false); - } - - /** - * Method to compare two {@code MicroarrayExperimentTO}s, to check for complete - * equality of each attribute. - * - * @param to1 A {@code MicroarrayExperimentTO} to be compared to {@code to2}. - * @param to2 A {@code MicroarrayExperimentTO} to be compared to {@code to1}. - * @param compareId A {@code boolean} defining whether IDs of {@code EntityTO}s should be - * used for comparisons. - * @return {@code true} if {@code to1} and {@code to2} have all attributes equal. - */ - private static boolean areTOsEqual(MicroarrayExperimentTO to1, MicroarrayExperimentTO to2, boolean compareId) { - log.entry(to1, to2); - if (areTOsEqual((ExperimentTO) to1, (ExperimentTO) to2, compareId)) { - return log.traceExit(true); - } - return log.traceExit(false); - } /** * Method to compare two {@code RNASeqExperimentTO}s, to check for complete * equality of each attribute. diff --git a/bgee-dao-api/src/test/java/org/bgee/model/dao/api/TOComparatorTest.java b/bgee-dao-api/src/test/java/org/bgee/model/dao/api/TOComparatorTest.java index 2d8d8d25a..b5cce643a 100644 --- a/bgee-dao-api/src/test/java/org/bgee/model/dao/api/TOComparatorTest.java +++ b/bgee-dao-api/src/test/java/org/bgee/model/dao/api/TOComparatorTest.java @@ -31,16 +31,9 @@ import org.bgee.model.dao.api.expressiondata.call.GlobalExpressionCallDAO.EntityMinMaxRanksTO; import org.bgee.model.dao.api.expressiondata.rawdata.RawDataCallSourceDAO.CallSourceDataTO.ExclusionReason; import org.bgee.model.dao.api.expressiondata.rawdata.RawDataConditionDAO.RawDataConditionTO; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTDAO.ESTTO; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTLibraryDAO.ESTLibraryTO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituEvidenceDAO.InSituEvidenceTO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituExperimentDAO.InSituExperimentTO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituSpotDAO.InSituSpotTO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipDAO.AffymetrixChipTO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipDAO.AffymetrixChipTO.DetectionType; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipDAO.AffymetrixChipTO.NormalizationType; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixProbesetDAO.AffymetrixProbesetTO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.MicroarrayExperimentDAO.MicroarrayExperimentTO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqExperimentDAO.RNASeqExperimentTO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqLibraryAnnotatedSampleDAO.RNASeqLibraryAnnotatedSampleTO.AbundanceUnit; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqResultAnnotatedSampleDAO.RNASeqResultAnnotatedSampleTO; @@ -95,27 +88,27 @@ protected Logger getLogger() { @Test public void testAreSpeciesTOEqual() { SpeciesTO to1 = new SpeciesTO(1, "name1", "genus1", "species1", 1, - 1, "path1", "version1", "assemblyXref", 2, 1); + 1, "path1", "version1", "assemblyXref", 2, 1, "xref1"); SpeciesTO to2 = new SpeciesTO(1, "name1", "genus1", "species1", 1, - 1, "path1", "version1", "assemblyXref", 2, 1); + 1, "path1", "version1", "assemblyXref", 2, 1, "xref1"); assertTrue(TOComparator.areTOsEqual(to1, to2, true)); assertTrue(TOComparator.areTOsEqual(to1, to2, false)); to2 = new SpeciesTO(2, "name1", "genus1", "species1", 1, - 1, "path1", "version1", "assemblyXref", 2, 1); + 1, "path1", "version1", "assemblyXref", 2, 1, "xref1"); assertFalse(TOComparator.areTOsEqual(to1, to2, true)); to2 = new SpeciesTO(2, "name1", "genus1", "species1", 1, - 1, "path1", "version1", "assemblyXref", 2, 1); + 1, "path1", "version1", "assemblyXref", 2, 1, "xref1"); assertFalse(TOComparator.areTOsEqual(to1, to2, true)); to2 = new SpeciesTO(2, "name1", "genus1", "species1", 1, - 1, "path1", "version1", "assemblyXref", 2, 1); + 1, "path1", "version1", "assemblyXref", 2, 1, "xref1"); assertFalse(TOComparator.areTOsEqual(to1, to2, true)); assertTrue(TOComparator.areTOsEqual(to1, to2, false)); to2 = new SpeciesTO(1, "name1", "genus1", "species1", 2, - 1, "path1", "version1", "assemblyXref", 2, 1); + 1, "path1", "version1", "assemblyXref", 2, 1, "xref1"); assertFalse(TOComparator.areTOsEqual(to1, to2, true)); assertFalse(TOComparator.areTOsEqual(to1, to2, false)); } @@ -164,19 +157,19 @@ public void testAreGOTermTOEqual() { */ @Test public void testAreGeneTOEqual() { - GeneTO to1 = new GeneTO(1, "ID1", "name1", "desc1", 1, 2, 3, true, 1, "expression summary"); - GeneTO to2 = new GeneTO(1, "ID1", "name1", "desc1", 1, 2, 3, true, 1, "expression summary"); + GeneTO to1 = new GeneTO(1, "ID1", "name1", "desc1", 1, 2, true, "reg1", 1, "expression summary"); + GeneTO to2 = new GeneTO(1, "ID1", "name1", "desc1", 1, 2, true, "reg1", 1, "expression summary"); assertTrue(TOComparator.areTOsEqual(to1, to2, true)); assertTrue(TOComparator.areTOsEqual(to1, to2, false)); - to2 = new GeneTO(1, "ID1", "name1", "desc1", 1, 2, 3, false, 1, "expression summary"); + to2 = new GeneTO(1, "ID1", "name1", "desc1", 1, 2, false, "reg1", 1, "expression summary"); assertFalse(TOComparator.areTOsEqual(to1, to2, true)); - to2 = new GeneTO(2, "ID1", "name1", "desc1", 1, 2, 3, true, 1, "expression summary"); + to2 = new GeneTO(2, "ID1", "name1", "desc1", 1, 2, true, "reg1", 1, "expression summary"); assertFalse(TOComparator.areTOsEqual(to1, to2, true)); assertTrue(TOComparator.areTOsEqual(to1, to2, false)); - to2 = new GeneTO(1, "ID1", "name1", "desc1", 1, 2, 3, true, 2, "expression summary"); + to2 = new GeneTO(1, "ID1", "name1", "desc1", 1, 2, true, "reg1", 2, "expression summary"); assertFalse(TOComparator.areTOsEqual(to1, to2, true)); } @@ -421,22 +414,22 @@ public void testAreRelationTOsEqual() { */ @Test public void testAreConditionRankInfoTOsEqual() { - ConditionRankInfoTO to1 = new ConditionRankInfoTO(DAODataType.AFFYMETRIX, new BigDecimal("1000"), new BigDecimal("10000")); - ConditionRankInfoTO to2 = new ConditionRankInfoTO(DAODataType.AFFYMETRIX, new BigDecimal("1000"), new BigDecimal("10000")); + ConditionRankInfoTO to1 = new ConditionRankInfoTO(DAODataType.RNA_SEQ, new BigDecimal("1000"), new BigDecimal("10000")); + ConditionRankInfoTO to2 = new ConditionRankInfoTO(DAODataType.RNA_SEQ, new BigDecimal("1000"), new BigDecimal("10000")); assertTrue(TOComparator.areTOsEqual(to1, to2)); //Check with BigDecimals of different scales - to2 = new ConditionRankInfoTO(DAODataType.AFFYMETRIX, new BigDecimal("1000.00"), new BigDecimal("10000.00")); + to2 = new ConditionRankInfoTO(DAODataType.RNA_SEQ, new BigDecimal("1000.00"), new BigDecimal("10000.00")); assertTrue(TOComparator.areTOsEqual(to1, to2)); //Check when they are not equal - to2 = new ConditionRankInfoTO(DAODataType.EST, new BigDecimal("1000"), new BigDecimal("10000")); + to2 = new ConditionRankInfoTO(DAODataType.IN_SITU, new BigDecimal("1000"), new BigDecimal("10000")); assertFalse(TOComparator.areTOsEqual(to1, to2, true)); - to2 = new ConditionRankInfoTO(DAODataType.AFFYMETRIX, new BigDecimal("5000"), new BigDecimal("10000")); + to2 = new ConditionRankInfoTO(DAODataType.RNA_SEQ, new BigDecimal("5000"), new BigDecimal("10000")); assertFalse(TOComparator.areTOsEqual(to1, to2, true)); - to2 = new ConditionRankInfoTO(DAODataType.AFFYMETRIX, new BigDecimal("1000"), new BigDecimal("50000")); + to2 = new ConditionRankInfoTO(DAODataType.RNA_SEQ, new BigDecimal("1000"), new BigDecimal("50000")); assertFalse(TOComparator.areTOsEqual(to1, to2, true)); } @@ -452,8 +445,8 @@ public void testAreConditionTOsEqual() { assertTrue(TOComparator.areTOsEqual(to1, to2, false)); Collection rankTOs = Arrays.asList( - new ConditionRankInfoTO(DAODataType.AFFYMETRIX, new BigDecimal("1000"), new BigDecimal("10000")), - new ConditionRankInfoTO(DAODataType.EST, new BigDecimal("1000"), new BigDecimal("10000"))); + new ConditionRankInfoTO(DAODataType.RNA_SEQ, new BigDecimal("1000"), new BigDecimal("10000")), + new ConditionRankInfoTO(DAODataType.IN_SITU, new BigDecimal("1000"), new BigDecimal("10000"))); to1 = new ConditionTO(1, "anatEntityId1", "stageId1", "cellTypeId1", ConditionTO.DAOSex.FEMALE, "wildtype", 99, rankTOs); to2 = new ConditionTO(1, "anatEntityId1", "stageId1", "cellTypeId1", ConditionTO.DAOSex.FEMALE, "wildtype", 99, rankTOs); assertTrue(TOComparator.areTOsEqual(to1, to2, true)); @@ -577,19 +570,16 @@ public void regressionTestAreTOCollectionsEqual() { public void testAreDiffExpressionCallTOEqual() { DiffExpressionCallTO to1 = new DiffExpressionCallTO(1, 1, 1, ComparisonFactor.ANATOMY, DiffExprCallType.NOT_DIFF_EXPRESSED, - DataState.HIGHQUALITY, 0.02f, 2, 0, DiffExprCallType.NOT_DIFF_EXPRESSED, DataState.LOWQUALITY, 0.05f, 1, 0); DiffExpressionCallTO to2 = new DiffExpressionCallTO(1, 1, 1, ComparisonFactor.ANATOMY, DiffExprCallType.NOT_DIFF_EXPRESSED, - DataState.HIGHQUALITY, 0.02f, 2, 0, DiffExprCallType.NOT_DIFF_EXPRESSED, DataState.LOWQUALITY, 0.05f, 1, 0); assertTrue(TOComparator.areTOsEqual(to1, to2, true)); assertTrue(TOComparator.areTOsEqual(to1, to2, false)); //Different diffExprCallTypeRNASeq to2 = new DiffExpressionCallTO(1, 1, 1, - ComparisonFactor.ANATOMY, DiffExprCallType.NOT_DIFF_EXPRESSED, - DataState.HIGHQUALITY, 0.02f, 2, 0, DiffExprCallType.UNDER_EXPRESSED, + ComparisonFactor.ANATOMY, DiffExprCallType.UNDER_EXPRESSED, DataState.LOWQUALITY, 0.05f, 1, 0); assertFalse(TOComparator.areTOsEqual(to1, to2, true)); assertFalse(TOComparator.areTOsEqual(to1, to2, false)); @@ -597,7 +587,6 @@ public void testAreDiffExpressionCallTOEqual() { //Different id to2 = new DiffExpressionCallTO(2, 1, 1, ComparisonFactor.ANATOMY, DiffExprCallType.NOT_DIFF_EXPRESSED, - DataState.HIGHQUALITY, 0.02f, 2, 0, DiffExprCallType.NOT_DIFF_EXPRESSED, DataState.LOWQUALITY, 0.05f, 1, 0); assertFalse(TOComparator.areTOsEqual(to1, to2, true)); assertTrue(TOComparator.areTOsEqual(to1, to2, false)); @@ -605,24 +594,11 @@ public void testAreDiffExpressionCallTOEqual() { //both best p-value are null to1 = new DiffExpressionCallTO(1, 1, 1, ComparisonFactor.ANATOMY, DiffExprCallType.NOT_DIFF_EXPRESSED, - DataState.HIGHQUALITY, null, 2, 0, DiffExprCallType.NOT_DIFF_EXPRESSED, DataState.LOWQUALITY, 0.05f, 1, 0); to2 = new DiffExpressionCallTO(1, 1, 1, ComparisonFactor.ANATOMY, DiffExprCallType.NOT_DIFF_EXPRESSED, - DataState.HIGHQUALITY, null, 2, 0, DiffExprCallType.NOT_DIFF_EXPRESSED, DataState.LOWQUALITY, 0.05f, 1, 0); assertTrue(TOComparator.areTOsEqual(to1, to2, true)); - - //best p-value for Affymetrix is null - to1 = new DiffExpressionCallTO(1, 1, 1, - ComparisonFactor.ANATOMY, DiffExprCallType.NOT_DIFF_EXPRESSED, - DataState.HIGHQUALITY, null, 2, 0, DiffExprCallType.NOT_DIFF_EXPRESSED, - DataState.LOWQUALITY, 0.05f, 1, 0); - to2 = new DiffExpressionCallTO(1, 1, 1, - ComparisonFactor.ANATOMY, DiffExprCallType.NOT_DIFF_EXPRESSED, - DataState.HIGHQUALITY, 0.05f, 2, 0, DiffExprCallType.NOT_DIFF_EXPRESSED, - DataState.LOWQUALITY, 0.05f, 1, 0); - assertFalse(TOComparator.areTOsEqual(to1, to2, true)); } /** @@ -712,99 +688,24 @@ public void testAreSourceTOEqual() { */ @Test public void testAreSourceToSpeciesTOEqual() { - SourceToSpeciesTO to1 = new SourceToSpeciesTO(1, 11, DAODataType.EST, InfoType.DATA); - SourceToSpeciesTO to2 = new SourceToSpeciesTO(1, 11, DAODataType.EST, InfoType.DATA); + SourceToSpeciesTO to1 = new SourceToSpeciesTO(1, 11, DAODataType.RNA_SEQ, InfoType.DATA); + SourceToSpeciesTO to2 = new SourceToSpeciesTO(1, 11, DAODataType.RNA_SEQ, InfoType.DATA); assertTrue(TOComparator.areTOsEqual(to1, to2)); - to2 = new SourceToSpeciesTO(1, 11, DAODataType.AFFYMETRIX, InfoType.DATA); + to2 = new SourceToSpeciesTO(1, 11, DAODataType.IN_SITU, InfoType.DATA); assertFalse(TOComparator.areTOsEqual(to1, to2)); - to2 = new SourceToSpeciesTO(1, 11, DAODataType.EST, InfoType.ANNOTATION); + to2 = new SourceToSpeciesTO(1, 11, DAODataType.RNA_SEQ, InfoType.ANNOTATION); assertFalse(TOComparator.areTOsEqual(to1, to2)); - to2 = new SourceToSpeciesTO(1, 21, DAODataType.EST, InfoType.DATA); + to2 = new SourceToSpeciesTO(1, 21, DAODataType.RNA_SEQ, InfoType.DATA); assertFalse(TOComparator.areTOsEqual(to1, to2)); - to2 = new SourceToSpeciesTO(2, 11, DAODataType.EST, InfoType.DATA); + to2 = new SourceToSpeciesTO(2, 11, DAODataType.RNA_SEQ, InfoType.DATA); assertFalse(TOComparator.areTOsEqual(to1, to2, true)); assertFalse(TOComparator.areTOsEqual(to1, to2, false)); } - /** - * Test the generic method {@link TOComparator#areTOsEqual(Object, Object)} - * using {@code AffymetrixProbesetTO}s. - */ - @Test - public void testAreAffymetrixProbesetTOEqual() { - AffymetrixProbesetTO to1 = new AffymetrixProbesetTO("A1", 1, 11, new BigDecimal("11.1"), new BigDecimal("0.5"), - new BigDecimal("0.9"), 110L, new BigDecimal("5.5"), DataState.HIGHQUALITY, ExclusionReason.NOT_EXCLUDED); - AffymetrixProbesetTO to2 = new AffymetrixProbesetTO("A1", 1, 11, new BigDecimal("11.1"), new BigDecimal("0.5"), - new BigDecimal("0.9"), 110L, new BigDecimal("5.5"), DataState.HIGHQUALITY, ExclusionReason.NOT_EXCLUDED); - assertTrue(TOComparator.areTOsEqual(to1, to2)); - - to2 = new AffymetrixProbesetTO("A2", 1, 11, new BigDecimal("11.1"), new BigDecimal("0.5"), - new BigDecimal("0.9"), 110L, new BigDecimal("5.5"), DataState.HIGHQUALITY, ExclusionReason.NOT_EXCLUDED); - assertFalse(TOComparator.areTOsEqual(to1, to2)); - assertTrue(TOComparator.areTOsEqual(to1, to2, false)); - - to2 = new AffymetrixProbesetTO("A1", 1, 11, new BigDecimal("11.01"), new BigDecimal("0.5"), - new BigDecimal("0.9"), 110L, new BigDecimal("5.5"), DataState.HIGHQUALITY, ExclusionReason.NOT_EXCLUDED); - assertFalse(TOComparator.areTOsEqual(to1, to2)); - - to2 = new AffymetrixProbesetTO("A1", 1, 11, new BigDecimal("11.1"), new BigDecimal("0.05"), - new BigDecimal("0.9"), 110L, new BigDecimal("5.5"), DataState.HIGHQUALITY, ExclusionReason.NOT_EXCLUDED); - assertFalse(TOComparator.areTOsEqual(to1, to2)); - - to2 = new AffymetrixProbesetTO("A1", 1, 11, new BigDecimal("11.1"), new BigDecimal("0.5"), - new BigDecimal("0.9"), 110L, new BigDecimal("5.5"), DataState.LOWQUALITY, ExclusionReason.NOT_EXCLUDED); - assertFalse(TOComparator.areTOsEqual(to1, to2)); - - to2 = new AffymetrixProbesetTO("A1", 1, 11, new BigDecimal("11.1"), new BigDecimal("0.5"), - new BigDecimal("0.9"), 110L, new BigDecimal("5.5"), DataState.HIGHQUALITY, ExclusionReason.PRE_FILTERING); - assertFalse(TOComparator.areTOsEqual(to1, to2, true)); - assertFalse(TOComparator.areTOsEqual(to1, to2, false)); - } - /** - * Test the generic method {@link TOComparator#areTOsEqual(Object, Object)} - * using {@code AffymetrixChipTO}s. - */ - @Test - public void testAreAffymetrixChipTOEqual() { - AffymetrixChipTO to1 = new AffymetrixChipTO(1, "Chip1", "Exp1", "ChipTypeId1", "2018-07-20", NormalizationType.GC_RMA, - DetectionType.SCHUSTER, 1, new BigDecimal("10"), new BigDecimal("95.5"), new BigDecimal("8557.5"), 9000); - AffymetrixChipTO to2 = new AffymetrixChipTO(1, "Chip1", "Exp1", "ChipTypeId1", "2018-07-20", NormalizationType.GC_RMA, - DetectionType.SCHUSTER, 1, new BigDecimal("10"), new BigDecimal("95.5"), new BigDecimal("8557.5"), 9000); - assertTrue(TOComparator.areTOsEqual(to1, to2, true)); - assertTrue(TOComparator.areTOsEqual(to1, to2, false)); - - to2 = new AffymetrixChipTO(2, "Chip1", "Exp1", "ChipTypeId1", "2018-07-20", NormalizationType.GC_RMA, - DetectionType.SCHUSTER, 1, new BigDecimal("10"), new BigDecimal("95.5"), new BigDecimal("8557.5"), 9000); - assertFalse(TOComparator.areTOsEqual(to1, to2, true)); - assertTrue(TOComparator.areTOsEqual(to1, to2, false)); - - to2 = new AffymetrixChipTO(1, "Chip1", "Exp1", "ChipTypeId1", "2017-07-20", NormalizationType.GC_RMA, - DetectionType.SCHUSTER, 1, new BigDecimal("10"), new BigDecimal("95.5"), new BigDecimal("8557.5"), 9000); - assertFalse(TOComparator.areTOsEqual(to1, to2)); - } - /** - * Test the generic method {@link TOComparator#areTOsEqual(Object, Object)} - * using {@code MicroarrayExperimentTO}s. - */ - @Test - public void testAreMicroarrayExperimentTOEqual() { - MicroarrayExperimentTO to1 = new MicroarrayExperimentTO("Exp1", "name", "description", 1); - MicroarrayExperimentTO to2 = new MicroarrayExperimentTO("Exp1", "name", "description", 1); - assertTrue(TOComparator.areTOsEqual(to1, to2)); - assertTrue(TOComparator.areTOsEqual(to1, to2, true)); - assertTrue(TOComparator.areTOsEqual(to1, to2, false)); - - to2 = new MicroarrayExperimentTO("Exp2", "name", "description", 1); - assertFalse(TOComparator.areTOsEqual(to1, to2, true)); - assertTrue(TOComparator.areTOsEqual(to1, to2, false)); - - to2 = new MicroarrayExperimentTO("Exp1", "name", "description", 2); - assertFalse(TOComparator.areTOsEqual(to1, to2)); - } /** * Test the generic method {@link TOComparator#areTOsEqual(Object, Object)} * using {@code RNASeqResultTO}s. @@ -878,55 +779,6 @@ public void testAreRNASeqExperimentTOEqual() { to2 = new RNASeqExperimentTO("Exp1", "name", "description", 2, true, 232, "DOI1"); assertFalse(TOComparator.areTOsEqual(to1, to2)); } - /** - * Test the generic method {@link TOComparator#areTOsEqual(Object, Object)} - * using {@code ESTLibraryTO}s. - */ - @Test - public void testAreESTLibraryTOEqual() { - ESTLibraryTO to1 = new ESTLibraryTO("Exp1", "name", "description", 1, 2); - ESTLibraryTO to2 = new ESTLibraryTO("Exp1", "name", "description", 1, 2); - assertTrue(TOComparator.areTOsEqual(to1, to2)); - assertTrue(TOComparator.areTOsEqual(to1, to2, true)); - assertTrue(TOComparator.areTOsEqual(to1, to2, false)); - - to2 = new ESTLibraryTO("Exp2", "name", "description", 1, 2); - assertFalse(TOComparator.areTOsEqual(to1, to2, true)); - assertTrue(TOComparator.areTOsEqual(to1, to2, false)); - - to2 = new ESTLibraryTO("Exp1", "name2", "description", 1, 2); - assertFalse(TOComparator.areTOsEqual(to1, to2)); - - to2 = new ESTLibraryTO("Exp1", "name", "description", 2, 2); - assertFalse(TOComparator.areTOsEqual(to1, to2)); - } - /** - * Test the generic method {@link TOComparator#areTOsEqual(Object, Object)} - * using {@code ESTTO}s. - */ - @Test - public void testAreESTTOEqual() { - ESTTO to1 = new ESTTO("ID1", "ID2", "LibId1", "clusterId1", 1, DataState.HIGHQUALITY, new BigDecimal(1), 110L); - ESTTO to2 = new ESTTO("ID1", "ID2", "LibId1", "clusterId1", 1, DataState.HIGHQUALITY, new BigDecimal(1), 110L); - assertTrue(TOComparator.areTOsEqual(to1, to2)); - - to2 = new ESTTO("ID2", "ID2", "LibId1", "clusterId1", 1, DataState.HIGHQUALITY, new BigDecimal(1), 110L); - assertFalse(TOComparator.areTOsEqual(to1, to2)); - assertTrue(TOComparator.areTOsEqual(to1, to2, false)); - - to2 = new ESTTO("ID1", "ID3", "LibId1", "clusterId1", 1, DataState.HIGHQUALITY, new BigDecimal(1), 110L); - assertFalse(TOComparator.areTOsEqual(to1, to2)); - - to2 = new ESTTO("ID1", "ID2", "LibId2", "clusterId1", 1, DataState.HIGHQUALITY, new BigDecimal(1), 110L); - assertFalse(TOComparator.areTOsEqual(to1, to2)); - - to2 = new ESTTO("ID1", "ID2", "LibId1", "clusterId2", 1, DataState.HIGHQUALITY, new BigDecimal(1), 110L); - assertFalse(TOComparator.areTOsEqual(to1, to2)); - - to2 = new ESTTO("ID1", "ID2", "LibId1", "clusterId1", 1, DataState.HIGHQUALITY, new BigDecimal(1), 1L); - assertFalse(TOComparator.areTOsEqual(to1, to2, true)); - assertFalse(TOComparator.areTOsEqual(to1, to2, false)); - } /** * Test the generic method {@link TOComparator#areTOsEqual(Object, Object)} * using {@code InSituExperimentTO}s. diff --git a/bgee-dao-api/src/test/java/org/bgee/model/dao/api/expressiondata/call/DiffExpressionCallDAOTest.java b/bgee-dao-api/src/test/java/org/bgee/model/dao/api/expressiondata/call/DiffExpressionCallDAOTest.java index 2ecb6839b..d8053c0f9 100644 --- a/bgee-dao-api/src/test/java/org/bgee/model/dao/api/expressiondata/call/DiffExpressionCallDAOTest.java +++ b/bgee-dao-api/src/test/java/org/bgee/model/dao/api/expressiondata/call/DiffExpressionCallDAOTest.java @@ -42,19 +42,15 @@ protected Logger getLogger() { @Test public void shouldExtractDataTypesToDataStates() { DiffExpressionCallTO callTO = new DiffExpressionCallTO(null, null, null, null, - null, DataState.HIGHQUALITY, null, null, null, null, null, null, null, null); Map expectedMap = new HashMap<>(); - expectedMap.put(DiffExpressionCallDAO.Attribute.DIFF_EXPR_AFFYMETRIX_DATA, DataState.HIGHQUALITY); expectedMap.put(DiffExpressionCallDAO.Attribute.DIFF_EXPR_RNA_SEQ_DATA, null); assertEquals("Incorrect data types to data states extracted", expectedMap, callTO.extractDataTypesToDataStates()); callTO = new DiffExpressionCallTO(null, null, null, null, - null, DataState.LOWQUALITY, null, null, null, null, DataState.NODATA, null, null, null); expectedMap = new HashMap<>(); - expectedMap.put(DiffExpressionCallDAO.Attribute.DIFF_EXPR_AFFYMETRIX_DATA, DataState.LOWQUALITY); expectedMap.put(DiffExpressionCallDAO.Attribute.DIFF_EXPR_RNA_SEQ_DATA, DataState.NODATA); assertEquals("Incorrect data types to data states extracted", expectedMap, callTO.extractDataTypesToDataStates()); @@ -75,27 +71,22 @@ public void shouldExtractDataTypesToDataStates() { @Test public void shouldRetrieveDiffExpressionFilteringDataTypes() { DiffExpressionCallTO callTO = new DiffExpressionCallTO(null, null, null, null, - null, DataState.LOWQUALITY, null, null, null, null, null, null, null, null); Map expectedMap = new EnumMap<>(DiffExpressionCallDAO.Attribute.class); - expectedMap.put(DiffExpressionCallDAO.Attribute.DIFF_EXPR_AFFYMETRIX_DATA, DataState.LOWQUALITY); assertEquals("Incorrect filtering data types retrieved", expectedMap, callTO.extractFilteringDataTypes()); callTO = new DiffExpressionCallTO(null, null, null, null, - null, DataState.LOWQUALITY, null, null, null, null, DataState.LOWQUALITY, null, null, null); expectedMap = new EnumMap<>(DiffExpressionCallDAO.Attribute.class); - expectedMap.put(DiffExpressionCallDAO.Attribute.DIFF_EXPR_AFFYMETRIX_DATA, DataState.LOWQUALITY); expectedMap.put(DiffExpressionCallDAO.Attribute.DIFF_EXPR_RNA_SEQ_DATA, DataState.LOWQUALITY); assertEquals("Incorrect filtering data types retrieved with all LOWQUALITY", expectedMap, callTO.extractFilteringDataTypes()); callTO = new DiffExpressionCallTO(null, null, null, null, - null, null, null, null, null, null, DataState.NODATA, null, null, null); expectedMap = new EnumMap<>(DiffExpressionCallDAO.Attribute.class); @@ -103,10 +94,8 @@ public void shouldRetrieveDiffExpressionFilteringDataTypes() { callTO.extractFilteringDataTypes()); callTO = new DiffExpressionCallTO(null, null, null, null, - null, DataState.HIGHQUALITY, null, null, null, null, DataState.HIGHQUALITY, null, null, null); expectedMap = new EnumMap<>(DiffExpressionCallDAO.Attribute.class); - expectedMap.put(DiffExpressionCallDAO.Attribute.DIFF_EXPR_AFFYMETRIX_DATA, DataState.HIGHQUALITY); expectedMap.put(DiffExpressionCallDAO.Attribute.DIFF_EXPR_RNA_SEQ_DATA, DataState.HIGHQUALITY); assertEquals("Incorrect filtering data types retrieved with all HIGHQUALITY", expectedMap, diff --git a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/connector/MySQLDAOManager.java b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/connector/MySQLDAOManager.java index a0d03b0c7..d26b72baf 100644 --- a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/connector/MySQLDAOManager.java +++ b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/connector/MySQLDAOManager.java @@ -29,15 +29,10 @@ import org.bgee.model.dao.api.anatdev.SexDAO; import org.bgee.model.dao.api.exception.DAOException; import org.bgee.model.dao.api.expressiondata.call.ConditionDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.SamplePValueDAO; import org.bgee.model.dao.api.expressiondata.rawdata.RawDataConditionDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTLibraryDAO; +import org.bgee.model.dao.api.expressiondata.rawdata.SamplePValueDAO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituEvidenceDAO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituExperimentDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipTypeDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.MicroarrayExperimentDAO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqExperimentDAO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqLibraryAnnotatedSampleDAO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqLibraryDAO; @@ -54,22 +49,16 @@ import org.bgee.model.dao.mysql.anatdev.mapping.MySQLRawSimilarityAnnotationDAO; import org.bgee.model.dao.mysql.anatdev.mapping.MySQLStageGroupingDAO; import org.bgee.model.dao.mysql.anatdev.mapping.MySQLSummarySimilarityAnnotationDAO; +import org.bgee.model.dao.mysql.expressiondata.MySQLObservedExpressionDAO; import org.bgee.model.dao.mysql.expressiondata.call.MySQLConditionDAO; import org.bgee.model.dao.mysql.expressiondata.call.MySQLDiffExpressionCallDAO; import org.bgee.model.dao.mysql.expressiondata.call.MySQLGlobalExpressionCallDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.MySQLRawDataConditionDAO; -import org.bgee.model.dao.mysql.expressiondata.rawdata.MySQLRawExpressionCallDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.MySQLSamplePValueDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.MysqlRawDataCountDAO; -import org.bgee.model.dao.mysql.expressiondata.rawdata.est.MySQLESTDAO; -import org.bgee.model.dao.mysql.expressiondata.rawdata.est.MySQLESTLibraryDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.insitu.MySQLInSituEvidenceDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.insitu.MySQLInSituExperimentDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.insitu.MySQLInSituSpotDAO; -import org.bgee.model.dao.mysql.expressiondata.rawdata.microarray.MySQLAffymetrixChipDAO; -import org.bgee.model.dao.mysql.expressiondata.rawdata.microarray.MySQLAffymetrixChipTypeDAO; -import org.bgee.model.dao.mysql.expressiondata.rawdata.microarray.MySQLAffymetrixProbesetDAO; -import org.bgee.model.dao.mysql.expressiondata.rawdata.microarray.MySQLMicroarrayExperimentDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.rnaseq.MySQLRNASeqExperimentDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.rnaseq.MySQLRNASeqLibraryAnnotatedSampleDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.rnaseq.MySQLRNASeqLibraryDAO; @@ -1056,10 +1045,15 @@ protected MySQLRelationDAO getNewRelationDAO() { log.traceEntry(); return log.traceExit(new MySQLRelationDAO(this)); } +// @Override +// protected MySQLRawExpressionCallDAO getNewRawExpressionCallDAO() { +// log.traceEntry(); +// return log.traceExit(new MySQLRawExpressionCallDAO(this)); +// } @Override - protected MySQLRawExpressionCallDAO getNewRawExpressionCallDAO() { + protected MySQLObservedExpressionDAO getNewObservedExpressionDAO() { log.traceEntry(); - return log.traceExit(new MySQLRawExpressionCallDAO(this)); + return log.traceExit(new MySQLObservedExpressionDAO(this)); } @Override protected MySQLGlobalExpressionCallDAO getNewGlobalExpressionCallDAO() { @@ -1087,11 +1081,6 @@ protected RawDataConditionDAO getNewRawDataConditionDAO() { return log.traceExit(new MySQLRawDataConditionDAO(this)); } @Override - protected MySQLAffymetrixProbesetDAO getNewAffymetrixProbesetDAO() { - log.traceEntry(); - return log.traceExit(new MySQLAffymetrixProbesetDAO(this)); - } - @Override protected InSituEvidenceDAO getNewInSituEvidenceDAO() { log.traceEntry(); return log.traceExit(new MySQLInSituEvidenceDAO(this)); @@ -1179,18 +1168,6 @@ protected SamplePValueDAO getNewSamplePValueDAO() { return log.traceExit(new MySQLSamplePValueDAO(this)); } - @Override - protected AffymetrixChipDAO getNewAffymetrixChipDAO() { - log.traceEntry(); - return log.traceExit(new MySQLAffymetrixChipDAO(this)); - } - - @Override - protected MicroarrayExperimentDAO getNewMicroarrayExperimentDAO() { - log.traceEntry(); - return log.traceExit(new MySQLMicroarrayExperimentDAO(this)); - } - @Override protected RNASeqExperimentDAO getNewRnaSeqExperimentDAO() { log.traceEntry(); @@ -1209,16 +1186,6 @@ protected RNASeqLibraryDAO getNewRnaSeqLibraryDAO() { return log.traceExit(new MySQLRNASeqLibraryDAO(this)); } @Override - protected ESTLibraryDAO getNewESTLibraryDAO() { - log.traceEntry(); - return log.traceExit(new MySQLESTLibraryDAO(this)); - } - @Override - protected ESTDAO getNewESTDAO() { - log.traceEntry(); - return log.traceExit(new MySQLESTDAO(this)); - } - @Override protected SexDAO getNewSexDAO() { log.traceEntry(); return log.traceExit(new MySQLSexDAO(this)); @@ -1230,9 +1197,4 @@ protected MysqlRawDataCountDAO getNewRawDataCountDAO() { return log.traceExit(new MysqlRawDataCountDAO(this)); } - @Override - protected AffymetrixChipTypeDAO getNewAffymetrixChipTypeDAO() { - log.traceEntry(); - return log.traceExit(new MySQLAffymetrixChipTypeDAO(this)); - } } diff --git a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/MySQLObservedExpressionDAO.java b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/MySQLObservedExpressionDAO.java new file mode 100644 index 000000000..c1e2323bb --- /dev/null +++ b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/MySQLObservedExpressionDAO.java @@ -0,0 +1,313 @@ +package org.bgee.model.dao.mysql.expressiondata; + +import java.math.BigDecimal; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bgee.model.dao.api.exception.DAOException; +import org.bgee.model.dao.api.expressiondata.DAODataType; +import org.bgee.model.dao.api.expressiondata.DAOObservedExpressionFilter; +import org.bgee.model.dao.api.expressiondata.ObservedExpressionDAO; +import org.bgee.model.dao.api.expressiondata.rawdata.RawDataConditionDAO; +import org.bgee.model.dao.mysql.MySQLDAO; +import org.bgee.model.dao.mysql.connector.BgeePreparedStatement; +import org.bgee.model.dao.mysql.connector.MySQLDAOManager; +import org.bgee.model.dao.mysql.connector.MySQLDAOResultSet; +import org.bgee.model.dao.mysql.exception.UnrecognizedColumnException; +import org.bgee.model.dao.mysql.expressiondata.rawdata.MySQLRawDataConditionDAO; + +public class MySQLObservedExpressionDAO extends MySQLDAO implements ObservedExpressionDAO{ + + private static final Logger log = LogManager.getLogger(MySQLObservedExpressionDAO.class.getName()); + private static final String TABLE_NAME = "expression"; + + public MySQLObservedExpressionDAO(MySQLDAOManager manager) throws IllegalArgumentException { + super(manager); + } + + @Override + public ObservedExpressionTOResultSet getObservedExpression(DAOObservedExpressionFilter observedExpressionFilter, + Collection attributes) { + log.traceEntry("{}, {}", observedExpressionFilter, attributes); + final Set clonedAttrs = Collections + .unmodifiableSet(attributes == null || attributes.isEmpty()? + EnumSet.allOf(ObservedExpressionDAO.Attribute.class) :EnumSet.copyOf(attributes)); + + StringBuilder sb = new StringBuilder(); + sb.append(generateSelectClause(TABLE_NAME, clonedAttrs.stream().collect( + Collectors.toMap(a -> a.getTOFieldName(), a -> a)), false)); + sb.append(generateFromClause(observedExpressionFilter)); + sb.append(generateWhereClause(observedExpressionFilter)); + try { + BgeePreparedStatement stmt = this.getManager().getConnection().prepareStatement(sb.toString()); + int paramIndex = 1; + if (!observedExpressionFilter.getBgeeGeneIds().isEmpty()) { + stmt.setIntegers(paramIndex, observedExpressionFilter.getBgeeGeneIds(), true); + paramIndex += observedExpressionFilter.getBgeeGeneIds().size(); + } + if (!observedExpressionFilter.getConditionIds().isEmpty()) { + stmt.setIntegers(paramIndex, observedExpressionFilter.getConditionIds(), true); + paramIndex += observedExpressionFilter.getConditionIds().size(); + } + if(observedExpressionFilter.getRawDataConditionFilter() != null) { + if (!observedExpressionFilter.getRawDataConditionFilter().getAnatEntityIds().isEmpty()) { + stmt.setStrings(paramIndex, observedExpressionFilter.getRawDataConditionFilter().getAnatEntityIds(), true); + paramIndex += observedExpressionFilter.getRawDataConditionFilter().getAnatEntityIds().size(); + } + if (!observedExpressionFilter.getRawDataConditionFilter().getDevStageIds().isEmpty()) { + stmt.setStrings(paramIndex, observedExpressionFilter.getRawDataConditionFilter().getDevStageIds(), true); + paramIndex += observedExpressionFilter.getRawDataConditionFilter().getDevStageIds().size(); + } + if (!observedExpressionFilter.getRawDataConditionFilter().getCellTypeIds().isEmpty()) { + stmt.setStrings(paramIndex, observedExpressionFilter.getRawDataConditionFilter().getCellTypeIds(), true); + paramIndex += observedExpressionFilter.getRawDataConditionFilter().getCellTypeIds().size(); + } + if (!observedExpressionFilter.getRawDataConditionFilter().getSexIds().isEmpty()) { + stmt.setStrings(paramIndex, observedExpressionFilter.getRawDataConditionFilter().getSexIds(), true); + paramIndex += observedExpressionFilter.getRawDataConditionFilter().getSexIds().size(); + } + if (!observedExpressionFilter.getRawDataConditionFilter().getStrainIds().isEmpty()) { + stmt.setStrings(paramIndex, observedExpressionFilter.getRawDataConditionFilter().getStrainIds(), true); + paramIndex += observedExpressionFilter.getRawDataConditionFilter().getStrainIds().size(); + } + if (!observedExpressionFilter.getRawDataConditionFilter().getSpeciesIds().isEmpty()) { + stmt.setIntegers(paramIndex, observedExpressionFilter.getRawDataConditionFilter().getSpeciesIds(), true); + paramIndex += observedExpressionFilter.getRawDataConditionFilter().getSpeciesIds().size(); + } + } + return log.traceExit(new MySQLObservedExpressionTOResultSet(stmt)); + } catch (SQLException e) { + throw log.throwing(new DAOException(e)); + } + } + + private String generateFromClause(DAOObservedExpressionFilter expressionFilter) { + log.traceEntry("{}", expressionFilter); + StringBuilder sb = new StringBuilder(" FROM "); + if (expressionFilter.getRawDataConditionFilter() != null && !(expressionFilter.getRawDataConditionFilter().areAllCondParamFiltersEmpty() && + expressionFilter.getRawDataConditionFilter().getSpeciesIds().isEmpty())) { + sb.append(" " + MySQLRawDataConditionDAO.TABLE_NAME + " INNER JOIN " + TABLE_NAME + + " ON " + MySQLRawDataConditionDAO.TABLE_NAME + "." + + RawDataConditionDAO.Attribute.ID.getTOFieldName() + " = " + TABLE_NAME + "." + + ObservedExpressionDAO.Attribute.CONDITION_ID.getTOFieldName()); + } else { + sb.append(" " + TABLE_NAME); + } + return sb.toString(); + } + + private String generateWhereClause(DAOObservedExpressionFilter filter) { + log.traceEntry("{}", filter); + //filter contains at least one filtering. These is always a WHERE clause + StringBuilder sb = new StringBuilder(" WHERE"); + boolean andClauseRequired = false; + if (! filter.getBgeeGeneIds().isEmpty()) { + sb.append(generateOneWhereClause(filter.getBgeeGeneIds(), + ObservedExpressionDAO.Attribute.BGEE_GENE_ID.getTOFieldName(), andClauseRequired, TABLE_NAME)); + andClauseRequired = true; + } + if (! filter.getConditionIds().isEmpty()) { + sb.append(generateOneWhereClause(filter.getConditionIds(), + ObservedExpressionDAO.Attribute.CONDITION_ID.getTOFieldName(), andClauseRequired, TABLE_NAME)); + andClauseRequired = true; + } + if (filter.getRawDataConditionFilter() != null && ! filter.getRawDataConditionFilter().getAnatEntityIds().isEmpty()) { + sb.append(generateOneWhereClause(filter.getRawDataConditionFilter().getAnatEntityIds(), + RawDataConditionDAO.Attribute.ANAT_ENTITY_ID.getTOFieldName(), andClauseRequired, + MySQLRawDataConditionDAO.TABLE_NAME)); + andClauseRequired = true; + } + if (filter.getRawDataConditionFilter() != null && ! filter.getRawDataConditionFilter().getDevStageIds().isEmpty()) { + sb.append(generateOneWhereClause(filter.getRawDataConditionFilter().getDevStageIds(), + RawDataConditionDAO.Attribute.STAGE_ID.getTOFieldName(), andClauseRequired, + MySQLRawDataConditionDAO.TABLE_NAME)); + andClauseRequired = true; + } + if (filter.getRawDataConditionFilter() != null && ! filter.getRawDataConditionFilter().getCellTypeIds().isEmpty()) { + sb.append(generateOneWhereClause(filter.getRawDataConditionFilter().getCellTypeIds(), + RawDataConditionDAO.Attribute.CELL_TYPE_ID.getTOFieldName(), andClauseRequired, + MySQLRawDataConditionDAO.TABLE_NAME)); + andClauseRequired = true; + } + if (filter.getRawDataConditionFilter() != null && ! filter.getRawDataConditionFilter().getSexIds().isEmpty()) { + sb.append(generateOneWhereClause(filter.getRawDataConditionFilter().getSexIds(), + RawDataConditionDAO.Attribute.SEX.getTOFieldName(), andClauseRequired, + MySQLRawDataConditionDAO.TABLE_NAME)); + andClauseRequired = true; + } + if (filter.getRawDataConditionFilter() != null && ! filter.getRawDataConditionFilter().getStrainIds().isEmpty()) { + sb.append(generateOneWhereClause(filter.getRawDataConditionFilter().getStrainIds(), + RawDataConditionDAO.Attribute.STRAIN.getTOFieldName(), andClauseRequired, + MySQLRawDataConditionDAO.TABLE_NAME)); + andClauseRequired = true; + } + if (filter.getRawDataConditionFilter() != null && ! filter.getRawDataConditionFilter().getSpeciesIds().isEmpty()) { + sb.append(generateOneWhereClause(filter.getRawDataConditionFilter().getSpeciesIds(), + RawDataConditionDAO.Attribute.SPECIES_ID.getTOFieldName(), andClauseRequired, + MySQLRawDataConditionDAO.TABLE_NAME)); + andClauseRequired = true; + } + if (!filter.getDatatypes().equals(EnumSet.allOf(DAODataType.class))) { + StringBuilder datatypeSb = new StringBuilder(); + boolean orRequired = false; + for (DAODataType datatype : filter.getDatatypes()) { + switch (datatype) { + case RNA_SEQ: + if (orRequired) datatypeSb.append(" OR"); + datatypeSb.append(" ").append(TABLE_NAME).append(".").append( + ObservedExpressionDAO.Attribute.BULK_NUM_OBS.getTOFieldName()).append(" IS NOT NULL"); + orRequired = true; + break; + case SC_RNA_SEQ: + if (orRequired) datatypeSb.append(" OR"); + datatypeSb.append(" ").append(TABLE_NAME).append(".").append( + ObservedExpressionDAO.Attribute.FULL_LENGTH_NUM_OBS.getTOFieldName()).append(" IS NOT NULL"); + datatypeSb.append(" OR ").append(TABLE_NAME).append(".").append( + ObservedExpressionDAO.Attribute.DROPLET_NUM_OBS.getTOFieldName()).append(" IS NOT NULL"); + orRequired = true; + break; + case IN_SITU: + if (orRequired) datatypeSb.append(" OR"); + datatypeSb.append(" ").append(TABLE_NAME).append(".").append( + ObservedExpressionDAO.Attribute.IN_SITU_NUM_OBS.getTOFieldName()).append(" IS NOT NULL"); + orRequired = true; + break; + default: + throw log.throwing(new IllegalStateException("Unsupported DAODataType: " + datatype)); + } + } + if (andClauseRequired) { + sb.append(" AND"); + } + sb.append(" (").append(datatypeSb).append(")"); + andClauseRequired = true; + } + return log.traceExit(sb.toString()); + } + + private String generateOneWhereClause(Set condParamsSet, String fieldName, + boolean andClauseRequired, String tableName) { + StringBuilder sb = new StringBuilder(); + if (andClauseRequired) { + sb.append(" AND"); + } + sb.append(" " + tableName).append(".").append(fieldName) + .append(" IN (") + .append(BgeePreparedStatement.generateParameterizedQueryString(condParamsSet.size())) + .append(")"); + return log.traceExit(sb.toString()); + } + + /** + * Implementation of the {@code ObservedExpressionTOResultSet}. + * + * @author Julien Wollbrett + * @version Bgee 16, Nov. 2025 + * @since Bgee 16, Nov. 2025 + */ + class MySQLObservedExpressionTOResultSet extends MySQLDAOResultSet + implements ObservedExpressionTOResultSet{ + + protected MySQLObservedExpressionTOResultSet(BgeePreparedStatement statement) { + super(statement); + } + + @Override + protected ObservedExpressionTO getNewTO() throws DAOException, UnrecognizedColumnException { + log.traceEntry(); + try { + final ResultSet currentResultSet = this.getCurrentResultSet(); + Integer id = null, conditionId = null, bgeeGeneId = null; + Integer bulkNumberObs = null, fullLengthNumberObs = null, dropletNumberObs = null, inSituNumberObs = null; + BigDecimal bulkRank = null, bulkPValue = null, bulkWeight = null; + BigDecimal fullLengthRank = null, fullLengthPValue = null, fullLengthWeight = null; + BigDecimal dropletRank = null, dropletPValue = null, dropletWeight = null; + BigDecimal inSituRank = null, inSituPValue = null, inSituWeight = null; + Map colNameToAttr = EnumSet.allOf(ObservedExpressionDAO.Attribute.class) + .stream().collect(Collectors.toMap(a -> a.getTOFieldName(), a -> a)); + COL: for (String columnName : this.getColumnLabels().values()) { + //don't use MySQLDAO.getAttributeFromColName because we don't cover all columns + //with ConditionDAO.Attributes (max rank columns) + ObservedExpressionDAO.Attribute attr = colNameToAttr.get(columnName); + if (attr == null) { + continue COL; + } + switch (attr) { + case EXPRESSION_ID: + id = currentResultSet.getInt(columnName); + break; + case CONDITION_ID: + conditionId = currentResultSet.getInt(columnName); + break; + case BGEE_GENE_ID: + bgeeGeneId = currentResultSet.getInt(columnName); + break; + case BULK_SCORE: + bulkRank = currentResultSet.getBigDecimal(columnName); + break; + case BULK_PVALUE: + bulkPValue = currentResultSet.getBigDecimal(columnName); + break; + case BULK_WEIGHT: + bulkWeight = currentResultSet.getBigDecimal(columnName); + break; + case BULK_NUM_OBS: + bulkNumberObs = currentResultSet.getInt(columnName); + break; + case FULL_LENGTH_SCORE: + fullLengthRank = currentResultSet.getBigDecimal(columnName); + break; + case FULL_LENGTH_PVALUE: + fullLengthPValue = currentResultSet.getBigDecimal(columnName); + break; + case FULL_LENGTH_WEIGHT: + fullLengthWeight = currentResultSet.getBigDecimal(columnName); + break; + case FULL_LENGTH_NUM_OBS: + fullLengthNumberObs = currentResultSet.getInt(columnName); + break; + case DROPLET_SCORE: + dropletRank = currentResultSet.getBigDecimal(columnName); + break; + case DROPLET_PVALUE: + dropletPValue = currentResultSet.getBigDecimal(columnName); + break; + case DROPLET_WEIGHT: + dropletWeight = currentResultSet.getBigDecimal(columnName); + break; + case DROPLET_NUM_OBS: + dropletNumberObs = currentResultSet.getInt(columnName); + break; + case IN_SITU_SCORE: + inSituRank = currentResultSet.getBigDecimal(columnName); + break; + case IN_SITU_PVALUE: + inSituPValue = currentResultSet.getBigDecimal(columnName); + break; + case IN_SITU_WEIGHT: + inSituWeight = currentResultSet.getBigDecimal(columnName); + break; + case IN_SITU_NUM_OBS: + inSituNumberObs = currentResultSet.getInt(columnName); + break; + default: + log.throwing(new UnrecognizedColumnException(columnName)); + } + } + return log.traceExit(new ObservedExpressionTO(id, conditionId, bgeeGeneId, bulkRank, bulkPValue, + bulkWeight, bulkNumberObs, fullLengthRank, fullLengthPValue, fullLengthWeight, + fullLengthNumberObs, dropletRank, dropletPValue, dropletWeight, dropletNumberObs, + inSituRank, inSituPValue, inSituWeight, inSituNumberObs)); + } catch (SQLException e) { + throw log.throwing(new DAOException(e)); + } + } } +} diff --git a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/call/MySQLConditionDAO.java b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/call/MySQLConditionDAO.java index d916c96e8..d22e0705e 100644 --- a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/call/MySQLConditionDAO.java +++ b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/call/MySQLConditionDAO.java @@ -24,7 +24,6 @@ import org.bgee.model.dao.api.exception.DAOException; import org.bgee.model.dao.api.expressiondata.DAODataType; import org.bgee.model.dao.api.expressiondata.call.ConditionDAO; -import org.bgee.model.dao.api.expressiondata.call.ConditionDAO.GlobalConditionToRawConditionTO.ConditionRelationOrigin; import org.bgee.model.dao.api.expressiondata.call.DAOCallFilter; import org.bgee.model.dao.api.expressiondata.call.DAOConditionFilter; import org.bgee.model.dao.api.expressiondata.call.DAOConditionFilter2; @@ -55,114 +54,103 @@ public class MySQLConditionDAO extends MySQLCallDAO impl public final static String SPECIES_ID = "speciesId"; public final static String RAW_COND_ID_FIELD = "conditionId"; public final static String GLOBAL_COND_ID_FIELD = "globalConditionId"; - private final static String COND_REL_ORIGIN_FIELD = "conditionRelationOrigin"; + private final static String COND_PARAM_SUBSET_MASK_FIELD = "subsetMask"; public final static String ANAT_ENTITY_ID_FIELD = "anatEntityId"; + public final static String SOURCE_COND_ID_RELATION = "sourceGlobalConditionId"; + public final static String TARGET_COND_ID_RELATION = "targetGlobalConditionId"; public final static String TABLE_NAME = "globalCond"; - - /** - * @param tableName A {@code String} that is the name of the global condition table - * in the SQL query. - * @param condParamCombination A {@code Collection} of {@code ConditionDAO.Attribute}s defining the - * condition parameters considered for aggregating the expression data - * (see {@link Attribute#isConditionParameter()}). - * @return A {@code String} that is the part of a WHERE clause - * allowing to select the conditions for the requested - * condition parameter combination. - * @throws IllegalArgumentException If {@code conditionParameters} is {@code null}, empty, - * or one of the {@code Attribute}s in {@code conditionParameters} - * is not a condition parameter attributes (see - * {@link ConditionDAO.Attribute#isConditionParameter()}). - */ - private static String getCondParamCombinationWhereClause(final String tableName, - Collection condParamCombination) throws IllegalArgumentException { - log.traceEntry("{}, {}", tableName, condParamCombination); - if (condParamCombination == null || condParamCombination.isEmpty()) { - throw log.throwing(new IllegalArgumentException( - "A condition parameter combination must be provided.")); - } - final Set condParams = EnumSet.copyOf(condParamCombination); - if (condParams.stream().anyMatch(a -> !a.isConditionParameter())) { - throw log.throwing(new IllegalArgumentException("The condition parameter combination " - + "contains some Attributes that are not condition parameters: " + condParams)); - } - - final Map colToAttr = getColToAttributesMap(); - return log.traceExit(EnumSet.allOf(ConditionDAO.Attribute.class).stream() - .filter(a -> a.isConditionParameter() && !condParams.contains(a)) - .map(a -> tableName + "." + getSelectExprFromAttribute(a, colToAttr) + " = '" - + a.getRootId() + "'") - .collect(Collectors.joining(" AND "))); - } + private final static String COND_REL_TABLE_NAME = "globalCondRelation"; + +// /** +// * @param tableName A {@code String} that is the name of the global condition table +// * in the SQL query. +// * @param condParamCombination A {@code Collection} of {@code ConditionDAO.Attribute}s defining the +// * condition parameters considered for aggregating the expression data +// * (see {@link Attribute#isConditionParameter()}). +// * @return A {@code String} that is the part of a WHERE clause +// * allowing to select the conditions for the requested +// * condition parameter combination. +// * @throws IllegalArgumentException If {@code conditionParameters} is {@code null}, empty, +// * or one of the {@code Attribute}s in {@code conditionParameters} +// * is not a condition parameter attributes (see +// * {@link ConditionDAO.Attribute#isConditionParameter()}). +// */ +// private static String getCondParamCombinationWhereClause(final String tableName, +// Collection condParamCombination) throws IllegalArgumentException { +// log.traceEntry("{}, {}", tableName, condParamCombination); +// if (condParamCombination == null || condParamCombination.isEmpty()) { +// throw log.throwing(new IllegalArgumentException( +// "A condition parameter combination must be provided.")); +// } +// final Set condParams = EnumSet.copyOf(condParamCombination); +// if (condParams.stream().anyMatch(a -> !a.isConditionParameter())) { +// throw log.throwing(new IllegalArgumentException("The condition parameter combination " +// + "contains some Attributes that are not condition parameters: " + condParams)); +// } +// +// final Map colToAttr = getColToAttributesMap(); +// return log.traceExit(EnumSet.allOf(ConditionDAO.Attribute.class).stream() +// .filter(a -> a.isConditionParameter() && !condParams.contains(a)) +// .map(a -> tableName + "." + getSelectExprFromAttribute(a, colToAttr) + " = '" +// + a.getRootId() + "'") +// .collect(Collectors.joining(" AND "))); +// } /** * Get a {@code Map} associating column names to corresponding {@code ConditionDAO.Attribute}. * - * @param global A {@code boolean} defining whether the global conditions (if {@code true}) - * were targeted, or the raw conditions (if {@code false}). * @return A {@code Map} where keys are {@code String}s that are column names, * the associated value being the corresponding {@code ConditionDAO.Attribute}. */ private static Map getColToAttributesMap() { log.traceEntry(); - Map colToAttributesMap = new HashMap<>(); - colToAttributesMap.put(GLOBAL_COND_ID_FIELD, ConditionDAO.Attribute.ID); - //only the original condition table containing all parameters has the field "exprMappedConditionId", - //allowing to map conditions used in annotations to conditions used in expression tables. - colToAttributesMap.put("anatEntityId", ConditionDAO.Attribute.ANAT_ENTITY_ID); - colToAttributesMap.put("stageId", ConditionDAO.Attribute.STAGE_ID); - colToAttributesMap.put("sex", ConditionDAO.Attribute.SEX_ID); - colToAttributesMap.put("strain", ConditionDAO.Attribute.STRAIN_ID); - colToAttributesMap.put("cellTypeId", ConditionDAO.Attribute.CELL_TYPE_ID); - colToAttributesMap.put(SPECIES_ID, ConditionDAO.Attribute.SPECIES_ID); -// if (!global) { -// colToAttributesMap.put("sexInferred", ConditionDAO.Attribute.SEX_INFERRED); -// } - - return log.traceExit(colToAttributesMap); + return log.traceExit(EnumSet.allOf(ConditionDAO.Attribute.class) + .stream() + .collect(Collectors.toMap(e -> e.getTOFieldName(), e -> e))); } public MySQLConditionDAO(MySQLDAOManager manager) throws IllegalArgumentException { super(manager); } - @Override - public GlobalConditionToRawConditionTOResultSet getGlobalCondToRawCondBySpeciesIds( - Collection speciesIds, Collection conditionParameters) - throws DAOException, IllegalArgumentException { - log.traceEntry("{}, {}", speciesIds, conditionParameters); - - final Set speIds = Collections.unmodifiableSet(speciesIds == null? new HashSet<>(): - new HashSet<>(speciesIds)); - String tableName = "globalCondToCond"; - String globalCondTableName = "globalCond"; - StringBuilder sb = new StringBuilder(); - sb.append("SELECT ").append(tableName).append(".* FROM ").append(tableName) - .append(" INNER JOIN ").append(globalCondTableName) - .append(" ON ").append(globalCondTableName).append(".globalConditionId = ") - .append(tableName).append(".globalConditionId"); - if (!conditionParameters.containsAll(ConditionDAO.Attribute.getCondParams()) || !speIds.isEmpty()) { - sb.append(" WHERE "); - } - sb.append(getCondParamCombinationWhereClause(globalCondTableName, conditionParameters)); - if (!speIds.isEmpty()) { - if (!conditionParameters.containsAll(ConditionDAO.Attribute.getCondParams())) { - sb.append(" AND "); - } - sb.append(globalCondTableName).append(".").append(SPECIES_ID).append(" IN (") - .append(BgeePreparedStatement.generateParameterizedQueryString(speIds.size())) - .append(")"); - } - try { - BgeePreparedStatement stmt = this.getManager().getConnection().prepareStatement(sb.toString()); - if (!speIds.isEmpty()) { - stmt.setIntegers(1, speIds, true); - } - return log.traceExit(new MySQLGlobalConditionToRawConditionTOResultSet(stmt)); - } catch (SQLException e) { - throw log.throwing(new DAOException(e)); - } - } +// @Override +// public GlobalConditionToRawConditionTOResultSet getGlobalCondToRawCondBySpeciesIds( +// Collection speciesIds, Collection conditionParameters) +// throws DAOException, IllegalArgumentException { +// log.traceEntry("{}, {}", speciesIds, conditionParameters); +// +// final Set speIds = Collections.unmodifiableSet(speciesIds == null? new HashSet<>(): +// new HashSet<>(speciesIds)); +// String tableName = "globalCondToCond"; +// String globalCondTableName = "globalCond"; +// StringBuilder sb = new StringBuilder(); +// sb.append("SELECT ").append(tableName).append(".* FROM ").append(tableName) +// .append(" INNER JOIN ").append(globalCondTableName) +// .append(" ON ").append(globalCondTableName).append(".globalConditionId = ") +// .append(tableName).append(".globalConditionId"); +// if (!conditionParameters.containsAll(ConditionDAO.Attribute.getCondParams()) || !speIds.isEmpty()) { +// sb.append(" WHERE "); +// } +// sb.append(getCondParamCombinationWhereClause(globalCondTableName, conditionParameters)); +// if (!speIds.isEmpty()) { +// if (!conditionParameters.containsAll(ConditionDAO.Attribute.getCondParams())) { +// sb.append(" AND "); +// } +// sb.append(globalCondTableName).append(".").append(SPECIES_ID).append(" IN (") +// .append(BgeePreparedStatement.generateParameterizedQueryString(speIds.size())) +// .append(")"); +// } +// try { +// BgeePreparedStatement stmt = this.getManager().getConnection().prepareStatement(sb.toString()); +// if (!speIds.isEmpty()) { +// stmt.setIntegers(1, speIds, true); +// } +// return log.traceExit(new MySQLGlobalConditionToRawConditionTOResultSet(stmt)); +// } catch (SQLException e) { +// throw log.throwing(new DAOException(e)); +// } +// } @Override @@ -822,16 +810,16 @@ public int insertGlobalConditions(Collection conditionTOs) throws D } @Override - public int insertGlobalConditionToRawCondition( - Collection globalCondToRawCondTOs) + public int insertRawConditionToSelfGlobalCondition( + Collection rawConditionToSelfGlobalConditionTOs) throws DAOException, IllegalArgumentException { - log.traceEntry("{}", globalCondToRawCondTOs); + log.traceEntry("{}", rawConditionToSelfGlobalConditionTOs); - if (globalCondToRawCondTOs == null || globalCondToRawCondTOs.isEmpty()) { + if (rawConditionToSelfGlobalConditionTOs == null || rawConditionToSelfGlobalConditionTOs.isEmpty()) { throw log.throwing(new IllegalArgumentException("No condition relation provided")); } - List toList = new ArrayList<>(globalCondToRawCondTOs); + List toList = new ArrayList<>(rawConditionToSelfGlobalConditionTOs); int maxElementCount = 5000; int iterationCount = toList.size() < maxElementCount? 1: (int) Math.ceil((float) toList.size()/(float) maxElementCount); @@ -840,11 +828,11 @@ public int insertGlobalConditionToRawCondition( (i + 1) * maxElementCount < toList.size()? (i + 1) * maxElementCount: toList.size())) //Warning, this stream must be sequential, our SQL accessors cannot be used in parallel .mapToInt((partition) -> { - StringBuilder sql = new StringBuilder(); - sql.append("INSERT INTO globalCondToCond (") + StringBuilder sql = new StringBuilder(); + sql.append("INSERT INTO condToSelfGlobalCond (") .append(RAW_COND_ID_FIELD).append(", ") .append(GLOBAL_COND_ID_FIELD).append(", ") - .append(COND_REL_ORIGIN_FIELD) + .append(COND_PARAM_SUBSET_MASK_FIELD) .append(") VALUES "); for (int i = 0; i < partition.size(); i++) { if (i > 0) { @@ -856,12 +844,12 @@ public int insertGlobalConditionToRawCondition( try (BgeePreparedStatement stmt = this.getManager().getConnection().prepareStatement(sql.toString())) { int paramIndex = 1; - for (GlobalConditionToRawConditionTO to: partition) { + for (RawConditionToSelfGlobalConditionTO to: partition) { stmt.setInt(paramIndex, to.getRawConditionId()); paramIndex++; stmt.setInt(paramIndex, to.getGlobalConditionId()); paramIndex++; - stmt.setString(paramIndex, to.getConditionRelationOrigin().getStringRepresentation()); + stmt.setInt(paramIndex, to.getCondParamBitwise()); paramIndex++; } return stmt.executeUpdate(); @@ -901,7 +889,9 @@ protected ConditionDAO.ConditionTO getNewTO() throws DAOException { Integer id = null, speciesId = null; String anatEntityId = null, stageId = null, cellTypeId = null, strainId = null; ConditionDAO.ConditionTO.DAOSex sex = null; + BigDecimal bulkMaxRank = null, singleCellMaxRank = null, inSituMaxRank = null; Map colToAttrMap = getColToAttributesMap(); + COL: for (String columnName : this.getColumnLabels().values()) { //don't use MySQLDAO.getAttributeFromColName because we don't cover all columns @@ -937,10 +927,9 @@ protected ConditionDAO.ConditionTO getNewTO() throws DAOException { log.throwing(new UnrecognizedColumnException(columnName)); } } - //XXX: retrieval of ConditionRankInfoTOs associated to a ConditionTO not yet implemented, - //to be added when needed. + Set condRankInfo = new HashSet<>(); return log.traceExit(new ConditionTO(id, anatEntityId, stageId, cellTypeId, sex, - strainId, speciesId, null)); + strainId, speciesId, condRankInfo)); } catch (SQLException e) { throw log.throwing(new DAOException(e)); } @@ -948,27 +937,27 @@ protected ConditionDAO.ConditionTO getNewTO() throws DAOException { } /** - * MySQL implementation of {@code GlobalConditionToRawConditionTOResultSet}. + * MySQL implementation of {@code MySQLRawConditionToSelfGlobalConditionTOResultSet}. * * @author Frederic Bastian * @version Bgee 14 Mar. 2017 * @since Bgee 14 Mar. 2017 */ - public class MySQLGlobalConditionToRawConditionTOResultSet - extends MySQLDAOResultSet - implements GlobalConditionToRawConditionTOResultSet { + class MySQLRawConditionToSelfGlobalConditionTOResultSet + extends MySQLDAOResultSet + implements RawConditionToSelfGlobalConditionTOResultSet { - private MySQLGlobalConditionToRawConditionTOResultSet(BgeePreparedStatement statement) { + private MySQLRawConditionToSelfGlobalConditionTOResultSet(BgeePreparedStatement statement) { super(statement); } @Override - protected GlobalConditionToRawConditionTO getNewTO() throws DAOException { + protected RawConditionToSelfGlobalConditionTO getNewTO() throws DAOException { log.traceEntry(); try { final ResultSet currentResultSet = this.getCurrentResultSet(); Integer rawConditionId = null, globalConditionId = null; - ConditionRelationOrigin relOrigin = null; + Integer condParamSubsetMask = null; for (String columnName: this.getColumnLabels().values()) { @@ -976,19 +965,160 @@ protected GlobalConditionToRawConditionTO getNewTO() throws DAOException { rawConditionId = currentResultSet.getInt(columnName); } else if (columnName.equals(GLOBAL_COND_ID_FIELD)) { globalConditionId = currentResultSet.getInt(columnName); - } else if (columnName.equals(COND_REL_ORIGIN_FIELD)) { - relOrigin = ConditionRelationOrigin.convertToCondRelOrigin( - currentResultSet.getString(columnName)); + } else if (columnName.equals(COND_PARAM_SUBSET_MASK_FIELD)) { + condParamSubsetMask = currentResultSet.getInt(columnName); } else { throw log.throwing(new UnrecognizedColumnException(columnName)); } } - return log.traceExit(new GlobalConditionToRawConditionTO( - rawConditionId, globalConditionId, relOrigin)); + return log.traceExit(new RawConditionToSelfGlobalConditionTO( + rawConditionId, globalConditionId, condParamSubsetMask)); } catch (SQLException e) { throw log.throwing(new DAOException(e)); } } } + + /** + * MySQL implementation of {@code GlobalConditionToDirectAncestorTOResultSet}. + * + * @author Julien Wollbrett + * @version Bgee 16 Oct. 2025 + * @since Bgee 16 Oct. 2025 + */ + class MySQLGlobalConditionToDirectAncestorTOResultSet + extends MySQLDAOResultSet + implements GlobalConditionToDirectAncestorTOResultSet { + + private MySQLGlobalConditionToDirectAncestorTOResultSet(BgeePreparedStatement statement) { + super(statement); + } + + @Override + protected GlobalConditionToDirectAncestorTO getNewTO() throws DAOException { + log.traceEntry(); + try { + final ResultSet currentResultSet = this.getCurrentResultSet(); + Integer sourceConditionId = null, targetConditionId = null; + + for (String columnName: this.getColumnLabels().values()) { + + if (columnName.equals(SOURCE_COND_ID_RELATION)) { + sourceConditionId = currentResultSet.getInt(columnName); + } else if (columnName.equals(TARGET_COND_ID_RELATION)) { + targetConditionId = currentResultSet.getInt(columnName); + } else { + throw log.throwing(new UnrecognizedColumnException(columnName)); + } + } + + return log.traceExit(new GlobalConditionToDirectAncestorTO( + sourceConditionId, targetConditionId)); + } catch (SQLException e) { + throw log.throwing(new DAOException(e)); + } + } + } + + @Override + public int insertcondIdToDirectAncestorId(Collection condIdToDirectAncestorTOs) + throws DAOException, IllegalArgumentException { + log.traceEntry("{}", condIdToDirectAncestorTOs); + + if (condIdToDirectAncestorTOs == null || condIdToDirectAncestorTOs.isEmpty()) { + throw log.throwing(new IllegalArgumentException("no condIdToDirectAncestorTOs provided")); + } + + List toList = new ArrayList<>(condIdToDirectAncestorTOs); + int maxElementCount = 5000; + int iterationCount = toList.size() < maxElementCount? 1: (int) Math.ceil((float) toList.size()/(float) maxElementCount); + + int countUpdated = IntStream.range(0, iterationCount) + .mapToObj(i -> toList.subList(i * maxElementCount, + (i + 1) * maxElementCount < toList.size()? (i + 1) * maxElementCount: toList.size())) + //Warning, this stream must be sequential, our SQL accessors cannot be used in parallel + .mapToInt((partition) -> { + StringBuilder sql = new StringBuilder(); + sql.append("INSERT INTO ").append(COND_REL_TABLE_NAME).append(" (") + .append(SOURCE_COND_ID_RELATION).append(", ") + .append(TARGET_COND_ID_RELATION) + .append(") VALUES "); + for (int i = 0; i < partition.size(); i++) { + if (i > 0) { + sql.append(", "); + } + sql.append("(").append(BgeePreparedStatement.generateParameterizedQueryString(2)) + .append(") "); + } + try (BgeePreparedStatement stmt = + this.getManager().getConnection().prepareStatement(sql.toString())) { + int paramIndex = 1; + for (GlobalConditionToDirectAncestorTO to: partition) { + stmt.setInt(paramIndex, to.getSourceConditionId()); + paramIndex++; + stmt.setInt(paramIndex, to.getTargetConditionId()); + paramIndex++; + } + return stmt.executeUpdate(); + } catch (SQLException e) { + throw log.throwing(new DAOException(e)); + } + }) + .sum(); + return log.traceExit(countUpdated); + } + + @Override + public GlobalConditionToDirectAncestorTOResultSet getGlobalConditionToDirectAncestor(Integer speciesId) + throws DAOException, IllegalArgumentException { + log.traceEntry("{}", speciesId); + if (speciesId == null || speciesId <= 0) { + throw log.throwing(new IllegalArgumentException("the speciesId can not be null or <= 0")); + } + //SELECT clause + StringBuilder sqlQuery = new StringBuilder("SELECT ").append(SOURCE_COND_ID_RELATION).append(", ") + .append(TARGET_COND_ID_RELATION); + //FROM clause + sqlQuery.append(" FROM ").append(COND_REL_TABLE_NAME) + .append(" AS ").append(COND_REL_TABLE_NAME) + .append(" INNER JOIN ").append(TABLE_NAME).append(" AS ").append(TABLE_NAME) + .append(" ON ").append(TABLE_NAME).append(".").append(GLOBAL_COND_ID_FIELD) + .append(" = ").append(COND_REL_TABLE_NAME).append(".").append(SOURCE_COND_ID_RELATION); + //WHERE clause + sqlQuery.append(" WHERE ").append(TABLE_NAME).append(".").append(SPECIES_ID).append(" = ?"); + try { + BgeePreparedStatement stmt = this.getManager().getConnection().prepareStatement(sqlQuery.toString()); + stmt.setInt(1, speciesId); + return log.traceExit(new MySQLGlobalConditionToDirectAncestorTOResultSet(stmt)); + } catch (SQLException e) { + throw log.throwing(new DAOException(e)); + } + } + + @Override + public RawConditionToSelfGlobalConditionTOResultSet getRawConditionToSelfGlobalConditionFromGlobalConditionIds( + Collection globalConditionIds, EnumSet condParams) throws DAOException { + log.traceEntry("{}, {}", globalConditionIds, condParams); + if (globalConditionIds == null || globalConditionIds.isEmpty()) { + throw log.throwing(new IllegalArgumentException("At least one conditionId should be provided")); + } + EnumSet processdDataTypes = condParams == null || condParams.isEmpty() ? + EnumSet.allOf(ConditionParameter.class) : condParams; + StringBuilder sqlQuery = new StringBuilder("SELECT ").append(RAW_COND_ID_FIELD).append(", ") + .append(GLOBAL_COND_ID_FIELD).append(", ").append(COND_PARAM_SUBSET_MASK_FIELD); + sqlQuery.append(" FROM condToSelfGlobalCond"); + sqlQuery.append(" WHERE ").append(GLOBAL_COND_ID_FIELD).append(" IN (") + .append(BgeePreparedStatement.generateParameterizedQueryString(globalConditionIds.size())).append(")") + .append(" AND ").append(COND_PARAM_SUBSET_MASK_FIELD).append(" = ") + .append(RawConditionToSelfGlobalConditionTO.fromCondParamToSubsetMask(processdDataTypes)); + try { + BgeePreparedStatement stmt = this.getManager().getConnection().prepareStatement(sqlQuery.toString()); + int paramIndex = 1; + stmt.setIntegers(paramIndex, globalConditionIds, true); + return log.traceExit(new MySQLRawConditionToSelfGlobalConditionTOResultSet(stmt)); + } catch (SQLException e) { + throw log.throwing(new DAOException(e)); + } + } } diff --git a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/call/MySQLDiffExpressionCallDAO.java b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/call/MySQLDiffExpressionCallDAO.java index 3eda85212..cfc847253 100644 --- a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/call/MySQLDiffExpressionCallDAO.java +++ b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/call/MySQLDiffExpressionCallDAO.java @@ -64,12 +64,6 @@ public MySQLDiffExpressionCallDAO(MySQLDAOManager manager) throws IllegalArgumen * species allowing to filter the calls to use. * @param factor A {@code ComparisonFactor} that is the comparison factor * allowing to filter the calls to use. - * @param diffExprCallTypeAffymetrix A {@code DiffExprCallType} that is the type of the - * differential expression calls to be used for Affymetrix - * data. - * @param includeAffymetrixTypes A {@code boolean} defining whether differential - * expression call types from Affymetrix data should be - * include or exclude. * @param diffExprCallTypeRNASeq A {@code DiffExprCallType} that is the type of the * differential expression calls to be used for RNA-seq * data. @@ -86,10 +80,9 @@ public MySQLDiffExpressionCallDAO(MySQLDAOManager manager) throws IllegalArgumen */ private DiffExpressionCallTOResultSet getDiffExpressionCalls( String omaTaxonId, Set speciesIds, - ComparisonFactor factor, Set diffExprCallTypeAffymetrix, - boolean includeAffymetrixTypes, Set diffExprCallTypeRNASeq, + ComparisonFactor factor,Set diffExprCallTypeRNASeq, boolean includeRnaSeqTypes, boolean isSatisfyAllCallTypeCondition) { - log.entry(omaTaxonId, speciesIds, factor, diffExprCallTypeAffymetrix, includeAffymetrixTypes, + log.entry(omaTaxonId, speciesIds, factor, diffExprCallTypeRNASeq, includeRnaSeqTypes, isSatisfyAllCallTypeCondition); // Construct sql query @@ -179,11 +172,9 @@ private DiffExpressionCallTOResultSet getDiffExpressionCalls( sql += diffExprTableName; } - boolean filterAffymetrixTypes = - (diffExprCallTypeAffymetrix != null && diffExprCallTypeAffymetrix.size() != 0 ); boolean filterRNASeqTypes = (diffExprCallTypeRNASeq != null && diffExprCallTypeRNASeq.size() != 0 ); - if (factor != null || filterAffymetrixTypes || filterRNASeqTypes) { + if (factor != null || filterRNASeqTypes) { if (hasSpecies && !hasOMATaxon && !orderTOsByOmaGroup) { sql += " AND "; } else { @@ -194,9 +185,6 @@ private DiffExpressionCallTOResultSet getDiffExpressionCalls( sql += diffExprTableName + ".comparisonFactor = ?"; } int nbFilterCallType = 0; - if (filterAffymetrixTypes) { - nbFilterCallType++; - } if (filterRNASeqTypes) { nbFilterCallType++; } @@ -207,14 +195,6 @@ private DiffExpressionCallTOResultSet getDiffExpressionCalls( if (nbFilterCallType > 1) { sql += " ("; } - if (filterAffymetrixTypes) { - sql += diffExprTableName + ".diffExprCallAffymetrix "; - if (!includeAffymetrixTypes) { - sql += "NOT"; - } - sql += " IN (" + BgeePreparedStatement.generateParameterizedQueryString( - diffExprCallTypeAffymetrix.size()) + ")"; - } if (nbFilterCallType > 1) { if (isSatisfyAllCallTypeCondition) { sql += " AND "; @@ -268,11 +248,6 @@ private DiffExpressionCallTOResultSet getDiffExpressionCalls( stmtIndex++; } - if (filterAffymetrixTypes) { - stmt.setEnumDAOFields(stmtIndex, diffExprCallTypeAffymetrix, true); - stmtIndex += diffExprCallTypeAffymetrix.size(); - } - if (filterRNASeqTypes) { stmt.setEnumDAOFields(stmtIndex, diffExprCallTypeRNASeq, true); stmtIndex += diffExprCallTypeRNASeq.size(); @@ -325,18 +300,6 @@ private String generateSelectClause( sql += "conditionId"; } else if (attribute.equals(DiffExpressionCallDAO.Attribute.COMPARISON_FACTOR)) { sql += "comparisonFactor"; - } else if (attribute.equals(DiffExpressionCallDAO.Attribute.DIFF_EXPR_CALL_AFFYMETRIX)) { - sql += "diffExprCallAffymetrix"; - } else if (attribute.equals(DiffExpressionCallDAO.Attribute.DIFF_EXPR_AFFYMETRIX_DATA)) { - sql += "diffExprAffymetrixData"; - } else if (attribute.equals(DiffExpressionCallDAO.Attribute.BEST_P_VALUE_AFFYMETRIX)) { - sql += "bestPValueAffymetrix"; - } else if (attribute.equals( - DiffExpressionCallDAO.Attribute.CONSISTENT_DEA_COUNT_AFFYMETRIX)) { - sql += "consistentDEACountAffymetrix"; - } else if (attribute.equals( - DiffExpressionCallDAO.Attribute.INCONSISTENT_DEA_COUNT_AFFYMETRIX)) { - sql += "inconsistentDEACountAffymetrix"; } else if (attribute.equals(DiffExpressionCallDAO.Attribute.DIFF_EXPR_CALL_RNA_SEQ)) { sql += "diffExprCallRNASeq"; } else if (attribute.equals(DiffExpressionCallDAO.Attribute.DIFF_EXPR_RNA_SEQ_DATA)) { @@ -413,11 +376,10 @@ protected DiffExpressionCallTO getNewTO() throws DAOException { Integer id = null, geneId = null, conditionId = null; ComparisonFactor comparisonFactor = null; - DataState diffExprAffymetrixData = null, diffExprRNASeqData = null; - DiffExprCallType diffExprCallTypeAffymetrix = null, diffExprCallTypeRNASeq = null; - Float bestPValueAffymetrix = null, bestPValueRNASeq = null; - Integer consistentDEACountAffymetrix = null, inconsistentDEACountAffymetrix = null, - consistentDEACountRNASeq = null, inconsistentDEACountRNASeq = null; + DataState diffExprRNASeqData = null; + DiffExprCallType diffExprCallTypeRNASeq = null; + Float bestPValueRNASeq = null; + Integer consistentDEACountRNASeq = null, inconsistentDEACountRNASeq = null; //every call to values() returns a newly cloned array, so we cache the array for (Entry column: this.getColumnLabels().entrySet()) { @@ -435,27 +397,7 @@ protected DiffExpressionCallTO getNewTO() throws DAOException { comparisonFactor = ComparisonFactor.convertToComparisonFactor( this.getCurrentResultSet().getString(column.getKey())); - } else if (column.getValue().equals("diffExprCallAffymetrix")) { - diffExprCallTypeAffymetrix = DiffExprCallType.convertToDiffExprCallType( - this.getCurrentResultSet().getString(column.getKey())); - - } else if (column.getValue().equals("diffExprAffymetrixData")) { - diffExprAffymetrixData = DataState.convertToDataState( - this.getCurrentResultSet().getString(column.getKey())); - - } else if (column.getValue().equals("bestPValueAffymetrix")) { - bestPValueAffymetrix = this.getCurrentResultSet().getFloat( - column.getKey()); - - } else if (column.getValue().equals("consistentDEACountAffymetrix")) { - consistentDEACountAffymetrix = this.getCurrentResultSet().getInt( - column.getKey()); - - } else if (column.getValue().equals("inconsistentDEACountAffymetrix")) { - inconsistentDEACountAffymetrix = this.getCurrentResultSet().getInt( - column.getKey()); - - } else if (column.getValue().equals("diffExprCallRNASeq")) { + }else if (column.getValue().equals("diffExprCallRNASeq")) { diffExprCallTypeRNASeq = DiffExprCallType.convertToDiffExprCallType( this.getCurrentResultSet().getString(column.getKey())); @@ -486,9 +428,7 @@ protected DiffExpressionCallTO getNewTO() throws DAOException { } } return log.traceExit(new DiffExpressionCallTO(id, geneId, conditionId, - comparisonFactor, diffExprCallTypeAffymetrix, diffExprAffymetrixData, - bestPValueAffymetrix, consistentDEACountAffymetrix, inconsistentDEACountAffymetrix, - diffExprCallTypeRNASeq, diffExprRNASeqData, bestPValueRNASeq, + comparisonFactor, diffExprCallTypeRNASeq, diffExprRNASeqData, bestPValueRNASeq, consistentDEACountRNASeq, inconsistentDEACountRNASeq)); } } diff --git a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/MySQLRawDataConditionDAO.java b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/MySQLRawDataConditionDAO.java index 1ccbcd908..c8fb49233 100644 --- a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/MySQLRawDataConditionDAO.java +++ b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/MySQLRawDataConditionDAO.java @@ -18,17 +18,13 @@ import org.bgee.model.dao.api.expressiondata.rawdata.DAORawDataConditionFilter; import org.bgee.model.dao.api.expressiondata.rawdata.DAORawDataFilter; import org.bgee.model.dao.api.expressiondata.rawdata.RawDataConditionDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTLibraryDAO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituSpotDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipDAO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqLibraryAnnotatedSampleDAO; import org.bgee.model.dao.mysql.connector.BgeePreparedStatement; import org.bgee.model.dao.mysql.connector.MySQLDAOManager; import org.bgee.model.dao.mysql.connector.MySQLDAOResultSet; import org.bgee.model.dao.mysql.exception.UnrecognizedColumnException; -import org.bgee.model.dao.mysql.expressiondata.rawdata.est.MySQLESTLibraryDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.insitu.MySQLInSituSpotDAO; -import org.bgee.model.dao.mysql.expressiondata.rawdata.microarray.MySQLAffymetrixChipDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.rnaseq.MySQLRNASeqLibraryAnnotatedSampleDAO; public class MySQLRawDataConditionDAO extends MySQLRawDataDAO @@ -149,51 +145,57 @@ public RawDataConditionTOResultSet getRawDataConditionsLinkedToDataType( // that are always propagated. boolean onlyUsedInPropagatedCalls = processedFilters.getRawDataFilters().stream() .allMatch(item -> item.usedInPropagatedCallsIsTheOnlyPotentialNotBlank()); - sb.append(" WHERE "); - if ( !processedFilters.getRawDataFilters().isEmpty() && !dataType.isAlwaysPropagated() || + boolean needFiltersClause = !processedFilters.getRawDataFilters().isEmpty() && !dataType.isAlwaysPropagated() || !onlyUsedInPropagatedCalls && dataType.isAlwaysPropagated() || - isSingleCell != null) { - sb.append("(") - .append(generateWhereClauseRawDataFilter(processedFilters, rawDataFiltersToDatabaseMapping, - isSingleCell)) - .append(") AND "); - } - //We at least always need to check that results are from conditions - //used in annotations of the requested data type. - //Since it is annoying to check whether generateFromClauseRawData made indeed a join - //to the assay table, we always add this clause + isSingleCell != null; + // Determine whether the assay table was already joined by generateFromClauseRawData. + // If so, the EXISTS check below is redundant and can be skipped. + boolean assayTableJoined; switch(dataType) { - case AFFYMETRIX: - sb.append(" EXISTS(SELECT 1 FROM ").append(MySQLAffymetrixChipDAO.TABLE_NAME) - .append(" WHERE ").append(MySQLAffymetrixChipDAO.TABLE_NAME).append(".") - .append(AffymetrixChipDAO.Attribute.CONDITION_ID.getTOFieldName()).append(" = ") - .append(TABLE_NAME).append(".").append(RawDataConditionDAO.Attribute.ID.getTOFieldName()) - .append(")"); - break; - case EST: - sb.append(" EXISTS(SELECT 1 FROM ").append(MySQLESTLibraryDAO.TABLE_NAME) - .append(" WHERE ").append(MySQLESTLibraryDAO.TABLE_NAME).append(".") - .append(ESTLibraryDAO.Attribute.CONDITION_ID.getTOFieldName()).append(" = ") - .append(TABLE_NAME).append(".").append(RawDataConditionDAO.Attribute.ID.getTOFieldName()) - .append(")"); - break; case IN_SITU: - sb.append(" EXISTS(SELECT 1 FROM ").append(MySQLInSituSpotDAO.TABLE_NAME) - .append(" WHERE ").append(MySQLInSituSpotDAO.TABLE_NAME).append(".") - .append(InSituSpotDAO.Attribute.CONDITION_ID.getTOFieldName()).append(" = ") - .append(TABLE_NAME).append(".").append(RawDataConditionDAO.Attribute.ID.getTOFieldName()) - .append(")"); + assayTableJoined = rawDataFiltersToDatabaseMapping.getJoinedTables() + .contains(MySQLInSituSpotDAO.TABLE_NAME); break; case RNA_SEQ: - sb.append(" EXISTS(SELECT 1 FROM ").append(MySQLRNASeqLibraryAnnotatedSampleDAO.TABLE_NAME) - .append(" WHERE ").append(MySQLRNASeqLibraryAnnotatedSampleDAO.TABLE_NAME).append(".") - .append(RNASeqLibraryAnnotatedSampleDAO.Attribute.CONDITION_ID.getTOFieldName()).append(" = ") - .append(TABLE_NAME).append(".").append(RawDataConditionDAO.Attribute.ID.getTOFieldName()) - .append(")"); + assayTableJoined = rawDataFiltersToDatabaseMapping.getJoinedTables() + .contains(MySQLRNASeqLibraryAnnotatedSampleDAO.TABLE_NAME); break; default: throw log.throwing(new IllegalStateException("Unsupported data type: " + dataType)); } + sb.append(" WHERE "); + if (needFiltersClause) { + sb.append("(") + .append(generateWhereClauseRawDataFilter(processedFilters, rawDataFiltersToDatabaseMapping, + isSingleCell)) + .append(")"); + } + // We at least always need to check that results are from conditions + // used in annotations of the requested data type, unless the assay table + // was already joined in the FROM clause (making this EXISTS redundant). + if (!assayTableJoined) { + if (needFiltersClause) { + sb.append(" AND "); + } + switch(dataType) { + case IN_SITU: + sb.append(" EXISTS(SELECT 1 FROM ").append(MySQLInSituSpotDAO.TABLE_NAME) + .append(" WHERE ").append(MySQLInSituSpotDAO.TABLE_NAME).append(".") + .append(InSituSpotDAO.Attribute.CONDITION_ID.getTOFieldName()).append(" = ") + .append(TABLE_NAME).append(".").append(RawDataConditionDAO.Attribute.ID.getTOFieldName()) + .append(")"); + break; + case RNA_SEQ: + sb.append(" EXISTS(SELECT 1 FROM ").append(MySQLRNASeqLibraryAnnotatedSampleDAO.TABLE_NAME) + .append(" WHERE ").append(MySQLRNASeqLibraryAnnotatedSampleDAO.TABLE_NAME).append(".") + .append(RNASeqLibraryAnnotatedSampleDAO.Attribute.CONDITION_ID.getTOFieldName()).append(" = ") + .append(TABLE_NAME).append(".").append(RawDataConditionDAO.Attribute.ID.getTOFieldName()) + .append(")"); + break; + default: + throw log.throwing(new IllegalStateException("Unsupported data type: " + dataType)); + } + } try { BgeePreparedStatement stmt = this.parameterizeQuery(sb.toString(), processedFilters, diff --git a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/MySQLRawDataDAO.java b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/MySQLRawDataDAO.java index 349331f8f..63fe44c26 100644 --- a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/MySQLRawDataDAO.java +++ b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/MySQLRawDataDAO.java @@ -21,14 +21,9 @@ import org.bgee.model.dao.api.expressiondata.rawdata.DAOProcessedRawDataFilter; import org.bgee.model.dao.api.expressiondata.rawdata.DAORawDataFilter; import org.bgee.model.dao.api.expressiondata.rawdata.RawDataConditionDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTLibraryDAO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituEvidenceDAO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituExperimentDAO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituSpotDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixProbesetDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.MicroarrayExperimentDAO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqExperimentDAO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqLibraryAnnotatedSampleDAO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqLibraryDAO; @@ -38,14 +33,9 @@ import org.bgee.model.dao.mysql.connector.BgeePreparedStatement; import org.bgee.model.dao.mysql.connector.MySQLDAOManager; import org.bgee.model.dao.mysql.expressiondata.rawdata.RawDataFiltersToDatabaseMapping.RawDataColumn; -import org.bgee.model.dao.mysql.expressiondata.rawdata.est.MySQLESTDAO; -import org.bgee.model.dao.mysql.expressiondata.rawdata.est.MySQLESTLibraryDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.insitu.MySQLInSituEvidenceDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.insitu.MySQLInSituExperimentDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.insitu.MySQLInSituSpotDAO; -import org.bgee.model.dao.mysql.expressiondata.rawdata.microarray.MySQLAffymetrixChipDAO; -import org.bgee.model.dao.mysql.expressiondata.rawdata.microarray.MySQLAffymetrixProbesetDAO; -import org.bgee.model.dao.mysql.expressiondata.rawdata.microarray.MySQLMicroarrayExperimentDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.rnaseq.MySQLRNASeqExperimentDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.rnaseq.MySQLRNASeqLibraryAnnotatedSampleDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.rnaseq.MySQLRNASeqLibraryDAO; @@ -224,22 +214,8 @@ protected > BgeePreparedStatement parameterizeQuery(Stri BgeePreparedStatement stmt = this.getManager().getConnection() .prepareStatement(query); int paramIndex = 1; - //ESTs can't have results if an experiment ID is requested - //(ESTs don't have experiments). - //If all filters request an experiment, we returned a FALSE clause, - //thus we have no parameters to set here; - //otherwise, we will discard the filters that have an experiment ID, - //because the method generateOneFilterWhereClause will skip the experimentId field - //for ESTs, and we would obtain some results why we should not - if (!DAODataType.EST.equals(dataType) || - !processedFilters.isAlwaysExactlyExperimentId()) { - - for (DAORawDataFilter rawDataFilter : processedFilters.getRawDataFilters()) { - //discard the filters that have an experiment ID for EST - if (DAODataType.EST.equals(dataType) && !rawDataFilter.getExperimentIds().isEmpty()) { - log.debug("Skipping DAORawDataFilter for EST because experiment IDs: {}", rawDataFilter); - continue; - } + + for (DAORawDataFilter rawDataFilter : processedFilters.getRawDataFilters()) { Set geneIds = rawDataFilter.getGeneIds(); Set speciesIds = rawDataFilter.getSpeciesIds(); @@ -261,8 +237,7 @@ protected > BgeePreparedStatement parameterizeQuery(Stri if (callTableAssayIds == null) { // parameterize expIds - // ESTs does not have experimentIds - if (!dataType.equals(DAODataType.EST) && !expIds.isEmpty()) { + if (!expIds.isEmpty()) { stmt.setStrings(paramIndex, expIds, true); paramIndex += expIds.size(); } @@ -273,11 +248,9 @@ protected > BgeePreparedStatement parameterizeQuery(Stri } //parameterize assay or experiment IDs if (!expOrAssayIds.isEmpty()) { - // ESTs does not have experimentIds - if (!dataType.equals(DAODataType.EST)) { - stmt.setStrings(paramIndex, expOrAssayIds, true); - paramIndex += expOrAssayIds.size(); - } + // add it twice to filter on experiment or assay + stmt.setStrings(paramIndex, expOrAssayIds, true); + paramIndex += expOrAssayIds.size(); stmt.setStrings(paramIndex, expOrAssayIds, true); paramIndex += expOrAssayIds.size(); } @@ -313,7 +286,6 @@ protected > BgeePreparedStatement parameterizeQuery(Stri stmt.setBoolean(paramIndex, isSingleCell); paramIndex++; } - } } // special cases outside of the DAORawDataFilters @@ -415,137 +387,19 @@ protected > RawDataFiltersToDatabaseMapping generateFrom log.traceEntry("{}, {}, {}, {}, {}", sb, processedFilters, isSingleCell, necessaryTables, datatype); Map ambiguousColToTable = new HashMap<>(); - if (datatype.equals(DAODataType.AFFYMETRIX)) { - ambiguousColToTable = generateFromClauseRawDataAffymetrix(sb, - (DAOProcessedRawDataFilter) processedFilters, necessaryTables); - } else if (datatype.equals(DAODataType.IN_SITU)) { + Set joinedTables = new LinkedHashSet<>(); + if (datatype.equals(DAODataType.IN_SITU)) { ambiguousColToTable = generateFromClauseRawDataInSitu(sb, - (DAOProcessedRawDataFilter) processedFilters, necessaryTables); - } else if (datatype.equals(DAODataType.EST)) { - ambiguousColToTable = generateFromClauseRawDataEst(sb, - (DAOProcessedRawDataFilter) processedFilters, necessaryTables); + (DAOProcessedRawDataFilter) processedFilters, necessaryTables, joinedTables); } else if (datatype.equals(DAODataType.RNA_SEQ)) { ambiguousColToTable = generateFromClauseRawDataRnaSeq(sb, (DAOProcessedRawDataFilter) processedFilters, isSingleCell, - necessaryTables); + necessaryTables, joinedTables); } else { throw log.throwing(new IllegalStateException("dataType " + datatype + "not recognized.")); } - return log.traceExit(new RawDataFiltersToDatabaseMapping(ambiguousColToTable, datatype)); - } - - /** - * Method, specific to affymetrix, allowing to add a FROM clause to a {@code StringBuilder} - * based on {@code DAORawDataFilter}s, and {@code boolean}s describing mandatory tables. - * It will return a {@code Map} with {@code RawDataColumn} corresponding to columns to use - * in the WHERE clause as keys and {@code String} corresponding to the table to use to - * retrieve the data as values. - * - * @param sb The {@code StringBuilder} for which the FROM clause will be - * created. - * @param processedFilters A {@code DAOProcessedRawDataFilter} to use to generate the - * FROM clause - * @param necessaryTables A {@code Set} of {@code String}s corresponding to the names - * of tables necessary to the creation of the FROM clause. - * {@code necessaryTables} must contain only the names - * of the tables used to retrieve necessary information - * in the SELECT clause, not the tables used for filtering results. - * Other tables will be automatically added to the clause - * by this method to satisfy the {@code filter}s. - * @return A {@code Map} with {@code RawDataColumn} as keys and - * {@code String} as value defining the table to use. - */ - private Map generateFromClauseRawDataAffymetrix(StringBuilder sb, - DAOProcessedRawDataFilter processedFilters, Set necessaryTables) { - log.traceEntry("{}, {}, {}", sb, processedFilters, necessaryTables); - - if (necessaryTables.size() == 2 && !necessaryTables.containsAll( - Set.of(MySQLAffymetrixProbesetDAO.TABLE_NAME, MySQLAffymetrixChipDAO.TABLE_NAME)) || - necessaryTables.size() > 2) { - throw log.throwing(new IllegalStateException("Combination of necessary tables unsupported: " - + necessaryTables)); - } - - Map colToTableMap = new LinkedHashMap<>(); - LinkedHashSet orderedTables = new LinkedHashSet<>(); - - // check needed tables - boolean geneTable = processedFilters.isNeedSpeciesId() && necessaryTables.size() == 1 && - necessaryTables.contains(MySQLAffymetrixProbesetDAO.TABLE_NAME) - && !processedFilters.isNeedAssayId() && !processedFilters.isNeedExperimentId() && - !processedFilters.isNeedConditionId(); - boolean condTable = processedFilters.isNeedSpeciesId() && !geneTable || - necessaryTables.contains(MySQLRawDataConditionDAO.TABLE_NAME); - assert !(geneTable && condTable): "We should never need both cond and gene table"; - boolean expTable = necessaryTables.contains(MySQLMicroarrayExperimentDAO.TABLE_NAME); - boolean probesetTable = necessaryTables.contains(MySQLAffymetrixProbesetDAO.TABLE_NAME) || - processedFilters.isNeedGeneId() || - processedFilters.getFilterToCallTableAssayIds() != null; - assert !(processedFilters.getFilterToCallTableAssayIds() != null && - !necessaryTables.contains(MySQLAffymetrixProbesetDAO.TABLE_NAME)): "affymetrixProbeset should" - + " be a mandatory table if filterToCallTableAssayIds is not null"; - boolean chipTable = necessaryTables.contains(MySQLAffymetrixChipDAO.TABLE_NAME) || - processedFilters.isNeedAssayId() || !expTable && processedFilters.isNeedExperimentId() || - !condTable && processedFilters.isNeedConditionId() || - expTable && condTable || expTable && probesetTable || condTable && probesetTable; - log.debug("geneTable: {}, condTable: {}, expTable: {}, probesetTable: {}, chipTable: {}", - geneTable, condTable, expTable, probesetTable, chipTable); - - - // first check if always require geneIds. - //XXX maybe overthinking as it is anyway not optimized for a Collection of filters - if (processedFilters.isAlwaysGeneId()) { - orderedTables.add(MySQLAffymetrixProbesetDAO.TABLE_NAME); - } - // then check table filtering on speciesId if any - if (processedFilters.isNeedSpeciesId()) { - if (geneTable) { - colToTableMap.put(RawDataColumn.SPECIES_ID, MySQLGeneDAO.TABLE_NAME); - orderedTables.add(MySQLGeneDAO.TABLE_NAME); - if (probesetTable) { - orderedTables.add(MySQLAffymetrixProbesetDAO.TABLE_NAME); - } - } else if (condTable) { - colToTableMap.put(RawDataColumn.SPECIES_ID, MySQLRawDataConditionDAO.TABLE_NAME); - orderedTables.add(MySQLRawDataConditionDAO.TABLE_NAME); - } - } - // then add chip table if required - if (chipTable) { - orderedTables.add(MySQLAffymetrixChipDAO.TABLE_NAME); - } - // then add probeset table. Not added if already inserted as we use a LinkedHashSet, - //and insertion order is not affected if an element is re-inserted into the set. - if (probesetTable) { - orderedTables.add(MySQLAffymetrixProbesetDAO.TABLE_NAME); - } - // then check if the experiment table was necessary and adapt expId table accordingly - if (expTable) { - orderedTables.add(MySQLMicroarrayExperimentDAO.TABLE_NAME); - if (processedFilters.isNeedExperimentId()) { - colToTableMap.put(RawDataColumn.EXPERIMENT_ID, MySQLMicroarrayExperimentDAO.TABLE_NAME); - } - } else if (processedFilters.isNeedExperimentId()) { - colToTableMap.put(RawDataColumn.EXPERIMENT_ID, MySQLAffymetrixChipDAO.TABLE_NAME); - } - - // finally check if the cond table has to be added. Not added if already inserted as we use - // a LinkedHashSet. Detect which table to use to retrieve the potential conditionId - if (condTable) { - orderedTables.add(MySQLRawDataConditionDAO.TABLE_NAME); - if (processedFilters.isNeedConditionId()) { - colToTableMap.put(RawDataColumn.COND_ID, MySQLRawDataConditionDAO.TABLE_NAME); - } - // if cond is not a necessary table it means conditionId can be retrieved from - } else if (processedFilters.isNeedConditionId()) { - colToTableMap.put(RawDataColumn.COND_ID, MySQLAffymetrixChipDAO.TABLE_NAME); - } - log.debug("orderedTables: {}", orderedTables); - - sb.append(writeFromClauseAffymetrix(orderedTables)); - - return log.traceExit(colToTableMap); + return log.traceExit(new RawDataFiltersToDatabaseMapping(ambiguousColToTable, datatype, joinedTables)); } /** @@ -570,7 +424,8 @@ private Map generateFromClauseRawDataAffymetrix(StringBui * {@code String} as value defining the table to use. */ private Map generateFromClauseRawDataInSitu(StringBuilder sb, - DAOProcessedRawDataFilter processedFilters, Set necessaryTables) { + DAOProcessedRawDataFilter processedFilters, Set necessaryTables, + Set joinedTablesOut) { log.traceEntry("{}, {}, {}", sb, processedFilters, necessaryTables); if (necessaryTables.size() == 2 && !necessaryTables.containsAll( @@ -661,279 +516,18 @@ private Map generateFromClauseRawDataInSitu(StringBuilder log.debug("orderedTables: {}", orderedTables); sb.append(writeFromClauseInSitu(orderedTables)); + joinedTablesOut.addAll(orderedTables); return log.traceExit(colToTableMap); } /** - * Method, specific to ESTs, allowing to add a FROM clause to a {@code StringBuilder} - * based on {@code DAORawDataFilter}s, and {@code boolean}s describing mandatory tables. - * It will return a {@code Map} with {@code RawDataColumn} corresponding to - * ambiguous columns to use in the WHERE clause as keys and {@code String} corresponding - * to the ambiguous tables to use to retrieve the data as values. - * - * @param sb The {@code StringBuilder} for which the FROM clause will be - * created. - * @param processedFilters The {@code DAOProcessedRawDataFilter} to use to generate the - * FROM clause - * @param necessaryTables A {@code Set} of {@code String}s corresponding to the names - * of tables necessary to the creation of the FROM clause. - * {@code necessaryTables} must contain only the names - * of the tables used to retrieve necessary information - * in the SELECT clause, not the tables used for filtering results. - * Other tables will be automatically added to the clause - * by this method to satisfy the {@code filter}s. - * @return A {@code Map} with {@code RawDataColumn} as keys and - * {@code String} as value defining the table to use. - */ - private Map generateFromClauseRawDataEst(StringBuilder sb, - DAOProcessedRawDataFilter processedFilters, Set necessaryTables) { - log.traceEntry("{}, {}, {}", sb, processedFilters, necessaryTables); - - if (necessaryTables.size() > 1) { - throw log.throwing(new IllegalStateException("Combination of necessary tables unsupported: " - + necessaryTables)); - } - - Map colToTableMap = new LinkedHashMap<>(); - LinkedHashSet orderedTables = new LinkedHashSet<>(); - - // check needed tables - boolean geneTable = processedFilters.isNeedSpeciesId() && necessaryTables.size() == 1 && - necessaryTables.contains(MySQLESTDAO.TABLE_NAME) - && !processedFilters.isNeedAssayId() && !processedFilters.isNeedConditionId(); - boolean condTable = processedFilters.isNeedSpeciesId() && !geneTable || necessaryTables - .contains(MySQLRawDataConditionDAO.TABLE_NAME); - assert !(geneTable && condTable): "We should never need both cond and gene table"; - boolean callTable = necessaryTables.contains(MySQLESTDAO.TABLE_NAME) || - processedFilters.isNeedGeneId() || - processedFilters.getFilterToCallTableAssayIds() != null; - assert !(processedFilters.getFilterToCallTableAssayIds() != null && - !necessaryTables.contains(MySQLESTDAO.TABLE_NAME)): "expressedSequenceTag should" - + " be a mandatory table if filterToCallTableAssayIds is not null"; - boolean assayTable = necessaryTables.contains(MySQLESTLibraryDAO.TABLE_NAME) || - !condTable && processedFilters.isNeedConditionId() || - condTable && callTable || processedFilters.isNeedAssayId(); - log.debug("geneTable: {}, condTable: {}, estTable: {}, estLibraryTable: {}", - geneTable, condTable, callTable, assayTable); - - // then check table filtering on speciesId if any - if (processedFilters.isNeedSpeciesId()) { - if (geneTable) { - colToTableMap.put(RawDataColumn.SPECIES_ID, MySQLGeneDAO.TABLE_NAME); - orderedTables.add(MySQLGeneDAO.TABLE_NAME); - if (callTable) { - orderedTables.add(MySQLESTDAO.TABLE_NAME); - } - } else if (condTable) { - colToTableMap.put(RawDataColumn.SPECIES_ID, MySQLRawDataConditionDAO.TABLE_NAME); - orderedTables.add(MySQLRawDataConditionDAO.TABLE_NAME); - } - } - // then add assay table if required - if (assayTable) { - orderedTables.add(MySQLESTLibraryDAO.TABLE_NAME); - if (processedFilters.isNeedAssayId()) { - colToTableMap.put(RawDataColumn.ASSAY_ID, MySQLESTLibraryDAO.TABLE_NAME); - } - // if assay is not a necessary table it means assayId can be retrieved from call table - } else if (processedFilters.isNeedAssayId()) { - colToTableMap.put(RawDataColumn.ASSAY_ID, MySQLESTDAO.TABLE_NAME); - } - // then add call table. Not added if already inserted as we use a LinkedHashSet, - //and insertion order is not affected if an element is re-inserted into the set. - if (callTable) { - orderedTables.add(MySQLESTDAO.TABLE_NAME); - } - // finally check if the cond table has to be added. Not added if already inserted as we use - // a LinkedHashSet. Detect which table to use to retrieve the potential conditionId - if (condTable) { - orderedTables.add(MySQLRawDataConditionDAO.TABLE_NAME); - if (processedFilters.isNeedConditionId()) { - colToTableMap.put(RawDataColumn.COND_ID, MySQLRawDataConditionDAO.TABLE_NAME); - } - // if cond is not a necessary table it means conditionId can be retrieved from - } else if (processedFilters.isNeedConditionId()) { - colToTableMap.put(RawDataColumn.COND_ID, MySQLESTLibraryDAO.TABLE_NAME); - } - log.debug("orderedTables: {}", orderedTables); - - sb.append(writeFromClauseEST(orderedTables)); - - return log.traceExit(colToTableMap); - } - - /** - * Generate the {@code StringBuilder} corresponding to the FROM clause of any EST - * query based on a {@code LinkedHashSet} containing tables to join in the proper order. - * - * @param tables A {@code LinkedHashSet} containing tables to join in the FROM clause in - * the proper order - * @return A {@code StringBuilder} corresponding to the FROM clause of any EST - * query - */ - private StringBuilder writeFromClauseEST(LinkedHashSet tables) { - log.traceEntry("{}", tables); - if (tables == null || tables.isEmpty()) { - throw log.throwing(new IllegalArgumentException("tables can not be null" - + " or empty.")); - } - Set previousTables = new HashSet<>(); - StringBuilder sb = new StringBuilder(); - sb.append(" FROM"); - for (String table : tables) { - if (previousTables.isEmpty()) { - sb.append(" " + table); - previousTables.add(table); - - //manage condition table - } else if (table.equals(MySQLRawDataConditionDAO.TABLE_NAME)) { - assert previousTables.contains(MySQLESTLibraryDAO.TABLE_NAME); - sb.append(" INNER JOIN " + table + " ON ") - .append(table + "." + RawDataConditionDAO.Attribute.ID.getTOFieldName() + " = ") - .append(MySQLESTLibraryDAO.TABLE_NAME + ".") - .append(ESTLibraryDAO.Attribute.CONDITION_ID.getTOFieldName()); - previousTables.add(table); - - // manage call table - } else if (table.equals(MySQLESTDAO.TABLE_NAME)) { - if (previousTables.contains(MySQLGeneDAO.TABLE_NAME)) { - sb.append(" INNER JOIN " + table + " ON ") - .append(MySQLGeneDAO.TABLE_NAME + "." + GeneDAO.Attribute.ID.getTOFieldName()) - .append(" = " + table + "." + ESTDAO.Attribute.BGEE_GENE_ID - .getTOFieldName()); - } else if (previousTables.contains(MySQLESTLibraryDAO.TABLE_NAME)) { - sb.append(" INNER JOIN " + table + " ON ") - .append(MySQLESTLibraryDAO.TABLE_NAME + "." + ESTLibraryDAO.Attribute - .ID.getTOFieldName() + " = ") - .append(table + "." + ESTDAO.Attribute.EST_LIBRARY_ID - .getTOFieldName()); - } else { - throw log.throwing(new IllegalStateException(table + " can not be join to an" - + " other table.")); - } - previousTables.add(table); - - // and finally manage assay table - } else if (table.equals(MySQLESTLibraryDAO.TABLE_NAME)) { - if (previousTables.contains(MySQLRawDataConditionDAO.TABLE_NAME)) { - sb.append(" INNER JOIN " + table + " ON ") - .append(MySQLRawDataConditionDAO.TABLE_NAME + ".") - .append(RawDataConditionDAO.Attribute.ID.getTOFieldName() + " = " + table + ".") - .append(ESTLibraryDAO.Attribute.CONDITION_ID.getTOFieldName()); - } else if (previousTables.contains(MySQLESTDAO.TABLE_NAME)) { - sb.append(" INNER JOIN " + table + " ON ") - .append(MySQLESTDAO.TABLE_NAME + ".") - .append(ESTDAO.Attribute.EST_LIBRARY_ID.getTOFieldName()) - .append(" = " + table + "." + ESTLibraryDAO.Attribute - .ID.getTOFieldName()); - } else { - throw log.throwing(new IllegalStateException(table + " can not be join to an" - + " other table.")); - } - previousTables.add(table); - } else { - throw log.throwing(new IllegalStateException( - table + " is not a proper table name or not in proper order. Previous tables: " - + previousTables)); - } - } - return log.traceExit(sb); - } - - /** - * Generate the {@code StringBuilder} corresponding to the FROM clause of any affymetrix - * query based on a {@code LinkedHashSet} containing tables to join in the proper order. - * - * @param tables A {@code LinkedHashSet} containing tables to join in the FROM clause in - * the proper order - * @return A {@code StringBuilder} corresponding to the FROM clause of any affymetrix - * query - */ - private StringBuilder writeFromClauseAffymetrix(LinkedHashSet tables) { - log.traceEntry("{}", tables); - if (tables == null || tables.isEmpty()) { - throw log.throwing(new IllegalArgumentException("tables can not be null" - + " or empty.")); - } - Set previousTables = new HashSet<>(); - StringBuilder sb = new StringBuilder(); - sb.append(" FROM"); - for (String table : tables) { - if (previousTables.isEmpty()) { - sb.append(" " + table); - previousTables.add(table); - - } else if (table.equals(MySQLRawDataConditionDAO.TABLE_NAME)) { - assert previousTables.contains(MySQLAffymetrixChipDAO.TABLE_NAME); - sb.append(" INNER JOIN " + table + " ON ") - .append(table + "." + RawDataConditionDAO.Attribute.ID.getTOFieldName() + " = ") - .append(MySQLAffymetrixChipDAO.TABLE_NAME + ".") - .append(AffymetrixChipDAO.Attribute.CONDITION_ID.getTOFieldName()); - previousTables.add(MySQLRawDataConditionDAO.TABLE_NAME); - - // manage experiment table - } else if (table.equals(MySQLMicroarrayExperimentDAO.TABLE_NAME)) { - assert previousTables.contains(MySQLAffymetrixChipDAO.TABLE_NAME); - sb.append(" INNER JOIN " + table + " ON ") - .append(table + "." + MicroarrayExperimentDAO.Attribute.ID.getTOFieldName() + " = ") - .append(MySQLAffymetrixChipDAO.TABLE_NAME + ".") - .append(AffymetrixChipDAO.Attribute.EXPERIMENT_ID.getTOFieldName()); - previousTables.add(MySQLMicroarrayExperimentDAO.TABLE_NAME); - - // manage probeset table - } else if (table.equals(MySQLAffymetrixProbesetDAO.TABLE_NAME)) { - if (previousTables.contains(MySQLGeneDAO.TABLE_NAME)) { - sb.append(" INNER JOIN " + table + " ON ") - .append(MySQLGeneDAO.TABLE_NAME + "." + GeneDAO.Attribute.ID.getTOFieldName()) - .append(" = " + table + "." + AffymetrixProbesetDAO.Attribute.BGEE_GENE_ID - .getTOFieldName()); - } else if (previousTables.contains(MySQLAffymetrixChipDAO.TABLE_NAME)) { - sb.append(" INNER JOIN " + table + " ON ") - .append(MySQLAffymetrixChipDAO.TABLE_NAME + "." + AffymetrixChipDAO.Attribute - .BGEE_AFFYMETRIX_CHIP_ID.getTOFieldName() + " = ") - .append(table + "." + AffymetrixProbesetDAO.Attribute.BGEE_AFFYMETRIX_CHIP_ID - .getTOFieldName()); - } else { - throw log.throwing(new IllegalStateException(table + " can not be join to an" - + " other table.")); - } - previousTables.add(MySQLAffymetrixProbesetDAO.TABLE_NAME); - - // and finally manage chip table - } else if (table.equals(MySQLAffymetrixChipDAO.TABLE_NAME)) { - if (previousTables.contains(MySQLRawDataConditionDAO.TABLE_NAME)) { - sb.append(" INNER JOIN " + table + " ON ") - .append(MySQLRawDataConditionDAO.TABLE_NAME + ".") - .append(RawDataConditionDAO.Attribute.ID.getTOFieldName() + " = " +table + ".") - .append(AffymetrixChipDAO.Attribute.CONDITION_ID.getTOFieldName()); - } else if (previousTables.contains(MySQLAffymetrixProbesetDAO.TABLE_NAME)) { - sb.append(" INNER JOIN " + table + " ON ") - .append(MySQLAffymetrixProbesetDAO.TABLE_NAME + ".") - .append(AffymetrixProbesetDAO.Attribute.BGEE_AFFYMETRIX_CHIP_ID.getTOFieldName()) - .append(" = " + table + "." + AffymetrixChipDAO.Attribute - .BGEE_AFFYMETRIX_CHIP_ID.getTOFieldName()); - } else { - throw log.throwing(new IllegalStateException(table + " can not be join to an" - + " other table.")); - } - previousTables.add(MySQLAffymetrixChipDAO.TABLE_NAME); - } else { - throw log.throwing(new IllegalStateException( - table + " is not a proper table name or not in proper order. Previous tables: " - + previousTables)); - } - } - return log.traceExit(sb); - } - - /** - * Generate the {@code StringBuilder} corresponding to the FROM clause of any affymetrix + * Generate the {@code StringBuilder} corresponding to the FROM clause of any in situ * query based on a {@code LinkedHashSet} containing tables to join in the proper order. * * @param tables A {@code LinkedHashSet} containing tables to join in the FROM clause in * the proper order - * @return A {@code StringBuilder} corresponding to the FROM clause of any affymetrix + * @return A {@code StringBuilder} corresponding to the FROM clause of any in situ * query */ private StringBuilder writeFromClauseInSitu(LinkedHashSet tables) { @@ -1040,7 +634,7 @@ private StringBuilder writeFromClauseInSitu(LinkedHashSet tables) { */ protected Map generateFromClauseRawDataRnaSeq(StringBuilder sb, DAOProcessedRawDataFilter processedFilters, Boolean isSingleCell, - Set necessaryTables) { + Set necessaryTables, Set joinedTablesOut) { log.traceEntry("{}, {}, {}, {}", sb, processedFilters, isSingleCell, necessaryTables); // possibilities : experiment or library or annotated samples or result or condition or @@ -1166,6 +760,7 @@ protected Map generateFromClauseRawDataRnaSeq(StringBuild log.debug("orderedTables: {}", orderedTables); sb.append(writeFromClauseRnaSeq(orderedTables)); + joinedTablesOut.addAll(orderedTables); return log.traceExit(colToTableMap); } @@ -1294,24 +889,8 @@ protected > String generateWhereClauseRawDataFilter( DAOProcessedRawDataFilter processedRawDataFilters, RawDataFiltersToDatabaseMapping filtersToDatabaseMapping, Boolean isSingleCell) { log.traceEntry("{}, {}, {}", processedRawDataFilters, filtersToDatabaseMapping, isSingleCell); - DAODataType dataType = filtersToDatabaseMapping.getDatatype(); - //ESTs can't have results if an experiment ID is requested - //(ESTs don't have experiments) - //If all filters request an experiment, we return a FALSE clause; - //otherwise, we discard the filters that have an experiment ID, - //because the method generateOneFilterWhereClause will skip the experimentId field - //for ESTs, and we would obtain some results why we should not - if (DAODataType.EST.equals(dataType) && - processedRawDataFilters.isAlwaysExactlyExperimentId()) { - log.debug("Returning FALSE where clause for EST because experiment IDs"); - return log.traceExit(" FALSE"); - } String whereClause = processedRawDataFilters.getRawDataFilters().stream() .filter(f -> { - if (DAODataType.EST.equals(dataType) && !f.getExperimentIds().isEmpty()) { - log.debug("Skipping DAORawDataFilter for EST because experiment IDs: {}", f); - return false; - } return true; }) .map(f -> this.generateOneFilterWhereClause(f, filtersToDatabaseMapping, @@ -1494,8 +1073,8 @@ private String generateExpAssayIdFilter(Set expIds, Set assayIds sb.append("("); } boolean filterFound = false; - // filter on experiment for all datatypes except est as no such concept exists - if (!expIds.isEmpty() && !filtersToDatabaseMapping.getDatatype().equals(DAODataType.EST)) { + // filter on experiment for all datatypes + if (!expIds.isEmpty()) { //retrieve table to use for experimentId sb.append(Optional.ofNullable(filtersToDatabaseMapping.getColToTableName() .get(RawDataColumn.EXPERIMENT_ID)) @@ -1537,22 +1116,21 @@ private String generateExpAssayIdFilter(Set expIds, Set assayIds if(filterFound) { sb.append(" OR "); } - // Once again, ESTs does not have experimentIds - if (!filtersToDatabaseMapping.getDatatype().equals(DAODataType.EST)) { - //try to find experimentIds - sb.append(Optional.ofNullable(filtersToDatabaseMapping.getColToTableName() - .get(RawDataColumn.EXPERIMENT_ID)) - .orElseThrow(() -> new IllegalStateException("no table associated to column" - + RawDataColumn.EXPERIMENT_ID))) - .append(".") - .append(Optional.ofNullable(filtersToDatabaseMapping.getColToColumnName() - .get(RawDataColumn.EXPERIMENT_ID)) - .orElseThrow(() -> new IllegalStateException("no column name associated to column" - + RawDataColumn.EXPERIMENT_ID))) - .append(" IN (") - .append(BgeePreparedStatement.generateParameterizedQueryString(expOrAssayIds.size())); - sb.append(") OR "); - } + + //try to find experimentIds + sb.append(Optional.ofNullable(filtersToDatabaseMapping.getColToTableName() + .get(RawDataColumn.EXPERIMENT_ID)) + .orElseThrow(() -> new IllegalStateException("no table associated to column" + + RawDataColumn.EXPERIMENT_ID))) + .append(".") + .append(Optional.ofNullable(filtersToDatabaseMapping.getColToColumnName() + .get(RawDataColumn.EXPERIMENT_ID)) + .orElseThrow(() -> new IllegalStateException("no column name associated to column" + + RawDataColumn.EXPERIMENT_ID))) + .append(" IN (") + .append(BgeePreparedStatement.generateParameterizedQueryString(expOrAssayIds.size())); + sb.append(") OR "); + // try to find assayIds sb.append(Optional.ofNullable(filtersToDatabaseMapping.getColToTableName() .get(RawDataColumn.ASSAY_ID)) diff --git a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/MySQLSamplePValueDAO.java b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/MySQLSamplePValueDAO.java index 1f42eda88..05622223f 100644 --- a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/MySQLSamplePValueDAO.java +++ b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/MySQLSamplePValueDAO.java @@ -95,80 +95,6 @@ public MySQLSamplePValueDAO(MySQLDAOManager manager) throws IllegalArgumentExcep super(manager); } - @Override - public SamplePValueTOResultSet getAffymetrixPValuesOrderedByGeneIdAndExprId( - Collection geneIds) throws DAOException, IllegalArgumentException { - - if (geneIds == null || geneIds.isEmpty() || geneIds.stream().anyMatch(id -> id == null)) { - throw log.throwing(new IllegalArgumentException("No gene IDs or null gene ID provided")); - } - Set clonedGeneIds = new HashSet<>(geneIds); - - String sampleTableName = "affymetrixProbeset"; - StringBuilder sb = new StringBuilder("SELECT ") - //geneId - .append(MySQLGeneDAO.BGEE_GENE_ID) - //sampleId - .append(", ").append("bgeeAffymetrixChipId").append(" AS ") - .append(getSelectExprFromAttribute(SamplePValueDAO.Attribute.SAMPLE_ID, colToAttrMap)) - //expressionId - .append(", ").append(getSelectExprFromAttribute(SamplePValueDAO.Attribute.EXPRESSION_ID, colToAttrMap)) - //Bonferroni corrected pvalue - .append(", MIN(").append(getSelectExprFromAttribute(SamplePValueDAO.Attribute.P_VALUE, colToAttrMap)) - //Deactivate bonferroni correction for now, since the p-value for MAS5 are 0.01, 0.05, 0.1 - //=> no p-values are going to be significant -// .append(") * COUNT(affymetrixProbesetId) AS ") - .append(") AS ") - .append(getSelectExprFromAttribute(SamplePValueDAO.Attribute.P_VALUE, colToAttrMap)) - .append(" FROM ").append(sampleTableName) - .append(getWhere(sampleTableName, geneIds)) - .append(" GROUP BY ") - .append(MySQLGeneDAO.BGEE_GENE_ID) - .append(", bgeeAffymetrixChipId ") - .append(", ") - .append(getSelectExprFromAttribute(SamplePValueDAO.Attribute.EXPRESSION_ID, colToAttrMap)) - .append(getOrderBy(sampleTableName)); - try { - BgeePreparedStatement stmt = this.getManager().getConnection().prepareStatement(sb.toString()); - stmt.setIntegers(1, clonedGeneIds, true); - return log.traceExit(new MySQLSamplePValueTOResultSet(stmt, String.class, Integer.class)); - } catch (SQLException e) { - throw log.throwing(new DAOException(e)); - } - } - - @Override - public SamplePValueTOResultSet getESTPValuesOrderedByGeneIdAndExprId(Collection geneIds) - throws DAOException, IllegalArgumentException { - if (geneIds == null || geneIds.isEmpty() || geneIds.stream().anyMatch(id -> id == null)) { - throw log.throwing(new IllegalArgumentException("No gene IDs or null gene ID provided")); - } - Set clonedGeneIds = new HashSet<>(geneIds); - - String sampleTableName = "expressedSequenceTag"; - - StringBuilder sb = new StringBuilder("SELECT DISTINCT ") - //geneId - .append(MySQLGeneDAO.BGEE_GENE_ID) - //sampleId - .append(", ").append("estLibraryId").append(" AS ") - .append(getSelectExprFromAttribute(SamplePValueDAO.Attribute.SAMPLE_ID, colToAttrMap)) - //expressionId - .append(", ").append(getSelectExprFromAttribute(SamplePValueDAO.Attribute.EXPRESSION_ID, colToAttrMap)) - //pvalue - .append(", ").append(getSelectExprFromAttribute(SamplePValueDAO.Attribute.P_VALUE, colToAttrMap)) - .append(" FROM ").append(sampleTableName) - .append(getWhere(sampleTableName, geneIds)) - .append(getOrderBy(sampleTableName)); - try { - BgeePreparedStatement stmt = this.getManager().getConnection().prepareStatement(sb.toString()); - stmt.setIntegers(1, clonedGeneIds, true); - return log.traceExit(new MySQLSamplePValueTOResultSet(stmt, String.class, String.class)); - } catch (SQLException e) { - throw log.throwing(new DAOException(e)); - } - } - @Override public SamplePValueTOResultSet getInSituPValuesOrderedByGeneIdAndExprId(Collection geneIds) throws DAOException, IllegalArgumentException { diff --git a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/MysqlRawDataCountDAO.java b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/MysqlRawDataCountDAO.java index 42111859c..089ff97b7 100644 --- a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/MysqlRawDataCountDAO.java +++ b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/MysqlRawDataCountDAO.java @@ -14,10 +14,8 @@ import org.bgee.model.dao.api.expressiondata.rawdata.DAOProcessedRawDataFilter; import org.bgee.model.dao.api.expressiondata.rawdata.DAORawDataFilter; import org.bgee.model.dao.api.expressiondata.rawdata.RawDataCountDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTDAO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituEvidenceDAO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituSpotDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipDAO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqLibraryAnnotatedSampleDAO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqLibraryDAO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqResultAnnotatedSampleDAO; @@ -25,12 +23,8 @@ import org.bgee.model.dao.mysql.connector.MySQLDAOManager; import org.bgee.model.dao.mysql.connector.MySQLDAOResultSet; import org.bgee.model.dao.mysql.exception.UnrecognizedColumnException; -import org.bgee.model.dao.mysql.expressiondata.rawdata.est.MySQLESTDAO; -import org.bgee.model.dao.mysql.expressiondata.rawdata.est.MySQLESTLibraryDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.insitu.MySQLInSituEvidenceDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.insitu.MySQLInSituSpotDAO; -import org.bgee.model.dao.mysql.expressiondata.rawdata.microarray.MySQLAffymetrixChipDAO; -import org.bgee.model.dao.mysql.expressiondata.rawdata.microarray.MySQLAffymetrixProbesetDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.rnaseq.MySQLRNASeqLibraryAnnotatedSampleDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.rnaseq.MySQLRNASeqLibraryDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.rnaseq.MySQLRNASeqResultAnnotatedSampleDAO; @@ -44,243 +38,6 @@ public MysqlRawDataCountDAO(MySQLDAOManager manager) throws IllegalArgumentExcep private static final Logger log = LogManager.getLogger(MysqlRawDataCountDAO.class.getName()); - @Override - public RawDataCountContainerTO getAffymetrixCount(Collection rawDataFilters, - boolean experimentCount, boolean assayCount, boolean callCount) { - log.traceEntry("{}, {},{}, {}", rawDataFilters, experimentCount, assayCount, callCount); - if (!experimentCount && !assayCount && !callCount) { - throw log.throwing(new IllegalArgumentException("experimentCount, assayCount and" - + " callsCount can not be all false at the same time")); - } - //If callsCount is requested along with other count, we separate that in two queries - //for faster results - boolean newCallCount = callCount; - RawDataCountContainerTO callCountTO = null; - if (callCount && (experimentCount || assayCount)) { - callCountTO = this.getAffymetrixCount(rawDataFilters, false, false, true); - newCallCount = false; - } - DAOProcessedRawDataFilter processedRawDataFilters = - new DAOProcessedRawDataFilter<>(rawDataFilters); - if (newCallCount) { - final MySQLAffymetrixChipDAO assayDAO = new MySQLAffymetrixChipDAO(this.getManager()); - processedRawDataFilters = this.processFilterForCallTableAssayIds( - processedRawDataFilters, - (s) -> assayDAO.getAffymetrixChips(s, null, null, - Set.of(AffymetrixChipDAO.Attribute.BGEE_AFFYMETRIX_CHIP_ID)) - .stream() - .map(to -> to.getId()) - . collect(Collectors.toSet()), - Integer.class, DAODataType.AFFYMETRIX, null); - //Means there is no result - if (processedRawDataFilters == null) { - return log.traceExit(new RawDataCountContainerTO( - experimentCount? 0: null, - assayCount? 0: null, - newCallCount? 0: null, - null, - null)); - } - } - StringBuilder sb = new StringBuilder(); - - boolean probesetTable = processedRawDataFilters.isNeedGeneId() || newCallCount; - - // generate SELECT clause - sb.append("SELECT "); - boolean previousCount = false; - if (experimentCount) { - sb.append(" count(distinct ").append(MySQLAffymetrixChipDAO.TABLE_NAME).append(".") - .append(AffymetrixChipDAO.Attribute.EXPERIMENT_ID.getTOFieldName()).append(") as ") - .append(RawDataCountDAO.Attribute.EXP_COUNT.getTOFieldName()); - previousCount = true; - } - if (assayCount) { - if(previousCount) { - sb.append(","); - } - sb.append(" count("); - if (!probesetTable) { - //count(*) is faster than count(columnName) - //(see https://stackoverflow.com/a/3003482). - //If the probesets were not required, then - //number of lines = number of bgeeAffymetrixChipId - //(primary key of the affymetrixChip table) - sb.append("*"); - } else { - //If probesets are needed, we need to add DISTINCT, - //because relation 1-to-many to table affymetrixProbeset. - sb.append("distinct ").append(MySQLAffymetrixProbesetDAO.TABLE_NAME).append(".") - .append(AffymetrixChipDAO.Attribute.BGEE_AFFYMETRIX_CHIP_ID.getTOFieldName()); - } - sb.append(") as ") - .append(RawDataCountDAO.Attribute.ASSAY_COUNT.getTOFieldName()); - previousCount = true; - } - if (newCallCount) { - assert probesetTable; - if(previousCount) { - sb.append(","); - } - //Here, number of lines always equal to - //count(distinct affymetrixProbesetId, bgeeAffymetrixChipId) - //(the primary key of the table that we need to count). - //We wouldn't need the DISTINCT. - //And count(*) is faster than count(columnName) - sb.append(" count(*) as ") - .append(RawDataCountDAO.Attribute.CALLS_COUNT.getTOFieldName()); - } - - // create the set of tables it is necessary to use in FROM clause even if no filter - // on those columns - Set necessaryTables = new HashSet<>(); - if (newCallCount) { - necessaryTables.add(MySQLAffymetrixProbesetDAO.TABLE_NAME); - } - if (experimentCount || assayCount && !newCallCount) { - necessaryTables.add(MySQLAffymetrixChipDAO.TABLE_NAME); - } - - // generate FROM clause - RawDataFiltersToDatabaseMapping filtersToDatabaseMapping = generateFromClauseRawData(sb, - processedRawDataFilters, null, necessaryTables, DAODataType.AFFYMETRIX); - - // generate WHERE CLAUSE - // usedInPropagatedCalls is only used for rna-seq data. If usedInPropagatedCalls is the only - // rawDataFilter not empty the query will not contain any where clause. That's why we check - // that any rawDataFilter variable except usedInPropagatedCalls is not empty. - boolean requireWhereClause = processedRawDataFilters.getRawDataFilters().stream() - .allMatch(item -> item.usedInPropagatedCallsIsTheOnlyPotentialNotBlank()) ? false : true; - if(requireWhereClause) { - sb.append(" WHERE ") - .append(generateWhereClauseRawDataFilter(processedRawDataFilters, - filtersToDatabaseMapping)); - } - try { - BgeePreparedStatement stmt = this.parameterizeQuery(sb.toString(), - processedRawDataFilters, DAODataType.AFFYMETRIX, null, null); - MySQLRawDataCountContainerTOResultSet resultSet = new MySQLRawDataCountContainerTOResultSet(stmt); - resultSet.next(); - RawDataCountContainerTO to = resultSet.getTO(); - resultSet.close(); - if (callCountTO != null) { - return log.traceExit(new RawDataCountContainerTO(to.getExperimentCount(), - to.getAssayCount(), callCountTO.getCallCount())); - } - return log.traceExit(to); - } catch (SQLException e) { - throw log.throwing(new DAOException(e)); - } - } - - public RawDataCountContainerTO getESTCount(Collection rawDataFilters, - boolean assayCount, boolean callCount) { - log.traceEntry("{}, {},{}, {}", rawDataFilters, assayCount, callCount); - if (!assayCount && !callCount) { - throw log.throwing(new IllegalArgumentException("experimentCount, assayCount and" - + " callsCount can not be all false at the same time")); - } - //If callsCount is requested along with other count, we separate that in two queries - //for faster results - boolean newCallCount = callCount; - RawDataCountContainerTO callCountTO = null; - if (callCount && assayCount) { - callCountTO = this.getESTCount(rawDataFilters, false, true); - newCallCount = false; - } - final DAOProcessedRawDataFilter processedRawDataFilters = - new DAOProcessedRawDataFilter<>(rawDataFilters); - //If all filters always target some experiment IDs, there can be no results, as EST - //don't have experiments. Since we don't have to return a ResultSet, - //we don't even bother performing a query with a FALSE where clause. - if (processedRawDataFilters.isAlwaysExactlyExperimentId()) { - return log.traceExit(new RawDataCountContainerTO(null, - assayCount? 0: null, callCount? 0: null)); - } - - StringBuilder sb = new StringBuilder(); - - boolean callTable = processedRawDataFilters.isNeedGeneId() || newCallCount; - - // generate SELECT clause - sb.append("SELECT "); - boolean previousCount = false; - if (assayCount) { - sb.append(" count("); - if (!callTable) { - //count(*) is faster than count(columnName) - //(see https://stackoverflow.com/a/3003482). - //If the calls were not required, then - //number of lines = number of estLibraryId - //(primary key of the estLibrary table) - sb.append("*"); - } else { - //If calls are needed, we need to add DISTINCT, - //because relation 1-to-many to table call. - sb.append("distinct ").append(MySQLESTDAO.TABLE_NAME).append(".") - .append(ESTDAO.Attribute.EST_LIBRARY_ID.getTOFieldName()); - } - sb.append(") as ") - .append(RawDataCountDAO.Attribute.ASSAY_COUNT.getTOFieldName()); - previousCount = true; - } - if (newCallCount) { - assert callTable; - if(previousCount) { - sb.append(","); - } - //Here, number of lines always equal to - //count(distinct estId) - //(the primary key of the table that we need to count). - //We wouldn't need the DISTINCT. - //And count(*) is faster than count(columnName) - sb.append(" count(*) as ") - .append(RawDataCountDAO.Attribute.CALLS_COUNT.getTOFieldName()); - } - - // create the set of tables it is necessary to use in FROM clause even if no filter - // on those columns. - // If newCallCount then all count can be retrieved from the call table - Set necessaryTables = new HashSet<>(); - if (newCallCount) { - necessaryTables.add(MySQLESTDAO.TABLE_NAME); - } else { - assert assayCount; - necessaryTables.add(MySQLESTLibraryDAO.TABLE_NAME); - } - - // generate FROM clause - RawDataFiltersToDatabaseMapping filtersToDatabaseMapping = generateFromClauseRawData(sb, - processedRawDataFilters, null, necessaryTables, DAODataType.EST); - - // generate WHERE CLAUSE - // usedInPropagatedCalls is only used for rna-seq data. If usedInPropagatedCalls is the only - // rawDataFilter not empty the query will not contain any where clause. That's why we check - // that any rawDataFilter variable except usedInPropagatedCalls is not empty. - boolean requireWhereClause = processedRawDataFilters.getRawDataFilters().stream() - .allMatch(item -> item.usedInPropagatedCallsIsTheOnlyPotentialNotBlank()) ? false : true; - if(requireWhereClause) { - sb.append(" WHERE ") - .append(generateWhereClauseRawDataFilter(processedRawDataFilters, - filtersToDatabaseMapping)); - } - try { - BgeePreparedStatement stmt = this.parameterizeQuery(sb.toString(), - processedRawDataFilters, DAODataType.EST, null, null); - MySQLRawDataCountContainerTOResultSet resultSet = new MySQLRawDataCountContainerTOResultSet(stmt); - resultSet.next(); - RawDataCountContainerTO to = resultSet.getTO(); - resultSet.close(); - if (callCountTO != null) { - return log.traceExit(new RawDataCountContainerTO(to.getExperimentCount(), - to.getAssayCount(), callCountTO.getCallCount())); - } - return log.traceExit(to); - } catch (SQLException e) { - throw log.throwing(new DAOException(e)); - } - } - @Override public RawDataCountContainerTO getInSituCount(Collection rawDataFilters, boolean experimentCount, boolean assayCount, boolean assayConditionCount, diff --git a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/RawDataFiltersToDatabaseMapping.java b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/RawDataFiltersToDatabaseMapping.java index f704eb1cf..aeebaf610 100644 --- a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/RawDataFiltersToDatabaseMapping.java +++ b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/RawDataFiltersToDatabaseMapping.java @@ -4,29 +4,22 @@ import java.util.HashMap; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bgee.model.dao.api.expressiondata.DAODataType; import org.bgee.model.dao.api.expressiondata.rawdata.RawDataConditionDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTLibraryDAO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituEvidenceDAO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituExperimentDAO; import org.bgee.model.dao.api.expressiondata.rawdata.insitu.InSituSpotDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixProbesetDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.MicroarrayExperimentDAO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqExperimentDAO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqLibraryDAO; import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqResultAnnotatedSampleDAO; import org.bgee.model.dao.api.gene.GeneDAO; import org.bgee.model.dao.api.species.SpeciesDAO; -import org.bgee.model.dao.mysql.expressiondata.rawdata.est.MySQLESTDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.insitu.MySQLInSituSpotDAO; -import org.bgee.model.dao.mysql.expressiondata.rawdata.microarray.MySQLAffymetrixChipDAO; -import org.bgee.model.dao.mysql.expressiondata.rawdata.microarray.MySQLAffymetrixProbesetDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.rnaseq.MySQLRNASeqResultAnnotatedSampleDAO; /** @@ -41,6 +34,7 @@ public class RawDataFiltersToDatabaseMapping { private final Map colToTableName; private final Map colToColumnName; private final DAODataType datatype; + private final Set joinedTables; private final static Logger log = LogManager.getLogger(RawDataFiltersToDatabaseMapping.class); @@ -56,7 +50,7 @@ public static enum RawDataColumn {SPECIES_ID, EXPERIMENT_ID, COND_ID, ASSAY_ID, CALL_TABLE_ASSAY_ID, GENE_ID} public RawDataFiltersToDatabaseMapping(Map ambiguousColToTableName, - DAODataType datatype) { + DAODataType datatype, Set joinedTables) { if (datatype == null) { throw log.throwing(new IllegalArgumentException("datatype can not be null")); } @@ -68,6 +62,8 @@ public RawDataFiltersToDatabaseMapping(Map ambiguousColTo this.colToColumnName = Collections.unmodifiableMap(RawDataFiltersToDatabaseMapping .generateColToColName(datatype)); this.datatype = datatype; + this.joinedTables = joinedTables == null ? + Collections.emptySet() : Collections.unmodifiableSet(joinedTables); log.debug(this.toString()); } @@ -75,6 +71,10 @@ public Map getColToTableName() { return colToTableName; } + public Set getJoinedTables() { + return joinedTables; + } + public Map getColToColumnName() { return colToColumnName; } @@ -90,16 +90,9 @@ private static Map generateColToTableName( finalColToTable.putAll(ambiguousColToTable.entrySet().stream() .collect(Collectors.toMap(e-> e.getKey(), e-> e.getValue()))); // then add tables that can not be ambiguous. It depends on the datatype - if (datatype.equals(DAODataType.AFFYMETRIX)) { - finalColToTable.put(RawDataColumn.ASSAY_ID, MySQLAffymetrixChipDAO.TABLE_NAME); - finalColToTable.put(RawDataColumn.GENE_ID, MySQLAffymetrixProbesetDAO.TABLE_NAME); - finalColToTable.put(RawDataColumn.CALL_TABLE_ASSAY_ID, MySQLAffymetrixProbesetDAO.TABLE_NAME); - } else if (datatype.equals(DAODataType.RNA_SEQ)) { + if (datatype.equals(DAODataType.RNA_SEQ)) { finalColToTable.put(RawDataColumn.GENE_ID, MySQLRNASeqResultAnnotatedSampleDAO.TABLE_NAME); finalColToTable.put(RawDataColumn.CALL_TABLE_ASSAY_ID, MySQLRNASeqResultAnnotatedSampleDAO.TABLE_NAME); - } else if (datatype.equals(DAODataType.EST)) { - finalColToTable.put(RawDataColumn.CALL_TABLE_ASSAY_ID, MySQLESTDAO.TABLE_NAME); - finalColToTable.put(RawDataColumn.GENE_ID, MySQLESTDAO.TABLE_NAME); } else if (datatype.equals(DAODataType.IN_SITU)) { finalColToTable.put(RawDataColumn.CALL_TABLE_ASSAY_ID, MySQLInSituSpotDAO.TABLE_NAME); finalColToTable.put(RawDataColumn.GENE_ID, MySQLInSituSpotDAO.TABLE_NAME); @@ -121,14 +114,7 @@ private static Map generateColToColName(DAODataType datat finalColToColName.put(RawDataColumn.SPECIES_ID, SpeciesDAO.Attribute .ID.getTOFieldName()); // then add column names that depend on the datatype - if (datatype.equals(DAODataType.AFFYMETRIX)) { - finalColToColName.put(RawDataColumn.ASSAY_ID, AffymetrixChipDAO.Attribute - .AFFYMETRIX_CHIP_ID.getTOFieldName()); - finalColToColName.put(RawDataColumn.CALL_TABLE_ASSAY_ID, AffymetrixProbesetDAO.Attribute - .BGEE_AFFYMETRIX_CHIP_ID.getTOFieldName()); - finalColToColName.put(RawDataColumn.EXPERIMENT_ID, MicroarrayExperimentDAO.Attribute - .ID.getTOFieldName()); - } else if (datatype.equals(DAODataType.RNA_SEQ)) { + if (datatype.equals(DAODataType.RNA_SEQ)) { finalColToColName.put(RawDataColumn.EXPERIMENT_ID, RNASeqExperimentDAO.Attribute .ID.getTOFieldName()); finalColToColName.put(RawDataColumn.CALL_TABLE_ASSAY_ID, RNASeqResultAnnotatedSampleDAO @@ -136,12 +122,6 @@ private static Map generateColToColName(DAODataType datat // for RNA-Seq the filtering is done on library IDs finalColToColName.put(RawDataColumn.ASSAY_ID, RNASeqLibraryDAO.Attribute .ID.getTOFieldName()); - } else if (datatype.equals(DAODataType.EST)) { - // for RNA-Seq the filtering is done on library IDs - finalColToColName.put(RawDataColumn.ASSAY_ID, ESTLibraryDAO.Attribute - .ID.getTOFieldName()); - finalColToColName.put(RawDataColumn.CALL_TABLE_ASSAY_ID, ESTDAO.Attribute - .EST_LIBRARY_ID.getTOFieldName()); } else if (datatype.equals(DAODataType.IN_SITU)) { finalColToColName.put(RawDataColumn.ASSAY_ID, InSituEvidenceDAO.Attribute .IN_SITU_EVIDENCE_ID.getTOFieldName()); @@ -158,7 +138,7 @@ private static Map generateColToColName(DAODataType datat @Override public int hashCode() { - return Objects.hash(colToColumnName, colToTableName, datatype); + return Objects.hash(colToColumnName, colToTableName, datatype, joinedTables); } @Override @@ -171,13 +151,14 @@ public boolean equals(Object obj) { return false; RawDataFiltersToDatabaseMapping other = (RawDataFiltersToDatabaseMapping) obj; return Objects.equals(colToColumnName, other.colToColumnName) - && Objects.equals(colToTableName, other.colToTableName) && datatype == other.datatype; + && Objects.equals(colToTableName, other.colToTableName) && datatype == other.datatype + && Objects.equals(joinedTables, other.joinedTables); } @Override public String toString() { return "RawDataFiltersToDatabaseMapping [colToTableName=" + colToTableName + ", colToColumnName=" - + colToColumnName + ", datatype=" + datatype + "]"; + + colToColumnName + ", datatype=" + datatype + ", joinedTables=" + joinedTables + "]"; } } diff --git a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/call/MySQLRawExpressionCallDAO.java b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/call/MySQLRawExpressionCallDAO.java new file mode 100644 index 000000000..ed84f2d7c --- /dev/null +++ b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/call/MySQLRawExpressionCallDAO.java @@ -0,0 +1,301 @@ +//package org.bgee.model.dao.mysql.expressiondata.rawdata.call; +// +//import java.math.BigDecimal; +//import java.sql.ResultSet; +//import java.sql.SQLException; +//import java.util.Collection; +//import java.util.EnumSet; +//import java.util.HashMap; +//import java.util.HashSet; +//import java.util.Map; +//import java.util.Set; +//import java.util.stream.Collectors; +// +//import org.apache.logging.log4j.LogManager; +//import org.apache.logging.log4j.Logger; +//import org.bgee.model.dao.api.exception.DAOException; +//import org.bgee.model.dao.api.expressiondata.DAODataType; +//import org.bgee.model.dao.api.expressiondata.rawdata.call.DAORawCallFilter; +//import org.bgee.model.dao.api.expressiondata.rawdata.call.DAORawCallValues; +//import org.bgee.model.dao.api.expressiondata.rawdata.call.RawExpressionCallDAO; +//import org.bgee.model.dao.api.gene.GeneDAO; +//import org.bgee.model.dao.mysql.connector.BgeePreparedStatement; +//import org.bgee.model.dao.mysql.connector.MySQLDAOManager; +//import org.bgee.model.dao.mysql.connector.MySQLDAOResultSet; +//import org.bgee.model.dao.mysql.exception.UnrecognizedColumnException; +//import org.bgee.model.dao.mysql.expressiondata.rawdata.MySQLRawDataConditionDAO; +//import org.bgee.model.dao.mysql.expressiondata.rawdata.MySQLRawDataDAO; +//import org.bgee.model.dao.mysql.gene.MySQLGeneDAO; +// +///** +// * A {@code RawExpressionCallDAO} for MySQL. +// * +// * @author Valentine Rech de Laval +// * @author Frederic Bastian +// * @version Bgee 14, Feb. 2017 +// * @see org.bgee.model.dao.api.expressiondata.rawdata.RawExpressionCallDAO.RawExpressionCallTO +// * @since Bgee 14, Feb. 2017 +// */ +//public class MySQLRawExpressionCallDAO extends MySQLRawDataDAO +// implements RawExpressionCallDAO { +// +// private final static Logger log = LogManager.getLogger(MySQLRawExpressionCallDAO.class.getName()); +// +// //TODO: the name of that table is temporary. Do not forget to update it while moving to Bgee 16 +// public final static String TABLE_NAME = "expression_otf"; +// +// public MySQLRawExpressionCallDAO(MySQLDAOManager manager) throws IllegalArgumentException { +// super(manager); +// } +// +// +// @Override +// public RawExpressionCallTOResultSet getExpressionCallsOrderedByGeneIdAndExprId( +// Collection geneIds) throws DAOException, IllegalArgumentException { +// log.traceEntry("{}", geneIds); +// +// if (geneIds == null || geneIds.isEmpty() || geneIds.stream().anyMatch(id -> id == null)) { +// throw log.throwing(new IllegalArgumentException("No gene IDs or null gene ID provided")); +// } +// Set clonedGeneIds = new HashSet<>(geneIds); +// +// StringBuilder sb = new StringBuilder(); +// sb.append("SELECT ").append(TABLE_NAME).append(".*") +// .append(" FROM ").append(TABLE_NAME) +// .append(" WHERE ").append(TABLE_NAME).append(".") +// .append(MySQLGeneDAO.BGEE_GENE_ID).append(" IN (") +// .append(BgeePreparedStatement.generateParameterizedQueryString(clonedGeneIds.size())).append(")") +// .append(" ORDER BY ").append(TABLE_NAME).append(".").append(MySQLGeneDAO.BGEE_GENE_ID) +// .append(", ").append(TABLE_NAME).append(".").append(RawExpressionCallDAO.Attribute.EXPRESSION_ID); +// try { +// BgeePreparedStatement stmt = this.getManager().getConnection().prepareStatement(sb.toString()); +// stmt.setIntegers(1, clonedGeneIds, true); +// return log.traceExit(new MySQLRawExpressionCallTOResultSet(stmt)); +// } catch (SQLException e) { +// throw log.throwing(new DAOException(e)); +// } +// } +// +// @Override +// public RawExpressionCallTOResultSet getRawExpressionCalls(DAORawCallFilter rawCallFilter) +// throws DAOException { +// log.traceEntry("{}", rawCallFilter); +// // at least one dataFilter should be provided +// if (rawCallFilter == null) { +// throw log.throwing(new IllegalArgumentException("At least one DAORawCallFilter should be provided to" +// + " retrieve raw expression calls")); +// } +// +// StringBuilder sb = new StringBuilder(); +// // generate SELECT clause +// sb.append("SELECT DISTINCT ") +// .append(MySQLRawExpressionCallDAO.TABLE_NAME + "." + RawExpressionCallDAO.Attribute.EXPRESSION_ID.getTOFieldName()).append(", ") +// .append(MySQLRawExpressionCallDAO.TABLE_NAME + "." + RawExpressionCallDAO.Attribute.CONDITION_ID.getTOFieldName()).append(", ") +// .append(MySQLRawExpressionCallDAO.TABLE_NAME + "." + RawExpressionCallDAO.Attribute.BGEE_GENE_ID.getTOFieldName()).append(", ") +// // generate select clause depending on data types +// .append(generateSelectClauseDependingOnDataTypes(rawCallFilter.getDataTypes())) +// // generate FROM CLAUSE +// .append(generateFromClause(rawCallFilter)) +// // generate WHERE CLAUSE. There is always a WHERE clause as we do not allow empty DAO data filters +// .append(" WHERE ").append(generateWhereClause(rawCallFilter)); +// +// try { +// BgeePreparedStatement stmt = this.parameterizeQuery(sb.toString(), rawCallFilter); +// return log.traceExit(new MySQLRawExpressionCallTOResultSet(stmt)); +// } catch (SQLException e) { +// throw log.throwing(new DAOException(e)); +// } +// } +// +// // method used to generate the select clause retrieving pValues, weight and expression score +// // for each datatype requested. +// private String generateSelectClauseDependingOnDataTypes(EnumSet dataTypes) { +// log.traceEntry("{}", dataTypes); +// if(dataTypes == null || dataTypes.isEmpty()) { +// throw log.throwing(new IllegalArgumentException("At least one DAODataType should be provided to" +// + " retrieve raw expression calls")); +// } +// +// // first retrieve datatype dependant attributes +// EnumSet datatypeDependentAttributes = +// RawExpressionCallDAO.Attribute.getDataTypeDependentAttributes(); +// +// // then retrieve all datatype dependant columns for which the datatype is requested +// return log.traceExit(dataTypes.stream() +// .map(dt -> { +// return datatypeDependentAttributes.stream().map(attr -> { +// return MySQLRawExpressionCallDAO.TABLE_NAME + "." + attr.getTOFieldName() + dt.getFieldNamePart(); +// }).collect(Collectors.joining(", ")); +// }).collect(Collectors.joining(", "))); +// +// } +// +// // generate from clause to retrieve raw expression calls. +// // The logic is the following: +// // - join to the condition table if condition filters are provided +// // - always join to gene table if speciesIds are provided +// // - otherwise only expression table is queried +// private String generateFromClause(DAORawCallFilter rawCallFilter) { +// log.traceEntry("{}", rawCallFilter); +// StringBuilder sb = new StringBuilder(); +// +// sb.append(" FROM " + TABLE_NAME + " AS " + TABLE_NAME); +// if (! rawCallFilter.getSpeciesIds().isEmpty()) { +// sb.append(" INNER JOIN " + MySQLGeneDAO.TABLE_NAME + " AS "+ MySQLGeneDAO.TABLE_NAME) +// .append(" ON " + MySQLGeneDAO.TABLE_NAME + "." + GeneDAO.Attribute.ID.getTOFieldName() + " = " + TABLE_NAME + "." + +// RawExpressionCallDAO.Attribute.BGEE_GENE_ID.getTOFieldName()); +// } +// if (! rawCallFilter.getConditionFilters().isEmpty()) { +// sb.append(" INNER JOIN " + MySQLRawDataConditionDAO.TABLE_NAME + " AS "+ MySQLRawDataConditionDAO.TABLE_NAME) +// .append(" ON " + MySQLRawDataConditionDAO.TABLE_NAME + ".exprMappedConditionId" + " = " + TABLE_NAME + "." + +// RawExpressionCallDAO.Attribute.CONDITION_ID.getTOFieldName()); +// } +// return log.traceExit(sb.toString()); +// } +// +// private String generateWhereClause(DAORawCallFilter rawCallFilter) { +// log.traceEntry("{}", rawCallFilter); +// boolean alreadyFiltered = false; +// StringBuilder filterSb = new StringBuilder(); +// if (! rawCallFilter.getGeneIds().isEmpty()) { +// filterSb.append(TABLE_NAME).append(".").append(RawExpressionCallDAO.Attribute.BGEE_GENE_ID.getTOFieldName()) +// .append(" IN ("); +// filterSb.append(BgeePreparedStatement.generateParameterizedQueryString(rawCallFilter.getGeneIds().size())) +// .append(")"); +// alreadyFiltered = true; +// } +// if (!rawCallFilter.getSpeciesIds().isEmpty()) { +// if (alreadyFiltered) { +// filterSb.append(" AND "); +// } +// filterSb.append(MySQLGeneDAO.TABLE_NAME).append(".").append(GeneDAO.Attribute.SPECIES_ID.getTOFieldName()) +// .append(" IN ("); +// filterSb.append(BgeePreparedStatement.generateParameterizedQueryString(rawCallFilter.getSpeciesIds().size())) +// .append(")"); +// alreadyFiltered = true; +// } +// if (!rawCallFilter.getConditionFilters().isEmpty()) { +// if (alreadyFiltered) { +// filterSb.append(" AND "); +// } +// filterSb.append(rawCallFilter.getConditionFilters().stream().map(cf -> { +// return MySQLRawDataConditionDAO.generateOneConditionFilter(cf); +// }).collect(Collectors.joining(" OR "))); +// } +// return filterSb.toString(); +// } +// +// private BgeePreparedStatement parameterizeQuery(String query, DAORawCallFilter rawCallFilter) throws SQLException { +// log.traceEntry("{}, {}", query, rawCallFilter); +// BgeePreparedStatement stmt = this.getManager().getConnection() +// .prepareStatement(query); +// int paramIndex = 1; +// if (! rawCallFilter.getGeneIds().isEmpty()) { +// stmt.setIntegers(paramIndex, rawCallFilter.getGeneIds(), true); +// paramIndex += rawCallFilter.getGeneIds().size(); +// } +// if (!rawCallFilter.getSpeciesIds().isEmpty()) { +// stmt.setIntegers(paramIndex, rawCallFilter.getSpeciesIds(), true); +// paramIndex += rawCallFilter.getSpeciesIds().size(); +// } +// if (!rawCallFilter.getConditionFilters().isEmpty()) { +// MySQLRawDataConditionDAO.configureRawDataConditionFiltersStmt(stmt, rawCallFilter.getConditionFilters(), +// paramIndex); +// } +// return stmt; +// } +// private static DAODataType detectDataTypeFromColumnName (String columnName) { +// log.traceEntry(); +// for (DAODataType dataType : EnumSet.allOf(DAODataType.class)) { +// if (dataType.equals(DAODataType.RNA_SEQ)) { +// String columnNameWithoutSingleCell = columnName +// .replace(DAODataType.SC_RNA_SEQ.getFieldNamePart(), ""); +// if (columnNameWithoutSingleCell.contains(dataType.getFieldNamePart())) { +// return dataType; +// } +// } +// if (columnName.contains(dataType.getFieldNamePart())) { +// return dataType; +// } +// } +// throw log.throwing(new IllegalArgumentException("Field name with no data type info: " +// + columnName)); +// } +// +// /** +// * Implementation of the {@code RawExpressionCallTOResultSet}. +// * +// * @author Frederic Bastian +// * @version Bgee 16.0 Mar. 2025 +// * @since Bgee 14 Feb. 2017 +// */ +// class MySQLRawExpressionCallTOResultSet extends MySQLDAOResultSet +// implements RawExpressionCallTOResultSet { +// +// /** +// * @param statement The {@code BgeePreparedStatement} +// * @param comb The {@code CondParamCombination} allowing to target the appropriate +// * field and table names. +// */ +// private MySQLRawExpressionCallTOResultSet(BgeePreparedStatement statement) { +// super(statement); +// } +// +// @Override +// protected RawExpressionCallDAO.RawExpressionCallTO getNewTO() throws DAOException { +// try { +// log.traceEntry(); +// final ResultSet currentResultSet = this.getCurrentResultSet(); +// Long id = null; +// Integer bgeeGeneId = null, conditionId = null; +// Map> rawCallValuePerDataTypePerAttribute = new HashMap<>(); +// for (String colName: this.getColumnLabels().values()) { +// if (colName.equals(RawExpressionCallDAO.Attribute.BGEE_GENE_ID.getTOFieldName())) { +// bgeeGeneId = currentResultSet.getInt(colName); +// } else if (colName.equals(RawExpressionCallDAO.Attribute.CONDITION_ID.getTOFieldName())) { +// conditionId = currentResultSet.getInt(colName); +// } else if (colName.equals(RawExpressionCallDAO.Attribute.EXPRESSION_ID.getTOFieldName())) { +// id = currentResultSet.getLong(colName); +// } else if (colName.startsWith(RawExpressionCallDAO.Attribute.SCORE.getTOFieldName())) { +// addDtDependantValue(rawCallValuePerDataTypePerAttribute, colName, RawExpressionCallDAO.Attribute.SCORE, currentResultSet); +//// rawCallValuePerDataTypePerAttribute.put(detectDataTypeFromColumnName(colName), +//// new HashMap<>(Map.of(RawExpressionCallDAO.Attribute.SCORE, currentResultSet.getBigDecimal(colName)))); +// } else if (colName.startsWith(RawExpressionCallDAO.Attribute.PVALUE.getTOFieldName())) { +// addDtDependantValue(rawCallValuePerDataTypePerAttribute, colName, RawExpressionCallDAO.Attribute.PVALUE, currentResultSet); +// } else if (colName.startsWith(RawExpressionCallDAO.Attribute.WEIGHT.getTOFieldName())) { +// addDtDependantValue(rawCallValuePerDataTypePerAttribute, colName, RawExpressionCallDAO.Attribute.WEIGHT, currentResultSet); +//// rawCallValuePerDataTypePerAttribute.put(detectDataTypeFromColumnName(colName), +//// new HashMap<>(Map.of(RawExpressionCallDAO.Attribute.WEIGHT, currentResultSet.getBigDecimal(colName)))); +// } else { +// throw log.throwing(new UnrecognizedColumnException(colName)); +// } +// } +// Map rawCallValuesPerDataType = +// rawCallValuePerDataTypePerAttribute.keySet().stream().collect(Collectors.toMap(dt -> dt, +// dt -> new DAORawCallValues(rawCallValuePerDataTypePerAttribute.get(dt).get(RawExpressionCallDAO.Attribute.SCORE), +// rawCallValuePerDataTypePerAttribute.get(dt).get(RawExpressionCallDAO.Attribute.PVALUE), +// rawCallValuePerDataTypePerAttribute.get(dt).get(RawExpressionCallDAO.Attribute.WEIGHT)))); +// return log.traceExit(new RawExpressionCallTO(id, bgeeGeneId, conditionId, rawCallValuesPerDataType)); +// } catch (SQLException e) { +// throw log.throwing(new DAOException(e)); +// } +// } +// +// private void addDtDependantValue ( +// Map> rawCallValuePerDataTypePerAttribute, +// String colName, RawExpressionCallDAO.Attribute attr, ResultSet currentResultSet) throws SQLException{ +// DAODataType dt = detectDataTypeFromColumnName(colName); +// if (rawCallValuePerDataTypePerAttribute.containsKey(dt)) { +// rawCallValuePerDataTypePerAttribute.get(dt).put(attr, currentResultSet.getBigDecimal(colName)); +// } else { +// Map tempMap = new HashMap<>(); +// tempMap.put(attr, currentResultSet.getBigDecimal(colName)); +// rawCallValuePerDataTypePerAttribute.put(detectDataTypeFromColumnName(colName), +// tempMap); +// } +//// return rawCallValuePerDataTypePerAttribute; +// } +// } +// +// +//} diff --git a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/est/MySQLESTDAO.java b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/est/MySQLESTDAO.java deleted file mode 100644 index 9862887d1..000000000 --- a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/est/MySQLESTDAO.java +++ /dev/null @@ -1,178 +0,0 @@ -package org.bgee.model.dao.mysql.expressiondata.rawdata.est; - -import java.math.BigDecimal; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Collection; -import java.util.Collections; -import java.util.EnumSet; -import java.util.Map.Entry; -import java.util.stream.Collectors; -import java.util.Set; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bgee.model.dao.api.exception.DAOException; -import org.bgee.model.dao.api.expressiondata.call.CallDAO.CallTO.DataState; -import org.bgee.model.dao.api.expressiondata.DAODataType; -import org.bgee.model.dao.api.expressiondata.rawdata.DAOProcessedRawDataFilter; -import org.bgee.model.dao.api.expressiondata.rawdata.DAORawDataFilter; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTLibraryDAO; -import org.bgee.model.dao.mysql.connector.BgeePreparedStatement; -import org.bgee.model.dao.mysql.connector.MySQLDAOManager; -import org.bgee.model.dao.mysql.connector.MySQLDAOResultSet; -import org.bgee.model.dao.mysql.exception.UnrecognizedColumnException; -import org.bgee.model.dao.mysql.expressiondata.rawdata.MySQLRawDataDAO; -import org.bgee.model.dao.mysql.expressiondata.rawdata.RawDataFiltersToDatabaseMapping; - -public class MySQLESTDAO extends MySQLRawDataDAO implements ESTDAO { - - /** - * {@code Logger} of the class. - */ - private final static Logger log = LogManager.getLogger(MySQLESTDAO.class.getName()); - public final static String TABLE_NAME = "expressedSequenceTag"; - - /** - * Constructor providing the {@code MySQLDAOManager} that this {@code MySQLDAO} - * will use to obtain {@code BgeeConnection}s. - * @param manager the {@code MySQLDAOManager} to use. - * @throws IllegalArgumentException If {@code manager} is {@code null}. - */ - public MySQLESTDAO(MySQLDAOManager manager) throws IllegalArgumentException { - super(manager); - } - - @Override - public ESTTOResultSet getESTs(Collection rawDataFilters, Long offset, Integer limit, - Collection attributes) throws DAOException { - log.traceEntry("{}, {}, {}, {}", rawDataFilters, offset, limit, attributes); - checkOffsetAndLimit(offset, limit); - - //It is very ugly, but for performance reasons, we use two queries: - //one for identifying the internal assay IDs, the second one to retrieve the calls. - //It is because the optimizer completely fail at generating a correct query plan, - //we really tried hard to fix this - //(see https://dba.stackexchange.com/questions/320207/optimization-with-subquery-not-working-as-expected). - //This logic is managed in the method processFilterForCallTableAssayIds, - //which returns the appropriate DAOProcessedRawDataFilter to be used in this method. - final MySQLESTLibraryDAO assayDAO = new MySQLESTLibraryDAO(this.getManager()); - DAOProcessedRawDataFilter processedFilters = this.processFilterForCallTableAssayIds( - new DAOProcessedRawDataFilter(rawDataFilters), - (s) -> assayDAO.getESTLibraries(s, null, null, - Set.of(ESTLibraryDAO.Attribute.ID)) - .stream() - .map(to -> to.getId()) - .collect(Collectors.toSet()), - String.class, DAODataType.EST, null); - if (processedFilters == null) { - try { - return log.traceExit(new MySQLESTTOResultSet( - this.getManager().getConnection().prepareStatement( - "SELECT NULL FROM " + TABLE_NAME + " WHERE FALSE"))); - } catch (SQLException e) { - throw log.throwing(new DAOException(e)); - } - } - - final Set clonedAttrs = Collections - .unmodifiableSet(attributes == null || attributes.isEmpty()? - EnumSet.allOf(ESTDAO.Attribute.class): EnumSet.copyOf(attributes)); - - StringBuilder sb = new StringBuilder(); - - // generate SELECT - sb.append(generateSelectClauseRawDataFilters(processedFilters, TABLE_NAME, - getColToAttributesMap(ESTDAO.Attribute.class), true, clonedAttrs)); - - // generate FROM - RawDataFiltersToDatabaseMapping filtersToDatabaseMapping = generateFromClauseRawData(sb, - processedFilters, null, Set.of(TABLE_NAME), DAODataType.EST); - - // generate WHERE CLAUSE - // usedInPropagatedCalls is not used to generate EST queries. If usedInPropagatedCalls is the only - // rawDataFilter not empty the query will not contain any where clause. That's why we check - // that any rawDataFilter variable except usedInPropagatedCalls is not empty. - boolean requireWhereClause = rawDataFilters.stream() - .allMatch(item -> item.usedInPropagatedCallsIsTheOnlyPotentialNotBlank()) ? false : true; - if(requireWhereClause) { - sb.append(" WHERE ").append(generateWhereClauseRawDataFilter(processedFilters, - filtersToDatabaseMapping)); - } - // generate ORDER BY - sb.append(" ORDER BY") - .append(" " + TABLE_NAME + "." + ESTDAO.Attribute.EST_LIBRARY_ID - .getTOFieldName()) - .append(", " + TABLE_NAME + "." + ESTDAO.Attribute.EST_ID.getTOFieldName()); - - //generate offset and limit - if (limit != null) { - sb.append(offset == null ? " LIMIT ?": " LIMIT ?, ?"); - } - try { - BgeePreparedStatement stmt = this.parameterizeQuery(sb.toString(), processedFilters, - DAODataType.EST, offset, limit); - return log.traceExit(new MySQLESTTOResultSet(stmt)); - } catch (SQLException e) { - throw log.throwing(new DAOException(e)); - } - } - - class MySQLESTTOResultSet extends MySQLDAOResultSet - implements ESTTOResultSet{ - - /** - * @param statement The {@code BgeePreparedStatement} - */ - private MySQLESTTOResultSet(BgeePreparedStatement statement) { - super(statement); - } - - @Override - protected ESTDAO.ESTTO getNewTO() throws DAOException { - log.traceEntry(); - try { - final ResultSet currentResultSet = this.getCurrentResultSet(); - BigDecimal pValue = null; - Integer bgeeGeneId = null; - Long expressionid = null; - String estId1 = null, estId2 = null, estLibraryId = null, uniGeneClusterId = null, - estData = null; - - for (Entry column : this.getColumnLabels().entrySet()) { - if (column.getValue().equals(ESTDAO.Attribute.EST_ID.getTOFieldName())) { - estId1 = currentResultSet.getString(column.getKey()); - } else if(column.getValue().equals(ESTDAO.Attribute.EST_ID2 - .getTOFieldName())) { - estId2 = currentResultSet.getString(column.getKey()); - } else if(column.getValue().equals(ESTDAO.Attribute.EST_LIBRARY_ID - .getTOFieldName())) { - estLibraryId = currentResultSet.getString(column.getKey()); - } else if(column.getValue().equals(ESTDAO.Attribute.BGEE_GENE_ID - .getTOFieldName())) { - bgeeGeneId = currentResultSet.getInt(column.getKey()); - } else if(column.getValue().equals(ESTDAO.Attribute.UNIGENE_CLUSTER_ID - .getTOFieldName())) { - uniGeneClusterId = currentResultSet.getString(column.getKey()); - } else if(column.getValue().equals(ESTDAO.Attribute.EXPRESSION_ID - .getTOFieldName())) { - expressionid = currentResultSet.getLong(column.getKey()); - } else if(column.getValue().equals(ESTDAO.Attribute.PVALUE - .getTOFieldName())) { - pValue = currentResultSet.getBigDecimal(column.getKey()); - } else if(column.getValue().equals(ESTDAO.Attribute.EST_DATA - .getTOFieldName())) { - estData = currentResultSet.getString(column.getKey()); - } else { - log.throwing(new UnrecognizedColumnException(column.getValue())); - } - } - return log.traceExit(new ESTTO(estId1, estId2, estLibraryId, uniGeneClusterId, - bgeeGeneId, DataState.convertToDataState(estData), pValue, expressionid)); - } catch (SQLException e) { - throw log.throwing(new DAOException(e)); - } - } - } -} diff --git a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/est/MySQLESTLibraryDAO.java b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/est/MySQLESTLibraryDAO.java deleted file mode 100644 index 4ed2ef456..000000000 --- a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/est/MySQLESTLibraryDAO.java +++ /dev/null @@ -1,131 +0,0 @@ -package org.bgee.model.dao.mysql.expressiondata.rawdata.est; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Collection; -import java.util.Collections; -import java.util.EnumSet; -import java.util.Map.Entry; -import java.util.Set; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bgee.model.dao.api.exception.DAOException; -import org.bgee.model.dao.api.expressiondata.DAODataType; -import org.bgee.model.dao.api.expressiondata.rawdata.DAOProcessedRawDataFilter; -import org.bgee.model.dao.api.expressiondata.rawdata.DAORawDataFilter; -import org.bgee.model.dao.api.expressiondata.rawdata.est.ESTLibraryDAO; -import org.bgee.model.dao.mysql.connector.BgeePreparedStatement; -import org.bgee.model.dao.mysql.connector.MySQLDAOManager; -import org.bgee.model.dao.mysql.connector.MySQLDAOResultSet; -import org.bgee.model.dao.mysql.exception.UnrecognizedColumnException; -import org.bgee.model.dao.mysql.expressiondata.rawdata.MySQLRawDataDAO; -import org.bgee.model.dao.mysql.expressiondata.rawdata.RawDataFiltersToDatabaseMapping; - -public class MySQLESTLibraryDAO extends MySQLRawDataDAO - implements ESTLibraryDAO{ - - private final static Logger log = LogManager.getLogger(MySQLESTLibraryDAO.class.getName()); - public final static String TABLE_NAME = "estLibrary"; - - public MySQLESTLibraryDAO(MySQLDAOManager manager) throws IllegalArgumentException { - super(manager); - } - - @Override - public ESTLibraryTOResultSet getESTLibraries(Collection rawDataFilters, - Long offset, Integer limit, Collection attrs) - throws DAOException { - log.traceEntry("{}, {}, {}, {}", rawDataFilters, offset, limit, attrs); - checkOffsetAndLimit(offset, limit); - - final DAOProcessedRawDataFilter processedFilters = - new DAOProcessedRawDataFilter<>(rawDataFilters); - final Set clonedAttrs = Collections - .unmodifiableSet(attrs == null || attrs.isEmpty()? - EnumSet.allOf(ESTLibraryDAO.Attribute.class): EnumSet.copyOf(attrs)); - - StringBuilder sb = new StringBuilder(); - - // generate SELECT - sb.append(generateSelectClauseRawDataFilters(processedFilters, TABLE_NAME, - getColToAttributesMap(ESTLibraryDAO.Attribute.class), true, clonedAttrs)); - - // generate FROM - RawDataFiltersToDatabaseMapping filtersToDatabaseMapping = generateFromClauseRawData(sb, - processedFilters, null, Set.of(TABLE_NAME), DAODataType.EST); - - // generate WHERE CLAUSE - // usedInPropagatedCalls is not used to generate EST queries. If usedInPropagatedCalls is the only - // rawDataFilter not empty the query will not contain any where clause. That's why we check - // that any rawDataFilter variable except usedInPropagatedCalls is not empty. - boolean requireWhereClause = rawDataFilters.stream() - .allMatch(item -> item.usedInPropagatedCallsIsTheOnlyPotentialNotBlank()) ? false : true; - if(requireWhereClause) { - sb.append(" WHERE ").append(generateWhereClauseRawDataFilter(processedFilters, - filtersToDatabaseMapping)); - } - - // generate ORDER BY - sb.append(" ORDER BY") - .append(" " + TABLE_NAME + "." + ESTLibraryDAO.Attribute.ID - .getTOFieldName()); - - //generate offset and limit - if (limit != null) { - sb.append(offset == null ? " LIMIT ?": " LIMIT ?, ?"); - } - try { - BgeePreparedStatement stmt = this.parameterizeQuery(sb.toString(), processedFilters, - DAODataType.EST, offset, limit); - return log.traceExit(new MySQLESTLibraryTOResultSet(stmt)); - } catch (SQLException e) { - throw log.throwing(new DAOException(e)); - } - } - - class MySQLESTLibraryTOResultSet extends MySQLDAOResultSet - implements ESTLibraryTOResultSet{ - - /** - * @param statement The {@code BgeePreparedStatement} - */ - private MySQLESTLibraryTOResultSet(BgeePreparedStatement statement) { - super(statement); - } - - @Override - protected ESTLibraryDAO.ESTLibraryTO getNewTO() throws DAOException { - log.traceEntry(); - try { - final ResultSet currentResultSet = this.getCurrentResultSet(); - Integer dataSourceid = null, conditionId = null; - String estLibraryId = null, estLibraryName = null, estLibraryDescription = null; - - for (Entry column : this.getColumnLabels().entrySet()) { - if (column.getValue().equals(ESTLibraryDAO.Attribute.ID.getTOFieldName())) { - estLibraryId = currentResultSet.getString(column.getKey()); - } else if(column.getValue().equals(ESTLibraryDAO.Attribute.NAME - .getTOFieldName())) { - estLibraryName = currentResultSet.getString(column.getKey()); - } else if(column.getValue().equals(ESTLibraryDAO.Attribute.DESCRIPTION - .getTOFieldName())) { - estLibraryDescription = currentResultSet.getString(column.getKey()); - } else if(column.getValue().equals(ESTLibraryDAO.Attribute.DATA_SOURCE_ID - .getTOFieldName())) { - dataSourceid = currentResultSet.getInt(column.getKey()); - } else if(column.getValue().equals(ESTLibraryDAO.Attribute.CONDITION_ID - .getTOFieldName())) { - conditionId = currentResultSet.getInt(column.getKey()); - } else { - log.throwing(new UnrecognizedColumnException(column.getValue())); - } - } - return log.traceExit(new ESTLibraryTO(estLibraryId, estLibraryName, - estLibraryDescription, dataSourceid, conditionId)); - } catch (SQLException e) { - throw log.throwing(new DAOException(e)); - } - } - } -} diff --git a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/microarray/MySQLAffymetrixChipDAO.java b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/microarray/MySQLAffymetrixChipDAO.java deleted file mode 100644 index cc4873c71..000000000 --- a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/microarray/MySQLAffymetrixChipDAO.java +++ /dev/null @@ -1,198 +0,0 @@ -package org.bgee.model.dao.mysql.expressiondata.rawdata.microarray; - -import java.math.BigDecimal; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Collection; -import java.util.Collections; -import java.util.EnumSet; -import java.util.Map.Entry; -import java.util.Set; -import java.util.stream.Collectors; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bgee.model.dao.api.exception.DAOException; -import org.bgee.model.dao.api.expressiondata.DAODataType; -import org.bgee.model.dao.api.expressiondata.rawdata.DAOProcessedRawDataFilter; -import org.bgee.model.dao.api.expressiondata.rawdata.DAORawDataFilter; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipDAO.AffymetrixChipTO.DetectionType; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipDAO.AffymetrixChipTO.NormalizationType; -import org.bgee.model.dao.mysql.connector.BgeePreparedStatement; -import org.bgee.model.dao.mysql.connector.MySQLDAOManager; -import org.bgee.model.dao.mysql.connector.MySQLDAOResultSet; -import org.bgee.model.dao.mysql.exception.UnrecognizedColumnException; -import org.bgee.model.dao.mysql.expressiondata.rawdata.MySQLRawDataDAO; -import org.bgee.model.dao.mysql.expressiondata.rawdata.RawDataFiltersToDatabaseMapping; - -public class MySQLAffymetrixChipDAO extends MySQLRawDataDAO - implements AffymetrixChipDAO{ - - private static final Logger log = LogManager.getLogger(MySQLAffymetrixChipDAO.class.getName()); - public static final String TABLE_NAME = "affymetrixChip"; - - public MySQLAffymetrixChipDAO(MySQLDAOManager manager) throws IllegalArgumentException { - super(manager); - } - - @Override - public AffymetrixChipTOResultSet getAffymetrixChips(Collection rawDataFilters, - Long offset, Integer limit, Collection attrs) - throws DAOException { - log.traceEntry("{}, {}, {}, {}", rawDataFilters, offset, limit, attrs); - checkOffsetAndLimit(offset, limit); - - final DAOProcessedRawDataFilter processedFilters = - new DAOProcessedRawDataFilter<>(rawDataFilters); - final Set clonedAttrs = Collections - .unmodifiableSet(attrs == null || attrs.isEmpty()? - EnumSet.allOf(AffymetrixChipDAO.Attribute.class): EnumSet.copyOf(attrs)); - - StringBuilder sb = new StringBuilder(); - - // generate SELECT - sb.append(generateSelectClauseRawDataFilters(processedFilters, TABLE_NAME, - getColToAttributesMap(AffymetrixChipDAO.Attribute.class), true, clonedAttrs)); - - // generate FROM - RawDataFiltersToDatabaseMapping filtersToDatabaseMapping = generateFromClauseRawData(sb, - processedFilters, null, Set.of(TABLE_NAME), DAODataType.AFFYMETRIX); - - // generate WHERE CLAUSE - // usedInPropagatedCalls is not used to generate affymetrix queries. If usedInPropagatedCalls is the only - // rawDataFilter not empty the query will not contain any where clause. That's why we check - // that any rawDataFilter variable except usedInPropagatedCalls is not empty. - boolean requireWhereClause = rawDataFilters.stream() - .allMatch(item -> item.usedInPropagatedCallsIsTheOnlyPotentialNotBlank()) ? false : true; - if(requireWhereClause) { - sb.append(" WHERE ").append(generateWhereClauseRawDataFilter(processedFilters, - filtersToDatabaseMapping)); - } - - // generate ORDER BY - sb.append(" ORDER BY") - .append(" " + TABLE_NAME + "." + AffymetrixChipDAO.Attribute.EXPERIMENT_ID - .getTOFieldName()) - .append(", " + TABLE_NAME + "." + AffymetrixChipDAO.Attribute.BGEE_AFFYMETRIX_CHIP_ID - .getTOFieldName()); - - //generate offset and limit - if (limit != null) { - sb.append(offset == null ? " LIMIT ?": " LIMIT ?, ?"); - } - try { - BgeePreparedStatement stmt = this.parameterizeQuery(sb.toString(), processedFilters, - DAODataType.AFFYMETRIX, offset, limit); - return log.traceExit(new MySQLAffymetrixChipTOResultSet(stmt)); - } catch (SQLException e) { - throw log.throwing(new DAOException(e)); - } - } - - @Override - public AffymetrixChipTOResultSet getAffymetrixChipsFromBgeeChipIds( - Collection bgeeChipIds, Collection attrs) - throws DAOException { - log.traceEntry("{}, {}", bgeeChipIds, attrs); - if (bgeeChipIds == null || bgeeChipIds.isEmpty()) { - throw log.throwing(new IllegalArgumentException("need to provide at least one" - + "bgeeChipId")); - } - final Set clonedAttrs = Collections - .unmodifiableSet(attrs == null || attrs.isEmpty()? - EnumSet.allOf(AffymetrixChipDAO.Attribute.class): EnumSet.copyOf(attrs)); - final Set clonedBgeeChipIds = Collections.unmodifiableSet(bgeeChipIds.stream() - .filter(id -> id != null).collect(Collectors.toSet())); - // generate SELECT - StringBuilder sb = new StringBuilder(); - sb.append(generateSelectClause(TABLE_NAME, getColToAttributesMap(AffymetrixChipDAO - .Attribute.class), true, clonedAttrs)) - .append(" FROM ").append(TABLE_NAME).append(" WHERE ") - .append(AffymetrixChipDAO.Attribute.BGEE_AFFYMETRIX_CHIP_ID.getTOFieldName()) - .append(" IN (") - .append(BgeePreparedStatement.generateParameterizedQueryString(clonedBgeeChipIds.size())) - .append(")"); - try { - BgeePreparedStatement stmt = this.getManager().getConnection() - .prepareStatement(sb.toString()); - stmt.setIntegers(1, clonedBgeeChipIds, true); - return log.traceExit(new MySQLAffymetrixChipTOResultSet(stmt)); - } catch (SQLException e) { - throw log.throwing(new DAOException(e)); - } - } - - class MySQLAffymetrixChipTOResultSet extends MySQLDAOResultSet - implements AffymetrixChipTOResultSet{ - - /** - * @param statement The {@code BgeePreparedStatement} - */ - private MySQLAffymetrixChipTOResultSet(BgeePreparedStatement statement) { - super(statement); - } - - @Override - protected AffymetrixChipDAO.AffymetrixChipTO getNewTO() throws DAOException { - log.traceEntry(); - try { - final ResultSet currentResultSet = this.getCurrentResultSet(); - Integer bgeeAffymetrixChipId = null, conditionId = null, chipDistinctRankCount = null; - String affymetrixChipId = null, microarrayExperimentId = null, chipTypeId = null, - scanDate = null, normalizationType = null, detectionType = null; - BigDecimal qualityScore = null, percentPresent = null, chipMaxRank = null; - - for (Entry column : this.getColumnLabels().entrySet()) { - if (column.getValue().equals(AffymetrixChipDAO.Attribute.BGEE_AFFYMETRIX_CHIP_ID - .getTOFieldName())) { - bgeeAffymetrixChipId = currentResultSet.getInt(column.getKey()); - } else if(column.getValue().equals(AffymetrixChipDAO.Attribute.AFFYMETRIX_CHIP_ID - .getTOFieldName())) { - affymetrixChipId = currentResultSet.getString(column.getKey()); - } else if(column.getValue().equals(AffymetrixChipDAO.Attribute.EXPERIMENT_ID - .getTOFieldName())) { - microarrayExperimentId = currentResultSet.getString(column.getKey()); - } else if(column.getValue().equals(AffymetrixChipDAO.Attribute.CHIP_TYPE_ID - .getTOFieldName())) { - chipTypeId = currentResultSet.getString(column.getKey()); - } else if(column.getValue().equals(AffymetrixChipDAO.Attribute.SCAN_DATE - .getTOFieldName())) { - scanDate = currentResultSet.getString(column.getKey()); - } else if(column.getValue().equals(AffymetrixChipDAO.Attribute.NORMALIZATION_TYPE - .getTOFieldName())) { - normalizationType = currentResultSet.getString(column.getKey()); - } else if(column.getValue().equals(AffymetrixChipDAO.Attribute.DETECTION_TYPE - .getTOFieldName())) { - detectionType = currentResultSet.getString(column.getKey()); - } else if(column.getValue().equals(AffymetrixChipDAO.Attribute.CONDITION_ID - .getTOFieldName())) { - conditionId = currentResultSet.getInt(column.getKey()); - } else if(column.getValue().equals(AffymetrixChipDAO.Attribute.QUALITY_SCORE - .getTOFieldName())) { - qualityScore = currentResultSet.getBigDecimal(column.getKey()); - } else if(column.getValue().equals(AffymetrixChipDAO.Attribute.PERCENT_PRESENT - .getTOFieldName())) { - percentPresent = currentResultSet.getBigDecimal(column.getKey()); - } else if(column.getValue().equals(AffymetrixChipDAO.Attribute.MAX_RANK - .getTOFieldName())) { - chipMaxRank = currentResultSet.getBigDecimal(column.getKey()); - } else if(column.getValue().equals(AffymetrixChipDAO.Attribute.DISTINCT_RANK_COUNT - .getTOFieldName())) { - chipDistinctRankCount = currentResultSet.getInt(column.getKey()); - } else { - log.throwing(new UnrecognizedColumnException(column.getValue())); - } - } - return log.traceExit(new AffymetrixChipTO(bgeeAffymetrixChipId, affymetrixChipId, - microarrayExperimentId, chipTypeId, scanDate, - NormalizationType.convertToNormalizationType(normalizationType), - DetectionType.convertToDetectionType(detectionType), conditionId, qualityScore, - percentPresent, chipMaxRank, chipDistinctRankCount)); - } catch (SQLException e) { - throw log.throwing(new DAOException(e)); - } - } - } - -} diff --git a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/microarray/MySQLAffymetrixChipTypeDAO.java b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/microarray/MySQLAffymetrixChipTypeDAO.java deleted file mode 100644 index 52c2c45a6..000000000 --- a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/microarray/MySQLAffymetrixChipTypeDAO.java +++ /dev/null @@ -1,126 +0,0 @@ -package org.bgee.model.dao.mysql.expressiondata.rawdata.microarray; - -import java.math.BigDecimal; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Collection; -import java.util.Collections; -import java.util.EnumSet; -import java.util.HashSet; -import java.util.Map.Entry; -import java.util.Set; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bgee.model.dao.api.exception.DAOException; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipTypeDAO; -import org.bgee.model.dao.mysql.connector.BgeePreparedStatement; -import org.bgee.model.dao.mysql.connector.MySQLDAOManager; -import org.bgee.model.dao.mysql.connector.MySQLDAOResultSet; -import org.bgee.model.dao.mysql.exception.UnrecognizedColumnException; -import org.bgee.model.dao.mysql.expressiondata.rawdata.MySQLRawDataDAO; - -public class MySQLAffymetrixChipTypeDAO extends MySQLRawDataDAO - implements AffymetrixChipTypeDAO{ - -public MySQLAffymetrixChipTypeDAO(MySQLDAOManager manager) throws IllegalArgumentException { - super(manager); - } - - private static final Logger log = LogManager.getLogger(MySQLAffymetrixChipTypeDAO.class.getName()); - public static final String TABLE_NAME = "chipType"; - - @Override - public MySQLAffymetrixChipTypeTOResultSet getAffymetrixChipTypes(Collection chipTypeIds, - Collection attributes) - throws DAOException { - log.traceEntry("{}, {}", chipTypeIds, attributes); - - final Set clonedChipTypeIds = Collections.unmodifiableSet( - chipTypeIds == null? - new HashSet<>(): new HashSet<>(chipTypeIds)); - final Set attrs = Collections.unmodifiableSet( - attributes == null || attributes.isEmpty()? - EnumSet.allOf(AffymetrixChipTypeDAO.Attribute.class): - EnumSet.copyOf(attributes)); - - // generate SELECT clause - StringBuilder sb = new StringBuilder(); - sb.append(generateSelectClause(TABLE_NAME, getColToAttributesMap(AffymetrixChipTypeDAO - .Attribute.class), true, attrs)) - .append(" FROM ").append(TABLE_NAME); - - // generate WHERE CLAUSE - if (!clonedChipTypeIds.isEmpty()) { - sb.append(" WHERE ") - .append(AffymetrixChipTypeDAO.Attribute.CHIP_TYPE_ID.getTOFieldName()) - .append(" IN (") - .append(BgeePreparedStatement.generateParameterizedQueryString(clonedChipTypeIds.size())) - .append(")"); - } - //parameterize query - try { - BgeePreparedStatement stmt = this.getManager().getConnection() - .prepareStatement(sb.toString()); - stmt.setStrings(1, clonedChipTypeIds, true); - return log.traceExit(new MySQLAffymetrixChipTypeTOResultSet(stmt)); - } catch (SQLException e) { - throw log.throwing(new DAOException(e)); - } - } - - class MySQLAffymetrixChipTypeTOResultSet extends MySQLDAOResultSet - implements AffymetrixChipTypeTOResultSet{ - - /** - * @param statement The {@code BgeePreparedStatement} - */ - private MySQLAffymetrixChipTypeTOResultSet(BgeePreparedStatement statement) { - super(statement); - } - - @Override - protected AffymetrixChipTypeDAO.AffymetrixChipTypeTO getNewTO() throws DAOException { - log.traceEntry(); - try { - final ResultSet currentResultSet = this.getCurrentResultSet(); - Boolean isCompatible = null; - String affymetrixChipTypeId = null, affymetrixChipTypeName = null, cdfName = null; - BigDecimal qualityScoreThreshold = null, percentPresentThreshold = null, - chipTypeMaxRank = null; - - for (Entry column : this.getColumnLabels().entrySet()) { - if (column.getValue().equals(AffymetrixChipTypeDAO.Attribute.CHIP_TYPE_ID - .getTOFieldName())) { - affymetrixChipTypeId = currentResultSet.getString(column.getKey()); - } else if(column.getValue().equals(AffymetrixChipTypeDAO.Attribute.CHIP_TYPE_NAME - .getTOFieldName())) { - affymetrixChipTypeName = currentResultSet.getString(column.getKey()); - } else if(column.getValue().equals(AffymetrixChipTypeDAO.Attribute.CDF_NAME - .getTOFieldName())) { - cdfName = currentResultSet.getString(column.getKey()); - } else if(column.getValue().equals(AffymetrixChipTypeDAO.Attribute.IS_COMPATIBLE - .getTOFieldName())) { - isCompatible = currentResultSet.getBoolean(column.getKey()); - } else if(column.getValue().equals(AffymetrixChipTypeDAO.Attribute - .QUALITY_SCORE_THRESHOLD.getTOFieldName())) { - qualityScoreThreshold = currentResultSet.getBigDecimal(column.getKey()); - } else if(column.getValue().equals(AffymetrixChipTypeDAO.Attribute - .PERCENT_PRESENT_THRESHOLD.getTOFieldName())) { - percentPresentThreshold = currentResultSet.getBigDecimal(column.getKey()); - } else if(column.getValue().equals(AffymetrixChipTypeDAO.Attribute.CHIP_TYPE_MAX_RANK - .getTOFieldName())) { - chipTypeMaxRank = currentResultSet.getBigDecimal(column.getKey()); - } else { - log.throwing(new UnrecognizedColumnException(column.getValue())); - } - } - return log.traceExit(new AffymetrixChipTypeTO(affymetrixChipTypeId, affymetrixChipTypeName, - cdfName, isCompatible, qualityScoreThreshold, percentPresentThreshold, - chipTypeMaxRank)); - } catch (SQLException e) { - throw log.throwing(new DAOException(e)); - } - } - } -} diff --git a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/microarray/MySQLAffymetrixProbesetDAO.java b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/microarray/MySQLAffymetrixProbesetDAO.java deleted file mode 100644 index 9622502a7..000000000 --- a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/microarray/MySQLAffymetrixProbesetDAO.java +++ /dev/null @@ -1,199 +0,0 @@ -package org.bgee.model.dao.mysql.expressiondata.rawdata.microarray; - -import java.math.BigDecimal; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Collection; -import java.util.Collections; -import java.util.EnumSet; -import java.util.Map.Entry; -import java.util.stream.Collectors; -import java.util.Set; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bgee.model.dao.api.exception.DAOException; -import org.bgee.model.dao.api.expressiondata.call.CallDAO.CallTO.DataState; -import org.bgee.model.dao.api.expressiondata.DAODataType; -import org.bgee.model.dao.api.expressiondata.rawdata.DAOProcessedRawDataFilter; -import org.bgee.model.dao.api.expressiondata.rawdata.DAORawDataFilter; -import org.bgee.model.dao.api.expressiondata.rawdata.RawDataCallSourceDAO.CallSourceDataTO.ExclusionReason; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixChipDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.AffymetrixProbesetDAO; -import org.bgee.model.dao.mysql.connector.BgeePreparedStatement; -import org.bgee.model.dao.mysql.connector.MySQLDAOManager; -import org.bgee.model.dao.mysql.connector.MySQLDAOResultSet; -import org.bgee.model.dao.mysql.exception.UnrecognizedColumnException; -import org.bgee.model.dao.mysql.expressiondata.rawdata.MySQLRawDataDAO; -import org.bgee.model.dao.mysql.expressiondata.rawdata.RawDataFiltersToDatabaseMapping; - - -public class MySQLAffymetrixProbesetDAO extends MySQLRawDataDAO - implements AffymetrixProbesetDAO { - - /** - * {@code Logger} of the class. - */ - private final static Logger log = LogManager.getLogger(MySQLAffymetrixProbesetDAO.class.getName()); - public final static String TABLE_NAME = "affymetrixProbeset"; - - /** - * Constructor providing the {@code MySQLDAOManager} that this {@code MySQLDAO} - * will use to obtain {@code BgeeConnection}s. - * @param manager the {@code MySQLDAOManager} to use. - * @throws IllegalArgumentException If {@code manager} is {@code null}. - */ - public MySQLAffymetrixProbesetDAO(MySQLDAOManager manager) { - super(manager); - } - - @Override - public AffymetrixProbesetTOResultSet getAffymetrixProbesets( - Collection rawDataFilters, Long offset, Integer limit, - Collection attrs) throws DAOException { - log.traceEntry("{}, {}, {}, {}", rawDataFilters, offset, limit, attrs); - checkOffsetAndLimit(offset, limit); - - //It is very ugly, but for performance reasons, we use two queries: - //one for identifying the internal assay IDs, the second one to retrieve the calls. - //It is because the optimizer completely fail at generating a correct query plan, - //we really tried hard to fix this - //(see https://dba.stackexchange.com/questions/320207/optimization-with-subquery-not-working-as-expected). - //This logic is managed in the method processFilterForCallTableAssayIds, - //which returns the appropriate DAOProcessedRawDataFilter to be used in this method. - final MySQLAffymetrixChipDAO assayDAO = new MySQLAffymetrixChipDAO(this.getManager()); - DAOProcessedRawDataFilter processedFilters = this.processFilterForCallTableAssayIds( - new DAOProcessedRawDataFilter(rawDataFilters), - (s) -> assayDAO.getAffymetrixChips(s, null, null, - Set.of(AffymetrixChipDAO.Attribute.BGEE_AFFYMETRIX_CHIP_ID)) - .stream() - .map(to -> to.getId()) - . collect(Collectors.toSet()), - Integer.class, DAODataType.AFFYMETRIX, null); - if (processedFilters == null) { - try { - return log.traceExit(new MySQLAffymetrixProbesetTOResultSet( - this.getManager().getConnection().prepareStatement( - "SELECT NULL FROM " + TABLE_NAME + " WHERE FALSE"))); - } catch (SQLException e) { - throw log.throwing(new DAOException(e)); - } - } - - final Set clonedAttrs = Collections - .unmodifiableSet(attrs == null || attrs.isEmpty()? - EnumSet.allOf(AffymetrixProbesetDAO.Attribute.class): EnumSet.copyOf(attrs)); - - StringBuilder sb = new StringBuilder(); - - // generate SELECT - sb.append(generateSelectClauseRawDataFilters(processedFilters, TABLE_NAME, - getColToAttributesMap(AffymetrixProbesetDAO.Attribute.class), false, clonedAttrs)); - - // generate FROM - RawDataFiltersToDatabaseMapping filtersToDatabaseMapping = generateFromClauseRawData(sb, - processedFilters, null, Set.of(TABLE_NAME), DAODataType.AFFYMETRIX); - - // generate WHERE - // usedInPropagatedCalls is not used to generate affymetrix queries. If usedInPropagatedCalls is the only - // rawDataFilter not empty the query will not contain any where clause. That's why we check - // that any rawDataFilter variable except usedInPropagatedCalls is not empty. - boolean requireWhereClause = rawDataFilters.stream() - .allMatch(item -> item.usedInPropagatedCallsIsTheOnlyPotentialNotBlank()) ? false : true; - if(requireWhereClause) { - sb.append(" WHERE ") - .append(generateWhereClauseRawDataFilter(processedFilters, filtersToDatabaseMapping)); - } - - // generate ORDER BY - sb.append(" ORDER BY") - .append(" ").append(TABLE_NAME).append(".") - .append(AffymetrixProbesetDAO.Attribute.BGEE_AFFYMETRIX_CHIP_ID.getTOFieldName()) - .append(", ").append(TABLE_NAME).append(".") - .append(AffymetrixProbesetDAO.Attribute.ID.getTOFieldName()) - .append(", ").append(TABLE_NAME).append(".") - .append(AffymetrixProbesetDAO.Attribute.BGEE_GENE_ID.getTOFieldName()); - - //generate offset and limit - if (limit != null) { - sb.append(offset == null ? " LIMIT ?": " LIMIT ?, ?"); - } - try { - BgeePreparedStatement stmt = this.parameterizeQuery(sb.toString(), processedFilters, - DAODataType.AFFYMETRIX, offset, limit); - return log.traceExit(new MySQLAffymetrixProbesetTOResultSet(stmt)); - } catch (SQLException e) { - throw log.throwing(new DAOException(e)); - } - } - - class MySQLAffymetrixProbesetTOResultSet extends MySQLDAOResultSet - implements AffymetrixProbesetTOResultSet{ - - /** - * @param statement The {@code BgeePreparedStatement} - */ - private MySQLAffymetrixProbesetTOResultSet(BgeePreparedStatement statement) { - super(statement); - } - - @Override - protected AffymetrixProbesetDAO.AffymetrixProbesetTO getNewTO() throws DAOException { - log.traceEntry(); - try { - final ResultSet currentResultSet = this.getCurrentResultSet(); - Integer bgeeAffymetrixChipId = null, bgeeGeneId = null; - String affymetrixProbesetId = null, affymetrixData = null, reasonForExclusion = null; - Long expressionId = null; - BigDecimal normalizedSignalIntensity = null, pValue = null, qValue = null, rank = null; - - for (Entry column : this.getColumnLabels().entrySet()) { - if (column.getValue().equals(AffymetrixProbesetDAO.Attribute.ID - .getTOFieldName())) { - affymetrixProbesetId = currentResultSet.getString(column.getKey()); - } else if(column.getValue().equals(AffymetrixProbesetDAO.Attribute - .BGEE_AFFYMETRIX_CHIP_ID.getTOFieldName())) { - bgeeAffymetrixChipId = currentResultSet.getInt(column.getKey()); - } else if(column.getValue().equals(AffymetrixProbesetDAO.Attribute.BGEE_GENE_ID - .getTOFieldName())) { - bgeeGeneId = currentResultSet.getInt(column.getKey()); - } else if(column.getValue().equals(AffymetrixProbesetDAO.Attribute - .NORMALIZED_SIGNAL_INTENSITY.getTOFieldName())) { - normalizedSignalIntensity = currentResultSet.getBigDecimal(column.getKey()); - } else if(column.getValue().equals(AffymetrixProbesetDAO.Attribute.PVALUE - .getTOFieldName())) { - pValue = currentResultSet.getBigDecimal(column.getKey()); - } else if(column.getValue().equals(AffymetrixProbesetDAO.Attribute.QVALUE - .getTOFieldName())) { - qValue = currentResultSet.getBigDecimal(column.getKey()); - } else if(column.getValue().equals(AffymetrixProbesetDAO.Attribute.EXPRESSION_ID - .getTOFieldName())) { - expressionId = currentResultSet.getLong(column.getKey()); - } else if(column.getValue().equals(AffymetrixProbesetDAO.Attribute.RANK - .getTOFieldName())) { - rank = currentResultSet.getBigDecimal(column.getKey()); - } else if(column.getValue().equals(AffymetrixProbesetDAO.Attribute.AFFYMETRIX_DATA - .getTOFieldName())) { - affymetrixData = currentResultSet.getString(column.getKey()); - } else if(column.getValue().equals(AffymetrixProbesetDAO.Attribute - .REASON_FOR_EXCLUSION.getTOFieldName())) { - reasonForExclusion = currentResultSet.getString(column.getKey()); - } else if (column.getValue().equals(AffymetrixProbesetDAO.Attribute - .RAW_DETECTION_FLAG.getTOFieldName())) { - //TODO the database schema still contain the column rawDetectionFlag that is not - // used and should be removed. Remove this condition when the schema is updated - } - else { - log.throwing(new UnrecognizedColumnException(column.getValue())); - } - } - return log.traceExit(new AffymetrixProbesetTO(affymetrixProbesetId, bgeeAffymetrixChipId, - bgeeGeneId, normalizedSignalIntensity, pValue, qValue, expressionId, rank, - DataState.convertToDataState(affymetrixData), - ExclusionReason.convertToExclusionReason(reasonForExclusion))); - } catch (SQLException e) { - throw log.throwing(new DAOException(e)); - } - } - } -} diff --git a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/microarray/MySQLMicroarrayExperimentDAO.java b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/microarray/MySQLMicroarrayExperimentDAO.java deleted file mode 100644 index 1bf0b9957..000000000 --- a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/microarray/MySQLMicroarrayExperimentDAO.java +++ /dev/null @@ -1,129 +0,0 @@ -package org.bgee.model.dao.mysql.expressiondata.rawdata.microarray; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Collection; -import java.util.Collections; -import java.util.EnumSet; -import java.util.Map.Entry; -import java.util.Set; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bgee.model.dao.api.exception.DAOException; -import org.bgee.model.dao.api.expressiondata.DAODataType; -import org.bgee.model.dao.api.expressiondata.rawdata.DAOProcessedRawDataFilter; -import org.bgee.model.dao.api.expressiondata.rawdata.DAORawDataFilter; -import org.bgee.model.dao.api.expressiondata.rawdata.microarray.MicroarrayExperimentDAO; -import org.bgee.model.dao.mysql.connector.BgeePreparedStatement; -import org.bgee.model.dao.mysql.connector.MySQLDAOManager; -import org.bgee.model.dao.mysql.connector.MySQLDAOResultSet; -import org.bgee.model.dao.mysql.exception.UnrecognizedColumnException; -import org.bgee.model.dao.mysql.expressiondata.rawdata.MySQLRawDataDAO; -import org.bgee.model.dao.mysql.expressiondata.rawdata.RawDataFiltersToDatabaseMapping; - -public class MySQLMicroarrayExperimentDAO extends MySQLRawDataDAO - implements MicroarrayExperimentDAO { - - private static final Logger log = LogManager.getLogger(MySQLMicroarrayExperimentDAO.class.getName()); - public static final String TABLE_NAME = "microarrayExperiment"; - - public MySQLMicroarrayExperimentDAO(MySQLDAOManager manager) throws IllegalArgumentException { - super(manager); - } - - @Override - public MicroarrayExperimentTOResultSet getExperiments(Collection rawDataFilters, - Long offset, Integer limit, Collection attrs) - throws DAOException { - log.traceEntry("{}, {}, {}, {}", rawDataFilters, offset, limit, attrs); - checkOffsetAndLimit(offset, limit); - - final DAOProcessedRawDataFilter processedFilters = - new DAOProcessedRawDataFilter<>(rawDataFilters); - final Set clonedAttrs = Collections - .unmodifiableSet(attrs == null || attrs.isEmpty()? - EnumSet.allOf(MicroarrayExperimentDAO.Attribute.class): EnumSet.copyOf(attrs)); - - StringBuilder sb = new StringBuilder(); - - // generate SELECT - sb.append(generateSelectClauseRawDataFilters(processedFilters, TABLE_NAME, - getColToAttributesMap(MicroarrayExperimentDAO.Attribute.class), true, - clonedAttrs)); - - //generate FROM clause - RawDataFiltersToDatabaseMapping filtersToDatabaseMapping = generateFromClauseRawData(sb, - processedFilters, null, Set.of(TABLE_NAME), DAODataType.AFFYMETRIX); - - // generate WHERE - // usedInPropagatedCalls is not used to generate affymetrix queries. If usedInPropagatedCalls is the only - // rawDataFilter not empty the query will not contain any where clause. That's why we check - // that any rawDataFilter variable except usedInPropagatedCalls is not empty. - boolean requireWhereClause = rawDataFilters.stream() - .allMatch(item -> item.usedInPropagatedCallsIsTheOnlyPotentialNotBlank()) ? false : true; - if(requireWhereClause) { - sb.append(" WHERE ").append(generateWhereClauseRawDataFilter(processedFilters, - filtersToDatabaseMapping)); - } - - // generate ORDER BY - sb.append(" ORDER BY") - .append(" " + TABLE_NAME + "." + MicroarrayExperimentDAO.Attribute.ID.getTOFieldName()); - - //generate offset and limit - if (limit != null) { - sb.append(offset == null ? " LIMIT ?": " LIMIT ?, ?"); - } - //add values to parameterized queries - try { - BgeePreparedStatement stmt = this.parameterizeQuery(sb.toString(), - processedFilters, DAODataType.AFFYMETRIX, offset, limit); - return log.traceExit(new MySQLMicroarrayExperimentTOResultSet(stmt)); - } catch (SQLException e) { - throw log.throwing(new DAOException(e)); - } - } - - class MySQLMicroarrayExperimentTOResultSet extends MySQLDAOResultSet - implements MicroarrayExperimentTOResultSet{ - - /** - * @param statement The {@code BgeePreparedStatement} - */ - private MySQLMicroarrayExperimentTOResultSet(BgeePreparedStatement statement) { - super(statement); - } - - @Override - protected MicroarrayExperimentDAO.MicroarrayExperimentTO getNewTO() throws DAOException { - log.traceEntry(); - try { - final ResultSet currentResultSet = this.getCurrentResultSet(); - Integer dataSourceId = null; - String id = null, name = null, description = null; - - for (Entry column : this.getColumnLabels().entrySet()) { - if (column.getValue().equals(MicroarrayExperimentDAO.Attribute.ID.getTOFieldName())) { - id = currentResultSet.getString(column.getKey()); - } else if(column.getValue().equals(MicroarrayExperimentDAO.Attribute.NAME - .getTOFieldName())) { - name = currentResultSet.getString(column.getKey()); - } else if(column.getValue().equals(MicroarrayExperimentDAO.Attribute.DESCRIPTION - .getTOFieldName())) { - description = currentResultSet.getString(column.getKey()); - } else if(column.getValue().equals(MicroarrayExperimentDAO.Attribute.DATA_SOURCE_ID - .getTOFieldName())) { - dataSourceId = currentResultSet.getInt(column.getKey()); - } else { - log.throwing(new UnrecognizedColumnException(column.getValue())); - } - } - return log.traceExit(new MicroarrayExperimentTO(id, name, description, dataSourceId)); - } catch (SQLException e) { - throw log.throwing(new DAOException(e)); - } - } - } - -} diff --git a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/rnaseq/MySQLRNASeqExperimentDAO.java b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/rnaseq/MySQLRNASeqExperimentDAO.java index ebe8f0bbd..326d9e771 100644 --- a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/rnaseq/MySQLRNASeqExperimentDAO.java +++ b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/expressiondata/rawdata/rnaseq/MySQLRNASeqExperimentDAO.java @@ -53,14 +53,23 @@ public RNASeqExperimentTOResultSet getExperiments(Collection r // add boolean isTargetBase to the SELECT clause //XXX in the future this boolean could be added to the DB - sb.append(", (CASE WHEN(SELECT distinct 1 from rnaSeqLibrary as t2 inner join rnaSeqLibraryAnnotatedSample " - + "as t3 on t2.rnaSeqLibraryId = t3.rnaSeqLibraryId where t2.rnaSeqExperimentId = rnaSeqExperiment.rnaSeqExperimentId " - + "and t3.multipleLibraryIndividualSample = 1) then 1 else 0 end) as isTargetBase "); + if (isSingleCell != null && isSingleCell) { + sb.append(", (CASE WHEN tb.rnaSeqExperimentId IS NOT NULL THEN 1 ELSE 0 END) AS isTargetBase "); + } // generate FROM RawDataFiltersToDatabaseMapping filtersToDatabaseMapping = generateFromClauseRawData(sb, processedFilters, isSingleCell, Set.of(TABLE_NAME), DAODataType.RNA_SEQ); + //left join clause used to quickly retrieve isTargetBased information + if (isSingleCell != null && isSingleCell) { + sb.append(" LEFT JOIN (") + .append(" SELECT DISTINCT l2.rnaSeqExperimentId ") + .append(" FROM rnaSeqLibrary AS l2 ") + .append(" INNER JOIN rnaSeqLibraryAnnotatedSample AS s ON l2.rnaSeqLibraryId = s.rnaSeqLibraryId ") + .append(" WHERE s.multipleLibraryIndividualSample = 1 ") + .append(") AS tb ON tb.rnaSeqExperimentId = rnaSeqExperiment.rnaSeqExperimentId"); + } // generate WHERE CLAUSE if (!processedFilters.getRawDataFilters().isEmpty() || isSingleCell != null) { diff --git a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/gene/MySQLGeneDAO.java b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/gene/MySQLGeneDAO.java index 429a011a8..fafe9b4d2 100644 --- a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/gene/MySQLGeneDAO.java +++ b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/gene/MySQLGeneDAO.java @@ -63,6 +63,7 @@ public class MySQLGeneDAO extends MySQLDAO implements GeneDAO columnToAttributesMap.put("speciesId", GeneDAO.Attribute.SPECIES_ID); columnToAttributesMap.put("geneBioTypeId", GeneDAO.Attribute.GENE_BIO_TYPE_ID); columnToAttributesMap.put("ensemblGene", GeneDAO.Attribute.ENSEMBL_GENE); + columnToAttributesMap.put("seqRegionName", GeneDAO.Attribute.SEQ_REGION_NAME); columnToAttributesMap.put("geneMappedToGeneIdCount", GeneDAO.Attribute.GENE_MAPPED_TO_SAME_GENE_ID_COUNT); columnToAttributesMap.put("expressionSummary", GeneDAO.Attribute.EXPRESSION_SUMMARY); } @@ -315,15 +316,6 @@ public int updateGenes(Collection genes, Collection a if (attributesToUpdate == null || attributesToUpdate.isEmpty()) { throw log.throwing(new IllegalArgumentException("No attribute is given, then no gene is updated")); } - // if - // (attributesToUpdate.contains(GeneDAO.Attribute.ANCESTRAL_OMA_NODE_ID) - // || - // attributesToUpdate.contains(GeneDAO.Attribute.ANCESTRAL_OMA_TAXON_ID)) - // { - // throw log.throwing(new IllegalArgumentException( - // "'Ancestral OMA' attributes are not store in database, then no gene - // is updated")); - // } int geneUpdatedCount = 0; // Construct sql query according to currents attributes @@ -411,8 +403,8 @@ private MySQLGeneTOResultSet(BgeePreparedStatement statement) { protected GeneTO getNewTO() { log.traceEntry(); String geneId = null, geneName = null, geneDescription = null, expressionSummary = null; - Integer id = null, speciesId = null, geneBioTypeId = null, OMAParentNodeId = null, - geneMappedToGeneIdCount = null; + String seqRegionName = null; + Integer id = null, speciesId = null, geneBioTypeId = null, geneMappedToGeneIdCount = null; Boolean ensemblGene = null; // Get results for (Entry column : this.getColumnLabels().entrySet()) { @@ -438,6 +430,9 @@ protected GeneTO getNewTO() { } else if (column.getValue().equals("ensemblGene")) { ensemblGene = this.getCurrentResultSet().getBoolean(column.getKey()); + } else if (column.getValue().equals("seqRegionName")) { + seqRegionName = this.getCurrentResultSet().getString(column.getKey()); + } else if (column.getValue().equals("geneMappedToGeneIdCount")) { geneMappedToGeneIdCount = this.getCurrentResultSet().getInt(column.getKey()); @@ -455,8 +450,8 @@ protected GeneTO getNewTO() { } } // Set GeneTO - return log.traceExit(new GeneTO(id, geneId, geneName, geneDescription, speciesId, geneBioTypeId, OMAParentNodeId, - ensemblGene, geneMappedToGeneIdCount, expressionSummary)); + return log.traceExit(new GeneTO(id, geneId, geneName, geneDescription, speciesId, geneBioTypeId, ensemblGene, + seqRegionName, geneMappedToGeneIdCount, expressionSummary)); } } diff --git a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/species/MySQLSpeciesDAO.java b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/species/MySQLSpeciesDAO.java index 8e50b1deb..06bfa304f 100644 --- a/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/species/MySQLSpeciesDAO.java +++ b/bgee-dao-sql/src/main/java/org/bgee/model/dao/mysql/species/MySQLSpeciesDAO.java @@ -38,16 +38,17 @@ public class MySQLSpeciesDAO extends MySQLDAO implements S static { COL_TO_ATTR_MAP = new HashMap<>(); COL_TO_ATTR_MAP.put("speciesId", SpeciesDAO.Attribute.ID); - COL_TO_ATTR_MAP.put("speciesCommonName", SpeciesDAO.Attribute.COMMON_NAME); COL_TO_ATTR_MAP.put("genus", SpeciesDAO.Attribute.GENUS); COL_TO_ATTR_MAP.put("species", SpeciesDAO.Attribute.SPECIES_NAME); + COL_TO_ATTR_MAP.put("speciesCommonName", SpeciesDAO.Attribute.COMMON_NAME); + COL_TO_ATTR_MAP.put("speciesDisplayOrder", SpeciesDAO.Attribute.DISPLAY_ORDER); COL_TO_ATTR_MAP.put("taxonId", SpeciesDAO.Attribute.PARENT_TAXON_ID); COL_TO_ATTR_MAP.put("genomeFilePath", SpeciesDAO.Attribute.GENOME_FILE_PATH); COL_TO_ATTR_MAP.put("genomeVersion", SpeciesDAO.Attribute.GENOME_VERSION); COL_TO_ATTR_MAP.put("genomeAssemblyXRef", SpeciesDAO.Attribute.GENOME_ASSEMBLY_XREF); COL_TO_ATTR_MAP.put("dataSourceId", SpeciesDAO.Attribute.DATA_SOURCE_ID); COL_TO_ATTR_MAP.put("genomeSpeciesId", SpeciesDAO.Attribute.GENOME_SPECIES_ID); - COL_TO_ATTR_MAP.put("speciesDisplayOrder", SpeciesDAO.Attribute.DISPLAY_ORDER); + COL_TO_ATTR_MAP.put("devOntologyXRef", SpeciesDAO.Attribute.DEV_ONTOLOGY_XREF); } /** * Constructor providing the {@code MySQLDAOManager} that this {@code MySQLDAO} @@ -264,7 +265,7 @@ protected SpeciesTO getNewTO() { log.traceEntry(); Integer speciesId = null, taxonId = null, genomeSpeciesId = null, displayOrder = null, dataSourceId = null; - String genus = null, species = null, speciesCommonName = null, + String genus = null, species = null, speciesCommonName = null, devOntologyXRef = null, genomeFilePath = null, genomeVersion = null, genomeAssemblyXRef = null; // Get results try { @@ -306,6 +307,9 @@ protected SpeciesTO getNewTO() { case GENOME_SPECIES_ID: genomeSpeciesId = this.getCurrentResultSet().getInt(columnIndex); break; + case DEV_ONTOLOGY_XREF: + devOntologyXRef = this.getCurrentResultSet().getString(columnIndex); + break; default: log.throwing(new UnrecognizedColumnException(columnName)); } @@ -316,7 +320,7 @@ protected SpeciesTO getNewTO() { //Set SpeciesTO return log.traceExit(new SpeciesTO(speciesId, speciesCommonName, genus, species, displayOrder, taxonId, genomeFilePath, genomeVersion, genomeAssemblyXRef, - dataSourceId, genomeSpeciesId)); + dataSourceId, genomeSpeciesId, devOntologyXRef)); } } } \ No newline at end of file diff --git a/bgee-dao-sql/src/test/java/org/bgee/model/dao/mysql/expressiondata/rawdata/microarray/MySQLAffymetrixProbesetDAOIT.java b/bgee-dao-sql/src/test/java/org/bgee/model/dao/mysql/expressiondata/rawdata/microarray/MySQLAffymetrixProbesetDAOIT.java deleted file mode 100644 index 467250c4c..000000000 --- a/bgee-dao-sql/src/test/java/org/bgee/model/dao/mysql/expressiondata/rawdata/microarray/MySQLAffymetrixProbesetDAOIT.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.bgee.model.dao.mysql.expressiondata.rawdata.microarray; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bgee.model.dao.mysql.MySQLITAncestor; -import org.junit.Rule; -import org.junit.rules.ExpectedException; - - -/** - * Integration tests for {@link MySQLAffymetrixProbesetDAO}, performed on a real MySQL database. - * See the documentation of {@link org.bgee.model.dao.mysql.MySQLITAncestor} for - * important information. - * - * @author Valentine Rech de Laval - * @version Bgee 13 - * @see org.bgee.model.dao.api.expressiondata.rawdata.affymetrix.AffymetrixProbesetDAO - * @since Bgee 13 - */ -public class MySQLAffymetrixProbesetDAOIT extends MySQLITAncestor { - - private final static Logger log = - LogManager.getLogger(MySQLAffymetrixProbesetDAOIT.class.getName()); - - public MySQLAffymetrixProbesetDAOIT() { - super(); - } - - @Override - protected Logger getLogger() { - return log; - } - - @Rule - public ExpectedException thrown = ExpectedException.none(); -} diff --git a/bgee-dao-sql/src/test/java/org/bgee/model/dao/mysql/file/MySQLDownloadFileDAOIT.java b/bgee-dao-sql/src/test/java/org/bgee/model/dao/mysql/file/MySQLDownloadFileDAOIT.java index cf72ef43c..4836824c5 100644 --- a/bgee-dao-sql/src/test/java/org/bgee/model/dao/mysql/file/MySQLDownloadFileDAOIT.java +++ b/bgee-dao-sql/src/test/java/org/bgee/model/dao/mysql/file/MySQLDownloadFileDAOIT.java @@ -100,7 +100,7 @@ public void testInsertDownloadFiles() throws SQLException { new DownloadFileTO(2, "file name 2", "file desc 2", "path/file2/xx", 2L, CategoryEnum.DIFF_EXPR_ANAT_COMPLETE, 11, null), new DownloadFileTO(3, "file name 3", "file desc 3", "path/file3", 10L, - CategoryEnum.AFFY_ANNOT, 22, null)); + CategoryEnum.RNASEQ_ANNOT, 22, null)); try { MySQLDownloadFileDAO dao = new MySQLDownloadFileDAO(this.getMySQLDAOManager()); assertEquals("Incorrect number of rows inserted", 3, @@ -140,7 +140,7 @@ public void testInsertDownloadFiles() throws SQLException { stmt.setString(3, "file desc 3"); stmt.setString(4, "path/file3"); stmt.setLong(5, 10L); - stmt.setEnumDAOField(6, CategoryEnum.AFFY_ANNOT); + stmt.setEnumDAOField(6, CategoryEnum.RNASEQ_ANNOT); stmt.setInt(7, 22); assertTrue("DownloadFileTO incorrectly inserted", stmt.getRealPreparedStatement().executeQuery().next()); diff --git a/bgee-dao-sql/src/test/java/org/bgee/model/dao/mysql/gene/MySQLGeneDAOIT.java b/bgee-dao-sql/src/test/java/org/bgee/model/dao/mysql/gene/MySQLGeneDAOIT.java index d9bfbd847..7cf475d57 100644 --- a/bgee-dao-sql/src/test/java/org/bgee/model/dao/mysql/gene/MySQLGeneDAOIT.java +++ b/bgee-dao-sql/src/test/java/org/bgee/model/dao/mysql/gene/MySQLGeneDAOIT.java @@ -63,10 +63,10 @@ public void shouldGetAllGenes() throws SQLException { // Generate manually expected result List expectedGenes = Arrays.asList( - new GeneTO(1, "ID1", "genN1", "genDesc1", 11, 12, 5, true, 1, "expression summary"), - new GeneTO(2, "ID2", "genN2", "genDesc2", 21, 0, 2, true, 1, "expression summary"), - new GeneTO(3, "ID3", "genN3", "genDesc3", 31, 0, 3, false, 1, "expression summary"), - new GeneTO(4, "ID4", "genN4", "genDesc4", 21, 0, 0, true, 1, "expression summary")); + new GeneTO(1, "ID1", "genN1", "genDesc1", 11, 12, true, "reg1", 1, "expression summary"), + new GeneTO(2, "ID2", "genN2", "genDesc2", 21, 0, true, "reg1", 1, "expression summary"), + new GeneTO(3, "ID3", "genN3", "genDesc3", 31, 0, false, "reg1", 1, "expression summary"), + new GeneTO(4, "ID4", "genN4", "genDesc4", 21, 0, true, "reg1", 1, "expression summary")); //Compare assertTrue("GeneTOs incorrectly retrieved", TOComparator.areTOCollectionsEqual(methGenes, expectedGenes)); @@ -119,8 +119,8 @@ public void shouldGetGenesBySpeciesIds() throws SQLException { dao.clearAttributes(); methGenes = dao.getGenesBySpeciesIds(speciesIds).getAllTOs(); expectedGenes = Arrays.asList( - new GeneTO(1, "ID1", "genN1", "genDesc1", 11, 12, 5, true, 1, "expression summary"), - new GeneTO(3, "ID3", "genN3", "genDesc3", 31, 0, 3, false, 1, "expression summary")); + new GeneTO(1, "ID1", "genN1", "genDesc1", 11, 12, true, "reg1", 1, "expression summary"), + new GeneTO(3, "ID3", "genN3", "genDesc3", 31, 0, false, "reg1", 1, "expression summary")); //Compare assertTrue("GeneTOs incorrectly retrieved", TOComparator.areTOCollectionsEqual(methGenes, expectedGenes)); @@ -201,8 +201,8 @@ public void shouldGetGenesBySpeciesAndGeneIds() throws SQLException { methGenes = dao.getGenesBySpeciesAndGeneIds(speciesIds.stream().collect( Collectors.toMap(id -> id, id -> null)), true).getAllTOs(); expectedGenes = Arrays.asList( - new GeneTO(1, "ID1", "genN1", "genDesc1", 11, 12, 5, true, 1, "expression summary"), - new GeneTO(3, "ID3", "genN3", "genDesc3", 31, 0, 3, false, 1, "expression summary")); + new GeneTO(1, "ID1", "genN1", "genDesc1", 11, 12, true, "reg1", 1, "expression summary"), + new GeneTO(3, "ID3", "genN3", "genDesc3", 31, 0, false, "reg1", 1, "expression summary")); //Compare assertTrue("GeneTOs incorrectly retrieved", TOComparator.areTOCollectionsEqual(methGenes, expectedGenes)); @@ -268,8 +268,8 @@ public void shouldUpdateGenes() throws SQLException { this.populateAndUseDatabase(); Collection geneTOs = Arrays.asList( - new GeneTO(1, "ID1", "GNMod1", "DescMod1", 31, 12, 7, true, 1, "expression summary"), - new GeneTO(2, "ID2", "GNMod2", "DescMod2", 11, 12, 6, false, 1, "expression summary")); + new GeneTO(1, "ID1", "GNMod1", "DescMod1", 31, 12, true, "reg1", 1, "expression summary"), + new GeneTO(2, "ID2", "GNMod2", "DescMod2", 11, 12, false, "reg1", 1, "expression summary")); Collection attributesToUpdate1 = Arrays.asList( GeneDAO.Attribute.GENE_BIO_TYPE_ID); diff --git a/bgee-dao-sql/src/test/java/org/bgee/model/dao/mysql/source/MySQLSourceToSpeciesDAOIT.java b/bgee-dao-sql/src/test/java/org/bgee/model/dao/mysql/source/MySQLSourceToSpeciesDAOIT.java index d0eca927d..e6d64fcdf 100644 --- a/bgee-dao-sql/src/test/java/org/bgee/model/dao/mysql/source/MySQLSourceToSpeciesDAOIT.java +++ b/bgee-dao-sql/src/test/java/org/bgee/model/dao/mysql/source/MySQLSourceToSpeciesDAOIT.java @@ -112,7 +112,7 @@ public void shouldGetSourceToSpecies() throws SQLException { // Test recovery of one attribute with filter on data source IDs and data types EnumSet attributes = EnumSet.of(SourceToSpeciesDAO.Attribute.DATA_TYPE); Set dataSourceIds = new HashSet<>(Arrays.asList(4, 99)); - EnumSet dataTypes = EnumSet.of(DAODataType.EST, DAODataType.IN_SITU); + EnumSet dataTypes = EnumSet.of(DAODataType.RNA_SEQ, DAODataType.IN_SITU); expectedTOs = this.getAllSourceToSpeciesTOs().stream() .filter(s -> dataSourceIds.contains(s.getDataSourceId())) .filter(s -> dataTypes.contains(s.getDataType())) @@ -143,14 +143,14 @@ public void shouldGetSourceToSpecies() throws SQLException { private List getAllSourceToSpeciesTOs() { return Arrays.asList( - new SourceToSpeciesTO(1, 11, DAODataType.AFFYMETRIX, InfoType.DATA), - new SourceToSpeciesTO(1, 11, DAODataType.AFFYMETRIX, InfoType.ANNOTATION), - new SourceToSpeciesTO(1, 21, DAODataType.AFFYMETRIX, InfoType.DATA), - new SourceToSpeciesTO(1, 21, DAODataType.AFFYMETRIX, InfoType.ANNOTATION), - new SourceToSpeciesTO(2, 11, DAODataType.EST, InfoType.DATA), - new SourceToSpeciesTO(3, 11, DAODataType.EST, InfoType.ANNOTATION), - new SourceToSpeciesTO(4, 11, DAODataType.EST, InfoType.DATA), - new SourceToSpeciesTO(4, 11, DAODataType.EST, InfoType.ANNOTATION), + new SourceToSpeciesTO(1, 11, DAODataType.RNA_SEQ, InfoType.DATA), + new SourceToSpeciesTO(1, 11, DAODataType.RNA_SEQ, InfoType.ANNOTATION), + new SourceToSpeciesTO(1, 21, DAODataType.RNA_SEQ, InfoType.DATA), + new SourceToSpeciesTO(1, 21, DAODataType.RNA_SEQ, InfoType.ANNOTATION), + new SourceToSpeciesTO(2, 11, DAODataType.IN_SITU, InfoType.DATA), + new SourceToSpeciesTO(3, 11, DAODataType.IN_SITU, InfoType.ANNOTATION), + new SourceToSpeciesTO(4, 11, DAODataType.IN_SITU, InfoType.DATA), + new SourceToSpeciesTO(4, 11, DAODataType.IN_SITU, InfoType.ANNOTATION), new SourceToSpeciesTO(4, 21, DAODataType.RNA_SEQ, InfoType.DATA), new SourceToSpeciesTO(4, 21, DAODataType.IN_SITU, InfoType.ANNOTATION)); } diff --git a/bgee-dao-sql/src/test/java/org/bgee/model/dao/mysql/species/MySQLSpeciesDAOIT.java b/bgee-dao-sql/src/test/java/org/bgee/model/dao/mysql/species/MySQLSpeciesDAOIT.java index 9d87e436f..6cba3b8fd 100644 --- a/bgee-dao-sql/src/test/java/org/bgee/model/dao/mysql/species/MySQLSpeciesDAOIT.java +++ b/bgee-dao-sql/src/test/java/org/bgee/model/dao/mysql/species/MySQLSpeciesDAOIT.java @@ -47,11 +47,11 @@ public void shouldInsertSpecies() throws SQLException { //create a Collection of SpeciesTOs to be inserted Collection speciesTOs = new ArrayList(); speciesTOs.add(new SpeciesTO(10, "commonName1", "genus1", "speciesName1", 2, - 100, "path/1", "1", "assemblyXRef", 1, null)); + 100, "path/1", "1", "assemblyXRef", 1, null, "xref1")); speciesTOs.add(new SpeciesTO(20, "commonName2", "genus2", "speciesName2", 1, - 120, "path/2", "2", "assemblyXRef", 1, 200)); + 120, "path/2", "2", "assemblyXRef", 1, 200, "xref1")); speciesTOs.add(new SpeciesTO(30, "commonName3", "genus3", "speciesName3", 3, - 500, "path/3", "3", "assemblyXRef", 1, 200)); + 500, "path/3", "3", "assemblyXRef", 1, 200, "xref1")); try { MySQLSpeciesDAO dao = new MySQLSpeciesDAO(this.getMySQLDAOManager()); assertEquals("Incorrect number of rows inserted", 3, @@ -130,22 +130,22 @@ public void shouldGetAllSpecies() throws SQLException { List expectedSpecies = Arrays.asList( new SpeciesTO(31, "spCName31", "gen31", "sp31", 1, 311, "gen31_sp31/gen31_sp31.genome31", "genome31", "assemblyXRef31", - 1, 0), + 1, 0, "xref1"), new SpeciesTO(41, "spCName41", "gen41", "sp41", 2, 411, "gen41_sp41/gen41_sp41.genome41", "genome41", "assemblyXRef41", - 1, 0), + 1, 0, "xref1"), new SpeciesTO(21, "spCName21", "gen21", "sp21", 3, 211, "gen51_sp51/gen51_sp51.genome51", "genome51", "assemblyXRef51", - 1, 51), + 1, 51, "xref1"), new SpeciesTO(11, "spCName11", "gen11", "sp11", 4, 111, "gen11_sp11/gen11_sp11.genome11", "genome11", "assemblyXRef11", - 1, 0), + 1, 0, "xref1"), new SpeciesTO(42, "spCName42", "gen41", "sp42", 5, 411, "gen41_sp41/gen41_sp41.genome41", "genome41", "assemblyXRef41", - 1, 41), + 1, 41, "xref1"), new SpeciesTO(51, "spCName51", "gen51", "sp51", 6, 511, "gen51_sp51/gen51_sp51.genome51", "genome51", "assemblyXRef51", - 1, 0)); + 1, 0, "xref1")); // Compare assertTrue("SpeciesTOs incorrectly retrieved", TOComparator.areTOCollectionsEqual(methSpecies, expectedSpecies)); @@ -156,12 +156,12 @@ public void shouldGetAllSpecies() throws SQLException { // Generate manually expected result expectedSpecies = Arrays.asList( - new SpeciesTO(null, "spCName11", null, null, null, null, null, null, null, null, null), - new SpeciesTO(null, "spCName21",null, null, null, null, null, null, null, null, null), - new SpeciesTO(null, "spCName31", null, null, null, null, null, null, null, null, null), - new SpeciesTO(null, "spCName41", null, null, null, null, null, null, null, null, null), - new SpeciesTO(null, "spCName42", null, null, null, null, null, null, null, null, null), - new SpeciesTO(null, "spCName51", null, null, null, null, null, null, null, null, null)); + new SpeciesTO(null, "spCName11", null, null, null, null, null, null, null, null, null, null), + new SpeciesTO(null, "spCName21",null, null, null, null, null, null, null, null, null, null), + new SpeciesTO(null, "spCName31", null, null, null, null, null, null, null, null, null, null), + new SpeciesTO(null, "spCName41", null, null, null, null, null, null, null, null, null, null), + new SpeciesTO(null, "spCName42", null, null, null, null, null, null, null, null, null, null), + new SpeciesTO(null, "spCName51", null, null, null, null, null, null, null, null, null, null)); // Compare assertTrue("SpeciesTOs incorrectly retrieved", TOComparator.areTOCollectionsEqual(methSpecies, expectedSpecies)); @@ -183,9 +183,9 @@ public void shouldGetSpeciesByIds() throws SQLException { // expected result List expectedSpeciesTOs = Arrays.asList( new SpeciesTO(11, "spCName11", "gen11", "sp11", 4, 111, - "gen11_sp11/gen11_sp11.genome11", "genome11", "assemblyXRef11", 1, 0), + "gen11_sp11/gen11_sp11.genome11", "genome11", "assemblyXRef11", 1, 0, null), new SpeciesTO(31, "spCName31", "gen31", "sp31", 1, 311, - "gen31_sp31/gen31_sp31.genome31", "genome31", "assemblyXRef31", 1, 0)); + "gen31_sp31/gen31_sp31.genome31", "genome31", "assemblyXRef31", 1, 0, null)); // Compare assertTrue("SpeciesTOs incorrectly retrieved, expected: " + expectedSpeciesTOs + ", but was: " + speciesTOs, @@ -197,8 +197,8 @@ public void shouldGetSpeciesByIds() throws SQLException { // Generate manually expected result expectedSpeciesTOs = Arrays.asList( - new SpeciesTO(11, null, "gen11", "sp11", null, null, null, null, null, null, null), - new SpeciesTO(31, null, "gen31", "sp31", null, null, null, null, null, null, null)); + new SpeciesTO(11, null, "gen11", "sp11", null, null, null, null, null, null, null, null), + new SpeciesTO(31, null, "gen31", "sp31", null, null, null, null, null, null, null, null)); // Compare assertTrue("SpeciesTOs incorrectly retrieved, expected: " + expectedSpeciesTOs + ", but was: " + speciesTOs, @@ -211,8 +211,8 @@ public void shouldGetSpeciesByIds() throws SQLException { // Generate manually expected result expectedSpeciesTOs = Arrays.asList( - new SpeciesTO(11, null, "gen11", "sp11", 4, null, null, null, null, null, null), - new SpeciesTO(31, null, "gen31", "sp31", 1, null, null, null, null, null, null)); + new SpeciesTO(11, null, "gen11", "sp11", 4, null, null, null, null, null, null, null), + new SpeciesTO(31, null, "gen31", "sp31", 1, null, null, null, null, null, null, null)); // Compare assertTrue("SpeciesTOs incorrectly retrieved, expected: " + expectedSpeciesTOs + ", but was: " + speciesTOs, @@ -230,11 +230,11 @@ public void testGetSpeciesInDataGroup() throws SQLException { // Generate manually expected result List expectedSpecies = Arrays.asList( new SpeciesTO(41, "spCName41", "gen41", "sp41", 2, 411, - "gen41_sp41/gen41_sp41.genome41", "genome41", "assemblyXRef41", 1, 0), + "gen41_sp41/gen41_sp41.genome41", "genome41", "assemblyXRef41", 1, 0, null), new SpeciesTO(21, "spCName21", "gen21", "sp21", 3, 211, - "gen51_sp51/gen51_sp51.genome51", "genome51", "assemblyXRef51", 1, 51), + "gen51_sp51/gen51_sp51.genome51", "genome51", "assemblyXRef51", 1, 51, null), new SpeciesTO(51, "spCName51", "gen51", "sp51", 6, 511, - "gen51_sp51/gen51_sp51.genome51", "genome51", "assemblyXRef51", 1, 0)); + "gen51_sp51/gen51_sp51.genome51", "genome51", "assemblyXRef51", 1, 0, null)); // Compare assertTrue("SpeciesTOs incorrectly retrieved", TOComparator.areTOCollectionsEqual(methSpecies, expectedSpecies)); @@ -244,9 +244,9 @@ public void testGetSpeciesInDataGroup() throws SQLException { // Generate manually expected result expectedSpecies = Arrays.asList( - new SpeciesTO(null, "spCName41", null, null, null, null, null, null, null, null, null), - new SpeciesTO(null, "spCName21",null, null, null, null, null, null, null, null, null), - new SpeciesTO(null, "spCName51", null, null, null, null, null, null, null, null, null)); + new SpeciesTO(null, "spCName41", null, null, null, null, null, null, null, null, null,null), + new SpeciesTO(null, "spCName21",null, null, null, null, null, null, null, null, null, null), + new SpeciesTO(null, "spCName51", null, null, null, null, null, null, null, null, null, null)); // Compare assertTrue("SpeciesTOs incorrectly retrieved", TOComparator.areTOCollectionsEqual(methSpecies, expectedSpecies)); diff --git a/bgee-dao-sql/src/test/resources/sql/bgeeConstraint.sql b/bgee-dao-sql/src/test/resources/sql/bgeeConstraint.sql index 31d9d1ae0..856bc17f8 100644 --- a/bgee-dao-sql/src/test/resources/sql/bgeeConstraint.sql +++ b/bgee-dao-sql/src/test/resources/sql/bgeeConstraint.sql @@ -154,12 +154,6 @@ add primary key (summarySimilarityAnnotationId, negated, ECOId, CIOId, reference -- **************************************************** -- GENE AND TRANSCRIPT INFO -- **************************************************** -/*!40000 ALTER TABLE `OMAHierarchicalGroup` DISABLE KEYS */; -alter table OMAHierarchicalGroup -add primary key (OMANodeId), -add unique (OMANodeLeftBound), -add unique (OMANodeRightBound); -/*!40000 ALTER TABLE `OMAHierarchicalGroup` ENABLE KEYS */; /*!40000 ALTER TABLE `geneOntologyTerm` DISABLE KEYS */; alter table geneOntologyTerm @@ -237,8 +231,8 @@ add unique(transcriptId, bgeeGeneId); -- for this design of PK and UNIQUE indexes alter table cond modify conditionId mediumint unsigned not null auto_increment, -add primary key(anatEntityId, stageId, speciesId, sex, sexInferred, strain), -add unique(conditionId); +add primary key(conditionId), +add unique(anatEntityId, cellTypeId, stageId, speciesId, sex, sexInferred, strain); /*!40000 ALTER TABLE `cond` ENABLE KEYS */; /*!40000 ALTER TABLE `remapCond` DISABLE KEYS */; @@ -246,20 +240,28 @@ alter table remapCond add primary key(incorrectConditionId, remappedConditionId); /*!40000 ALTER TABLE `remapCond` ENABLE KEYS */; +/*!40000 ALTER TABLE `remapExpression` DISABLE KEYS */; +alter table remapExpression +add primary key(incorrectExpressionId, remappedExpressionId); +/*!40000 ALTER TABLE `remapExpression` ENABLE KEYS */; + /*!40000 ALTER TABLE `globalCond` DISABLE KEYS */; alter table globalCond modify globalConditionId mediumint unsigned not null auto_increment primary key, -- not a primary key as for table cond, because some field can be null -add unique(anatEntityId, stageId, speciesId, sex, strain); +add unique(anatEntityId, cellTypeId, stageId, speciesId, sex, strain); /*!40000 ALTER TABLE `globalCond` ENABLE KEYS */; -/*!40000 ALTER TABLE `globalCondToCond` DISABLE KEYS */; -alter table globalCondToCond +/*!40000 ALTER TABLE `globalCondRelation` DISABLE KEYS */; +alter table globalCondRelation +add primary key (sourceGlobalConditionId, targetGlobalConditionId); +/*!40000 ALTER TABLE `globalCondRelation` ENABLE KEYS */; + +/*!40000 ALTER TABLE `condToSelfGlobalCond` DISABLE KEYS */; +alter table condToSelfGlobalCond -- we set up this primary key using conditionRelationOrigin to benefit from the clustered index -add primary key (globalConditionId, conditionId, conditionRelationOrigin), --- but actually the unique constraint is on globalConditionId and conditionId -add unique(globalConditionId, conditionId); -/*!40000 ALTER TABLE `globalCondToCond` ENABLE KEYS */; +add primary key (conditionId, subsetMask, globalConditionId); +/*!40000 ALTER TABLE `condToSelfGlobalCond` ENABLE KEYS */; -- **************************************************** -- EXPRESSION DATA @@ -269,115 +271,11 @@ add unique(globalConditionId, conditionId); -- for this design of PK and UNIQUE indexes alter table expression modify expressionId int unsigned not null auto_increment, -add primary key(bgeeGeneId, conditionId), +-- we added the primary key in this order to be able to parallelize insertion per conditionId +add primary key(conditionId, bgeeGeneId), add unique(expressionId); /*!40000 ALTER TABLE `expression` ENABLE KEYS */; -/*!40000 ALTER TABLE `globalExpression` DISABLE KEYS */; --- See https://stackoverflow.com/a/42822691/1768736 for motivation --- for this design of PK and UNIQUE indexes -alter table globalExpression -modify globalExpressionId int unsigned not null auto_increment, -add primary key(bgeeGeneId, globalConditionId), -add unique(globalExpressionId); -/*!40000 ALTER TABLE `globalExpression` ENABLE KEYS */; - --- **************************************************** --- DIFFERENTIAL EXPRESSION DATA --- **************************************************** -/*!40000 ALTER TABLE `differentialExpression` DISABLE KEYS */; --- See https://stackoverflow.com/a/42822691/1768736 for motivation --- for this design of PK and UNIQUE indexes -alter table differentialExpression -modify differentialExpressionId int unsigned not null auto_increment, --- TODO: manage maxNumberOfConditions -add primary key(bgeeGeneId, conditionId, comparisonFactor), -add unique(differentialExpressionId); -/*!40000 ALTER TABLE `differentialExpression` ENABLE KEYS */; - -/*!40000 ALTER TABLE `differentialExpressionAnalysis` DISABLE KEYS */; -alter table differentialExpressionAnalysis -modify deaId smallint unsigned not null auto_increment primary key; -/*!40000 ALTER TABLE `differentialExpressionAnalysis` ENABLE KEYS */; - -/*!40000 ALTER TABLE `deaSampleGroup` DISABLE KEYS */; -alter table deaSampleGroup -modify deaSampleGroupId mediumint unsigned not null auto_increment primary key; -/*!40000 ALTER TABLE `deaSampleGroup` ENABLE KEYS */; - --- **************************************************** --- RAW EST DATA --- **************************************************** -/*!40000 ALTER TABLE `estLibrary` DISABLE KEYS */; -alter table estLibrary -add primary key (estLibraryId); -/*!40000 ALTER TABLE `estLibrary` ENABLE KEYS */; - -/*!40000 ALTER TABLE `estLibraryToKeyword` DISABLE KEYS */; -alter table estLibraryToKeyword -add primary key (estLibraryId, keywordId); -/*!40000 ALTER TABLE `estLibraryToKeyword` ENABLE KEYS */; - -/*!40000 ALTER TABLE `expressedSequenceTag` DISABLE KEYS */; -alter table expressedSequenceTag -add primary key (estId); -/*!40000 ALTER TABLE `expressedSequenceTag` ENABLE KEYS */; - -/*!40000 ALTER TABLE `estLibraryExpression` DISABLE KEYS */; -alter table estLibraryExpression -add primary key (expressionId, estLibraryId); -/*!40000 ALTER TABLE `estLibraryExpression` ENABLE KEYS */; - --- **************************************************** --- RAW AFFYMETRIX DATA --- **************************************************** -/*!40000 ALTER TABLE `microarrayExperiment` DISABLE KEYS */; -alter table microarrayExperiment -add primary key (microarrayExperimentId); -/*!40000 ALTER TABLE `microarrayExperiment` ENABLE KEYS */; - -/*!40000 ALTER TABLE `microarrayExperimentToKeyword` DISABLE KEYS */; -alter table microarrayExperimentToKeyword -add primary key (microarrayExperimentId, keywordId); -/*!40000 ALTER TABLE `microarrayExperimentToKeyword` ENABLE KEYS */; - --- chipTypes in this table don't represent affymetrix chips only --- (could for instance represent cDNA chip) -/*!40000 ALTER TABLE `chipType` DISABLE KEYS */; -alter table chipType -add unique(chipTypeName), -add unique(cdfName), -add primary key (chipTypeId); -/*!40000 ALTER TABLE `chipType` ENABLE KEYS */; - -/*!40000 ALTER TABLE `affymetrixChip` DISABLE KEYS */; -alter table affymetrixChip -modify bgeeAffymetrixChipId smallint unsigned not null auto_increment primary key, -add unique (affymetrixChipId, microarrayExperimentId); -/*!40000 ALTER TABLE `affymetrixChip` ENABLE KEYS */; - -/*!40000 ALTER TABLE `affymetrixProbeset` DISABLE KEYS */; -alter table affymetrixProbeset -add primary key (affymetrixProbesetId, bgeeAffymetrixChipId); -/*!40000 ALTER TABLE `affymetrixProbeset` ENABLE KEYS */; - -/*!40000 ALTER TABLE `microarrayExperimentExpression` DISABLE KEYS */; -alter table microarrayExperimentExpression -add primary key (expressionId, microarrayExperimentId); -/*!40000 ALTER TABLE `microarrayExperimentExpression` ENABLE KEYS */; - --- ****** for diff expression ******** - -/*!40000 ALTER TABLE `deaSampleGroupToAffymetrixChip` DISABLE KEYS */; -alter table deaSampleGroupToAffymetrixChip -add primary key (bgeeAffymetrixChipId, deaSampleGroupId); -/*!40000 ALTER TABLE `deaSampleGroupToAffymetrixChip` ENABLE KEYS */; - -/*!40000 ALTER TABLE `deaAffymetrixProbesetSummary` DISABLE KEYS */; -alter table deaAffymetrixProbesetSummary -add primary key (deaAffymetrixProbesetSummaryId, deaSampleGroupId); -/*!40000 ALTER TABLE `deaAffymetrixProbesetSummary` ENABLE KEYS */; - -- **************************************************** -- RAW IN SITU DATA -- **************************************************** @@ -409,6 +307,7 @@ add primary key (expressionId, inSituExperimentId); -- **************************************************** -- RAW RNA-SEQ DATA -- **************************************************** + /*!40000 ALTER TABLE `rnaSeqExperiment` DISABLE KEYS */; alter table rnaSeqExperiment add primary key (rnaSeqExperimentId); @@ -419,52 +318,56 @@ alter table rnaSeqExperimentToKeyword add primary key (rnaSeqExperimentId, keywordId); /*!40000 ALTER TABLE `rnaSeqExperimentToKeyword` ENABLE KEYS */; -/*!40000 ALTER TABLE `rnaSeqPlatform` DISABLE KEYS */; -alter table rnaSeqPlatform -add primary key (rnaSeqPlatformId); -/*!40000 ALTER TABLE `rnaSeqPlatform` ENABLE KEYS */; - /*!40000 ALTER TABLE `rnaSeqLibrary` DISABLE KEYS */; alter table rnaSeqLibrary add primary key (rnaSeqLibraryId); /*!40000 ALTER TABLE `rnaSeqLibrary` ENABLE KEYS */; -/*!40000 ALTER TABLE `rnaSeqRun` DISABLE KEYS */; -alter table rnaSeqRun -add primary key (rnaSeqRunId); -/*!40000 ALTER TABLE `rnaSeqRun` ENABLE KEYS */; - /*!40000 ALTER TABLE `rnaSeqLibraryDiscarded` DISABLE KEYS */; alter table rnaSeqLibraryDiscarded add primary key (rnaSeqLibraryId); /*!40000 ALTER TABLE `rnaSeqLibraryDiscarded` ENABLE KEYS */; -/*!40000 ALTER TABLE `rnaSeqResult` DISABLE KEYS */; -alter table rnaSeqResult -add primary key (bgeeGeneId, rnaSeqLibraryId); -/*!40000 ALTER TABLE `rnaSeqResult` ENABLE KEYS */; - -/*!40000 ALTER TABLE `rnaSeqTranscriptResult` DISABLE KEYS */; -alter table rnaSeqTranscriptResult -add primary key (bgeeTranscriptId, rnaSeqLibraryId); -/*!40000 ALTER TABLE `rnaSeqTranscriptResult` ENABLE KEYS */; - -/*!40000 ALTER TABLE `rnaSeqExperimentExpression` DISABLE KEYS */; -alter table rnaSeqExperimentExpression -add primary key (expressionId, rnaSeqExperimentId); -/*!40000 ALTER TABLE `rnaSeqExperimentExpression` ENABLE KEYS */; - --- ****** for diff expression ******** - -/*!40000 ALTER TABLE `deaSampleGroupToRnaSeqLibrary` DISABLE KEYS */; -alter table deaSampleGroupToRnaSeqLibrary -add primary key (rnaSeqLibraryId, deaSampleGroupId); -/*!40000 ALTER TABLE `deaSampleGroupToRnaSeqLibrary` ENABLE KEYS */; +/*!40000 ALTER TABLE `rnaSeqRun` DISABLE KEYS */; +alter table rnaSeqRun +add primary key (rnaSeqRunId); +/*!40000 ALTER TABLE `rnaSeqRun` ENABLE KEYS */; -/*!40000 ALTER TABLE `deaRNASeqSummary` DISABLE KEYS */; -alter table deaRNASeqSummary -add primary key (geneSummaryId, deaSampleGroupId); -/*!40000 ALTER TABLE `deaRNASeqSummary` ENABLE KEYS */; +/*!40000 ALTER TABLE `rnaSeqLibraryAnnotatedSample` DISABLE KEYS */; +alter table rnaSeqLibraryAnnotatedSample +modify rnaSeqLibraryAnnotatedSampleId mediumint unsigned not null auto_increment primary key, +add unique (rnaSeqLibraryId, conditionId, cellTypeAuthorAnnotation); +/*!40000 ALTER TABLE `rnaSeqLibraryAnnotatedSample` ENABLE KEYS */; + +/*!40000 ALTER TABLE `rnaSeqLibraryAnnotatedSampleGeneResult` DISABLE KEYS */; +alter table rnaSeqLibraryAnnotatedSampleGeneResult +add primary key (rnaSeqLibraryAnnotatedSampleId, bgeeGeneId); +/*!40000 ALTER TABLE `rnaSeqLibraryAnnotatedSampleGeneResult` ENABLE KEYS */; + +/*!40000 ALTER TABLE `rnaSeqLibraryIndividualSample` DISABLE KEYS */; +alter table rnaSeqLibraryIndividualSample +modify rnaSeqLibraryIndividualSampleId int unsigned not null auto_increment primary key; +/*!40000 ALTER TABLE `rnaSeqLibraryIndividualSample` ENABLE KEYS */; + +/*!40000 ALTER TABLE `rnaSeqLibraryIndividualSampleGeneResult` DISABLE KEYS */; +alter table rnaSeqLibraryIndividualSampleGeneResult +add primary key (rnaSeqLibraryIndividualSampleId, bgeeGeneId); +/*!40000 ALTER TABLE `rnaSeqLibraryIndividualSampleGeneResult` ENABLE KEYS */; + +/*!40000 ALTER TABLE `rnaSeqPopulationCapture` DISABLE KEYS */; +alter table rnaSeqPopulationCapture +add primary key (rnaSeqPopulationCaptureId); +/*!40000 ALTER TABLE `rnaSeqPopulationCapture` ENABLE KEYS */; + +/*!40000 ALTER TABLE `rnaSeqPopulationCaptureToBiotypeExcludedAbsentCalls` DISABLE KEYS */; +alter table rnaSeqPopulationCaptureToBiotypeExcludedAbsentCalls +add primary key (rnaSeqPopulationCaptureId, geneBioTypeId); +/*!40000 ALTER TABLE `rnaSeqPopulationCaptureToBiotypeExcludedAbsentCalls` ENABLE KEYS */; + +/*!40000 ALTER TABLE `rnaSeqPopulationCaptureSpeciesMaxRank` DISABLE KEYS */; +alter table rnaSeqPopulationCaptureSpeciesMaxRank +add primary key (speciesId, rnaSeqPopulationCaptureId, rnaSeqTechnologyIsSingleCell, sampleMultiplexing); +/*!40000 ALTER TABLE `rnaSeqPopulationCaptureSpeciesMaxRank` ENABLE KEYS */; -- ***************************************** -- DOWNLOAD FILES diff --git a/bgee-dao-sql/src/test/resources/sql/bgeeForeignKey.sql b/bgee-dao-sql/src/test/resources/sql/bgeeForeignKey.sql index 1c490cbea..463fe1ee0 100644 --- a/bgee-dao-sql/src/test/resources/sql/bgeeForeignKey.sql +++ b/bgee-dao-sql/src/test/resources/sql/bgeeForeignKey.sql @@ -1,11 +1,11 @@ --- this file contains the foreign key constraints. +-- this file contains the foreign key constraints. -- **************************************************** -- GENERAL -- **************************************************** /*!40000 ALTER TABLE `dataSourceToSpecies` DISABLE KEYS */; -alter table dataSourceToSpecies -add foreign key (dataSourceId) references dataSource(dataSourceId) on delete cascade, +alter table dataSourceToSpecies +add foreign key (dataSourceId) references dataSource(dataSourceId) on delete cascade, add foreign key (speciesId) references species(speciesId) on delete cascade; /*!40000 ALTER TABLE `dataSourceToSpecies` ENABLE KEYS */; @@ -112,10 +112,6 @@ add foreign key (CIOId) references CIOStatement(CIOId) on delete cascade; -- **************************************************** -- GENE AND TRANSCRIPT INFO -- **************************************************** -/*!40000 ALTER TABLE `OMAHierarchicalGroup` DISABLE KEYS */; -alter table OMAHierarchicalGroup -add foreign key (taxonId) references taxon(taxonId) on delete set null; -/*!40000 ALTER TABLE `OMAHierarchicalGroup` ENABLE KEYS */; /*!40000 ALTER TABLE `geneOntologyTermAltId` DISABLE KEYS */; alter table geneOntologyTermAltId @@ -145,8 +141,7 @@ add foreign key (taxonId) references taxon(taxonId) on delete cascade; /*!40000 ALTER TABLE `gene` DISABLE KEYS */; alter table gene add foreign key (speciesId) references species(speciesId) on delete cascade, -add foreign key (geneBioTypeId) references geneBioType(geneBioTypeId) on delete set null, -add foreign key (OMAParentNodeId) references OMAHierarchicalGroup(OMANodeId) on delete set null; +add foreign key (geneBioTypeId) references geneBioType(geneBioTypeId) on delete set null; /*!40000 ALTER TABLE `gene` ENABLE KEYS */; /*!40000 ALTER TABLE `geneNameSynonym` DISABLE KEYS */; @@ -180,10 +175,11 @@ add foreign key (bgeeGeneId) references gene(bgeeGeneId) on delete cascade; -- CONDITIONS -- **************************************************** /*!40000 ALTER TABLE `cond` DISABLE KEYS */; -alter table cond -add foreign key (exprMappedConditionId) references cond(conditionId) on delete cascade, +alter table cond +add foreign key (exprMappedConditionId) references cond(conditionId) on delete cascade, add foreign key (anatEntityId) references anatEntity(anatEntityId) on delete cascade, -add foreign key (stageId) references stage(stageId) on delete cascade, +add foreign key (cellTypeId) references anatEntity(anatEntityId) on delete cascade, +add foreign key (stageId) references stage(stageId) on delete cascade, add foreign key (speciesId) references species(speciesId) on delete cascade; /*!40000 ALTER TABLE `cond` ENABLE KEYS */; @@ -192,18 +188,30 @@ alter table remapCond add foreign key (remappedConditionId) references cond(conditionId) on delete cascade; /*!40000 ALTER TABLE `remapCond` ENABLE KEYS */; +/*!40000 ALTER TABLE `remapExpression` DISABLE KEYS */; +alter table remapExpression +add foreign key (remappedExpressionId) references expression(expressionId) on delete cascade; +/*!40000 ALTER TABLE `remapExpression` ENABLE KEYS */; + /*!40000 ALTER TABLE `globalCond` DISABLE KEYS */; alter table globalCond add foreign key (anatEntityId) references anatEntity(anatEntityId) on delete cascade, -add foreign key (stageId) references stage(stageId) on delete cascade, +add foreign key (cellTypeId) references anatEntity(anatEntityId) on delete cascade, +add foreign key (stageId) references stage(stageId) on delete cascade, add foreign key (speciesId) references species(speciesId) on delete cascade; /*!40000 ALTER TABLE `globalCond` ENABLE KEYS */; -/*!40000 ALTER TABLE `globalCondToCond` DISABLE KEYS */; -alter table globalCondToCond +/*!40000 ALTER TABLE `globalCondRelation` DISABLE KEYS */; +alter table globalCondRelation +add foreign key (sourceGlobalConditionId) references globalCond(globalConditionId) on delete cascade, +add foreign key (targetGlobalConditionId) references globalCond(globalConditionId) on delete cascade; +/*!40000 ALTER TABLE `globalCondRelation` ENABLE KEYS */; + +/*!40000 ALTER TABLE `condToSelfGlobalCond` DISABLE KEYS */; +alter table condToSelfGlobalCond add foreign key (conditionId) references cond(conditionId) on delete cascade, add foreign key (globalConditionId) references globalCond(globalConditionId) on delete cascade; -/*!40000 ALTER TABLE `globalCondToCond` ENABLE KEYS */; +/*!40000 ALTER TABLE `condToSelfGlobalCond` ENABLE KEYS */; -- **************************************************** -- EXPRESSION DATA @@ -214,110 +222,6 @@ add foreign key (bgeeGeneId) references gene(bgeeGeneId) on delete cascade, add foreign key (conditionId) references cond(conditionId) on delete cascade; /*!40000 ALTER TABLE `expression` ENABLE KEYS */; -/*!40000 ALTER TABLE `globalExpression` DISABLE KEYS */; -alter table globalExpression -add foreign key (bgeeGeneId) references gene(bgeeGeneId) on delete cascade, -add foreign key (globalConditionId) references globalCond(globalConditionId) on delete cascade; -/*!40000 ALTER TABLE `globalExpression` ENABLE KEYS */; - --- **************************************************** --- DIFFERENTIAL EXPRESSION DATA --- **************************************************** -/*!40000 ALTER TABLE `differentialExpression` DISABLE KEYS */; -alter table differentialExpression -add foreign key (bgeeGeneId) references gene(bgeeGeneId) on delete cascade, -add foreign key (conditionId) references cond(conditionId) on delete cascade; -/*!40000 ALTER TABLE `differentialExpression` ENABLE KEYS */; - -/*!40000 ALTER TABLE `differentialExpressionAnalysis` DISABLE KEYS */; -alter table differentialExpressionAnalysis -add foreign key (microarrayExperimentId) references microarrayExperiment(microarrayExperimentId) on delete cascade, -add foreign key (rnaSeqExperimentId) references rnaSeqExperiment(rnaSeqExperimentId) on delete cascade; -/*!40000 ALTER TABLE `differentialExpressionAnalysis` ENABLE KEYS */; - -/*!40000 ALTER TABLE `deaSampleGroup` DISABLE KEYS */; -alter table deaSampleGroup -add foreign key (deaId) references differentialExpressionAnalysis(deaId) on delete cascade, -add foreign key (conditionId) references cond(conditionId) on delete cascade; -/*!40000 ALTER TABLE `deaSampleGroup` ENABLE KEYS */; - --- **************************************************** --- RAW EST DATA --- **************************************************** -/*!40000 ALTER TABLE `estLibrary` DISABLE KEYS */; -alter table estLibrary -add foreign key (conditionId) references cond(conditionId) on delete cascade, -add foreign key (dataSourceId) references dataSource(dataSourceId); -/*!40000 ALTER TABLE `estLibrary` ENABLE KEYS */; - -/*!40000 ALTER TABLE `estLibraryToKeyword` DISABLE KEYS */; -alter table estLibraryToKeyword -add foreign key (estLibraryId) references estLibrary(estLibraryId) on delete cascade, -add foreign key (keywordId) references keyword(keywordId) on delete cascade; -/*!40000 ALTER TABLE `estLibraryToKeyword` ENABLE KEYS */; - -/*!40000 ALTER TABLE `expressedSequenceTag` DISABLE KEYS */; -alter table expressedSequenceTag -add foreign key (estLibraryId) references estLibrary(estLibraryId) on delete cascade, -add foreign key (bgeeGeneId) references gene(bgeeGeneId) on delete cascade, -add foreign key (expressionId) references expression(expressionId) on delete set null; -/*!40000 ALTER TABLE `expressedSequenceTag` ENABLE KEYS */; - -/*!40000 ALTER TABLE `estLibraryExpression` DISABLE KEYS */; -alter table estLibraryExpression -add foreign key (expressionId) references expression(expressionId) on delete cascade, -add foreign key (estLibraryId) references estLibrary(estLibraryId) on delete cascade; -/*!40000 ALTER TABLE `estLibraryExpression` ENABLE KEYS */; --- **************************************************** --- RAW AFFYMETRIX DATA --- **************************************************** -/*!40000 ALTER TABLE `microarrayExperiment` DISABLE KEYS */; -alter table microarrayExperiment -add foreign key (dataSourceId) references dataSource(dataSourceId); -/*!40000 ALTER TABLE `microarrayExperiment` ENABLE KEYS */; - -/*!40000 ALTER TABLE `microarrayExperimentToKeyword` DISABLE KEYS */; -alter table microarrayExperimentToKeyword -add foreign key (microarrayExperimentId) references microarrayExperiment(microarrayExperimentId) on delete cascade, -add foreign key (keywordId) references keyword(keywordId) on delete cascade; -/*!40000 ALTER TABLE `microarrayExperimentToKeyword` ENABLE KEYS */; - -/*!40000 ALTER TABLE `affymetrixChip` DISABLE KEYS */; -alter table affymetrixChip -add foreign key (microarrayExperimentId) references microarrayExperiment(microarrayExperimentId) on delete cascade, -add foreign key (chipTypeId) references chipType(chipTypeId) on delete set null, -add foreign key (conditionId) references cond(conditionId) on delete cascade; -/*!40000 ALTER TABLE `affymetrixChip` ENABLE KEYS */; - -/*!40000 ALTER TABLE `affymetrixProbeset` DISABLE KEYS */; -alter table affymetrixProbeset -add foreign key (bgeeAffymetrixChipId) references affymetrixChip(bgeeAffymetrixChipId) on delete cascade, -add foreign key (bgeeGeneId) references gene(bgeeGeneId) on delete cascade, -add foreign key (expressionId) references expression(expressionId) on delete set null; -/*!40000 ALTER TABLE `affymetrixProbeset` ENABLE KEYS */; - -/*!40000 ALTER TABLE `microarrayExperimentExpression` DISABLE KEYS */; -alter table microarrayExperimentExpression -add foreign key (expressionId) references expression(expressionId) on delete cascade, -add foreign key (microarrayExperimentId) references microarrayExperiment(microarrayExperimentId) on delete cascade; -/*!40000 ALTER TABLE `microarrayExperimentExpression` ENABLE KEYS */; - --- ****** for diff expression ******** - -/*!40000 ALTER TABLE `deaSampleGroupToAffymetrixChip` DISABLE KEYS */; -alter table deaSampleGroupToAffymetrixChip -add foreign key (deaSampleGroupId) references deaSampleGroup(deaSampleGroupId) on delete cascade, -add foreign key (bgeeAffymetrixChipId) references affymetrixChip(bgeeAffymetrixChipId) on delete cascade; -/*!40000 ALTER TABLE `deaSampleGroupToAffymetrixChip` ENABLE KEYS */; - -/*!40000 ALTER TABLE `deaAffymetrixProbesetSummary` DISABLE KEYS */; -alter table deaAffymetrixProbesetSummary -add foreign key (deaAffymetrixProbesetSummaryId) references affymetrixProbeset(affymetrixProbesetId) on delete cascade, -add foreign key (deaSampleGroupId) references deaSampleGroup(deaSampleGroupId) on delete cascade, -add foreign key (bgeeGeneId) references gene(bgeeGeneId) on delete cascade, -add foreign key (differentialExpressionId) references differentialExpression(differentialExpressionId) on delete set null; -/*!40000 ALTER TABLE `deaAffymetrixProbesetSummary` ENABLE KEYS */; - -- **************************************************** -- RAW IN SITU DATA -- **************************************************** @@ -354,6 +258,7 @@ add foreign key (inSituExperimentId) references inSituExperiment(inSituExperimen -- **************************************************** -- RAW RNA-SEQ DATA -- **************************************************** + /*!40000 ALTER TABLE `rnaSeqExperiment` DISABLE KEYS */; alter table rnaSeqExperiment add foreign key (dataSourceId) references dataSource(dataSourceId); @@ -368,8 +273,7 @@ add foreign key (keywordId) references keyword(keywordId) on delete cascade; /*!40000 ALTER TABLE `rnaSeqLibrary` DISABLE KEYS */; alter table rnaSeqLibrary add foreign key (rnaSeqExperimentId) references rnaSeqExperiment(rnaSeqExperimentId) on delete cascade, -add foreign key (rnaSeqPlatformId) references rnaSeqPlatform(rnaSeqPlatformId) on delete cascade, -add foreign key (conditionId) references cond(conditionId) on delete cascade; +add foreign key (rnaSeqPopulationCaptureId) references rnaSeqPopulationCapture(rnaSeqPopulationCaptureId) on delete cascade; /*!40000 ALTER TABLE `rnaSeqLibrary` ENABLE KEYS */; /*!40000 ALTER TABLE `rnaSeqRun` DISABLE KEYS */; @@ -377,39 +281,41 @@ alter table rnaSeqRun add foreign key (rnaSeqLibraryId) references rnaSeqLibrary(rnaSeqLibraryId) on delete cascade; /*!40000 ALTER TABLE `rnaSeqRun` ENABLE KEYS */; -/*!40000 ALTER TABLE `rnaSeqResult` DISABLE KEYS */; -alter table rnaSeqResult +/*!40000 ALTER TABLE `rnaSeqLibraryAnnotatedSample` DISABLE KEYS */; +alter table rnaSeqLibraryAnnotatedSample add foreign key (rnaSeqLibraryId) references rnaSeqLibrary(rnaSeqLibraryId) on delete cascade, +add foreign key (conditionId) references cond(conditionId) on delete cascade; +/*!40000 ALTER TABLE `rnaSeqLibraryAnnotatedSample` ENABLE KEYS */; + +/*!40000 ALTER TABLE `rnaSeqLibraryAnnotatedSampleGeneResult` DISABLE KEYS */; +alter table rnaSeqLibraryAnnotatedSampleGeneResult +add foreign key (rnaSeqLibraryAnnotatedSampleId) references rnaSeqLibraryAnnotatedSample(rnaSeqLibraryAnnotatedSampleId) on delete cascade, add foreign key (bgeeGeneId) references gene(bgeeGeneId) on delete cascade, add foreign key (expressionId) references expression(expressionId) on delete set null; -/*!40000 ALTER TABLE `rnaSeqResult` ENABLE KEYS */; +/*!40000 ALTER TABLE `rnaSeqLibraryAnnotatedSampleGeneResult` ENABLE KEYS */; -/*!40000 ALTER TABLE `rnaSeqTranscriptResult` DISABLE KEYS */; -alter table rnaSeqTranscriptResult -add foreign key (rnaSeqLibraryId) references rnaSeqLibrary(rnaSeqLibraryId) on delete cascade, -add foreign key (bgeeTranscriptId) references transcript(bgeeTranscriptId) on delete cascade; -/*!40000 ALTER TABLE `rnaSeqTranscriptResult` ENABLE KEYS */; +/*!40000 ALTER TABLE `rnaSeqLibraryIndividualSample` DISABLE KEYS */; +alter table rnaSeqLibraryIndividualSample +add foreign key (rnaSeqLibraryAnnotatedSampleId) references rnaSeqLibraryAnnotatedSample(rnaSeqLibraryAnnotatedSampleId) on delete cascade; +/*!40000 ALTER TABLE `rnaSeqLibraryIndividualSample` ENABLE KEYS */; -/*!40000 ALTER TABLE `rnaSeqExperimentExpression` DISABLE KEYS */; -alter table rnaSeqExperimentExpression -add foreign key (expressionId) references expression(expressionId) on delete cascade, -add foreign key (rnaSeqExperimentId) references rnaSeqExperiment(rnaSeqExperimentId) on delete cascade; -/*!40000 ALTER TABLE `rnaSeqExperimentExpression` ENABLE KEYS */; +/*!40000 ALTER TABLE `rnaSeqLibraryIndividualSampleGeneResult` DISABLE KEYS */; +alter table rnaSeqLibraryIndividualSampleGeneResult +add foreign key (rnaSeqLibraryIndividualSampleId) references rnaSeqLibraryIndividualSample(rnaSeqLibraryIndividualSampleId) on delete cascade, +add foreign key (bgeeGeneId) references gene(bgeeGeneId) on delete cascade; +/*!40000 ALTER TABLE `rnaSeqLibraryIndividualSampleGeneResult` ENABLE KEYS */; --- ****** for diff expression ******** +/*!40000 ALTER TABLE `rnaSeqPopulationCaptureToBiotypeExcludedAbsentCalls` DISABLE KEYS */; +alter table rnaSeqPopulationCaptureToBiotypeExcludedAbsentCalls +add foreign key (rnaSeqPopulationCaptureId) references rnaSeqPopulationCapture(rnaSeqPopulationCaptureId) on delete cascade, +add foreign key (geneBioTypeId) references geneBioType(geneBioTypeId) on delete cascade; +/*!40000 ALTER TABLE `rnaSeqPopulationCaptureToBiotypeExcludedAbsentCalls` ENABLE KEYS */; -/*!40000 ALTER TABLE `deaSampleGroupToRnaSeqLibrary` DISABLE KEYS */; -alter table deaSampleGroupToRnaSeqLibrary -add foreign key (deaSampleGroupId) references deaSampleGroup(deaSampleGroupId) on delete cascade, -add foreign key (rnaSeqLibraryId) references rnaSeqLibrary(rnaSeqLibraryId) on delete cascade; -/*!40000 ALTER TABLE `deaSampleGroupToRnaSeqLibrary` ENABLE KEYS */; - -/*!40000 ALTER TABLE `deaRNASeqSummary` DISABLE KEYS */; -alter table deaRNASeqSummary -add foreign key (geneSummaryId) references rnaSeqResult(bgeeGeneId) on delete cascade, -add foreign key (deaSampleGroupId) references deaSampleGroup(deaSampleGroupId) on delete cascade, -add foreign key (differentialExpressionId) references differentialExpression(differentialExpressionId) on delete set null; -/*!40000 ALTER TABLE `deaRNASeqSummary` ENABLE KEYS */; +/*!40000 ALTER TABLE `rnaSeqPopulationCaptureSpeciesMaxRank` DISABLE KEYS */; +alter table rnaSeqPopulationCaptureSpeciesMaxRank +add foreign key (rnaSeqPopulationCaptureId) references rnaSeqPopulationCapture(rnaSeqPopulationCaptureId) on delete cascade, +add foreign key (speciesId) references species(speciesId) on delete cascade; +/*!40000 ALTER TABLE `rnaSeqPopulationCaptureSpeciesMaxRank` ENABLE KEYS */; /*!40000 ALTER TABLE `downloadFile` DISABLE KEYS */; alter table downloadFile diff --git a/bgee-dao-sql/src/test/resources/sql/bgeeIndex.sql b/bgee-dao-sql/src/test/resources/sql/bgeeIndex.sql index d062696b6..e3fe7b6d3 100644 --- a/bgee-dao-sql/src/test/resources/sql/bgeeIndex.sql +++ b/bgee-dao-sql/src/test/resources/sql/bgeeIndex.sql @@ -1,6 +1,9 @@ -- this file contains the indexes that do not add any constraints, defined solely --- for performance issues (unique indexes are therefore not present in this file, --- but in bgeeConstraint.sql) +-- for performance issues or for FK definitions from other tables +-- (unique indexes are therefore not present in this file, but in bgeeConstraint.sql) --- index needed to improve performances when inserting ranks in affymetrixProbeset table -ALTER TABLE affymetrixProbeset ADD INDEX (bgeeGeneId, bgeeAffymetrixChipId); +-- index generated to fasten the retrieval of raw data as proposed in the +-- DBA StackExchange issue : https://dba.stackexchange.com/questions/320207/optimization-with-subquery-not-working-as-expected +-- the improvement provided by these index has not been tested +ALTER TABLE rnaSeqLibraryAnnotatedSampleGeneResult ADD INDEX (rnaSeqLibraryAnnotatedSampleId, expressionId, bgeeGeneId, abundance); +ALTER TABLE cond ADD INDEX(speciesId, conditionId); \ No newline at end of file diff --git a/bgee-dao-sql/src/test/resources/sql/bgeeSchema.sql b/bgee-dao-sql/src/test/resources/sql/bgeeSchema.sql index e56ade701..bba8313b0 100644 --- a/bgee-dao-sql/src/test/resources/sql/bgeeSchema.sql +++ b/bgee-dao-sql/src/test/resources/sql/bgeeSchema.sql @@ -26,42 +26,41 @@ ALTER DATABASE CHARACTER SET utf8 COLLATE utf8_general_ci; -- GENERAL -- **************************************************** create table author ( - authorId smallInt unsigned not null, - authorName varchar(255) not null COMMENT 'Bgee team author names' + authorId smallInt unsigned not null, + authorName varchar(255) not null COMMENT 'Bgee team author names' ) engine = innodb; create table dataSource ( - dataSourceId smallInt unsigned not null, - dataSourceName varchar(255) not null COMMENT 'Data source name', - XRefUrl varchar(255) not null default '' COMMENT 'URL for cross-references to data sources', + dataSourceId smallInt unsigned not null, + dataSourceName varchar(55) not null COMMENT 'Data source name', + XRefUrl varchar(255) not null default '' COMMENT 'URL for cross-references to data sources', -- path to experiment for expression data sources (ArrayExpress, GEO, NCBI, in situ databases, ...) -- parameters such as experimentId are defined by the syntax [experimentId] for instance - experimentUrl varchar(255) not null default '' COMMENT 'URL to experiment for expression data sources', --- path to in situ evidence for in situ databases, --- to Affymetrix chips for affymetrix data + experimentUrl varchar(100) not null default '' COMMENT 'URL to experiment for expression data sources', +-- path to in situ evidence for in situ databases -- parameters such as experimentId are defined by the syntax [experimentId] for instance - evidenceUrl varchar(255) not null default '' COMMENT 'URL to evidence for expression data sources', + evidenceUrl varchar(100) not null default '' COMMENT 'URL to evidence for expression data sources', -- url to the home page of the ressource - baseUrl varchar(255) not null default '' COMMENT 'URL to the home page of data sources', - releaseDate date null COMMENT 'Date of data source used', + baseUrl varchar(100) not null default '' COMMENT 'URL to the home page of data sources', + releaseDate date null COMMENT 'Date of data source used', -- e.g.: Ensembl 87, git version xxx - releaseVersion varchar(255) not null default '' COMMENT 'Version of data source used', - dataSourceDescription TEXT COMMENT 'Description of data source', + releaseVersion varchar(35) not null default '' COMMENT 'Version of data source used', + dataSourceDescription varchar(200) COMMENT 'Description of data source', -- to define if this dataSource should be displayed on the page listing data sources - toDisplay boolean not null default 0 COMMENT 'Display this data source in listing data source page?', + toDisplay boolean not null default 0 COMMENT 'Display this data source in listing data source page?', -- a cat to organize the display - category enum('', 'Genomics database', 'Proteomics database', - 'In situ data source', 'Affymetrix data source', 'EST data source', 'RNA-Seq data source', - 'Ontology') COMMENT 'Data source category to organize the display', + category enum('', 'Genomics database', 'Proteomics database', + 'In situ data source', 'RNA-Seq data source', + 'Single-cell RNA-Seq data source', 'Ontology') COMMENT 'Data source category to organize the display', -- to organize the display. Default value is the highest value, so that this field is the last to be displayed - displayOrder tinyint unsigned not null default 255 COMMENT 'Data source display ordering' + displayOrder tinyint unsigned not null default 255 COMMENT 'Data source display ordering' ) engine = innodb; create table dataSourceToSpecies ( - dataSourceId smallInt unsigned not null COMMENT 'Data source id', - speciesId mediumint unsigned not null COMMENT 'NCBI species taxon id', - dataType enum('affymetrix', 'est', 'in situ', 'rna-seq') not null COMMENT 'Data type', - infoType enum('data', 'annotation') not null COMMENT 'Information type' + dataSourceId smallInt unsigned not null COMMENT 'Data source id', + speciesId mediumint unsigned not null COMMENT 'NCBI species taxon id', + dataType enum('in situ', 'rna-seq', 'single-cell RNA-Seq') not null COMMENT 'Data type', + infoType enum('data', 'annotation') not null COMMENT 'Information type' ) engine = innodb; create table keyword ( @@ -125,6 +124,7 @@ create table species ( -- (for instance, chimp genome for bonobo species). genomeFilePath varchar(100) not null COMMENT 'GTF annotation path used to map this species in Ensembl FTP', genomeVersion varchar(50) not null, + genomeAssemblyXRef varchar(250) not null default '' COMMENT 'XRef to the genome assembly', dataSourceId smallInt unsigned not null COMMENT 'source for genome information', -- ID of the species whose the genome was used for this species. This is used -- when a genome is not in Ensembl. For instance, for bonobo (ID 9597), we use the chimp @@ -132,7 +132,8 @@ create table species ( -- We don't use a foreign key constraint here, because maybe the species whose the genome -- was used does not have any data in Bgee, and thus is not in the taxon table. -- If the correct genome of the species was used, the value of this field is 0. - genomeSpeciesId mediumint unsigned not null default 0 COMMENT 'NCBI species taxon id used for mapping (0 if the same species)' + genomeSpeciesId mediumint unsigned not null default 0 COMMENT 'NCBI species taxon id used for mapping (0 if the same species)', + devOntologyXRef varchar(250) not null default '' COMMENT 'XRef to the developmental stage ontology of that species' ) engine = innodb; -- which sex values are permitted for each species. @@ -283,6 +284,27 @@ create table anatEntityNameSynonym ( anatEntityNameSynonym varchar(255) not null COMMENT 'Anatomical entity name synonym' ) engine = innodb; +-- Note: +-- * query to obtain list of tissues: +-- we select the list of terms that are descendants of 'UBERON:0001062 anatomical entity', +-- and that are not part of the cell type graph. Indeed, a term such as +-- 'CL:0002252 epithelial cell of esophagus' is both a cell type, +-- and part of esophagus. We don't want to retrieve those. The query is: +-- ``` +-- select distinct t1.anatEntityId from anatEntity as t1 +-- inner join anatEntityRelation as t2 on t1.anatEntityId = t2.anatEntitySourceId +-- and t2.anatEntityTargetId = 'UBERON:0001062' and t2.relationType = 'is_a part_of' +-- left outer join anatEntityRelation as t3 on t1.anatEntityId = t3.anatEntitySourceId +-- and t3.anatEntityTargetId = 'GO:0005575' and t3.relationType = 'is_a part_of' +-- where t3.anatEntitySourceId is null; +-- ``` +-- * query to obtain list of cell types: +-- simply retrieve the descendants of 'GO:0005575 cellular component'. The query is: +-- ``` +-- select distinct t1.anatEntityId from anatEntity as t1 +-- inner join anatEntityRelation as t2 on t1.anatEntityId = t2.anatEntitySourceId +-- where t2.anatEntityTargetId = 'GO:0005575' and relationType = 'is_a part_of'; +-- ``` create table anatEntityRelation ( anatEntityRelationId int unsigned not null, anatEntitySourceId varchar(20) not null COMMENT 'Anatomical entity source id', @@ -417,29 +439,6 @@ create table rawSimilarityAnnotation ( -- **************************************************** -- GENE AND TRANSCRIPT INFO -- **************************************************** --- Hierarchical Orthologous Groups from OMA. - --- All the nodes of a particular group are stored in a nested set model. --- A node in the tree could be a speciation node or a duplication node. --- The OMANodeLeftBound and OMANodeRightBound correspond to the left and right bound IDs of the nested set model. --- Note: to use the nested set model, we often need to join this table to itself, --- using a range condition on left and right bounds for the join clause; sadly, --- there is a performance issue for such queries in MySQL, see --- http://www.percona.com/blog/2010/05/17/joining-on-range-wrong/ -create table OMAHierarchicalGroup ( - -- A unique ID for each node inside an OMA Hierarchical Orthologous Group. - -- Auto generated by us, unique over all groups (use as primary key) - OMANodeId int unsigned not null COMMENT 'OMA Hierarchical Orthologous node id', - -- The ID of Hierarchical Orthologous Group as provided by OMA. - -- Only for Xref purpose. - OMAGroupId varchar(255) not null COMMENT 'OMA Hierarchical Orthologous Group id', - -- Bounds generated over all groups. - OMANodeLeftBound int unsigned not null COMMENT 'OMA left bound id in the nested set model', - OMANodeRightBound int unsigned not null COMMENT 'OMA right bound id in the nested set model', - -- The ID corresponding to the level of taxonomy as in NCBI. - -- Some nodes have no taxonomy ID because they correspond to a duplication node (paralogy group). - taxonId mediumint unsigned COMMENT 'NCBI taxon id corresponding to the level of taxonomy' -) engine = innodb; create table geneOntologyTerm ( goId char(10) not null COMMENT 'Gene Ontology id', @@ -481,21 +480,19 @@ create table geneParalogs ( create table gene ( -- warning, maybe this bgeeGeneId will need to be changed to an 'int' when we reach around 200 species bgeeGeneId mediumint unsigned not null COMMENT 'Numeric internal gene ID used for improving performances', - geneId varchar(20) not null COMMENT 'Real gene id', + geneId varchar(64) not null COMMENT 'Real gene id', geneName varchar(255) not null default '' COMMENT 'Gene name', geneDescription TEXT COMMENT 'Gene description', speciesId mediumint unsigned not null COMMENT 'NCBI species taxon id this gene belongs to', -- TODO: check if we should add 'not null' to geneBioTypeId. -- This depends on pipeline. If we update biotype after insertion of gene, it's not possible to set 'not null'. geneBioTypeId smallint unsigned COMMENT 'Gene BioType id (type of gene)', --- can be null if the gene does not belong to a hierarchical group --- a gene can belong to one and only one group --- OMA parent node ID instead of OMA node ID to avoid create group for all genes - OMAParentNodeId int unsigned default null COMMENT 'OMA Hierarchical Orthologous parent node id', -- defines whether the gene ID is present in Ensembl. For some species, they are not -- (for instance, bonobo; we use chimp genome) ensemblGene boolean not null default 1 COMMENT 'Is the gene in Ensembl (default) (= 1), if not (= 0)', - geneMappedToGeneIdCount tinyint unsigned not null default 1 COMMENT 'number of genes in the Bgee database with the same gene ID. In Bgee, for some species with no genome available, we use the genome of a closely-related species, such as chimpanzee genome for analyzing bonobo data. For this reason, a same gene ID can be mapped to several species in Bgee. The value returned here is equal to 1 when the gene ID is uniquely used in the Bgee database.' + seqRegionName varchar(255) not null default '' COMMENT 'Chromosomal or assembly name where this gene is', + geneMappedToGeneIdCount tinyint unsigned not null default 1 COMMENT 'number of genes in the Bgee database with the same Ensembl gene ID. In Bgee, for some species with no genome available, we use the genome of a closely-related species, such as chimpanzee genome for analyzing bonobo data. For this reason, a same Ensembl gene ID can be mapped to several species in Bgee. The value returned here is equal to 1 when the Ensembl gene ID is uniquely used in the Bgee database.', + expressionSummary varchar(255) not null default '' COMMENT 'Sentence generated from propagated expression calls that summarizes the expression for the anatomical entity and the celltype' ) engine = innodb; create table geneNameSynonym ( @@ -542,11 +539,12 @@ create table transcript ( -- **************************************************** -- 'condition' is a reserved keyword in MySQL, we can't use it as table name create table cond ( - conditionId mediumint unsigned not null COMMENT 'Internal condition ID. Each condition is species-specific', - exprMappedConditionId mediumint unsigned not null COMMENT 'the condition ID that should be used for insertion into the expression table: too-granular conditions (e.g., 43 yo human stage, or sexInferred=1) are mapped to less granular conditions for summary. Equal to conditionId if condition is not too granular.', - anatEntityId varchar(20) not null COMMENT 'Uberon anatomical entity ID', - stageId varchar(20) not null COMMENT 'Uberon stage ID', - speciesId mediumint unsigned not null COMMENT 'NCBI species taxon ID', + conditionId mediumint unsigned not null COMMENT 'Internal condition ID. Each condition is species-specific', + exprMappedConditionId mediumint unsigned not null COMMENT 'the condition ID that should be used for insertion into the expression table: too-granular conditions (e.g., 43 yo human stage, or sexInferred=1) are mapped to less granular conditions for summary. Equal to conditionId if condition is not too granular.', + anatEntityId varchar(20) not null COMMENT 'Uberon anatomical entity ID', + cellTypeId varchar(20) default null COMMENT 'A second uberon anatomical entity ID used to manage composition of anatomical entities. Used only for single cell data for postcomposition of anatomical entity ID and cell type ID', + stageId varchar(20) not null COMMENT 'Uberon stage ID', + speciesId mediumint unsigned not null COMMENT 'NCBI species taxon ID', -- NA: not available from source information -- not annotated: information not captured by Bgee -- If an ENUM column is declared NOT NULL, its default value is the first element of the list @@ -566,18 +564,23 @@ create table remapCond ( remappedConditionId mediumint unsigned not null ) engine = innodb COMMENT 'This table is used as an intermediary step for condition remapping, see remap_conditions.pl'; +create table remapExpression ( + incorrectExpressionId int unsigned not null, + remappedExpressionId int unsigned not null +) engine = innodb COMMENT 'This table is used as an intermediary step for condition remapping, see remap_conditions.sql'; + create table globalCond ( globalConditionId mediumint unsigned not null, anatEntityId varchar(20) COMMENT 'Uberon anatomical entity ID. Can be null in this table if this condition aggregates data according to other condition parameters (e.g., grouping all data in a same stage whatever the organ is).', + cellTypeId varchar(20) default null COMMENT 'A second uberon anatomical entity ID used to manage composition of anatomical entities. Used only for single cell data for postcomposition of anatomical entity ID and cell type ID', stageId varchar(20) COMMENT 'Uberon stage ID. Can be null in this table if this condition aggregates data according to other condition parameters (e.g., grouping all data in a same organ whatever the dev. stage is).', speciesId mediumint unsigned not null COMMENT 'NCBI species taxon ID', -- NA: not available from source information -- not annotated: information not captured by Bgee -- If an ENUM column is declared NOT NULL, its default value is the first element of the list --- In this table, only 'not annotated' is used to replace 'NA', as for conditions --- used in expression table - sex enum('not annotated', 'hermaphrodite', 'female', 'male', 'mixed') - COMMENT 'Sex information. NA: not available from source information; not annotated: not used in this table, since all conditions used in the expression tables have "NA" replaced with "not annotated". Can be null in this table if this condition aggregates data according to other condition parameters (e.g., grouping all data in a same organ whatever the sex is).', +-- In this table, only 'any' is used to replace 'not annotated', 'NA', 'mixed' +-- and also represents the propagation of calls along the sex 'ontology'. + sex enum('any', 'hermaphrodite', 'female', 'male'), -- For now, strains are captured as free-text format, only 4 term are "standardized": -- 'NA', 'not annotated', 'wild-type', 'confidential_restricted_data'. -- In this table, only 'wild-type' is used to replace 'NA', 'not annotated', and @@ -587,211 +590,30 @@ create table globalCond ( -- ** RANKS ** -- max ranks in each data type and condition, notably used to allow normalization --- between data types and conditions. For EST and in situ data, they are also used for computation --- of weighted mean between data types: for these data types, because we pool together all data +-- between data types and conditions. For in situ data, they are also used for computation +-- of weighted mean between data types: for these data type, because we pool together all data -- in a same condition, instead of computing a mean between samples, and because we use "dense ranking" -- instead of fractional ranking (so that the max rank is equal to the number of distinct ranks), -- it is irrelevant to consider a sum of the number of distinct ranks in each sample for weighting --- the mean, as for Affymetrix and EST data. +-- the mean. -- Note: these values are the same for all genes in a condition-species, this is why they are stored in this table. - affymetrixMaxRank decimal(9,2) unsigned, - rnaSeqMaxRank decimal(9,2) unsigned, - estMaxRank decimal(9,2) unsigned, - inSituMaxRank decimal(9,2) unsigned, - - affymetrixGlobalMaxRank decimal(9,2) unsigned COMMENT 'This max rank is computed by taking into account all data in this condition, but also in all child conditions.', - rnaSeqGlobalMaxRank decimal(9,2) unsigned COMMENT 'This max rank is computed by taking into account all data in this condition, but also in all child conditions.', - estGlobalMaxRank decimal(9,2) unsigned COMMENT 'This max rank is computed by taking into account all data in this condition, but also in all child conditions.', - inSituGlobalMaxRank decimal(9,2) unsigned COMMENT 'This max rank is computed by taking into account all data in this condition, but also in all child conditions.' -) engine = innodb COMMENT 'This table contains all condition used in the globalExpression table. It thus includes "real" conditions used in the raw expression table, but mostly conditions resulting from the propagation of expression calls in the globalExpression table. It results from the computation of propagated calls according to different condition parameters combination (e.g., grouping all data in a same anat. entity, or all data in a same anat. entity - stage, or data in anat. entity - sex). This is why the fields anatEntityId, stageId, sex, strain, can be null in this table (but not all of them at the same time).'; - -create table globalCondToCond ( - globalConditionId mediumint unsigned not null, - conditionId mediumint unsigned not null, - conditionRelationOrigin enum('self', 'descendant', 'parent') not null COMMENT 'Define whether the data from the raw conditions used for production of global calls in this global condition comes from raw conditions mapped to the globalCondition itself, a descendant global condition, or a parent global condition.' -) engine = innodb -comment = 'this table allows to link globalConditions to the raw conditions that were aggregated to produce global expression calls in the globalExpression table.'; - --- **************************************************** --- RAW EST DATA --- **************************************************** -create table estLibrary ( - estLibraryId varchar(50) not null, - estLibraryName varchar(255) not null, - estLibraryDescription text, - conditionId mediumint unsigned not null, - dataSourceId smallInt unsigned not null -) engine = innodb; - -create table estLibraryToKeyword ( - estLibraryId varchar(50) not null, - keywordId int unsigned not null -) engine = innodb; - -create table expressedSequenceTag ( - estId varchar(50) not null, --- ESTs have two IDs in Unigene - estId2 varchar(50) not null default '', - estLibraryId varchar(50) not null, - bgeeGeneId mediumint unsigned not null COMMENT 'Internal gene ID', - UniGeneClusterId varchar(70) not null default '', - expressionId int unsigned, --- Warning, qualities must be ordered, the index in the enum is used in many queries - estData enum('no data', 'poor quality', 'high quality') default 'no data' -) engine = innodb; - -create table estLibraryExpression ( - expressionId int unsigned not null, - estLibraryId varchar(50) not null, - estCount mediumint unsigned not null default 0 - comment 'number of ESTs in this library mapped to the gene associated to this expressionId', --- no 'callDirection' column for ESTs, only 'present' calls are generated from ESTs - estLibraryCallQuality enum('poor quality', 'high quality') not null - comment 'Inferred quality for this call based on this library (based on the number of ESTs mapped to gene, see Audic and Claverie 1997). Value "poor quality" instead of "low quality" for historical reasons.' -) engine = innodb -comment = 'This table stores information about expression calls produced from EST libraries, that is then used in Bgee to compute global summary expression calls and qualities. Only "present" calls are generated from ESTs (no "absent" calls).'; - --- **************************************************** --- RAW AFFYMETRIX DATA --- **************************************************** -create table microarrayExperiment ( - microarrayExperimentId varchar(70) not null, - microarrayExperimentName varchar(255) not null default '', - microarrayExperimentDescription text, - dataSourceId smallInt unsigned not null -) engine = innodb; + bulkMaxRank decimal(9,2) unsigned, + singleCellMaxRank decimal(9,2) unsigned, + inSituMaxRank decimal(9,2) unsigned +) engine = innodb COMMENT 'This table includes "real" conditions used in the raw expression table, but mostly conditions resulting from the propagation of expression calls. It results from the computation of propagated calls according to different condition parameters combination (e.g., grouping all data in a same anat. entity, or all data in a same anat. entity - stage, or data in anat. entity - sex). This is why the fields anatEntityId, stageId, sex, strain, can be null in this table (but not all of them at the same time).'; -create table microarrayExperimentToKeyword ( - microarrayExperimentId varchar(70) not null, - keywordId int unsigned not null -) engine = innodb; - -create table chipType ( - chipTypeId varchar(70) not null, - chipTypeName varchar(255) not null, - cdfName varchar(255) not null, - isCompatible tinyint(1) not null default 1, - qualityScoreThreshold decimal(10, 2) unsigned not null default 0, --- percentage of present probesets --- 100.00 - percentPresentThreshold decimal(5, 2) unsigned not null default 0, - --- this field is used for rank computations, and is set after all expression data insertion, --- this is why null value is permitted. - chipTypeMaxRank decimal(9,2) unsigned COMMENT 'The max fractional rank in this chip type (see `rank` field in affymetrixProbeset table)' -) engine = innodb; +CREATE TABLE globalCondRelation ( + sourceGlobalConditionId mediumint unsigned NOT NULL, + targetGlobalConditionId mediumint unsigned NOT NULL +) engine = innodb COMMENT 'This table stores the relations between global conditions, allowing to reconstruct the global condition graph used for call propagation. A relation exists between a source global condition and a target global condition, when the target is a parent of the source in the global condition graph (e.g., when the anatEntityId of the target is a parent of the anatEntityId of the source in the anatomical entity ontology).'; --- this table represents mapping of affymetrix probesets in general, --- not constrainted by the tables chipType and afymetrixProbeset --- (that means for instance that you can insert in this table a mapping --- for a probeset not present in the table affymetrixProbeset) --- => so, NO foreign keys to the tables affymetrixProbeset and chipType. --- moreover, the probeset mapping can be use for other tables --- (deaAffymetrixProbesetGroups) --- create table affymetrixProbesetMapping( --- chipTypeId varchar(70) not null, --- affymetrixProbesetId varchar(70) not null, --- bgeeGeneId mediumint unsigned not null COMMENT 'Internal gene ID' --- ) engine = innodb; - -create table affymetrixChip ( --- affymetrixChipId are not unique (couple affymetrixChipId - microarrayExperimentId is) --- then we need an internal ID to link to affymetrixProbeset --- warning, SMALLINT UNSIGNED only allows for 65535 chips to be inserted (we have 12,996 as of Bgee 14) - bgeeAffymetrixChipId smallint unsigned not null, - affymetrixChipId varchar(255) not null, - microarrayExperimentId varchar(70) not null, --- define only if CEL file available, normalization gcRMA, detection schuster - chipTypeId varchar(70), - scanDate varchar(70) not null default '', --- An enum listing the different methods used ib Bgee --- to normalize Affymetrix data: --- * MAS5: normalization using the MAS5 software. Using --- this naormalization usually means that only the processed MAS5 files --- were available, otherwise another method would be used. --- * RMA: normalization by RMA method. --- * gcRMA: normalization by gcRMA method. This is the default --- method in Bgee when raw data are available. - normalizationType enum('MAS5', 'RMA', 'gcRMA') not null, --- An enum listing the different methods to generate expression calls --- on Affymetrix chips: --- * MAS5: expression calls from the MAS5 software. Such calls --- are usually taken from a processed MAS5 file, and imply that the data --- were also normalizd using MAS5. --- * Schuster: Wilcoxon test on the signal of probesets --- against a subset of weakly expressed probesets, to generate expression calls --- (see https://www.ncbi.nlm.nih.gov/pubmed/17594492). Such calls usually implies --- that raw data were available, and were normalized using gcRMA. - detectionType enum('MAS5', 'Schuster') not null, +create table condToSelfGlobalCond ( conditionId mediumint unsigned not null, --- arIQR_score Marta score --- can be set to 0 if it is a MAS5 file --- 99999999.99 - qualityScore decimal(10, 2) unsigned not null default 0, --- percentage of present probesets --- 100.00 - percentPresent decimal(5, 2) unsigned not null, - --- the following fields are used for rank computations, and are set after all expression data insertion, --- this is why null value is permitted. - chipMaxRank decimal(9,2) unsigned COMMENT 'The max fractional rank in this chip (see `rank` field in affymetrixProbeset table)', - chipDistinctRankCount mediumint unsigned COMMENT 'The count of distinct rank in this chip (see `rank` field in affymetrixProbeset table, used for weighted mean rank computations)' -) engine = innodb; - -create table affymetrixProbeset ( - affymetrixProbesetId varchar(70) not null, - bgeeAffymetrixChipId smallint unsigned not null, - bgeeGeneId mediumint unsigned not null COMMENT 'Internal gene ID', - normalizedSignalIntensity decimal(13,5) unsigned not null default 0, --- Warning, flags must be ordered, the index in the enum is used in many queries - detectionFlag enum('undefined', 'absent', 'marginal', 'present') not null default 'undefined', - expressionId int unsigned, --- rank is not "not null" because we update this information afterwards. --- note that this corresponds to the rank of the gene, not of the probeset --- (so, all probesets mapped to a same gene have the same rank, based on its highest signal intensity) - rank decimal(9, 2) unsigned, --- Warning, qualities must be ordered, the index in the enum is used in many queries - affymetrixData enum('no data', 'poor quality', 'high quality') not null default 'no data', --- When expressionId is null, the result is not used for the summary of expression. --- Reasons are: --- * pre filtering: Probesets always seen as "absent" or "marginal" over the whole dataset are removed --- * noExpression conflict: a "noExpression" result has been removed because of expression in a sub-condition. --- Note: as of Bgee 14, we haven't remove this reason for exclusion, but we don't use it for now, --- as we might want to take into account noExpression in parent conditions for generating --- a global expression calls, where there is expression in a sub-condition. --- Maybe we'll discard them again, but I don't think so, it'll allow to present absolutely --- all data available about a call to users. --- * undefined: only 'undefined' calls have been seen --- --- Note that, as of Bgee 14, 2 reasons for exclusion were removed: 'bronze quality' and 'absent low quality'. --- 'bronze quality' exclusion was removed, because now we always propagate expression evidence, --- so a 'bronze quality' call can provide additional evidence to a parent structure. --- 'bronze quality' used to be: for a gene/condition, no "present high" and mix of "present low" and "absent". --- 'absent low quality' was removed, because we now use a same consistent mechanism for present/absent calls, --- taking also into account 'absent low quality' evidence. --- 'absent low quality' used to be: probesets always "absent" for this gene/condition, --- but only seen by MAS5 (that we do not trust = "low quality" - "noExpression" should always be "high quality"). - reasonForExclusion enum('not excluded', 'pre-filtering', - 'noExpression conflict', 'undefined') not null default 'not excluded' -) engine = innodb; - -create table microarrayExperimentExpression ( - expressionId int unsigned not null, - microarrayExperimentId varchar(70) not null, - presentHighMicroarrayChipCount smallint unsigned not null default 0 - comment 'number of chips in this experiment that produced this call as present high quality', - presentLowMicroarrayChipCount smallint unsigned not null default 0 - comment 'number of chips in this experiment that produced this call as present low quality', - absentHighMicroarrayChipCount smallint unsigned not null default 0 - comment 'number of chips in this experiment that produced this call as absent high quality', - absentLowMicroarrayChipCount smallint unsigned not null default 0 - comment 'number of chips in this experiment that produced this call as absent low quality', - microarrayExperimentCallDirection enum('present', 'absent') not null - comment 'Inferred direction for this call based on this experiment ("present" chips always win over "absent" chips)', - microarrayExperimentCallQuality enum('poor quality', 'high quality') not null - comment 'Inferred quality for this call based on this experiment (from all chips, "present high" > "present low" > "absent high" > "absent low"). Value "poor quality" instead of "low quality" for historical reasons.' + globalConditionId mediumint unsigned not null, + -- subsetMask indicates which condition parameters were used to map an observed condition to the corresponding globalCondition. For instance, a subset mask of 3 (binary 11000) indicates that only anatEntityId and stageId were used to define the globalCondition for this mapping. A subset Mask of 7 (binary 11100) indicates that anatEntityId, stageId and cellTypeId were used, and so on. It is really useful to retrieve all observed global condition for a given subset of condition parameters and then be able to subset the condition graph only for these global conditions and their parents. + subsetMask tinyint unsigned NOT NULL COMMENT '5-bit mask, values 1..31. bit 1: anatEntityId, bit 2: stageId, bit 3: celltypeId, bit 4: sex, bit 5: strain' ) engine = innodb -comment = 'This table stores information about expression calls produced from microarray experiments, that is then used in Bgee to compute global summary expression calls and qualities.'; +comment = 'this table allows to link cond to their self globalCondition depending on the subset of condition parameters used to define the globalCondition'; -- **************************************************** -- IN SITU HYBRIDIZATION DATA @@ -836,6 +658,7 @@ create table inSituSpot ( expressionId int unsigned, -- Warning, qualities must be ordered, the index in the enum is used in many queries inSituData enum('no data', 'poor quality', 'high quality') default 'no data', + pValue decimal(31, 30) unsigned default null, -- When expressionId is null, the result is not used for the summary of expression. -- Reasons are: -- * pre filtering: Probesets always seen as "absent" or "marginal" over the whole dataset are removed @@ -855,8 +678,7 @@ create table inSituSpot ( -- taking also into account 'absent low quality' evidence. -- 'absent low quality' used to be: probesets always "absent" for this gene/condition, -- but only seen by MAS5 (that we do not trust = "low quality" - "noExpression" should always be "high quality"). - reasonForExclusion enum('not excluded', 'pre-filtering', - 'noExpression conflict', 'undefined') not null default 'not excluded' + reasonForExclusion enum('not excluded', 'pre-filtering', 'undefined') not null default 'not excluded' ) engine = innodb; create table inSituExperimentExpression ( @@ -877,15 +699,53 @@ create table inSituExperimentExpression ( ) engine = innodb comment = 'This table stores information about expression calls produced from in situ hybridization experiments, that is then used in Bgee to compute global summary expression calls and qualities.'; --- **************************************************** --- RNA-Seq DATA --- **************************************************** +-- this table contains counts and abundance level for each gene at the level of an annotated +-- sample. Each pair of bgeeGeneId and rnaSeqLibraryAnnotatedSampleId is unique. +-- * for bulk RNA-Seq one result corresponds to one gene at one organ level. +-- * for BRB-Seq one result corresponds to one gene at one organ level (after demultiplexing of pooled libraries) +-- * for full length single cell RNA-Seq one result corresponds to one gene at one cell level +-- * for droplet base single cell RNA-Seq one result corresponds to one gene at one cell-type population level (combine all counts of same cell-type per library) +create table rnaSeqLibraryAnnotatedSampleGeneResult ( + rnaSeqLibraryAnnotatedSampleId mediumint unsigned not null COMMENT 'Internal ID used to define one library at one annotated condition', + bgeeGeneId mediumint unsigned not null COMMENT 'Internal gene ID', +-- abundance values inserted here are NOT TMM normalized, +-- these are the raw data before any normalization + abundanceUnit enum ('tpm','cpm'), + abundance decimal(16, 6) not null COMMENT 'abundance values, NOT log transformed', +-- rawRank is not "not null" because we update this information afterwards + rawRank decimal(9, 2) unsigned, +-- for information, measure not normalized for reads or genes lengths + readsCount decimal(16, 6) unsigned not null COMMENT 'As of Bgee 14, read counts are "estimated counts" produced using the Kallisto software. They are not normalized for read or gene lengths.', + UMIsCount decimal(16, 6) unsigned not null COMMENT 'As of Bgee 15, UMI counts are "estimated counts" produced using the Kallisto software. They are not normalized for read or gene lengths.', +-- zScore can be negative + zScore decimal(35, 30), + pValue decimal(31, 30) unsigned default null COMMENT 'present calls are based on the pValue', + expressionId int unsigned, +-- TODO: to remove as not used anymore since Bgee 15. pValues are now used to consider a call as present/absent. + detectionFlag enum('undefined', 'absent', 'present') default 'undefined', +-- When expressionId is null, the result is not used for the summary of expression. +-- Reasons are: +-- * pre filtering: Probesets always seen as "absent" or "marginal" over the whole dataset are removed +-- * noExpression conflict: a "noExpression" result has been removed because of expression in a sub-condition. +-- Note: as of Bgee 14, we haven't remove this reason for exclusion, but we don't use it for now, +-- as we might want to take into account noExpression in parent conditions for generating +-- a global expression calls, where there is expression in a sub-condition. +-- Maybe we'll discard them again, but I don't think so, it'll allow to present absolutely +-- all data available about a call to users. +-- * undefined: only 'undefined' calls have been seen + rnaSeqData enum('no data','poor quality','high quality') default 'no data', + reasonForExclusion enum('not excluded', 'pre-filtering', 'biotype not targeted', + 'undefined') not null default 'not excluded' +) engine = innodb; + create table rnaSeqExperiment ( -- primary exp ID, from GEO, patterns GSExxx rnaSeqExperimentId varchar(70) not null, rnaSeqExperimentName varchar(255) not null default '', rnaSeqExperimentDescription text, - dataSourceId smallInt unsigned not null + dataSourceId smallInt unsigned not null, + numberOfAnnotatedCells int unsigned not null default 0, + DOI varchar(255) ) engine = innodb; create table rnaSeqExperimentToKeyword ( @@ -893,33 +753,28 @@ create table rnaSeqExperimentToKeyword ( keywordId int unsigned not null ) engine = innodb; -create table rnaSeqPlatform ( - rnaSeqPlatformId varchar(255) not null, - rnaSeqPlatformDescription text -) engine = innodb; - --- corresponds to one sample +-- Corresponds to one library in the sense of one sequencing library. It can contains several +-- sample libraries each one of them potentially having different condition in case of library (e.g BRB-Seq) +-- or sample (e.g 10x) multiplexing -- uses to produce several runs create table rnaSeqLibrary ( -- primary ID, from GEO, pattern GSMxxx rnaSeqLibraryId varchar(70) not null, rnaSeqExperimentId varchar(70) not null, - rnaSeqPlatformId varchar(255) not null, - conditionId mediumint unsigned not null, --- TMM normalization factor - tmmFactor decimal(8, 6) not null default 1.0, --- FPKM threshold to consider a gene as expressed - fpkmThreshold decimal(16, 6) not null, --- TPM threshold to consider a gene as expressed - tpmThreshold decimal(16, 6) not null, - allGenesPercentPresent decimal(5, 2) unsigned not null default 0, - proteinCodingGenesPercentPresent decimal(5, 2) unsigned not null default 0, - intergenicRegionsPercentPresent decimal(5, 2) unsigned not null default 0, - thresholdRatioIntergenicCodingPercent decimal(5, 2) unsigned not null default 0 - COMMENT 'Proportion intergenic/coding region used to define the threshold to consider a gene as expressed (should always be 5%, but some libraries do not allow to reach this value)', --- total number of reads in library, including those not mapped. --- In case of paired-end libraries, it's the number of pairs of reads; --- In case of single read, it's the total number of reads + rnaSeqSequencerName varchar(255) not null, + rnaSeqTechnologyName varchar(255) not null, + rnaSeqTechnologyIsSingleCell tinyint(1) not null, + sampleMultiplexing boolean not null default 0, + libraryMultiplexing boolean not null default 0, +-- **** Columns related to the sampling protocol *** +-- TODO: check validity of enum + strandSelection enum ('NA', 'forward', 'revert', 'unstranded'), + cellCompartment enum('NA', 'nucleus', 'cell'), + sequencedTranscriptPart enum ('NA', '3prime', '5prime', 'full length'), + fragmentation smallint unsigned not null default 0 COMMENT 'corresponds to fragmentation of cDNA. 0 for long reads', + rnaSeqPopulationCaptureId varchar(255) not null, + genotype varchar(70), + -- In case of single read, it's the total number of reads allReadsCount bigint unsigned not null default 0, -- total number of reads in library that were mapped to anything. -- if it is not a paired-end library, this number is equal to leftMappedReadsCount @@ -928,15 +783,22 @@ create table rnaSeqLibrary ( -- so we store the min and max read lengths minReadLength int unsigned not null default 0, maxReadLength int unsigned not null default 0, --- Is the library built using paired end? + -- Is the library built using paired end? -- NA: info not used for pseudo-mapping. Default value in an enum is the first one. libraryType enum('NA', 'single', 'paired') not null, - libraryOrientation enum('NA', 'forward', 'reverse', 'unstranded') not null, + usedInPropagatedCalls tinyint(1) not null default 0 +) engine = innodb; --- the following fields are used for rank computations, and are set after all expression data insertion, --- this is why null value is permitted. - rnaSeqLibraryAnnotatedSampleMaxRank decimal(9,2) unsigned COMMENT 'The max fractional rank in this library (see `rank` field in rnaSeqResult table)', - rnaSeqLibraryAnnotatedSampleDistinctRankCount mediumint unsigned COMMENT 'The count of distinct rank in this library (see `rank` field in rnaSeqResult table, used for weighted mean rank computations)' +-- XXX should we keep discarded info at rnaSeqLibrary level, at rnaSeqLibraryAnnotatedSample level, +-- or at both levels? IfrnaSeqLibraryAnnotatedSample level or both do we want to provide condition ? +-- +-- We sometimes discard some runs associated to a library, because of low mappability. +-- We keep track of these discarded runs in this table. +-- UPDATE Bgee 14: for pseudo-mapping using Kallisto, runs are pooled, so we can only exclude libraries, +-- not specific runs. +create table rnaSeqLibraryDiscarded ( + rnaSeqLibraryId varchar(70) not null, + rnaSeqLibraryDiscardReason varchar(255) not null default '' ) engine = innodb; -- Store the information of runs used, pool together to generate the results @@ -947,194 +809,106 @@ create table rnaSeqRun ( rnaSeqLibraryId varchar(70) not null ) engine = innodb; --- We sometimes discard some runs associated to a library, because of low mappability. --- We keep track of these discarded runs in this table. --- UPDATE Bgee 14: for pseudo-mapping using Kallisto, runs are pooled, so we can only exclude libraries, --- not specific runs. -create table rnaSeqLibraryDiscarded ( - rnaSeqLibraryId varchar(70) not null -) engine = innodb; - --- This table contains TPM/RPKM/read count values for each gene for each library --- and link them to an expressionId -create table rnaSeqResult ( +-- corresponds to one library as it was annotated +-- * for bulk RNA-Seq one library corresponds to one sample. For such data there is a +-- 1-to-1 relation between rnaSeqLibrary and rnaSeqLibraryAnnotatedSample. +-- * for BRB-Seq one library contains several libraries pooled together. Each pooled library has its +-- own annotation. For such data there will be 1-to-many relation between rnaSeqLibrary and +-- rnaSeqLibraryAnnotatedSample. +-- * for full length single cell RNA-Seq one library corresponds to one sample. For such data +-- there is a 1-to-1 relation between rnaSeqLibrary and rnaSeqLibraryAnnotatedSample. +-- * for droplet base single cell RNA-Seq one library corresponds to a cell population. Each cell has +-- been annotated with a different barcode and each barcode has its own annotation. For such data +-- there will be 1-to-many relation between rnaSeqLibrary and rnaSeqLibraryAnnotatedSample. +create table rnaSeqLibraryAnnotatedSample ( + rnaSeqLibraryAnnotatedSampleId mediumint unsigned not null, rnaSeqLibraryId varchar(70) not null, + conditionId mediumint unsigned not null, +-- all *AuthorAnnotation columns correspond to free text retrieved from paper by Bgee curators. +-- anatEntityAuthorAnnotation and stageAuthorAnnotation are at the library level they are unique +-- for a given combination of rnaSeqLibraryId and conditionId. However, it is possible to have different +-- cellTypeAuthorAnnotation for a given combination of rnaSeqLibraryId and conditionId as different +-- cellTypeAuthorAnnotation can be annotated with the same cellTypeId, especially when the cell ontology does +-- not contain terms precise enough. + cellTypeAuthorAnnotation varchar(255) not null, + anatEntityAuthorAnnotation varchar(255) not null, + stageAuthorAnnotation varchar(255) not null, + abundanceUnit enum('tpm', 'cpm'), + meanAbundanceReferenceIntergenicDistribution decimal(16, 6) not null default -1 COMMENT 'mean TPM of the distribution of the reference intergenics regions in this library, NOT log transformed', + sdAbundanceReferenceIntergenicDistribution decimal(16, 6) not null default -1 COMMENT 'standard deviation in TPM of the distribution of the reference intergenics regions in this library, NOT log transformed', +-- TMM normalization factor + tmmFactor decimal(8, 6) not null default 1.0, +-- abundance threshold to consider a gene as expressed + abundanceThreshold decimal(16, 6) not null default -1, + allGenesPercentPresent decimal(5, 2) unsigned not null default 0, + proteinCodingGenesPercentPresent decimal(5, 2) unsigned not null default 0, + intergenicRegionsPercentPresent decimal(5, 2) unsigned not null default 0, + pValueThreshold decimal(5, 4) unsigned not null default 0 COMMENT 'pValue threshold used to consider genes present/absent. (for Bgee15 this threshold should always be 0.05)', +-- total number of reads in library, including those not mapped. +-- In case of paired-end libraries, it's the number of pairs of reads; +-- total number of UMIs in library, including those not mapped. + allUMIsCount int unsigned not null default 0, +-- total number of UMIs in library that were mapped to anything. + mappedUMIsCount int unsigned not null default 0, +-- the following fields are used for rank computations, and are set after all expression data insertion, +-- this is why null value is permitted. + rnaSeqLibraryAnnotatedSampleMaxRank decimal(9,2) unsigned COMMENT 'The max fractional rank in this sample (see `rank` field in rnaSeqLibraryAnnotatedSampleGeneResult table)', + rnaSeqLibraryAnnotatedSampleDistinctRankCount mediumint unsigned COMMENT 'The count of distinct rank in this sample (see `rank` field in rnaSeqLibraryAnnotatedSampleGeneResult table, used for weighted mean rank computations)', + multipleLibraryIndividualSample boolean not null default 0 COMMENT 'boolean true if the annotated sample contains several individual samples. e.g true for 10x as one annotated sample corresponds to one cell population and individual sample will correspond to each cell of this cell population', + -- can be null as it is applicable only to pooled bulk samples like BRB-Seq + barcode varchar(70) COMMENT 'barcode used to pool several samples in the same library', +-- these 3 columns have been added to be able to insert precise Salmon condition information + time decimal(5, 2) unsigned default null, + timeUnit varchar(35) default null, + physiologicalStatus varchar(255) default null +) engine = innodb; + +-- TO CLARIFY: +-- * comment from Fred :comes from sample and library demultiplexing. In scRNA-Seq, 1 sample = 1 cell. In bulk, 1 sample = 1 organ for instance) +-- * my feeling : comes only from sample demultiplexing with barcodes describing each cell. For library demultiplexing like BRB-Seq all librariesq +-- are already described in the table `rnaSeqLibraryAnnotatedSample` So for me for BRB-Seq +-- rnaSeqLibraryAnnotatedSample.multipleLibraryIndividualSample == 0. +create table rnaSeqLibraryIndividualSample ( + rnaSeqLibraryIndividualSampleId int unsigned not null, + rnaSeqLibraryAnnotatedSampleId mediumint unsigned not null, + barcode varchar(70) COMMENT 'barcode used to pool several samples in the same library', + sampleName varchar(70), + -- total number of UMIs in library that were mapped to this individual sample + mappedUMIsCount int unsigned not null default 0 +) engine = innodb; + +-- gene result at individual sample level (e.g for each cell for 10x) +create table rnaSeqLibraryIndividualSampleGeneResult ( + rnaSeqLibraryIndividualSampleId int unsigned not null COMMENT 'Internal ID used to define one individual sample', bgeeGeneId mediumint unsigned not null COMMENT 'Internal gene ID', --- FPRKM and TPM values inserted here are NOT TMM normalized, --- these are the raw data before any normalization - fpkm decimal(16, 6) not null COMMENT 'FPKM values, NOT log transformed', - tpm decimal(16, 6) not null COMMENT 'TPM values, NOT log transformed', --- rank is not "not null" because we update this information afterwards - rank decimal(9, 2) unsigned, --- for information, measure not normalized for reads or genes lengths + abundanceUnit enum ('tpm','cpm'), + abundance decimal(16, 6) not null COMMENT 'abundance values, NOT log transformed', readsCount decimal(16, 6) unsigned not null COMMENT 'As of Bgee 14, read counts are "estimated counts" produced using the Kallisto software. They are not normalized for read or gene lengths.', - expressionId int unsigned, - detectionFlag enum('undefined', 'absent', 'present') default 'undefined', --- Warning, qualities must be ordered, the index in the enum is used in many queries. --- We should only see genes with 'high quality' here - rnaSeqData enum('no data', 'poor quality', 'high quality') default 'no data', --- When expressionId is null, the result is not used for the summary of expression. --- Reasons are: --- * pre filtering: Probesets always seen as "absent" or "marginal" over the whole dataset are removed --- * noExpression conflict: a "noExpression" result has been removed because of expression in a sub-condition. --- Note: as of Bgee 14, we haven't remove this reason for exclusion, but we don't use it for now, --- as we might want to take into account noExpression in parent conditions for generating --- a global expression calls, where there is expression in a sub-condition. --- Maybe we'll discard them again, but I don't think so, it'll allow to present absolutely --- all data available about a call to users. --- * undefined: only 'undefined' calls have been seen --- --- Note that, as of Bgee 14, 2 reasons for exclusion were removed: 'bronze quality' and 'absent low quality'. --- 'bronze quality' exclusion was removed, because now we always propagate expression evidence, --- so a 'bronze quality' call can provide additional evidence to a parent structure. --- 'bronze quality' used to be: for a gene/condition, no "present high" and mix of "present low" and "absent". --- 'absent low quality' was removed, because we now use a same consistent mechanism for present/absent calls, --- taking also into account 'absent low quality' evidence. --- 'absent low quality' used to be: probesets always "absent" for this gene/condition, --- but only seen by MAS5 (that we do not trust = "low quality" - "noExpression" should always be "high quality"). - reasonForExclusion enum('not excluded', 'pre-filtering', - 'noExpression conflict', 'undefined') not null default 'not excluded' -) engine = innodb; - --- This table contains TPM/RPKM/read count values for each transcript for each library --- NOTE Bgee 14: as of Bgee 14 this table is not filled -create table rnaSeqTranscriptResult ( - rnaSeqLibraryId varchar(70) not null, - bgeeTranscriptId int unsigned not null COMMENT 'Internal transcript ID', - fpkm decimal(16, 6) not null, - tpm decimal(16, 6) not null, --- for information, measure not normalized for reads or genes lengths - readsCount int unsigned not null -) engine = innodb; - -create table rnaSeqExperimentExpression ( - expressionId int unsigned not null, - rnaSeqExperimentId varchar(70) not null, - presentHighRNASeqLibraryCount smallint unsigned not null default 0 - comment 'number of RNA-Seq libraries in this experiment that produced this call as present high quality', - presentLowRNASeqLibraryCount smallint unsigned not null default 0 - comment 'number of RNA-Seq libraries in this experiment that produced this call as present low quality', - absentHighRNASeqLibraryCount smallint unsigned not null default 0 - comment 'number of RNA-Seq libraries in this experiment that produced this call as absent high quality', - absentLowRNASeqLibraryCount smallint unsigned not null default 0 - comment 'number of RNA-Seq libraries in this experiment that produced this call as absent low quality', - rnaSeqExperimentCallDirection enum('present', 'absent') not null - comment 'Inferred direction for this call based on this experiment ("present" libraries always win over "absent" libraries)', - rnaSeqExperimentCallQuality enum('poor quality', 'high quality') not null - comment 'Inferred quality for this call based on this experiment (from all libraries, "present high" > "present low" > "absent high" > "absent low"). Value "poor quality" instead of "low quality" for historical reasons.' -) engine = innodb -comment = 'This table stores information about expression calls produced from RNA-Seq experiments, that is then used in Bgee to compute global summary expression calls and qualities.'; - --- **************************************************** --- RAW DIFFERENTIAL EXPRESSION ANALYSES --- Note: dea = Differential Expression Analyses ;) --- **************************************************** - --- several differential expression analyses can be performed --- on the same experiment -create table differentialExpressionAnalysis ( - deaId smallint unsigned not null, - detectionType enum('Limma - MCM'), --- defines whether different organs at a same (broad) developmental stage --- were compared ('anatomy'), or a same organ at different developmental stages --- ('development') - comparisonFactor enum('anatomy', 'development'), --- microarrayExperimentId and rnaSeqExperimentId cannot be both null, ot both not null --- at the same time. We use these fields rather than an association table, --- because a DEA can belong to only one experiment, and because this would make --- one join less needed. - microarrayExperimentId varchar(70) default null, - rnaSeqExperimentId varchar(70) default null -) engine = innodb; - --- a DEA can only be performed by comparing different conditions --- (a condition being an organ at a developmental stage), with each condition --- represented by several replicates. Such a group of replicates of a same condition --- in a same DEA is a 'deaSampleGroup'. --- While it would be possible to determine the condition (anatEntityId + stageId) --- of a deaSampleGroup by looking at the individual samples (for instance, --- looking at the condition of an affymetrixChip member of a deaSampleGroup), --- this information is also present in this table (see anatEntityId and stageId fields). --- This is because, for the sake of performing the analyses, too granular --- developmental stages can be mapped to a broader parent stage (for instance, --- mapping '24 yo human' to 'young adult'), otherwise the analyses could be --- meaningless (e.g., performing a DEA on '24 yo human' vs. '25 yo human'). --- So the anatEntityId and stageId in this table can actually be different than --- the annotated anatDevId and stageId of the samples (meaning, different than --- in the table affymetrixChip or rnaSeqLibrary). --- As of Bgee 13, a deaSampleGroup can either be a group of affymetrixChips, --- or a group of rnaSeqLibraries. Their related samples will then be find --- respectively in deaSampleGroupToAffymetrixChip, or deaSampleGroupToRnaSeqLibrary. --- this can be determined by checking in the table differentialExpressionAnalysis --- the fields microarrayExperimentId and rnaSeqExperimentId, to determine whether --- the DEA was using Affymetrix, or RNA-Seq. -create table deaSampleGroup ( - deaSampleGroupId mediumint unsigned not null, - deaId smallint unsigned not null, - conditionId mediumint unsigned not null + UMIsCount decimal(16, 6) unsigned not null , + rnaSeqData enum('no data','poor quality','high quality') default 'no data', + reasonForExclusion enum('not excluded', 'pre-filtering', 'biotype not targeted', + 'undefined') not null default 'not excluded' ) engine = innodb; --- An association table to link an affymetrixChip to the deaSampleGroup it belongs to. --- A same chip can be part of several groups, for instance if it was use for DEAs --- with different comparisonFactors. But all the affymetrixChips inside a deaSampleGroup --- are unique -create table deaSampleGroupToAffymetrixChip ( - deaSampleGroupId mediumint unsigned not null, - bgeeAffymetrixChipId smallint unsigned not null -) engine = innodb; - --- An association table to link a rnaSeqLibrary to the deaSampleGroup it belongs to. --- A same library can be part of several groups, for instance if it was use for DEAs --- with different comparisonFactors. But all the rnaSeqLibraries inside a deaSampleGroup --- are unique -create table deaSampleGroupToRnaSeqLibrary ( - deaSampleGroupId mediumint unsigned not null, - rnaSeqLibraryId varchar(70) not null +-- called protocol until Bgee 15 but updated the name as protocol now regroup +-- a lot of different parameters (e.g population captured, strand, fragmentation size, ...) +create table rnaSeqPopulationCapture ( + rnaSeqPopulationCaptureId varchar(255) not null ) engine = innodb; --- differentialExpressionAnalysisProbesetsSummary --- a line in this table is a summary of a set of probesets, used for the --- differential expression analysis, belonging to different --- affymetrix chips, corresponding to one group of chips -create table deaAffymetrixProbesetSummary ( --- deaAffymetrixProbesetSummaryId corresponds to the IDs of the probesets used for this summary --- (all of them have the same of course). These probesets belong to the affymetrix chips, retrieved using the field `deaChipsGroupId` --- and the table `deaChipsGroupToAffymetrixChip` - deaAffymetrixProbesetSummaryId varchar(70) not null, - deaSampleGroupId mediumint unsigned not null, - bgeeGeneId mediumint unsigned not null COMMENT 'Internal gene ID', - foldChange decimal(7,2) not null default 0, - differentialExpressionId int unsigned, --- Warning, qualities must be ordered, the index in the enum is used in many queries --- 'not expressed' = gene never seen as 'expressed' in the conditions studied ('marginal' is not considered) --- 'no diff expression' = gene has expression, but no significant fold change observed - differentialExpressionAffymetrixData enum('no data', 'not expressed', 'no diff expression', 'poor quality', 'high quality') default 'no data', --- p-value adjusted by Benjamini-Hochberg procedure --- "number of digits to the right of the decimal point (the scale). It has a range of 0 to 30" - deaRawPValue decimal(31, 30) unsigned not null default 1, --- excluded if not expressed in ALL samples in a given analysis --- (it is not excluded if expressed in at least one condition) - reasonForExclusion enum('not excluded', 'not expressed') not null default 'not excluded' +-- this table stores the biotypes for which calls will be generated in a given population captured (e.g. polyA, lncRNA, etc). For instance, for polyA, we might want to generate calls only for biotype having a polyA tail. This table allows to manage this information. +create table rnaSeqPopulationCaptureToBiotype ( + rnaSeqPopulationCaptureId varchar(255) not null COMMENT 'protocol ID for which a biotype will not be used to generate absent calls', + geneBioTypeId smallint unsigned not null COMMENT 'biotype ID for which absent calls will not be generated.' ) engine = innodb; --- deaRNASeqSummary --- a line in this table is a summary of a set of RNA-Seq results, used for the --- differential expression analysis, belonging to different runs, corresponding to one group of runs -create table deaRNASeqSummary ( - geneSummaryId mediumint unsigned not null, - deaSampleGroupId mediumint unsigned not null, - foldChange decimal(7,2) not null default 0, - differentialExpressionId int unsigned, --- Warning, qualities must be ordered, the index in the enum is used in many queries --- 'not expressed' = gene never seen as 'expressed' in the conditions studied ('marginal' is not considered) --- 'no diff expression' = gene has expression, but no significant fold change observed - differentialExpressionRNASeqData enum('no data', 'not expressed', 'no diff expression', 'poor quality', 'high quality') default 'no data', --- p-value adjusted by Benjamini-Hochberg procedure --- "number of digits to the right of the decimal point (the scale). It has a range of 0 to 30" - deaRawPValue decimal(31, 30) unsigned not null default 1, --- excluded if not expressed in ALL samples in a given analysis --- (it is not excluded if expressed in at least one condition) - reasonForExclusion enum('not excluded', 'not expressed') not null default 'not excluded' +-- this table stores the max rank for each population captured , presence or not of multiplexing, is single cell, and species. This max rank is used for normalization. +create table rnaSeqPopulationCaptureSpeciesMaxRank ( + rnaSeqPopulationCaptureId varchar(255) not null, + rnaSeqTechnologyIsSingleCell tinyint unsigned not null, + sampleMultiplexing tinyint unsigned not null, + speciesId mediumint unsigned not null, + maxRank decimal(9,2) unsigned not null COMMENT 'The max fractional rank in this protocol and species (see `rank` field in rnaSeqLibraryAnnotatedSampleGeneResult table)' ) engine = innodb; -- **************************************************** @@ -1147,286 +921,33 @@ create table deaRNASeqSummary ( create table expression ( expressionId int unsigned not null COMMENT 'Internal expression ID, not stable between releases.', bgeeGeneId mediumint unsigned not null COMMENT 'Internal gene ID, not stable between releases.', - conditionId mediumint unsigned not null COMMENT 'ID of condition in the related condition table ("cond"), not stable between releases.' + conditionId mediumint unsigned not null COMMENT 'ID of condition in the related condition table ("cond"), not stable between releases.', + bulkScore decimal(9,2) unsigned not null COMMENT 'The score of the expression call in the bulk RNA-Seq protocol', + bulkPValue decimal(31,30) unsigned not null COMMENT 'The p-value of the expression call in the bulk RNA-Seq protocol', + bulkWeight bigint unsigned not null COMMENT 'The weight of the expression call in the bulk RNA-Seq protocol', + bulkNumberObs smallint unsigned not null COMMENT 'The number of observations for the expression call in the bulk RNA-Seq protocol', + fullLengthScore decimal(9,2) unsigned not null COMMENT 'The score of the expression call in the full-length single-cell RNA-Seq protocol', + fullLengthPValue decimal(31,30) unsigned not null COMMENT 'The p-value of the expression call in the full-length single-cell RNA-Seq protocol', + fullLengthWeight bigint unsigned not null COMMENT 'The weight of the expression call in the full-length single-cell RNA-Seq protocol', + fullLengthNumberObs smallint unsigned not null COMMENT 'The number of observations for the expression call in the full-length single-cell RNA-Seq protocol', + dropletScore decimal(9,2) unsigned not null COMMENT 'The score of the expression call in the droplet-based single-cell RNA-Seq protocol', + dropletPValue decimal(31,30) unsigned not null COMMENT 'The p-value of the expression call in the droplet-based single-cell RNA-Seq protocol', + dropletWeight bigint unsigned not null COMMENT 'The weight of the expression call in the droplet-based single-cell RNA-Seq protocol', + dropletNumberObs smallint unsigned not null COMMENT 'The number of observations for the expression call in the droplet-based single-cell RNA-Seq protocol', + inSituScore decimal(9,2) unsigned not null COMMENT 'The score of the expression call in the in situ hybridization protocol', + inSituPValue decimal(31,30) unsigned not null COMMENT 'The p-value of the expression call in the in situ hybridization protocol', + inSituWeight bigint unsigned not null COMMENT 'The weight of the expression call in the in situ hybridization protocol', + inSituNumberObs smallint unsigned not null COMMENT 'The number of observations for the expression call in the in situ hybridization protocol' ) engine = innodb comment = 'This table is a summary of expression calls for a given gene-condition (anatomical entity - developmental stage - sex- strain), over all the experiments and data types, with no propagation nor experiment expression summary.'; --- This table is a summary of expression calls for a given gene-condition --- gene - anatomical entity - developmental stage - sex- strain, over all the experiments --- for all data types, with all data propagated and reconciled, with experiment expression summaries computed. --- DESIGN note: this table uses an ugly design with enumerated columns. For a discussion about this decision, --- see http://stackoverflow.com/q/42781299/1768736 -create table globalExpression ( - globalExpressionId int unsigned not null COMMENT 'Internal expression ID, not stable between releases.', - bgeeGeneId mediumint unsigned not null COMMENT 'Internal gene ID, not stable between releases.', - globalConditionId mediumint unsigned not null COMMENT 'ID of condition in the related condition table ("cond"), not stable between releases.', - --- ** OBSERVED DATA STATES ** -- --- It is not enough to only know that there are some data in the condition itself --- (see 'SELF' expression summaries below), because we need to distinguish between --- data propagated along a condition parameter (e.g., a developmental stage), --- but observed in another (e.g., an anat. entity). --- Note that these enum values must stay in sync with org.bgee.model.dao.api.expressiondata.call.DAOPropagationState. --- And these enum are null when the expression calls were propagated without taking into account --- the related condition parameter. --- Note that for EST data, there is no propagation of ABSENT calls from parent conditions, --- so this simplifies its enum. - estAnatEntityPropagationState ENUM('self', 'descendant', 'self and descendant') COMMENT 'The origin of the propagated EST data related to the anatomical entity of the related condition. If null, it means that anatomical entities were not considered in this propagated call, or that the call is not supported by any EST data.', - estStagePropagationState ENUM('self', 'descendant', 'self and descendant') COMMENT 'The origin of the propagated EST data related to the developmental stage of the related condition. If null, it means that developmental stages were not considered in this propagated call, or that the call is not supported by any EST data.', - estConditionObservedData BOOLEAN COMMENT 'Whether some EST data were observed in this condition itself. If null, it means that the call is not supported by any EST data. This field is redundant as compared to the "self" experiment counts below, but is more practical to use.', - - affymetrixAnatEntityPropagationState ENUM('all', 'self', 'ancestor', 'descendant', 'self and ancestor', - 'self and descendant', 'ancestor and descendant') COMMENT 'The origin of the propagated Affymetrix data related to the anatomical entity of the related condition. If null, it means that anatomical entities were not considered in this propagated call, or that the call is not supported by any Affymetrix data.', - affymetrixStagePropagationState ENUM('all', 'self', 'ancestor', 'descendant', 'self and ancestor', - 'self and descendant', 'ancestor and descendant') COMMENT 'The origin of the propagated Affymetrix data related to the developmental stage of the related condition. If null, it means that developmental stages were not considered in this propagated call, or that the call is not supported by any Affymetrix data.', - affymetrixConditionObservedData BOOLEAN COMMENT 'Whether some Affymetrix data were observed in this condition itself. If null, it means that the call is not supported by any Affymetrix data. This field is redundant as compared to the "self" experiment counts below, but is more practical to use.', - - inSituAnatEntityPropagationState ENUM('all', 'self', 'ancestor', 'descendant', 'self and ancestor', - 'self and descendant', 'ancestor and descendant') COMMENT 'The origin of the propagated in situ hybridization data related to the anatomical entity of the related condition. If null, it means that anatomical entities were not considered in this propagated call, or that the call is not supported by any in situ hybridization data.', - inSituStagePropagationState ENUM('all', 'self', 'ancestor', 'descendant', 'self and ancestor', - 'self and descendant', 'ancestor and descendant') COMMENT 'The origin of the propagated in situ hybridization data related to the developmental stage of the related condition. If null, it means that developmental stages were not considered in this propagated call, or that the call is not supported by any in situ hybridization data.', - inSituConditionObservedData BOOLEAN COMMENT 'Whether some in situ hybridization data were observed in this condition itself. If null, it means that the call is not supported by any in situ hybridization data. This field is redundant as compared to the "self" experiment counts below, but is more practical to use.', - - rnaSeqAnatEntityPropagationState ENUM('all', 'self', 'ancestor', 'descendant', 'self and ancestor', - 'self and descendant', 'ancestor and descendant') COMMENT 'The origin of the propagated RNA-Seq data related to the anatomical entity of the related condition. If null, it means that anatomical entities were not considered in this propagated call, or that the call is not supported by any RNA-Seq data.', - rnaSeqStagePropagationState ENUM('all', 'self', 'ancestor', 'descendant', 'self and ancestor', - 'self and descendant', 'ancestor and descendant') COMMENT 'The origin of the propagated RNA-Seq data related to the developmental stage of the related condition. If null, it means that developmental stages were not considered in this propagated call, or that the call is not supported by any RNA-Seq data.', - rnaSeqConditionObservedData BOOLEAN COMMENT 'Whether some RNA-Seq data were observed in this condition itself. If null, it means that the call is not supported by any RNA-Seq data. This field is redundant as compared to the "self" experiment counts below, but is more practical to use.', - --- ** EXPRESSION SUMMARIES ** --- Note: EST data are not used to produce no-expression calls - estLibPresentHighSelfCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of EST libraries showing expression of this gene in this condition (not taking into account sub-conditions) with a high quality.', - estLibPresentLowSelfCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of EST libraries showing expression of this gene in this condition (not taking into account sub-conditions) with a low quality.', - estLibPresentHighDescendantCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of EST libraries showing expression of this gene, solely in the sub-conditions of this condition, with a high quality.', - estLibPresentLowDescendantCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of EST libraries showing expression of this gene, solely in the sub-conditions of this condition, with a low quality.', - estLibPresentHighTotalCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of EST libraries showing expression of this gene in this condition or in sub-conditions with a high quality.', - estLibPresentLowTotalCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of EST libraries showing expression of this gene in this condition or in sub-conditions with a low quality.', - estLibPropagatedCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of EST libraries used to show presence of expression (low or high) in sub-conditions of this condition.', - - affymetrixExpPresentHighSelfCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of Affymetrix experiments showing expression of this gene in this condition (not taking into account sub-conditions) with a high quality.', - affymetrixExpPresentLowSelfCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of Affymetrix experiments showing expression of this gene in this condition (not taking into account sub-conditions) with a low quality.', - affymetrixExpAbsentHighSelfCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of Affymetrix experiments showing absence of expression of this gene in this condition (not taking into account parent conditions) with a high quality.', - affymetrixExpAbsentLowSelfCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of Affymetrix experiments showing absence of expression of this gene in this condition (not taking into account parent conditions) with a low quality.', - affymetrixExpPresentHighDescendantCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of Affymetrix experiments showing expression of this gene, solely in the sub-conditions of this condition, with a high quality.', - affymetrixExpPresentLowDescendantCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of Affymetrix experiments showing expression of this gene, solely in the sub-conditions of this condition, with a low quality.', - affymetrixExpAbsentHighParentCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of Affymetrix experiments showing absence of expression of this gene, solely in the valid parent conditions of this condition, with a high quality.', - affymetrixExpAbsentLowParentCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of Affymetrix experiments showing absence of expression of this gene, solely in the valid parent conditions of this condition, with a low quality.', - affymetrixExpPresentHighTotalCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of Affymetrix experiments showing expression of this gene in this condition or in sub-conditions with a high quality.', - affymetrixExpPresentLowTotalCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of Affymetrix experiments showing expression of this gene in this condition or in sub-conditions with a low quality.', - affymetrixExpAbsentHighTotalCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of Affymetrix experiments showing absence of expression of this gene in this condition or valid parent conditions with a high quality.', - affymetrixExpAbsentLowTotalCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of Affymetrix experiments showing absence of expression of this gene in this condition or valid parent conditions with a low quality.', - affymetrixExpPropagatedCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of Affymetrix experiments used either to show presence of expression (low or high) in sub-conditions of this condition, or absence of expression (low or high) in parent conditions of this condition.', - - inSituExpPresentHighSelfCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of in situ hybridization experiments showing expression of this gene in this condition (not taking into account sub-conditions) with a high quality.', - inSituExpPresentLowSelfCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of in situ hybridization experiments showing expression of this gene in this condition (not taking into account sub-conditions) with a low quality.', - inSituExpAbsentHighSelfCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of in situ hybridization experiments showing absence of expression of this gene in this condition (not taking into account parent conditions) with a high quality.', - inSituExpAbsentLowSelfCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of in situ hybridization experiments showing absence of expression of this gene in this condition (not taking into account parent conditions) with a low quality.', - inSituExpPresentHighDescendantCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of in situ hybridization experiments showing expression of this gene, solely in the sub-conditions of this condition, with a high quality.', - inSituExpPresentLowDescendantCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of in situ hybridization experiments showing expression of this gene, solely in the sub-conditions of this condition, with a low quality.', - inSituExpAbsentHighParentCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of in situ hybridization experiments showing absence of expression of this gene, solely in the valid parent conditions of this condition, with a high quality.', - inSituExpAbsentLowParentCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of in situ hybridization experiments showing absence of expression of this gene, solely in the valid parent conditions of this condition, with a low quality.', - inSituExpPresentHighTotalCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of in situ hybridization experiments showing expression of this gene in this condition or in sub-conditions with a high quality.', - inSituExpPresentLowTotalCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of in situ hybridization experiments showing expression of this gene in this condition or in sub-conditions with a low quality.', - inSituExpAbsentHighTotalCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of in situ hybridization experiments showing absence of expression of this gene in this condition or valid parent conditions with a high quality.', - inSituExpAbsentLowTotalCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of in situ hybridization experiments showing absence of expression of this gene in this condition or valid parent conditions with a low quality.', - inSituExpPropagatedCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of in situ hybridization experiments used either to show presence of expression (low or high) in sub-conditions of this condition, or absence of expression (low or high) in parent conditions of this condition.', - - rnaSeqExpPresentHighSelfCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of RNA-Seq experiments showing expression of this gene in this condition (not taking into account sub-conditions) with a high quality.', - rnaSeqExpPresentLowSelfCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of RNA-Seq experiments showing expression of this gene in this condition (not taking into account sub-conditions) with a low quality.', - rnaSeqExpAbsentHighSelfCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of RNA-Seq experiments showing absence of expression of this gene in this condition (not taking into account parent conditions) with a high quality.', - rnaSeqExpAbsentLowSelfCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of RNA-Seq experiments showing absence of expression of this gene in this condition (not taking into account parent conditions) with a low quality.', - rnaSeqExpPresentHighDescendantCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of RNA-Seq experiments showing expression of this gene, solely in the sub-conditions of this condition, with a high quality.', - rnaSeqExpPresentLowDescendantCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of RNA-Seq experiments showing expression of this gene, solely in the sub-conditions of this condition, with a low quality.', - rnaSeqExpAbsentHighParentCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of RNA-Seq experiments showing absence of expression of this gene, solely in the valid parent conditions of this condition, with a high quality.', - rnaSeqExpAbsentLowParentCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of RNA-Seq experiments showing absence of expression of this gene, solely in the valid parent conditions of this condition, with a low quality.', - rnaSeqExpPresentHighTotalCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of RNA-Seq experiments showing expression of this gene in this condition or in sub-conditions with a high quality.', - rnaSeqExpPresentLowTotalCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of RNA-Seq experiments showing expression of this gene in this condition or in sub-conditions with a low quality.', - rnaSeqExpAbsentHighTotalCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of RNA-Seq experiments showing absence of expression of this gene in this condition or valid parent conditions with a high quality.', - rnaSeqExpAbsentLowTotalCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of RNA-Seq experiments showing absence of expression of this gene in this condition or valid parent conditions with a low quality.', - rnaSeqExpPropagatedCount SMALLINT UNSIGNED NOT NULL DEFAULT 0 COMMENT 'Count of RNA-Seq experiments used either to show presence of expression (low or high) in sub-conditions of this condition, or absence of expression (low or high) in parent conditions of this condition.', - --- ** RANKS ** --- For RNA-Seq data: mean ranks before normalization between data types and conditions. --- Used for convenience during rank computations. It corresponds to the following: --- gene ranks are computed for each sample, then a mean is computed for each gene and condition --- of the expression table, weighted by the number of distinct ranks in each sample. - rnaSeqMeanRank decimal(9, 2) unsigned COMMENT 'RNA-Seq mean rank for this gene-condition before normalization over all data types, conditions and species.', --- For Affymetrix: --- mean ranks *after within-datatype normalization*, before normalization between data types and conditions. --- Used for convenience during rank computations. It corresponds to the following: --- ranks are computed for each sample, then "normalized" between samples in a same condition --- of the expression table ("within-datatype normalization", based on the genomic coverage of each chip type); --- then a mean is computed for each gene and condition, weighted by the number of distinct ranks --- in each sample. - affymetrixMeanRank decimal(9, 2) unsigned COMMENT 'Affymetrix mean rank for this gene-condition, after normalization between different chip types, but before normalization over all data types, conditions and species.', --- For EST and in situ data: ranks before normalization between data types and conditions. --- Used for convenience during rank computations. It corresponds to the following: --- For each condition of the expression table, all data are pooled together; they are not first --- analyzed independently per libraries or experiments, as for Affymetrix and RNA-Seq data. --- This is because the genomic coverage of EST or in situ experiments is usually very low, --- and highly variable. Genes are ranked based on number of ESTs or of in situ evidence in each condition. --- They are ranked using "dense ranking" instead of fractional ranking. - estRank decimal(9, 2) unsigned COMMENT 'EST rank for this gene-condition before normalization over all data types, conditions and species. All EST libraries in a same condition are pulled together, so there is no concept of "mean", only a single rank is computed from EST data for each gene-condition.', - inSituRank decimal(9, 2) unsigned COMMENT 'In situ hybridization rank for this gene-condition before normalization over all data types, conditions and species. All in situ evidence in a same condition are pulled together, so there is no concept of "mean", only a single rank is computed from in situ data for each gene-condition.', - --- All ranks are normalized between all data types and conditons, this is what we use to compute --- the global mean rank of a gene in a condition. Basically, the max rank over all data types --- and all conditions is retrieved, and used to normalize all ranks. --- normalized rank = rank * (max of max rank over all conditions and data types) / (max rank for this condition and data type) - rnaSeqMeanRankNorm decimal(9, 2) unsigned COMMENT 'RNA-Seq normalized mean rank for this gene-condition after normalization over all data types, conditions and species, computed from the field rnaSeqMeanRank, and rnaSeqMaxRank in the related condition table.', - affymetrixMeanRankNorm decimal(9, 2) unsigned COMMENT 'Affymetrix normalized mean rank for this gene-condition after normalization over all data types, conditions and species, computed from the field affymetrixMeanRank, and affymetrixMaxRank in the related condition table.', --- For EST and in situ, the rank is not a mean - estRankNorm decimal(9, 2) unsigned COMMENT 'EST normalized rank for this gene-condition after normalization over all data types, conditions and species, computed from the field estRank, and estMaxRank in the related condition table.', - inSituRankNorm decimal(9, 2) unsigned COMMENT 'In situ hybridization normalized rank for this gene-condition after normalization over all data types, conditions and species, computed from the field inSituRank, and inSituMaxRank in the related condition table.', - --- For Affymetrix and RNA-Seq data: sum of the number of distinct ranks in each sample --- where this gene is considered, in this condition and data type (for RNA-Seq: the same set of genes --- is considered in all conditions, so these values are all the same for all genes in a same condition-species; --- for Affymetrix, it depends on the chip types, so it can vary between genes of same condition-species ). --- Distinct ranks in samples are used to weight the mean rank of genes for each data type and condition. --- By storing the sum of the distinct rank count, we will be able to compute the weighted mean --- over all data types in a condition. --- XXX: shoud we store this information in the condition table for RNA-Seq? --- Or maybe we shouldn't constrain to have the same genomic coverage in all libraries of a condition? --- --- For EST and in situ data, this is irrelevant as we pool all data for a same condition together, --- and use dense ranking instead of fractional ranking. As a result, the max rank in each condition --- is used for weighted mean computation between data types. - rnaSeqDistinctRankSum int unsigned COMMENT 'Factor used to weight the RNA-Seq normalized mean rank (rnaSeqMeanRankNorm), to compute a global weighted mean rank between all data types. Corresponds to the sum of distinct ranks in each library mapped to this condition. Note that for EST and in situ data, the max rank found in the related condition table is instead used to compute the weighted mean between data types.', - affymetrixDistinctRankSum int unsigned COMMENT 'Factor used to weight the Affymetrix normalized mean rank (affymetrixMeanRankNorm), to compute a global weighted mean rank between all data types. Corresponds to the sum of distinct ranks in each chip mapped to this condition. Note that for EST and in situ data, the max rank found in the related condition table is instead used to compute the weighted mean between data types.', - --- Same fields, but dedicated to "global" ranks, computed by taking into account --- all data in a condition, but also all data in its descendant conditions. - rnaSeqGlobalMeanRank decimal(9, 2) unsigned COMMENT 'RNA-Seq global mean rank for this gene in this condition and all its descendant conditions, before normalization over all data types, conditions and species.', - affymetrixGlobalMeanRank decimal(9, 2) unsigned COMMENT 'Affymetrix global mean rank for this gene in this condition and all its descendant conditions, after normalization between different chip types, but before normalization over all data types, conditions and species.', - estGlobalRank decimal(9, 2) unsigned COMMENT 'EST global rank for this gene in this condition and all its descendant conditions, before normalization over all data types, conditions and species. All EST libraries in a same condition in this condition and its descendant conditions are pulled together, so there is no concept of "mean", only a single rank is computed from EST data for each gene-condition.', - inSituGlobalRank decimal(9, 2) unsigned COMMENT 'In situ hybridization global rank for this gene in this condition and all its descendant conditions, before normalization over all data types, conditions and species. All in situ evidence in a same condition and its descendant conditions are pulled together, so there is no concept of "mean", only a single rank is computed from in situ data for each gene-condition.', - - rnaSeqGlobalMeanRankNorm decimal(9, 2) unsigned COMMENT 'RNA-Seq normalized mean rank for this gene in this condition and all its descendant conditions, after normalization over all data types, conditions and species, computed from the field rnaSeqMeanRank, and rnaSeqMaxRank in the related condition table.', - affymetrixGlobalMeanRankNorm decimal(9, 2) unsigned COMMENT 'Affymetrix normalized mean rank for this gene in this condition and all its descendant conditions, after normalization over all data types, conditions and species, computed from the field affymetrixMeanRank, and affymetrixMaxRank in the related condition table.', - estGlobalRankNorm decimal(9, 2) unsigned COMMENT 'EST normalized rank for this gene in this condition and all its descendant conditions, after normalization over all data types, conditions and species, computed from the field estRank, and estMaxRank in the related condition table.', - inSituGlobalRankNorm decimal(9, 2) unsigned COMMENT 'In situ hybridization normalized rank for this gene in this condition and all its descendant conditions, after normalization over all data types, conditions and species, computed from the field inSituRank, and inSituMaxRank in the related condition table.', - - rnaSeqGlobalDistinctRankSum int unsigned COMMENT 'Factor used to weight the RNA-Seq normalized global mean rank (rnaSeqGlobalMeanRankNorm), to compute a global weighted mean rank between all data types. Corresponds to the sum of distinct ranks in each library mapped to this condition and all its descendant conditions. Note that for EST and in situ data, the global max rank found in the related condition table is instead used to compute the weighted mean between data types.', - affymetrixGlobalDistinctRankSum int unsigned COMMENT 'Factor used to weight the Affymetrix normalized global mean rank (affymetrixGlobalMeanRankNorm), to compute a global weighted mean rank between all data types. Corresponds to the sum of distinct ranks in each chip mapped to this condition and all its descendant conditions. Note that for EST and in situ data, the global max rank found in the related condition table is instead used to compute the weighted mean between data types.' -) engine = innodb -comment = 'This table is a summary of expression calls for a given gene-condition (anatomical entity - developmental stage - sex- strain), over all the experiments and data types, with all data propagated and reconciled, and with experiment expression summaries computed.'; - --- **************************************************** --- SUMMARY DIFF EXPRESSION CALLS --- **************************************************** - -create table differentialExpression ( - differentialExpressionId int unsigned not null, - bgeeGeneId mediumint unsigned not null COMMENT 'Internal gene ID', - conditionId mediumint unsigned not null, --- defines whether different organs at a same (broad) developmental stage --- were compared ('anatomy'), or a same organ at different developmental stages --- ('development') - comparisonFactor enum('anatomy', 'development'), --- *** Affymetrix *** --- the diff expression call generated by Affymetrix --- 'not expressed' = gene never seen as 'expressed' in the conditions studied ('marginal' is not considered) --- 'no diff expression' = gene has expression, but no significant fold change observed - diffExprCallAffymetrix enum('no data', 'not expressed', 'no diff expression', 'under-expression', 'over-expression') not null default 'no data', --- confidence in the call generated by Affymetrix data --- 'no data' is redundant but it is kept to keep the same indexes for all data states (for instance, rnaSeqData in expression table) - diffExprAffymetrixData enum('no data', 'poor quality', 'high quality') default 'no data', --- among all the analyses using Affymetrix comparing this condition, best p-value associated to this call --- "number of digits to the right of the decimal point (the scale). It has a range of 0 to 30" - bestPValueAffymetrix decimal(31, 30) unsigned not null default 1, --- number of analyses using Affymetrix data where the same call is found - consistentDEACountAffymetrix smallint unsigned not null default 0, --- number of analyses using Affymetrix data where a different call is found - inconsistentDEACountAffymetrix smallint unsigned not null default 0, --- *** RNA-Seq *** --- the diff expression call generated by RNA-Seq --- 'not expressed' = gene never seen as 'expressed' in the conditions studied ('marginal' is not considered) --- 'no diff expression' = gene has expression, but no significant fold change observed - diffExprCallRNASeq enum('no data','not expressed', 'no diff expression', 'under-expression', 'over-expression') not null default 'no data', --- confidence in the call generated by RNA-Seq data --- 'no data' is redundant but it is kept to keep the same indexes for all data states (for instance, rnaSeqData in expression table) - diffExprRNASeqData enum('no data', 'poor quality', 'high quality') default 'no data', --- among all the analyses using RNA-Seq comparing this condition, best p-value associated to this call --- "number of digits to the right of the decimal point (the scale). It has a range of 0 to 30" - bestPValueRNASeq decimal(31, 30) unsigned not null default 1, --- number of analyses using RNA-Seq data where the same call is found - consistentDEACountRNASeq smallint unsigned not null default 0, --- number of analyses using RNA-Seq data where a different call is found - inconsistentDEACountRNASeq smallint unsigned not null default 0 -) engine = innodb; - --- this version of the diff expression table is not considered as of Bgee 13 -/*create table differentialExpression ( - differentialExpressionId int unsigned not null, - bgeeGeneId mediumint unsigned not null COMMENT 'Internal gene ID', - conditionId mediumint unsigned not null, --- defines whether different organs at a same (broad) developmental stage --- were compared ('anatomy'), or a same organ at different developmental stages --- ('development') - comparisonFactor enum('anatomy', 'development'), --- Warning, differentialExpressionCall must be ordered this way, the index in the enum --- is used in many queries - differentialExpressionCall enum('no diff expression', 'under-expression', 'over-expression'), --- the maximum number of conditions compared for which this differential expression call --- is valid. For instance, if a differential expression analysis comparing 3 conditions --- generated a call for a given gene-organ-stage, and another analysis comparing --- 6 conditions generated another call for the same gene-organ-stage (with different --- direction and/or qualities), then maxNumberOfConditions will be 3 for the call --- generated by the first analysis, and 6 for the other call. --- --- But if the two analyses were generating the same call, then they would be only --- one call in this table for the given gene-organ-stade, with a maxNumberOfConditions --- equals to 6 --- --- default is 3, because as of Bgee 13, this is the minimum number of conditions --- to perform a diff expression analysis --- --- Examples of queries: --- --- query to retrieve diff expression calls for a given Gene with no minimum number --- of conditions compared requested: --- select * from differentialExpression as t1 inner join --- ( --- select geneId, organId, stageId, comparisonFactor, min(maxNumberOfConditions) as min --- from differentialExpression where geneId = ? group by geneId, organId, stageId, --- comparisonFactor) --- ) as t2 on t1.geneId = t2.geneId and t1.organId = t2.organId and t1.stageId = t2.stageId --- and t1.comparisonFactor = t2.comparisonFactor and t1.maxNumberOfConditions = t2.min --- where t1.geneId = ?; --- --- Alternatively: TO TEST, IT SEEMS WRONG --- --- select * from differentialExpression as t1 --- where t1.geneId = ? and t1.maxNumberOfConditions = --- (select min(maxNumberOfConditions) from differentialExpression as t2 where --- t2.geneId = t1.geneId and t2.organId = t1.organId and t2.stageId = t1.geneId and --- t2.comparisonFactor = t1.comparisonFactor); --- --- Example of query to select the calls with the maximum number of conditions compared --- for a given gene-organ-stage, with no minimum defined (select only the "best" calls): --- --- select * from differentialExpression as t1 inner join --- ( --- select geneId, organId, stageId, comparisonFactor, max(maxNumberOfConditions) as max --- from differentialExpression where geneId = ? group by geneId, organId, stageId, --- comparisonFactor) --- ) as t2 on t1.geneId = t2.geneId and t1.organId = t2.organId and t1.stageId = t2.stageId --- and t1.comparisonFactor = t2.comparisonFactor and t1.maxNumberOfConditions = t2.max --- where t1.geneId = ?; - maxNumberOfConditions smallint unsigned not null default 3, --- Warning, qualities must be ordered this way, the index in the enum is used in many queries - differentialExpressionAffymetrixData enum('no data', 'poor quality', 'high quality') default 'no data', - differentialExpressionRnaSeqData enum('no data', 'poor quality', 'high quality') default 'no data' -) engine = innodb;*/ - -- select((select count(1) from rnaSeqExperiment) + (select count(1) from rnaSeqLibrary) + (select count(1) from rnaSeqResults) + (select count(1) from rnaSeqExperimentToKeyword) + (select count(1) from affymetrixChip) + (select count(1) from affymetrixProbeset) + (select count(1) from author) + (select count(1) from chipType) + (select count(1) from dataSource) + (select count(1) from dataType) + (select count(1) from deaAffymetrixProbesetSummary) + (select count(1) from deaChipsGroup) + (select count(1) from deaChipsGroupToAffymetrixChip) + (select count(1) from detectionType) + (select count(1) from differentialExpression) + (select count(1) from differentialExpressionAnalysis) + (select count(1) from differentialExpressionAnalysisType) + (select count(1) from estLibrary) + (select count(1) from estLibraryToKeyword) + (select count(1) from expressedSequenceTag) + (select count(1) from expression) + (select count(1) from gene) + (select count(1) from geneBioType) + (select count(1) from geneFamily) + (select count(1) from geneFamilyPredictionMethod) + (select count(1) from geneNameSynonym) + (select count(1) from geneOntologyDescendants) + (select count(1) from geneOntologyTerm) + (select count(1) from geneToTerm) + (select count(1) from geneXRef) + (select count(1) from globalExpression) + (select count(1) from globalExpressionToExpression) + (select count(1) from hogDescendants) + (select count(1) from hogExpression) + (select count(1) from hogExpressionSummary) + (select count(1) from hogExpressionToExpression) + (select count(1) from hogNameSynonym) + (select count(1) from hogRelationship) + (select count(1) from hogXRef) + (select count(1) from homologousOrgansGroup) + (select count(1) from inSituEvidence) + (select count(1) from inSituExperiment) + (select count(1) from inSituExperimentToKeyword) + (select count(1) from inSituSpot) + (select count(1) from keyword) + (select count(1) from metaStage) + (select count(1) from metaStageNameSynonym) + (select count(1) from microarrayExperiment) + (select count(1) from microarrayExperimentToKeyword) + (select count(1) from normalizationType) + (select count(1) from organ) + (select count(1) from organDescendants) + (select count(1) from organNameSynonym) + (select count(1) from organRelationship) + (select count(1) from species) + (select count(1) from stage) + (select count(1) from stageNameSynonym) + (select count(1) from stageXRef)); -- ****************************************** -- AVAILABLE FILES FOR DOWNLOAD -- ****************************************** --- see (https://github.com/BgeeDB/bgee_apps/issues/31) +-- see (https://gitlab.sib.swiss/Bgee/bgee_apps/issues/31) create table downloadFile ( downloadFileId mediumint unsigned not null, -- path relative to the root of the download file directory, including file name @@ -1434,12 +955,13 @@ create table downloadFile ( -- currently, just the name of the file downloadFileName varchar(255) not null, downloadFileDescription text, - downloadFileCategory enum("expr_simple", "expr_complete", "diff_expr_anatomy_complete", "diff_expr_anatomy_simple" - , "diff_expr_dev_complete", "diff_expr_dev_simple", "ortholog", - "affy_annot","rnaseq_annot","affy_data","rnaseq_data"), + downloadFileCategory enum('expr_simple', 'expr_complete', 'diff_expr_anatomy_complete', 'diff_expr_anatomy_simple' + , 'diff_expr_dev_complete', 'diff_expr_dev_simple', 'ortholog', + 'affy_annot','rnaseq_annot','affy_data','rnaseq_data', 'full_length_annot', 'full_length_data', 'droplet_based_annot', + 'droplet_based_data', 'droplet_based_h5ad', 'full_length_h5ad'), speciesDataGroupId mediumint unsigned not null, downloadFileSize int unsigned not null, - downloadFileConditionParameters set('anatomicalEntity', 'developmentalStage') + downloadFileConditionParameters set('anatomicalEntity', 'developmentalStage', 'sex', 'strain') ) engine = innodb; -- ***************************************** diff --git a/bgee-pipeline/src/main/java/org/bgee/pipeline/CommandRunner.java b/bgee-pipeline/src/main/java/org/bgee/pipeline/CommandRunner.java index 4cbf68bd0..ad6ad84b5 100644 --- a/bgee-pipeline/src/main/java/org/bgee/pipeline/CommandRunner.java +++ b/bgee-pipeline/src/main/java/org/bgee/pipeline/CommandRunner.java @@ -13,14 +13,14 @@ import org.bgee.pipeline.annotations.SimilarityAnnotation; import org.bgee.pipeline.easybgee.BgeeToEasyBgee; import org.bgee.pipeline.expression.GenoFishProject; -import org.bgee.pipeline.expression.InsertPropagatedCalls; +import org.bgee.pipeline.expression.InsertPropagatedConditions; +//import org.bgee.pipeline.expression.InsertPropagatedCalls; import org.bgee.pipeline.expression.downloadfile.GenerateExprFile2; import org.bgee.pipeline.expression.downloadfile.GenerateXRefsFilesWithExprInfo; import org.bgee.pipeline.expression.downloadfile.collaboration.GenerateBioSODAFile; import org.bgee.pipeline.expression.downloadfile.collaboration.GenerateOncoMXFile; -import org.bgee.pipeline.expression.downloadfile.GenerateDiffExprFile; +//import org.bgee.pipeline.expression.downloadfile.GenerateDiffExprFile; import org.bgee.pipeline.gene.InsertGO; -import org.bgee.pipeline.gene.ParseOrthoXML; import org.bgee.pipeline.ontologycommon.InsertCIO; import org.bgee.pipeline.ontologycommon.InsertECO; import org.bgee.pipeline.ontologycommon.OntologyTools; @@ -249,10 +249,14 @@ public static void main(String[] args) throws IllegalArgumentException, Exceptio break; //---------- Hierarchical groups ----------- - case "ParseOrthoXML": - ParseOrthoXML.main(newArgs); - break; +// case "ParseOrthoXML": +// ParseOrthoXML.main(newArgs); +// break; + //---------- Condition propagation ----------- + case "InsertPropagatedConditions": + InsertPropagatedConditions.main(newArgs); + break; //---------- Call propagation ----------- case "InsertGlobalCalls": throw log.throwing(new UnsupportedOperationException("Method disabled while updated")); @@ -262,17 +266,17 @@ public static void main(String[] args) throws IllegalArgumentException, Exceptio throw log.throwing(new UnsupportedOperationException("Method disabled while updated")); // FilterNoExprCalls.main(newArgs); // break; - case "InsertPropagatedCalls": - InsertPropagatedCalls.main(newArgs); - break; +// case "InsertPropagatedCalls": +// InsertPropagatedCalls.main(newArgs); +// break; case "CorrectTaxonConstraints": CorrectTaxonConstraints.main(newArgs); break; //---------- Download file generation ----------- - case "GenerateDiffExprFile": - GenerateDiffExprFile.main(newArgs); - break; +// case "GenerateDiffExprFile": +// GenerateDiffExprFile.main(newArgs); +// break; case "GenerateBasicExprFile": GenerateExprFile2.main(newArgs); break; diff --git a/bgee-pipeline/src/main/java/org/bgee/pipeline/MySQLDAOUser.java b/bgee-pipeline/src/main/java/org/bgee/pipeline/MySQLDAOUser.java index cabff54a1..2353dd9e1 100644 --- a/bgee-pipeline/src/main/java/org/bgee/pipeline/MySQLDAOUser.java +++ b/bgee-pipeline/src/main/java/org/bgee/pipeline/MySQLDAOUser.java @@ -16,7 +16,6 @@ import org.bgee.model.dao.mysql.expressiondata.call.MySQLConditionDAO; import org.bgee.model.dao.mysql.expressiondata.call.MySQLDiffExpressionCallDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.MySQLRawExpressionCallDAO; -import org.bgee.model.dao.mysql.expressiondata.rawdata.microarray.MySQLAffymetrixProbesetDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.insitu.MySQLInSituSpotDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.rnaseq.MySQLRNASeqResultAnnotatedSampleDAO; import org.bgee.model.dao.mysql.file.MySQLDownloadFileDAO; @@ -184,12 +183,12 @@ protected MySQLConditionDAO getConditionDAO() { protected MySQLDiffExpressionCallDAO getDiffExpressionCallDAO() { return (MySQLDiffExpressionCallDAO) this.manager.getDiffExpressionCallDAO(); } - /** - * @return A {@code MySQLNoExpressionCallDAO}. - */ - protected MySQLRawExpressionCallDAO getNewRawExpressionCallDAO() { - return (MySQLRawExpressionCallDAO) this.manager.getRawExpressionCallDAO(); - } +// /** +// * @return A {@code MySQLNoExpressionCallDAO}. +// */ +// protected MySQLRawExpressionCallDAO getNewRawExpressionCallDAO() { +// return (MySQLRawExpressionCallDAO) this.manager.getRawExpressionCallDAO(); +// } /** * @return A {@code MySQLAnatEntityDAO}. */ @@ -208,12 +207,6 @@ protected MySQLSpeciesDataGroupDAO getSpeciesDataGroupDAO() { protected MySQLDownloadFileDAO getDownloadFileDAO() { return (MySQLDownloadFileDAO) this.manager.getDownloadFileDAO(); } - /** - * @return A {@code MySQLAffymetrixProbesetDAO}. - */ - protected MySQLAffymetrixProbesetDAO getAffymetrixProbesetDAO() { - return (MySQLAffymetrixProbesetDAO) this.manager.getAffymetrixProbesetDAO(); - } /** * @return A {@code MySQLInSituSpotDAO}. */ diff --git a/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/InsertPropagatedCalls.java b/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/InsertPropagatedCalls.java index 607c9e7b1..48e4c8366 100644 --- a/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/InsertPropagatedCalls.java +++ b/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/InsertPropagatedCalls.java @@ -1,3365 +1,3365 @@ -package org.bgee.pipeline.expression; - -import java.math.BigDecimal; -import java.sql.Connection; -import java.sql.SQLException; -import java.util.AbstractMap; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.NoSuchElementException; -import java.util.Optional; -import java.util.Set; -import java.util.Spliterator; -import java.util.Spliterators; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.LinkedBlockingDeque; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Consumer; -import java.util.function.Function; -import java.util.function.Supplier; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; - -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.Level; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; -import org.apache.logging.log4j.MarkerManager; -import org.bgee.model.ServiceFactory; -import org.bgee.model.anatdev.AnatEntity; -import org.bgee.model.anatdev.DevStage; -import org.bgee.model.dao.api.DAO; -import org.bgee.model.dao.api.DAOManager; -import org.bgee.model.dao.api.exception.DAOException; -import org.bgee.model.dao.api.expressiondata.call.ConditionDAO; -import org.bgee.model.dao.api.expressiondata.call.ConditionDAO.ConditionTO; -import org.bgee.model.dao.api.expressiondata.call.ConditionDAO.GlobalConditionToRawConditionTO; -import org.bgee.model.dao.api.expressiondata.DAODataType; -import org.bgee.model.dao.api.expressiondata.call.DAOFDRPValue; -import org.bgee.model.dao.api.expressiondata.call.GlobalExpressionCallDAO; -import org.bgee.model.dao.api.expressiondata.call.GlobalExpressionCallDAO.GlobalExpressionCallDataTO; -import org.bgee.model.dao.api.expressiondata.call.GlobalExpressionCallDAO.GlobalExpressionCallTO; -import org.bgee.model.dao.api.expressiondata.rawdata.RawExpressionCallDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.RawExpressionCallDAO.RawExpressionCallTO; -import org.bgee.model.dao.api.expressiondata.rawdata.SamplePValueDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.SamplePValueDAO.SamplePValueTO; -import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqResultAnnotatedSampleDAO; -import org.bgee.model.dao.api.expressiondata.rawdata.DAORawDataConditionFilter; -import org.bgee.model.dao.api.expressiondata.rawdata.DAORawDataFilter; -import org.bgee.model.dao.api.expressiondata.rawdata.RawDataCallSourceDAO.CallSourceTO; -import org.bgee.model.dao.api.expressiondata.rawdata.RawDataConditionDAO.RawDataConditionTO; -import org.bgee.model.dao.api.expressiondata.rawdata.RawDataConditionDAO.RawDataConditionTO.DAORawDataSex; -import org.bgee.model.dao.api.expressiondata.rawdata.RawDataConditionDAO.RawDataConditionTOResultSet; -import org.bgee.model.dao.mysql.connector.MySQLDAOManager; -import org.bgee.model.expressiondata.call.Call.ExpressionCall; -import org.bgee.model.expressiondata.call.CallData.ExpressionCallData; -import org.bgee.model.expressiondata.call.CallService; -import org.bgee.model.expressiondata.call.CallServiceUtils; -import org.bgee.model.expressiondata.call.Condition; -import org.bgee.model.expressiondata.call.ConditionGraph; -import org.bgee.model.expressiondata.call.ConditionGraphService; -import org.bgee.model.expressiondata.baseelements.DataPropagation; -import org.bgee.model.expressiondata.baseelements.DataType; -import org.bgee.model.expressiondata.baseelements.FDRPValue; -import org.bgee.model.expressiondata.baseelements.FDRPValueCondition; -import org.bgee.model.expressiondata.baseelements.PropagationState; -import org.bgee.model.expressiondata.rawdata.baseelements.RawDataCondition; -import org.bgee.model.species.Species; -import org.bgee.pipeline.BgeeDBUtils; -import org.bgee.pipeline.CommandRunner; - -import com.google.common.base.Objects; - -/** - * Class responsible for inserting the propagated expression into the Bgee database. - * - * @author Valentine Rech de Laval - * @author Frederic Bastian - * @version Bgee 15.0, Apr. 2021 - * @since Bgee 14, Jan. 2017 - */ -public class InsertPropagatedCalls extends CallService { - private final static Logger log = LogManager.getLogger(InsertPropagatedCalls.class.getName()); - private static final Marker BLOCKING_QUEUE_MARKER = MarkerManager.getMarker("BLOCKING_QUEUE_MARKER"); - private static final Marker INSERTION_MARKER = MarkerManager.getMarker("INSERTION_MARKER"); - private static final Marker COMPUTE_MARKER = MarkerManager.getMarker("COMPUTE_MARKER"); - - /** - * An {@code int} that is the maximum number of levels to propagate expression calls - * to descendant conditions. - */ - public final static int NB_SUBLEVELS_MAX = 1; - /** - * An {@code int} that is the number of genes to load at a same time to propagate calls for, - * and to run computations in parallel between groups of genes of this size. - * The lower this number the higher the number of query to the database, but then they should be fast, - * and the number of threads working in parallel until the end will be higher (for not waiting, - * e.g., that remaining threads handle 2000 genes.) - */ - public final static int GENE_PARALLEL_GROUP_SIZE = 200; - - /** - * The maximum number of {@code Set}s that can be stored in {@link #callsToInsert}. - * If this threshold is exceeded, computation threads will wait for {@link #insertThread} - * to deal with the {@code Set}s already present for insertion (computations can be faster - * than insertion in some cases). - */ - private final static int MAX_NUMBER_OF_CALLS_TO_INSERT = 100; - - //As of Bgee 15, we only need one combination of condition parameters, - //because anyway all calls will be propagated to the root of each parameter. - //So, to retrieve expression in an organ for, e.g., any stage, - //it is possible to simply target the root of all dev. stages. - //For this to work, each ontology must have only one root, so for instance - //we added a unique root to the Uberon anatomical ontology. - private final static Set COND_PARAMS = Collections.unmodifiableSet( - EnumSet.allOf(ConditionDAO.Attribute.class).stream().filter(p -> p.isConditionParameter()) - .collect(Collectors.toSet())); - - private final static AtomicInteger COND_ID_COUNTER = new AtomicInteger(0); - private final static AtomicLong EXPR_ID_COUNTER = new AtomicLong(0); - private final static BigDecimal ZERO_BIGDECIMAL = new BigDecimal("0"); - private final static BigDecimal ABOVE_ZERO_BIGDECIMAL = new BigDecimal("0.000000000000000000000000000001"); - private final static BigDecimal MIN_FDR_BIGDECIMAL = new BigDecimal("0.00000000000001"); - - /** - * A {@code Set} of {@code String}s storing the IDs of anatomical terms corresponding to - * the concept "unknown". To allow a simple blacklisting of "unknown" terms, we will remap them - * to the root of the anat. entity ontology. - */ - private final static Set UNKNOWN_ANAT_ENTITY_IDS = Collections.unmodifiableSet( - new HashSet<>(Arrays.asList("XAO:0003003", "ZFA:0001093"))); - /** - * An {@code AnatEntity} that is the root of the anat. entity ontology. - */ - private final static AnatEntity ROOT_ANAT_ENTITY = new AnatEntity(ConditionDAO.ANAT_ENTITY_ROOT_ID); - - /** - * Main method to insert propagated calls in Bgee database, see {@link #insert(List, Collection)}. - * Parameters that must be provided in order in {@code args} are: - *
      - *
    1. a list of NCBI species IDs (for instance, {@code 9606} for human) that will be used to - * propagate expression, separated by the {@code String} {@link CommandRunner#LIST_SEPARATOR}. - * If empty (see {@link CommandRunner#EMPTY_LIST}), all species in database will be used. - *
    2. a {@code Map} where keys are whatever, and each value is a set of strings, - * corresponding to {@code ConditionDAO.Attribute}s, allowing to target a specific - * condition parameter combination. Example: 1//ANAT_ENTITY_ID,2//ANAT_ENTITY_ID--STAGE_ID - *
    3. an {@code int} defining the offset of the first gene to retrieve, - * for each of the requested species independently. For instance, if two species - * and an offset of 1000 were requested, the first gene retrieved for the first species - * will have offset 1000 among the genes of that species, the first gene retrieved - * for the second species will have offset 1000 among the genes of that other species. - * Can be {@code null} (see {@link CommandRunner#EMPTY_ARG}). - *
    4. an {@code int} defining the count of genes to retrieve, - * for each of the requested species independently. For instance, if two species - * and an gene count of 1000 were requested, 1000 genes will be retrieved for the first species, - * and 1000 genes will be retrieved for the second species. Can be {@code null} - * (see {@link CommandRunner#EMPTY_ARG}). A value of 0 is equivalent of a {@code null} value - * (no effect, all genes for each species are retrieved). - *
    5. A {@code boolean} defining whether global conditions should be computed and inserted - * along with the propagation of calls (if {@code true}), or if there were already computed - * and inserted, and should be retrieved from the database to propagate the calls (if {@code false}). - *
    - * - * @param args An {@code Array} of {@code String}s containing the requested parameters. - * @throws DAOException If an error occurred while inserting the data into the Bgee database. - */ - public static void main(String[] args) throws DAOException { - log.traceEntry("{}", (Object[]) args); - - if (args[0].equals("insertCalls")) { - int expectedArgLength = 6; - - if (args.length != expectedArgLength) { - throw log.throwing(new IllegalArgumentException("Incorrect number of arguments " + - "provided, expected " + expectedArgLength + " arguments, " + args.length + - " provided.")); - } - - List speciesIds = CommandRunner.parseListArgumentAsInt(args[1]); - List condParamArg = CommandRunner.parseListArgument(args[2]); - int geneOffset = CommandRunner.parseArgument(args[3]) == null ? - 0 : Integer.parseInt(CommandRunner.parseArgument(args[3])); - int geneRowCount = CommandRunner.parseArgument(args[4]) == null ? - 0 : Integer.parseInt(CommandRunner.parseArgument(args[4])); - boolean computeInsertGlobalCond = CommandRunner.parseArgumentAsBoolean(args[5]); - //we keep the order of combinations requested by the user - Set condParams = getCondParamsFromArg(condParamArg); - - InsertPropagatedCalls.insert(speciesIds, geneOffset, geneRowCount, computeInsertGlobalCond, - condParams); - } else if (args[0].equals("insertGlobalConditions")) { - int expectedArgLength = 3; - if (args.length != expectedArgLength) { - throw log.throwing(new IllegalArgumentException("Incorrect number of arguments " + - "provided, expected " + expectedArgLength + " arguments, " + args.length + - " provided.")); - } - - List speciesIds = CommandRunner.parseListArgumentAsInt(args[1]); - List condParamArg = CommandRunner.parseListArgument(args[2]); - InsertPropagatedCalls.insertGlobalConditions(speciesIds, getCondParamsFromArg(condParamArg), - DAOManager::getDAOManager, ServiceFactory::new); - } else { - throw log.throwing(new IllegalArgumentException("Unrecognized action: " + args[0])); - } - - log.traceExit(); - } - private static Set getCondParamsFromArg(List arg) { - log.traceEntry("{}", arg); - Set condParams = arg.stream() - .distinct() - .map(p -> ConditionDAO.Attribute.valueOf(p)) - .collect(Collectors.toSet()); - if (condParams.isEmpty()) { - condParams = COND_PARAMS; - } - if (!COND_PARAMS.containsAll(condParams)) { - condParams.removeAll(COND_PARAMS); - throw log.throwing(new IllegalArgumentException("Unrecognized condition parameters: " - + condParams)); - } - return log.traceExit(condParams); - } - - /** - * A {@code Spliterator} allowing to stream over grouped data according - * to provided {@code Comparator} obtained from a main {@code Stream} of {@code CallTO}s - * and one or several {@code Stream}s of {@code ExperimentExpressionTO}s - * and one or several {@code Stream}s of {@code SamplePValueTO}s. - *

    - * This {@code Spliterator} is ordered, sorted, immutable, unsized, and - * contains unique and not {@code null} elements. - * - * @author Valentine Rech de Laval - * @author Frederic Bastian - * @version Bgee 15.0, Mar. 2021 - * @since Bgee 13, Oct. 2016 - * - * @param The type of the objects returned by this {@code CallSpliterator}. - */ - public class CallSpliterator> - extends Spliterators.AbstractSpliterator { - - /** - * A {@code Comparator} only to verify that {@code RawExpressionCallTO} - * {@code Stream} elements are properly ordered. - */ - final private Comparator CALL_TO_COMPARATOR = - Comparator.comparing(RawExpressionCallTO::getBgeeGeneId, Comparator.nullsLast(Comparator.naturalOrder())) - .thenComparing(RawExpressionCallTO::getId, Comparator.nullsLast(Comparator.naturalOrder())); +//package org.bgee.pipeline.expression; +// +//import java.math.BigDecimal; +//import java.sql.Connection; +//import java.sql.SQLException; +//import java.util.AbstractMap; +//import java.util.Arrays; +//import java.util.Collection; +//import java.util.Collections; +//import java.util.Comparator; +//import java.util.EnumSet; +//import java.util.HashMap; +//import java.util.HashSet; +//import java.util.Iterator; +//import java.util.LinkedHashMap; +//import java.util.List; +//import java.util.Map; +//import java.util.Map.Entry; +//import java.util.NoSuchElementException; +//import java.util.Optional; +//import java.util.Set; +//import java.util.Spliterator; +//import java.util.Spliterators; +//import java.util.concurrent.BlockingQueue; +//import java.util.concurrent.ConcurrentHashMap; +//import java.util.concurrent.ConcurrentMap; +//import java.util.concurrent.LinkedBlockingDeque; +//import java.util.concurrent.atomic.AtomicBoolean; +//import java.util.concurrent.atomic.AtomicInteger; +//import java.util.concurrent.atomic.AtomicLong; +//import java.util.function.Consumer; +//import java.util.function.Function; +//import java.util.function.Supplier; +//import java.util.stream.Collectors; +//import java.util.stream.IntStream; +//import java.util.stream.Stream; +//import java.util.stream.StreamSupport; +// +//import org.apache.commons.lang3.StringUtils; +//import org.apache.logging.log4j.Level; +//import org.apache.logging.log4j.LogManager; +//import org.apache.logging.log4j.Logger; +//import org.apache.logging.log4j.Marker; +//import org.apache.logging.log4j.MarkerManager; +//import org.bgee.model.ServiceFactory; +//import org.bgee.model.anatdev.AnatEntity; +//import org.bgee.model.anatdev.DevStage; +//import org.bgee.model.dao.api.DAO; +//import org.bgee.model.dao.api.DAOManager; +//import org.bgee.model.dao.api.exception.DAOException; +//import org.bgee.model.dao.api.expressiondata.call.ConditionDAO; +//import org.bgee.model.dao.api.expressiondata.call.ConditionDAO.ConditionTO; +//import org.bgee.model.dao.api.expressiondata.call.ConditionDAO.GlobalConditionToRawConditionTO; +//import org.bgee.model.dao.api.expressiondata.DAODataType; +//import org.bgee.model.dao.api.expressiondata.call.DAOFDRPValue; +//import org.bgee.model.dao.api.expressiondata.call.GlobalExpressionCallDAO; +//import org.bgee.model.dao.api.expressiondata.call.GlobalExpressionCallDAO.GlobalExpressionCallDataTO; +//import org.bgee.model.dao.api.expressiondata.call.GlobalExpressionCallDAO.GlobalExpressionCallTO; +//import org.bgee.model.dao.api.expressiondata.rawdata.RawExpressionCallDAO; +//import org.bgee.model.dao.api.expressiondata.rawdata.RawExpressionCallDAO.RawExpressionCallTO; +//import org.bgee.model.dao.api.expressiondata.rawdata.SamplePValueDAO; +//import org.bgee.model.dao.api.expressiondata.rawdata.SamplePValueDAO.SamplePValueTO; +//import org.bgee.model.dao.api.expressiondata.rawdata.rnaseq.RNASeqResultAnnotatedSampleDAO; +//import org.bgee.model.dao.api.expressiondata.rawdata.DAORawDataConditionFilter; +//import org.bgee.model.dao.api.expressiondata.rawdata.DAORawDataFilter; +//import org.bgee.model.dao.api.expressiondata.rawdata.RawDataCallSourceDAO.CallSourceTO; +//import org.bgee.model.dao.api.expressiondata.rawdata.RawDataConditionDAO.RawDataConditionTO; +//import org.bgee.model.dao.api.expressiondata.rawdata.RawDataConditionDAO.RawDataConditionTO.DAORawDataSex; +//import org.bgee.model.dao.api.expressiondata.rawdata.RawDataConditionDAO.RawDataConditionTOResultSet; +//import org.bgee.model.dao.mysql.connector.MySQLDAOManager; +//import org.bgee.model.expressiondata.call.Call.ExpressionCall; +//import org.bgee.model.expressiondata.call.CallData.ExpressionCallData; +//import org.bgee.model.expressiondata.call.CallService; +//import org.bgee.model.expressiondata.call.CallServiceUtils; +//import org.bgee.model.expressiondata.call.Condition; +//import org.bgee.model.expressiondata.call.ConditionGraph; +//import org.bgee.model.expressiondata.call.ConditionGraphService; +//import org.bgee.model.expressiondata.baseelements.DataPropagation; +//import org.bgee.model.expressiondata.baseelements.DataType; +//import org.bgee.model.expressiondata.baseelements.FDRPValue; +//import org.bgee.model.expressiondata.baseelements.FDRPValueCondition; +//import org.bgee.model.expressiondata.baseelements.PropagationState; +//import org.bgee.model.expressiondata.rawdata.baseelements.RawDataCondition; +//import org.bgee.model.species.Species; +//import org.bgee.pipeline.BgeeDBUtils; +//import org.bgee.pipeline.CommandRunner; +// +//import com.google.common.base.Objects; +// +///** +// * Class responsible for inserting the propagated expression into the Bgee database. +// * +// * @author Valentine Rech de Laval +// * @author Frederic Bastian +// * @version Bgee 15.0, Apr. 2021 +// * @since Bgee 14, Jan. 2017 +// */ +//public class InsertPropagatedCalls extends CallService { +// private final static Logger log = LogManager.getLogger(InsertPropagatedCalls.class.getName()); +// private static final Marker BLOCKING_QUEUE_MARKER = MarkerManager.getMarker("BLOCKING_QUEUE_MARKER"); +// private static final Marker INSERTION_MARKER = MarkerManager.getMarker("INSERTION_MARKER"); +// private static final Marker COMPUTE_MARKER = MarkerManager.getMarker("COMPUTE_MARKER"); +// +// /** +// * An {@code int} that is the maximum number of levels to propagate expression calls +// * to descendant conditions. +// */ +// public final static int NB_SUBLEVELS_MAX = 1; +// /** +// * An {@code int} that is the number of genes to load at a same time to propagate calls for, +// * and to run computations in parallel between groups of genes of this size. +// * The lower this number the higher the number of query to the database, but then they should be fast, +// * and the number of threads working in parallel until the end will be higher (for not waiting, +// * e.g., that remaining threads handle 2000 genes.) +// */ +// public final static int GENE_PARALLEL_GROUP_SIZE = 200; +// +// /** +// * The maximum number of {@code Set}s that can be stored in {@link #callsToInsert}. +// * If this threshold is exceeded, computation threads will wait for {@link #insertThread} +// * to deal with the {@code Set}s already present for insertion (computations can be faster +// * than insertion in some cases). +// */ +// private final static int MAX_NUMBER_OF_CALLS_TO_INSERT = 100; +// +// //As of Bgee 15, we only need one combination of condition parameters, +// //because anyway all calls will be propagated to the root of each parameter. +// //So, to retrieve expression in an organ for, e.g., any stage, +// //it is possible to simply target the root of all dev. stages. +// //For this to work, each ontology must have only one root, so for instance +// //we added a unique root to the Uberon anatomical ontology. +// private final static Set COND_PARAMS = Collections.unmodifiableSet( +// EnumSet.allOf(ConditionDAO.Attribute.class).stream().filter(p -> p.isConditionParameter()) +// .collect(Collectors.toSet())); +// +// private final static AtomicInteger COND_ID_COUNTER = new AtomicInteger(0); +// private final static AtomicLong EXPR_ID_COUNTER = new AtomicLong(0); +// private final static BigDecimal ZERO_BIGDECIMAL = new BigDecimal("0"); +// private final static BigDecimal ABOVE_ZERO_BIGDECIMAL = new BigDecimal("0.000000000000000000000000000001"); +// private final static BigDecimal MIN_FDR_BIGDECIMAL = new BigDecimal("0.00000000000001"); +// +// /** +// * A {@code Set} of {@code String}s storing the IDs of anatomical terms corresponding to +// * the concept "unknown". To allow a simple blacklisting of "unknown" terms, we will remap them +// * to the root of the anat. entity ontology. +// */ +// private final static Set UNKNOWN_ANAT_ENTITY_IDS = Collections.unmodifiableSet( +// new HashSet<>(Arrays.asList("XAO:0003003", "ZFA:0001093"))); +// /** +// * An {@code AnatEntity} that is the root of the anat. entity ontology. +// */ +// private final static AnatEntity ROOT_ANAT_ENTITY = new AnatEntity(ConditionDAO.ANAT_ENTITY_ROOT_ID); +// +// /** +// * Main method to insert propagated calls in Bgee database, see {@link #insert(List, Collection)}. +// * Parameters that must be provided in order in {@code args} are: +// *

      +// *
    1. a list of NCBI species IDs (for instance, {@code 9606} for human) that will be used to +// * propagate expression, separated by the {@code String} {@link CommandRunner#LIST_SEPARATOR}. +// * If empty (see {@link CommandRunner#EMPTY_LIST}), all species in database will be used. +// *
    2. a {@code Map} where keys are whatever, and each value is a set of strings, +// * corresponding to {@code ConditionDAO.Attribute}s, allowing to target a specific +// * condition parameter combination. Example: 1//ANAT_ENTITY_ID,2//ANAT_ENTITY_ID--STAGE_ID +// *
    3. an {@code int} defining the offset of the first gene to retrieve, +// * for each of the requested species independently. For instance, if two species +// * and an offset of 1000 were requested, the first gene retrieved for the first species +// * will have offset 1000 among the genes of that species, the first gene retrieved +// * for the second species will have offset 1000 among the genes of that other species. +// * Can be {@code null} (see {@link CommandRunner#EMPTY_ARG}). +// *
    4. an {@code int} defining the count of genes to retrieve, +// * for each of the requested species independently. For instance, if two species +// * and an gene count of 1000 were requested, 1000 genes will be retrieved for the first species, +// * and 1000 genes will be retrieved for the second species. Can be {@code null} +// * (see {@link CommandRunner#EMPTY_ARG}). A value of 0 is equivalent of a {@code null} value +// * (no effect, all genes for each species are retrieved). +// *
    5. A {@code boolean} defining whether global conditions should be computed and inserted +// * along with the propagation of calls (if {@code true}), or if there were already computed +// * and inserted, and should be retrieved from the database to propagate the calls (if {@code false}). +// *
    +// * +// * @param args An {@code Array} of {@code String}s containing the requested parameters. +// * @throws DAOException If an error occurred while inserting the data into the Bgee database. +// */ +// public static void main(String[] args) throws DAOException { +// log.traceEntry("{}", (Object[]) args); +// +// if (args[0].equals("insertCalls")) { +// int expectedArgLength = 6; +// +// if (args.length != expectedArgLength) { +// throw log.throwing(new IllegalArgumentException("Incorrect number of arguments " + +// "provided, expected " + expectedArgLength + " arguments, " + args.length + +// " provided.")); +// } +// +// List speciesIds = CommandRunner.parseListArgumentAsInt(args[1]); +// List condParamArg = CommandRunner.parseListArgument(args[2]); +// int geneOffset = CommandRunner.parseArgument(args[3]) == null ? +// 0 : Integer.parseInt(CommandRunner.parseArgument(args[3])); +// int geneRowCount = CommandRunner.parseArgument(args[4]) == null ? +// 0 : Integer.parseInt(CommandRunner.parseArgument(args[4])); +// boolean computeInsertGlobalCond = CommandRunner.parseArgumentAsBoolean(args[5]); +// //we keep the order of combinations requested by the user +// Set condParams = getCondParamsFromArg(condParamArg); +// +// InsertPropagatedCalls.insert(speciesIds, geneOffset, geneRowCount, computeInsertGlobalCond, +// condParams); +// } else if (args[0].equals("insertGlobalConditions")) { +// int expectedArgLength = 3; +// if (args.length != expectedArgLength) { +// throw log.throwing(new IllegalArgumentException("Incorrect number of arguments " + +// "provided, expected " + expectedArgLength + " arguments, " + args.length + +// " provided.")); +// } +// +// List speciesIds = CommandRunner.parseListArgumentAsInt(args[1]); +// List condParamArg = CommandRunner.parseListArgument(args[2]); +// InsertPropagatedCalls.insertGlobalConditions(speciesIds, getCondParamsFromArg(condParamArg), +// DAOManager::getDAOManager, ServiceFactory::new); +// } else { +// throw log.throwing(new IllegalArgumentException("Unrecognized action: " + args[0])); +// } +// +// log.traceExit(); +// } +// private static Set getCondParamsFromArg(List arg) { +// log.traceEntry("{}", arg); +// Set condParams = arg.stream() +// .distinct() +// .map(p -> ConditionDAO.Attribute.valueOf(p)) +// .collect(Collectors.toSet()); +// if (condParams.isEmpty()) { +// condParams = COND_PARAMS; +// } +// if (!COND_PARAMS.containsAll(condParams)) { +// condParams.removeAll(COND_PARAMS); +// throw log.throwing(new IllegalArgumentException("Unrecognized condition parameters: " +// + condParams)); +// } +// return log.traceExit(condParams); +// } +// +// /** +// * A {@code Spliterator} allowing to stream over grouped data according +// * to provided {@code Comparator} obtained from a main {@code Stream} of {@code CallTO}s +// * and one or several {@code Stream}s of {@code ExperimentExpressionTO}s +// * and one or several {@code Stream}s of {@code SamplePValueTO}s. +// *

    +// * This {@code Spliterator} is ordered, sorted, immutable, unsized, and +// * contains unique and not {@code null} elements. +// * +// * @author Valentine Rech de Laval +// * @author Frederic Bastian +// * @version Bgee 15.0, Mar. 2021 +// * @since Bgee 13, Oct. 2016 +// * +// * @param The type of the objects returned by this {@code CallSpliterator}. +// */ +// public class CallSpliterator> +// extends Spliterators.AbstractSpliterator { +// +// /** +// * A {@code Comparator} only to verify that {@code RawExpressionCallTO} +// * {@code Stream} elements are properly ordered. +// */ +// final private Comparator CALL_TO_COMPARATOR = +// Comparator.comparing(RawExpressionCallTO::getBgeeGeneId, Comparator.nullsLast(Comparator.naturalOrder())) +// .thenComparing(RawExpressionCallTO::getId, Comparator.nullsLast(Comparator.naturalOrder())); +//// /** +//// * A {@code Comparator} only to verify that {@code ExperimentExpressionTO} +//// * {@code Stream} elements are properly ordered. This {@code Comparator} is valid only +//// * to compare {@code ExperimentExpressionTO}s for one specific data type and one specific gene. +//// */ +//// final private Comparator EXP_EXPR_TO_COMPARATOR = +//// Comparator.comparing(ExperimentExpressionTO::getExpressionId, +//// Comparator.nullsLast(Comparator.naturalOrder())); +// +// final private Stream callTOs; +// //TODO: javadoc: not final for lazy loading +// private Iterator itCallTOs; +// private RawExpressionCallTO lastCallTO; +// final private Map>> samplePValueTOsByDataType; +// //TODO: javadoc: this map is NOT immutable (but reference is final) +// final private Map>> mapDataTypeToSamplePValueTOIt; +// //TODO: javadoc: this map is NOT immutable (but reference is final) +// final private Map> mapDataTypeToLastSamplePValueTO; +// +// private boolean isInitiated; +// private boolean isClosed; +// +// /** +// * Default constructor. +// * +// * @param callTOs A {@code Stream} of {@code T}s that is the stream of calls. +// * @param samplePValueTOsByDataType A {@code Map} where keys are {@code DataType}s +// * defining data types, the associated value being a +// * {@code Stream} of {@code SamplePValueTO}s +// * storing expression p-values associated with each call. +// */ +// public CallSpliterator(Stream callTOs, +// Map>> samplePValueTOsByDataType) { +// super(Long.MAX_VALUE, Spliterator.ORDERED | Spliterator.IMMUTABLE +// | Spliterator.DISTINCT | Spliterator.NONNULL | Spliterator.SORTED); +// //experimentExprTOsByDataType can be null since we don't use them as of Bge 15.0, +// //but we keep the possibility of easily reactivating it +// if (callTOs == null || /*experimentExprTOsByDataType == null || +// experimentExprTOsByDataType.entrySet().stream() +// .anyMatch(e -> e == null || e.getValue() == null) ||*/ +// samplePValueTOsByDataType == null || +// samplePValueTOsByDataType.entrySet().stream() +// .anyMatch(e -> e == null || e.getValue() == null)) { +// throw new IllegalArgumentException("Provided streams cannot be null"); +// } +// +// this.callTOs = callTOs; +// this.itCallTOs = null; +// this.lastCallTO = null; +// this.samplePValueTOsByDataType = Collections.unmodifiableMap(samplePValueTOsByDataType); +// this.isInitiated = false; +// this.isClosed = false; +// this.mapDataTypeToLastSamplePValueTO = new HashMap<>(); +// this.mapDataTypeToSamplePValueTOIt = new HashMap<>(); +// } +// +// //the line 'action.accept((U) data);' generates a warning for unchecked cast. +// //to avoid it. we would need to parameterize each class definition used in the Map generated +// //by this Spliterator, and provide their class at instantiation (RawExpressionCallTO.class, +// //DataType.class, ExperimentExpressionTO.class, etc): boring. +// @SuppressWarnings("unchecked") +// @Override +// public boolean tryAdvance(Consumer action) { +// log.traceEntry("{}", action); +// +// if (this.isClosed) { +// throw log.throwing(new IllegalStateException("Already close")); +// } +// +// // Lazy loading: we do not get stream iterators (terminal operation) +// // before tryAdvance() is called. +// if (!this.isInitiated) { +// //set it first because method can return false and exist the block +// this.isInitiated = true; +// +// this.itCallTOs = this.callTOs.iterator(); +// try { +// this.lastCallTO = this.itCallTOs.next(); +// } catch (NoSuchElementException e) { +// log.catching(Level.DEBUG, e); +// return log.traceExit(false); +// } +// +// for (Entry>> entry: +// this.samplePValueTOsByDataType.entrySet()) { +// Iterator> it = entry.getValue().iterator(); +// try { +// this.mapDataTypeToLastSamplePValueTO.put(entry.getKey(), it.next()); +// log.trace("mapDataTypeToLastSamplePValueTO: {}", +// this.mapDataTypeToLastSamplePValueTO.get(entry.getKey())); +// //don't store the iterator if there is no element (catch clause) +// this.mapDataTypeToSamplePValueTOIt.put(entry.getKey(), it); +// } catch (NoSuchElementException e) { +// //it's OK to have no element for a given data type +// log.catching(Level.TRACE, e); +// } +// } +// //We should have at least one data type with supporting data +// if (this.mapDataTypeToSamplePValueTOIt.isEmpty()) { +// throw log.throwing(new IllegalStateException("Missing supporting data")); +// } +// } +// +// //if already initialized, no calls retrieved, but method called again (should never happen, +// //as the method would have returned false during initialization above) +// if (this.lastCallTO == null) { +// log.warn("Stream used again despite having no elements."); +// return log.traceExit(false); +// } +// +// //This Set is the element generated by this Stream, on which the Consumer is applied. +// //It retrieves all RawExpressionCallTOs for one given gene, and associates them +// //to their relative ExperimentExpressionTOs and SamplePValueTOs, per data type, +// //into RawExpressionCallDataTOs. +// final Set data = new HashSet<>(); +// +// //we iterate the CallTO ResultSet and stop when we reach the next gene, or when +// //there is no more element, then we do a last iteration after the last TO is +// //retrieved, to properly group all the calls. This is why we use the boolean currentGeneIteration. +// //This loop always work on this.lastCallTO, which has already been populated at this point. +// boolean currentGeneIteration = true; +// while (currentGeneIteration) { +// if (this.lastCallTO.getBgeeGeneId() == null || this.lastCallTO.getId() == null) { +// throw log.throwing(new IllegalStateException("Missing attributes in raw call: " +// + this.lastCallTO)); +// } +// log.trace("currentGeneIteration, lastCallTO: {}", this.lastCallTO); +// // We add the previous ExperimentExpressionTOs to the group +// assert data.stream().noneMatch(rawData -> rawData.getRawExpressionCallTO().getId() +// .equals(this.lastCallTO.getId())); +// data.add(new RawExpressionCallData(this.lastCallTO, +// this.getPValues(this.lastCallTO.getId()))); +// +// RawExpressionCallTO currentCallTO = null; +// //try-catch to avoid calling both hasNext and next +// try { +// currentCallTO = this.itCallTOs.next(); +// currentGeneIteration = true; +// } catch (NoSuchElementException e) { +// currentGeneIteration = false; +// } +// //the calls are supposed to be ordered by ascending gene ID - expression ID +// if (currentCallTO != null && CALL_TO_COMPARATOR.compare(this.lastCallTO, currentCallTO) > 0) { +// throw log.throwing(new IllegalStateException("The expression calls " +// + "were not retrieved in correct order, which is mandatory " +// + "for proper generation of data: previous call: " +// + this.lastCallTO + ", current call: " + currentCallTO)); +// } +// log.trace("Previous call={} - Current call={}", this.lastCallTO, currentCallTO); +// +// //if the gene changes, or if it is the latest iteration (one iteration after +// //the last CallTO was retrieved, this.itCallTOs.next() threw an exception), +// //we generate the data Map for the previous gene, as all data were iterated for that gene. +// if (!currentGeneIteration || !currentCallTO.getBgeeGeneId().equals(this.lastCallTO.getBgeeGeneId())) { +// assert (currentGeneIteration && currentCallTO != null) || (!currentGeneIteration && currentCallTO == null); +// currentGeneIteration = false; +// action.accept((U) data); //method will exit after accepting the action +// log.trace("Done accumulating data for {}", this.lastCallTO.getBgeeGeneId()); +// } +// +// //Important that this line is executed at every iteration, +// //so that it is set to null when there is no more data +// this.lastCallTO = currentCallTO; +// } +// +// if (this.lastCallTO != null) { +// return log.traceExit(true); +// } +// return log.traceExit(false); +// } +// +// /** +// * Get {@code SamplePValueTO}s grouped by {@code DataType}s +// * corresponding to the provided expression ID. +// *

    +// * Related {@code Iterator}s are modified. +// * +// * @param expressionId An {@code Integer} that is the ID of the expression. +// * @return A {@code Map} where keys are {@code DataType}s, the associated values +// * are {@code Set}s of {@code SamplePValueTO}s. +// */ +// private Map>> getPValues(Long expressionId) { +// log.traceEntry("{}", expressionId); +// +// log.debug(COMPUTE_MARKER, "Retrieve pvalues for expressionId: {}", expressionId); +// Map>> samplePValueTOsByDataType = new HashMap<>(); +// for (Entry>> entry: mapDataTypeToSamplePValueTOIt.entrySet()) { +// DataType currentDataType = entry.getKey(); +// log.debug(COMPUTE_MARKER, "PValues for DataType: {}", currentDataType); +// Iterator> it = entry.getValue(); +// SamplePValueTO currentTO = mapDataTypeToLastSamplePValueTO.get(currentDataType); +// log.debug(COMPUTE_MARKER, "CurrentTO: {}", currentTO); +// Set> samplePValueTOs = new HashSet<>(); +// while (currentTO != null && expressionId.equals(currentTO.getExpressionId())) { +// log.debug(COMPUTE_MARKER, "CurrentTO: {} - expressionId: {}", currentTO, expressionId); +// // We should not have 2 identical TOs +// assert samplePValueTOsByDataType.get(currentDataType) == null || +// !samplePValueTOsByDataType.get(currentDataType).contains(currentTO); +// +// //if it is the first iteration for this datatype and expressionId, +// //we store the associated SamplePValueTO Set. +// if (samplePValueTOs.isEmpty()) { +// samplePValueTOsByDataType.put(currentDataType, samplePValueTOs); +// } +// +// samplePValueTOs.add(currentTO); +// +// //try-catch to avoid calling both next and hasNext +// try { +// SamplePValueTO nextTO = it.next(); +// //the TOs are supposed to be ordered by ascending expression ID +// //for a specific data type and a specific gene +// //Note: actually, we can't do this check, because we can't know +// //with this implementation whether there was a switch of gene, +// //in which case it would be valid to have a smaller expression ID +//// if (EXP_EXPR_TO_COMPARATOR.compare(currentTO, nextTO) > 0) { +//// throw log.throwing(new IllegalStateException("The expression calls " +//// + "were not retrieved in correct order, which is mandatory " +//// + "for proper generation of data: previous TO: " +//// + currentTO + ", next TO: " + nextTO)); +//// } +// log.debug(COMPUTE_MARKER, "Previous TO={}, Current TO={}", currentTO, nextTO); +// currentTO = nextTO; +// } catch (NoSuchElementException e) { +// currentTO = null; +// log.catching(Level.TRACE, e); +// } +// } +// mapDataTypeToLastSamplePValueTO.put(currentDataType, currentTO); +// log.debug(COMPUTE_MARKER, "Storing for data type {} last SamplePValuesTO: {}", currentDataType, currentTO); +// } +// if (samplePValueTOsByDataType.isEmpty()) { +// throw log.throwing(new IllegalStateException("No supporting data for expression ID " +// + expressionId)); +// } +// +// return log.traceExit(samplePValueTOsByDataType); +// } +// +// /** +// * Return {@code null}, because a {@code CallSpliterator} does not have +// * the capability of being accessed in parallel. +// * +// * @return The {@code Spliterator} that is {@code null}. +// */ +// @Override +// public Spliterator trySplit() { +// log.traceEntry(); +// return log.traceExit((Spliterator) null); +// } +// +// @Override +// public Comparator getComparator() { +// log.traceEntry(); +// //An element of the Stream is a Set of RawExpressionCallData each containing +// //one RawExpressionCallTOs for one specific gene, +// //so retrieving the RawExpressionCallTO of the first RawExpressionCallData +// //is enough to retrieve the gene ID and order the Maps +// return log.traceExit(Comparator.comparing(s -> s.stream().findFirst().get() +// .getRawExpressionCallTO().getBgeeGeneId(), +// Comparator.nullsLast(Comparator.naturalOrder()))); +// } +// +// /** +// * Close {@code Stream}s provided at instantiation. +// */ +// public void close() { +// log.traceEntry(); +// if (!isClosed){ +// try { +// callTOs.close(); +// samplePValueTOsByDataType.values().stream().forEach(s -> s.close()); +// } finally { +// this.isClosed = true; +// } +// } +// log.traceExit(); +// } +// } +// +// /** +// * This class describes the calls related to gene baseline expression specific to pipeline. +// *

    +// * Warning: this class must override hashCode/equals from ExpressionCall class, +// * we want each PipelineCall to be considered unique, otherwise this would result in incorrect +// * generation of propagated calls. +// * +// * @author Valentine Rech de Laval +// * @version Bgee 14, Jan. 2017 +// * @since Bgee 14, Jan. 2017 +// */ +// private static class PipelineCall extends ExpressionCall { +// +// private int bgeeGeneId; +// +// private final Set parentSourceCallTOs; +// +// private final Set selfSourceCallTOs; +// +// private final Set descendantSourceCallTOs; +// +// private PipelineCall(int bgeeGeneId, Condition condition, +// Set selfSourceCallTOs) { +// this(bgeeGeneId, condition, null, null, null, null, selfSourceCallTOs, null); +// } +// private PipelineCall(int bgeeGeneId, Condition condition, +// Collection callData, +// Collection pValues, Collection bestDescendantPValues, +// Set parentSourceCallTOs, Set selfSourceCallTOs, +// Set descendantSourceCallTOs) { +// super(null, condition, null, pValues, bestDescendantPValues, null, null, +// callData, null, null); +// this.bgeeGeneId = bgeeGeneId; +// this.parentSourceCallTOs = parentSourceCallTOs == null? null: +// Collections.unmodifiableSet(new HashSet<>(parentSourceCallTOs)); +// this.selfSourceCallTOs = selfSourceCallTOs == null? null: +// Collections.unmodifiableSet(new HashSet<>(selfSourceCallTOs)); +// this.descendantSourceCallTOs = descendantSourceCallTOs == null? null: +// Collections.unmodifiableSet(new HashSet<>(descendantSourceCallTOs)); +// } +// +// +// /** +// * @return The {@code int} that is the bgee gene ID. +// */ +// public int getBgeeGeneId() { +// return bgeeGeneId; +// } +// /** +// * @return The {@code Set} of {@code RawExpressionCallTO}s corresponding to source call TOs +// * of parent calls of this {@code ExpressionCall}. +// */ +// public Set getParentSourceCallTOs() { +// return parentSourceCallTOs; +// } +// /** +// * @return The {@code Set} of {@code RawExpressionCallTO}s corresponding to source call TOs +// * of self calls of this {@code ExpressionCall}. +// */ +// public Set getSelfSourceCallTOs() { +// return selfSourceCallTOs; +// } +// /** +// * @return The {@code Set} of {@code RawExpressionCallTO}s corresponding to source call TOs +// * of descendant calls of this {@code ExpressionCall}. +// */ +// public Set getDescendantSourceCallTOs() { +// return descendantSourceCallTOs; +// } +// +// /** +// * Override method implemented in {@code ExpressionCall} to restore default {@code Object#hashCode()} behavior. +// */ +// @Override +// public int hashCode() { +// return System.identityHashCode(this); +// } +// /** +// * Override method implemented in {@code ExpressionCall} to restore default {@code Object#equals(Object)} behavior. +// */ +// @Override +// public boolean equals(Object obj) { +// return this == obj; +// } +// @Override +// public String toString() { +// StringBuilder builder = new StringBuilder(); +// builder.append("PipelineCall [bgeeGeneId=").append(bgeeGeneId) +// .append(", parentSourceCallTOs=").append(parentSourceCallTOs) +// .append(", selfSourceCallTOs=").append(selfSourceCallTOs) +// .append(", descendantSourceCallTOs=").append(descendantSourceCallTOs) +// .append(", pValues=").append(getPValues()) +// .append(", bestDescendantPValues=").append(getBestDescendantPValues()) +// .append(", dataPropagation=").append(getDataPropagation()) +// .append(", callData=").append(getCallData()) +// .append("]"); +// return builder.toString(); +// } +// +// } +// +// /** +// * This class describes the expression state related to gene baseline expression specific to pipeline. +// * Do not override hashCode/equals for proper call reconciliation. +// * +// * @param The type of experiment ID of the {@code SamplePValueTO}s contained +// * in this {@code PipelineCallData}. +// * @param The type of sample ID of the {@code SamplePValueTO}s contained +// * in this {@code PipelineCallData}. +// * @author Valentine Rech de Laval +// * @author Frederic Bastian +// * @version Bgee 15.0, Mar. 2021 +// * @since Bgee 14, Jan. 2017 +// */ +// private static class PipelineCallData, U extends Comparable> { +// +// final private DataType dataType; +// +// final private Set> parentPValues; +// //this stores the "self" p-values (in the condition itself) +// //for all possible combination of condition parameters +// final private Map, Set>> +// selfPValuesPerCondParamCombinations; +// final private Set> descendantPValues; +// +// private PipelineCallData(DataType dataType, +// Set> parentPValues, +// Map, Set>> +// selfPValuesPerCondParamCombinations, +// Set> descendantPValues) { +// if (selfPValuesPerCondParamCombinations != null && +// !selfPValuesPerCondParamCombinations.keySet().equals( +// CallService.Attribute.getAllPossibleCondParamCombinations())) { +// throw log.throwing(new IllegalArgumentException("Invalid condition parameters.")); +// } +// if (selfPValuesPerCondParamCombinations != null && +// selfPValuesPerCondParamCombinations.values().stream() +// .anyMatch(v -> v == null)) { +// throw log.throwing(new IllegalArgumentException("Invalid null values.")); +// } +// +// this.dataType = dataType; +// this.parentPValues = Collections.unmodifiableSet(parentPValues == null? +// new HashSet<>(): new HashSet<>(parentPValues)); +// //we will use defensive copying, there is no unmodifiableEnumSet +// this.selfPValuesPerCondParamCombinations = selfPValuesPerCondParamCombinations == null? +// new HashMap<>(): selfPValuesPerCondParamCombinations.entrySet().stream() +// .collect(Collectors.toMap( +// e -> EnumSet.copyOf(e.getKey()), +// e -> new HashSet<>(e.getValue()))); +// this.descendantPValues = Collections.unmodifiableSet(descendantPValues == null? +// new HashSet<>(): new HashSet<>(descendantPValues)); +// } +// +// public DataType getDataType() { +// return dataType; +// } +// public Set> getParentPValues() { +// return parentPValues; +// } +// public Map, Set>> +// getSelfPValuesPerCondParamCombinations() { +// //defensive copying, there is no unmodifiableEnumSet +// return selfPValuesPerCondParamCombinations.entrySet().stream() +// .collect(Collectors.toMap( +// e -> EnumSet.copyOf(e.getKey()), +// e -> new HashSet<>(e.getValue()))); +// } +// public Set> getDescendantPValues() { +// return descendantPValues; +// } +// +// //Note: do not implement hashCode/equals, otherwise we could discard different +// //ExperimentExpressionCount from same experiment, in different conditions being aggregated. +// @Override +// public String toString() { +// StringBuilder builder = new StringBuilder(); +// builder.append("PipelineCallData [dataType=").append(dataType) +// .append(", parentPValues=").append(parentPValues) +// .append(", selfPValuesPerCondParamCombinations=").append(selfPValuesPerCondParamCombinations) +// .append(", descendantPValues=").append(descendantPValues) +// .append("]"); +// return builder.toString(); +// } +// } +// +// /** +// * {@code TransferObject}s do not implement equals/hashCode, and we need it for inserting +// * {@code GlobalConditionToRawConditionTO}s, so we extend this class and implements hashCode/Equals. +// */ +// private static class PipelineGlobalCondToRawCondTO extends GlobalConditionToRawConditionTO { +// private static final long serialVersionUID = -4710796651567000694L; +// +// public PipelineGlobalCondToRawCondTO(GlobalConditionToRawConditionTO to) { +// this(to.getRawConditionId(), to.getGlobalConditionId(), to.getConditionRelationOrigin()); +// } +// public PipelineGlobalCondToRawCondTO(Integer rawConditionId, Integer globalConditionId, +// ConditionRelationOrigin conditionRelationOrigin) { +// super(rawConditionId, globalConditionId, conditionRelationOrigin); +// } +// +// @Override +// public int hashCode() { +// final int prime = 31; +// int result = 1; +// result = prime * result + ((this.getRawConditionId() == null) ? 0 : +// this.getRawConditionId().hashCode()); +// result = prime * result + ((this.getGlobalConditionId() == null) ? 0 : +// this.getGlobalConditionId().hashCode()); +// result = prime * result + ((this.getConditionRelationOrigin() == null) ? 0 : +// this.getConditionRelationOrigin().hashCode()); +// return result; +// } +// @Override +// public boolean equals(Object obj) { +// if (this == obj) { +// return true; +// } +// if (obj == null) { +// return false; +// } +// if (getClass() != obj.getClass()) { +// return false; +// } +// PipelineGlobalCondToRawCondTO other = (PipelineGlobalCondToRawCondTO) obj; +// if (this.getRawConditionId() == null) { +// if (other.getRawConditionId() != null) { +// return false; +// } +// } else if (!this.getRawConditionId().equals(other.getRawConditionId())) { +// return false; +// } +// if (this.getGlobalConditionId() == null) { +// if (other.getGlobalConditionId() != null) { +// return false; +// } +// } else if (!this.getGlobalConditionId().equals(other.getGlobalConditionId())) { +// return false; +// } +// if (this.getConditionRelationOrigin() == null) { +// if (other.getConditionRelationOrigin() != null) { +// return false; +// } +// } else if (!this.getConditionRelationOrigin().equals(other.getConditionRelationOrigin())) { +// return false; +// } +// return true; +// } +// } +// +// /** +// * Class used to store a {@code RawExpressionCallTO} associated with +// * its {@code ExperimentExpressionTO}s per {@code DataType} and +// * {@code SamplePValueTO}s per {@code DataType}. +// * +// * @author Frederic Bastian +// * @version Bgee 15.0, Mar 2021 +// * @since Bgee 15.0, Mar 2021 +// */ +// private static class RawExpressionCallData { +// private final RawExpressionCallTO rawExpressionCallTO; +// private final Map>> samplePValueTOsPerDataType; +// +// public RawExpressionCallData(RawExpressionCallTO rawExpressionCallTO, +// Map>> samplePValueTOsPerDataType) { +// this.rawExpressionCallTO = rawExpressionCallTO; +// this.samplePValueTOsPerDataType = samplePValueTOsPerDataType; +// } +// +// public RawExpressionCallTO getRawExpressionCallTO() { +// return rawExpressionCallTO; +// } +// public Map>> getSamplePValueTOsPerDataType() { +// return samplePValueTOsPerDataType; +// } +// } +// +// /** +// * Class solely created to implement hashCode/equals on {@code SamplePValueTO} +// * based on {@code expressionId}, {@code experimentId}, {@code sampleId}. +// * +// * @param The type of experiment ID +// * @param The type of sample ID +// * @author Frederic Bastian +// * @version Bgee 15.0, Mar 2021 +// * @since Bgee 15.0, Mar 2021 +// */ +// public static class PipelineSamplePValueTO, U extends Comparable> +// extends SamplePValueTO { +// private static final long serialVersionUID = 6552984802761656993L; +// +// public PipelineSamplePValueTO(SamplePValueTO samplePValueTO) { +// super(samplePValueTO.getExpressionId(), samplePValueTO.getExperimentId(), +// samplePValueTO.getSampleId(), samplePValueTO.getpValue()); +// } +// public PipelineSamplePValueTO(CallSourceTO callSourceTO) { +// super(callSourceTO.getExpressionId(), null, callSourceTO.getAssayId(), callSourceTO.getPValue()); +// } +// +// @Override +// public int hashCode() { +// final int prime = 31; +// int result = 1; +// result = prime * result + ((this.getExpressionId() == null) ? 0 : this.getExpressionId().hashCode()); +// result = prime * result + ((this.getExperimentId() == null) ? 0 : this.getExperimentId().hashCode()); +// result = prime * result + ((this.getSampleId() == null) ? 0 : this.getSampleId().hashCode()); +// return result; +// } +// @Override +// public boolean equals(Object obj) { +// if (this == obj) { +// return true; +// } +// if (obj == null) { +// return false; +// } +// if (!(obj instanceof SamplePValueTO)) { +// return false; +// } +// SamplePValueTO other = (SamplePValueTO) obj; +// if (!Objects.equal(this.getExpressionId(), other.getExpressionId())) { +// return false; +// } +// if (!Objects.equal(this.getExperimentId(), other.getExperimentId())) { +// return false; +// } +// if (!Objects.equal(this.getSampleId(), other.getSampleId())) { +// return false; +// } +// return true; +// } +// } +// +// /** +// * Class responsible for running in a separate thread the insertions to database +// * for a specific species ID and combination of condition parameters, +// * to be able to have a single transaction to insert these data. +// * This should not impact performances, as anyway INSERT statements are executed +// * sequentially in MySQL. +// *

    +// * This thread is also for killing all queries performed by different threads +// * when an error occurs in any thread. +// * +// * @author Frederic Bastian +// * @version Bgee 14 Feb. 2017 +// * @since Bgee 14 Feb. 2017 +// */ +// private static class InsertJob implements Runnable { +// /** +// * The {@code InsertPropagatedCalls} object that launched this tread. Allows this thread +// * to be notified on error or job completion, and to share variables between threads +// * used by this object. +// */ +// private final InsertPropagatedCalls callPropagator; +// /** +// * A {@code Map} where keys are {@code Condition}s already inserted into the database +// * for the requested species before doing the call propagation, the associating value +// * being an {@code Integer} that is the related global condition ID. +// */ +// private final Map globalCondsAlreadyInsertedMap; // /** -// * A {@code Comparator} only to verify that {@code ExperimentExpressionTO} -// * {@code Stream} elements are properly ordered. This {@code Comparator} is valid only -// * to compare {@code ExperimentExpressionTO}s for one specific data type and one specific gene. +// * A {@code Set} of {@code GlobalCondToRawCondTO}s already inserted into the database +// * for the requested species before doing the call propagation. // */ -// final private Comparator EXP_EXPR_TO_COMPARATOR = -// Comparator.comparing(ExperimentExpressionTO::getExpressionId, -// Comparator.nullsLast(Comparator.naturalOrder())); - - final private Stream callTOs; - //TODO: javadoc: not final for lazy loading - private Iterator itCallTOs; - private RawExpressionCallTO lastCallTO; - final private Map>> samplePValueTOsByDataType; - //TODO: javadoc: this map is NOT immutable (but reference is final) - final private Map>> mapDataTypeToSamplePValueTOIt; - //TODO: javadoc: this map is NOT immutable (but reference is final) - final private Map> mapDataTypeToLastSamplePValueTO; - - private boolean isInitiated; - private boolean isClosed; - - /** - * Default constructor. - * - * @param callTOs A {@code Stream} of {@code T}s that is the stream of calls. - * @param samplePValueTOsByDataType A {@code Map} where keys are {@code DataType}s - * defining data types, the associated value being a - * {@code Stream} of {@code SamplePValueTO}s - * storing expression p-values associated with each call. - */ - public CallSpliterator(Stream callTOs, - Map>> samplePValueTOsByDataType) { - super(Long.MAX_VALUE, Spliterator.ORDERED | Spliterator.IMMUTABLE - | Spliterator.DISTINCT | Spliterator.NONNULL | Spliterator.SORTED); - //experimentExprTOsByDataType can be null since we don't use them as of Bge 15.0, - //but we keep the possibility of easily reactivating it - if (callTOs == null || /*experimentExprTOsByDataType == null || - experimentExprTOsByDataType.entrySet().stream() - .anyMatch(e -> e == null || e.getValue() == null) ||*/ - samplePValueTOsByDataType == null || - samplePValueTOsByDataType.entrySet().stream() - .anyMatch(e -> e == null || e.getValue() == null)) { - throw new IllegalArgumentException("Provided streams cannot be null"); - } - - this.callTOs = callTOs; - this.itCallTOs = null; - this.lastCallTO = null; - this.samplePValueTOsByDataType = Collections.unmodifiableMap(samplePValueTOsByDataType); - this.isInitiated = false; - this.isClosed = false; - this.mapDataTypeToLastSamplePValueTO = new HashMap<>(); - this.mapDataTypeToSamplePValueTOIt = new HashMap<>(); - } - - //the line 'action.accept((U) data);' generates a warning for unchecked cast. - //to avoid it. we would need to parameterize each class definition used in the Map generated - //by this Spliterator, and provide their class at instantiation (RawExpressionCallTO.class, - //DataType.class, ExperimentExpressionTO.class, etc): boring. - @SuppressWarnings("unchecked") - @Override - public boolean tryAdvance(Consumer action) { - log.traceEntry("{}", action); - - if (this.isClosed) { - throw log.throwing(new IllegalStateException("Already close")); - } - - // Lazy loading: we do not get stream iterators (terminal operation) - // before tryAdvance() is called. - if (!this.isInitiated) { - //set it first because method can return false and exist the block - this.isInitiated = true; - - this.itCallTOs = this.callTOs.iterator(); - try { - this.lastCallTO = this.itCallTOs.next(); - } catch (NoSuchElementException e) { - log.catching(Level.DEBUG, e); - return log.traceExit(false); - } - - for (Entry>> entry: - this.samplePValueTOsByDataType.entrySet()) { - Iterator> it = entry.getValue().iterator(); - try { - this.mapDataTypeToLastSamplePValueTO.put(entry.getKey(), it.next()); - log.trace("mapDataTypeToLastSamplePValueTO: {}", - this.mapDataTypeToLastSamplePValueTO.get(entry.getKey())); - //don't store the iterator if there is no element (catch clause) - this.mapDataTypeToSamplePValueTOIt.put(entry.getKey(), it); - } catch (NoSuchElementException e) { - //it's OK to have no element for a given data type - log.catching(Level.TRACE, e); - } - } - //We should have at least one data type with supporting data - if (this.mapDataTypeToSamplePValueTOIt.isEmpty()) { - throw log.throwing(new IllegalStateException("Missing supporting data")); - } - } - - //if already initialized, no calls retrieved, but method called again (should never happen, - //as the method would have returned false during initialization above) - if (this.lastCallTO == null) { - log.warn("Stream used again despite having no elements."); - return log.traceExit(false); - } - - //This Set is the element generated by this Stream, on which the Consumer is applied. - //It retrieves all RawExpressionCallTOs for one given gene, and associates them - //to their relative ExperimentExpressionTOs and SamplePValueTOs, per data type, - //into RawExpressionCallDataTOs. - final Set data = new HashSet<>(); - - //we iterate the CallTO ResultSet and stop when we reach the next gene, or when - //there is no more element, then we do a last iteration after the last TO is - //retrieved, to properly group all the calls. This is why we use the boolean currentGeneIteration. - //This loop always work on this.lastCallTO, which has already been populated at this point. - boolean currentGeneIteration = true; - while (currentGeneIteration) { - if (this.lastCallTO.getBgeeGeneId() == null || this.lastCallTO.getId() == null) { - throw log.throwing(new IllegalStateException("Missing attributes in raw call: " - + this.lastCallTO)); - } - log.trace("currentGeneIteration, lastCallTO: {}", this.lastCallTO); - // We add the previous ExperimentExpressionTOs to the group - assert data.stream().noneMatch(rawData -> rawData.getRawExpressionCallTO().getId() - .equals(this.lastCallTO.getId())); - data.add(new RawExpressionCallData(this.lastCallTO, - this.getPValues(this.lastCallTO.getId()))); - - RawExpressionCallTO currentCallTO = null; - //try-catch to avoid calling both hasNext and next - try { - currentCallTO = this.itCallTOs.next(); - currentGeneIteration = true; - } catch (NoSuchElementException e) { - currentGeneIteration = false; - } - //the calls are supposed to be ordered by ascending gene ID - expression ID - if (currentCallTO != null && CALL_TO_COMPARATOR.compare(this.lastCallTO, currentCallTO) > 0) { - throw log.throwing(new IllegalStateException("The expression calls " - + "were not retrieved in correct order, which is mandatory " - + "for proper generation of data: previous call: " - + this.lastCallTO + ", current call: " + currentCallTO)); - } - log.trace("Previous call={} - Current call={}", this.lastCallTO, currentCallTO); - - //if the gene changes, or if it is the latest iteration (one iteration after - //the last CallTO was retrieved, this.itCallTOs.next() threw an exception), - //we generate the data Map for the previous gene, as all data were iterated for that gene. - if (!currentGeneIteration || !currentCallTO.getBgeeGeneId().equals(this.lastCallTO.getBgeeGeneId())) { - assert (currentGeneIteration && currentCallTO != null) || (!currentGeneIteration && currentCallTO == null); - currentGeneIteration = false; - action.accept((U) data); //method will exit after accepting the action - log.trace("Done accumulating data for {}", this.lastCallTO.getBgeeGeneId()); - } - - //Important that this line is executed at every iteration, - //so that it is set to null when there is no more data - this.lastCallTO = currentCallTO; - } - - if (this.lastCallTO != null) { - return log.traceExit(true); - } - return log.traceExit(false); - } - - /** - * Get {@code SamplePValueTO}s grouped by {@code DataType}s - * corresponding to the provided expression ID. - *

    - * Related {@code Iterator}s are modified. - * - * @param expressionId An {@code Integer} that is the ID of the expression. - * @return A {@code Map} where keys are {@code DataType}s, the associated values - * are {@code Set}s of {@code SamplePValueTO}s. - */ - private Map>> getPValues(Long expressionId) { - log.traceEntry("{}", expressionId); - - log.debug(COMPUTE_MARKER, "Retrieve pvalues for expressionId: {}", expressionId); - Map>> samplePValueTOsByDataType = new HashMap<>(); - for (Entry>> entry: mapDataTypeToSamplePValueTOIt.entrySet()) { - DataType currentDataType = entry.getKey(); - log.debug(COMPUTE_MARKER, "PValues for DataType: {}", currentDataType); - Iterator> it = entry.getValue(); - SamplePValueTO currentTO = mapDataTypeToLastSamplePValueTO.get(currentDataType); - log.debug(COMPUTE_MARKER, "CurrentTO: {}", currentTO); - Set> samplePValueTOs = new HashSet<>(); - while (currentTO != null && expressionId.equals(currentTO.getExpressionId())) { - log.debug(COMPUTE_MARKER, "CurrentTO: {} - expressionId: {}", currentTO, expressionId); - // We should not have 2 identical TOs - assert samplePValueTOsByDataType.get(currentDataType) == null || - !samplePValueTOsByDataType.get(currentDataType).contains(currentTO); - - //if it is the first iteration for this datatype and expressionId, - //we store the associated SamplePValueTO Set. - if (samplePValueTOs.isEmpty()) { - samplePValueTOsByDataType.put(currentDataType, samplePValueTOs); - } - - samplePValueTOs.add(currentTO); - - //try-catch to avoid calling both next and hasNext - try { - SamplePValueTO nextTO = it.next(); - //the TOs are supposed to be ordered by ascending expression ID - //for a specific data type and a specific gene - //Note: actually, we can't do this check, because we can't know - //with this implementation whether there was a switch of gene, - //in which case it would be valid to have a smaller expression ID -// if (EXP_EXPR_TO_COMPARATOR.compare(currentTO, nextTO) > 0) { -// throw log.throwing(new IllegalStateException("The expression calls " -// + "were not retrieved in correct order, which is mandatory " -// + "for proper generation of data: previous TO: " -// + currentTO + ", next TO: " + nextTO)); +// private final Set globalCondToRawConds; +// +// private InsertJob(InsertPropagatedCalls callPropagator, +// Map globalCondsAlreadyInsertedMap, +// Set globalCondToRawConds) { +// log.traceEntry("{}, {}, {}", callPropagator, globalCondsAlreadyInsertedMap, globalCondToRawConds); +// if (!callPropagator.computeAndInsertGlobalCond && +// (globalCondsAlreadyInsertedMap == null || globalCondsAlreadyInsertedMap.isEmpty() || +// globalCondToRawConds == null || globalCondToRawConds.isEmpty())) { +// throw log.throwing(new IllegalArgumentException( +// "Some global conditions should have already been computed and inserted.")); +// } +// this.callPropagator = callPropagator; +// this.globalCondsAlreadyInsertedMap = Collections.unmodifiableMap( +// globalCondsAlreadyInsertedMap == null ? new HashMap<>() : +// new HashMap<>(globalCondsAlreadyInsertedMap)); +// this.globalCondToRawConds = Collections.unmodifiableSet(globalCondToRawConds == null ? +// new HashSet<>() : new HashSet<>(globalCondToRawConds)); +// } +// +// @Override +// public void run() { +// log.traceEntry(); +// +// //We need a new connection to the database for each thread, so we use +// //the ServiceFactory Supplier +// final ServiceFactory factory = this.callPropagator.serviceFactorySupplier.get(); +// final DAOManager daoManager = factory.getDAOManager(); +// final ConditionDAO condDAO = daoManager.getConditionDAO(); +// final GlobalExpressionCallDAO exprDAO = daoManager.getGlobalExpressionCallDAO(); +// //in order to insert globalConditions +// final Map insertedCondMap = new HashMap<>( +// this.globalCondsAlreadyInsertedMap); +// //relations between globalConditions and raw conditions +// final Set globalCondToRawConds = new HashSet<>( +// this.globalCondToRawConds); +// +// boolean errorInThisThread = false; +// int groupsInserted = 0; +// try { +// //If all the global conds should have been inserted already +// if (!this.callPropagator.computeAndInsertGlobalCond && +// condDAO.getGlobalConditions( +// Collections.singleton(this.callPropagator.speciesId), +// generateDAOConditionFilters(null, this.callPropagator.condParams), +// null) +// .stream().noneMatch(e -> true)) { +// throw log.throwing(new IllegalStateException( +// "Global conditions should have been inserted for species " + +// this.callPropagator.speciesId)); +// } +// +// boolean firstInsert = true; +// INSERT: while ((!this.callPropagator.jobCompleted || +// //important to check that there is no remaining calls to insert, +// //as other thread might set the jobCompleted flag to true +// //before this thread finishes to insert all data. +// !this.callPropagator.callsToInsert.isEmpty()) && +// //but if an error occurred, we stop immediately in any case. +// this.callPropagator.errorOccured == null) { +// +// //wait for consuming new data +// Set toInsert = null; +// try { +// log.trace(BLOCKING_QUEUE_MARKER, "Trying to take Set of PipelineCalls"); +// //here we ask to wait indefinitely +// toInsert = this.callPropagator.callsToInsert.take(); +// log.trace(BLOCKING_QUEUE_MARKER, "Done taking Set of {} PipelineCalls", +// toInsert.size()); +// } catch (InterruptedException e) { +// //this Thread will be interrupted if an error occurred in an other Thread +// //or if all computations are finished and this thread is waiting +// //for more data to consume. +// log.catching(Level.DEBUG, e); +// continue INSERT; +// } +// assert toInsert != null; +// +// //wait for receiving data for starting the transaction, +// //otherwise there might be some lock issues +// if (firstInsert) { +// startTransaction((MySQLDAOManager) daoManager); +// firstInsert = false; +// } +// +// // Here, we insert new conditions, and add them to the known conditions +// Map newCondMap = InsertPropagatedCalls +// .insertNewGlobalConditions(toInsert.stream() +// .flatMap(c -> { +// Set callConds = c.getBestDescendantPValues().stream() +// .map(p -> p.getCondition()) +// .collect(Collectors.toSet()); +// callConds.add(c.getCondition()); +// return callConds.stream(); +// }) +// .collect(Collectors.toSet()), +// insertedCondMap.keySet(), condDAO); +// if (!this.callPropagator.computeAndInsertGlobalCond && !newCondMap.isEmpty()) { +// throw log.throwing(new IllegalStateException( +// "All globalConditions should have been inserted already")); +// } +// if (!Collections.disjoint(insertedCondMap.keySet(), newCondMap.keySet())) { +// throw log.throwing(new IllegalStateException("Error, new conditions already seen. " +// + "new conditions: " + newCondMap.keySet() + " - existing conditions: " +// + insertedCondMap.keySet())); +// } +// if (!Collections.disjoint(insertedCondMap.values(), newCondMap.values())) { +// throw log.throwing(new IllegalStateException("Error, condition IDs reused. " +// + "new IDs: " + newCondMap.values() + " - existing IDs: " +// + insertedCondMap.values())); +// } +// insertedCondMap.putAll(newCondMap); +// +// //Now, we insert relations between globalConditions and source raw conditions, +// //to be able to later retrieve relations between globalExpressions to expressions, +// //without needing the table globalExpressionToExpression, that was very much too large +// //(more than 10 billions rows for 29 species). +// Set newGlobalCondToRawConds = +// InsertPropagatedCalls.insertGlobalCondToRawCondsFromCalls(toInsert, +// globalCondToRawConds, insertedCondMap, condDAO); +// if (!this.callPropagator.computeAndInsertGlobalCond && +// !newGlobalCondToRawConds.isEmpty()) { +// throw log.throwing(new IllegalStateException( +// "All globalCondToConds should have been inserted already")); +// } +// if (!Collections.disjoint(globalCondToRawConds, newGlobalCondToRawConds)) { +// throw log.throwing(new IllegalStateException("Error, new condition relations already seen. " +// + "new relations: " + newGlobalCondToRawConds + " - existing relations: " +// + globalCondToRawConds)); +// } +// //Deactivate this assert, it is very slow and, anyway, there is +// //a primary key(globalConditionId, conditionId) which makes +// //this situation impossible. +// // //We're not supposed to generate a same relation between a global condition +// // //and a raw condition having different conditionRelationOrigins. +// // //Since conditionRelationOrigin is taken into account in equals/hashCode, +// // //make an assert here based solely on global condition ID and raw condition ID. +// // assert newGlobalCondToRawConds.stream() +// // .noneMatch(r1 -> globalCondToRawConds.stream() +// // .anyMatch(r2 -> r1.getRawConditionId().equals(r2.getRawConditionId()) && +// // r1.getGlobalConditionId().equals(r2.getGlobalConditionId()))): +// // "Incorrect new relations: " + newGlobalCondToRawConds + " - " + globalCondToRawConds; +// +// globalCondToRawConds.addAll(newGlobalCondToRawConds); +// +// +// // And we finish by inserting the computed calls +// insertPropagatedCalls(toInsert, insertedCondMap, exprDAO); +// if (log.isDebugEnabled()) { +// log.debug("{} calls inserted for gene {}", toInsert.size(), +// toInsert.iterator().next().getBgeeGeneId()); +// } +// +// log.trace(INSERTION_MARKER, "Calls inserted."); +// groupsInserted++; +// if (log.isInfoEnabled() && groupsInserted % 100 == 0) { +// log.info(INSERTION_MARKER, "{} genes inserted.", groupsInserted); +// } +// } +// +// } catch (Exception e) { +// errorInThisThread = true; +// if (this.callPropagator.errorOccured == null) { +// this.callPropagator.errorOccured = e; +// } +// if (e instanceof RuntimeException) { +// throw log.throwing((RuntimeException) e); +// } +// throw log.throwing(new IllegalStateException(e)); +// +// } finally { +// assert this.callPropagator.jobCompleted || +// this.callPropagator.errorOccured != null; +// //we assume the insertion is done using MySQL, and we commit/rollback the transaction +// try { +// this.killAllDAOManagersIfNeeded(); +// } finally { +// try { +// //recheck the jobCompleted flag in case this Thread was interrupted +// //for unknown reason +// if (this.callPropagator.jobCompleted && this.callPropagator.errorOccured == null) { +// log.info("{} genes inserted, committing transaction", groupsInserted); +// ((MySQLDAOManager) daoManager).getConnection().getRealConnection().commit(); +// ((MySQLDAOManager) daoManager).getConnection().getRealConnection().setAutoCommit(true); +// } else { +// log.info("Rollbacking transaction"); +// ((MySQLDAOManager) daoManager).getConnection().getRealConnection().rollback(); +// ((MySQLDAOManager) daoManager).getConnection().getRealConnection().setAutoCommit(true); // } - log.debug(COMPUTE_MARKER, "Previous TO={}, Current TO={}", currentTO, nextTO); - currentTO = nextTO; - } catch (NoSuchElementException e) { - currentTO = null; - log.catching(Level.TRACE, e); - } - } - mapDataTypeToLastSamplePValueTO.put(currentDataType, currentTO); - log.debug(COMPUTE_MARKER, "Storing for data type {} last SamplePValuesTO: {}", currentDataType, currentTO); - } - if (samplePValueTOsByDataType.isEmpty()) { - throw log.throwing(new IllegalStateException("No supporting data for expression ID " - + expressionId)); - } - - return log.traceExit(samplePValueTOsByDataType); - } - - /** - * Return {@code null}, because a {@code CallSpliterator} does not have - * the capability of being accessed in parallel. - * - * @return The {@code Spliterator} that is {@code null}. - */ - @Override - public Spliterator trySplit() { - log.traceEntry(); - return log.traceExit((Spliterator) null); - } - - @Override - public Comparator getComparator() { - log.traceEntry(); - //An element of the Stream is a Set of RawExpressionCallData each containing - //one RawExpressionCallTOs for one specific gene, - //so retrieving the RawExpressionCallTO of the first RawExpressionCallData - //is enough to retrieve the gene ID and order the Maps - return log.traceExit(Comparator.comparing(s -> s.stream().findFirst().get() - .getRawExpressionCallTO().getBgeeGeneId(), - Comparator.nullsLast(Comparator.naturalOrder()))); - } - - /** - * Close {@code Stream}s provided at instantiation. - */ - public void close() { - log.traceEntry(); - if (!isClosed){ - try { - callTOs.close(); - samplePValueTOsByDataType.values().stream().forEach(s -> s.close()); - } finally { - this.isClosed = true; - } - } - log.traceExit(); - } - } - - /** - * This class describes the calls related to gene baseline expression specific to pipeline. - *

    - * Warning: this class must override hashCode/equals from ExpressionCall class, - * we want each PipelineCall to be considered unique, otherwise this would result in incorrect - * generation of propagated calls. - * - * @author Valentine Rech de Laval - * @version Bgee 14, Jan. 2017 - * @since Bgee 14, Jan. 2017 - */ - private static class PipelineCall extends ExpressionCall { - - private int bgeeGeneId; - - private final Set parentSourceCallTOs; - - private final Set selfSourceCallTOs; - - private final Set descendantSourceCallTOs; - - private PipelineCall(int bgeeGeneId, Condition condition, - Set selfSourceCallTOs) { - this(bgeeGeneId, condition, null, null, null, null, selfSourceCallTOs, null); - } - private PipelineCall(int bgeeGeneId, Condition condition, - Collection callData, - Collection pValues, Collection bestDescendantPValues, - Set parentSourceCallTOs, Set selfSourceCallTOs, - Set descendantSourceCallTOs) { - super(null, condition, null, pValues, bestDescendantPValues, null, null, - callData, null, null); - this.bgeeGeneId = bgeeGeneId; - this.parentSourceCallTOs = parentSourceCallTOs == null? null: - Collections.unmodifiableSet(new HashSet<>(parentSourceCallTOs)); - this.selfSourceCallTOs = selfSourceCallTOs == null? null: - Collections.unmodifiableSet(new HashSet<>(selfSourceCallTOs)); - this.descendantSourceCallTOs = descendantSourceCallTOs == null? null: - Collections.unmodifiableSet(new HashSet<>(descendantSourceCallTOs)); - } - - - /** - * @return The {@code int} that is the bgee gene ID. - */ - public int getBgeeGeneId() { - return bgeeGeneId; - } - /** - * @return The {@code Set} of {@code RawExpressionCallTO}s corresponding to source call TOs - * of parent calls of this {@code ExpressionCall}. - */ - public Set getParentSourceCallTOs() { - return parentSourceCallTOs; - } - /** - * @return The {@code Set} of {@code RawExpressionCallTO}s corresponding to source call TOs - * of self calls of this {@code ExpressionCall}. - */ - public Set getSelfSourceCallTOs() { - return selfSourceCallTOs; - } - /** - * @return The {@code Set} of {@code RawExpressionCallTO}s corresponding to source call TOs - * of descendant calls of this {@code ExpressionCall}. - */ - public Set getDescendantSourceCallTOs() { - return descendantSourceCallTOs; - } - - /** - * Override method implemented in {@code ExpressionCall} to restore default {@code Object#hashCode()} behavior. - */ - @Override - public int hashCode() { - return System.identityHashCode(this); - } - /** - * Override method implemented in {@code ExpressionCall} to restore default {@code Object#equals(Object)} behavior. - */ - @Override - public boolean equals(Object obj) { - return this == obj; - } - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("PipelineCall [bgeeGeneId=").append(bgeeGeneId) - .append(", parentSourceCallTOs=").append(parentSourceCallTOs) - .append(", selfSourceCallTOs=").append(selfSourceCallTOs) - .append(", descendantSourceCallTOs=").append(descendantSourceCallTOs) - .append(", pValues=").append(getPValues()) - .append(", bestDescendantPValues=").append(getBestDescendantPValues()) - .append(", dataPropagation=").append(getDataPropagation()) - .append(", callData=").append(getCallData()) - .append("]"); - return builder.toString(); - } - - } - - /** - * This class describes the expression state related to gene baseline expression specific to pipeline. - * Do not override hashCode/equals for proper call reconciliation. - * - * @param The type of experiment ID of the {@code SamplePValueTO}s contained - * in this {@code PipelineCallData}. - * @param The type of sample ID of the {@code SamplePValueTO}s contained - * in this {@code PipelineCallData}. - * @author Valentine Rech de Laval - * @author Frederic Bastian - * @version Bgee 15.0, Mar. 2021 - * @since Bgee 14, Jan. 2017 - */ - private static class PipelineCallData, U extends Comparable> { - - final private DataType dataType; - - final private Set> parentPValues; - //this stores the "self" p-values (in the condition itself) - //for all possible combination of condition parameters - final private Map, Set>> - selfPValuesPerCondParamCombinations; - final private Set> descendantPValues; - - private PipelineCallData(DataType dataType, - Set> parentPValues, - Map, Set>> - selfPValuesPerCondParamCombinations, - Set> descendantPValues) { - if (selfPValuesPerCondParamCombinations != null && - !selfPValuesPerCondParamCombinations.keySet().equals( - CallService.Attribute.getAllPossibleCondParamCombinations())) { - throw log.throwing(new IllegalArgumentException("Invalid condition parameters.")); - } - if (selfPValuesPerCondParamCombinations != null && - selfPValuesPerCondParamCombinations.values().stream() - .anyMatch(v -> v == null)) { - throw log.throwing(new IllegalArgumentException("Invalid null values.")); - } - - this.dataType = dataType; - this.parentPValues = Collections.unmodifiableSet(parentPValues == null? - new HashSet<>(): new HashSet<>(parentPValues)); - //we will use defensive copying, there is no unmodifiableEnumSet - this.selfPValuesPerCondParamCombinations = selfPValuesPerCondParamCombinations == null? - new HashMap<>(): selfPValuesPerCondParamCombinations.entrySet().stream() - .collect(Collectors.toMap( - e -> EnumSet.copyOf(e.getKey()), - e -> new HashSet<>(e.getValue()))); - this.descendantPValues = Collections.unmodifiableSet(descendantPValues == null? - new HashSet<>(): new HashSet<>(descendantPValues)); - } - - public DataType getDataType() { - return dataType; - } - public Set> getParentPValues() { - return parentPValues; - } - public Map, Set>> - getSelfPValuesPerCondParamCombinations() { - //defensive copying, there is no unmodifiableEnumSet - return selfPValuesPerCondParamCombinations.entrySet().stream() - .collect(Collectors.toMap( - e -> EnumSet.copyOf(e.getKey()), - e -> new HashSet<>(e.getValue()))); - } - public Set> getDescendantPValues() { - return descendantPValues; - } - - //Note: do not implement hashCode/equals, otherwise we could discard different - //ExperimentExpressionCount from same experiment, in different conditions being aggregated. - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("PipelineCallData [dataType=").append(dataType) - .append(", parentPValues=").append(parentPValues) - .append(", selfPValuesPerCondParamCombinations=").append(selfPValuesPerCondParamCombinations) - .append(", descendantPValues=").append(descendantPValues) - .append("]"); - return builder.toString(); - } - } - - /** - * {@code TransferObject}s do not implement equals/hashCode, and we need it for inserting - * {@code GlobalConditionToRawConditionTO}s, so we extend this class and implements hashCode/Equals. - */ - private static class PipelineGlobalCondToRawCondTO extends GlobalConditionToRawConditionTO { - private static final long serialVersionUID = -4710796651567000694L; - - public PipelineGlobalCondToRawCondTO(GlobalConditionToRawConditionTO to) { - this(to.getRawConditionId(), to.getGlobalConditionId(), to.getConditionRelationOrigin()); - } - public PipelineGlobalCondToRawCondTO(Integer rawConditionId, Integer globalConditionId, - ConditionRelationOrigin conditionRelationOrigin) { - super(rawConditionId, globalConditionId, conditionRelationOrigin); - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((this.getRawConditionId() == null) ? 0 : - this.getRawConditionId().hashCode()); - result = prime * result + ((this.getGlobalConditionId() == null) ? 0 : - this.getGlobalConditionId().hashCode()); - result = prime * result + ((this.getConditionRelationOrigin() == null) ? 0 : - this.getConditionRelationOrigin().hashCode()); - return result; - } - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - PipelineGlobalCondToRawCondTO other = (PipelineGlobalCondToRawCondTO) obj; - if (this.getRawConditionId() == null) { - if (other.getRawConditionId() != null) { - return false; - } - } else if (!this.getRawConditionId().equals(other.getRawConditionId())) { - return false; - } - if (this.getGlobalConditionId() == null) { - if (other.getGlobalConditionId() != null) { - return false; - } - } else if (!this.getGlobalConditionId().equals(other.getGlobalConditionId())) { - return false; - } - if (this.getConditionRelationOrigin() == null) { - if (other.getConditionRelationOrigin() != null) { - return false; - } - } else if (!this.getConditionRelationOrigin().equals(other.getConditionRelationOrigin())) { - return false; - } - return true; - } - } - - /** - * Class used to store a {@code RawExpressionCallTO} associated with - * its {@code ExperimentExpressionTO}s per {@code DataType} and - * {@code SamplePValueTO}s per {@code DataType}. - * - * @author Frederic Bastian - * @version Bgee 15.0, Mar 2021 - * @since Bgee 15.0, Mar 2021 - */ - private static class RawExpressionCallData { - private final RawExpressionCallTO rawExpressionCallTO; - private final Map>> samplePValueTOsPerDataType; - - public RawExpressionCallData(RawExpressionCallTO rawExpressionCallTO, - Map>> samplePValueTOsPerDataType) { - this.rawExpressionCallTO = rawExpressionCallTO; - this.samplePValueTOsPerDataType = samplePValueTOsPerDataType; - } - - public RawExpressionCallTO getRawExpressionCallTO() { - return rawExpressionCallTO; - } - public Map>> getSamplePValueTOsPerDataType() { - return samplePValueTOsPerDataType; - } - } - - /** - * Class solely created to implement hashCode/equals on {@code SamplePValueTO} - * based on {@code expressionId}, {@code experimentId}, {@code sampleId}. - * - * @param The type of experiment ID - * @param The type of sample ID - * @author Frederic Bastian - * @version Bgee 15.0, Mar 2021 - * @since Bgee 15.0, Mar 2021 - */ - public static class PipelineSamplePValueTO, U extends Comparable> - extends SamplePValueTO { - private static final long serialVersionUID = 6552984802761656993L; - - public PipelineSamplePValueTO(SamplePValueTO samplePValueTO) { - super(samplePValueTO.getExpressionId(), samplePValueTO.getExperimentId(), - samplePValueTO.getSampleId(), samplePValueTO.getpValue()); - } - public PipelineSamplePValueTO(CallSourceTO callSourceTO) { - super(callSourceTO.getExpressionId(), null, callSourceTO.getAssayId(), callSourceTO.getPValue()); - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((this.getExpressionId() == null) ? 0 : this.getExpressionId().hashCode()); - result = prime * result + ((this.getExperimentId() == null) ? 0 : this.getExperimentId().hashCode()); - result = prime * result + ((this.getSampleId() == null) ? 0 : this.getSampleId().hashCode()); - return result; - } - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (!(obj instanceof SamplePValueTO)) { - return false; - } - SamplePValueTO other = (SamplePValueTO) obj; - if (!Objects.equal(this.getExpressionId(), other.getExpressionId())) { - return false; - } - if (!Objects.equal(this.getExperimentId(), other.getExperimentId())) { - return false; - } - if (!Objects.equal(this.getSampleId(), other.getSampleId())) { - return false; - } - return true; - } - } - - /** - * Class responsible for running in a separate thread the insertions to database - * for a specific species ID and combination of condition parameters, - * to be able to have a single transaction to insert these data. - * This should not impact performances, as anyway INSERT statements are executed - * sequentially in MySQL. - *

    - * This thread is also for killing all queries performed by different threads - * when an error occurs in any thread. - * - * @author Frederic Bastian - * @version Bgee 14 Feb. 2017 - * @since Bgee 14 Feb. 2017 - */ - private static class InsertJob implements Runnable { - /** - * The {@code InsertPropagatedCalls} object that launched this tread. Allows this thread - * to be notified on error or job completion, and to share variables between threads - * used by this object. - */ - private final InsertPropagatedCalls callPropagator; - /** - * A {@code Map} where keys are {@code Condition}s already inserted into the database - * for the requested species before doing the call propagation, the associating value - * being an {@code Integer} that is the related global condition ID. - */ - private final Map globalCondsAlreadyInsertedMap; - /** - * A {@code Set} of {@code GlobalCondToRawCondTO}s already inserted into the database - * for the requested species before doing the call propagation. - */ - private final Set globalCondToRawConds; - - private InsertJob(InsertPropagatedCalls callPropagator, - Map globalCondsAlreadyInsertedMap, - Set globalCondToRawConds) { - log.traceEntry("{}, {}, {}", callPropagator, globalCondsAlreadyInsertedMap, globalCondToRawConds); - if (!callPropagator.computeAndInsertGlobalCond && - (globalCondsAlreadyInsertedMap == null || globalCondsAlreadyInsertedMap.isEmpty() || - globalCondToRawConds == null || globalCondToRawConds.isEmpty())) { - throw log.throwing(new IllegalArgumentException( - "Some global conditions should have already been computed and inserted.")); - } - this.callPropagator = callPropagator; - this.globalCondsAlreadyInsertedMap = Collections.unmodifiableMap( - globalCondsAlreadyInsertedMap == null ? new HashMap<>() : - new HashMap<>(globalCondsAlreadyInsertedMap)); - this.globalCondToRawConds = Collections.unmodifiableSet(globalCondToRawConds == null ? - new HashSet<>() : new HashSet<>(globalCondToRawConds)); - } - - @Override - public void run() { - log.traceEntry(); - - //We need a new connection to the database for each thread, so we use - //the ServiceFactory Supplier - final ServiceFactory factory = this.callPropagator.serviceFactorySupplier.get(); - final DAOManager daoManager = factory.getDAOManager(); - final ConditionDAO condDAO = daoManager.getConditionDAO(); - final GlobalExpressionCallDAO exprDAO = daoManager.getGlobalExpressionCallDAO(); - //in order to insert globalConditions - final Map insertedCondMap = new HashMap<>( - this.globalCondsAlreadyInsertedMap); - //relations between globalConditions and raw conditions - final Set globalCondToRawConds = new HashSet<>( - this.globalCondToRawConds); - - boolean errorInThisThread = false; - int groupsInserted = 0; - try { - //If all the global conds should have been inserted already - if (!this.callPropagator.computeAndInsertGlobalCond && - condDAO.getGlobalConditions( - Collections.singleton(this.callPropagator.speciesId), - generateDAOConditionFilters(null, this.callPropagator.condParams), - null) - .stream().noneMatch(e -> true)) { - throw log.throwing(new IllegalStateException( - "Global conditions should have been inserted for species " + - this.callPropagator.speciesId)); - } - - boolean firstInsert = true; - INSERT: while ((!this.callPropagator.jobCompleted || - //important to check that there is no remaining calls to insert, - //as other thread might set the jobCompleted flag to true - //before this thread finishes to insert all data. - !this.callPropagator.callsToInsert.isEmpty()) && - //but if an error occurred, we stop immediately in any case. - this.callPropagator.errorOccured == null) { - - //wait for consuming new data - Set toInsert = null; - try { - log.trace(BLOCKING_QUEUE_MARKER, "Trying to take Set of PipelineCalls"); - //here we ask to wait indefinitely - toInsert = this.callPropagator.callsToInsert.take(); - log.trace(BLOCKING_QUEUE_MARKER, "Done taking Set of {} PipelineCalls", - toInsert.size()); - } catch (InterruptedException e) { - //this Thread will be interrupted if an error occurred in an other Thread - //or if all computations are finished and this thread is waiting - //for more data to consume. - log.catching(Level.DEBUG, e); - continue INSERT; - } - assert toInsert != null; - - //wait for receiving data for starting the transaction, - //otherwise there might be some lock issues - if (firstInsert) { - startTransaction((MySQLDAOManager) daoManager); - firstInsert = false; - } - - // Here, we insert new conditions, and add them to the known conditions - Map newCondMap = InsertPropagatedCalls - .insertNewGlobalConditions(toInsert.stream() - .flatMap(c -> { - Set callConds = c.getBestDescendantPValues().stream() - .map(p -> p.getCondition()) - .collect(Collectors.toSet()); - callConds.add(c.getCondition()); - return callConds.stream(); - }) - .collect(Collectors.toSet()), - insertedCondMap.keySet(), condDAO); - if (!this.callPropagator.computeAndInsertGlobalCond && !newCondMap.isEmpty()) { - throw log.throwing(new IllegalStateException( - "All globalConditions should have been inserted already")); - } - if (!Collections.disjoint(insertedCondMap.keySet(), newCondMap.keySet())) { - throw log.throwing(new IllegalStateException("Error, new conditions already seen. " - + "new conditions: " + newCondMap.keySet() + " - existing conditions: " - + insertedCondMap.keySet())); - } - if (!Collections.disjoint(insertedCondMap.values(), newCondMap.values())) { - throw log.throwing(new IllegalStateException("Error, condition IDs reused. " - + "new IDs: " + newCondMap.values() + " - existing IDs: " - + insertedCondMap.values())); - } - insertedCondMap.putAll(newCondMap); - - //Now, we insert relations between globalConditions and source raw conditions, - //to be able to later retrieve relations between globalExpressions to expressions, - //without needing the table globalExpressionToExpression, that was very much too large - //(more than 10 billions rows for 29 species). - Set newGlobalCondToRawConds = - InsertPropagatedCalls.insertGlobalCondToRawCondsFromCalls(toInsert, - globalCondToRawConds, insertedCondMap, condDAO); - if (!this.callPropagator.computeAndInsertGlobalCond && - !newGlobalCondToRawConds.isEmpty()) { - throw log.throwing(new IllegalStateException( - "All globalCondToConds should have been inserted already")); - } - if (!Collections.disjoint(globalCondToRawConds, newGlobalCondToRawConds)) { - throw log.throwing(new IllegalStateException("Error, new condition relations already seen. " - + "new relations: " + newGlobalCondToRawConds + " - existing relations: " - + globalCondToRawConds)); - } - //Deactivate this assert, it is very slow and, anyway, there is - //a primary key(globalConditionId, conditionId) which makes - //this situation impossible. - // //We're not supposed to generate a same relation between a global condition - // //and a raw condition having different conditionRelationOrigins. - // //Since conditionRelationOrigin is taken into account in equals/hashCode, - // //make an assert here based solely on global condition ID and raw condition ID. - // assert newGlobalCondToRawConds.stream() - // .noneMatch(r1 -> globalCondToRawConds.stream() - // .anyMatch(r2 -> r1.getRawConditionId().equals(r2.getRawConditionId()) && - // r1.getGlobalConditionId().equals(r2.getGlobalConditionId()))): - // "Incorrect new relations: " + newGlobalCondToRawConds + " - " + globalCondToRawConds; - - globalCondToRawConds.addAll(newGlobalCondToRawConds); - - - // And we finish by inserting the computed calls - insertPropagatedCalls(toInsert, insertedCondMap, exprDAO); - if (log.isDebugEnabled()) { - log.debug("{} calls inserted for gene {}", toInsert.size(), - toInsert.iterator().next().getBgeeGeneId()); - } - - log.trace(INSERTION_MARKER, "Calls inserted."); - groupsInserted++; - if (log.isInfoEnabled() && groupsInserted % 100 == 0) { - log.info(INSERTION_MARKER, "{} genes inserted.", groupsInserted); - } - } - - } catch (Exception e) { - errorInThisThread = true; - if (this.callPropagator.errorOccured == null) { - this.callPropagator.errorOccured = e; - } - if (e instanceof RuntimeException) { - throw log.throwing((RuntimeException) e); - } - throw log.throwing(new IllegalStateException(e)); - - } finally { - assert this.callPropagator.jobCompleted || - this.callPropagator.errorOccured != null; - //we assume the insertion is done using MySQL, and we commit/rollback the transaction - try { - this.killAllDAOManagersIfNeeded(); - } finally { - try { - //recheck the jobCompleted flag in case this Thread was interrupted - //for unknown reason - if (this.callPropagator.jobCompleted && this.callPropagator.errorOccured == null) { - log.info("{} genes inserted, committing transaction", groupsInserted); - ((MySQLDAOManager) daoManager).getConnection().getRealConnection().commit(); - ((MySQLDAOManager) daoManager).getConnection().getRealConnection().setAutoCommit(true); - } else { - log.info("Rollbacking transaction"); - ((MySQLDAOManager) daoManager).getConnection().getRealConnection().rollback(); - ((MySQLDAOManager) daoManager).getConnection().getRealConnection().setAutoCommit(true); - } - } catch (SQLException e) { - if (errorInThisThread) { - //we are already going to throw an exception, so that's enough - log.catching(e); - } else { - if (this.callPropagator.errorOccured == null) { - this.callPropagator.errorOccured = e; - } - throw log.throwing(new IllegalStateException(e)); - } - } finally { - //notify the producer that the insertion is completed - synchronized(this.callPropagator.insertFinished) { - this.callPropagator.insertFinished.set(true); - this.callPropagator.insertFinished.notifyAll(); - } - //close connection - daoManager.close(); - } - } - } - - log.debug("Insert thread shut down"); - log.traceExit(); - } - - /** - * Kill the running queries to data source launched by other threads if an error occurred - * in any thread. - */ - private void killAllDAOManagersIfNeeded() { - log.traceEntry(); - if (this.callPropagator.errorOccured == null) { - log.traceExit(); return; - } - log.debug("Killing all DAO managers"); - this.callPropagator.daoManagers.stream() - .filter(dm -> !dm.isClosed() && !dm.isKilled()) - .forEach(dm -> dm.kill()); - - log.traceExit(); - } - - private void insertPropagatedCalls(Set propagatedCalls, - Map condMap, GlobalExpressionCallDAO dao) { - log.traceEntry("{}, {}, {}", propagatedCalls, condMap, dao); - - //Now, insert. We associate each PipelineCall to its generated TO for easier retrieval - Map callMap = propagatedCalls.stream() - .collect(Collectors.toMap( - c -> convertPipelineCallToGlobalExprCallTO( - EXPR_ID_COUNTER.incrementAndGet(), - condMap, c), - c -> c - )); - log.trace("Inserting {} GlobalExpressionCallTOs", callMap.keySet().size()); - //Maybe it would generate a query too large to insert all calls for one gene - //at once. But I think our max_allowed_packet_size is big enough and should be OK. - //Worst case scenario we'll add a loop here. - assert !callMap.isEmpty(); - dao.insertGlobalCalls(callMap.keySet()); - log.trace("Done inserting GlobalExpressionCallTOs"); - - //Note: actually, we don't fill this globalExpressionToExpression table anymore, - //it is very much too large (more than 10 billions rows for 29 species). - //We now retrieve the relations between globalExpression and expression - //through relations between globalConditions and conditions. -// //insert the relations between global expr IDs and raw expr IDs. -// //Note that we insert all relation, even the "invalid" ones (ABSENT calls in descendant, -// //PRESENT calls in parents; having all relations for descendants is essential for computing -// //a global rank score) -// log.trace("Start inserting GlobalExpressionToRawExpressionTOs"); -// Set globalToRawTOs = callMap.entrySet().stream() -// .flatMap(e -> { -// int globalExprId = e.getKey().getId(); -// PipelineCall call = e.getValue(); -// -// Set tos = new HashSet<>(); -// if (call.getSelfSourceCallTOs() != null) { -// tos.addAll(call.getSelfSourceCallTOs().stream() -// .map(p -> new GlobalExpressionToRawExpressionTO(p.getId(), -// globalExprId, GlobalExpressionToRawExpressionTO.CallOrigin.SELF)) -// .collect(Collectors.toSet())); +// } catch (SQLException e) { +// if (errorInThisThread) { +// //we are already going to throw an exception, so that's enough +// log.catching(e); +// } else { +// if (this.callPropagator.errorOccured == null) { +// this.callPropagator.errorOccured = e; +// } +// throw log.throwing(new IllegalStateException(e)); // } -// if (call.getParentSourceCallTOs() != null) { -// tos.addAll(call.getParentSourceCallTOs().stream() -// .map(p -> new GlobalExpressionToRawExpressionTO(p.getId(), -// globalExprId, GlobalExpressionToRawExpressionTO.CallOrigin.PARENT)) -// .collect(Collectors.toSet())); +// } finally { +// //notify the producer that the insertion is completed +// synchronized(this.callPropagator.insertFinished) { +// this.callPropagator.insertFinished.set(true); +// this.callPropagator.insertFinished.notifyAll(); // } -// if (call.getDescendantSourceCallTOs() != null) { -// tos.addAll(call.getDescendantSourceCallTOs().stream() -// .map(d -> new GlobalExpressionToRawExpressionTO(d.getId(), -// globalExprId, GlobalExpressionToRawExpressionTO.CallOrigin.DESCENDANT)) -// .collect(Collectors.toSet())); +// //close connection +// daoManager.close(); +// } +// } +// } +// +// log.debug("Insert thread shut down"); +// log.traceExit(); +// } +// +// /** +// * Kill the running queries to data source launched by other threads if an error occurred +// * in any thread. +// */ +// private void killAllDAOManagersIfNeeded() { +// log.traceEntry(); +// if (this.callPropagator.errorOccured == null) { +// log.traceExit(); return; +// } +// log.debug("Killing all DAO managers"); +// this.callPropagator.daoManagers.stream() +// .filter(dm -> !dm.isClosed() && !dm.isKilled()) +// .forEach(dm -> dm.kill()); +// +// log.traceExit(); +// } +// +// private void insertPropagatedCalls(Set propagatedCalls, +// Map condMap, GlobalExpressionCallDAO dao) { +// log.traceEntry("{}, {}, {}", propagatedCalls, condMap, dao); +// +// //Now, insert. We associate each PipelineCall to its generated TO for easier retrieval +// Map callMap = propagatedCalls.stream() +// .collect(Collectors.toMap( +// c -> convertPipelineCallToGlobalExprCallTO( +// EXPR_ID_COUNTER.incrementAndGet(), +// condMap, c), +// c -> c +// )); +// log.trace("Inserting {} GlobalExpressionCallTOs", callMap.keySet().size()); +// //Maybe it would generate a query too large to insert all calls for one gene +// //at once. But I think our max_allowed_packet_size is big enough and should be OK. +// //Worst case scenario we'll add a loop here. +// assert !callMap.isEmpty(); +// dao.insertGlobalCalls(callMap.keySet()); +// log.trace("Done inserting GlobalExpressionCallTOs"); +// +// //Note: actually, we don't fill this globalExpressionToExpression table anymore, +// //it is very much too large (more than 10 billions rows for 29 species). +// //We now retrieve the relations between globalExpression and expression +// //through relations between globalConditions and conditions. +//// //insert the relations between global expr IDs and raw expr IDs. +//// //Note that we insert all relation, even the "invalid" ones (ABSENT calls in descendant, +//// //PRESENT calls in parents; having all relations for descendants is essential for computing +//// //a global rank score) +//// log.trace("Start inserting GlobalExpressionToRawExpressionTOs"); +//// Set globalToRawTOs = callMap.entrySet().stream() +//// .flatMap(e -> { +//// int globalExprId = e.getKey().getId(); +//// PipelineCall call = e.getValue(); +//// +//// Set tos = new HashSet<>(); +//// if (call.getSelfSourceCallTOs() != null) { +//// tos.addAll(call.getSelfSourceCallTOs().stream() +//// .map(p -> new GlobalExpressionToRawExpressionTO(p.getId(), +//// globalExprId, GlobalExpressionToRawExpressionTO.CallOrigin.SELF)) +//// .collect(Collectors.toSet())); +//// } +//// if (call.getParentSourceCallTOs() != null) { +//// tos.addAll(call.getParentSourceCallTOs().stream() +//// .map(p -> new GlobalExpressionToRawExpressionTO(p.getId(), +//// globalExprId, GlobalExpressionToRawExpressionTO.CallOrigin.PARENT)) +//// .collect(Collectors.toSet())); +//// } +//// if (call.getDescendantSourceCallTOs() != null) { +//// tos.addAll(call.getDescendantSourceCallTOs().stream() +//// .map(d -> new GlobalExpressionToRawExpressionTO(d.getId(), +//// globalExprId, GlobalExpressionToRawExpressionTO.CallOrigin.DESCENDANT)) +//// .collect(Collectors.toSet())); +//// } +//// +//// return tos.stream(); +//// }) +//// .collect(Collectors.toSet()); +//// assert !globalToRawTOs.isEmpty(); +//// dao.insertGlobalExpressionToRawExpression(globalToRawTOs); +//// log.trace("Done inserting {} GlobalExpressionToRawExpressionTOs", globalToRawTOs.size()); +// log.traceExit(); +// } +// +// private GlobalExpressionCallTO convertPipelineCallToGlobalExprCallTO(long exprId, +// Map condMap, PipelineCall pipelineCall) { +// log.traceEntry("{}, {}, {}", exprId, condMap, pipelineCall); +// +// return log.traceExit(new GlobalExpressionCallTO(exprId, pipelineCall.getBgeeGeneId(), +// condMap.get(pipelineCall.getCondition()), +// //GlobalMeanRank: not a real attribute of the table. Maybe we should +// //create a subclass of GlobalExpressionCallTO to be returned by getGlobalExpressionCalls +// null, +// //GlobalExpressionCallDataTOs +// convertPipelineCallToExpressionCallDataTOs(pipelineCall), +// convertFDRPValuesToDAOFDRPValues(pipelineCall.getPValues(), condMap), +// convertFDRPValuesToDAOFDRPValues(pipelineCall.getBestDescendantPValues(), condMap))); +// } +// +// private static Set convertFDRPValuesToDAOFDRPValues( +// Set pValues, Map condMap) { +// log.traceEntry("{}", pValues); +// return log.traceExit(pValues.stream().map( p -> new DAOFDRPValue(p.getPValue(), +// (p instanceof FDRPValueCondition)? +// condMap.get(((FDRPValueCondition) p).getCondition()): null, +// p.getDataTypes().stream().map( dt -> { +// switch (dt) { +// case AFFYMETRIX: +// return DAODataType.AFFYMETRIX; +// case EST: +// return DAODataType.EST; +// case RNA_SEQ: +// return DAODataType.RNA_SEQ; +// case IN_SITU: +// return DAODataType.IN_SITU; +// case SC_RNA_SEQ: +// return DAODataType.SC_RNA_SEQ; +// default: +// throw log.throwing(new IllegalStateException( +// "Unsupported condition parameter: " + dt)); // } +// }).collect(Collectors.toSet()))) +// .collect(Collectors.toSet())); +// +// } +// +// private Set convertPipelineCallToExpressionCallDataTOs( +// PipelineCall pipelineCall) { +// log.traceEntry("{}", pipelineCall); +// +// return log.traceExit(pipelineCall.getCallData().stream() +// .map(cd -> { +// +// //Rank info: computed by the Perl pipeline after generation +// //of these global calls +//// BigDecimal meanRank = cd.getRank(); +//// BigDecimal meanRankNorm = cd.getNormalizedRank(); +//// BigDecimal weightForMeanRank = cd.getWeightForMeanRank(); +// +// return new GlobalExpressionCallDataTO( +// //data type +// this.callPropagator.utils.convertDataTypeToDAODataType( +// Collections.singleton(cd.getDataType())).iterator().next(), +// //self p-value observation counts +// cd.getDataPropagation().getSelfObservationCounts().entrySet().stream() +// .collect(Collectors.toMap( +// e -> convertCondParamAttrsToCondDAOAttrs(e.getKey()), +// e -> e.getValue())), +// //descendant p-value observation counts +// cd.getDataPropagation().getDescendantObservationCounts().entrySet().stream() +// .collect(Collectors.toMap( +// e -> convertCondParamAttrsToCondDAOAttrs(e.getKey()), +// e -> e.getValue())), +// //FDR-corrected p-values for individual data type: +// //they are not produced and stored in database in this way +// null, null, +// //rank info: computed by the Perl pipeline after generation +// //of these global calls +//// meanRank, meanRankNorm, weightForMeanRank +// null, null, null +// ); +// }).collect(Collectors.toSet())); +// } +// } +// +// public static void insertGlobalConditions(List speciesIds, +// Set condParams, final Supplier daoManagerSupplier, +// final Function serviceFactoryProvider) { +// log.traceEntry("{}, {}, {}, {}", speciesIds, condParams, daoManagerSupplier, serviceFactoryProvider); +// +// final Set clonedCondParams = Collections.unmodifiableSet( +// condParams.stream().distinct().collect(Collectors.toSet())); +// try(DAOManager commonManager = daoManagerSupplier.get()) { +// final List speciesIdsToUse = BgeeDBUtils.checkAndGetSpeciesIds(speciesIds, +// commonManager.getSpeciesDAO()); +// COND_ID_COUNTER.set(commonManager.getConditionDAO().getMaxGlobalConditionId()); +// +// //close connection immediately, but do not close the manager because of +// //the try-with-resource clause. +// commonManager.releaseResources(); +// +// speciesIdsToUse.parallelStream().forEach(speciesId -> { +// //Give as argument a Supplier of ServiceFactory so that this object +// //can provide a new connection to each parallel thread. +// InsertPropagatedCalls insert = new InsertPropagatedCalls( +// () -> serviceFactoryProvider.apply(daoManagerSupplier.get()), +// clonedCondParams, speciesId, 0, 0, false); +// try { +// insert.insertGlobalConditionsForOneSpecies(); +// } catch (Exception e) { +// throw log.throwing(new IllegalStateException(e)); +// } +// }); +// } +// } +// /** +// * +// * @param speciesIds +// * @param geneOffset An {@code int} that is the offset parameter to retrieve genes +// * to insert data for, for each of the requested species +// * independently. For instance, if two species and an offset +// * of 1000 were requested, the first gene retrieved +// * for the first species will have offset 1000 +// * among the genes of that species, the first gene retrieved +// * for the second species will have offset 1000 among the genes +// * of that other species. +// * @param geneRowCount An {@code int} that is the row_count parameter to retrieve genes +// * to insert data for, for each of the requested species +// * independently. For instance, if two species and a row count +// * of 1000 were requested, 1000 genes will be retrieved +// * for the first species, and 1000 genes for the second species. +// * If 0, all genes for the requested species are retrieved. +// * @param computeInsertGlobalCond A {@code boolean} defining whether global conditions +// * should be computed and inserted at the same time as +// * the propagated calls (if {@code true}), or whether they were +// * already computed and should be retrieved from the database +// * (if {@code false}). +// * @param condParams A {@code Set} of {@code ConditionDAO.Attribute}s, +// * defining the condition parameters that +// * are requested for queries, allowing to determine +// * which condition and expression information to target. +// */ +// public static void insert(List speciesIds, int geneOffset, int geneRowCount, +// boolean computeInsertGlobalCond, Set condParams) { +// log.traceEntry("{}, {}, {}, {}, {}", speciesIds, geneOffset, geneRowCount, +// computeInsertGlobalCond, condParams); +// InsertPropagatedCalls.insert(speciesIds, geneOffset, geneRowCount, computeInsertGlobalCond, +// condParams, DAOManager::getDAOManager, ServiceFactory::new); +// log.traceExit(); +// } +// /** +// * +// *

    +// * We need suppliers rather than already instantiated {@code DAOManager}s and {@code ServiceFactory}s +// * to provide new ones to each thread, in case we make a parallel implementation of this code. +// * +// * @param speciesIds +// * @param geneOffset An {@code int} that is the offset parameter to retrieve genes +// * to insert data for, for each of the requested species +// * independently. For instance, if two species and an offset +// * of 1000 were requested, the first gene retrieved +// * for the first species will have offset 1000 +// * among the genes of that species, the first gene retrieved +// * for the second species will have offset 1000 among the genes +// * of that other species. +// * @param geneRowCount An {@code int} that is the row_count parameter to retrieve genes +// * to insert data for, for each of the requested species +// * independently. For instance, if two species and a row count +// * of 1000 were requested, 1000 genes will be retrieved +// * for the first species, and 1000 genes for the second species. +// * If 0, all genes for the requested species are retrieved. +// * @param computeInsertGlobalCond A {@code boolean} defining whether global conditions +// * should be computed and inserted at the same time as +// * the propagated calls (if {@code true}), or whether they were +// * already computed and should be retrieved from the database +// * (if {@code false}). +// * @param condParams A {@code Set} of {@code ConditionDAO.Attribute}s, +// * defining the condition parameters that +// * are requested for queries, allowing to determine +// * which condition and expression information to target. +// * @param daoManagerSupplier The {@code Supplier} of {@code DAOManager} to use. +// * @param serviceFactoryProvider The {@code Function} accepting a {@code DAOManager} as argument +// * and returning a new {@code ServiceFactory}. +// */ +// public static void insert(List speciesIds, int geneOffset, int geneRowCount, +// boolean computeInsertGlobalCond, Set condParams, +// final Supplier daoManagerSupplier, +// final Function serviceFactoryProvider) { +// log.traceEntry("{}, {}, {}, {}, {}, {}, {}", speciesIds, geneOffset, geneRowCount, +// computeInsertGlobalCond, condParams, daoManagerSupplier, serviceFactoryProvider); +// +// // Sanity checks on attributes +// if (condParams == null || condParams.isEmpty()) { +// throw log.throwing(new IllegalArgumentException("Condition attributes should not be empty")); +// } +// final Set clonedCondParams = Collections.unmodifiableSet( +// condParams.stream().distinct().collect(Collectors.toSet())); +// +// +// try(DAOManager commonManager = daoManagerSupplier.get()) { +// //You can set max number of parallel threads from common pool. +// //we'll use the common pool and not a forked pool because forked pool +// //can't currently define parallelism for Streams +// //(see http://stackoverflow.com/questions/28985704/parallel-stream-from-a-hashset-doesnt-run-in-parallel) +// //use the sys prop "java.util.concurrent.ForkJoinPool.common.parallelism" in command line argument. +// +// // Get all species in Bgee even if some species IDs were provided, to check user input. +// // We need a specific DAOManager for this (commonManager), to not use the same as the one used +// // for each species. +// final List speciesIdsToUse = BgeeDBUtils.checkAndGetSpeciesIds(speciesIds, +// commonManager.getSpeciesDAO()); +// +// //we also need to set the max condition ID and max expression ID +// ConditionDAO condDAO = commonManager.getConditionDAO(); +//// GlobalExpressionCallDAO exprDAO = commonManager.getGlobalExpressionCallDAO(); +// COND_ID_COUNTER.set(condDAO.getMaxGlobalConditionId()); +//// EXPR_ID_COUNTER.set(exprDAO.getMaxGlobalExprId()); +// condDAO = null; +//// exprDAO = null; +// +// //close connection immediately, but do not close the manager because of +// //the try-with-resource clause. +// commonManager.releaseResources(); +// +// +// //Note: no parallel streams here, because the different streams would lock the same tables +// //in database. Parallel tasks are used per species. +// speciesIdsToUse.stream().forEach(speciesId -> { +// //Give as argument a Supplier of ServiceFactory so that this object +// //can provide a new connection to each parallel thread. +// InsertPropagatedCalls insert = new InsertPropagatedCalls( +// () -> serviceFactoryProvider.apply(daoManagerSupplier.get()), +// clonedCondParams, speciesId, geneOffset, geneRowCount, computeInsertGlobalCond); +// insert.insertOneSpecies(); +// }); +// } +// log.traceExit(); +// } +// +// private static void startTransaction(MySQLDAOManager daoManager) throws Exception { +// log.traceEntry("{}", daoManager); +// //we assume the insertion is done using MySQL, and we start a transaction +// log.debug(INSERTION_MARKER, "Trying to start transaction..."); +// //try several attempts in case the first SELECT queries lock relevant tables +// int maxAttempt = 10; +// int i = 0; +// TRANSACTION: while (true) { +// try { +// //TODO: reimplement properly in MySQLDAOManager. +// //I do it here because I want to turn autocommit to true before setting the transaction level, +// //to be sure it's properly set for the next transaction +// daoManager.getConnection().getRealConnection().setAutoCommit(true); +// daoManager.getConnection().getRealConnection() +// .setTransactionIsolation(Connection.TRANSACTION_READ_UNCOMMITTED); +// daoManager.getConnection().getRealConnection().setAutoCommit(false); +// break TRANSACTION; +// } catch (Exception e) { +// if (i < maxAttempt) { +// log.catching(Level.DEBUG, e); +// log.debug(INSERTION_MARKER, +// "Trying to start transaction failed, {} try over {}", +// i + 1, maxAttempt); +// try { +// Thread.sleep(2000); +// } catch(InterruptedException ex) { +// log.catching(ex); +// Thread.currentThread().interrupt(); +// throw log.throwing(ex); +// } +// } else { +// log.debug(INSERTION_MARKER, +// "Starting transaction failed, {} try over {}", +// i + 1, maxAttempt); +// //that was the last try, throw exception +// throw e; +// } +// } +// i++; +// } +// +// log.info(INSERTION_MARKER, "Starting transaction"); +// log.traceExit(); +// } +// +// private static ConditionGraph loadConditionGraph(ConditionGraphService condGraphService, +// Set conds, boolean inferConditions) { +// log.traceEntry("{}, {}, {}", condGraphService, conds, inferConditions); +// +// if (!inferConditions) { +// //If we don't infer conditions they were already pre-computed +// //and we have nothing more to do +// return log.traceExit(condGraphService.loadConditionGraph(conds, +// false, false)); +// } +// //Infer conditions. +// //Of note, non-informative anat. entities/cell types are not considered when inferring +// //propagated conditions (except roots, or terms used in annotations). +// ConditionGraph conditionGraph = condGraphService.loadConditionGraph( +// conds, +// true, //propagate to ancestor conditions +// false //We do not propagate to descendant conditions anymore +// ); +// //Since we propagate only to ancestor as of Bgee 15.0, +// //we don't need to filter out descendant propagated strains, stages, sexes +// return log.traceExit(conditionGraph); +// } +// +// private static Map insertNewGlobalConditions(Set condsToInsert, +// Set insertedGlobalConditions, ConditionDAO condDAO) { +// log.traceEntry("{}, {}, {}", condsToInsert, insertedGlobalConditions, condDAO); +// +// //First, we retrieve the conditions not already present in the database +// Set conds = new HashSet<>(condsToInsert); +// conds.removeAll(insertedGlobalConditions); +// +// //now we create the Map associating each Condition to insert to a generated ID for insertion +// Map newConds = conds.stream() +// .collect(Collectors.toMap(c -> c, c -> COND_ID_COUNTER.incrementAndGet())); +// +// //now we insert the conditions +// Set condTOs = newConds.entrySet().stream() +// .map(e -> mapConditionToConditionTO(e.getValue(), e.getKey())) +// .collect(Collectors.toSet()); +// if (!condTOs.isEmpty()) { +// condDAO.insertGlobalConditions(condTOs); +// } +// +// //return new conditions with IDs +// return log.traceExit(newConds); +// } +// +// private static Set insertGlobalCondToRawCondsFromCalls( +// Set propagatedCalls, Set insertedRels, +// Map condMap, ConditionDAO condDAO) { +// log.traceEntry("{}, {}, {}, {}", propagatedCalls, insertedRels, condMap, condDAO); +// +// //We map PipelineCalls to GlobalConditionToRawConditionTOs +// Set toInsert = propagatedCalls.stream() +// .flatMap(c -> { +// Integer globalCondId = condMap.get(c.getCondition()); +// if (globalCondId == null) { +// throw log.throwing(new IllegalArgumentException("Missing inserted condition: " +// + c.getCondition())); +// } +// +// Set relTOs = new HashSet<>(); +// if (c.getParentSourceCallTOs() != null) { +// relTOs.addAll(c.getParentSourceCallTOs().stream() +// .map(source -> new PipelineGlobalCondToRawCondTO( +// source.getConditionId(), +// globalCondId, +// GlobalConditionToRawConditionTO.ConditionRelationOrigin.PARENT)) +// .collect(Collectors.toSet())); +// } +// if (c.getSelfSourceCallTOs() != null) { +// relTOs.addAll(c.getSelfSourceCallTOs().stream() +// .map(source -> new PipelineGlobalCondToRawCondTO( +// source.getConditionId(), +// globalCondId, +// GlobalConditionToRawConditionTO.ConditionRelationOrigin.SELF)) +// .collect(Collectors.toSet())); +// } +// if (c.getDescendantSourceCallTOs() != null) { +// relTOs.addAll(c.getDescendantSourceCallTOs().stream() +// .map(source -> new PipelineGlobalCondToRawCondTO( +// source.getConditionId(), +// globalCondId, +// GlobalConditionToRawConditionTO.ConditionRelationOrigin.DESCENDANT)) +// .collect(Collectors.toSet())); +// } +// return relTOs.stream(); +// }).collect(Collectors.toSet()); +// +// return log.traceExit(insertGlobalCondToRawConds(toInsert, insertedRels, condDAO)); +// } +// +// private static Set insertGlobalCondToRawConds( +// Set toInsert, Set insertedRels, +// ConditionDAO condDAO) { +// log.traceEntry("{}, {}, {}, {}", toInsert, insertedRels, condDAO); +// +// //We remove those already inserted +// Set newRels = new HashSet<>(toInsert); +// newRels.removeAll(insertedRels); +// +// //Deactivate this assert, it is very slow and, anyway, there is +// //a primary key(globalConditionId, conditionId) which makes +// //this situation impossible. +//// //We're not supposed to generate a same relation between a global condition +//// //and a raw condition having different conditionRelationOrigins. +//// //Since conditionRelationOrigin is taken into account in equals/hashCode, +//// //make an assert here based solely on global condition ID and raw condition ID. +//// assert newRels.stream() +//// .noneMatch(r1 -> newRels.stream() +//// .filter(r2 -> r2 != r1) +//// .anyMatch(r2 -> r1.getRawConditionId().equals(r2.getRawConditionId()) && +//// r1.getGlobalConditionId().equals(r2.getGlobalConditionId()))): +//// "Incorrect new relations: " + newRels; +// +// //now we insert the relations +// if (!newRels.isEmpty()) { +// condDAO.insertGlobalConditionToRawCondition(newRels.stream() +// .map(c -> (GlobalConditionToRawConditionTO) c) +// .collect(Collectors.toSet())); +// } +// +// //return new rels +// return log.traceExit(newRels); +// } +// +// /** +// * An {@code int} that is the offset parameter to retrieve genes to insert data for. +// * @see #geneRowCount; +// */ +// private final int geneOffset; +// /** +// * An {@code int} that is the row_count parameter to retrieve genes to insert data for. +// * If 0, all genes are retrieved. +// * @see #geneOffset; +// */ +// private final int geneRowCount; +// /** +// * A {@code boolean} defining whether global conditions should be computed and inserted +// * at the same time as the propagated calls (if {@code true}), or whether they were +// * already computed and should be retrieved from the database (if {@code false}). +// */ +// private final boolean computeAndInsertGlobalCond; +// /** +// * A {@code volatile} {@code Throwable} allowing to notify all threads when an error occurs, +// * and to store the actual error that occurred. +// */ +// private volatile Throwable errorOccured; +// /** +// * A {@code volatile} {@code boolean} allowing to notify all threads that the insertion +// * of all calls for a species is finished. +// */ +// private volatile boolean jobCompleted; +// /** +// * A {@code Supplier} of {@code ServiceFactory}s to be acquired from different threads. +// */ +// private final Supplier serviceFactorySupplier; +// /** +// * A {@code BlockingQueue} containing {@code Set}s of {@code PipelineCall}s to be inserted. +// * Each contained {@code Set} is inserted into the database in a single INSERT statement. +// * Propagating threads will add new {@code PipelineCall}s to be inserted to this queue, +// * and the insertion thread will remove them from the queue for insertion. +// */ +// private final BlockingQueue> callsToInsert; +// /** +// * An {@code AtomicBoolean} that will allow the main thread to acquire a lock and wait on it, +// * to be notified by the insert thread when all calls are inserted (computations can be faster +// * than insertions). +// */ +// private final AtomicBoolean insertFinished; +// /** +// * A concurrent {@code Set} of {@code DAOManager}s backed by a {@code ConcurrentMap}, +// * in order to kill queries run in different threads in case of error in any thread. +// * The killing will be performed by {@link #insertThread}, as we know this thread +// * will be running during the whole process and will performing fast queries only. +// */ +// private final Set daoManagers; +// /** +// * A {@code Set} of {@code ConditionDAO.Attribute}s defining the condition parameters +// * that were requested for queries, allowing to determine how the data should be aggregated. +// */ +// private final EnumSet condParams; +// /** +// * An {@code int} that is the ID of the species to propagate calls for. +// */ +// private final int speciesId; +// /** +// * A {@code ConcurrentMap} where keys are {@code Condition}s, the associated value +// * being a {@code Set} of {@code Condition}s that are their ancestral conditions, +// * as retrieved in the method {@link #propagatePipelineCalls(Map, ConditionGraph)}. +// * This {@code ConcurrentMap} serves as a cache, to not query the {@code ConditionGraph} +// * each time. +// */ +// //Should a similar mechanism be directly implemented in ConditionGraph? +// //Seems complicated and might not worth it in all situations. +// //But it seems to result in a 30% speed increase in this class. +// private final ConcurrentMap> condToAncestors; +// /** +// * A {@code ConcurrentMap} where keys are {@code Condition}s, the associated value +// * being a {@code Set} of {@code Condition}s that are their descendant conditions, +// * as retrieved in the method {@link #propagatePipelineCalls(Map, ConditionGraph)}. +// * This {@code ConcurrentMap} serves as a cache, to not query the {@code ConditionGraph} +// * each time. +// */ +// //Should a similar mechanism be directly implemented in ConditionGraph? +// //Seems complicated and might not worth it in all situations. +// //But it seems to result in a 30% speed increase in this class. +// //Note: actually as of Bgee 14.2 we do not propagate absent calls to substructures anymore +//// private final ConcurrentMap> condToDescendants; // -// return tos.stream(); -// }) -// .collect(Collectors.toSet()); -// assert !globalToRawTOs.isEmpty(); -// dao.insertGlobalExpressionToRawExpression(globalToRawTOs); -// log.trace("Done inserting {} GlobalExpressionToRawExpressionTOs", globalToRawTOs.size()); - log.traceExit(); - } - - private GlobalExpressionCallTO convertPipelineCallToGlobalExprCallTO(long exprId, - Map condMap, PipelineCall pipelineCall) { - log.traceEntry("{}, {}, {}", exprId, condMap, pipelineCall); - - return log.traceExit(new GlobalExpressionCallTO(exprId, pipelineCall.getBgeeGeneId(), - condMap.get(pipelineCall.getCondition()), - //GlobalMeanRank: not a real attribute of the table. Maybe we should - //create a subclass of GlobalExpressionCallTO to be returned by getGlobalExpressionCalls - null, - //GlobalExpressionCallDataTOs - convertPipelineCallToExpressionCallDataTOs(pipelineCall), - convertFDRPValuesToDAOFDRPValues(pipelineCall.getPValues(), condMap), - convertFDRPValuesToDAOFDRPValues(pipelineCall.getBestDescendantPValues(), condMap))); - } - - private static Set convertFDRPValuesToDAOFDRPValues( - Set pValues, Map condMap) { - log.traceEntry("{}", pValues); - return log.traceExit(pValues.stream().map( p -> new DAOFDRPValue(p.getPValue(), - (p instanceof FDRPValueCondition)? - condMap.get(((FDRPValueCondition) p).getCondition()): null, - p.getDataTypes().stream().map( dt -> { - switch (dt) { - case AFFYMETRIX: - return DAODataType.AFFYMETRIX; - case EST: - return DAODataType.EST; - case RNA_SEQ: - return DAODataType.RNA_SEQ; - case IN_SITU: - return DAODataType.IN_SITU; - case SC_RNA_SEQ: - return DAODataType.SC_RNA_SEQ; - default: - throw log.throwing(new IllegalStateException( - "Unsupported condition parameter: " + dt)); - } - }).collect(Collectors.toSet()))) - .collect(Collectors.toSet())); - - } - - private Set convertPipelineCallToExpressionCallDataTOs( - PipelineCall pipelineCall) { - log.traceEntry("{}", pipelineCall); - - return log.traceExit(pipelineCall.getCallData().stream() - .map(cd -> { - - //Rank info: computed by the Perl pipeline after generation - //of these global calls -// BigDecimal meanRank = cd.getRank(); -// BigDecimal meanRankNorm = cd.getNormalizedRank(); -// BigDecimal weightForMeanRank = cd.getWeightForMeanRank(); - - return new GlobalExpressionCallDataTO( - //data type - this.callPropagator.utils.convertDataTypeToDAODataType( - Collections.singleton(cd.getDataType())).iterator().next(), - //self p-value observation counts - cd.getDataPropagation().getSelfObservationCounts().entrySet().stream() - .collect(Collectors.toMap( - e -> convertCondParamAttrsToCondDAOAttrs(e.getKey()), - e -> e.getValue())), - //descendant p-value observation counts - cd.getDataPropagation().getDescendantObservationCounts().entrySet().stream() - .collect(Collectors.toMap( - e -> convertCondParamAttrsToCondDAOAttrs(e.getKey()), - e -> e.getValue())), - //FDR-corrected p-values for individual data type: - //they are not produced and stored in database in this way - null, null, - //rank info: computed by the Perl pipeline after generation - //of these global calls -// meanRank, meanRankNorm, weightForMeanRank - null, null, null - ); - }).collect(Collectors.toSet())); - } - } - - public static void insertGlobalConditions(List speciesIds, - Set condParams, final Supplier daoManagerSupplier, - final Function serviceFactoryProvider) { - log.traceEntry("{}, {}, {}, {}", speciesIds, condParams, daoManagerSupplier, serviceFactoryProvider); - - final Set clonedCondParams = Collections.unmodifiableSet( - condParams.stream().distinct().collect(Collectors.toSet())); - try(DAOManager commonManager = daoManagerSupplier.get()) { - final List speciesIdsToUse = BgeeDBUtils.checkAndGetSpeciesIds(speciesIds, - commonManager.getSpeciesDAO()); - COND_ID_COUNTER.set(commonManager.getConditionDAO().getMaxGlobalConditionId()); - - //close connection immediately, but do not close the manager because of - //the try-with-resource clause. - commonManager.releaseResources(); - - speciesIdsToUse.parallelStream().forEach(speciesId -> { - //Give as argument a Supplier of ServiceFactory so that this object - //can provide a new connection to each parallel thread. - InsertPropagatedCalls insert = new InsertPropagatedCalls( - () -> serviceFactoryProvider.apply(daoManagerSupplier.get()), - clonedCondParams, speciesId, 0, 0, false); - try { - insert.insertGlobalConditionsForOneSpecies(); - } catch (Exception e) { - throw log.throwing(new IllegalStateException(e)); - } - }); - } - } - /** - * - * @param speciesIds - * @param geneOffset An {@code int} that is the offset parameter to retrieve genes - * to insert data for, for each of the requested species - * independently. For instance, if two species and an offset - * of 1000 were requested, the first gene retrieved - * for the first species will have offset 1000 - * among the genes of that species, the first gene retrieved - * for the second species will have offset 1000 among the genes - * of that other species. - * @param geneRowCount An {@code int} that is the row_count parameter to retrieve genes - * to insert data for, for each of the requested species - * independently. For instance, if two species and a row count - * of 1000 were requested, 1000 genes will be retrieved - * for the first species, and 1000 genes for the second species. - * If 0, all genes for the requested species are retrieved. - * @param computeInsertGlobalCond A {@code boolean} defining whether global conditions - * should be computed and inserted at the same time as - * the propagated calls (if {@code true}), or whether they were - * already computed and should be retrieved from the database - * (if {@code false}). - * @param condParams A {@code Set} of {@code ConditionDAO.Attribute}s, - * defining the condition parameters that - * are requested for queries, allowing to determine - * which condition and expression information to target. - */ - public static void insert(List speciesIds, int geneOffset, int geneRowCount, - boolean computeInsertGlobalCond, Set condParams) { - log.traceEntry("{}, {}, {}, {}, {}", speciesIds, geneOffset, geneRowCount, - computeInsertGlobalCond, condParams); - InsertPropagatedCalls.insert(speciesIds, geneOffset, geneRowCount, computeInsertGlobalCond, - condParams, DAOManager::getDAOManager, ServiceFactory::new); - log.traceExit(); - } - /** - * - *

    - * We need suppliers rather than already instantiated {@code DAOManager}s and {@code ServiceFactory}s - * to provide new ones to each thread, in case we make a parallel implementation of this code. - * - * @param speciesIds - * @param geneOffset An {@code int} that is the offset parameter to retrieve genes - * to insert data for, for each of the requested species - * independently. For instance, if two species and an offset - * of 1000 were requested, the first gene retrieved - * for the first species will have offset 1000 - * among the genes of that species, the first gene retrieved - * for the second species will have offset 1000 among the genes - * of that other species. - * @param geneRowCount An {@code int} that is the row_count parameter to retrieve genes - * to insert data for, for each of the requested species - * independently. For instance, if two species and a row count - * of 1000 were requested, 1000 genes will be retrieved - * for the first species, and 1000 genes for the second species. - * If 0, all genes for the requested species are retrieved. - * @param computeInsertGlobalCond A {@code boolean} defining whether global conditions - * should be computed and inserted at the same time as - * the propagated calls (if {@code true}), or whether they were - * already computed and should be retrieved from the database - * (if {@code false}). - * @param condParams A {@code Set} of {@code ConditionDAO.Attribute}s, - * defining the condition parameters that - * are requested for queries, allowing to determine - * which condition and expression information to target. - * @param daoManagerSupplier The {@code Supplier} of {@code DAOManager} to use. - * @param serviceFactoryProvider The {@code Function} accepting a {@code DAOManager} as argument - * and returning a new {@code ServiceFactory}. - */ - public static void insert(List speciesIds, int geneOffset, int geneRowCount, - boolean computeInsertGlobalCond, Set condParams, - final Supplier daoManagerSupplier, - final Function serviceFactoryProvider) { - log.traceEntry("{}, {}, {}, {}, {}, {}, {}", speciesIds, geneOffset, geneRowCount, - computeInsertGlobalCond, condParams, daoManagerSupplier, serviceFactoryProvider); - - // Sanity checks on attributes - if (condParams == null || condParams.isEmpty()) { - throw log.throwing(new IllegalArgumentException("Condition attributes should not be empty")); - } - final Set clonedCondParams = Collections.unmodifiableSet( - condParams.stream().distinct().collect(Collectors.toSet())); - - - try(DAOManager commonManager = daoManagerSupplier.get()) { - //You can set max number of parallel threads from common pool. - //we'll use the common pool and not a forked pool because forked pool - //can't currently define parallelism for Streams - //(see http://stackoverflow.com/questions/28985704/parallel-stream-from-a-hashset-doesnt-run-in-parallel) - //use the sys prop "java.util.concurrent.ForkJoinPool.common.parallelism" in command line argument. - - // Get all species in Bgee even if some species IDs were provided, to check user input. - // We need a specific DAOManager for this (commonManager), to not use the same as the one used - // for each species. - final List speciesIdsToUse = BgeeDBUtils.checkAndGetSpeciesIds(speciesIds, - commonManager.getSpeciesDAO()); - - //we also need to set the max condition ID and max expression ID - ConditionDAO condDAO = commonManager.getConditionDAO(); -// GlobalExpressionCallDAO exprDAO = commonManager.getGlobalExpressionCallDAO(); - COND_ID_COUNTER.set(condDAO.getMaxGlobalConditionId()); -// EXPR_ID_COUNTER.set(exprDAO.getMaxGlobalExprId()); - condDAO = null; -// exprDAO = null; - - //close connection immediately, but do not close the manager because of - //the try-with-resource clause. - commonManager.releaseResources(); - - - //Note: no parallel streams here, because the different streams would lock the same tables - //in database. Parallel tasks are used per species. - speciesIdsToUse.stream().forEach(speciesId -> { - //Give as argument a Supplier of ServiceFactory so that this object - //can provide a new connection to each parallel thread. - InsertPropagatedCalls insert = new InsertPropagatedCalls( - () -> serviceFactoryProvider.apply(daoManagerSupplier.get()), - clonedCondParams, speciesId, geneOffset, geneRowCount, computeInsertGlobalCond); - insert.insertOneSpecies(); - }); - } - log.traceExit(); - } - - private static void startTransaction(MySQLDAOManager daoManager) throws Exception { - log.traceEntry("{}", daoManager); - //we assume the insertion is done using MySQL, and we start a transaction - log.debug(INSERTION_MARKER, "Trying to start transaction..."); - //try several attempts in case the first SELECT queries lock relevant tables - int maxAttempt = 10; - int i = 0; - TRANSACTION: while (true) { - try { - //TODO: reimplement properly in MySQLDAOManager. - //I do it here because I want to turn autocommit to true before setting the transaction level, - //to be sure it's properly set for the next transaction - daoManager.getConnection().getRealConnection().setAutoCommit(true); - daoManager.getConnection().getRealConnection() - .setTransactionIsolation(Connection.TRANSACTION_READ_UNCOMMITTED); - daoManager.getConnection().getRealConnection().setAutoCommit(false); - break TRANSACTION; - } catch (Exception e) { - if (i < maxAttempt) { - log.catching(Level.DEBUG, e); - log.debug(INSERTION_MARKER, - "Trying to start transaction failed, {} try over {}", - i + 1, maxAttempt); - try { - Thread.sleep(2000); - } catch(InterruptedException ex) { - log.catching(ex); - Thread.currentThread().interrupt(); - throw log.throwing(ex); - } - } else { - log.debug(INSERTION_MARKER, - "Starting transaction failed, {} try over {}", - i + 1, maxAttempt); - //that was the last try, throw exception - throw e; - } - } - i++; - } - - log.info(INSERTION_MARKER, "Starting transaction"); - log.traceExit(); - } - - private static ConditionGraph loadConditionGraph(ConditionGraphService condGraphService, - Set conds, boolean inferConditions) { - log.traceEntry("{}, {}, {}", condGraphService, conds, inferConditions); - - if (!inferConditions) { - //If we don't infer conditions they were already pre-computed - //and we have nothing more to do - return log.traceExit(condGraphService.loadConditionGraph(conds, - false, false)); - } - //Infer conditions. - //Of note, non-informative anat. entities/cell types are not considered when inferring - //propagated conditions (except roots, or terms used in annotations). - ConditionGraph conditionGraph = condGraphService.loadConditionGraph( - conds, - true, //propagate to ancestor conditions - false //We do not propagate to descendant conditions anymore - ); - //Since we propagate only to ancestor as of Bgee 15.0, - //we don't need to filter out descendant propagated strains, stages, sexes - return log.traceExit(conditionGraph); - } - - private static Map insertNewGlobalConditions(Set condsToInsert, - Set insertedGlobalConditions, ConditionDAO condDAO) { - log.traceEntry("{}, {}, {}", condsToInsert, insertedGlobalConditions, condDAO); - - //First, we retrieve the conditions not already present in the database - Set conds = new HashSet<>(condsToInsert); - conds.removeAll(insertedGlobalConditions); - - //now we create the Map associating each Condition to insert to a generated ID for insertion - Map newConds = conds.stream() - .collect(Collectors.toMap(c -> c, c -> COND_ID_COUNTER.incrementAndGet())); - - //now we insert the conditions - Set condTOs = newConds.entrySet().stream() - .map(e -> mapConditionToConditionTO(e.getValue(), e.getKey())) - .collect(Collectors.toSet()); - if (!condTOs.isEmpty()) { - condDAO.insertGlobalConditions(condTOs); - } - - //return new conditions with IDs - return log.traceExit(newConds); - } - - private static Set insertGlobalCondToRawCondsFromCalls( - Set propagatedCalls, Set insertedRels, - Map condMap, ConditionDAO condDAO) { - log.traceEntry("{}, {}, {}, {}", propagatedCalls, insertedRels, condMap, condDAO); - - //We map PipelineCalls to GlobalConditionToRawConditionTOs - Set toInsert = propagatedCalls.stream() - .flatMap(c -> { - Integer globalCondId = condMap.get(c.getCondition()); - if (globalCondId == null) { - throw log.throwing(new IllegalArgumentException("Missing inserted condition: " - + c.getCondition())); - } - - Set relTOs = new HashSet<>(); - if (c.getParentSourceCallTOs() != null) { - relTOs.addAll(c.getParentSourceCallTOs().stream() - .map(source -> new PipelineGlobalCondToRawCondTO( - source.getConditionId(), - globalCondId, - GlobalConditionToRawConditionTO.ConditionRelationOrigin.PARENT)) - .collect(Collectors.toSet())); - } - if (c.getSelfSourceCallTOs() != null) { - relTOs.addAll(c.getSelfSourceCallTOs().stream() - .map(source -> new PipelineGlobalCondToRawCondTO( - source.getConditionId(), - globalCondId, - GlobalConditionToRawConditionTO.ConditionRelationOrigin.SELF)) - .collect(Collectors.toSet())); - } - if (c.getDescendantSourceCallTOs() != null) { - relTOs.addAll(c.getDescendantSourceCallTOs().stream() - .map(source -> new PipelineGlobalCondToRawCondTO( - source.getConditionId(), - globalCondId, - GlobalConditionToRawConditionTO.ConditionRelationOrigin.DESCENDANT)) - .collect(Collectors.toSet())); - } - return relTOs.stream(); - }).collect(Collectors.toSet()); - - return log.traceExit(insertGlobalCondToRawConds(toInsert, insertedRels, condDAO)); - } - - private static Set insertGlobalCondToRawConds( - Set toInsert, Set insertedRels, - ConditionDAO condDAO) { - log.traceEntry("{}, {}, {}, {}", toInsert, insertedRels, condDAO); - - //We remove those already inserted - Set newRels = new HashSet<>(toInsert); - newRels.removeAll(insertedRels); - - //Deactivate this assert, it is very slow and, anyway, there is - //a primary key(globalConditionId, conditionId) which makes - //this situation impossible. -// //We're not supposed to generate a same relation between a global condition -// //and a raw condition having different conditionRelationOrigins. -// //Since conditionRelationOrigin is taken into account in equals/hashCode, -// //make an assert here based solely on global condition ID and raw condition ID. -// assert newRels.stream() -// .noneMatch(r1 -> newRels.stream() -// .filter(r2 -> r2 != r1) -// .anyMatch(r2 -> r1.getRawConditionId().equals(r2.getRawConditionId()) && -// r1.getGlobalConditionId().equals(r2.getGlobalConditionId()))): -// "Incorrect new relations: " + newRels; - - //now we insert the relations - if (!newRels.isEmpty()) { - condDAO.insertGlobalConditionToRawCondition(newRels.stream() - .map(c -> (GlobalConditionToRawConditionTO) c) - .collect(Collectors.toSet())); - } - - //return new rels - return log.traceExit(newRels); - } - - /** - * An {@code int} that is the offset parameter to retrieve genes to insert data for. - * @see #geneRowCount; - */ - private final int geneOffset; - /** - * An {@code int} that is the row_count parameter to retrieve genes to insert data for. - * If 0, all genes are retrieved. - * @see #geneOffset; - */ - private final int geneRowCount; - /** - * A {@code boolean} defining whether global conditions should be computed and inserted - * at the same time as the propagated calls (if {@code true}), or whether they were - * already computed and should be retrieved from the database (if {@code false}). - */ - private final boolean computeAndInsertGlobalCond; - /** - * A {@code volatile} {@code Throwable} allowing to notify all threads when an error occurs, - * and to store the actual error that occurred. - */ - private volatile Throwable errorOccured; - /** - * A {@code volatile} {@code boolean} allowing to notify all threads that the insertion - * of all calls for a species is finished. - */ - private volatile boolean jobCompleted; - /** - * A {@code Supplier} of {@code ServiceFactory}s to be acquired from different threads. - */ - private final Supplier serviceFactorySupplier; - /** - * A {@code BlockingQueue} containing {@code Set}s of {@code PipelineCall}s to be inserted. - * Each contained {@code Set} is inserted into the database in a single INSERT statement. - * Propagating threads will add new {@code PipelineCall}s to be inserted to this queue, - * and the insertion thread will remove them from the queue for insertion. - */ - private final BlockingQueue> callsToInsert; - /** - * An {@code AtomicBoolean} that will allow the main thread to acquire a lock and wait on it, - * to be notified by the insert thread when all calls are inserted (computations can be faster - * than insertions). - */ - private final AtomicBoolean insertFinished; - /** - * A concurrent {@code Set} of {@code DAOManager}s backed by a {@code ConcurrentMap}, - * in order to kill queries run in different threads in case of error in any thread. - * The killing will be performed by {@link #insertThread}, as we know this thread - * will be running during the whole process and will performing fast queries only. - */ - private final Set daoManagers; - /** - * A {@code Set} of {@code ConditionDAO.Attribute}s defining the condition parameters - * that were requested for queries, allowing to determine how the data should be aggregated. - */ - private final EnumSet condParams; - /** - * An {@code int} that is the ID of the species to propagate calls for. - */ - private final int speciesId; - /** - * A {@code ConcurrentMap} where keys are {@code Condition}s, the associated value - * being a {@code Set} of {@code Condition}s that are their ancestral conditions, - * as retrieved in the method {@link #propagatePipelineCalls(Map, ConditionGraph)}. - * This {@code ConcurrentMap} serves as a cache, to not query the {@code ConditionGraph} - * each time. - */ - //Should a similar mechanism be directly implemented in ConditionGraph? - //Seems complicated and might not worth it in all situations. - //But it seems to result in a 30% speed increase in this class. - private final ConcurrentMap> condToAncestors; - /** - * A {@code ConcurrentMap} where keys are {@code Condition}s, the associated value - * being a {@code Set} of {@code Condition}s that are their descendant conditions, - * as retrieved in the method {@link #propagatePipelineCalls(Map, ConditionGraph)}. - * This {@code ConcurrentMap} serves as a cache, to not query the {@code ConditionGraph} - * each time. - */ - //Should a similar mechanism be directly implemented in ConditionGraph? - //Seems complicated and might not worth it in all situations. - //But it seems to result in a 30% speed increase in this class. - //Note: actually as of Bgee 14.2 we do not propagate absent calls to substructures anymore -// private final ConcurrentMap> condToDescendants; - - public InsertPropagatedCalls(Supplier serviceFactorySupplier, - Set condParams, int speciesId, int geneOffset, int geneRowCount, - boolean computeAndInsertGlobalCond) { - this(serviceFactorySupplier, condParams, speciesId, geneOffset, geneRowCount, - computeAndInsertGlobalCond, new CallServiceUtils()); - } - public InsertPropagatedCalls(Supplier serviceFactorySupplier, - Set condParams, int speciesId, int geneOffset, int geneRowCount, - boolean computeAndInsertGlobalCond, CallServiceUtils utils) { - super(serviceFactorySupplier.get(), utils); - if (condParams == null || condParams.isEmpty()) { - throw log.throwing(new IllegalArgumentException("Condition attributes should not be empty")); - } - if (geneOffset < 0 || geneRowCount < 0) { - throw log.throwing(new IllegalArgumentException( - "geneOffset and geneRowCount cannot be negative")); - } - if (geneOffset > 0 && geneRowCount == 0) { - throw log.throwing(new IllegalArgumentException( - "geneRowCount must be provided if geneOffset is provided")); - } - this.serviceFactorySupplier = serviceFactorySupplier; - this.condParams = EnumSet.copyOf(condParams); - this.speciesId = speciesId; - this.geneOffset = geneOffset; - this.geneRowCount = geneRowCount; - this.computeAndInsertGlobalCond = computeAndInsertGlobalCond; - //use a LinkedBlockingDeque because we are going to do lots of insert/remove, - //and because we don't care about element order. We are going to block - //if there are too many results waiting to be inserted, to not overload the memory - this.callsToInsert = new LinkedBlockingDeque<>(MAX_NUMBER_OF_CALLS_TO_INSERT); - this.insertFinished = new AtomicBoolean(false); - this.daoManagers = Collections.newSetFromMap(new ConcurrentHashMap<>()); - this.errorOccured = null; - this.jobCompleted = false; - - this.condToAncestors = new ConcurrentHashMap<>(); -// this.condToDescendants = new ConcurrentHashMap<>(); - } - - private void insertGlobalConditionsForOneSpecies() throws Exception { - log.traceEntry(); - log.info("Start inserting global conditions for the species {} with combinations of condition parameters {}...", - this.speciesId, this.condParams); - - try (DAOManager mainManager = this.getDaoManager()) { - ConditionDAO condDAO = mainManager.getConditionDAO(); - - Species species = this.getServiceFactory().getSpeciesService().loadSpeciesByIds( - Collections.singleton(this.speciesId), false).iterator().next(); - - //First, we retrieve the raw conditions already present in database. - final Map rawCondMap = Collections.unmodifiableMap( - this.loadRawConditionMap(Collections.singleton(species))); - log.info("{} raw data conditions for species {}", rawCondMap.size(), speciesId); - - // We use all existing conditions in the species, and infer all propagated conditions - log.info("Starting condition inference for species {}...", this.speciesId); - Map> globalCondToSelfRawCondIds = rawCondMap.entrySet() - .stream() - .map(e -> new AbstractMap.SimpleEntry<>( - mapRawDataConditionToCondition(e.getValue()), - new HashSet<>(Arrays.asList(e.getKey())))) - .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue(), - (v1, v2) -> {v1.addAll(v2); return v1;})); - assert globalCondToSelfRawCondIds.values().stream().flatMap(s -> s.stream()) - .collect(Collectors.toSet()).equals(rawCondMap.keySet()); - - final ConditionGraph conditionGraph = loadConditionGraph( - this.getServiceFactory().getConditionGraphService(), - globalCondToSelfRawCondIds.keySet(), - true); - log.info("Done condition inference for species {}.", this.speciesId); - - startTransaction((MySQLDAOManager) mainManager); - - Map globalCondsInserted = InsertPropagatedCalls - .insertNewGlobalConditions(conditionGraph.getConditions(), - new HashSet<>(), condDAO); - log.info("{} conditions inserted for species {}", globalCondsInserted.size(), this.speciesId); - assert conditionGraph.getConditions().equals(globalCondsInserted.keySet()); - - Set toInsert = new HashSet<>(); - //SELF ConditionRelationOrigin - toInsert.addAll(globalCondsInserted.entrySet().stream() - .flatMap(e -> globalCondToSelfRawCondIds.getOrDefault(e.getKey(), new HashSet<>()) - .stream() - .map(rawCondId -> new PipelineGlobalCondToRawCondTO(rawCondId, e.getValue(), - GlobalConditionToRawConditionTO.ConditionRelationOrigin.SELF))) - .collect(Collectors.toSet())); - //DESCENDANT ConditionRelationOrigin - toInsert.addAll(globalCondsInserted.entrySet().stream() - //get the ancestors of the iterated condition - .flatMap(e -> conditionGraph.getAncestorConditions(e.getKey()) - .stream() - //retrieve the raw condition IDs associated to the iterated condition - .flatMap(ancestor -> globalCondToSelfRawCondIds - .getOrDefault(e.getKey(), new HashSet<>()) - .stream() - //Create an association from the ancestor condition ID - //to the raw condition IDs of the iterated condition - //with ConditionOrigin DESCENDANT - .map(rawCondId -> new PipelineGlobalCondToRawCondTO(rawCondId, - globalCondsInserted.get(ancestor), - GlobalConditionToRawConditionTO.ConditionRelationOrigin.DESCENDANT)))) - .collect(Collectors.toSet())); - Set inserted = InsertPropagatedCalls - .insertGlobalCondToRawConds(toInsert, new HashSet<>(), condDAO); - - assert toInsert.stream().map(to -> to.getGlobalConditionId()).collect(Collectors.toSet()) - .equals(new HashSet<>(globalCondsInserted.values())); - assert toInsert.stream().map(to -> to.getRawConditionId()).collect(Collectors.toSet()) - .equals(globalCondToSelfRawCondIds.values().stream() - .flatMap(s -> s.stream()).collect(Collectors.toSet())); - assert inserted.equals(toInsert); - - ((MySQLDAOManager) mainManager).getConnection().getRealConnection().commit(); - ((MySQLDAOManager) mainManager).getConnection().getRealConnection().setAutoCommit(true); - - log.info("{} GlobalCondToRawCondTOs inserted for species {}", toInsert.size(), this.speciesId); - } - log.traceExit(); - } - - private void insertOneSpecies() { - log.traceEntry(); - - log.info("Start inserting of propagated calls for the species {} with combinations of condition parameters {}...", - this.speciesId, this.condParams); - - Thread insertThread = null; - // close connection to database between each species, to avoid idle - // connection reset or for parallel execution - try (DAOManager mainManager = this.getDaoManager()) { - - Species species = this.getServiceFactory().getSpeciesService().loadSpeciesByIds( - Collections.singleton(this.speciesId), false).iterator().next(); - - //First, we retrieve the raw conditions already present in database. - final Map rawCondMap = Collections.unmodifiableMap( - this.loadRawConditionMap(Collections.singleton(species))); - log.info("{} Conditions for species {}", rawCondMap.size(), speciesId); - //Retrieve the global conditions and mappings to raw conditions already inserted - final Map globalCondAlreadyInserted = loadGlobalConditionMap( - Collections.singleton(species), - generateDAOConditionFilters(null, this.condParams), - null, - mainManager.getConditionDAO(), - this.getServiceFactory().getAnatEntityService(), - this.getServiceFactory().getDevStageService(), - this.getServiceFactory().getSexService(), - this.getServiceFactory().getStrainService()) - .entrySet().stream() - .collect(Collectors.toMap(e -> e.getValue(), e -> e.getKey())); - final Set globalCondToCondAlreadyInserted = - mainManager.getConditionDAO().getGlobalCondToRawCondBySpeciesIds( - Collections.singleton(this.speciesId), this.condParams) - .stream().map(gctc -> new PipelineGlobalCondToRawCondTO(gctc)) - .collect(Collectors.toSet()); - - // We use all existing conditions in the species, and infer all propagated conditions - log.info("Starting condition inference..."); - ConditionGraphService condGraphService = this.getServiceFactory().getConditionGraphService(); - final ConditionGraph conditionGraph = this.computeAndInsertGlobalCond ? - loadConditionGraph(condGraphService, - rawCondMap.values() - .stream().map(rawCond -> mapRawDataConditionToCondition(rawCond)) - .collect(Collectors.toSet()), - true) : - //In case the global conditions were pre-computed - loadConditionGraph(condGraphService, globalCondAlreadyInserted.keySet(), - false); - log.info("Done condition inference."); - - //we retrieve the IDs of genes with expression data. This is because making the computation - //a whole species at a time can use too much memory for species with large amount of data. - //Also, the computations for those species are slow so we want to go parallel. - final List bgeeGeneIds = Collections.unmodifiableList( - mainManager.getGeneDAO() - .getGenesWithDataBySpeciesIdsOrdered(Collections.singleton(speciesId), - this.geneOffset, this.geneRowCount) - .stream().map(g -> g.getId()) - .collect(Collectors.toList())); - log.info("{} genes with data retrieved for species {}", bgeeGeneIds.size(), speciesId); - - //Remaining computations/insertions will be made in separate threads - //with a separate database connection, so we close the main connection immediately, - //but do not close the manager because of the try-with-resource clause. - mainManager.releaseResources(); - - //PARALLEL EXECUTION: here we create the independent thread responsible for - //inserting the data into the data source - insertThread = new Thread(new InsertJob(this, globalCondAlreadyInserted, - globalCondToCondAlreadyInserted)); - //just to be accessible from the Stream to notify of exceptions - final Thread localInsertThread = insertThread; - //PARALLEL EXECUTION: start the insertion Thread - insertThread.start(); - - //PARALLEL EXECUTION: we generate groups of genes of size GENES_PER_ITERATION - //and run the computations in parallel between groups - //(important to convert to float here before dividing, otherwise the rounding could be incorrect) - int iterationCount = (int) Math.ceil((float) bgeeGeneIds.size()/(float) GENE_PARALLEL_GROUP_SIZE); - IntStream.range(0, iterationCount).parallel() - .mapToObj(i -> bgeeGeneIds.subList(i * GENE_PARALLEL_GROUP_SIZE, - ((i + 1) * GENE_PARALLEL_GROUP_SIZE) > bgeeGeneIds.size()? - bgeeGeneIds.size(): ((i + 1) * GENE_PARALLEL_GROUP_SIZE))) - .forEach(subsetGeneIds -> { - //check at each iteration if an error occurred in another thread - this.checkErrorOccurred(); - - //We need a new connection to the database for each thread, so we use - //a ServiceFactory Supplier - final ServiceFactory threadServiceFactory = this.serviceFactorySupplier.get(); - - try (DAOManager threadDAOManager = threadServiceFactory.getDAOManager()) { - //PARALLEL EXECUTION: each thread-specific DAOManager is registered - //to be able to kill all queries in case of error in any thread. - //The killing will be performed by this.insertThread, as we know this thread - //will be running during the whole process and will be performing fast queries only. - this.daoManagers.add(threadDAOManager); - - log.debug("Processing {} genes...", subsetGeneIds.size()); - // We propagate calls. Each Map contains all propagated calls for one gene - final Stream> propagatedCalls = - this.generatePropagatedCalls( - new HashSet<>(subsetGeneIds), rawCondMap, conditionGraph, - threadDAOManager); - - //Provide the calls to insert to the Thread managing the insertions - //through the dedicated BlockingQueue - propagatedCalls.forEach(set -> { - //Check error status - this.checkErrorOccurred(); - try { - //wait indefinitely for space in the queue to be available - //(to not overload the memory) - log.trace(BLOCKING_QUEUE_MARKER, "Offering Set of {} PipelineCalls", - set.size()); - this.callsToInsert.put(set); - } catch (InterruptedException e) { - this.exceptionOccurs(e, localInsertThread); - } - }); - - log.debug("Done processing {} genes.", subsetGeneIds.size()); - } catch (Exception e) { - this.exceptionOccurs(e, localInsertThread); - } - }); - - //very important to set this flag here for the insertion thread to know it should quit. - this.jobCompleted = true; - - } catch (Exception e) { - this.exceptionOccurs(e, insertThread); - } finally { - //if there are no more data to be inserted, - //wake up the insert thread that might still be waiting for new data to insert - this.interruptInsertIfNeeded(insertThread); - } - assert this.jobCompleted || this.errorOccured != null; - - //now we need to wait for the Insert thread to complete the call insertions - //before quitting: moving to another species while we still lock the tables would be bad. - //If we run the computations with a high enough number of threads, - //the computations are faster than the insertions - log.info("Computations finished, continuing insertion."); - synchronized(this.insertFinished) { - while (!this.insertFinished.get()) { - try { - this.insertFinished.wait(); - } catch (InterruptedException e) { - throw log.throwing(new IllegalStateException(e)); - } - } - } - - - log.info("Done inserting of propagated calls for the species {} with combinations of condition parameters {}...", - this.speciesId, this.condParams); - - log.traceExit(); - } - - /** - * Method to check if an {@code Exception} occurred in a different {@code Thread} - * than the caller {@code Thread}, launched by this {@code InsertPropagatedCalls} object. - * @throws IllegalStateException If an {@code Exception} occurred in a different {@code Thread}. - */ - private void checkErrorOccurred() throws IllegalStateException { - log.traceEntry(); - if (this.errorOccured != null) { - log.debug("Stop execution following error in other Thread."); - throw new IllegalStateException("Exception thrown in another thread, stop job."); - } - log.traceExit(); - } - - /** - * Method rethrowing any {@code Exception} as a {@code RuntimeException} and storing - * it in {@link #errorOccured} and notifying {@link #insertThread} that an error occurred. - * @param e - * @param insertThread - * @throws RuntimeException - */ - private void exceptionOccurs(Exception e, Thread insertThread) throws RuntimeException { - log.traceEntry("{}, {}", e, insertThread); - //set errorOccured for all threads to know there was an error - if (this.errorOccured == null) { - this.errorOccured = e; - } - //wake up the insert thread that might be waiting to consume new data. - //important to set errorOccured before calling this method. - this.interruptInsertIfNeeded(insertThread); - //throw exception appropriately - if (e instanceof RuntimeException) { - throw log.throwing((RuntimeException) e); - } - throw log.throwing(new IllegalStateException(e)); - } - - private void interruptInsertIfNeeded(Thread insertThread) { - log.traceEntry("{}", insertThread); - Set waitingStates = EnumSet.of(Thread.State.BLOCKED, Thread.State.WAITING, - Thread.State.TIMED_WAITING); - if (insertThread != null && waitingStates.contains(insertThread.getState()) && - (this.errorOccured != null || (this.jobCompleted && this.callsToInsert.isEmpty()))) { - log.debug("Interrupting insert thread"); - insertThread.interrupt(); - } - log.traceExit(); - } - - private Map loadRawConditionMap(Collection species) { - log.traceEntry("{}", species); - - //TODO: to refactor with method org.bgee.model.CommonService.loadConditionMapFromResultSet - Map speMap = species.stream() - .collect(Collectors.toMap(s -> s.getId(), s -> s, (s1, s2) -> s1)); - Set anatEntityIds = new HashSet<>(); - Set stageIds = new HashSet<>(); - Set cellTypeIds = new HashSet<>(); - Set sexIds = new HashSet<>(); - Set strainIds = new HashSet<>(); - Set conditionTOs = new HashSet<>(); - //check that we have covered all condition parameters - if (EnumSet.allOf(ConditionDAO.Attribute.class).stream() - .filter(c -> c.isConditionParameter()).count() != 5) { - throw log.throwing(new IllegalStateException("Some condition parameters not covered")); - } - - RawDataConditionTOResultSet rs = this.getDaoManager().getRawDataConditionDAO() - .getRawDataConditionsFromRawConditionFilters( - Set.of(new DAORawDataConditionFilter(speMap.keySet(), - null, null, null, null, null)), - null); - - while (rs.next()) { - RawDataConditionTO condTO = rs.getTO(); - if (!speMap.keySet().contains(condTO.getSpeciesId())) { - throw log.throwing(new IllegalArgumentException( - "The retrieved ConditionTOs do not match the provided Species.")); - } - conditionTOs.add(condTO); - //As of Bgee 15.0, only the cellTypeId could be null - assert condTO.getAnatEntityId() != null; - assert condTO.getStageId() != null; - assert condTO.getSex() != null; - assert condTO.getStrainId() != null; - if (condTO.getAnatEntityId() != null) { - anatEntityIds.add(condTO.getAnatEntityId()); - } else { - anatEntityIds.add(ConditionDAO.ANAT_ENTITY_ROOT_ID); - } - if (condTO.getStageId() != null) { - stageIds.add(condTO.getStageId()); - } else { - stageIds.add(ConditionDAO.DEV_STAGE_ROOT_ID); - } - if (condTO.getCellTypeId() != null) { - cellTypeIds.add(condTO.getCellTypeId()); - } else { - cellTypeIds.add(ConditionDAO.CELL_TYPE_ROOT_ID); - } - if (condTO.getSex() != null) { - sexIds.add(condTO.getSex().getStringRepresentation()); - } else { - sexIds.add(DAORawDataSex.NA.getStringRepresentation()); - } - if (condTO.getStrainId() != null) { - strainIds.add(condTO.getStrainId()); - } else { - strainIds.add(ConditionDAO.STRAIN_ROOT_ID); - } - } - - Set allAnatEntityIds = new HashSet<>(anatEntityIds); - allAnatEntityIds.addAll(cellTypeIds); - final Map anatMap = allAnatEntityIds.isEmpty()? new HashMap<>(): - this.getServiceFactory().getAnatEntityService().loadAnatEntities( - speMap.keySet(), true, allAnatEntityIds, false) - .collect(Collectors.toMap(a -> a.getId(), a -> a)); - if (!allAnatEntityIds.isEmpty() && anatMap.size() != allAnatEntityIds.size()) { - allAnatEntityIds.removeAll(anatMap.keySet()); - throw log.throwing(new IllegalStateException("Some anat. entities used in a condition " - + "are not supposed to exist in the related species. Species: " + speMap.keySet() - + " - anat. entities: " + allAnatEntityIds)); - } - final Map stageMap = stageIds.isEmpty()? new HashMap<>(): - this.getServiceFactory().getDevStageService().loadDevStages( - speMap.keySet(), true, stageIds, false) - .collect(Collectors.toMap(s -> s.getId(), s -> s)); - if (!stageIds.isEmpty() && stageMap.size() != stageIds.size()) { - stageIds.removeAll(stageMap.keySet()); - throw log.throwing(new IllegalStateException("Some stages used in a condition " - + "are not supposed to exist in the related species. Species: " + speMap.keySet() - + " - stages: " + stageIds)); - } - - return log.traceExit(conditionTOs.stream() - .collect(Collectors.toMap(cTO -> cTO.getId(), - cTO -> new RawDataCondition( - Optional.ofNullable(anatMap.get(cTO.getAnatEntityId() == null ? - ConditionDAO.ANAT_ENTITY_ROOT_ID : cTO.getAnatEntityId())) - .orElseThrow(() -> new IllegalStateException("Anat. entity not found: " - + cTO.getAnatEntityId())), - Optional.ofNullable(stageMap.get(cTO.getStageId() == null ? - ConditionDAO.DEV_STAGE_ROOT_ID : cTO.getStageId())) - .orElseThrow(() -> new IllegalStateException("Stage not found: " - + cTO.getStageId())), - Optional.ofNullable(anatMap.get(cTO.getCellTypeId() == null ? - ConditionDAO.CELL_TYPE_ROOT_ID : cTO.getCellTypeId())) - .orElseThrow(() -> new IllegalStateException("Cell type not found: " - + cTO.getCellTypeId())), - mapDAORawDataSexToRawDataSex(cTO.getSex() == null ? - DAORawDataSex.NA : cTO.getSex()), - mapDAORawDataStrainToRawDataStrain(cTO.getStrainId() == null ? - ConditionDAO.STRAIN_ROOT_ID : cTO.getStrainId()), - Optional.ofNullable(speMap.get(cTO.getSpeciesId())).orElseThrow( - () -> new IllegalStateException("Species not found: " - + cTO.getSpeciesId()))) - )) - ); - } - - /** - * Generate propagated and reconciled expression calls. - * - * @param geneIds A {@code Collection} of {@code Integer}s that are the Bgee IDs - * of the genes for which to return the {@code ExpressionCall}s. - * @param condMap A {@code Map} where keys are {@code Integer}s that are condition IDs, - * the associated value being the corresponding {@code RawDataCondition} - * with attributes populated according to the requested - * condition parameters. - * @param conditionGraph A {@code ConditionGraph} containing the {@code Condition}s - * and relations considering attributes according - * to the requested condition parameters. - * @param daoManager The {@code DAOManager} to use to retrieve - * {@code DAO}s to perform queries to the data source. - * @return A {@code Stream} of {@code Map}s where keys are {@code Set} of - * {@code ConditionDAO.Attribute}s representing combinations of - * condition parameters, the associated value being a {@code Set} - * of {@code ExpressionCall}s that are propagated and reconciled - * expression calls for one gene according to the associated combination. - */ - private Stream> generatePropagatedCalls( - Set geneIds, Map condMap, ConditionGraph conditionGraph, - DAOManager daoManager) { - log.traceEntry("{}, {}, {}, {}", geneIds, condMap, conditionGraph, daoManager); - - log.trace(COMPUTE_MARKER, "Creating Splitereator with DAO queries..."); - this.checkErrorOccurred(); - final RawExpressionCallDAO rawCallDAO = daoManager.getRawExpressionCallDAO(); - final Stream streamRawCallTOs = - this.performsRawExpressionCallTOQuery(geneIds, rawCallDAO); - - this.checkErrorOccurred(); - final Map>> samplePValueTOsByDataType = - performsSamplePValueQuery(geneIds, daoManager); - - final CallSpliterator> spliterator = - new CallSpliterator<>(streamRawCallTOs, samplePValueTOsByDataType); - final Stream> callTOsByGeneStream = - StreamSupport.stream(spliterator, false).onClose(() -> spliterator.close()); - - log.trace(COMPUTE_MARKER, "Done creating Splitereator with DAO queries."); - - Stream> reconciledCalls = callTOsByGeneStream - // First we convert each Set for a gene - // into one Map> having source RawExpressionCallTO, - .map(geneData -> geneData.stream() - .collect(Collectors.toMap( - rawExprCallData -> mapRawCallTOToPipelineCall( - rawExprCallData.getRawExpressionCallTO(), - condMap.get(rawExprCallData.getRawExpressionCallTO().getConditionId()), - this.condParams), - rawExprCallData -> mapExpExprTOsToPipelineCallData( - rawExprCallData.getSamplePValueTOsPerDataType(), - this.condParams)))) - - //Now, we group all PipelineCalls and PipelineCallDatas mapped to a same Condition - //g: Map> - //NOTE: there can still be key collision after Bgee 15.0 because of merge of, e.g., - //raw data sexes 'not annotated' and 'mixed' into the data sex 'ANY'. - .map(g -> g.entrySet().stream().collect(Collectors - //we group the entries Entry> by condition - //and merge them. - .toMap( - e -> e.getKey().getCondition(), - e -> e, - (e1, e2) -> { - PipelineCall call1 = e1.getKey(); - PipelineCall call2 = e2.getKey(); - assert call1.getParentSourceCallTOs() == null || - call1.getParentSourceCallTOs().isEmpty(); - assert call1.getDescendantSourceCallTOs() == null || - call1.getDescendantSourceCallTOs().isEmpty(); - assert call1.getSelfSourceCallTOs() != null && - !call1.getSelfSourceCallTOs().isEmpty(); - assert call2.getParentSourceCallTOs() == null || - call2.getParentSourceCallTOs().isEmpty(); - assert call2.getDescendantSourceCallTOs() == null || - call2.getDescendantSourceCallTOs().isEmpty(); - assert call2.getSelfSourceCallTOs() != null && - !call2.getSelfSourceCallTOs().isEmpty(); - - assert Integer.compare(call1.getBgeeGeneId(), call2.getBgeeGeneId()) == 0; - assert call1.getCondition().equals(call2.getCondition()); - assert call1.getDataPropagation().getCondParamCombinations() - .equals(call2.getDataPropagation().getCondParamCombinations()); - assert call1.getDataPropagation().getCondParamCombinations().stream() - .map(comb -> call1.getDataPropagation().getPropagationState(comb)) - .allMatch(propState -> PropagationState.SELF.equals(propState)); - assert call2.getDataPropagation().getCondParamCombinations().stream() - .map(comb -> call2.getDataPropagation().getPropagationState(comb)) - .allMatch(propState -> PropagationState.SELF.equals(propState)); - - Set combinedTOs = - new HashSet<>(call1.getSelfSourceCallTOs()); - combinedTOs.addAll(call2.getSelfSourceCallTOs()); - PipelineCall combinedCall = new PipelineCall( - call1.getBgeeGeneId(), call1.getCondition(), combinedTOs); - - Set> combinedData = new HashSet<>(e1.getValue()); - combinedData.addAll(e2.getValue()); - - return new AbstractMap.SimpleEntry<>(combinedCall, combinedData); - } - ) - ) - //Now retrieve the Entries that were reduced, and collect them into a Map. - //The returned value of this map function is of the same type as the input element: - //Map> - .values().stream().collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue())) - ) - //then we propagate all PipelineCalls of the Map (associated to one gene only), - //and retrieve the original and the propagated calls. - //g: Map> - .map(g -> { - //propagatePipelineCalls returns only the new propagated calls, - //we need to add the original calls to the Map for following steps - Map>> calls = - this.propagatePipelineCalls(g, conditionGraph); - calls.putAll(g); - return calls; - }) - - //then we reconcile calls for a same gene-condition - //g: Map> - .map(g -> { - log.trace(COMPUTE_MARKER, "Starting to reconcile {} PipelineCalls.", g.size()); - this.checkErrorOccurred(); - //group calls per Condition (they all are about the same gene already) - final Map> callGroup = g.entrySet().stream() - .collect(Collectors.groupingBy(e -> e.getKey().getCondition(), - Collectors.mapping(e2 -> e2.getKey(), Collectors.toSet()))); - //group CallData per Condition (they all are about the same gene already) - final Map>> callDataGroup = g.entrySet().stream() - .collect(Collectors.groupingBy(e -> e.getKey().getCondition(), - Collectors.mapping(e2 -> e2.getValue(), Collectors.toSet()))) // produce Map> - .entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue() - .stream().flatMap(ps -> ps.stream()).collect(Collectors.toSet()))); // produce Map> - - // Reconcile calls and return all of them in one Set - Set s = callGroup.keySet().stream() - .map(c -> reconcileGeneCalls(callGroup.get(c), callDataGroup.get(c))) - //reconcileGeneCalls return null if there was no valid data to propagate - //(e.g., only "present" calls in parent conditions) - .filter(c -> c != null) - .collect(Collectors.toSet()); - log.trace(COMPUTE_MARKER, "Done reconciliation, {} PipelineCalls produced.", s.size()); - return s; - }) - - //Now we have a final step since Bgee 15.0: For each call, we have computed - //FDR-corrected p-values for all combinations of data types, considering all p-values - //in the condition itself and in its descendant conditions. - //Now for each call and each combination of data types, we need to find - //the best corrected p-value among the descendant conditions - //s: Set - .map(s -> { - log.trace(COMPUTE_MARKER, "Finding best descendant p-values for {} PipelineCalls.", s.size()); - //First we create a Map to more easily retrieve calls from a condition - Map callPerCondition = s.stream() - .map(c -> new AbstractMap.SimpleEntry<>(c.getCondition(), c)) - //At this point there should be only one call per condition, - //and thus no key collision - .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue())); - //Now, for each call, and for each combinations of data types, - //we are going to retrieve the best corrected p-value among the calls - //in descendant conditions. - //Compute a Map parent -> descendants - Map> parentToDescendantConds = callPerCondition.keySet() - .stream() - .flatMap(cond -> this.condToAncestors.computeIfAbsent( - cond, - k -> conditionGraph.getAncestorConditions(k)).stream() - .filter(parent -> callPerCondition.containsKey(parent)) - .map(parent -> new AbstractMap.SimpleEntry<>(parent, - new HashSet<>(Arrays.asList(cond))))) - .collect(Collectors.toMap(e -> e.getKey(), - e -> e.getValue(), - (v1, v2) -> {v1.addAll(v2); return v1;})); - Set> allDataTypeCombs = DataType.getAllPossibleDataTypeCombinations(); - return s.stream().map(c -> { - Map, FDRPValueCondition> bestPValuePerDataTypeComb = new HashMap<>(); - Set descendantCalls = parentToDescendantConds - //Some conditions have no descendant conditions obviously - .getOrDefault(c.getCondition(), new HashSet<>()) - .stream() - .map(cond -> callPerCondition.get(cond)) - .filter(descCond -> descCond != null) - .collect(Collectors.toSet()); - for (PipelineCall descendantCall: descendantCalls) { - Map, FDRPValue> pValuePerDataTypeComb = - descendantCall.getPValues().stream() - .map(p -> new AbstractMap.SimpleEntry<>(p.getDataTypes(), p)) - .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue())); - for (EnumSet comb: allDataTypeCombs) { - EnumSet bestMatchComb = DataType - .findCombinationWithGreatestOverlap( - pValuePerDataTypeComb.keySet(), comb); - if (bestMatchComb != null) { - FDRPValue existingPVal = pValuePerDataTypeComb.get(bestMatchComb); - FDRPValueCondition newPVal = new FDRPValueCondition( - existingPVal.getPValue(), comb, descendantCall.getCondition()); - bestPValuePerDataTypeComb.merge(comb, newPVal, - (p1, p2) -> p1.getPValue().compareTo(p2.getPValue()) == -1? - p1: p2); - } - } - } - log.trace("Done searching best descendant p-values for call: {}", c); - return new PipelineCall(c.getBgeeGeneId(), c.getCondition(), c.getCallData(), - c.getPValues(), bestPValuePerDataTypeComb.values(), - c.getParentSourceCallTOs(), c.getSelfSourceCallTOs(), - c.getDescendantSourceCallTOs()); - }).collect(Collectors.toSet()); - }); - - return log.traceExit(reconciledCalls); - } - - //************************************************************************* - // METHODS PERFORMING THE QUERIES TO THE DAOs - //************************************************************************* - /** - * Perform query to retrieve expressed calls without the post-processing of - * the results returned by {@code DAO}s. - * - * @param geneIds A {@code Collection} of {@code Integer}s that are the Bgee IDs of the genes - * for which to return the {@code RawExpressionCallTO}s. - * @param rawCallDAO The {@code RawExpressionCallDAO} to use to retrieve {@code RawExpressionCallTO}s - * from data source. - * @return The {@code Stream} of {@code RawExpressionCallTO}s. - */ - private Stream performsRawExpressionCallTOQuery(Set geneIds, - RawExpressionCallDAO rawCallDAO) throws IllegalArgumentException { - log.traceEntry("{}, {}", geneIds, rawCallDAO); - - Stream expr = rawCallDAO - .getExpressionCallsOrderedByGeneIdAndExprId(geneIds) - //retrieve the Stream resulting from the query. Note that the query is not executed - //as long as the Stream is not consumed (lazy-loading). - .stream(); - - return log.traceExit(expr); - } - - private Map>> performsSamplePValueQuery( - Set geneIds, DAOManager daoManager) throws IllegalArgumentException { - log.traceEntry("{}, {}", geneIds, daoManager); - - //TODO: Former DAO to access these data, we still use it for all data but RNA-Seq and scRNA-Seq, - //to update to use the new system for all data types. - //Of note, SamplePValueDAO could be completely deleted, - //and SamplePValueTO replaced with PipelineSamplePValueTO defined in this class. - //PipelineSamplePValueTO would not have to extend SamplePValueTO, - //but would implement the same attributes - SamplePValueDAO samplePValueDAO = daoManager.getSamplePValueDAO(); - - //New system to access this information - Set rawDataFilters = Set.of(new DAORawDataFilter(geneIds, null, null)); - // ************ RNA-Seq and scRNA-Seq ************ - RNASeqResultAnnotatedSampleDAO rnaSeqResultDAO = daoManager.getRnaSeqResultAnnotatedSampleDAO(); - EnumSet rnaSeqAttrs = EnumSet.of( - RNASeqResultAnnotatedSampleDAO.Attribute.BGEE_GENE_ID, - RNASeqResultAnnotatedSampleDAO.Attribute.LIBRARY_ANNOTATED_SAMPLE_ID, - RNASeqResultAnnotatedSampleDAO.Attribute.EXPRESSION_ID, - RNASeqResultAnnotatedSampleDAO.Attribute.PVALUE); - LinkedHashMap rnaSeqOrderingAttrs = - new LinkedHashMap<>(); - rnaSeqOrderingAttrs.put( - RNASeqResultAnnotatedSampleDAO.OrderingAttribute.BGEE_GENE_ID, - DAO.Direction.ASC); - rnaSeqOrderingAttrs.put( - RNASeqResultAnnotatedSampleDAO.OrderingAttribute.EXPRESSION_ID, - DAO.Direction.ASC); - - Map>> map = new HashMap<>(); - for (DataType dt: DataType.values()) { - switch (dt) { - case AFFYMETRIX: - map.put(dt, samplePValueDAO.getAffymetrixPValuesOrderedByGeneIdAndExprId(geneIds).stream() - .map(p -> p)); - break; - case EST: - map.put(dt, samplePValueDAO.getESTPValuesOrderedByGeneIdAndExprId(geneIds).stream() - .map(p -> p)); - break; - case IN_SITU: - map.put(dt, samplePValueDAO.getInSituPValuesOrderedByGeneIdAndExprId(geneIds).stream() - .map(p -> p)); - break; - case RNA_SEQ: - map.put(dt, rnaSeqResultDAO.getResultAnnotatedSamples(rawDataFilters, false, true, null, null, - rnaSeqAttrs, rnaSeqOrderingAttrs).stream() - //We don't have experimentId in the returned TOs, - //but annotatedSampleIds are unique in RNA-Seq data. - //We arbitrarily defined the experimentId type - //needed by PipelineSamplePValueTO as String - .map(to -> new PipelineSamplePValueTO(to))); - break; - case SC_RNA_SEQ: - map.put(dt, rnaSeqResultDAO.getResultAnnotatedSamples(rawDataFilters, true, true, null, null, - rnaSeqAttrs, rnaSeqOrderingAttrs).stream() - //We don't have experimentId in the returned TOs, - //but annotatedSampleIds are unique in RNA-Seq data. - //We arbitrarily defined the experimentId type - //needed by PipelineSamplePValueTO as String - .map(to -> new PipelineSamplePValueTO(to))); - break; - default: - throw log.throwing(new IllegalStateException("Unsupported DataType: " + dt)); - } - } - - return log.traceExit(map); - } - - //************************************************************************* - // METHODS PROPAGATION: from CallTOs to propagated Calls - //************************************************************************* - - /** - * Propagate {@code ExpressionCall}s to descendant and ancestor conditions - * from {@code conditionGraph}. - *

    - * Returned {@code ExpressionCall}s have {@code DataPropagation}, {@code ExpressionSummary}, - * and {@code DataQuality} equal to {@code null}. - * - * @param calls A {@code Collection} of {@code ExpressionCall}s to be propagated. - * @param conditionGraph A {@code ConditionGraph} containing at least anat. entity - * {@code Ontology} to use for the propagation. - * @return A {@code Map} where keys are {@code PipelineCall}, the associated - * values are {@code Set}s of {@code PipelineCallData}. - * @throws IllegalArgumentException If {@code calls} or {@code conditionGraph} are {@code null}, - * empty. - */ - private Map>> propagatePipelineCalls( - Map>> data, ConditionGraph conditionGraph) - throws IllegalArgumentException { - log.traceEntry("{}, {}", data, conditionGraph); - log.trace(COMPUTE_MARKER, "Starting to propagate {} PipelineCalls.", data.size()); - - Map>> propagatedData = new HashMap<>(); - this.checkErrorOccurred(); - - assert data != null && !data.isEmpty(); - assert conditionGraph != null; - - Set calls = data.keySet(); - - // Here, no calls should have PropagationState which is not SELF - assert calls.stream().allMatch(c -> - c.getDataPropagation().getCondParamCombinations().stream() - .allMatch(comb -> PropagationState.SELF.equals(c.getDataPropagation().getPropagationState(comb)))); - // Check conditionGraph contains all conditions of calls - assert conditionGraph.getConditions().containsAll( - calls.stream().map(c -> c.getCondition()).collect(Collectors.toSet())); - - //***************************** - // PROPAGATE CALLS - //***************************** - - // Counts for log tracing - int callCount = calls.size(); - int analyzedCallCount = 0; - - for (Entry>> entry: data.entrySet()) { - this.checkErrorOccurred(); - if (log.isTraceEnabled() && analyzedCallCount % 100 == 0) { - log.trace("{}/{} expression calls analyzed.", analyzedCallCount, callCount); - } - analyzedCallCount++; - - ExpressionCall curCall = entry.getKey(); - log.trace(COMPUTE_MARKER, "Propagation for call: {}", curCall); - - // Retrieve conditions - log.trace(COMPUTE_MARKER, "Starting to retrieve ancestral conditions for {}.", - curCall.getCondition()); - Set ancestorConditions = this.condToAncestors.computeIfAbsent( - curCall.getCondition(), - k -> conditionGraph.getAncestorConditions(k)); - log.trace(COMPUTE_MARKER, "Done retrieving ancestral conditions for {}: {}.", - curCall.getCondition(), ancestorConditions.size()); - log.trace("Ancestor conditions: {}", ancestorConditions); - if (!ancestorConditions.isEmpty()) { - Map>> ancestorCalls = - propagatePipelineData(entry, ancestorConditions, true); - assert !ancestorCalls.isEmpty(); - propagatedData.putAll(ancestorCalls); - } - - //Note: actually as of Bgee 14.2 we do not propagate absent calls to substructures anymore - Set descendantConditions = new HashSet<>(); -// log.trace(COMPUTE_MARKER, "Starting to retrieve descendant conditions for {}.", -// curCall.getCondition()); -// Set descendantConditions = this.condToDescendants.computeIfAbsent( -// curCall.getCondition(), -// k -> conditionGraph.getDescendantConditions( -// k, false, false, NB_SUBLEVELS_MAX, null)); -// log.trace(COMPUTE_MARKER, "Done retrieving descendant conditions for {}: {}.", -// curCall.getCondition(), descendantConditions.size()); -// log.trace("Descendant conditions: {}", descendantConditions); - if (!descendantConditions.isEmpty()) { - Map>> descendantCalls = - propagatePipelineData(entry, descendantConditions, false); - assert !descendantCalls.isEmpty(); - propagatedData.putAll(descendantCalls); - } - - log.trace(COMPUTE_MARKER, "Done propagation for call: {}", curCall); - } - - log.trace(COMPUTE_MARKER, "Done propagating {} PipelineCalls, {} new propagated PipelineCalls.", - data.size(), propagatedData.size()); - return log.traceExit(propagatedData); - } - - /** - * Propagate calls to provided {@code propagatedConds}. - * - * @param data An {@code Entry} where keys are {@code PipelineCall}, - * the associated value being a {@code Set} of {@code PipelineCallData} - * that is the call to be propagated. - * @param propagatedConds A {@code Collection} of {@code Condition}s that are the conditions - * for which the propagation have to be done. - * @param areAncestors A {@code boolean} defining whether the {@code propagatedConds} - * are ancestors or descendants. If {@code true}, it is ancestors. - * @return A {@code Set} of {@code ExpressionCall}s that are propagated calls - * from provided {@code data}, without including calls in {@code data}. - */ - private Map>> propagatePipelineData( - Entry>> data, Set propagatedConds, - boolean areAncestors) { - log.traceEntry("{}, {}, {}", data, propagatedConds, areAncestors); - log.trace(COMPUTE_MARKER, "Start to propagate PipelineData, to ancestor? {}.", areAncestors); - - Map>> map = new HashMap<>(); - this.checkErrorOccurred(); - - if (propagatedConds.isEmpty()) { - throw log.throwing(new IllegalArgumentException("No provided propagated conditions")); - } - PipelineCall call = data.getKey(); - Condition callCondition = call.getCondition(); - - //For each propagated condition (not including the original condition), - //create a new PipelineCall, with source CallTOs stored in the appropriate attribute, - //and with associated PipelineCallData updated. - for (Condition condition : propagatedConds) { - log.trace("Propagation of the current call to condition: {}", condition); - assert !callCondition.equals(condition); - - Set> relativeData = new HashSet<>(); - - //for each original PipelineCallData, create a new PipelineCallData with DataPropagation updated - //and ExperimentExpressionTOs stored in the appropriate attributes - for (PipelineCallData pipelineData: data.getValue()) { - this.checkErrorOccurred(); - - // Here, we define propagation states. - // A state should stay to null if we do not have this information in call condition. - PropagationState anatEntityPropagationState = null; - PropagationState devStagePropagationState = null; - PropagationState cellTypePropagationState = null; - PropagationState sexPropagationState = null; - PropagationState strainPropagationState = null; - if (areAncestors) { - if (callCondition.getAnatEntityId() != null) - anatEntityPropagationState = PropagationState.DESCENDANT; - if (callCondition.getDevStageId() != null) - devStagePropagationState = PropagationState.DESCENDANT; - if (callCondition.getCellTypeId() != null) - cellTypePropagationState = PropagationState.DESCENDANT; - if (callCondition.getSexId() != null) - sexPropagationState = PropagationState.DESCENDANT; - if (callCondition.getStrainId() != null) - strainPropagationState = PropagationState.DESCENDANT; - } else { - if (callCondition.getAnatEntityId() != null) { - anatEntityPropagationState = PropagationState.ANCESTOR; - } - //no propagation to substages etc, only to substructures, but it does not hurt - //and the value is changed just below - if (callCondition.getDevStageId() != null) { - devStagePropagationState = PropagationState.ANCESTOR; - } - if (callCondition.getCellTypeId() != null) { - cellTypePropagationState = PropagationState.ANCESTOR; - } - if (callCondition.getSexId() != null) { - sexPropagationState = PropagationState.ANCESTOR; - } - if (callCondition.getStrainId() != null) { - strainPropagationState = PropagationState.ANCESTOR; - } - } - - if (callCondition.getAnatEntityId() != null && - callCondition.getAnatEntityId().equals(condition.getAnatEntityId())) { - anatEntityPropagationState = PropagationState.SELF; - } - if (callCondition.getDevStageId() != null && - callCondition.getDevStageId().equals(condition.getDevStageId())) { - devStagePropagationState = PropagationState.SELF; - } - if (callCondition.getCellTypeId() != null && - callCondition.getCellTypeId().equals(condition.getCellTypeId())) { - cellTypePropagationState = PropagationState.SELF; - } - if (callCondition.getSexId() != null && - callCondition.getSexId().equals(condition.getSexId())) { - sexPropagationState = PropagationState.SELF; - } - if (callCondition.getStrainId() != null && - callCondition.getStrainId().equals(condition.getStrainId())) { - strainPropagationState = PropagationState.SELF; - } - assert anatEntityPropagationState != PropagationState.SELF || - devStagePropagationState != PropagationState.SELF || - cellTypePropagationState != PropagationState.SELF || - sexPropagationState != PropagationState.SELF || - strainPropagationState != PropagationState.SELF; - - //We want to count the number of self p-values for all combinations of condition parameters - EnumSet selfPropStateParams = EnumSet.noneOf( - CallService.Attribute.class); - for (CallService.Attribute condParam: CallService.Attribute.getAllConditionParameters()) { - switch(condParam) { - case ANAT_ENTITY_ID: - if (anatEntityPropagationState.equals(PropagationState.SELF)) { - selfPropStateParams.add(condParam); - } - break; - case CELL_TYPE_ID: - if (cellTypePropagationState.equals(PropagationState.SELF)) { - selfPropStateParams.add(condParam); - } - break; - case DEV_STAGE_ID: - if (devStagePropagationState.equals(PropagationState.SELF)) { - selfPropStateParams.add(condParam); - } - break; - case SEX_ID: - if (sexPropagationState.equals(PropagationState.SELF)) { - selfPropStateParams.add(condParam); - } - break; - case STRAIN_ID: - if (strainPropagationState.equals(PropagationState.SELF)) { - selfPropStateParams.add(condParam); - } - break; - default: - throw log.throwing(new IllegalStateException("Unsupported condition parameter: " - + condParam)); - } - } - Set> selfPropStateParamCombinations = - selfPropStateParams.isEmpty()? new HashSet<>(): - CallService.Attribute.getAllPossibleCondParamCombinations(selfPropStateParams); - assert !selfPropStateParamCombinations.contains( - CallService.Attribute.getAllConditionParameters()); - Set> notSelfPropStateParamCombinations = - CallService.Attribute.getAllPossibleCondParamCombinations(); - notSelfPropStateParamCombinations.removeAll(selfPropStateParamCombinations); - assert notSelfPropStateParamCombinations.contains( - CallService.Attribute.getAllConditionParameters()); - - switch(pipelineData.getDataType()) { - case EST: - case IN_SITU: - case RNA_SEQ: - case SC_RNA_SEQ: - //We know the generic types depending on the data types - @SuppressWarnings("unchecked") - Set> localPValues = pipelineData - .getSelfPValuesPerCondParamCombinations().get( - CallService.Attribute.getAllConditionParameters()) - .stream() - .map(pval -> (SamplePValueTO) pval) - .collect(Collectors.toSet()); - assert !localPValues.isEmpty(); - - Map, Set>> - selfPValuesPerCondParamCombinations = selfPropStateParamCombinations.stream() - .collect(Collectors.toMap(comb -> comb, comb -> localPValues)); - selfPValuesPerCondParamCombinations.putAll(notSelfPropStateParamCombinations.stream() - .collect(Collectors.toMap(comb -> comb, comb -> new HashSet<>()))); - Set> parentPValues = null; - Set> descendantPValues = null; - if (areAncestors) { - descendantPValues = localPValues; - } else { - parentPValues = localPValues; - } - relativeData.add(new PipelineCallData<>(pipelineData.getDataType(), - parentPValues, selfPValuesPerCondParamCombinations, descendantPValues)); - break; - case AFFYMETRIX: - //We know the generic types depending on the data types - @SuppressWarnings("unchecked") - Set> localPValues2 = pipelineData - .getSelfPValuesPerCondParamCombinations().get( - CallService.Attribute.getAllConditionParameters()) - .stream() - .map(pval -> (SamplePValueTO) pval) - .collect(Collectors.toSet()); - assert !localPValues2.isEmpty(); - - Map, Set>> - selfPValuesPerCondParamCombinations2 = selfPropStateParamCombinations.stream() - .collect(Collectors.toMap(comb -> comb, comb -> localPValues2)); - selfPValuesPerCondParamCombinations2.putAll(notSelfPropStateParamCombinations.stream() - .collect(Collectors.toMap(comb -> comb, comb -> new HashSet<>()))); - Set> parentPValues2 = null; - Set> descendantPValues2 = null; - if (areAncestors) { - descendantPValues2 = localPValues2; - } else { - parentPValues2 = localPValues2; - } - relativeData.add(new PipelineCallData<>(pipelineData.getDataType(), - parentPValues2, selfPValuesPerCondParamCombinations2, descendantPValues2)); - break; - } - } - - // Add propagated expression call. - Set ancestorCallTOs = null; - Set descendantCallTOs = null; - if (areAncestors) { - descendantCallTOs = call.getSelfSourceCallTOs(); - } else { - ancestorCallTOs = call.getSelfSourceCallTOs(); - } - - PipelineCall propagatedCall = new PipelineCall( - call.getBgeeGeneId(), - condition, - null, // Collection callData (update after the propagation), - null, null, //corrected p-values - ancestorCallTOs, null, descendantCallTOs); - - log.trace("Add the propagated call: {}", propagatedCall); - map.put(propagatedCall, relativeData); - } - if (map.isEmpty()) { - throw log.throwing(new IllegalStateException("No propagated calls")); - } - - log.trace(COMPUTE_MARKER, "Done propagating PipelineData, to ancestor? {}.", areAncestors); - return log.traceExit(map); - } - - /** - * Reconcile several pipeline calls into one pipeline call. - *

    - * Return the representative {@code PipelineCall} (with reconciled quality per data types, - * observed data state, conflict status etc. - * - * @param calls A {@code Set} of {@code PipelineCall}s that are the calls to be reconciled. - * @param pipelineData A {@code Set} of {@code PipelineCallData} that are the pipeline call data - * to be used for reconciliation. - * @return The representative {@code ExpressionCall}. - */ - //We return PipelineCall rather than ExpressionCall to be able to keep bgeeGeneId - private PipelineCall reconcileGeneCalls(Set calls, - Set> pipelineData) { - log.traceEntry("{}, {}", calls, pipelineData); - - this.checkErrorOccurred(); - - assert calls != null && !calls.isEmpty(); - assert pipelineData != null && !pipelineData.isEmpty(); - - Set geneIds = calls.stream().map(c -> c.getBgeeGeneId()).collect(Collectors.toSet()); - if (geneIds.size() != 1 || geneIds.contains(null)) { - throw log.throwing(new IllegalArgumentException( - "None or several genes are found in provided PipelineCalls")); - } - int geneId = geneIds.iterator().next(); - - Set conditions = calls.stream().map(c -> c.getCondition()).collect(Collectors.toSet()); - if (conditions.size() != 1 || conditions.contains(null)) { - throw log.throwing(new IllegalArgumentException( - "None or several conditions are found in provided PipelineCalls")); - } - Condition condition = conditions.iterator().next(); - - Map>> pipelineDataByDataTypes = pipelineData.stream() - .collect(Collectors.groupingBy(PipelineCallData::getDataType, Collectors.toSet())); - - Set expressionCallData = new HashSet<>(); - for (Entry>> entry: pipelineDataByDataTypes.entrySet()) { - ExpressionCallData cd = mergePipelineCallDataIntoExpressionCallData( - entry.getKey(), entry.getValue()); - //the returned callData is null if there was no valid data to propagate - //(e.g., only "present" expression calls in parent conditions) - if (cd != null) { - expressionCallData.add(cd); - } - } - if (expressionCallData.isEmpty()) { - log.trace("No valid data to propagate"); - return log.traceExit((PipelineCall) null); - } - - //************************ - // Data propagation - //************************ - -// assert expressionCallData.stream() -// .flatMap(ecd -> ecd.getExperimentCounts(PropagationState.ALL).stream()) -// .mapToInt(c -> c.getCount()).sum() != 0; -// assert Boolean.TRUE.equals(dataProp.isIncludingObservedData()) && expressionCallData.stream() -// .flatMap(ecd -> ecd.getExperimentCounts(PropagationState.SELF).stream()) -// .mapToInt(c -> c.getCount()).sum() > 0 || -// Boolean.FALSE.equals(dataProp.isIncludingObservedData()) && expressionCallData.stream() -// .flatMap(ecd -> ecd.getExperimentCounts(PropagationState.SELF).stream()) -// .mapToInt(c -> c.getCount()).sum() == 0 && -// expressionCallData.stream().mapToInt(ecd -> ecd.getPropagatedExperimentCount()).sum() > 0; - - - Set selfSourceCallTOs = calls.stream() - .map(PipelineCall::getSelfSourceCallTOs) - .filter(s -> s != null) - .flatMap(s -> s.stream()) - .collect(Collectors.toSet()); - - //************************ - // FDR-corrected p-values - //************************ - //We compute the corrected p-values for all combination of data types with data for this call. - //First, we retrieve all possible combinations of the data types used. - Set> usedDataTypeCombs = DataType.getAllPossibleDataTypeCombinations( - expressionCallData.stream().map(ecd -> ecd.getDataType()).collect(Collectors.toSet())); - //And now we correct the p-values for all possible combination of the data types used - Set correctedPValues = usedDataTypeCombs.stream() - .map(dtComb -> { - //Use List to not loose equal pvalues - List pValues = expressionCallData.stream() - .filter(ecd -> dtComb.contains(ecd.getDataType())) - .flatMap(ecd -> ecd.getAllPValues().stream()) - .collect(Collectors.toList()); - return new FDRPValue(computeFDRCorrectedPValue(pValues), dtComb); - }) - .collect(Collectors.toSet()); - //now we need to complement the combinations of data types by copying the computed p-values: - //for instance, if there was only RNA-Seq and Affymetrix data for this call, - //then the combination EST-RNA-Seq-Affymetrix will have the exact same corrected p-value as - //RNA-Seq-Affymetrix. We need to get all combinations with any data available. - //So this otherDataTypeCombs Set will contains all combination with at least one DataType - //with no associated data for this call. - Set> otherDataTypeCombs = DataType.getAllPossibleDataTypeCombinations().stream() - .filter(s -> !usedDataTypeCombs.contains(s)) - .collect(Collectors.toSet()); - //For each of these combinations, we'll try to find the combination with a computed p-value - //with the most overlap in data types and with all its data types contained. - Map, FDRPValue> pValuePerDataTypeComb = correctedPValues.stream() - .map(p -> new AbstractMap.SimpleEntry<>(p.getDataTypes(), p)) - .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue())); - Set otherCorrectedPValues = otherDataTypeCombs.stream() - .map(otherDTComb -> { - EnumSet mostMatchedDataTypesCombination = - DataType.findCombinationWithGreatestOverlap( - pValuePerDataTypeComb.keySet(), otherDTComb); - if (mostMatchedDataTypesCombination == null) { - return null; - } - return new FDRPValue(pValuePerDataTypeComb.get(mostMatchedDataTypesCombination) - .getPValue(), otherDTComb); - }) - //If we couldn't find any overlap with a computed combination, the previous mapping - //returned null, we filter that out. - .filter(p -> p != null) - .collect(Collectors.toSet()); - - assert Collections.disjoint(correctedPValues, otherCorrectedPValues): - "Corrected pvalues: " + correctedPValues + " - others: " + otherCorrectedPValues; - Set allCorrectedPValues = new HashSet<>(correctedPValues); - allCorrectedPValues.addAll(otherCorrectedPValues); - - // It is not necessary to infer ExpressionSummary, SummaryQuality using - // CallService.inferSummaryXXX(), because they will not be inserted in the db, - // we solely store p-values - return log.traceExit(new PipelineCall(geneId, condition, expressionCallData, - allCorrectedPValues, null, - calls.stream().map(PipelineCall::getParentSourceCallTOs) - .filter(s -> s != null) - .flatMap(Set::stream).collect(Collectors.toSet()), - selfSourceCallTOs, - calls.stream().map(PipelineCall::getDescendantSourceCallTOs) - .filter(s -> s != null) - .flatMap(Set::stream).collect(Collectors.toSet()))); - } - // code taken from https://github.com/cBioPortal/cbioportal/blob/master/core/src/main/java/ - // org/mskcc/cbio/portal/stats/BenjaminiHochbergFDR.java - private static BigDecimal computeFDRCorrectedPValue(List pValues) { - log.traceEntry("{}", pValues); - - int m = pValues.size(); - Double[] pValuesDouble = - pValues.stream() - .map(p -> p.compareTo(ZERO_BIGDECIMAL) == 0 ? ABOVE_ZERO_BIGDECIMAL : p) - .map(p -> p.doubleValue()) - .toArray(length -> new Double[length]); - double[] adjustedPValues = new double[m]; - - Arrays.sort(pValuesDouble); - // iterate through all p-values: largest to smallest - for (int i = m - 1; i >= 0; i--) { - if (i == m - 1) { - adjustedPValues[i] = pValuesDouble[i]; - } else { - double unadjustedPvalue = pValuesDouble[i]; - int divideByM = i + 1; - double left = adjustedPValues[i + 1]; - double right = (m / (double) divideByM) * unadjustedPvalue; - adjustedPValues[i] = Math.min(left, right); - } - } - //Find the smallest corrected p-value - BigDecimal fdr = BigDecimal.valueOf(Arrays.stream(adjustedPValues).min().getAsDouble()); - //If the FDR is less than MIN_FDR_BIGDECIMAL, change it to MIN_FDR_BIGDECIMAL - //(in order to avoid having fields in the globalExpression table with too much precision) - if (fdr.compareTo(MIN_FDR_BIGDECIMAL) < 0) { - fdr = MIN_FDR_BIGDECIMAL; - } - return log.traceExit(fdr); - } - - /** - * Merge a {@code Set} of {@code PipelineCallData} into one {@code ExpressionCallData}. - * - * @param dataType A {@code DataType} that is the data type of {@code pipelineCallData}. - * @param pipelineCallData A {@code Set} of {@code PipelineCallData} to be used to - * build the {@code ExpressionCallData}. - * on propagated data. - */ - private ExpressionCallData mergePipelineCallDataIntoExpressionCallData(DataType dataType, - Set> pipelineCallData) { - log.traceEntry("{}, {}", dataType, pipelineCallData); - - this.checkErrorOccurred(); - - assert pipelineCallData.stream().noneMatch(pcd -> !dataType.equals(pcd.getDataType())); - //at this point, we have only propagated one call at a time, so we should have - //p-values in only one of these 3 attributes - assert pipelineCallData.stream().allMatch(pcd -> - (/*(pcd.getSelfPValues() != null && !pcd.getSelfPValues().isEmpty()) &&*/ - (pcd.getParentPValues() == null || pcd.getParentPValues().isEmpty()) && - (pcd.getDescendantPValues() == null || pcd.getDescendantPValues().isEmpty()) || - - /*(pcd.getSelfPValues() == null || pcd.getSelfPValues().isEmpty()) &&*/ - (pcd.getParentPValues() != null && !pcd.getParentPValues().isEmpty()) && - (pcd.getDescendantPValues() == null || pcd.getDescendantPValues().isEmpty()) || - - /*(pcd.getSelfPValues() == null || pcd.getSelfPValues().isEmpty()) &&*/ - (pcd.getParentPValues() == null || pcd.getParentPValues().isEmpty()) && - (pcd.getDescendantPValues() != null && !pcd.getDescendantPValues().isEmpty()))); - - - - //Rank info: computed by the Perl pipeline after insertion of these global calls -// BigDecimal rank = null; -// BigDecimal rankNorm = null; -// BigDecimal rankSum = null; -// if (selfSourceCallTO != null) { -// switch(dataType) { -// case AFFYMETRIX: -// rank = selfSourceCallTO.getAffymetrixMeanRank(); -// rankNorm = selfSourceCallTO.getAffymetrixMeanRankNorm(); -// rankSum = selfSourceCallTO.getAffymetrixDistinctRankSum(); -// break; -// case RNA_SEQ: -// rank = selfSourceCallTO.getRNASeqMeanRank(); -// rankNorm = selfSourceCallTO.getRNASeqMeanRankNorm(); -// rankSum = selfSourceCallTO.getRNASeqDistinctRankSum(); -// break; -// case EST: -// rank = selfSourceCallTO.getESTRank(); -// rankNorm = selfSourceCallTO.getESTRankNorm(); -// break; -// case IN_SITU: -// rank = selfSourceCallTO.getInSituRank(); -// rankNorm = selfSourceCallTO.getInSituRankNorm(); -// break; -// default: -// log.throwing(new IllegalStateException("Unsupported data type: " + dataType)); -// } +// public InsertPropagatedCalls(Supplier serviceFactorySupplier, +// Set condParams, int speciesId, int geneOffset, int geneRowCount, +// boolean computeAndInsertGlobalCond) { +// this(serviceFactorySupplier, condParams, speciesId, geneOffset, geneRowCount, +// computeAndInsertGlobalCond, new CallServiceUtils()); +// } +// public InsertPropagatedCalls(Supplier serviceFactorySupplier, +// Set condParams, int speciesId, int geneOffset, int geneRowCount, +// boolean computeAndInsertGlobalCond, CallServiceUtils utils) { +// super(serviceFactorySupplier.get(), utils); +// if (condParams == null || condParams.isEmpty()) { +// throw log.throwing(new IllegalArgumentException("Condition attributes should not be empty")); // } - - - EnumSet allCondParams = CallService.Attribute.getAllConditionParameters(); - //We map to PipelineSamplePValueTO because it implements hashCode/equals, - //taking into account the experiment and sample IDs, so that we can be sure - //we don't count a p-value coming from a same observation several times. - Map, Set>> selfPValuesPerCondParamComb = - pipelineCallData.stream() - .flatMap(pcd -> pcd.getSelfPValuesPerCondParamCombinations().entrySet().stream()) - .collect(Collectors.toMap( - e -> e.getKey(), - e -> e.getValue().stream().map(p -> new PipelineSamplePValueTO<>(p)) - .collect(Collectors.toSet()), - (v1, v2) -> {v1.addAll(v2); return v1;})); - Set> descendantPValues = pipelineCallData.stream() - .flatMap(pcd -> pcd.getDescendantPValues().stream() - .map(p -> new PipelineSamplePValueTO<>(p))) - .collect(Collectors.toSet()); - Set> selfPValues = selfPValuesPerCondParamComb.get(allCondParams); - - assert Stream.concat(selfPValues.stream(), descendantPValues.stream()) - .collect(Collectors.toSet()).containsAll(selfPValuesPerCondParamComb.values().stream() - .flatMap(s -> s.stream()).collect(Collectors.toSet())); - if (!Collections.disjoint(selfPValues, descendantPValues)) { - selfPValues.retainAll(descendantPValues); - throw log.throwing(new IllegalStateException( - "self and desendant p-values should always be disjoined, p-values in common: " - + selfPValues)); - } - - Map, Integer> selfObservationCounts = - selfPValuesPerCondParamComb.entrySet().stream() - .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue().size())); - assert selfObservationCounts.keySet().contains(CallService.Attribute.getAllConditionParameters()); - //We collect to a List to not eliminate equals pvalues - List mappedDescendantPValues = descendantPValues.stream().map(p -> p.getpValue()) - .collect(Collectors.toList()); - //For descendant observation counts, we store in database only the count - //for all condition parameters. But for creating the DataPropagation object - //we need to have the same keyset in both Maps. - Map, Integer> descendantObservationCounts = - selfObservationCounts.keySet().stream().collect(Collectors.toMap( - k -> k, - k -> k.equals(CallService.Attribute.getAllConditionParameters())? - mappedDescendantPValues.size(): 0)); - DataPropagation dataProp = new DataPropagation(selfObservationCounts, - descendantObservationCounts); - - log.trace(COMPUTE_MARKER, "ExpressionCallData to be created: {} - {} - {} - {}", - dataType, selfPValuesPerCondParamComb, descendantPValues, dataProp); - return log.traceExit(new ExpressionCallData(dataType, - //We collect to a List to not eliminate equals pvalues - selfPValues.stream().map(p -> p.getpValue()).collect(Collectors.toList()), - mappedDescendantPValues, - null, null, null, - dataProp)); - } - - //************************************************************************* - // METHODS MAPPING dao-api objects to bgee-core objects - //************************************************************************* - - private static Set> mapExpExprTOsToPipelineCallData( - Map>> pValuesByDataTypes, - Set condParams) { - log.traceEntry("{}, {}, {}", pValuesByDataTypes, condParams); - EnumSet dataTypes = EnumSet.noneOf(DataType.class); - dataTypes.addAll(pValuesByDataTypes.keySet()); - return log.traceExit(dataTypes.stream() - .map(dt -> { - return new PipelineCallData<>( - dt, null, - CallService.Attribute.getAllPossibleCondParamCombinations().stream() - .collect(Collectors.toMap(comb -> comb , - comb -> pValuesByDataTypes.get(dt).stream() - .map(pval -> pval) - .collect(Collectors.toSet()))), - null); -// switch(dt) { -// case EST: -// case IN_SITU: -// case RNA_SEQ: -// case SC_RNA_SEQ: -// //The cast of the generic types of SamplePValueTOs can be determined by the data type -// @SuppressWarnings("unchecked") -// PipelineCallData pipelineCallData = new PipelineCallData<>( -// dt, getSelfDataProp(condParams), -// null, expExprsByDataTypes == null? null: expExprsByDataTypes.get(dt), null, -// null, -// pValuesByDataTypes.get(dt).stream() -// .map(pval -> (SamplePValueTO) pval) -// .collect(Collectors.toSet()), -// null); -// return pipelineCallData; -// case AFFYMETRIX: -// //The cast of the generic types of SamplePValueTOs can be determined by the data type -// @SuppressWarnings("unchecked") -// PipelineCallData pipelineCallData2 = new PipelineCallData<>( -// dt, getSelfDataProp(condParams), -// null, expExprsByDataTypes == null? null: expExprsByDataTypes.get(dt), null, -// null, -// pValuesByDataTypes.get(dt).stream() -// .map(pval -> (SamplePValueTO) pval) +// if (geneOffset < 0 || geneRowCount < 0) { +// throw log.throwing(new IllegalArgumentException( +// "geneOffset and geneRowCount cannot be negative")); +// } +// if (geneOffset > 0 && geneRowCount == 0) { +// throw log.throwing(new IllegalArgumentException( +// "geneRowCount must be provided if geneOffset is provided")); +// } +// this.serviceFactorySupplier = serviceFactorySupplier; +// this.condParams = EnumSet.copyOf(condParams); +// this.speciesId = speciesId; +// this.geneOffset = geneOffset; +// this.geneRowCount = geneRowCount; +// this.computeAndInsertGlobalCond = computeAndInsertGlobalCond; +// //use a LinkedBlockingDeque because we are going to do lots of insert/remove, +// //and because we don't care about element order. We are going to block +// //if there are too many results waiting to be inserted, to not overload the memory +// this.callsToInsert = new LinkedBlockingDeque<>(MAX_NUMBER_OF_CALLS_TO_INSERT); +// this.insertFinished = new AtomicBoolean(false); +// this.daoManagers = Collections.newSetFromMap(new ConcurrentHashMap<>()); +// this.errorOccured = null; +// this.jobCompleted = false; +// +// this.condToAncestors = new ConcurrentHashMap<>(); +//// this.condToDescendants = new ConcurrentHashMap<>(); +// } +// +// private void insertGlobalConditionsForOneSpecies() throws Exception { +// log.traceEntry(); +// log.info("Start inserting global conditions for the species {} with combinations of condition parameters {}...", +// this.speciesId, this.condParams); +// +// try (DAOManager mainManager = this.getDaoManager()) { +// ConditionDAO condDAO = mainManager.getConditionDAO(); +// +// Species species = this.getServiceFactory().getSpeciesService().loadSpeciesByIds( +// Collections.singleton(this.speciesId), false).iterator().next(); +// +// //First, we retrieve the raw conditions already present in database. +// final Map rawCondMap = Collections.unmodifiableMap( +// this.loadRawConditionMap(Collections.singleton(species))); +// log.info("{} raw data conditions for species {}", rawCondMap.size(), speciesId); +// +// // We use all existing conditions in the species, and infer all propagated conditions +// log.info("Starting condition inference for species {}...", this.speciesId); +// Map> globalCondToSelfRawCondIds = rawCondMap.entrySet() +// .stream() +// .map(e -> new AbstractMap.SimpleEntry<>( +// mapRawDataConditionToCondition(e.getValue()), +// new HashSet<>(Arrays.asList(e.getKey())))) +// .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue(), +// (v1, v2) -> {v1.addAll(v2); return v1;})); +// assert globalCondToSelfRawCondIds.values().stream().flatMap(s -> s.stream()) +// .collect(Collectors.toSet()).equals(rawCondMap.keySet()); +// +// final ConditionGraph conditionGraph = loadConditionGraph( +// this.getServiceFactory().getConditionGraphService(), +// globalCondToSelfRawCondIds.keySet(), +// true); +// log.info("Done condition inference for species {}.", this.speciesId); +// +// startTransaction((MySQLDAOManager) mainManager); +// +// Map globalCondsInserted = InsertPropagatedCalls +// .insertNewGlobalConditions(conditionGraph.getConditions(), +// new HashSet<>(), condDAO); +// log.info("{} conditions inserted for species {}", globalCondsInserted.size(), this.speciesId); +// assert conditionGraph.getConditions().equals(globalCondsInserted.keySet()); +// +// Set toInsert = new HashSet<>(); +// //SELF ConditionRelationOrigin +// toInsert.addAll(globalCondsInserted.entrySet().stream() +// .flatMap(e -> globalCondToSelfRawCondIds.getOrDefault(e.getKey(), new HashSet<>()) +// .stream() +// .map(rawCondId -> new PipelineGlobalCondToRawCondTO(rawCondId, e.getValue(), +// GlobalConditionToRawConditionTO.ConditionRelationOrigin.SELF))) +// .collect(Collectors.toSet())); +// //DESCENDANT ConditionRelationOrigin +// toInsert.addAll(globalCondsInserted.entrySet().stream() +// //get the ancestors of the iterated condition +// .flatMap(e -> conditionGraph.getAncestorConditions(e.getKey()) +// .stream() +// //retrieve the raw condition IDs associated to the iterated condition +// .flatMap(ancestor -> globalCondToSelfRawCondIds +// .getOrDefault(e.getKey(), new HashSet<>()) +// .stream() +// //Create an association from the ancestor condition ID +// //to the raw condition IDs of the iterated condition +// //with ConditionOrigin DESCENDANT +// .map(rawCondId -> new PipelineGlobalCondToRawCondTO(rawCondId, +// globalCondsInserted.get(ancestor), +// GlobalConditionToRawConditionTO.ConditionRelationOrigin.DESCENDANT)))) +// .collect(Collectors.toSet())); +// Set inserted = InsertPropagatedCalls +// .insertGlobalCondToRawConds(toInsert, new HashSet<>(), condDAO); +// +// assert toInsert.stream().map(to -> to.getGlobalConditionId()).collect(Collectors.toSet()) +// .equals(new HashSet<>(globalCondsInserted.values())); +// assert toInsert.stream().map(to -> to.getRawConditionId()).collect(Collectors.toSet()) +// .equals(globalCondToSelfRawCondIds.values().stream() +// .flatMap(s -> s.stream()).collect(Collectors.toSet())); +// assert inserted.equals(toInsert); +// +// ((MySQLDAOManager) mainManager).getConnection().getRealConnection().commit(); +// ((MySQLDAOManager) mainManager).getConnection().getRealConnection().setAutoCommit(true); +// +// log.info("{} GlobalCondToRawCondTOs inserted for species {}", toInsert.size(), this.speciesId); +// } +// log.traceExit(); +// } +// +// private void insertOneSpecies() { +// log.traceEntry(); +// +// log.info("Start inserting of propagated calls for the species {} with combinations of condition parameters {}...", +// this.speciesId, this.condParams); +// +// Thread insertThread = null; +// // close connection to database between each species, to avoid idle +// // connection reset or for parallel execution +// try (DAOManager mainManager = this.getDaoManager()) { +// +// Species species = this.getServiceFactory().getSpeciesService().loadSpeciesByIds( +// Collections.singleton(this.speciesId), false).iterator().next(); +// +// //First, we retrieve the raw conditions already present in database. +// final Map rawCondMap = Collections.unmodifiableMap( +// this.loadRawConditionMap(Collections.singleton(species))); +// log.info("{} Conditions for species {}", rawCondMap.size(), speciesId); +// //Retrieve the global conditions and mappings to raw conditions already inserted +// final Map globalCondAlreadyInserted = loadGlobalConditionMap( +// Collections.singleton(species), +// generateDAOConditionFilters(null, this.condParams), +// null, +// mainManager.getConditionDAO(), +// this.getServiceFactory().getAnatEntityService(), +// this.getServiceFactory().getDevStageService(), +// this.getServiceFactory().getSexService(), +// this.getServiceFactory().getStrainService()) +// .entrySet().stream() +// .collect(Collectors.toMap(e -> e.getValue(), e -> e.getKey())); +// final Set globalCondToCondAlreadyInserted = +// mainManager.getConditionDAO().getGlobalCondToRawCondBySpeciesIds( +// Collections.singleton(this.speciesId), this.condParams) +// .stream().map(gctc -> new PipelineGlobalCondToRawCondTO(gctc)) +// .collect(Collectors.toSet()); +// +// // We use all existing conditions in the species, and infer all propagated conditions +// log.info("Starting condition inference..."); +// ConditionGraphService condGraphService = this.getServiceFactory().getConditionGraphService(); +// final ConditionGraph conditionGraph = this.computeAndInsertGlobalCond ? +// loadConditionGraph(condGraphService, +// rawCondMap.values() +// .stream().map(rawCond -> mapRawDataConditionToCondition(rawCond)) // .collect(Collectors.toSet()), -// null); -// return pipelineCallData2; -// default: -// throw log.throwing(new IllegalStateException("Unsupported data type: " + dt)); +// true) : +// //In case the global conditions were pre-computed +// loadConditionGraph(condGraphService, globalCondAlreadyInserted.keySet(), +// false); +// log.info("Done condition inference."); +// +// //we retrieve the IDs of genes with expression data. This is because making the computation +// //a whole species at a time can use too much memory for species with large amount of data. +// //Also, the computations for those species are slow so we want to go parallel. +// final List bgeeGeneIds = Collections.unmodifiableList( +// mainManager.getGeneDAO() +// .getGenesWithDataBySpeciesIdsOrdered(Collections.singleton(speciesId), +// this.geneOffset, this.geneRowCount) +// .stream().map(g -> g.getId()) +// .collect(Collectors.toList())); +// log.info("{} genes with data retrieved for species {}", bgeeGeneIds.size(), speciesId); +// +// //Remaining computations/insertions will be made in separate threads +// //with a separate database connection, so we close the main connection immediately, +// //but do not close the manager because of the try-with-resource clause. +// mainManager.releaseResources(); +// +// //PARALLEL EXECUTION: here we create the independent thread responsible for +// //inserting the data into the data source +// insertThread = new Thread(new InsertJob(this, globalCondAlreadyInserted, +// globalCondToCondAlreadyInserted)); +// //just to be accessible from the Stream to notify of exceptions +// final Thread localInsertThread = insertThread; +// //PARALLEL EXECUTION: start the insertion Thread +// insertThread.start(); +// +// //PARALLEL EXECUTION: we generate groups of genes of size GENES_PER_ITERATION +// //and run the computations in parallel between groups +// //(important to convert to float here before dividing, otherwise the rounding could be incorrect) +// int iterationCount = (int) Math.ceil((float) bgeeGeneIds.size()/(float) GENE_PARALLEL_GROUP_SIZE); +// IntStream.range(0, iterationCount).parallel() +// .mapToObj(i -> bgeeGeneIds.subList(i * GENE_PARALLEL_GROUP_SIZE, +// ((i + 1) * GENE_PARALLEL_GROUP_SIZE) > bgeeGeneIds.size()? +// bgeeGeneIds.size(): ((i + 1) * GENE_PARALLEL_GROUP_SIZE))) +// .forEach(subsetGeneIds -> { +// //check at each iteration if an error occurred in another thread +// this.checkErrorOccurred(); +// +// //We need a new connection to the database for each thread, so we use +// //a ServiceFactory Supplier +// final ServiceFactory threadServiceFactory = this.serviceFactorySupplier.get(); +// +// try (DAOManager threadDAOManager = threadServiceFactory.getDAOManager()) { +// //PARALLEL EXECUTION: each thread-specific DAOManager is registered +// //to be able to kill all queries in case of error in any thread. +// //The killing will be performed by this.insertThread, as we know this thread +// //will be running during the whole process and will be performing fast queries only. +// this.daoManagers.add(threadDAOManager); +// +// log.debug("Processing {} genes...", subsetGeneIds.size()); +// // We propagate calls. Each Map contains all propagated calls for one gene +// final Stream> propagatedCalls = +// this.generatePropagatedCalls( +// new HashSet<>(subsetGeneIds), rawCondMap, conditionGraph, +// threadDAOManager); +// +// //Provide the calls to insert to the Thread managing the insertions +// //through the dedicated BlockingQueue +// propagatedCalls.forEach(set -> { +// //Check error status +// this.checkErrorOccurred(); +// try { +// //wait indefinitely for space in the queue to be available +// //(to not overload the memory) +// log.trace(BLOCKING_QUEUE_MARKER, "Offering Set of {} PipelineCalls", +// set.size()); +// this.callsToInsert.put(set); +// } catch (InterruptedException e) { +// this.exceptionOccurs(e, localInsertThread); +// } +// }); +// +// log.debug("Done processing {} genes.", subsetGeneIds.size()); +// } catch (Exception e) { +// this.exceptionOccurs(e, localInsertThread); +// } +// }); +// +// //very important to set this flag here for the insertion thread to know it should quit. +// this.jobCompleted = true; +// +// } catch (Exception e) { +// this.exceptionOccurs(e, insertThread); +// } finally { +// //if there are no more data to be inserted, +// //wake up the insert thread that might still be waiting for new data to insert +// this.interruptInsertIfNeeded(insertThread); +// } +// assert this.jobCompleted || this.errorOccured != null; +// +// //now we need to wait for the Insert thread to complete the call insertions +// //before quitting: moving to another species while we still lock the tables would be bad. +// //If we run the computations with a high enough number of threads, +// //the computations are faster than the insertions +// log.info("Computations finished, continuing insertion."); +// synchronized(this.insertFinished) { +// while (!this.insertFinished.get()) { +// try { +// this.insertFinished.wait(); +// } catch (InterruptedException e) { +// throw log.throwing(new IllegalStateException(e)); // } - }) - .collect(Collectors.toSet())); - } - - private static PipelineCall mapRawCallTOToPipelineCall(RawExpressionCallTO callTO, - RawDataCondition cond, Set condParams) { - log.traceEntry("{}, {}, {}", callTO, cond, condParams); - - if (cond == null) { - throw log.throwing(new IllegalArgumentException("No Condition provided for CallTO: " - + callTO)); - } - assert callTO.getBgeeGeneId() != null; - assert callTO.getConditionId() != null; - - return log.traceExit(new PipelineCall( - callTO.getBgeeGeneId(), - mapRawDataConditionToCondition(cond), - // At this point, we do not generate data state, quality, and CallData, - // as we haven't reconcile data. - Collections.singleton(callTO))); - } - - private static Condition mapRawDataConditionToCondition(RawDataCondition rawCond) { - log.traceEntry("{}", rawCond); - if (rawCond == null) { - return log.traceExit((Condition) null); - } - //All the elements must be non-null, otherwise the propagation will end up - //with not comparable conditions between elements mapped to the root - //and element mapped to null. - assert rawCond.getAnatEntity() != null; - assert rawCond.getDevStage() != null; - assert rawCond.getCellType() != null; - assert rawCond.getSex() != null; - assert rawCond.getStrain() != null; - AnatEntity anatEntityToUse = rawCond.getAnatEntity(); - //Quick and dirty blacklisting of "unknown" terms, we remap them to the root of the anatEntities - if (UNKNOWN_ANAT_ENTITY_IDS.contains(anatEntityToUse.getId())) { - anatEntityToUse = ROOT_ANAT_ENTITY; - } - return log.traceExit(new Condition(anatEntityToUse, rawCond.getDevStage(), - rawCond.getCellType(), mapRawDataSexToSex(rawCond.getSex()), - mapRawDataStrainToStrain(rawCond.getStrain()), rawCond.getSpecies())); - } - - - //************************************************************************* - // METHODS COUTING EXPERIMENTS FOR DIFFERENT CALL TYPES - // As of Bgee 15.0, not used anymore - //************************************************************************* +// } +// } +// +// +// log.info("Done inserting of propagated calls for the species {} with combinations of condition parameters {}...", +// this.speciesId, this.condParams); +// +// log.traceExit(); +// } +// +// /** +// * Method to check if an {@code Exception} occurred in a different {@code Thread} +// * than the caller {@code Thread}, launched by this {@code InsertPropagatedCalls} object. +// * @throws IllegalStateException If an {@code Exception} occurred in a different {@code Thread}. +// */ +// private void checkErrorOccurred() throws IllegalStateException { +// log.traceEntry(); +// if (this.errorOccured != null) { +// log.debug("Stop execution following error in other Thread."); +// throw new IllegalStateException("Exception thrown in another thread, stop job."); +// } +// log.traceExit(); +// } +// +// /** +// * Method rethrowing any {@code Exception} as a {@code RuntimeException} and storing +// * it in {@link #errorOccured} and notifying {@link #insertThread} that an error occurred. +// * @param e +// * @param insertThread +// * @throws RuntimeException +// */ +// private void exceptionOccurs(Exception e, Thread insertThread) throws RuntimeException { +// log.traceEntry("{}, {}", e, insertThread); +// //set errorOccured for all threads to know there was an error +// if (this.errorOccured == null) { +// this.errorOccured = e; +// } +// //wake up the insert thread that might be waiting to consume new data. +// //important to set errorOccured before calling this method. +// this.interruptInsertIfNeeded(insertThread); +// //throw exception appropriately +// if (e instanceof RuntimeException) { +// throw log.throwing((RuntimeException) e); +// } +// throw log.throwing(new IllegalStateException(e)); +// } +// +// private void interruptInsertIfNeeded(Thread insertThread) { +// log.traceEntry("{}", insertThread); +// Set waitingStates = EnumSet.of(Thread.State.BLOCKED, Thread.State.WAITING, +// Thread.State.TIMED_WAITING); +// if (insertThread != null && waitingStates.contains(insertThread.getState()) && +// (this.errorOccured != null || (this.jobCompleted && this.callsToInsert.isEmpty()))) { +// log.debug("Interrupting insert thread"); +// insertThread.interrupt(); +// } +// log.traceExit(); +// } +// +// private Map loadRawConditionMap(Collection species) { +// log.traceEntry("{}", species); +// +// //TODO: to refactor with method org.bgee.model.CommonService.loadConditionMapFromResultSet +// Map speMap = species.stream() +// .collect(Collectors.toMap(s -> s.getId(), s -> s, (s1, s2) -> s1)); +// Set anatEntityIds = new HashSet<>(); +// Set stageIds = new HashSet<>(); +// Set cellTypeIds = new HashSet<>(); +// Set sexIds = new HashSet<>(); +// Set strainIds = new HashSet<>(); +// Set conditionTOs = new HashSet<>(); +// //check that we have covered all condition parameters +// if (EnumSet.allOf(ConditionDAO.Attribute.class).stream() +// .filter(c -> c.isConditionParameter()).count() != 5) { +// throw log.throwing(new IllegalStateException("Some condition parameters not covered")); +// } +// +// RawDataConditionTOResultSet rs = this.getDaoManager().getRawDataConditionDAO() +// .getRawDataConditionsFromRawConditionFilters( +// Set.of(new DAORawDataConditionFilter(speMap.keySet(), +// null, null, null, null, null)), +// null); +// +// while (rs.next()) { +// RawDataConditionTO condTO = rs.getTO(); +// if (!speMap.keySet().contains(condTO.getSpeciesId())) { +// throw log.throwing(new IllegalArgumentException( +// "The retrieved ConditionTOs do not match the provided Species.")); +// } +// conditionTOs.add(condTO); +// //As of Bgee 15.0, only the cellTypeId could be null +// assert condTO.getAnatEntityId() != null; +// assert condTO.getStageId() != null; +// assert condTO.getSex() != null; +// assert condTO.getStrainId() != null; +// if (condTO.getAnatEntityId() != null) { +// anatEntityIds.add(condTO.getAnatEntityId()); +// } else { +// anatEntityIds.add(ConditionDAO.ANAT_ENTITY_ROOT_ID); +// } +// if (condTO.getStageId() != null) { +// stageIds.add(condTO.getStageId()); +// } else { +// stageIds.add(ConditionDAO.DEV_STAGE_ROOT_ID); +// } +// if (condTO.getCellTypeId() != null) { +// cellTypeIds.add(condTO.getCellTypeId()); +// } else { +// cellTypeIds.add(ConditionDAO.CELL_TYPE_ROOT_ID); +// } +// if (condTO.getSex() != null) { +// sexIds.add(condTO.getSex().getStringRepresentation()); +// } else { +// sexIds.add(DAORawDataSex.NA.getStringRepresentation()); +// } +// if (condTO.getStrainId() != null) { +// strainIds.add(condTO.getStrainId()); +// } else { +// strainIds.add(ConditionDAO.STRAIN_ROOT_ID); +// } +// } +// +// Set allAnatEntityIds = new HashSet<>(anatEntityIds); +// allAnatEntityIds.addAll(cellTypeIds); +// final Map anatMap = allAnatEntityIds.isEmpty()? new HashMap<>(): +// this.getServiceFactory().getAnatEntityService().loadAnatEntities( +// speMap.keySet(), true, allAnatEntityIds, false) +// .collect(Collectors.toMap(a -> a.getId(), a -> a)); +// if (!allAnatEntityIds.isEmpty() && anatMap.size() != allAnatEntityIds.size()) { +// allAnatEntityIds.removeAll(anatMap.keySet()); +// throw log.throwing(new IllegalStateException("Some anat. entities used in a condition " +// + "are not supposed to exist in the related species. Species: " + speMap.keySet() +// + " - anat. entities: " + allAnatEntityIds)); +// } +// final Map stageMap = stageIds.isEmpty()? new HashMap<>(): +// this.getServiceFactory().getDevStageService().loadDevStages( +// speMap.keySet(), true, stageIds, false) +// .collect(Collectors.toMap(s -> s.getId(), s -> s)); +// if (!stageIds.isEmpty() && stageMap.size() != stageIds.size()) { +// stageIds.removeAll(stageMap.keySet()); +// throw log.throwing(new IllegalStateException("Some stages used in a condition " +// + "are not supposed to exist in the related species. Species: " + speMap.keySet() +// + " - stages: " + stageIds)); +// } +// +// return log.traceExit(conditionTOs.stream() +// .collect(Collectors.toMap(cTO -> cTO.getId(), +// cTO -> new RawDataCondition( +// Optional.ofNullable(anatMap.get(cTO.getAnatEntityId() == null ? +// ConditionDAO.ANAT_ENTITY_ROOT_ID : cTO.getAnatEntityId())) +// .orElseThrow(() -> new IllegalStateException("Anat. entity not found: " +// + cTO.getAnatEntityId())), +// Optional.ofNullable(stageMap.get(cTO.getStageId() == null ? +// ConditionDAO.DEV_STAGE_ROOT_ID : cTO.getStageId())) +// .orElseThrow(() -> new IllegalStateException("Stage not found: " +// + cTO.getStageId())), +// Optional.ofNullable(anatMap.get(cTO.getCellTypeId() == null ? +// ConditionDAO.CELL_TYPE_ROOT_ID : cTO.getCellTypeId())) +// .orElseThrow(() -> new IllegalStateException("Cell type not found: " +// + cTO.getCellTypeId())), +// mapDAORawDataSexToRawDataSex(cTO.getSex() == null ? +// DAORawDataSex.NA : cTO.getSex()), +// mapDAORawDataStrainToRawDataStrain(cTO.getStrainId() == null ? +// ConditionDAO.STRAIN_ROOT_ID : cTO.getStrainId()), +// Optional.ofNullable(speMap.get(cTO.getSpeciesId())).orElseThrow( +// () -> new IllegalStateException("Species not found: " +// + cTO.getSpeciesId()))) +// )) +// ); +// } +// // /** -// * Count the number of experiments for a combination of self/descendant/ancestor, -// * present/absent, and high/low. -// * -// * @param pipelineCallData A {@code Set} of {@code PipelineCallData}. -// * @param funCallDataToEETO A {@code Function} accepting a {@code PipelineCallData} returning -// * a specific {@code Set} of {@code ExperimentExpressionTO}s. -// * @param callQuality A {@code CallQuality} that is quality allowing to filter. -// * {@code ExperimentExpressionTO}s. -// * @param callDirection A {@code CallDirection} that is direction allowing to filter. -// * {@code ExperimentExpressionTO}s -// * @return The {@code int} that is the number of experiments for a combination. +// * Generate propagated and reconciled expression calls. +// * +// * @param geneIds A {@code Collection} of {@code Integer}s that are the Bgee IDs +// * of the genes for which to return the {@code ExpressionCall}s. +// * @param condMap A {@code Map} where keys are {@code Integer}s that are condition IDs, +// * the associated value being the corresponding {@code RawDataCondition} +// * with attributes populated according to the requested +// * condition parameters. +// * @param conditionGraph A {@code ConditionGraph} containing the {@code Condition}s +// * and relations considering attributes according +// * to the requested condition parameters. +// * @param daoManager The {@code DAOManager} to use to retrieve +// * {@code DAO}s to perform queries to the data source. +// * @return A {@code Stream} of {@code Map}s where keys are {@code Set} of +// * {@code ConditionDAO.Attribute}s representing combinations of +// * condition parameters, the associated value being a {@code Set} +// * of {@code ExpressionCall}s that are propagated and reconciled +// * expression calls for one gene according to the associated combination. // */ -// private static int getSpecificCount(Set> pipelineCallData, -// Function, Set> funCallDataToEETO, -// CallDirection callDirection, CallQuality callQuality) { -// log.traceEntry("{}, {}, {}, {}", pipelineCallData, funCallDataToEETO, callQuality, callDirection); +// private Stream> generatePropagatedCalls( +// Set geneIds, Map condMap, ConditionGraph conditionGraph, +// DAOManager daoManager) { +// log.traceEntry("{}, {}, {}, {}", geneIds, condMap, conditionGraph, daoManager); +// +// log.trace(COMPUTE_MARKER, "Creating Splitereator with DAO queries..."); +// this.checkErrorOccurred(); +// final RawExpressionCallDAO rawCallDAO = daoManager.getRawExpressionCallDAO(); +// final Stream streamRawCallTOs = +// this.performsRawExpressionCallTOQuery(geneIds, rawCallDAO); +// +// this.checkErrorOccurred(); +// final Map>> samplePValueTOsByDataType = +// performsSamplePValueQuery(geneIds, daoManager); // -// //to count each experiment only once in a given set of "self", "parent" or "descendant" attributes, -// //we keep its "best" call from all ExperimentExpressionTOs. -// Set bestSelectedEETOs = getBestSelectedEETOs(pipelineCallData, -// funCallDataToEETO); +// final CallSpliterator> spliterator = +// new CallSpliterator<>(streamRawCallTOs, samplePValueTOsByDataType); +// final Stream> callTOsByGeneStream = +// StreamSupport.stream(spliterator, false).onClose(() -> spliterator.close()); // -// return log.traceExit((int) bestSelectedEETOs.stream() -// .filter(eeTo -> callDirection.equals(eeTo.getCallDirection()) -// && callQuality.equals(eeTo.getCallQuality())) -// .map(ExperimentExpressionTO::getExperimentId) -// .distinct() -// .count()); +// log.trace(COMPUTE_MARKER, "Done creating Splitereator with DAO queries."); +// +// Stream> reconciledCalls = callTOsByGeneStream +// // First we convert each Set for a gene +// // into one Map> having source RawExpressionCallTO, +// .map(geneData -> geneData.stream() +// .collect(Collectors.toMap( +// rawExprCallData -> mapRawCallTOToPipelineCall( +// rawExprCallData.getRawExpressionCallTO(), +// condMap.get(rawExprCallData.getRawExpressionCallTO().getConditionId()), +// this.condParams), +// rawExprCallData -> mapExpExprTOsToPipelineCallData( +// rawExprCallData.getSamplePValueTOsPerDataType(), +// this.condParams)))) +// +// //Now, we group all PipelineCalls and PipelineCallDatas mapped to a same Condition +// //g: Map> +// //NOTE: there can still be key collision after Bgee 15.0 because of merge of, e.g., +// //raw data sexes 'not annotated' and 'mixed' into the data sex 'ANY'. +// .map(g -> g.entrySet().stream().collect(Collectors +// //we group the entries Entry> by condition +// //and merge them. +// .toMap( +// e -> e.getKey().getCondition(), +// e -> e, +// (e1, e2) -> { +// PipelineCall call1 = e1.getKey(); +// PipelineCall call2 = e2.getKey(); +// assert call1.getParentSourceCallTOs() == null || +// call1.getParentSourceCallTOs().isEmpty(); +// assert call1.getDescendantSourceCallTOs() == null || +// call1.getDescendantSourceCallTOs().isEmpty(); +// assert call1.getSelfSourceCallTOs() != null && +// !call1.getSelfSourceCallTOs().isEmpty(); +// assert call2.getParentSourceCallTOs() == null || +// call2.getParentSourceCallTOs().isEmpty(); +// assert call2.getDescendantSourceCallTOs() == null || +// call2.getDescendantSourceCallTOs().isEmpty(); +// assert call2.getSelfSourceCallTOs() != null && +// !call2.getSelfSourceCallTOs().isEmpty(); +// +// assert Integer.compare(call1.getBgeeGeneId(), call2.getBgeeGeneId()) == 0; +// assert call1.getCondition().equals(call2.getCondition()); +// assert call1.getDataPropagation().getCondParamCombinations() +// .equals(call2.getDataPropagation().getCondParamCombinations()); +// assert call1.getDataPropagation().getCondParamCombinations().stream() +// .map(comb -> call1.getDataPropagation().getPropagationState(comb)) +// .allMatch(propState -> PropagationState.SELF.equals(propState)); +// assert call2.getDataPropagation().getCondParamCombinations().stream() +// .map(comb -> call2.getDataPropagation().getPropagationState(comb)) +// .allMatch(propState -> PropagationState.SELF.equals(propState)); +// +// Set combinedTOs = +// new HashSet<>(call1.getSelfSourceCallTOs()); +// combinedTOs.addAll(call2.getSelfSourceCallTOs()); +// PipelineCall combinedCall = new PipelineCall( +// call1.getBgeeGeneId(), call1.getCondition(), combinedTOs); +// +// Set> combinedData = new HashSet<>(e1.getValue()); +// combinedData.addAll(e2.getValue()); +// +// return new AbstractMap.SimpleEntry<>(combinedCall, combinedData); +// } +// ) +// ) +// //Now retrieve the Entries that were reduced, and collect them into a Map. +// //The returned value of this map function is of the same type as the input element: +// //Map> +// .values().stream().collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue())) +// ) +// //then we propagate all PipelineCalls of the Map (associated to one gene only), +// //and retrieve the original and the propagated calls. +// //g: Map> +// .map(g -> { +// //propagatePipelineCalls returns only the new propagated calls, +// //we need to add the original calls to the Map for following steps +// Map>> calls = +// this.propagatePipelineCalls(g, conditionGraph); +// calls.putAll(g); +// return calls; +// }) +// +// //then we reconcile calls for a same gene-condition +// //g: Map> +// .map(g -> { +// log.trace(COMPUTE_MARKER, "Starting to reconcile {} PipelineCalls.", g.size()); +// this.checkErrorOccurred(); +// //group calls per Condition (they all are about the same gene already) +// final Map> callGroup = g.entrySet().stream() +// .collect(Collectors.groupingBy(e -> e.getKey().getCondition(), +// Collectors.mapping(e2 -> e2.getKey(), Collectors.toSet()))); +// //group CallData per Condition (they all are about the same gene already) +// final Map>> callDataGroup = g.entrySet().stream() +// .collect(Collectors.groupingBy(e -> e.getKey().getCondition(), +// Collectors.mapping(e2 -> e2.getValue(), Collectors.toSet()))) // produce Map> +// .entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue() +// .stream().flatMap(ps -> ps.stream()).collect(Collectors.toSet()))); // produce Map> +// +// // Reconcile calls and return all of them in one Set +// Set s = callGroup.keySet().stream() +// .map(c -> reconcileGeneCalls(callGroup.get(c), callDataGroup.get(c))) +// //reconcileGeneCalls return null if there was no valid data to propagate +// //(e.g., only "present" calls in parent conditions) +// .filter(c -> c != null) +// .collect(Collectors.toSet()); +// log.trace(COMPUTE_MARKER, "Done reconciliation, {} PipelineCalls produced.", s.size()); +// return s; +// }) +// +// //Now we have a final step since Bgee 15.0: For each call, we have computed +// //FDR-corrected p-values for all combinations of data types, considering all p-values +// //in the condition itself and in its descendant conditions. +// //Now for each call and each combination of data types, we need to find +// //the best corrected p-value among the descendant conditions +// //s: Set +// .map(s -> { +// log.trace(COMPUTE_MARKER, "Finding best descendant p-values for {} PipelineCalls.", s.size()); +// //First we create a Map to more easily retrieve calls from a condition +// Map callPerCondition = s.stream() +// .map(c -> new AbstractMap.SimpleEntry<>(c.getCondition(), c)) +// //At this point there should be only one call per condition, +// //and thus no key collision +// .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue())); +// //Now, for each call, and for each combinations of data types, +// //we are going to retrieve the best corrected p-value among the calls +// //in descendant conditions. +// //Compute a Map parent -> descendants +// Map> parentToDescendantConds = callPerCondition.keySet() +// .stream() +// .flatMap(cond -> this.condToAncestors.computeIfAbsent( +// cond, +// k -> conditionGraph.getAncestorConditions(k)).stream() +// .filter(parent -> callPerCondition.containsKey(parent)) +// .map(parent -> new AbstractMap.SimpleEntry<>(parent, +// new HashSet<>(Arrays.asList(cond))))) +// .collect(Collectors.toMap(e -> e.getKey(), +// e -> e.getValue(), +// (v1, v2) -> {v1.addAll(v2); return v1;})); +// Set> allDataTypeCombs = DataType.getAllPossibleDataTypeCombinations(); +// return s.stream().map(c -> { +// Map, FDRPValueCondition> bestPValuePerDataTypeComb = new HashMap<>(); +// Set descendantCalls = parentToDescendantConds +// //Some conditions have no descendant conditions obviously +// .getOrDefault(c.getCondition(), new HashSet<>()) +// .stream() +// .map(cond -> callPerCondition.get(cond)) +// .filter(descCond -> descCond != null) +// .collect(Collectors.toSet()); +// for (PipelineCall descendantCall: descendantCalls) { +// Map, FDRPValue> pValuePerDataTypeComb = +// descendantCall.getPValues().stream() +// .map(p -> new AbstractMap.SimpleEntry<>(p.getDataTypes(), p)) +// .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue())); +// for (EnumSet comb: allDataTypeCombs) { +// EnumSet bestMatchComb = DataType +// .findCombinationWithGreatestOverlap( +// pValuePerDataTypeComb.keySet(), comb); +// if (bestMatchComb != null) { +// FDRPValue existingPVal = pValuePerDataTypeComb.get(bestMatchComb); +// FDRPValueCondition newPVal = new FDRPValueCondition( +// existingPVal.getPValue(), comb, descendantCall.getCondition()); +// bestPValuePerDataTypeComb.merge(comb, newPVal, +// (p1, p2) -> p1.getPValue().compareTo(p2.getPValue()) == -1? +// p1: p2); +// } +// } +// } +// log.trace("Done searching best descendant p-values for call: {}", c); +// return new PipelineCall(c.getBgeeGeneId(), c.getCondition(), c.getCallData(), +// c.getPValues(), bestPValuePerDataTypeComb.values(), +// c.getParentSourceCallTOs(), c.getSelfSourceCallTOs(), +// c.getDescendantSourceCallTOs()); +// }).collect(Collectors.toSet()); +// }); +// +// return log.traceExit(reconciledCalls); // } // -// private static Set getBestSelectedEETOs(Set> pipelineCallData, -// Function, Set> funCallDataToEETO) { -// log.traceEntry("{}, {}", pipelineCallData, funCallDataToEETO); -// if (pipelineCallData == null || pipelineCallData.isEmpty()) { -// return log.traceExit(new HashSet<>()); -// } -// return log.traceExit(getBestExperimentExpressionTOs( -// pipelineCallData.stream() -// .map(p -> funCallDataToEETO.apply(p)) -// .filter(s -> s != null) -// .flatMap(Set::stream) -// .collect(Collectors.toSet()) -// )); +// //************************************************************************* +// // METHODS PERFORMING THE QUERIES TO THE DAOs +// //************************************************************************* +// /** +// * Perform query to retrieve expressed calls without the post-processing of +// * the results returned by {@code DAO}s. +// * +// * @param geneIds A {@code Collection} of {@code Integer}s that are the Bgee IDs of the genes +// * for which to return the {@code RawExpressionCallTO}s. +// * @param rawCallDAO The {@code RawExpressionCallDAO} to use to retrieve {@code RawExpressionCallTO}s +// * from data source. +// * @return The {@code Stream} of {@code RawExpressionCallTO}s. +// */ +// private Stream performsRawExpressionCallTOQuery(Set geneIds, +// RawExpressionCallDAO rawCallDAO) throws IllegalArgumentException { +// log.traceEntry("{}, {}", geneIds, rawCallDAO); +// +// Stream expr = rawCallDAO +// .getExpressionCallsOrderedByGeneIdAndExprId(geneIds) +// //retrieve the Stream resulting from the query. Note that the query is not executed +// //as long as the Stream is not consumed (lazy-loading). +// .stream(); +// +// return log.traceExit(expr); +// } +// +// private Map>> performsSamplePValueQuery( +// Set geneIds, DAOManager daoManager) throws IllegalArgumentException { +// log.traceEntry("{}, {}", geneIds, daoManager); +// +// //TODO: Former DAO to access these data, we still use it for all data but RNA-Seq and scRNA-Seq, +// //to update to use the new system for all data types. +// //Of note, SamplePValueDAO could be completely deleted, +// //and SamplePValueTO replaced with PipelineSamplePValueTO defined in this class. +// //PipelineSamplePValueTO would not have to extend SamplePValueTO, +// //but would implement the same attributes +// SamplePValueDAO samplePValueDAO = daoManager.getSamplePValueDAO(); +// +// //New system to access this information +// Set rawDataFilters = Set.of(new DAORawDataFilter(geneIds, null, null)); +// // ************ RNA-Seq and scRNA-Seq ************ +// RNASeqResultAnnotatedSampleDAO rnaSeqResultDAO = daoManager.getRnaSeqResultAnnotatedSampleDAO(); +// EnumSet rnaSeqAttrs = EnumSet.of( +// RNASeqResultAnnotatedSampleDAO.Attribute.BGEE_GENE_ID, +// RNASeqResultAnnotatedSampleDAO.Attribute.LIBRARY_ANNOTATED_SAMPLE_ID, +// RNASeqResultAnnotatedSampleDAO.Attribute.EXPRESSION_ID, +// RNASeqResultAnnotatedSampleDAO.Attribute.PVALUE); +// LinkedHashMap rnaSeqOrderingAttrs = +// new LinkedHashMap<>(); +// rnaSeqOrderingAttrs.put( +// RNASeqResultAnnotatedSampleDAO.OrderingAttribute.BGEE_GENE_ID, +// DAO.Direction.ASC); +// rnaSeqOrderingAttrs.put( +// RNASeqResultAnnotatedSampleDAO.OrderingAttribute.EXPRESSION_ID, +// DAO.Direction.ASC); +// +// Map>> map = new HashMap<>(); +// for (DataType dt: DataType.values()) { +// switch (dt) { +// case AFFYMETRIX: +// map.put(dt, samplePValueDAO.getAffymetrixPValuesOrderedByGeneIdAndExprId(geneIds).stream() +// .map(p -> p)); +// break; +// case EST: +// map.put(dt, samplePValueDAO.getESTPValuesOrderedByGeneIdAndExprId(geneIds).stream() +// .map(p -> p)); +// break; +// case IN_SITU: +// map.put(dt, samplePValueDAO.getInSituPValuesOrderedByGeneIdAndExprId(geneIds).stream() +// .map(p -> p)); +// break; +// case RNA_SEQ: +// map.put(dt, rnaSeqResultDAO.getResultAnnotatedSamples(rawDataFilters, false, true, null, null, +// rnaSeqAttrs, rnaSeqOrderingAttrs).stream() +// //We don't have experimentId in the returned TOs, +// //but annotatedSampleIds are unique in RNA-Seq data. +// //We arbitrarily defined the experimentId type +// //needed by PipelineSamplePValueTO as String +// .map(to -> new PipelineSamplePValueTO(to))); +// break; +// case SC_RNA_SEQ: +// map.put(dt, rnaSeqResultDAO.getResultAnnotatedSamples(rawDataFilters, true, true, null, null, +// rnaSeqAttrs, rnaSeqOrderingAttrs).stream() +// //We don't have experimentId in the returned TOs, +// //but annotatedSampleIds are unique in RNA-Seq data. +// //We arbitrarily defined the experimentId type +// //needed by PipelineSamplePValueTO as String +// .map(to -> new PipelineSamplePValueTO(to))); +// break; +// default: +// throw log.throwing(new IllegalStateException("Unsupported DataType: " + dt)); +// } +// } +// +// return log.traceExit(map); // } // -// /** -// * Count the number of experiments for total counts (combination of present/absent and high/low). +// //************************************************************************* +// // METHODS PROPAGATION: from CallTOs to propagated Calls +// //************************************************************************* +// +// /** +// * Propagate {@code ExpressionCall}s to descendant and ancestor conditions +// * from {@code conditionGraph}. +// *

    +// * Returned {@code ExpressionCall}s have {@code DataPropagation}, {@code ExpressionSummary}, +// * and {@code DataQuality} equal to {@code null}. // * -// * @param pipelineCallData A {@code Set} of {@code PipelineCallData}. -// * @param callQuality A {@code CallQuality} that is quality allowing to filter. -// * {@code ExperimentExpressionTO}s. -// * @param callDirection A {@code CallDirection} that is direction allowing to filter. -// * {@code ExperimentExpressionTO}s -// * @return The {@code int} that is the number of experiments for total counts. +// * @param calls A {@code Collection} of {@code ExpressionCall}s to be propagated. +// * @param conditionGraph A {@code ConditionGraph} containing at least anat. entity +// * {@code Ontology} to use for the propagation. +// * @return A {@code Map} where keys are {@code PipelineCall}, the associated +// * values are {@code Set}s of {@code PipelineCallData}. +// * @throws IllegalArgumentException If {@code calls} or {@code conditionGraph} are {@code null}, +// * empty. // */ -// private static int getTotalCount(Set> pipelineCallData, -// final CallDirection callDirection, CallQuality callQuality) { -// log.traceEntry("{}, {}, {}", pipelineCallData, callQuality, callDirection); +// private Map>> propagatePipelineCalls( +// Map>> data, ConditionGraph conditionGraph) +// throws IllegalArgumentException { +// log.traceEntry("{}, {}", data, conditionGraph); +// log.trace(COMPUTE_MARKER, "Starting to propagate {} PipelineCalls.", data.size()); +// +// Map>> propagatedData = new HashMap<>(); +// this.checkErrorOccurred(); // -// //to count each experiment only once in different "total" attributes, -// //we keep its "best" call from all ExperimentExpressionTOs. -// Set bestSelfAndRelatedEETOs = getBestTotalEETOs(pipelineCallData); +// assert data != null && !data.isEmpty(); +// assert conditionGraph != null; // -// return log.traceExit((int) bestSelfAndRelatedEETOs.stream() -// .filter(eeTo -> callDirection.equals(eeTo.getCallDirection()) -// && callQuality.equals(eeTo.getCallQuality())) -// .map(ExperimentExpressionTO::getExperimentId) -// .distinct() -// .count()); +// Set calls = data.keySet(); +// +// // Here, no calls should have PropagationState which is not SELF +// assert calls.stream().allMatch(c -> +// c.getDataPropagation().getCondParamCombinations().stream() +// .allMatch(comb -> PropagationState.SELF.equals(c.getDataPropagation().getPropagationState(comb)))); +// // Check conditionGraph contains all conditions of calls +// assert conditionGraph.getConditions().containsAll( +// calls.stream().map(c -> c.getCondition()).collect(Collectors.toSet())); +// +// //***************************** +// // PROPAGATE CALLS +// //***************************** +// +// // Counts for log tracing +// int callCount = calls.size(); +// int analyzedCallCount = 0; +// +// for (Entry>> entry: data.entrySet()) { +// this.checkErrorOccurred(); +// if (log.isTraceEnabled() && analyzedCallCount % 100 == 0) { +// log.trace("{}/{} expression calls analyzed.", analyzedCallCount, callCount); +// } +// analyzedCallCount++; +// +// ExpressionCall curCall = entry.getKey(); +// log.trace(COMPUTE_MARKER, "Propagation for call: {}", curCall); +// +// // Retrieve conditions +// log.trace(COMPUTE_MARKER, "Starting to retrieve ancestral conditions for {}.", +// curCall.getCondition()); +// Set ancestorConditions = this.condToAncestors.computeIfAbsent( +// curCall.getCondition(), +// k -> conditionGraph.getAncestorConditions(k)); +// log.trace(COMPUTE_MARKER, "Done retrieving ancestral conditions for {}: {}.", +// curCall.getCondition(), ancestorConditions.size()); +// log.trace("Ancestor conditions: {}", ancestorConditions); +// if (!ancestorConditions.isEmpty()) { +// Map>> ancestorCalls = +// propagatePipelineData(entry, ancestorConditions, true); +// assert !ancestorCalls.isEmpty(); +// propagatedData.putAll(ancestorCalls); +// } +// +// //Note: actually as of Bgee 14.2 we do not propagate absent calls to substructures anymore +// Set descendantConditions = new HashSet<>(); +//// log.trace(COMPUTE_MARKER, "Starting to retrieve descendant conditions for {}.", +//// curCall.getCondition()); +//// Set descendantConditions = this.condToDescendants.computeIfAbsent( +//// curCall.getCondition(), +//// k -> conditionGraph.getDescendantConditions( +//// k, false, false, NB_SUBLEVELS_MAX, null)); +//// log.trace(COMPUTE_MARKER, "Done retrieving descendant conditions for {}: {}.", +//// curCall.getCondition(), descendantConditions.size()); +//// log.trace("Descendant conditions: {}", descendantConditions); +// if (!descendantConditions.isEmpty()) { +// Map>> descendantCalls = +// propagatePipelineData(entry, descendantConditions, false); +// assert !descendantCalls.isEmpty(); +// propagatedData.putAll(descendantCalls); +// } +// +// log.trace(COMPUTE_MARKER, "Done propagation for call: {}", curCall); +// } +// +// log.trace(COMPUTE_MARKER, "Done propagating {} PipelineCalls, {} new propagated PipelineCalls.", +// data.size(), propagatedData.size()); +// return log.traceExit(propagatedData); // } // // /** -// * Retrieve the {@code ExperimentExpressionTO}s from valid attributes of the provided -// * {@code PipelineCallData} depending on their {@code CallDirection}, then keep only -// * for each experiment the "best" {@code ExperimentExpressionTO}. +// * Propagate calls to provided {@code propagatedConds}. // * -// * @param pipelineCallData -// * @return +// * @param data An {@code Entry} where keys are {@code PipelineCall}, +// * the associated value being a {@code Set} of {@code PipelineCallData} +// * that is the call to be propagated. +// * @param propagatedConds A {@code Collection} of {@code Condition}s that are the conditions +// * for which the propagation have to be done. +// * @param areAncestors A {@code boolean} defining whether the {@code propagatedConds} +// * are ancestors or descendants. If {@code true}, it is ancestors. +// * @return A {@code Set} of {@code ExpressionCall}s that are propagated calls +// * from provided {@code data}, without including calls in {@code data}. // */ -// private static Set getBestTotalEETOs(Set> pipelineCallData) { -// log.traceEntry("{}", pipelineCallData); -// if (pipelineCallData == null || pipelineCallData.isEmpty()) { -// return log.traceExit(new HashSet<>()); -// } -// return log.traceExit(getBestExperimentExpressionTOs( -// pipelineCallData.stream() -// .map(p -> { -// Set exps = new HashSet<>(); -// -// if (p.getSelfExperimentExpr() != null) { -// exps.addAll(p.getSelfExperimentExpr()); +// private Map>> propagatePipelineData( +// Entry>> data, Set propagatedConds, +// boolean areAncestors) { +// log.traceEntry("{}, {}, {}", data, propagatedConds, areAncestors); +// log.trace(COMPUTE_MARKER, "Start to propagate PipelineData, to ancestor? {}.", areAncestors); +// +// Map>> map = new HashMap<>(); +// this.checkErrorOccurred(); +// +// if (propagatedConds.isEmpty()) { +// throw log.throwing(new IllegalArgumentException("No provided propagated conditions")); +// } +// PipelineCall call = data.getKey(); +// Condition callCondition = call.getCondition(); +// +// //For each propagated condition (not including the original condition), +// //create a new PipelineCall, with source CallTOs stored in the appropriate attribute, +// //and with associated PipelineCallData updated. +// for (Condition condition : propagatedConds) { +// log.trace("Propagation of the current call to condition: {}", condition); +// assert !callCondition.equals(condition); +// +// Set> relativeData = new HashSet<>(); +// +// //for each original PipelineCallData, create a new PipelineCallData with DataPropagation updated +// //and ExperimentExpressionTOs stored in the appropriate attributes +// for (PipelineCallData pipelineData: data.getValue()) { +// this.checkErrorOccurred(); +// +// // Here, we define propagation states. +// // A state should stay to null if we do not have this information in call condition. +// PropagationState anatEntityPropagationState = null; +// PropagationState devStagePropagationState = null; +// PropagationState cellTypePropagationState = null; +// PropagationState sexPropagationState = null; +// PropagationState strainPropagationState = null; +// if (areAncestors) { +// if (callCondition.getAnatEntityId() != null) +// anatEntityPropagationState = PropagationState.DESCENDANT; +// if (callCondition.getDevStageId() != null) +// devStagePropagationState = PropagationState.DESCENDANT; +// if (callCondition.getCellTypeId() != null) +// cellTypePropagationState = PropagationState.DESCENDANT; +// if (callCondition.getSexId() != null) +// sexPropagationState = PropagationState.DESCENDANT; +// if (callCondition.getStrainId() != null) +// strainPropagationState = PropagationState.DESCENDANT; +// } else { +// if (callCondition.getAnatEntityId() != null) { +// anatEntityPropagationState = PropagationState.ANCESTOR; +// } +// //no propagation to substages etc, only to substructures, but it does not hurt +// //and the value is changed just below +// if (callCondition.getDevStageId() != null) { +// devStagePropagationState = PropagationState.ANCESTOR; +// } +// if (callCondition.getCellTypeId() != null) { +// cellTypePropagationState = PropagationState.ANCESTOR; +// } +// if (callCondition.getSexId() != null) { +// sexPropagationState = PropagationState.ANCESTOR; +// } +// if (callCondition.getStrainId() != null) { +// strainPropagationState = PropagationState.ANCESTOR; // } +// } +// +// if (callCondition.getAnatEntityId() != null && +// callCondition.getAnatEntityId().equals(condition.getAnatEntityId())) { +// anatEntityPropagationState = PropagationState.SELF; +// } +// if (callCondition.getDevStageId() != null && +// callCondition.getDevStageId().equals(condition.getDevStageId())) { +// devStagePropagationState = PropagationState.SELF; +// } +// if (callCondition.getCellTypeId() != null && +// callCondition.getCellTypeId().equals(condition.getCellTypeId())) { +// cellTypePropagationState = PropagationState.SELF; +// } +// if (callCondition.getSexId() != null && +// callCondition.getSexId().equals(condition.getSexId())) { +// sexPropagationState = PropagationState.SELF; +// } +// if (callCondition.getStrainId() != null && +// callCondition.getStrainId().equals(condition.getStrainId())) { +// strainPropagationState = PropagationState.SELF; +// } +// assert anatEntityPropagationState != PropagationState.SELF || +// devStagePropagationState != PropagationState.SELF || +// cellTypePropagationState != PropagationState.SELF || +// sexPropagationState != PropagationState.SELF || +// strainPropagationState != PropagationState.SELF; // -// //Note: as of Bgee 14.2, we don't propagate absent calls to sub-structures anymore. -// //Former comment: -//// //we keep only ABSENT calls from parent structures, so that -//// //getBestExperimentExpressionTOs does not discard an experiment -//// //showing expression of the gene in one parent, and absence of expression -//// //in another parent: in that case, we want to propagate only the absence -//// //of expression, since we don't propagate presence of expression -//// //to descendant conditions -// if (p.getParentExperimentExpr() != null) { -// exps.addAll(p.getParentExperimentExpr().stream() -// //Note: as of Bgee 14.2, we don't propagate absent calls to sub-structures anymore. -//// .filter(eeTO -> CallDirection.ABSENT.equals(eeTO.getCallDirection())) -// .filter(eeTO -> false) -// .collect(Collectors.toSet())); +// //We want to count the number of self p-values for all combinations of condition parameters +// EnumSet selfPropStateParams = EnumSet.noneOf( +// CallService.Attribute.class); +// for (CallService.Attribute condParam: CallService.Attribute.getAllConditionParameters()) { +// switch(condParam) { +// case ANAT_ENTITY_ID: +// if (anatEntityPropagationState.equals(PropagationState.SELF)) { +// selfPropStateParams.add(condParam); +// } +// break; +// case CELL_TYPE_ID: +// if (cellTypePropagationState.equals(PropagationState.SELF)) { +// selfPropStateParams.add(condParam); +// } +// break; +// case DEV_STAGE_ID: +// if (devStagePropagationState.equals(PropagationState.SELF)) { +// selfPropStateParams.add(condParam); +// } +// break; +// case SEX_ID: +// if (sexPropagationState.equals(PropagationState.SELF)) { +// selfPropStateParams.add(condParam); +// } +// break; +// case STRAIN_ID: +// if (strainPropagationState.equals(PropagationState.SELF)) { +// selfPropStateParams.add(condParam); +// } +// break; +// default: +// throw log.throwing(new IllegalStateException("Unsupported condition parameter: " +// + condParam)); // } -// -// //we do not propagate ABSENT calls to parent condition, -// //so here we keep only PRESENT calls -// if (p.getDescendantExperimentExpr() != null) { -// exps.addAll(p.getDescendantExperimentExpr().stream() -// .filter(eeTO -> CallDirection.PRESENT.equals(eeTO.getCallDirection())) -// .collect(Collectors.toSet())); +// } +// Set> selfPropStateParamCombinations = +// selfPropStateParams.isEmpty()? new HashSet<>(): +// CallService.Attribute.getAllPossibleCondParamCombinations(selfPropStateParams); +// assert !selfPropStateParamCombinations.contains( +// CallService.Attribute.getAllConditionParameters()); +// Set> notSelfPropStateParamCombinations = +// CallService.Attribute.getAllPossibleCondParamCombinations(); +// notSelfPropStateParamCombinations.removeAll(selfPropStateParamCombinations); +// assert notSelfPropStateParamCombinations.contains( +// CallService.Attribute.getAllConditionParameters()); +// +// switch(pipelineData.getDataType()) { +// case EST: +// case IN_SITU: +// case RNA_SEQ: +// case SC_RNA_SEQ: +// //We know the generic types depending on the data types +// @SuppressWarnings("unchecked") +// Set> localPValues = pipelineData +// .getSelfPValuesPerCondParamCombinations().get( +// CallService.Attribute.getAllConditionParameters()) +// .stream() +// .map(pval -> (SamplePValueTO) pval) +// .collect(Collectors.toSet()); +// assert !localPValues.isEmpty(); +// +// Map, Set>> +// selfPValuesPerCondParamCombinations = selfPropStateParamCombinations.stream() +// .collect(Collectors.toMap(comb -> comb, comb -> localPValues)); +// selfPValuesPerCondParamCombinations.putAll(notSelfPropStateParamCombinations.stream() +// .collect(Collectors.toMap(comb -> comb, comb -> new HashSet<>()))); +// Set> parentPValues = null; +// Set> descendantPValues = null; +// if (areAncestors) { +// descendantPValues = localPValues; +// } else { +// parentPValues = localPValues; // } -// return exps; -// }) -// .flatMap(Set::stream) -// .collect(Collectors.toSet()) -// )); +// relativeData.add(new PipelineCallData<>(pipelineData.getDataType(), +// parentPValues, selfPValuesPerCondParamCombinations, descendantPValues)); +// break; +// case AFFYMETRIX: +// //We know the generic types depending on the data types +// @SuppressWarnings("unchecked") +// Set> localPValues2 = pipelineData +// .getSelfPValuesPerCondParamCombinations().get( +// CallService.Attribute.getAllConditionParameters()) +// .stream() +// .map(pval -> (SamplePValueTO) pval) +// .collect(Collectors.toSet()); +// assert !localPValues2.isEmpty(); +// +// Map, Set>> +// selfPValuesPerCondParamCombinations2 = selfPropStateParamCombinations.stream() +// .collect(Collectors.toMap(comb -> comb, comb -> localPValues2)); +// selfPValuesPerCondParamCombinations2.putAll(notSelfPropStateParamCombinations.stream() +// .collect(Collectors.toMap(comb -> comb, comb -> new HashSet<>()))); +// Set> parentPValues2 = null; +// Set> descendantPValues2 = null; +// if (areAncestors) { +// descendantPValues2 = localPValues2; +// } else { +// parentPValues2 = localPValues2; +// } +// relativeData.add(new PipelineCallData<>(pipelineData.getDataType(), +// parentPValues2, selfPValuesPerCondParamCombinations2, descendantPValues2)); +// break; +// } +// } +// +// // Add propagated expression call. +// Set ancestorCallTOs = null; +// Set descendantCallTOs = null; +// if (areAncestors) { +// descendantCallTOs = call.getSelfSourceCallTOs(); +// } else { +// ancestorCallTOs = call.getSelfSourceCallTOs(); +// } +// +// PipelineCall propagatedCall = new PipelineCall( +// call.getBgeeGeneId(), +// condition, +// null, // Collection callData (update after the propagation), +// null, null, //corrected p-values +// ancestorCallTOs, null, descendantCallTOs); +// +// log.trace("Add the propagated call: {}", propagatedCall); +// map.put(propagatedCall, relativeData); +// } +// if (map.isEmpty()) { +// throw log.throwing(new IllegalStateException("No propagated calls")); +// } +// +// log.trace(COMPUTE_MARKER, "Done propagating PipelineData, to ancestor? {}.", areAncestors); +// return log.traceExit(map); // } // -// /** -// * Retrieve for each experiment ID the {@code ExperimentExpressionTO} corresponding to the best call, -// * among the {@code ExperimentExpressionTO}s in {@code eeTO}s. +// /** +// * Reconcile several pipeline calls into one pipeline call. +// *

    +// * Return the representative {@code PipelineCall} (with reconciled quality per data types, +// * observed data state, conflict status etc. // * -// * @param eeTOs -// * @return +// * @param calls A {@code Set} of {@code PipelineCall}s that are the calls to be reconciled. +// * @param pipelineData A {@code Set} of {@code PipelineCallData} that are the pipeline call data +// * to be used for reconciliation. +// * @return The representative {@code ExpressionCall}. // */ -// private static Set getBestExperimentExpressionTOs( -// Collection eeTOs) { -// log.traceEntry("{}", eeTOs); +// //We return PipelineCall rather than ExpressionCall to be able to keep bgeeGeneId +// private PipelineCall reconcileGeneCalls(Set calls, +// Set> pipelineData) { +// log.traceEntry("{}, {}", calls, pipelineData); +// +// this.checkErrorOccurred(); +// +// assert calls != null && !calls.isEmpty(); +// assert pipelineData != null && !pipelineData.isEmpty(); +// +// Set geneIds = calls.stream().map(c -> c.getBgeeGeneId()).collect(Collectors.toSet()); +// if (geneIds.size() != 1 || geneIds.contains(null)) { +// throw log.throwing(new IllegalArgumentException( +// "None or several genes are found in provided PipelineCalls")); +// } +// int geneId = geneIds.iterator().next(); // -// return log.traceExit(new HashSet<>(eeTOs.stream() -// //we create a Map experimentId -> ExperimentExpressionTO, -// //and keep the ExperimentExpressionTO corresponding to the best call -// //when there is a key collision +// Set conditions = calls.stream().map(c -> c.getCondition()).collect(Collectors.toSet()); +// if (conditions.size() != 1 || conditions.contains(null)) { +// throw log.throwing(new IllegalArgumentException( +// "None or several conditions are found in provided PipelineCalls")); +// } +// Condition condition = conditions.iterator().next(); +// +// Map>> pipelineDataByDataTypes = pipelineData.stream() +// .collect(Collectors.groupingBy(PipelineCallData::getDataType, Collectors.toSet())); +// +// Set expressionCallData = new HashSet<>(); +// for (Entry>> entry: pipelineDataByDataTypes.entrySet()) { +// ExpressionCallData cd = mergePipelineCallDataIntoExpressionCallData( +// entry.getKey(), entry.getValue()); +// //the returned callData is null if there was no valid data to propagate +// //(e.g., only "present" expression calls in parent conditions) +// if (cd != null) { +// expressionCallData.add(cd); +// } +// } +// if (expressionCallData.isEmpty()) { +// log.trace("No valid data to propagate"); +// return log.traceExit((PipelineCall) null); +// } +// +// //************************ +// // Data propagation +// //************************ +// +//// assert expressionCallData.stream() +//// .flatMap(ecd -> ecd.getExperimentCounts(PropagationState.ALL).stream()) +//// .mapToInt(c -> c.getCount()).sum() != 0; +//// assert Boolean.TRUE.equals(dataProp.isIncludingObservedData()) && expressionCallData.stream() +//// .flatMap(ecd -> ecd.getExperimentCounts(PropagationState.SELF).stream()) +//// .mapToInt(c -> c.getCount()).sum() > 0 || +//// Boolean.FALSE.equals(dataProp.isIncludingObservedData()) && expressionCallData.stream() +//// .flatMap(ecd -> ecd.getExperimentCounts(PropagationState.SELF).stream()) +//// .mapToInt(c -> c.getCount()).sum() == 0 && +//// expressionCallData.stream().mapToInt(ecd -> ecd.getPropagatedExperimentCount()).sum() > 0; +// +// +// Set selfSourceCallTOs = calls.stream() +// .map(PipelineCall::getSelfSourceCallTOs) +// .filter(s -> s != null) +// .flatMap(s -> s.stream()) +// .collect(Collectors.toSet()); +// +// //************************ +// // FDR-corrected p-values +// //************************ +// //We compute the corrected p-values for all combination of data types with data for this call. +// //First, we retrieve all possible combinations of the data types used. +// Set> usedDataTypeCombs = DataType.getAllPossibleDataTypeCombinations( +// expressionCallData.stream().map(ecd -> ecd.getDataType()).collect(Collectors.toSet())); +// //And now we correct the p-values for all possible combination of the data types used +// Set correctedPValues = usedDataTypeCombs.stream() +// .map(dtComb -> { +// //Use List to not loose equal pvalues +// List pValues = expressionCallData.stream() +// .filter(ecd -> dtComb.contains(ecd.getDataType())) +// .flatMap(ecd -> ecd.getAllPValues().stream()) +// .collect(Collectors.toList()); +// return new FDRPValue(computeFDRCorrectedPValue(pValues), dtComb); +// }) +// .collect(Collectors.toSet()); +// //now we need to complement the combinations of data types by copying the computed p-values: +// //for instance, if there was only RNA-Seq and Affymetrix data for this call, +// //then the combination EST-RNA-Seq-Affymetrix will have the exact same corrected p-value as +// //RNA-Seq-Affymetrix. We need to get all combinations with any data available. +// //So this otherDataTypeCombs Set will contains all combination with at least one DataType +// //with no associated data for this call. +// Set> otherDataTypeCombs = DataType.getAllPossibleDataTypeCombinations().stream() +// .filter(s -> !usedDataTypeCombs.contains(s)) +// .collect(Collectors.toSet()); +// //For each of these combinations, we'll try to find the combination with a computed p-value +// //with the most overlap in data types and with all its data types contained. +// Map, FDRPValue> pValuePerDataTypeComb = correctedPValues.stream() +// .map(p -> new AbstractMap.SimpleEntry<>(p.getDataTypes(), p)) +// .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue())); +// Set otherCorrectedPValues = otherDataTypeCombs.stream() +// .map(otherDTComb -> { +// EnumSet mostMatchedDataTypesCombination = +// DataType.findCombinationWithGreatestOverlap( +// pValuePerDataTypeComb.keySet(), otherDTComb); +// if (mostMatchedDataTypesCombination == null) { +// return null; +// } +// return new FDRPValue(pValuePerDataTypeComb.get(mostMatchedDataTypesCombination) +// .getPValue(), otherDTComb); +// }) +// //If we couldn't find any overlap with a computed combination, the previous mapping +// //returned null, we filter that out. +// .filter(p -> p != null) +// .collect(Collectors.toSet()); +// +// assert Collections.disjoint(correctedPValues, otherCorrectedPValues): +// "Corrected pvalues: " + correctedPValues + " - others: " + otherCorrectedPValues; +// Set allCorrectedPValues = new HashSet<>(correctedPValues); +// allCorrectedPValues.addAll(otherCorrectedPValues); +// +// // It is not necessary to infer ExpressionSummary, SummaryQuality using +// // CallService.inferSummaryXXX(), because they will not be inserted in the db, +// // we solely store p-values +// return log.traceExit(new PipelineCall(geneId, condition, expressionCallData, +// allCorrectedPValues, null, +// calls.stream().map(PipelineCall::getParentSourceCallTOs) +// .filter(s -> s != null) +// .flatMap(Set::stream).collect(Collectors.toSet()), +// selfSourceCallTOs, +// calls.stream().map(PipelineCall::getDescendantSourceCallTOs) +// .filter(s -> s != null) +// .flatMap(Set::stream).collect(Collectors.toSet()))); +// } +// // code taken from https://github.com/cBioPortal/cbioportal/blob/master/core/src/main/java/ +// // org/mskcc/cbio/portal/stats/BenjaminiHochbergFDR.java +// private static BigDecimal computeFDRCorrectedPValue(List pValues) { +// log.traceEntry("{}", pValues); +// +// int m = pValues.size(); +// Double[] pValuesDouble = +// pValues.stream() +// .map(p -> p.compareTo(ZERO_BIGDECIMAL) == 0 ? ABOVE_ZERO_BIGDECIMAL : p) +// .map(p -> p.doubleValue()) +// .toArray(length -> new Double[length]); +// double[] adjustedPValues = new double[m]; +// +// Arrays.sort(pValuesDouble); +// // iterate through all p-values: largest to smallest +// for (int i = m - 1; i >= 0; i--) { +// if (i == m - 1) { +// adjustedPValues[i] = pValuesDouble[i]; +// } else { +// double unadjustedPvalue = pValuesDouble[i]; +// int divideByM = i + 1; +// double left = adjustedPValues[i + 1]; +// double right = (m / (double) divideByM) * unadjustedPvalue; +// adjustedPValues[i] = Math.min(left, right); +// } +// } +// //Find the smallest corrected p-value +// BigDecimal fdr = BigDecimal.valueOf(Arrays.stream(adjustedPValues).min().getAsDouble()); +// //If the FDR is less than MIN_FDR_BIGDECIMAL, change it to MIN_FDR_BIGDECIMAL +// //(in order to avoid having fields in the globalExpression table with too much precision) +// if (fdr.compareTo(MIN_FDR_BIGDECIMAL) < 0) { +// fdr = MIN_FDR_BIGDECIMAL; +// } +// return log.traceExit(fdr); +// } +// +// /** +// * Merge a {@code Set} of {@code PipelineCallData} into one {@code ExpressionCallData}. +// * +// * @param dataType A {@code DataType} that is the data type of {@code pipelineCallData}. +// * @param pipelineCallData A {@code Set} of {@code PipelineCallData} to be used to +// * build the {@code ExpressionCallData}. +// * on propagated data. +// */ +// private ExpressionCallData mergePipelineCallDataIntoExpressionCallData(DataType dataType, +// Set> pipelineCallData) { +// log.traceEntry("{}, {}", dataType, pipelineCallData); +// +// this.checkErrorOccurred(); +// +// assert pipelineCallData.stream().noneMatch(pcd -> !dataType.equals(pcd.getDataType())); +// //at this point, we have only propagated one call at a time, so we should have +// //p-values in only one of these 3 attributes +// assert pipelineCallData.stream().allMatch(pcd -> +// (/*(pcd.getSelfPValues() != null && !pcd.getSelfPValues().isEmpty()) &&*/ +// (pcd.getParentPValues() == null || pcd.getParentPValues().isEmpty()) && +// (pcd.getDescendantPValues() == null || pcd.getDescendantPValues().isEmpty()) || +// +// /*(pcd.getSelfPValues() == null || pcd.getSelfPValues().isEmpty()) &&*/ +// (pcd.getParentPValues() != null && !pcd.getParentPValues().isEmpty()) && +// (pcd.getDescendantPValues() == null || pcd.getDescendantPValues().isEmpty()) || +// +// /*(pcd.getSelfPValues() == null || pcd.getSelfPValues().isEmpty()) &&*/ +// (pcd.getParentPValues() == null || pcd.getParentPValues().isEmpty()) && +// (pcd.getDescendantPValues() != null && !pcd.getDescendantPValues().isEmpty()))); +// +// +// +// //Rank info: computed by the Perl pipeline after insertion of these global calls +//// BigDecimal rank = null; +//// BigDecimal rankNorm = null; +//// BigDecimal rankSum = null; +//// if (selfSourceCallTO != null) { +//// switch(dataType) { +//// case AFFYMETRIX: +//// rank = selfSourceCallTO.getAffymetrixMeanRank(); +//// rankNorm = selfSourceCallTO.getAffymetrixMeanRankNorm(); +//// rankSum = selfSourceCallTO.getAffymetrixDistinctRankSum(); +//// break; +//// case RNA_SEQ: +//// rank = selfSourceCallTO.getRNASeqMeanRank(); +//// rankNorm = selfSourceCallTO.getRNASeqMeanRankNorm(); +//// rankSum = selfSourceCallTO.getRNASeqDistinctRankSum(); +//// break; +//// case EST: +//// rank = selfSourceCallTO.getESTRank(); +//// rankNorm = selfSourceCallTO.getESTRankNorm(); +//// break; +//// case IN_SITU: +//// rank = selfSourceCallTO.getInSituRank(); +//// rankNorm = selfSourceCallTO.getInSituRankNorm(); +//// break; +//// default: +//// log.throwing(new IllegalStateException("Unsupported data type: " + dataType)); +//// } +//// } +// +// +// EnumSet allCondParams = CallService.Attribute.getAllConditionParameters(); +// //We map to PipelineSamplePValueTO because it implements hashCode/equals, +// //taking into account the experiment and sample IDs, so that we can be sure +// //we don't count a p-value coming from a same observation several times. +// Map, Set>> selfPValuesPerCondParamComb = +// pipelineCallData.stream() +// .flatMap(pcd -> pcd.getSelfPValuesPerCondParamCombinations().entrySet().stream()) // .collect(Collectors.toMap( -// eeTO -> eeTO.getExperimentId(), -// eeTO -> eeTO, -// (v1, v2) -> { -// //"present" calls always win over "absent" calls -// if (!v1.getCallDirection().equals(v2.getCallDirection())) { -// if (v1.getCallDirection().equals(CallDirection.PRESENT)) { -// return v1; -// } -// return v2; -// } -// //high quality win over low quality -// if (!v1.getCallQuality().equals(v2.getCallQuality())) { -// if (v1.getCallQuality().ordinal() > v2.getCallQuality().ordinal()) { -// return v1; -// } -// return v2; -// } -// //equal calls, return v1 -// return v1; -// })) -// .values())); +// e -> e.getKey(), +// e -> e.getValue().stream().map(p -> new PipelineSamplePValueTO<>(p)) +// .collect(Collectors.toSet()), +// (v1, v2) -> {v1.addAll(v2); return v1;})); +// Set> descendantPValues = pipelineCallData.stream() +// .flatMap(pcd -> pcd.getDescendantPValues().stream() +// .map(p -> new PipelineSamplePValueTO<>(p))) +// .collect(Collectors.toSet()); +// Set> selfPValues = selfPValuesPerCondParamComb.get(allCondParams); +// +// assert Stream.concat(selfPValues.stream(), descendantPValues.stream()) +// .collect(Collectors.toSet()).containsAll(selfPValuesPerCondParamComb.values().stream() +// .flatMap(s -> s.stream()).collect(Collectors.toSet())); +// if (!Collections.disjoint(selfPValues, descendantPValues)) { +// selfPValues.retainAll(descendantPValues); +// throw log.throwing(new IllegalStateException( +// "self and desendant p-values should always be disjoined, p-values in common: " +// + selfPValues)); +// } +// +// Map, Integer> selfObservationCounts = +// selfPValuesPerCondParamComb.entrySet().stream() +// .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue().size())); +// assert selfObservationCounts.keySet().contains(CallService.Attribute.getAllConditionParameters()); +// //We collect to a List to not eliminate equals pvalues +// List mappedDescendantPValues = descendantPValues.stream().map(p -> p.getpValue()) +// .collect(Collectors.toList()); +// //For descendant observation counts, we store in database only the count +// //for all condition parameters. But for creating the DataPropagation object +// //we need to have the same keyset in both Maps. +// Map, Integer> descendantObservationCounts = +// selfObservationCounts.keySet().stream().collect(Collectors.toMap( +// k -> k, +// k -> k.equals(CallService.Attribute.getAllConditionParameters())? +// mappedDescendantPValues.size(): 0)); +// DataPropagation dataProp = new DataPropagation(selfObservationCounts, +// descendantObservationCounts); +// +// log.trace(COMPUTE_MARKER, "ExpressionCallData to be created: {} - {} - {} - {}", +// dataType, selfPValuesPerCondParamComb, descendantPValues, dataProp); +// return log.traceExit(new ExpressionCallData(dataType, +// //We collect to a List to not eliminate equals pvalues +// selfPValues.stream().map(p -> p.getpValue()).collect(Collectors.toList()), +// mappedDescendantPValues, +// null, null, null, +// dataProp)); +// } +// +// //************************************************************************* +// // METHODS MAPPING dao-api objects to bgee-core objects +// //************************************************************************* +// +// private static Set> mapExpExprTOsToPipelineCallData( +// Map>> pValuesByDataTypes, +// Set condParams) { +// log.traceEntry("{}, {}, {}", pValuesByDataTypes, condParams); +// EnumSet dataTypes = EnumSet.noneOf(DataType.class); +// dataTypes.addAll(pValuesByDataTypes.keySet()); +// return log.traceExit(dataTypes.stream() +// .map(dt -> { +// return new PipelineCallData<>( +// dt, null, +// CallService.Attribute.getAllPossibleCondParamCombinations().stream() +// .collect(Collectors.toMap(comb -> comb , +// comb -> pValuesByDataTypes.get(dt).stream() +// .map(pval -> pval) +// .collect(Collectors.toSet()))), +// null); +//// switch(dt) { +//// case EST: +//// case IN_SITU: +//// case RNA_SEQ: +//// case SC_RNA_SEQ: +//// //The cast of the generic types of SamplePValueTOs can be determined by the data type +//// @SuppressWarnings("unchecked") +//// PipelineCallData pipelineCallData = new PipelineCallData<>( +//// dt, getSelfDataProp(condParams), +//// null, expExprsByDataTypes == null? null: expExprsByDataTypes.get(dt), null, +//// null, +//// pValuesByDataTypes.get(dt).stream() +//// .map(pval -> (SamplePValueTO) pval) +//// .collect(Collectors.toSet()), +//// null); +//// return pipelineCallData; +//// case AFFYMETRIX: +//// //The cast of the generic types of SamplePValueTOs can be determined by the data type +//// @SuppressWarnings("unchecked") +//// PipelineCallData pipelineCallData2 = new PipelineCallData<>( +//// dt, getSelfDataProp(condParams), +//// null, expExprsByDataTypes == null? null: expExprsByDataTypes.get(dt), null, +//// null, +//// pValuesByDataTypes.get(dt).stream() +//// .map(pval -> (SamplePValueTO) pval) +//// .collect(Collectors.toSet()), +//// null); +//// return pipelineCallData2; +//// default: +//// throw log.throwing(new IllegalStateException("Unsupported data type: " + dt)); +//// } +// }) +// .collect(Collectors.toSet())); +// } +// +// private static PipelineCall mapRawCallTOToPipelineCall(RawExpressionCallTO callTO, +// RawDataCondition cond, Set condParams) { +// log.traceEntry("{}, {}, {}", callTO, cond, condParams); +// +// if (cond == null) { +// throw log.throwing(new IllegalArgumentException("No Condition provided for CallTO: " +// + callTO)); +// } +// assert callTO.getBgeeGeneId() != null; +// assert callTO.getConditionId() != null; +// +// return log.traceExit(new PipelineCall( +// callTO.getBgeeGeneId(), +// mapRawDataConditionToCondition(cond), +// // At this point, we do not generate data state, quality, and CallData, +// // as we haven't reconcile data. +// Collections.singleton(callTO))); +// } +// +// private static Condition mapRawDataConditionToCondition(RawDataCondition rawCond) { +// log.traceEntry("{}", rawCond); +// if (rawCond == null) { +// return log.traceExit((Condition) null); +// } +// //All the elements must be non-null, otherwise the propagation will end up +// //with not comparable conditions between elements mapped to the root +// //and element mapped to null. +// assert rawCond.getAnatEntity() != null; +// assert rawCond.getDevStage() != null; +// assert rawCond.getCellType() != null; +// assert rawCond.getSex() != null; +// assert rawCond.getStrain() != null; +// AnatEntity anatEntityToUse = rawCond.getAnatEntity(); +// //Quick and dirty blacklisting of "unknown" terms, we remap them to the root of the anatEntities +// if (UNKNOWN_ANAT_ENTITY_IDS.contains(anatEntityToUse.getId())) { +// anatEntityToUse = ROOT_ANAT_ENTITY; +// } +// return log.traceExit(new Condition(anatEntityToUse, rawCond.getDevStage(), +// rawCond.getCellType(), mapRawDataSexToSex(rawCond.getSex()), +// mapRawDataStrainToStrain(rawCond.getStrain()), rawCond.getSpecies())); +// } +// +// +// //************************************************************************* +// // METHODS COUTING EXPERIMENTS FOR DIFFERENT CALL TYPES +// // As of Bgee 15.0, not used anymore +// //************************************************************************* +//// /** +//// * Count the number of experiments for a combination of self/descendant/ancestor, +//// * present/absent, and high/low. +//// * +//// * @param pipelineCallData A {@code Set} of {@code PipelineCallData}. +//// * @param funCallDataToEETO A {@code Function} accepting a {@code PipelineCallData} returning +//// * a specific {@code Set} of {@code ExperimentExpressionTO}s. +//// * @param callQuality A {@code CallQuality} that is quality allowing to filter. +//// * {@code ExperimentExpressionTO}s. +//// * @param callDirection A {@code CallDirection} that is direction allowing to filter. +//// * {@code ExperimentExpressionTO}s +//// * @return The {@code int} that is the number of experiments for a combination. +//// */ +//// private static int getSpecificCount(Set> pipelineCallData, +//// Function, Set> funCallDataToEETO, +//// CallDirection callDirection, CallQuality callQuality) { +//// log.traceEntry("{}, {}, {}, {}", pipelineCallData, funCallDataToEETO, callQuality, callDirection); +//// +//// //to count each experiment only once in a given set of "self", "parent" or "descendant" attributes, +//// //we keep its "best" call from all ExperimentExpressionTOs. +//// Set bestSelectedEETOs = getBestSelectedEETOs(pipelineCallData, +//// funCallDataToEETO); +//// +//// return log.traceExit((int) bestSelectedEETOs.stream() +//// .filter(eeTo -> callDirection.equals(eeTo.getCallDirection()) +//// && callQuality.equals(eeTo.getCallQuality())) +//// .map(ExperimentExpressionTO::getExperimentId) +//// .distinct() +//// .count()); +//// } +//// +//// private static Set getBestSelectedEETOs(Set> pipelineCallData, +//// Function, Set> funCallDataToEETO) { +//// log.traceEntry("{}, {}", pipelineCallData, funCallDataToEETO); +//// if (pipelineCallData == null || pipelineCallData.isEmpty()) { +//// return log.traceExit(new HashSet<>()); +//// } +//// return log.traceExit(getBestExperimentExpressionTOs( +//// pipelineCallData.stream() +//// .map(p -> funCallDataToEETO.apply(p)) +//// .filter(s -> s != null) +//// .flatMap(Set::stream) +//// .collect(Collectors.toSet()) +//// )); +//// } +//// +//// /** +//// * Count the number of experiments for total counts (combination of present/absent and high/low). +//// * +//// * @param pipelineCallData A {@code Set} of {@code PipelineCallData}. +//// * @param callQuality A {@code CallQuality} that is quality allowing to filter. +//// * {@code ExperimentExpressionTO}s. +//// * @param callDirection A {@code CallDirection} that is direction allowing to filter. +//// * {@code ExperimentExpressionTO}s +//// * @return The {@code int} that is the number of experiments for total counts. +//// */ +//// private static int getTotalCount(Set> pipelineCallData, +//// final CallDirection callDirection, CallQuality callQuality) { +//// log.traceEntry("{}, {}, {}", pipelineCallData, callQuality, callDirection); +//// +//// //to count each experiment only once in different "total" attributes, +//// //we keep its "best" call from all ExperimentExpressionTOs. +//// Set bestSelfAndRelatedEETOs = getBestTotalEETOs(pipelineCallData); +//// +//// return log.traceExit((int) bestSelfAndRelatedEETOs.stream() +//// .filter(eeTo -> callDirection.equals(eeTo.getCallDirection()) +//// && callQuality.equals(eeTo.getCallQuality())) +//// .map(ExperimentExpressionTO::getExperimentId) +//// .distinct() +//// .count()); +//// } +//// +//// /** +//// * Retrieve the {@code ExperimentExpressionTO}s from valid attributes of the provided +//// * {@code PipelineCallData} depending on their {@code CallDirection}, then keep only +//// * for each experiment the "best" {@code ExperimentExpressionTO}. +//// * +//// * @param pipelineCallData +//// * @return +//// */ +//// private static Set getBestTotalEETOs(Set> pipelineCallData) { +//// log.traceEntry("{}", pipelineCallData); +//// if (pipelineCallData == null || pipelineCallData.isEmpty()) { +//// return log.traceExit(new HashSet<>()); +//// } +//// return log.traceExit(getBestExperimentExpressionTOs( +//// pipelineCallData.stream() +//// .map(p -> { +//// Set exps = new HashSet<>(); +//// +//// if (p.getSelfExperimentExpr() != null) { +//// exps.addAll(p.getSelfExperimentExpr()); +//// } +//// +//// //Note: as of Bgee 14.2, we don't propagate absent calls to sub-structures anymore. +//// //Former comment: +////// //we keep only ABSENT calls from parent structures, so that +////// //getBestExperimentExpressionTOs does not discard an experiment +////// //showing expression of the gene in one parent, and absence of expression +////// //in another parent: in that case, we want to propagate only the absence +////// //of expression, since we don't propagate presence of expression +////// //to descendant conditions +//// if (p.getParentExperimentExpr() != null) { +//// exps.addAll(p.getParentExperimentExpr().stream() +//// //Note: as of Bgee 14.2, we don't propagate absent calls to sub-structures anymore. +////// .filter(eeTO -> CallDirection.ABSENT.equals(eeTO.getCallDirection())) +//// .filter(eeTO -> false) +//// .collect(Collectors.toSet())); +//// } +//// +//// //we do not propagate ABSENT calls to parent condition, +//// //so here we keep only PRESENT calls +//// if (p.getDescendantExperimentExpr() != null) { +//// exps.addAll(p.getDescendantExperimentExpr().stream() +//// .filter(eeTO -> CallDirection.PRESENT.equals(eeTO.getCallDirection())) +//// .collect(Collectors.toSet())); +//// } +//// return exps; +//// }) +//// .flatMap(Set::stream) +//// .collect(Collectors.toSet()) +//// )); +//// } +//// +//// /** +//// * Retrieve for each experiment ID the {@code ExperimentExpressionTO} corresponding to the best call, +//// * among the {@code ExperimentExpressionTO}s in {@code eeTO}s. +//// * +//// * @param eeTOs +//// * @return +//// */ +//// private static Set getBestExperimentExpressionTOs( +//// Collection eeTOs) { +//// log.traceEntry("{}", eeTOs); +//// +//// return log.traceExit(new HashSet<>(eeTOs.stream() +//// //we create a Map experimentId -> ExperimentExpressionTO, +//// //and keep the ExperimentExpressionTO corresponding to the best call +//// //when there is a key collision +//// .collect(Collectors.toMap( +//// eeTO -> eeTO.getExperimentId(), +//// eeTO -> eeTO, +//// (v1, v2) -> { +//// //"present" calls always win over "absent" calls +//// if (!v1.getCallDirection().equals(v2.getCallDirection())) { +//// if (v1.getCallDirection().equals(CallDirection.PRESENT)) { +//// return v1; +//// } +//// return v2; +//// } +//// //high quality win over low quality +//// if (!v1.getCallQuality().equals(v2.getCallQuality())) { +//// if (v1.getCallQuality().ordinal() > v2.getCallQuality().ordinal()) { +//// return v1; +//// } +//// return v2; +//// } +//// //equal calls, return v1 +//// return v1; +//// })) +//// .values())); +//// } +// +// private static String mapDAORawDataStrainToRawDataStrain(String daoStrain) { +// log.traceEntry("{}", daoStrain); +// if (StringUtils.isBlank(daoStrain)) { +// return log.traceExit((String) null); +// } +// return log.traceExit(daoStrain); // } - - private static String mapDAORawDataStrainToRawDataStrain(String daoStrain) { - log.traceEntry("{}", daoStrain); - if (StringUtils.isBlank(daoStrain)) { - return log.traceExit((String) null); - } - return log.traceExit(daoStrain); - } -} \ No newline at end of file +//} \ No newline at end of file diff --git a/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/InsertPropagatedConditions.java b/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/InsertPropagatedConditions.java new file mode 100644 index 000000000..476e0bde3 --- /dev/null +++ b/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/InsertPropagatedConditions.java @@ -0,0 +1,698 @@ +package org.bgee.pipeline.expression; + +import java.sql.Connection; +import java.util.AbstractMap; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.MarkerManager; +import org.bgee.model.ServiceFactory; +import org.bgee.model.anatdev.AnatEntity; +import org.bgee.model.anatdev.DevStage; +import org.bgee.model.anatdev.Sex; +import org.bgee.model.anatdev.Strain; +import org.bgee.model.dao.api.DAOManager; +import org.bgee.model.dao.api.exception.DAOException; +import org.bgee.model.dao.api.expressiondata.call.ConditionDAO; +import org.bgee.model.dao.api.expressiondata.call.ConditionDAO.ConditionParameter; +import org.bgee.model.dao.api.expressiondata.call.ConditionDAO.ConditionTO; +import org.bgee.model.dao.api.expressiondata.call.ConditionDAO.GlobalConditionToDirectAncestorTO; +import org.bgee.model.dao.api.expressiondata.call.ConditionDAO.RawConditionToSelfGlobalConditionTO; +import org.bgee.model.dao.api.expressiondata.rawdata.DAORawDataConditionFilter; +import org.bgee.model.dao.api.expressiondata.rawdata.RawDataConditionDAO.RawDataConditionTO; +import org.bgee.model.dao.api.expressiondata.rawdata.RawDataConditionDAO.RawDataConditionTO.DAORawDataSex; +import org.bgee.model.dao.api.expressiondata.rawdata.RawDataConditionDAO.RawDataConditionTOResultSet; +import org.bgee.model.dao.mysql.connector.MySQLDAOManager; +import org.bgee.model.expressiondata.call.CallService; +import org.bgee.model.expressiondata.call.CallServiceUtils; +import org.bgee.model.expressiondata.call.Condition; +import org.bgee.model.expressiondata.call.ConditionGraph; +import org.bgee.model.expressiondata.call.ConditionGraphService; +import org.bgee.model.expressiondata.rawdata.baseelements.RawDataCondition; +import org.bgee.model.species.Species; +import org.bgee.pipeline.BgeeDBUtils; +import org.bgee.pipeline.CommandRunner; + +/** + * Class responsible for inserting the propagated Conditions into the Bgee database. + * + * @author Julien Wollbrett + * @author Frederic Bastian + * @author Valentine Rech de Laval + * @version Bgee 16, Nov. 2025 + * @since Bgee 16, Nov. 2025 + */ +public class InsertPropagatedConditions extends CallService { + private final static Logger log = LogManager.getLogger(InsertPropagatedConditions.class.getName()); + private static final Marker INSERTION_MARKER = MarkerManager.getMarker("INSERTION_MARKER"); + + private final static Set COND_PARAMS = Collections.unmodifiableSet( + EnumSet.allOf(ConditionDAO.Attribute.class).stream().filter(p -> p.isConditionParameter()) + .collect(Collectors.toSet())); + + private final static AtomicInteger COND_ID_COUNTER = new AtomicInteger(0); + + /** + * A {@code Set} of {@code String}s storing the IDs of anatomical terms corresponding to + * the concept "unknown". To allow a simple blacklisting of "unknown" terms, we will remap them + * to the root of the anat. entity ontology. + */ + private final static Set UNKNOWN_ANAT_ENTITY_IDS = Collections.unmodifiableSet( + new HashSet<>(Arrays.asList("XAO:0003003", "ZFA:0001093"))); + /** + * An {@code AnatEntity} that is the root of the anat. entity ontology. + */ + private final static AnatEntity ROOT_ANAT_ENTITY = new AnatEntity(ConditionDAO.ANAT_ENTITY_ROOT_ID); + private final static DevStage ROOT_DEV_STAGE = new DevStage(ConditionDAO.DEV_STAGE_ROOT_ID); + private final static AnatEntity ROOT_CELL_TYPE = new AnatEntity(ConditionDAO.CELL_TYPE_ROOT_ID); + private final static Sex ROOT_SEX = new Sex(ConditionDAO.SEX_ROOT_ID); + private final static Strain ROOT_STRAIN = new Strain(ConditionDAO.STRAIN_ROOT_ID); + + + /** + * Main method to insert propagated conditions in Bgee database, see {@link #insert(List, Collection)}. + * Parameters that must be provided in order in {@code args} are: + *

      + *
    1. a list of NCBI species IDs (for instance, {@code 9606} for human) that will be used to + * propagate expression, separated by the {@code String} {@link CommandRunner#LIST_SEPARATOR}. + * If empty (see {@link CommandRunner#EMPTY_LIST}), all species in database will be used. + *
    2. a {@code Map} where keys are whatever, and each value is a set of strings, + * corresponding to {@code ConditionDAO.Attribute}s, allowing to target a specific + * condition parameter combination. Example: 1//ANAT_ENTITY_ID,2//ANAT_ENTITY_ID--STAGE_ID + *
    + * + * @param args An {@code Array} of {@code String}s containing the requested parameters. + * @throws DAOException If an error occurred while inserting the data into the Bgee database. + */ + public static void main(String[] args) throws DAOException { + log.traceEntry("{}", (Object[]) args); + + if (args[0].equals("insertGlobalConditions")) { + int expectedArgLength = 3; + if (args.length != expectedArgLength) { + throw log.throwing(new IllegalArgumentException("Incorrect number of arguments " + + "provided, expected " + expectedArgLength + " arguments, " + args.length + + " provided.")); + } + + List speciesIds = CommandRunner.parseListArgumentAsInt(args[1]); + List condParamArg = CommandRunner.parseListArgument(args[2]); + InsertPropagatedConditions.insertGlobalConditions(speciesIds, getCondParamsFromArg(condParamArg), + DAOManager::getDAOManager, ServiceFactory::new); + } else { + throw log.throwing(new IllegalArgumentException("Unrecognized action: " + args[0])); + } + + log.traceExit(); + } + private static Set getCondParamsFromArg(List arg) { + log.traceEntry("{}", arg); + Set condParams = arg.stream() + .distinct() + .map(p -> ConditionDAO.Attribute.valueOf(p)) + .collect(Collectors.toSet()); + if (condParams.isEmpty()) { + condParams = COND_PARAMS; + } + if (!COND_PARAMS.containsAll(condParams)) { + condParams.removeAll(COND_PARAMS); + throw log.throwing(new IllegalArgumentException("Unrecognized condition parameters: " + + condParams)); + } + return log.traceExit(condParams); + } + + /** + * {@code TransferObject}s do not implement equals/hashCode, and we need it for inserting + * {@code RawConditionToSelfGlobalConditionTO}s, so we extend this class and implements hashCode/Equals. + */ + private static class PipelineRawConditionToSelfGlobalConditionTO extends RawConditionToSelfGlobalConditionTO { + private static final long serialVersionUID = -4710796651567000694L; + + public PipelineRawConditionToSelfGlobalConditionTO(Integer rawConditionId, Integer globalConditionId, + EnumSet conditionParameters) { + super(rawConditionId, globalConditionId, conditionParameters); + } + + } + + public static void insertGlobalConditions(List speciesIds, + Set condParams, final Supplier daoManagerSupplier, + final Function serviceFactoryProvider) { + log.traceEntry("{}, {}, {}, {}", speciesIds, condParams, daoManagerSupplier, serviceFactoryProvider); + + final Set clonedCondParams = Collections.unmodifiableSet( + condParams.stream().distinct().collect(Collectors.toSet())); + try(DAOManager commonManager = daoManagerSupplier.get()) { + final List speciesIdsToUse = BgeeDBUtils.checkAndGetSpeciesIds(speciesIds, + commonManager.getSpeciesDAO()); + COND_ID_COUNTER.set(commonManager.getConditionDAO().getMaxGlobalConditionId()); + + //close connection immediately, but do not close the manager because of + //the try-with-resource clause. + commonManager.releaseResources(); + + // do not use parallelstream here as we already parallelize the insertion of direct ancestor relationships, which is the most time-consuming part of the process and whom time is really different between species. We want to process species one by one to avoid having some threads idle while waiting for the longest one to finish. + speciesIdsToUse.stream().forEach(speciesId -> { + //Give as argument a Supplier of ServiceFactory so that this object + //can provide a new connection to each parallel thread. + InsertPropagatedConditions insert = new InsertPropagatedConditions( + () -> serviceFactoryProvider.apply(daoManagerSupplier.get()), + clonedCondParams, speciesId, 0, 0); + try { + insert.insertGlobalConditionsForOneSpecies(); + } catch (Exception e) { + throw log.throwing(new IllegalStateException(e)); + } + }); + } + } + + private static void startTransaction(MySQLDAOManager daoManager) throws Exception { + log.traceEntry("{}", daoManager); + //we assume the insertion is done using MySQL, and we start a transaction + log.debug(INSERTION_MARKER, "Trying to start transaction..."); + //try several attempts in case the first SELECT queries lock relevant tables + int maxAttempt = 10; + int i = 0; + TRANSACTION: while (true) { + try { + //TODO: reimplement properly in MySQLDAOManager. + //I do it here because I want to turn autocommit to true before setting the transaction level, + //to be sure it's properly set for the next transaction + daoManager.getConnection().getRealConnection().setAutoCommit(true); + daoManager.getConnection().getRealConnection() + .setTransactionIsolation(Connection.TRANSACTION_READ_UNCOMMITTED); + daoManager.getConnection().getRealConnection().setAutoCommit(false); + break TRANSACTION; + } catch (Exception e) { + if (i < maxAttempt) { + log.catching(Level.DEBUG, e); + log.debug(INSERTION_MARKER, + "Trying to start transaction failed, {} try over {}", + i + 1, maxAttempt); + try { + Thread.sleep(2000); + } catch(InterruptedException ex) { + log.catching(ex); + Thread.currentThread().interrupt(); + throw log.throwing(ex); + } + } else { + log.debug(INSERTION_MARKER, + "Starting transaction failed, {} try over {}", + i + 1, maxAttempt); + //that was the last try, throw exception + throw e; + } + } + i++; + } + + log.info(INSERTION_MARKER, "Starting transaction"); + log.traceExit(); + } + + private static ConditionGraph loadConditionGraph(ConditionGraphService condGraphService, + Set conds, boolean inferConditions) { + log.traceEntry("{}, {}, {}", condGraphService, conds, inferConditions); + + if (!inferConditions) { + //If we don't infer conditions they were already pre-computed + //and we have nothing more to do + return log.traceExit(condGraphService.loadConditionGraph(conds, + false, false)); + } + //Infer conditions. + //Of note, non-informative anat. entities/cell types are not considered when inferring + //propagated conditions (except roots, or terms used in annotations). + ConditionGraph conditionGraph = condGraphService.loadConditionGraph( + conds, + true, //propagate to ancestor conditions + false //We do not propagate to descendant conditions anymore + ); + //Since we propagate only to ancestor as of Bgee 15.0, + //we don't need to filter out descendant propagated strains, stages, sexes + return log.traceExit(conditionGraph); + } + + private static Map insertNewGlobalConditions(Set condsToInsert, + Set insertedGlobalConditions, ConditionDAO condDAO) { + log.traceEntry("{}, {}, {}", condsToInsert, insertedGlobalConditions, condDAO); + + //First, we retrieve the conditions not already present in the database + Set conds = new HashSet<>(condsToInsert); + conds.removeAll(insertedGlobalConditions); + + //now we create the Map associating each Condition to insert to a generated ID for insertion + Map newConds = conds.stream() + .collect(Collectors.toMap(c -> c, c -> COND_ID_COUNTER.incrementAndGet())); + + //now we insert the conditions + Set condTOs = newConds.entrySet().stream() + .map(e -> mapConditionToConditionTO(e.getValue(), e.getKey())) + .collect(Collectors.toSet()); + if (!condTOs.isEmpty()) { + condDAO.insertGlobalConditions(condTOs); + } + + //return new conditions with IDs + return log.traceExit(newConds); + } + + private static void insertRawCondToSelfGlobalCond( + Set toInsert, ConditionDAO condDAO) { + log.traceEntry("{}, {}, {}, {}", toInsert, condDAO); + + //now we insert the relations + if (!toInsert.isEmpty()) { + condDAO.insertRawConditionToSelfGlobalCondition(toInsert.stream() + .map(c -> (RawConditionToSelfGlobalConditionTO) c) + .collect(Collectors.toSet())); + } + log.traceExit(); + } + + private static Set insertGlobalConditionDirectRelation(ConditionGraph conditionGraph, + Map globalCondToCondId, MySQLDAOManager daoManager, ConditionDAO condDAO) + throws Exception { + log.traceEntry("{}, {}", conditionGraph, globalCondToCondId); + + int total = globalCondToCondId.size(); + log.info("Computing direct ancestor relationships for {} conditions...", total); + long startTime = System.currentTimeMillis(); + AtomicInteger processedCount = new AtomicInteger(0); + + // Release the DB connection before the long parallel computation. Without this the + // connection sits idle long enough for MySQL wait_timeout to expire. + daoManager.getConnection().getRealConnection().setAutoCommit(true); + daoManager.releaseResources(); + + // Use parallelStream so each condition is processed independently on separate threads. + // getAncestorConditions(directRelOnly=true) computes all ancestor conditions then applies + // a pairwise two-check filter (direct ontology step OR no intermediate in the graph). + // Memory usage is O(|relativeConds|) per call — no shared cache, so no risk of OOM. + // getAncestorConditions only reads immutable/unmodifiable data → thread-safe. + Set globalConditionToDirectAncestorTOs = globalCondToCondId.entrySet() + .parallelStream() + .flatMap(entry -> { + Integer condId = entry.getValue(); + int count = processedCount.incrementAndGet(); + if (count % 10_000 == 0) { + log.info("Progress: {}/{} conditions processed ({} ms elapsed)", + count, total, System.currentTimeMillis() - startTime); + } + return conditionGraph.getAncestorConditions(entry.getKey(), true) + .stream() + .map(globalCondToCondId::get) + // Guard: ancestors are always in globalCondToCondId, but skip nulls + // defensively to avoid inserting corrupt TOs. + .filter(java.util.Objects::nonNull) + .map(parentId -> new GlobalConditionToDirectAncestorTO(condId, parentId)); + }) + .collect(Collectors.toSet()); + + log.info("Generated {} ancestor relationships in {} ms total", + globalConditionToDirectAncestorTOs.size(), System.currentTimeMillis() - startTime); + + // Re-acquire a fresh connection and start a transaction before the INSERT. + startTransaction(daoManager); + + // then insert GlobalConditionDirectRelationTO + if (!globalConditionToDirectAncestorTOs.isEmpty()) { + condDAO.insertcondIdToDirectAncestorId(globalConditionToDirectAncestorTOs); + } + + return globalConditionToDirectAncestorTOs; + } + + /** + * A {@code Set} of {@code ConditionDAO.Attribute}s defining the condition parameters + * that were requested for queries, allowing to determine how the data should be aggregated. + */ + private final EnumSet condParams; + /** + * An {@code int} that is the ID of the species to propagate calls for. + */ + private final int speciesId; + + public InsertPropagatedConditions(Supplier serviceFactorySupplier, + Set condParams, int speciesId, int geneOffset, int geneRowCount) { + this(serviceFactorySupplier, condParams, speciesId, geneOffset, geneRowCount, new CallServiceUtils()); + } + public InsertPropagatedConditions(Supplier serviceFactorySupplier, + Set condParams, int speciesId, int geneOffset, int geneRowCount, + CallServiceUtils utils) { + super(serviceFactorySupplier.get(), utils); + if (condParams == null || condParams.isEmpty()) { + throw log.throwing(new IllegalArgumentException("Condition attributes should not be empty")); + } + if (geneOffset < 0 || geneRowCount < 0) { + throw log.throwing(new IllegalArgumentException( + "geneOffset and geneRowCount cannot be negative")); + } + if (geneOffset > 0 && geneRowCount == 0) { + throw log.throwing(new IllegalArgumentException( + "geneRowCount must be provided if geneOffset is provided")); + } + this.condParams = EnumSet.copyOf(condParams); + this.speciesId = speciesId; + } + + private void insertGlobalConditionsForOneSpecies() throws Exception { + log.traceEntry(); + log.info("Start inserting global conditions for the species {} with combinations of condition parameters {}...", + this.speciesId, this.condParams); + + try (DAOManager mainManager = this.getDaoManager()) { + ConditionDAO condDAO = mainManager.getConditionDAO(); + + Species species = this.getServiceFactory().getSpeciesService().loadSpeciesByIds( + Collections.singleton(this.speciesId), false).iterator().next(); + + //First, we retrieve the raw conditions already present in database. + final Map rawCondIdToRawCondMap = Collections.unmodifiableMap( + this.loadRawConditionMap(Collections.singleton(species))); + log.info("{} raw data conditions for species {}", rawCondIdToRawCondMap.size(), speciesId); + + // We use all existing conditions in the species, and infer all propagated conditions + log.info("Starting condition inference for species {}...", this.speciesId); + Map> globalCondToSelfRawCondIds = rawCondIdToRawCondMap.entrySet() + .stream() + .map(e -> new AbstractMap.SimpleEntry<>( + mapRawDataConditionToCondition(e.getValue()), + new HashSet<>(Arrays.asList(e.getKey())))) + .collect(Collectors.toMap(e -> e.getKey(), e -> e.getValue(), + (v1, v2) -> {v1.addAll(v2); return v1;})); + assert globalCondToSelfRawCondIds.values().stream().flatMap(s -> s.stream()) + .collect(Collectors.toSet()).equals(rawCondIdToRawCondMap.keySet()); + + // Release the DB connection before the long CPU-only computation (condition graph + // inference). Without this the connection sits idle long enough for MySQL + // wait_timeout to expire, causing a CommunicationsException on the next query. + ((MySQLDAOManager) mainManager).getConnection().getRealConnection().setAutoCommit(true); + mainManager.releaseResources(); + + final ConditionGraph conditionGraph = loadConditionGraph( + this.getServiceFactory().getConditionGraphService(), + globalCondToSelfRawCondIds.keySet(), + true); + log.info("Done condition inference for species {}.", this.speciesId); + + startTransaction((MySQLDAOManager) mainManager); + + //Insert propagated conditions + Map globalCondsToglobalCondId = InsertPropagatedConditions + .insertNewGlobalConditions(conditionGraph.getConditions(), + new HashSet<>(), condDAO); + ((MySQLDAOManager) mainManager).getConnection().getRealConnection().commit(); + assert conditionGraph.getConditions().equals(globalCondsToglobalCondId.keySet()); + log.info("{} global conditions inserted for species {}", globalCondsToglobalCondId.size(), this.speciesId); + + //Insert relations between raw condition and self propagated conditions for all combinations + //of condition parameters + Set rawCondToSelfCondTOs = generateRawConditionToSelfGlobalCondition( + rawCondIdToRawCondMap, globalCondToSelfRawCondIds, globalCondsToglobalCondId); + InsertPropagatedConditions.insertRawCondToSelfGlobalCond(rawCondToSelfCondTOs, condDAO); + ((MySQLDAOManager) mainManager).getConnection().getRealConnection().commit(); + assert rawCondToSelfCondTOs.stream().map(to -> to.getGlobalConditionId()).collect(Collectors.toSet()) + .equals(new HashSet<>(globalCondsToglobalCondId.values())); + assert rawCondToSelfCondTOs.stream().map(to -> to.getRawConditionId()).collect(Collectors.toSet()) + .equals(globalCondToSelfRawCondIds.values().stream() + .flatMap(s -> s.stream()).collect(Collectors.toSet())); + log.info("{} relations between raw condition and self propagated conditions for all combinations" + + " of condition parameters have been inserted for species {}", rawCondToSelfCondTOs.size(), this.speciesId); + + //Finally insert direct relations between propagated conditions + Set globalCondToDirectAncestorTOs = InsertPropagatedConditions + .insertGlobalConditionDirectRelation(conditionGraph, globalCondsToglobalCondId, + (MySQLDAOManager) mainManager, condDAO); + log.info("{} relations between global conndition and their direct ancestors have been inserted for species {}", + globalCondToDirectAncestorTOs.size(), speciesId); + + + ((MySQLDAOManager) mainManager).getConnection().getRealConnection().commit(); + ((MySQLDAOManager) mainManager).getConnection().getRealConnection().setAutoCommit(true); + } + log.traceExit(); + } + + private Set generateRawConditionToSelfGlobalCondition( + Map rawCondIdToRawCondMap, Map> globalCondToSelfRawCondIds, + Map globalCondToGlobalCondIdMap) { + log.traceEntry("{}, {}, {}", rawCondIdToRawCondMap, globalCondToSelfRawCondIds, globalCondToGlobalCondIdMap); + // first need raw cond and corresponding global condition ID. + Map rawCondIdToSelfGlobalCond = + globalCondToSelfRawCondIds.entrySet() + .stream() + .flatMap(entry -> entry.getValue().stream() + .map(i -> Map.entry(i, entry.getKey()))) + .collect(Collectors.toMap( + Map.Entry::getKey, + Map.Entry::getValue + )); + // keep only IDs for each conditionParameter + Map globalCondParamIdsToGlobalCondIdMap = + globalCondToGlobalCondIdMap.entrySet().stream().map(es -> { + Condition cond = new Condition(new AnatEntity(es.getKey().getAnatEntityId()), + new DevStage(es.getKey().getDevStageId()), new AnatEntity(es.getKey().getCellTypeId()), + new Sex(es.getKey().getSexId()), new Strain(es.getKey().getStrainId()), new Species(es.getKey().getSpeciesId())); + return Map.entry(cond, es.getValue()); + }).collect(Collectors.toMap(ae -> ae.getKey(), ae -> ae.getValue())); + + // for each raw cond Id + Set allRawCondToSelfGlobalCond = + rawCondIdToSelfGlobalCond.entrySet().stream().map(rcm -> { + Integer rawConditionId = rcm.getKey(); + Condition globalCondition = rcm.getValue(); + Set rawCondToSelfGlobalCond = new HashSet<>(); + // generate PipelineRawConditionToSelfGlobalConditionTO for each combination of condition parameter + for (int subsetMask = 1; subsetMask <= ConditionDAO.MAX_MASK; subsetMask++) { + EnumSet condParams = RawConditionToSelfGlobalConditionTO.fromSubsetMaskToCondParam(subsetMask); + //TODO: init all these root terms at instanciation + AnatEntity anatEntity = InsertPropagatedConditions.ROOT_ANAT_ENTITY; + DevStage stage = InsertPropagatedConditions.ROOT_DEV_STAGE; + AnatEntity cellType = InsertPropagatedConditions.ROOT_CELL_TYPE; + Sex sex = InsertPropagatedConditions.ROOT_SEX; + Strain strain = InsertPropagatedConditions.ROOT_STRAIN; + if (condParams.contains(ConditionParameter.ANAT_ENTITY)) { + anatEntity = new AnatEntity(globalCondition.getAnatEntityId()); + } + if (condParams.contains(ConditionParameter.STAGE)) { + stage = new DevStage(globalCondition.getDevStageId()); + } + if (condParams.contains(ConditionParameter.CELL_TYPE)) { + cellType = new AnatEntity(globalCondition.getCellTypeId()); + } + if (condParams.contains(ConditionParameter.SEX)) { + sex = new Sex(globalCondition.getSexId()); + } + if (condParams.contains(ConditionParameter.STRAIN)) { + strain = new Strain(globalCondition.getStrainId()); + } + Condition currentCond = new Condition(anatEntity, stage, cellType, sex, strain, new Species(globalCondition.getSpeciesId())); + Integer currentGlobalCondId = globalCondParamIdsToGlobalCondIdMap.get(currentCond); + rawCondToSelfGlobalCond.add(new PipelineRawConditionToSelfGlobalConditionTO(rawConditionId, currentGlobalCondId, condParams)); + + } + return rawCondToSelfGlobalCond; + }).flatMap(e -> e.stream()). collect(Collectors.toSet()); + return allRawCondToSelfGlobalCond; + } + + private Map loadRawConditionMap(Collection species) { + log.traceEntry("{}", species); + + //TODO: to refactor with method org.bgee.model.CommonService.loadConditionMapFromResultSet + Map speMap = species.stream() + .collect(Collectors.toMap(s -> s.getId(), s -> s, (s1, s2) -> s1)); + Set anatEntityIds = new HashSet<>(); + Set stageIds = new HashSet<>(); + Set cellTypeIds = new HashSet<>(); + Set sexIds = new HashSet<>(); + Set strainIds = new HashSet<>(); + Set conditionTOs = new HashSet<>(); + //check that we have covered all condition parameters + if (EnumSet.allOf(ConditionDAO.Attribute.class).stream() + .filter(c -> c.isConditionParameter()).count() != 5) { + throw log.throwing(new IllegalStateException("Some condition parameters not covered")); + } + + RawDataConditionTOResultSet rs = this.getDaoManager().getRawDataConditionDAO() + .getRawDataConditionsFromRawConditionFilters( + Set.of(new DAORawDataConditionFilter(speMap.keySet(), + null, null, null, null, null)), + null); + + while (rs.next()) { + RawDataConditionTO condTO = rs.getTO(); + if (!speMap.keySet().contains(condTO.getSpeciesId())) { + throw log.throwing(new IllegalArgumentException( + "The retrieved ConditionTOs do not match the provided Species.")); + } + // We propagate only conditions that are mapped to expression + if (!condTO.getId().equals(condTO.getExprMappedConditionId())) { + continue; + } + conditionTOs.add(condTO); + //As of Bgee 15.0, only the cellTypeId could be null + assert condTO.getAnatEntityId() != null; + assert condTO.getStageId() != null; + assert condTO.getSex() != null; + assert condTO.getStrainId() != null; + if (condTO.getAnatEntityId() != null) { + anatEntityIds.add(condTO.getAnatEntityId()); + } else { + anatEntityIds.add(ConditionDAO.ANAT_ENTITY_ROOT_ID); + } + if (condTO.getStageId() != null) { + stageIds.add(condTO.getStageId()); + } else { + stageIds.add(ConditionDAO.DEV_STAGE_ROOT_ID); + } + if (condTO.getCellTypeId() != null) { + cellTypeIds.add(condTO.getCellTypeId()); + } else { + cellTypeIds.add(ConditionDAO.CELL_TYPE_ROOT_ID); + } + if (condTO.getSex() != null) { + sexIds.add(condTO.getSex().getStringRepresentation()); + } else { + sexIds.add(DAORawDataSex.NA.getStringRepresentation()); + } + if (condTO.getStrainId() != null) { + strainIds.add(condTO.getStrainId()); + } else { + strainIds.add(ConditionDAO.STRAIN_ROOT_ID); + } + } + + Set allAnatEntityIds = new HashSet<>(anatEntityIds); + allAnatEntityIds.addAll(cellTypeIds); + final Map anatMap = allAnatEntityIds.isEmpty()? new HashMap<>(): + this.getServiceFactory().getAnatEntityService().loadAnatEntities( + speMap.keySet(), true, allAnatEntityIds, false) + .collect(Collectors.toMap(a -> a.getId(), a -> a)); + if (!allAnatEntityIds.isEmpty() && anatMap.size() != allAnatEntityIds.size()) { + allAnatEntityIds.removeAll(anatMap.keySet()); + throw log.throwing(new IllegalStateException("Some anat. entities used in a condition " + + "are not supposed to exist in the related species. Species: " + speMap.keySet() + + " - anat. entities: " + allAnatEntityIds)); + } + final Map stageMap = stageIds.isEmpty()? new HashMap<>(): + this.getServiceFactory().getDevStageService().loadDevStages( + speMap.keySet(), true, stageIds, false) + .collect(Collectors.toMap(s -> s.getId(), s -> s)); + if (!stageIds.isEmpty() && stageMap.size() != stageIds.size()) { + stageIds.removeAll(stageMap.keySet()); + throw log.throwing(new IllegalStateException("Some stages used in a condition " + + "are not supposed to exist in the related species. Species: " + speMap.keySet() + + " - stages: " + stageIds)); + } + + return log.traceExit(conditionTOs.stream() + .collect(Collectors.toMap(cTO -> cTO.getId(), + cTO -> new RawDataCondition( + Optional.ofNullable(anatMap.get(cTO.getAnatEntityId() == null ? + ConditionDAO.ANAT_ENTITY_ROOT_ID : cTO.getAnatEntityId())) + .orElseThrow(() -> new IllegalStateException("Anat. entity not found: " + + cTO.getAnatEntityId())), + Optional.ofNullable(stageMap.get(cTO.getStageId() == null ? + ConditionDAO.DEV_STAGE_ROOT_ID : cTO.getStageId())) + .orElseThrow(() -> new IllegalStateException("Stage not found: " + + cTO.getStageId())), + Optional.ofNullable(anatMap.get(cTO.getCellTypeId() == null ? + ConditionDAO.CELL_TYPE_ROOT_ID : cTO.getCellTypeId())) + .orElseThrow(() -> new IllegalStateException("Cell type not found: " + + cTO.getCellTypeId())), + mapDAORawDataSexToRawDataSex(cTO.getSex() == null ? + DAORawDataSex.NA : cTO.getSex()), + mapDAORawDataStrainToRawDataStrain(cTO.getStrainId() == null ? + ConditionDAO.STRAIN_ROOT_ID : cTO.getStrainId()), + Optional.ofNullable(speMap.get(cTO.getSpeciesId())).orElseThrow( + () -> new IllegalStateException("Species not found: " + + cTO.getSpeciesId()))) + )) + ); + } + + + + //************************************************************************* + // METHODS PERFORMING THE QUERIES TO THE DAOs + //************************************************************************* + /** + * Perform query to retrieve expressed calls without the post-processing of + * the results returned by {@code DAO}s. + * + * @param geneIds A {@code Collection} of {@code Integer}s that are the Bgee IDs of the genes + * for which to return the {@code RawExpressionCallTO}s. + * @param rawCallDAO The {@code RawExpressionCallDAO} to use to retrieve {@code RawExpressionCallTO}s + * from data source. + * @return The {@code Stream} of {@code RawExpressionCallTO}s. + */ + + + //************************************************************************* + // METHODS PROPAGATION: from CallTOs to propagated Calls + //************************************************************************* + + + + /** + * Merge a {@code Set} of {@code PipelineCallData} into one {@code ExpressionCallData}. + * + * @param dataType A {@code DataType} that is the data type of {@code pipelineCallData}. + * @param pipelineCallData A {@code Set} of {@code PipelineCallData} to be used to + * build the {@code ExpressionCallData}. + * on propagated data. + */ + private static Condition mapRawDataConditionToCondition(RawDataCondition rawCond) { + log.traceEntry("{}", rawCond); + if (rawCond == null) { + return log.traceExit((Condition) null); + } + //All the elements must be non-null, otherwise the propagation will end up + //with not comparable conditions between elements mapped to the root + //and element mapped to null. + assert rawCond.getAnatEntity() != null; + assert rawCond.getDevStage() != null; + assert rawCond.getCellType() != null; + assert rawCond.getSex() != null; + assert rawCond.getStrain() != null; + AnatEntity anatEntityToUse = rawCond.getAnatEntity(); + //Quick and dirty blacklisting of "unknown" terms, we remap them to the root of the anatEntities + if (UNKNOWN_ANAT_ENTITY_IDS.contains(anatEntityToUse.getId())) { + anatEntityToUse = ROOT_ANAT_ENTITY; + } + return log.traceExit(new Condition(anatEntityToUse, rawCond.getDevStage(), + rawCond.getCellType(), mapRawDataSexToSex(rawCond.getSex()), + mapRawDataStrainToStrain(rawCond.getStrain()), rawCond.getSpecies())); + } + + + private static String mapDAORawDataStrainToRawDataStrain(String daoStrain) { + log.traceEntry("{}", daoStrain); + if (StringUtils.isBlank(daoStrain)) { + return log.traceExit((String) null); + } + return log.traceExit(daoStrain); + } +} \ No newline at end of file diff --git a/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/GenerateDiffExprFile.java b/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/GenerateDiffExprFile.java index 896bcca77..8420459ac 100644 --- a/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/GenerateDiffExprFile.java +++ b/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/GenerateDiffExprFile.java @@ -1,1130 +1,1130 @@ -package org.bgee.pipeline.expression.downloadfile; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bgee.model.dao.api.exception.DAOException; -import org.bgee.model.dao.api.expressiondata.call.CallDAO.CallTO.DataState; -import org.bgee.model.dao.api.expressiondata.call.DiffExpressionCallDAO; -import org.bgee.model.dao.api.expressiondata.call.DiffExpressionCallDAO.DiffExpressionCallTO; -import org.bgee.model.dao.api.expressiondata.call.DiffExpressionCallDAO.DiffExpressionCallTO.ComparisonFactor; -import org.bgee.model.dao.api.expressiondata.call.DiffExpressionCallDAO.DiffExpressionCallTO.DiffExprCallType; -import org.bgee.model.dao.mysql.connector.MySQLDAOManager; -import org.bgee.model.file.SpeciesDownloadFile.Category; -import org.bgee.pipeline.BgeeDBUtils; -import org.bgee.pipeline.CommandRunner; -import org.bgee.pipeline.Utils; -import org.supercsv.cellprocessor.constraint.DMinMax; -import org.supercsv.cellprocessor.constraint.IsElementOf; -import org.supercsv.cellprocessor.constraint.LMinMax; -import org.supercsv.cellprocessor.constraint.NotNull; -import org.supercsv.cellprocessor.constraint.StrNotNullOrEmpty; -import org.supercsv.cellprocessor.ift.CellProcessor; -import org.supercsv.io.CsvMapWriter; -import org.supercsv.io.ICsvMapWriter; - -/** - * Class used to generate differential expression TSV download files (simple and advanced files) - * from the Bgee database. - * - * @author Valentine Rech de Laval - * @version Bgee 13 - * @since Bgee 13 - */ -//TODO: stop using these awful Maps, use a BeanReader/BeanWriter instead, -//see org.bgee.pipeline.annotations.SimilarityAnnotationUtils -public class GenerateDiffExprFile extends GenerateDownloadFile { - - /** - * {@code Logger} of the class. - */ - private final static Logger log = LogManager.getLogger(GenerateDiffExprFile.class.getName()); - - /** - * A {@code String} that is the name of the column containing best p-value using Affymetrix, - * in the download file. - */ - public final static String AFFYMETRIX_P_VALUE_COLUMN_NAME = "Affymetrix best supporting p-value"; - /** - * A {@code String} that is the name of the column containing the number of analysis using - * Affymetrix data where the same call is found, in the download file. - */ - //XXX: maybe we should also provide number of probesets, not only number of analysis - public final static String AFFYMETRIX_CONSISTENT_DEA_COUNT_COLUMN_NAME = - "Affymetrix analysis count supporting Affymetrix call"; - /** - * A {@code String} that is the name of the column containing the number of analysis using - * Affymetrix data where a different call is found, in the download file. - */ - //XXX: maybe we should also provide number of probesets, not only number of analysis - public final static String AFFYMETRIX_INCONSISTENT_DEA_COUNT_COLUMN_NAME = - "Affymetrix analysis count in conflict with Affymetrix call"; - /** - * A {@code String} that is the name of the column containing best p-value using RNA-Seq, - * in the download file. - */ - public final static String RNASEQ_P_VALUE_COLUMN_NAME = "RNA-Seq best supporting p-value"; - /** - * A {@code String} that is the name of the column containing the number of analysis using - * RNA-Seq data where the same call is found, in the download file. - */ - public final static String RNASEQ_CONSISTENT_DEA_COUNT_COLUMN_NAME = - "RNA-Seq analysis count supporting RNA-Seq call"; - /** - * A {@code String} that is the name of the column containing the number of analysis using - * RNA-Seq data where a different call is found, in the download file. - */ - public final static String RNASEQ_INCONSISTENT_DEA_COUNT_COLUMN_NAME = - "RNA-Seq analysis count in conflict with RNA-Seq call"; - - /** - * A {@code String} that is the name of the column containing merged differential expressions - * from different data types, in the download file. - */ - public final static String DIFFEXPRESSION_COLUMN_NAME = "Differential expression"; - - /** - * An {@code Enum} used to define the possible differential expression file types to be - * generated, as class arguments. - *
      - *
    • {@code DIFF_EXPR_ANATOMY_SIMPLE}: differential expression based on comparison - * of several anatomical entities at a same - * (broad) developmental stage, - * in a simple download file. - *
    • {@code DIFF_EXPR_ANATOMY_COMPLETE}: differential expression based on comparison - * of several anatomical entities at a same - * (broad) developmental stage, - * in an advanced download file. - *
    • {@code DIFF_EXPR_DEVELOPMENT_SIMPLE}: differential expression based on comparison - * of a same anatomical entity at different - * developmental stages, - * in a simple download file. - *
    • {@code DIFF_EXPR_DEVELOPMENT_COMPLETE}: differential expression based on comparison - * of a same anatomical entity at different - * developmental stages, - * in an advanced download file. - *
    - * - * @author Valentine Rech de Laval - * @version Bgee 13 - * @since Bgee 13 - */ - public enum SingleSpDiffExprFileType implements DiffExprFileType { - DIFF_EXPR_ANATOMY_SIMPLE(Category.DIFF_EXPR_ANAT_SIMPLE, - true, ComparisonFactor.ANATOMY), - DIFF_EXPR_ANATOMY_COMPLETE(Category.DIFF_EXPR_ANAT_COMPLETE, - false, ComparisonFactor.ANATOMY), - DIFF_EXPR_DEVELOPMENT_SIMPLE(Category.DIFF_EXPR_DEV_SIMPLE, - true, ComparisonFactor.DEVELOPMENT), - DIFF_EXPR_DEVELOPMENT_COMPLETE(Category.DIFF_EXPR_DEV_COMPLETE, - false, ComparisonFactor.DEVELOPMENT); - - /** - * A {@code Category} that is the category of files of this type. - */ - private final Category category; - - /** - * A {@code boolean} defining whether this {@code DiffExprFileType} is a simple file type. - */ - private final boolean simpleFileType; - - /** - * A {@code ComparisonFactor} defining what is the compared experimental factor that - * generated the differential expression calls. - */ - //XXX: I find it a bit weird to use the ComparisonFactor of DiffExpressionCallTO at this point, - //because it is not a class related to a DAO... - private final ComparisonFactor comparisonFactor; - - /** - * Constructor providing the {@code Category} of this {@code DiffExprFileType}, - * a {@code boolean} defining whether this {@code DiffExprFileType} is a simple file type, - * and a {@code ComparisonFactor} defining what is the experimental factor compared - * that generated the differential expression calls. - * - * @param category A {@code Category} corresponding to this - * {@code DiffExprFileType}. - * @param isSimpleFileType A {@code boolean} defining whether this - * {@code DiffExprFileType} is a simple file type. - * @param comparisonFactor A {@code ComparisonFactor} defining what is the - * experimental factor compared that generated the - * differential expression calls. - */ - private SingleSpDiffExprFileType(Category category, boolean isSimpleFileType, - ComparisonFactor comparisonFactor) { - this.category = category; - this.simpleFileType = isSimpleFileType; - this.comparisonFactor = comparisonFactor; - } - - @Override - public boolean isSimpleFileType() { - return this.simpleFileType; - } - @Override - public ComparisonFactor getComparisonFactor() { - return this.comparisonFactor; - } - @Override - public Category getCategory() { - return this.category; - } - @Override - public String getStringRepresentation() { - return this.category.getStringRepresentation(); - } - } - - /** - * An {@code Enum} used to define, for each data type (Affymetrix and RNA-Seq), - * as well as for the summary column, the data state of the call. - *
      - *
    • {@code NO_DATA}: means that the call has never been observed - * for the related data type. - *
    • {@code NOT_EXPRESSED}: means that the related gene was never seen - * as 'expressed' in any of the samples used - * in the analysis for the related data type, - * it was then not tested for differential expression. - *
    • {@code OVER_EXPRESSION}: over-expressed calls. - *
    • {@code UNDER_EXPRESSION}: under-expressed calls. - *
    • {@code NOT_DIFF_EXPRESSION}: means that the gene was tested for differential - * expression, but no significant fold change observed. - *
    • {@code WEAK_AMBIGUITY}: different data types are not completely coherent: a data - * type says over or under-expressed, while the other says - * 'not differentially expressed'; or a data type says - * over-expressed, while the other data type says 'not - * expressed'. - *
    • {@code STRONG_AMBIGUITY}: different data types are not coherent: a data type says over - * or under-expressed, while the other data says the opposite. - *
    - * - * @author Valentine Rech de Laval - * @version Bgee 13 - * @since Bgee 13 - */ - //TODO: what level of ambiguity of 'no diff expressed' vs. 'not expressed'? no ambiguity? - //lower qual? - //TODO: actually, for weak ambiguity, shouldn't we provide the direction of the diff expression? - //There is kind of a "winning" call in case of weak ambiguity - public enum DiffExpressionData { - NO_DATA("no data"), NOT_EXPRESSED("not expressed"), OVER_EXPRESSION("over-expression"), - UNDER_EXPRESSION("under-expression"), NOT_DIFF_EXPRESSION("no diff expression"), - WEAK_AMBIGUITY(GenerateDownloadFile.WEAK_AMBIGUITY), - STRONG_AMBIGUITY(GenerateDownloadFile.STRONG_AMBIGUITY); - - private final String stringRepresentation; - - /** - * Constructor providing the {@code String} representation - * of this {@code DiffExpressionData}. - * - * @param stringRepresentation A {@code String} corresponding to this - * {@code DiffExpressionData}. - */ - private DiffExpressionData(String stringRepresentation) { - this.stringRepresentation = stringRepresentation; - } - - public String getStringRepresentation() { - return this.stringRepresentation; - } - - @Override - public String toString() { - return this.getStringRepresentation(); - } - } - - /** - * Main method to trigger the generate differential expression TSV download files (simple and - * advanced) from Bgee database. Parameters that must be provided in order in {@code args} are: - *
      - *
    1. a list of NCBI species IDs (for instance, {@code 9606} for human) that will be used to - * generate download files, separated by the {@code String} {@link CommandRunner#LIST_SEPARATOR}. - * If an empty list is provided (see {@link CommandRunner#EMPTY_LIST}), all species - * contained in database will be used. - *
    2. a list of files types that will be generated ('diffexpr-anatomy-simple' for - * {@link SingleSpDiffExprFileType DIFF_EXPR_ANATOMY_SIMPLE}, 'diffexpr-anatomy-complete' for - * {@link SingleSpDiffExprFileType DIFF_EXPR_ANATOMY_COMPLETE}, 'diffexpr-development-simple' - * for {@link SingleSpDiffExprFileType DIFF_EXPR_DEVELOPMENT_SIMPLE}, and - * 'diffexpr-development-complete' for - * {@link SingleSpDiffExprFileType DIFF_EXPR_DEVELOPMENT_COMPLETE}), separated by the - * {@code String} {@link CommandRunner#LIST_SEPARATOR}. If an empty list is provided - * (see {@link CommandRunner#EMPTY_LIST}), all possible file types will be generated. - *
    3. the directory path that will be used to generate download files. - *
    - * - * @param args An {@code Array} of {@code String}s containing the requested parameters. - * @throws IllegalArgumentException If incorrect parameters were provided. - * @throws IOException If an error occurred while trying to write generated files. - */ - public static void main(String[] args) throws IllegalArgumentException, IOException { - log.entry((Object[]) args); - - int expectedArgLength = 3; - if (args.length != expectedArgLength) { - throw log.throwing(new IllegalArgumentException( - "Incorrect number of arguments provided, expected " + - expectedArgLength + " arguments, " + args.length + " provided.")); - } - - GenerateDiffExprFile generator = new GenerateDiffExprFile( - CommandRunner.parseListArgumentAsInt(args[0]), - GenerateDownloadFile.convertToFileTypes( - CommandRunner.parseListArgument(args[1]), SingleSpDiffExprFileType.class), - args[2]); - generator.generateDiffExprFiles(); - log.traceExit(); - } - /** - * Default constructor. - */ - //suppress warning as this default constructor should not be used. - @SuppressWarnings("unused") - private GenerateDiffExprFile() { - this(null, null, null, null); - } - /** - * Constructor providing parameters to generate files, and using the default {@code DAOManager}. - * - * @param speciesIds A {@code List} of {@code String}s that are the IDs of species - * we want to generate data for. If {@code null} or empty, all species - * are used. - * @param fileTypes A {@code Set} of {@code DiffExprFileType}s that are the types - * of files we want to generate. If {@code null} or empty, - * all {@code DiffExprFileType}s are generated . - * @param directory A {@code String} that is the directory where to store files. - * @throws IllegalArgumentException If {@code directory} is {@code null} or blank. - */ - public GenerateDiffExprFile(List speciesIds, Set fileTypes, - String directory) throws IllegalArgumentException { - this(null, speciesIds, fileTypes, directory); - } - /** - * Constructor providing the {@code MySQLDAOManager} that will be used by - * this object to perform queries to the database. This is useful for unit testing. - * - * @param manager the {@code MySQLDAOManager} to use. - * @param speciesIds A {@code List} of {@code String}s that are the IDs of species - * we want to generate data for. If {@code null} or empty, all species - * are used. - * @param fileTypes A {@code Set} of {@code DiffExprFileType}s that are the types - * of files we want to generate. If {@code null} or empty, - * all {@code DiffExprFileType}s are generated . - * @param directory A {@code String} that is the directory where to store files. - * @throws IllegalArgumentException If {@code directory} is {@code null} or blank. - */ - public GenerateDiffExprFile(MySQLDAOManager manager, List speciesIds, - Set fileTypes, String directory) - throws IllegalArgumentException { - super(manager, speciesIds, fileTypes, directory); - } - - /** - * Generate differential expression files, for the species and file types - * provided at instantiation, in the directory provided at instantiation. - * - * @throws IOException If an error occurred while trying to write generated files. - */ - //TODO: add OMA node ID in complete files - public void generateDiffExprFiles() throws IOException { - log.entry(this.speciesIds, this.fileTypes, this.directory); - - Set setSpecies = new HashSet<>(); - if (this.speciesIds != null) { - setSpecies = new HashSet<>(this.speciesIds); - } - - // Check user input, retrieve info for generating file names - // Retrieve species names and IDs (all species names if speciesIds is null or empty) - // FIXME give a service to checkAndGetLatinNamesBySpeciesIds - Map speciesNamesForFilesByIds = - Utils.checkAndGetLatinNamesBySpeciesIds(setSpecies, null); - assert speciesNamesForFilesByIds.size() >= setSpecies.size(); - - // If no file types are given by user, we set all file types - if (this.fileTypes == null || this.fileTypes.isEmpty()) { - this.fileTypes = EnumSet.allOf(SingleSpDiffExprFileType.class); - } - - // Retrieve gene names, stage names, anat. entity names, once for all species - Map geneNamesByIds = - BgeeDBUtils.getGeneNamesByIds(setSpecies, this.getGeneDAO()); - Map stageNamesByIds = - BgeeDBUtils.getStageNamesByIds(setSpecies, this.getStageDAO()); - Map anatEntityNamesByIds = - BgeeDBUtils.getAnatEntityNamesByIds(setSpecies, this.getAnatEntityDAO()); - - // Split file types according to comparison factor - Map> factorsToFileTypes = - new HashMap>(); - for (FileType fileType: this.fileTypes) { - Set types = factorsToFileTypes.get( - ((SingleSpDiffExprFileType) fileType).getComparisonFactor()); - if (types == null) { - types = EnumSet.noneOf(SingleSpDiffExprFileType.class); - factorsToFileTypes.put( - ((SingleSpDiffExprFileType) fileType).getComparisonFactor(), types); - } - types.add((SingleSpDiffExprFileType) fileType); - } - - // Generate differential expression files, species by species. - for (int speciesId: speciesNamesForFilesByIds.keySet()) { - log.info("Start generating of differential expresion files for the species {}...", - speciesId); - - try { - //generate files grouped by ComparisonFactor (the queries are not the same - //depending on the comparison factor) - for (Set groupedFileTypes: factorsToFileTypes.values()) { - this.generateDiffExprFiles( - speciesNamesForFilesByIds.get(speciesId), groupedFileTypes, speciesId, - geneNamesByIds, stageNamesByIds, anatEntityNamesByIds); - } - } finally { - //close connection to database between each species, to avoid idle connection reset - this.getManager().releaseResources(); - } - log.info("Done generating of differential expresion files for the species {}.", - speciesId); - } - - log.traceExit(); - } - - /** - * Generate download files containing differential expression calls - * for a single species and a single comparison factor. - * This method is responsible for retrieving data from the data source, and then - * to write them into files. Files are written in directory provided at instantiation. - *

    - * Note that all {@code DiffExprFileType}s in {@code fileTypes} should have the same value - * returned by {@code getComparisonFactor}, otherwise an {@code IllegalArgumentException} - * is thrown. This is because the queries used are not the same for different - * comparison factors. For several comparison factors, you must call this method - * several times. - * - * @param fileNamePrefix A {@code String} to be used as a prefix of the names - * of the generated files (usually containing the species name). - * @param fileTypes A {@code Set} of {@code DiffExprFileType}s that are the file - * types to be generated, with equal comparison factors, - * as returned by {@link SingleSpDiffExprFileType#getComparisonFactor()}. - * @param speciesId A {@code String} that is the ID of the species for which - * files are being generated. - * @param geneNamesByIds A {@code Map} where keys are {@code String}s corresponding to - * gene IDs, the associated values being {@code String}s - * corresponding to gene names. - * @param stageNamesByIds A {@code Map} where keys are {@code String}s corresponding to - * stage IDs, the associated values being {@code String}s - * corresponding to stage names. - * @param anatEntityNamesByIds A {@code Map} where keys are {@code String}s corresponding to - * anatomical entity IDs, the associated values being - * {@code String}s corresponding to anatomical entity names. - * @throws IOException If an error occurred while trying to write the - * {@code outputFile}. - * @throws IllegalArgumentException If incorrect {@code DiffExprFileType}s provided. - */ - private void generateDiffExprFiles(String fileNamePrefix, - Set fileTypes, int speciesId, - Map geneNamesByIds, Map stageNamesByIds, - Map anatEntityNamesByIds) throws IOException, IllegalArgumentException { - log.entry(this.directory, fileNamePrefix, fileTypes, speciesId, - geneNamesByIds, stageNamesByIds, anatEntityNamesByIds); - - log.debug("Start generating download files for the species {} and file types {}...", - speciesId, fileTypes); - - if (fileTypes == null || fileTypes.isEmpty()) { - throw log.throwing(new IllegalArgumentException("No provided file types to be generated")); - } - - // We check that all file types have the same comparison factor and retrieve informations: - // comparison factor and if there is an advanced file to be generated. - //TODO: accept more than one comparison factor over all possible file types, - //if we generate multi-species diff expression files over DEVELOPMENT. - //the expression query should be performed once per comparison factor - boolean generateCompleteFile = false; - ComparisonFactor factor = null; - for (SingleSpDiffExprFileType fileType: fileTypes) { - if (factor == null) { - factor = fileType.getComparisonFactor(); - } else if (!fileType.getComparisonFactor().equals(factor)) { - throw log.throwing(new IllegalArgumentException( - "All file types do not have the same comparison factor: " + fileTypes)); - } - if (!fileType.isSimpleFileType()) { - generateCompleteFile = true; - } - } - assert factor != null; - - //******************************** - // RETRIEVE DATA FROM DATA SOURCE - //******************************** - Set speciesFilter = new HashSet<>(); - speciesFilter.add(speciesId); - - //Load differential expression calls. - List diffExprTOs = - this.loadDiffExprCalls(speciesFilter, factor, generateCompleteFile); - - log.trace("Done retrieving data for differential expression files for the species {}.", - speciesId); - - //TODO test no generated file with absolutely no data in it. - if (diffExprTOs.isEmpty()) { - log.trace("No data retrieved for differential expression files for the species {} and file types {}...", - speciesId, fileTypes); - return; - } - //**************************** - // PRODUCE AND WRITE DATA - //**************************** - log.trace("Start generating and writing file content for species {} and file types {}...", - speciesId, fileTypes); - - //now, we write all requested differential expression files at once. This way, we will - //generate the data only once, and we will not have to store them in memory (the memory - //usage could be huge). - //XXX: well, you do store them in memory here, all CallTOs are loaded into a List. - //Should we retrieve the DAOResultSet instead, and make the query use an ORDER BY? - //(This would require to implement a same mechanism as DAO#setAttributes - //for setting ORDER BY clause - it seems the best way to go, rather than storing - //results in memory just by laziness of creating this mechanism :p) - - //OK, first we allow to store file names, writers, etc, associated to a DiffExprFileType, - //for the catch and finally clauses. - Map generatedFileNames = new HashMap(); - - //we will write results in temporary files that we will rename at the end - //if everything is correct - String tmpExtension = ".tmp"; - - //in order to close all writers in a finally clause - Map writersUsed = - new HashMap(); - try { - //************************** - // OPEN FILES, CREATE WRITERS, WRITE HEADERS - //************************** - Map processors = - new HashMap(); - Map headers = - new HashMap(); - - for (SingleSpDiffExprFileType fileType: fileTypes) { - assert fileType.getComparisonFactor().equals(factor); - - String[] fileTypeHeaders = this.generateDiffExprFileHeader(fileType); - headers.put(fileType, fileTypeHeaders); - CellProcessor[] fileTypeProcessors = - this.generateDiffExprFileCellProcessors(fileType, fileTypeHeaders); - processors.put(fileType, fileTypeProcessors); - - //Create file name - String fileName = this.formatString(fileNamePrefix + "_" + - fileType.getStringRepresentation() + EXTENSION); - generatedFileNames.put(fileType, fileName); - - //write in temp file - File file = new File(this.directory, fileName + tmpExtension); - //override any existing file - if (file.exists()) { - file.delete(); - } - - //create writer and write header - ICsvMapWriter mapWriter = new CsvMapWriter(new FileWriter(file), - Utils.getCsvPreferenceWithQuote(this.generateQuoteMode(fileTypeHeaders))); - mapWriter.writeHeader(fileTypeHeaders); - writersUsed.put(fileType, mapWriter); - } - - //**************************** - // WRITE ROWS - //**************************** - // Now, we write the rows in all files - this.writeDiffExprRows(geneNamesByIds, stageNamesByIds, anatEntityNamesByIds, - writersUsed, processors, headers, diffExprTOs); - - } catch (Exception e) { - this.deleteTempFiles(generatedFileNames, tmpExtension); - throw e; - } finally { - for (ICsvMapWriter writer: writersUsed.values()) { - writer.close(); - } - } - //now, if everything went fine, we rename the temporary files - this.renameTempFiles(generatedFileNames, tmpExtension); - - log.traceExit(); - } - - /** - * Retrieves all differential expression calls for the requested species from the Bgee data - * source. - * - * @param speciesIds A {@code Set} of {@code String}s that are the IDs of species - * allowing to filter the expression calls to retrieve. - * @param factor A {@code ComparisonFactor}s that is the comparison factor - * allowing to filter the calls to use. - * @param generateAdvancedFile A {@code boolean} defining whether data for an advanced - * differential expression file are necessary. - * @return A {@code List} of {@code DiffExpressionCallTO}s that are - * all differential expression calls for the requested species. - * @throws DAOException If an error occurred while getting the data from the Bgee data source. - */ - private List loadDiffExprCalls(Set speciesIds, - ComparisonFactor factor, boolean generateAdvancedFile) throws DAOException { - log.entry(speciesIds, factor, generateAdvancedFile); - - log.debug("Start retrieving differential expression calls for the species IDs {}...", - speciesIds); - - DiffExpressionCallDAO dao = this.getDiffExpressionCallDAO(); - //do not retrieve the internal diff. expression IDs - dao.setAttributes(EnumSet.complementOf(EnumSet.of(DiffExpressionCallDAO.Attribute.ID))); - -// DiffExpressionCallParams params = new DiffExpressionCallParams(); -// params.addAllSpeciesIds(speciesIds); -// params.setComparisonFactor(factor); -// // If the advanced file won't be generated, we do not retrieve calls without at least -// // one data type with over- or under-expression. -// if (!generateAdvancedFile) { -// params.setSatisfyAllCallTypeConditions(false); -// params.setIncludeAffymetrixTypes(true); -// params.addAllAffymetrixDiffExprCallTypes( -// EnumSet.of(DiffExprCallType.OVER_EXPRESSED, DiffExprCallType.UNDER_EXPRESSED)); -// params.setIncludeRNASeqTypes(true); -// params.addAllRNASeqDiffExprCallTypes( -// EnumSet.of(DiffExprCallType.OVER_EXPRESSED, DiffExprCallType.UNDER_EXPRESSED)); +//package org.bgee.pipeline.expression.downloadfile; +// +//import java.io.File; +//import java.io.FileWriter; +//import java.io.IOException; +//import java.util.ArrayList; +//import java.util.Collections; +//import java.util.EnumSet; +//import java.util.HashMap; +//import java.util.HashSet; +//import java.util.List; +//import java.util.Map; +//import java.util.Map.Entry; +//import java.util.Set; +// +//import org.apache.logging.log4j.LogManager; +//import org.apache.logging.log4j.Logger; +//import org.bgee.model.dao.api.exception.DAOException; +//import org.bgee.model.dao.api.expressiondata.call.CallDAO.CallTO.DataState; +//import org.bgee.model.dao.api.expressiondata.call.DiffExpressionCallDAO; +//import org.bgee.model.dao.api.expressiondata.call.DiffExpressionCallDAO.DiffExpressionCallTO; +//import org.bgee.model.dao.api.expressiondata.call.DiffExpressionCallDAO.DiffExpressionCallTO.ComparisonFactor; +//import org.bgee.model.dao.api.expressiondata.call.DiffExpressionCallDAO.DiffExpressionCallTO.DiffExprCallType; +//import org.bgee.model.dao.mysql.connector.MySQLDAOManager; +//import org.bgee.model.file.SpeciesDownloadFile.Category; +//import org.bgee.pipeline.BgeeDBUtils; +//import org.bgee.pipeline.CommandRunner; +//import org.bgee.pipeline.Utils; +//import org.supercsv.cellprocessor.constraint.DMinMax; +//import org.supercsv.cellprocessor.constraint.IsElementOf; +//import org.supercsv.cellprocessor.constraint.LMinMax; +//import org.supercsv.cellprocessor.constraint.NotNull; +//import org.supercsv.cellprocessor.constraint.StrNotNullOrEmpty; +//import org.supercsv.cellprocessor.ift.CellProcessor; +//import org.supercsv.io.CsvMapWriter; +//import org.supercsv.io.ICsvMapWriter; +// +///** +// * Class used to generate differential expression TSV download files (simple and advanced files) +// * from the Bgee database. +// * +// * @author Valentine Rech de Laval +// * @version Bgee 13 +// * @since Bgee 13 +// */ +////TODO: stop using these awful Maps, use a BeanReader/BeanWriter instead, +////see org.bgee.pipeline.annotations.SimilarityAnnotationUtils +//public class GenerateDiffExprFile extends GenerateDownloadFile { +// +// /** +// * {@code Logger} of the class. +// */ +// private final static Logger log = LogManager.getLogger(GenerateDiffExprFile.class.getName()); +// +// /** +// * A {@code String} that is the name of the column containing best p-value using Affymetrix, +// * in the download file. +// */ +// public final static String AFFYMETRIX_P_VALUE_COLUMN_NAME = "Affymetrix best supporting p-value"; +// /** +// * A {@code String} that is the name of the column containing the number of analysis using +// * Affymetrix data where the same call is found, in the download file. +// */ +// //XXX: maybe we should also provide number of probesets, not only number of analysis +// public final static String AFFYMETRIX_CONSISTENT_DEA_COUNT_COLUMN_NAME = +// "Affymetrix analysis count supporting Affymetrix call"; +// /** +// * A {@code String} that is the name of the column containing the number of analysis using +// * Affymetrix data where a different call is found, in the download file. +// */ +// //XXX: maybe we should also provide number of probesets, not only number of analysis +// public final static String AFFYMETRIX_INCONSISTENT_DEA_COUNT_COLUMN_NAME = +// "Affymetrix analysis count in conflict with Affymetrix call"; +// /** +// * A {@code String} that is the name of the column containing best p-value using RNA-Seq, +// * in the download file. +// */ +// public final static String RNASEQ_P_VALUE_COLUMN_NAME = "RNA-Seq best supporting p-value"; +// /** +// * A {@code String} that is the name of the column containing the number of analysis using +// * RNA-Seq data where the same call is found, in the download file. +// */ +// public final static String RNASEQ_CONSISTENT_DEA_COUNT_COLUMN_NAME = +// "RNA-Seq analysis count supporting RNA-Seq call"; +// /** +// * A {@code String} that is the name of the column containing the number of analysis using +// * RNA-Seq data where a different call is found, in the download file. +// */ +// public final static String RNASEQ_INCONSISTENT_DEA_COUNT_COLUMN_NAME = +// "RNA-Seq analysis count in conflict with RNA-Seq call"; +// +// /** +// * A {@code String} that is the name of the column containing merged differential expressions +// * from different data types, in the download file. +// */ +// public final static String DIFFEXPRESSION_COLUMN_NAME = "Differential expression"; +// +// /** +// * An {@code Enum} used to define the possible differential expression file types to be +// * generated, as class arguments. +// *

      +// *
    • {@code DIFF_EXPR_ANATOMY_SIMPLE}: differential expression based on comparison +// * of several anatomical entities at a same +// * (broad) developmental stage, +// * in a simple download file. +// *
    • {@code DIFF_EXPR_ANATOMY_COMPLETE}: differential expression based on comparison +// * of several anatomical entities at a same +// * (broad) developmental stage, +// * in an advanced download file. +// *
    • {@code DIFF_EXPR_DEVELOPMENT_SIMPLE}: differential expression based on comparison +// * of a same anatomical entity at different +// * developmental stages, +// * in a simple download file. +// *
    • {@code DIFF_EXPR_DEVELOPMENT_COMPLETE}: differential expression based on comparison +// * of a same anatomical entity at different +// * developmental stages, +// * in an advanced download file. +// *
    +// * +// * @author Valentine Rech de Laval +// * @version Bgee 13 +// * @since Bgee 13 +// */ +// public enum SingleSpDiffExprFileType implements DiffExprFileType { +// DIFF_EXPR_ANATOMY_SIMPLE(Category.DIFF_EXPR_ANAT_SIMPLE, +// true, ComparisonFactor.ANATOMY), +// DIFF_EXPR_ANATOMY_COMPLETE(Category.DIFF_EXPR_ANAT_COMPLETE, +// false, ComparisonFactor.ANATOMY), +// DIFF_EXPR_DEVELOPMENT_SIMPLE(Category.DIFF_EXPR_DEV_SIMPLE, +// true, ComparisonFactor.DEVELOPMENT), +// DIFF_EXPR_DEVELOPMENT_COMPLETE(Category.DIFF_EXPR_DEV_COMPLETE, +// false, ComparisonFactor.DEVELOPMENT); +// +// /** +// * A {@code Category} that is the category of files of this type. +// */ +// private final Category category; +// +// /** +// * A {@code boolean} defining whether this {@code DiffExprFileType} is a simple file type. +// */ +// private final boolean simpleFileType; +// +// /** +// * A {@code ComparisonFactor} defining what is the compared experimental factor that +// * generated the differential expression calls. +// */ +// //XXX: I find it a bit weird to use the ComparisonFactor of DiffExpressionCallTO at this point, +// //because it is not a class related to a DAO... +// private final ComparisonFactor comparisonFactor; +// +// /** +// * Constructor providing the {@code Category} of this {@code DiffExprFileType}, +// * a {@code boolean} defining whether this {@code DiffExprFileType} is a simple file type, +// * and a {@code ComparisonFactor} defining what is the experimental factor compared +// * that generated the differential expression calls. +// * +// * @param category A {@code Category} corresponding to this +// * {@code DiffExprFileType}. +// * @param isSimpleFileType A {@code boolean} defining whether this +// * {@code DiffExprFileType} is a simple file type. +// * @param comparisonFactor A {@code ComparisonFactor} defining what is the +// * experimental factor compared that generated the +// * differential expression calls. +// */ +// private SingleSpDiffExprFileType(Category category, boolean isSimpleFileType, +// ComparisonFactor comparisonFactor) { +// this.category = category; +// this.simpleFileType = isSimpleFileType; +// this.comparisonFactor = comparisonFactor; +// } +// +// @Override +// public boolean isSimpleFileType() { +// return this.simpleFileType; +// } +// @Override +// public ComparisonFactor getComparisonFactor() { +// return this.comparisonFactor; +// } +// @Override +// public Category getCategory() { +// return this.category; +// } +// @Override +// public String getStringRepresentation() { +// return this.category.getStringRepresentation(); +// } +// } +// +// /** +// * An {@code Enum} used to define, for each data type (Affymetrix and RNA-Seq), +// * as well as for the summary column, the data state of the call. +// *
      +// *
    • {@code NO_DATA}: means that the call has never been observed +// * for the related data type. +// *
    • {@code NOT_EXPRESSED}: means that the related gene was never seen +// * as 'expressed' in any of the samples used +// * in the analysis for the related data type, +// * it was then not tested for differential expression. +// *
    • {@code OVER_EXPRESSION}: over-expressed calls. +// *
    • {@code UNDER_EXPRESSION}: under-expressed calls. +// *
    • {@code NOT_DIFF_EXPRESSION}: means that the gene was tested for differential +// * expression, but no significant fold change observed. +// *
    • {@code WEAK_AMBIGUITY}: different data types are not completely coherent: a data +// * type says over or under-expressed, while the other says +// * 'not differentially expressed'; or a data type says +// * over-expressed, while the other data type says 'not +// * expressed'. +// *
    • {@code STRONG_AMBIGUITY}: different data types are not coherent: a data type says over +// * or under-expressed, while the other data says the opposite. +// *
    +// * +// * @author Valentine Rech de Laval +// * @version Bgee 13 +// * @since Bgee 13 +// */ +// //TODO: what level of ambiguity of 'no diff expressed' vs. 'not expressed'? no ambiguity? +// //lower qual? +// //TODO: actually, for weak ambiguity, shouldn't we provide the direction of the diff expression? +// //There is kind of a "winning" call in case of weak ambiguity +// public enum DiffExpressionData { +// NO_DATA("no data"), NOT_EXPRESSED("not expressed"), OVER_EXPRESSION("over-expression"), +// UNDER_EXPRESSION("under-expression"), NOT_DIFF_EXPRESSION("no diff expression"), +// WEAK_AMBIGUITY(GenerateDownloadFile.WEAK_AMBIGUITY), +// STRONG_AMBIGUITY(GenerateDownloadFile.STRONG_AMBIGUITY); +// +// private final String stringRepresentation; +// +// /** +// * Constructor providing the {@code String} representation +// * of this {@code DiffExpressionData}. +// * +// * @param stringRepresentation A {@code String} corresponding to this +// * {@code DiffExpressionData}. +// */ +// private DiffExpressionData(String stringRepresentation) { +// this.stringRepresentation = stringRepresentation; +// } +// +// public String getStringRepresentation() { +// return this.stringRepresentation; +// } +// +// @Override +// public String toString() { +// return this.getStringRepresentation(); +// } +// } +// +// /** +// * Main method to trigger the generate differential expression TSV download files (simple and +// * advanced) from Bgee database. Parameters that must be provided in order in {@code args} are: +// *
      +// *
    1. a list of NCBI species IDs (for instance, {@code 9606} for human) that will be used to +// * generate download files, separated by the {@code String} {@link CommandRunner#LIST_SEPARATOR}. +// * If an empty list is provided (see {@link CommandRunner#EMPTY_LIST}), all species +// * contained in database will be used. +// *
    2. a list of files types that will be generated ('diffexpr-anatomy-simple' for +// * {@link SingleSpDiffExprFileType DIFF_EXPR_ANATOMY_SIMPLE}, 'diffexpr-anatomy-complete' for +// * {@link SingleSpDiffExprFileType DIFF_EXPR_ANATOMY_COMPLETE}, 'diffexpr-development-simple' +// * for {@link SingleSpDiffExprFileType DIFF_EXPR_DEVELOPMENT_SIMPLE}, and +// * 'diffexpr-development-complete' for +// * {@link SingleSpDiffExprFileType DIFF_EXPR_DEVELOPMENT_COMPLETE}), separated by the +// * {@code String} {@link CommandRunner#LIST_SEPARATOR}. If an empty list is provided +// * (see {@link CommandRunner#EMPTY_LIST}), all possible file types will be generated. +// *
    3. the directory path that will be used to generate download files. +// *
    +// * +// * @param args An {@code Array} of {@code String}s containing the requested parameters. +// * @throws IllegalArgumentException If incorrect parameters were provided. +// * @throws IOException If an error occurred while trying to write generated files. +// */ +// public static void main(String[] args) throws IllegalArgumentException, IOException { +// log.entry((Object[]) args); +// +// int expectedArgLength = 3; +// if (args.length != expectedArgLength) { +// throw log.throwing(new IllegalArgumentException( +// "Incorrect number of arguments provided, expected " + +// expectedArgLength + " arguments, " + args.length + " provided.")); // } // -// List diffExpressionCallTOs = -// dao.getDiffExpressionCalls(params).getAllTOs(); - List diffExpressionCallTOs = null; - log.debug("Done retrieving global expression calls, {} calls found", - diffExpressionCallTOs.size()); - - return log.traceExit(diffExpressionCallTOs); - } - - /** - * Generates an {@code Array} of {@code CellProcessor}s used to process - * a differential expression TSV file of type {@code DiffExprFileType}. - * - * @param fileType The {@code DiffExprFileType} of the file to be generated. - * @param header An {@code Array} of {@code String}s representing the names - * of the columns of a differential expression file. - * @return An {@code Array} of {@code CellProcessor}s used to process - * a differential expression file. - * @throw IllegalArgumentException If {@code fileType} is not managed by this method. - */ - private CellProcessor[] generateDiffExprFileCellProcessors( - SingleSpDiffExprFileType fileType, String[] header) { - log.entry(fileType, header); - - List data = new ArrayList(); - for (DiffExpressionData diffExprData: DiffExpressionData.values()) { - data.add(diffExprData.getStringRepresentation()); - } - - List specificTypeQualities = new ArrayList(); - specificTypeQualities.add(GenerateDownloadFile.convertDataStateToString(DataState.HIGHQUALITY)); - specificTypeQualities.add(GenerateDownloadFile.convertDataStateToString(DataState.LOWQUALITY)); - specificTypeQualities.add(GenerateDownloadFile.NA_VALUE); - - List resumeQualities = new ArrayList(); - resumeQualities.add(GenerateDownloadFile.convertDataStateToString(DataState.HIGHQUALITY)); - resumeQualities.add(GenerateDownloadFile.convertDataStateToString(DataState.LOWQUALITY)); - resumeQualities.add(GenerateDownloadFile.NA_VALUE); - - //Then, we build the CellProcessor - CellProcessor[] processors = new CellProcessor[header.length]; - for (int i = 0; i < header.length; i++) { - switch (header[i]) { - // *** CellProcessors common to all file types *** - case GENE_ID_COLUMN_NAME: - case ANAT_ENTITY_ID_COLUMN_NAME: - case ANAT_ENTITY_NAME_COLUMN_NAME: - case STAGE_ID_COLUMN_NAME: - case STAGE_NAME_COLUMN_NAME: - processors[i] = new StrNotNullOrEmpty(); - break; - case GENE_NAME_COLUMN_NAME: - processors[i] = new NotNull(); - break; - case DIFFEXPRESSION_COLUMN_NAME: - processors[i] = new IsElementOf(data); - break; - case QUALITY_COLUMN_NAME: - processors[i] = new IsElementOf(resumeQualities); - break; - } - - // If it was one of the column common to all file types, - // iterate next column name - if (processors[i] != null) { - continue; - } - - if (!fileType.isSimpleFileType()) { - // *** Attributes specific to complete file *** - // TODO: when relaxed in situ will be in the database, uncomment commented lines - switch (header[i]) { - -// case AFFYMETRIX_DATA_COLUMN_NAME: -// case RNASEQ_DATA_COLUMN_NAME: -// processors[i] = new IsElementOf(data); -// break; -// case AFFYMETRIX_QUAL_COLUMN_NAME: -// case RNASEQ_QUAL_COLUMN_NAME: -// processors[i] = new IsElementOf(specificTypeQualities); -// break; -// case AFFYMETRIX_P_VALUE_COLUMN_NAME: -// case RNASEQ_P_VALUE_COLUMN_NAME: -// processors[i] = new DMinMax(0, 1); -// break; -// case AFFYMETRIX_CONSISTENT_DEA_COUNT_COLUMN_NAME: -// case AFFYMETRIX_INCONSISTENT_DEA_COUNT_COLUMN_NAME: -// case RNASEQ_CONSISTENT_DEA_COUNT_COLUMN_NAME: -// case RNASEQ_INCONSISTENT_DEA_COUNT_COLUMN_NAME: -// processors[i] = new LMinMax(0, Long.MAX_VALUE); -// break; - } - } - - if (processors[i] == null) { - throw log.throwing(new IllegalArgumentException("Unrecognized header: " - + header[i] + " for file type: " + fileType.getStringRepresentation())); - } - } - return log.traceExit(processors); - } - - /** - * Generates an {@code Array} of {@code String}s used to generate the header of - * a differential expression TSV file of type {@code fileType}. - * - * @param fileType The {@code DiffExprFileType} of the file to be generated. - * @return An {@code Array} of {@code String}s used to produce the header. - */ - private String[] generateDiffExprFileHeader(SingleSpDiffExprFileType fileType) { - log.entry(fileType); - - String[] headers = null; - int nbColumns = 8; - if (!fileType.isSimpleFileType()) { - nbColumns = 18; - } - headers = new String[nbColumns]; - - // *** Headers common to all file types *** - headers[0] = GENE_ID_COLUMN_NAME; - headers[1] = GENE_NAME_COLUMN_NAME; - headers[2] = ANAT_ENTITY_ID_COLUMN_NAME; - headers[3] = ANAT_ENTITY_NAME_COLUMN_NAME; - headers[4] = STAGE_ID_COLUMN_NAME; - headers[5] = STAGE_NAME_COLUMN_NAME; - headers[6] = DIFFEXPRESSION_COLUMN_NAME; - headers[7] = QUALITY_COLUMN_NAME; - - if (!fileType.isSimpleFileType()) { - // *** Headers specific to complete file *** - headers[8] = AFFYMETRIX_DATA_COLUMN_NAME; - headers[9] = AFFYMETRIX_QUAL_COLUMN_NAME; - headers[10] = AFFYMETRIX_P_VALUE_COLUMN_NAME; - headers[11] = AFFYMETRIX_CONSISTENT_DEA_COUNT_COLUMN_NAME; - headers[12] = AFFYMETRIX_INCONSISTENT_DEA_COUNT_COLUMN_NAME; - headers[13] = RNASEQ_DATA_COLUMN_NAME; - headers[14] = RNASEQ_QUAL_COLUMN_NAME; - headers[15] = RNASEQ_P_VALUE_COLUMN_NAME; - headers[16] = RNASEQ_CONSISTENT_DEA_COUNT_COLUMN_NAME; - headers[17] = RNASEQ_INCONSISTENT_DEA_COUNT_COLUMN_NAME; - } - - return log.traceExit(headers); - } - - /** - * Generate {@code Array} of {@code boolean}s (one per CSV column) indicating - * whether the quoting of the corresponding column should be forced. - * - * @param headers An {@code Array} of {@code String}s representing the names of the columns. - * @return the {@code Array} of {@code boolean}s (one per CSV column) indicating - * whether quoting of the corresponding column should be forced. - * @see Utils#getCsvPreferenceWithQuote(Object[]) - */ - private boolean[] generateQuoteMode(String[] headers) { - log.entry((Object[]) headers); - - boolean[] quoteMode = new boolean[headers.length]; - for (int i = 0; i < headers.length; i++) { - switch (headers[i]) { +// GenerateDiffExprFile generator = new GenerateDiffExprFile( +// CommandRunner.parseListArgumentAsInt(args[0]), +// GenerateDownloadFile.convertToFileTypes( +// CommandRunner.parseListArgument(args[1]), SingleSpDiffExprFileType.class), +// args[2]); +// generator.generateDiffExprFiles(); +// log.traceExit(); +// } +// /** +// * Default constructor. +// */ +// //suppress warning as this default constructor should not be used. +// @SuppressWarnings("unused") +// private GenerateDiffExprFile() { +// this(null, null, null, null); +// } +// /** +// * Constructor providing parameters to generate files, and using the default {@code DAOManager}. +// * +// * @param speciesIds A {@code List} of {@code String}s that are the IDs of species +// * we want to generate data for. If {@code null} or empty, all species +// * are used. +// * @param fileTypes A {@code Set} of {@code DiffExprFileType}s that are the types +// * of files we want to generate. If {@code null} or empty, +// * all {@code DiffExprFileType}s are generated . +// * @param directory A {@code String} that is the directory where to store files. +// * @throws IllegalArgumentException If {@code directory} is {@code null} or blank. +// */ +// public GenerateDiffExprFile(List speciesIds, Set fileTypes, +// String directory) throws IllegalArgumentException { +// this(null, speciesIds, fileTypes, directory); +// } +// /** +// * Constructor providing the {@code MySQLDAOManager} that will be used by +// * this object to perform queries to the database. This is useful for unit testing. +// * +// * @param manager the {@code MySQLDAOManager} to use. +// * @param speciesIds A {@code List} of {@code String}s that are the IDs of species +// * we want to generate data for. If {@code null} or empty, all species +// * are used. +// * @param fileTypes A {@code Set} of {@code DiffExprFileType}s that are the types +// * of files we want to generate. If {@code null} or empty, +// * all {@code DiffExprFileType}s are generated . +// * @param directory A {@code String} that is the directory where to store files. +// * @throws IllegalArgumentException If {@code directory} is {@code null} or blank. +// */ +// public GenerateDiffExprFile(MySQLDAOManager manager, List speciesIds, +// Set fileTypes, String directory) +// throws IllegalArgumentException { +// super(manager, speciesIds, fileTypes, directory); +// } +// +// /** +// * Generate differential expression files, for the species and file types +// * provided at instantiation, in the directory provided at instantiation. +// * +// * @throws IOException If an error occurred while trying to write generated files. +// */ +// //TODO: add OMA node ID in complete files +// public void generateDiffExprFiles() throws IOException { +// log.entry(this.speciesIds, this.fileTypes, this.directory); +// +// Set setSpecies = new HashSet<>(); +// if (this.speciesIds != null) { +// setSpecies = new HashSet<>(this.speciesIds); +// } +// +// // Check user input, retrieve info for generating file names +// // Retrieve species names and IDs (all species names if speciesIds is null or empty) +// // FIXME give a service to checkAndGetLatinNamesBySpeciesIds +// Map speciesNamesForFilesByIds = +// Utils.checkAndGetLatinNamesBySpeciesIds(setSpecies, null); +// assert speciesNamesForFilesByIds.size() >= setSpecies.size(); +// +// // If no file types are given by user, we set all file types +// if (this.fileTypes == null || this.fileTypes.isEmpty()) { +// this.fileTypes = EnumSet.allOf(SingleSpDiffExprFileType.class); +// } +// +// // Retrieve gene names, stage names, anat. entity names, once for all species +// Map geneNamesByIds = +// BgeeDBUtils.getGeneNamesByIds(setSpecies, this.getGeneDAO()); +// Map stageNamesByIds = +// BgeeDBUtils.getStageNamesByIds(setSpecies, this.getStageDAO()); +// Map anatEntityNamesByIds = +// BgeeDBUtils.getAnatEntityNamesByIds(setSpecies, this.getAnatEntityDAO()); +// +// // Split file types according to comparison factor +// Map> factorsToFileTypes = +// new HashMap>(); +// for (FileType fileType: this.fileTypes) { +// Set types = factorsToFileTypes.get( +// ((SingleSpDiffExprFileType) fileType).getComparisonFactor()); +// if (types == null) { +// types = EnumSet.noneOf(SingleSpDiffExprFileType.class); +// factorsToFileTypes.put( +// ((SingleSpDiffExprFileType) fileType).getComparisonFactor(), types); +// } +// types.add((SingleSpDiffExprFileType) fileType); +// } +// +// // Generate differential expression files, species by species. +// for (int speciesId: speciesNamesForFilesByIds.keySet()) { +// log.info("Start generating of differential expresion files for the species {}...", +// speciesId); +// +// try { +// //generate files grouped by ComparisonFactor (the queries are not the same +// //depending on the comparison factor) +// for (Set groupedFileTypes: factorsToFileTypes.values()) { +// this.generateDiffExprFiles( +// speciesNamesForFilesByIds.get(speciesId), groupedFileTypes, speciesId, +// geneNamesByIds, stageNamesByIds, anatEntityNamesByIds); +// } +// } finally { +// //close connection to database between each species, to avoid idle connection reset +// this.getManager().releaseResources(); +// } +// log.info("Done generating of differential expresion files for the species {}.", +// speciesId); +// } +// +// log.traceExit(); +// } +// +// /** +// * Generate download files containing differential expression calls +// * for a single species and a single comparison factor. +// * This method is responsible for retrieving data from the data source, and then +// * to write them into files. Files are written in directory provided at instantiation. +// *

    +// * Note that all {@code DiffExprFileType}s in {@code fileTypes} should have the same value +// * returned by {@code getComparisonFactor}, otherwise an {@code IllegalArgumentException} +// * is thrown. This is because the queries used are not the same for different +// * comparison factors. For several comparison factors, you must call this method +// * several times. +// * +// * @param fileNamePrefix A {@code String} to be used as a prefix of the names +// * of the generated files (usually containing the species name). +// * @param fileTypes A {@code Set} of {@code DiffExprFileType}s that are the file +// * types to be generated, with equal comparison factors, +// * as returned by {@link SingleSpDiffExprFileType#getComparisonFactor()}. +// * @param speciesId A {@code String} that is the ID of the species for which +// * files are being generated. +// * @param geneNamesByIds A {@code Map} where keys are {@code String}s corresponding to +// * gene IDs, the associated values being {@code String}s +// * corresponding to gene names. +// * @param stageNamesByIds A {@code Map} where keys are {@code String}s corresponding to +// * stage IDs, the associated values being {@code String}s +// * corresponding to stage names. +// * @param anatEntityNamesByIds A {@code Map} where keys are {@code String}s corresponding to +// * anatomical entity IDs, the associated values being +// * {@code String}s corresponding to anatomical entity names. +// * @throws IOException If an error occurred while trying to write the +// * {@code outputFile}. +// * @throws IllegalArgumentException If incorrect {@code DiffExprFileType}s provided. +// */ +// private void generateDiffExprFiles(String fileNamePrefix, +// Set fileTypes, int speciesId, +// Map geneNamesByIds, Map stageNamesByIds, +// Map anatEntityNamesByIds) throws IOException, IllegalArgumentException { +// log.entry(this.directory, fileNamePrefix, fileTypes, speciesId, +// geneNamesByIds, stageNamesByIds, anatEntityNamesByIds); +// +// log.debug("Start generating download files for the species {} and file types {}...", +// speciesId, fileTypes); +// +// if (fileTypes == null || fileTypes.isEmpty()) { +// throw log.throwing(new IllegalArgumentException("No provided file types to be generated")); +// } +// +// // We check that all file types have the same comparison factor and retrieve informations: +// // comparison factor and if there is an advanced file to be generated. +// //TODO: accept more than one comparison factor over all possible file types, +// //if we generate multi-species diff expression files over DEVELOPMENT. +// //the expression query should be performed once per comparison factor +// boolean generateCompleteFile = false; +// ComparisonFactor factor = null; +// for (SingleSpDiffExprFileType fileType: fileTypes) { +// if (factor == null) { +// factor = fileType.getComparisonFactor(); +// } else if (!fileType.getComparisonFactor().equals(factor)) { +// throw log.throwing(new IllegalArgumentException( +// "All file types do not have the same comparison factor: " + fileTypes)); +// } +// if (!fileType.isSimpleFileType()) { +// generateCompleteFile = true; +// } +// } +// assert factor != null; +// +// //******************************** +// // RETRIEVE DATA FROM DATA SOURCE +// //******************************** +// Set speciesFilter = new HashSet<>(); +// speciesFilter.add(speciesId); +// +// //Load differential expression calls. +// List diffExprTOs = +// this.loadDiffExprCalls(speciesFilter, factor, generateCompleteFile); +// +// log.trace("Done retrieving data for differential expression files for the species {}.", +// speciesId); +// +// //TODO test no generated file with absolutely no data in it. +// if (diffExprTOs.isEmpty()) { +// log.trace("No data retrieved for differential expression files for the species {} and file types {}...", +// speciesId, fileTypes); +// return; +// } +// //**************************** +// // PRODUCE AND WRITE DATA +// //**************************** +// log.trace("Start generating and writing file content for species {} and file types {}...", +// speciesId, fileTypes); +// +// //now, we write all requested differential expression files at once. This way, we will +// //generate the data only once, and we will not have to store them in memory (the memory +// //usage could be huge). +// //XXX: well, you do store them in memory here, all CallTOs are loaded into a List. +// //Should we retrieve the DAOResultSet instead, and make the query use an ORDER BY? +// //(This would require to implement a same mechanism as DAO#setAttributes +// //for setting ORDER BY clause - it seems the best way to go, rather than storing +// //results in memory just by laziness of creating this mechanism :p) +// +// //OK, first we allow to store file names, writers, etc, associated to a DiffExprFileType, +// //for the catch and finally clauses. +// Map generatedFileNames = new HashMap(); +// +// //we will write results in temporary files that we will rename at the end +// //if everything is correct +// String tmpExtension = ".tmp"; +// +// //in order to close all writers in a finally clause +// Map writersUsed = +// new HashMap(); +// try { +// //************************** +// // OPEN FILES, CREATE WRITERS, WRITE HEADERS +// //************************** +// Map processors = +// new HashMap(); +// Map headers = +// new HashMap(); +// +// for (SingleSpDiffExprFileType fileType: fileTypes) { +// assert fileType.getComparisonFactor().equals(factor); +// +// String[] fileTypeHeaders = this.generateDiffExprFileHeader(fileType); +// headers.put(fileType, fileTypeHeaders); +// CellProcessor[] fileTypeProcessors = +// this.generateDiffExprFileCellProcessors(fileType, fileTypeHeaders); +// processors.put(fileType, fileTypeProcessors); +// +// //Create file name +// String fileName = this.formatString(fileNamePrefix + "_" + +// fileType.getStringRepresentation() + EXTENSION); +// generatedFileNames.put(fileType, fileName); +// +// //write in temp file +// File file = new File(this.directory, fileName + tmpExtension); +// //override any existing file +// if (file.exists()) { +// file.delete(); +// } +// +// //create writer and write header +// ICsvMapWriter mapWriter = new CsvMapWriter(new FileWriter(file), +// Utils.getCsvPreferenceWithQuote(this.generateQuoteMode(fileTypeHeaders))); +// mapWriter.writeHeader(fileTypeHeaders); +// writersUsed.put(fileType, mapWriter); +// } +// +// //**************************** +// // WRITE ROWS +// //**************************** +// // Now, we write the rows in all files +// this.writeDiffExprRows(geneNamesByIds, stageNamesByIds, anatEntityNamesByIds, +// writersUsed, processors, headers, diffExprTOs); +// +// } catch (Exception e) { +// this.deleteTempFiles(generatedFileNames, tmpExtension); +// throw e; +// } finally { +// for (ICsvMapWriter writer: writersUsed.values()) { +// writer.close(); +// } +// } +// //now, if everything went fine, we rename the temporary files +// this.renameTempFiles(generatedFileNames, tmpExtension); +// +// log.traceExit(); +// } +// +// /** +// * Retrieves all differential expression calls for the requested species from the Bgee data +// * source. +// * +// * @param speciesIds A {@code Set} of {@code String}s that are the IDs of species +// * allowing to filter the expression calls to retrieve. +// * @param factor A {@code ComparisonFactor}s that is the comparison factor +// * allowing to filter the calls to use. +// * @param generateAdvancedFile A {@code boolean} defining whether data for an advanced +// * differential expression file are necessary. +// * @return A {@code List} of {@code DiffExpressionCallTO}s that are +// * all differential expression calls for the requested species. +// * @throws DAOException If an error occurred while getting the data from the Bgee data source. +// */ +// private List loadDiffExprCalls(Set speciesIds, +// ComparisonFactor factor, boolean generateAdvancedFile) throws DAOException { +// log.entry(speciesIds, factor, generateAdvancedFile); +// +// log.debug("Start retrieving differential expression calls for the species IDs {}...", +// speciesIds); +// +// DiffExpressionCallDAO dao = this.getDiffExpressionCallDAO(); +// //do not retrieve the internal diff. expression IDs +// dao.setAttributes(EnumSet.complementOf(EnumSet.of(DiffExpressionCallDAO.Attribute.ID))); +// +//// DiffExpressionCallParams params = new DiffExpressionCallParams(); +//// params.addAllSpeciesIds(speciesIds); +//// params.setComparisonFactor(factor); +//// // If the advanced file won't be generated, we do not retrieve calls without at least +//// // one data type with over- or under-expression. +//// if (!generateAdvancedFile) { +//// params.setSatisfyAllCallTypeConditions(false); +//// params.setIncludeAffymetrixTypes(true); +//// params.addAllAffymetrixDiffExprCallTypes( +//// EnumSet.of(DiffExprCallType.OVER_EXPRESSED, DiffExprCallType.UNDER_EXPRESSED)); +//// params.setIncludeRNASeqTypes(true); +//// params.addAllRNASeqDiffExprCallTypes( +//// EnumSet.of(DiffExprCallType.OVER_EXPRESSED, DiffExprCallType.UNDER_EXPRESSED)); +//// } +//// +//// List diffExpressionCallTOs = +//// dao.getDiffExpressionCalls(params).getAllTOs(); +// List diffExpressionCallTOs = null; +// log.debug("Done retrieving global expression calls, {} calls found", +// diffExpressionCallTOs.size()); +// +// return log.traceExit(diffExpressionCallTOs); +// } +// +// /** +// * Generates an {@code Array} of {@code CellProcessor}s used to process +// * a differential expression TSV file of type {@code DiffExprFileType}. +// * +// * @param fileType The {@code DiffExprFileType} of the file to be generated. +// * @param header An {@code Array} of {@code String}s representing the names +// * of the columns of a differential expression file. +// * @return An {@code Array} of {@code CellProcessor}s used to process +// * a differential expression file. +// * @throw IllegalArgumentException If {@code fileType} is not managed by this method. +// */ +// private CellProcessor[] generateDiffExprFileCellProcessors( +// SingleSpDiffExprFileType fileType, String[] header) { +// log.entry(fileType, header); +// +// List data = new ArrayList(); +// for (DiffExpressionData diffExprData: DiffExpressionData.values()) { +// data.add(diffExprData.getStringRepresentation()); +// } +// +// List specificTypeQualities = new ArrayList(); +// specificTypeQualities.add(GenerateDownloadFile.convertDataStateToString(DataState.HIGHQUALITY)); +// specificTypeQualities.add(GenerateDownloadFile.convertDataStateToString(DataState.LOWQUALITY)); +// specificTypeQualities.add(GenerateDownloadFile.NA_VALUE); +// +// List resumeQualities = new ArrayList(); +// resumeQualities.add(GenerateDownloadFile.convertDataStateToString(DataState.HIGHQUALITY)); +// resumeQualities.add(GenerateDownloadFile.convertDataStateToString(DataState.LOWQUALITY)); +// resumeQualities.add(GenerateDownloadFile.NA_VALUE); +// +// //Then, we build the CellProcessor +// CellProcessor[] processors = new CellProcessor[header.length]; +// for (int i = 0; i < header.length; i++) { +// switch (header[i]) { +// // *** CellProcessors common to all file types *** // case GENE_ID_COLUMN_NAME: // case ANAT_ENTITY_ID_COLUMN_NAME: +// case ANAT_ENTITY_NAME_COLUMN_NAME: // case STAGE_ID_COLUMN_NAME: +// case STAGE_NAME_COLUMN_NAME: +// processors[i] = new StrNotNullOrEmpty(); +// break; +// case GENE_NAME_COLUMN_NAME: +// processors[i] = new NotNull(); +// break; // case DIFFEXPRESSION_COLUMN_NAME: +// processors[i] = new IsElementOf(data); +// break; // case QUALITY_COLUMN_NAME: -// case AFFYMETRIX_DATA_COLUMN_NAME: -// case AFFYMETRIX_QUAL_COLUMN_NAME: -// case AFFYMETRIX_P_VALUE_COLUMN_NAME: -// case AFFYMETRIX_CONSISTENT_DEA_COUNT_COLUMN_NAME: -// case AFFYMETRIX_INCONSISTENT_DEA_COUNT_COLUMN_NAME: -// case RNASEQ_DATA_COLUMN_NAME: -// case RNASEQ_QUAL_COLUMN_NAME: -// case RNASEQ_P_VALUE_COLUMN_NAME: -// case RNASEQ_CONSISTENT_DEA_COUNT_COLUMN_NAME: -// case RNASEQ_INCONSISTENT_DEA_COUNT_COLUMN_NAME: -// quoteMode[i] = false; +// processors[i] = new IsElementOf(resumeQualities); // break; -// case GENE_NAME_COLUMN_NAME: -// case ANAT_ENTITY_NAME_COLUMN_NAME: -// case STAGE_NAME_COLUMN_NAME: -// quoteMode[i] = true; +// } +// +// // If it was one of the column common to all file types, +// // iterate next column name +// if (processors[i] != null) { +// continue; +// } +// +// if (!fileType.isSimpleFileType()) { +// // *** Attributes specific to complete file *** +// // TODO: when relaxed in situ will be in the database, uncomment commented lines +// switch (header[i]) { +// +//// case AFFYMETRIX_DATA_COLUMN_NAME: +//// case RNASEQ_DATA_COLUMN_NAME: +//// processors[i] = new IsElementOf(data); +//// break; +//// case AFFYMETRIX_QUAL_COLUMN_NAME: +//// case RNASEQ_QUAL_COLUMN_NAME: +//// processors[i] = new IsElementOf(specificTypeQualities); +//// break; +//// case AFFYMETRIX_P_VALUE_COLUMN_NAME: +//// case RNASEQ_P_VALUE_COLUMN_NAME: +//// processors[i] = new DMinMax(0, 1); +//// break; +//// case AFFYMETRIX_CONSISTENT_DEA_COUNT_COLUMN_NAME: +//// case AFFYMETRIX_INCONSISTENT_DEA_COUNT_COLUMN_NAME: +//// case RNASEQ_CONSISTENT_DEA_COUNT_COLUMN_NAME: +//// case RNASEQ_INCONSISTENT_DEA_COUNT_COLUMN_NAME: +//// processors[i] = new LMinMax(0, Long.MAX_VALUE); +//// break; +// } +// } +// +// if (processors[i] == null) { +// throw log.throwing(new IllegalArgumentException("Unrecognized header: " +// + header[i] + " for file type: " + fileType.getStringRepresentation())); +// } +// } +// return log.traceExit(processors); +// } +// +// /** +// * Generates an {@code Array} of {@code String}s used to generate the header of +// * a differential expression TSV file of type {@code fileType}. +// * +// * @param fileType The {@code DiffExprFileType} of the file to be generated. +// * @return An {@code Array} of {@code String}s used to produce the header. +// */ +// private String[] generateDiffExprFileHeader(SingleSpDiffExprFileType fileType) { +// log.entry(fileType); +// +// String[] headers = null; +// int nbColumns = 8; +// if (!fileType.isSimpleFileType()) { +// nbColumns = 18; +// } +// headers = new String[nbColumns]; +// +// // *** Headers common to all file types *** +// headers[0] = GENE_ID_COLUMN_NAME; +// headers[1] = GENE_NAME_COLUMN_NAME; +// headers[2] = ANAT_ENTITY_ID_COLUMN_NAME; +// headers[3] = ANAT_ENTITY_NAME_COLUMN_NAME; +// headers[4] = STAGE_ID_COLUMN_NAME; +// headers[5] = STAGE_NAME_COLUMN_NAME; +// headers[6] = DIFFEXPRESSION_COLUMN_NAME; +// headers[7] = QUALITY_COLUMN_NAME; +// +// if (!fileType.isSimpleFileType()) { +// // *** Headers specific to complete file *** +// headers[8] = AFFYMETRIX_DATA_COLUMN_NAME; +// headers[9] = AFFYMETRIX_QUAL_COLUMN_NAME; +// headers[10] = AFFYMETRIX_P_VALUE_COLUMN_NAME; +// headers[11] = AFFYMETRIX_CONSISTENT_DEA_COUNT_COLUMN_NAME; +// headers[12] = AFFYMETRIX_INCONSISTENT_DEA_COUNT_COLUMN_NAME; +// headers[13] = RNASEQ_DATA_COLUMN_NAME; +// headers[14] = RNASEQ_QUAL_COLUMN_NAME; +// headers[15] = RNASEQ_P_VALUE_COLUMN_NAME; +// headers[16] = RNASEQ_CONSISTENT_DEA_COUNT_COLUMN_NAME; +// headers[17] = RNASEQ_INCONSISTENT_DEA_COUNT_COLUMN_NAME; +// } +// +// return log.traceExit(headers); +// } +// +// /** +// * Generate {@code Array} of {@code boolean}s (one per CSV column) indicating +// * whether the quoting of the corresponding column should be forced. +// * +// * @param headers An {@code Array} of {@code String}s representing the names of the columns. +// * @return the {@code Array} of {@code boolean}s (one per CSV column) indicating +// * whether quoting of the corresponding column should be forced. +// * @see Utils#getCsvPreferenceWithQuote(Object[]) +// */ +// private boolean[] generateQuoteMode(String[] headers) { +// log.entry((Object[]) headers); +// +// boolean[] quoteMode = new boolean[headers.length]; +// for (int i = 0; i < headers.length; i++) { +// switch (headers[i]) { +//// case GENE_ID_COLUMN_NAME: +//// case ANAT_ENTITY_ID_COLUMN_NAME: +//// case STAGE_ID_COLUMN_NAME: +//// case DIFFEXPRESSION_COLUMN_NAME: +//// case QUALITY_COLUMN_NAME: +//// case AFFYMETRIX_DATA_COLUMN_NAME: +//// case AFFYMETRIX_QUAL_COLUMN_NAME: +//// case AFFYMETRIX_P_VALUE_COLUMN_NAME: +//// case AFFYMETRIX_CONSISTENT_DEA_COUNT_COLUMN_NAME: +//// case AFFYMETRIX_INCONSISTENT_DEA_COUNT_COLUMN_NAME: +//// case RNASEQ_DATA_COLUMN_NAME: +//// case RNASEQ_QUAL_COLUMN_NAME: +//// case RNASEQ_P_VALUE_COLUMN_NAME: +//// case RNASEQ_CONSISTENT_DEA_COUNT_COLUMN_NAME: +//// case RNASEQ_INCONSISTENT_DEA_COUNT_COLUMN_NAME: +//// quoteMode[i] = false; +//// break; +//// case GENE_NAME_COLUMN_NAME: +//// case ANAT_ENTITY_NAME_COLUMN_NAME: +//// case STAGE_NAME_COLUMN_NAME: +//// quoteMode[i] = true; +//// break; +// default: +// throw log.throwing(new IllegalArgumentException( +// "Unrecognized header: " + headers[i] + " for diff. expression file.")); +// } +// } +// +// return log.traceExit(quoteMode); +// } +// +// /** +// * Generate rows to be written and write them in a file. This methods will notably use +// * {@code callTOs} to produce information, that is different depending on {@code fileType}. +// * Note that order of elements in {{@code callTOs} will be modified as a result of +// * the call to this method. +// *

    +// * Information that will be generated is provided in the given {@code processors}. +// * +// * @param geneId A {@code String} that is the ID of the gene considered. +// * @param geneNamesByIds A {@code Map} where keys are {@code String}s corresponding to +// * gene IDs, the associated values being {@code String}s +// * corresponding to gene names. +// * @param stageNamesByIds A {@code Map} where keys are {@code String}s corresponding to +// * stage IDs, the associated values being {@code String}s +// * corresponding to stage names. +// * @param anatEntityNamesByIds A {@code Map} where keys are {@code String}s corresponding to +// * anatomical entity IDs, the associated values being +// * {@code String}s corresponding to anatomical entity names. +// * @param processors A {@code Map} where keys are {@code DiffExprFileType}s +// * corresponding to which type of file should be generated, the +// * associated values being an {@code Array} of +// * {@code CellProcessor}s used to process a file. +// * @param headers A {@code Map} where keys are {@code DiffExprFileType}s +// * corresponding to which type of file should be generated, the +// * associated values being an {@code Array} of {@code String}s +// * used to produce the header. +// * @param writersUsed A {@code Map} where keys are {@code DiffExprFileType}s +// * corresponding to which type of file should be generated, the +// * associated values being {@code ICsvMapWriter}s corresponding to +// * the writers. +// * @param allCallTOs A {@code List} of {@code DiffExpressionCallTO}s that are +// * calls to be written. Elements in this {@code List} will be +// * re-ordered. +// * @throws IOException If an error occurred while trying to write the {@code outputFile}. +// */ +// private void writeDiffExprRows(Map geneNamesByIds, +// Map stageNamesByIds, Map anatEntityNamesByIds, +// Map writersUsed, +// Map processors, +// Map headers, List allCallTOs) +// throws IOException { +// log.entry(geneNamesByIds, stageNamesByIds, anatEntityNamesByIds, writersUsed, +// processors, headers, allCallTOs); +// +// +// for (Entry writerFileType: writersUsed.entrySet()) { +// // We order TOs, according to the values returned by the methods {@code CallTO#getGeneId()}, +// // {@code CallTO#getAnatEntityId()}, and {@code CallTO#getStageId()}. +// // we do not copy the List to save memory, so the provided argument will be modified. +// // We do not order in the same way depending on the comparison factor. +// Boolean orderByAnatomy = null; +// if (writerFileType.getKey().getComparisonFactor().equals(ComparisonFactor.ANATOMY)) { +// //ComparisonFactor = anatomy means that we compared different organs +// //at a same stage, so we want to group the organs by stage, thus, ordering +// //by stage first. +// orderByAnatomy = false; +// } else if (writerFileType.getKey().getComparisonFactor().equals( +// ComparisonFactor.DEVELOPMENT)) { +// //ComparisonFactor = development means that we compared a same organ +// //at different stages, so we want to group by organs +// orderByAnatomy = true; +// } else { +// throw log.throwing(new AssertionError("Unsupported ComparisonFactor.")); +// } +// //FIXME: to reactivate +// //Collections.sort(allCallTOs, new CallTOComparator(orderByAnatomy)); +// +// Map row = null; +// try { +// int callCount = 0; +// int callTotalCount = allCallTOs.size(); +// for (DiffExpressionCallTO callTO: allCallTOs) { +// callCount++; +// if (log.isDebugEnabled() && callCount % 10000 == 0) { +// log.debug("Iterating call {} over {}", callCount, callTotalCount); +// } +// row = this.generateDiffExprRow(geneNamesByIds, stageNamesByIds, +// anatEntityNamesByIds, callTO, writerFileType.getKey()); +// if (row != null) { +// log.trace("Write row: {} - using writer: {}", row, +// writerFileType.getValue()); +// writerFileType.getValue().write(row, +// headers.get(writerFileType.getKey()), +// processors.get(writerFileType.getKey())); +// } +// } +// } catch (IllegalArgumentException e) { +// //any IllegalArgumentException thrown by generateExprRow should come +// //from a problem in the data, thus from an illegal state +// throw log.throwing(new IllegalStateException("Incorrect data state", e)); +// } +// } +// log.traceExit(); +// } +// /** +// * Generate a row to be written in a differential expression download file. This methods will +// * notably use {@code to} to produce differential expression information, that is different +// * depending on {@code fileType}. The results are returned as a {@code Map}; it can be +// * {@code null} if the {@code DiffExpressionCallTO} provided do not allow to generate +// * information to be included in the file of the given {@code DiffExprFileType}. +// *

    +// *

      +// *
    • information that will be generated in any case: entries with keys equal to +// * {@link #GENE_ID_COLUMN_NAME}, {@link #GENE_NAME_COLUMN_NAME}, +// * {@link #STAGE_ID_COLUMN_NAME}, {@link #STAGE_NAME_COLUMN_NAME}, +// * {@link #ANAT_ENTITY_ID_COLUMN_NAME}, {@link #ANAT_ENTITY_NAME_COLUMN_NAME}, +// * {@link #DIFFEXPRESSION_COLUMN_NAME}, {@link #QUALITY_COLUMN_NAME}. +// *
    • information generated for files of the type +// * {@link SingleSpDiffExprFileType DIFF_EXPR_ANATOMY_COMPLETE} or +// * {@link SingleSpDiffExprFileType DIFF_EXPR_DEVELOPMENT_COMPLETE}: +// * entries with keys equal to {@link #AFFYMETRIX_DATA_COLUMN_NAME}, +// * {@link #AFFYMETRIX_CALL_QUALITY_COLUMN_NAME}, {@link #AFFYMETRIX_P_VALUE_COLUMN_NAME}, +// * {@link #AFFYMETRIX_CONSISTENT_DEA_COUNT_COLUMN_NAME}, +// * {@link #AFFYMETRIX_INCONSISTENT_DEA_COUNT_COLUMN_NAME}, {@link #RNASEQ_DATA_COLUMN_NAME}, +// * {@link #RNASEQ_CALL_QUALITY_COLUMN_NAME}, {@link #RNASEQ_P_VALUE_COLUMN_NAME}, +// * {@link #RNASEQ_CONSISTENT_DEA_COUNT_COLUMN_NAME} +// * {@link #RNASEQ_INCONSISTENT_DEA_COUNT_COLUMN_NAME}. +// *
    +// * +// * @param geneNamesByIds A {@code Map} where keys are {@code String}s corresponding to +// * gene IDs, the associated values being {@code String}s +// * corresponding to gene names. +// * @param stageNamesByIds A {@code Map} where keys are {@code String}s corresponding to +// * stage IDs, the associated values being {@code String}s +// * corresponding to stage names. +// * @param anatEntityNamesByIds A {@code Map} where keys are {@code String}s corresponding to +// * anatomical entity IDs, the associated values being +// * {@code String}s corresponding to anatomical entity names. +// * @param to A {@code DiffExpressionCallTO} that is the call to be written. +// * @param fileType The {@code DiffExprFileType} defining which type of file should +// * be generated. +// * @return A {@code Map} containing the generated information. {@code null} +// * if no information should be generated for the provided +// * {@code fileType}. +// * @throws IllegalArgumentException If the {@code DiffExpressionCallTO} provided +// * provides inconsistent data. +// */ +// private Map generateDiffExprRow(Map geneNamesByIds, +// Map stageNamesByIds, Map anatEntityNamesByIds, +// DiffExpressionCallTO to, SingleSpDiffExprFileType fileType) +// throws IllegalArgumentException { +// log.entry(geneNamesByIds, stageNamesByIds, anatEntityNamesByIds, to, fileType); +// +// Map row = new HashMap(); +// +// // ******************************** +// // Set IDs and names +// // ******************************** +// //FIXME: to reactivate +//// this.addIdsAndNames(row, to.getBgeeGeneId(), geneNamesByIds.get(to.getBgeeGeneId()), +//// to.getAnatEntityId(), anatEntityNamesByIds.get(to.getAnatEntityId()), +//// to.getStageId(), stageNamesByIds.get(to.getStageId())); +// +// // ******************************** +// // Set simple file columns +// // ******************************** +// boolean dataAdded = this.addDiffExprCallMergedDataToRow(fileType, row, +// to.getDiffExprCallTypeAffymetrix(), to.getAffymetrixData(), +// to.getDiffExprCallTypeRNASeq(), to.getRNASeqData()); +// +// if (!dataAdded) { +// return log.traceExit((Map) null); +// } +// +// // ******************************** +// // Set advance file columns +// // ******************************** +// if (!fileType.isSimpleFileType()) { +// row.put(AFFYMETRIX_DATA_COLUMN_NAME, +// to.getDiffExprCallTypeAffymetrix().getStringRepresentation()); +// row.put(AFFYMETRIX_QUAL_COLUMN_NAME, +// GenerateDownloadFile.convertDataStateToString(to.getAffymetrixData())); +// +// row.put(AFFYMETRIX_P_VALUE_COLUMN_NAME, String.valueOf(to.getBestPValueAffymetrix())); +// row.put(AFFYMETRIX_CONSISTENT_DEA_COUNT_COLUMN_NAME, +// String.valueOf(to.getConsistentDEACountAffymetrix())); +// row.put(AFFYMETRIX_INCONSISTENT_DEA_COUNT_COLUMN_NAME, +// String.valueOf(to.getInconsistentDEACountAffymetrix())); +// +// row.put(RNASEQ_DATA_COLUMN_NAME, +// to.getDiffExprCallTypeRNASeq().getStringRepresentation()); +// row.put(RNASEQ_QUAL_COLUMN_NAME, +// GenerateDownloadFile.convertDataStateToString(to.getRNASeqData())); +// +// row.put(RNASEQ_P_VALUE_COLUMN_NAME, String.valueOf(to.getBestPValueRNASeq())); +// row.put(RNASEQ_CONSISTENT_DEA_COUNT_COLUMN_NAME, +// String.valueOf(to.getConsistentDEACountRNASeq())); +// row.put(RNASEQ_INCONSISTENT_DEA_COUNT_COLUMN_NAME, +// String.valueOf(to.getInconsistentDEACountRNASeq())); +// } +// +// return log.traceExit(row); +// } +// +// /** +// * Add to the provided {@code row} merged {@code DataState}s and qualities. +// *

    +// * The provided {@code Map} will be modified. +// * +// * @param fileType The {@code DiffExprFileType} defining which type of file should be +// * generated. +// * @param row A {@code Map} where keys are {@code String}s that are column names, +// * the associated values being a {@code String} that is the value +// * for the call. +// * @param affymetrixType A {@code DiffExprCallType} that is the differential expression call +// * type of Affymetrix data to be merged with {@code rnaSeqType}. +// * @param affymetrixQuality A {@code DataState} that is the Affymetrix call quality to be merged +// * with {@code rnaSeqQuality}. +// * @param rnaSeqType A {@code DiffExprCallType} that is the differential expression call +// * type of RNA-Seq data to be merged with {@code affymetrixType}. +// * @param rnaSeqQuality A {@code DataState} that is the RNA-Seq call quality to be merged +// * with {@code affymetrixQuality}. +// * @return A {@code boolean} that is {@code true} if data added to {@code row}. +// * @throws IllegalArgumentException If call data are inconsistent (for instance, without any data). +// */ +// private boolean addDiffExprCallMergedDataToRow(SingleSpDiffExprFileType fileType, +// Map row, DiffExprCallType affymetrixType, DataState affymetrixQuality, +// DiffExprCallType rnaSeqType, DataState rnaSeqQuality) throws IllegalArgumentException { +// log.entry(fileType, row, affymetrixType, affymetrixQuality, rnaSeqType, rnaSeqQuality); +// +// DiffExpressionData summary = DiffExpressionData.NO_DATA; +// String quality = GenerateDownloadFile.convertDataStateToString(DataState.NODATA); +// +// Set allType = EnumSet.of(affymetrixType, rnaSeqType); +// +// // Sanity check on data: one call should't be only no data and/or not_expressed data. +// if ((affymetrixType.equals(DiffExprCallType.NOT_EXPRESSED) || +// affymetrixType.equals(DiffExprCallType.NO_DATA)) && +// (rnaSeqType.equals(DiffExprCallType.NOT_EXPRESSED) || +// rnaSeqType.equals(DiffExprCallType.NO_DATA))) { +// throw log.throwing(new IllegalArgumentException("One call should not be only "+ +// DiffExprCallType.NOT_EXPRESSED.getStringRepresentation() + " and/or " + +// DiffExprCallType.NO_DATA.getStringRepresentation())); +// } +// +// // One call containing over- AND under- expression returns STRONG_AMBIGUITY. +// if ((allType.contains(DiffExprCallType.UNDER_EXPRESSED) && +// allType.contains(DiffExprCallType.OVER_EXPRESSED))) { +// summary = DiffExpressionData.STRONG_AMBIGUITY; +// quality = GenerateDownloadFile.convertDataStateToString(DataState.NODATA); +// +// // Both data types are equals or only one is set to 'no data': +// // we choose the data which is not 'no data'. +// } else if (affymetrixType.equals(rnaSeqType) || allType.contains(DiffExprCallType.NO_DATA)) { +// DiffExprCallType type = affymetrixType; +// if (affymetrixType.equals(DiffExprCallType.NO_DATA)) { +// type = rnaSeqType; +// } +// assert !type.equals(DiffExprCallType.NO_DATA); +// +// //store only quality of data different from NO_DATA +// Set allDataQuality = EnumSet.noneOf(DataState.class); +// if (!affymetrixType.equals(DiffExprCallType.NO_DATA)) { +// allDataQuality.add(affymetrixQuality); +// } +// if (!rnaSeqType.equals(DiffExprCallType.NO_DATA)) { +// allDataQuality.add(rnaSeqQuality); +// } +// assert allDataQuality.size() >=1 && allDataQuality.size() <= 2; +// +// switch (type) { +// case OVER_EXPRESSED: +// summary = DiffExpressionData.OVER_EXPRESSION; // break; - default: - throw log.throwing(new IllegalArgumentException( - "Unrecognized header: " + headers[i] + " for diff. expression file.")); - } - } - - return log.traceExit(quoteMode); - } - - /** - * Generate rows to be written and write them in a file. This methods will notably use - * {@code callTOs} to produce information, that is different depending on {@code fileType}. - * Note that order of elements in {{@code callTOs} will be modified as a result of - * the call to this method. - *

    - * Information that will be generated is provided in the given {@code processors}. - * - * @param geneId A {@code String} that is the ID of the gene considered. - * @param geneNamesByIds A {@code Map} where keys are {@code String}s corresponding to - * gene IDs, the associated values being {@code String}s - * corresponding to gene names. - * @param stageNamesByIds A {@code Map} where keys are {@code String}s corresponding to - * stage IDs, the associated values being {@code String}s - * corresponding to stage names. - * @param anatEntityNamesByIds A {@code Map} where keys are {@code String}s corresponding to - * anatomical entity IDs, the associated values being - * {@code String}s corresponding to anatomical entity names. - * @param processors A {@code Map} where keys are {@code DiffExprFileType}s - * corresponding to which type of file should be generated, the - * associated values being an {@code Array} of - * {@code CellProcessor}s used to process a file. - * @param headers A {@code Map} where keys are {@code DiffExprFileType}s - * corresponding to which type of file should be generated, the - * associated values being an {@code Array} of {@code String}s - * used to produce the header. - * @param writersUsed A {@code Map} where keys are {@code DiffExprFileType}s - * corresponding to which type of file should be generated, the - * associated values being {@code ICsvMapWriter}s corresponding to - * the writers. - * @param allCallTOs A {@code List} of {@code DiffExpressionCallTO}s that are - * calls to be written. Elements in this {@code List} will be - * re-ordered. - * @throws IOException If an error occurred while trying to write the {@code outputFile}. - */ - private void writeDiffExprRows(Map geneNamesByIds, - Map stageNamesByIds, Map anatEntityNamesByIds, - Map writersUsed, - Map processors, - Map headers, List allCallTOs) - throws IOException { - log.entry(geneNamesByIds, stageNamesByIds, anatEntityNamesByIds, writersUsed, - processors, headers, allCallTOs); - - - for (Entry writerFileType: writersUsed.entrySet()) { - // We order TOs, according to the values returned by the methods {@code CallTO#getGeneId()}, - // {@code CallTO#getAnatEntityId()}, and {@code CallTO#getStageId()}. - // we do not copy the List to save memory, so the provided argument will be modified. - // We do not order in the same way depending on the comparison factor. - Boolean orderByAnatomy = null; - if (writerFileType.getKey().getComparisonFactor().equals(ComparisonFactor.ANATOMY)) { - //ComparisonFactor = anatomy means that we compared different organs - //at a same stage, so we want to group the organs by stage, thus, ordering - //by stage first. - orderByAnatomy = false; - } else if (writerFileType.getKey().getComparisonFactor().equals( - ComparisonFactor.DEVELOPMENT)) { - //ComparisonFactor = development means that we compared a same organ - //at different stages, so we want to group by organs - orderByAnatomy = true; - } else { - throw log.throwing(new AssertionError("Unsupported ComparisonFactor.")); - } - //FIXME: to reactivate - //Collections.sort(allCallTOs, new CallTOComparator(orderByAnatomy)); - - Map row = null; - try { - int callCount = 0; - int callTotalCount = allCallTOs.size(); - for (DiffExpressionCallTO callTO: allCallTOs) { - callCount++; - if (log.isDebugEnabled() && callCount % 10000 == 0) { - log.debug("Iterating call {} over {}", callCount, callTotalCount); - } - row = this.generateDiffExprRow(geneNamesByIds, stageNamesByIds, - anatEntityNamesByIds, callTO, writerFileType.getKey()); - if (row != null) { - log.trace("Write row: {} - using writer: {}", row, - writerFileType.getValue()); - writerFileType.getValue().write(row, - headers.get(writerFileType.getKey()), - processors.get(writerFileType.getKey())); - } - } - } catch (IllegalArgumentException e) { - //any IllegalArgumentException thrown by generateExprRow should come - //from a problem in the data, thus from an illegal state - throw log.throwing(new IllegalStateException("Incorrect data state", e)); - } - } - log.traceExit(); - } - /** - * Generate a row to be written in a differential expression download file. This methods will - * notably use {@code to} to produce differential expression information, that is different - * depending on {@code fileType}. The results are returned as a {@code Map}; it can be - * {@code null} if the {@code DiffExpressionCallTO} provided do not allow to generate - * information to be included in the file of the given {@code DiffExprFileType}. - *

    - *

      - *
    • information that will be generated in any case: entries with keys equal to - * {@link #GENE_ID_COLUMN_NAME}, {@link #GENE_NAME_COLUMN_NAME}, - * {@link #STAGE_ID_COLUMN_NAME}, {@link #STAGE_NAME_COLUMN_NAME}, - * {@link #ANAT_ENTITY_ID_COLUMN_NAME}, {@link #ANAT_ENTITY_NAME_COLUMN_NAME}, - * {@link #DIFFEXPRESSION_COLUMN_NAME}, {@link #QUALITY_COLUMN_NAME}. - *
    • information generated for files of the type - * {@link SingleSpDiffExprFileType DIFF_EXPR_ANATOMY_COMPLETE} or - * {@link SingleSpDiffExprFileType DIFF_EXPR_DEVELOPMENT_COMPLETE}: - * entries with keys equal to {@link #AFFYMETRIX_DATA_COLUMN_NAME}, - * {@link #AFFYMETRIX_CALL_QUALITY_COLUMN_NAME}, {@link #AFFYMETRIX_P_VALUE_COLUMN_NAME}, - * {@link #AFFYMETRIX_CONSISTENT_DEA_COUNT_COLUMN_NAME}, - * {@link #AFFYMETRIX_INCONSISTENT_DEA_COUNT_COLUMN_NAME}, {@link #RNASEQ_DATA_COLUMN_NAME}, - * {@link #RNASEQ_CALL_QUALITY_COLUMN_NAME}, {@link #RNASEQ_P_VALUE_COLUMN_NAME}, - * {@link #RNASEQ_CONSISTENT_DEA_COUNT_COLUMN_NAME} - * {@link #RNASEQ_INCONSISTENT_DEA_COUNT_COLUMN_NAME}. - *
    - * - * @param geneNamesByIds A {@code Map} where keys are {@code String}s corresponding to - * gene IDs, the associated values being {@code String}s - * corresponding to gene names. - * @param stageNamesByIds A {@code Map} where keys are {@code String}s corresponding to - * stage IDs, the associated values being {@code String}s - * corresponding to stage names. - * @param anatEntityNamesByIds A {@code Map} where keys are {@code String}s corresponding to - * anatomical entity IDs, the associated values being - * {@code String}s corresponding to anatomical entity names. - * @param to A {@code DiffExpressionCallTO} that is the call to be written. - * @param fileType The {@code DiffExprFileType} defining which type of file should - * be generated. - * @return A {@code Map} containing the generated information. {@code null} - * if no information should be generated for the provided - * {@code fileType}. - * @throws IllegalArgumentException If the {@code DiffExpressionCallTO} provided - * provides inconsistent data. - */ - private Map generateDiffExprRow(Map geneNamesByIds, - Map stageNamesByIds, Map anatEntityNamesByIds, - DiffExpressionCallTO to, SingleSpDiffExprFileType fileType) - throws IllegalArgumentException { - log.entry(geneNamesByIds, stageNamesByIds, anatEntityNamesByIds, to, fileType); - - Map row = new HashMap(); - - // ******************************** - // Set IDs and names - // ******************************** - //FIXME: to reactivate -// this.addIdsAndNames(row, to.getBgeeGeneId(), geneNamesByIds.get(to.getBgeeGeneId()), -// to.getAnatEntityId(), anatEntityNamesByIds.get(to.getAnatEntityId()), -// to.getStageId(), stageNamesByIds.get(to.getStageId())); - - // ******************************** - // Set simple file columns - // ******************************** - boolean dataAdded = this.addDiffExprCallMergedDataToRow(fileType, row, - to.getDiffExprCallTypeAffymetrix(), to.getAffymetrixData(), - to.getDiffExprCallTypeRNASeq(), to.getRNASeqData()); - - if (!dataAdded) { - return log.traceExit((Map) null); - } - - // ******************************** - // Set advance file columns - // ******************************** - if (!fileType.isSimpleFileType()) { - row.put(AFFYMETRIX_DATA_COLUMN_NAME, - to.getDiffExprCallTypeAffymetrix().getStringRepresentation()); - row.put(AFFYMETRIX_QUAL_COLUMN_NAME, - GenerateDownloadFile.convertDataStateToString(to.getAffymetrixData())); - - row.put(AFFYMETRIX_P_VALUE_COLUMN_NAME, String.valueOf(to.getBestPValueAffymetrix())); - row.put(AFFYMETRIX_CONSISTENT_DEA_COUNT_COLUMN_NAME, - String.valueOf(to.getConsistentDEACountAffymetrix())); - row.put(AFFYMETRIX_INCONSISTENT_DEA_COUNT_COLUMN_NAME, - String.valueOf(to.getInconsistentDEACountAffymetrix())); - - row.put(RNASEQ_DATA_COLUMN_NAME, - to.getDiffExprCallTypeRNASeq().getStringRepresentation()); - row.put(RNASEQ_QUAL_COLUMN_NAME, - GenerateDownloadFile.convertDataStateToString(to.getRNASeqData())); - - row.put(RNASEQ_P_VALUE_COLUMN_NAME, String.valueOf(to.getBestPValueRNASeq())); - row.put(RNASEQ_CONSISTENT_DEA_COUNT_COLUMN_NAME, - String.valueOf(to.getConsistentDEACountRNASeq())); - row.put(RNASEQ_INCONSISTENT_DEA_COUNT_COLUMN_NAME, - String.valueOf(to.getInconsistentDEACountRNASeq())); - } - - return log.traceExit(row); - } - - /** - * Add to the provided {@code row} merged {@code DataState}s and qualities. - *

    - * The provided {@code Map} will be modified. - * - * @param fileType The {@code DiffExprFileType} defining which type of file should be - * generated. - * @param row A {@code Map} where keys are {@code String}s that are column names, - * the associated values being a {@code String} that is the value - * for the call. - * @param affymetrixType A {@code DiffExprCallType} that is the differential expression call - * type of Affymetrix data to be merged with {@code rnaSeqType}. - * @param affymetrixQuality A {@code DataState} that is the Affymetrix call quality to be merged - * with {@code rnaSeqQuality}. - * @param rnaSeqType A {@code DiffExprCallType} that is the differential expression call - * type of RNA-Seq data to be merged with {@code affymetrixType}. - * @param rnaSeqQuality A {@code DataState} that is the RNA-Seq call quality to be merged - * with {@code affymetrixQuality}. - * @return A {@code boolean} that is {@code true} if data added to {@code row}. - * @throws IllegalArgumentException If call data are inconsistent (for instance, without any data). - */ - private boolean addDiffExprCallMergedDataToRow(SingleSpDiffExprFileType fileType, - Map row, DiffExprCallType affymetrixType, DataState affymetrixQuality, - DiffExprCallType rnaSeqType, DataState rnaSeqQuality) throws IllegalArgumentException { - log.entry(fileType, row, affymetrixType, affymetrixQuality, rnaSeqType, rnaSeqQuality); - - DiffExpressionData summary = DiffExpressionData.NO_DATA; - String quality = GenerateDownloadFile.convertDataStateToString(DataState.NODATA); - - Set allType = EnumSet.of(affymetrixType, rnaSeqType); - - // Sanity check on data: one call should't be only no data and/or not_expressed data. - if ((affymetrixType.equals(DiffExprCallType.NOT_EXPRESSED) || - affymetrixType.equals(DiffExprCallType.NO_DATA)) && - (rnaSeqType.equals(DiffExprCallType.NOT_EXPRESSED) || - rnaSeqType.equals(DiffExprCallType.NO_DATA))) { - throw log.throwing(new IllegalArgumentException("One call should not be only "+ - DiffExprCallType.NOT_EXPRESSED.getStringRepresentation() + " and/or " + - DiffExprCallType.NO_DATA.getStringRepresentation())); - } - - // One call containing over- AND under- expression returns STRONG_AMBIGUITY. - if ((allType.contains(DiffExprCallType.UNDER_EXPRESSED) && - allType.contains(DiffExprCallType.OVER_EXPRESSED))) { - summary = DiffExpressionData.STRONG_AMBIGUITY; - quality = GenerateDownloadFile.convertDataStateToString(DataState.NODATA); - - // Both data types are equals or only one is set to 'no data': - // we choose the data which is not 'no data'. - } else if (affymetrixType.equals(rnaSeqType) || allType.contains(DiffExprCallType.NO_DATA)) { - DiffExprCallType type = affymetrixType; - if (affymetrixType.equals(DiffExprCallType.NO_DATA)) { - type = rnaSeqType; - } - assert !type.equals(DiffExprCallType.NO_DATA); - - //store only quality of data different from NO_DATA - Set allDataQuality = EnumSet.noneOf(DataState.class); - if (!affymetrixType.equals(DiffExprCallType.NO_DATA)) { - allDataQuality.add(affymetrixQuality); - } - if (!rnaSeqType.equals(DiffExprCallType.NO_DATA)) { - allDataQuality.add(rnaSeqQuality); - } - assert allDataQuality.size() >=1 && allDataQuality.size() <= 2; - - switch (type) { - case OVER_EXPRESSED: - summary = DiffExpressionData.OVER_EXPRESSION; - break; - case UNDER_EXPRESSED: - summary = DiffExpressionData.UNDER_EXPRESSION; - break; - case NOT_DIFF_EXPRESSED: - summary = DiffExpressionData.NOT_DIFF_EXPRESSION; - // We don't write 'not expressed' calls in simple file, we need to check it - // again because when simple file is generated in same time as advanced file - // we do not filter these calls when retrieving calls from database. - if (fileType.isSimpleFileType()) { - return log.traceExit(false); - } - break; - default: - throw log.throwing(new AssertionError( - "Both DiffExprCallType are set to 'no data' or 'not expressed'")); - } - if (allDataQuality.contains(DataState.HIGHQUALITY)) { - quality = GenerateDownloadFile.convertDataStateToString(DataState.HIGHQUALITY); - } else { - quality = GenerateDownloadFile.convertDataStateToString(DataState.LOWQUALITY); - } - - // All possible cases where the summary is WEAK_AMBIGUITY: - // - NOT_DIFF_EXPRESSED and (OVER_EXPRESSED or UNDER_EXPRESSED) - // - NOT_EXPRESSED and OVER_EXPRESSED - // - NOT_EXPRESSED and NOT_DIFF_EXPRESSED - //XXX: actually, I think that there are no NOT_EXPRESSED case inserted, - //but it doesn't hurt to keep this code - } else if ((allType.contains(DiffExprCallType.NOT_DIFF_EXPRESSED) && - (allType.contains(DiffExprCallType.OVER_EXPRESSED) || - allType.contains(DiffExprCallType.UNDER_EXPRESSED))) || - (allType.contains(DiffExprCallType.NOT_EXPRESSED) && - (allType.contains(DiffExprCallType.OVER_EXPRESSED)) || - allType.contains(DiffExprCallType.NOT_DIFF_EXPRESSED))) { - summary = DiffExpressionData.WEAK_AMBIGUITY; - quality = GenerateDownloadFile.convertDataStateToString(DataState.NODATA); - - // One call containing NOT_EXPRESSED and UNDER_EXPRESSED returns - // UNDER_EXPRESSION with LOWQUALITY - //XXX: actually, I think that there are no NOT_EXPRESSED case inserted, - //but it doesn't hurt to keep this code - } else if (allType.contains(DiffExprCallType.NOT_EXPRESSED) && - allType.contains(DiffExprCallType.UNDER_EXPRESSED)) { - summary = DiffExpressionData.UNDER_EXPRESSION; - quality = GenerateDownloadFile.convertDataStateToString(DataState.LOWQUALITY); - - } else { - throw log.throwing(new AssertionError("All logical conditions should have been checked.")); - } - assert !summary.equals(DiffExpressionData.NO_DATA); - - row.put(DIFFEXPRESSION_COLUMN_NAME, summary.getStringRepresentation()); - row.put(QUALITY_COLUMN_NAME, quality); - - return log.traceExit(true); - } -} +// case UNDER_EXPRESSED: +// summary = DiffExpressionData.UNDER_EXPRESSION; +// break; +// case NOT_DIFF_EXPRESSED: +// summary = DiffExpressionData.NOT_DIFF_EXPRESSION; +// // We don't write 'not expressed' calls in simple file, we need to check it +// // again because when simple file is generated in same time as advanced file +// // we do not filter these calls when retrieving calls from database. +// if (fileType.isSimpleFileType()) { +// return log.traceExit(false); +// } +// break; +// default: +// throw log.throwing(new AssertionError( +// "Both DiffExprCallType are set to 'no data' or 'not expressed'")); +// } +// if (allDataQuality.contains(DataState.HIGHQUALITY)) { +// quality = GenerateDownloadFile.convertDataStateToString(DataState.HIGHQUALITY); +// } else { +// quality = GenerateDownloadFile.convertDataStateToString(DataState.LOWQUALITY); +// } +// +// // All possible cases where the summary is WEAK_AMBIGUITY: +// // - NOT_DIFF_EXPRESSED and (OVER_EXPRESSED or UNDER_EXPRESSED) +// // - NOT_EXPRESSED and OVER_EXPRESSED +// // - NOT_EXPRESSED and NOT_DIFF_EXPRESSED +// //XXX: actually, I think that there are no NOT_EXPRESSED case inserted, +// //but it doesn't hurt to keep this code +// } else if ((allType.contains(DiffExprCallType.NOT_DIFF_EXPRESSED) && +// (allType.contains(DiffExprCallType.OVER_EXPRESSED) || +// allType.contains(DiffExprCallType.UNDER_EXPRESSED))) || +// (allType.contains(DiffExprCallType.NOT_EXPRESSED) && +// (allType.contains(DiffExprCallType.OVER_EXPRESSED)) || +// allType.contains(DiffExprCallType.NOT_DIFF_EXPRESSED))) { +// summary = DiffExpressionData.WEAK_AMBIGUITY; +// quality = GenerateDownloadFile.convertDataStateToString(DataState.NODATA); +// +// // One call containing NOT_EXPRESSED and UNDER_EXPRESSED returns +// // UNDER_EXPRESSION with LOWQUALITY +// //XXX: actually, I think that there are no NOT_EXPRESSED case inserted, +// //but it doesn't hurt to keep this code +// } else if (allType.contains(DiffExprCallType.NOT_EXPRESSED) && +// allType.contains(DiffExprCallType.UNDER_EXPRESSED)) { +// summary = DiffExpressionData.UNDER_EXPRESSION; +// quality = GenerateDownloadFile.convertDataStateToString(DataState.LOWQUALITY); +// +// } else { +// throw log.throwing(new AssertionError("All logical conditions should have been checked.")); +// } +// assert !summary.equals(DiffExpressionData.NO_DATA); +// +// row.put(DIFFEXPRESSION_COLUMN_NAME, summary.getStringRepresentation()); +// row.put(QUALITY_COLUMN_NAME, quality); +// +// return log.traceExit(true); +// } +//} diff --git a/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/GenerateDownloadFile.java b/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/GenerateDownloadFile.java index 2167eb2a7..57fc6c790 100644 --- a/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/GenerateDownloadFile.java +++ b/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/GenerateDownloadFile.java @@ -107,116 +107,6 @@ public abstract class GenerateDownloadFile extends MySQLDAOUser { "Descendant observation count "; public final static String SELF_OBSERVATION_COUNT_PREFIX = "Self observation count "; - //TODO: manage all these attributes by generalizing over DataType enum, e.g., - //using a Map> - /** - * A {@code String} that is the name of the column containing expression, no-expression or - * differential expression found with Affymetrix experiment, in the download file. - */ - public final static String AFFYMETRIX_DATA_COLUMN_NAME = - DataType.AFFYMETRIX.getStringRepresentation() + CALL_TYPE_COLUMN_NAME_SUFFIX; - /** - * A {@code String} that is the name of the column containing call quality found with - * Affymetrix experiment, in the download file. - */ - public final static String AFFYMETRIX_QUAL_COLUMN_NAME = - DataType.AFFYMETRIX.getStringRepresentation() + CALL_QUALITY_COLUMN_NAME_SUFFIX; - - /** - * A {@code String} that is the name of the column containing FDR pvalue found with - * Affymetrix experiment, in the download file. - */ - public final static String AFFYMETRIX_FDR_COLUMN_NAME = - DataType.AFFYMETRIX.getStringRepresentation() + FDR_COLUMN_NAME_SUFFIX; - /** - * A {@code String} that is the name of the column containing if an Affymetrix experiment - * is observed, in the download file. - */ - public final static String AFFYMETRIX_OBSERVED_DATA_COLUMN_NAME = - OBSERVED_DATA_COLUMN_NAME_PREFIX + DataType.AFFYMETRIX.getStringRepresentation() + OBSERVED_DATA_COLUMN_NAME_SUFFIX; - /** - * A {@code String} that is the name of the column containing number of pvalues coming from - * self observed Affymetrix data, in the download file. - */ - public final static String AFFYMETRIX_SELF_OBSERVATION_COUNT_COLUMN_NAME = - SELF_OBSERVATION_COUNT_PREFIX + DataType.AFFYMETRIX.getStringRepresentation(); - /** - * A {@code String} that is the name of the column containing number of pvalues coming from - * descendant observed Affymetrix data, in the download file. - */ - public final static String AFFYMETRIX_DESCENDANT_OBSERVATION_COUNT_COLUMN_NAME = - DESCENDANT_OBSERVATION_COUNT_PREFIX + DataType.AFFYMETRIX.getStringRepresentation(); - /** - * A {@code String} that is the name of the column containing - * the expression rank from Affymetrix data in the download file. - */ - public final static String AFFYMETRIX_EXPRESSION_RANK_COLUMN_NAME = DataType.AFFYMETRIX.getStringRepresentation() - + EXPRESSION_RANK_COLUMN_NAME_SUFFIX; - /** - * A {@code String} that is the name of the column containing - * the expression score from Affymetrix data in the download file. - */ - public final static String AFFYMETRIX_EXPRESSION_SCORE_COLUMN_NAME = DataType.AFFYMETRIX.getStringRepresentation() - + EXPRESSION_SCORE_COLUMN_NAME_SUFFIX; - /** - * A {@code String} that is the name of the column containing in the download file - * the weight for Affymetrix data when computing mean rank/score. - */ - public final static String AFFYMETRIX_WEIGHT_COLUMN_NAME = DataType.AFFYMETRIX.getStringRepresentation() - + WEIGHT_COLUMN_NAME_SUFFIX; - - /** - * A {@code String} that is the name of the column containing expression/no-expression found - * with EST experiment, in the download file. - */ - public final static String EST_DATA_COLUMN_NAME = DataType.EST.getStringRepresentation() + CALL_TYPE_COLUMN_NAME_SUFFIX; - /** - * A {@code String} that is the name of the column containing call quality found with - * EST experiment, in the download file. - */ - public final static String EST_QUAL_COLUMN_NAME = DataType.EST.getStringRepresentation() + CALL_QUALITY_COLUMN_NAME_SUFFIX; - /** - * A {@code String} that is the name of the column containing FDR pvalue found with - * EST experiment, in the download file. - */ - public final static String EST_FDR_COLUMN_NAME = DataType.EST.getStringRepresentation() + FDR_COLUMN_NAME_SUFFIX; - /** - * A {@code String} that is the name of the column containing if an EST experiment is observed, - * in the download file. - */ - public final static String EST_OBSERVED_DATA_COLUMN_NAME = - OBSERVED_DATA_COLUMN_NAME_PREFIX + DataType.EST.getStringRepresentation() + OBSERVED_DATA_COLUMN_NAME_SUFFIX; - /** - * A {@code String} that is the name of the column containing number of pvalues coming from - * self observed EST data, in the download file. - */ - public final static String EST_SELF_OBSERVATION_COUNT_COLUMN_NAME = - SELF_OBSERVATION_COUNT_PREFIX + DataType.EST.getStringRepresentation(); - /** - * A {@code String} that is the name of the column containing number of pvalues coming from - * descendant observed EST data, in the download file. - */ - public final static String EST_DESCENDANT_OBSERVATION_COUNT_COLUMN_NAME = - DESCENDANT_OBSERVATION_COUNT_PREFIX + DataType.EST.getStringRepresentation(); - /** - * A {@code String} that is the name of the column containing - * the expression rank from EST data in the download file. - */ - public final static String EST_EXPRESSION_RANK_COLUMN_NAME = DataType.EST.getStringRepresentation() - + EXPRESSION_RANK_COLUMN_NAME_SUFFIX; - /** - * A {@code String} that is the name of the column containing - * the expression score from EST data in the download file. - */ - public final static String EST_EXPRESSION_SCORE_COLUMN_NAME = DataType.EST.getStringRepresentation() - + EXPRESSION_SCORE_COLUMN_NAME_SUFFIX; - /** - * A {@code String} that is the name of the column containing in the download file - * the weight for EST data when computing mean rank/score. - */ - public final static String EST_WEIGHT_COLUMN_NAME = DataType.EST.getStringRepresentation() - + WEIGHT_COLUMN_NAME_SUFFIX; - /** * A {@code String} that is the name of the column containing expression/no-expression * found with in situ experiment, in the download file. diff --git a/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/GenerateExprFile2.java b/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/GenerateExprFile2.java index 5662f6fa3..dd1cbc958 100644 --- a/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/GenerateExprFile2.java +++ b/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/GenerateExprFile2.java @@ -776,23 +776,15 @@ private CellProcessor[] generateExprFileCellProcessors( if (!fileType.isSimpleFileType()) { // *** Attributes specific to complete file *** - if (header[i].equals(AFFYMETRIX_DATA_COLUMN_NAME) || - header[i].equals(EST_DATA_COLUMN_NAME) || - header[i].equals(IN_SITU_DATA_COLUMN_NAME) || + if (header[i].equals(IN_SITU_DATA_COLUMN_NAME) || header[i].equals(RNASEQ_DATA_COLUMN_NAME) || header[i].equals(SC_RNA_SEQ_DATA_COLUMN_NAME)) { processors[i] = new IsElementOf(expressionSummaries); - } else if (header[i].equals(AFFYMETRIX_QUAL_COLUMN_NAME) || - header[i].equals(EST_QUAL_COLUMN_NAME) || - header[i].equals(IN_SITU_QUAL_COLUMN_NAME) || + } else if (header[i].equals(IN_SITU_QUAL_COLUMN_NAME) || header[i].equals(RNASEQ_QUAL_COLUMN_NAME) || header[i].equals(SC_RNA_SEQ_QUAL_COLUMN_NAME)) { processors[i] = new IsElementOf(qualitySummaries); - } else if (header[i].equals(AFFYMETRIX_SELF_OBSERVATION_COUNT_COLUMN_NAME) || - header[i].equals(AFFYMETRIX_DESCENDANT_OBSERVATION_COUNT_COLUMN_NAME) || - header[i].equals(EST_SELF_OBSERVATION_COUNT_COLUMN_NAME) || - header[i].equals(EST_DESCENDANT_OBSERVATION_COUNT_COLUMN_NAME) || - header[i].equals(IN_SITU_SELF_OBSERVATION_COUNT_COLUMN_NAME) || + } else if (header[i].equals(IN_SITU_SELF_OBSERVATION_COUNT_COLUMN_NAME) || header[i].equals(IN_SITU_DESCENDANT_OBSERVATION_COUNT_COLUMN_NAME) || header[i].equals(RNASEQ_SELF_OBSERVATION_COUNT_COLUMN_NAME) || header[i].equals(RNASEQ_DESCENDANT_OBSERVATION_COUNT_COLUMN_NAME) || @@ -801,22 +793,12 @@ private CellProcessor[] generateExprFileCellProcessors( header[i].equals(SELF_OBSERVATION_COUNT_COLUMN_NAME) || header[i].equals(DESCENDANT_OBSERVATION_COUNT_COLUMN_NAME)) { processors[i] = new LMinMax(0, Long.MAX_VALUE); - } else if (header[i].equals(AFFYMETRIX_OBSERVED_DATA_COLUMN_NAME) || - header[i].equals(EST_OBSERVED_DATA_COLUMN_NAME) || - header[i].equals(IN_SITU_OBSERVED_DATA_COLUMN_NAME) || + } else if (header[i].equals(IN_SITU_OBSERVED_DATA_COLUMN_NAME) || header[i].equals(RNASEQ_OBSERVED_DATA_COLUMN_NAME) || header[i].equals(SC_RNA_SEQ_OBSERVED_DATA_COLUMN_NAME) || header[i].equals(INCLUDING_OBSERVED_DATA_COLUMN_NAME)) { processors[i] = new IsElementOf(originValues); - } else if (header[i].equals(AFFYMETRIX_EXPRESSION_SCORE_COLUMN_NAME) || - header[i].equals(AFFYMETRIX_EXPRESSION_RANK_COLUMN_NAME) || - header[i].equals(AFFYMETRIX_WEIGHT_COLUMN_NAME) || - header[i].equals(AFFYMETRIX_FDR_COLUMN_NAME) || - header[i].equals(EST_EXPRESSION_SCORE_COLUMN_NAME) || - header[i].equals(EST_EXPRESSION_RANK_COLUMN_NAME) || - header[i].equals(EST_WEIGHT_COLUMN_NAME) || - header[i].equals(EST_FDR_COLUMN_NAME) || - header[i].equals(IN_SITU_EXPRESSION_SCORE_COLUMN_NAME) || + } else if (header[i].equals(IN_SITU_EXPRESSION_SCORE_COLUMN_NAME) || header[i].equals(IN_SITU_EXPRESSION_RANK_COLUMN_NAME) || header[i].equals(IN_SITU_WEIGHT_COLUMN_NAME) || header[i].equals(IN_SITU_FDR_COLUMN_NAME) || @@ -905,24 +887,6 @@ private String[] generateExprFileHeader(SingleSpExprFileType2 fileType) { headers[idx++] = INCLUDING_OBSERVED_DATA_COLUMN_NAME; headers[idx++] = SELF_OBSERVATION_COUNT_COLUMN_NAME; headers[idx++] = DESCENDANT_OBSERVATION_COUNT_COLUMN_NAME; - headers[idx++] = AFFYMETRIX_DATA_COLUMN_NAME; - headers[idx++] = AFFYMETRIX_QUAL_COLUMN_NAME; - headers[idx++] = AFFYMETRIX_FDR_COLUMN_NAME; - headers[idx++] = AFFYMETRIX_EXPRESSION_SCORE_COLUMN_NAME; - headers[idx++] = AFFYMETRIX_EXPRESSION_RANK_COLUMN_NAME; - headers[idx++] = AFFYMETRIX_WEIGHT_COLUMN_NAME; - headers[idx++] = AFFYMETRIX_OBSERVED_DATA_COLUMN_NAME; - headers[idx++] = AFFYMETRIX_SELF_OBSERVATION_COUNT_COLUMN_NAME; - headers[idx++] = AFFYMETRIX_DESCENDANT_OBSERVATION_COUNT_COLUMN_NAME; - headers[idx++] = EST_DATA_COLUMN_NAME; - headers[idx++] = EST_QUAL_COLUMN_NAME; - headers[idx++] = EST_FDR_COLUMN_NAME; - headers[idx++] = EST_EXPRESSION_SCORE_COLUMN_NAME; - headers[idx++] = EST_EXPRESSION_RANK_COLUMN_NAME; - headers[idx++] = EST_WEIGHT_COLUMN_NAME; - headers[idx++] = EST_OBSERVED_DATA_COLUMN_NAME; - headers[idx++] = EST_SELF_OBSERVATION_COUNT_COLUMN_NAME; - headers[idx++] = EST_DESCENDANT_OBSERVATION_COUNT_COLUMN_NAME; headers[idx++] = IN_SITU_DATA_COLUMN_NAME; headers[idx++] = IN_SITU_QUAL_COLUMN_NAME; headers[idx++] = IN_SITU_FDR_COLUMN_NAME; @@ -1144,25 +1108,6 @@ private boolean[] generateQuoteMode(String[] headers) { headers[i].equals(EXPRESSION_RANK_COLUMN_NAME) || headers[i].equals(EXPRESSION_SCORE_COLUMN_NAME) || headers[i].equals(INCLUDING_OBSERVED_DATA_COLUMN_NAME) || - headers[i].equals(AFFYMETRIX_DATA_COLUMN_NAME) || - headers[i].equals(AFFYMETRIX_QUAL_COLUMN_NAME) || - headers[i].equals(AFFYMETRIX_FDR_COLUMN_NAME) || - headers[i].equals(AFFYMETRIX_EXPRESSION_RANK_COLUMN_NAME) || - headers[i].equals(AFFYMETRIX_EXPRESSION_SCORE_COLUMN_NAME) || - headers[i].equals(AFFYMETRIX_WEIGHT_COLUMN_NAME) || - headers[i].equals(AFFYMETRIX_OBSERVED_DATA_COLUMN_NAME) || - headers[i].equals(AFFYMETRIX_SELF_OBSERVATION_COUNT_COLUMN_NAME) || - headers[i].equals(AFFYMETRIX_DESCENDANT_OBSERVATION_COUNT_COLUMN_NAME) || - - headers[i].equals(EST_DATA_COLUMN_NAME) || - headers[i].equals(EST_QUAL_COLUMN_NAME) || - headers[i].equals(EST_FDR_COLUMN_NAME) || - headers[i].equals(EST_EXPRESSION_RANK_COLUMN_NAME) || - headers[i].equals(EST_EXPRESSION_SCORE_COLUMN_NAME) || - headers[i].equals(EST_WEIGHT_COLUMN_NAME) || - headers[i].equals(EST_OBSERVED_DATA_COLUMN_NAME) || - headers[i].equals(EST_SELF_OBSERVATION_COUNT_COLUMN_NAME) || - headers[i].equals(EST_DESCENDANT_OBSERVATION_COUNT_COLUMN_NAME) || headers[i].equals(IN_SITU_DATA_COLUMN_NAME) || headers[i].equals(IN_SITU_QUAL_COLUMN_NAME) || diff --git a/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/GenerateMultiSpeciesExprFile.java b/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/GenerateMultiSpeciesExprFile.java index 4eabb9f01..17af38ea5 100644 --- a/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/GenerateMultiSpeciesExprFile.java +++ b/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/GenerateMultiSpeciesExprFile.java @@ -932,7 +932,6 @@ private String[] generateHeader(MultiSpExprFileType fileType) STAGE_ID_COLUMN_NAME, STAGE_NAME_COLUMN_NAME, SPECIES_LATIN_NAME_COLUMN_NAME, CIO_ID_COLUMN_NAME, CIO_NAME_ID_COLUMN_NAME, EXPRESSION_COLUMN_NAME, QUALITY_COLUMN_NAME, INCLUDING_OBSERVED_DATA_COLUMN_NAME, - AFFYMETRIX_DATA_COLUMN_NAME, AFFYMETRIX_QUAL_COLUMN_NAME, EST_DATA_COLUMN_NAME, EST_CALL_QUALITY_COLUMN_NAME, INSITU_DATA_COLUMN_NAME, INSITU_CALL_QUALITY_COLUMN_NAME, RNASEQ_DATA_COLUMN_NAME, RNASEQ_QUAL_COLUMN_NAME}); @@ -1036,8 +1035,6 @@ private boolean[] generateQuoteMode(String[] headers) { headers[i].equals(EXPRESSION_COLUMN_NAME) || headers[i].equals(QUALITY_COLUMN_NAME) || headers[i].equals(INCLUDING_OBSERVED_DATA_COLUMN_NAME) || - headers[i].equals(AFFYMETRIX_DATA_COLUMN_NAME) || - headers[i].equals(AFFYMETRIX_QUAL_COLUMN_NAME) || headers[i].equals(EST_DATA_COLUMN_NAME) || headers[i].equals(EST_CALL_QUALITY_COLUMN_NAME) || headers[i].equals(INSITU_DATA_COLUMN_NAME) || @@ -1127,18 +1124,10 @@ private String[] generateFieldMapping(MultiSpExprFileType fileType, String[] hea // *** Attributes specific to complete file *** if (header[i].equals(INCLUDING_OBSERVED_DATA_COLUMN_NAME)) { mapping[i] = "includingObservedData"; - } else if (header[i].equals(AFFYMETRIX_DATA_COLUMN_NAME)) { - mapping[i] = "affymetrixData"; - } else if (header[i].equals(AFFYMETRIX_QUAL_COLUMN_NAME)) { - mapping[i] = "affymetrixCallQuality"; - } else if (header[i].equals(AFFYMETRIX_OBSERVED_DATA_COLUMN_NAME)) { - mapping[i] = "includingAffymetrixObservedData"; } else if (header[i].equals(EST_DATA_COLUMN_NAME)) { mapping[i] = "estData"; } else if (header[i].equals(EST_CALL_QUALITY_COLUMN_NAME)) { mapping[i] = "estCallQuality"; - } else if (header[i].equals(EST_OBSERVED_DATA_COLUMN_NAME)) { - mapping[i] = "includingEstObservedData"; } else if (header[i].equals(INSITU_DATA_COLUMN_NAME)) { mapping[i] = "inSituData"; } else if (header[i].equals(INSITU_CALL_QUALITY_COLUMN_NAME)) { diff --git a/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/GenerateRankFile.java b/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/GenerateRankFile.java index e1d802120..bc3f7d748 100644 --- a/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/GenerateRankFile.java +++ b/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/GenerateRankFile.java @@ -93,8 +93,6 @@ public static class ExpressionCallBean { private final String devStageId; private final String devStageName; private final String formattedRank; - private final boolean affymetrixData; - private final boolean estData; private final boolean inSituData; private final boolean rnaSeqData; private final boolean singleCellRnaSeqFullLengthData; @@ -103,7 +101,7 @@ public static class ExpressionCallBean { public ExpressionCallBean(String geneId, String geneName, String anatEntityId, String anatEntityName, String devStageId, String devStageName, String formattedRank, - boolean affymetrixData, boolean estData, boolean inSituData, boolean rnaSeqData, + boolean inSituData, boolean rnaSeqData, boolean singleCellRnaSeqFullLengthData, boolean redundant, List btoXRefs) { this.geneId = geneId; @@ -113,8 +111,6 @@ public ExpressionCallBean(String geneId, String geneName, String anatEntityId, this.devStageId = devStageId; this.devStageName = devStageName; this.formattedRank = formattedRank; - this.affymetrixData = affymetrixData; - this.estData = estData; this.inSituData = inSituData; this.rnaSeqData = rnaSeqData; this.singleCellRnaSeqFullLengthData = singleCellRnaSeqFullLengthData; @@ -143,12 +139,6 @@ public String getDevStageName() { public String getFormattedRank() { return formattedRank; } - public boolean isAffymetrixData() { - return affymetrixData; - } - public boolean isEstData() { - return estData; - } public boolean isInSituData() { return inSituData; } @@ -839,8 +829,6 @@ private Stream mapCallsToBeans(List singleGe cond.getAnatEntityId(), anatEntity == null? null: anatEntity.getName(), cond.getDevStageId(), devStage == null? null: devStage.getName(), FORMATTER.apply(c.getMeanRank()), - dataTypeToStatus.get(DataType.AFFYMETRIX), - dataTypeToStatus.get(DataType.EST), dataTypeToStatus.get(DataType.IN_SITU), dataTypeToStatus.get(DataType.RNA_SEQ), dataTypeToStatus.get(DataType.SC_RNA_SEQ), diff --git a/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/InsertSpeciesDataGroups.java b/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/InsertSpeciesDataGroups.java index 0dc014616..252b25747 100644 --- a/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/InsertSpeciesDataGroups.java +++ b/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/InsertSpeciesDataGroups.java @@ -600,8 +600,6 @@ // case DIFF_EXPR_DEV_COMPLETE: // case DIFF_EXPR_DEV_SIMPLE: // case ORTHOLOG: -// case AFFY_ANNOT: -// case AFFY_DATA: // case RNASEQ_ANNOT: // case RNASEQ_DATA: // return log.traceExit(DownloadFileTO.CategoryEnum.convertToCategoryEnum( diff --git a/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/collaboration/GenerateOncoMXFile.java b/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/collaboration/GenerateOncoMXFile.java index 733ce1b00..e6a0748c8 100644 --- a/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/collaboration/GenerateOncoMXFile.java +++ b/bgee-pipeline/src/main/java/org/bgee/pipeline/expression/downloadfile/collaboration/GenerateOncoMXFile.java @@ -72,7 +72,7 @@ public class GenerateOncoMXFile { /** * A {@code Set} of {DataType}s used to build the data for OncoMX. */ - private final static Set DATA_TYPES = EnumSet.of(DataType.RNA_SEQ, DataType.AFFYMETRIX); + private final static Set DATA_TYPES = EnumSet.of(DataType.RNA_SEQ); /** * Launches the generation of the files used by OncoMX. diff --git a/bgee-pipeline/src/main/java/org/bgee/pipeline/gene/ParseOrthoXML.java b/bgee-pipeline/src/main/java/org/bgee/pipeline/gene/ParseOrthoXML.java index 13039a49a..23d352339 100644 --- a/bgee-pipeline/src/main/java/org/bgee/pipeline/gene/ParseOrthoXML.java +++ b/bgee-pipeline/src/main/java/org/bgee/pipeline/gene/ParseOrthoXML.java @@ -210,29 +210,29 @@ public ParseOrthoXML(MySQLDAOManager manager) { * by the OrthoXMLReader. * @throws IOException If the mapping file could not be read. */ - public static void main(String[] args) throws IllegalArgumentException, DAOException, - XMLStreamException, XMLParseException, IOException { - log.entry((Object[]) args); - - int expectedArgLengthWithoutMapping = 1; - int expectedArgLengthWithMapping = 2; - if (args.length != expectedArgLengthWithoutMapping && - args.length != expectedArgLengthWithMapping) { - throw log.throwing(new IllegalArgumentException("Incorrect number of " + - "arguments provided, expected " + expectedArgLengthWithoutMapping + - "or " + expectedArgLengthWithMapping + " arguments, " + - args.length + " provided.")); - } - - ParseOrthoXML parser = new ParseOrthoXML(); - if (args.length == expectedArgLengthWithoutMapping) { - parser.parseXML(args[0], null); - } else { - parser.parseXML(args[0], args[1]); - } - - log.traceExit(); - } +// public static void main(String[] args) throws IllegalArgumentException, DAOException, +// XMLStreamException, XMLParseException, IOException { +// log.entry((Object[]) args); +// +// int expectedArgLengthWithoutMapping = 1; +// int expectedArgLengthWithMapping = 2; +// if (args.length != expectedArgLengthWithoutMapping && +// args.length != expectedArgLengthWithMapping) { +// throw log.throwing(new IllegalArgumentException("Incorrect number of " + +// "arguments provided, expected " + expectedArgLengthWithoutMapping + +// "or " + expectedArgLengthWithMapping + " arguments, " + +// args.length + " provided.")); +// } +// +// ParseOrthoXML parser = new ParseOrthoXML(); +// if (args.length == expectedArgLengthWithoutMapping) { +// parser.parseXML(args[0], null); +// } else { +// parser.parseXML(args[0], args[1]); +// } +// +// log.traceExit(); +// } /** * Performs the complete task of reading the Hierarchical Groups orthoxml file and @@ -259,67 +259,67 @@ public static void main(String[] args) throws IllegalArgumentException, DAOExcep * OrthoXMLReader. * @throws IOException If the mapping file could not be read. */ - public void parseXML(String orthoXMLFile, String geneMappingFile) - throws DAOException, XMLStreamException, XMLParseException, IOException { - log.entry(orthoXMLFile, geneMappingFile); - log.info("Start parsing of OrthoXML file..."); - - // First, if provided, we read mapping file save data in a map - Map geneMapping = new HashMap(); - if (geneMappingFile != null) { - geneMapping = this.readMappingFile(geneMappingFile); - } - // Catch any IllegalStateException to wrap it into a IllegalArgumentException - // (a IllegalStateException would be generated because the OrthoXML groups - // loaded from the file would be invalid, so it would be a wrong argument). - try { - // Retrieve gene IDs of the Bgee database to be able to check if OMA genes are - // in Bgee and to update OMAParentNodeId in gene table. - this.loadGeneIdsFromDb(); - - // Retrieve taxon IDs of the Bgee database to be able to check if OMA HOGs - // correspond to taxa present in Bgee. - this.loadTaxonIdsFromDb(); - - // Retrieve species from Bgee that use genome of another species. - this.loadMappingSpeciesToGenomeSpecies(); - - // Construct HierarchicalNodeTOs and GeneTOs - this.generateTOsFromFile(orthoXMLFile, geneMapping); - - // Start a transaction to insert HierarchicalNodeTOs and update GeneTOs - // in the Bgee data source. Note that we do not need to call rollback if - // an error occurs, calling closeDAO will rollback any ongoing transaction. - int nbInsertedGroups = 0, nbUpdatedGenes = 0, nbInsertedGroupToGene = 0; - - this.startTransaction(); - - log.info("Start inserting of hierarchical groups..."); - nbInsertedGroups = this.getHierarchicalGroupDAO() - .insertHierarchicalNodes(this.hierarchicalNodeTOs); - log.info("Done inserting hierarchical groups"); - - log.info("Start updating genes..."); -// nbUpdatedGenes = this.getGeneDAO().updateGenes(this.geneTOs, -// Arrays.asList(GeneDAO.Attribute.OMA_PARENT_NODE_ID)); - log.info("Done updating genes."); - log.info("Start inserting gene to hierarchical group mapping..."); - nbInsertedGroupToGene = this.getHierarchicalGroupDAO() - .insertHierarchicalNodeToGene(this.hierarchicalNodeToGeneTOs); - log.info("Done inserting gene to hierarchical group mapping."); - - this.commit(); - log.info("Done parsing of OrthoXML file: {} hierarchical groups inserted " + - ",{} genes updated, and {} mapping between hierarchical group and genes inserted.", nbInsertedGroups, nbUpdatedGenes, nbInsertedGroupToGene); - } catch (IllegalStateException e) { - log.catching(e); - throw log.throwing(new IllegalArgumentException( - "The OrthoXML file provided is invalid", e)); - } finally { - this.closeDAO(); - } - log.traceExit(); - } +// public void parseXML(String orthoXMLFile, String geneMappingFile) +// throws DAOException, XMLStreamException, XMLParseException, IOException { +// log.entry(orthoXMLFile, geneMappingFile); +// log.info("Start parsing of OrthoXML file..."); +// +// // First, if provided, we read mapping file save data in a map +// Map geneMapping = new HashMap(); +// if (geneMappingFile != null) { +// geneMapping = this.readMappingFile(geneMappingFile); +// } +// // Catch any IllegalStateException to wrap it into a IllegalArgumentException +// // (a IllegalStateException would be generated because the OrthoXML groups +// // loaded from the file would be invalid, so it would be a wrong argument). +// try { +// // Retrieve gene IDs of the Bgee database to be able to check if OMA genes are +// // in Bgee and to update OMAParentNodeId in gene table. +// this.loadGeneIdsFromDb(); +// +// // Retrieve taxon IDs of the Bgee database to be able to check if OMA HOGs +// // correspond to taxa present in Bgee. +// this.loadTaxonIdsFromDb(); +// +// // Retrieve species from Bgee that use genome of another species. +// this.loadMappingSpeciesToGenomeSpecies(); +// +// // Construct HierarchicalNodeTOs and GeneTOs +// this.generateTOsFromFile(orthoXMLFile, geneMapping); +// +// // Start a transaction to insert HierarchicalNodeTOs and update GeneTOs +// // in the Bgee data source. Note that we do not need to call rollback if +// // an error occurs, calling closeDAO will rollback any ongoing transaction. +// int nbInsertedGroups = 0, nbUpdatedGenes = 0, nbInsertedGroupToGene = 0; +// +// this.startTransaction(); +// +// log.info("Start inserting of hierarchical groups..."); +// nbInsertedGroups = this.getHierarchicalGroupDAO() +// .insertHierarchicalNodes(this.hierarchicalNodeTOs); +// log.info("Done inserting hierarchical groups"); +// +// log.info("Start updating genes..."); +//// nbUpdatedGenes = this.getGeneDAO().updateGenes(this.geneTOs, +//// Arrays.asList(GeneDAO.Attribute.OMA_PARENT_NODE_ID)); +// log.info("Done updating genes."); +// log.info("Start inserting gene to hierarchical group mapping..."); +// nbInsertedGroupToGene = this.getHierarchicalGroupDAO() +// .insertHierarchicalNodeToGene(this.hierarchicalNodeToGeneTOs); +// log.info("Done inserting gene to hierarchical group mapping."); +// +// this.commit(); +// log.info("Done parsing of OrthoXML file: {} hierarchical groups inserted " + +// ",{} genes updated, and {} mapping between hierarchical group and genes inserted.", nbInsertedGroups, nbUpdatedGenes, nbInsertedGroupToGene); +// } catch (IllegalStateException e) { +// log.catching(e); +// throw log.throwing(new IllegalArgumentException( +// "The OrthoXML file provided is invalid", e)); +// } finally { +// this.closeDAO(); +// } +// log.traceExit(); +// } /** * Extract the information about the gene mapping to include in Bgee from the provided @@ -497,50 +497,50 @@ private void loadMappingSpeciesToGenomeSpecies() throws DAOException, IllegalArg * @throws XMLParseException If there is an error in parsing the XML retrieved * by the OrthoXMLReader. */ - private void generateTOsFromFile(String orthoXMLFile, Map geneMapping) - throws FileNotFoundException, - XMLStreamException, XMLParseException { - log.entry(orthoXMLFile, geneMapping); - OrthoXMLReader reader = new OrthoXMLReader(new File(orthoXMLFile)); - List speciesInFile = reader.getSpecies(); - List speciesIdsInFile = new ArrayList(); - for (Species species : speciesInFile) { - speciesIdsInFile.add(species.getNcbiTaxId()); - } - // Common species - List common = new ArrayList(this.speciesIdsInBgee); - common.retainAll(speciesIdsInFile); - if (!common.isEmpty()) { - log.trace("The common species between Bgee and OMA file species are: {}", common); - } else { - throw log.throwing(new IllegalArgumentException( - "There is no common species between Bgee and OMA file species.")); - } - - // Species in Bgee but not in provided OMA file - List speciesBgeeSpecific = new ArrayList(this.speciesIdsInBgee); - speciesBgeeSpecific.removeAll(speciesIdsInFile); - if (!speciesBgeeSpecific.isEmpty()) { - log.trace("The species specific to Bgee are: {}", speciesBgeeSpecific); - } - - // Species in provided OMA file but not in Bgee - List speciesOMASpecific = new ArrayList(speciesIdsInFile); - speciesOMASpecific.removeAll(this.speciesIdsInBgee); - if (!speciesOMASpecific.isEmpty()) { - log.trace("The species specific to OMA file are: {}", speciesOMASpecific); - } - - // Read all the groups in the file iteratively - Group group = null; - while ((group = reader.next()) != null) { - this.generateTOsFromGroup(group, group.getId(), geneMapping); - // We increment the nestedSetBoundSeed because we move to the next OMA group. - this.nestedSetBoundSeed++; - } - log.info("Done retrieving hierarchical groups."); - log.traceExit(); - } +// private void generateTOsFromFile(String orthoXMLFile, Map geneMapping) +// throws FileNotFoundException, +// XMLStreamException, XMLParseException { +// log.entry(orthoXMLFile, geneMapping); +// OrthoXMLReader reader = new OrthoXMLReader(new File(orthoXMLFile)); +// List speciesInFile = reader.getSpecies(); +// List speciesIdsInFile = new ArrayList(); +// for (Species species : speciesInFile) { +// speciesIdsInFile.add(species.getNcbiTaxId()); +// } +// // Common species +// List common = new ArrayList(this.speciesIdsInBgee); +// common.retainAll(speciesIdsInFile); +// if (!common.isEmpty()) { +// log.trace("The common species between Bgee and OMA file species are: {}", common); +// } else { +// throw log.throwing(new IllegalArgumentException( +// "There is no common species between Bgee and OMA file species.")); +// } +// +// // Species in Bgee but not in provided OMA file +// List speciesBgeeSpecific = new ArrayList(this.speciesIdsInBgee); +// speciesBgeeSpecific.removeAll(speciesIdsInFile); +// if (!speciesBgeeSpecific.isEmpty()) { +// log.trace("The species specific to Bgee are: {}", speciesBgeeSpecific); +// } +// +// // Species in provided OMA file but not in Bgee +// List speciesOMASpecific = new ArrayList(speciesIdsInFile); +// speciesOMASpecific.removeAll(this.speciesIdsInBgee); +// if (!speciesOMASpecific.isEmpty()) { +// log.trace("The species specific to OMA file are: {}", speciesOMASpecific); +// } +// +// // Read all the groups in the file iteratively +// Group group = null; +// while ((group = reader.next()) != null) { +// this.generateTOsFromGroup(group, group.getId(), geneMapping); +// // We increment the nestedSetBoundSeed because we move to the next OMA group. +// this.nestedSetBoundSeed++; +// } +// log.info("Done retrieving hierarchical groups."); +// log.traceExit(); +// } /** * Extract all relevant information from a {@code Group} and create a @@ -556,84 +556,84 @@ private void generateTOsFromFile(String orthoXMLFile, Map geneMap * {@code HierarchicalNodeTO} has been added representing * the given {@code Group}. */ - private boolean generateTOsFromGroup(Group group, String omaXrefId, - Map geneMapping) { - log.entry(group, omaXrefId, geneMapping); - // First, we check if the group represents a taxon presents in Bgee or if it's a - // paralog group. If wrong, we don't insert a hierarchical groupTO. - String groupTaxId = group.getProperty(TAX_ID_ATTRIBUTE); - if (groupTaxId != null && !this.taxonIdsInBgee.contains(Integer.parseInt(groupTaxId))) { - log.warn("{} ({}) isn't a taxon relevant to Bgee", - group.getProperty(TAX_RANGE_ATTRIBUTE), groupTaxId); - return log.traceExit(false); - } - - // Second, we increment the nestedSetBoundSeed because we will create a new - // hierarchical group - this.nestedSetBoundSeed++; - - // We add a HierarchicalNodeTO in collection containing hierarchical groups to be - // inserted into the Bgee database - // The last argument is the number of children of the HierarchicalNodeTO to create. - // So, we need to remove 1 to countGroups() to subtract the current group. - log.debug("add OMAHierarchicalGroup and GeneToOma for group {}",group.getId()); - this.addHierarchicalNodeTO(this.omaNodeId, omaXrefId, this.nestedSetBoundSeed, - group.getProperty(TAX_ID_ATTRIBUTE), countGroups(group) - 1); - this.addHierarchicalNodeToGeneTO( - group.getProperty(TAX_ID_ATTRIBUTE), group.getNestedGenes(), - this.omaNodeId); - // Then, we retrieve gene data. - if (group.getGenes() != null) { - log.debug("Retrieving genes from group {}", group); - for (sbc.orthoxml.Gene groupGene : group.getGenes()) { - log.debug("Retrieving gene with identifier {}", groupGene.getGeneIdentifier()); - boolean isInBgee = false ; - for (String omaGeneId : retrieveSplittedGeneIdentifier(groupGene)) { - log.debug("Examining OMA geneId {}", omaGeneId); - if(idToBgeeIdInBgee.containsKey(omaGeneId)){ - for (Integer bgeeGeneId : idToBgeeIdInBgee.get(omaGeneId)){ - if (this.addGeneTO(new GeneTO(bgeeGeneId,omaGeneId, null, null, null, null, - this.omaNodeId,null, null, null), - omaXrefId)) { - isInBgee = true; - } else if (!geneMapping.isEmpty()) { - log.debug("Trying to find a x-ref for geneId {} from mapping file", omaGeneId); - if (geneMapping.containsKey(omaGeneId)) { - String currentGeneId = geneMapping.get(omaGeneId); - log.debug("Mapping found for geneId {}: {}", omaGeneId, currentGeneId); - if (this.addGeneTO(new GeneTO(bgeeGeneId, currentGeneId, null, null, null, null, - this.omaNodeId, null, null, null), omaXrefId)) { - isInBgee = true; - } - } - } - } - } - - } - if (!isInBgee) { - log.warn("No gene ID {} found in Bgee for the node {}", - groupGene.getGeneIdentifier(), this.omaNodeId); - } - } - } - - // Incrementing the node ID. Done after to be able to set OMA parent node ID - // into gene table - this.omaNodeId++; - - if (group.getChildren() != null && group.getChildren().size() > 0) { - for (Group childGroup : group.getChildren()) { - // Recurse - if (generateTOsFromGroup(childGroup, omaXrefId, geneMapping)) { - // We increment the nestedSetBoundSeed because we are at a leaf of the - // nested set model - this.nestedSetBoundSeed++; - } - } - } - return log.traceExit(true); - } +// private boolean generateTOsFromGroup(Group group, String omaXrefId, +// Map geneMapping) { +// log.entry(group, omaXrefId, geneMapping); +// // First, we check if the group represents a taxon presents in Bgee or if it's a +// // paralog group. If wrong, we don't insert a hierarchical groupTO. +// String groupTaxId = group.getProperty(TAX_ID_ATTRIBUTE); +// if (groupTaxId != null && !this.taxonIdsInBgee.contains(Integer.parseInt(groupTaxId))) { +// log.warn("{} ({}) isn't a taxon relevant to Bgee", +// group.getProperty(TAX_RANGE_ATTRIBUTE), groupTaxId); +// return log.traceExit(false); +// } +// +// // Second, we increment the nestedSetBoundSeed because we will create a new +// // hierarchical group +// this.nestedSetBoundSeed++; +// +// // We add a HierarchicalNodeTO in collection containing hierarchical groups to be +// // inserted into the Bgee database +// // The last argument is the number of children of the HierarchicalNodeTO to create. +// // So, we need to remove 1 to countGroups() to subtract the current group. +// log.debug("add OMAHierarchicalGroup and GeneToOma for group {}",group.getId()); +// this.addHierarchicalNodeTO(this.omaNodeId, omaXrefId, this.nestedSetBoundSeed, +// group.getProperty(TAX_ID_ATTRIBUTE), countGroups(group) - 1); +// this.addHierarchicalNodeToGeneTO( +// group.getProperty(TAX_ID_ATTRIBUTE), group.getNestedGenes(), +// this.omaNodeId); +// // Then, we retrieve gene data. +// if (group.getGenes() != null) { +// log.debug("Retrieving genes from group {}", group); +// for (sbc.orthoxml.Gene groupGene : group.getGenes()) { +// log.debug("Retrieving gene with identifier {}", groupGene.getGeneIdentifier()); +// boolean isInBgee = false ; +// for (String omaGeneId : retrieveSplittedGeneIdentifier(groupGene)) { +// log.debug("Examining OMA geneId {}", omaGeneId); +// if(idToBgeeIdInBgee.containsKey(omaGeneId)){ +// for (Integer bgeeGeneId : idToBgeeIdInBgee.get(omaGeneId)){ +// if (this.addGeneTO(new GeneTO(bgeeGeneId,omaGeneId, null, null, null, null, +// this.omaNodeId,null, null, null), +// omaXrefId)) { +// isInBgee = true; +// } else if (!geneMapping.isEmpty()) { +// log.debug("Trying to find a x-ref for geneId {} from mapping file", omaGeneId); +// if (geneMapping.containsKey(omaGeneId)) { +// String currentGeneId = geneMapping.get(omaGeneId); +// log.debug("Mapping found for geneId {}: {}", omaGeneId, currentGeneId); +// if (this.addGeneTO(new GeneTO(bgeeGeneId, currentGeneId, null, null, null, null, +// this.omaNodeId, null, null, null), omaXrefId)) { +// isInBgee = true; +// } +// } +// } +// } +// } +// +// } +// if (!isInBgee) { +// log.warn("No gene ID {} found in Bgee for the node {}", +// groupGene.getGeneIdentifier(), this.omaNodeId); +// } +// } +// } +// +// // Incrementing the node ID. Done after to be able to set OMA parent node ID +// // into gene table +// this.omaNodeId++; +// +// if (group.getChildren() != null && group.getChildren().size() > 0) { +// for (Group childGroup : group.getChildren()) { +// // Recurse +// if (generateTOsFromGroup(childGroup, omaXrefId, geneMapping)) { +// // We increment the nestedSetBoundSeed because we are at a leaf of the +// // nested set model +// this.nestedSetBoundSeed++; +// } +// } +// } +// return log.traceExit(true); +// } /** * Retrieves the split {@code String} of OMA Gene Identifier. diff --git a/bgee-pipeline/src/main/java/org/bgee/pipeline/species/InsertTaxa.java b/bgee-pipeline/src/main/java/org/bgee/pipeline/species/InsertTaxa.java index 1028df4ce..a13cc417f 100644 --- a/bgee-pipeline/src/main/java/org/bgee/pipeline/species/InsertTaxa.java +++ b/bgee-pipeline/src/main/java/org/bgee/pipeline/species/InsertTaxa.java @@ -147,6 +147,8 @@ public class InsertTaxa extends MySQLDAOUser { * we use the chimp genome (ID 9598), because bonobo is not in Ensembl. */ public static final String SPECIES_GENOME_ID_KEY= "genomeSpeciesId"; + + public static final String DEV_ONTOLOGY_XREF_KEY= "devOntologyXRef"; /** * A {@code String} that is the key to retrieve the fake prefix of genes for species * whose genome is not in Ensembl, and that are used in Bgee, @@ -689,9 +691,11 @@ private Set getSpeciesTOs(Collection> allSpecies) Integer genomeSpeciesId = (Integer) species.get(SPECIES_GENOME_ID_KEY); + String devOntologyXRef = (String) species.get(DEV_ONTOLOGY_XREF_KEY); + speciesTOs.add(new SpeciesTO(speciesId, commonName, genus, speciesName, displayOrder, parentTaxonId, genomeFilePath, genomeVersion, genomeAssemblyXRef, dataSourceId, - genomeSpeciesId)); + genomeSpeciesId, devOntologyXRef)); } if (speciesTOs.size() != allSpecies.size()) { throw log.throwing(new IllegalStateException("The taxonomy ontology " + diff --git a/bgee-pipeline/src/test/java/org/bgee/pipeline/BgeeDBUtilsTest.java b/bgee-pipeline/src/test/java/org/bgee/pipeline/BgeeDBUtilsTest.java index e4146877f..c54b1bf74 100644 --- a/bgee-pipeline/src/test/java/org/bgee/pipeline/BgeeDBUtilsTest.java +++ b/bgee-pipeline/src/test/java/org/bgee/pipeline/BgeeDBUtilsTest.java @@ -138,9 +138,9 @@ private SpeciesTOResultSet mockGetAllSpecies(MockDAOManager mockManager) { // We need a mock MySQLSpeciesTOResultSet to mock the return of getAllSpecies(). SpeciesTOResultSet mockSpeciesTORs = this.createMockDAOResultSet( Arrays.asList( - new SpeciesTO(21, null, null, null, null, null, null, null, null, null, null), - new SpeciesTO(11, null, null, null, null, null, null, null, null, null, null), - new SpeciesTO(30, null, null, null, null, null, null, null, null, null, null)), + new SpeciesTO(21, null, null, null, null, null, null, null, null, null, null, null), + new SpeciesTO(11, null, null, null, null, null, null, null, null, null, null, null), + new SpeciesTO(30, null, null, null, null, null, null, null, null, null, null, null)), MySQLSpeciesTOResultSet.class); when(mockManager.mockSpeciesDAO.getAllSpecies(EnumSet.of(SpeciesDAO.Attribute.ID))) .thenReturn(mockSpeciesTORs); @@ -372,9 +372,9 @@ public void shouldGetGeneNamesByIds() { public void shouldGetGeneTOsByIds() { try (MockDAOManager mockManager = new MockDAOManager()) { - GeneTO gene1 = new GeneTO(1, "1", "gene A", "desc A", 1, 1, 1, true, 1, null); - GeneTO gene2 = new GeneTO(2, "2", "gene B", "desc B", 2, 1, 1, true, 1, null); - GeneTO gene3 = new GeneTO(3, "3", "gene C", "desc C", 1, 2, 2, true, 1, null); + GeneTO gene1 = new GeneTO(1, "1", "gene A", "desc A", 1, 1, true, "reg1", 1, null); + GeneTO gene2 = new GeneTO(2, "2", "gene B", "desc B", 2, 1, true, "reg1", 1, null); + GeneTO gene3 = new GeneTO(3, "3", "gene C", "desc C", 1, 2, true, "reg1", 1, null); List returnedGeneTOs = Arrays.asList(gene1, gene2, gene3); @@ -422,8 +422,8 @@ public void shouldGetGeneTOsByIds() { try (MockDAOManager mockManager = new MockDAOManager()) { List returnedGeneTOs = Arrays.asList( - new GeneTO(1, "1", "gene A", "desc A", 1, 1, 1, true, 2, null), - new GeneTO(1, "1", "gene B", "desc B", 1, 1, 1, true, 2, null)); + new GeneTO(1, "1", "gene A", "desc A", 1, 1, true, "reg1", 2, null), + new GeneTO(1, "1", "gene B", "desc B", 1, 1, true, "reg1", 2, null)); GeneTOResultSet mockRS = this.createMockDAOResultSet( returnedGeneTOs, MySQLGeneTOResultSet.class); diff --git a/bgee-pipeline/src/test/java/org/bgee/pipeline/TestAncestor.java b/bgee-pipeline/src/test/java/org/bgee/pipeline/TestAncestor.java index 5bd6db943..aa96f37b1 100644 --- a/bgee-pipeline/src/test/java/org/bgee/pipeline/TestAncestor.java +++ b/bgee-pipeline/src/test/java/org/bgee/pipeline/TestAncestor.java @@ -22,7 +22,6 @@ import org.bgee.model.dao.mysql.connector.BgeeConnection; import org.bgee.model.dao.mysql.connector.MySQLDAOManager; import org.bgee.model.dao.mysql.expressiondata.call.MySQLDiffExpressionCallDAO; -import org.bgee.model.dao.mysql.expressiondata.rawdata.microarray.MySQLAffymetrixProbesetDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.insitu.MySQLInSituSpotDAO; import org.bgee.model.dao.mysql.expressiondata.rawdata.rnaseq.MySQLRNASeqResultAnnotatedSampleDAO; import org.bgee.model.dao.mysql.file.MySQLDownloadFileDAO; @@ -170,8 +169,6 @@ protected class MockDAOManager extends MySQLDAOManager { public final MySQLSpeciesDataGroupDAO mockSpeciesDataGroupDAO = mock(MySQLSpeciesDataGroupDAO.class); public final MySQLDownloadFileDAO mockDownloadFileDAO = mock(MySQLDownloadFileDAO.class); - public final MySQLAffymetrixProbesetDAO mockAffymetrixProbesetDAO = - mock(MySQLAffymetrixProbesetDAO.class); public final MySQLInSituSpotDAO mockInSituSpotDAO = mock(MySQLInSituSpotDAO.class); public final MySQLRNASeqResultAnnotatedSampleDAO mockRNASeqResultDAO = mock(MySQLRNASeqResultAnnotatedSampleDAO.class); @@ -272,10 +269,6 @@ protected MySQLDownloadFileDAO getNewDownloadFileDAO() { return this.mockDownloadFileDAO; } @Override - protected MySQLAffymetrixProbesetDAO getNewAffymetrixProbesetDAO() { - return this.mockAffymetrixProbesetDAO; - } - @Override protected MySQLInSituSpotDAO getNewInSituSpotDAO() { return this.mockInSituSpotDAO; } diff --git a/bgee-pipeline/src/test/java/org/bgee/pipeline/expression/downloadfile/GenerateDiffExprFileTest.java b/bgee-pipeline/src/test/java/org/bgee/pipeline/expression/downloadfile/GenerateDiffExprFileTest.java index 2a1ef56b0..3f605e04e 100644 --- a/bgee-pipeline/src/test/java/org/bgee/pipeline/expression/downloadfile/GenerateDiffExprFileTest.java +++ b/bgee-pipeline/src/test/java/org/bgee/pipeline/expression/downloadfile/GenerateDiffExprFileTest.java @@ -36,8 +36,8 @@ import org.bgee.model.dao.mysql.expressiondata.call.MySQLDiffExpressionCallDAO.MySQLDiffExpressionCallTOResultSet; import org.bgee.model.dao.mysql.species.MySQLSpeciesDAO.MySQLSpeciesTOResultSet; import org.bgee.pipeline.TestAncestor; -import org.bgee.pipeline.expression.downloadfile.GenerateDiffExprFile.DiffExpressionData; -import org.bgee.pipeline.expression.downloadfile.GenerateDiffExprFile.SingleSpDiffExprFileType; +//import org.bgee.pipeline.expression.downloadfile.GenerateDiffExprFile.DiffExpressionData; +//import org.bgee.pipeline.expression.downloadfile.GenerateDiffExprFile.SingleSpDiffExprFileType; //import org.bgee.pipeline.expression.downloadfile.GenerateExprFile.SingleSpExprFileType; import org.junit.Test; import org.supercsv.cellprocessor.constraint.DMinMax; diff --git a/bgee-pipeline/src/test/java/org/bgee/pipeline/expression/downloadfile/InsertSpeciesDataGroupsTest.java b/bgee-pipeline/src/test/java/org/bgee/pipeline/expression/downloadfile/InsertSpeciesDataGroupsTest.java index 37ddf11f2..7fae85c6c 100644 --- a/bgee-pipeline/src/test/java/org/bgee/pipeline/expression/downloadfile/InsertSpeciesDataGroupsTest.java +++ b/bgee-pipeline/src/test/java/org/bgee/pipeline/expression/downloadfile/InsertSpeciesDataGroupsTest.java @@ -159,7 +159,7 @@ protected Logger getLogger() { // groupToCategories.put("groupOneSpecies1", new HashSet(Arrays.asList( // CategoryEnum.DIFF_EXPR_ANAT_COMPLETE.getStringRepresentation(), // CategoryEnum.DIFF_EXPR_ANAT_SIMPLE.getStringRepresentation(), -// CategoryEnum.AFFY_ANNOT.getStringRepresentation()))); +// CategoryEnum.RNASEQ_ANNOT.getStringRepresentation()))); // groupToCategories.put("groupOneSpecies2", new HashSet(Arrays.asList( // CategoryEnum.EXPR_CALLS_COMPLETE.getStringRepresentation(), // CategoryEnum.RNASEQ_DATA.getStringRepresentation()))); @@ -186,7 +186,7 @@ protected Logger getLogger() { // singleSpCategoryToFilePathPattern.put(CategoryEnum.DIFF_EXPR_ANAT_SIMPLE.getStringRepresentation(), // tmpPath.relativize(file2.toPath()).toString() // .replaceAll("sp1", InsertSpeciesDataGroups.STRING_TO_REPLACE)); -// singleSpCategoryToFilePathPattern.put(CategoryEnum.AFFY_ANNOT.getStringRepresentation(), +// singleSpCategoryToFilePathPattern.put(CategoryEnum.RNASEQ_ANNOT.getStringRepresentation(), // tmpPath.relativize(file3.toPath()).toString() // .replaceAll("sp1", InsertSpeciesDataGroups.STRING_TO_REPLACE)); // singleSpCategoryToFilePathPattern.put(CategoryEnum.EXPR_CALLS_COMPLETE.getStringRepresentation(), @@ -263,7 +263,7 @@ protected Logger getLogger() { //// file2.length(), CategoryEnum.DIFF_EXPR_ANAT_SIMPLE, "1"), // new DownloadFileTO("3", file3.getName(), null, // tmpPath.relativize(file3.toPath()).toString(), -// file3.length(), CategoryEnum.AFFY_ANNOT, "1"), +// file3.length(), CategoryEnum.RNASEQ_ANNOT, "1"), // new DownloadFileTO("4", file4.getName(), null, // tmpPath.relativize(file4.toPath()).toString(), // file4.length(), CategoryEnum.EXPR_CALLS_COMPLETE, "2"), diff --git a/bgee-pipeline/src/test/java/org/bgee/pipeline/species/InsertTaxaTest.java b/bgee-pipeline/src/test/java/org/bgee/pipeline/species/InsertTaxaTest.java index ca0e2d13d..415bed0ff 100644 --- a/bgee-pipeline/src/test/java/org/bgee/pipeline/species/InsertTaxaTest.java +++ b/bgee-pipeline/src/test/java/org/bgee/pipeline/species/InsertTaxaTest.java @@ -117,13 +117,13 @@ public void shouldInsertSpeciesAndTaxa() throws FileNotFoundException, Set expectedSpeciesTOs = new HashSet(); expectedSpeciesTOs.add( new SpeciesTO(8, "my common nameA", "my genusA", "my speciesA", 2, 16, - "file/pathA", "versionA", "assemblyXRefA", 2, null)); + "file/pathA", "versionA", "assemblyXRefA", 2, null, null)); expectedSpeciesTOs.add( new SpeciesTO(13, "my common nameB", "my genusB", "my speciesB", 1, 12, - "file/pathB", "versionB", "assemblyXRefB", 24, 20)); + "file/pathB", "versionB", "assemblyXRefB", 24, 20, null)); expectedSpeciesTOs.add( new SpeciesTO(15, "", "my genusC", "my speciesC", 3, 14, - "file/pathC", "versionC", "assemblyXRefC", 2, null)); + "file/pathC", "versionC", "assemblyXRefC", 2, null, null)); ArgumentCaptor speciesTOsArg = ArgumentCaptor.forClass(Set.class); verify(mockManager.mockSpeciesDAO).insertSpecies(speciesTOsArg.capture()); if (!this.areSpeciesTOCollectionsEqual( diff --git a/bgee-webapp/src/main/java/org/bgee/controller/CommandData.java b/bgee-webapp/src/main/java/org/bgee/controller/CommandData.java index d166bf435..0282e3b26 100644 --- a/bgee-webapp/src/main/java/org/bgee/controller/CommandData.java +++ b/bgee-webapp/src/main/java/org/bgee/controller/CommandData.java @@ -1,7 +1,26 @@ package org.bgee.controller; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.EnumMap; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import javax.servlet.http.HttpServletResponse; + import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bgee.controller.exception.InvalidRequestException; @@ -10,34 +29,18 @@ import org.bgee.controller.utils.BgeeCacheService; import org.bgee.controller.utils.BgeeCacheService.CacheDefinition; import org.bgee.controller.utils.BgeeCacheService.CacheType; -import org.bgee.model.BgeeEnum; +import org.bgee.model.ComposedEntity; import org.bgee.model.ServiceFactory; import org.bgee.model.anatdev.AnatEntity; import org.bgee.model.anatdev.DevStage; import org.bgee.model.anatdev.Sex; import org.bgee.model.anatdev.Sex.SexEnum; -import org.bgee.model.dao.api.expressiondata.call.ConditionDAO; -import org.bgee.model.expressiondata.BaseConditionFilter2.ComposedFilterIds; -import org.bgee.model.expressiondata.BaseConditionFilter2.FilterIds; import org.bgee.model.expressiondata.baseelements.ConditionParameter; import org.bgee.model.expressiondata.baseelements.DataType; -import org.bgee.model.expressiondata.baseelements.SummaryCallType.ExpressionSummary; -import org.bgee.model.expressiondata.baseelements.SummaryQuality; -import org.bgee.model.expressiondata.call.Call.ExpressionCall2; import org.bgee.model.expressiondata.call.CallFilter.ExpressionCallFilter2; -import org.bgee.model.expressiondata.call.ConditionFilter2; import org.bgee.model.expressiondata.call.ExpressionCallLoader; import org.bgee.model.expressiondata.call.ExpressionCallPostFilter; -import org.bgee.model.expressiondata.call.ExpressionCallProcessedFilter; -import org.bgee.model.expressiondata.call.ExpressionCallProcessedFilter.ExpressionCallProcessedFilterConditionPart; -import org.bgee.model.expressiondata.call.ExpressionCallService; -import org.bgee.model.expressiondata.rawdata.baseelements.Assay; -import org.bgee.model.expressiondata.rawdata.baseelements.Experiment; -import org.bgee.model.expressiondata.rawdata.baseelements.ExperimentAssay; -import org.bgee.model.expressiondata.rawdata.baseelements.RawDataContainer; -import org.bgee.model.expressiondata.rawdata.baseelements.RawDataContainerWithExperiment; -import org.bgee.model.expressiondata.rawdata.baseelements.RawDataCountContainer; -import org.bgee.model.expressiondata.rawdata.baseelements.RawDataDataType; +import org.bgee.model.expressiondata.call.OTFExpressionCall; import org.bgee.model.expressiondata.rawdata.RawDataConditionFilter; import org.bgee.model.expressiondata.rawdata.RawDataFilter; import org.bgee.model.expressiondata.rawdata.RawDataLoader; @@ -46,6 +49,13 @@ import org.bgee.model.expressiondata.rawdata.RawDataProcessedFilter; import org.bgee.model.expressiondata.rawdata.RawDataProcessedFilter.RawDataProcessedFilterConditionPart; import org.bgee.model.expressiondata.rawdata.RawDataService; +import org.bgee.model.expressiondata.rawdata.baseelements.Assay; +import org.bgee.model.expressiondata.rawdata.baseelements.Experiment; +import org.bgee.model.expressiondata.rawdata.baseelements.ExperimentAssay; +import org.bgee.model.expressiondata.rawdata.baseelements.RawDataContainer; +import org.bgee.model.expressiondata.rawdata.baseelements.RawDataContainerWithExperiment; +import org.bgee.model.expressiondata.rawdata.baseelements.RawDataCountContainer; +import org.bgee.model.expressiondata.rawdata.baseelements.RawDataDataType; import org.bgee.model.gene.Gene; import org.bgee.model.gene.GeneFilter; import org.bgee.model.job.Job; @@ -59,26 +69,6 @@ import org.bgee.view.DataDisplay; import org.bgee.view.ViewFactory; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.EnumMap; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import javax.servlet.http.HttpServletResponse; - /** * Controller that handles requests for the raw data page. * @@ -86,16 +76,16 @@ * @version Bgee 15.0, Jan. 2023 * @since Bgee 15.0, Oct. 2022 */ -public class CommandData extends CommandParent { +public class CommandData extends CommandExpressionSupport { private final static Logger log = LogManager.getLogger(CommandData.class.getName()); public static class ExpressionCallResponse { - private final List calls; + private final List calls; private final LinkedHashSet> condParams; private final EnumSet requestedDataTypes; - public ExpressionCallResponse(List calls, + public ExpressionCallResponse(List calls, LinkedHashSet> condParams, EnumSet requestedDataTypes) { this.calls = calls; @@ -103,7 +93,7 @@ public ExpressionCallResponse(List calls, this.requestedDataTypes = requestedDataTypes; } - public List getCalls() { + public List getCalls() { return calls; } public LinkedHashSet> getCondParams() { @@ -513,56 +503,6 @@ public String toString() { return builder.toString(); } } - public static class ExprCallResultCacheKey { - - private final ExpressionCallFilter2 sourceFilter; - private final Long offset; - private final Integer limit; - - public ExprCallResultCacheKey(ExpressionCallFilter2 sourceFilter, Long offset, Integer limit) { - this.sourceFilter = sourceFilter; - this.offset = offset; - this.limit = limit; - } - - public ExpressionCallFilter2 getSourceFilter() { - return sourceFilter; - } - public Long getOffset() { - return offset; - } - public Integer getLimit() { - return limit; - } - - @Override - public int hashCode() { - return Objects.hash(limit, offset, sourceFilter); - } - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - ExprCallResultCacheKey other = (ExprCallResultCacheKey) obj; - return Objects.equals(limit, other.limit) && Objects.equals(offset, other.offset) - && Objects.equals(sourceFilter, other.sourceFilter); - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("ExprCallResultCacheKey [") - .append("offset=").append(offset) - .append(", limit=").append(limit) - .append(", sourceFilter=").append(sourceFilter) - .append("]"); - return builder.toString(); - } - } public static class RawDataCondPartProcessingCacheKey { private final Set condFilters; public RawDataCondPartProcessingCacheKey(Set condFilters) { @@ -593,36 +533,6 @@ public String toString() { return builder.toString(); } } - public static class ExprCallCondPartProcessingCacheKey { - private final Set condFilters; - public ExprCallCondPartProcessingCacheKey(Set condFilters) { - this.condFilters = condFilters; - } - public Set getCondFilters() { - return condFilters; - } - @Override - public int hashCode() { - return Objects.hash(condFilters); - } - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - ExprCallCondPartProcessingCacheKey other = (ExprCallCondPartProcessingCacheKey) obj; - return Objects.equals(condFilters, other.condFilters); - } - @Override - public String toString() { - StringBuilder builder = new StringBuilder(); - builder.append("ExprCallCondPartProcessingCacheKey [condFilters=").append(condFilters).append("]"); - return builder.toString(); - } - } /** * An {@code int} that is the maximum allowed number of results @@ -642,7 +552,6 @@ public String toString() { * for convenience when comparing to start and end times provided as {@code long}. * * @see #loadRawDataCounts(RawDataLoader, EnumSet) - * @see #loadExprCallCounts(ExpressionCallLoader) */ private final static long COMPUTE_TIME_COUNT_CACHE_MS = 1000L; @@ -650,10 +559,6 @@ public String toString() { RAW_DATA_COUNT_CACHE_DEF = new CacheDefinition<>("rawDataCountCache", RawDataCacheKey.class, RawDataCountContainer.class, CacheType.LRU, 300); - private final static CacheDefinition - EXPR_CALL_COUNT_CACHE_DEF = new CacheDefinition<>("exprCallCountCache", - ExpressionCallFilter2.class, Long.class, CacheType.LRU, 60); - /** * A {@code long} that is the execution time in milliseconds of the processing of * the part of a filter related to condition, that triggers storing the result in cache. @@ -667,7 +572,6 @@ public String toString() { * @see #loadRawDataLoader(RawDataFilter) * @see #loadExprCallLoader(ExpressionCallFilter2) */ - private final static long COMPUTE_TIME_PROCESSED_COND_PART_CACHE_MS = 1000L; private final static CacheDefinition RAW_DATA_PROCESSED_COND_PART_CACHE_DEF = new CacheDefinition<>("rawDataProcessedCondPartCache", @@ -675,12 +579,6 @@ public String toString() { RawDataProcessedFilter.RawDataProcessedFilterConditionPart.class, CacheType.LRU, 20); - private final static CacheDefinition - EXPR_CALL_PROCESSED_COND_PART_CACHE_DEF = new CacheDefinition<>("exprCallProcessedCondPartCache", - ExprCallCondPartProcessingCacheKey.class, - ExpressionCallProcessedFilter.ExpressionCallProcessedFilterConditionPart.class, - CacheType.LRU, 20); - /** * A {@code long} that is the execution time in milliseconds of the processing of results * that triggers storing the result in cache. Defined as {@code long} @@ -698,13 +596,6 @@ public String toString() { RAW_DATA_RESULT_CACHE_DEF = new CacheDefinition<>("rawDataResultCache", RawDataResultCacheKey.class, RawDataContainer.class, CacheType.LRU, 100); - //Suppress warning for List generic type to have inference - //working with 'List.class' - @SuppressWarnings("rawtypes") - private final static CacheDefinition - EXPR_CALL_RESULT_CACHE_DEF = new CacheDefinition<>("exprCallResultCache", - ExprCallResultCacheKey.class, List.class, CacheType.LRU, 20); - /** * A {@code long} that is the execution time in milliseconds of the generation of post-filters * that triggers storing the result in cache. Defined as {@code long} @@ -719,32 +610,8 @@ public String toString() { RAW_DATA_POST_FILTER_CACHE_DEF = new CacheDefinition<>("rawDataPostFilterCache", RawDataCacheKey.class, RawDataPostFilter.class, CacheType.LRU, 100); - private final static CacheDefinition - EXPR_CALL_POST_FILTER_CACHE_DEF = new CacheDefinition<>("exprCallPostFilterCache", - ExpressionCallFilter2.class, ExpressionCallPostFilter.class, CacheType.LRU, 20); - - /** - * A {@code String} to recognize the action of requesting an experiment page - * (there is no corresponding action in {@code RequestParameter}, it is triggered - * when the URL parameter {@code exp_id} is provided). - */ private final static String EXPERIMENT_PAGE_ACTION = "experiment"; - private final static String ID_PARAM_SUMMARY_VALUE = "SUMMARY"; - private final static Set SUMMARY_ANAT_ENTITY_IDS = Set.of( - "UBERON:0001062", - "UBERON:0000010", "UBERON:0000211", "UBERON:0000309", "UBERON:0000468", - "UBERON:0000949", "UBERON:0000990", "UBERON:0001004", "UBERON:0001007", - "UBERON:0001008", "UBERON:0001009", "UBERON:0001015", "UBERON:0001017", - "UBERON:0001032", "UBERON:0001434", "UBERON:0002193", "UBERON:0002330", - "UBERON:0002384", "UBERON:0002405", "UBERON:0002416", "UBERON:0015204"); - private final static String SUMMARY_ANAT_ENTITY_ROOT_ID = "UBERON:0001062"; - private final static Set SUMMARY_DISCARD_ANAT_ENTITY_AND_CHILDREN_IDS = - Collections.unmodifiableSet( - SUMMARY_ANAT_ENTITY_IDS.stream().filter(id -> !id.equals(SUMMARY_ANAT_ENTITY_ROOT_ID)) - .collect(Collectors.toSet())); - private final static Set SUMMARY_CELL_TYPE_IDS = Set.of(ConditionDAO.CELL_TYPE_ROOT_ID); - //Static initializer { if (LIMIT_MAX > RawDataLoader.LIMIT_MAX) { @@ -769,7 +636,7 @@ public CommandData(HttpServletResponse response, RequestParameters requestParame BgeeProperties prop, ViewFactory viewFactory, ServiceFactory serviceFactory, JobService jobService, BgeeCacheService cacheService, User user) { super(response, requestParameters, prop, viewFactory, serviceFactory, jobService, - cacheService, user, null, null); + cacheService, user); this.speciesService = this.serviceFactory.getSpeciesService(); } @@ -909,7 +776,7 @@ private void processExprCallPage(List speciesList, DataFormDetails form log.debug("Action identified: {}", this.requestParameters.getAction()); List colDescriptions = null; - List calls = null; + List calls = null; Long count = null; ExpressionCallPostFilter postFilter = null; @@ -941,8 +808,8 @@ private void processExprCallPage(List speciesList, DataFormDetails form //to set them in a ConditionFilter without providing a species ID, //so we don't have to explicitly check here if some are provided. if (this.requestParameters.getSpeciesId() != null || - !condParams.isEmpty() && !condParams.containsAll(ConditionParameter.allOf()) || - !dataTypes.isEmpty() && !dataTypes.equals(EnumSet.allOf(DataType.class))) { + (!condParams.isEmpty() && !condParams.containsAll(ConditionParameter.allOf())) || + (!dataTypes.isEmpty() && !dataTypes.equals(EnumSet.allOf(DataType.class)))) { if (this.requestParameters.getGeneIds() == null || this.requestParameters.getGeneIds().isEmpty()) { @@ -964,30 +831,52 @@ else if (this.requestParameters.isGetFilters()) { job = this.jobService.registerNewJob(this.user.getUUID().toString()); job.startJob(); //If filters are provided, they will be considered with this ExpressionCallLoader + long startTimeLoader = System.currentTimeMillis(); ExpressionCallLoader callLoader = this.loadExprCallLoader(true, condParams, dataTypes); - - //results - if (this.requestParameters.isGetResults()) { - calls = this.loadExprCallResults(callLoader); + log.debug("ExpressionCallLoader generated in {} ms", System.currentTimeMillis() - startTimeLoader); + + // Run OTF propagation once and reuse the result for results, count, and post-filters. + List allOtfCalls = null; + if (this.requestParameters.isGetResults() || this.requestParameters.isGetResultCount() + || (this.requestParameters.isGetFilters() && postFilter == null)) { + long startTimeOtf = System.currentTimeMillis(); + allOtfCalls = this.loadExprCallResults(callLoader, DEFAULT_LIMIT, LIMIT_MAX); + log.debug("loadDataOnTheFly() completed in {} ms, {} calls retrieved", + System.currentTimeMillis() - startTimeOtf, allOtfCalls.size()); } - //Raw data counts + + //Count derived from OTF result size if (this.requestParameters.isGetResultCount()) { - count = this.loadExprCallCount(callLoader); + count = (long) allOtfCalls.size(); + log.debug("Result count: {}", count); } - //Filters. PostFilter is not null and is an empty filter if no genes are specified, + + //Post-filters derived from OTF result conditions. + //PostFilter is not null and is an empty filter if no genes are specified, //in that case we don't retrieve filters. if (this.requestParameters.isGetFilters() && postFilter == null) { - //For requesting getFilters, well, the filter parameters must be ignored - ExpressionCallLoader loaderToUse = callLoader; - ExpressionCallFilter2 noFilterParamFilter = this.loadExprCallFilter( - false, condParams, dataTypes); - //We try to avoid requesting a ProcessedFilter if not necessary, - //by comparing the RawDataFilters - if (!callLoader.getProcessedFilter() - .getSourceFilter().equals(noFilterParamFilter)) { - loaderToUse = this.loadExprCallLoader(noFilterParamFilter); + long startTimePostFilter = System.currentTimeMillis(); + postFilter = this.buildPostFilterFromOtfCalls(allOtfCalls, condParams); + log.debug("Post-filter built in {} ms", System.currentTimeMillis() - startTimePostFilter); + } + + //Paginated results + if (this.requestParameters.isGetResults()) { + long offset = this.requestParameters.getOffset() == null? 0L: + this.requestParameters.getOffset(); + if (offset < 0) { + throw log.throwing(new InvalidRequestException("Offset must be non-negative.")); } - postFilter = this.loadExprCallPostFilters(loaderToUse); + int limit = this.requestParameters.getLimit() == null? DEFAULT_LIMIT: + this.requestParameters.getLimit(); + if (limit > LIMIT_MAX) { + throw log.throwing(new InvalidRequestException( + "Limit cannot be greater than " + LIMIT_MAX)); + } + long startTimePagination = System.currentTimeMillis(); + calls = allOtfCalls.stream().skip(offset).limit(limit).collect(Collectors.toList()); + log.debug("Pagination (offset={}, limit={}) completed in {} ms", + offset, limit, System.currentTimeMillis() - startTimePagination); } job.completeWithSuccess(); @@ -1262,14 +1151,7 @@ private RawDataLoader loadRawDataLoader(boolean consideringFilters) { return log.traceExit(this.loadRawDataLoader(this.loadRawDataFilter(consideringFilters))); } - private ExpressionCallLoader loadExprCallLoader(boolean consideringFilters, - Set> condParams, EnumSet dataTypes) - throws InvalidRequestException { - log.traceEntry("{}, {}, {}", consideringFilters, condParams, dataTypes); - - return log.traceExit(this.loadExprCallLoader( - this.loadExprCallFilter(consideringFilters, condParams, dataTypes))); - } + private RawDataLoader loadRawDataLoader(RawDataFilter filter) { log.traceEntry("{}", filter); @@ -1287,22 +1169,6 @@ private RawDataLoader loadRawDataLoader(RawDataFilter filter) { return log.traceExit(rawDataService.getRawDataLoader(processedFilter)); } - private ExpressionCallLoader loadExprCallLoader(ExpressionCallFilter2 filter) { - log.traceEntry("{}", filter); - - ExpressionCallService callService = this.serviceFactory.getExpressionCallService(); - //Try to get the processed condition part of the processed filter from cache - ExpressionCallProcessedFilter processedFilter = this.cacheService.useCacheNonAtomic( - EXPR_CALL_PROCESSED_COND_PART_CACHE_DEF, - new ExprCallCondPartProcessingCacheKey(filter.getConditionFilters()), - () -> callService.processExpressionCallFilter(filter), - pf -> pf.getConditionPart(), - condPart -> callService.processExpressionCallFilter(filter, - null, condPart, null), - COMPUTE_TIME_PROCESSED_COND_PART_CACHE_MS); - - return log.traceExit(callService.getCallLoader(processedFilter)); - } private RawDataFilter loadRawDataFilter(boolean consideringFilters) { log.traceEntry("{}", consideringFilters); @@ -1407,222 +1273,7 @@ private RawDataFilter loadRawDataFilter(boolean consideringFilters) { // for now we do not allow to retrieve only not propagated raw data. onlyPropagatedParam == false ? null: true)); } - private ExpressionCallFilter2 loadExprCallFilter(boolean consideringFilters, - Set> condParams, EnumSet dataTypes) - throws InvalidRequestException { - log.traceEntry("{}, {}, {}", consideringFilters, condParams, dataTypes); - - //Either there is no filtering at all, or some genes must be requested. - //Checks are made in method #processExprCallPage() - Integer speciesId = this.requestParameters.getSpeciesId(); - if (speciesId == null) { - log.debug("No filter present, returning an empty ExpressionCallFilter2"); - return log.traceExit(new ExpressionCallFilter2()); - } - GeneFilter geneFilter = new GeneFilter(speciesId, this.requestParameters.getGeneIds()); - if (geneFilter.getGeneIds().isEmpty()) { - throw log.throwing(new InvalidRequestException("Some genes must be selected.")); - } - - //Currently there is only one filter for both anat. entities and cell types - List filterAnatEntityCellTypeIds = !consideringFilters? null: - this.requestParameters.getValues( - this.requestParameters.getUrlParametersInstance().getParamFilterAnatEntity()); - List filterDevStageIds = !consideringFilters? null: - this.requestParameters.getValues( - this.requestParameters.getUrlParametersInstance().getParamFilterDevStage()); - List filterSexIds = !consideringFilters? null: - this.requestParameters.getValues( - this.requestParameters.getUrlParametersInstance().getParamFilterSex()); - List filterStrains = !consideringFilters? null: - this.requestParameters.getValues( - this.requestParameters.getUrlParametersInstance().getParamFilterStrain()); - - List sexes = this.requestParameters.getSex(); - if (sexes != null && (sexes.contains(RequestParameters.ALL_VALUE) || - sexes.containsAll( - EnumSet.allOf(SexEnum.class) - .stream() - .map(e -> e.name()) - .collect(Collectors.toSet())))) { - sexes = null; - } - - Map, ComposedFilterIds> condParamToComposedFilterIds = - new HashMap<>(); - - //-------------- - //Management of "magic" values: - //If we receive the magic value "SUMMARY", we'll use a fix list of terms. - List anatEntityIds = this.requestParameters.getAnatEntity() == null? new ArrayList<>(): - new ArrayList<>(this.requestParameters.getAnatEntity()); - boolean summaryTermsRequested = false; - if (anatEntityIds.contains(ID_PARAM_SUMMARY_VALUE)) { - summaryTermsRequested = true; - anatEntityIds.addAll(SUMMARY_ANAT_ENTITY_IDS); - anatEntityIds.remove(ID_PARAM_SUMMARY_VALUE); - } - List cellTypeIds = this.requestParameters.getCellType() == null? new ArrayList<>(): - new ArrayList<>(this.requestParameters.getCellType()); - if (cellTypeIds.contains(ID_PARAM_SUMMARY_VALUE)) { - cellTypeIds.addAll(SUMMARY_CELL_TYPE_IDS); - cellTypeIds.remove(ID_PARAM_SUMMARY_VALUE); - } - List discardAnatEntityIds = this.requestParameters.getDiscardAnatEntity() == null? - new ArrayList<>(): new ArrayList<>(this.requestParameters.getDiscardAnatEntity()); - if (discardAnatEntityIds.contains(ID_PARAM_SUMMARY_VALUE)) { - discardAnatEntityIds.addAll(SUMMARY_DISCARD_ANAT_ENTITY_AND_CHILDREN_IDS); - discardAnatEntityIds.remove(ID_PARAM_SUMMARY_VALUE); - if (!summaryTermsRequested) { - discardAnatEntityIds.removeAll(anatEntityIds); - } - } - boolean requestedAnatEntityDescendant = Boolean.TRUE.equals(this.requestParameters.getFirstValue( - this.requestParameters.getUrlParametersInstance().getParamAnatEntityDescendant())); - if (!anatEntityIds.isEmpty() && !discardAnatEntityIds.isEmpty() && !requestedAnatEntityDescendant) { - throw log.throwing(new InvalidRequestException("Only when anat. entity descendants are requested " - + "it is possible to exclude anat. entities and their children.")); - } - //And we never include child terms when the parameter comes from a filter. - boolean anatEntityDescendant = - filterAnatEntityCellTypeIds != null && !filterAnatEntityCellTypeIds.isEmpty() || - anatEntityIds.isEmpty()? false: - Boolean.TRUE.equals(this.requestParameters.getFirstValue( - this.requestParameters.getUrlParametersInstance() - .getParamAnatEntityDescendant())); - //-------------- - - //ANAT ENTITY AND CELL TYPE - FilterIds anatEntityFilter = new FilterIds<>( - //Filters override the related parameter from the form - filterAnatEntityCellTypeIds != null && !filterAnatEntityCellTypeIds.isEmpty()? - filterAnatEntityCellTypeIds: anatEntityIds, - anatEntityDescendant, - filterAnatEntityCellTypeIds != null && !filterAnatEntityCellTypeIds.isEmpty()? - null: discardAnatEntityIds, - null); - FilterIds cellTypeFilter = new FilterIds<>( - //Filters override the related parameter from the form - filterAnatEntityCellTypeIds != null && !filterAnatEntityCellTypeIds.isEmpty()? - filterAnatEntityCellTypeIds: cellTypeIds, - //And we never include child terms when the parameter comes from a filter. - filterAnatEntityCellTypeIds != null && !filterAnatEntityCellTypeIds.isEmpty() || - cellTypeIds.isEmpty()? - false: Boolean.TRUE.equals(this.requestParameters.getFirstValue( - this.requestParameters.getUrlParametersInstance() - .getParamCellTypeDescendant()))); - - - List> composedFilterIds = new ArrayList<>(List.of(anatEntityFilter)); - //In case we used the filters, anatEntityFilter and cellTypeFilter should be equal, - //and we thus don't use the cellTypeFilter - if (!anatEntityFilter.equals(cellTypeFilter)) { - composedFilterIds.add(cellTypeFilter); - } - ComposedFilterIds anatComposedFilter = new ComposedFilterIds<>( - composedFilterIds.stream() - .filter(f -> !f.isEmpty()) - .collect(Collectors.toList())); - condParamToComposedFilterIds.put(ConditionParameter.ANAT_ENTITY_CELL_TYPE, anatComposedFilter); - - //DEV. STAGE - FilterIds devStageFilter = new FilterIds<>( - //Filters override the related parameter from the form - filterDevStageIds != null && !filterDevStageIds.isEmpty()? - filterDevStageIds: this.requestParameters.getDevStage(), - //And we never include child terms when the parameter comes from a filter. - filterDevStageIds != null && !filterDevStageIds.isEmpty() || - this.requestParameters.getDevStage() == null || - this.requestParameters.getDevStage().isEmpty()? - false: Boolean.TRUE.equals(this.requestParameters.getFirstValue( - this.requestParameters.getUrlParametersInstance() - .getParamStageDescendant()))); - condParamToComposedFilterIds.put(ConditionParameter.DEV_STAGE, - new ComposedFilterIds<>(devStageFilter)); - - //SEX - FilterIds sexFilter = new FilterIds<>( - //Filters override the related parameter from the form - filterSexIds != null && !filterSexIds.isEmpty()? - filterSexIds: sexes, - //sex descendant always false: requesting descendants of the root is equivalent - //to request all sexes, in which case we don't provide requested sex IDs - false); - condParamToComposedFilterIds.put(ConditionParameter.SEX, - new ComposedFilterIds<>(sexFilter)); - - //STRAIN - FilterIds strainFilter = new FilterIds<>( - //Filters override the related parameter from the form - filterStrains != null && !filterStrains.isEmpty()? - filterStrains: this.requestParameters.getStrain(), - //strain descendant always false: requesting descendants of the root is equivalent - //to request all strains, in which case we don't provide requested strains - false); - condParamToComposedFilterIds.put(ConditionParameter.STRAIN, - new ComposedFilterIds<>(strainFilter)); - - ConditionFilter2 condFilter = null; - try { - condFilter = new ConditionFilter2(speciesId, - condParamToComposedFilterIds, - condParams, - null, - this.requestParameters.isExcludeNonInformative()); - if (condFilter.areAllFiltersExceptSpeciesEmpty()) { - //To request a species a GeneFilter is mandatory, - //so if there are no other filters, we can discard this ConditionFilter - condFilter = null; - } - } catch (IllegalArgumentException e) { - log.catching(e); - throw log.throwing(new InvalidRequestException(e.getMessage())); - } - - //ExpressionSummary and SummaryQuality - SummaryQuality tmpQual = SummaryQuality.values()[0]; - if (this.requestParameters.getDataQuality() != null && - !this.requestParameters.getDataQuality().isBlank()) { - try { - tmpQual = BgeeEnum.convert(SummaryQuality.class, this.requestParameters.getDataQuality()); - } catch (IllegalArgumentException e) { - log.catching(Level.DEBUG, e); - throw log.throwing(new InvalidRequestException( - "Unrecognized data quality: " + this.requestParameters.getDataQuality())); - } - } - SummaryQuality qual = tmpQual; - Map summaryCallTypeQualityFilter = new HashMap<>(); - if (this.requestParameters.getExprType() == null || this.requestParameters.getExprType().isEmpty() || - this.requestParameters.getExprType().contains(RequestParameters.ALL_VALUE)) { - summaryCallTypeQualityFilter = EnumSet.allOf(ExpressionSummary.class).stream() - .collect(Collectors.toMap(es -> es, es -> qual)); - } else { - try { - summaryCallTypeQualityFilter = this.requestParameters.getExprType().stream() - .collect(Collectors.toMap( - s -> BgeeEnum.convert(ExpressionSummary.class, s), - s -> qual)); - } catch (IllegalArgumentException e) { - log.catching(Level.DEBUG, e); - throw log.throwing(new InvalidRequestException( - "Unrecognized call types: " + this.requestParameters.getExprType())); - } - } - try { - return log.traceExit(new ExpressionCallFilter2( - summaryCallTypeQualityFilter, - geneFilter, - condFilter != null? Set.of(condFilter): null, - dataTypes, - condParams, - this.requestParameters.getObservedData() == null? null: condParams, - this.requestParameters.getObservedData())); - } catch (IllegalArgumentException e) { - log.catching(Level.ERROR, e); - throw log.throwing(new InvalidRequestException("Incorrect parameters")); - } - } + private EnumMap> loadRawDataResults(RawDataLoader rawDataLoader, EnumSet dataTypes, InformationType infoType) throws InvalidRequestException { @@ -1664,34 +1315,6 @@ private ExpressionCallFilter2 loadExprCallFilter(boolean consideringFilters, () -> new EnumMap<>(DataType.class)))); } - private List loadExprCallResults(ExpressionCallLoader callLoader) - throws InvalidRequestException { - log.traceEntry("{}", callLoader); - - Integer limit = this.requestParameters.getLimit() == null? DEFAULT_LIMIT: - this.requestParameters.getLimit(); - if (limit > LIMIT_MAX) { - throw log.throwing(new InvalidRequestException("It is not possible to request more than " - + LIMIT_MAX + " results.")); - } - Long offset = this.requestParameters.getOffset() == null? 0: - this.requestParameters.getOffset(); - if (offset != null && offset < 0) { - throw log.throwing(new InvalidRequestException("Offset cannot be less than 0.")); - } - ExprCallResultCacheKey cacheKey = new ExprCallResultCacheKey( - callLoader.getProcessedFilter().getSourceFilter(), - offset, limit); - //Suppress warnings because we are responsible for the insertion and know the generic type - @SuppressWarnings("unchecked") - List results = this.cacheService.useCacheNonAtomic( - EXPR_CALL_RESULT_CACHE_DEF, - cacheKey, - () -> callLoader.loadData(offset, limit), - COMPUTE_TIME_RESULT_CACHE_MS); - return log.traceExit(results); - } - private EnumMap loadRawDataCounts(RawDataLoader rawDataLoader, EnumSet dataTypes, EnumSet infoTypes) { log.traceEntry("{}, {}, {}", rawDataLoader, dataTypes, infoTypes); @@ -1751,14 +1374,6 @@ private EnumMap loadRawDataCounts(RawDataLoader return log.traceExit(counts); } - private long loadExprCallCount(ExpressionCallLoader callLoader) { - log.traceEntry("{}", callLoader); - return log.traceExit(this.cacheService.useCacheNonAtomic( - EXPR_CALL_COUNT_CACHE_DEF, - callLoader.getProcessedFilter().getSourceFilter(), - () -> callLoader.loadDataCount(), - COMPUTE_TIME_COUNT_CACHE_MS)); - } private EnumMap loadRawDataPostFilters(RawDataLoader rawDataLoader, EnumSet dataTypes, InformationType infoType) { @@ -1785,14 +1400,33 @@ private EnumMap loadRawDataPostFilters(RawDataLoade (v1, v2) -> {throw new IllegalStateException("Key collision impossible");}, () -> new EnumMap<>(DataType.class)))); } - private ExpressionCallPostFilter loadExprCallPostFilters(ExpressionCallLoader callLoader) { - log.traceEntry("{}", callLoader); - return log.traceExit(this.cacheService.useCacheNonAtomic( - EXPR_CALL_POST_FILTER_CACHE_DEF, - callLoader.getProcessedFilter().getSourceFilter(), - () -> callLoader.loadPostFilter(), - COMPUTE_TIME_POST_FILTER_CACHE_MS - )); + + /** + * Build an {@link ExpressionCallPostFilter} by extracting the distinct condition-parameter + * entities that appear in the given OTF propagation results. + */ + private ExpressionCallPostFilter buildPostFilterFromOtfCalls(List allCalls, + Set> condParams) { + log.traceEntry("{}, {}", allCalls, condParams); + if (allCalls == null || allCalls.isEmpty()) { + return log.traceExit(new ExpressionCallPostFilter()); + } + Map, Set> condParamEntities = new HashMap<>(); + for (ConditionParameter cp : condParams) { + Set entities = new HashSet<>(); + for (OTFExpressionCall c : allCalls) { + if (c.getCondition() == null) continue; + ComposedEntity compEnt = c.getCondition().getConditionParameterValue(cp); + if (compEnt == null) continue; + for (Object e : compEnt.getEntities()) { + if (e != null) entities.add(e); + } + } + if (!entities.isEmpty()) { + condParamEntities.put(cp, entities); + } + } + return log.traceExit(new ExpressionCallPostFilter(condParamEntities)); } private EnumMap> getColumnDescriptions(String action, @@ -1805,37 +1439,24 @@ private EnumMap> getColumnDescriptions(String if (RequestParameters.ACTION_RAW_DATA_ANNOTS.equals(action) || EXPERIMENT_PAGE_ACTION.equals(action)) { boolean withExpInfo = RequestParameters.ACTION_RAW_DATA_ANNOTS.equals(action); - dataTypeTolDescrSupplier.put(DataType.AFFYMETRIX, - () -> getAffymetrixRawDataAnnotsColumnDescriptions(withExpInfo)); dataTypeTolDescrSupplier.put(DataType.RNA_SEQ, () -> getRnaSeqRawDataAnnotsColumnDescriptions(false, withExpInfo)); dataTypeTolDescrSupplier.put(DataType.SC_RNA_SEQ, () -> getRnaSeqRawDataAnnotsColumnDescriptions(true, withExpInfo)); - //Of note, there's no experiment page for EST - dataTypeTolDescrSupplier.put(DataType.EST, - () -> getESTRawDataAnnotsColumnDescriptions()); dataTypeTolDescrSupplier.put(DataType.IN_SITU, () -> getInSituRawDataAnnotsColumnDescriptions(withExpInfo)); } else if (RequestParameters.ACTION_PROC_EXPR_VALUES.equals(action)) { - dataTypeTolDescrSupplier.put(DataType.AFFYMETRIX, - () -> getAffymetrixProcExprValuesColumnDescriptions()); dataTypeTolDescrSupplier.put(DataType.RNA_SEQ, () -> getRnaSeqProcExprValuesColumnDescriptions(false)); dataTypeTolDescrSupplier.put(DataType.SC_RNA_SEQ, () -> getRnaSeqProcExprValuesColumnDescriptions(true)); - dataTypeTolDescrSupplier.put(DataType.EST, - () -> getESTProcExprValuesColumnDescriptions()); dataTypeTolDescrSupplier.put(DataType.IN_SITU, () -> getInSituProcExprValuesColumnDescriptions()); } else if (RequestParameters.ACTION_EXPERIMENTS.equals(action)) { - dataTypeTolDescrSupplier.put(DataType.AFFYMETRIX, - () -> getAffymetrixExperimentsColumnDescriptions()); dataTypeTolDescrSupplier.put(DataType.RNA_SEQ, () -> getRnaSeqExperimentsColumnDescriptions(false)); dataTypeTolDescrSupplier.put(DataType.SC_RNA_SEQ, () -> getRnaSeqExperimentsColumnDescriptions(true)); - dataTypeTolDescrSupplier.put(DataType.EST, - () -> getESTExperimentsColumnDescriptions()); dataTypeTolDescrSupplier.put(DataType.IN_SITU, () -> getInSituExperimentsColumnDescriptions()); } else { @@ -1961,31 +1582,6 @@ private List getExprCallColumnDescriptions(Set getAffymetrixRawDataAnnotsColumnDescriptions( - boolean withExperimentInfo) { - log.traceEntry("{}", withExperimentInfo); - List colDescr = new ArrayList<>(); - if (withExperimentInfo) { - colDescr.add(new ColumnDescription("Experiment ID", null, - List.of("result.experiment.id"), - ColumnDescription.ColumnType.INTERNAL_LINK, - ColumnDescription.INTERNAL_LINK_TARGET_EXP, null, true, null, null)); - colDescr.add(new ColumnDescription("Experiment name", null, - List.of("result.experiment.name"), - ColumnDescription.ColumnType.STRING, - null, null, true, null, null)); - } - colDescr.add(new ColumnDescription("Chip ID", "Identifier of the Affymetrix chip", - List.of("result.id"), - ColumnDescription.ColumnType.STRING, - null, null, true, null, null)); - - colDescr.addAll(getConditionColumnDescriptions("result", false, false)); - colDescr.add(getAnnotsToProcExprValuesColDesc("result.experiment.id", "result.id", - null, false)); - - return log.traceExit(colDescr); - } private List getRnaSeqRawDataAnnotsColumnDescriptions(boolean isSingleCell, boolean withExperimentInfo) { log.traceEntry("{}, {}", isSingleCell, withExperimentInfo); @@ -2091,28 +1687,6 @@ private List getRnaSeqRawDataAnnotsColumnDescriptions(boolean return log.traceExit(colDescr); } - private List getESTRawDataAnnotsColumnDescriptions() { - log.traceEntry(); - List colDescr = new ArrayList<>(); - colDescr.add(new ColumnDescription("Library ID", null, - List.of("result.id"), - ColumnDescription.ColumnType.STRING, - null, null, true, null, null)); - colDescr.add(new ColumnDescription("Library name", null, - List.of("result.name"), - ColumnDescription.ColumnType.STRING, - null, null, true, null, null)); - colDescr.add(new ColumnDescription("Description", null, - List.of("result.description"), - ColumnDescription.ColumnType.STRING, - null, null, true, null, null)); - - colDescr.addAll(getConditionColumnDescriptions("result", false, false)); - colDescr.add(getAnnotsToProcExprValuesColDesc(null, "result.id", - null, false)); - - return log.traceExit(colDescr); - } private List getInSituRawDataAnnotsColumnDescriptions( boolean withExperimentInfo) { log.traceEntry("{}", withExperimentInfo); @@ -2235,47 +1809,6 @@ private ColumnDescription getExprCallsToProcExprValuesColDesc(Collection getAffymetrixProcExprValuesColumnDescriptions() { - log.traceEntry(); - List colDescr = new ArrayList<>(); - colDescr.add(new ColumnDescription("Experiment ID", null, - List.of("result.assay.experiment.id"), - ColumnDescription.ColumnType.INTERNAL_LINK, - ColumnDescription.INTERNAL_LINK_TARGET_EXP, null, true, null, null)); - colDescr.add(new ColumnDescription("Chip ID", "Identifier of the Affymetrix chip", - List.of("result.assay.id"), - ColumnDescription.ColumnType.STRING, - null, null, true, null, null)); - colDescr.add(new ColumnDescription("Probeset ID", "Identifier of the probeset for the chip type", - List.of("result.id"), - ColumnDescription.ColumnType.STRING, - null, null, true, null, null)); - colDescr.add(new ColumnDescription("Gene ID", null, - List.of("result.expressionCall.gene.geneId"), - ColumnDescription.ColumnType.INTERNAL_LINK, - ColumnDescription.INTERNAL_LINK_TARGET_GENE, null, true, - "result.expressionCall.gene.geneMappedToSameGeneIdCount", - "result.expressionCall.gene.species.id")); - colDescr.add(new ColumnDescription("Gene name", null, - List.of("result.expressionCall.gene.name"), - ColumnDescription.ColumnType.STRING, - null, null, true, null, null)); - colDescr.add(new ColumnDescription("Signal intensity", - "Normalized signal intensity of the probeset", - List.of("result.normalizedSignalIntensity"), - ColumnDescription.ColumnType.NUMERIC, - null, null, true, null, null)); - colDescr.add(new ColumnDescription("Expression p-value", - "P-value for the test of expression signal of the gene " - + "significantly different from background expression", - List.of("result.expressionCall.pValue"), - ColumnDescription.ColumnType.STRING, - null, null, true, null, null)); - - colDescr.addAll(getConditionColumnDescriptions("result.assay", false, false)); - - return log.traceExit(colDescr); - } private List getRnaSeqProcExprValuesColumnDescriptions(boolean isSingleCell) { log.traceEntry(); List colDescr = new ArrayList<>(); @@ -2329,42 +1862,6 @@ private List getRnaSeqProcExprValuesColumnDescriptions(boolea return log.traceExit(colDescr); } - private List getESTProcExprValuesColumnDescriptions() { - log.traceEntry(); - List colDescr = new ArrayList<>(); - colDescr.add(new ColumnDescription("Library ID", null, - List.of("result.assay.id"), - ColumnDescription.ColumnType.STRING, - null, null, true, null, null)); - colDescr.add(new ColumnDescription("Library name", null, - List.of("result.assay.name"), - ColumnDescription.ColumnType.STRING, - null, null, true, null, null)); - colDescr.add(new ColumnDescription("EST ID", "Identifier of the Expressed Sequence Tag", - List.of("result.id"), - ColumnDescription.ColumnType.STRING, - null, null, true, null, null)); - colDescr.add(new ColumnDescription("Gene ID", null, - List.of("result.rawCall.gene.geneId"), - ColumnDescription.ColumnType.INTERNAL_LINK, - ColumnDescription.INTERNAL_LINK_TARGET_GENE, null, true, - "result.rawCall.gene.geneMappedToSameGeneIdCount", - "result.rawCall.gene.species.id")); - colDescr.add(new ColumnDescription("Gene name", null, - List.of("result.rawCall.gene.name"), - ColumnDescription.ColumnType.STRING, - null, null, true, null, null)); - colDescr.add(new ColumnDescription("Expression p-value", - "P-value for the test of expression signal of the gene " - + "significantly different from background expression", - List.of("result.rawCall.pValue"), - ColumnDescription.ColumnType.STRING, - null, null, true, null, null)); - - colDescr.addAll(getConditionColumnDescriptions("result.assay", false, false)); - - return log.traceExit(colDescr); - } private List getInSituProcExprValuesColumnDescriptions() { log.traceEntry(); List colDescr = new ArrayList<>(); @@ -2397,25 +1894,6 @@ private List getInSituProcExprValuesColumnDescriptions() { return log.traceExit(colDescr); } - private List getAffymetrixExperimentsColumnDescriptions() { - log.traceEntry(); - - List colDescr = new ArrayList<>(); - colDescr.add(new ColumnDescription("Experiment ID", null, - List.of("result.id"), - ColumnDescription.ColumnType.INTERNAL_LINK, - ColumnDescription.INTERNAL_LINK_TARGET_EXP, null, true, null, null)); - colDescr.add(new ColumnDescription("Experiment name", null, - List.of("result.name"), - ColumnDescription.ColumnType.STRING, - null, null, true, null, null)); - colDescr.add(new ColumnDescription("Description", null, - List.of("result.description"), - ColumnDescription.ColumnType.STRING, - null, null, true, null, null)); - colDescr.add(getExpToAnnotsColDesc("result.id")); - return log.traceExit(colDescr); - } private List getRnaSeqExperimentsColumnDescriptions(boolean isSingleCell) { log.traceEntry("{}", isSingleCell); @@ -2446,34 +1924,6 @@ private List getRnaSeqExperimentsColumnDescriptions(boolean i colDescr.add(getExpToAnnotsColDesc("result.id")); return log.traceExit(colDescr); } - private List getESTExperimentsColumnDescriptions() { - log.traceEntry(); - - List colDescr = new ArrayList<>(); - colDescr.add(new ColumnDescription("Library ID", null, - List.of("result.id"), - ColumnDescription.ColumnType.STRING, - null, null, true, null, null)); - colDescr.add(new ColumnDescription("Library name", null, - List.of("result.name"), - ColumnDescription.ColumnType.STRING, - null, null, true, null, null)); - colDescr.add(new ColumnDescription("Description", null, - List.of("result.description"), - ColumnDescription.ColumnType.STRING, - null, null, true, null, null)); - //We don't use the method getExprToAnnotsColDesc here, - //because EST data have no concept of experiment. - colDescr.add(new ColumnDescription("Link to raw data annotations", - "See the raw data annotation results for this library", - null, - ColumnDescription.ColumnType.LINK_TO_RAW_DATA_ANNOTS, - null, List.of(new ColumnDescription.FilterTarget("result.id", - this.requestParameters.getUrlParametersInstance() - .getParamFilterAssayId().getName())), false, null, null)); - - return log.traceExit(colDescr); - } private List getInSituExperimentsColumnDescriptions() { log.traceEntry(); diff --git a/bgee-webapp/src/main/java/org/bgee/controller/CommandExpressionSupport.java b/bgee-webapp/src/main/java/org/bgee/controller/CommandExpressionSupport.java new file mode 100644 index 000000000..024bcef31 --- /dev/null +++ b/bgee-webapp/src/main/java/org/bgee/controller/CommandExpressionSupport.java @@ -0,0 +1,469 @@ +package org.bgee.controller; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import javax.servlet.http.HttpServletResponse; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bgee.controller.exception.InvalidRequestException; +import org.bgee.controller.user.User; +import org.bgee.controller.utils.BgeeCacheService; +import org.bgee.controller.utils.BgeeCacheService.CacheDefinition; +import org.bgee.controller.utils.BgeeCacheService.CacheType; +import org.bgee.model.BgeeEnum; +import org.bgee.model.ServiceFactory; +import org.bgee.model.anatdev.Sex.SexEnum; +import org.bgee.model.dao.api.expressiondata.call.ConditionDAO; +import org.bgee.model.expressiondata.BaseConditionFilter2.ComposedFilterIds; +import org.bgee.model.expressiondata.BaseConditionFilter2.FilterIds; +import org.bgee.model.expressiondata.baseelements.ConditionParameter; +import org.bgee.model.expressiondata.baseelements.DataType; +import org.bgee.model.expressiondata.baseelements.SummaryQuality; +import org.bgee.model.expressiondata.baseelements.SummaryCallType.ExpressionSummary; +import org.bgee.model.expressiondata.call.ConditionFilter2; +import org.bgee.model.expressiondata.call.ExpressionCallLoader; +import org.bgee.model.expressiondata.call.ExpressionCallProcessedFilter; +import org.bgee.model.expressiondata.call.ExpressionCallService; +import org.bgee.model.expressiondata.call.OTFExpressionCall; +import org.bgee.model.expressiondata.call.CallFilter.ExpressionCallFilter2; +import org.bgee.model.expressiondata.call.ExpressionCallProcessedFilter.ExpressionCallProcessedFilterConditionPart; +import org.bgee.model.gene.GeneFilter; +import org.bgee.model.job.JobService; +import org.bgee.view.ViewFactory; + +public abstract class CommandExpressionSupport extends CommandParent{ + + private final static Logger log = LogManager.getLogger(CommandExpressionSupport.class.getName()); + + public static class ExprCallCondPartProcessingCacheKey { + private final Set condFilters; + public ExprCallCondPartProcessingCacheKey(Set condFilters) { + this.condFilters = condFilters; + } + public Set getCondFilters() { + return condFilters; + } + @Override + public int hashCode() { + return Objects.hash(condFilters); + } + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + ExprCallCondPartProcessingCacheKey other = (ExprCallCondPartProcessingCacheKey) obj; + return Objects.equals(condFilters, other.condFilters); + } + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("ExprCallCondPartProcessingCacheKey [condFilters=") + .append(condFilters) + .append("]"); + return builder.toString(); + } + } + + public static class ExprCallResultCacheKey { + + private final ExpressionCallFilter2 sourceFilter; + private final Long offset; + private final Integer limit; + + public ExprCallResultCacheKey(ExpressionCallFilter2 sourceFilter, Long offset, Integer limit) { + this.sourceFilter = sourceFilter; + this.offset = offset; + this.limit = limit; + } + + public ExpressionCallFilter2 getSourceFilter() { + return sourceFilter; + } + public Long getOffset() { + return offset; + } + public Integer getLimit() { + return limit; + } + + @Override + public int hashCode() { + return Objects.hash(limit, offset, sourceFilter); + } + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + ExprCallResultCacheKey other = (ExprCallResultCacheKey) obj; + return Objects.equals(limit, other.limit) && Objects.equals(offset, other.offset) + && Objects.equals(sourceFilter, other.sourceFilter); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("ExprCallResultCacheKey [") + .append("offset=").append(offset) + .append(", limit=").append(limit) + .append(", sourceFilter=").append(sourceFilter) + .append("]"); + return builder.toString(); + } + } + + private final static CacheDefinition + EXPR_CALL_PROCESSED_COND_PART_CACHE_DEF = new CacheDefinition<>("exprCallProcessedCondPartCache", + ExprCallCondPartProcessingCacheKey.class, + ExpressionCallProcessedFilter.ExpressionCallProcessedFilterConditionPart.class, + CacheType.LRU, 20); + + //Suppress warning for List generic type to have inference working with 'List.class' + @SuppressWarnings("rawtypes") + private final static CacheDefinition + EXPR_CALL_RESULT_CACHE_DEF = new CacheDefinition<>("exprCallResultCache", + ExprCallResultCacheKey.class, List.class, CacheType.LRU, 20); + + /** + * A {@code String} to recognize the action of requesting an experiment page + * (there is no corresponding action in {@code RequestParameter}, it is triggered + * when the URL parameter {@code exp_id} is provided). + */ + + protected final static long COMPUTE_TIME_PROCESSED_COND_PART_CACHE_MS = 1000L; + private final static long COMPUTE_TIME_RESULT_CACHE_MS = 2000L; + + private final static String ID_PARAM_SUMMARY_VALUE = "SUMMARY"; + private final static Set SUMMARY_ANAT_ENTITY_IDS = Set.of( + "UBERON:0001062", + "UBERON:0000010", "UBERON:0000211", "UBERON:0000309", "UBERON:0000468", + "UBERON:0000949", "UBERON:0000990", "UBERON:0001004", "UBERON:0001007", + "UBERON:0001008", "UBERON:0001009", "UBERON:0001015", "UBERON:0001017", + "UBERON:0001032", "UBERON:0001434", "UBERON:0002193", "UBERON:0002330", + "UBERON:0002384", "UBERON:0002405", "UBERON:0002416", "UBERON:0015204"); + private final static String SUMMARY_ANAT_ENTITY_ROOT_ID = "UBERON:0001062"; + private final static Set SUMMARY_DISCARD_ANAT_ENTITY_AND_CHILDREN_IDS = + Collections.unmodifiableSet( + SUMMARY_ANAT_ENTITY_IDS.stream().filter(id -> !id.equals(SUMMARY_ANAT_ENTITY_ROOT_ID)) + .collect(Collectors.toSet())); + private final static Set SUMMARY_CELL_TYPE_IDS = Set.of(ConditionDAO.CELL_TYPE_ROOT_ID); + + public CommandExpressionSupport(HttpServletResponse response, RequestParameters requestParameters, + BgeeProperties prop, ViewFactory viewFactory, ServiceFactory serviceFactory, + JobService jobService, BgeeCacheService cacheService, User user) { + super(response, requestParameters, prop, viewFactory, serviceFactory, jobService, + cacheService, user, null, null); + } + + public CommandExpressionSupport(HttpServletResponse response, RequestParameters requestParameters, + BgeeProperties prop, ViewFactory viewFactory, ServiceFactory serviceFactory, + BgeeCacheService cacheService) { + super(response, requestParameters, prop, viewFactory, serviceFactory, null, cacheService, + null, null, null); + } + + protected ExpressionCallLoader loadExprCallLoader(boolean consideringFilters, + Set> condParams, EnumSet dataTypes) + throws InvalidRequestException { + log.traceEntry("{}, {}, {}", consideringFilters, condParams, dataTypes); + + long startTimeFilter = System.currentTimeMillis(); + ExpressionCallFilter2 filter = this.loadExprCallFilter(consideringFilters, condParams, dataTypes); + log.debug("ExpressionCallFilter2 built in {} ms", System.currentTimeMillis() - startTimeFilter); + + return log.traceExit(this.loadExprCallLoader(filter)); + } + + protected ExpressionCallLoader loadExprCallLoader(ExpressionCallFilter2 filter) { + log.traceEntry("{}", filter); + + ExpressionCallService callService = this.serviceFactory.getExpressionCallService(); + //Try to get the processed condition part of the processed filter from cache + long startTimeProcessFilter = System.currentTimeMillis(); + //TODO: remove includeChildTerms (and not excludeTermsAndChildrenIds) from the filter used + // to retrieve the cache key of the processedfilters. With OTF propagations all conditions from the condition + // graph have to be processed and the removal of children has to be done after the propagation. + // Of course filterIds has to be kept as part of the caching key. + ExpressionCallProcessedFilter processedFilter = this.cacheService.useCacheNonAtomic( + EXPR_CALL_PROCESSED_COND_PART_CACHE_DEF, + new ExprCallCondPartProcessingCacheKey(filter.getConditionFilters()), + () -> callService.processExpressionCallFilter(filter), + pf -> pf.getConditionPart(), + condPart -> callService.processExpressionCallFilter(filter, + null, condPart, null), + COMPUTE_TIME_PROCESSED_COND_PART_CACHE_MS); + log.debug("processExpressionCallFilter (via cache) completed in {} ms", + System.currentTimeMillis() - startTimeProcessFilter); + + long startTimeGetLoader = System.currentTimeMillis(); + ExpressionCallLoader loader = callService.getCallLoader(processedFilter); + log.debug("getCallLoader() completed in {} ms", System.currentTimeMillis() - startTimeGetLoader); + + return log.traceExit(loader); + } + + protected List loadExprCallResults(ExpressionCallLoader callLoader, + int defaultLimit, int limitMax) throws InvalidRequestException { + log.traceEntry("{}, {}, {}", callLoader, defaultLimit, limitMax); + + Integer limit = this.requestParameters.getLimit() == null? defaultLimit: + this.requestParameters.getLimit(); + if (limit > limitMax) { + throw log.throwing(new InvalidRequestException("It is not possible to request more than " + + limitMax + " results.")); + } + Long offset = this.requestParameters.getOffset() == null? 0: + this.requestParameters.getOffset(); + if (offset != null && offset < 0) { + throw log.throwing(new InvalidRequestException("Offset cannot be less than 0.")); + } + ExprCallResultCacheKey cacheKey = new ExprCallResultCacheKey( + callLoader.getProcessedFilter().getSourceFilter(), + offset, limit); + //Suppress warnings because we are responsible for the insertion and know the generic type + @SuppressWarnings("unchecked") + List results = this.cacheService.useCacheNonAtomic( + EXPR_CALL_RESULT_CACHE_DEF, + cacheKey, + () -> callLoader.loadDataOnTheFly().values().stream() + .flatMap(List::stream) + .collect(Collectors.toList()), + COMPUTE_TIME_RESULT_CACHE_MS); + return log.traceExit(results); + } + + private ExpressionCallFilter2 loadExprCallFilter(boolean consideringFilters, + Set> condParams, EnumSet dataTypes) + throws InvalidRequestException { + log.traceEntry("{}, {}, {}", consideringFilters, condParams, dataTypes); + + //Either there is no filtering at all, or some genes must be requested. + //Checks are made in method #processExprCallPage() + Integer speciesId = this.requestParameters.getSpeciesId(); + if (speciesId == null) { + log.debug("No filter present, returning an empty ExpressionCallFilter2"); + return log.traceExit(new ExpressionCallFilter2()); + } + GeneFilter geneFilter = new GeneFilter(speciesId, this.requestParameters.getGeneIds()); + if (geneFilter.getGeneIds().isEmpty()) { + throw log.throwing(new InvalidRequestException("Some genes must be selected.")); + } + + //Currently there is only one filter for both anat. entities and cell types + List filterAnatEntityCellTypeIds = !consideringFilters? null: + this.requestParameters.getValues( + this.requestParameters.getUrlParametersInstance().getParamFilterAnatEntity()); + List filterDevStageIds = !consideringFilters? null: + this.requestParameters.getValues( + this.requestParameters.getUrlParametersInstance().getParamFilterDevStage()); + List filterSexIds = !consideringFilters? null: + this.requestParameters.getValues( + this.requestParameters.getUrlParametersInstance().getParamFilterSex()); + List filterStrains = !consideringFilters? null: + this.requestParameters.getValues( + this.requestParameters.getUrlParametersInstance().getParamFilterStrain()); + + List sexes = this.requestParameters.getSex(); + if (sexes != null && (sexes.contains(RequestParameters.ALL_VALUE) || + sexes.containsAll( + EnumSet.allOf(SexEnum.class) + .stream() + .map(e -> e.name()) + .collect(Collectors.toSet())))) { + sexes = null; + } + + Map, ComposedFilterIds> condParamToComposedFilterIds = + new HashMap<>(); + + //-------------- + //Management of "magic" values: + //If we receive the magic value "SUMMARY", we'll use a fix list of terms. + List anatEntityIds = this.requestParameters.getAnatEntity() == null? new ArrayList<>(): + new ArrayList<>(this.requestParameters.getAnatEntity()); + boolean summaryTermsRequested = false; + if (anatEntityIds.contains(ID_PARAM_SUMMARY_VALUE)) { + summaryTermsRequested = true; + anatEntityIds.addAll(SUMMARY_ANAT_ENTITY_IDS); + anatEntityIds.remove(ID_PARAM_SUMMARY_VALUE); + } + List cellTypeIds = this.requestParameters.getCellType() == null? new ArrayList<>(): + new ArrayList<>(this.requestParameters.getCellType()); + if (cellTypeIds.contains(ID_PARAM_SUMMARY_VALUE)) { + cellTypeIds.addAll(SUMMARY_CELL_TYPE_IDS); + cellTypeIds.remove(ID_PARAM_SUMMARY_VALUE); + } + List discardAnatEntityIds = this.requestParameters.getDiscardAnatEntity() == null? + new ArrayList<>(): new ArrayList<>(this.requestParameters.getDiscardAnatEntity()); + if (discardAnatEntityIds.contains(ID_PARAM_SUMMARY_VALUE)) { + discardAnatEntityIds.addAll(SUMMARY_DISCARD_ANAT_ENTITY_AND_CHILDREN_IDS); + discardAnatEntityIds.remove(ID_PARAM_SUMMARY_VALUE); + if (!summaryTermsRequested) { + discardAnatEntityIds.removeAll(anatEntityIds); + } + } + boolean requestedAnatEntityDescendant = Boolean.TRUE.equals(this.requestParameters.getFirstValue( + this.requestParameters.getUrlParametersInstance().getParamAnatEntityDescendant())); + if (!anatEntityIds.isEmpty() && !discardAnatEntityIds.isEmpty() && !requestedAnatEntityDescendant) { + throw log.throwing(new InvalidRequestException("Only when anat. entity descendants are requested " + + "it is possible to exclude anat. entities and their children.")); + } + //And we never include child terms when the parameter comes from a filter. + boolean anatEntityDescendant = + filterAnatEntityCellTypeIds != null && !filterAnatEntityCellTypeIds.isEmpty() || + anatEntityIds.isEmpty()? false: + Boolean.TRUE.equals(this.requestParameters.getFirstValue( + this.requestParameters.getUrlParametersInstance() + .getParamAnatEntityDescendant())); + //-------------- + + //ANAT ENTITY AND CELL TYPE + FilterIds anatEntityFilter = new FilterIds<>( + //Filters override the related parameter from the form + filterAnatEntityCellTypeIds != null && !filterAnatEntityCellTypeIds.isEmpty()? + filterAnatEntityCellTypeIds: anatEntityIds, + anatEntityDescendant, + filterAnatEntityCellTypeIds != null && !filterAnatEntityCellTypeIds.isEmpty()? + null: discardAnatEntityIds, + null); + FilterIds cellTypeFilter = new FilterIds<>( + //Filters override the related parameter from the form + filterAnatEntityCellTypeIds != null && !filterAnatEntityCellTypeIds.isEmpty()? + filterAnatEntityCellTypeIds: cellTypeIds, + //And we never include child terms when the parameter comes from a filter. + filterAnatEntityCellTypeIds != null && !filterAnatEntityCellTypeIds.isEmpty() || + cellTypeIds.isEmpty()? + false: Boolean.TRUE.equals(this.requestParameters.getFirstValue( + this.requestParameters.getUrlParametersInstance() + .getParamCellTypeDescendant()))); + + + List> composedFilterIds = new ArrayList<>(List.of(anatEntityFilter)); + //In case we used the filters, anatEntityFilter and cellTypeFilter should be equal, + //and we thus don't use the cellTypeFilter + if (!anatEntityFilter.equals(cellTypeFilter)) { + composedFilterIds.add(cellTypeFilter); + } + ComposedFilterIds anatComposedFilter = new ComposedFilterIds<>( + composedFilterIds.stream() + .filter(f -> !f.isEmpty()) + .collect(Collectors.toList())); + condParamToComposedFilterIds.put(ConditionParameter.ANAT_ENTITY_CELL_TYPE, anatComposedFilter); + + //DEV. STAGE + FilterIds devStageFilter = new FilterIds<>( + //Filters override the related parameter from the form + filterDevStageIds != null && !filterDevStageIds.isEmpty()? + filterDevStageIds: this.requestParameters.getDevStage(), + //And we never include child terms when the parameter comes from a filter. + filterDevStageIds != null && !filterDevStageIds.isEmpty() || + this.requestParameters.getDevStage() == null || + this.requestParameters.getDevStage().isEmpty()? + false: Boolean.TRUE.equals(this.requestParameters.getFirstValue( + this.requestParameters.getUrlParametersInstance() + .getParamStageDescendant()))); + condParamToComposedFilterIds.put(ConditionParameter.DEV_STAGE, + new ComposedFilterIds<>(devStageFilter)); + + //SEX + FilterIds sexFilter = new FilterIds<>( + //Filters override the related parameter from the form + filterSexIds != null && !filterSexIds.isEmpty()? + filterSexIds: sexes, + //sex descendant always false: requesting descendants of the root is equivalent + //to request all sexes, in which case we don't provide requested sex IDs + false); + condParamToComposedFilterIds.put(ConditionParameter.SEX, + new ComposedFilterIds<>(sexFilter)); + + //STRAIN + FilterIds strainFilter = new FilterIds<>( + //Filters override the related parameter from the form + filterStrains != null && !filterStrains.isEmpty()? + filterStrains: this.requestParameters.getStrain(), + //strain descendant always false: requesting descendants of the root is equivalent + //to request all strains, in which case we don't provide requested strains + false); + condParamToComposedFilterIds.put(ConditionParameter.STRAIN, + new ComposedFilterIds<>(strainFilter)); + + ConditionFilter2 condFilter = null; + try { + condFilter = new ConditionFilter2(speciesId, + condParamToComposedFilterIds, + condParams, + null, + this.requestParameters.isExcludeNonInformative()); + if (condFilter.areAllFiltersExceptSpeciesEmpty()) { + //To request a species a GeneFilter is mandatory, + //so if there are no other filters, we can discard this ConditionFilter + condFilter = null; + } + } catch (IllegalArgumentException e) { + log.catching(e); + throw log.throwing(new InvalidRequestException(e.getMessage())); + } + + //ExpressionSummary and SummaryQuality + SummaryQuality tmpQual = SummaryQuality.values()[0]; + if (this.requestParameters.getDataQuality() != null && + !this.requestParameters.getDataQuality().isBlank()) { + try { + tmpQual = BgeeEnum.convert(SummaryQuality.class, this.requestParameters.getDataQuality()); + } catch (IllegalArgumentException e) { + log.catching(Level.DEBUG, e); + throw log.throwing(new InvalidRequestException( + "Unrecognized data quality: " + this.requestParameters.getDataQuality())); + } + } + SummaryQuality qual = tmpQual; + Map summaryCallTypeQualityFilter = new HashMap<>(); + if (this.requestParameters.getExprType() == null || this.requestParameters.getExprType().isEmpty() || + this.requestParameters.getExprType().contains(RequestParameters.ALL_VALUE)) { + summaryCallTypeQualityFilter = EnumSet.allOf(ExpressionSummary.class).stream() + .collect(Collectors.toMap(es -> es, es -> qual)); + } else { + try { + summaryCallTypeQualityFilter = this.requestParameters.getExprType().stream() + .collect(Collectors.toMap( + s -> BgeeEnum.convert(ExpressionSummary.class, s), + s -> qual)); + } catch (IllegalArgumentException e) { + log.catching(Level.DEBUG, e); + throw log.throwing(new InvalidRequestException( + "Unrecognized call types: " + this.requestParameters.getExprType())); + } + } + try { + return log.traceExit(new ExpressionCallFilter2( + summaryCallTypeQualityFilter, + geneFilter, + condFilter != null? Set.of(condFilter): null, + dataTypes, + condParams, + this.requestParameters.getObservedData() == null? null: condParams, + this.requestParameters.getObservedData())); + } catch (IllegalArgumentException e) { + log.catching(Level.ERROR, e); + throw log.throwing(new InvalidRequestException("Incorrect parameters")); + } + } + +} diff --git a/bgee-webapp/src/main/java/org/bgee/controller/CommandGene.java b/bgee-webapp/src/main/java/org/bgee/controller/CommandGene.java index 566179a79..6bcd3c3ee 100644 --- a/bgee-webapp/src/main/java/org/bgee/controller/CommandGene.java +++ b/bgee-webapp/src/main/java/org/bgee/controller/CommandGene.java @@ -20,13 +20,21 @@ import org.apache.logging.log4j.Logger; import org.bgee.controller.exception.InvalidRequestException; import org.bgee.controller.exception.PageNotFoundException; +import org.bgee.controller.utils.BgeeCacheService; import org.bgee.model.ServiceFactory; +import org.bgee.model.expressiondata.BaseConditionFilter2.ComposedFilterIds; import org.bgee.model.expressiondata.call.Call.ExpressionCall; import org.bgee.model.expressiondata.call.Call.ExpressionCall.ClusteringMethod; +import org.bgee.model.expressiondata.call.CallFilter.ExpressionCallFilter2; import org.bgee.model.expressiondata.call.CallService; +import org.bgee.model.expressiondata.call.ConditionFilter2; +import org.bgee.model.expressiondata.call.ExpressionCallLoader; +import org.bgee.model.expressiondata.call.ExpressionCallService; +import org.bgee.model.expressiondata.call.OTFExpressionCall; +import org.bgee.model.expressiondata.baseelements.ConditionParameter; import org.bgee.model.expressiondata.baseelements.DataType; -import org.bgee.model.expressiondata.baseelements.SummaryCallType; import org.bgee.model.expressiondata.baseelements.SummaryCallType.ExpressionSummary; +import org.bgee.model.expressiondata.baseelements.SummaryQuality; import org.bgee.model.gene.Gene; import org.bgee.model.gene.GeneFilter; import org.bgee.model.gene.GeneHomologs; @@ -48,7 +56,7 @@ * @version Bgee 15.1, Jan. 2024 * @since Bgee 13, Nov. 2015 */ -public class CommandGene extends CommandParent { +public class CommandGene extends CommandExpressionSupport { private final static Logger log = LogManager.getLogger(CommandGene.class.getName()); @@ -62,11 +70,10 @@ public class CommandGene extends CommandParent { * to retrieve in one request. Value: 100. */ private final static int DEFAULT_LIMIT = 100; - public static class GeneExpressionResponse { //Deactivated as long as we don't retrieve the Gene when there is no expression data // private final Gene gene; - private final List calls; + private final List calls; private final ExpressionSummary callType; private final EnumSet condParams; private final EnumSet dataTypes; @@ -79,7 +86,7 @@ public static class GeneExpressionResponse { * @param includingAllRedundantCalls See {@link #isIncludingAllRedundantCalls()}. * @param clustering See {@link #getClustering()}. */ - public GeneExpressionResponse(List calls, ExpressionSummary callType, + public GeneExpressionResponse(List calls, ExpressionSummary callType, EnumSet condParams, EnumSet dataTypes, boolean includingAllRedundantCalls, Map clustering) { this.includingAllRedundantCalls = includingAllRedundantCalls; @@ -98,7 +105,7 @@ public GeneExpressionResponse(List calls, ExpressionSummary call * depending on {@link #isIncludingAllRedundantCalls()}. * @see #isIncludingAllRedundantCalls() */ - public List getCalls() { + public List getCalls() { return calls; } /** @@ -217,8 +224,9 @@ public String toString() { * @param serviceFactory A {@code ServiceFactory} that provides bgee services. */ public CommandGene(HttpServletResponse response, RequestParameters requestParameters, - BgeeProperties prop, ViewFactory viewFactory, ServiceFactory serviceFactory) { - super(response, requestParameters, prop, viewFactory, serviceFactory); + BgeeProperties prop, ViewFactory viewFactory, ServiceFactory serviceFactory, + BgeeCacheService cacheService) { + super(response, requestParameters, prop, viewFactory, serviceFactory, cacheService); } @Override @@ -232,6 +240,7 @@ public void processRequest() throws Exception { GeneService geneService = serviceFactory.getGeneService(); GeneHomologsService geneHomologsService = serviceFactory.getGeneHomologsService(); CallService callService = serviceFactory.getCallService(); + ExpressionCallService expressionCallService = serviceFactory.getExpressionCallService(); //******************************************* // GENE SEARCHES @@ -285,7 +294,7 @@ public void processRequest() throws Exception { log.traceExit(); return; } if (RequestParameters.ACTION_GENE_EXPRESSION.equals(action)) { - this.processExpressionRequest(callService, display); + this.processExpressionRequest(callService, expressionCallService, display); log.traceExit(); return; } @@ -418,11 +427,13 @@ private void processXRefsRequest(GeneService geneService, GeneDisplay display) * @throws InvalidRequestException * @throws PageNotFoundException */ - private void processExpressionRequest(CallService callService, GeneDisplay display) + private void processExpressionRequest(CallService callService, + ExpressionCallService expressionCallService, GeneDisplay display) throws InvalidRequestException, PageNotFoundException { - log.traceEntry("{}, {}", callService, display); + log.traceEntry("{}, {}, {}", callService, expressionCallService, display); String geneId = requestParameters.getGeneId(); Integer speciesId = requestParameters.getSpeciesId(); + long startTime = System.currentTimeMillis(); URLParameters urlParameters = requestParameters.getUrlParametersInstance(); //Condition parameters @@ -444,11 +455,21 @@ private void processExpressionRequest(CallService callService, GeneDisplay displ throw log.throwing(new InvalidRequestException("Only one expression type can be provided")); } String requestedCallType = this.requestParameters.getExprType().iterator().next(); - try { - callType = SummaryCallType.ExpressionSummary.convertToExpression(requestedCallType); - } catch (IllegalArgumentException e) { - log.catching(e); - throw log.throwing(new InvalidRequestException("Unkown call type: " + requestedCallType)); + String expressedValue = ExpressionSummary.EXPRESSED.getStringRepresentation(); + String notExpressedValue = ExpressionSummary.NOT_EXPRESSED.getStringRepresentation(); + if (RequestParameters.ALL_VALUE.equalsIgnoreCase(requestedCallType)) { + throw log.throwing(new InvalidRequestException( + "Expression type 'all' is not supported for this endpoint. " + + "Please use either '" + expressedValue + "' or '" + notExpressedValue + "'.")); + } + if (expressedValue.equalsIgnoreCase(requestedCallType)) { + callType = ExpressionSummary.EXPRESSED; + } else if (notExpressedValue.equalsIgnoreCase(requestedCallType)) { + callType = ExpressionSummary.NOT_EXPRESSED; + } else { + throw log.throwing(new InvalidRequestException( + "Unknown call type: " + requestedCallType + ". " + + "Accepted values are '" + expressedValue + "' and '" + notExpressedValue + "'.")); } } @@ -459,10 +480,19 @@ private void processExpressionRequest(CallService callService, GeneDisplay displ if (speciesId == null || speciesId < 1) { throw log.throwing(new InvalidRequestException("Invalid species ID argument: " + speciesId)); } +// GeneExpressionResponse exprResponse = loadExpression(callType, geneId, speciesId, condParamAttrs, +// dataTypes, callService, expressionCallService, getClusteringFunction()); + log.debug("request parameters retrieved in {} ms", + System.currentTimeMillis() - startTime); + startTime = System.currentTimeMillis(); GeneExpressionResponse exprResponse = loadExpression(callType, geneId, speciesId, condParamAttrs, - dataTypes, callService, getClusteringFunction()); + dataTypes, callService, expressionCallService, null); + log.debug("expression data loaded in {} ms", + System.currentTimeMillis() - startTime); + startTime = System.currentTimeMillis(); display.displayGeneExpression(exprResponse); - + log.debug("expression data displayed in {} ms", + System.currentTimeMillis() - startTime); log.traceExit(); } @@ -547,17 +577,33 @@ private static GeneHomologs loadHomologs(String geneId, Integer speciesId, GeneH } } - private static GeneExpressionResponse loadExpression(ExpressionSummary callType, + private GeneExpressionResponse loadExpression(ExpressionSummary callType, String geneId, Integer speciesId, EnumSet condParamAttrs, EnumSet dataTypes, CallService callService, - Function, Map> clusteringFunction) - throws PageNotFoundException { - log.traceEntry("{}, {}, {}, {}, {}, {}, {}", callType, geneId, speciesId, condParamAttrs, - dataTypes, callService, clusteringFunction); + ExpressionCallService expressionCallService, + Function, Map> clusteringFunction) + throws PageNotFoundException, InvalidRequestException { + log.traceEntry("{}, {}, {}, {}, {}, {}, {}, {}", callType, geneId, speciesId, + condParamAttrs, dataTypes, callService, expressionCallService, clusteringFunction); try { - List calls = callService.loadSilverCondObservedCalls( - new GeneFilter(speciesId, geneId), condParamAttrs, callType, dataTypes); + Set> condParams = convertCondParamAttrsToCondParams(condParamAttrs); + // Build and execute an ExpressionCallLoader as in CommandData. + ExpressionCallFilter2 exprCallFilter = new ExpressionCallFilter2( + Collections.singletonMap( + ExpressionSummary.NOT_EXPRESSED.equals(callType)? + ExpressionSummary.NOT_EXPRESSED: ExpressionSummary.EXPRESSED, + SummaryQuality.SILVER), + new GeneFilter(speciesId, geneId), + buildConditionFilters(speciesId, condParams), + dataTypes, + condParams, + condParams, + true); + ExpressionCallLoader callLoader = this.loadExprCallLoader(exprCallFilter); + List calls = this.loadExprCallResults( + callLoader, DEFAULT_LIMIT, LIMIT_MAX); + if (calls.isEmpty()) { log.debug("No calls for gene {} in species {}", geneId, speciesId); //XXX: maybe we should retrieve the gene here with the method loadGenes @@ -571,10 +617,12 @@ private static GeneExpressionResponse loadExpression(ExpressionSummary callType, } //Store a clustering of ExpressionCalls - Map clustering = clusteringFunction.apply(calls); +// Map clustering = clusteringFunction.apply(calls); +// return log.traceExit(new GeneExpressionResponse(calls, callType, condParamAttrs, dataTypes, +// true, clustering)); return log.traceExit(new GeneExpressionResponse(calls, callType, condParamAttrs, dataTypes, - true, clustering)); + true, null)); //FIXME: actually catching IllegalArgumentException leads to masking real errors. //I think it was done because a missing gene can lead to an IllegalArgumentException. //To deactivate catching of IllegalArgumentException and to check! @@ -584,7 +632,51 @@ private static GeneExpressionResponse loadExpression(ExpressionSummary callType, + (speciesId != null && speciesId > 0? " in species " + speciesId: ""))); } } - + + private static Set> convertCondParamAttrsToCondParams( + Set condParamAttrs) { + log.traceEntry("{}", condParamAttrs); + Set> condParams = new HashSet<>(); + if (condParamAttrs == null || condParamAttrs.isEmpty()) { + condParams.addAll(ConditionParameter.allOf()); + return log.traceExit(condParams); + } + if (condParamAttrs.contains(CallService.Attribute.ANAT_ENTITY_ID) || + condParamAttrs.contains(CallService.Attribute.CELL_TYPE_ID)) { + condParams.add(ConditionParameter.ANAT_ENTITY_CELL_TYPE); + } + if (condParamAttrs.contains(CallService.Attribute.DEV_STAGE_ID)) { + condParams.add(ConditionParameter.DEV_STAGE); + } + if (condParamAttrs.contains(CallService.Attribute.SEX_ID)) { + condParams.add(ConditionParameter.SEX); + } + if (condParamAttrs.contains(CallService.Attribute.STRAIN_ID)) { + condParams.add(ConditionParameter.STRAIN); + } + if (condParams.isEmpty()) { + condParams.addAll(ConditionParameter.allOf()); + } + return log.traceExit(condParams); + } + + private static Set buildConditionFilters(Integer speciesId, + Set> condParams) { + log.traceEntry("{}, {}", speciesId, condParams); + + Map, ComposedFilterIds> condParamToComposedFilterIds = + new HashMap<>(); + for (ConditionParameter condParam: ConditionParameter.allOf()) { + condParamToComposedFilterIds.put(condParam, new ComposedFilterIds<>()); + } + ConditionFilter2 condFilter = new ConditionFilter2(speciesId, + condParamToComposedFilterIds, + condParams, + null, + false); + return log.traceExit(condFilter.areAllFiltersExceptSpeciesEmpty()? null: Set.of(condFilter)); + } + /** * Return the {@code Function} corresponding to the clustering method to used, * based on the properties {@link BgeeProperties#getGeneScoreClusteringMethod()} @@ -598,6 +690,7 @@ private static GeneExpressionResponse loadExpression(ExpressionSummary callType, * allowing to parameterize the clustering function. * @see ExpressionCall#generateMeanRankScoreClustering(List, ClusteringMethod, double) */ + @SuppressWarnings("unused") private Function, Map> getClusteringFunction() throws IllegalStateException { log.traceEntry(); diff --git a/bgee-webapp/src/main/java/org/bgee/controller/CommandRPackage.java b/bgee-webapp/src/main/java/org/bgee/controller/CommandRPackage.java index 9d02147fa..a4ccf8260 100644 --- a/bgee-webapp/src/main/java/org/bgee/controller/CommandRPackage.java +++ b/bgee-webapp/src/main/java/org/bgee/controller/CommandRPackage.java @@ -560,8 +560,6 @@ private void processGetAllSpecies() throws IOException { requestedAttrs.add(SPECIES_GENUS_PARAM); requestedAttrs.add(SPECIES_NAME_PARAM); requestedAttrs.add(SPECIES_COMMON_NAME_PARAM); - requestedAttrs.add(DataType.AFFYMETRIX.toString()); - requestedAttrs.add(DataType.EST.toString()); requestedAttrs.add(DataType.IN_SITU.toString()); requestedAttrs.add(DataType.RNA_SEQ.toString()); requestedAttrs.add(DataType.SC_RNA_SEQ.toString()); @@ -771,8 +769,6 @@ private static void checkSpeciesAttrs(List rqAttrs){ && !rqAttr.equals(SPECIES_GENUS_PARAM) && !rqAttr.equals(SPECIES_NAME_PARAM) && !rqAttr.equals(SPECIES_COMMON_NAME_PARAM) - && !rqAttr.equals(DataType.AFFYMETRIX.toString()) - && !rqAttr.equals(DataType.EST.toString()) && !rqAttr.equals(DataType.IN_SITU.toString()) && !rqAttr.equals(DataType.RNA_SEQ.toString()) && !rqAttr.equals(DataType.SC_RNA_SEQ.toString())){ diff --git a/bgee-webapp/src/main/java/org/bgee/controller/FrontController.java b/bgee-webapp/src/main/java/org/bgee/controller/FrontController.java index 933ad42bb..a03cdc7a1 100644 --- a/bgee-webapp/src/main/java/org/bgee/controller/FrontController.java +++ b/bgee-webapp/src/main/java/org/bgee/controller/FrontController.java @@ -1,8 +1,10 @@ package org.bgee.controller; import java.io.IOException; +import java.util.List; import java.util.Properties; import java.util.function.Supplier; +import java.util.stream.Collectors; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; @@ -27,6 +29,7 @@ import org.bgee.controller.exception.JobResultNotFoundException; import org.bgee.model.ServiceFactory; import org.bgee.model.dao.api.exception.QueryInterruptedException; +import org.bgee.model.expressiondata.call.ConditionGraphCacheService; import org.bgee.model.gene.GeneNotFoundException; import org.bgee.model.job.JobService; import org.bgee.model.job.exception.TooManyJobsException; @@ -268,7 +271,7 @@ public void doRequest(final HttpServletRequest request, final HttpServletRespons } else if (requestParameters.isAGenePageCategory()){ controller = new CommandGene(response, requestParameters, this.prop, factory, - serviceFactory); + serviceFactory, cacheService); } else if (requestParameters.isAExprComparisonPageCategory()){ controller = new CommandExpressionComparison(response, requestParameters, this.prop, factory, serviceFactory); @@ -422,6 +425,18 @@ public void initializeCaches(long sleepBetweenComputeMs) throws InterruptedExcep CommandData commandData = this.getPartialCommandData(); commandData.initializeCaches(sleepBetweenComputeMs); + + // --- Add condition graph cache initialization here --- + ConditionGraphCacheService cacheManager = new ConditionGraphCacheService(serviceFactoryProvider.get()); + + // Fetch all species IDs from DB + List speciesIds = this.serviceFactoryProvider.get().getSpeciesService() + .loadSpeciesByIds(null, false).stream() + .map(s -> s.getId()).collect(Collectors.toList()); + + log.info("Loading condition graph cache for {} species...", speciesIds.size()); + cacheManager.loadAllSpeciesGraphs(speciesIds); + log.info("ConditionGraphCache successfully initialized."); log.traceExit(); } diff --git a/bgee-webapp/src/main/java/org/bgee/view/JsonHelper.java b/bgee-webapp/src/main/java/org/bgee/view/JsonHelper.java index fc6499334..992cf10cb 100644 --- a/bgee-webapp/src/main/java/org/bgee/view/JsonHelper.java +++ b/bgee-webapp/src/main/java/org/bgee/view/JsonHelper.java @@ -7,14 +7,13 @@ import org.bgee.controller.BgeeProperties; import org.bgee.controller.RequestParameters; import org.bgee.model.XRef; -import org.bgee.model.expressiondata.rawdata.baseelements.ExperimentAssay; -import org.bgee.model.expressiondata.rawdata.baseelements.RawCall; -import org.bgee.model.expressiondata.rawdata.baseelements.RawDataAnnotation; -import org.bgee.model.expressiondata.rawdata.est.ESTCountContainer; import org.bgee.model.expressiondata.baseelements.ConditionParameter; import org.bgee.model.expressiondata.call.Condition2; import org.bgee.model.expressiondata.call.ExpressionCallPostFilter; import org.bgee.model.expressiondata.rawdata.RawDataPostFilter; +import org.bgee.model.expressiondata.rawdata.baseelements.ExperimentAssay; +import org.bgee.model.expressiondata.rawdata.baseelements.RawCall; +import org.bgee.model.expressiondata.rawdata.baseelements.RawDataAnnotation; import org.bgee.model.file.SpeciesDownloadFile; import org.bgee.model.job.Job; import org.bgee.model.species.Species; @@ -22,18 +21,17 @@ import org.bgee.view.json.adapters.BgeeTypeAdapterFactory; import org.bgee.view.json.adapters.Condition2TypeAdapter; import org.bgee.view.json.adapters.ConditionParameterTypeAdapter; -import org.bgee.view.json.adapters.SpeciesDownloadFileTypeAdapter; -import org.bgee.view.json.adapters.ESTCountContainerTypeAdapter; import org.bgee.view.json.adapters.ExperimentAssayTypeAdapter; import org.bgee.view.json.adapters.ExpressionCallPostFilterTypeAdapter; -import org.bgee.view.json.adapters.XRefTypeAdapter; import org.bgee.view.json.adapters.JobTypeAdapter; import org.bgee.view.json.adapters.RawCallTypeAdapter; import org.bgee.view.json.adapters.RawDataAnnotationTypeAdapter; import org.bgee.view.json.adapters.RawDataPostFilterTypeAdapter; import org.bgee.view.json.adapters.RequestParameterTypeAdapter; +import org.bgee.view.json.adapters.SpeciesDownloadFileTypeAdapter; import org.bgee.view.json.adapters.TopAnatResultsTypeAdapter; import org.bgee.view.json.adapters.TypeAdaptersUtils; +import org.bgee.view.json.adapters.XRefTypeAdapter; import com.google.gson.ExclusionStrategy; import com.google.gson.FieldAttributes; @@ -164,7 +162,6 @@ public JsonHelper(BgeeProperties props, RequestParameters requestParameters, this.utils, this.requestParameters.getUrlParametersInstance())) .registerTypeAdapter(ExperimentAssay.class, new ExperimentAssayTypeAdapter( this.utils)) - .registerTypeAdapter(ESTCountContainer.class, new ESTCountContainerTypeAdapter()) .registerTypeAdapter(Condition2.class, new Condition2TypeAdapter(this.utils)) .registerTypeAdapter(ConditionParameter.class, new ConditionParameterTypeAdapter()) .registerTypeAdapterFactory(new BgeeTypeAdapterFactory(s -> this.urlEncode(s), diff --git a/bgee-webapp/src/main/java/org/bgee/view/csv/CsvRPackageDisplay.java b/bgee-webapp/src/main/java/org/bgee/view/csv/CsvRPackageDisplay.java index e1d7c1088..3f9b45203 100644 --- a/bgee-webapp/src/main/java/org/bgee/view/csv/CsvRPackageDisplay.java +++ b/bgee-webapp/src/main/java/org/bgee/view/csv/CsvRPackageDisplay.java @@ -154,26 +154,6 @@ public void displaySpecies(List attrs, List speciesList) { speMap.put(header[columnNumber], species.getName()); columnNumber++; break; - case CommandRPackage.SPECIES_AFFYMETRIX_PARAM: - if (species.getDataTypesByDataSourcesForData().values().stream().flatMap(dt -> dt.stream()) - .filter(dt -> dt.equals(DataType.AFFYMETRIX)).collect(Collectors.toSet()).size() > 0) { - speMap.put(header[columnNumber], "T"); - columnNumber++; - break; - } - speMap.put(header[columnNumber], "F"); - columnNumber++; - break; - case CommandRPackage.SPECIES_EST_PARAM: - if (species.getDataTypesByDataSourcesForData().values().stream().flatMap(dt -> dt.stream()) - .filter(dt -> dt.equals(DataType.EST)).collect(Collectors.toSet()).size() > 0) { - speMap.put(header[columnNumber], "T"); - columnNumber++; - break; - } - speMap.put(header[columnNumber], "F"); - columnNumber++; - break; case CommandRPackage.SPECIES_IN_SITU_PARAM: if (species.getDataTypesByDataSourcesForData().values().stream().flatMap(dt -> dt.stream()) .filter(dt -> dt.equals(DataType.IN_SITU)).collect(Collectors.toSet()).size() > 0) { diff --git a/bgee-webapp/src/main/java/org/bgee/view/html/HtmlParentDisplay.java b/bgee-webapp/src/main/java/org/bgee/view/html/HtmlParentDisplay.java index 625ab1d09..0b0a48e3d 100644 --- a/bgee-webapp/src/main/java/org/bgee/view/html/HtmlParentDisplay.java +++ b/bgee-webapp/src/main/java/org/bgee/view/html/HtmlParentDisplay.java @@ -1363,7 +1363,7 @@ protected String getDatasetSchemaId(Integer speciesId, SpeciesDownloadFile.Categ case EXPR_CALLS_COMPLETE: hash = "expr-calls"; break; - case AFFY_DATA: + case DROPLET_BASED_DATA: hash = "proc-values-affymetrix"; break; case RNASEQ_DATA: @@ -1390,7 +1390,7 @@ protected String getDatasetSchemaName(Integer speciesId, SpeciesDownloadFile.Cat "CategoryEnum can not be null " + category)); } else if (category == SpeciesDownloadFile.Category.EXPR_CALLS_COMPLETE) { return log.traceExit("expr-calls "+speciesId); - } else if (category == SpeciesDownloadFile.Category.AFFY_DATA) { + } else if (category == SpeciesDownloadFile.Category.DROPLET_BASED_DATA) { return log.traceExit("proc-values-affymetrix "+speciesId); } else if (category == SpeciesDownloadFile.Category.RNASEQ_DATA) { return log.traceExit("proc-values-rna-seq "+speciesId); @@ -1414,8 +1414,8 @@ protected String getDatasetSchemaDescription(Integer speciesId, SpeciesDownloadF "CategoryEnum can not be null " + category)); } else if (category == SpeciesDownloadFile.Category.EXPR_CALLS_COMPLETE) { return log.traceExit("Expression calls generated by Bgee for the species "+speciesId); - } else if (category == SpeciesDownloadFile.Category.AFFY_DATA) { - return log.traceExit("Affymetrix expression values processed for the species "+speciesId); + } else if (category == SpeciesDownloadFile.Category.DROPLET_BASED_DATA) { + return log.traceExit("droplet-based single-cell expression values processed for the species "+speciesId); } else if (category == SpeciesDownloadFile.Category.RNASEQ_DATA) { return log.traceExit("RNA-Seq expression values processed for the species "+speciesId); } else { diff --git a/bgee-webapp/src/main/java/org/bgee/view/json/adapters/AffymetrixChipTypeAdapter.java b/bgee-webapp/src/main/java/org/bgee/view/json/adapters/AffymetrixChipTypeAdapter.java deleted file mode 100644 index b677c893d..000000000 --- a/bgee-webapp/src/main/java/org/bgee/view/json/adapters/AffymetrixChipTypeAdapter.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.bgee.view.json.adapters; - -import java.io.IOException; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bgee.model.expressiondata.rawdata.baseelements.RawDataAnnotation; -import org.bgee.model.expressiondata.rawdata.microarray.AffymetrixChip; - -import com.google.gson.Gson; -import com.google.gson.TypeAdapter; -import com.google.gson.stream.JsonReader; -import com.google.gson.stream.JsonWriter; - -public class AffymetrixChipTypeAdapter extends TypeAdapter { - private static final Logger log = LogManager.getLogger(AffymetrixChipTypeAdapter.class.getName()); - - private final Gson gson; - private final TypeAdaptersUtils utils; - - public AffymetrixChipTypeAdapter(Gson gson, TypeAdaptersUtils utils) { - this.gson = gson; - this.utils = utils; - } - - @Override - public void write(JsonWriter out, AffymetrixChip value) throws IOException { - log.traceEntry("{}, {}", out, value); - if (value == null) { - out.nullValue(); - log.traceExit(); return; - } - out.beginObject(); - - out.name("id").value(value.getId()); - out.name("experiment"); - this.utils.writeSimplifiedNamedEntity(out, value.getExperiment()); - out.name("annotation"); - this.gson.getAdapter(RawDataAnnotation.class).write(out, value.getAnnotation()); - - out.endObject(); - log.traceExit(); - } - - @Override - public AffymetrixChip read(JsonReader in) throws IOException { - throw log.throwing(new UnsupportedOperationException("No custom JSON reader for AffymetrixChip.")); - } -} \ No newline at end of file diff --git a/bgee-webapp/src/main/java/org/bgee/view/json/adapters/AffymetrixProbesetTypeAdapter.java b/bgee-webapp/src/main/java/org/bgee/view/json/adapters/AffymetrixProbesetTypeAdapter.java deleted file mode 100644 index 6587265bc..000000000 --- a/bgee-webapp/src/main/java/org/bgee/view/json/adapters/AffymetrixProbesetTypeAdapter.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.bgee.view.json.adapters; - -import java.io.IOException; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bgee.model.expressiondata.rawdata.baseelements.RawCall; -import org.bgee.model.expressiondata.rawdata.microarray.AffymetrixChip; -import org.bgee.model.expressiondata.rawdata.microarray.AffymetrixProbeset; - -import com.google.gson.Gson; -import com.google.gson.TypeAdapter; -import com.google.gson.stream.JsonReader; -import com.google.gson.stream.JsonWriter; - -public class AffymetrixProbesetTypeAdapter extends TypeAdapter { - private static final Logger log = LogManager.getLogger(AffymetrixProbesetTypeAdapter.class.getName()); - - private final Gson gson; - - public AffymetrixProbesetTypeAdapter(Gson gson) { - this.gson = gson; - } - - @Override - public void write(JsonWriter out, AffymetrixProbeset value) throws IOException { - log.traceEntry("{}, {}", out, value); - if (value == null) { - out.nullValue(); - log.traceExit(); return; - } - out.beginObject(); - - out.name("id").value(value.getId()); - out.name("assay"); - this.gson.getAdapter(AffymetrixChip.class).write(out, value.getAssay()); - out.name("normalizedSignalIntensity").value(value.getNormalizedSignalIntensity()); - out.name("expressionCall"); - this.gson.getAdapter(RawCall.class).write(out, value.getRawCall()); - - out.endObject(); - log.traceExit(); - } - - @Override - public AffymetrixProbeset read(JsonReader in) throws IOException { - throw log.throwing(new UnsupportedOperationException("No custom JSON reader for AffymetrixProbeset.")); - } -} diff --git a/bgee-webapp/src/main/java/org/bgee/view/json/adapters/BgeeTypeAdapterFactory.java b/bgee-webapp/src/main/java/org/bgee/view/json/adapters/BgeeTypeAdapterFactory.java index 0003a96e7..8f8f97676 100644 --- a/bgee-webapp/src/main/java/org/bgee/view/json/adapters/BgeeTypeAdapterFactory.java +++ b/bgee-webapp/src/main/java/org/bgee/view/json/adapters/BgeeTypeAdapterFactory.java @@ -12,8 +12,6 @@ import org.bgee.controller.RequestParameters; import org.bgee.model.anatdev.multispemapping.AnatEntitySimilarityAnalysis; import org.bgee.model.expressiondata.call.MultiGeneExprAnalysis; -import org.bgee.model.expressiondata.rawdata.microarray.AffymetrixChip; -import org.bgee.model.expressiondata.rawdata.microarray.AffymetrixProbeset; import org.bgee.model.gene.Gene; import org.bgee.model.gene.GeneHomologs; import org.bgee.model.ontology.Ontology; @@ -115,16 +113,6 @@ public TypeAdapter create(Gson gson, TypeToken typeToken) { TypeAdapter result = (TypeAdapter) new AnatEntitySimilarityAnalysisTypeAdapter(gson, this.utils); return log.traceExit(result); } - if (AffymetrixChip.class.isAssignableFrom(rawClass)) { - @SuppressWarnings("unchecked") - TypeAdapter result = (TypeAdapter) new AffymetrixChipTypeAdapter(gson, this.utils); - return log.traceExit(result); - } - if (AffymetrixProbeset.class.isAssignableFrom(rawClass)) { - @SuppressWarnings("unchecked") - TypeAdapter result = (TypeAdapter) new AffymetrixProbesetTypeAdapter(gson); - return log.traceExit(result); - } if (ExpressionCallResponse.class.isAssignableFrom(rawClass)) { @SuppressWarnings("unchecked") TypeAdapter result = (TypeAdapter) new ExpressionCallResponseTypeAdapter(gson, this.utils); diff --git a/bgee-webapp/src/main/java/org/bgee/view/json/adapters/Condition2TypeAdapter.java b/bgee-webapp/src/main/java/org/bgee/view/json/adapters/Condition2TypeAdapter.java index 64390a1d8..52d0f847b 100644 --- a/bgee-webapp/src/main/java/org/bgee/view/json/adapters/Condition2TypeAdapter.java +++ b/bgee-webapp/src/main/java/org/bgee/view/json/adapters/Condition2TypeAdapter.java @@ -7,6 +7,7 @@ import org.bgee.model.ComposedEntity; import org.bgee.model.NamedEntity; import org.bgee.model.anatdev.AnatEntity; +import org.bgee.model.dao.api.expressiondata.call.ConditionDAO; import org.bgee.model.expressiondata.baseelements.ConditionParameter; import org.bgee.model.expressiondata.call.Condition2; @@ -26,6 +27,12 @@ public Condition2TypeAdapter(TypeAdaptersUtils utils) { @Override public void write(JsonWriter out, Condition2 value) throws IOException { log.traceEntry("{}, {}", out, value); + write(out, value, false); + log.traceExit(); + } + + public void write(JsonWriter out, Condition2 value, boolean removeRootCellType) throws IOException { + log.traceEntry("{}, {}", out, value, removeRootCellType); if (value == null) { out.nullValue(); log.traceExit(); return; @@ -54,13 +61,15 @@ public void write(JsonWriter out, Condition2 value) throws IOException { } else { out.value("NA"); } - out.name("cellType"); - //We don't write NA anymore instead of the root of the cell types, - //because we need all values to link to processed expression values through filters - if (cellType != null/* && !ConditionDAO.CELL_TYPE_ROOT_ID.equals(cellType.getId())*/) { - this.utils.writeSimplifiedNamedEntity(out, cellType); - } else { - out.value("NA"); + if (!( removeRootCellType && cellType != null && ConditionDAO.CELL_TYPE_ROOT_ID.equals(cellType.getId()))) { + out.name("cellType"); + //We don't write NA anymore instead of the root of the cell types, + //because we need all values to link to processed expression values through filters + if (cellType != null/* && !ConditionDAO.CELL_TYPE_ROOT_ID.equals(cellType.getId())*/) { + this.utils.writeSimplifiedNamedEntity(out, cellType); + } else { + out.value("NA"); + } } } else { //For now none of the remaining cond params cannot be post-composed diff --git a/bgee-webapp/src/main/java/org/bgee/view/json/adapters/ESTCountContainerTypeAdapter.java b/bgee-webapp/src/main/java/org/bgee/view/json/adapters/ESTCountContainerTypeAdapter.java deleted file mode 100644 index 7072cfca2..000000000 --- a/bgee-webapp/src/main/java/org/bgee/view/json/adapters/ESTCountContainerTypeAdapter.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.bgee.view.json.adapters; - -import java.io.IOException; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.bgee.model.expressiondata.rawdata.est.ESTCountContainer; - -import com.google.gson.TypeAdapter; -import com.google.gson.stream.JsonReader; -import com.google.gson.stream.JsonWriter; - -/** - * We create this adapter because an {@code ESTCountContainer} does not have - * an {@code experimentCount} attribute, while all other count containers extend - * {@code RawDataCountContainerWithExperiment}, with a {@code experimentCount} attribute. - * This causes inconsistency in the responses, and we create an adapter to add a "fake" - * {@code experimentCount} attribute in the response for {@code ESTCountContainer}. - * - * @author Frederic Bastian - * @version Bgee 15.0 Dec. 2022 - * @since Bgee 15.0 Dec. 2022 - */ -public class ESTCountContainerTypeAdapter extends TypeAdapter { - private static final Logger log = LogManager.getLogger(ESTCountContainerTypeAdapter.class.getName()); - - @Override - public void write(JsonWriter out, ESTCountContainer value) throws IOException { - log.traceEntry("{}, {}", out, value); - if (value == null) { - out.nullValue(); - log.traceExit(); return; - } - out.beginObject(); - - if (value.getAssayCount() != null) { - out.name("experimentCount").value(value.getAssayCount()); - out.name("assayCount").value(value.getAssayCount()); - } - if (value.getCallCount() != null) { - out.name("callCount").value(value.getCallCount()); - } - out.name("resultFound").value(value.isResultFound()); - - out.endObject(); - log.traceExit(); - } - - @Override - public ESTCountContainer read(JsonReader in) throws IOException { - //for now, we never read JSON values - throw log.throwing(new UnsupportedOperationException("No custom JSON reader for ESTCountContainer.")); - } - -} diff --git a/bgee-webapp/src/main/java/org/bgee/view/json/adapters/ExpressionCallResponseTypeAdapter.java b/bgee-webapp/src/main/java/org/bgee/view/json/adapters/ExpressionCallResponseTypeAdapter.java index c70dc55b5..ef113bf0e 100644 --- a/bgee-webapp/src/main/java/org/bgee/view/json/adapters/ExpressionCallResponseTypeAdapter.java +++ b/bgee-webapp/src/main/java/org/bgee/view/json/adapters/ExpressionCallResponseTypeAdapter.java @@ -14,6 +14,7 @@ import org.bgee.model.expressiondata.baseelements.DataType; import org.bgee.model.expressiondata.baseelements.SummaryQuality; import org.bgee.model.expressiondata.call.Condition2; +import org.bgee.model.expressiondata.call.OTFExpressionCall; import com.google.gson.Gson; import com.google.gson.TypeAdapter; @@ -58,15 +59,13 @@ public void write(JsonWriter out, ExpressionCallResponse value) throws IOExcepti if (value.getCalls() != null) { out.name("expressionCalls"); out.beginArray(); - for (ExpressionCall2 call: value.getCalls()) { - EnumSet dataTypes = call.getCallData().stream().map(ExpressionCallData2::getDataType) - .collect(Collectors.toCollection(() -> EnumSet.noneOf(DataType.class))); + for (OTFExpressionCall call: value.getCalls()) { + EnumSet dataTypes = call.getSupportingDataTypes(); boolean highQualScore = false; - if (!SummaryQuality.BRONZE.equals(call.getSummaryQuality()) && - (dataTypes.contains(DataType.AFFYMETRIX) || - dataTypes.contains(DataType.RNA_SEQ) || - dataTypes.contains(DataType.SC_RNA_SEQ) || - call.getMeanRank().compareTo(BigDecimal.valueOf(20000)) < 0)) { + //FXIME: Need to consider the SummaryQuality once it is implemented. TO be done before Bgee 16.0 release + if (/*!SummaryQuality.BRONZE.equals(call.()) &&*/ + (dataTypes.contains(DataType.RNA_SEQ) || + dataTypes.contains(DataType.SC_RNA_SEQ))) { highQualScore = true; } out.beginObject(); @@ -78,7 +77,7 @@ public void write(JsonWriter out, ExpressionCallResponse value) throws IOExcepti out.name("expressionScore"); out.beginObject(); - out.name("expressionScore").value(call.getFormattedExpressionScore()); + out.name("expressionScore").value(call.getExpressionScore()); out.name("expressionScoreConfidence"); if (highQualScore) { out.value("high"); @@ -87,8 +86,7 @@ public void write(JsonWriter out, ExpressionCallResponse value) throws IOExcepti } out.endObject(); - String fdr = call.getPValueWithEqualDataTypes(value.getRequestedDataTypes()) - .getFormattedPValue(); + String fdr = call.getFormattedAllDatatypePValue(); out.name("fdr").value(fdr); out.name("dataTypesWithData"); @@ -103,9 +101,11 @@ public void write(JsonWriter out, ExpressionCallResponse value) throws IOExcepti out.name(d.name()).value(dataTypes.contains(d)); } out.endObject(); - - out.name("expressionState").value(call.getSummaryCallType().toString().toLowerCase()); - out.name("expressionQuality").value(call.getSummaryQuality().toString().toLowerCase()); + //FIXME: Need to be reactivated before Bgee 16.0 release. +// out.name("expressionState").value(call.getSummaryCallType().toString().toLowerCase()); +// out.name("expressionQuality").value(call.getSummaryQuality().toString().toLowerCase()); + out.name("expressionState").value(call.getAllDataTypePValue().compareTo(new BigDecimal(0.05)) <= 0 ? "expressed" : "not_expressed"); + out.name("expressionQuality").value("gold"); out.endObject(); } diff --git a/bgee-webapp/src/main/java/org/bgee/view/json/adapters/GeneExpressionResponseTypeAdapter.java b/bgee-webapp/src/main/java/org/bgee/view/json/adapters/GeneExpressionResponseTypeAdapter.java index c3fabf3bd..6f34bbab7 100644 --- a/bgee-webapp/src/main/java/org/bgee/view/json/adapters/GeneExpressionResponseTypeAdapter.java +++ b/bgee-webapp/src/main/java/org/bgee/view/json/adapters/GeneExpressionResponseTypeAdapter.java @@ -2,18 +2,22 @@ import java.io.IOException; import java.math.BigDecimal; +import java.util.Collection; import java.util.EnumSet; import java.util.Set; -import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.bgee.controller.CommandGene.GeneExpressionResponse; +import org.bgee.model.ComposedEntity; +import org.bgee.model.NamedEntity; +import org.bgee.model.anatdev.AnatEntity; +import org.bgee.model.dao.api.expressiondata.call.ConditionDAO; +import org.bgee.model.expressiondata.baseelements.ConditionParameter; import org.bgee.model.expressiondata.call.CallService; -import org.bgee.model.expressiondata.call.Call.ExpressionCall; -import org.bgee.model.expressiondata.call.CallData.ExpressionCallData; +import org.bgee.model.expressiondata.call.Condition2; +import org.bgee.model.expressiondata.call.OTFExpressionCall; import org.bgee.model.expressiondata.baseelements.DataType; -import org.bgee.model.expressiondata.baseelements.SummaryQuality; import com.google.gson.TypeAdapter; import com.google.gson.stream.JsonReader; @@ -62,27 +66,26 @@ public void write(JsonWriter out, GeneExpressionResponse value) throws IOExcepti EnumSet dataTypesWithData = EnumSet.noneOf(DataType.class); out.name("calls"); out.beginArray(); - for (ExpressionCall call: value.getCalls()) { - Set dataTypes = call.getCallData().stream().map(ExpressionCallData::getDataType) - .collect(Collectors.toCollection(() -> EnumSet.noneOf(DataType.class))); + for (OTFExpressionCall call: value.getCalls()) { + Set dataTypes = call.getSupportingDataTypes(); dataTypesWithData.addAll(dataTypes); boolean highQualScore = false; - if (!SummaryQuality.BRONZE.equals(call.getSummaryQuality()) && - (dataTypes.contains(DataType.AFFYMETRIX) || - dataTypes.contains(DataType.RNA_SEQ) || - dataTypes.contains(DataType.SC_RNA_SEQ) || - call.getMeanRank().compareTo(BigDecimal.valueOf(20000)) < 0)) { - highQualScore = true; - } + //FIXME: commented for testing OTF propagation. Summary quality should be calculated + // as part of OTF propagation before Bgee 16 release. +// if (!SummaryQuality.BRONZE.equals(call.getSummaryQuality()) && +// (dataTypes.contains(DataType.RNA_SEQ) || +// dataTypes.contains(DataType.SC_RNA_SEQ) || +// call.getMeanRank().compareTo(BigDecimal.valueOf(20000)) < 0)) { +// highQualScore = true; +// } out.beginObject(); - out.name("condition"); - this.utils.writeSimplifiedCondition(out, call.getCondition(), condParams); + this.writeGeneExpressionCondition(out, call.getCondition(), condParams); out.name("expressionScore"); out.beginObject(); - out.name("expressionScore").value(call.getFormattedExpressionScore()); + out.name("expressionScore").value(call.getExpressionScore()); out.name("expressionScoreConfidence"); if (highQualScore) { out.value("high"); @@ -91,8 +94,7 @@ public void write(JsonWriter out, GeneExpressionResponse value) throws IOExcepti } out.endObject(); - String fdr = call.getPValueWithEqualDataTypes(value.getDataTypes()) - .getFormattedPValue(); + String fdr = call.getFormattedAllDatatypePValue(); out.name("fdr").value(fdr); out.name("dataTypesWithData"); @@ -108,9 +110,11 @@ public void write(JsonWriter out, GeneExpressionResponse value) throws IOExcepti } out.endArray(); - out.name("expressionState").value(call.getSummaryCallType().toString().toLowerCase()); - out.name("expressionQuality").value(call.getSummaryQuality().toString().toLowerCase()); - out.name("clusterIndex").value(value.getClustering().get(call)); + //FIXME: these 3 values are hardcoded for the sake of testing OTF propagation. Should be computed from the data. + //FIXME: TODO BEFORE BGEE 16 RELEASE + out.name("expressionState").value(call.getExpressionScore().compareTo(new BigDecimal(0.05)) < 0 ? "not expressed" : "expressed"); + out.name("expressionQuality").value("gold"); + out.name("clusterIndex").value(0); out.endObject(); } @@ -127,6 +131,64 @@ public void write(JsonWriter out, GeneExpressionResponse value) throws IOExcepti log.traceExit(); } + private void writeGeneExpressionCondition(JsonWriter out, Condition2 condition, + Collection requestedCondParams) throws IOException { + log.traceEntry("{}, {}, {}", out, condition, requestedCondParams); + if (condition == null) { + out.nullValue(); + log.traceExit(); + return; + } + + out.beginObject(); + + boolean anatRequested = requestedCondParams == null + || requestedCondParams.contains(CallService.Attribute.ANAT_ENTITY_ID); + boolean cellTypeRequested = requestedCondParams == null + || requestedCondParams.contains(CallService.Attribute.CELL_TYPE_ID); + boolean devStageRequested = requestedCondParams == null + || requestedCondParams.contains(CallService.Attribute.DEV_STAGE_ID); + boolean sexRequested = requestedCondParams == null + || requestedCondParams.contains(CallService.Attribute.SEX_ID); + boolean strainRequested = requestedCondParams == null + || requestedCondParams.contains(CallService.Attribute.STRAIN_ID); + + ComposedEntity anatCellValue = + condition.getConditionParameterValue(ConditionParameter.ANAT_ENTITY_CELL_TYPE); + if (!anatCellValue.isEmpty()) { + AnatEntity anatEntity = anatCellValue.size() > 1? anatCellValue.getEntity(1): anatCellValue.getEntity(0); + AnatEntity cellType = anatCellValue.size() > 1? anatCellValue.getEntity(0): null; + if (anatRequested) { + out.name("anatEntity"); + this.utils.writeSimplifiedNamedEntity(out, anatEntity); + } + if (cellTypeRequested && cellType != null && + !ConditionDAO.CELL_TYPE_ROOT_ID.equals(cellType.getId())) { + out.name("cellType"); + this.utils.writeSimplifiedNamedEntity(out, cellType); + } + } + + if (devStageRequested && !condition.getConditionParameterValue(ConditionParameter.DEV_STAGE).isEmpty()) { + out.name(ConditionParameter.DEV_STAGE.getAttributeName()); + this.utils.writeSimplifiedNamedEntity(out, + condition.getConditionParameterValue(ConditionParameter.DEV_STAGE).getEntity(0)); + } + if (sexRequested && !condition.getConditionParameterValue(ConditionParameter.SEX).isEmpty()) { + out.name(ConditionParameter.SEX.getAttributeName()); + NamedEntity sexEntity = condition.getConditionParameterValue(ConditionParameter.SEX).getEntity(0); + out.value(sexEntity.getName()); + } + if (strainRequested && !condition.getConditionParameterValue(ConditionParameter.STRAIN).isEmpty()) { + out.name(ConditionParameter.STRAIN.getAttributeName()); + NamedEntity strainEntity = condition.getConditionParameterValue(ConditionParameter.STRAIN).getEntity(0); + out.value(strainEntity.getName()); + } + + out.endObject(); + log.traceExit(); + } + @Override public GeneExpressionResponse read(JsonReader in) throws IOException { //for now, we never read JSON values diff --git a/bgee-webapp/src/main/webapp/js/download.js b/bgee-webapp/src/main/webapp/js/download.js index f4b752c78..eae4299e0 100644 --- a/bgee-webapp/src/main/webapp/js/download.js +++ b/bgee-webapp/src/main/webapp/js/download.js @@ -548,13 +548,6 @@ var download = { var bgeeFullLengthDataRootURL = getUrlForFileCategory(files, "full_length_root"); var bgeeFullLengthDataFileSize = getSizeForFileCategory(files, "full_length_data"); var bgeeFullLengthAnnotFileSize = getSizeForFileCategory(files, "full_length_annot"); - - // Affymetrix processed expression values - var bgeeAffyDataFileUrl = getUrlForFileCategory(files, "affy_data"); - var bgeeAffyAnnotFileUrl = getUrlForFileCategory(files, "affy_annot"); - var bgeeAffyDataRootURL = getUrlForFileCategory(files, "affy_root"); - var bgeeAffyDataFileSize = getSizeForFileCategory(files, "affy_data"); - var bgeeAffyAnnotFileSize = getSizeForFileCategory(files, "affy_annot"); // In situ processed expression values var bgeeInSituDataFileUrl = $currentSpecies.data( "bgeeinsitudatafileurl" ); @@ -562,12 +555,6 @@ var download = { var bgeeInSituDataFileSize = $currentSpecies.data( "bgeeinsitudatafilesize" ); var bgeeInSituAnnotFileSize = $currentSpecies.data( "bgeeinsituannotfilesize" ); - // EST processed expression values - var bgeeEstDataFileUrl = $currentSpecies.data( "bgeeestdatafileurl" ); - var bgeeEstAnnotFileUrl = $currentSpecies.data( "bgeeestannotfileurl" ); - var bgeeEstDataFileSize = $currentSpecies.data( "bgeeestdatafilesize" ); - var bgeeEstAnnotFileSize = $currentSpecies.data( "bgeeestannotfilesize" ); - // Proceed to the update var numberOfSpecies = groupData.members.length; var namesOfAllSpecies = ""; diff --git a/bgee-webapp/src/test/java/org/bgee/controller/CommandDownloadTest.java b/bgee-webapp/src/test/java/org/bgee/controller/CommandDownloadTest.java index a06186d2c..82bf6f085 100644 --- a/bgee-webapp/src/test/java/org/bgee/controller/CommandDownloadTest.java +++ b/bgee-webapp/src/test/java/org/bgee/controller/CommandDownloadTest.java @@ -139,9 +139,9 @@ public static List getTestGroups() { new SpeciesDownloadFile("my/path/fileg3_2.tsv.zip", "fileg3_2.tsv.zip", null, 5000L, Category.DIFF_EXPR_ANAT_COMPLETE, 33), new SpeciesDownloadFile("my/path/fileg3_3.tsv.zip", "fileg3_3.tsv.zip", - null, 5000L, Category.AFFY_DATA, 33), + null, 5000L, Category.RNASEQ_DATA, 33), new SpeciesDownloadFile("my/path/fileg3_4.tsv.zip", "fileg3_4.tsv.zip", - null, 5000L, Category.AFFY_ANNOT, 33) + null, 5000L, Category.RNASEQ_ANNOT, 33) )); Set dlFileGroup4 = new HashSet<>(Arrays.asList( new SpeciesDownloadFile("my/path/fileg4_1.tsv.zip", "fileg4_1.tsv.zip", @@ -149,9 +149,9 @@ public static List getTestGroups() { new SpeciesDownloadFile("my/path/fileg4_2.tsv.zip", "fileg4_2.tsv.zip", null, 55000L, Category.DIFF_EXPR_ANAT_COMPLETE, 44), new SpeciesDownloadFile("my/path/fileg4_3.tsv.zip", "fileg4_3.tsv.zip", - null, 55000L, Category.AFFY_DATA, 44), + null, 55000L, Category.RNASEQ_DATA, 44), new SpeciesDownloadFile("my/path/fileg4_4.tsv.zip", "fileg4_4.tsv.zip", - null, 55000L, Category.AFFY_ANNOT, 44), + null, 55000L, Category.RNASEQ_ANNOT, 44), new SpeciesDownloadFile("my/path/fileg4_5.tsv.zip", "fileg4_5.tsv.zip", null, 55000L, Category.RNASEQ_ANNOT, 44), new SpeciesDownloadFile("my/path/fileg4_6.tsv.zip", "fileg4_6.tsv.zip",