Implement OTF (On-The-Fly) expression call propagation and remove Affymetrix/EST datatypes

bgee-core/src/main/java/org/bgee/model/CommonService.java

@@ -317,10 +317,6 @@ protected static GeneBioType mapGeneBioTypeTOToGeneBioType(GeneBioTypeTO geneBio
     protected static DataType convertDaoDataTypeToDataType(DAODataType dt) {
         log.traceEntry("{}", dt);
         switch(dt) {
-            case AFFYMETRIX:
-                return log.traceExit(DataType.AFFYMETRIX);
-            case EST:
-                return log.traceExit(DataType.EST);
             case IN_SITU:
                 return log.traceExit(DataType.IN_SITU);
             case RNA_SEQ:
@@ -342,10 +338,6 @@ public static DAODataType convertDataTypeToDAODataType(DataType dt)
             return log.traceExit((DAODataType) null);
         }
         switch(dt) {
-        case AFFYMETRIX: 
-            return log.traceExit(DAODataType.AFFYMETRIX);
-        case EST: 
-            return log.traceExit(DAODataType.EST);
         case IN_SITU: 
             return log.traceExit(DAODataType.IN_SITU);
         case RNA_SEQ: 
@@ -402,6 +394,11 @@ protected static RawDataSex mapDAORawDataSexToRawDataSex(DAORawDataSex daoRawDat
             throw log.throwing(new IllegalStateException("Unrecognized DAORawDataSex: " + daoRawDataSex));
         }
     }
+    public static DAOSex convertDAORawDataSexToDAOSex(DAORawDataSex daoRawDataSex) {
+        log.traceEntry("{}", daoRawDataSex);
+        return log.traceExit(convertSexToDAOSex(mapRawDataSexToSex(
+                mapDAORawDataSexToRawDataSex(daoRawDataSex))));
+    }
     protected static Sex mapRawDataSexToSex(RawDataSex daoRawDataSex) {
         log.traceEntry("{}", daoRawDataSex);
         if (daoRawDataSex == null) {
@@ -1054,7 +1051,7 @@ protected static DAOConditionFilter generateDAOConditionFilter(ConditionFilter c
         return log.traceExit(daoCondFilter);
     }
 
-    protected static Strain mapRawDataStrainToStrain(String strain) {
+    public static Strain mapRawDataStrainToStrain(String strain) {
         log.traceEntry("{}", strain);
         if (StringUtils.isBlank(strain)) {
             log.traceExit(); return null;

bgee-core/src/main/java/org/bgee/model/ServiceFactory.java

@@ -13,6 +13,7 @@
 import org.bgee.model.anatdev.multispemapping.DevStageSimilarityService;
 import org.bgee.model.dao.api.DAOManager;
 import org.bgee.model.expressiondata.call.CallService;
+import org.bgee.model.expressiondata.call.ConditionGraphCacheService;
 import org.bgee.model.expressiondata.call.ConditionGraphService;
 import org.bgee.model.expressiondata.call.ConditionService;
 import org.bgee.model.expressiondata.call.ExpressionCallService;
@@ -304,6 +305,14 @@ public ConditionService getConditionService() {
         return log.traceExit(new ConditionService(this));
     }
 
+    /**
+     * @return A newly instantiated {@code ConditionGraphCacheService}
+     */
+    public ConditionGraphCacheService getConditionGraphCacheService() {
+        log.traceEntry();
+        return log.traceExit(new ConditionGraphCacheService(this));
+    }
+
     /**
      * @return A newly instantiated {@code ConditionGraphService}
      */

bgee-core/src/main/java/org/bgee/model/expressiondata/baseelements/CallType.java

@@ -45,7 +45,7 @@ public interface CallType {
     public static enum Expression implements CallType, BgeeEnumField {
         EXPRESSED(Collections.unmodifiableSet(EnumSet.allOf(DataType.class))),
         NOT_EXPRESSED(Collections.unmodifiableSet(
-                EnumSet.of(DataType.AFFYMETRIX, DataType.IN_SITU, DataType.RNA_SEQ)));
+                EnumSet.of(DataType.IN_SITU, DataType.RNA_SEQ)));
         private final static Logger log = LogManager.getLogger(Expression.class.getName());
 
         /**
@@ -167,7 +167,7 @@ public static enum DiffExpression implements CallType, BgeeEnumField {
          * @see #getAllowedDataTypes()
          */
         private static final Set<DataType> DIFF_EXPR_DATA_TYPES = 
-                Collections.unmodifiableSet(EnumSet.of(DataType.AFFYMETRIX, DataType.RNA_SEQ));
+                Collections.unmodifiableSet(EnumSet.of(DataType.RNA_SEQ));
 
         @Override
         public Set<DataType> getAllowedDataTypes() {

bgee-core/src/main/java/org/bgee/model/expressiondata/baseelements/DataType.java

@@ -37,7 +37,6 @@
 //TODO: why don't we have a "ALL" data type?? This would be much cleaner than having to provide "null" 
 //everywhere...
 public enum DataType implements BgeeEnumField {
-    AFFYMETRIX("Affymetrix", true, null, true), EST("EST", false, null, false),
     IN_SITU("in situ hybridization", true, null, true),
     //Note: It is important to keep SC_RNA_SEQ before RNA_SEQ until we fix the issue of retrieving
     // experiment information consisting in different datatype (e.g both RNA_SEQ and SC_RNA_SEQ) as

bgee-core/src/main/java/org/bgee/model/expressiondata/call/CallMapping.java

@@ -320,10 +320,6 @@ private static Set<DataType> mapDAODataTypeToDataType(Set<DAODataType> dts,
             mappedDataTypes = dts.stream()
                     .map(dt -> {
                         switch(dt) {
-                        case AFFYMETRIX:
-                            return log.traceExit(DataType.AFFYMETRIX);
-                        case EST:
-                            return log.traceExit(DataType.EST);
                         case IN_SITU:
                             return log.traceExit(DataType.IN_SITU);
                         case RNA_SEQ:

bgee-core/src/main/java/org/bgee/model/expressiondata/call/CallService.java

@@ -1880,10 +1880,6 @@ private static Set<DataType> mapDAODataTypeToDataType(Set<DAODataType> dts,
             mappedDataTypes = dts.stream()
                     .map(dt -> {
                         switch(dt) {
-                        case AFFYMETRIX:
-                            return log.traceExit(DataType.AFFYMETRIX);
-                        case EST:
-                            return log.traceExit(DataType.EST);
                         case IN_SITU:
                             return log.traceExit(DataType.IN_SITU);
                         case RNA_SEQ:

bgee-core/src/main/java/org/bgee/model/expressiondata/call/CallServiceUtils.java

@@ -30,11 +30,11 @@
 import org.bgee.model.dao.api.expressiondata.DAODataType;
 import org.bgee.model.dao.api.expressiondata.call.CallObservedDataDAOFilter2;
 import org.bgee.model.dao.api.expressiondata.call.ConditionDAO;
+import org.bgee.model.dao.api.expressiondata.call.ConditionDAO.ConditionTO;
+import org.bgee.model.dao.api.expressiondata.call.ConditionDAO.ConditionTOResultSet;
 import org.bgee.model.dao.api.expressiondata.call.DAOConditionFilter2;
 import org.bgee.model.dao.api.expressiondata.call.DAOFDRPValueFilter2;
 import org.bgee.model.dao.api.expressiondata.call.DAOPropagationState;
-import org.bgee.model.dao.api.expressiondata.call.ConditionDAO.ConditionTO;
-import org.bgee.model.dao.api.expressiondata.call.ConditionDAO.ConditionTOResultSet;
 import org.bgee.model.expressiondata.BaseConditionFilter2.FilterIds;
 import org.bgee.model.expressiondata.baseelements.ConditionParameter;
 import org.bgee.model.expressiondata.baseelements.DataType;
@@ -301,7 +301,10 @@ public Set<DAOConditionFilter2> convertConditionFiltersToDAOConditionFilters(
             AnatEntityService anatEntityService, Set<Integer> consideredSpeciesIds) {
         log.traceEntry("{}, {}, {}, {}", condFilters, ontService, anatEntityService, consideredSpeciesIds);
         if (condFilters == null || condFilters.isEmpty()) {
-            return log.traceExit(new HashSet<>());
+            if(consideredSpeciesIds == null || consideredSpeciesIds.isEmpty()) {
+                return log.traceExit(new HashSet<>());
+            }
+            return log.traceExit(Set.of(new DAOConditionFilter2(consideredSpeciesIds, null, null, null, null, null, null, null)));
         }
 
         //First, in order to load appropriately the ontologies,
@@ -318,12 +321,17 @@ public Set<DAOConditionFilter2> convertConditionFiltersToDAOConditionFilters(
                     ConditionParameter.ANAT_ENTITY_CELL_TYPE).getFilterIds(0);
             FilterIds<String> cellTypeFilterIds = filter.getComposedFilterIds(
                     ConditionParameter.ANAT_ENTITY_CELL_TYPE).getFilterIds(1);
-            if (anatEntityFilterIds != null && anatEntityFilterIds.isIncludeChildTerms()) {
+            //XXX: we used to consider isIncludeChildTerms == true detect terms to retrieve for anat. enttiy,
+            //     Cell type and dev. stage.
+            //     Since Bgee 16.0 we generate the calls on the fly. All descendant condition of a requested
+            //     one have to be processed. Then we always need to retrieve all child terms of a requested term.
+            //     The subseting of condition is done later once the propagation has been done.
+            if (anatEntityFilterIds != null) {
                 anatEntityAndCellTypeIdsWithChildrenRequested.addAll(anatEntityFilterIds.getIds());
                 anatEntityAndCellTypeIdsWithChildrenRequested.addAll(anatEntityFilterIds.getExcludeTermsAndChildrenIds());
                 speciesIdsWithAnatCellChildrenRequested.add(filter.getSpeciesId());
             }
-            if (cellTypeFilterIds != null && cellTypeFilterIds.isIncludeChildTerms()) {
+            if (cellTypeFilterIds != null) {
                 anatEntityAndCellTypeIdsWithChildrenRequested.addAll(cellTypeFilterIds.getIds());
                 anatEntityAndCellTypeIdsWithChildrenRequested.addAll(cellTypeFilterIds.getExcludeTermsAndChildrenIds());
                 speciesIdsWithAnatCellChildrenRequested.add(filter.getSpeciesId());
@@ -332,24 +340,36 @@ public Set<DAOConditionFilter2> convertConditionFiltersToDAOConditionFilters(
             assert !filter.getComposedFilterIds(ConditionParameter.DEV_STAGE).isComposed();
             FilterIds<String> devStageFilterIds = filter.getComposedFilterIds(
                     ConditionParameter.DEV_STAGE).getFilterIds(0);
-            if (devStageFilterIds != null && devStageFilterIds.isIncludeChildTerms()) {
+            if (devStageFilterIds != null) {
                 devStageIdsWithChildrenRequested.addAll(devStageFilterIds.getIds());
                 devStageIdsWithChildrenRequested.addAll(devStageFilterIds.getExcludeTermsAndChildrenIds());
                 speciesIdsWithDevStageChildrenRequested.add(filter.getSpeciesId());
             }
         }
 
         //Now we load the ontologies if needed
+        long t0 = System.currentTimeMillis();
         MultiSpeciesOntology<AnatEntity, String> anatOntology = anatEntityAndCellTypeIdsWithChildrenRequested.isEmpty()?
                 null: ontService.getAnatEntityOntology(
                         speciesIdsWithAnatCellChildrenRequested, anatEntityAndCellTypeIdsWithChildrenRequested,
                         EnumSet.of(RelationType.ISA_PARTOF), false, true);
+        log.debug("getAnatEntityOntology() completed in {} ms (requested {} terms in {} species)",
+                System.currentTimeMillis() - t0,
+                anatEntityAndCellTypeIdsWithChildrenRequested.size(),
+                speciesIdsWithAnatCellChildrenRequested.size());
+        t0 = System.currentTimeMillis();
         MultiSpeciesOntology<DevStage, String> stageOntology = devStageIdsWithChildrenRequested.isEmpty()?
                 null: ontService.getDevStageOntology(
                         speciesIdsWithDevStageChildrenRequested, devStageIdsWithChildrenRequested, false, true);
+        log.debug("getDevStageOntology() completed in {} ms (requested {} terms in {} species)",
+                System.currentTimeMillis() - t0,
+                devStageIdsWithChildrenRequested.size(),
+                speciesIdsWithDevStageChildrenRequested.size());
         //There is no ontology for RawDataSex and RawDataStrain (String), really it's simply one root
         //with all other terms at the first level.
 
+        t0 = System.currentTimeMillis();
+        t0 = System.currentTimeMillis();
         Map<Integer, Set<String>> nonInformativePerSpeciesId = condFilters.stream()
                 .filter(f -> f.isExcludeNonInformative())
                 .map(f -> f.getSpeciesId()).distinct()
@@ -363,8 +383,11 @@ public Set<DAOConditionFilter2> convertConditionFiltersToDAOConditionFilters(
                               .filter(aeid -> !aeid.equals(ConditionDAO.ANAT_ENTITY_ROOT_ID) &&
                                       !aeid.equals(ConditionDAO.CELL_TYPE_ROOT_ID))
                               .collect(Collectors.toSet())));
+        log.debug("loadNonInformativeAnatEntities() completed in {} ms ({} species with exclusion)",
+                System.currentTimeMillis() - t0, nonInformativePerSpeciesId.size());
 
         //Now we have everything we need to create the DAO filters
+        t0 = System.currentTimeMillis();
         Set<DAOConditionFilter2> daoCondFilters = new HashSet<>();
         for (ConditionFilter2 filter: condFilters) {
             Set<String> anatEntityIds = new HashSet<>();
@@ -380,15 +403,13 @@ public Set<DAOConditionFilter2> convertConditionFiltersToDAOConditionFilters(
                     ConditionParameter.ANAT_ENTITY_CELL_TYPE).getFilterIds(1);
             if (anatEntityFilterIds != null) {
                 anatEntityIds.addAll(anatEntityFilterIds.getIds());
-                if (anatEntityFilterIds.isIncludeChildTerms()) {
-                    anatEntityIds.addAll(
-                            anatEntityFilterIds.getIds().stream()
-                            .flatMap(id -> anatOntology.getDescendantIds(
-                                    id, false, Collections.singleton(filter.getSpeciesId()))
-                                    .stream())
-                            .collect(Collectors.toSet())
-                    );
-                }
+                anatEntityIds.addAll(
+                        anatEntityFilterIds.getIds().stream()
+                        .flatMap(id -> anatOntology.getDescendantIds(
+                                id, false, Collections.singleton(filter.getSpeciesId()))
+                                .stream())
+                        .collect(Collectors.toSet())
+                        );
                 if (!anatEntityFilterIds.getExcludeTermsAndChildrenIds().isEmpty()) {
                     Set<String> anatEntityIdsToExclude = new HashSet<>();
                     anatEntityIdsToExclude.addAll(anatEntityFilterIds.getExcludeTermsAndChildrenIds());
@@ -398,7 +419,7 @@ public Set<DAOConditionFilter2> convertConditionFiltersToDAOConditionFilters(
                                     id, false, Collections.singleton(filter.getSpeciesId()))
                                     .stream())
                             .collect(Collectors.toSet())
-                    );
+                            );
                     anatEntityIdsToExclude.removeAll(anatEntityFilterIds.getNotToExcludeIds());
                     if (anatEntityIds.removeAll(anatEntityIdsToExclude) && anatEntityIds.isEmpty()) {
                         throw log.throwing(new IllegalArgumentException(
@@ -408,15 +429,13 @@ public Set<DAOConditionFilter2> convertConditionFiltersToDAOConditionFilters(
             }
             if (cellTypeFilterIds != null) {
                 cellTypeIds.addAll(cellTypeFilterIds.getIds());
-                if (cellTypeFilterIds.isIncludeChildTerms()) {
-                    cellTypeIds.addAll(
-                            cellTypeFilterIds.getIds().stream()
-                            .flatMap(id -> anatOntology.getDescendantIds(
-                                    id, false, Collections.singleton(filter.getSpeciesId()))
-                                    .stream())
-                            .collect(Collectors.toSet())
-                    );
-                }
+                cellTypeIds.addAll(
+                        cellTypeFilterIds.getIds().stream()
+                        .flatMap(id -> anatOntology.getDescendantIds(
+                                id, false, Collections.singleton(filter.getSpeciesId()))
+                                .stream())
+                        .collect(Collectors.toSet())
+                        );
                 if (!cellTypeFilterIds.getExcludeTermsAndChildrenIds().isEmpty()) {
                     Set<String> cellTypeIdsToExclude = new HashSet<>();
                     cellTypeIdsToExclude.addAll(cellTypeFilterIds.getExcludeTermsAndChildrenIds());
@@ -426,7 +445,7 @@ public Set<DAOConditionFilter2> convertConditionFiltersToDAOConditionFilters(
                                     id, false, Collections.singleton(filter.getSpeciesId()))
                                     .stream())
                             .collect(Collectors.toSet())
-                    );
+                            );
                     //we don't want to exclude the selected terms themselves
                     cellTypeIdsToExclude.removeAll(cellTypeFilterIds.getNotToExcludeIds());
                     if (cellTypeIds.removeAll(cellTypeIdsToExclude) && cellTypeIds.isEmpty()) {
@@ -442,32 +461,13 @@ public Set<DAOConditionFilter2> convertConditionFiltersToDAOConditionFilters(
                     ConditionParameter.DEV_STAGE).getFilterIds(0);
             if (devStageFilterIds != null) {
                 devStageIds.addAll(devStageFilterIds.getIds());
-                if (devStageFilterIds.isIncludeChildTerms()) {
                     devStageIds.addAll(
                             devStageFilterIds.getIds().stream()
                             .flatMap(id -> stageOntology.getDescendantIds(
                                     id, false, Collections.singleton(filter.getSpeciesId()))
                                     .stream())
                             .collect(Collectors.toSet())
                     );
-                }
-                if (!devStageFilterIds.getExcludeTermsAndChildrenIds().isEmpty()) {
-                    Set<String> devStageIdsToExclude = new HashSet<>();
-                    devStageIdsToExclude.addAll(devStageFilterIds.getExcludeTermsAndChildrenIds());
-                    devStageIdsToExclude.addAll(
-                            devStageFilterIds.getExcludeTermsAndChildrenIds().stream()
-                            .flatMap(id -> stageOntology.getDescendantIds(
-                                    id, false, Collections.singleton(filter.getSpeciesId()))
-                                    .stream())
-                            .collect(Collectors.toSet())
-                    );
-                    //we don't want to exclude the selected terms themselves
-                    devStageIdsToExclude.removeAll(devStageFilterIds.getNotToExcludeIds());
-                    if (devStageIds.removeAll(devStageIdsToExclude) && devStageIds.isEmpty()) {
-                        throw log.throwing(new IllegalArgumentException(
-                                "No result should be retrieved because of dev. stage exclusion"));
-                    }
-                }
             }
 
             //For now we consider there is no composition for sexes and strains
@@ -504,6 +504,8 @@ public Set<DAOConditionFilter2> convertConditionFiltersToDAOConditionFilters(
                     filter, condParamComb, daoCondFilter);
             daoCondFilters.add(daoCondFilter);
         }
+        log.debug("DAOConditionFilter2 construction loop completed in {} ms ({} filters built)",
+                System.currentTimeMillis() - t0, daoCondFilters.size());
 
         //Now we filter the daoCondFilters: if one of them target a species with no additional parameters,
         //then we discard any other filter targeting the same species