diff --git a/module/ablation.amlg b/module/ablation.amlg index d85485c63..9057a5725 100644 --- a/module/ablation.amlg +++ b/module/ablation.amlg @@ -622,6 +622,9 @@ ;Optional task_id to track progress of this long running operation. If unspecified, will use .null as the task_id. #{type "string"} task_id .null + ;optional flag, specifying whether reduced cases should be removed completely (default behavior), or when true, removed into a storage container + #{type "boolean"} + remove_cases_into_storage .false ;optional flag specifying whether this method was called internally by another method. ;Defaults to .false, assuming this will be called by an external API/user. #{type "boolean"} @@ -924,6 +927,11 @@ cases_to_remove (contained_entities (query_equals ".keeping" .null) ) )) + ;remove called by hierarchy clustering, ensure all clusters have at least 30 cases in them + (if (and remove_cases_into_storage !autoHierarchyEnabled) + (call !FilterCasesToRemoveForClustering) + ) + (if !tsTimeFeature (call !FilterCasesToRemoveForTimeSeries (assoc ensure_enough_to_remove .true)) ) @@ -934,6 +942,7 @@ (call !RemoveCases (assoc cases cases_to_remove distribute_weight_feature distribute_weight_feature + remove_cases_into_storage remove_cases_into_storage )) ;else no cases to remove, clear caches for these added features so they can be cleared out quickly @@ -1069,6 +1078,67 @@ ) ) + ;helper method to ensure that all clusters have at least 30 cases remaining in them by removing smaller cluster cases + ;from cases_to_remove and adding the latest cases that were added to the core set to cases_to_remove + !FilterCasesToRemoveForClustering + (let + (assoc + small_cluster_map + (filter + (lambda (< (current_value) 30)) + (compute_on_contained_entities + (query_not_in_entity_list cases_to_remove) + (query_value_masses ".cluster_id") + ) + ) + ) + + ;nothing to do since all clusters have at least 30 cases + (if (= 0 (size small_cluster_map)) + (conclude) + ) + + (declare (assoc + cluster_cases_to_keep + (apply "append" + (map + (lambda (let + (assoc + num_needed (current_value 1) + cluster_id (current_index 1) + ) + + (contained_entities + (query_in_entity_list cases_to_remove) + (query_equals ".cluster_id" cluster_id) + (query_max ".keeping" num_needed) + ) + )) + ;a map of cluster_id -> number of needed cases + (map (lambda (- 30 (current_value))) small_cluster_map) + ) + ) + ;last added cases for keeping will be used to pad cases for removal to replace the cluster cases needed to be kept + cases_to_pad_for_removal + (contained_entities + (query_not_in_entity_list cases_to_remove) + (query_max ".keeping" (size cluster_cases_to_keep)) + ) + )) + + (assign (assoc + cases_to_remove + (append + cases_to_pad_for_removal + (contained_entities + (query_in_entity_list cases_to_remove) + (query_not_in_entity_list cluster_cases_to_keep) + ) + ) + )) + ) + + ;helper method that merges duplicate cases during the reduce_data flow ;uses the precomputed min_neighbor_surprisal_map and neighbor_surprisals_map to find and merge duplicates diff --git a/module/analysis.amlg b/module/analysis.amlg index a5d42e028..c6852d897 100644 --- a/module/analysis.amlg +++ b/module/analysis.amlg @@ -8,6 +8,9 @@ ;Optional task_id to track progress of this long running operation. If unspecified, will use .null as the task_id. #{type "string"} task_id .null + ;optional flag, if true analyzes all subtrainees in hierarchy + #{type "boolean"} + full_hierarchy .false ) (declare (assoc saved_analyze_parameters_map (retrieve_from_entity "!savedAnalyzeParameterMap") )) @@ -19,7 +22,13 @@ task_id task_id )) - (call analyze (append saved_analyze_parameters_map {"task_id" task_id}) ) + (call analyze + (append + saved_analyze_parameters_map + {"task_id" task_id} + {"full_hierarchy" full_hierarchy} + ) + ) ) ) @@ -116,6 +125,9 @@ ;Defaults to .false, assuming this will be called by an external API/user. #{type "boolean"} internal .false + ;optional flag, if true analyzes all subtrainees in hierarchy + #{type "boolean"} + full_hierarchy .false ) (call !ValidateParameters) (call !ValidateFeatures) @@ -563,6 +575,29 @@ )) ) ) + + (if (and full_hierarchy !autoHierarchyEnabled !autoAblationEnabled) + (map + (lambda (let + (assoc + sub_hierarchy + (get + (call_entity [!traineeContainer (current_value 2)] "get_hierarchy" (assoc + internal_only .true + )) + [1 "payload" "children"] + ) + ) + ;only need to analyze subtrainee if they have their own subtrainees + (if (size sub_hierarchy) + (call_entity [!traineeContainer (current_value 1)] "auto_analyze" (assoc + full_hierarchy .true + )) + ) + )) + !autoHierarchyIds + ) + ) ) ;called on none or one action feature at a time by Analyze() @@ -861,6 +896,16 @@ )) ) + ;full_hierarchy analyze should automatically include .cluster_id if there are subtrainees + (if (and full_hierarchy !autoHierarchyEnabled !autoAblationEnabled) + (if (= 0 (size context_features)) + (assign (assoc context_features (append !trainedFeatures ".cluster_id") )) + + (not (contains_value context_features ".cluster_id")) + (accum (assoc context_features ".cluster_id")) + ) + ) + ;if there are computed features that are to be analyzed (e.g., 'similarity_conviction') append them to context_features (if (size (get !computedFeaturesMap "computed_features")) (assign (assoc @@ -1023,38 +1068,46 @@ ) )) - ;check if auto analysis is enabled and the analysis threshold should be increased - (if (and - !autoAnalyzeEnabled - (>= !dataMassChangeSinceLastAnalyze !autoAnalyzeThreshold) - ) - (let - (assoc - total_case_mass - (if use_case_weights - (compute_on_contained_entities - (query_not_equals weight_feature .null) - (query_sum weight_feature) - ) - - ;else it's 1 per case, thus the total mass is num_cases - num_cases + (declare (assoc + total_case_mass + (if use_case_weights + (compute_on_contained_entities + (query_not_equals weight_feature .null) + (query_sum weight_feature) ) + + ;else it's 1 per case, thus the total mass is num_cases + num_cases ) + )) - (assign_to_entities (assoc - !autoAnalyzeThreshold - ;!autoAnalyzeThreshold is a delta from the previous threshold. - ;if several case deletions cause the above formula to return < 0, default - ; to the default !autoAnalyzeThreshold. - (max - (- (* total_case_mass !autoAnalyzeGrowthFactorAmount) !autoAnalyzeThreshold) - 100 - ) - )) - ) + ;if there are cases that were reduced to storage prior to this analyze call, count them as well + (if (size (contained_entities !reducedCasesStorage)) + (accum (assoc + total_case_mass (size (contained_entities !reducedCasesStorage)) + )) ) + (assign_to_entities (assoc + !autoAnalyzeThreshold + (if !autoAnalyzeEnabled + ;!autoAnalyzeThreshold is a delta from the previous threshold. + ;if several case deletions cause the above formula to return < 0, default + ; to the default !autoAnalyzeThreshold. + (max + (- (* total_case_mass !autoAnalyzeGrowthFactorAmount) !autoAnalyzeThreshold) + 100 + ) + + ;else auto analyze is not enabled, this analyze was called 'out of cycle' of auto analyze + ;update the threshold relative to the current dataset mass + (max + (- (* total_case_mass !autoAnalyzeGrowthFactorAmount) total_case_mass) + 100 + ) + ) + )) + ;reset !dataMassChangeSinceLastAnalyze since we are now analyzing (assign_to_entities (assoc !dataMassChangeSinceLastAnalyze 0.0 )) diff --git a/module/dataparameters.amlg b/module/dataparameters.amlg index d451b42c5..c8e719bfc 100644 --- a/module/dataparameters.amlg +++ b/module/dataparameters.amlg @@ -483,7 +483,7 @@ (if (or (= context_features .null) (= context_features (list))) !trainedFeaturesContextKey - ;targetless, predicting a specific feature, pull the analyzed targetless parameters that included that feature + ;targetless, predicting a specific feature, pull the analyzed targetless parameters that included that feature (and (!= .null feature) (= ["targetless"] target_mode)) (call !BuildContextFeaturesKey (assoc context_features (values (append context_features feature) .true) @@ -1045,6 +1045,20 @@ ; above the threshold (in the case of rmse and mae) or below the threshold (otherwise). #{ref "AblationThresholdMap"} rel_threshold_map (assoc) + ;Ratio of smallest mass (number) of cases to total cases needed that can be considered an individual cluster when clustering + ;for auto hierarchy. When unspecified defaults to 0.05. Applicable when auto_hierarchy_enabled is true + #{type "number" exclusive_min 0} + hierarchy_clustering_ratio 0.05 + ;flag, default is false. when true, enables automatic internal hierarchy creation via clustering + #{type "boolean"} + auto_hierarchy_enabled .false + ;max depth of hierarchy, specifying the number of levels of subtrainees allowed + #{type "number" min 1} + auto_hierarchy_max_depth 5 + ;max breadth of hierarchy, specifying the number of subtrainees allowed + #{type "number" min 2} + auto_hierarchy_max_breadth 5 + ) (call !ValidateParameters) @@ -1072,6 +1086,11 @@ !autoAblationRelThresholdMap rel_threshold_map !ablationBatchSize batch_size !ablatedCasesDistributionBatchSize ablated_cases_distribution_batch_size + + !autoHierarchyEnabled auto_hierarchy_enabled + !autoHierarchyClusteringRatio hierarchy_clustering_ratio + !autoHierarchyMaxDepth auto_hierarchy_max_depth + !autoHierarchyMaxBreadth auto_hierarchy_max_breadth )) (accum_to_entities (assoc !revision 1)) diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index 6dfc43808..f5de932dc 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -13,8 +13,17 @@ ;path to this trainee as a list of path labels #{ref "TraineePath"} path_list [] + ;flag, default to false. if set to true will only output simplified internal hierarchy + #{type "boolean"} + internal_only .false ) - (call !Return (assoc payload (call !GetHierarchy) )) + (call !Return (assoc + payload + (if internal_only + (call !GetInternalHierarchy) + (call !GetHierarchy) + ) + )) ) ;Returns the full entity path to a child trainee provided its unique trainee id if it is contained in the hierarchy. @@ -203,7 +212,10 @@ (if (!= .null result) (seq - (call_entity trainee_path "initialize" (assoc trainee_id child_id)) + (call_entity trainee_path "initialize" (assoc + trainee_id child_id + filepath (or filepath (retrieve_from_entity "filepath")) + )) (call !AddChildTraineeReferences (assoc child_id child_id @@ -1245,7 +1257,10 @@ (assign (assoc entity_path_id id)) ) - (call_entity [!traineeContainer entity_path_id] "set_parent_id" (assoc parent_id !traineeId)) + (call_entity [!traineeContainer entity_path_id] "set_parent_id" (assoc + parent_id !traineeId + is_contained .true + )) (accum_to_entities (assoc !containedTraineeNameToIdMap (associate entity_path_id id) @@ -1257,12 +1272,370 @@ (call !Return) ) + !ReduceIntoHierarchy + (seq + + ;if subtrainees have not been created yet, trainee should be analyzed if it hasn't been analyzed yet + (if (and (= 0 (size !autoHierarchyIds)) (= .null !savedAnalyzeParameterMap)) + (call analyze (assoc "internal" .true)) + ) + + (declare (assoc + ;flag set to true if subtrainees will need to be created + create_subtrainees (= 0 (size !autoHierarchyIds)) + + ;copy of this trainee's parameters that would be copied to newly created subtrainees + trainee_dataparams_map (retrieve_from_entity "!dataParametersMap") + trainee_dataparams_paths (retrieve_from_entity "!!dataParametersPaths") + trainee_marginal_stats_map (retrieve_from_entity "!featureMarginalStatsMap") + trainee_expected_values_map (retrieve_from_entity "!expectedValuesMap") + )) + + (declare (assoc + reduce_payload + (call reduce_data (assoc + features features + skip_auto_analyze .true + internal .true + remove_cases_into_storage .true + )) + )) + + ;hierarchy max depth level has been reached + (if (and create_subtrainees (= !autoHierarchyDepthLevel !autoHierarchyMaxDepth)) + (seq + (assign_to_entities (assoc + !autoHierarchyCaseLoadMap + { + "self" (call !GetNumTrainingCases) + "total" (+ (get !autoHierarchyCaseLoadMap "total") (size cases)) + } + )) + + (if (not !autoHierarchyDepthReached) + (seq + ;update analyze parameters with case_weights and enable auto ablation + (assign_to_entities (assoc + !savedAnalyzeParameterMap + (modify !savedAnalyzeParameterMap + "use_case_weights" .true + "weight_feature" ".case_weight" + ) + !autoAblationEnabled .true + !autoHierarchyDepthReached .true + )) + + (call !Analyze (append + !savedAnalyzeParameterMap + (assoc + "from_react_into_features" .true + "internal" .true + ) + )) + ) + ) + + ;remove !reducedCasesStorage + (destroy_entities !reducedCasesStorage) + (conclude) + ) + ) + + ;create subtrainees from clusters if they haven't beet created yet + (declare (assoc + subtrainee_ids + (if create_subtrainees + (seq + (call react_into_features (assoc + clustering .true + clustering_min_cluster_mass (ceil (* num_cases !autoHierarchyClusteringRatio)) + use_case_weights .true + internal .true + )) + + ;ensure ablation is enabled going forward for this trainee + (assign_to_entities (assoc !autoAblationEnabled .true)) + + (assign_to_entities (assoc + !savedAnalyzeParameterMap + (modify !savedAnalyzeParameterMap + "context_features" (append (get !savedAnalyzeParameterMap "context_features") ".cluster_id") + "use_case_weights" .true + "weight_feature" ".case_weight" + ) + )) + ;prevent calling react_into_features during the analyze() + (call !Analyze (append + !savedAnalyzeParameterMap + (assoc + "from_react_into_features" .true + "internal" .true + ) + )) + + (map + (lambda (concat !traineeId "_sub" (current_value)) ) + (indices (compute_on_contained_entities (query_value_masses ".cluster_id"))) + ) + ) + + ;else re-analyze after the latest reduce if needed and use the already created subtrainee ids + (seq + (if (>= !dataMassChangeSinceLastAnalyze !autoAnalyzeThreshold) + (call !Analyze (append + !savedAnalyzeParameterMap + (assoc + ;prevent calling react_into_features during the analyze() + "from_react_into_features" .true + "internal" .true + "full_hierarchy" .true + ) + )) + ) + + !autoHierarchyIds + ) + ) + )) + + ;create subtrainees + (if create_subtrainees + (let + (assoc + cluster_ids (indices (compute_on_contained_entities (query_value_masses ".cluster_id"))) + ) + + ;combine unclustered cases into existing clusters + (if (and (contains_value cluster_ids 0) (> (size cluster_ids) 1)) + (let + (assoc + unclustered_cases (contained_entities (query_equals ".cluster_id" 0)) + ) + + (declare (assoc + forced_clusters + ;if there's only 1 actual cluster and a set of unclustered, move all the unclustered into the one cluster + (if (= 2 (size cluster_ids_map)) + (let + (assoc cluster_id (first (indices (remove cluster_ids_map 0))) ) + (map cluster_id unclustered_cases) + ) + + ||(map + (lambda (first + (call !ReactDiscriminative (assoc + context_features !trainedFeatures + context_values (retrieve_from_entity (current_value 1) !trainedFeatures) + action_features [".cluster_id"] + return_action_values_only .true + use_case_weights .true + filtering_queries [(query_not_equals ".cluster_id" 0) (query_not_in_entity_list [(current_value 3)])] + )) + )) + unclustered_cases + ) + ) + )) + + ;update cluster id for each unclustered case + (map + (lambda + (assign_to_entities (current_index) (assoc ".cluster_id" (current_value 1))) + ) + (zip unclustered_cases forced_clusters) + ) + + (assign (assoc + subtrainee_ids + (map + (lambda (concat !traineeId "_sub" (current_value)) ) + (indices (compute_on_contained_entities (query_value_masses ".cluster_id"))) + ) + )) + ) + ) + + ;if there are more clusters than allowed, redistribute the smallest clusters among the larger ones + (if (> (size subtrainee_ids) !autoHierarchyMaxBreadth) + (let + (assoc + cluster_ids_map (compute_on_contained_entities (query_value_masses ".cluster_id")) + ) + + (declare (assoc + cluster_ids_by_size + (sort + (lambda + ;decreasing order + (< + (get cluster_ids_map [(current_value 1)]) + (get cluster_ids_map [(current_value 2)]) + ) + ) + (indices cluster_ids_map) + ) + extra_cluster_ids (tail cluster_ids_by_size (- (size subtrainee_ids) !autoHierarchyMaxBreadth)) + distribute_cluster_cases (contained_entities (query_among ".cluster_id" extra_cluster_ids)) + )) + + ;predict the most similar cluster that is not among the extra_cluster_ids + (declare (assoc + forced_clusters + ||(map + (lambda (first + (call !ReactDiscriminative (assoc + context_features !trainedFeatures + context_values (retrieve_from_entity (current_value 1) !trainedFeatures) + action_features [".cluster_id"] + return_action_values_only .true + use_case_weights .true + filtering_queries [(query_not_among ".cluster_id" extra_cluster_ids) (query_not_in_entity_list [(current_value 3)])] + )) + )) + distribute_cluster_cases + ) + + )) + + ;update cluster id for each of the distribute cluster cases + (map + (lambda + (assign_to_entities (current_index) (assoc ".cluster_id" (current_value 1))) + ) + (zip distribute_cluster_cases forced_clusters) + ) + + (assign (assoc + subtrainee_ids + (map + (lambda (concat !traineeId "_sub" (current_value)) ) + (trunc cluster_ids_by_size !autoHierarchyMaxBreadth) + ) + )) + ) + ) + + (map + (lambda (let + (assoc subtrainee_id (current_value 1)) + (call create_subtrainee (assoc + filepath (retrieve_from_entity "filepath") + path [subtrainee_id] + child_id subtrainee_id + )) + + (set_entity_permissions [!traineeContainer subtrainee_id] .true) + + (call_entity [!traineeContainer subtrainee_id] "set_feature_attributes" (assoc + feature_attributes (keep !featureAttributes features) + )) + + ;set the minimal params needed for reacts on subtrainee by copying this trainee's parameters + (call_entity [!traineeContainer subtrainee_id] "set_subtrainee_params_for_react" (assoc + trainee_dataparams_map trainee_dataparams_map + trainee_dataparams_paths trainee_dataparams_paths + trainee_marginal_stats_map trainee_marginal_stats_map + trainee_expected_values_map trainee_expected_values_map + auto_hierarchy_depth_level (+ 1 !autoHierarchyDepthLevel) + )) + + (call_entity [!traineeContainer subtrainee_id] "set_auto_ablation_params" (assoc + ;auto_ablation_enabled .false + min_num_cases !autoAblationMinNumCases + hierarchy_clustering_ratio !autoHierarchyClusteringRatio + auto_hierarchy_enabled .true + max_num_cases !autoAblationMaxNumCases + reduce_max_cases !postReduceMaxCases + auto_hierarchy_max_depth !autoHierarchyMaxDepth + auto_hierarchy_max_breadth !autoHierarchyMaxBreadth + )) + )) + subtrainee_ids + ) + + (assign_to_entities (assoc !autoHierarchyIds (sort subtrainee_ids) )) + ) + ) + + ;grab all the reduces cases slated for removal from storage + (declare (assoc + reduced_cases_values + (map + (lambda (retrieve_from_entity [!reducedCasesStorage (current_value 1)] features)) + (contained_entities !reducedCasesStorage) + ) + )) + + ;train subtrainees + (call !TrainSubtraineeClusterIndices (assoc + cases reduced_cases_values + ;predict clusters for all cases in (contained_entities !reducedCasesStorage) + predicted_clusters + ||(map + (lambda (first + (call !ReactDiscriminative (assoc + context_features features + context_values (current_value 1) + action_features [".cluster_id"] + return_action_values_only .true + use_case_weights .true + )) + )) + reduced_cases_values + ) + )) + + (assign_to_entities (assoc + !autoHierarchyCaseLoadMap + { + "self" (call !GetNumTrainingCases) + "total" (+ (get !autoHierarchyCaseLoadMap "total") (size cases)) + } + )) + + ;remove !reducedCasesStorage + (destroy_entities !reducedCasesStorage) + ) + + !TrainSubtraineeClusterIndices + (map + (lambda (let + (assoc + cluster_id (current_value 1) + subtrainee_id (concat !traineeId "_sub" (current_value 1)) + cluster_indices + (filter + (lambda (= cluster_id (get predicted_clusters (current_value)))) + (indices predicted_clusters) + ) + ) + + (call_entity [!traineeContainer subtrainee_id] "train" (assoc + features features + cases (unzip cases cluster_indices) + session "none" + ;allow training on reserved features to preserve all case data + allow_training_reserved_features .true + )) + )) + (values predicted_clusters .true) + ) + + + set_subtrainee_params_for_react + (assign_to_entities (assoc + !dataParametersMap trainee_dataparams_map + !dataParametersPaths trainee_dataparams_paths + !featureMarginalStatsMap trainee_marginal_stats_map + !expectedValuesMap trainee_expected_values_map + !autoHierarchyDepthLevel auto_hierarchy_depth_level + )) ;Helper method to create the actual full contained entity path by interleaving !traineeContainer with each name in the path !ConvertNamePathToEntityPath (weave (range !traineeContainer 1 (size path) 1) path) - ;method to recurse down the the hierarchy to output the currently contained hierarchy as a nested assoc + ;method to recurse down the hierarchy to output the currently contained hierarchy as a nested assoc ;with .false for trainees that are stored independently !GetHierarchy (append @@ -1270,7 +1643,8 @@ (assoc "id" (retrieve_from_entity "!traineeId") "path" path_list - "contained" .true + "contained" !traineeIsContained + "ablation" !autoAblationEnabled "children" (map (lambda @@ -1296,14 +1670,35 @@ ) "contained" .false "children" [] + "num_cases" .null ) ) ) (indices !childTraineeIsContainedMap) ) + "num_cases" (call !GetNumTrainingCases) ) ) + ;recurse down the hierarchy to output the internally contained hierarchy as a nested assoc with only the trainees and number of cases for each + !GetInternalHierarchy + (assoc + "num_cases" (call !GetNumTrainingCases) + "children" + (map + (lambda + (get + (call_entity (list !traineeContainer (current_index 1)) "get_hierarchy" (assoc + internal_only .true + )) + [1 "payload"] + ) + ) + ;only keep internal subtrainees + (filter .true !childTraineeIsContainedMap .true) + ) + ) + ;Returns the full entity path to a child trainee provided its unique trainee id if it is contained in the hierarchy. ;Iterates down the hierarchy searching for a trainee that matches the specified id, returns null if not found or ;a string error if found but trainee is stored externally as an independent trainee. @@ -1514,6 +1909,7 @@ ;set the !parentId for the child (call_entity child_entity_path "set_parent_id" (assoc parent_id (call_entity path_to_parent "get_trainee_id") + is_contained is_contained )) ) diff --git a/module/react.amlg b/module/react.amlg index ce4d1f9af..4afe306a2 100644 --- a/module/react.amlg +++ b/module/react.amlg @@ -1735,21 +1735,28 @@ exp_value ) - ;else interpolate the result from the nearest neighbors - (call !InterpolateActionValues (assoc - action_feature (first action_features) - candidate_case_ids (first local_data_cases_tuple) - candidate_case_weights (get local_data_cases_tuple 1) - candidate_case_values (last local_data_cases_tuple) - allow_nulls allow_nulls - feature_has_continuous_nulls - (if allow_nulls - (and - (not (contains_index !nominalsMap (first action_features))) - (!= .false (get !featureNullRatiosMap [(first action_features) "has_nulls"])) + (if output_raw_influential_cases_only + (zip + (first local_data_cases_tuple) + (get local_data_cases_tuple 1) + ) + + ;else interpolate the result from the nearest neighbors + (call !InterpolateActionValues (assoc + action_feature (first action_features) + candidate_case_ids (first local_data_cases_tuple) + candidate_case_weights (get local_data_cases_tuple 1) + candidate_case_values (last local_data_cases_tuple) + allow_nulls allow_nulls + feature_has_continuous_nulls + (if allow_nulls + (and + (not (contains_index !nominalsMap (first action_features))) + (!= .false (get !featureNullRatiosMap [(first action_features) "has_nulls"])) + ) ) - ) - )) + )) + ) ) ) diff --git a/module/react_discriminative.amlg b/module/react_discriminative.amlg index dd00b1639..dbfcbf808 100644 --- a/module/react_discriminative.amlg +++ b/module/react_discriminative.amlg @@ -1,5 +1,40 @@ ;Contains methods for discriminative (prediction) reacting. { + !ReactDiscriminativeParameters + (assoc + output_raw_influential_cases_only output_raw_influential_cases_only + context_features context_features + context_values context_values + action_features action_features + action_values action_values + details details + skip_encoding skip_encoding + skip_decoding skip_decoding + num_features_returned num_features_returned + extra_features extra_features + match_on_context_features match_on_context_features + ignore_case ignore_case + focal_case focal_case + tie_break_random_seed tie_break_random_seed + allow_nulls allow_nulls + return_action_values_only return_action_values_only + force_targetless force_targetless + data_params_map data_params_map + weight_feature weight_feature + use_case_weights use_case_weights + case_indices case_indices + leave_case_out leave_case_out + goal_features_map goal_features_map + preserve_feature_values preserve_feature_values + new_case_threshold new_case_threshold + has_dependent_features has_dependent_features + impute_react impute_react + filtering_queries filtering_queries + use_enabled_hierarchy use_enabled_hierarchy + ) + + ReactDiscriminative (call !ReactDiscriminative @(target .true "!ReactDiscriminativeParameters") ) + ;reacts to the context specified: computes the next action from replays given the current context ; context_features: list of context features ; context_values: current values of the world state @@ -78,6 +113,7 @@ has_dependent_features !hasDependentFeatures impute_react .false filtering_queries (list) + use_enabled_hierarchy .true ;local variables, should not be passed in as a parameter valid_weight_feature .false @@ -330,6 +366,123 @@ ) )) + + (if (and use_enabled_hierarchy !autoHierarchyEnabled (size !autoHierarchyIds) (!= ".cluster_id" (first action_features))) + (let + (assoc + clustering_prediction + (call !ReactDiscriminative (assoc + context_features context_features + context_values context_values + action_features [".cluster_id"] + skip_encoding .true + skip_decoding .true + details { "categorical_action_probabilities" .true} + use_case_weights .true + output_raw_influential_cases_only .false + use_enabled_hierarchy .false + )) + ) + + (declare (assoc + cap (get clustering_prediction ["categorical_action_probabilities" ".cluster_id"]) + )) + + (conclude + ;iterpolate among the involved clusters, union of all the raw influences + (let + (assoc + relevant_cluster_cases_map + (map + (lambda (let + (assoc subtrainee_id (concat !traineeId "_sub" (current_index 1)) ) + (get + (call_entity [!traineeContainer subtrainee_id] + "ReactDiscriminative" + (modify + @(target .true "!ReactDiscriminativeParameters") + "output_raw_influential_cases_only" .true + ) + ) + ["action_values" 0] + ) + )) + cap + ) + local_cases_map + (get + (call !ReactDiscriminative + (modify + @(target .true "!ReactDiscriminativeParameters") + "output_raw_influential_cases_only" .true + "use_enabled_hierarchy" .false + "use_case_weights" .false + ) + ) + ["action_values" 0] + ) + ) + + (declare (assoc + combined_influences_map + (append + ;combine all the subtrainee cases and influences into one assoc where the indices are pair of [subtrainee_id case_id] + (apply "append" (values + (map + (lambda (let + (assoc sub_id (concat !traineeId "_sub" (current_index 1)) ) + + ;change the assoc indices to be a pair such that the assoc is now [subtrainee_id case_id] -> influence weight + (zip + (map (lambda (append !traineeContainer sub_id (current_value )) ) (indices (current_value))) + (values (current_value)) + ) + )) + relevant_cluster_cases_map + ) + )) + local_cases_map + ) + k_parameter (get data_params_map "k") + )) + + ;sort subtrainee cases by their influences + (declare (assoc + sub_cases + (sort + (lambda + ;decreasing order + (< + (get combined_influences_map [(current_value 1)]) + (get combined_influences_map [(current_value 2)]) + ) + ) + (indices combined_influences_map) + ) + )) + + ;output only the 20 most relevant cases + (if output_raw_influential_cases_only + { "action_values" [ + (keep combined_influences_map (trunc sub_cases 20)) + ]} + + ;else interpolate between the results + (call !DynamicBandwidthFilterAndInterpolate (assoc + action_feature (first action_features) + sub_cases sub_cases + probs (unzip combined_influences_map sub_cases) + marginal_cutoff (if (~ [] k_parameter) (first k_parameter) ) + min_k (if (~ [] k_parameter) (get k_parameter 1) ) + static_k (if (~ 0 k_parameter) k_parameter) + )) + ) + ) + + ) + ) + ) + ;if there are dependent context features, and the action is dependent, precompute all residuals for dependent continuous features (if dependent_features_map (if (contains_index dependent_features_map (first action_features)) diff --git a/module/react_utilities.amlg b/module/react_utilities.amlg index 1e200b176..9fb0ed5e4 100644 --- a/module/react_utilities.amlg +++ b/module/react_utilities.amlg @@ -825,6 +825,17 @@ (assoc set_valid_weight_feature .true ) + + (if output_raw_influential_cases_only + ;this is a hierarchy leaf node that is not splitting anymore, use weights for reacts + (if (and use_enabled_hierarchy !autoHierarchyDepthReached) + (assign (assoc + use_case_weights .true + weight_feature ".case_weight" + )) + ) + ) + (if (= .null use_case_weights) (if (or (= (get data_params_map "paramPath") [".default"]) (= (get data_params_map "paramPath") .null)) ;if default params, no case weights @@ -1469,4 +1480,87 @@ (zip features) ) + + !DynamicBandwidthFilterAndInterpolate + (declare + (assoc + action_feature .null + sub_cases [] + probs [] + marginal_cutoff 0.05 + min_k 5 + ) + + (declare (assoc + num_probs (size probs) + total_prob (first probs) + first_prob (first probs) + )) + + (declare (assoc + num_relevant + (while (< (current_index) num_probs) + (if (> (current_index) 0) + (let + (assoc prob (get probs (current_index 1)) ) + (accum (assoc total_prob prob )) + + (if (and + (> total_prob 1.25) + (or + ;marginal probability is less than marginal cutoff + (< (/ prob total_prob) marginal_cutoff) + ;delta to first prob is >= 20 %, i.e. the difference is large enough to not have to accumulate to reach the marginal cutoff. + (>= (/ (- first_prob prob) first_prob) 0.2) + ) + ;and it's different from the previous probability + ;to ensure all equidistant cases are considered + (!= prob (get probs (- (current_index) 1) ) ) + ) + (conclude (current_index) ) + ) + ) + ) + + (+ (current_index) 1) + ) + )) + + (if (< num_relevant min_k) + (assign (assoc num_relevant min_k)) + ) + + (if (!= num_relevant num_probs) + (assign (assoc + sub_cases (trunc sub_cases num_relevant) + probs (trunc probs num_relevant) + )) + ) + + ;output same format as ReactDiscriminative + { "action_values" [ + (call !InterpolateActionValues (assoc + action_feature action_feature + candidate_case_ids sub_cases + candidate_case_weights probs + candidate_case_values + (map + (lambda + ;retrieve action feature value from subtrainee case or from current dataset case + (retrieve_from_entity (current_value) action_feature) + ) + sub_cases + ) + allow_nulls allow_nulls + feature_has_continuous_nulls + (if allow_nulls + (and + (not (contains_index !nominalsMap action_feature)) + (!= .false (get !featureNullRatiosMap [action_feature "has_nulls"])) + ) + ) + )) + ] } + ) + } \ No newline at end of file diff --git a/module/remove_cases.amlg b/module/remove_cases.amlg index 819d155dd..afb8f8305 100644 --- a/module/remove_cases.amlg +++ b/module/remove_cases.amlg @@ -176,11 +176,13 @@ ;parameters: ; cases: list of case ids to remove ; distribute_weight_feature: name of feature into which to distribute the removed cases' weights to their neighbors. + ; remove_cases_into_storage: flag, when true will remove cases into a storage container instead of deleting them !RemoveCases (declare (assoc cases (list) distribute_weight_feature .null + remove_cases_into_storage .false ) (if (= 0 (size cases)) (conclude)) @@ -308,7 +310,20 @@ ;remove all the cases after clearing the query caches (reclaim_resources .null .false .true) - (apply "destroy_entities" cases) + (if remove_cases_into_storage + (seq + (if (not (contains_entity !reducedCasesStorage)) + (create_entities !reducedCasesStorage {}) + ) + (map + (lambda (move_entities (current_value) [!reducedCasesStorage (current_value 1)])) + cases + ) + ) + + ;else simply delete the cases + (apply "destroy_entities" cases) + ) ;dataset has changed so clear out these cached value diff --git a/module/train.amlg b/module/train.amlg index 30c4b4170..e6b6a08ea 100644 --- a/module/train.amlg +++ b/module/train.amlg @@ -53,11 +53,43 @@ (call !ValidateParameters) + + (declare (assoc num_cases (call !GetNumTrainingCases) )) + + ;if going to be training more cases than are allowed to fit into a dataset for reduction, + ;split the training data into chunks to allow reduction and hierarchy to activate + (if (and + !autoHierarchyEnabled + (not !autoAblationEnabled) + (> (+ num_cases (size cases)) !autoAblationMaxNumCases) + (not (and train_weights_only accumulate_weight_feature)) + ) + (conclude (call !SplitLargeTrainForHierarchy)) + ) + ;unsure that session is set to some string value (if (= .null session) (assign (assoc session "none")) ) + (declare (assoc + predicted_clusters + (if (and !autoHierarchyEnabled !autoAblationEnabled) + ||(map + (lambda (first + (call !ReactDiscriminative (assoc + context_features features + context_values (current_value 1) + action_features [".cluster_id"] + return_action_values_only .true + use_case_weights .true + )) + )) + cases + ) + ) + )) + (assign (assoc accumulate_weight_feature (if (and @@ -303,8 +335,6 @@ ) ) - (declare (assoc num_cases (call !GetNumTrainingCases) )) - (declare (assoc skip_ablation (call !CanTrainAblationBeSkipped) ;if accumulating weight feature, store the index of that weight feature @@ -456,6 +486,39 @@ ".next_trained_index" (+ next_trained_index (size cases)) )) + ;update num_cases after training + (assign (assoc num_cases (call !GetNumTrainingCases) )) + + ;check if hierarchy is enabled and there are now enough cases to activate reduction and enable hierarchy + (if (and !autoHierarchyEnabled (>= num_cases !autoAblationMaxNumCases)) + (call !ReduceIntoHierarchy) + + !autoHierarchyEnabled + (assign_to_entities (assoc + !autoHierarchyCaseLoadMap + { + "self" num_cases + "total" + ;if there are subtrainees, accumulate num of all cases (trained and ablated) to the total + (if (size !autoHierarchyIds) + (+ (get !autoHierarchyCaseLoadMap "total") (size cases) ) + + ;else total is same as the number of cases in this trainee + num_cases + ) + } + )) + ) + + ;if hierarchy and ablation are enabled, ablated cases are trained into subtrainees + (if (and !autoHierarchyEnabled (size ablated_indices_list) (not !autoHierarchyDepthReached)) + ;only keep the ablated cases and only those predicted_clusters matching the ablated cases + (call !TrainSubtraineeClusterIndices (assoc + cases (keep cases ablated_indices_list) + predicted_clusters (keep predicted_clusters ablated_indices_list) + )) + ) + (accum_to_entities (assoc !revision 1)) ;return response @@ -863,6 +926,8 @@ (not skip_reduce_data) (>= (call !GetNumTrainingCases) !autoAblationMaxNumCases) (size !dataParametersMap) + ;allow ReduceIntoHierearchy to run the reduce flow if hierarchy is enabled instead of doing it here + (not !autoHierarchyEnabled) ) (seq ;clear progress from auto analyze if there was one @@ -1040,6 +1105,11 @@ )) ) + ;prepend cluster id feature to features + (if (and !autoHierarchyEnabled (size predicted_clusters)) + (assign (assoc ablate_train_features (append ".cluster_id" ablate_train_features) )) + ) + ;create the cases (declare (assoc output_cases @@ -1058,6 +1128,12 @@ )) )) ) + + ;prepend cluster id to values + (if (and !autoHierarchyEnabled (size predicted_clusters)) + (assign (assoc feature_values (append (get predicted_clusters (current_value 1)) feature_values) )) + ) + (if rebalance_weights (seq (accum (assoc diff --git a/module/train_utilities.amlg b/module/train_utilities.amlg index 104465959..635368f2a 100644 --- a/module/train_utilities.amlg +++ b/module/train_utilities.amlg @@ -913,4 +913,91 @@ )) )) + !SplitLargeTrainForHierarchy + (let + (assoc + num_new (size cases) + payload [1 {"payload" {"num_trained" 0 "ablated_indices" [] "status" .null}}] + new_payload .null + start 0 + end -1 + ) + + (while (> num_new 0) + (assign (assoc + num_cases (call !GetNumTrainingCases) + num_to_train + ;if trainee is ablating cases into subrainees, can train a larger load, + ;otherwise only train up to the amount needed to start the reduce and make a sub hierarchy + (if (and !autoAblationEnabled (size !autoHierarchyIds)) + (min !autoAblationMaxNumCases num_new) + (min (- !autoAblationMaxNumCases num_cases) num_new) + ) + start (+ end 1) + end (+ start num_to_train -1) + new_payload + (call train (assoc + ;;;TODO: don't split time series in middle of a series + cases (unzip cases (range start end)) + features features + derived_features derived_features + session session + series series + input_is_substituted input_is_substituted + allow_training_reserved_features allow_training_reserved_features + skip_auto_analyze skip_auto_analyze + skip_reduce_data skip_reduce_data + start_index start_index + )) + + start_index (if start_index (+ start_index num_to_train)) + num_new (- num_new num_to_train) + )) + + ;accumulate payload + (call !AccumulateTrainPayloads) + + ;stop train on errors + (if (= 0 (first payload)) (conclude) ) + ) + + ;output accumulated payload + payload + ) + + ;accumulate engine output specific to the train method 'new_payload' with 'payload' + !AccumulateTrainPayloads + (assign (assoc + payload + [ + ;if errors, output a 0 + (if (or (= 0 (first payload)) (= 0 (first new_payload))) + 0 + 1 + ) + (assoc + "payload" + { + "num_trained" + (+ + (get payload [1 "payload" "num_trained"]) + (get new_payload [1 "payload" "num_trained"]) + ) + "ablated_indices" + (append + (get payload [1 "payload" "ablated_indices"]) + (get new_payload [1 "payload" "ablated_indices"]) + ) + "status" + (or + (get payload [1 "payload" "status"]) + (get new_payload [1 "payload" "status"]) + ) + } + "warnings" (filter (append (get payload [1 "warnings"]) (get new_payload [1 "warnings"]))) + "errors" (filter (append (get payload [1 "errors"]) (get new_payload [1 "errors"]))) + ) + ] + )) + } \ No newline at end of file diff --git a/module/trainee.amlg b/module/trainee.amlg index cf78fa793..198810145 100644 --- a/module/trainee.amlg +++ b/module/trainee.amlg @@ -44,6 +44,8 @@ ;the name of the entity that contains subtrainees !traineeContainer ".trainee_container" + ;the name of the entity containing cases slated for removal + !reducedCasesStorage ".reduced_cases_storage" ;the supported prediction stats that users can request from react_aggregate !supportedPredictionStats (list "mae" "confusion_matrix" "r2" "rmse" "adjusted_smape" "smape" "spearman_coeff" "precision" "recall" "accuracy" "mcc" "all" "missing_value_accuracy") @@ -146,12 +148,36 @@ ; false means it is in the hierarchy but not a contained trainee and communication with it requires routing outside of this trainee !childTraineeIsContainedMap (assoc) + !autoHierarchyIds .null + + !autoHierarchyEnabled .false + + !autoHierarchyClusteringRatio 0.05 + + ;number of cases in self vs total traned into hierarchy + !autoHierarchyCaseLoadMap { "self" 0 "total" 0 } + + ;current hierarchy depth level, where 0 = the top level trainee + !autoHierarchyDepthLevel 0 + + ;max depth of hierarchy, specifying the number of levels of subtrainees allowed + !autoHierarchyMaxDepth 5 + + ;max number of subtrainees + !autoHierarchyMaxBreadth 5 + + ;flage, set to true if this is a leaf node in a hierarchy and max depth has been reached, no more subtrainees should be created + !autoHierarchyDepthReached .false + ;unique id of parent trainee if this trainee is a subtrainee in a hierarchy !parentId .null ;unique id of this trainee !traineeId trainee_id + ;flag set to true if trainee is contained inside their parent + !traineeIsContained .false + ;amount of total influence weight to accumulate among nearest neighbors before stopping (for influential cases) !influenceWeightThreshold .99 @@ -695,9 +721,15 @@ ;the unique string identifier for the parent of the trainee #{type ["string" "null"]} parent_id .null + ;flag, no default. If specified, sets the flag specifying whether this trainee is contained inside the specified parent trainee. + #{type "boolean"} + is_contained .null ) (call !ValidateParameters) (assign_to_entities (assoc !parentId parent_id)) + (if (!= .null is_contained) + (assign_to_entities (assoc !traineeIsContained is_contained)) + ) (accum_to_entities (assoc !revision 1)) (call !Return) ) diff --git a/performance_tests/bank_hierarchy.amlg b/performance_tests/bank_hierarchy.amlg new file mode 100644 index 000000000..c24bfdaa4 --- /dev/null +++ b/performance_tests/bank_hierarchy.amlg @@ -0,0 +1,213 @@ +(seq + (load_entity "../howso.amlg" "howso" + .null .false {escape_resource_name .false escape_contained_resource_names .false} + ) + (set_entity_permissions "howso" .true) + (call_entity "howso" "initialize" (assoc trainee_id "hbank" filepath "../" print_progress .true)) + + (declare (assoc + data (load "performance_data/bank-full.csv") + train_size 2000 + test_size 10000 + )) + + (declare (assoc + features (first data) + cases_data (tail data) + )) + (declare (assoc + training_indices (rand (indices cases_data) train_size .true) + )) + (declare (assoc + training_data (unzip cases_data training_indices) + other_data (remove cases_data training_indices) + start (system_time) + )) + (declare (assoc + test_indices (rand (indices other_data) test_size .true) + )) + (declare (assoc + test_data (unzip other_data test_indices) + )) + (assign (assoc + other_data (remove other_data test_indices) + )) + + (call_entity "howso" "set_feature_attributes" (assoc + feature_attributes + (assoc + "job" (assoc "type" "nominal" "id_feature" .true) + "marital" (assoc "type" "nominal") + "education" (assoc "type" "nominal") + "default" (assoc "type" "nominal") + "housing" (assoc "type" "nominal") + "loan" (assoc "type" "nominal") + "contact" (assoc "type" "nominal") + "month" (assoc "type" "nominal") + "poutcome" (assoc "type" "nominal") + "y" (assoc "type" "nominal") + "pdays" (assoc "type" "ordinal" "data_type" "number") + "age" (assoc "type" "ordinal" "data_type" "number") + "day" (assoc "type" "ordinal" "data_type" "number") + "campaign" (assoc "type" "ordinal" "data_type" "number") + ) + )) + + (call_entity "howso" "set_auto_ablation_params" (assoc + min_num_cases 5000 + hierarchy_clustering_ratio 0.05 + auto_hierarchy_enabled .true + max_num_cases 10000 + reduce_max_cases 5000 + )) + + (call_entity "howso" "train" (assoc + features features + cases (append training_data other_data) + )) + + (declare (assoc + load_time (- (system_time) start) + num_cases (get (call_entity "howso" "get_num_training_cases") (list 1 "payload" "count")) + )) + (print "Loaded Bank: " num_cases "\n") + (print "Load time: " load_time "\n") + +(print (call_entity "howso" "get_hierarchy" (assoc internal_only .true)) ) + + + (call_entity "howso" "analyze" (assoc full_hierarchy .true)) + + + (declare (assoc + predictions + (get + (call_entity "howso" "react" (assoc + action_features [(last features)] + context_features (trunc features) + context_values (map (lambda (trunc (current_value))) test_data) + )) + [1 "payload" "action_values"] + ) + )) + + (print "hierarchy accuracy: " + (/ + (size (filter + (lambda (let + (assoc + actual (last (current_value 1)) + predicted (get predictions [(current_index 2) 0]) + ) + (= actual predicted) + )) + test_data + )) + test_size + ) + "\n\n" + ) + + + ; (call_entity "howso" "set_auto_ablation_params" (assoc + ; auto_ablation_enabled .false + ; auto_hierarchy_enabled .false + ; )) + (assign (assoc + predictions + (get + (call_entity "howso" "react" (assoc + action_features [(last features)] + context_features (trunc features) + context_values (map (lambda (trunc (current_value))) test_data) + )) + [1 "payload" "action_values"] + ) + )) + + (print "NO hierarchy accuracy (reduce/ablate base only): " + (/ + (size (filter + (lambda (let + (assoc + actual (last (current_value 1)) + predicted (get predictions [(current_index 2) 0]) + ) + (= actual predicted) + )) + test_data + )) + test_size + ) + "\n\n" + ) + + + (destroy_entities "howso") + + + (load_entity "../howso.amlg" "howso" + .null .false {escape_resource_name .false escape_contained_resource_names .false} + ) + (set_entity_permissions "howso" .true) + (call_entity "howso" "initialize" (assoc trainee_id "hbank" filepath "../" print_progress .true)) + + (call_entity "howso" "set_feature_attributes" (assoc + feature_attributes + (assoc + "job" (assoc "type" "nominal" "id_feature" .true) + "marital" (assoc "type" "nominal") + "education" (assoc "type" "nominal") + "default" (assoc "type" "nominal") + "housing" (assoc "type" "nominal") + "loan" (assoc "type" "nominal") + "contact" (assoc "type" "nominal") + "month" (assoc "type" "nominal") + "poutcome" (assoc "type" "nominal") + "y" (assoc "type" "nominal") + "pdays" (assoc "type" "ordinal" "data_type" "number") + "age" (assoc "type" "ordinal" "data_type" "number") + "day" (assoc "type" "ordinal" "data_type" "number") + "campaign" (assoc "type" "ordinal" "data_type" "number") + ) + )) + + (call_entity "howso" "train" (assoc + features features + cases (append training_data other_data) + )) + (assign (assoc + num_cases (get (call_entity "howso" "get_num_training_cases") (list 1 "payload" "count")) + )) + (print "Reloaded Bank: " num_cases "\n") + (call_entity "howso" "analyze") + + (assign (assoc + predictions + (get + (call_entity "howso" "react" (assoc + action_features [(last features)] + context_features (trunc features) + context_values (map (lambda (trunc (current_value))) test_data) + )) + [1 "payload" "action_values"] + ) + )) + + (print "non-hierarchy full dataset accuracy: " + (/ + (size (filter + (lambda (let + (assoc + actual (last (current_value 1)) + predicted (get predictions [(current_index 2) 0]) + ) + (= actual predicted) + )) + test_data + )) + test_size + ) + "\n" + ) +)