From c881e52bd1786c1dc2a666634f5aa49ccc0eef6b Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Mon, 24 Aug 2026 10:24:37 -0400 Subject: [PATCH 01/42] 25938: Wip, auto hierarchy sandbox --- module/hierarchy.amlg | 85 +++++++++++++++++++++ module/react_discriminative.amlg | 33 ++++++++ module/train.amlg | 59 +++++++++++++++ module/trainee.amlg | 2 + performance_tests/bank_hierarchy.amlg | 104 ++++++++++++++++++++++++++ 5 files changed, 283 insertions(+) create mode 100644 performance_tests/bank_hierarchy.amlg diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index 6dfc43808..3531c383f 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -1257,6 +1257,91 @@ (call !Return) ) + make_hierarchy + (declare + (assoc + clustering_min_cluster_mass 10 + use_case_weights .null + ) + + (call react_into_features (assoc + clustering .true + clustering_min_cluster_mass clustering_min_cluster_mass + use_case_weights use_case_weights + )) + + (declare (assoc + cluster_ids_map (compute_on_contained_entities (query_value_masses ".cluster_id")) + + )) + + (print cluster_ids_map "\n") + + (map + (lambda (let + (assoc + cluster_id (current_value 1) + subtrainee_id (concat "child" (current_value 1)) + ) + + (call create_subtrainee (assoc + path [subtrainee_id] + child_id subtrainee_id + )) + + ; (call move_cases (assoc + ; target_id subtrainee_id + ; condition { ".cluster_id" cluster_id } + ; )) + + ;copy cases into corresponding subtrainee + (declare (assoc + cases (contained_entities (query_equals ".cluster_id" cluster_id)) + )) + (declare (assoc + case_features (indices (remove (retrieve_entity_root (first cases)) [".session" ".session_training_index" ".cluster_id"])) + )) + + (call_entity [!traineeContainer subtrainee_id] "train" (assoc + features case_features + cases + (map + (lambda (retrieve_from_entity (current_value) case_features)) + cases + ) + session "none" + ;allow training on reserved features to preserve all case data + allow_training_reserved_features .true + )) + + (call_entity [!traineeContainer subtrainee_id] "prep_subtrainee" (assoc + features case_features + use_case_weights use_case_weights + data_parameters_map !dataParametersMap + data_parameters_paths !dataParametersPaths + )) + + )) + (indices cluster_ids_map) + ) + + (assign_to_entities (assoc !autoHierarchyIds (sort (indices cluster_ids_map)) )) + + (call !Return) + ) + + prep_subtrainee + (seq + (assign_to_entities (assoc + !dataParametersMap data_parameters_map + !dataParametersPaths data_parameters_paths + )) + (call !UpdateInactiveFeatures) + (call !CacheExpectedValuesAndProbabilities (assoc + features features + use_case_weights use_case_weights + )) + ) ;Helper method to create the actual full contained entity path by interleaving !traineeContainer with each name in the path !ConvertNamePathToEntityPath diff --git a/module/react_discriminative.amlg b/module/react_discriminative.amlg index dd00b1639..86c3ca6bf 100644 --- a/module/react_discriminative.amlg +++ b/module/react_discriminative.amlg @@ -330,6 +330,39 @@ ) )) + + (if (and (size !autoHierarchyIds) (!= ".cluster_id" (first action_features))) + (let + (assoc + clustering_prediction + (call !ReactDiscriminative (assoc + context_features context_features + context_values context_values + action_features [".cluster_id"] + skip_encoding .true + skip_decoding .true + details { "categorical_action_probabilities" .true} + )) + ) + + (declare (assoc + cap (get clustering_prediction ["categorical_action_probabilities" ".cluster_id"]) + )) + + ;if single cluster, just react against that one cluster + (if (= 1 (size cap)) + (conclude + (call_entity [!traineeContainer (concat "child" (first (indices cap)))] "ReactDiscriminative") + ) + + ;else TOOD: call react against all the needed custers and weigh results according to their CAPs? + ;or according to the union of all the neighbors from the clusters? + + ) + ) + ) + + ;if there are dependent context features, and the action is dependent, precompute all residuals for dependent continuous features (if dependent_features_map (if (contains_index dependent_features_map (first action_features)) diff --git a/module/train.amlg b/module/train.amlg index b10d58921..64f436f86 100644 --- a/module/train.amlg +++ b/module/train.amlg @@ -58,6 +58,65 @@ (assign (assoc session "none")) ) + + (if (size !autoHierarchyIds) + (let + (assoc + predicted_clusters + ||(map + (lambda (first + (call !ReactDiscriminative (assoc + context_features features + context_values (current_value 1) + action_features [".cluster_id"] + skip_encoding .true + skip_decoding .true + return_action_values_only .true + )) + )) + cases + ) + ) + + (map + (lambda (let + (assoc cluster_id (current_value 1)) + + (declare (assoc + cluster_indices + (filter + (lambda (= cluster_id (get predicted_clusters (current_value)))) + (indices predicted_clusters) + ) + )) + + (call_entity [!traineeContainer (concat "child" cluster_id)] "train" (assoc + features features + cases (unzip cases cluster_indices) + session session + )) + )) + (values predicted_clusters .true) + ) + + + (map + (lambda + (print + "child" (current_value) ": " + (get + (call_entity [!traineeContainer (concat "child" (current_value 1))] "get_num_training_cases") + [1 "payload" "count"] + ) "\n" + ) + ) + !autoHierarchyIds + ) + + (conclude (call !Return)) ;;TODO: add appropriate output, total trained from above train call? + ) + ) + (assign (assoc accumulate_weight_feature (if (and diff --git a/module/trainee.amlg b/module/trainee.amlg index f4bf1b13f..cb07a55ad 100644 --- a/module/trainee.amlg +++ b/module/trainee.amlg @@ -143,6 +143,8 @@ ; false means it is in the hierarchy but not a contained trainee and communication with it requires routing outside of this trainee !childTraineeIsContainedMap (assoc) + !autoHierarchyIds .null + ;unique id of parent trainee if this trainee is a subtrainee in a hierarchy !parentId .null diff --git a/performance_tests/bank_hierarchy.amlg b/performance_tests/bank_hierarchy.amlg new file mode 100644 index 000000000..1133d4c49 --- /dev/null +++ b/performance_tests/bank_hierarchy.amlg @@ -0,0 +1,104 @@ +(seq + (load_entity "../howso.amlg" "howso" + .null .false {escape_resource_name .false escape_contained_resource_names .false} + ) + (set_entity_permissions "howso" .true) + (call_entity "howso" "initialize" (assoc trainee_id "model" filepath "../" print_progress .true)) + + (declare (assoc + data (load "performance_data/bank-full.csv") + train_size 2000 + test_size 5000 + )) + + (declare (assoc + features (first data) + cases_data (tail data) + )) + (declare (assoc + training_indices (rand (indices cases_data) train_size .true) + )) + (declare (assoc + training_data (unzip cases_data training_indices) + other_data (remove cases_data training_indices) + start (system_time) + )) + (declare (assoc + test_indices (rand (indices other_data) test_size .true) + )) + (declare (assoc + test_data (unzip other_data test_indices) + )) + (assign (assoc + other_data (remove other_data test_indices) + )) + + (call_entity "howso" "set_feature_attributes" (assoc + feature_attributes + (assoc + "job" (assoc "type" "nominal" "id_feature" .true) + "marital" (assoc "type" "nominal") + "education" (assoc "type" "nominal") + "default" (assoc "type" "nominal") + "housing" (assoc "type" "nominal") + "loan" (assoc "type" "nominal") + "contact" (assoc "type" "nominal") + "month" (assoc "type" "nominal") + "poutcome" (assoc "type" "nominal") + "y" (assoc "type" "nominal") + "pdays" (assoc "type" "ordinal" "data_type" "number") + "age" (assoc "type" "ordinal" "data_type" "number") + "day" (assoc "type" "ordinal" "data_type" "number") + "campaign" (assoc "type" "ordinal" "data_type" "number") + ) + )) + + (call_entity "howso" "train" (assoc + features features + cases training_data + )) + + (declare (assoc + load_time (- (system_time) start) + num_cases (get (call_entity "howso" "get_num_training_cases") (list 1 "payload" "count")) + )) + (print "Loaded Bank: " num_cases "\n") + (print "Load time: " load_time "\n") + + + (print "analyzing Bank ...\n") + (assign (assoc start (system_time) )) + (call_entity "howso" "analyze" (assoc + context_features features + targeted_model "targetless" + convergence_threshold 0 + )) + (declare (assoc analyze_time (- (system_time) start) )) + (print "Analyze time: " analyze_time "\n" ) + + + (assign (assoc start (system_time) )) + (print "making hierarchy...\n") + (call_entity "howso" "make_hierarchy" (assoc + clustering_min_cluster_mass 10 + )) + (print "hierarchy: " (- (system_time) start) "\n") + + (print + ;(call_entity "howso" "get_hierarchy") + "Base dataset num cases: " (get (call_entity "howso" "get_num_training_cases") [1 "payload" "count"]) "\n" + ) + + (call_entity "howso" "train" (assoc + features features + cases other_data + )) + + (call_entity "howso" "react" (assoc + action_features [(last features)] + context_features (trunc features) + context_values [(trunc (first test_data))] + )) + + (destroy_entities "howso") +) From 2ce1de3046c1a54e5b0b3910f65a727f9f86b4bc Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 25 Aug 2026 13:48:12 -0400 Subject: [PATCH 02/42] wip --- module/dataparameters.amlg | 2 + module/hierarchy.amlg | 72 ++++++------- module/react_discriminative.amlg | 34 +++++- module/train.amlg | 142 ++++++++++++++++---------- module/trainee.amlg | 2 + performance_tests/bank_hierarchy.amlg | 20 ++-- 6 files changed, 171 insertions(+), 101 deletions(-) diff --git a/module/dataparameters.amlg b/module/dataparameters.amlg index 8b95fc668..dc7732fe8 100644 --- a/module/dataparameters.amlg +++ b/module/dataparameters.amlg @@ -1111,6 +1111,8 @@ !autoAblationRelThresholdMap rel_threshold_map !ablationBatchSize batch_size !ablatedCasesDistributionBatchSize ablated_cases_distribution_batch_size + + !autoHierarchyEnabled .true )) (accum_to_entities (assoc !revision 1)) diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index 3531c383f..3b6e022f4 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -1257,17 +1257,15 @@ (call !Return) ) - make_hierarchy + !make_hierarchy (declare (assoc clustering_min_cluster_mass 10 - use_case_weights .null ) - (call react_into_features (assoc clustering .true clustering_min_cluster_mass clustering_min_cluster_mass - use_case_weights use_case_weights + use_case_weights .false )) (declare (assoc @@ -1289,59 +1287,53 @@ child_id subtrainee_id )) + ;move cases into corresponding subtrainee ; (call move_cases (assoc ; target_id subtrainee_id ; condition { ".cluster_id" cluster_id } ; )) + ; ;copy cases into corresponding subtrainee - (declare (assoc - cases (contained_entities (query_equals ".cluster_id" cluster_id)) - )) - (declare (assoc - case_features (indices (remove (retrieve_entity_root (first cases)) [".session" ".session_training_index" ".cluster_id"])) - )) + ; (declare (assoc + ; cases (contained_entities (query_equals ".cluster_id" cluster_id)) + ; )) + ; (declare (assoc + ; case_features (indices (remove (retrieve_entity_root (first cases)) [".session" ".session_training_index" ".cluster_id"])) + ; )) + ; (call_entity [!traineeContainer subtrainee_id] "train" (assoc + ; features case_features + ; cases + ; (map + ; (lambda (retrieve_from_entity (current_value) case_features)) + ; cases + ; ) + ; session "none" + ; ;allow training on reserved features to preserve all case data + ; allow_training_reserved_features .true + ; )) - (call_entity [!traineeContainer subtrainee_id] "train" (assoc - features case_features - cases - (map - (lambda (retrieve_from_entity (current_value) case_features)) - cases - ) - session "none" - ;allow training on reserved features to preserve all case data - allow_training_reserved_features .true - )) - (call_entity [!traineeContainer subtrainee_id] "prep_subtrainee" (assoc - features case_features - use_case_weights use_case_weights - data_parameters_map !dataParametersMap - data_parameters_paths !dataParametersPaths - )) + ;copy all the attributes into subtrainee + (assign_entity_roots [!traineeContainer subtrainee_id] (retrieve_entity_root)) + + (call_entity [!traineeContainer subtrainee_id] "set_parent_id" (assoc parent_id !traineeId)) + (call_entity [!traineeContainer subtrainee_id] "set_trainee_id" (assoc trainee_id subtrainee_id)) + ;todo: remove this so subtrainees can automatically subdivide into their own hierarchies + (call_entity [!traineeContainer subtrainee_id] "set_auto_ablation_params" (assoc auto_ablation_enabled .false)) )) (indices cluster_ids_map) ) (assign_to_entities (assoc !autoHierarchyIds (sort (indices cluster_ids_map)) )) - - (call !Return) ) prep_subtrainee - (seq - (assign_to_entities (assoc - !dataParametersMap data_parameters_map - !dataParametersPaths data_parameters_paths - )) - (call !UpdateInactiveFeatures) - (call !CacheExpectedValuesAndProbabilities (assoc - features features - use_case_weights use_case_weights - )) - ) + (assign_to_entities (assoc + !traineeId trainee_id + !parentId parent_id + )) ;Helper method to create the actual full contained entity path by interleaving !traineeContainer with each name in the path !ConvertNamePathToEntityPath diff --git a/module/react_discriminative.amlg b/module/react_discriminative.amlg index 86c3ca6bf..8b14797da 100644 --- a/module/react_discriminative.amlg +++ b/module/react_discriminative.amlg @@ -1,5 +1,8 @@ ;Contains methods for discriminative (prediction) reacting. { + + ReactDiscriminative (call !ReactDiscriminative) + ;reacts to the context specified: computes the next action from replays given the current context ; context_features: list of context features ; context_values: current values of the world state @@ -352,7 +355,36 @@ ;if single cluster, just react against that one cluster (if (= 1 (size cap)) (conclude - (call_entity [!traineeContainer (concat "child" (first (indices cap)))] "ReactDiscriminative") + ;explicitly pass in all the parameters into react + (call_entity [!traineeContainer (concat "child" (first (indices cap)))] "ReactDiscriminative" (assoc + context_features context_features + context_values context_values + action_features action_features + action_values action_values + details details + skip_encoding skip_encoding + skip_decoding skip_decoding + num_features_returned num_features_returned + extra_features extra_features + match_on_context_features match_on_context_features + ignore_case ignore_case + focal_case focal_case + tie_break_random_seed tie_break_random_seed + allow_nulls allow_nulls + return_action_values_only return_action_values_only + force_targetless force_targetless + data_params_map data_params_map + weight_feature weight_feature + use_case_weights use_case_weights + case_indices case_indices + leave_case_out leave_case_out + goal_features_map goal_features_map + preserve_feature_values preserve_feature_values + new_case_threshold new_case_threshold + has_dependent_features has_dependent_features + impute_react impute_react + filtering_queries filtering_queries + )) ) ;else TOOD: call react against all the needed custers and weigh results according to their CAPs? diff --git a/module/train.amlg b/module/train.amlg index 64f436f86..8fb35f273 100644 --- a/module/train.amlg +++ b/module/train.amlg @@ -58,64 +58,22 @@ (assign (assoc session "none")) ) - - (if (size !autoHierarchyIds) - (let - (assoc - predicted_clusters - ||(map - (lambda (first - (call !ReactDiscriminative (assoc - context_features features - context_values (current_value 1) - action_features [".cluster_id"] - skip_encoding .true - skip_decoding .true - return_action_values_only .true - )) + (declare (assoc + predicted_clusters + (if (and !autoHierarchyEnabled (size !autoHierarchyIds)) + ||(map + (lambda (first + (call !ReactDiscriminative (assoc + context_features features + context_values (current_value 1) + action_features [".cluster_id"] + return_action_values_only .true )) - cases - ) - ) - - (map - (lambda (let - (assoc cluster_id (current_value 1)) - - (declare (assoc - cluster_indices - (filter - (lambda (= cluster_id (get predicted_clusters (current_value)))) - (indices predicted_clusters) - ) - )) - - (call_entity [!traineeContainer (concat "child" cluster_id)] "train" (assoc - features features - cases (unzip cases cluster_indices) - session session )) - )) - (values predicted_clusters .true) - ) - - - (map - (lambda - (print - "child" (current_value) ": " - (get - (call_entity [!traineeContainer (concat "child" (current_value 1))] "get_num_training_cases") - [1 "payload" "count"] - ) "\n" - ) + cases ) - !autoHierarchyIds ) - - (conclude (call !Return)) ;;TODO: add appropriate output, total trained from above train call? - ) - ) + )) (assign (assoc accumulate_weight_feature @@ -515,6 +473,71 @@ ".next_trained_index" (+ next_trained_index (size cases)) )) + (if (and !autoHierarchyEnabled (size ablated_indices_list)) + (seq + ;only keep the ablated cases + (assign (assoc cases (keep cases ablated_indices_list) )) + + (if (= 0 (size !autoHierarchyIds)) + (seq + (call !make_hierarchy) + + (assign (assoc + predicted_clusters + ||(map + (lambda (first + (call !ReactDiscriminative (assoc + context_features features + context_values (current_value 1) + action_features [".cluster_id"] + return_action_values_only .true + )) + )) + cases + ) + )) + ) + + ;else hierarchy already exists, only keep those predicted_clusters matching the ablated cases + (assign (assoc predicted_clusters (keep predicted_clusters ablated_indices_list))) + ) + + (map + (lambda (let + (assoc cluster_id (current_value 1)) + + (declare (assoc + cluster_indices + (filter + (lambda (= cluster_id (get predicted_clusters (current_value)))) + (indices predicted_clusters) + ) + )) + + (call_entity [!traineeContainer (concat "child" cluster_id)] "train" (assoc + features features + cases (unzip cases cluster_indices) + session session + )) + )) + (values predicted_clusters .true) + ) + + (map + (lambda + (print + "child" (current_value) ": " + (get + (call_entity [!traineeContainer (concat "child" (current_value 1))] "get_num_training_cases") + [1 "payload" "count"] + ) "\n" + ) + ) + !autoHierarchyIds + ) + ) + ) + (call !ProgressClear) (accum_to_entities (assoc !revision 1)) @@ -1099,6 +1122,11 @@ )) ) + ;prepend cluster id feature to features + (if (and !autoHierarchyEnabled (size predicted_clusters)) + (assign (assoc ablate_train_features (append ".cluster_id" ablate_train_features) )) + ) + ;create the cases (declare (assoc output_cases @@ -1117,6 +1145,12 @@ )) )) ) + + ;prepend cluster id to values + (if (and !autoHierarchyEnabled (size predicted_clusters)) + (assign (assoc feature_values (append (get predicted_clusters (current_value 1)) feature_values) )) + ) + (if rebalance_weights (seq (accum (assoc diff --git a/module/trainee.amlg b/module/trainee.amlg index cb07a55ad..b22aef83b 100644 --- a/module/trainee.amlg +++ b/module/trainee.amlg @@ -145,6 +145,8 @@ !autoHierarchyIds .null + !autoHierarchyEnabled .false + ;unique id of parent trainee if this trainee is a subtrainee in a hierarchy !parentId .null diff --git a/performance_tests/bank_hierarchy.amlg b/performance_tests/bank_hierarchy.amlg index 1133d4c49..538524ee8 100644 --- a/performance_tests/bank_hierarchy.amlg +++ b/performance_tests/bank_hierarchy.amlg @@ -53,6 +53,15 @@ ) )) + (call_entity "howso" "set_auto_ablation_params" (assoc + auto_ablation_enabled .true + min_num_cases 2000 + + ;todo: add parameters? + ; clustering_min_cluster_mass 10 + ; auto_hierarchy_enabled .true + )) + (call_entity "howso" "train" (assoc features features cases training_data @@ -78,11 +87,6 @@ (assign (assoc start (system_time) )) - (print "making hierarchy...\n") - (call_entity "howso" "make_hierarchy" (assoc - clustering_min_cluster_mass 10 - )) - (print "hierarchy: " (- (system_time) start) "\n") (print ;(call_entity "howso" "get_hierarchy") @@ -91,7 +95,11 @@ (call_entity "howso" "train" (assoc features features - cases other_data + cases (trunc other_data 1000) ;only the first 1000 + )) + (call_entity "howso" "train" (assoc + features features + cases (tail other_data -1000) ;everything but the first 1000 )) (call_entity "howso" "react" (assoc From 69d6d44f11d17631f1df6c891e9dd25798da8837 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 25 Aug 2026 16:34:10 -0400 Subject: [PATCH 03/42] wip --- module/dataparameters.amlg | 28 +++-- module/hierarchy.amlg | 162 ++++++++++++++++++-------- module/react_discriminative.amlg | 3 +- module/train.amlg | 28 ++--- performance_tests/bank_hierarchy.amlg | 2 +- 5 files changed, 149 insertions(+), 74 deletions(-) diff --git a/module/dataparameters.amlg b/module/dataparameters.amlg index dc7732fe8..294e0eb40 100644 --- a/module/dataparameters.amlg +++ b/module/dataparameters.amlg @@ -505,17 +505,6 @@ ) ) - ;there are multiple sets of analyze params, we must determine the best - (declare (assoc - context_key - (if (or (= context_features .null) (= context_features (list))) - !trainedFeaturesContextKey - - ;else context_features were passed - (call !BuildContextFeaturesKey (assoc context_features context_features)) - ) - )) - ;if feature is specified as "", this will default to ["targetless"] since "" can't be a targeted action feature (declare (assoc target_mode @@ -531,6 +520,22 @@ ) )) + ;there are multiple sets of analyze params, we must determine the best + (declare (assoc + context_key + (if (or (= context_features .null) (= context_features (list))) + !trainedFeaturesContextKey + + ;targetless, predicting a specific feature, pull the analyzed parameters that included that feature + (and (!= .null feature) (= ["targetless"] target_mode)) + (call !BuildContextFeaturesKey (assoc + context_features (values (append context_features feature) .true) + )) + + ;else context_features were passed + (call !BuildContextFeaturesKey (assoc context_features context_features)) + ) + )) ;if we have completely correct HPs, return them (if (contains_index !dataParametersMap (append target_mode context_key weight_feature)) @@ -551,7 +556,6 @@ ;naively set the context to the longest option (assign (assoc context_key (first context_key_options) )) - (while (< (current_index) (size context_key_options)) (assign (assoc candidate_context_key (get context_key_options (current_index 1)) )) diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index 3b6e022f4..52cf35d47 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -1270,63 +1270,131 @@ (declare (assoc cluster_ids_map (compute_on_contained_entities (query_value_masses ".cluster_id")) - )) +;;;debug +(print cluster_ids_map "\n") - (print cluster_ids_map "\n") + (if (!= .null !savedAnalyzeParameterMap) + (seq + (assign_to_entities (assoc + !savedAnalyzeParameterMap + (modify + !savedAnalyzeParameterMap + "context_features" + (append (get !savedAnalyzeParameterMap "context_features") ".cluster_id") + ) + )) + ;prevent calling react_into_features during the analyze() + (call !Analyze (append !savedAnalyzeParameterMap (assoc "from_react_into_features" .true) )) + ) - (map - (lambda (let + (call !Analyze (assoc + targeted_model "targetless" + context_features (concat !trainedFeatures ".cluster_id") + weight_feature + (if (and !autoAblationEnabled !autoAblationWeightFeature) + !autoAblationWeightFeature + weight_feature + ) + use_case_weights (or (and !autoAblationEnabled !autoAblationWeightFeature) use_case_weights) + k_folds 1 + ;prevent calling react_into_features during the analyze() + from_react_into_features .true + )) + ) + + ;there are unclustered cases, forcibly move them into clusters + (if (contains_index cluster_ids_map 0) + (let (assoc - cluster_id (current_value 1) - subtrainee_id (concat "child" (current_value 1)) + unclustered_cases (contained_entities (query_equals ".cluster_id" 0)) ) - (call create_subtrainee (assoc - path [subtrainee_id] - child_id subtrainee_id + (declare (assoc + forced_clusters + ||(map + (lambda (first + (call !ReactDiscriminative (assoc + context_features !trainedFeatures + context_values (retrieve_from_entity (current_value 1) !trainedFeatures) + action_features [".cluster_id"] + return_action_values_only .true + use_case_weights .true + filtering_queries [(query_not_equals ".cluster_id" 0) (query_not_in_entity_list [(current_value 3)])] + )) + )) + unclustered_cases + ) )) - ;move cases into corresponding subtrainee - ; (call move_cases (assoc - ; target_id subtrainee_id - ; condition { ".cluster_id" cluster_id } - ; )) - ; - - ;copy cases into corresponding subtrainee - ; (declare (assoc - ; cases (contained_entities (query_equals ".cluster_id" cluster_id)) - ; )) - ; (declare (assoc - ; case_features (indices (remove (retrieve_entity_root (first cases)) [".session" ".session_training_index" ".cluster_id"])) - ; )) - ; (call_entity [!traineeContainer subtrainee_id] "train" (assoc - ; features case_features - ; cases - ; (map - ; (lambda (retrieve_from_entity (current_value) case_features)) - ; cases - ; ) - ; session "none" - ; ;allow training on reserved features to preserve all case data - ; allow_training_reserved_features .true - ; )) - - - ;copy all the attributes into subtrainee - (assign_entity_roots [!traineeContainer subtrainee_id] (retrieve_entity_root)) - - (call_entity [!traineeContainer subtrainee_id] "set_parent_id" (assoc parent_id !traineeId)) - (call_entity [!traineeContainer subtrainee_id] "set_trainee_id" (assoc trainee_id subtrainee_id)) - - ;todo: remove this so subtrainees can automatically subdivide into their own hierarchies - (call_entity [!traineeContainer subtrainee_id] "set_auto_ablation_params" (assoc auto_ablation_enabled .false)) - )) - (indices cluster_ids_map) + ;update cluster id for each unclustered case + (map + (lambda + (assign_to_entities (current_index) (assoc ".cluster_id" (current_value 1))) + ) + (zip unclustered_cases forced_clusters) + ) + ) ) - (assign_to_entities (assoc !autoHierarchyIds (sort (indices cluster_ids_map)) )) + (assign (assoc + cluster_ids_map (compute_on_contained_entities (query_value_masses ".cluster_id")) + )) +;;;debug +(print "updated: " cluster_ids_map "\n") + + + (declare (assoc + subtrainee_ids + (map + (lambda (let + (assoc + cluster_id (current_value 1) + subtrainee_id (concat !traineeId "_sub" (current_value 1)) + ) + + (call create_subtrainee (assoc + path [subtrainee_id] + child_id subtrainee_id + )) + + ;copy cases into corresponding subtrainee + (declare (assoc + cases (contained_entities (query_equals ".cluster_id" cluster_id)) + )) + (declare (assoc + case_features (indices (remove (retrieve_entity_root (first cases)) [".session" ".session_training_index" ".cluster_id"])) + )) + (call_entity [!traineeContainer subtrainee_id] "train" (assoc + features case_features + cases + (map + (lambda (retrieve_from_entity (current_value) case_features)) + cases + ) + session "none" + ;allow training on reserved features to preserve all case data + allow_training_reserved_features .true + )) + + + ;copy all the attributes into subtrainee + (assign_entity_roots [!traineeContainer subtrainee_id] (retrieve_entity_root)) + + (call_entity [!traineeContainer subtrainee_id] "set_parent_id" (assoc parent_id !traineeId)) + (call_entity [!traineeContainer subtrainee_id] "set_trainee_id" (assoc trainee_id subtrainee_id)) + + ;todo: remove this so subtrainees can automatically subdivide into their own hierarchies + (call_entity [!traineeContainer subtrainee_id] "set_auto_ablation_params" (assoc auto_ablation_enabled .false)) + + ;output id + subtrainee_id + )) + (indices cluster_ids_map) + ) + )) + + (assign_to_entities (assoc !autoHierarchyIds (sort subtrainee_ids) )) ) prep_subtrainee diff --git a/module/react_discriminative.amlg b/module/react_discriminative.amlg index 8b14797da..00a1d3fae 100644 --- a/module/react_discriminative.amlg +++ b/module/react_discriminative.amlg @@ -345,6 +345,7 @@ skip_encoding .true skip_decoding .true details { "categorical_action_probabilities" .true} + use_case_weights .true )) ) @@ -356,7 +357,7 @@ (if (= 1 (size cap)) (conclude ;explicitly pass in all the parameters into react - (call_entity [!traineeContainer (concat "child" (first (indices cap)))] "ReactDiscriminative" (assoc + (call_entity [!traineeContainer (concat !traineeId "_sub" (first (indices cap)))] "ReactDiscriminative" (assoc context_features context_features context_values context_values action_features action_features diff --git a/module/train.amlg b/module/train.amlg index 8fb35f273..59a2e55e1 100644 --- a/module/train.amlg +++ b/module/train.amlg @@ -68,6 +68,7 @@ context_values (current_value 1) action_features [".cluster_id"] return_action_values_only .true + use_case_weights .true )) )) cases @@ -491,6 +492,7 @@ context_values (current_value 1) action_features [".cluster_id"] return_action_values_only .true + use_case_weights .true )) )) cases @@ -514,7 +516,7 @@ ) )) - (call_entity [!traineeContainer (concat "child" cluster_id)] "train" (assoc + (call_entity [!traineeContainer (concat !traineeId "_sub" cluster_id)] "train" (assoc features features cases (unzip cases cluster_indices) session session @@ -523,18 +525,18 @@ (values predicted_clusters .true) ) - (map - (lambda - (print - "child" (current_value) ": " - (get - (call_entity [!traineeContainer (concat "child" (current_value 1))] "get_num_training_cases") - [1 "payload" "count"] - ) "\n" - ) - ) - !autoHierarchyIds - ) +;;;debug +(map + (lambda + (print (current_value) ": " + (get + (call_entity [!traineeContainer (current_value 1)] "get_num_training_cases") + [1 "payload" "count"] + ) "\n" + ) + ) + !autoHierarchyIds +) ) ) diff --git a/performance_tests/bank_hierarchy.amlg b/performance_tests/bank_hierarchy.amlg index 538524ee8..09fd1ab55 100644 --- a/performance_tests/bank_hierarchy.amlg +++ b/performance_tests/bank_hierarchy.amlg @@ -3,7 +3,7 @@ .null .false {escape_resource_name .false escape_contained_resource_names .false} ) (set_entity_permissions "howso" .true) - (call_entity "howso" "initialize" (assoc trainee_id "model" filepath "../" print_progress .true)) + (call_entity "howso" "initialize" (assoc trainee_id "hbank" filepath "../" print_progress .true)) (declare (assoc data (load "performance_data/bank-full.csv") From fe8d561de382a5701565263a7d028499aefc66c7 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Wed, 26 Aug 2026 11:28:34 -0400 Subject: [PATCH 04/42] wip, adding output_raw_influential_cases_only --- module/hierarchy.amlg | 6 +++--- module/react.amlg | 4 ++++ module/react_discriminative.amlg | 16 ++++++++++------ 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index 52cf35d47..b2475416f 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -1274,6 +1274,7 @@ ;;;debug (print cluster_ids_map "\n") + ;analyze with .cluster_id as a feature (if (!= .null !savedAnalyzeParameterMap) (seq (assign_to_entities (assoc @@ -1343,7 +1344,7 @@ ;;;debug (print "updated: " cluster_ids_map "\n") - + ;create and populate subtrainees, one per cluster (declare (assoc subtrainee_ids (map @@ -1377,14 +1378,13 @@ allow_training_reserved_features .true )) - ;copy all the attributes into subtrainee (assign_entity_roots [!traineeContainer subtrainee_id] (retrieve_entity_root)) (call_entity [!traineeContainer subtrainee_id] "set_parent_id" (assoc parent_id !traineeId)) (call_entity [!traineeContainer subtrainee_id] "set_trainee_id" (assoc trainee_id subtrainee_id)) - ;todo: remove this so subtrainees can automatically subdivide into their own hierarchies + ;;;todo: remove this so subtrainees can automatically subdivide into their own hierarchies (call_entity [!traineeContainer subtrainee_id] "set_auto_ablation_params" (assoc auto_ablation_enabled .false)) ;output id diff --git a/module/react.amlg b/module/react.amlg index 59c528dbd..fcc84e5c2 100644 --- a/module/react.amlg +++ b/module/react.amlg @@ -1682,6 +1682,10 @@ exp_value ) + (if output_raw_influential_cases_only + (conclude (zip (first local_data_cases_tuple) (get local_data_cases_tuple 1)) ) + ) + ;else interpolate the result from the nearest neighbors (call !InterpolateActionValues (assoc action_feature (first action_features) diff --git a/module/react_discriminative.amlg b/module/react_discriminative.amlg index 00a1d3fae..83f5c54de 100644 --- a/module/react_discriminative.amlg +++ b/module/react_discriminative.amlg @@ -354,10 +354,12 @@ )) ;if single cluster, just react against that one cluster - (if (= 1 (size cap)) - (conclude + (map + (lambda (let + (assoc subtrainee_id (concat !traineeId "_sub" (current_index 1)) ) ;explicitly pass in all the parameters into react - (call_entity [!traineeContainer (concat !traineeId "_sub" (first (indices cap)))] "ReactDiscriminative" (assoc + (call_entity [!traineeContainer subtrainee_id] "ReactDiscriminative" (assoc + output_raw_influential_cases_only .true context_features context_features context_values context_values action_features action_features @@ -386,12 +388,14 @@ impute_react impute_react filtering_queries filtering_queries )) - ) + )) + cap + ) ;else TOOD: call react against all the needed custers and weigh results according to their CAPs? - ;or according to the union of all the neighbors from the clusters? + ;or according to the union of all the raw influences from all the clusters? + - ) ) ) From 34bfb8ea3af9d93fa96f404c8e787d39feee9cfa Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Fri, 28 Aug 2026 10:11:22 -0400 Subject: [PATCH 05/42] react among sub trainees --- module/react.amlg | 35 +++--- module/react_discriminative.amlg | 187 ++++++++++++++++++++++++------- module/react_utilities.amlg | 79 +++++++++++++ 3 files changed, 245 insertions(+), 56 deletions(-) diff --git a/module/react.amlg b/module/react.amlg index fcc84e5c2..fa75a7aa5 100644 --- a/module/react.amlg +++ b/module/react.amlg @@ -1683,24 +1683,27 @@ ) (if output_raw_influential_cases_only - (conclude (zip (first local_data_cases_tuple) (get local_data_cases_tuple 1)) ) - ) + (zip + (first local_data_cases_tuple) + (get local_data_cases_tuple 1) + ) - ;else interpolate the result from the nearest neighbors - (call !InterpolateActionValues (assoc - action_feature (first action_features) - candidate_case_ids (first local_data_cases_tuple) - candidate_case_weights (get local_data_cases_tuple 1) - candidate_case_values (last local_data_cases_tuple) - allow_nulls allow_nulls - feature_has_continuous_nulls - (if allow_nulls - (and - (not (contains_index !nominalsMap (first action_features))) - (!= .false (get !featureNullRatiosMap [(first action_features) "has_nulls"])) + ;else interpolate the result from the nearest neighbors + (call !InterpolateActionValues (assoc + action_feature (first action_features) + candidate_case_ids (first local_data_cases_tuple) + candidate_case_weights (get local_data_cases_tuple 1) + candidate_case_values (last local_data_cases_tuple) + allow_nulls allow_nulls + feature_has_continuous_nulls + (if allow_nulls + (and + (not (contains_index !nominalsMap (first action_features))) + (!= .false (get !featureNullRatiosMap [(first action_features) "has_nulls"])) + ) ) - ) - )) + )) + ) ) ) diff --git a/module/react_discriminative.amlg b/module/react_discriminative.amlg index 83f5c54de..680f28d03 100644 --- a/module/react_discriminative.amlg +++ b/module/react_discriminative.amlg @@ -1,7 +1,37 @@ ;Contains methods for discriminative (prediction) reacting. { - ReactDiscriminative (call !ReactDiscriminative) + ReactDiscriminative + (call !ReactDiscriminative (assoc + output_raw_influential_cases_only output_raw_influential_cases_only + context_features context_features + context_values context_values + action_features action_features + action_values action_values + details details + skip_encoding skip_encoding + skip_decoding skip_decoding + num_features_returned num_features_returned + extra_features extra_features + match_on_context_features match_on_context_features + ignore_case ignore_case + focal_case focal_case + tie_break_random_seed tie_break_random_seed + allow_nulls allow_nulls + return_action_values_only return_action_values_only + force_targetless force_targetless + data_params_map data_params_map + weight_feature weight_feature + use_case_weights use_case_weights + case_indices case_indices + leave_case_out leave_case_out + goal_features_map goal_features_map + preserve_feature_values preserve_feature_values + new_case_threshold new_case_threshold + has_dependent_features has_dependent_features + impute_react impute_react + filtering_queries filtering_queries + )) ;reacts to the context specified: computes the next action from replays given the current context ; context_features: list of context features @@ -354,52 +384,129 @@ )) ;if single cluster, just react against that one cluster - (map - (lambda (let - (assoc subtrainee_id (concat !traineeId "_sub" (current_index 1)) ) - ;explicitly pass in all the parameters into react - (call_entity [!traineeContainer subtrainee_id] "ReactDiscriminative" (assoc - output_raw_influential_cases_only .true - context_features context_features - context_values context_values - action_features action_features - action_values action_values - details details - skip_encoding skip_encoding - skip_decoding skip_decoding - num_features_returned num_features_returned - extra_features extra_features - match_on_context_features match_on_context_features - ignore_case ignore_case - focal_case focal_case - tie_break_random_seed tie_break_random_seed - allow_nulls allow_nulls - return_action_values_only return_action_values_only - force_targetless force_targetless - data_params_map data_params_map - weight_feature weight_feature - use_case_weights use_case_weights - case_indices case_indices - leave_case_out leave_case_out - goal_features_map goal_features_map - preserve_feature_values preserve_feature_values - new_case_threshold new_case_threshold - has_dependent_features has_dependent_features - impute_react impute_react - filtering_queries filtering_queries - )) + (if (= 1 (size cap)) + (call_entity [!traineeContainer (concat !traineeId "_sub" (first (indices cap)))] "ReactDiscriminative" (assoc + context_features context_features + context_values context_values + action_features action_features + action_values action_values + details details + skip_encoding skip_encoding + skip_decoding skip_decoding + num_features_returned num_features_returned + extra_features extra_features + match_on_context_features match_on_context_features + ignore_case ignore_case + focal_case focal_case + tie_break_random_seed tie_break_random_seed + allow_nulls allow_nulls + return_action_values_only return_action_values_only + force_targetless force_targetless + data_params_map data_params_map + weight_feature weight_feature + use_case_weights use_case_weights + case_indices case_indices + leave_case_out leave_case_out + goal_features_map goal_features_map + preserve_feature_values preserve_feature_values + new_case_threshold new_case_threshold + has_dependent_features has_dependent_features + impute_react impute_react + filtering_queries filtering_queries )) - cap - ) - ;else TOOD: call react against all the needed custers and weigh results according to their CAPs? - ;or according to the union of all the raw influences from all the clusters? + ;else iterpolate among the involved clusters, union of all the raw influences + (let + (assoc + relevant_cluster_cases_map + (map + (lambda (let + (assoc subtrainee_id (concat !traineeId "_sub" (current_index 1)) ) + (get + (call_entity [!traineeContainer subtrainee_id] "ReactDiscriminative" (assoc + output_raw_influential_cases_only .true + context_features context_features + context_values context_values + action_features action_features + action_values action_values + details details + skip_encoding skip_encoding + skip_decoding skip_decoding + num_features_returned num_features_returned + extra_features extra_features + match_on_context_features match_on_context_features + ignore_case ignore_case + focal_case focal_case + tie_break_random_seed tie_break_random_seed + allow_nulls allow_nulls + return_action_values_only return_action_values_only + force_targetless force_targetless + data_params_map data_params_map + weight_feature weight_feature + use_case_weights use_case_weights + case_indices case_indices + leave_case_out leave_case_out + goal_features_map goal_features_map + preserve_feature_values preserve_feature_values + new_case_threshold new_case_threshold + has_dependent_features has_dependent_features + impute_react impute_react + filtering_queries filtering_queries + )) + ["action_values" 0] + ) + )) + cap + ) + ) + (declare (assoc + combined_influences_map + ;combine all the subtrainee cases and influences into one assoc where the indices are pair of [subtrainee_id case_id] + (apply "append" (values + (map + (lambda (let + (assoc sub_id (concat !traineeId "_sub" (current_index 1)) ) + + ;change the assoc indices to be a pair such that the assoc is now [subtrainee_id case_id] -> influence weight + (zip + (map (lambda [sub_id (current_value 1)] ) (indices (current_value))) + (values (current_value)) + ) + )) + relevant_cluster_cases_map + ) + )) + k_parameter (get data_params_map "k") + )) + + ;sort subtrainee cases by their influences + (declare (assoc + sub_cases + (sort + (lambda + (< + (get combined_influences_map [(current_value 1)]) + (get combined_influences_map [(current_value 2)]) + ) + ) + (indices combined_influences_map) + ) + )) + (call !DynamicBandwidthFilterAndInterpolate (assoc + action_feature (first action_features) + sub_cases sub_cases + probs (unzip combined_influences_map sub_cases) + marginal_cutoff (if (~ [] k_parameter) (first k_parameter) ) + min_k (if (~ [] k_parameter) (get k_parameter 1) ) + static_k (if (~ 0 k_parameter) k_parameter) + )) + ) + ) ) ) - ;if there are dependent context features, and the action is dependent, precompute all residuals for dependent continuous features (if dependent_features_map (if (contains_index dependent_features_map (first action_features)) diff --git a/module/react_utilities.amlg b/module/react_utilities.amlg index 7bcf8b4f8..45ce7a215 100644 --- a/module/react_utilities.amlg +++ b/module/react_utilities.amlg @@ -1457,4 +1457,83 @@ (zip features) ) + + !DynamicBandwidthFilterAndInterpolate + (declare + (assoc + action_feature .null + sub_cases [] + probs [] + marginal_cutoff 0.05 + min_k 5 + ) + + (declare (assoc + num_probs (size probs) + total_prob (first probs) + first_prob (first probs) + )) + + (declare (assoc + num_relevant + (while (< (current_index) num_probs) + (if (> (current_index) 0) + (let + (assoc prob (get probs (current_index 1)) ) + (accum (assoc total_prob prob )) + + (if (and + (> total_prob 1.25) + (or + ;marginal probability is less than marginal cutoff + (< (/ prob total_prob) marginal_cutoff) + ;delta to first prob is >= 20 %, i.e. the difference is large enough to not have to accumulate to reach the marginal cutoff. + (>= (/ (- first_prob prob) first_prob) 0.2) + ) + ;and it's different from the previous probability + ;to ensure all equidistant cases are considered + (!= prob (get probs (- (current_index) 1) ) ) + ) + (conclude (current_index) ) + ) + ) + ) + + (+ (current_index) 1) + ) + )) + + (if (< num_relevant min_k) + (assign (assoc num_relevant min_k)) + ) + + (if (!= num_relevant num_probs) + (assign (assoc + sub_cases (trunc sub_cases num_relevant) + probs (trunc probs num_relevant) + )) + ) + + (call !InterpolateActionValues (assoc + action_feature action_feature + candidate_case_ids sub_cases + candidate_case_weights probs + candidate_case_values + (map + (lambda + (retrieve_from_entity [!traineeContainer (first (current_value 1)) (last (current_value 1))] action_feature) + ) + sub_cases + ) + allow_nulls allow_nulls + feature_has_continuous_nulls + (if allow_nulls + (and + (not (contains_index !nominalsMap action_feature)) + (!= .false (get !featureNullRatiosMap [action_feature "has_nulls"])) + ) + ) + )) + ) + } \ No newline at end of file From d29cc2b09bf2a0028120f5d4e0827759bf0a0448 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Fri, 28 Aug 2026 13:29:39 -0400 Subject: [PATCH 06/42] react fixes --- module/react_discriminative.amlg | 231 +++++++++++--------------- module/react_utilities.amlg | 39 +++-- performance_tests/bank_hierarchy.amlg | 5 +- 3 files changed, 117 insertions(+), 158 deletions(-) diff --git a/module/react_discriminative.amlg b/module/react_discriminative.amlg index 680f28d03..6bf7690c5 100644 --- a/module/react_discriminative.amlg +++ b/module/react_discriminative.amlg @@ -1,37 +1,38 @@ ;Contains methods for discriminative (prediction) reacting. { + !ReactDiscriminativeParameters + (assoc + output_raw_influential_cases_only output_raw_influential_cases_only + context_features context_features + context_values context_values + action_features action_features + action_values action_values + details details + skip_encoding skip_encoding + skip_decoding skip_decoding + num_features_returned num_features_returned + extra_features extra_features + match_on_context_features match_on_context_features + ignore_case ignore_case + focal_case focal_case + tie_break_random_seed tie_break_random_seed + allow_nulls allow_nulls + return_action_values_only return_action_values_only + force_targetless force_targetless + data_params_map data_params_map + weight_feature weight_feature + use_case_weights use_case_weights + case_indices case_indices + leave_case_out leave_case_out + goal_features_map goal_features_map + preserve_feature_values preserve_feature_values + new_case_threshold new_case_threshold + has_dependent_features has_dependent_features + impute_react impute_react + filtering_queries filtering_queries + ) - ReactDiscriminative - (call !ReactDiscriminative (assoc - output_raw_influential_cases_only output_raw_influential_cases_only - context_features context_features - context_values context_values - action_features action_features - action_values action_values - details details - skip_encoding skip_encoding - skip_decoding skip_decoding - num_features_returned num_features_returned - extra_features extra_features - match_on_context_features match_on_context_features - ignore_case ignore_case - focal_case focal_case - tie_break_random_seed tie_break_random_seed - allow_nulls allow_nulls - return_action_values_only return_action_values_only - force_targetless force_targetless - data_params_map data_params_map - weight_feature weight_feature - use_case_weights use_case_weights - case_indices case_indices - leave_case_out leave_case_out - goal_features_map goal_features_map - preserve_feature_values preserve_feature_values - new_case_threshold new_case_threshold - has_dependent_features has_dependent_features - impute_react impute_react - filtering_queries filtering_queries - )) + ReactDiscriminative (call !ReactDiscriminative @(target .true "!ReactDiscriminativeParameters") ) ;reacts to the context specified: computes the next action from replays given the current context ; context_features: list of context features @@ -384,124 +385,78 @@ )) ;if single cluster, just react against that one cluster - (if (= 1 (size cap)) - (call_entity [!traineeContainer (concat !traineeId "_sub" (first (indices cap)))] "ReactDiscriminative" (assoc - context_features context_features - context_values context_values - action_features action_features - action_values action_values - details details - skip_encoding skip_encoding - skip_decoding skip_decoding - num_features_returned num_features_returned - extra_features extra_features - match_on_context_features match_on_context_features - ignore_case ignore_case - focal_case focal_case - tie_break_random_seed tie_break_random_seed - allow_nulls allow_nulls - return_action_values_only return_action_values_only - force_targetless force_targetless - data_params_map data_params_map - weight_feature weight_feature - use_case_weights use_case_weights - case_indices case_indices - leave_case_out leave_case_out - goal_features_map goal_features_map - preserve_feature_values preserve_feature_values - new_case_threshold new_case_threshold - has_dependent_features has_dependent_features - impute_react impute_react - filtering_queries filtering_queries - )) - - ;else iterpolate among the involved clusters, union of all the raw influences - (let - (assoc - relevant_cluster_cases_map - (map - (lambda (let - (assoc subtrainee_id (concat !traineeId "_sub" (current_index 1)) ) - (get - (call_entity [!traineeContainer subtrainee_id] "ReactDiscriminative" (assoc - output_raw_influential_cases_only .true - context_features context_features - context_values context_values - action_features action_features - action_values action_values - details details - skip_encoding skip_encoding - skip_decoding skip_decoding - num_features_returned num_features_returned - extra_features extra_features - match_on_context_features match_on_context_features - ignore_case ignore_case - focal_case focal_case - tie_break_random_seed tie_break_random_seed - allow_nulls allow_nulls - return_action_values_only return_action_values_only - force_targetless force_targetless - data_params_map data_params_map - weight_feature weight_feature - use_case_weights use_case_weights - case_indices case_indices - leave_case_out leave_case_out - goal_features_map goal_features_map - preserve_feature_values preserve_feature_values - new_case_threshold new_case_threshold - has_dependent_features has_dependent_features - impute_react impute_react - filtering_queries filtering_queries - )) - ["action_values" 0] - ) - )) - cap - ) + (conclude + (if (= 1 (size cap)) + (call_entity [!traineeContainer (concat !traineeId "_sub" (first (indices cap)))] + "ReactDiscriminative" + @(target .true "!ReactDiscriminativeParameters") ) - (declare (assoc - combined_influences_map - ;combine all the subtrainee cases and influences into one assoc where the indices are pair of [subtrainee_id case_id] - (apply "append" (values + ;else iterpolate among the involved clusters, union of all the raw influences + (let + (assoc + relevant_cluster_cases_map (map (lambda (let - (assoc sub_id (concat !traineeId "_sub" (current_index 1)) ) - - ;change the assoc indices to be a pair such that the assoc is now [subtrainee_id case_id] -> influence weight - (zip - (map (lambda [sub_id (current_value 1)] ) (indices (current_value))) - (values (current_value)) + (assoc subtrainee_id (concat !traineeId "_sub" (current_index 1)) ) + (get + (call_entity [!traineeContainer subtrainee_id] + "ReactDiscriminative" + (modify + @(target .true "!ReactDiscriminativeParameters") + "output_raw_influential_cases_only" .true + ) + ) + ["action_values" 0] ) )) - relevant_cluster_cases_map + cap ) - )) - k_parameter (get data_params_map "k") - )) + ) - ;sort subtrainee cases by their influences - (declare (assoc - sub_cases - (sort - (lambda - (< - (get combined_influences_map [(current_value 1)]) - (get combined_influences_map [(current_value 2)]) + (declare (assoc + combined_influences_map + ;combine all the subtrainee cases and influences into one assoc where the indices are pair of [subtrainee_id case_id] + (apply "append" (values + (map + (lambda (let + (assoc sub_id (concat !traineeId "_sub" (current_index 1)) ) + + ;change the assoc indices to be a pair such that the assoc is now [subtrainee_id case_id] -> influence weight + (zip + (map (lambda [sub_id (current_value 1)] ) (indices (current_value))) + (values (current_value)) + ) + )) + relevant_cluster_cases_map + ) + )) + k_parameter (get data_params_map "k") + )) + + ;sort subtrainee cases by their influences + (declare (assoc + sub_cases + (sort + (lambda + (< + (get combined_influences_map [(current_value 1)]) + (get combined_influences_map [(current_value 2)]) + ) ) + (indices combined_influences_map) ) - (indices combined_influences_map) - ) - )) + )) - (call !DynamicBandwidthFilterAndInterpolate (assoc - action_feature (first action_features) - sub_cases sub_cases - probs (unzip combined_influences_map sub_cases) - marginal_cutoff (if (~ [] k_parameter) (first k_parameter) ) - min_k (if (~ [] k_parameter) (get k_parameter 1) ) - static_k (if (~ 0 k_parameter) k_parameter) - )) + (call !DynamicBandwidthFilterAndInterpolate (assoc + action_feature (first action_features) + sub_cases sub_cases + probs (unzip combined_influences_map sub_cases) + marginal_cutoff (if (~ [] k_parameter) (first k_parameter) ) + min_k (if (~ [] k_parameter) (get k_parameter 1) ) + static_k (if (~ 0 k_parameter) k_parameter) + )) + ) ) ) ) diff --git a/module/react_utilities.amlg b/module/react_utilities.amlg index 45ce7a215..7ca91a7d6 100644 --- a/module/react_utilities.amlg +++ b/module/react_utilities.amlg @@ -1514,26 +1514,29 @@ )) ) - (call !InterpolateActionValues (assoc - action_feature action_feature - candidate_case_ids sub_cases - candidate_case_weights probs - candidate_case_values - (map - (lambda - (retrieve_from_entity [!traineeContainer (first (current_value 1)) (last (current_value 1))] action_feature) + ;output same format as ReactDiscriminative + { "action_values" [ + (call !InterpolateActionValues (assoc + action_feature action_feature + candidate_case_ids sub_cases + candidate_case_weights probs + candidate_case_values + (map + (lambda + (retrieve_from_entity [!traineeContainer (first (current_value 1)) (last (current_value 1))] action_feature) + ) + sub_cases ) - sub_cases - ) - allow_nulls allow_nulls - feature_has_continuous_nulls - (if allow_nulls - (and - (not (contains_index !nominalsMap action_feature)) - (!= .false (get !featureNullRatiosMap [action_feature "has_nulls"])) + allow_nulls allow_nulls + feature_has_continuous_nulls + (if allow_nulls + (and + (not (contains_index !nominalsMap action_feature)) + (!= .false (get !featureNullRatiosMap [action_feature "has_nulls"])) + ) ) - ) - )) + )) + ] } ) } \ No newline at end of file diff --git a/performance_tests/bank_hierarchy.amlg b/performance_tests/bank_hierarchy.amlg index 09fd1ab55..09e0cd2e8 100644 --- a/performance_tests/bank_hierarchy.amlg +++ b/performance_tests/bank_hierarchy.amlg @@ -102,11 +102,12 @@ cases (tail other_data -1000) ;everything but the first 1000 )) +(print (call_entity "howso" "react" (assoc action_features [(last features)] context_features (trunc features) - context_values [(trunc (first test_data))] + context_values (map (lambda (trunc (current_value))) (trunc test_data 20)) )) - +) (destroy_entities "howso") ) From e8ac4e8bfee8da6ac0386a1daeee4ffee2d81d1f Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Thu, 3 Sep 2026 07:53:59 -0400 Subject: [PATCH 07/42] update sandbox test --- performance_tests/bank_hierarchy.amlg | 71 ++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 7 deletions(-) diff --git a/performance_tests/bank_hierarchy.amlg b/performance_tests/bank_hierarchy.amlg index 09e0cd2e8..0ce63e7bc 100644 --- a/performance_tests/bank_hierarchy.amlg +++ b/performance_tests/bank_hierarchy.amlg @@ -8,7 +8,7 @@ (declare (assoc data (load "performance_data/bank-full.csv") train_size 2000 - test_size 5000 + test_size 10000 )) (declare (assoc @@ -102,12 +102,69 @@ cases (tail other_data -1000) ;everything but the first 1000 )) -(print - (call_entity "howso" "react" (assoc - action_features [(last features)] - context_features (trunc features) - context_values (map (lambda (trunc (current_value))) (trunc test_data 20)) + + (declare (assoc + predictions + (get + (call_entity "howso" "react" (assoc + action_features [(last features)] + context_features (trunc features) + context_values (map (lambda (trunc (current_value))) test_data) + )) + [1 "payload" "action_values"] + ) )) -) + + (print "accuracy: " + (/ + (size (filter + (lambda (let + (assoc + actual (last (current_value 1)) + predicted (get predictions [(current_index 2) 0]) + ) + (= actual predicted) + )) + test_data + )) + test_size + ) + "\n" + ) + + + + (call_entity "howso" "analyze") + + (assign (assoc + predictions + (get + (call_entity "howso" "react" (assoc + action_features [(last features)] + context_features (trunc features) + context_values (map (lambda (trunc (current_value))) test_data) + )) + [1 "payload" "action_values"] + ) + )) + + (print "post analyze accuracy: " + (/ + (size (filter + (lambda (let + (assoc + actual (last (current_value 1)) + predicted (get predictions [(current_index 2) 0]) + ) + (= actual predicted) + )) + test_data + )) + test_size + ) + "\n" + ) + + (destroy_entities "howso") ) From 4f22dda20dd8dfab3c01a631f4a56e9ab9ed8de0 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Thu, 3 Sep 2026 13:24:36 -0400 Subject: [PATCH 08/42] test update --- performance_tests/bank_hierarchy.amlg | 67 +++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/performance_tests/bank_hierarchy.amlg b/performance_tests/bank_hierarchy.amlg index 0ce63e7bc..cb76b0a96 100644 --- a/performance_tests/bank_hierarchy.amlg +++ b/performance_tests/bank_hierarchy.amlg @@ -167,4 +167,71 @@ (destroy_entities "howso") + + + + (load_entity "../howso.amlg" "howso" + .null .false {escape_resource_name .false escape_contained_resource_names .false} + ) + (set_entity_permissions "howso" .true) + (call_entity "howso" "initialize" (assoc trainee_id "hbank" filepath "../" print_progress .true)) + + (call_entity "howso" "set_feature_attributes" (assoc + feature_attributes + (assoc + "job" (assoc "type" "nominal" "id_feature" .true) + "marital" (assoc "type" "nominal") + "education" (assoc "type" "nominal") + "default" (assoc "type" "nominal") + "housing" (assoc "type" "nominal") + "loan" (assoc "type" "nominal") + "contact" (assoc "type" "nominal") + "month" (assoc "type" "nominal") + "poutcome" (assoc "type" "nominal") + "y" (assoc "type" "nominal") + "pdays" (assoc "type" "ordinal" "data_type" "number") + "age" (assoc "type" "ordinal" "data_type" "number") + "day" (assoc "type" "ordinal" "data_type" "number") + "campaign" (assoc "type" "ordinal" "data_type" "number") + ) + )) + + (call_entity "howso" "train" (assoc + features features + cases (append training_data other_data) + )) + (assign (assoc + num_cases (get (call_entity "howso" "get_num_training_cases") (list 1 "payload" "count")) + )) + (print "Reloaded Bank: " num_cases "\n") + (call_entity "howso" "analyze") + + (assign (assoc + predictions + (get + (call_entity "howso" "react" (assoc + action_features [(last features)] + context_features (trunc features) + context_values (map (lambda (trunc (current_value))) test_data) + )) + [1 "payload" "action_values"] + ) + )) + + (print "non-hierarchy accuracy: " + (/ + (size (filter + (lambda (let + (assoc + actual (last (current_value 1)) + predicted (get predictions [(current_index 2) 0]) + ) + (= actual predicted) + )) + test_data + )) + test_size + ) + "\n" + ) ) From 6b5b8372b60b18e0f8dd526881f87007a6150b62 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Fri, 4 Sep 2026 10:05:12 -0400 Subject: [PATCH 09/42] cleanup --- module/train.amlg | 2 +- performance_tests/bank_hierarchy.amlg | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/module/train.amlg b/module/train.amlg index d662e1e04..54041d7da 100644 --- a/module/train.amlg +++ b/module/train.amlg @@ -528,7 +528,7 @@ ;;;debug (map (lambda - (print (current_value) ": " + (print (current_value) " trained: " (get (call_entity [!traineeContainer (current_value 1)] "get_num_training_cases") [1 "payload" "count"] diff --git a/performance_tests/bank_hierarchy.amlg b/performance_tests/bank_hierarchy.amlg index cb76b0a96..f59b050ac 100644 --- a/performance_tests/bank_hierarchy.amlg +++ b/performance_tests/bank_hierarchy.amlg @@ -97,6 +97,9 @@ features features cases (trunc other_data 1000) ;only the first 1000 )) + + (print "\n") + (call_entity "howso" "train" (assoc features features cases (tail other_data -1000) ;everything but the first 1000 @@ -203,7 +206,7 @@ (assign (assoc num_cases (get (call_entity "howso" "get_num_training_cases") (list 1 "payload" "count")) )) - (print "Reloaded Bank: " num_cases "\n") + (print "\nReloaded Bank: " num_cases "\n") (call_entity "howso" "analyze") (assign (assoc From 3aa05abfb4de9d7b811d9306f946d7c6db353696 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Fri, 4 Sep 2026 11:32:12 -0400 Subject: [PATCH 10/42] auto hierarchy via parameters --- module/conviction.amlg | 2 +- module/dataparameters.amlg | 11 ++++++++++- module/hierarchy.amlg | 16 +++++++++------- module/train.amlg | 2 +- module/trainee.amlg | 2 ++ 5 files changed, 23 insertions(+), 10 deletions(-) diff --git a/module/conviction.amlg b/module/conviction.amlg index 8d214d6fc..8ebb9aeb7 100644 --- a/module/conviction.amlg +++ b/module/conviction.amlg @@ -303,7 +303,7 @@ ;!computedFeaturesMap so the weight_feature key is correct. (call !UpdateCaseWeightParameters) - (if (= .null case_ids) + (if (and (= .null case_ids) (size computed_map)) (assign_to_entities (assoc !computedFeaturesMap (assoc diff --git a/module/dataparameters.amlg b/module/dataparameters.amlg index c1b4feef8..323c2f932 100644 --- a/module/dataparameters.amlg +++ b/module/dataparameters.amlg @@ -1047,6 +1047,14 @@ ; above the threshold (in the case of rmse and mae) or below the threshold (otherwise). #{ref "AblationThresholdMap"} rel_threshold_map (assoc) + ;Smallest mass (number) of cases needed that can be considered an individual cluster when clustering + ;for auto hierarchy. When unspecified defaults to 5. Applicable when auto_hierarchy_enabled is true + #{type "number" exclusive_min 0} + clustering_min_cluster_mass .null + ;flag, default is false. when true, enables automatic internal hierarchy creation via clustering + #{type "boolean"} + auto_hierarchy_enabled .false + ) (call !ValidateParameters) @@ -1075,7 +1083,8 @@ !ablationBatchSize batch_size !ablatedCasesDistributionBatchSize ablated_cases_distribution_batch_size - !autoHierarchyEnabled .true + !autoHierarchyEnabled auto_hierarchy_enabled + !autoHierarchyClusteringMinMass clustering_min_cluster_mass )) (accum_to_entities (assoc !revision 1)) diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index b2475416f..8b2fdaee7 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -1257,14 +1257,11 @@ (call !Return) ) - !make_hierarchy - (declare - (assoc - clustering_min_cluster_mass 10 - ) + !MakeHierarchy + (seq (call react_into_features (assoc clustering .true - clustering_min_cluster_mass clustering_min_cluster_mass + clustering_min_cluster_mass !autoHierarchyClusteringMinMass use_case_weights .false )) @@ -1272,7 +1269,7 @@ cluster_ids_map (compute_on_contained_entities (query_value_masses ".cluster_id")) )) ;;;debug -(print cluster_ids_map "\n") +(print "clustered: " cluster_ids_map "\n") ;analyze with .cluster_id as a feature (if (!= .null !savedAnalyzeParameterMap) @@ -1304,6 +1301,11 @@ )) ) + (call !ProgressClear (assoc + internal .false + task_id .null + )) + ;there are unclustered cases, forcibly move them into clusters (if (contains_index cluster_ids_map 0) (let diff --git a/module/train.amlg b/module/train.amlg index 54041d7da..a015b0673 100644 --- a/module/train.amlg +++ b/module/train.amlg @@ -481,7 +481,7 @@ (if (= 0 (size !autoHierarchyIds)) (seq - (call !make_hierarchy) + (call !MakeHierarchy) (assign (assoc predicted_clusters diff --git a/module/trainee.amlg b/module/trainee.amlg index 3b6375dec..c2b1cee03 100644 --- a/module/trainee.amlg +++ b/module/trainee.amlg @@ -150,6 +150,8 @@ !autoHierarchyEnabled .false + !autoHierarchyClusteringMinMass 5 + ;unique id of parent trainee if this trainee is a subtrainee in a hierarchy !parentId .null From 64ba9c12469b15395ee894b34a0bcbc2f87701ff Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Fri, 4 Sep 2026 12:11:31 -0400 Subject: [PATCH 11/42] improved get_hierarchy, added is_contained flag to trainee --- module/hierarchy.amlg | 25 ++++++++++++++++----- module/trainee.amlg | 9 ++++++++ performance_tests/bank_hierarchy.amlg | 32 ++++++++++++++------------- 3 files changed, 45 insertions(+), 21 deletions(-) diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index 8b2fdaee7..ab934650d 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -1245,7 +1245,10 @@ (assign (assoc entity_path_id id)) ) - (call_entity [!traineeContainer entity_path_id] "set_parent_id" (assoc parent_id !traineeId)) + (call_entity [!traineeContainer entity_path_id] "set_parent_id" (assoc + parent_id !traineeId + is_contained .true + )) (accum_to_entities (assoc !containedTraineeNameToIdMap (associate entity_path_id id) @@ -1383,12 +1386,18 @@ ;copy all the attributes into subtrainee (assign_entity_roots [!traineeContainer subtrainee_id] (retrieve_entity_root)) - (call_entity [!traineeContainer subtrainee_id] "set_parent_id" (assoc parent_id !traineeId)) + (call_entity [!traineeContainer subtrainee_id] "set_parent_id" (assoc + parent_id !traineeId + is_contained .true + )) (call_entity [!traineeContainer subtrainee_id] "set_trainee_id" (assoc trainee_id subtrainee_id)) ;;;todo: remove this so subtrainees can automatically subdivide into their own hierarchies (call_entity [!traineeContainer subtrainee_id] "set_auto_ablation_params" (assoc auto_ablation_enabled .false)) + ;clear out hierarchy on the subtrainee since everything else was copied + (call_entity [!traineeContainer subtrainee_id] "prep_hierarchy_subtrainee") + ;output id subtrainee_id )) @@ -1399,10 +1408,11 @@ (assign_to_entities (assoc !autoHierarchyIds (sort subtrainee_ids) )) ) - prep_subtrainee + prep_hierarchy_subtrainee (assign_to_entities (assoc - !traineeId trainee_id - !parentId parent_id + !containedTraineeIdToNameMap (assoc) + !containedTraineeNameToIdMap (assoc) + !childTraineeIsContainedMap (assoc) )) ;Helper method to create the actual full contained entity path by interleaving !traineeContainer with each name in the path @@ -1417,7 +1427,7 @@ (assoc "id" (retrieve_from_entity "!traineeId") "path" path_list - "contained" .true + "contained" !traineeIsContained "children" (map (lambda @@ -1443,11 +1453,13 @@ ) "contained" .false "children" [] + "num_cases" .null ) ) ) (indices !childTraineeIsContainedMap) ) + "num_cases" (call !GetNumTrainingCases) ) ) @@ -1661,6 +1673,7 @@ ;set the !parentId for the child (call_entity child_entity_path "set_parent_id" (assoc parent_id (call_entity path_to_parent "get_trainee_id") + is_contained is_contained )) ) diff --git a/module/trainee.amlg b/module/trainee.amlg index c2b1cee03..53d0cbd67 100644 --- a/module/trainee.amlg +++ b/module/trainee.amlg @@ -158,6 +158,9 @@ ;unique id of this trainee !traineeId trainee_id + ;flag set to true if trainee is contained inside their parent + !traineeIsContained .false + ;amount of total influence weight to accumulate among nearest neighbors before stopping (for influential cases) !influenceWeightThreshold .99 @@ -701,9 +704,15 @@ ;the unique string identifier for the parent of the trainee #{type ["string" "null"]} parent_id .null + ;flag, no default. If specified, sets the flas specifying whether this trainee is contained inside the specified parent trainee. + #{type "boolean"} + is_contained .null ) (call !ValidateParameters) (assign_to_entities (assoc !parentId parent_id)) + (if (!= .null is_contained) + (assign_to_entities (assoc !traineeIsContained is_contained)) + ) (accum_to_entities (assoc !revision 1)) (call !Return) ) diff --git a/performance_tests/bank_hierarchy.amlg b/performance_tests/bank_hierarchy.amlg index f59b050ac..6677ef900 100644 --- a/performance_tests/bank_hierarchy.amlg +++ b/performance_tests/bank_hierarchy.amlg @@ -57,9 +57,8 @@ auto_ablation_enabled .true min_num_cases 2000 - ;todo: add parameters? - ; clustering_min_cluster_mass 10 - ; auto_hierarchy_enabled .true + clustering_min_cluster_mass 10 + auto_hierarchy_enabled .true )) (call_entity "howso" "train" (assoc @@ -80,7 +79,7 @@ (call_entity "howso" "analyze" (assoc context_features features targeted_model "targetless" - convergence_threshold 0 + ; convergence_threshold 0 )) (declare (assoc analyze_time (- (system_time) start) )) (print "Analyze time: " analyze_time "\n" ) @@ -93,18 +92,21 @@ "Base dataset num cases: " (get (call_entity "howso" "get_num_training_cases") [1 "payload" "count"]) "\n" ) - (call_entity "howso" "train" (assoc - features features - cases (trunc other_data 1000) ;only the first 1000 - )) - - (print "\n") + ; (call_entity "howso" "train" (assoc + ; features features + ; cases (trunc other_data 1000) ;only the first 1000 + ; )) + ; (print "\n") + ; (call_entity "howso" "train" (assoc + ; features features + ; cases (tail other_data -1000) ;everything but the first 1000 + ; )) (call_entity "howso" "train" (assoc features features - cases (tail other_data -1000) ;everything but the first 1000 + cases other_data )) - + (print (call_entity "howso" "get_hierarchy") ) (declare (assoc predictions @@ -132,7 +134,7 @@ )) test_size ) - "\n" + "\n\n" ) @@ -165,7 +167,7 @@ )) test_size ) - "\n" + "\n\n" ) @@ -206,7 +208,7 @@ (assign (assoc num_cases (get (call_entity "howso" "get_num_training_cases") (list 1 "payload" "count")) )) - (print "\nReloaded Bank: " num_cases "\n") + (print "Reloaded Bank: " num_cases "\n") (call_entity "howso" "analyze") (assign (assoc From 86df4cfa8e3e383e22683b2ea1588ff364ba7c41 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Fri, 4 Sep 2026 21:36:24 -0400 Subject: [PATCH 12/42] multi-layered hierarchy --- module/hierarchy.amlg | 86 +++++++++++++++++++-------- module/train.amlg | 55 +++++++++++++++-- performance_tests/bank_hierarchy.amlg | 2 +- 3 files changed, 111 insertions(+), 32 deletions(-) diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index ab934650d..4df2b6ab0 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -203,7 +203,10 @@ (if (!= .null result) (seq - (call_entity trainee_path "initialize" (assoc trainee_id child_id)) + (call_entity trainee_path "initialize" (assoc + trainee_id child_id + filepath (or filepath (retrieve_from_entity "filepath")) + )) (call !AddChildTraineeReferences (assoc child_id child_id @@ -1360,10 +1363,13 @@ ) (call create_subtrainee (assoc + filepath (retrieve_from_entity "filepath") path [subtrainee_id] child_id subtrainee_id )) + (set_entity_permissions [!traineeContainer subtrainee_id] .true) + ;copy cases into corresponding subtrainee (declare (assoc cases (contained_entities (query_equals ".cluster_id" cluster_id)) @@ -1371,32 +1377,64 @@ (declare (assoc case_features (indices (remove (retrieve_entity_root (first cases)) [".session" ".session_training_index" ".cluster_id"])) )) - (call_entity [!traineeContainer subtrainee_id] "train" (assoc - features case_features - cases - (map - (lambda (retrieve_from_entity (current_value) case_features)) - cases - ) - session "none" - ;allow training on reserved features to preserve all case data - allow_training_reserved_features .true - )) - ;copy all the attributes into subtrainee - (assign_entity_roots [!traineeContainer subtrainee_id] (retrieve_entity_root)) + (call_entity [!traineeContainer subtrainee_id] "set_feature_attributes" (assoc + feature_attributes (keep !featureAttributes case_features) + )) - (call_entity [!traineeContainer subtrainee_id] "set_parent_id" (assoc - parent_id !traineeId - is_contained .true + ;subtrainees can automatically subdivide into their own hierarchies as necessary but at a factor of 4x size of this trainee + (call_entity [!traineeContainer subtrainee_id] "set_auto_ablation_params" (assoc + auto_ablation_enabled .true + min_num_cases !autoAblationMinNumCases + clustering_min_cluster_mass !autoHierarchyClusteringMinMass + auto_hierarchy_enabled .true )) - (call_entity [!traineeContainer subtrainee_id] "set_trainee_id" (assoc trainee_id subtrainee_id)) - ;;;todo: remove this so subtrainees can automatically subdivide into their own hierarchies - (call_entity [!traineeContainer subtrainee_id] "set_auto_ablation_params" (assoc auto_ablation_enabled .false)) +;;; (call_entity [!traineeContainer subtrainee_id] "prep_hierarchy_subtrainee") + + ;if there are more cases than ablation thershold, split into: train min amount, analyze, train remainder + (if (> (size cases) !autoAblationMinNumCases) + (seq + (call_entity [!traineeContainer subtrainee_id] "train" (assoc + features case_features + cases + (map + (lambda (retrieve_from_entity (current_value) case_features)) + (trunc cases !autoAblationMinNumCases) + ) + session "none" + ;allow training on reserved features to preserve all case data + allow_training_reserved_features .true + )) + + (call_entity [!traineeContainer subtrainee_id] "analyze") - ;clear out hierarchy on the subtrainee since everything else was copied - (call_entity [!traineeContainer subtrainee_id] "prep_hierarchy_subtrainee") + (call_entity [!traineeContainer subtrainee_id] "train" (assoc + features case_features + cases + (map + (lambda (retrieve_from_entity (current_value) case_features)) + (tail cases (- !autoAblationMinNumCases)) + ) + session "none" + ;allow training on reserved features to preserve all case data + allow_training_reserved_features .true + )) + ) + + ;else train all the cases into the subtrainee + (call_entity [!traineeContainer subtrainee_id] "train" (assoc + features case_features + cases + (map + (lambda (retrieve_from_entity (current_value) case_features)) + cases + ) + session "none" + ;allow training on reserved features to preserve all case data + allow_training_reserved_features .true + )) + ) ;output id subtrainee_id @@ -1410,9 +1448,7 @@ prep_hierarchy_subtrainee (assign_to_entities (assoc - !containedTraineeIdToNameMap (assoc) - !containedTraineeNameToIdMap (assoc) - !childTraineeIsContainedMap (assoc) + ;copy data params ? data paths? cached residuals / expected )) ;Helper method to create the actual full contained entity path by interleaving !traineeContainer with each name in the path diff --git a/module/train.amlg b/module/train.amlg index a015b0673..3811c8286 100644 --- a/module/train.amlg +++ b/module/train.amlg @@ -506,7 +506,10 @@ (map (lambda (let - (assoc cluster_id (current_value 1)) + (assoc + cluster_id (current_value 1) + subtrainee_id (concat !traineeId "_sub" (current_value 1)) + ) (declare (assoc cluster_indices @@ -514,13 +517,53 @@ (lambda (= cluster_id (get predicted_clusters (current_value)))) (indices predicted_clusters) ) + num_cases_in_subtrainee + (get + (call_entity [!traineeContainer subtrainee_id] "get_num_training_cases") + ["1" "payload" "count"] + ) )) - (call_entity [!traineeContainer (concat !traineeId "_sub" cluster_id)] "train" (assoc - features features - cases (unzip cases cluster_indices) - session session - )) + ;training enough cases that the subtrainee should be split into its own hierarchy + ;split cluster_indices into however many needed to reach min num cases + ;analyze + ;train the remainder + (if (and + ;the trainee doesn't have enough cases in it already + (< num_cases_in_subtrainee !autoAblationMinNumCases) + (>= (+ num_cases_in_subtrainee (size cluster_indices)) !autoAblationMinNumCases) + ) + (let + (assoc + cases_for_subtrainee (unzip cases cluster_indices) + ;number of cases to reach the ablation/hierarchy threshold + first_train_amount (- !autoAblationMinNumCases num_cases_in_subtrainee) + ) +(print "enabling hierarchy for " subtrainee_id "\n") + (call_entity [!traineeContainer subtrainee_id] "train" (assoc + features features + ;train only the initial set of cases needed to reach the threshold + cases (trunc cases_for_subtrainee first_train_amount) + session session + )) + + (call_entity [!traineeContainer subtrainee_id] "analyze") + + (call_entity [!traineeContainer subtrainee_id] "train" (assoc + features features + ;everything but that initial set of cases + cases (tail cases_for_subtrainee (- first_train_amount)) + session session + )) + ) + + ;else train the cases into the subtrainee + (call_entity [!traineeContainer subtrainee_id] "train" (assoc + features features + cases (unzip cases cluster_indices) + session session + )) + ) )) (values predicted_clusters .true) ) diff --git a/performance_tests/bank_hierarchy.amlg b/performance_tests/bank_hierarchy.amlg index 6677ef900..0350f5ea3 100644 --- a/performance_tests/bank_hierarchy.amlg +++ b/performance_tests/bank_hierarchy.amlg @@ -55,7 +55,7 @@ (call_entity "howso" "set_auto_ablation_params" (assoc auto_ablation_enabled .true - min_num_cases 2000 + min_num_cases train_size ; 2000 cases clustering_min_cluster_mass 10 auto_hierarchy_enabled .true From 96fd2ae4737e258314b6a3bada1eab40dd10acb1 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 8 Sep 2026 09:58:36 -0400 Subject: [PATCH 13/42] hierarchy react when autohierarchy is enabled only --- module/react_discriminative.amlg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/react_discriminative.amlg b/module/react_discriminative.amlg index 6bf7690c5..c07598d51 100644 --- a/module/react_discriminative.amlg +++ b/module/react_discriminative.amlg @@ -365,7 +365,7 @@ )) - (if (and (size !autoHierarchyIds) (!= ".cluster_id" (first action_features))) + (if (and !autoHierarchyEnabled (size !autoHierarchyIds) (!= ".cluster_id" (first action_features))) (let (assoc clustering_prediction From 6b5d24d113f8269923f0262a558129dcf6c7d25e Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 8 Sep 2026 13:45:00 -0400 Subject: [PATCH 14/42] changes hierarchy min cluster size to use ratio to total instead of fixed value --- module/dataparameters.amlg | 8 +++---- module/hierarchy.amlg | 48 +++++++++++++++++++++++++++----------- module/train.amlg | 40 ++++++++++++++++++++----------- module/trainee.amlg | 2 +- 4 files changed, 66 insertions(+), 32 deletions(-) diff --git a/module/dataparameters.amlg b/module/dataparameters.amlg index 323c2f932..635dad29b 100644 --- a/module/dataparameters.amlg +++ b/module/dataparameters.amlg @@ -1047,10 +1047,10 @@ ; above the threshold (in the case of rmse and mae) or below the threshold (otherwise). #{ref "AblationThresholdMap"} rel_threshold_map (assoc) - ;Smallest mass (number) of cases needed that can be considered an individual cluster when clustering - ;for auto hierarchy. When unspecified defaults to 5. Applicable when auto_hierarchy_enabled is true + ;Ratio of smallest mass (number) of cases to total cases needed that can be considered an individual cluster when clustering + ;for auto hierarchy. When unspecified defaults to 0.1. Applicable when auto_hierarchy_enabled is true #{type "number" exclusive_min 0} - clustering_min_cluster_mass .null + hierarchy_clustering_ratio .null ;flag, default is false. when true, enables automatic internal hierarchy creation via clustering #{type "boolean"} auto_hierarchy_enabled .false @@ -1084,7 +1084,7 @@ !ablatedCasesDistributionBatchSize ablated_cases_distribution_batch_size !autoHierarchyEnabled auto_hierarchy_enabled - !autoHierarchyClusteringMinMass clustering_min_cluster_mass + !autoHierarchyClusteringRatio hierarchy_clustering_ratio )) (accum_to_entities (assoc !revision 1)) diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index 4df2b6ab0..7a7297b2c 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -1265,9 +1265,12 @@ !MakeHierarchy (seq + (declare (assoc + num_cases (call !GetNumTrainingCases) + )) (call react_into_features (assoc clustering .true - clustering_min_cluster_mass !autoHierarchyClusteringMinMass + clustering_min_cluster_mass (ceil (* num_cases !autoHierarchyClusteringRatio)) use_case_weights .false )) @@ -1277,6 +1280,11 @@ ;;;debug (print "clustered: " cluster_ids_map "\n") + ;don't create subtrainees if unable to cluster cases + (if (and (= 1 (size cluster_ids_map)) (contains_index cluster_ids_map 0)) + (conclude) + ) + ;analyze with .cluster_id as a feature (if (!= .null !savedAnalyzeParameterMap) (seq @@ -1321,18 +1329,26 @@ (declare (assoc forced_clusters - ||(map - (lambda (first - (call !ReactDiscriminative (assoc - context_features !trainedFeatures - context_values (retrieve_from_entity (current_value 1) !trainedFeatures) - action_features [".cluster_id"] - return_action_values_only .true - use_case_weights .true - filtering_queries [(query_not_equals ".cluster_id" 0) (query_not_in_entity_list [(current_value 3)])] + ;if there's only 1 actual cluster and a set of unclustered, move all the unclustered into the one cluster + (if (= 2 (size cluster_ids_map)) + (let + (assoc cluster_id (first (indices (remove cluster_ids_map 0))) ) + (map cluster_id unclustered_cases) + ) + + ||(map + (lambda (first + (call !ReactDiscriminative (assoc + context_features !trainedFeatures + context_values (retrieve_from_entity (current_value 1) !trainedFeatures) + action_features [".cluster_id"] + return_action_values_only .true + use_case_weights .true + filtering_queries [(query_not_equals ".cluster_id" 0) (query_not_in_entity_list [(current_value 3)])] + )) )) - )) - unclustered_cases + unclustered_cases + ) ) )) @@ -1352,6 +1368,12 @@ ;;;debug (print "updated: " cluster_ids_map "\n") + + ;don't create subtrainees if unable to cluster cases + (if (= 1 (size cluster_ids_map)) + (conclude) + ) + ;create and populate subtrainees, one per cluster (declare (assoc subtrainee_ids @@ -1386,7 +1408,7 @@ (call_entity [!traineeContainer subtrainee_id] "set_auto_ablation_params" (assoc auto_ablation_enabled .true min_num_cases !autoAblationMinNumCases - clustering_min_cluster_mass !autoHierarchyClusteringMinMass + hierarchy_clustering_ratio !autoHierarchyClusteringRatio auto_hierarchy_enabled .true )) diff --git a/module/train.amlg b/module/train.amlg index 3811c8286..581095d79 100644 --- a/module/train.amlg +++ b/module/train.amlg @@ -483,21 +483,33 @@ (seq (call !MakeHierarchy) - (assign (assoc - predicted_clusters - ||(map - (lambda (first - (call !ReactDiscriminative (assoc - context_features features - context_values (current_value 1) - action_features [".cluster_id"] - return_action_values_only .true - use_case_weights .true + ;unable to cluster cases, need to train all the would-be ablated cases into this trainee + (if (= 0 (size !autoHierarchyIds)) + (seq + + ;;;TODO: force-train the would-be ablated cases? undo their weight distribution? create a single sub cluster? + + ;empty predicted clusters + (assign (assoc predicted_clusters [])) + ) + + ;else predict which cluster each ablated case should go into + (assign (assoc + predicted_clusters + ||(map + (lambda (first + (call !ReactDiscriminative (assoc + context_features features + context_values (current_value 1) + action_features [".cluster_id"] + return_action_values_only .true + use_case_weights .true + )) )) - )) - cases - ) - )) + cases + ) + )) + ) ) ;else hierarchy already exists, only keep those predicted_clusters matching the ablated cases diff --git a/module/trainee.amlg b/module/trainee.amlg index 53d0cbd67..baa9112c2 100644 --- a/module/trainee.amlg +++ b/module/trainee.amlg @@ -150,7 +150,7 @@ !autoHierarchyEnabled .false - !autoHierarchyClusteringMinMass 5 + !autoHierarchyClusteringRatio 5 ;unique id of parent trainee if this trainee is a subtrainee in a hierarchy !parentId .null From 6bc93cd26ebac9a7f0390247ec30c97bcc338cd9 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 8 Sep 2026 18:35:59 -0400 Subject: [PATCH 15/42] initial support for single cluster subtrainee --- module/hierarchy.amlg | 96 +++++++++++++++++++++++-------------------- module/train.amlg | 86 +++++++++++++++++++++++++------------- 2 files changed, 109 insertions(+), 73 deletions(-) diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index 7a7297b2c..cf9c36009 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -1265,9 +1265,6 @@ !MakeHierarchy (seq - (declare (assoc - num_cases (call !GetNumTrainingCases) - )) (call react_into_features (assoc clustering .true clustering_min_cluster_mass (ceil (* num_cases !autoHierarchyClusteringRatio)) @@ -1280,41 +1277,45 @@ ;;;debug (print "clustered: " cluster_ids_map "\n") - ;don't create subtrainees if unable to cluster cases - (if (and (= 1 (size cluster_ids_map)) (contains_index cluster_ids_map 0)) - (conclude) - ) + ;don't analyze for clusters if unable to cluster cases + (if (not (or + (= 1 (size cluster_ids_map)) + ;only 1 actual cluster with some unclustered + (and + (= 2 (size cluster_ids_map)) + (contains_index cluster_ids_map 0) + ) + )) + ;analyze with .cluster_id as a feature + (if (!= .null !savedAnalyzeParameterMap) + (seq + (assign_to_entities (assoc + !savedAnalyzeParameterMap + (modify + !savedAnalyzeParameterMap + "context_features" + (append (get !savedAnalyzeParameterMap "context_features") ".cluster_id") + ) + )) + ;prevent calling react_into_features during the analyze() + (call !Analyze (append !savedAnalyzeParameterMap (assoc "from_react_into_features" .true) )) + ) - ;analyze with .cluster_id as a feature - (if (!= .null !savedAnalyzeParameterMap) - (seq - (assign_to_entities (assoc - !savedAnalyzeParameterMap - (modify - !savedAnalyzeParameterMap - "context_features" - (append (get !savedAnalyzeParameterMap "context_features") ".cluster_id") + (call !Analyze (assoc + targeted_model "targetless" + context_features (concat !trainedFeatures ".cluster_id") + weight_feature + (if (and !autoAblationEnabled !autoAblationWeightFeature) + !autoAblationWeightFeature + weight_feature ) + use_case_weights (or (and !autoAblationEnabled !autoAblationWeightFeature) use_case_weights) + k_folds 1 + ;prevent calling react_into_features during the analyze() + from_react_into_features .true )) - ;prevent calling react_into_features during the analyze() - (call !Analyze (append !savedAnalyzeParameterMap (assoc "from_react_into_features" .true) )) ) - - (call !Analyze (assoc - targeted_model "targetless" - context_features (concat !trainedFeatures ".cluster_id") - weight_feature - (if (and !autoAblationEnabled !autoAblationWeightFeature) - !autoAblationWeightFeature - weight_feature - ) - use_case_weights (or (and !autoAblationEnabled !autoAblationWeightFeature) use_case_weights) - k_folds 1 - ;prevent calling react_into_features during the analyze() - from_react_into_features .true - )) ) - (call !ProgressClear (assoc internal .false task_id .null @@ -1369,11 +1370,6 @@ (print "updated: " cluster_ids_map "\n") - ;don't create subtrainees if unable to cluster cases - (if (= 1 (size cluster_ids_map)) - (conclude) - ) - ;create and populate subtrainees, one per cluster (declare (assoc subtrainee_ids @@ -1404,13 +1400,23 @@ feature_attributes (keep !featureAttributes case_features) )) - ;subtrainees can automatically subdivide into their own hierarchies as necessary but at a factor of 4x size of this trainee - (call_entity [!traineeContainer subtrainee_id] "set_auto_ablation_params" (assoc - auto_ablation_enabled .true - min_num_cases !autoAblationMinNumCases - hierarchy_clustering_ratio !autoHierarchyClusteringRatio - auto_hierarchy_enabled .true - )) + ;do not enable auto ablation if there's only one subtrainee (cluster) + (if (= 1 (size cluster_ids_map)) + (call_entity [!traineeContainer subtrainee_id] "set_auto_ablation_params" (assoc + auto_ablation_enabled .false + min_num_cases !autoAblationMinNumCases + hierarchy_clustering_ratio !autoHierarchyClusteringRatio + auto_hierarchy_enabled .true + )) + + ;subtrainees can automatically subdivide into their own hierarchies as necessary but at a factor of 4x size of this trainee + (call_entity [!traineeContainer subtrainee_id] "set_auto_ablation_params" (assoc + auto_ablation_enabled .true + min_num_cases !autoAblationMinNumCases + hierarchy_clustering_ratio !autoHierarchyClusteringRatio + auto_hierarchy_enabled .true + )) + ) ;;; (call_entity [!traineeContainer subtrainee_id] "prep_hierarchy_subtrainee") diff --git a/module/train.amlg b/module/train.amlg index 581095d79..9b41d5199 100644 --- a/module/train.amlg +++ b/module/train.amlg @@ -60,7 +60,7 @@ (declare (assoc predicted_clusters - (if (and !autoHierarchyEnabled (size !autoHierarchyIds)) + (if (and !autoHierarchyEnabled !autoAblationEnabled) ||(map (lambda (first (call !ReactDiscriminative (assoc @@ -474,7 +474,48 @@ ".next_trained_index" (+ next_trained_index (size cases)) )) - (if (and !autoHierarchyEnabled (size ablated_indices_list)) + (declare (assoc + num_cases (call !GetNumTrainingCases) + )) + + ;hierarchy is enabled but not auto ablation, meaning this is a single cluster subtrainee, check if it can be clustered into more + (if (and + !autoHierarchyEnabled + (not !autoAblationEnabled) + (> num_cases !autoAblationMinNumCases) + (= 0 (size !autoHierarchyIds)) + ) + (seq + ;TODO: check if need to analyze here? + + (call react_into_features (assoc + clustering .true + clustering_min_cluster_mass (ceil (* num_cases !autoHierarchyClusteringRatio)) + use_case_weights .false + )) + + (declare (assoc + cluster_ids_map (compute_on_contained_entities (query_value_masses ".cluster_id")) + )) + + ;if clustered into multiple clusters + (if (or + (>= (size cluster_ids_map) 3) + (and + (= (size cluster_ids_map) 2) + (not (contains_index cluster_ids_map 0)) + ) + ) + ;;;TODO: + ;enable auto ablation + ;create sibling cluster ids in parent + ;update parent's hierarchy + ;move cases to siblings + ;update parent's cluster ids by predicting parent's cases + ) + ) + + (and !autoHierarchyEnabled (size ablated_indices_list)) (seq ;only keep the ablated cases (assign (assoc cases (keep cases ablated_indices_list) )) @@ -483,33 +524,22 @@ (seq (call !MakeHierarchy) - ;unable to cluster cases, need to train all the would-be ablated cases into this trainee - (if (= 0 (size !autoHierarchyIds)) - (seq - - ;;;TODO: force-train the would-be ablated cases? undo their weight distribution? create a single sub cluster? - - ;empty predicted clusters - (assign (assoc predicted_clusters [])) - ) - - ;else predict which cluster each ablated case should go into - (assign (assoc - predicted_clusters - ||(map - (lambda (first - (call !ReactDiscriminative (assoc - context_features features - context_values (current_value 1) - action_features [".cluster_id"] - return_action_values_only .true - use_case_weights .true - )) + ;else predict which cluster each ablated case should go into + (assign (assoc + predicted_clusters + ||(map + (lambda (first + (call !ReactDiscriminative (assoc + context_features features + context_values (current_value 1) + action_features [".cluster_id"] + return_action_values_only .true + use_case_weights .true )) - cases - ) - )) - ) + )) + cases + ) + )) ) ;else hierarchy already exists, only keep those predicted_clusters matching the ablated cases From 9839f70e437cdf44b3b84e72be0a019490490a86 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Wed, 9 Sep 2026 10:11:52 -0400 Subject: [PATCH 16/42] create subhiearrchy for singleton subtrainees --- module/hierarchy.amlg | 26 +++++++++++++++++--------- module/train.amlg | 19 +++++++++++-------- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index cf9c36009..2d1cfdaf2 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -1264,16 +1264,24 @@ ) !MakeHierarchy - (seq - (call react_into_features (assoc - clustering .true - clustering_min_cluster_mass (ceil (* num_cases !autoHierarchyClusteringRatio)) - use_case_weights .false - )) + (declare + (assoc + cluster_ids_map .null + ) - (declare (assoc - cluster_ids_map (compute_on_contained_entities (query_value_masses ".cluster_id")) - )) + (if (= .null cluster_ids_map) + (seq + (call react_into_features (assoc + clustering .true + clustering_min_cluster_mass (ceil (* num_cases !autoHierarchyClusteringRatio)) + use_case_weights .false + )) + + (assign (assoc + cluster_ids_map (compute_on_contained_entities (query_value_masses ".cluster_id")) + )) + ) + ) ;;;debug (print "clustered: " cluster_ids_map "\n") diff --git a/module/train.amlg b/module/train.amlg index 9b41d5199..0c3e56473 100644 --- a/module/train.amlg +++ b/module/train.amlg @@ -478,7 +478,8 @@ num_cases (call !GetNumTrainingCases) )) - ;hierarchy is enabled but not auto ablation, meaning this is a single cluster subtrainee, check if it can be clustered into more + ;hierarchy is enabled but not auto ablation, meaning this is a single cluster subtrainee, + ;check if it can be split into multiple clusters (if (and !autoHierarchyEnabled (not !autoAblationEnabled) @@ -498,7 +499,7 @@ cluster_ids_map (compute_on_contained_entities (query_value_masses ".cluster_id")) )) - ;if clustered into multiple clusters + ;if cases were clustered into multiple clusters (if (or (>= (size cluster_ids_map) 3) (and @@ -506,15 +507,17 @@ (not (contains_index cluster_ids_map 0)) ) ) - ;;;TODO: - ;enable auto ablation - ;create sibling cluster ids in parent - ;update parent's hierarchy - ;move cases to siblings - ;update parent's cluster ids by predicting parent's cases + (seq + ;enable auto ablation and create hierarchy so that + ;future trained cases are trained into subtrainees + (assign_to_entities (assoc !autoAblationEnabled .true)) + + (call !MakeHierarchy (assoc cluster_ids_map cluster_ids_map)) + ) ) ) + ;else hierarchy and ablation are enabled, ablated cases are trained into subtrainees (and !autoHierarchyEnabled (size ablated_indices_list)) (seq ;only keep the ablated cases From b5894e22e44b991a041c590cfebf172a61c7603a Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Wed, 9 Sep 2026 10:13:04 -0400 Subject: [PATCH 17/42] update test --- performance_tests/bank_hierarchy.amlg | 77 ++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/performance_tests/bank_hierarchy.amlg b/performance_tests/bank_hierarchy.amlg index 0350f5ea3..55d7b1d76 100644 --- a/performance_tests/bank_hierarchy.amlg +++ b/performance_tests/bank_hierarchy.amlg @@ -57,7 +57,7 @@ auto_ablation_enabled .true min_num_cases train_size ; 2000 cases - clustering_min_cluster_mass 10 + hierarchy_clustering_ratio 0.1 auto_hierarchy_enabled .true )) @@ -136,7 +136,47 @@ ) "\n\n" ) + (call_entity "howso" "set_auto_ablation_params" (assoc + auto_ablation_enabled .false + min_num_cases train_size ; 2000 cases + hierarchy_clustering_ratio 0.1 + auto_hierarchy_enabled .true + )) + (assign (assoc + predictions + (get + (call_entity "howso" "react" (assoc + action_features [(last features)] + context_features (trunc features) + context_values (map (lambda (trunc (current_value))) test_data) + )) + [1 "payload" "action_values"] + ) + )) + + (print "no hierarchy accuracy: " + (/ + (size (filter + (lambda (let + (assoc + actual (last (current_value 1)) + predicted (get predictions [(current_index 2) 0]) + ) + (= actual predicted) + )) + test_data + )) + test_size + ) + "\n\n" + ) + (call_entity "howso" "set_auto_ablation_params" (assoc + auto_ablation_enabled .true + min_num_cases train_size ; 2000 cases + hierarchy_clustering_ratio 0.1 + auto_hierarchy_enabled .true + )) (call_entity "howso" "analyze") @@ -170,6 +210,41 @@ "\n\n" ) + (call_entity "howso" "set_auto_ablation_params" (assoc + auto_ablation_enabled .false + min_num_cases train_size ; 2000 cases + hierarchy_clustering_ratio 0.1 + auto_hierarchy_enabled .true + )) + (assign (assoc + predictions + (get + (call_entity "howso" "react" (assoc + action_features [(last features)] + context_features (trunc features) + context_values (map (lambda (trunc (current_value))) test_data) + )) + [1 "payload" "action_values"] + ) + )) + + (print "post-analyze no hierarchy accuracy: " + (/ + (size (filter + (lambda (let + (assoc + actual (last (current_value 1)) + predicted (get predictions [(current_index 2) 0]) + ) + (= actual predicted) + )) + test_data + )) + test_size + ) + "\n\n" + ) + (destroy_entities "howso") From e20df94394bebd964d4b4ee5da24a376f2f55f8e Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Wed, 9 Sep 2026 17:17:02 -0400 Subject: [PATCH 18/42] fixes --- module/analysis.amlg | 5 ++ module/hierarchy.amlg | 111 +++++++++++++++++++++++++++++++++--------- module/train.amlg | 15 +++--- 3 files changed, 100 insertions(+), 31 deletions(-) diff --git a/module/analysis.amlg b/module/analysis.amlg index 9e1881704..8e4dccf67 100644 --- a/module/analysis.amlg +++ b/module/analysis.amlg @@ -883,6 +883,11 @@ (assign (assoc context_features_key (call !BuildContextFeaturesKey (assoc context_features context_features)) )) ) + ;update saved analyze parameters with the possibly changed context features + (assign_to_entities (assoc + !savedAnalyzeParameterMap (modify !savedAnalyzeParameterMap "context_features" context_features) + )) + ;if auto ablation is on and the user has not overridden the default, set use_case_weights to true (if (and !autoAblationEnabled (!= .false use_case_weights) ) (assign (assoc use_case_weights .true) ) diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index 2d1cfdaf2..f532851a3 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -1330,7 +1330,7 @@ )) ;there are unclustered cases, forcibly move them into clusters - (if (contains_index cluster_ids_map 0) + (if (and (contains_index cluster_ids_map 0) (> (size cluster_ids_map) 1)) (let (assoc unclustered_cases (contained_entities (query_equals ".cluster_id" 0)) @@ -1352,7 +1352,7 @@ context_values (retrieve_from_entity (current_value 1) !trainedFeatures) action_features [".cluster_id"] return_action_values_only .true - use_case_weights .true + use_case_weights !autoAblationEnabled filtering_queries [(query_not_equals ".cluster_id" 0) (query_not_in_entity_list [(current_value 3)])] )) )) @@ -1401,7 +1401,11 @@ cases (contained_entities (query_equals ".cluster_id" cluster_id)) )) (declare (assoc - case_features (indices (remove (retrieve_entity_root (first cases)) [".session" ".session_training_index" ".cluster_id"])) + case_features + (indices (remove + (retrieve_entity_root (first cases)) + [".session" ".session_training_index" ".cluster_id" ".case_weight" ".influence_weight_entropy"] + )) )) (call_entity [!traineeContainer subtrainee_id] "set_feature_attributes" (assoc @@ -1426,8 +1430,6 @@ )) ) -;;; (call_entity [!traineeContainer subtrainee_id] "prep_hierarchy_subtrainee") - ;if there are more cases than ablation thershold, split into: train min amount, analyze, train remainder (if (> (size cases) !autoAblationMinNumCases) (seq @@ -1445,31 +1447,45 @@ (call_entity [!traineeContainer subtrainee_id] "analyze") + ;split the remaining cases into chunks enabling checking whether this one subtrainee can be clustered + (if (= 1 (size cluster_ids_map)) + (call !TrainSubtraineeCasesInChunks (assoc + remaining_cases (tail cases (- !autoAblationMinNumCases)) + )) + + ;else train the remaining cases in a single call + (call_entity [!traineeContainer subtrainee_id] "train" (assoc + features case_features + cases + (map + (lambda (retrieve_from_entity (current_value) case_features)) + (tail cases (- !autoAblationMinNumCases)) + ) + session "none" + ;allow training on reserved features to preserve all case data + allow_training_reserved_features .true + )) + ) + ) + + ;else train all the cases into the subtrainee + (if (= 1 (size cluster_ids_map)) + (call !TrainSubtraineeCasesInChunks (assoc + remaining_cases cases + )) + (call_entity [!traineeContainer subtrainee_id] "train" (assoc features case_features cases (map (lambda (retrieve_from_entity (current_value) case_features)) - (tail cases (- !autoAblationMinNumCases)) + cases ) session "none" ;allow training on reserved features to preserve all case data allow_training_reserved_features .true )) ) - - ;else train all the cases into the subtrainee - (call_entity [!traineeContainer subtrainee_id] "train" (assoc - features case_features - cases - (map - (lambda (retrieve_from_entity (current_value) case_features)) - cases - ) - session "none" - ;allow training on reserved features to preserve all case data - allow_training_reserved_features .true - )) ) ;output id @@ -1482,10 +1498,59 @@ (assign_to_entities (assoc !autoHierarchyIds (sort subtrainee_ids) )) ) - prep_hierarchy_subtrainee - (assign_to_entities (assoc - ;copy data params ? data paths? cached residuals / expected - )) + !TrainSubtraineeCasesInChunks + (declare + (assoc remaining_cases [] ) + ;number of train chunks needed to train all the data if every chunk doubles in size + ;starting with the first chunk of size 2 * !autoAblationMinNumCases + (declare (assoc + num_train_chunks (ceil (- (log (size remaining_cases) 2) (log !autoAblationMinNumCases 2))) + start 0 + end 0 + num_remaining_cases (size remaining_cases) + chunk_size !autoAblationMinNumCases + )) + + (if (> num_train_chunks 0) + (while (< (current_index) num_train_chunks) + (assign (assoc + chunk_size (* 2 chunk_size) + start end + )) + (assign (assoc + end (+ start (- chunk_size 1)) + )) + (if (>= end num_remaining_cases) + (assign (assoc end (- num_remaining_cases 1) )) + ) + + (call_entity [!traineeContainer subtrainee_id] "train" (assoc + features case_features + cases + (map + (lambda (retrieve_from_entity (current_value) case_features)) + (unzip remaining_cases (range start end)) + ) + session "none" + ;allow training on reserved features to preserve all case data + allow_training_reserved_features .true + )) + ) + + ;else no chunking is needed, just train the remainder + (call_entity [!traineeContainer subtrainee_id] "train" (assoc + features case_features + cases + (map + (lambda (retrieve_from_entity (current_value) case_features)) + remaining_cases + ) + session "none" + ;allow training on reserved features to preserve all case data + allow_training_reserved_features .true + )) + ) + ) ;Helper method to create the actual full contained entity path by interleaving !traineeContainer with each name in the path !ConvertNamePathToEntityPath diff --git a/module/train.amlg b/module/train.amlg index 0c3e56473..7a3b74387 100644 --- a/module/train.amlg +++ b/module/train.amlg @@ -474,7 +474,7 @@ ".next_trained_index" (+ next_trained_index (size cases)) )) - (declare (assoc + (assign (assoc num_cases (call !GetNumTrainingCases) )) @@ -483,16 +483,15 @@ (if (and !autoHierarchyEnabled (not !autoAblationEnabled) - (> num_cases !autoAblationMinNumCases) + (>= num_cases !autoAblationMinNumCases) (= 0 (size !autoHierarchyIds)) ) (seq - ;TODO: check if need to analyze here? - (call react_into_features (assoc clustering .true clustering_min_cluster_mass (ceil (* num_cases !autoHierarchyClusteringRatio)) use_case_weights .false + overwrite .true )) (declare (assoc @@ -508,11 +507,11 @@ ) ) (seq + (call !MakeHierarchy (assoc cluster_ids_map cluster_ids_map)) + ;enable auto ablation and create hierarchy so that ;future trained cases are trained into subtrainees (assign_to_entities (assoc !autoAblationEnabled .true)) - - (call !MakeHierarchy (assoc cluster_ids_map cluster_ids_map)) ) ) ) @@ -594,7 +593,7 @@ (call_entity [!traineeContainer subtrainee_id] "analyze") - (call_entity [!traineeContainer subtrainee_id] "train" (assoc + (call_entity [!traineeContainer subtrainee_id] "train" (assoc ;;;TODO: chunk cases features features ;everything but that initial set of cases cases (tail cases_for_subtrainee (- first_train_amount)) @@ -603,7 +602,7 @@ ) ;else train the cases into the subtrainee - (call_entity [!traineeContainer subtrainee_id] "train" (assoc + (call_entity [!traineeContainer subtrainee_id] "train" (assoc ;;;TODO: chunk cases features features cases (unzip cases cluster_indices) session session From a9213c4c61d276d06d5b8bf9383ebf83413cbb9a Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Thu, 10 Sep 2026 10:33:27 -0400 Subject: [PATCH 19/42] adds simplified hierarchy --- module/hierarchy.amlg | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index f532851a3..91b891bf1 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -13,8 +13,17 @@ ;path to this trainee as a list of path labels #{ref "TraineePath"} path_list [] + ;flag, default to false. if set to true will only output simplified internal hierarchy + #{type "boolean"} + internal_only .false ) - (call !Return (assoc payload (call !GetHierarchy) )) + (call !Return (assoc + payload + (if internal_only + (call !GetInternalHierarchy) + (call !GetHierarchy) + ) + )) ) ;Returns the full entity path to a child trainee provided its unique trainee id if it is contained in the hierarchy. @@ -1285,7 +1294,7 @@ ;;;debug (print "clustered: " cluster_ids_map "\n") - ;don't analyze for clusters if unable to cluster cases + ;don't analyze for clusters if unable to cluster cases into two or more (if (not (or (= 1 (size cluster_ids_map)) ;only 1 actual cluster with some unclustered @@ -1556,7 +1565,7 @@ !ConvertNamePathToEntityPath (weave (range !traineeContainer 1 (size path) 1) path) - ;method to recurse down the the hierarchy to output the currently contained hierarchy as a nested assoc + ;method to recurse down the hierarchy to output the currently contained hierarchy as a nested assoc ;with .false for trainees that are stored independently !GetHierarchy (append @@ -1600,6 +1609,25 @@ ) ) + ;recurse down the hierarchy to output the internally contained hierarchy as a nested assoc with only the trainees and number of cases for each + !GetInternalHierarchy + (assoc + "num_cases" (call !GetNumTrainingCases) + "children" + (map + (lambda + (get + (call_entity (list !traineeContainer (current_index 1)) "get_hierarchy" (assoc + internal_only .true + )) + [1 "payload"] + ) + ) + ;only keep internal subtrainees + (filter .true !childTraineeIsContainedMap .true) + ) + ) + ;Returns the full entity path to a child trainee provided its unique trainee id if it is contained in the hierarchy. ;Iterates down the hierarchy searching for a trainee that matches the specified id, returns null if not found or ;a string error if found but trainee is stored externally as an independent trainee. From 93d97827bfd5e33d907225bfcf0ca75f629337e0 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Thu, 10 Sep 2026 16:10:55 -0400 Subject: [PATCH 20/42] hierarchy fixes for single subtrainee --- module/hierarchy.amlg | 38 ++++++++++++----- module/train.amlg | 96 +++++++++++++++++++++---------------------- 2 files changed, 76 insertions(+), 58 deletions(-) diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index 91b891bf1..3e3dbc7ff 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -1292,7 +1292,7 @@ ) ) ;;;debug -(print "clustered: " cluster_ids_map "\n") +(print !traineeId " clustered: " cluster_ids_map "\n") ;don't analyze for clusters if unable to cluster cases into two or more (if (not (or @@ -1315,7 +1315,10 @@ ) )) ;prevent calling react_into_features during the analyze() - (call !Analyze (append !savedAnalyzeParameterMap (assoc "from_react_into_features" .true) )) + (call !Analyze (append + !savedAnalyzeParameterMap + (assoc "from_react_into_features" .true) + )) ) (call !Analyze (assoc @@ -1421,7 +1424,7 @@ feature_attributes (keep !featureAttributes case_features) )) - ;do not enable auto ablation if there's only one subtrainee (cluster) + ;do not enable auto ablation if there's only one subtrainee (cluster) until that one subtrainee is ready for its on hierearchy (if (= 1 (size cluster_ids_map)) (call_entity [!traineeContainer subtrainee_id] "set_auto_ablation_params" (assoc auto_ablation_enabled .false @@ -1507,15 +1510,20 @@ (assign_to_entities (assoc !autoHierarchyIds (sort subtrainee_ids) )) ) + ;Trains sub trainee using growing chunks of remaining_cases + ;assumes cases are specified as entity ids, if cases_are_ids is set to false, remaining_cases are rows of feature values instead !TrainSubtraineeCasesInChunks (declare - (assoc remaining_cases [] ) + (assoc + remaining_cases [] + cases_are_ids .true + ) ;number of train chunks needed to train all the data if every chunk doubles in size ;starting with the first chunk of size 2 * !autoAblationMinNumCases (declare (assoc num_train_chunks (ceil (- (log (size remaining_cases) 2) (log !autoAblationMinNumCases 2))) start 0 - end 0 + end -1 num_remaining_cases (size remaining_cases) chunk_size !autoAblationMinNumCases )) @@ -1524,7 +1532,7 @@ (while (< (current_index) num_train_chunks) (assign (assoc chunk_size (* 2 chunk_size) - start end + start (+ end 1) )) (assign (assoc end (+ start (- chunk_size 1)) @@ -1536,8 +1544,13 @@ (call_entity [!traineeContainer subtrainee_id] "train" (assoc features case_features cases - (map - (lambda (retrieve_from_entity (current_value) case_features)) + (if cases_are_ids + (map + (lambda (retrieve_from_entity (current_value) case_features)) + (unzip remaining_cases (range start end)) + ) + + ;else remaining_cases are already rows of feature values (unzip remaining_cases (range start end)) ) session "none" @@ -1550,8 +1563,13 @@ (call_entity [!traineeContainer subtrainee_id] "train" (assoc features case_features cases - (map - (lambda (retrieve_from_entity (current_value) case_features)) + (if cases_are_ids + (map + (lambda (retrieve_from_entity (current_value) case_features)) + remaining_cases + ) + + ;else remaining_cases are already rows of feature values remaining_cases ) session "none" diff --git a/module/train.amlg b/module/train.amlg index 7a3b74387..b032a8604 100644 --- a/module/train.amlg +++ b/module/train.amlg @@ -57,21 +57,27 @@ (if (= .null session) (assign (assoc session "none")) ) - +(print !traineeId " training " (size cases) " ablation: " !autoAblationEnabled " subs: " (size !autoHierarchyIds) "\n") (declare (assoc predicted_clusters (if (and !autoHierarchyEnabled !autoAblationEnabled) - ||(map - (lambda (first - (call !ReactDiscriminative (assoc - context_features features - context_values (current_value 1) - action_features [".cluster_id"] - return_action_values_only .true - use_case_weights .true + (if (> (size !autoHierarchyIds) 1) + ||(map + (lambda (first + (call !ReactDiscriminative (assoc + context_features features + context_values (current_value 1) + action_features [".cluster_id"] + return_action_values_only .true + use_case_weights .true + )) )) - )) - cases + cases + ) + + ;else only one cluster + (= (size !autoHierarchyIds) 1) + (map 0 cases) ;;;;TODO: verify that it's always 0 ) ) )) @@ -497,7 +503,7 @@ (declare (assoc cluster_ids_map (compute_on_contained_entities (query_value_masses ".cluster_id")) )) - +(print !traineeId ": " num_cases " " cluster_ids_map "\n") ;if cases were clustered into multiple clusters (if (or (>= (size cluster_ids_map) 3) @@ -507,11 +513,13 @@ ) ) (seq - (call !MakeHierarchy (assoc cluster_ids_map cluster_ids_map)) - - ;enable auto ablation and create hierarchy so that + ;enable auto ablation and create hierarchy after making hierarchy so that ;future trained cases are trained into subtrainees (assign_to_entities (assoc !autoAblationEnabled .true)) + + (call !MakeHierarchy (assoc + cluster_ids_map cluster_ids_map + )) ) ) ) @@ -529,17 +537,23 @@ ;else predict which cluster each ablated case should go into (assign (assoc predicted_clusters - ||(map - (lambda (first - (call !ReactDiscriminative (assoc - context_features features - context_values (current_value 1) - action_features [".cluster_id"] - return_action_values_only .true - use_case_weights .true + (if (> (size !autoHierarchyIds) 1) + ||(map + (lambda (first + (call !ReactDiscriminative (assoc + context_features features + context_values (current_value 1) + action_features [".cluster_id"] + return_action_values_only .true + use_case_weights .true + )) )) - )) - cases + cases + ) + + ;else only one cluster + (= (size !autoHierarchyIds) 1) + (map 0 cases) ;;;;TODO: verify that it's always 0 ) )) ) @@ -593,42 +607,28 @@ (call_entity [!traineeContainer subtrainee_id] "analyze") - (call_entity [!traineeContainer subtrainee_id] "train" (assoc ;;;TODO: chunk cases - features features - ;everything but that initial set of cases - cases (tail cases_for_subtrainee (- first_train_amount)) - session session + (call !TrainSubtraineeCasesInChunks (assoc + remaining_cases (tail cases_for_subtrainee (- first_train_amount)) + case_features features + cases_are_ids .false )) ) ;else train the cases into the subtrainee - (call_entity [!traineeContainer subtrainee_id] "train" (assoc ;;;TODO: chunk cases - features features - cases (unzip cases cluster_indices) - session session + (call !TrainSubtraineeCasesInChunks (assoc + remaining_cases (unzip cases cluster_indices) + case_features features + cases_are_ids .false )) ) )) (values predicted_clusters .true) ) - -;;;debug -(map - (lambda - (print (current_value) " trained: " - (get - (call_entity [!traineeContainer (current_value 1)] "get_num_training_cases") - [1 "payload" "count"] - ) "\n" - ) - ) - !autoHierarchyIds -) ) ) (accum_to_entities (assoc !revision 1)) - +(print !traineeId " num_trained: " (size new_case_ids) " num_ablated: " (size ablated_indices_list) " into: " (values predicted_clusters .true) "\n") ;return response (call !Return (assoc warnings (if (size warnings) (indices warnings)) From 76212ed3f8871b422cd0c1402b653214b104758c Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Fri, 11 Sep 2026 17:07:49 -0400 Subject: [PATCH 21/42] WIP: refactoring to use reduce --- module/ablation.amlg | 4 + module/remove_cases.amlg | 15 +++- module/train.amlg | 174 ++++++++++++++++++++++++++++----------- module/trainee.amlg | 38 +++++++++ 4 files changed, 180 insertions(+), 51 deletions(-) diff --git a/module/ablation.amlg b/module/ablation.amlg index 9acccd9c5..abf64cf91 100644 --- a/module/ablation.amlg +++ b/module/ablation.amlg @@ -622,6 +622,9 @@ ;Optional task_id to track progress of this long running operation. If unspecified, will use .null as the task_id. #{type "string"} task_id .null + ;optional flag, specifying whether reduced cases should be removed completely (default behavior), or when true, removed into a storage container + #{type "boolean"} + remove_case_into_storage .false ;optional flag specifying whether this method was called internally by another method. ;Defaults to .false, assuming this will be called by an external API/user. #{type "boolean"} @@ -933,6 +936,7 @@ (call !RemoveCases (assoc cases cases_to_remove distribute_weight_feature distribute_weight_feature + remove_case_into_storage remove_case_into_storage )) ;else no cases to remove, clear caches for these added features so they can be cleared out quickly diff --git a/module/remove_cases.amlg b/module/remove_cases.amlg index 819d155dd..78eaea171 100644 --- a/module/remove_cases.amlg +++ b/module/remove_cases.amlg @@ -176,11 +176,13 @@ ;parameters: ; cases: list of case ids to remove ; distribute_weight_feature: name of feature into which to distribute the removed cases' weights to their neighbors. + ; remove_case_into_storage: flag, when true will remove cases into a storage container instead of deleting them !RemoveCases (declare (assoc cases (list) distribute_weight_feature .null + remove_case_into_storage .false ) (if (= 0 (size cases)) (conclude)) @@ -308,7 +310,18 @@ ;remove all the cases after clearing the query caches (reclaim_resources .null .false .true) - (apply "destroy_entities" cases) + (if remove_case_into_storage + (seq + (create_entities !reducedCasesStorage) + (map + (lambda (move_entities (current_value) [!reducedCasesStorage (current_value 1)])) + cases + ) + ) + + ;else simply delete the cases + (apply "destroy_entities" cases) + ) ;dataset has changed so clear out these cached value diff --git a/module/train.amlg b/module/train.amlg index b032a8604..0dcefec6b 100644 --- a/module/train.amlg +++ b/module/train.amlg @@ -53,6 +53,60 @@ (call !ValidateParameters) + + (declare (assoc num_cases (call !GetNumTrainingCases) )) + + ;if going to be training more cases that are allowed to fit into a dataset, split the training data + (if (and + !autoHierarchyEnabled + (> (+ num_cases (size cases)) !autoAblationMaxNumCases) + (not (and train_weights_only accumulate_weight_feature)) + ) + (let + (assoc + num_new (size cases) + payload [1 {"payload" {"num_trained" 0 "ablated_indices" [] "status" .null}}] + new_payload .null + start 0 + end -1 + ) + + (while (> num_new 0) + (assign (assoc + num_cases (call !GetNumTrainingCases) + num_to_train (min (- !autoAblationMaxNumCases num_cases) num_new) + start (+ end 1) + end (+ start num_to_train -1) + new_payload + (call !train (assoc + ;;;TODO: don't split time series in middle of a series + cases (unzip cases (range start end)) + features features + derived_features derived_features + session session + series series + input_is_substituted input_is_substituted + allow_training_reserved_features allow_training_reserved_features + skip_auto_analyze skip_auto_analyze + skip_reduce_data skip_reduce_data + start_index start_index + )) + + start_index (if start_index (+ start_index num_to_train)) + num_new (- num_new num_to_train) + )) + + ;accumulate payload + (call !AccumulateTrainPayloads) + + ;stop train on errors + (if (= 0 (first payload)) (conclude) ) + ) + + (conclude payload) + ) + ) + ;unsure that session is set to some string value (if (= .null session) (assign (assoc session "none")) @@ -61,23 +115,17 @@ (declare (assoc predicted_clusters (if (and !autoHierarchyEnabled !autoAblationEnabled) - (if (> (size !autoHierarchyIds) 1) - ||(map - (lambda (first - (call !ReactDiscriminative (assoc - context_features features - context_values (current_value 1) - action_features [".cluster_id"] - return_action_values_only .true - use_case_weights .true - )) + ||(map + (lambda (first + (call !ReactDiscriminative (assoc + context_features features + context_values (current_value 1) + action_features [".cluster_id"] + return_action_values_only .true + use_case_weights .true )) - cases - ) - - ;else only one cluster - (= (size !autoHierarchyIds) 1) - (map 0 cases) ;;;;TODO: verify that it's always 0 + )) + cases ) ) )) @@ -327,8 +375,6 @@ ) ) - (declare (assoc num_cases (call !GetNumTrainingCases) )) - (declare (assoc skip_ablation (call !CanTrainAblationBeSkipped) ;if accumulating weight feature, store the index of that weight feature @@ -484,48 +530,76 @@ num_cases (call !GetNumTrainingCases) )) - ;hierarchy is enabled but not auto ablation, meaning this is a single cluster subtrainee, - ;check if it can be split into multiple clusters - (if (and - !autoHierarchyEnabled - (not !autoAblationEnabled) - (>= num_cases !autoAblationMinNumCases) - (= 0 (size !autoHierarchyIds)) - ) - (seq - (call react_into_features (assoc - clustering .true - clustering_min_cluster_mass (ceil (* num_cases !autoHierarchyClusteringRatio)) - use_case_weights .false - overwrite .true - )) + (if (and !autoHierarchyEnabled (>= num_cases !autoAblationMaxNumCases)) + (let + (assoc + reduce_payload + (call reduce_data (assoc + features features + skip_auto_analyze .true + internal .true + remove_case_into_storage .true + )) + ) + + (if (= 0 (size !autoHierarchyIds)) + (call react_into_features (assoc + clustering .true + clustering_min_cluster_mass (ceil (* num_cases !autoHierarchyClusteringRatio)) + use_case_weights .true + )) + ) (declare (assoc - cluster_ids_map (compute_on_contained_entities (query_value_masses ".cluster_id")) - )) -(print !traineeId ": " num_cases " " cluster_ids_map "\n") - ;if cases were clustered into multiple clusters - (if (or - (>= (size cluster_ids_map) 3) - (and - (= (size cluster_ids_map) 2) - (not (contains_index cluster_ids_map 0)) + subtrainee_ids + (if (= 0 (size !autoHierarchyIds)) + (map + (lambda (concat !traineeId "_sub" (current_value)) ) + (indices (compute_on_contained_entities (query_value_masses ".cluster_id"))) + ) + + !autoHierarchyIds ) - ) + )) + + (if (= 0 (size !autoHierarchyIds)) (seq - ;enable auto ablation and create hierarchy after making hierarchy so that - ;future trained cases are trained into subtrainees - (assign_to_entities (assoc !autoAblationEnabled .true)) + (map + (lambda (let + (assoc subtrainee_id (current_value 1)) + (call create_subtrainee (assoc + filepath (retrieve_from_entity "filepath") + path [subtrainee_id] + child_id subtrainee_id + )) - (call !MakeHierarchy (assoc - cluster_ids_map cluster_ids_map - )) + (set_entity_permissions [!traineeContainer subtrainee_id] .true) + + (call_entity [!traineeContainer subtrainee_id] "set_feature_attributes" (assoc + feature_attributes (keep !featureAttributes features) + )) + + (call_entity [!traineeContainer subtrainee_id] "set_auto_ablation_params" (assoc + ;auto_ablation_enabled .false + min_num_cases !autoAblationMinNumCases + hierarchy_clustering_ratio !autoHierarchyClusteringRatio + auto_hierarchy_enabled .true + )) + )) + subtrainee_ids + ) + (assign_to_entities (assoc !autoHierarchyIds (sort subtrainee_ids) )) ) ) + + ;;;TODO: predict clusters for all cases in (contained_entities !reducedCasesStorage) + ;;;TODO: train subtrainees + ;;;TODO: remove !reducedCasesStorage ) + ) ;else hierarchy and ablation are enabled, ablated cases are trained into subtrainees - (and !autoHierarchyEnabled (size ablated_indices_list)) + (if (and !autoHierarchyEnabled (size ablated_indices_list)) (seq ;only keep the ablated cases (assign (assoc cases (keep cases ablated_indices_list) )) diff --git a/module/trainee.amlg b/module/trainee.amlg index baa9112c2..7e1df3bd1 100644 --- a/module/trainee.amlg +++ b/module/trainee.amlg @@ -44,6 +44,8 @@ ;the name of the entity that contains subtrainees !traineeContainer ".trainee_container" + ;the name of the entity containing cases slated for removal + !reducedCasesStorage ".reduced_cases_storage" ;the supported prediction stats that users can request from react_aggregate !supportedPredictionStats (list "mae" "confusion_matrix" "r2" "rmse" "adjusted_smape" "smape" "spearman_coeff" "precision" "recall" "accuracy" "mcc" "all" "missing_value_accuracy") @@ -960,4 +962,40 @@ (list 1 (assoc "payload" payload) ) ) + ;accumulate new_payload with payload + !AccumulateTrainPayloads + (assign (assoc + payload + [ + ;if errors, output a 0 + (if (or (= 0 (first payload)) (= 0 (first new_payload))) + 0 + 1 + ) + (assoc + "payload" + { + "num_trained" + (+ + (get payload [1 "payload" "num_trained"]) + (get new_payload [1 "payload" "num_trained"]) + ) + "ablated_indices" + (append + (get payload [1 "payload" "ablated_indices"]) + (get new_payload [1 "payload" "ablated_indices"]) + ) + "status" + (or + (get payload [1 "payload" "status"]) + (get new_payload [1 "payload" "status"]) + ) + } + "warnings" (filter (append (get payload [1 "warnings"]) (get new_payload [1 "warnings"]))) + "errors" (filter (append (get payload [1 "errors"]) (get new_payload [1 "errors"]))) + ) + ] + )) + + } From aadbab4df919ab150a8d7ac430aecefbb7f828b6 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Mon, 14 Sep 2026 16:50:43 -0400 Subject: [PATCH 22/42] redo to use reduction --- module/ablation.amlg | 4 +- module/hierarchy.amlg | 495 +++++++++++++++++++-------------------- module/progress.amlg | 3 + module/remove_cases.amlg | 10 +- module/train.amlg | 171 +------------- 5 files changed, 261 insertions(+), 422 deletions(-) diff --git a/module/ablation.amlg b/module/ablation.amlg index abf64cf91..986dfc9e3 100644 --- a/module/ablation.amlg +++ b/module/ablation.amlg @@ -624,7 +624,7 @@ task_id .null ;optional flag, specifying whether reduced cases should be removed completely (default behavior), or when true, removed into a storage container #{type "boolean"} - remove_case_into_storage .false + remove_cases_into_storage .false ;optional flag specifying whether this method was called internally by another method. ;Defaults to .false, assuming this will be called by an external API/user. #{type "boolean"} @@ -936,7 +936,7 @@ (call !RemoveCases (assoc cases cases_to_remove distribute_weight_feature distribute_weight_feature - remove_case_into_storage remove_case_into_storage + remove_cases_into_storage remove_cases_into_storage )) ;else no cases to remove, clear caches for these added features so they can be cleared out quickly diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index 3e3dbc7ff..fc04ebad1 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -1273,133 +1273,147 @@ ) !MakeHierarchy - (declare - (assoc - cluster_ids_map .null - ) + (seq - (if (= .null cluster_ids_map) - (seq - (call react_into_features (assoc - clustering .true - clustering_min_cluster_mass (ceil (* num_cases !autoHierarchyClusteringRatio)) - use_case_weights .false - )) + ;if subtrainees have not been created yet, trainee should be analyzed if it hasn't been analyzed yet + (if (and (= 0 (size !autoHierarchyIds)) (= .null !savedAnalyzeParameterMap)) + (call analyze) - (assign (assoc - cluster_ids_map (compute_on_contained_entities (query_value_masses ".cluster_id")) - )) - ) + !autoAnalyzeEnabled + (call !Analyze !savedAnalyzeParameterMap) ) -;;;debug -(print !traineeId " clustered: " cluster_ids_map "\n") - - ;don't analyze for clusters if unable to cluster cases into two or more - (if (not (or - (= 1 (size cluster_ids_map)) - ;only 1 actual cluster with some unclustered - (and - (= 2 (size cluster_ids_map)) - (contains_index cluster_ids_map 0) - ) - )) - ;analyze with .cluster_id as a feature - (if (!= .null !savedAnalyzeParameterMap) - (seq - (assign_to_entities (assoc - !savedAnalyzeParameterMap - (modify - !savedAnalyzeParameterMap - "context_features" - (append (get !savedAnalyzeParameterMap "context_features") ".cluster_id") - ) - )) - ;prevent calling react_into_features during the analyze() - (call !Analyze (append - !savedAnalyzeParameterMap - (assoc "from_react_into_features" .true) - )) - ) - (call !Analyze (assoc - targeted_model "targetless" - context_features (concat !trainedFeatures ".cluster_id") - weight_feature - (if (and !autoAblationEnabled !autoAblationWeightFeature) - !autoAblationWeightFeature - weight_feature - ) - use_case_weights (or (and !autoAblationEnabled !autoAblationWeightFeature) use_case_weights) - k_folds 1 - ;prevent calling react_into_features during the analyze() - from_react_into_features .true - )) - ) - ) (call !ProgressClear (assoc internal .false task_id .null )) - ;there are unclustered cases, forcibly move them into clusters - (if (and (contains_index cluster_ids_map 0) (> (size cluster_ids_map) 1)) - (let - (assoc - unclustered_cases (contained_entities (query_equals ".cluster_id" 0)) - ) + (declare (assoc + reduce_payload + (call reduce_data (assoc + features features + skip_auto_analyze .true + internal .true + remove_cases_into_storage .true + )) + )) - (declare (assoc - forced_clusters - ;if there's only 1 actual cluster and a set of unclustered, move all the unclustered into the one cluster - (if (= 2 (size cluster_ids_map)) - (let - (assoc cluster_id (first (indices (remove cluster_ids_map 0))) ) - (map cluster_id unclustered_cases) - ) + (declare (assoc + subtrainee_ids + (if (= 0 (size !autoHierarchyIds)) + (seq + (call react_into_features (assoc + clustering .true + clustering_min_cluster_mass (ceil (* num_cases !autoHierarchyClusteringRatio)) + use_case_weights .true + )) - ||(map - (lambda (first - (call !ReactDiscriminative (assoc - context_features !trainedFeatures - context_values (retrieve_from_entity (current_value 1) !trainedFeatures) - action_features [".cluster_id"] - return_action_values_only .true - use_case_weights !autoAblationEnabled - filtering_queries [(query_not_equals ".cluster_id" 0) (query_not_in_entity_list [(current_value 3)])] - )) + (if (!= .null !savedAnalyzeParameterMap) + (seq + (assign_to_entities (assoc + !savedAnalyzeParameterMap + (modify !savedAnalyzeParameterMap + "context_features" (append (get !savedAnalyzeParameterMap "context_features") ".cluster_id") + "use_case_weights" .true + "weight_feature" ".case_weight" + ) + )) + ;prevent calling react_into_features during the analyze() + (call !Analyze (append + !savedAnalyzeParameterMap + (assoc "from_react_into_features" .true) )) - unclustered_cases ) + + (call !Analyze (assoc + targeted_model "targetless" + context_features (concat !trainedFeatures ".cluster_id") + weight_feature + (if (and !autoAblationEnabled !autoAblationWeightFeature) + !autoAblationWeightFeature + weight_feature + ) + use_case_weights .true + k_folds 1 + ;prevent calling react_into_features during the analyze() + from_react_into_features .true + )) ) - )) - ;update cluster id for each unclustered case - (map - (lambda - (assign_to_entities (current_index) (assoc ".cluster_id" (current_value 1))) + (call !ProgressClear (assoc + internal .false + task_id .null + )) + + (map + (lambda (concat !traineeId "_sub" (current_value)) ) + (indices (compute_on_contained_entities (query_value_masses ".cluster_id"))) + ) ) - (zip unclustered_cases forced_clusters) - ) - ) - ) - (assign (assoc - cluster_ids_map (compute_on_contained_entities (query_value_masses ".cluster_id")) + ;else use the already created subtrainee ids + !autoHierarchyIds + ) )) -;;;debug -(print "updated: " cluster_ids_map "\n") + (if (= 0 (size !autoHierarchyIds)) + (let + (assoc + cluster_ids (indices (compute_on_contained_entities (query_value_masses ".cluster_id"))) + ) - ;create and populate subtrainees, one per cluster - (declare (assoc - subtrainee_ids - (map - (lambda (let + (if (and (contains_value cluster_ids 0) (> (size cluster_ids) 1)) + (let (assoc - cluster_id (current_value 1) - subtrainee_id (concat !traineeId "_sub" (current_value 1)) + unclustered_cases (contained_entities (query_equals ".cluster_id" 0)) ) + (declare (assoc + forced_clusters + ;if there's only 1 actual cluster and a set of unclustered, move all the unclustered into the one cluster + (if (= 2 (size cluster_ids_map)) + (let + (assoc cluster_id (first (indices (remove cluster_ids_map 0))) ) + (map cluster_id unclustered_cases) + ) + + ||(map + (lambda (first + (call !ReactDiscriminative (assoc + context_features !trainedFeatures + context_values (retrieve_from_entity (current_value 1) !trainedFeatures) + action_features [".cluster_id"] + return_action_values_only .true + use_case_weights .true + filtering_queries [(query_not_equals ".cluster_id" 0) (query_not_in_entity_list [(current_value 3)])] + )) + )) + unclustered_cases + ) + ) + )) + + ;update cluster id for each unclustered case + (map + (lambda + (assign_to_entities (current_index) (assoc ".cluster_id" (current_value 1))) + ) + (zip unclustered_cases forced_clusters) + ) + + (assign (assoc + subtrainee_ids + (map + (lambda (concat !traineeId "_sub" (current_value)) ) + (indices (compute_on_contained_entities (query_value_masses ".cluster_id"))) + ) + )) + ) + ) + + (map + (lambda (let + (assoc subtrainee_id (current_value 1)) (call create_subtrainee (assoc filepath (retrieve_from_entity "filepath") path [subtrainee_id] @@ -1408,177 +1422,152 @@ (set_entity_permissions [!traineeContainer subtrainee_id] .true) - ;copy cases into corresponding subtrainee - (declare (assoc - cases (contained_entities (query_equals ".cluster_id" cluster_id)) - )) - (declare (assoc - case_features - (indices (remove - (retrieve_entity_root (first cases)) - [".session" ".session_training_index" ".cluster_id" ".case_weight" ".influence_weight_entropy"] - )) - )) - (call_entity [!traineeContainer subtrainee_id] "set_feature_attributes" (assoc - feature_attributes (keep !featureAttributes case_features) + feature_attributes (keep !featureAttributes features) )) - ;do not enable auto ablation if there's only one subtrainee (cluster) until that one subtrainee is ready for its on hierearchy - (if (= 1 (size cluster_ids_map)) - (call_entity [!traineeContainer subtrainee_id] "set_auto_ablation_params" (assoc - auto_ablation_enabled .false - min_num_cases !autoAblationMinNumCases - hierarchy_clustering_ratio !autoHierarchyClusteringRatio - auto_hierarchy_enabled .true - )) - - ;subtrainees can automatically subdivide into their own hierarchies as necessary but at a factor of 4x size of this trainee - (call_entity [!traineeContainer subtrainee_id] "set_auto_ablation_params" (assoc - auto_ablation_enabled .true - min_num_cases !autoAblationMinNumCases - hierarchy_clustering_ratio !autoHierarchyClusteringRatio - auto_hierarchy_enabled .true - )) - ) - - ;if there are more cases than ablation thershold, split into: train min amount, analyze, train remainder - (if (> (size cases) !autoAblationMinNumCases) - (seq - (call_entity [!traineeContainer subtrainee_id] "train" (assoc - features case_features - cases - (map - (lambda (retrieve_from_entity (current_value) case_features)) - (trunc cases !autoAblationMinNumCases) - ) - session "none" - ;allow training on reserved features to preserve all case data - allow_training_reserved_features .true - )) - - (call_entity [!traineeContainer subtrainee_id] "analyze") - - ;split the remaining cases into chunks enabling checking whether this one subtrainee can be clustered - (if (= 1 (size cluster_ids_map)) - (call !TrainSubtraineeCasesInChunks (assoc - remaining_cases (tail cases (- !autoAblationMinNumCases)) - )) - - ;else train the remaining cases in a single call - (call_entity [!traineeContainer subtrainee_id] "train" (assoc - features case_features - cases - (map - (lambda (retrieve_from_entity (current_value) case_features)) - (tail cases (- !autoAblationMinNumCases)) - ) - session "none" - ;allow training on reserved features to preserve all case data - allow_training_reserved_features .true - )) - ) - ) - - ;else train all the cases into the subtrainee - (if (= 1 (size cluster_ids_map)) - (call !TrainSubtraineeCasesInChunks (assoc - remaining_cases cases - )) - - (call_entity [!traineeContainer subtrainee_id] "train" (assoc - features case_features - cases - (map - (lambda (retrieve_from_entity (current_value) case_features)) - cases - ) - session "none" - ;allow training on reserved features to preserve all case data - allow_training_reserved_features .true - )) - ) - ) - - ;output id - subtrainee_id + (call_entity [!traineeContainer subtrainee_id] "set_auto_ablation_params" (assoc + ;auto_ablation_enabled .false + min_num_cases !autoAblationMinNumCases + hierarchy_clustering_ratio !autoHierarchyClusteringRatio + auto_hierarchy_enabled .true + )) )) - (indices cluster_ids_map) + subtrainee_ids ) - )) - (assign_to_entities (assoc !autoHierarchyIds (sort subtrainee_ids) )) - ) - - ;Trains sub trainee using growing chunks of remaining_cases - ;assumes cases are specified as entity ids, if cases_are_ids is set to false, remaining_cases are rows of feature values instead - !TrainSubtraineeCasesInChunks - (declare - (assoc - remaining_cases [] - cases_are_ids .true + (assign_to_entities (assoc !autoHierarchyIds (sort subtrainee_ids) )) + ) ) - ;number of train chunks needed to train all the data if every chunk doubles in size - ;starting with the first chunk of size 2 * !autoAblationMinNumCases + (declare (assoc - num_train_chunks (ceil (- (log (size remaining_cases) 2) (log !autoAblationMinNumCases 2))) - start 0 - end -1 - num_remaining_cases (size remaining_cases) - chunk_size !autoAblationMinNumCases + reduced_cases_values + (map + (lambda (retrieve_from_entity [!reducedCasesStorage (current_value 1)] features)) + (contained_entities !reducedCasesStorage) + ) )) - (if (> num_train_chunks 0) - (while (< (current_index) num_train_chunks) - (assign (assoc - chunk_size (* 2 chunk_size) - start (+ end 1) - )) - (assign (assoc - end (+ start (- chunk_size 1)) - )) - (if (>= end num_remaining_cases) - (assign (assoc end (- num_remaining_cases 1) )) + ;train subtrainees + (call !TrainSubtraineeClusterIndices (assoc + cases reduced_cases_values + ;predict clusters for all cases in (contained_entities !reducedCasesStorage) + predicted_clusters + ||(map + (lambda (first + (call !ReactDiscriminative (assoc + context_features features + context_values (current_value 1) + action_features [".cluster_id"] + return_action_values_only .true + use_case_weights .true + )) + )) + reduced_cases_values ) + )) - (call_entity [!traineeContainer subtrainee_id] "train" (assoc - features case_features - cases - (if cases_are_ids - (map - (lambda (retrieve_from_entity (current_value) case_features)) - (unzip remaining_cases (range start end)) - ) + ;ensure ablation is enabled going forward + (assign (assoc !autoAblationEnabled .true)) - ;else remaining_cases are already rows of feature values - (unzip remaining_cases (range start end)) - ) - session "none" - ;allow training on reserved features to preserve all case data - allow_training_reserved_features .true - )) + ;remove !reducedCasesStorage + (destroy_entities !reducedCasesStorage) + ) + + !TrainSubtraineeClusterIndices + (map + (lambda (let + (assoc + cluster_id (current_value 1) + subtrainee_id (concat !traineeId "_sub" (current_value 1)) + cluster_indices + (filter + (lambda (= cluster_id (get predicted_clusters (current_value)))) + (indices predicted_clusters) + ) ) - ;else no chunking is needed, just train the remainder (call_entity [!traineeContainer subtrainee_id] "train" (assoc - features case_features - cases - (if cases_are_ids - (map - (lambda (retrieve_from_entity (current_value) case_features)) - remaining_cases - ) - - ;else remaining_cases are already rows of feature values - remaining_cases - ) + features features + cases (unzip cases cluster_indices) session "none" ;allow training on reserved features to preserve all case data allow_training_reserved_features .true )) - ) + )) + (values predicted_clusters .true) ) + + ; ;Trains sub trainee using growing chunks of remaining_cases + ; ;assumes cases are specified as entity ids, if cases_are_ids is set to false, remaining_cases are rows of feature values instead + ; !TrainSubtraineeCasesInChunks + ; (declare + ; (assoc + ; remaining_cases [] + ; cases_are_ids .true + ; ) + ; ;number of train chunks needed to train all the data if every chunk doubles in size + ; ;starting with the first chunk of size 2 * !autoAblationMinNumCases + ; (declare (assoc + ; num_train_chunks (ceil (- (log (size remaining_cases) 2) (log !autoAblationMinNumCases 2))) + ; start 0 + ; end -1 + ; num_remaining_cases (size remaining_cases) + ; chunk_size !autoAblationMinNumCases + ; )) + + ; (if (> num_train_chunks 0) + ; (while (< (current_index) num_train_chunks) + ; (assign (assoc + ; chunk_size (* 2 chunk_size) + ; start (+ end 1) + ; )) + ; (assign (assoc + ; end (+ start (- chunk_size 1)) + ; )) + ; (if (>= end num_remaining_cases) + ; (assign (assoc end (- num_remaining_cases 1) )) + ; ) + + ; (call_entity [!traineeContainer subtrainee_id] "train" (assoc + ; features case_features + ; cases + ; (if cases_are_ids + ; (map + ; (lambda (retrieve_from_entity (current_value) case_features)) + ; (unzip remaining_cases (range start end)) + ; ) + + ; ;else remaining_cases are already rows of feature values + ; (unzip remaining_cases (range start end)) + ; ) + ; session "none" + ; ;allow training on reserved features to preserve all case data + ; allow_training_reserved_features .true + ; )) + ; ) + + ; ;else no chunking is needed, just train the remainder + ; (call_entity [!traineeContainer subtrainee_id] "train" (assoc + ; features case_features + ; cases + ; (if cases_are_ids + ; (map + ; (lambda (retrieve_from_entity (current_value) case_features)) + ; remaining_cases + ; ) + + ; ;else remaining_cases are already rows of feature values + ; remaining_cases + ; ) + ; session "none" + ; ;allow training on reserved features to preserve all case data + ; allow_training_reserved_features .true + ; )) + ; ) + ; ) + ;Helper method to create the actual full contained entity path by interleaving !traineeContainer with each name in the path !ConvertNamePathToEntityPath (weave (range !traineeContainer 1 (size path) 1) path) diff --git a/module/progress.amlg b/module/progress.amlg index 3bfbb9e4b..c10b130ae 100644 --- a/module/progress.amlg +++ b/module/progress.amlg @@ -50,8 +50,11 @@ (assoc total .null details .null + internal .false ) + (if internal (conclude)) + (assign_to_entities (assoc !progressMap (modify diff --git a/module/remove_cases.amlg b/module/remove_cases.amlg index 78eaea171..afb8f8305 100644 --- a/module/remove_cases.amlg +++ b/module/remove_cases.amlg @@ -176,13 +176,13 @@ ;parameters: ; cases: list of case ids to remove ; distribute_weight_feature: name of feature into which to distribute the removed cases' weights to their neighbors. - ; remove_case_into_storage: flag, when true will remove cases into a storage container instead of deleting them + ; remove_cases_into_storage: flag, when true will remove cases into a storage container instead of deleting them !RemoveCases (declare (assoc cases (list) distribute_weight_feature .null - remove_case_into_storage .false + remove_cases_into_storage .false ) (if (= 0 (size cases)) (conclude)) @@ -310,9 +310,11 @@ ;remove all the cases after clearing the query caches (reclaim_resources .null .false .true) - (if remove_case_into_storage + (if remove_cases_into_storage (seq - (create_entities !reducedCasesStorage) + (if (not (contains_entity !reducedCasesStorage)) + (create_entities !reducedCasesStorage {}) + ) (map (lambda (move_entities (current_value) [!reducedCasesStorage (current_value 1)])) cases diff --git a/module/train.amlg b/module/train.amlg index 0dcefec6b..a2e89c7fb 100644 --- a/module/train.amlg +++ b/module/train.amlg @@ -78,7 +78,7 @@ start (+ end 1) end (+ start num_to_train -1) new_payload - (call !train (assoc + (call train (assoc ;;;TODO: don't split time series in middle of a series cases (unzip cases (range start end)) features features @@ -531,173 +531,18 @@ )) (if (and !autoHierarchyEnabled (>= num_cases !autoAblationMaxNumCases)) - (let - (assoc - reduce_payload - (call reduce_data (assoc - features features - skip_auto_analyze .true - internal .true - remove_case_into_storage .true - )) - ) - - (if (= 0 (size !autoHierarchyIds)) - (call react_into_features (assoc - clustering .true - clustering_min_cluster_mass (ceil (* num_cases !autoHierarchyClusteringRatio)) - use_case_weights .true - )) - ) - - (declare (assoc - subtrainee_ids - (if (= 0 (size !autoHierarchyIds)) - (map - (lambda (concat !traineeId "_sub" (current_value)) ) - (indices (compute_on_contained_entities (query_value_masses ".cluster_id"))) - ) - - !autoHierarchyIds - ) - )) - - (if (= 0 (size !autoHierarchyIds)) - (seq - (map - (lambda (let - (assoc subtrainee_id (current_value 1)) - (call create_subtrainee (assoc - filepath (retrieve_from_entity "filepath") - path [subtrainee_id] - child_id subtrainee_id - )) - - (set_entity_permissions [!traineeContainer subtrainee_id] .true) - - (call_entity [!traineeContainer subtrainee_id] "set_feature_attributes" (assoc - feature_attributes (keep !featureAttributes features) - )) - - (call_entity [!traineeContainer subtrainee_id] "set_auto_ablation_params" (assoc - ;auto_ablation_enabled .false - min_num_cases !autoAblationMinNumCases - hierarchy_clustering_ratio !autoHierarchyClusteringRatio - auto_hierarchy_enabled .true - )) - )) - subtrainee_ids - ) - (assign_to_entities (assoc !autoHierarchyIds (sort subtrainee_ids) )) - ) - ) - - ;;;TODO: predict clusters for all cases in (contained_entities !reducedCasesStorage) - ;;;TODO: train subtrainees - ;;;TODO: remove !reducedCasesStorage - ) + (call !MakeHierarchy) ) - ;else hierarchy and ablation are enabled, ablated cases are trained into subtrainees + ;hierarchy and ablation are enabled, ablated cases are trained into subtrainees (if (and !autoHierarchyEnabled (size ablated_indices_list)) (seq - ;only keep the ablated cases - (assign (assoc cases (keep cases ablated_indices_list) )) - - (if (= 0 (size !autoHierarchyIds)) - (seq - (call !MakeHierarchy) - - ;else predict which cluster each ablated case should go into - (assign (assoc - predicted_clusters - (if (> (size !autoHierarchyIds) 1) - ||(map - (lambda (first - (call !ReactDiscriminative (assoc - context_features features - context_values (current_value 1) - action_features [".cluster_id"] - return_action_values_only .true - use_case_weights .true - )) - )) - cases - ) - - ;else only one cluster - (= (size !autoHierarchyIds) 1) - (map 0 cases) ;;;;TODO: verify that it's always 0 - ) - )) - ) - - ;else hierarchy already exists, only keep those predicted_clusters matching the ablated cases - (assign (assoc predicted_clusters (keep predicted_clusters ablated_indices_list))) - ) - - (map - (lambda (let - (assoc - cluster_id (current_value 1) - subtrainee_id (concat !traineeId "_sub" (current_value 1)) - ) - - (declare (assoc - cluster_indices - (filter - (lambda (= cluster_id (get predicted_clusters (current_value)))) - (indices predicted_clusters) - ) - num_cases_in_subtrainee - (get - (call_entity [!traineeContainer subtrainee_id] "get_num_training_cases") - ["1" "payload" "count"] - ) - )) - - ;training enough cases that the subtrainee should be split into its own hierarchy - ;split cluster_indices into however many needed to reach min num cases - ;analyze - ;train the remainder - (if (and - ;the trainee doesn't have enough cases in it already - (< num_cases_in_subtrainee !autoAblationMinNumCases) - (>= (+ num_cases_in_subtrainee (size cluster_indices)) !autoAblationMinNumCases) - ) - (let - (assoc - cases_for_subtrainee (unzip cases cluster_indices) - ;number of cases to reach the ablation/hierarchy threshold - first_train_amount (- !autoAblationMinNumCases num_cases_in_subtrainee) - ) -(print "enabling hierarchy for " subtrainee_id "\n") - (call_entity [!traineeContainer subtrainee_id] "train" (assoc - features features - ;train only the initial set of cases needed to reach the threshold - cases (trunc cases_for_subtrainee first_train_amount) - session session - )) - - (call_entity [!traineeContainer subtrainee_id] "analyze") - - (call !TrainSubtraineeCasesInChunks (assoc - remaining_cases (tail cases_for_subtrainee (- first_train_amount)) - case_features features - cases_are_ids .false - )) - ) + ;only keep the ablated cases and only those predicted_clusters matching the ablated cases + (assign (assoc + predicted_clusters (keep predicted_clusters ablated_indices_list) + )) - ;else train the cases into the subtrainee - (call !TrainSubtraineeCasesInChunks (assoc - remaining_cases (unzip cases cluster_indices) - case_features features - cases_are_ids .false - )) - ) - )) - (values predicted_clusters .true) - ) + (call !TrainSubtraineeClusterIndices (assoc cases (keep cases ablated_indices_list) )) ) ) From 713b33e3ee3da1cbfd8d019d7eca9033f734cfd3 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 15 Sep 2026 12:01:26 -0400 Subject: [PATCH 23/42] train hierarchy fix --- module/hierarchy.amlg | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index fc04ebad1..d0ac50f27 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -1431,6 +1431,8 @@ min_num_cases !autoAblationMinNumCases hierarchy_clustering_ratio !autoHierarchyClusteringRatio auto_hierarchy_enabled .true + max_num_cases !autoAblationMaxNumCases + reduce_max_cases !postReduceMaxCases )) )) subtrainee_ids @@ -1468,7 +1470,7 @@ )) ;ensure ablation is enabled going forward - (assign (assoc !autoAblationEnabled .true)) + (assign_to_entities (assoc !autoAblationEnabled .true)) ;remove !reducedCasesStorage (destroy_entities !reducedCasesStorage) @@ -1581,6 +1583,7 @@ "id" (retrieve_from_entity "!traineeId") "path" path_list "contained" !traineeIsContained + "ablation" !autoAblationEnabled "children" (map (lambda From 5f335c59f68cbee292962e4115c2fab22dee85a6 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 15 Sep 2026 16:28:33 -0400 Subject: [PATCH 24/42] enable reacts, fix inf recursive trains --- module/hierarchy.amlg | 6 +- module/react_discriminative.amlg | 99 ++++++++++++++++++-------------- module/react_utilities.amlg | 6 +- module/train.amlg | 11 +++- 4 files changed, 75 insertions(+), 47 deletions(-) diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index d0ac50f27..c866cc706 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -1308,6 +1308,9 @@ use_case_weights .true )) + ;ensure ablation is enabled going forward for this trainee + (assign_to_entities (assoc !autoAblationEnabled .true)) + (if (!= .null !savedAnalyzeParameterMap) (seq (assign_to_entities (assoc @@ -1469,9 +1472,6 @@ ) )) - ;ensure ablation is enabled going forward - (assign_to_entities (assoc !autoAblationEnabled .true)) - ;remove !reducedCasesStorage (destroy_entities !reducedCasesStorage) ) diff --git a/module/react_discriminative.amlg b/module/react_discriminative.amlg index c07598d51..d9476af33 100644 --- a/module/react_discriminative.amlg +++ b/module/react_discriminative.amlg @@ -30,6 +30,7 @@ has_dependent_features has_dependent_features impute_react impute_react filtering_queries filtering_queries + use_enabled_hierarchy use_enabled_hierarchy ) ReactDiscriminative (call !ReactDiscriminative @(target .true "!ReactDiscriminativeParameters") ) @@ -112,6 +113,7 @@ has_dependent_features !hasDependentFeatures impute_react .false filtering_queries (list) + use_enabled_hierarchy .true ;local variables, should not be passed in as a parameter valid_weight_feature .false @@ -365,7 +367,7 @@ )) - (if (and !autoHierarchyEnabled (size !autoHierarchyIds) (!= ".cluster_id" (first action_features))) + (if (and use_enabled_hierarchy !autoHierarchyEnabled (size !autoHierarchyIds) (!= ".cluster_id" (first action_features))) (let (assoc clustering_prediction @@ -377,6 +379,8 @@ skip_decoding .true details { "categorical_action_probabilities" .true} use_case_weights .true + output_raw_influential_cases_only .false + use_enabled_hierarchy .false )) ) @@ -384,38 +388,41 @@ cap (get clustering_prediction ["categorical_action_probabilities" ".cluster_id"]) )) - ;if single cluster, just react against that one cluster (conclude - (if (= 1 (size cap)) - (call_entity [!traineeContainer (concat !traineeId "_sub" (first (indices cap)))] - "ReactDiscriminative" - @(target .true "!ReactDiscriminativeParameters") - ) - - ;else iterpolate among the involved clusters, union of all the raw influences - (let - (assoc - relevant_cluster_cases_map - (map - (lambda (let - (assoc subtrainee_id (concat !traineeId "_sub" (current_index 1)) ) - (get - (call_entity [!traineeContainer subtrainee_id] - "ReactDiscriminative" - (modify - @(target .true "!ReactDiscriminativeParameters") - "output_raw_influential_cases_only" .true - ) + ;iterpolate among the involved clusters, union of all the raw influences + (let + (assoc + relevant_cluster_cases_map + (map + (lambda (let + (assoc subtrainee_id (concat !traineeId "_sub" (current_index 1)) ) + (get + (call_entity [!traineeContainer subtrainee_id] + "ReactDiscriminative" + (modify + @(target .true "!ReactDiscriminativeParameters") + "output_raw_influential_cases_only" .true ) - ["action_values" 0] ) - )) - cap + ["action_values" 0] + ) + )) + cap + ) + local_cases_map + (call !ReactDiscriminative + (modify + @(target .true "!ReactDiscriminativeParameters") + "output_raw_influential_cases_only" .true + "use_enabled_hierarchy" .false + "use_case_weights" .true ) - ) + ) + ) - (declare (assoc - combined_influences_map + (declare (assoc + combined_influences_map + (append ;combine all the subtrainee cases and influences into one assoc where the indices are pair of [subtrainee_id case_id] (apply "append" (values (map @@ -424,30 +431,37 @@ ;change the assoc indices to be a pair such that the assoc is now [subtrainee_id case_id] -> influence weight (zip - (map (lambda [sub_id (current_value 1)] ) (indices (current_value))) + (map (lambda (append sub_id (current_value )) ) (indices (current_value))) (values (current_value)) ) )) relevant_cluster_cases_map ) )) - k_parameter (get data_params_map "k") - )) + (get local_cases_map ["action_values" 0]) + ) + k_parameter (get data_params_map "k") + )) - ;sort subtrainee cases by their influences - (declare (assoc - sub_cases - (sort - (lambda - (< - (get combined_influences_map [(current_value 1)]) - (get combined_influences_map [(current_value 2)]) - ) + ;sort subtrainee cases by their influences + (declare (assoc + sub_cases + (sort + (lambda + (< + (get combined_influences_map [(current_value 1)]) + (get combined_influences_map [(current_value 2)]) ) - (indices combined_influences_map) ) - )) + (indices combined_influences_map) + ) + )) + + ;output only the 20 most relevant cases + (if output_raw_influential_cases_only + (keep combined_influences_map (trunc sub_cases 20)) + ;else interpolate between the results (call !DynamicBandwidthFilterAndInterpolate (assoc action_feature (first action_features) sub_cases sub_cases @@ -458,6 +472,7 @@ )) ) ) + ) ) ) diff --git a/module/react_utilities.amlg b/module/react_utilities.amlg index f24580036..1c76ac72e 100644 --- a/module/react_utilities.amlg +++ b/module/react_utilities.amlg @@ -1535,7 +1535,11 @@ candidate_case_values (map (lambda - (retrieve_from_entity [!traineeContainer (first (current_value 1)) (last (current_value 1))] action_feature) + ;retrieve action feature value from subtrainee case or from current dataset case + (if (~ [] (current_value)) + (retrieve_from_entity (append !traineeContainer (current_value)) action_feature) + (retrieve_from_entity (current_value) action_feature) + ) ) sub_cases ) diff --git a/module/train.amlg b/module/train.amlg index a2e89c7fb..a4fa79f58 100644 --- a/module/train.amlg +++ b/module/train.amlg @@ -59,6 +59,7 @@ ;if going to be training more cases that are allowed to fit into a dataset, split the training data (if (and !autoHierarchyEnabled + (not !autoAblationEnabled) (> (+ num_cases (size cases)) !autoAblationMaxNumCases) (not (and train_weights_only accumulate_weight_feature)) ) @@ -71,10 +72,17 @@ end -1 ) +(print !traineeId " splitting large train payload " (size cases) "\n") (while (> num_new 0) (assign (assoc num_cases (call !GetNumTrainingCases) - num_to_train (min (- !autoAblationMaxNumCases num_cases) num_new) + num_to_train + ;if trainee is ablating cases into subrainees, can train a larger load, + ;otherwise only train up to the amount needed to start the reduce and make a sub hierarchy + (if (and !autoAblationEnabled (size !autoHierarchyIds)) + (min !autoAblationMaxNumCases num_new) + (min (- !autoAblationMaxNumCases num_cases) num_new) + ) start (+ end 1) end (+ start num_to_train -1) new_payload @@ -537,6 +545,7 @@ ;hierarchy and ablation are enabled, ablated cases are trained into subtrainees (if (and !autoHierarchyEnabled (size ablated_indices_list)) (seq +(print !traineeId " splitting ablated into subtrainees: " (size ablated_indices_list) "\n") ;only keep the ablated cases and only those predicted_clusters matching the ablated cases (assign (assoc predicted_clusters (keep predicted_clusters ablated_indices_list) From 406d356697bb0475bfefa13032515e3a05fb7b6d Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Wed, 16 Sep 2026 10:04:53 -0400 Subject: [PATCH 25/42] cleanup and internal progress fixes --- module/ablation.amlg | 7 ++++--- module/analysis.amlg | 7 ++++--- module/analysis_weights.amlg | 3 ++- module/conviction.amlg | 19 ++++++++++--------- module/hierarchy.amlg | 19 +++++++------------ module/mda_weight.amlg | 4 ++-- module/progress.amlg | 3 --- module/residuals.amlg | 2 +- module/train.amlg | 6 ++---- 9 files changed, 32 insertions(+), 38 deletions(-) diff --git a/module/ablation.amlg b/module/ablation.amlg index 986dfc9e3..9ee936942 100644 --- a/module/ablation.amlg +++ b/module/ablation.amlg @@ -746,9 +746,10 @@ num_progress_updates 1 progress_update_size (/ !autoAblationMinNumCases 11) + track_progress (not internal) )) - (call !ProgressUpdate (assoc details "Selecting cases to keep")) + (if track_progress (call !ProgressUpdate (assoc details "Selecting cases to keep")) ) (while (not done) (let @@ -847,7 +848,7 @@ ) ;update progress 10 times - (if (>= coreset_size (* num_progress_updates progress_update_size)) + (if (and track_progress (>= coreset_size (* num_progress_updates progress_update_size)) ) (if (<= num_progress_updates 10) (seq (call !ProgressUpdate (assoc details "Selecting cases to keep")) @@ -930,7 +931,7 @@ (call !FilterCasesToRemoveForTimeSeries (assoc ensure_enough_to_remove .true)) ) - (call !ProgressUpdate (assoc details "Removing cases")) + (if track_progress (call !ProgressUpdate (assoc details "Removing cases")) ) (if (size cases_to_remove) (call !RemoveCases (assoc diff --git a/module/analysis.amlg b/module/analysis.amlg index 8e4dccf67..2a2e779d4 100644 --- a/module/analysis.amlg +++ b/module/analysis.amlg @@ -609,6 +609,7 @@ ;name of deviation subtrainee, will be assigned if subtrainee is stored deviation_subtrainee_name .null + track_progress (not internal) )) (call !InitAnalyze) @@ -657,7 +658,7 @@ (call !GridSearch) - (call !ProgressUpdate (assoc details "Computing feature weights")) + (if track_progress (call !ProgressUpdate (assoc details "Computing feature weights")) ) (call !ComputeAndUseWeights) (call !TestAccuracyAndKeepOrRevertDataParameters) ;w or w/o weights @@ -673,11 +674,11 @@ )) (call !ConvergeResiduals (assoc use_deviations .true)) - (call !ProgressUpdate (assoc details "Updating parameters")) + (if track_progress (call !ProgressUpdate (assoc details "Updating parameters")) ) (call !GridSearch) ;now with weights - (call !ProgressUpdate (assoc details "Computing feature weights with deviations")) + (if track_progress (call !ProgressUpdate (assoc details "Computing feature weights with deviations")) ) (call !ComputeAndUseWeights) (call !TestAccuracyAndKeepOrRevertDataParameters) ;deviations w or w/o weights diff --git a/module/analysis_weights.amlg b/module/analysis_weights.amlg index aae8dd7f5..2bda329fd 100644 --- a/module/analysis_weights.amlg +++ b/module/analysis_weights.amlg @@ -21,7 +21,7 @@ residuals_map (list) )) - (call !ProgressUpdate (assoc details "Computing residuals")) + (if track_progress (call !ProgressUpdate (assoc details "Computing residuals")) ) ;possibly increase the number of samples if nominals are present (if (and !tsTimeFeature (size !sharedDeviationsMap)) @@ -508,6 +508,7 @@ ;for reduce_only or extra wide datasets, output a progress step for each converge deviation step (if (and + track_progress (> iteration 0) (= "deviations" computation_type) (or diff --git a/module/conviction.amlg b/module/conviction.amlg index 69ab925c3..6ce4132e1 100644 --- a/module/conviction.amlg +++ b/module/conviction.amlg @@ -133,6 +133,7 @@ ;one update every 10000 (or more) cases, for a max of 10 times for SC and RContrib flows progress_update_size (max 10000 (floor (/ num_reacts (min num_reacts 10)))) max_progress_updates (min (ceil (/ num_reacts 10000)) 10) + track_progress (not internal) )) (if familiarity_conviction_addition (accum (assoc num_steps 1)) ) @@ -386,7 +387,7 @@ ;will need to compute distance_contributions if caching values even if distance_contribution were not explicitly requested (if (or distance_contribution cache_values) (seq - (call !ProgressUpdate (assoc details "Computing distance contributions")) + (if track_progress (call !ProgressUpdate (assoc details "Computing distance contributions")) ) (assign (assoc case_to_dc_map @@ -505,7 +506,7 @@ (if (or familiarity_conviction_addition p_value_of_addition) ;calculates the conviction for each case, making a map of case id -> conviction (seq - (call !ProgressUpdate (assoc details "Computing familiarity conviction of addition")) + (if track_progress (call !ProgressUpdate (assoc details "Computing familiarity conviction of addition")) ) (declare (assoc entropies_map (compute_on_contained_entities @@ -581,7 +582,7 @@ (if (or familiarity_conviction_removal p_value_of_removal) ;calculates the conviction for each case, making a map of case id -> conviction (seq - (call !ProgressUpdate (assoc details "Computing familiarity conviction of removal")) + (if track_progress (call !ProgressUpdate (assoc details "Computing familiarity conviction of removal")) ) (declare (assoc entropies_map (compute_on_contained_entities @@ -678,7 +679,7 @@ ;update progress every 'progress_update_size' number of reacts (if (= 0 (mod (current_index) progress_update_size)) (if (< num_progress_updates max_progress_updates) - (call !ProgressLockAndUpdate) + (if track_progress (call !ProgressLockAndUpdate) ) ) ) (get @@ -698,7 +699,7 @@ ) )) - (call !ProgressUpdate (assoc details progress_detail_text)) + (if track_progress (call !ProgressUpdate (assoc details progress_detail_text)) ) (call !StoreCaseValues (assoc label_name similarity_conviction @@ -713,7 +714,7 @@ (if influence_weight_entropy (seq - (call !ProgressUpdate (assoc details "Computing influence weight entropies")) + (if track_progress (call !ProgressUpdate (assoc details "Computing influence weight entropies")) ) (call !ComputeAndStoreInfluenceWeightEntropies (assoc features features label_name influence_weight_entropy @@ -729,7 +730,7 @@ ;need to be analyzed before (if clustering (seq - (call !ProgressUpdate (assoc details "Clustering")) + (if track_progress (call !ProgressUpdate (assoc details "Clustering")) ) (call !Clustering) (if (not (contains_value !reactIntoFeaturesList clustering)) (accum_to_entities (assoc !reactIntoFeaturesList clustering)) @@ -771,7 +772,7 @@ ;update progress every 'progress_update_size' number of reacts (if (= 0 (mod (current_index) progress_update_size)) (if (< num_progress_updates max_progress_updates) - (call !ProgressLockAndUpdate) + (if track_progress (call !ProgressLockAndUpdate) ) ) ) (call !ResidualContribution (assoc @@ -789,7 +790,7 @@ ) ) - (call !ProgressUpdate (assoc details progress_detail_text)) + (if track_progress (call !ProgressUpdate (assoc details progress_detail_text)) ) (call !StoreCaseValues (assoc label_name residual_contribution diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index c866cc706..a71cd6aac 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -1277,17 +1277,12 @@ ;if subtrainees have not been created yet, trainee should be analyzed if it hasn't been analyzed yet (if (and (= 0 (size !autoHierarchyIds)) (= .null !savedAnalyzeParameterMap)) - (call analyze) + (call analyze (assoc internal .true)) !autoAnalyzeEnabled (call !Analyze !savedAnalyzeParameterMap) ) - (call !ProgressClear (assoc - internal .false - task_id .null - )) - (declare (assoc reduce_payload (call reduce_data (assoc @@ -1306,6 +1301,7 @@ clustering .true clustering_min_cluster_mass (ceil (* num_cases !autoHierarchyClusteringRatio)) use_case_weights .true + internal .true )) ;ensure ablation is enabled going forward for this trainee @@ -1324,7 +1320,10 @@ ;prevent calling react_into_features during the analyze() (call !Analyze (append !savedAnalyzeParameterMap - (assoc "from_react_into_features" .true) + (assoc + "from_react_into_features" .true + "internal" .true + ) )) ) @@ -1340,14 +1339,10 @@ k_folds 1 ;prevent calling react_into_features during the analyze() from_react_into_features .true + internal .true )) ) - (call !ProgressClear (assoc - internal .false - task_id .null - )) - (map (lambda (concat !traineeId "_sub" (current_value)) ) (indices (compute_on_contained_entities (query_value_masses ".cluster_id"))) diff --git a/module/mda_weight.amlg b/module/mda_weight.amlg index 7eef3aad5..88afb310f 100644 --- a/module/mda_weight.amlg +++ b/module/mda_weight.amlg @@ -354,7 +354,7 @@ ;update progress every 'progress_update_size' number of reacts (if (= 0 (mod (current_index) progress_update_size)) (if (< num_progress_updates max_progress_updates) - (call !ProgressLockAndUpdate) + (if (not internal) (call !ProgressLockAndUpdate) ) ) ) @@ -747,7 +747,7 @@ ;update progress here because tehre are less than 50 features and it won't be updated during the actual compute (if (= 1 max_progress_updates) - (call !ProgressUpdate (assoc details "Computing accuracy contribution matrix")) + (if (not internal) (call !ProgressUpdate (assoc details "Computing accuracy contribution matrix")) ) ) (declare (assoc all_sample_indices (range 0 (- (size (first feature_residuals_lists)) 1)) )) diff --git a/module/progress.amlg b/module/progress.amlg index c10b130ae..3bfbb9e4b 100644 --- a/module/progress.amlg +++ b/module/progress.amlg @@ -50,11 +50,8 @@ (assoc total .null details .null - internal .false ) - (if internal (conclude)) - (assign_to_entities (assoc !progressMap (modify diff --git a/module/residuals.amlg b/module/residuals.amlg index 3163ed553..2438ee436 100644 --- a/module/residuals.amlg +++ b/module/residuals.amlg @@ -678,7 +678,7 @@ (if (= 0 (mod (+ total_samples (current_index)) progress_update_size)) (if (<= num_progress_updates max_progress_updates) - (call !ProgressLockAndUpdate) + (if (not internal) (call !ProgressLockAndUpdate) ) ) ) diff --git a/module/train.amlg b/module/train.amlg index a4fa79f58..84cc8f3f3 100644 --- a/module/train.amlg +++ b/module/train.amlg @@ -72,7 +72,6 @@ end -1 ) -(print !traineeId " splitting large train payload " (size cases) "\n") (while (> num_new 0) (assign (assoc num_cases (call !GetNumTrainingCases) @@ -119,7 +118,7 @@ (if (= .null session) (assign (assoc session "none")) ) -(print !traineeId " training " (size cases) " ablation: " !autoAblationEnabled " subs: " (size !autoHierarchyIds) "\n") + (declare (assoc predicted_clusters (if (and !autoHierarchyEnabled !autoAblationEnabled) @@ -545,7 +544,6 @@ ;hierarchy and ablation are enabled, ablated cases are trained into subtrainees (if (and !autoHierarchyEnabled (size ablated_indices_list)) (seq -(print !traineeId " splitting ablated into subtrainees: " (size ablated_indices_list) "\n") ;only keep the ablated cases and only those predicted_clusters matching the ablated cases (assign (assoc predicted_clusters (keep predicted_clusters ablated_indices_list) @@ -556,7 +554,7 @@ ) (accum_to_entities (assoc !revision 1)) -(print !traineeId " num_trained: " (size new_case_ids) " num_ablated: " (size ablated_indices_list) " into: " (values predicted_clusters .true) "\n") + ;return response (call !Return (assoc warnings (if (size warnings) (indices warnings)) From 98dcae9b931dd4ec16a0dfd24155b198c69101dc Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Wed, 16 Sep 2026 10:38:35 -0400 Subject: [PATCH 26/42] cleanup train code --- module/train.amlg | 74 +++++-------------------------- module/train_utilities.amlg | 87 +++++++++++++++++++++++++++++++++++++ module/trainee.amlg | 36 --------------- 3 files changed, 99 insertions(+), 98 deletions(-) diff --git a/module/train.amlg b/module/train.amlg index 84cc8f3f3..6d6032228 100644 --- a/module/train.amlg +++ b/module/train.amlg @@ -56,62 +56,15 @@ (declare (assoc num_cases (call !GetNumTrainingCases) )) - ;if going to be training more cases that are allowed to fit into a dataset, split the training data + ;if going to be training more cases than are allowed to fit into a dataset for reduction, + ;split the training data into chunks to allow reduction and hierarchy to activate (if (and !autoHierarchyEnabled (not !autoAblationEnabled) (> (+ num_cases (size cases)) !autoAblationMaxNumCases) (not (and train_weights_only accumulate_weight_feature)) ) - (let - (assoc - num_new (size cases) - payload [1 {"payload" {"num_trained" 0 "ablated_indices" [] "status" .null}}] - new_payload .null - start 0 - end -1 - ) - - (while (> num_new 0) - (assign (assoc - num_cases (call !GetNumTrainingCases) - num_to_train - ;if trainee is ablating cases into subrainees, can train a larger load, - ;otherwise only train up to the amount needed to start the reduce and make a sub hierarchy - (if (and !autoAblationEnabled (size !autoHierarchyIds)) - (min !autoAblationMaxNumCases num_new) - (min (- !autoAblationMaxNumCases num_cases) num_new) - ) - start (+ end 1) - end (+ start num_to_train -1) - new_payload - (call train (assoc - ;;;TODO: don't split time series in middle of a series - cases (unzip cases (range start end)) - features features - derived_features derived_features - session session - series series - input_is_substituted input_is_substituted - allow_training_reserved_features allow_training_reserved_features - skip_auto_analyze skip_auto_analyze - skip_reduce_data skip_reduce_data - start_index start_index - )) - - start_index (if start_index (+ start_index num_to_train)) - num_new (- num_new num_to_train) - )) - - ;accumulate payload - (call !AccumulateTrainPayloads) - - ;stop train on errors - (if (= 0 (first payload)) (conclude) ) - ) - - (conclude payload) - ) + (conclude (call !SplitLargeTrainForHierarchy)) ) ;unsure that session is set to some string value @@ -533,24 +486,21 @@ ".next_trained_index" (+ next_trained_index (size cases)) )) - (assign (assoc - num_cases (call !GetNumTrainingCases) - )) + ;update num_cases after training + (assign (assoc num_cases (call !GetNumTrainingCases) )) + ;check if hierarchy is enabled and there are now enough cases to activate reduction and enable hierarchy (if (and !autoHierarchyEnabled (>= num_cases !autoAblationMaxNumCases)) (call !MakeHierarchy) ) - ;hierarchy and ablation are enabled, ablated cases are trained into subtrainees + ;if hierarchy and ablation are enabled, ablated cases are trained into subtrainees (if (and !autoHierarchyEnabled (size ablated_indices_list)) - (seq - ;only keep the ablated cases and only those predicted_clusters matching the ablated cases - (assign (assoc - predicted_clusters (keep predicted_clusters ablated_indices_list) - )) - - (call !TrainSubtraineeClusterIndices (assoc cases (keep cases ablated_indices_list) )) - ) + ;only keep the ablated cases and only those predicted_clusters matching the ablated cases + (call !TrainSubtraineeClusterIndices (assoc + cases (keep cases ablated_indices_list) + predicted_clusters (keep predicted_clusters ablated_indices_list) + )) ) (accum_to_entities (assoc !revision 1)) diff --git a/module/train_utilities.amlg b/module/train_utilities.amlg index ba7a8bbe5..ec6b8b9d7 100644 --- a/module/train_utilities.amlg +++ b/module/train_utilities.amlg @@ -913,4 +913,91 @@ )) )) + !SplitLargeTrainForHierarchy + (let + (assoc + num_new (size cases) + payload [1 {"payload" {"num_trained" 0 "ablated_indices" [] "status" .null}}] + new_payload .null + start 0 + end -1 + ) + + (while (> num_new 0) + (assign (assoc + num_cases (call !GetNumTrainingCases) + num_to_train + ;if trainee is ablating cases into subrainees, can train a larger load, + ;otherwise only train up to the amount needed to start the reduce and make a sub hierarchy + (if (and !autoAblationEnabled (size !autoHierarchyIds)) + (min !autoAblationMaxNumCases num_new) + (min (- !autoAblationMaxNumCases num_cases) num_new) + ) + start (+ end 1) + end (+ start num_to_train -1) + new_payload + (call train (assoc + ;;;TODO: don't split time series in middle of a series + cases (unzip cases (range start end)) + features features + derived_features derived_features + session session + series series + input_is_substituted input_is_substituted + allow_training_reserved_features allow_training_reserved_features + skip_auto_analyze skip_auto_analyze + skip_reduce_data skip_reduce_data + start_index start_index + )) + + start_index (if start_index (+ start_index num_to_train)) + num_new (- num_new num_to_train) + )) + + ;accumulate payload + (call !AccumulateTrainPayloads) + + ;stop train on errors + (if (= 0 (first payload)) (conclude) ) + ) + + ;output accumulated payload + payload + ) + + ;accumulate engine output specific to the train method 'new_payload' with 'payload' + !AccumulateTrainPayloads + (assign (assoc + payload + [ + ;if errors, output a 0 + (if (or (= 0 (first payload)) (= 0 (first new_payload))) + 0 + 1 + ) + (assoc + "payload" + { + "num_trained" + (+ + (get payload [1 "payload" "num_trained"]) + (get new_payload [1 "payload" "num_trained"]) + ) + "ablated_indices" + (append + (get payload [1 "payload" "ablated_indices"]) + (get new_payload [1 "payload" "ablated_indices"]) + ) + "status" + (or + (get payload [1 "payload" "status"]) + (get new_payload [1 "payload" "status"]) + ) + } + "warnings" (filter (append (get payload [1 "warnings"]) (get new_payload [1 "warnings"]))) + "errors" (filter (append (get payload [1 "errors"]) (get new_payload [1 "errors"]))) + ) + ] + )) + } \ No newline at end of file diff --git a/module/trainee.amlg b/module/trainee.amlg index 7e1df3bd1..ad475ba89 100644 --- a/module/trainee.amlg +++ b/module/trainee.amlg @@ -962,40 +962,4 @@ (list 1 (assoc "payload" payload) ) ) - ;accumulate new_payload with payload - !AccumulateTrainPayloads - (assign (assoc - payload - [ - ;if errors, output a 0 - (if (or (= 0 (first payload)) (= 0 (first new_payload))) - 0 - 1 - ) - (assoc - "payload" - { - "num_trained" - (+ - (get payload [1 "payload" "num_trained"]) - (get new_payload [1 "payload" "num_trained"]) - ) - "ablated_indices" - (append - (get payload [1 "payload" "ablated_indices"]) - (get new_payload [1 "payload" "ablated_indices"]) - ) - "status" - (or - (get payload [1 "payload" "status"]) - (get new_payload [1 "payload" "status"]) - ) - } - "warnings" (filter (append (get payload [1 "warnings"]) (get new_payload [1 "warnings"]))) - "errors" (filter (append (get payload [1 "errors"]) (get new_payload [1 "errors"]))) - ) - ] - )) - - } From 894be3f350838948e0bf72113eb3e7c21a550ba2 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Wed, 16 Sep 2026 11:37:28 -0400 Subject: [PATCH 27/42] typos, minor fixes --- module/dataparameters.amlg | 4 ++-- module/hierarchy.amlg | 4 ++-- module/trainee.amlg | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/module/dataparameters.amlg b/module/dataparameters.amlg index 635dad29b..7cd727c88 100644 --- a/module/dataparameters.amlg +++ b/module/dataparameters.amlg @@ -1048,9 +1048,9 @@ #{ref "AblationThresholdMap"} rel_threshold_map (assoc) ;Ratio of smallest mass (number) of cases to total cases needed that can be considered an individual cluster when clustering - ;for auto hierarchy. When unspecified defaults to 0.1. Applicable when auto_hierarchy_enabled is true + ;for auto hierarchy. When unspecified defaults to 0.05. Applicable when auto_hierarchy_enabled is true #{type "number" exclusive_min 0} - hierarchy_clustering_ratio .null + hierarchy_clustering_ratio 0.05 ;flag, default is false. when true, enables automatic internal hierarchy creation via clustering #{type "boolean"} auto_hierarchy_enabled .false diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index a71cd6aac..3ea46e6ac 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -1277,10 +1277,10 @@ ;if subtrainees have not been created yet, trainee should be analyzed if it hasn't been analyzed yet (if (and (= 0 (size !autoHierarchyIds)) (= .null !savedAnalyzeParameterMap)) - (call analyze (assoc internal .true)) + (call analyze (assoc "internal" .true)) !autoAnalyzeEnabled - (call !Analyze !savedAnalyzeParameterMap) + (call !Analyze (append !savedAnalyzeParameterMap (assoc "internal" .true))) ) (declare (assoc diff --git a/module/trainee.amlg b/module/trainee.amlg index ad475ba89..b31d1e3fa 100644 --- a/module/trainee.amlg +++ b/module/trainee.amlg @@ -152,7 +152,7 @@ !autoHierarchyEnabled .false - !autoHierarchyClusteringRatio 5 + !autoHierarchyClusteringRatio 0.05 ;unique id of parent trainee if this trainee is a subtrainee in a hierarchy !parentId .null @@ -706,7 +706,7 @@ ;the unique string identifier for the parent of the trainee #{type ["string" "null"]} parent_id .null - ;flag, no default. If specified, sets the flas specifying whether this trainee is contained inside the specified parent trainee. + ;flag, no default. If specified, sets the flag specifying whether this trainee is contained inside the specified parent trainee. #{type "boolean"} is_contained .null ) From c5c0251a71e4230c5f414abb5de881036ac5d9b4 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Wed, 16 Sep 2026 12:28:26 -0400 Subject: [PATCH 28/42] renames and fixes --- module/hierarchy.amlg | 6 +++++- module/train.amlg | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index 3ea46e6ac..a345d6a0a 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -1272,7 +1272,7 @@ (call !Return) ) - !MakeHierarchy + !ReduceIntoHierarchy (seq ;if subtrainees have not been created yet, trainee should be analyzed if it hasn't been analyzed yet @@ -1293,6 +1293,7 @@ )) )) + ;create subtrainees from clusters if they haven't beet created yet (declare (assoc subtrainee_ids (if (= 0 (size !autoHierarchyIds)) @@ -1354,12 +1355,14 @@ ) )) + ;create subtrainees (if (= 0 (size !autoHierarchyIds)) (let (assoc cluster_ids (indices (compute_on_contained_entities (query_value_masses ".cluster_id"))) ) + ;combine unclustered cases into existing clusters (if (and (contains_value cluster_ids 0) (> (size cluster_ids) 1)) (let (assoc @@ -1440,6 +1443,7 @@ ) ) + ;grab all the reduces cases slated for removal from storage (declare (assoc reduced_cases_values (map diff --git a/module/train.amlg b/module/train.amlg index 6d6032228..9e601f28d 100644 --- a/module/train.amlg +++ b/module/train.amlg @@ -491,7 +491,7 @@ ;check if hierarchy is enabled and there are now enough cases to activate reduction and enable hierarchy (if (and !autoHierarchyEnabled (>= num_cases !autoAblationMaxNumCases)) - (call !MakeHierarchy) + (call !ReduceIntoHierarchy) ) ;if hierarchy and ablation are enabled, ablated cases are trained into subtrainees @@ -910,6 +910,8 @@ (not skip_reduce_data) (>= (call !GetNumTrainingCases) !autoAblationMaxNumCases) (size !dataParametersMap) + ;allow ReduceIntoHierearchy to run the reduce flow if hierarchy is enabled instead of doing it here + (not !autoHierarchyEnabled) ) (seq ;clear progress from auto analyze if there was one @@ -919,6 +921,7 @@ (call reduce_data (assoc features features skip_auto_analyze skip_auto_analyze + internal .true )) ) From bea6c02680f2ec7b006afc13b5f7631b0d441f79 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Wed, 16 Sep 2026 13:55:06 -0400 Subject: [PATCH 29/42] hierarchy testi --- performance_tests/bank_hierarchy.amlg | 126 ++------------------------ 1 file changed, 10 insertions(+), 116 deletions(-) diff --git a/performance_tests/bank_hierarchy.amlg b/performance_tests/bank_hierarchy.amlg index 55d7b1d76..3e002caa7 100644 --- a/performance_tests/bank_hierarchy.amlg +++ b/performance_tests/bank_hierarchy.amlg @@ -54,16 +54,16 @@ )) (call_entity "howso" "set_auto_ablation_params" (assoc - auto_ablation_enabled .true - min_num_cases train_size ; 2000 cases - - hierarchy_clustering_ratio 0.1 + min_num_cases 5000 + hierarchy_clustering_ratio 0.05 auto_hierarchy_enabled .true + max_num_cases 10000 + reduce_max_cases 5000 )) (call_entity "howso" "train" (assoc features features - cases training_data + cases (append training_data other_data) )) (declare (assoc @@ -73,40 +73,9 @@ (print "Loaded Bank: " num_cases "\n") (print "Load time: " load_time "\n") +(print (call_entity "howso" "get_hierarchy" (assoc internal_only .true)) ) - (print "analyzing Bank ...\n") - (assign (assoc start (system_time) )) - (call_entity "howso" "analyze" (assoc - context_features features - targeted_model "targetless" - ; convergence_threshold 0 - )) - (declare (assoc analyze_time (- (system_time) start) )) - (print "Analyze time: " analyze_time "\n" ) - - - (assign (assoc start (system_time) )) - (print - ;(call_entity "howso" "get_hierarchy") - "Base dataset num cases: " (get (call_entity "howso" "get_num_training_cases") [1 "payload" "count"]) "\n" - ) - - ; (call_entity "howso" "train" (assoc - ; features features - ; cases (trunc other_data 1000) ;only the first 1000 - ; )) - ; (print "\n") - ; (call_entity "howso" "train" (assoc - ; features features - ; cases (tail other_data -1000) ;everything but the first 1000 - ; )) - - (call_entity "howso" "train" (assoc - features features - cases other_data - )) - (print (call_entity "howso" "get_hierarchy") ) (declare (assoc predictions @@ -120,41 +89,7 @@ ) )) - (print "accuracy: " - (/ - (size (filter - (lambda (let - (assoc - actual (last (current_value 1)) - predicted (get predictions [(current_index 2) 0]) - ) - (= actual predicted) - )) - test_data - )) - test_size - ) - "\n\n" - ) - (call_entity "howso" "set_auto_ablation_params" (assoc - auto_ablation_enabled .false - min_num_cases train_size ; 2000 cases - hierarchy_clustering_ratio 0.1 - auto_hierarchy_enabled .true - )) - (assign (assoc - predictions - (get - (call_entity "howso" "react" (assoc - action_features [(last features)] - context_features (trunc features) - context_values (map (lambda (trunc (current_value))) test_data) - )) - [1 "payload" "action_values"] - ) - )) - - (print "no hierarchy accuracy: " + (print "hierarchy accuracy: " (/ (size (filter (lambda (let @@ -171,50 +106,10 @@ "\n\n" ) - (call_entity "howso" "set_auto_ablation_params" (assoc - auto_ablation_enabled .true - min_num_cases train_size ; 2000 cases - hierarchy_clustering_ratio 0.1 - auto_hierarchy_enabled .true - )) - - - (call_entity "howso" "analyze") - - (assign (assoc - predictions - (get - (call_entity "howso" "react" (assoc - action_features [(last features)] - context_features (trunc features) - context_values (map (lambda (trunc (current_value))) test_data) - )) - [1 "payload" "action_values"] - ) - )) - - (print "post analyze accuracy: " - (/ - (size (filter - (lambda (let - (assoc - actual (last (current_value 1)) - predicted (get predictions [(current_index 2) 0]) - ) - (= actual predicted) - )) - test_data - )) - test_size - ) - "\n\n" - ) (call_entity "howso" "set_auto_ablation_params" (assoc auto_ablation_enabled .false - min_num_cases train_size ; 2000 cases - hierarchy_clustering_ratio 0.1 - auto_hierarchy_enabled .true + auto_hierarchy_enabled .false )) (assign (assoc predictions @@ -228,7 +123,7 @@ ) )) - (print "post-analyze no hierarchy accuracy: " + (print "NO hierarchy accuracy (reduce/ablate base only): " (/ (size (filter (lambda (let @@ -249,7 +144,6 @@ (destroy_entities "howso") - (load_entity "../howso.amlg" "howso" .null .false {escape_resource_name .false escape_contained_resource_names .false} ) @@ -298,7 +192,7 @@ ) )) - (print "non-hierarchy accuracy: " + (print "non-hierarchy full dataset accuracy: " (/ (size (filter (lambda (let From a09f32485441502edd9e8d65201ffdd37380ded5 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Mon, 21 Sep 2026 14:14:53 -0400 Subject: [PATCH 30/42] add full_hierarchy flag to analyze --- module/analysis.amlg | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/module/analysis.amlg b/module/analysis.amlg index 2a2e779d4..9398458e0 100644 --- a/module/analysis.amlg +++ b/module/analysis.amlg @@ -8,6 +8,9 @@ ;Optional task_id to track progress of this long running operation. If unspecified, will use .null as the task_id. #{type "string"} task_id .null + ;optional flag, if true analyzes all subtrainees in hierarchy + #{type "boolean"} + full_hierarchy .false ) (declare (assoc saved_analyze_parameters_map (retrieve_from_entity "!savedAnalyzeParameterMap") )) @@ -19,7 +22,13 @@ task_id task_id )) - (call analyze (append saved_analyze_parameters_map {"task_id" task_id}) ) + (call analyze + (append + saved_analyze_parameters_map + {"task_id" task_id} + {"full_hierarchy" full_hierarchy} + ) + ) ) ) @@ -116,6 +125,9 @@ ;Defaults to .false, assuming this will be called by an external API/user. #{type "boolean"} internal .false + ;optional flag, if true analyzes all subtrainees in hierarchy + #{type "boolean"} + full_hierarchy .false ) (call !ValidateParameters) (call !ValidateFeatures) @@ -563,6 +575,17 @@ )) ) ) + + (if (and full_hierarchy !autoHierarchyEnabled !autoAblationEnabled) + (map + (lambda + (call_entity [!traineeContainer (current_value 1)] "auto_analyze" (assoc + full_hierarchy .true + )) + ) + !autoHierarchyIds + ) + ) ) ;called on none or one action feature at a time by Analyze() @@ -861,6 +884,16 @@ )) ) + ;full_hierarchy analyze should automatically include .cluster_id if there are subtrainees + (if (and full_hierarchy !autoHierarchyEnabled !autoAblationEnabled) + (if (= 0 (size context_features)) + (assign (assoc context_features (append !trainedFeatures ".cluster_id") )) + + (not (contains_value context_features ".cluster_id")) + (accum (assoc context_features ".cluster_id")) + ) + ) + ;if there are computed features that are to be analyzed (e.g., 'similarity_conviction') append them to context_features (if (get !computedFeaturesMap "analyze") (assign (assoc From 341428c0be6f8a4491b9b2dcbe9b480b284e0e89 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Mon, 21 Sep 2026 15:07:16 -0400 Subject: [PATCH 31/42] react fixes --- module/react_discriminative.amlg | 25 ++++++++++++++++--------- module/react_utilities.amlg | 5 +---- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/module/react_discriminative.amlg b/module/react_discriminative.amlg index d9476af33..3a3112341 100644 --- a/module/react_discriminative.amlg +++ b/module/react_discriminative.amlg @@ -410,13 +410,17 @@ cap ) local_cases_map - (call !ReactDiscriminative - (modify - @(target .true "!ReactDiscriminativeParameters") - "output_raw_influential_cases_only" .true - "use_enabled_hierarchy" .false - "use_case_weights" .true + (get + (call !ReactDiscriminative + (modify + @(target .true "!ReactDiscriminativeParameters") + "output_raw_influential_cases_only" .true + "use_enabled_hierarchy" .false + "use_case_weights" .false + ;;;"use_case_weights" .true + ) ) + ["action_values" 0] ) ) @@ -431,16 +435,17 @@ ;change the assoc indices to be a pair such that the assoc is now [subtrainee_id case_id] -> influence weight (zip - (map (lambda (append sub_id (current_value )) ) (indices (current_value))) + (map (lambda (append !traineeContainer sub_id (current_value )) ) (indices (current_value))) (values (current_value)) ) )) relevant_cluster_cases_map ) )) - (get local_cases_map ["action_values" 0]) + local_cases_map ) k_parameter (get data_params_map "k") + )) ;sort subtrainee cases by their influences @@ -459,7 +464,9 @@ ;output only the 20 most relevant cases (if output_raw_influential_cases_only - (keep combined_influences_map (trunc sub_cases 20)) + { "action_values" [ + (keep combined_influences_map (trunc sub_cases 20)) + ]} ;else interpolate between the results (call !DynamicBandwidthFilterAndInterpolate (assoc diff --git a/module/react_utilities.amlg b/module/react_utilities.amlg index 1c76ac72e..082035833 100644 --- a/module/react_utilities.amlg +++ b/module/react_utilities.amlg @@ -1536,10 +1536,7 @@ (map (lambda ;retrieve action feature value from subtrainee case or from current dataset case - (if (~ [] (current_value)) - (retrieve_from_entity (append !traineeContainer (current_value)) action_feature) - (retrieve_from_entity (current_value) action_feature) - ) + (retrieve_from_entity (current_value) action_feature) ) sub_cases ) From c91f0680bfd6cb008d8a195504d150c8dc1284b1 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Mon, 21 Sep 2026 21:53:35 -0400 Subject: [PATCH 32/42] fixes --- module/hierarchy.amlg | 10 +++++++++- module/react_discriminative.amlg | 2 -- module/train.amlg | 16 ++++++++++++++++ module/trainee.amlg | 3 +++ 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index a345d6a0a..dba47df96 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -1471,6 +1471,14 @@ ) )) + (assign_to_entities (assoc + !autoHierarchyCaseLoadMap + { + "self" (call !GetNumTrainingCases) + "total" (+ (get !autoHierarchyCaseLoadMap "total") (size cases)) + } + )) + ;remove !reducedCasesStorage (destroy_entities !reducedCasesStorage) ) @@ -1621,7 +1629,7 @@ ;recurse down the hierarchy to output the internally contained hierarchy as a nested assoc with only the trainees and number of cases for each !GetInternalHierarchy (assoc - "num_cases" (call !GetNumTrainingCases) + "num_cases" !autoHierarchyCaseLoadMap "children" (map (lambda diff --git a/module/react_discriminative.amlg b/module/react_discriminative.amlg index 3a3112341..60d154d6d 100644 --- a/module/react_discriminative.amlg +++ b/module/react_discriminative.amlg @@ -417,7 +417,6 @@ "output_raw_influential_cases_only" .true "use_enabled_hierarchy" .false "use_case_weights" .false - ;;;"use_case_weights" .true ) ) ["action_values" 0] @@ -445,7 +444,6 @@ local_cases_map ) k_parameter (get data_params_map "k") - )) ;sort subtrainee cases by their influences diff --git a/module/train.amlg b/module/train.amlg index 9e601f28d..79c53c93e 100644 --- a/module/train.amlg +++ b/module/train.amlg @@ -492,6 +492,22 @@ ;check if hierarchy is enabled and there are now enough cases to activate reduction and enable hierarchy (if (and !autoHierarchyEnabled (>= num_cases !autoAblationMaxNumCases)) (call !ReduceIntoHierarchy) + + !autoHierarchyEnabled + (assign_to_entities (assoc + !autoHierarchyCaseLoadMap + { + "self" num_cases + "total" + ;if there are subtrainees, accumulate num of all cases (trained and ablated) to the total + (if (size !autoHierarchyIds) + (+ (get !autoHierarchyCaseLoadMap "total") (size cases) ) + + ;else total is same as the number of cases in this trainee + num_cases + ) + } + )) ) ;if hierarchy and ablation are enabled, ablated cases are trained into subtrainees diff --git a/module/trainee.amlg b/module/trainee.amlg index fbd4a23d8..665fc7419 100644 --- a/module/trainee.amlg +++ b/module/trainee.amlg @@ -154,6 +154,9 @@ !autoHierarchyClusteringRatio 0.05 + ;number of cases in self vs total traned into hierarchy + !autoHierarchyCaseLoadMap { "self" 0 "total" 0} + ;unique id of parent trainee if this trainee is a subtrainee in a hierarchy !parentId .null From 0eb43bfdcf7a72d104e7f1a74d7cebfb3e760adf Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:01:51 -0400 Subject: [PATCH 33/42] leaf trainees copy dataparams from parent on creation --- module/analysis.amlg | 22 +++++++--- module/hierarchy.amlg | 98 +++++++++++++------------------------------ 2 files changed, 45 insertions(+), 75 deletions(-) diff --git a/module/analysis.amlg b/module/analysis.amlg index 9398458e0..110d9239e 100644 --- a/module/analysis.amlg +++ b/module/analysis.amlg @@ -578,11 +578,23 @@ (if (and full_hierarchy !autoHierarchyEnabled !autoAblationEnabled) (map - (lambda - (call_entity [!traineeContainer (current_value 1)] "auto_analyze" (assoc - full_hierarchy .true - )) - ) + (lambda (let + (assoc + sub_hierarchy + (get + (call_entity [!traineeContainer (current_value 2)] "get_hierarchy" (assoc + internal_only .true + )) + [1 "payload" "children"] + ) + ) + ;only need to analyze subtrainee if they have their own subtrainees + (if (size sub_hierarchy) + (call_entity [!traineeContainer (current_value 1)] "auto_analyze" (assoc + full_hierarchy .true + )) + ) + )) !autoHierarchyIds ) ) diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index dba47df96..0ce7224bd 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -1283,6 +1283,17 @@ (call !Analyze (append !savedAnalyzeParameterMap (assoc "internal" .true))) ) + (declare (assoc + ;flag set to true if subtrainees will need to be created + create_subtrainees (= 0 (size !autoHierarchyIds)) + + ;copy of this trainee's parameters that would be copied to newly created subtrainees + trainee_dataparams_map (retrieve_from_entity "!dataParametersMap") + trainee_dataparams_paths (retrieve_from_entity "!!dataParametersPaths") + trainee_marginal_stats_map (retrieve_from_entity "!featureMarginalStatsMap") + trainee_expected_values_map (retrieve_from_entity "!expectedValuesMap") + )) + (declare (assoc reduce_payload (call reduce_data (assoc @@ -1296,7 +1307,7 @@ ;create subtrainees from clusters if they haven't beet created yet (declare (assoc subtrainee_ids - (if (= 0 (size !autoHierarchyIds)) + (if create_subtrainees (seq (call react_into_features (assoc clustering .true @@ -1356,7 +1367,7 @@ )) ;create subtrainees - (if (= 0 (size !autoHierarchyIds)) + (if create_subtrainees (let (assoc cluster_ids (indices (compute_on_contained_entities (query_value_masses ".cluster_id"))) @@ -1427,6 +1438,14 @@ feature_attributes (keep !featureAttributes features) )) + ;set the minimal params needed for reacts on subtrainee by copying this trainee's parameters + (call_entity [!traineeContainer subtrainee_id] "set_subtrainee_params_for_react" (assoc + trainee_dataparams_map trainee_dataparams_map + trainee_dataparams_paths trainee_dataparams_paths + trainee_marginal_stats_map trainee_marginal_stats_map + trainee_expected_values_map trainee_expected_values_map + )) + (call_entity [!traineeContainer subtrainee_id] "set_auto_ablation_params" (assoc ;auto_ablation_enabled .false min_num_cases !autoAblationMinNumCases @@ -1508,74 +1527,13 @@ ) - ; ;Trains sub trainee using growing chunks of remaining_cases - ; ;assumes cases are specified as entity ids, if cases_are_ids is set to false, remaining_cases are rows of feature values instead - ; !TrainSubtraineeCasesInChunks - ; (declare - ; (assoc - ; remaining_cases [] - ; cases_are_ids .true - ; ) - ; ;number of train chunks needed to train all the data if every chunk doubles in size - ; ;starting with the first chunk of size 2 * !autoAblationMinNumCases - ; (declare (assoc - ; num_train_chunks (ceil (- (log (size remaining_cases) 2) (log !autoAblationMinNumCases 2))) - ; start 0 - ; end -1 - ; num_remaining_cases (size remaining_cases) - ; chunk_size !autoAblationMinNumCases - ; )) - - ; (if (> num_train_chunks 0) - ; (while (< (current_index) num_train_chunks) - ; (assign (assoc - ; chunk_size (* 2 chunk_size) - ; start (+ end 1) - ; )) - ; (assign (assoc - ; end (+ start (- chunk_size 1)) - ; )) - ; (if (>= end num_remaining_cases) - ; (assign (assoc end (- num_remaining_cases 1) )) - ; ) - - ; (call_entity [!traineeContainer subtrainee_id] "train" (assoc - ; features case_features - ; cases - ; (if cases_are_ids - ; (map - ; (lambda (retrieve_from_entity (current_value) case_features)) - ; (unzip remaining_cases (range start end)) - ; ) - - ; ;else remaining_cases are already rows of feature values - ; (unzip remaining_cases (range start end)) - ; ) - ; session "none" - ; ;allow training on reserved features to preserve all case data - ; allow_training_reserved_features .true - ; )) - ; ) - - ; ;else no chunking is needed, just train the remainder - ; (call_entity [!traineeContainer subtrainee_id] "train" (assoc - ; features case_features - ; cases - ; (if cases_are_ids - ; (map - ; (lambda (retrieve_from_entity (current_value) case_features)) - ; remaining_cases - ; ) - - ; ;else remaining_cases are already rows of feature values - ; remaining_cases - ; ) - ; session "none" - ; ;allow training on reserved features to preserve all case data - ; allow_training_reserved_features .true - ; )) - ; ) - ; ) + set_subtrainee_params_for_react + (assign_to_entities (assoc + !dataParametersMap trainee_dataparams_map + !dataParametersPaths trainee_dataparams_paths + !featureMarginalStatsMap trainee_marginal_stats_map + !expectedValuesMap trainee_expected_values_map + )) ;Helper method to create the actual full contained entity path by interleaving !traineeContainer with each name in the path !ConvertNamePathToEntityPath From 008e6ef8e3481e6a0fa6440a262542adec773c19 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 22 Sep 2026 15:28:14 -0400 Subject: [PATCH 34/42] added support for max hierarchy depth --- module/hierarchy.amlg | 24 ++++++++++++++++++++++++ module/react_utilities.amlg | 11 +++++++++++ module/trainee.amlg | 14 +++++++++++++- 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index 0ce7224bd..44a7a2281 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -1304,6 +1304,26 @@ )) )) + ;hierarchy max depth level has been reached + (if (and create_subtrainees (= !autoHierarchyDepthLevel !autoHierarchyMaxDepth)) + (seq + (assign_to_entities (assoc + !autoHierarchyDepthReached .true + !autoHierarchyCaseLoadMap + { + "self" (call !GetNumTrainingCases) + "total" (+ (get !autoHierarchyCaseLoadMap "total") (size cases)) + } + )) + + ;;;TODO: update analyze parameters with case_weights and enable auto ablation? + + ;remove !reducedCasesStorage + (destroy_entities !reducedCasesStorage) + (conclude) + ) + ) + ;create subtrainees from clusters if they haven't beet created yet (declare (assoc subtrainee_ids @@ -1444,6 +1464,7 @@ trainee_dataparams_paths trainee_dataparams_paths trainee_marginal_stats_map trainee_marginal_stats_map trainee_expected_values_map trainee_expected_values_map + auto_hierarchy_depth_level (+ 1 !autoHierarchyDepthLevel) )) (call_entity [!traineeContainer subtrainee_id] "set_auto_ablation_params" (assoc @@ -1453,6 +1474,8 @@ auto_hierarchy_enabled .true max_num_cases !autoAblationMaxNumCases reduce_max_cases !postReduceMaxCases + auto_hierarchy_max_depth !autoHierarchyMaxDepth + auto_hierarchy_max_breadth !autoHierarchyMaxBreadth )) )) subtrainee_ids @@ -1533,6 +1556,7 @@ !dataParametersPaths trainee_dataparams_paths !featureMarginalStatsMap trainee_marginal_stats_map !expectedValuesMap trainee_expected_values_map + !autoHierarchyDepthLevel auto_hierarchy_depth_level )) ;Helper method to create the actual full contained entity path by interleaving !traineeContainer with each name in the path diff --git a/module/react_utilities.amlg b/module/react_utilities.amlg index 082035833..9fb0ed5e4 100644 --- a/module/react_utilities.amlg +++ b/module/react_utilities.amlg @@ -825,6 +825,17 @@ (assoc set_valid_weight_feature .true ) + + (if output_raw_influential_cases_only + ;this is a hierarchy leaf node that is not splitting anymore, use weights for reacts + (if (and use_enabled_hierarchy !autoHierarchyDepthReached) + (assign (assoc + use_case_weights .true + weight_feature ".case_weight" + )) + ) + ) + (if (= .null use_case_weights) (if (or (= (get data_params_map "paramPath") [".default"]) (= (get data_params_map "paramPath") .null)) ;if default params, no case weights diff --git a/module/trainee.amlg b/module/trainee.amlg index 665fc7419..b291f1c32 100644 --- a/module/trainee.amlg +++ b/module/trainee.amlg @@ -155,7 +155,19 @@ !autoHierarchyClusteringRatio 0.05 ;number of cases in self vs total traned into hierarchy - !autoHierarchyCaseLoadMap { "self" 0 "total" 0} + !autoHierarchyCaseLoadMap { "self" 0 "total" 0 } + + ;current hierarchy depth level, where 0 = the top level trainee + !autoHierarchyDepthLevel 0 + + ;max number of hierarchy levels, specifying the number of generations of children allowed + !autoHierarchyMaxDepth 5 + + ;max number of subtrainees + !autoHierarchyMaxBreadth 5 + + ;flage, set to true if this is a leaf node in a hierarchy and max depth has been reached, no more subtrainees should be created + !autoHierarchyDepthReached .false ;unique id of parent trainee if this trainee is a subtrainee in a hierarchy !parentId .null From 081742ab98a08ebfa3e813b3cc91c60c5e79113e Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Tue, 22 Sep 2026 17:35:38 -0400 Subject: [PATCH 35/42] missed parameters in setter --- module/dataparameters.amlg | 8 ++++++++ module/trainee.amlg | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/module/dataparameters.amlg b/module/dataparameters.amlg index 5de710b50..10357503d 100644 --- a/module/dataparameters.amlg +++ b/module/dataparameters.amlg @@ -1054,6 +1054,12 @@ ;flag, default is false. when true, enables automatic internal hierarchy creation via clustering #{type "boolean"} auto_hierarchy_enabled .false + ;max depth of hierarchy, specifying the number of levels of subtrainees allowed + #{type "number" min 1} + auto_hierarchy_max_depth 5 + ;max breadth of hierarchy, specifying the number of subtrainees allowed + #{type "number" min 2} + auto_hierarchy_max_breadth 5 ) (call !ValidateParameters) @@ -1085,6 +1091,8 @@ !autoHierarchyEnabled auto_hierarchy_enabled !autoHierarchyClusteringRatio hierarchy_clustering_ratio + !autoHierarchyMaxDepth auto_hierarchy_max_depth + !autoHierarchyMaxBreadth auto_hierarchy_max_breadth )) (accum_to_entities (assoc !revision 1)) diff --git a/module/trainee.amlg b/module/trainee.amlg index b291f1c32..3d8f972fb 100644 --- a/module/trainee.amlg +++ b/module/trainee.amlg @@ -160,7 +160,7 @@ ;current hierarchy depth level, where 0 = the top level trainee !autoHierarchyDepthLevel 0 - ;max number of hierarchy levels, specifying the number of generations of children allowed + ;max depth of hierarchy, specifying the number of levels of subtrainees allowed !autoHierarchyMaxDepth 5 ;max number of subtrainees From f35835cf4b6064de118e5b353e49c3c4cc8c7ed3 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Wed, 23 Sep 2026 09:20:48 -0400 Subject: [PATCH 36/42] adds a reanalyze --- module/hierarchy.amlg | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index 44a7a2281..96ca1e354 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -1381,8 +1381,20 @@ ) ) - ;else use the already created subtrainee ids - !autoHierarchyIds + ;else re-analyze after the latest reduce use the already created subtrainee ids + (seq + (call !Analyze (append + !savedAnalyzeParameterMap + (assoc + ;prevent calling react_into_features during the analyze() + "from_react_into_features" .true + "internal" .true + "full_hierarchy" .true + ) + )) + + !autoHierarchyIds + ) ) )) @@ -1611,7 +1623,7 @@ ;recurse down the hierarchy to output the internally contained hierarchy as a nested assoc with only the trainees and number of cases for each !GetInternalHierarchy (assoc - "num_cases" !autoHierarchyCaseLoadMap + "num_cases" (call !GetNumTrainingCases) "children" (map (lambda From 91f5d918ff79cdd537b838ecc4b9f03e2a0f82ba Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Wed, 23 Sep 2026 14:37:02 -0400 Subject: [PATCH 37/42] wip, support for max breadth --- module/hierarchy.amlg | 30 +++++++++++++++++++++++++++++- module/train.amlg | 2 +- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index 96ca1e354..6c83a45c7 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -1316,7 +1316,23 @@ } )) - ;;;TODO: update analyze parameters with case_weights and enable auto ablation? + ;update analyze parameters with case_weights and enable auto ablation + (assign_to_entities (assoc + !savedAnalyzeParameterMap + (modify !savedAnalyzeParameterMap + "use_case_weights" .true + "weight_feature" ".case_weight" + ) + !autoAblationEnabled .true + )) + + (call !Analyze (append + !savedAnalyzeParameterMap + (assoc + "from_react_into_features" .true + "internal" .true + ) + )) ;remove !reducedCasesStorage (destroy_entities !reducedCasesStorage) @@ -1455,6 +1471,18 @@ ) ) + ;if there are more clusters than allowed, redistribute the smallest clusters among the larger ones + (if (> (size cluster_ids) !autoHierarchyMaxBreadth) + (let + (assoc + cluster_ids_map (compute_on_contained_entities (query_value_masses ".cluster_id")) + ) + + ;;; WIP: todo: distrubitue + + ) + ) + (map (lambda (let (assoc subtrainee_id (current_value 1)) diff --git a/module/train.amlg b/module/train.amlg index 79c53c93e..e6b6a08ea 100644 --- a/module/train.amlg +++ b/module/train.amlg @@ -511,7 +511,7 @@ ) ;if hierarchy and ablation are enabled, ablated cases are trained into subtrainees - (if (and !autoHierarchyEnabled (size ablated_indices_list)) + (if (and !autoHierarchyEnabled (size ablated_indices_list) (not !autoHierarchyDepthReached)) ;only keep the ablated cases and only those predicted_clusters matching the ablated cases (call !TrainSubtraineeClusterIndices (assoc cases (keep cases ablated_indices_list) From aed113619725e4acf7bdb7559058206bbd9237b7 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Wed, 23 Sep 2026 14:37:21 -0400 Subject: [PATCH 38/42] testing --- performance_tests/bank_hierarchy.amlg | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/performance_tests/bank_hierarchy.amlg b/performance_tests/bank_hierarchy.amlg index 3e002caa7..c24bfdaa4 100644 --- a/performance_tests/bank_hierarchy.amlg +++ b/performance_tests/bank_hierarchy.amlg @@ -76,6 +76,8 @@ (print (call_entity "howso" "get_hierarchy" (assoc internal_only .true)) ) + (call_entity "howso" "analyze" (assoc full_hierarchy .true)) + (declare (assoc predictions @@ -107,10 +109,10 @@ ) - (call_entity "howso" "set_auto_ablation_params" (assoc - auto_ablation_enabled .false - auto_hierarchy_enabled .false - )) + ; (call_entity "howso" "set_auto_ablation_params" (assoc + ; auto_ablation_enabled .false + ; auto_hierarchy_enabled .false + ; )) (assign (assoc predictions (get From c36032e64f1636099c2fabdc60f6a634572a860c Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Wed, 23 Sep 2026 17:08:59 -0400 Subject: [PATCH 39/42] add max breadth enforcement --- module/hierarchy.amlg | 42 +++++++++++++++++++++++++++++++- module/react_discriminative.amlg | 1 + 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index 6c83a45c7..7067c420f 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -1478,8 +1478,48 @@ cluster_ids_map (compute_on_contained_entities (query_value_masses ".cluster_id")) ) - ;;; WIP: todo: distrubitue + (declare (assoc + cluster_ids_by_size + (sort + (lambda + ;decreasing order + (< + (get cluster_ids_map [(current_value 1)]) + (get cluster_ids_map [(current_value 2)]) + ) + ) + (indices cluster_ids_map) + ) + extra_cluster_ids (tail cluster_ids_by_size (- (size cluster_ids) !autoHierarchyMaxBreadth)) + distribute_cluster_cases (contained_entities (query_among ".cluster_id" extra_cluster_ids)) + )) + + ;predict the most similar cluster that is not among the extra_cluster_ids + (declare (assoc + forced_clusters + ||(map + (lambda (first + (call !ReactDiscriminative (assoc + context_features !trainedFeatures + context_values (retrieve_from_entity (current_value 1) !trainedFeatures) + action_features [".cluster_id"] + return_action_values_only .true + use_case_weights .true + filtering_queries [(query_not_among ".cluster_id" extra_cluster_ids) (query_not_in_entity_list [(current_value 3)])] + )) + )) + distribute_cluster_cases + ) + )) + + ;update cluster id for each of the distribute cluster cases + (map + (lambda + (assign_to_entities (current_index) (assoc ".cluster_id" (current_value 1))) + ) + (zip distribute_cluster_cases forced_clusters) + ) ) ) diff --git a/module/react_discriminative.amlg b/module/react_discriminative.amlg index 60d154d6d..dbfcbf808 100644 --- a/module/react_discriminative.amlg +++ b/module/react_discriminative.amlg @@ -451,6 +451,7 @@ sub_cases (sort (lambda + ;decreasing order (< (get combined_influences_map [(current_value 1)]) (get combined_influences_map [(current_value 2)]) From 67c9c4da4a0dfe2a7d872a1b2261b68bd1574eb4 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Thu, 24 Sep 2026 08:18:01 -0400 Subject: [PATCH 40/42] fix to breadth code --- module/hierarchy.amlg | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index 7067c420f..09ce7e645 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -1472,7 +1472,7 @@ ) ;if there are more clusters than allowed, redistribute the smallest clusters among the larger ones - (if (> (size cluster_ids) !autoHierarchyMaxBreadth) + (if (> (size subtrainee_ids) !autoHierarchyMaxBreadth) (let (assoc cluster_ids_map (compute_on_contained_entities (query_value_masses ".cluster_id")) @@ -1490,7 +1490,7 @@ ) (indices cluster_ids_map) ) - extra_cluster_ids (tail cluster_ids_by_size (- (size cluster_ids) !autoHierarchyMaxBreadth)) + extra_cluster_ids (tail cluster_ids_by_size (- (size subtrainee_ids) !autoHierarchyMaxBreadth)) distribute_cluster_cases (contained_entities (query_among ".cluster_id" extra_cluster_ids)) )) @@ -1520,6 +1520,14 @@ ) (zip distribute_cluster_cases forced_clusters) ) + + (assign (assoc + subtrainee_ids + (map + (lambda (concat !traineeId "_sub" (current_value)) ) + (trunc cluster_ids_by_size !autoHierarchyMaxBreadth) + ) + )) ) ) From 27ffbb320c4e941c00ce43f10bedbb22a9631014 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Thu, 24 Sep 2026 13:01:51 -0400 Subject: [PATCH 41/42] ensure reduce() doesn't blow away clusters for hierarchy --- module/ablation.amlg | 66 +++++++++++++++++++++++++ module/hierarchy.amlg | 111 +++++++++++++++++++----------------------- 2 files changed, 115 insertions(+), 62 deletions(-) diff --git a/module/ablation.amlg b/module/ablation.amlg index 9ee936942..9057a5725 100644 --- a/module/ablation.amlg +++ b/module/ablation.amlg @@ -927,6 +927,11 @@ cases_to_remove (contained_entities (query_equals ".keeping" .null) ) )) + ;remove called by hierarchy clustering, ensure all clusters have at least 30 cases in them + (if (and remove_cases_into_storage !autoHierarchyEnabled) + (call !FilterCasesToRemoveForClustering) + ) + (if !tsTimeFeature (call !FilterCasesToRemoveForTimeSeries (assoc ensure_enough_to_remove .true)) ) @@ -1073,6 +1078,67 @@ ) ) + ;helper method to ensure that all clusters have at least 30 cases remaining in them by removing smaller cluster cases + ;from cases_to_remove and adding the latest cases that were added to the core set to cases_to_remove + !FilterCasesToRemoveForClustering + (let + (assoc + small_cluster_map + (filter + (lambda (< (current_value) 30)) + (compute_on_contained_entities + (query_not_in_entity_list cases_to_remove) + (query_value_masses ".cluster_id") + ) + ) + ) + + ;nothing to do since all clusters have at least 30 cases + (if (= 0 (size small_cluster_map)) + (conclude) + ) + + (declare (assoc + cluster_cases_to_keep + (apply "append" + (map + (lambda (let + (assoc + num_needed (current_value 1) + cluster_id (current_index 1) + ) + + (contained_entities + (query_in_entity_list cases_to_remove) + (query_equals ".cluster_id" cluster_id) + (query_max ".keeping" num_needed) + ) + )) + ;a map of cluster_id -> number of needed cases + (map (lambda (- 30 (current_value))) small_cluster_map) + ) + ) + ;last added cases for keeping will be used to pad cases for removal to replace the cluster cases needed to be kept + cases_to_pad_for_removal + (contained_entities + (query_not_in_entity_list cases_to_remove) + (query_max ".keeping" (size cluster_cases_to_keep)) + ) + )) + + (assign (assoc + cases_to_remove + (append + cases_to_pad_for_removal + (contained_entities + (query_in_entity_list cases_to_remove) + (query_not_in_entity_list cluster_cases_to_keep) + ) + ) + )) + ) + + ;helper method that merges duplicate cases during the reduce_data flow ;uses the precomputed min_neighbor_surprisal_map and neighbor_surprisals_map to find and merge duplicates diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index 09ce7e645..3c746aaa2 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -1279,8 +1279,8 @@ (if (and (= 0 (size !autoHierarchyIds)) (= .null !savedAnalyzeParameterMap)) (call analyze (assoc "internal" .true)) - !autoAnalyzeEnabled - (call !Analyze (append !savedAnalyzeParameterMap (assoc "internal" .true))) + ; !autoAnalyzeEnabled + ; (call !Analyze (append !savedAnalyzeParameterMap (assoc "internal" .true))) ) (declare (assoc @@ -1296,7 +1296,7 @@ (declare (assoc reduce_payload - (call reduce_data (assoc + (call reduce_data (assoc ;;;TODO: verify subsequent calls with case weights features features skip_auto_analyze .true internal .true @@ -1308,7 +1308,6 @@ (if (and create_subtrainees (= !autoHierarchyDepthLevel !autoHierarchyMaxDepth)) (seq (assign_to_entities (assoc - !autoHierarchyDepthReached .true !autoHierarchyCaseLoadMap { "self" (call !GetNumTrainingCases) @@ -1316,23 +1315,28 @@ } )) - ;update analyze parameters with case_weights and enable auto ablation - (assign_to_entities (assoc - !savedAnalyzeParameterMap - (modify !savedAnalyzeParameterMap - "use_case_weights" .true - "weight_feature" ".case_weight" - ) - !autoAblationEnabled .true - )) + (if (not !autoHierarchyDepthReached) + (seq + ;update analyze parameters with case_weights and enable auto ablation + (assign_to_entities (assoc + !savedAnalyzeParameterMap + (modify !savedAnalyzeParameterMap + "use_case_weights" .true + "weight_feature" ".case_weight" + ) + !autoAblationEnabled .true + !autoHierarchyDepthReached .true + )) - (call !Analyze (append - !savedAnalyzeParameterMap - (assoc - "from_react_into_features" .true - "internal" .true + (call !Analyze (append + !savedAnalyzeParameterMap + (assoc + "from_react_into_features" .true + "internal" .true + ) + )) ) - )) + ) ;remove !reducedCasesStorage (destroy_entities !reducedCasesStorage) @@ -1355,41 +1359,22 @@ ;ensure ablation is enabled going forward for this trainee (assign_to_entities (assoc !autoAblationEnabled .true)) - (if (!= .null !savedAnalyzeParameterMap) - (seq - (assign_to_entities (assoc - !savedAnalyzeParameterMap - (modify !savedAnalyzeParameterMap - "context_features" (append (get !savedAnalyzeParameterMap "context_features") ".cluster_id") - "use_case_weights" .true - "weight_feature" ".case_weight" - ) - )) - ;prevent calling react_into_features during the analyze() - (call !Analyze (append - !savedAnalyzeParameterMap - (assoc - "from_react_into_features" .true - "internal" .true - ) - )) + (assign_to_entities (assoc + !savedAnalyzeParameterMap + (modify !savedAnalyzeParameterMap + "context_features" (append (get !savedAnalyzeParameterMap "context_features") ".cluster_id") + "use_case_weights" .true + "weight_feature" ".case_weight" + ) + )) + ;prevent calling react_into_features during the analyze() + (call !Analyze (append + !savedAnalyzeParameterMap + (assoc + "from_react_into_features" .true + "internal" .true ) - - (call !Analyze (assoc - targeted_model "targetless" - context_features (concat !trainedFeatures ".cluster_id") - weight_feature - (if (and !autoAblationEnabled !autoAblationWeightFeature) - !autoAblationWeightFeature - weight_feature - ) - use_case_weights .true - k_folds 1 - ;prevent calling react_into_features during the analyze() - from_react_into_features .true - internal .true - )) - ) + )) (map (lambda (concat !traineeId "_sub" (current_value)) ) @@ -1399,15 +1384,17 @@ ;else re-analyze after the latest reduce use the already created subtrainee ids (seq - (call !Analyze (append - !savedAnalyzeParameterMap - (assoc - ;prevent calling react_into_features during the analyze() - "from_react_into_features" .true - "internal" .true - "full_hierarchy" .true - ) - )) + (if !autoAnalyzeEnabled + (call !Analyze (append + !savedAnalyzeParameterMap + (assoc + ;prevent calling react_into_features during the analyze() + "from_react_into_features" .true + "internal" .true + "full_hierarchy" .true + ) + )) + ) !autoHierarchyIds ) From 3535722e0096b31e4323320e7a15793ef3aea909 Mon Sep 17 00:00:00 2001 From: howsoRes <144272317+howsoRes@users.noreply.github.com> Date: Fri, 25 Sep 2026 11:45:34 -0400 Subject: [PATCH 42/42] update analyze threshold compute --- module/analysis.amlg | 62 ++++++++++++++++++++++++------------------- module/hierarchy.amlg | 9 +++---- 2 files changed, 38 insertions(+), 33 deletions(-) diff --git a/module/analysis.amlg b/module/analysis.amlg index 110d9239e..1feadc065 100644 --- a/module/analysis.amlg +++ b/module/analysis.amlg @@ -1061,38 +1061,46 @@ ) )) - ;check if auto analysis is enabled and the analysis threshold should be increased - (if (and - !autoAnalyzeEnabled - (>= !dataMassChangeSinceLastAnalyze !autoAnalyzeThreshold) - ) - (let - (assoc - total_case_mass - (if use_case_weights - (compute_on_contained_entities - (query_not_equals weight_feature .null) - (query_sum weight_feature) - ) - - ;else it's 1 per case, thus the total mass is num_cases - num_cases + (declare (assoc + total_case_mass + (if use_case_weights + (compute_on_contained_entities + (query_not_equals weight_feature .null) + (query_sum weight_feature) ) + + ;else it's 1 per case, thus the total mass is num_cases + num_cases ) + )) - (assign_to_entities (assoc - !autoAnalyzeThreshold - ;!autoAnalyzeThreshold is a delta from the previous threshold. - ;if several case deletions cause the above formula to return < 0, default - ; to the default !autoAnalyzeThreshold. - (max - (- (* total_case_mass !autoAnalyzeGrowthFactorAmount) !autoAnalyzeThreshold) - 100 - ) - )) - ) + ;if there are cases that were reduced to storage prior to this analyze call, count them as well + (if (size (contained_entities !reducedCasesStorage)) + (accum (assoc + total_case_mass (size (contained_entities !reducedCasesStorage)) + )) ) + (assign_to_entities (assoc + !autoAnalyzeThreshold + (if !autoAnalyzeEnabled + ;!autoAnalyzeThreshold is a delta from the previous threshold. + ;if several case deletions cause the above formula to return < 0, default + ; to the default !autoAnalyzeThreshold. + (max + (- (* total_case_mass !autoAnalyzeGrowthFactorAmount) !autoAnalyzeThreshold) + 100 + ) + + ;else auto analyze is not enabled, this analyze was called 'out of cycle' of auto analyze + ;update the threshold relative to the current dataset mass + (max + (- (* total_case_mass !autoAnalyzeGrowthFactorAmount) total_case_mass) + 100 + ) + ) + )) + ;reset !dataMassChangeSinceLastAnalyze since we are now analyzing (assign_to_entities (assoc !dataMassChangeSinceLastAnalyze 0.0 )) diff --git a/module/hierarchy.amlg b/module/hierarchy.amlg index 3c746aaa2..f5de932dc 100644 --- a/module/hierarchy.amlg +++ b/module/hierarchy.amlg @@ -1278,9 +1278,6 @@ ;if subtrainees have not been created yet, trainee should be analyzed if it hasn't been analyzed yet (if (and (= 0 (size !autoHierarchyIds)) (= .null !savedAnalyzeParameterMap)) (call analyze (assoc "internal" .true)) - - ; !autoAnalyzeEnabled - ; (call !Analyze (append !savedAnalyzeParameterMap (assoc "internal" .true))) ) (declare (assoc @@ -1296,7 +1293,7 @@ (declare (assoc reduce_payload - (call reduce_data (assoc ;;;TODO: verify subsequent calls with case weights + (call reduce_data (assoc features features skip_auto_analyze .true internal .true @@ -1382,9 +1379,9 @@ ) ) - ;else re-analyze after the latest reduce use the already created subtrainee ids + ;else re-analyze after the latest reduce if needed and use the already created subtrainee ids (seq - (if !autoAnalyzeEnabled + (if (>= !dataMassChangeSinceLastAnalyze !autoAnalyzeThreshold) (call !Analyze (append !savedAnalyzeParameterMap (assoc