25938: WIP, Hierarchy sandbox

module/ablation.amlg

@@ -622,6 +622,9 @@
 			;Optional task_id to track progress of this long running operation. If unspecified, will use .null as the task_id.
 			#{type "string"}
 			task_id .null
+			;optional flag, specifying whether reduced cases should be removed completely (default behavior), or when true, removed into a storage container
+			#{type "boolean"}
+			remove_cases_into_storage .false
 			;optional flag specifying whether this method was called internally by another method.
 			;Defaults to .false, assuming this will be called by an external API/user.
 			#{type "boolean"}
@@ -924,6 +927,11 @@
 			cases_to_remove (contained_entities (query_equals ".keeping" .null) )
 		))
 
+		;remove called by hierarchy clustering, ensure all clusters have at least 30 cases in them
+		(if (and remove_cases_into_storage !autoHierarchyEnabled)
+			(call !FilterCasesToRemoveForClustering)
+		)
+
 		(if !tsTimeFeature
 			(call !FilterCasesToRemoveForTimeSeries (assoc ensure_enough_to_remove .true))
 		)
@@ -934,6 +942,7 @@
 			(call !RemoveCases (assoc
 				cases cases_to_remove
 				distribute_weight_feature distribute_weight_feature
+				remove_cases_into_storage remove_cases_into_storage
 			))
 
 			;else no cases to remove, clear caches for these added features so they can be cleared out quickly
@@ -1069,6 +1078,67 @@
 		)
 	)
 
+	;helper method to ensure that all clusters have at least 30 cases remaining in them by removing smaller cluster cases
+	;from cases_to_remove and adding the latest cases that were added to the core set to cases_to_remove
+	!FilterCasesToRemoveForClustering
+	(let
+		(assoc
+			small_cluster_map
+				(filter
+					(lambda (< (current_value) 30))
+					(compute_on_contained_entities
+						(query_not_in_entity_list cases_to_remove)
+						(query_value_masses ".cluster_id")
+					)
+				)
+		)
+
+		;nothing to do since all clusters have at least 30 cases
+		(if (= 0 (size small_cluster_map))
+			(conclude)
+		)
+
+		(declare (assoc
+			cluster_cases_to_keep
+				(apply "append"
+					(map
+						(lambda (let
+							(assoc
+								num_needed (current_value 1)
+								cluster_id (current_index 1)
+							)
+
+							(contained_entities
+								(query_in_entity_list cases_to_remove)
+								(query_equals ".cluster_id" cluster_id)
+								(query_max ".keeping" num_needed)
+							)
+						))
+						;a map of cluster_id -> number of needed cases
+						(map (lambda (- 30 (current_value))) small_cluster_map)
+					)
+				)
+			;last added cases for keeping will be used to pad cases for removal to replace the cluster cases needed to be kept
+			cases_to_pad_for_removal
+				(contained_entities
+					(query_not_in_entity_list cases_to_remove)
+					(query_max ".keeping" (size cluster_cases_to_keep))
+				)
+		))
+
+		(assign (assoc
+			cases_to_remove
+				(append
+					cases_to_pad_for_removal
+					(contained_entities
+						(query_in_entity_list cases_to_remove)
+						(query_not_in_entity_list cluster_cases_to_keep)
+					)
+				)
+		))
+	)
+
+
 
 	;helper method that merges duplicate cases during the reduce_data flow
 	;uses the precomputed min_neighbor_surprisal_map and neighbor_surprisals_map to find and merge duplicates

module/analysis.amlg

@@ -8,6 +8,9 @@
 			;Optional task_id to track progress of this long running operation. If unspecified, will use .null as the task_id.
 			#{type "string"}
 			task_id .null
+			;optional flag, if true analyzes all subtrainees in hierarchy
+			#{type "boolean"}
+			full_hierarchy .false
 		)
 
 		(declare (assoc saved_analyze_parameters_map (retrieve_from_entity "!savedAnalyzeParameterMap") ))
@@ -19,7 +22,13 @@
 				task_id task_id
 			))
 
-			(call analyze (append saved_analyze_parameters_map {"task_id" task_id}) )
+			(call analyze
+				(append
+					saved_analyze_parameters_map
+					{"task_id" task_id}
+					{"full_hierarchy" full_hierarchy}
+				)
+			)
 		)
 	)
 
@@ -116,6 +125,9 @@
 			;Defaults to .false, assuming this will be called by an external API/user.
 			#{type "boolean"}
 			internal .false
+			;optional flag, if true analyzes all subtrainees in hierarchy
+			#{type "boolean"}
+			full_hierarchy .false
 		)
 		(call !ValidateParameters)
 		(call !ValidateFeatures)
@@ -563,6 +575,29 @@
 				))
 			)
 		)
+
+		(if (and full_hierarchy !autoHierarchyEnabled !autoAblationEnabled)
+			(map
+				(lambda (let
+					(assoc
+						sub_hierarchy
+							(get
+								(call_entity [!traineeContainer (current_value 2)] "get_hierarchy" (assoc
+									internal_only .true
+								))
+								[1 "payload" "children"]
+							)
+					)
+					;only need to analyze subtrainee if they have their own subtrainees
+					(if (size sub_hierarchy)
+						(call_entity [!traineeContainer (current_value 1)] "auto_analyze" (assoc
+							full_hierarchy .true
+						))
+					)
+				))
+				!autoHierarchyIds
+			)
+		)
 	)
 
 	;called on none or one action feature at a time by Analyze()
@@ -861,6 +896,16 @@
 			))
 		)
 
+		;full_hierarchy analyze should automatically include .cluster_id if there are subtrainees
+		(if (and full_hierarchy !autoHierarchyEnabled !autoAblationEnabled)
+			(if (= 0 (size context_features))
+				(assign (assoc context_features (append !trainedFeatures ".cluster_id") ))
+
+				(not (contains_value context_features ".cluster_id"))
+				(accum (assoc context_features ".cluster_id"))
+			)
+		)
+
 		;if there are computed features that are to be analyzed (e.g., 'similarity_conviction') append them to context_features
 		(if (size (get !computedFeaturesMap "computed_features"))
 			(assign (assoc
@@ -1023,38 +1068,46 @@
 				)
 		))
 
-		;check if auto analysis is enabled and the analysis threshold should be increased
-		(if (and
-				!autoAnalyzeEnabled
-				(>= !dataMassChangeSinceLastAnalyze !autoAnalyzeThreshold)
-			)
-			(let
-				(assoc
-					total_case_mass
-						(if use_case_weights
-							(compute_on_contained_entities
-								(query_not_equals weight_feature .null)
-								(query_sum weight_feature)
-							)
-
-							;else it's 1 per case, thus the total mass is num_cases
-							num_cases
+		(declare (assoc
+			total_case_mass
+				(if use_case_weights
+					(compute_on_contained_entities
+						(query_not_equals weight_feature .null)
+						(query_sum weight_feature)
 					)
+
+					;else it's 1 per case, thus the total mass is num_cases
+					num_cases
 				)
+		))
 
-				(assign_to_entities (assoc
-					!autoAnalyzeThreshold
-						;!autoAnalyzeThreshold is a delta from the previous threshold.
-						;if several case deletions cause the above formula to return < 0, default
-						; to the default !autoAnalyzeThreshold.
-						(max
-							(- (* total_case_mass !autoAnalyzeGrowthFactorAmount) !autoAnalyzeThreshold)
-							100
-						)
-				))
-			)
+		;if there are cases that were reduced to storage prior to this analyze call, count them as well
+		(if (size (contained_entities !reducedCasesStorage))
+			(accum (assoc
+				total_case_mass (size (contained_entities !reducedCasesStorage))
+			))
 		)
 
+		(assign_to_entities (assoc
+			!autoAnalyzeThreshold
+				(if !autoAnalyzeEnabled
+					;!autoAnalyzeThreshold is a delta from the previous threshold.
+					;if several case deletions cause the above formula to return < 0, default
+					; to the default !autoAnalyzeThreshold.
+					(max
+						(- (* total_case_mass !autoAnalyzeGrowthFactorAmount) !autoAnalyzeThreshold)
+						100
+					)
+
+					;else auto analyze is not enabled, this analyze was called 'out of cycle' of auto analyze
+					;update the threshold relative to the current dataset mass
+					(max
+						(- (* total_case_mass !autoAnalyzeGrowthFactorAmount) total_case_mass)
+						100
+					)
+				)
+		))
+
 		;reset !dataMassChangeSinceLastAnalyze since we are now analyzing
 		(assign_to_entities (assoc !dataMassChangeSinceLastAnalyze 0.0 ))
 

module/dataparameters.amlg

@@ -483,7 +483,7 @@
 				(if (or (= context_features .null) (= context_features (list)))
 					!trainedFeaturesContextKey
 
-					;targetless, predicting a specific feature, pull the analyzed  targetless parameters that included that feature
+					;targetless, predicting a specific feature, pull the analyzed targetless parameters that included that feature
 					(and (!= .null feature) (= ["targetless"] target_mode))
 					(call !BuildContextFeaturesKey (assoc
 						context_features (values (append context_features feature) .true)
@@ -1045,6 +1045,20 @@
 			; above the threshold (in the case of rmse and mae) or below the threshold (otherwise).
 			#{ref "AblationThresholdMap"}
 			rel_threshold_map (assoc)
+			;Ratio of smallest mass (number) of cases to total cases needed that can be considered an individual cluster when clustering
+			;for auto hierarchy. When unspecified defaults to 0.05. Applicable when auto_hierarchy_enabled is true
+			#{type "number" exclusive_min 0}
+			hierarchy_clustering_ratio 0.05
+			;flag, default is false. when true, enables automatic internal hierarchy creation via clustering
+			#{type "boolean"}
+			auto_hierarchy_enabled .false
+			;max depth of hierarchy, specifying the number of levels of subtrainees allowed
+			#{type "number" min 1}
+			auto_hierarchy_max_depth 5
+			;max breadth of hierarchy, specifying the number of subtrainees allowed
+			#{type "number" min 2}
+			auto_hierarchy_max_breadth 5
+
 		)
 		(call !ValidateParameters)
 
@@ -1072,6 +1086,11 @@
 			!autoAblationRelThresholdMap rel_threshold_map
 			!ablationBatchSize batch_size
 			!ablatedCasesDistributionBatchSize ablated_cases_distribution_batch_size
+
+			!autoHierarchyEnabled auto_hierarchy_enabled
+			!autoHierarchyClusteringRatio hierarchy_clustering_ratio
+			!autoHierarchyMaxDepth auto_hierarchy_max_depth
+			!autoHierarchyMaxBreadth auto_hierarchy_max_breadth
 		))
 
 		(accum_to_entities (assoc !revision 1))