Skip to content

Commit

Permalink
22071: Updates react_aggregate return type to be more similar to reac…
Browse files Browse the repository at this point in the history 
…t, MAJOR (#347)
  • Loading branch information
@jdbeel
jdbeel authored Nov 7, 2024
1 parent e087016 commit 2d69489
Show file tree
Hide file tree
Showing 13 changed files with 395 additions and 325 deletions.
31 changes: 0 additions & 31 deletions howso/react_aggregate.amlg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -641,37 +641,6 @@
)
)

;transform stats to be per-feature for output and filter out any empty assocs
(assign (assoc
output
(filter
(lambda (and (!= (assoc) (current_value)) (!= (null) (current_value)) ))
(map
(lambda (let
(assoc feature (current_index 1))

(if (contains_index !uniqueNominalsSet feature)
;if unique nominal, we want to bring out the nulls, but only for those that we gave a null intentionally (nominal stuff)
(filter
(lambda (contains_index (get output (current_index)) feature))
(map
(lambda (get (current_value) feature))
output
)
)

;else standard filter-map flow
(filter (map
(lambda (get (current_value) feature))
output
))
)
))
(zip action_features)
)
)
))

(if holdout_entity_name
(call !RestoreHeldOutCases (assoc holdout_entity_name holdout_entity_name))
)
Expand Down
231 changes: 148 additions & 83 deletions howso/return_typing.amlg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -495,131 +495,196 @@
ReactAggregateResponse
{
type "assoc"
description "Map of feature names to maps of requested metric names to their computed values for the specified feature."
additional_indices {
type "assoc"
description "Map of metric names to their computed values."
additional_indices (false)
indices {
"feature_residuals_full" {
description "Map of requested detail names to maps of feature names to their computed values for the specified feature."
additional_indices (false)
indices {
"feature_residuals_full" {
type "assoc"
description "The mean absolute error of predicting each feature using the full set of context features."
additional_indices {
type ["number" "null"]
description "The mean absolute error of predicting this feature using the full set of context features."
}
"feature_residuals_robust" {
}
"feature_residuals_robust" {
type "assoc"
description "The mean absolute error of predicting each feature using samples from the power-set of context features."
additional_indices {
type ["number" "null"]
description "The mean absolute error of predicting this feature using samples from the power-set of context features."
}
"feature_contributions_full" {
}
"feature_contributions_full" {
type "assoc"
description
(concat
"The mean absolute difference of predicting the specified action feature with and without each feature while "
"using the full set of remaining context features."
)
additional_indices {
type ["number" "null"]
description
(concat
"The mean absolute difference of predicting the specified action feature with and without this feature while "
"using the full set of remaining context features."
)
}
"directional_feature_contributions_full" {
}
"directional_feature_contributions_full" {
type "assoc"
description
(concat
"The mean difference of predicting the specified action feature with and without each feature while "
"using the full set of remaining context features."
)
additional_indices {
type ["number" "null"]
description
(concat
"The mean difference of predicting the specified action feature with and without this feature while "
"using the full set of remaining context features."
)
}
"feature_contributions_robust" {
}
"feature_contributions_robust" {
type "assoc"
description
(concat
"The mean absolute difference of predicting the specified action feature with and without each feature while "
"using samples from the power-set of remaining context features."
)
additional_indices {
type ["number" "null"]
description
(concat
"The mean absolute difference of predicting the specified action feature with and without this feature while "
"using samples from the power-set of remaining context features."
)
}
"directional_feature_contributions_robust" {
}
"directional_feature_contributions_robust" {
type "assoc"
description
(concat
"The mean difference of predicting the specified action feature with and without each feature while "
"using samples from the power-set of remaining context features."
)
additional_indices {
type ["number" "null"]
description
(concat
"The mean difference of predicting the specified action feature with and without this feature while "
"using samples from the power-set of remaining context features."
)
}
"feature_mda_full" {
}
"feature_mda_full" {
type "assoc"
additional_indices {
type ["number" "null"]
description
(concat
"The mean decrease in accuracy of predicting the specified action feature without this feature versus "
"with this feature while using full set of remaining context features."
)
}
"feature_mda_robust" {
description
(concat
"The mean decrease in accuracy of predicting the specified action feature without each feature versus "
"with each feature while using full set of remaining context features."
)
}
"feature_mda_robust" {
type "assoc"
additional_indices {
type ["number" "null"]
description
(concat
"The mean decrease in accuracy of predicting the specified action feature without this feature versus "
"with this feature while using samples from the power-set of remaining context features."
)
}
"feature_mda_permutation_full" {
description
(concat
"The mean decrease in accuracy of predicting the specified action feature without each feature versus "
"with each feature while using samples from the power-set of remaining context features."
)
}
"feature_mda_permutation_full" {
type "assoc"
additional_indices {
type ["number" "null"]
description
(concat
"The mean decrease in accuracy of predicting the specified action feature using scrambled "
"values for this feature versus non-scrambled values for this feature while using the full "
"set of remaining context features."
)
}
"feature_mda_permutation_robust" {
description
(concat
"The mean decrease in accuracy of predicting the specified action feature using scrambled "
"values for each feature versus non-scrambled values for each feature while using the full "
"set of remaining context features."
)
}
"feature_mda_permutation_robust" {
type "assoc"
additional_indices {
type ["number" "null"]
description
(concat
"The mean decrease in accuracy of predicting the specified action feature using scrambled "
"values for this feature versus non-scrambled values for this feature while using samples "
"from the power-set of remaining context features."
)
}
"adjusted_smape" {
description
(concat
"The mean decrease in accuracy of predicting the specified action feature using scrambled "
"values for each feature versus non-scrambled values for each feature while using samples "
"from the power-set of remaining context features."
)
}
"adjusted_smape" {
type "assoc"
additional_indices {
type ["number" "null"]
description "The symmetric mean absolute percentage error with added the min gap / 2 to the actual and predicted values."
}
"smape" {
description "The symmetric mean absolute percentage error with added the min gap / 2 to the actual and predicted values for each feature."
}
"smape" {
type "assoc"
additional_indices {
type ["number" "null"]
description "The symmetric mean absolute percentage error of predicting the feature."
}
"mae" {
description "The symmetric mean absolute percentage error of predicting each feature."
}
"mae" {
type "assoc"
additional_indices {
type ["number" "null"]
description "The mean absolute error of predicting the feature."
}
"recall" {
description "The mean absolute error of predicting each feature."
}
"recall" {
type "assoc"
additional_indices {
type ["number" "null"]
description "The recall of predicting the feature."
}
"precision" {
description "The recall of predicting each feature."
}
"precision" {
type "assoc"
additional_indices {
type ["number" "null"]
description "The precision of predicting the feature."
}
"accuracy" {
description "The precision of predicting each feature."
}
"accuracy" {
type "assoc"
additional_indices {
type ["number" "null"]
description "The accuracy of predicting the feature."
}
"r2" {
description "The accuracy of predicting each feature."
}
"r2" {
type "assoc"
additional_indices {
type ["number" "null"]
description "The R^2 of predicting the feature."
}
"rmse" {
description "The R^2 of predicting each feature."
}
"rmse" {
type "assoc"
additional_indices {
type ["number" "null"]
description "The RMSE of predicting the feature."
}
"spearman_coeff" {
description "The RMSE of predicting each feature."
}
"spearman_coeff" {
type "assoc"
additional_indices {
type ["number" "null"]
description "The Spearman's coefficient of predicting the feature."
}
"mcc" {
description "The Spearman's coefficient of predicting each feature."
}
"mcc" {
type "assoc"
additional_indices {
type ["number" "null"]
description "The MCC of predicting the feature."
}
"missing_value_accuracy" {
description "The MCC of predicting each feature."
}
"missing_value_accuracy" {
type "assoc"
additional_indices {
type ["number" "null"]
description "The proportion of missing values that were correctly predicted as missing for the feature."
}
"confusion_matrix" {ref "ConfusionMatrix"}
description "The proportion of missing values that were correctly predicted as missing for each feature."
}
"confusion_matrix" {
type "assoc"
additional_indices {
ref "ConfusionMatrix"
}
description "The confusion matrix for each feature."
}
}
}
Expand Down
26 changes: 13 additions & 13 deletions unit_tests/ut_h_analyze.amlg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,14 +283,13 @@
))
))
(call keep_result_payload)
(assign (assoc result (map (lambda (get (current_value) "feature_contributions_full")) result) ))
(print "computed contributions: ")
(call assert_same (assoc
obs (sort (indices result))
obs (sort (indices (get result "feature_contributions_full") ))
exp (list "height" "length" "size" "sweet" "tart" "weight" "width")
))
(call assert_approximate (assoc
obs result
obs (get result "feature_contributions_full")
exp
(assoc
height 0.01
Expand All @@ -315,15 +314,17 @@

(assign (assoc
result2
(call_entity "howso" "react_aggregate" (assoc
context_features features
details (assoc feature_contributions_full (true))
feature_influences_action_feature "fruit"
hyperparameter_param_path ["targeted" "width" "fruit.height.length.size.sweet.tart.weight." ".none"]
sample_model_fraction 1.0
))
(get
(call_entity "howso" "react_aggregate" (assoc
context_features features
details (assoc feature_contributions_full (true))
feature_influences_action_feature "fruit"
hyperparameter_param_path ["targeted" "width" "fruit.height.length.size.sweet.tart.weight." ".none"]
sample_model_fraction 1.0
))
[1 "payload" "feature_contributions_full"]
)
))
(assign (assoc result2 (map (lambda (get (current_value) "feature_contributions_full")) (get result2 [1 "payload"])) ))
(print "computed contributions with custom hp map: ")
(call assert_same (assoc
obs (sort (indices result2))
Expand All @@ -345,10 +346,9 @@
))
))
(call keep_result_payload)
(assign (assoc result (map (lambda (get (current_value) "feature_contributions_full")) result) ))
(print "Computed contributions for action feature height: ")
(call assert_approximate (assoc
obs result
obs (get result "feature_contributions_full")
exp (assoc
fruit 0.03
length 0.11
Expand Down
2 changes: 1 addition & 1 deletion unit_tests/ut_h_dynamic_deviations.amlg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@
(print "Price MAE with Dynamic Deviations is as expected: ")
(call assert_approximate (assoc
exp 400
obs (get dd_agg_stats ["price" "feature_residuals_full"])
obs (get dd_agg_stats ["feature_residuals_full" "price"])
thresh 300
))
(call exit_if_failures (assoc msg "MAE with DD is not as expected."))
Expand Down
Loading

0 comments on commit 2d69489

Please sign in to comment.