From 628e455269c7f5501509e33ae471670c78bc1301 Mon Sep 17 00:00:00 2001 From: juacrumar Date: Fri, 6 Sep 2024 12:33:09 +0200 Subject: [PATCH 1/6] first attempt --- n3fit/src/n3fit/hyper_optimization/rewards.py | 19 +++++++++++++------ n3fit/src/n3fit/model_trainer.py | 3 ++- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/n3fit/src/n3fit/hyper_optimization/rewards.py b/n3fit/src/n3fit/hyper_optimization/rewards.py index 7513ea4a6a..8b125490d8 100644 --- a/n3fit/src/n3fit/hyper_optimization/rewards.py +++ b/n3fit/src/n3fit/hyper_optimization/rewards.py @@ -44,7 +44,7 @@ log = logging.getLogger(__name__) -def _average_best(fold_losses: np.ndarray, proportion: float = 0.9, axis: int = 0) -> float: +def _average_best(fold_losses: np.ndarray, proportion: float = 0.05, axis: int = 0) -> float: """ Compute the average of the input array along the specified axis, among the best `proportion` of replicas. @@ -195,7 +195,8 @@ def __init__( def compute_loss( self, penalties: dict[str, np.ndarray], - experimental_loss: np.ndarray, + validation_loss: np.ndarray, + kfold_loss: np.ndarray, pdf_object: N3PDF, experimental_data: list[DataGroupSpec], fold_idx: int = 0, @@ -250,20 +251,26 @@ def compute_loss( # update hyperopt metrics # these are saved in the phi_vector and chi2_matrix attributes, excluding penalties - self._save_hyperopt_metrics(phi_per_fold, experimental_loss, penalties, fold_idx) + self._save_hyperopt_metrics(phi_per_fold, kfold_loss, penalties, fold_idx) # Prepare the output loss, including penalties if necessary if self._penalties_in_loss: # include penalties to experimental loss - experimental_loss += sum(penalties.values()) + kfold_loss += sum(penalties.values()) # add penalties to phi in the form of a sum of per-replicas averages phi_per_fold += sum(np.mean(penalty) for penalty in penalties.values()) # define loss for hyperopt according to the chosen loss_type if self.loss_type == "chi2": - # calculate statistics of chi2 over replicas for a given k-fold - loss = self.reduce_over_replicas(experimental_loss) + # calculate statistics of chi2 over replicas for a given k-fold_statistic + + ### Experiment: + # Use the validation loss as the loss + # summed with how far from 2 are we for the kfold + validation_loss_average = self.reduce_over_replicas(validation_loss, proportion=0.9) + kfold_loss_average = self.reduce_over_replicas(kfold_loss, proportion=0.1) + loss = validation_loss_average + (max(kfold_loss_average, 2.0) - 2.0) elif self.loss_type == "phi2": loss = phi_per_fold**2 diff --git a/n3fit/src/n3fit/model_trainer.py b/n3fit/src/n3fit/model_trainer.py index 5b52422dda..82ccb77a48 100644 --- a/n3fit/src/n3fit/model_trainer.py +++ b/n3fit/src/n3fit/model_trainer.py @@ -1014,7 +1014,8 @@ def hyperparametrizable(self, params): # Compute per replica hyper losses hyper_loss = self._hyper_loss.compute_loss( penalties=penalties, - experimental_loss=experimental_loss, + kfold_loss=experimental_loss, + validation_loss=validation_loss, pdf_object=vplike_pdf, experimental_data=experimental_data, fold_idx=k, From 98a3dc1379e256647f4b5557cbfec4f5b99cb608 Mon Sep 17 00:00:00 2001 From: Tanjona Rabemananjara Date: Fri, 16 Aug 2024 11:42:32 +0200 Subject: [PATCH 2/6] filter positivity datapoints --- ...restricted_search_space_renew_hyperopt.yml | 50 +++++++++++++------ 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/n3fit/runcards/hyperopt_studies/restricted_search_space_renew_hyperopt.yml b/n3fit/runcards/hyperopt_studies/restricted_search_space_renew_hyperopt.yml index 29fbf41c49..acca5bd474 100644 --- a/n3fit/runcards/hyperopt_studies/restricted_search_space_renew_hyperopt.yml +++ b/n3fit/runcards/hyperopt_studies/restricted_search_space_renew_hyperopt.yml @@ -97,7 +97,7 @@ datacuts: ############################################################ theory: - theoryid: 700 # database id + theoryid: 40_000_000 hyperscan_config: architecture: @@ -243,27 +243,29 @@ fitting: savepseudodata: false fitbasis: EVOL basis: - - {fl: sng, trainable: false, smallx: [1.091, 1.119], largex: [1.471, 3.021]} - - {fl: g, trainable: false, smallx: [0.7795, 1.095], largex: [2.742, 5.547]} - - {fl: v, trainable: false, smallx: [0.472, 0.7576], largex: [1.571, 3.559]} - - {fl: v3, trainable: false, smallx: [0.07483, 0.4501], largex: [1.714, 3.467]} - - {fl: v8, trainable: false, smallx: [0.5731, 0.779], largex: [1.555, 3.465]} - - {fl: t3, trainable: false, smallx: [-0.5498, 1.0], largex: [1.778, 3.5]} - - {fl: t8, trainable: false, smallx: [0.5469, 0.857], largex: [1.555, 3.391]} - - {fl: t15, trainable: false, smallx: [1.081, 1.142], largex: [1.491, 3.092]} + - {fl: sng, trainable: false, smallx: [1.089, 1.119], largex: [1.475, 3.119]} + - {fl: g, trainable: false, smallx: [0.7504, 1.098], largex: [2.814, 5.669]} + - {fl: v, trainable: false, smallx: [0.479, 0.7384], largex: [1.549, 3.532]} + - {fl: v3, trainable: false, smallx: [0.1073, 0.4397], largex: [1.733, 3.458]} + - {fl: v8, trainable: false, smallx: [0.5507, 0.7837], largex: [1.516, 3.356]} + - {fl: t3, trainable: false, smallx: [-0.4506, 0.9305], largex: [1.745, 3.424]} + - {fl: t8, trainable: false, smallx: [0.5877, 0.8687], largex: [1.522, 3.515]} + - {fl: t15, trainable: false, smallx: [1.089, 1.141], largex: [1.492, 3.222]} -############################################################ +################################################################################ positivity: posdatasets: - - {dataset: NNPDF_POS_2P24GEV_F2U, maxlambda: 1e6} # Positivity Lagrange Multiplier + # Positivity Lagrange Multiplier + - {dataset: NNPDF_POS_2P24GEV_F2U, maxlambda: 1e6} - {dataset: NNPDF_POS_2P24GEV_F2D, maxlambda: 1e6} - {dataset: NNPDF_POS_2P24GEV_F2S, maxlambda: 1e6} - - {dataset: NNPDF_POS_2P24GEV_FLL-19PTS, maxlambda: 1e6} + - {dataset: NNPDF_POS_2P24GEV_FLL, maxlambda: 1e6} - {dataset: NNPDF_POS_2P24GEV_DYU, maxlambda: 1e10} - {dataset: NNPDF_POS_2P24GEV_DYD, maxlambda: 1e10} - {dataset: NNPDF_POS_2P24GEV_DYS, maxlambda: 1e10} - - {dataset: NNPDF_POS_2P24GEV_F2C-17PTS, maxlambda: 1e6} - - {dataset: NNPDF_POS_2P24GEV_XUQ, maxlambda: 1e6} # Positivity of MSbar PDFs + - {dataset: NNPDF_POS_2P24GEV_F2C, maxlambda: 1e6} + # Positivity of MSbar PDFs + - {dataset: NNPDF_POS_2P24GEV_XUQ, maxlambda: 1e6} - {dataset: NNPDF_POS_2P24GEV_XUB, maxlambda: 1e6} - {dataset: NNPDF_POS_2P24GEV_XDQ, maxlambda: 1e6} - {dataset: NNPDF_POS_2P24GEV_XDB, maxlambda: 1e6} @@ -271,6 +273,26 @@ positivity: - {dataset: NNPDF_POS_2P24GEV_XSB, maxlambda: 1e6} - {dataset: NNPDF_POS_2P24GEV_XGL, maxlambda: 1e6} +added_filter_rules: + - dataset: NNPDF_POS_2P24GEV_FLL + rule: "x > 5.0e-7" + - dataset: NNPDF_POS_2P24GEV_F2C + rule: "x < 0.74" + - dataset: NNPDF_POS_2P24GEV_XGL + rule: "x > 0.1" + - dataset: NNPDF_POS_2P24GEV_XUQ + rule: "x > 0.1" + - dataset: NNPDF_POS_2P24GEV_XUB + rule: "x > 0.1" + - dataset: NNPDF_POS_2P24GEV_XDQ + rule: "x > 0.1" + - dataset: NNPDF_POS_2P24GEV_XDB + rule: "x > 0.1" + - dataset: NNPDF_POS_2P24GEV_XSQ + rule: "x > 0.1" + - dataset: NNPDF_POS_2P24GEV_XSB + rule: "x > 0.1" + ############################################################ integrability: integdatasets: From a6c8bda85235f5a6dcea156921cf059f1b597814 Mon Sep 17 00:00:00 2001 From: Tanjona Rabemananjara Date: Sun, 18 Aug 2024 22:46:44 +0200 Subject: [PATCH 3/6] remove penalties in losses --- .../hyperopt_studies/restricted_search_space_renew_hyperopt.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/n3fit/runcards/hyperopt_studies/restricted_search_space_renew_hyperopt.yml b/n3fit/runcards/hyperopt_studies/restricted_search_space_renew_hyperopt.yml index acca5bd474..cad7defc0b 100644 --- a/n3fit/runcards/hyperopt_studies/restricted_search_space_renew_hyperopt.yml +++ b/n3fit/runcards/hyperopt_studies/restricted_search_space_renew_hyperopt.yml @@ -128,7 +128,6 @@ kfold: loss_type: chi2 replica_statistic: average_best fold_statistic: average - penalties_in_loss: True penalties: - saturation - patience From 4e398cf4822f25406e8e559401ac60860f894ac1 Mon Sep 17 00:00:00 2001 From: Tanjona Rabemananjara Date: Fri, 13 Sep 2024 11:58:38 +0200 Subject: [PATCH 4/6] completely remove penalties & increase threshold --- .../restricted_search_space_renew_hyperopt.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/n3fit/runcards/hyperopt_studies/restricted_search_space_renew_hyperopt.yml b/n3fit/runcards/hyperopt_studies/restricted_search_space_renew_hyperopt.yml index cad7defc0b..4e6924139e 100644 --- a/n3fit/runcards/hyperopt_studies/restricted_search_space_renew_hyperopt.yml +++ b/n3fit/runcards/hyperopt_studies/restricted_search_space_renew_hyperopt.yml @@ -128,11 +128,7 @@ kfold: loss_type: chi2 replica_statistic: average_best fold_statistic: average - penalties: - - saturation - - patience - - integrability - threshold: 10 + threshold: 20 partitions: - datasets: # DIS From 983f2f247e94cf18ee3ac7d3fb4f09109edb7fbf Mon Sep 17 00:00:00 2001 From: Tanjona Rabemananjara Date: Fri, 13 Sep 2024 13:10:48 +0200 Subject: [PATCH 5/6] change validation to 80% --- n3fit/src/n3fit/hyper_optimization/rewards.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/hyper_optimization/rewards.py b/n3fit/src/n3fit/hyper_optimization/rewards.py index 8b125490d8..d1bb9a49de 100644 --- a/n3fit/src/n3fit/hyper_optimization/rewards.py +++ b/n3fit/src/n3fit/hyper_optimization/rewards.py @@ -268,7 +268,7 @@ def compute_loss( ### Experiment: # Use the validation loss as the loss # summed with how far from 2 are we for the kfold - validation_loss_average = self.reduce_over_replicas(validation_loss, proportion=0.9) + validation_loss_average = self.reduce_over_replicas(validation_loss, proportion=0.8) kfold_loss_average = self.reduce_over_replicas(kfold_loss, proportion=0.1) loss = validation_loss_average + (max(kfold_loss_average, 2.0) - 2.0) elif self.loss_type == "phi2": From 14c202aec127316fc566536b69c741b721808cf2 Mon Sep 17 00:00:00 2001 From: juacrumar Date: Sun, 15 Sep 2024 21:08:18 +0200 Subject: [PATCH 6/6] explain choice of % in validation/fold --- n3fit/src/n3fit/hyper_optimization/rewards.py | 17 +++++++++++------ n3fit/src/n3fit/tests/test_hyperopt.py | 6 +++++- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/n3fit/src/n3fit/hyper_optimization/rewards.py b/n3fit/src/n3fit/hyper_optimization/rewards.py index d1bb9a49de..6a0a85e083 100644 --- a/n3fit/src/n3fit/hyper_optimization/rewards.py +++ b/n3fit/src/n3fit/hyper_optimization/rewards.py @@ -72,7 +72,7 @@ def _average_best(fold_losses: np.ndarray, proportion: float = 0.05, axis: int = return _average(best_losses, axis=axis) -def _average(fold_losses: np.ndarray, axis: int = 0) -> float: +def _average(fold_losses: np.ndarray, axis: int = 0, **kwargs) -> float: """ Compute the average of the input array along the specified axis. @@ -90,7 +90,7 @@ def _average(fold_losses: np.ndarray, axis: int = 0) -> float: return np.average(fold_losses, axis=axis).item() -def _best_worst(fold_losses: np.ndarray, axis: int = 0) -> float: +def _best_worst(fold_losses: np.ndarray, axis: int = 0, **kwargs) -> float: """ Compute the maximum value of the input array along the specified axis. @@ -108,7 +108,7 @@ def _best_worst(fold_losses: np.ndarray, axis: int = 0) -> float: return np.max(fold_losses, axis=axis).item() -def _std(fold_losses: np.ndarray, axis: int = 0) -> float: +def _std(fold_losses: np.ndarray, axis: int = 0, **kwargs) -> float: """ Compute the standard deviation of the input array along the specified axis. @@ -265,9 +265,14 @@ def compute_loss( if self.loss_type == "chi2": # calculate statistics of chi2 over replicas for a given k-fold_statistic - ### Experiment: - # Use the validation loss as the loss - # summed with how far from 2 are we for the kfold + # Construct the final loss as a sum of + # 1. The validation chi2 + # 2. The distance to 2 for the kfold chi2 + # If a proportion allow as a keyword argument, use 80% and 10% + # as a proxy of + # "80% of the replicas should be good, but only a small % has to cover the folds" + # The values of 80% and 10% are completely empirical and should be investigated further + validation_loss_average = self.reduce_over_replicas(validation_loss, proportion=0.8) kfold_loss_average = self.reduce_over_replicas(kfold_loss, proportion=0.1) loss = validation_loss_average + (max(kfold_loss_average, 2.0) - 2.0) diff --git a/n3fit/src/n3fit/tests/test_hyperopt.py b/n3fit/src/n3fit/tests/test_hyperopt.py index 274394b25a..68bfae960c 100644 --- a/n3fit/src/n3fit/tests/test_hyperopt.py +++ b/n3fit/src/n3fit/tests/test_hyperopt.py @@ -79,7 +79,11 @@ def test_compute_per_fold_loss(loss_type, replica_statistic, expected_per_fold_l # calculate statistic loss for one specific fold pdf_object = N3PDF(pdf_model.split_replicas()) predicted_per_fold_loss = loss.compute_loss( - penalties, experimental_loss, pdf_object, experimental_data + penalties, + kfold_loss=experimental_loss, + validation_loss=experimental_loss, + pdf_object=pdf_object, + experimental_data=experimental_data, ) # Assert