Merge pull request #130 from perib/new_search_space_def

fix num_nodes test, add update checker, setup.py change

setup.py

@@ -42,12 +42,10 @@ def calculate_version():
                       'lightgbm>=3.3.3',
                       'optuna>=3.0.5',
                       'baikal>=0.4.2',
-                      'jupyter>=1.0.0',
                       'networkx>=3.0',
                       'dask>=2024.4.2',
                       'distributed>=2024.4.2',
                       'dask-expr>=1.0.12',
-                      'dask-ml>=2023.4.20',
                       'dask-jobqueue>=0.8.5',
                       'func_timeout>=4.3.5',
                       'configspace>=0.7.1',

tpot2/__init__.py

@@ -18,3 +18,7 @@
 
 
 from .tpot_estimator import TPOTClassifier, TPOTRegressor, TPOTEstimator, TPOTEstimatorSteadyState
+
+from update_checker import update_check
+from ._version import __version__
+update_check("tpot2",__version__)

tpot2/_version.py

@@ -1 +1 @@
-__version__ = '0.1.5-alpha'
+__version__ = '0.1.7-alpha'

tpot2/config/classifiers.py

@@ -406,7 +406,6 @@ def get_HistGradientBoostingClassifier_ConfigurationSpace(n_features, random_sta
     validation_fraction_cond = EqualsCondition(validation_fraction, early_stop, "valid")
 
     space = {
-        'loss': Categorical("loss", ['log_loss', 'exponential']),
         'learning_rate': Float("learning_rate", bounds=(1e-3, 1), log=True),
         'min_samples_leaf': Integer("min_samples_leaf", bounds=(1, 200)),
         'max_features': Float("max_features", bounds=(0.1,1.0)), 
@@ -432,7 +431,6 @@ def get_HistGradientBoostingClassifier_ConfigurationSpace(n_features, random_sta
 def HistGradientBoostingClassifier_hyperparameter_parser(params):
 
     final_params = {
-        'loss': params['loss'],
         'learning_rate': params['learning_rate'],
         'min_samples_leaf': params['min_samples_leaf'],
         'max_features': params['max_features'],
@@ -447,7 +445,7 @@ def HistGradientBoostingClassifier_hyperparameter_parser(params):
 
 
     if params['early_stop'] == 'off':
-        final_params['n_iter_no_change'] = None
+        final_params['n_iter_no_change'] = 0
         final_params['validation_fraction'] = None
         final_params['early_stopping'] = False
     elif params['early_stop'] == 'valid':
@@ -477,12 +475,14 @@ def get_MLPClassifier_ConfigurationSpace(random_state):
 
     n_hidden_layers = Integer("n_hidden_layers", bounds=(1, 3))
     n_nodes_per_layer = Integer("n_nodes_per_layer", bounds=(16, 512))
-    activation = Categorical("activation", ['tanh', 'relu'])
-    alpha = Float("alpha", bounds=(1e-7, 1e-1), log=True)
-    learning_rate = Float("learning_rate", bounds=(1e-4, 1e-1), log=True)
+    activation = Categorical("activation", ["identity", "logistic",'tanh', 'relu'])
+    alpha = Float("alpha", bounds=(1e-4, 1e-1), log=True)
     early_stopping = Categorical("early_stopping", [True,False])
 
-    cs.add_hyperparameters([n_hidden_layers, n_nodes_per_layer, activation, alpha, learning_rate, early_stopping])
+    learning_rate_init = Float("learning_rate_init", bounds=(1e-4, 1e-1), log=True)
+    learning_rate = Categorical("learning_rate", ['constant', 'invscaling', 'adaptive'])
+
+    cs.add_hyperparameters([n_hidden_layers, n_nodes_per_layer, activation, alpha, learning_rate, early_stopping, learning_rate_init])
 
     return cs
 
@@ -492,8 +492,10 @@ def MLPClassifier_hyperparameter_parser(params):
         'hidden_layer_sizes' : [params['n_nodes_per_layer']]*params['n_hidden_layers'],
         'activation': params['activation'],
         'alpha': params['alpha'],
-        'learning_rate': params['learning_rate'],
         'early_stopping': params['early_stopping'],
+
+        'learning_rate_init': params['learning_rate_init'],
+        'learning_rate': params['learning_rate'],
     }
 
     if 'random_state' in params:

tpot2/config/get_configspace.py

@@ -104,12 +104,15 @@
 
 
 # not including "PassiveAggressiveClassifier" in classifiers since it is mainly for larger than memory datasets/online use cases
+
+# TODO need to subclass "GaussianProcessClassifier" and 'GaussianProcessRegressor'. These require n_features as a parameter for the kernel, but n_features may be different depending on selection functions or transformations previously in the pipeline.
+
 GROUPNAMES = {
         "selectors": ["SelectFwe", "SelectPercentile", "VarianceThreshold",],
         "selectors_classification": ["SelectFwe", "SelectPercentile", "VarianceThreshold", "RFE_classification", "SelectFromModel_classification"],
         "selectors_regression": ["SelectFwe", "SelectPercentile", "VarianceThreshold", "RFE_regression", "SelectFromModel_regression"],
-        "classifiers" :  ["LGBMRegressor", "BaggingClassifier", "GaussianProcessClassifier", 'AdaBoostClassifier', 'BernoulliNB', 'DecisionTreeClassifier', 'ExtraTreesClassifier', 'GaussianNB', 'HistGradientBoostingClassifier', 'KNeighborsClassifier','LinearDiscriminantAnalysis', 'LogisticRegression', "LinearSVC", "SVC", 'MLPClassifier', 'MultinomialNB',  "QuadraticDiscriminantAnalysis", 'RandomForestClassifier', 'SGDClassifier', 'XGBClassifier'],
-        "regressors" : ['AdaBoostRegressor', "ARDRegression", 'DecisionTreeRegressor', 'ExtraTreesRegressor', 'GaussianProcessRegressor', 'HistGradientBoostingRegressor', 'KNeighborsRegressor',  'LinearSVR', "MLPRegressor", 'RandomForestRegressor', 'SGDRegressor', 'SVR', 'XGBRegressor'],
+        "classifiers" :  ["LGBMRegressor", "BaggingClassifier", 'AdaBoostClassifier', 'BernoulliNB', 'DecisionTreeClassifier', 'ExtraTreesClassifier', 'GaussianNB', 'HistGradientBoostingClassifier', 'KNeighborsClassifier','LinearDiscriminantAnalysis', 'LogisticRegression', "LinearSVC", "SVC", 'MLPClassifier', 'MultinomialNB',  "QuadraticDiscriminantAnalysis", 'RandomForestClassifier', 'SGDClassifier', 'XGBClassifier'],
+        "regressors" : ['AdaBoostRegressor', "ARDRegression", 'DecisionTreeRegressor', 'ExtraTreesRegressor', 'HistGradientBoostingRegressor', 'KNeighborsRegressor',  'LinearSVR', "MLPRegressor", 'RandomForestRegressor', 'SGDRegressor', 'SVR', 'XGBRegressor'],
 
 
         "transformers":  ["Binarizer", "PCA", "ZeroCount", "ColumnOneHotEncoder", "FastICA", "FeatureAgglomeration", "Nystroem", "RBFSampler", "QuantileTransformer", "PowerTransformer"],
@@ -263,7 +266,7 @@ def get_configspace(name, n_classes=3, n_samples=100, n_features=100, random_sta
         case "FastICA":
             return transformers.get_FastICA_configspace(n_features=n_features, random_state=random_state)
         case "FeatureAgglomeration":
-            return transformers.get_FeatureAgglomeration_configspace(n_features=n_features,)
+            return transformers.get_FeatureAgglomeration_configspace(n_samples=n_samples)
         case "Nystroem":
             return transformers.get_Nystroem_configspace(n_features=n_features, random_state=random_state)
         case "RBFSampler":
@@ -435,9 +438,12 @@ def get_node(name, n_classes=3, n_samples=100, n_features=100, random_state=None
     if name == "HistGradientBoostingClassifier":
         configspace = get_configspace(name, n_classes=n_classes, n_samples=n_samples, random_state=random_state)
         return EstimatorNode(STRING_TO_CLASS[name], configspace, hyperparameter_parser=classifiers.HistGradientBoostingClassifier_hyperparameter_parser)
-    if name == "GradientBoostingRegressor" or name == "HistGradientBoostingRegressor":
+    if name == "GradientBoostingRegressor":
         configspace = get_configspace(name, n_classes=n_classes, n_samples=n_samples, random_state=random_state)
         return EstimatorNode(STRING_TO_CLASS[name], configspace, hyperparameter_parser=regressors.GradientBoostingRegressor_hyperparameter_parser)
+    if  name == "HistGradientBoostingRegressor":
+        configspace = get_configspace(name, n_classes=n_classes, n_samples=n_samples, random_state=random_state)
+        return EstimatorNode(STRING_TO_CLASS[name], configspace, hyperparameter_parser=regressors.HistGradientBoostingRegressor_hyperparameter_parser)
     if name == "MLPClassifier":
         configspace = get_configspace(name, n_classes=n_classes, n_samples=n_samples, random_state=random_state)
         return EstimatorNode(STRING_TO_CLASS[name], configspace, hyperparameter_parser=classifiers.MLPClassifier_hyperparameter_parser)

tpot2/config/regressors.py

@@ -412,6 +412,36 @@ def get_GradientBoostingRegressor_ConfigurationSpace(n_features, random_state):
     cs.add_conditions([validation_fraction_cond, n_iter_no_change_cond])
     return cs
 
+def GradientBoostingRegressor_hyperparameter_parser(params):
+
+    final_params = {
+        'loss': params['loss'],
+        'learning_rate': params['learning_rate'],
+        'min_samples_leaf': params['min_samples_leaf'],
+        'min_samples_split': params['min_samples_split'],
+        'max_features': params['max_features'],
+        'max_leaf_nodes': params['max_leaf_nodes'],
+        'max_depth': params['max_depth'],
+        'tol': params['tol'],
+        'subsample': params['subsample']
+    }
+
+    if 'random_state' in params:
+        final_params['random_state'] = params['random_state']
+
+    if params['early_stop'] == 'off':
+        final_params['n_iter_no_change'] = None
+        final_params['validation_fraction'] = None
+    elif params['early_stop'] == 'valid':
+        final_params['n_iter_no_change'] = params['n_iter_no_change']
+        final_params['validation_fraction'] = params['validation_fraction']
+    elif params['early_stop'] == 'train':
+        final_params['n_iter_no_change'] = params['n_iter_no_change']
+        final_params['validation_fraction'] = None
+
+
+    return final_params
+
 #only difference is l2_regularization
 def get_HistGradientBoostingRegressor_ConfigurationSpace(n_features, random_state):
     early_stop = Categorical("early_stop", ["off", "valid", "train"])
@@ -443,39 +473,40 @@ def get_HistGradientBoostingRegressor_ConfigurationSpace(n_features, random_stat
 
     return cs
 
-def GradientBoostingRegressor_hyperparameter_parser(params):
+
+def HistGradientBoostingRegressor_hyperparameter_parser(params):
 
     final_params = {
-        'loss': params['loss'],
         'learning_rate': params['learning_rate'],
         'min_samples_leaf': params['min_samples_leaf'],
         'max_features': params['max_features'],
         'max_leaf_nodes': params['max_leaf_nodes'],
         'max_depth': params['max_depth'],
         'tol': params['tol'],
+        'l2_regularization': params['l2_regularization']
     }
 
-    if "l2_regularization" in params:
-        final_params['l2_regularization'] = params['l2_regularization']
-
     if 'random_state' in params:
         final_params['random_state'] = params['random_state']
 
+
     if params['early_stop'] == 'off':
-        final_params['n_iter_no_change'] = None
+        final_params['n_iter_no_change'] = 0
         final_params['validation_fraction'] = None
+        final_params['early_stopping'] = False
     elif params['early_stop'] == 'valid':
         final_params['n_iter_no_change'] = params['n_iter_no_change']
         final_params['validation_fraction'] = params['validation_fraction']
+        final_params['early_stopping'] = True
     elif params['early_stop'] == 'train':
         final_params['n_iter_no_change'] = params['n_iter_no_change']
         final_params['validation_fraction'] = None
+        final_params['early_stopping'] = True
 
 
     return final_params
 
 
-
 ###
 
 def get_MLPRegressor_ConfigurationSpace(random_state):
@@ -495,7 +526,10 @@ def get_MLPRegressor_ConfigurationSpace(random_state):
     learning_rate = Float("learning_rate", bounds=(1e-4, 1e-1), log=True)
     early_stopping = Categorical("early_stopping", [True,False])
 
-    cs.add_hyperparameters([n_hidden_layers, n_nodes_per_layer, activation, alpha, learning_rate, early_stopping])
+    learning_rate_init = Float("learning_rate_init", bounds=(1e-4, 1e-1), log=True)
+    learning_rate = Categorical("learning_rate", ['constant', 'invscaling', 'adaptive'])
+
+    cs.add_hyperparameters([n_hidden_layers, n_nodes_per_layer, activation, alpha, learning_rate, early_stopping, learning_rate_init])
 
     return cs
 
@@ -505,8 +539,9 @@ def MLPRegressor_hyperparameter_parser(params):
         'hidden_layer_sizes' : [params['n_nodes_per_layer']]*params['n_hidden_layers'],
         'activation': params['activation'],
         'alpha': params['alpha'],
-        'learning_rate': params['learning_rate'],
         'early_stopping': params['early_stopping'],
+        'learning_rate_init': params['learning_rate_init'],
+        'learning_rate': params['learning_rate'],
     }
 
     if 'random_state' in params:

tpot2/config/transformers.py

@@ -44,11 +44,11 @@ def get_FastICA_configspace(n_features=100, random_state=None):
 
     )
 
-def get_FeatureAgglomeration_configspace(n_features=100):
+def get_FeatureAgglomeration_configspace(n_samples):
 
     linkage = Categorical('linkage', ['ward', 'complete', 'average'])
     metric = Categorical('metric', ['euclidean', 'l1', 'l2', 'manhattan', 'cosine'])
-    n_clusters = Integer('n_clusters', bounds=(2, 400))
+    n_clusters = Integer('n_clusters', bounds=(2, min(n_samples,400)))
     pooling_func = Categorical('pooling_func', ['mean', 'median', 'max'])
 
     metric_condition = NotEqualsCondition(metric, linkage, 'ward')

tpot2/objectives/number_of_nodes.py

@@ -4,7 +4,7 @@
 
 def number_of_nodes_objective(est):
     if isinstance(est, GraphPipeline):
-        return sum(node["instance"] for node in est.graph.nodes)
+        return sum(number_of_nodes_objective(est.graph.nodes[node]["instance"]) for node in est.graph.nodes)
     if isinstance(est, Pipeline):
         return sum(number_of_nodes_objective(estimator) for _,estimator in est.steps)
     if isinstance(est, sklearn.pipeline.FeatureUnion):

tpot2/objectives/tests/test_complexity_objective.py

@@ -1 +0,0 @@
-from ..complexity import BernoulliNB_Complexity, GaussianNB_Complexity, MultinomialNB_Complexity

tpot2/search_spaces/pipelines/graph.py

@@ -750,7 +750,11 @@ def generate(self, rng=None):
                                        self.cross_val_predict_cv, self.method, self.memory, self.use_label_encoder, rng=rng)  
             # if user specified limit, grab a random number between that limit
 
-        n_nodes = min(rng.integers(1, self.max_size), 5)
+        if self.max_size is None or self.max_size == np.inf:
+            n_nodes = rng.integers(1, 5)
+        else:
+            n_nodes = min(rng.integers(1, self.max_size), 5)
+
         starting_ops = []
         if self.inner_search_space is not None:
             starting_ops.append(ind._mutate_insert_inner_node)

tpot2/tpot_estimator/templates/tpottemplates.py

@@ -30,6 +30,7 @@ def __init__(       self,
                         memory_limit = "4GB",
                         client = None,
                         random_state=None,
+                        allow_inner_regressors=True,
                         **tpotestimator_kwargs,
         ):
         """
@@ -58,6 +59,7 @@ def __init__(       self,
         self.memory_limit = memory_limit
         self.client = client
         self.random_state = random_state
+        self.allow_inner_regressors = allow_inner_regressors
         self.tpotestimator_kwargs = tpotestimator_kwargs
 
         self.initialized = False
@@ -71,13 +73,18 @@ def fit(self, X, y):
                                         "n_features":X.shape[1], 
                                         "random_state":self.random_state}
 
-            search_space = tpot2.search_spaces.pipelines.GraphPipeline(
-                root_search_space= tpot2.config.get_search_space("regressors", **get_search_space_params),
-                leaf_search_space = None, 
-                inner_search_space = tpot2.config.get_search_space(["selectors","transformers","regressors","scalers"],**get_search_space_params),
-                max_size = 10,
-            )
-
+            if self.allow_inner_regressors:
+                search_space = tpot2.search_spaces.pipelines.GraphPipeline(
+                    root_search_space= tpot2.config.get_search_space("regressors", **get_search_space_params),
+                    leaf_search_space = None, 
+                    inner_search_space = tpot2.config.get_search_space(["selectors","transformers","regressors","scalers"],**get_search_space_params),
+                )
+            else:
+                search_space = tpot2.search_spaces.pipelines.GraphPipeline(
+                    root_search_space= tpot2.config.get_search_space("regressors", **get_search_space_params),
+                    leaf_search_space = None, 
+                    inner_search_space = tpot2.config.get_search_space(["selectors","transformers","scalers"],**get_search_space_params),
+                )
 
             super(TPOTRegressor,self).__init__(
                 search_space=search_space,
@@ -134,6 +141,7 @@ def __init__(       self,
                         memory_limit = "4GB",
                         client = None,
                         random_state=None,
+                        allow_inner_classifiers=True,
                         **tpotestimator_kwargs,
 
         ):
@@ -164,6 +172,7 @@ def __init__(       self,
         self.client = client
         self.random_state = random_state
         self.tpotestimator_kwargs = tpotestimator_kwargs
+        self.allow_inner_classifiers = allow_inner_classifiers
 
         self.initialized = False
 
@@ -176,12 +185,18 @@ def fit(self, X, y):
                                        "n_features":X.shape[1], 
                                        "random_state":self.random_state}
 
-            search_space = tpot2.search_spaces.pipelines.GraphPipeline(
-                root_search_space= tpot2.config.get_search_space("classifiers", **get_search_space_params),
-                leaf_search_space = None, 
-                inner_search_space = tpot2.config.get_search_space(["selectors","transformers","classifiers", "scalers"], **get_search_space_params),
-                max_size = 10,
-            )
+            if self.allow_inner_classifiers:
+                search_space = tpot2.search_spaces.pipelines.GraphPipeline(
+                    root_search_space= tpot2.config.get_search_space("classifiers", **get_search_space_params),
+                    leaf_search_space = None, 
+                    inner_search_space = tpot2.config.get_search_space(["selectors","transformers","classifiers", "scalers"], **get_search_space_params),
+                )
+            else:
+                search_space = tpot2.search_spaces.pipelines.GraphPipeline(
+                    root_search_space= tpot2.config.get_search_space("classifiers", **get_search_space_params),
+                    leaf_search_space = None, 
+                    inner_search_space = tpot2.config.get_search_space(["selectors","transformers","scalers"], **get_search_space_params),
+                )
 
 
             super(TPOTClassifier,self).__init__(