[ADD] Get incumbent config (#175)

* In progress get_incumbent_results

* [Add] get_incumbent_results to base task, changed additional info in abstract evaluator, and  tests

* In progress addressing fransisco's comment

* Proper check for include_traditional

* Fix flake

* Mock search of estimator

* Fixed path of run history test_api

* Addressed comments from Fransisco, making better tests

* fix flake

* After rebase fix issues

* fix flake

* Added debug information for API

* filtering only successful runs in get_incumbent_results

* Address comments from fransisco

* Revert changes made to run history assertion in base taks #1257

* fix flake issue

Author: ravinkohli

autoPyTorch/api/base_task.py

@@ -12,7 +12,7 @@
 import unittest.mock
 import warnings
 from abc import abstractmethod
-from typing import Any, Callable, Dict, List, Optional, Union, cast
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast
 
 from ConfigSpace.configuration_space import Configuration, ConfigurationSpace
 
@@ -223,9 +223,7 @@ def build_pipeline(self, dataset_properties: Dict[str, Any]) -> BasePipeline:
         """
         raise NotImplementedError
 
-    def set_pipeline_config(
-            self,
-            **pipeline_config_kwargs: Any) -> None:
+    def set_pipeline_config(self, **pipeline_config_kwargs: Any) -> None:
         """
         Check whether arguments are valid and
         then sets them to the current pipeline
@@ -259,12 +257,6 @@ def get_pipeline_options(self) -> dict:
         """
         return self.pipeline_options
 
-    # def set_search_space(self, search_space: ConfigurationSpace) -> None:
-    #     """
-    #     Update the search space.
-    #     """
-    #     raise NotImplementedError
-    #
     def get_search_space(self, dataset: BaseDataset = None) -> ConfigurationSpace:
         """
         Returns the current search space as ConfigurationSpace object.
@@ -406,9 +398,9 @@ def _close_dask_client(self) -> None:
             None
         """
         if (
-                hasattr(self, '_is_dask_client_internally_created')
-                and self._is_dask_client_internally_created
-                and self._dask_client
+            hasattr(self, '_is_dask_client_internally_created')
+            and self._is_dask_client_internally_created
+            and self._dask_client
         ):
             self._dask_client.shutdown()
             self._dask_client.close()
@@ -661,10 +653,11 @@ def _do_traditional_prediction(self, time_left: int, func_eval_time_limit_secs:
                             f"Fitting {cls} took {runtime}s, performance:{cost}/{additional_info}")
                         configuration = additional_info['pipeline_configuration']
                         origin = additional_info['configuration_origin']
+                        additional_info.pop('pipeline_configuration')
                         run_history.add(config=configuration, cost=cost,
                                         time=runtime, status=status, seed=self.seed,
                                         starttime=starttime, endtime=starttime + runtime,
-                                        origin=origin)
+                                        origin=origin, additional_info=additional_info)
                     else:
                         if additional_info.get('exitcode') == -6:
                             self._logger.error(
@@ -710,6 +703,7 @@ def _search(
         memory_limit: Optional[int] = 4096,
         smac_scenario_args: Optional[Dict[str, Any]] = None,
         get_smac_object_callback: Optional[Callable] = None,
+        tae_func: Optional[Callable] = None,
         all_supported_metrics: bool = True,
         precision: int = 32,
         disable_file_output: List = [],
@@ -777,6 +771,10 @@ def _search(
                 instances, num_params, runhistory, seed and ta. This is
                 an advanced feature. Use only if you are familiar with
                 [SMAC](https://automl.github.io/SMAC3/master/index.html).
+            tae_func (Optional[Callable]):
+                TargetAlgorithm to be optimised. If None, `eval_function`
+                available in autoPyTorch/evaluation/train_evaluator is used.
+                Must be child class of AbstractEvaluator.
             all_supported_metrics (bool), (default=True): if True, all
                 metrics supporting current task will be calculated
                 for each pipeline and results will be available via cv_results
@@ -988,7 +986,7 @@ def _search(
             )
             try:
                 run_history, self.trajectory, budget_type = \
-                    _proc_smac.run_smbo()
+                    _proc_smac.run_smbo(func=tae_func)
                 self.run_history.update(run_history, DataOrigin.INTERNAL)
                 trajectory_filename = os.path.join(
                     self._backend.get_smac_output_directory_for_run(self.seed),
@@ -1042,10 +1040,10 @@ def _search(
         return self
 
     def refit(
-            self,
-            dataset: BaseDataset,
-            budget_config: Dict[str, Union[int, str]] = {},
-            split_id: int = 0
+        self,
+        dataset: BaseDataset,
+        budget_config: Dict[str, Union[int, str]] = {},
+        split_id: int = 0
     ) -> "BaseTask":
         """
         Refit all models found with fit to new data.
@@ -1181,10 +1179,10 @@ def fit(self,
         return pipeline
 
     def predict(
-            self,
-            X_test: np.ndarray,
-            batch_size: Optional[int] = None,
-            n_jobs: int = 1
+        self,
+        X_test: np.ndarray,
+        batch_size: Optional[int] = None,
+        n_jobs: int = 1
     ) -> np.ndarray:
         """Generate the estimator predictions.
         Generate the predictions based on the given examples from the test set.
@@ -1234,9 +1232,9 @@ def predict(
         return predictions
 
     def score(
-            self,
-            y_pred: np.ndarray,
-            y_test: Union[np.ndarray, pd.DataFrame]
+        self,
+        y_pred: np.ndarray,
+        y_test: Union[np.ndarray, pd.DataFrame]
     ) -> Dict[str, float]:
         """Calculate the score on the test set.
         Calculate the evaluation measure on the test set.
@@ -1277,17 +1275,37 @@ def __del__(self) -> None:
         if hasattr(self, '_backend'):
             self._backend.context.delete_directories(force=False)
 
-    @typing.no_type_check
     def get_incumbent_results(
-            self
-    ):
-        pass
+        self,
+        include_traditional: bool = False
+    ) -> Tuple[Configuration, Dict[str, Union[int, str, float]]]:
+        """
+        Get Incumbent config and the corresponding results
+        Args:
+            include_traditional: Whether to include results from tradtional pipelines
 
-    @typing.no_type_check
-    def get_incumbent_config(
-            self
-    ):
-        pass
+        Returns:
+
+        """
+        assert self.run_history is not None, "No Run History found, search has not been called."
+        if self.run_history.empty():
+            raise ValueError("Run History is empty. Something went wrong, "
+                             "smac was not able to fit any model?")
+
+        run_history_data = self.run_history.data
+        if not include_traditional:
+            # traditional classifiers have trainer_configuration in their additional info
+            run_history_data = dict(
+                filter(lambda elem: elem[1].additional_info is not None and elem[1].
+                       additional_info['configuration_origin'] != 'traditional',
+                       run_history_data.items()))
+        run_history_data = dict(
+            filter(lambda elem: 'SUCCESS' in str(elem[1].status), run_history_data.items()))
+        sorted_runvalue_by_cost = sorted(run_history_data.items(), key=lambda item: item[1].cost)
+        incumbent_run_key, incumbent_run_value = sorted_runvalue_by_cost[0]
+        incumbent_config = self.run_history.ids_config[incumbent_run_key.config_id]
+        incumbent_results = incumbent_run_value.additional_info
+        return incumbent_config, incumbent_results
 
     def get_models_with_weights(self) -> List:
         if self.models_ is None or len(self.models_) == 0 or \

autoPyTorch/evaluation/abstract_evaluator.py

@@ -114,7 +114,8 @@ def get_additional_run_info(self) -> Dict[str, Any]:  # pylint: disable=R0201
                     Can be found in autoPyTorch/pipeline/components/setup/traditional_ml/classifier_configs
         """
         return {'pipeline_configuration': self.configuration,
-                'trainer_configuration': self.pipeline.named_steps['model_trainer'].choice.model.get_config()}
+                'trainer_configuration': self.pipeline.named_steps['model_trainer'].choice.model.get_config(),
+                'configuration_origin': 'traditional'}
 
     def get_pipeline_representation(self) -> Dict[str, str]:
         return self.pipeline.get_pipeline_representation()
@@ -178,7 +179,7 @@ def estimator_supports_iterative_fit(self) -> bool:  # pylint: disable=R0201
         return False
 
     def get_additional_run_info(self) -> Dict:  # pylint: disable=R0201
-        return {}
+        return {'configuration_origin': 'DUMMY'}
 
     def get_pipeline_representation(self) -> Dict[str, str]:
         return {
@@ -237,7 +238,7 @@ def estimator_supports_iterative_fit(self) -> bool:  # pylint: disable=R0201
         return False
 
     def get_additional_run_info(self) -> Dict:  # pylint: disable=R0201
-        return {}
+        return {'configuration_origin': 'DUMMY'}
 
     @staticmethod
     def get_default_pipeline_options() -> Dict[str, Any]:
@@ -611,8 +612,7 @@ def finish_up(self, loss: Dict[str, float], train_loss: Dict[str, float],
         additional_run_info = (
             {} if additional_run_info is None else additional_run_info
         )
-        for metric_name, value in loss.items():
-            additional_run_info[metric_name] = value
+        additional_run_info['opt_loss'] = loss
         additional_run_info['duration'] = self.duration
         additional_run_info['num_run'] = self.num_run
         if train_loss is not None:

autoPyTorch/evaluation/train_evaluator.py

@@ -172,10 +172,11 @@ def fit_predict_and_loss(self) -> None:
 
             status = StatusType.SUCCESS
 
-            self.logger.debug("In train evaluator fit_predict_and_loss, num_run: {} loss:{}".format(
-                self.num_run,
-                loss
-            ))
+            self.logger.debug("In train evaluator fit_predict_and_loss, num_run: {} loss:{},"
+                              " additional run info:{}, status: {}".format(self.num_run,
+                                                                           loss,
+                                                                           additional_run_info,
+                                                                           status))
             self.finish_up(
                 loss=loss,
                 train_loss=train_loss,