Coverage for mlos_core/mlos_core/tests/optimizers/optimizer_multiobj_test.py: 100%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5"""Test multi-target optimization.""" 

6 

7import logging 

8from typing import List, Optional, Type 

9 

10import ConfigSpace as CS 

11import numpy as np 

12import pandas as pd 

13import pytest 

14 

15from mlos_core.optimizers import BaseOptimizer, OptimizerType 

16from mlos_core.tests import SEED 

17 

18_LOG = logging.getLogger(__name__) 

19 

20 

21@pytest.mark.parametrize( 

22 ("optimizer_class", "kwargs"), 

23 [ 

24 *[(member.value, {}) for member in OptimizerType], 

25 ], 

26) 

27def test_multi_target_opt_wrong_weights( 

28 optimizer_class: Type[BaseOptimizer], 

29 kwargs: dict, 

30) -> None: 

31 """Make sure that the optimizer raises an error if the number of objective weights 

32 does not match the number of optimization targets. 

33 """ 

34 with pytest.raises(ValueError): 

35 optimizer_class( 

36 parameter_space=CS.ConfigurationSpace(seed=SEED), 

37 optimization_targets=["main_score", "other_score"], 

38 objective_weights=[1], 

39 **kwargs, 

40 ) 

41 

42 

43@pytest.mark.parametrize( 

44 ("objective_weights"), 

45 [ 

46 [2, 1], 

47 [0.5, 0.5], 

48 None, 

49 ], 

50) 

51@pytest.mark.parametrize( 

52 ("optimizer_class", "kwargs"), 

53 [ 

54 *[(member.value, {}) for member in OptimizerType], 

55 ], 

56) 

57def test_multi_target_opt( 

58 objective_weights: Optional[List[float]], 

59 optimizer_class: Type[BaseOptimizer], 

60 kwargs: dict, 

61) -> None: 

62 """Toy multi-target optimization problem to test the optimizers with mixed numeric 

63 types to ensure that original dtypes are retained. 

64 """ 

65 # pylint: disable=too-many-locals 

66 max_iterations = 10 

67 

68 def objective(point: pd.DataFrame) -> pd.DataFrame: 

69 # mix of hyperparameters, optimal is to select the highest possible 

70 return pd.DataFrame( 

71 { 

72 "main_score": point.x + point.y, 

73 "other_score": point.x**2 + point.y**2, 

74 } 

75 ) 

76 

77 input_space = CS.ConfigurationSpace(seed=SEED) 

78 # add a mix of numeric datatypes 

79 input_space.add(CS.UniformIntegerHyperparameter(name="x", lower=0, upper=5)) 

80 input_space.add(CS.UniformFloatHyperparameter(name="y", lower=0.0, upper=5.0)) 

81 

82 optimizer = optimizer_class( 

83 parameter_space=input_space, 

84 optimization_targets=["main_score", "other_score"], 

85 objective_weights=objective_weights, 

86 **kwargs, 

87 ) 

88 

89 with pytest.raises(ValueError, match="No observations"): 

90 optimizer.get_best_observations() 

91 

92 with pytest.raises(ValueError, match="No observations"): 

93 optimizer.get_observations() 

94 

95 for _ in range(max_iterations): 

96 suggestion, metadata = optimizer.suggest() 

97 assert isinstance(suggestion, pd.DataFrame) 

98 assert metadata is None or isinstance(metadata, pd.DataFrame) 

99 assert set(suggestion.columns) == {"x", "y"} 

100 # Check suggestion values are the expected dtype 

101 assert isinstance(suggestion.x.iloc[0], np.integer) 

102 assert isinstance(suggestion.y.iloc[0], np.floating) 

103 # Check that suggestion is in the space 

104 test_configuration = CS.Configuration( 

105 optimizer.parameter_space, suggestion.astype("O").iloc[0].to_dict() 

106 ) 

107 # Raises an error if outside of configuration space 

108 test_configuration.check_valid_configuration() 

109 # Test registering the suggested configuration with a score. 

110 observation = objective(suggestion) 

111 assert isinstance(observation, pd.DataFrame) 

112 assert set(observation.columns) == {"main_score", "other_score"} 

113 optimizer.register(configs=suggestion, scores=observation) 

114 

115 (best_config, best_score, best_context) = optimizer.get_best_observations() 

116 assert isinstance(best_config, pd.DataFrame) 

117 assert isinstance(best_score, pd.DataFrame) 

118 assert best_context is None 

119 assert set(best_config.columns) == {"x", "y"} 

120 assert set(best_score.columns) == {"main_score", "other_score"} 

121 assert best_config.shape == (1, 2) 

122 assert best_score.shape == (1, 2) 

123 

124 (all_configs, all_scores, all_contexts) = optimizer.get_observations() 

125 assert isinstance(all_configs, pd.DataFrame) 

126 assert isinstance(all_scores, pd.DataFrame) 

127 assert all_contexts is None 

128 assert set(all_configs.columns) == {"x", "y"} 

129 assert set(all_scores.columns) == {"main_score", "other_score"} 

130 assert all_configs.shape == (max_iterations, 2) 

131 assert all_scores.shape == (max_iterations, 2)