Coverage for mlos_core/mlos_core/tests/optimizers/optimizer_multiobj_test.py: 100%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5"""Test multi-target optimization.""" 

6 

7import logging 

8 

9import ConfigSpace as CS 

10import pandas as pd 

11import pytest 

12 

13from mlos_core.data_classes import Observations, Suggestion 

14from mlos_core.optimizers import BaseOptimizer, OptimizerType 

15from mlos_core.tests import SEED 

16 

17_LOG = logging.getLogger(__name__) 

18 

19 

20@pytest.mark.parametrize( 

21 ("optimizer_class", "kwargs"), 

22 [ 

23 *[(member.value, {}) for member in OptimizerType], 

24 ], 

25) 

26def test_multi_target_opt_wrong_weights( 

27 optimizer_class: type[BaseOptimizer], 

28 kwargs: dict, 

29) -> None: 

30 """Make sure that the optimizer raises an error if the number of objective weights 

31 does not match the number of optimization targets. 

32 """ 

33 with pytest.raises(ValueError): 

34 optimizer_class( 

35 parameter_space=CS.ConfigurationSpace(seed=SEED), 

36 optimization_targets=["main_score", "other_score"], 

37 objective_weights=[1], 

38 **kwargs, 

39 ) 

40 

41 

42@pytest.mark.parametrize( 

43 ("objective_weights"), 

44 [ 

45 [2, 1], 

46 [0.5, 0.5], 

47 None, 

48 ], 

49) 

50@pytest.mark.parametrize( 

51 ("optimizer_class", "kwargs"), 

52 [ 

53 *[(member.value, {}) for member in OptimizerType], 

54 ], 

55) 

56def test_multi_target_opt( 

57 objective_weights: list[float] | None, 

58 optimizer_class: type[BaseOptimizer], 

59 kwargs: dict, 

60) -> None: 

61 """Toy multi-target optimization problem to test the optimizers with mixed numeric 

62 types to ensure that original dtypes are retained. 

63 """ 

64 # pylint: disable=too-many-locals 

65 max_iterations = 10 

66 

67 def objective(point: pd.Series) -> pd.Series: 

68 # mix of hyperparameters, optimal is to select the highest possible 

69 ret: pd.Series = pd.Series( 

70 { 

71 "main_score": point.x + point.y, 

72 "other_score": point.x**2 + point.y**2, 

73 } 

74 ) 

75 return ret 

76 

77 input_space = CS.ConfigurationSpace(seed=SEED) 

78 # add a mix of numeric datatypes 

79 input_space.add(CS.UniformIntegerHyperparameter(name="x", lower=0, upper=5)) 

80 input_space.add(CS.UniformFloatHyperparameter(name="y", lower=0.0, upper=5.0)) 

81 

82 optimizer = optimizer_class( 

83 parameter_space=input_space, 

84 optimization_targets=["main_score", "other_score"], 

85 objective_weights=objective_weights, 

86 **kwargs, 

87 ) 

88 

89 with pytest.raises(ValueError, match="No observations"): 

90 optimizer.get_best_observations() 

91 

92 with pytest.raises(ValueError, match="No observations"): 

93 optimizer.get_observations() 

94 

95 for _ in range(max_iterations): 

96 suggestion = optimizer.suggest() 

97 assert isinstance(suggestion, Suggestion) 

98 assert isinstance(suggestion.config, pd.Series) 

99 assert suggestion.metadata is None or isinstance(suggestion.metadata, pd.Series) 

100 assert set(suggestion.config.index) == {"x", "y"} 

101 # Check suggestion values are the expected dtype 

102 assert isinstance(suggestion.config["x"], int) 

103 assert isinstance(suggestion.config["y"], float) 

104 # Check that suggestion is in the space 

105 config_dict: dict = suggestion.config.to_dict() 

106 test_configuration = CS.Configuration(optimizer.parameter_space, config_dict) 

107 # Raises an error if outside of configuration space 

108 test_configuration.check_valid_configuration() 

109 # Test registering the suggested configuration with a score. 

110 observation = objective(suggestion.config) 

111 assert isinstance(observation, pd.Series) 

112 assert set(observation.index) == {"main_score", "other_score"} 

113 optimizer.register(observations=suggestion.complete(observation)) 

114 

115 best_observations = optimizer.get_best_observations() 

116 assert isinstance(best_observations, Observations) 

117 assert isinstance(best_observations.configs, pd.DataFrame) 

118 assert isinstance(best_observations.scores, pd.DataFrame) 

119 assert best_observations.contexts is None 

120 assert set(best_observations.configs.columns) == {"x", "y"} 

121 assert set(best_observations.scores.columns) == {"main_score", "other_score"} 

122 assert best_observations.configs.shape == (1, 2) 

123 assert best_observations.scores.shape == (1, 2) 

124 

125 all_observations = optimizer.get_observations() 

126 assert isinstance(all_observations, Observations) 

127 assert isinstance(all_observations.configs, pd.DataFrame) 

128 assert isinstance(all_observations.scores, pd.DataFrame) 

129 assert all_observations.contexts is None 

130 assert set(all_observations.configs.columns) == {"x", "y"} 

131 assert set(all_observations.scores.columns) == {"main_score", "other_score"} 

132 assert all_observations.configs.shape == (max_iterations, 2) 

133 assert all_observations.scores.shape == (max_iterations, 2)