Coverage for mlos_bench/mlos_bench/tests/optimizers/mlos_core_opt_df_test.py: 100%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5"""Unit tests for internal methods of the `MlosCoreOptimizer`.""" 

6 

7 

8import pandas 

9import pytest 

10 

11from mlos_bench.optimizers.mlos_core_optimizer import MlosCoreOptimizer 

12from mlos_bench.tests import SEED 

13from mlos_bench.tunables.tunable_groups import TunableGroups 

14 

15# pylint: disable=redefined-outer-name, protected-access 

16 

17 

18@pytest.fixture 

19def mlos_core_optimizer(tunable_groups: TunableGroups) -> MlosCoreOptimizer: 

20 """An instance of a mlos_core optimizer (FLAML-based).""" 

21 test_opt_config = { 

22 "optimizer_type": "FLAML", 

23 "max_suggestions": 10, 

24 "seed": SEED, 

25 "optimization_targets": { 

26 "latency": "min", 

27 "throughput": "max", 

28 }, 

29 } 

30 return MlosCoreOptimizer(tunable_groups, test_opt_config) 

31 

32 

33def test_df(mlos_core_optimizer: MlosCoreOptimizer, mock_configs: list[dict]) -> None: 

34 """Test `MlosCoreOptimizer._to_df()` method on tunables that have special values.""" 

35 df_config = mlos_core_optimizer._to_df(mock_configs) 

36 assert isinstance(df_config, pandas.DataFrame) 

37 assert df_config.shape == (4, 6) 

38 assert set(df_config.columns) == { 

39 "kernel_sched_latency_ns", 

40 "kernel_sched_migration_cost_ns", 

41 "kernel_sched_migration_cost_ns!type", 

42 "kernel_sched_migration_cost_ns!special", 

43 "idle", 

44 "vmSize", 

45 } 

46 assert df_config.to_dict(orient="records") == [ 

47 { 

48 "idle": "halt", 

49 "kernel_sched_latency_ns": 1000000, 

50 "kernel_sched_migration_cost_ns": 50000, 

51 "kernel_sched_migration_cost_ns!special": None, 

52 "kernel_sched_migration_cost_ns!type": "range", 

53 "vmSize": "Standard_B4ms", 

54 }, 

55 { 

56 "idle": "halt", 

57 "kernel_sched_latency_ns": 2000000, 

58 "kernel_sched_migration_cost_ns": 40000, 

59 "kernel_sched_migration_cost_ns!special": None, 

60 "kernel_sched_migration_cost_ns!type": "range", 

61 "vmSize": "Standard_B4ms", 

62 }, 

63 { 

64 "idle": "mwait", 

65 "kernel_sched_latency_ns": 3000000, 

66 "kernel_sched_migration_cost_ns": None, # The value is special! 

67 "kernel_sched_migration_cost_ns!special": -1, 

68 "kernel_sched_migration_cost_ns!type": "special", 

69 "vmSize": "Standard_B4ms", 

70 }, 

71 { 

72 "idle": "mwait", 

73 "kernel_sched_latency_ns": 4000000, 

74 "kernel_sched_migration_cost_ns": 200000, 

75 "kernel_sched_migration_cost_ns!special": None, 

76 "kernel_sched_migration_cost_ns!type": "range", 

77 "vmSize": "Standard_B2s", 

78 }, 

79 ] 

80 

81 

82def test_df_str(mlos_core_optimizer: MlosCoreOptimizer, mock_configs: list[dict]) -> None: 

83 """Test `MlosCoreOptimizer._to_df()` type coercion on tunables with string 

84 values. 

85 """ 

86 df_config_orig = mlos_core_optimizer._to_df(mock_configs) 

87 df_config_str = mlos_core_optimizer._to_df( 

88 [{key: str(val) for (key, val) in config.items()} for config in mock_configs] 

89 ) 

90 assert df_config_orig.equals(df_config_str) 

91 

92 

93def test_adjust_signs_df(mlos_core_optimizer: MlosCoreOptimizer) -> None: 

94 """Test `MlosCoreOptimizer._adjust_signs_df()` on different types of inputs.""" 

95 df_scores_input = pandas.DataFrame( 

96 { 

97 "latency": [88.88, 66.66, 99.99, None], 

98 "throughput": [111, 222, 333, None], 

99 } 

100 ) 

101 

102 df_scores_output = pandas.DataFrame( 

103 { 

104 "latency": [88.88, 66.66, 99.99, float("NaN")], 

105 "throughput": [-111, -222, -333, float("NaN")], 

106 } 

107 ) 

108 

109 # Make sure we adjust the signs for minimization. 

110 df_scores = mlos_core_optimizer._adjust_signs_df(df_scores_input) 

111 assert df_scores.equals(df_scores_output) 

112 

113 # Check that the same operation works for string inputs. 

114 df_scores = mlos_core_optimizer._adjust_signs_df(df_scores_input.astype(str)) 

115 assert df_scores.equals(df_scores_output) 

116 

117 

118def test_adjust_signs_df_nan(mlos_core_optimizer: MlosCoreOptimizer) -> None: 

119 """Test `MlosCoreOptimizer._adjust_signs_df()` handling None, NaN, and Inf 

120 values. 

121 """ 

122 df_scores = mlos_core_optimizer._adjust_signs_df( 

123 pandas.DataFrame( 

124 { 

125 "latency": ["88.88", "NaN", "Inf", "-Inf", None], 

126 "throughput": ["111", "NaN", "Inf", "-Inf", None], 

127 } 

128 ) 

129 ) 

130 

131 assert df_scores.equals( 

132 pandas.DataFrame( 

133 { 

134 "latency": [88.88, float("NaN"), float("Inf"), float("-Inf"), float("NaN")], 

135 "throughput": [-111, float("NaN"), float("-Inf"), float("Inf"), float("NaN")], 

136 } 

137 ) 

138 ) 

139 

140 

141def test_adjust_signs_df_invalid(mlos_core_optimizer: MlosCoreOptimizer) -> None: 

142 """Test `MlosCoreOptimizer._adjust_signs_df()` on invalid inputs.""" 

143 with pytest.raises(ValueError): 

144 mlos_core_optimizer._adjust_signs_df( 

145 pandas.DataFrame( 

146 { 

147 "latency": ["INVALID"], 

148 "throughput": ["no input"], 

149 } 

150 ) 

151 ) 

152 

153 with pytest.raises(ValueError): 

154 mlos_core_optimizer._adjust_signs_df( 

155 pandas.DataFrame( 

156 { 

157 "latency": ["88.88", ""], 

158 "throughput": ["111", ""], 

159 } 

160 ) 

161 )