Coverage for mlos_bench/mlos_bench/tests/optimizers/mlos_core_opt_df_test.py: 100%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5"""Unit tests for internal methods of the `MlosCoreOptimizer`.""" 

6 

7from typing import List 

8 

9import pandas 

10import pytest 

11 

12from mlos_bench.optimizers.mlos_core_optimizer import MlosCoreOptimizer 

13from mlos_bench.tests import SEED 

14from mlos_bench.tunables.tunable_groups import TunableGroups 

15 

16# pylint: disable=redefined-outer-name, protected-access 

17 

18 

19@pytest.fixture 

20def mlos_core_optimizer(tunable_groups: TunableGroups) -> MlosCoreOptimizer: 

21 """An instance of a mlos_core optimizer (FLAML-based).""" 

22 test_opt_config = { 

23 "optimizer_type": "FLAML", 

24 "max_suggestions": 10, 

25 "seed": SEED, 

26 "optimization_targets": { 

27 "latency": "min", 

28 "throughput": "max", 

29 }, 

30 } 

31 return MlosCoreOptimizer(tunable_groups, test_opt_config) 

32 

33 

34def test_df(mlos_core_optimizer: MlosCoreOptimizer, mock_configs: List[dict]) -> None: 

35 """Test `MlosCoreOptimizer._to_df()` method on tunables that have special values.""" 

36 df_config = mlos_core_optimizer._to_df(mock_configs) 

37 assert isinstance(df_config, pandas.DataFrame) 

38 assert df_config.shape == (4, 6) 

39 assert set(df_config.columns) == { 

40 "kernel_sched_latency_ns", 

41 "kernel_sched_migration_cost_ns", 

42 "kernel_sched_migration_cost_ns!type", 

43 "kernel_sched_migration_cost_ns!special", 

44 "idle", 

45 "vmSize", 

46 } 

47 assert df_config.to_dict(orient="records") == [ 

48 { 

49 "idle": "halt", 

50 "kernel_sched_latency_ns": 1000000, 

51 "kernel_sched_migration_cost_ns": 50000, 

52 "kernel_sched_migration_cost_ns!special": None, 

53 "kernel_sched_migration_cost_ns!type": "range", 

54 "vmSize": "Standard_B4ms", 

55 }, 

56 { 

57 "idle": "halt", 

58 "kernel_sched_latency_ns": 2000000, 

59 "kernel_sched_migration_cost_ns": 40000, 

60 "kernel_sched_migration_cost_ns!special": None, 

61 "kernel_sched_migration_cost_ns!type": "range", 

62 "vmSize": "Standard_B4ms", 

63 }, 

64 { 

65 "idle": "mwait", 

66 "kernel_sched_latency_ns": 3000000, 

67 "kernel_sched_migration_cost_ns": None, # The value is special! 

68 "kernel_sched_migration_cost_ns!special": -1, 

69 "kernel_sched_migration_cost_ns!type": "special", 

70 "vmSize": "Standard_B4ms", 

71 }, 

72 { 

73 "idle": "mwait", 

74 "kernel_sched_latency_ns": 4000000, 

75 "kernel_sched_migration_cost_ns": 200000, 

76 "kernel_sched_migration_cost_ns!special": None, 

77 "kernel_sched_migration_cost_ns!type": "range", 

78 "vmSize": "Standard_B2s", 

79 }, 

80 ] 

81 

82 

83def test_df_str(mlos_core_optimizer: MlosCoreOptimizer, mock_configs: List[dict]) -> None: 

84 """Test `MlosCoreOptimizer._to_df()` type coercion on tunables with string 

85 values. 

86 """ 

87 df_config_orig = mlos_core_optimizer._to_df(mock_configs) 

88 df_config_str = mlos_core_optimizer._to_df( 

89 [{key: str(val) for (key, val) in config.items()} for config in mock_configs] 

90 ) 

91 assert df_config_orig.equals(df_config_str) 

92 

93 

94def test_adjust_signs_df(mlos_core_optimizer: MlosCoreOptimizer) -> None: 

95 """Test `MlosCoreOptimizer._adjust_signs_df()` on different types of inputs.""" 

96 df_scores_input = pandas.DataFrame( 

97 { 

98 "latency": [88.88, 66.66, 99.99, None], 

99 "throughput": [111, 222, 333, None], 

100 } 

101 ) 

102 

103 df_scores_output = pandas.DataFrame( 

104 { 

105 "latency": [88.88, 66.66, 99.99, float("NaN")], 

106 "throughput": [-111, -222, -333, float("NaN")], 

107 } 

108 ) 

109 

110 # Make sure we adjust the signs for minimization. 

111 df_scores = mlos_core_optimizer._adjust_signs_df(df_scores_input) 

112 assert df_scores.equals(df_scores_output) 

113 

114 # Check that the same operation works for string inputs. 

115 df_scores = mlos_core_optimizer._adjust_signs_df(df_scores_input.astype(str)) 

116 assert df_scores.equals(df_scores_output) 

117 

118 

119def test_adjust_signs_df_nan(mlos_core_optimizer: MlosCoreOptimizer) -> None: 

120 """Test `MlosCoreOptimizer._adjust_signs_df()` handling None, NaN, and Inf 

121 values. 

122 """ 

123 df_scores = mlos_core_optimizer._adjust_signs_df( 

124 pandas.DataFrame( 

125 { 

126 "latency": ["88.88", "NaN", "Inf", "-Inf", None], 

127 "throughput": ["111", "NaN", "Inf", "-Inf", None], 

128 } 

129 ) 

130 ) 

131 

132 assert df_scores.equals( 

133 pandas.DataFrame( 

134 { 

135 "latency": [88.88, float("NaN"), float("Inf"), float("-Inf"), float("NaN")], 

136 "throughput": [-111, float("NaN"), float("-Inf"), float("Inf"), float("NaN")], 

137 } 

138 ) 

139 ) 

140 

141 

142def test_adjust_signs_df_invalid(mlos_core_optimizer: MlosCoreOptimizer) -> None: 

143 """Test `MlosCoreOptimizer._adjust_signs_df()` on invalid inputs.""" 

144 with pytest.raises(ValueError): 

145 mlos_core_optimizer._adjust_signs_df( 

146 pandas.DataFrame( 

147 { 

148 "latency": ["INVALID"], 

149 "throughput": ["no input"], 

150 } 

151 ) 

152 ) 

153 

154 with pytest.raises(ValueError): 

155 mlos_core_optimizer._adjust_signs_df( 

156 pandas.DataFrame( 

157 { 

158 "latency": ["88.88", ""], 

159 "throughput": ["111", ""], 

160 } 

161 ) 

162 )