Coverage for mlos_bench/mlos_bench/tests/optimizers/mlos_core_opt_smac_test.py: 98%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5"""Unit tests for mock mlos_bench optimizer.""" 

6import os 

7import shutil 

8import sys 

9 

10import pytest 

11 

12from mlos_bench.optimizers.mlos_core_optimizer import MlosCoreOptimizer 

13from mlos_bench.tests import SEED 

14from mlos_bench.tunables.tunable_groups import TunableGroups 

15from mlos_bench.util import path_join 

16from mlos_core.optimizers.bayesian_optimizers.smac_optimizer import SmacOptimizer 

17 

18_OUTPUT_DIR_PATH_BASE = r"c:/temp" if sys.platform == "win32" else "/tmp/" 

19_OUTPUT_DIR = "_test_output_dir" # Will be deleted after the test. 

20 

21 

22def test_init_mlos_core_smac_opt_bad_trial_count(tunable_groups: TunableGroups) -> None: 

23 """Test invalid max_trials initialization of mlos_core SMAC optimizer.""" 

24 test_opt_config = { 

25 "optimizer_type": "SMAC", 

26 "max_trials": 10, 

27 "max_suggestions": 11, 

28 "seed": SEED, 

29 } 

30 with pytest.raises(AssertionError): 

31 opt = MlosCoreOptimizer(tunable_groups, test_opt_config) 

32 assert opt is None 

33 

34 

35def test_init_mlos_core_smac_opt_max_trials(tunable_groups: TunableGroups) -> None: 

36 """Test max_trials initialization of mlos_core SMAC optimizer.""" 

37 test_opt_config = { 

38 "optimizer_type": "SMAC", 

39 "max_suggestions": 123, 

40 "seed": SEED, 

41 } 

42 opt = MlosCoreOptimizer(tunable_groups, test_opt_config) 

43 # pylint: disable=protected-access 

44 assert isinstance(opt._opt, SmacOptimizer) 

45 assert opt._opt.base_optimizer.scenario.n_trials == test_opt_config["max_suggestions"] 

46 

47 

48def test_init_mlos_core_smac_absolute_output_directory(tunable_groups: TunableGroups) -> None: 

49 """Test absolute path output directory initialization of mlos_core SMAC 

50 optimizer. 

51 """ 

52 output_dir = path_join(_OUTPUT_DIR_PATH_BASE, _OUTPUT_DIR) 

53 test_opt_config = { 

54 "optimizer_type": "SMAC", 

55 "output_directory": output_dir, 

56 "seed": SEED, 

57 } 

58 opt = MlosCoreOptimizer(tunable_groups, test_opt_config) 

59 assert isinstance(opt, MlosCoreOptimizer) 

60 # pylint: disable=protected-access 

61 assert isinstance(opt._opt, SmacOptimizer) 

62 # Final portions of the path are generated by SMAC when run_name is not specified. 

63 assert path_join(str(opt._opt.base_optimizer.scenario.output_directory)).startswith( 

64 str(test_opt_config["output_directory"]) 

65 ) 

66 shutil.rmtree(output_dir) 

67 

68 

69def test_init_mlos_core_smac_relative_output_directory(tunable_groups: TunableGroups) -> None: 

70 """Test relative path output directory initialization of mlos_core SMAC 

71 optimizer. 

72 """ 

73 test_opt_config = { 

74 "optimizer_type": "SMAC", 

75 "output_directory": _OUTPUT_DIR, 

76 "seed": SEED, 

77 } 

78 opt = MlosCoreOptimizer(tunable_groups, test_opt_config) 

79 assert isinstance(opt, MlosCoreOptimizer) 

80 # pylint: disable=protected-access 

81 assert isinstance(opt._opt, SmacOptimizer) 

82 assert path_join(str(opt._opt.base_optimizer.scenario.output_directory)).startswith( 

83 path_join(os.getcwd(), str(test_opt_config["output_directory"])) 

84 ) 

85 shutil.rmtree(_OUTPUT_DIR) 

86 

87 

88def test_init_mlos_core_smac_relative_output_directory_with_run_name( 

89 tunable_groups: TunableGroups, 

90) -> None: 

91 """Test relative path output directory initialization of mlos_core SMAC 

92 optimizer. 

93 """ 

94 test_opt_config = { 

95 "optimizer_type": "SMAC", 

96 "output_directory": _OUTPUT_DIR, 

97 "run_name": "test_run", 

98 "seed": SEED, 

99 } 

100 opt = MlosCoreOptimizer(tunable_groups, test_opt_config) 

101 assert isinstance(opt, MlosCoreOptimizer) 

102 # pylint: disable=protected-access 

103 assert isinstance(opt._opt, SmacOptimizer) 

104 assert path_join(str(opt._opt.base_optimizer.scenario.output_directory)).startswith( 

105 path_join( 

106 os.getcwd(), str(test_opt_config["output_directory"]), str(test_opt_config["run_name"]) 

107 ) 

108 ) 

109 shutil.rmtree(_OUTPUT_DIR) 

110 

111 

112def test_init_mlos_core_smac_relative_output_directory_with_experiment_id( 

113 tunable_groups: TunableGroups, 

114) -> None: 

115 """Test relative path output directory initialization of mlos_core SMAC 

116 optimizer. 

117 """ 

118 test_opt_config = { 

119 "optimizer_type": "SMAC", 

120 "output_directory": _OUTPUT_DIR, 

121 "seed": SEED, 

122 } 

123 global_config = { 

124 "experiment_id": "experiment_id", 

125 } 

126 opt = MlosCoreOptimizer(tunable_groups, test_opt_config, global_config) 

127 assert isinstance(opt, MlosCoreOptimizer) 

128 # pylint: disable=protected-access 

129 assert isinstance(opt._opt, SmacOptimizer) 

130 assert path_join(str(opt._opt.base_optimizer.scenario.output_directory)).startswith( 

131 path_join( 

132 os.getcwd(), 

133 str(test_opt_config["output_directory"]), 

134 global_config["experiment_id"], 

135 ) 

136 ) 

137 shutil.rmtree(_OUTPUT_DIR) 

138 

139 

140def test_init_mlos_core_smac_temp_output_directory(tunable_groups: TunableGroups) -> None: 

141 """Test random output directory initialization of mlos_core SMAC optimizer.""" 

142 test_opt_config = { 

143 "optimizer_type": "SMAC", 

144 "output_directory": None, 

145 "seed": SEED, 

146 } 

147 opt = MlosCoreOptimizer(tunable_groups, test_opt_config) 

148 assert isinstance(opt, MlosCoreOptimizer) 

149 # pylint: disable=protected-access 

150 assert isinstance(opt._opt, SmacOptimizer) 

151 assert opt._opt.base_optimizer.scenario.output_directory is not None