Coverage for mlos_bench/mlos_bench/tests/config/cli/test_load_cli_config_examples.py: 97%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5"""Tests for loading storage config examples.""" 

6 

7import logging 

8from importlib.resources import files 

9 

10import pytest 

11 

12from mlos_bench.config.schemas import ConfigSchema 

13from mlos_bench.environments import Environment 

14from mlos_bench.launcher import Launcher 

15from mlos_bench.optimizers import Optimizer 

16from mlos_bench.schedulers import Scheduler 

17from mlos_bench.services.config_persistence import ConfigPersistenceService 

18from mlos_bench.storage import Storage 

19from mlos_bench.tests import check_class_name 

20from mlos_bench.tests.config import BUILTIN_TEST_CONFIG_PATH, locate_config_examples 

21from mlos_bench.util import path_join 

22 

23_LOG = logging.getLogger(__name__) 

24_LOG.setLevel(logging.DEBUG) 

25 

26 

27# Get the set of configs to test. 

28CONFIG_TYPE = "cli" 

29 

30 

31def filter_configs(configs_to_filter: list[str]) -> list[str]: 

32 """If necessary, filter out json files that aren't for the module we're testing.""" 

33 return configs_to_filter 

34 

35 

36configs = [ 

37 *locate_config_examples( 

38 ConfigPersistenceService.BUILTIN_CONFIG_PATH, 

39 CONFIG_TYPE, 

40 filter_configs, 

41 ), 

42 *locate_config_examples( 

43 BUILTIN_TEST_CONFIG_PATH, 

44 CONFIG_TYPE, 

45 filter_configs, 

46 ), 

47] 

48assert configs 

49 

50 

51@pytest.mark.skip(reason="Use full Launcher test (below) instead now.") 

52@pytest.mark.parametrize("config_path", configs) 

53def test_load_cli_config_examples( 

54 config_loader_service: ConfigPersistenceService, 

55 config_path: str, 

56) -> None: # pragma: no cover 

57 """Tests loading a config example.""" 

58 # pylint: disable=too-complex 

59 config = config_loader_service.load_config(config_path, ConfigSchema.CLI) 

60 assert isinstance(config, dict) 

61 

62 if config_paths := config.get("config_path"): 

63 assert isinstance(config_paths, list) 

64 config_paths.reverse() 

65 for path in config_paths: 

66 config_loader_service._config_path.insert(0, path) # pylint: disable=protected-access 

67 

68 # Foreach arg that references another file, see if we can at least load that too. 

69 args_to_skip = { 

70 "config_path", # handled above 

71 "log_file", 

72 "log_level", 

73 "experiment_id", 

74 "trial_id", 

75 "teardown", 

76 } 

77 for arg in config: 

78 if arg in args_to_skip: 

79 continue 

80 

81 if arg == "globals": 

82 for path in config[arg]: 

83 sub_config = config_loader_service.load_config(path, ConfigSchema.GLOBALS) 

84 assert isinstance(sub_config, dict) 

85 elif arg == "environment": 

86 sub_config = config_loader_service.load_config(config[arg], ConfigSchema.ENVIRONMENT) 

87 assert isinstance(sub_config, dict) 

88 elif arg == "optimizer": 

89 sub_config = config_loader_service.load_config(config[arg], ConfigSchema.OPTIMIZER) 

90 assert isinstance(sub_config, dict) 

91 elif arg == "storage": 

92 sub_config = config_loader_service.load_config(config[arg], ConfigSchema.STORAGE) 

93 assert isinstance(sub_config, dict) 

94 elif arg == "tunable_values": 

95 for path in config[arg]: 

96 sub_config = config_loader_service.load_config(path, ConfigSchema.TUNABLE_VALUES) 

97 assert isinstance(sub_config, dict) 

98 else: 

99 raise NotImplementedError(f"Unhandled arg {arg} in config {config_path}") 

100 

101 

102@pytest.mark.parametrize("config_path", configs) 

103def test_load_cli_config_examples_via_launcher( 

104 config_loader_service: ConfigPersistenceService, 

105 config_path: str, 

106) -> None: 

107 """Tests loading a config example via the Launcher.""" 

108 config = config_loader_service.load_config(config_path, ConfigSchema.CLI) 

109 assert isinstance(config, dict) 

110 

111 # Try to load the CLI config by instantiating a launcher. 

112 # To do this we need to make sure to give it a few extra paths and globals 

113 # to look for for our examples. 

114 cli_args = ( 

115 # pylint: disable=inconsistent-quotes 

116 f"--config {config_path}" 

117 f" --config-path {files('mlos_bench.config')} " 

118 f" --config-path {files('mlos_bench.tests.config')}" 

119 f" --config-path {path_join(str(files('mlos_bench.tests.config')), 'globals')}" 

120 f" --globals {files('mlos_bench.tests.config')}/experiments/experiment_test_config.jsonc" 

121 ) 

122 launcher = Launcher(description=__name__, long_text=config_path, argv=cli_args.split()) 

123 assert launcher 

124 

125 # Check that some parts of that config are loaded. 

126 

127 assert ConfigPersistenceService.BUILTIN_CONFIG_PATH in launcher.config_loader.config_paths 

128 if config_paths := config.get("config_path"): 

129 assert isinstance(config_paths, list) 

130 for path in config_paths: 

131 # Note: Checks that the order is maintained are handled in launcher_parse_args.py 

132 assert any( 

133 config_path.endswith(path) for config_path in launcher.config_loader.config_paths 

134 ), f"Expected {path} to be in {launcher.config_loader.config_paths}" 

135 

136 if "experiment_id" in config: 

137 assert launcher.global_config["experiment_id"] == config["experiment_id"] 

138 if "trial_id" in config: 

139 assert launcher.global_config["trial_id"] == config["trial_id"] 

140 

141 expected_log_level = logging.getLevelName(config.get("log_level", "INFO")) 

142 if isinstance(expected_log_level, int): # type: ignore[unreachable] 

143 expected_log_level = logging.getLevelName(expected_log_level) # type: ignore[unreachable] 

144 current_log_level = logging.getLevelName(logging.root.getEffectiveLevel()) 

145 assert current_log_level == expected_log_level 

146 

147 # TODO: Check that the log_file handler is set correctly. 

148 

149 expected_teardown = config.get("teardown", True) 

150 assert launcher.teardown == expected_teardown 

151 

152 # Note: Testing of "globals" processing handled in launcher_parse_args_test.py 

153 

154 # Instead of just checking that the config is loaded, check that the 

155 # Launcher loaded the expected types as well. 

156 

157 assert isinstance(launcher.root_environment, Environment) 

158 env_config = launcher.config_loader.load_config( 

159 config["environment"], 

160 ConfigSchema.ENVIRONMENT, 

161 ) 

162 assert check_class_name(launcher.root_environment, env_config["class"]) 

163 

164 assert isinstance(launcher.optimizer, Optimizer) 

165 if "optimizer" in config: 

166 opt_config = launcher.config_loader.load_config( 

167 config["optimizer"], 

168 ConfigSchema.OPTIMIZER, 

169 ) 

170 assert check_class_name(launcher.optimizer, opt_config["class"]) 

171 

172 assert isinstance(launcher.storage, Storage) 

173 if "storage" in config: 

174 storage_config = launcher.config_loader.load_config( 

175 config["storage"], 

176 ConfigSchema.STORAGE, 

177 ) 

178 assert check_class_name(launcher.storage, storage_config["class"]) 

179 

180 assert isinstance(launcher.scheduler, Scheduler) 

181 if "scheduler" in config: 

182 scheduler_config = launcher.config_loader.load_config( 

183 config["scheduler"], 

184 ConfigSchema.SCHEDULER, 

185 ) 

186 assert check_class_name(launcher.scheduler, scheduler_config["class"]) 

187 

188 # TODO: Check that the launcher assigns the tunables values as expected.