Coverage for mlos_bench/mlos_bench/tests/config/cli/test_load_cli_config_examples.py: 97%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5"""Tests for loading storage config examples.""" 

6 

7import logging 

8import sys 

9from typing import List 

10 

11import pytest 

12 

13from mlos_bench.config.schemas import ConfigSchema 

14from mlos_bench.environments import Environment 

15from mlos_bench.launcher import Launcher 

16from mlos_bench.optimizers import Optimizer 

17from mlos_bench.schedulers import Scheduler 

18from mlos_bench.services.config_persistence import ConfigPersistenceService 

19from mlos_bench.storage import Storage 

20from mlos_bench.tests import check_class_name 

21from mlos_bench.tests.config import BUILTIN_TEST_CONFIG_PATH, locate_config_examples 

22from mlos_bench.util import path_join 

23 

24if sys.version_info < (3, 10): 

25 from importlib_resources import files 

26else: 

27 from importlib.resources import files 

28 

29 

30_LOG = logging.getLogger(__name__) 

31_LOG.setLevel(logging.DEBUG) 

32 

33 

34# Get the set of configs to test. 

35CONFIG_TYPE = "cli" 

36 

37 

38def filter_configs(configs_to_filter: List[str]) -> List[str]: 

39 """If necessary, filter out json files that aren't for the module we're testing.""" 

40 return configs_to_filter 

41 

42 

43configs = [ 

44 *locate_config_examples( 

45 ConfigPersistenceService.BUILTIN_CONFIG_PATH, 

46 CONFIG_TYPE, 

47 filter_configs, 

48 ), 

49 *locate_config_examples( 

50 BUILTIN_TEST_CONFIG_PATH, 

51 CONFIG_TYPE, 

52 filter_configs, 

53 ), 

54] 

55assert configs 

56 

57 

58@pytest.mark.skip(reason="Use full Launcher test (below) instead now.") 

59@pytest.mark.parametrize("config_path", configs) 

60def test_load_cli_config_examples( 

61 config_loader_service: ConfigPersistenceService, 

62 config_path: str, 

63) -> None: # pragma: no cover 

64 """Tests loading a config example.""" 

65 # pylint: disable=too-complex 

66 config = config_loader_service.load_config(config_path, ConfigSchema.CLI) 

67 assert isinstance(config, dict) 

68 

69 if config_paths := config.get("config_path"): 

70 assert isinstance(config_paths, list) 

71 config_paths.reverse() 

72 for path in config_paths: 

73 config_loader_service._config_path.insert(0, path) # pylint: disable=protected-access 

74 

75 # Foreach arg that references another file, see if we can at least load that too. 

76 args_to_skip = { 

77 "config_path", # handled above 

78 "log_file", 

79 "log_level", 

80 "experiment_id", 

81 "trial_id", 

82 "teardown", 

83 } 

84 for arg in config: 

85 if arg in args_to_skip: 

86 continue 

87 

88 if arg == "globals": 

89 for path in config[arg]: 

90 sub_config = config_loader_service.load_config(path, ConfigSchema.GLOBALS) 

91 assert isinstance(sub_config, dict) 

92 elif arg == "environment": 

93 sub_config = config_loader_service.load_config(config[arg], ConfigSchema.ENVIRONMENT) 

94 assert isinstance(sub_config, dict) 

95 elif arg == "optimizer": 

96 sub_config = config_loader_service.load_config(config[arg], ConfigSchema.OPTIMIZER) 

97 assert isinstance(sub_config, dict) 

98 elif arg == "storage": 

99 sub_config = config_loader_service.load_config(config[arg], ConfigSchema.STORAGE) 

100 assert isinstance(sub_config, dict) 

101 elif arg == "tunable_values": 

102 for path in config[arg]: 

103 sub_config = config_loader_service.load_config(path, ConfigSchema.TUNABLE_VALUES) 

104 assert isinstance(sub_config, dict) 

105 else: 

106 raise NotImplementedError(f"Unhandled arg {arg} in config {config_path}") 

107 

108 

109@pytest.mark.parametrize("config_path", configs) 

110def test_load_cli_config_examples_via_launcher( 

111 config_loader_service: ConfigPersistenceService, 

112 config_path: str, 

113) -> None: 

114 """Tests loading a config example via the Launcher.""" 

115 config = config_loader_service.load_config(config_path, ConfigSchema.CLI) 

116 assert isinstance(config, dict) 

117 

118 # Try to load the CLI config by instantiating a launcher. 

119 # To do this we need to make sure to give it a few extra paths and globals 

120 # to look for for our examples. 

121 cli_args = ( 

122 f"--config {config_path}" 

123 f" --config-path {files('mlos_bench.config')} " 

124 f" --config-path {files('mlos_bench.tests.config')}" 

125 f" --config-path {path_join(str(files('mlos_bench.tests.config')), 'globals')}" 

126 f" --globals {files('mlos_bench.tests.config')}/experiments/experiment_test_config.jsonc" 

127 ) 

128 launcher = Launcher(description=__name__, long_text=config_path, argv=cli_args.split()) 

129 assert launcher 

130 

131 # Check that some parts of that config are loaded. 

132 

133 assert ConfigPersistenceService.BUILTIN_CONFIG_PATH in launcher.config_loader.config_paths 

134 if config_paths := config.get("config_path"): 

135 assert isinstance(config_paths, list) 

136 for path in config_paths: 

137 # Note: Checks that the order is maintained are handled in launcher_parse_args.py 

138 assert any( 

139 config_path.endswith(path) for config_path in launcher.config_loader.config_paths 

140 ), f"Expected {path} to be in {launcher.config_loader.config_paths}" 

141 

142 if "experiment_id" in config: 

143 assert launcher.global_config["experiment_id"] == config["experiment_id"] 

144 if "trial_id" in config: 

145 assert launcher.global_config["trial_id"] == config["trial_id"] 

146 

147 expected_log_level = logging.getLevelName(config.get("log_level", "INFO")) 

148 if isinstance(expected_log_level, int): 

149 expected_log_level = logging.getLevelName(expected_log_level) 

150 current_log_level = logging.getLevelName(logging.root.getEffectiveLevel()) 

151 assert current_log_level == expected_log_level 

152 

153 # TODO: Check that the log_file handler is set correctly. 

154 

155 expected_teardown = config.get("teardown", True) 

156 assert launcher.teardown == expected_teardown 

157 

158 # Note: Testing of "globals" processing handled in launcher_parse_args_test.py 

159 

160 # Instead of just checking that the config is loaded, check that the 

161 # Launcher loaded the expected types as well. 

162 

163 assert isinstance(launcher.environment, Environment) 

164 env_config = launcher.config_loader.load_config( 

165 config["environment"], 

166 ConfigSchema.ENVIRONMENT, 

167 ) 

168 assert check_class_name(launcher.environment, env_config["class"]) 

169 

170 assert isinstance(launcher.optimizer, Optimizer) 

171 if "optimizer" in config: 

172 opt_config = launcher.config_loader.load_config( 

173 config["optimizer"], 

174 ConfigSchema.OPTIMIZER, 

175 ) 

176 assert check_class_name(launcher.optimizer, opt_config["class"]) 

177 

178 assert isinstance(launcher.storage, Storage) 

179 if "storage" in config: 

180 storage_config = launcher.config_loader.load_config( 

181 config["storage"], 

182 ConfigSchema.STORAGE, 

183 ) 

184 assert check_class_name(launcher.storage, storage_config["class"]) 

185 

186 assert isinstance(launcher.scheduler, Scheduler) 

187 if "scheduler" in config: 

188 scheduler_config = launcher.config_loader.load_config( 

189 config["scheduler"], 

190 ConfigSchema.SCHEDULER, 

191 ) 

192 assert check_class_name(launcher.scheduler, scheduler_config["class"]) 

193 

194 # TODO: Check that the launcher assigns the tunables values as expected.