Coverage for mlos_bench/mlos_bench/tests/launcher_parse_args_test.py: 99%
111 statements
« prev ^ index » next coverage.py v7.6.10, created at 2025-01-21 01:50 +0000
1#
2# Copyright (c) Microsoft Corporation.
3# Licensed under the MIT License.
4#
5"""
6Unit tests to check the Launcher CLI arg parsing
7See Also: test_load_cli_config_examples.py
8"""
10import os
11import sys
12from getpass import getuser
13from importlib.resources import files
15import pytest
17from mlos_bench.config.schemas import ConfigSchema
18from mlos_bench.launcher import Launcher
19from mlos_bench.optimizers import MlosCoreOptimizer, OneShotOptimizer
20from mlos_bench.os_environ import environ
21from mlos_bench.schedulers import SyncScheduler
22from mlos_bench.services.types import (
23 SupportsAuth,
24 SupportsConfigLoading,
25 SupportsFileShareOps,
26 SupportsLocalExec,
27 SupportsRemoteExec,
28)
29from mlos_bench.tests import check_class_name
30from mlos_bench.util import path_join
32# pylint: disable=redefined-outer-name
35@pytest.fixture
36def config_paths() -> list[str]:
37 """
38 Returns a list of config paths.
40 Returns
41 -------
42 list[str]
43 """
44 return [
45 path_join(os.getcwd(), abs_path=True),
46 str(files("mlos_bench.config")),
47 str(files("mlos_bench.tests.config")),
48 ]
51# This is part of the minimal required args by the Launcher.
52ENV_CONF_PATH = "environments/mock/mock_env.jsonc"
55def _get_launcher(desc: str, cli_args: str) -> Launcher:
56 # The VSCode pytest wrapper actually starts in a different directory before
57 # changing into the code directory, but doesn't update the PWD environment
58 # variable so we use a separate variable.
59 # See global_test_config.jsonc for more details.
60 environ["CUSTOM_PATH_FROM_ENV"] = os.getcwd()
61 if sys.platform == "win32":
62 # Some env tweaks for platform compatibility.
63 environ["USER"] = environ["USERNAME"]
64 launcher = Launcher(description=desc, argv=cli_args.split())
65 # Check the basic parent service
66 assert isinstance(launcher.service, SupportsConfigLoading) # built-in
67 assert isinstance(launcher.service, SupportsLocalExec) # built-in
68 # All trial runners should have the same Environment class.
69 assert (
70 len({trial_runner.environment.__class__ for trial_runner in launcher.trial_runners}) == 1
71 )
72 expected_trial_runner_ids = set(range(1, len(launcher.trial_runners) + 1))
73 # Make sure that each trial runner has a unique ID.
74 assert {
75 trial_runner.trial_runner_id for trial_runner in launcher.trial_runners
76 } == expected_trial_runner_ids, "Need unique trial_runner_id in TrialRunners"
77 # Make sure that each trial runner environment has a unique ID.
78 assert {
79 trial_runner.environment.const_args["trial_runner_id"]
80 for trial_runner in launcher.trial_runners
81 } == expected_trial_runner_ids, "Need unique trial_runner_id in Environments"
82 # Make sure that each trial runner and environment trial_runner_id match.
83 assert all(
84 trial_runner.trial_runner_id == trial_runner.environment.const_args["trial_runner_id"]
85 for trial_runner in launcher.trial_runners
86 ), "TrialRunner and Environment trial_runner_id mismatch"
88 return launcher
91def test_launcher_args_parse_defaults(config_paths: list[str]) -> None:
92 """Test that we get the defaults we expect when using minimal config arg
93 examples.
94 """
95 cli_args = (
96 "--config-paths "
97 + " ".join(config_paths)
98 + f" --environment {ENV_CONF_PATH}"
99 + " --globals globals/global_test_config.jsonc"
100 )
101 launcher = _get_launcher(__name__, cli_args)
102 # Check that the first --globals file is loaded and $var expansion is handled.
103 assert launcher.global_config["experiment_id"] == "MockExperiment"
104 assert launcher.global_config["testVmName"] == "MockExperiment-vm"
105 # Check that secondary expansion also works.
106 assert launcher.global_config["testVnetName"] == "MockExperiment-vm-vnet"
107 # Check that we can expand a $var in a config file that references an environment variable.
108 assert path_join(launcher.global_config["pathVarWithEnvVarRef"], abs_path=True) == path_join(
109 os.getcwd(), "foo", abs_path=True
110 )
111 assert launcher.global_config["varWithEnvVarRef"] == f"user:{getuser()}"
112 assert launcher.teardown # defaults
113 # Make sure we have the right number of trial runners.
114 assert len(launcher.trial_runners) == 1 # defaults
115 # Check that the environment that got loaded looks to be of the right type.
116 env_config = launcher.config_loader.load_config(ENV_CONF_PATH, ConfigSchema.ENVIRONMENT)
117 assert env_config["class"] == "mlos_bench.environments.mock_env.MockEnv"
118 # All TrialRunners should get the same Environment.
119 assert all(
120 check_class_name(trial_runner.environment, env_config["class"])
121 for trial_runner in launcher.trial_runners
122 )
123 # Check that the optimizer looks right.
124 assert isinstance(launcher.optimizer, OneShotOptimizer)
125 # Check that the optimizer got initialized with defaults.
126 assert launcher.optimizer.tunable_params.is_defaults()
127 assert launcher.optimizer.max_suggestions == 1 # value for OneShotOptimizer
128 # Check that we pick up the right scheduler config:
129 assert isinstance(launcher.scheduler, SyncScheduler)
130 assert launcher.scheduler.trial_config_repeat_count == 1 # default
131 assert launcher.scheduler.max_trials == -1 # default
134def test_launcher_args_parse_1(config_paths: list[str]) -> None:
135 """
136 Test that using multiple --globals arguments works and that multiple space separated
137 options to --config-paths works.
139 Check $var expansion and Environment loading.
140 """
141 # Here we have multiple paths following --config-paths and --service.
142 cli_args = (
143 "--config-paths "
144 + " ".join(config_paths)
145 + " --num-trial-runners 5"
146 + " --service services/remote/mock/mock_auth_service.jsonc"
147 " services/remote/mock/mock_remote_exec_service.jsonc"
148 " --scheduler schedulers/sync_scheduler.jsonc"
149 f" --environment {ENV_CONF_PATH}"
150 " --globals globals/global_test_config.jsonc"
151 " --globals globals/global_test_extra_config.jsonc"
152 " --test_global_value_2 from-args"
153 )
154 launcher = _get_launcher(__name__, cli_args)
155 # Check some additional features of the the parent service
156 assert isinstance(launcher.service, SupportsAuth) # from --service
157 assert isinstance(launcher.service, SupportsRemoteExec) # from --service
158 # Check that the first --globals file is loaded and $var expansion is handled.
159 assert launcher.global_config["experiment_id"] == "MockExperiment"
160 assert launcher.global_config["testVmName"] == "MockExperiment-vm"
161 # Check that secondary expansion also works.
162 assert launcher.global_config["testVnetName"] == "MockExperiment-vm-vnet"
163 # Check that the second --globals file is loaded.
164 assert launcher.global_config["test_global_value"] == "from-file"
165 # Check overriding values in a file from the command line.
166 assert launcher.global_config["test_global_value_2"] == "from-args"
167 # Check that we can expand a $var in a config file that references an environment variable.
168 assert path_join(launcher.global_config["pathVarWithEnvVarRef"], abs_path=True) == path_join(
169 os.getcwd(), "foo", abs_path=True
170 )
171 assert launcher.global_config["varWithEnvVarRef"] == f"user:{getuser()}"
172 assert launcher.teardown
173 # Make sure we have the right number of trial runners.
174 assert len(launcher.trial_runners) == 5 # from cli args
175 # Check that the environment that got loaded looks to be of the right type.
176 env_config = launcher.config_loader.load_config(ENV_CONF_PATH, ConfigSchema.ENVIRONMENT)
177 assert env_config["class"] == "mlos_bench.environments.mock_env.MockEnv"
178 # All TrialRunners should get the same Environment.
179 assert all(
180 check_class_name(trial_runner.environment, env_config["class"])
181 for trial_runner in launcher.trial_runners
182 )
183 # Check that the optimizer looks right.
184 assert isinstance(launcher.optimizer, OneShotOptimizer)
185 # Check that the optimizer got initialized with defaults.
186 assert launcher.optimizer.tunable_params.is_defaults()
187 assert launcher.optimizer.max_suggestions == 1 # value for OneShotOptimizer
188 # Check that we pick up the right scheduler config:
189 assert isinstance(launcher.scheduler, SyncScheduler)
190 assert (
191 launcher.scheduler.trial_config_repeat_count == 3
192 ) # from the custom sync_scheduler.jsonc config
193 assert launcher.scheduler.max_trials == -1
196def test_launcher_args_parse_2(config_paths: list[str]) -> None:
197 """Test multiple --config-path instances, --config file vs --arg, --var=val
198 overrides, $var templates, option args, --random-init, etc.
199 """
200 config_file = "cli/test-cli-config.jsonc"
201 globals_file = "globals/global_test_config.jsonc"
202 # Here we have multiple --config-path and --service args, each with their own path.
203 cli_args = (
204 " ".join([f"--config-path {config_path}" for config_path in config_paths])
205 + f" --config {config_file}"
206 " --service services/remote/mock/mock_auth_service.jsonc"
207 " --service services/remote/mock/mock_remote_exec_service.jsonc"
208 f" --globals {globals_file}"
209 " --experiment_id MockeryExperiment"
210 " --no-teardown"
211 " --random-init"
212 " --random-seed 1234"
213 " --trial-config-repeat-count 5"
214 " --max_trials 200"
215 )
216 launcher = _get_launcher(__name__, cli_args)
217 # Check some additional features of the the parent service
218 assert isinstance(launcher.service, SupportsAuth) # from --service
219 assert isinstance(launcher.service, SupportsFileShareOps) # from --config
220 assert isinstance(launcher.service, SupportsRemoteExec) # from --service
221 # Check that the --globals file is loaded and $var expansion is handled
222 # using the value provided on the CLI.
223 assert launcher.global_config["experiment_id"] == "MockeryExperiment"
224 assert launcher.global_config["testVmName"] == "MockeryExperiment-vm"
225 # Check that secondary expansion also works.
226 assert launcher.global_config["testVnetName"] == "MockeryExperiment-vm-vnet"
227 # Check that we can expand a $var in a config file that references an environment variable.
228 assert path_join(launcher.global_config["pathVarWithEnvVarRef"], abs_path=True) == path_join(
229 os.getcwd(), "foo", abs_path=True
230 )
231 assert launcher.global_config["varWithEnvVarRef"] == f"user:{getuser()}"
232 assert not launcher.teardown
234 config = launcher.config_loader.load_config(config_file, ConfigSchema.CLI)
235 assert launcher.config_loader.config_paths == [
236 path_join(path, abs_path=True) for path in config_paths + config["config_path"]
237 ]
239 # Make sure we have the right number of trial runners.
240 assert len(launcher.trial_runners) == 3 # from test-cli-config.jsonc
241 # Check that the environment that got loaded looks to be of the right type.
242 env_config_file = config["environment"]
243 env_config = launcher.config_loader.load_config(env_config_file, ConfigSchema.ENVIRONMENT)
244 # All TrialRunners should get the same Environment.
245 assert all(
246 check_class_name(trial_runner.environment, env_config["class"])
247 for trial_runner in launcher.trial_runners
248 )
250 # Check that the optimizer looks right.
251 assert isinstance(launcher.optimizer, MlosCoreOptimizer)
252 opt_config_file = config["optimizer"]
253 opt_config = launcher.config_loader.load_config(opt_config_file, ConfigSchema.OPTIMIZER)
254 globals_file_config = launcher.config_loader.load_config(globals_file, ConfigSchema.GLOBALS)
255 # The actual global_config gets overwritten as a part of processing, so to test
256 # this we read the original value out of the source files.
257 orig_max_iters = globals_file_config.get(
258 "max_suggestions", opt_config.get("config", {}).get("max_suggestions", 100)
259 )
260 assert (
261 launcher.optimizer.max_suggestions
262 == orig_max_iters
263 == launcher.global_config["max_suggestions"]
264 )
266 # Check that the optimizer got initialized with random values instead of the defaults.
267 # Note: the environment doesn't get updated until suggest() is called to
268 # return these values in run.py.
269 assert not launcher.optimizer.tunable_params.is_defaults()
271 # TODO: Add a check that this flows through and replaces other seed config
272 # values through the stack.
273 # See Also: #495
275 # Check that CLI parameter overrides JSON config:
276 assert isinstance(launcher.scheduler, SyncScheduler)
277 assert launcher.scheduler.trial_config_repeat_count == 5 # from cli args
278 assert launcher.scheduler.max_trials == 200
280 # Check that the value from the file is overridden by the CLI arg.
281 assert config["random_seed"] == 42
282 # TODO: This isn't actually respected yet because the `--random-init` only
283 # applies to a temporary Optimizer used to populate the initial values via
284 # random sampling.
285 # assert launcher.optimizer.seed == 1234
288def test_launcher_args_parse_3(config_paths: list[str]) -> None:
289 """Check that cli file values take precedence over other values."""
290 config_file = "cli/test-cli-config.jsonc"
291 globals_file = "globals/global_test_config.jsonc"
292 # Here we don't override values in test-cli-config with cli args but ensure that
293 # those take precedence over other config files.
294 cli_args = (
295 " ".join([f"--config-path {config_path}" for config_path in config_paths])
296 + f" --config {config_file}"
297 f" --globals {globals_file}"
298 " --max-suggestions 10" # check for - to _ conversion too
299 )
300 launcher = _get_launcher(__name__, cli_args)
302 assert launcher.optimizer.max_suggestions == 10 # from CLI args
304 # Check that CLI file parameter overrides JSON config:
305 assert isinstance(launcher.scheduler, SyncScheduler)
306 # from test-cli-config.jsonc (should override scheduler config file)
307 assert launcher.scheduler.trial_config_repeat_count == 2
310if __name__ == "__main__":
311 pytest.main([__file__, "-n0"])