Coverage for mlos_bench/mlos_bench/tests/launcher_parse_args_test.py: 99%
103 statements
« prev ^ index » next coverage.py v7.6.1, created at 2024-10-07 01:52 +0000
1#
2# Copyright (c) Microsoft Corporation.
3# Licensed under the MIT License.
4#
5"""
6Unit tests to check the Launcher CLI arg parsing
7See Also: test_load_cli_config_examples.py
8"""
10import os
11import sys
12from getpass import getuser
13from typing import List
15import pytest
17from mlos_bench.config.schemas import ConfigSchema
18from mlos_bench.launcher import Launcher
19from mlos_bench.optimizers import MlosCoreOptimizer, OneShotOptimizer
20from mlos_bench.os_environ import environ
21from mlos_bench.schedulers import SyncScheduler
22from mlos_bench.services.types import (
23 SupportsAuth,
24 SupportsConfigLoading,
25 SupportsFileShareOps,
26 SupportsLocalExec,
27 SupportsRemoteExec,
28)
29from mlos_bench.tests import check_class_name
30from mlos_bench.util import path_join
32if sys.version_info < (3, 10):
33 from importlib_resources import files
34else:
35 from importlib.resources import files
37# pylint: disable=redefined-outer-name
40@pytest.fixture
41def config_paths() -> List[str]:
42 """
43 Returns a list of config paths.
45 Returns
46 -------
47 List[str]
48 """
49 return [
50 path_join(os.getcwd(), abs_path=True),
51 str(files("mlos_bench.config")),
52 str(files("mlos_bench.tests.config")),
53 ]
56# This is part of the minimal required args by the Launcher.
57ENV_CONF_PATH = "environments/mock/mock_env.jsonc"
60def _get_launcher(desc: str, cli_args: str) -> Launcher:
61 # The VSCode pytest wrapper actually starts in a different directory before
62 # changing into the code directory, but doesn't update the PWD environment
63 # variable so we use a separate variable.
64 # See global_test_config.jsonc for more details.
65 environ["CUSTOM_PATH_FROM_ENV"] = os.getcwd()
66 if sys.platform == "win32":
67 # Some env tweaks for platform compatibility.
68 environ["USER"] = environ["USERNAME"]
69 launcher = Launcher(description=desc, argv=cli_args.split())
70 # Check the basic parent service
71 assert isinstance(launcher.service, SupportsConfigLoading) # built-in
72 assert isinstance(launcher.service, SupportsLocalExec) # built-in
73 return launcher
76def test_launcher_args_parse_defaults(config_paths: List[str]) -> None:
77 """Test that we get the defaults we expect when using minimal config arg
78 examples.
79 """
80 cli_args = (
81 "--config-paths "
82 + " ".join(config_paths)
83 + f" --environment {ENV_CONF_PATH}"
84 + " --globals globals/global_test_config.jsonc"
85 )
86 launcher = _get_launcher(__name__, cli_args)
87 # Check that the first --globals file is loaded and $var expansion is handled.
88 assert launcher.global_config["experiment_id"] == "MockExperiment"
89 assert launcher.global_config["testVmName"] == "MockExperiment-vm"
90 # Check that secondary expansion also works.
91 assert launcher.global_config["testVnetName"] == "MockExperiment-vm-vnet"
92 # Check that we can expand a $var in a config file that references an environment variable.
93 assert path_join(launcher.global_config["pathVarWithEnvVarRef"], abs_path=True) == path_join(
94 os.getcwd(), "foo", abs_path=True
95 )
96 assert launcher.global_config["varWithEnvVarRef"] == f"user:{getuser()}"
97 assert launcher.teardown # defaults
98 # Check that the environment that got loaded looks to be of the right type.
99 env_config = launcher.config_loader.load_config(ENV_CONF_PATH, ConfigSchema.ENVIRONMENT)
100 assert env_config["class"] == "mlos_bench.environments.mock_env.MockEnv"
101 assert check_class_name(launcher.environment, env_config["class"])
102 # Check that the optimizer looks right.
103 assert isinstance(launcher.optimizer, OneShotOptimizer)
104 # Check that the optimizer got initialized with defaults.
105 assert launcher.optimizer.tunable_params.is_defaults()
106 assert launcher.optimizer.max_suggestions == 1 # value for OneShotOptimizer
107 # Check that we pick up the right scheduler config:
108 assert isinstance(launcher.scheduler, SyncScheduler)
109 assert launcher.scheduler.trial_config_repeat_count == 1 # default
110 assert launcher.scheduler.max_trials == -1 # default
113def test_launcher_args_parse_1(config_paths: List[str]) -> None:
114 """
115 Test that using multiple --globals arguments works and that multiple space separated
116 options to --config-paths works.
118 Check $var expansion and Environment loading.
119 """
120 # Here we have multiple paths following --config-paths and --service.
121 cli_args = (
122 "--config-paths "
123 + " ".join(config_paths)
124 + " --service services/remote/mock/mock_auth_service.jsonc"
125 + " services/remote/mock/mock_remote_exec_service.jsonc"
126 + " --scheduler schedulers/sync_scheduler.jsonc"
127 + f" --environment {ENV_CONF_PATH}"
128 + " --globals globals/global_test_config.jsonc"
129 + " --globals globals/global_test_extra_config.jsonc"
130 " --test_global_value_2 from-args"
131 )
132 launcher = _get_launcher(__name__, cli_args)
133 # Check some additional features of the the parent service
134 assert isinstance(launcher.service, SupportsAuth) # from --service
135 assert isinstance(launcher.service, SupportsRemoteExec) # from --service
136 # Check that the first --globals file is loaded and $var expansion is handled.
137 assert launcher.global_config["experiment_id"] == "MockExperiment"
138 assert launcher.global_config["testVmName"] == "MockExperiment-vm"
139 # Check that secondary expansion also works.
140 assert launcher.global_config["testVnetName"] == "MockExperiment-vm-vnet"
141 # Check that the second --globals file is loaded.
142 assert launcher.global_config["test_global_value"] == "from-file"
143 # Check overriding values in a file from the command line.
144 assert launcher.global_config["test_global_value_2"] == "from-args"
145 # Check that we can expand a $var in a config file that references an environment variable.
146 assert path_join(launcher.global_config["pathVarWithEnvVarRef"], abs_path=True) == path_join(
147 os.getcwd(), "foo", abs_path=True
148 )
149 assert launcher.global_config["varWithEnvVarRef"] == f"user:{getuser()}"
150 assert launcher.teardown
151 # Check that the environment that got loaded looks to be of the right type.
152 env_config = launcher.config_loader.load_config(ENV_CONF_PATH, ConfigSchema.ENVIRONMENT)
153 assert env_config["class"] == "mlos_bench.environments.mock_env.MockEnv"
154 # Check that the optimizer looks right.
155 assert isinstance(launcher.optimizer, OneShotOptimizer)
156 # Check that the optimizer got initialized with defaults.
157 assert launcher.optimizer.tunable_params.is_defaults()
158 assert launcher.optimizer.max_suggestions == 1 # value for OneShotOptimizer
159 # Check that we pick up the right scheduler config:
160 assert isinstance(launcher.scheduler, SyncScheduler)
161 assert (
162 launcher.scheduler.trial_config_repeat_count == 3
163 ) # from the custom sync_scheduler.jsonc config
164 assert launcher.scheduler.max_trials == -1
167def test_launcher_args_parse_2(config_paths: List[str]) -> None:
168 """Test multiple --config-path instances, --config file vs --arg, --var=val
169 overrides, $var templates, option args, --random-init, etc.
170 """
171 config_file = "cli/test-cli-config.jsonc"
172 globals_file = "globals/global_test_config.jsonc"
173 # Here we have multiple --config-path and --service args, each with their own path.
174 cli_args = (
175 " ".join([f"--config-path {config_path}" for config_path in config_paths])
176 + f" --config {config_file}"
177 + " --service services/remote/mock/mock_auth_service.jsonc"
178 + " --service services/remote/mock/mock_remote_exec_service.jsonc"
179 + f" --globals {globals_file}"
180 + " --experiment_id MockeryExperiment"
181 + " --no-teardown"
182 + " --random-init"
183 + " --random-seed 1234"
184 + " --trial-config-repeat-count 5"
185 + " --max_trials 200"
186 )
187 launcher = _get_launcher(__name__, cli_args)
188 # Check some additional features of the the parent service
189 assert isinstance(launcher.service, SupportsAuth) # from --service
190 assert isinstance(launcher.service, SupportsFileShareOps) # from --config
191 assert isinstance(launcher.service, SupportsRemoteExec) # from --service
192 # Check that the --globals file is loaded and $var expansion is handled
193 # using the value provided on the CLI.
194 assert launcher.global_config["experiment_id"] == "MockeryExperiment"
195 assert launcher.global_config["testVmName"] == "MockeryExperiment-vm"
196 # Check that secondary expansion also works.
197 assert launcher.global_config["testVnetName"] == "MockeryExperiment-vm-vnet"
198 # Check that we can expand a $var in a config file that references an environment variable.
199 assert path_join(launcher.global_config["pathVarWithEnvVarRef"], abs_path=True) == path_join(
200 os.getcwd(), "foo", abs_path=True
201 )
202 assert launcher.global_config["varWithEnvVarRef"] == f"user:{getuser()}"
203 assert not launcher.teardown
205 config = launcher.config_loader.load_config(config_file, ConfigSchema.CLI)
206 assert launcher.config_loader.config_paths == [
207 path_join(path, abs_path=True) for path in config_paths + config["config_path"]
208 ]
210 # Check that the environment that got loaded looks to be of the right type.
211 env_config_file = config["environment"]
212 env_config = launcher.config_loader.load_config(env_config_file, ConfigSchema.ENVIRONMENT)
213 assert check_class_name(launcher.environment, env_config["class"])
215 # Check that the optimizer looks right.
216 assert isinstance(launcher.optimizer, MlosCoreOptimizer)
217 opt_config_file = config["optimizer"]
218 opt_config = launcher.config_loader.load_config(opt_config_file, ConfigSchema.OPTIMIZER)
219 globals_file_config = launcher.config_loader.load_config(globals_file, ConfigSchema.GLOBALS)
220 # The actual global_config gets overwritten as a part of processing, so to test
221 # this we read the original value out of the source files.
222 orig_max_iters = globals_file_config.get(
223 "max_suggestions", opt_config.get("config", {}).get("max_suggestions", 100)
224 )
225 assert (
226 launcher.optimizer.max_suggestions
227 == orig_max_iters
228 == launcher.global_config["max_suggestions"]
229 )
231 # Check that the optimizer got initialized with random values instead of the defaults.
232 # Note: the environment doesn't get updated until suggest() is called to
233 # return these values in run.py.
234 assert not launcher.optimizer.tunable_params.is_defaults()
236 # TODO: Add a check that this flows through and replaces other seed config
237 # values through the stack.
238 # See Also: #495
240 # Check that CLI parameter overrides JSON config:
241 assert isinstance(launcher.scheduler, SyncScheduler)
242 assert launcher.scheduler.trial_config_repeat_count == 5 # from cli args
243 assert launcher.scheduler.max_trials == 200
245 # Check that the value from the file is overridden by the CLI arg.
246 assert config["random_seed"] == 42
247 # TODO: This isn't actually respected yet because the `--random-init` only
248 # applies to a temporary Optimizer used to populate the initial values via
249 # random sampling.
250 # assert launcher.optimizer.seed == 1234
253def test_launcher_args_parse_3(config_paths: List[str]) -> None:
254 """Check that cli file values take precedence over other values."""
255 config_file = "cli/test-cli-config.jsonc"
256 globals_file = "globals/global_test_config.jsonc"
257 # Here we don't override values in test-cli-config with cli args but ensure that
258 # those take precedence over other config files.
259 cli_args = (
260 " ".join([f"--config-path {config_path}" for config_path in config_paths])
261 + f" --config {config_file}"
262 + f" --globals {globals_file}"
263 + " --max-suggestions 10" # check for - to _ conversion too
264 )
265 launcher = _get_launcher(__name__, cli_args)
267 assert launcher.optimizer.max_suggestions == 10 # from CLI args
269 # Check that CLI file parameter overrides JSON config:
270 assert isinstance(launcher.scheduler, SyncScheduler)
271 # from test-cli-config.jsonc (should override scheduler config file)
272 assert launcher.scheduler.trial_config_repeat_count == 2
275if __name__ == "__main__":
276 pytest.main([__file__, "-n0"])