Coverage for mlos_core/mlos_core/tests/spaces/adapters/llamatune_test.py: 99%
171 statements
« prev ^ index » next coverage.py v7.6.1, created at 2024-10-07 01:52 +0000
1#
2# Copyright (c) Microsoft Corporation.
3# Licensed under the MIT License.
4#
5"""Tests for LlamaTune space adapter."""
7# pylint: disable=missing-function-docstring
9from typing import Any, Dict, Iterator, List, Set
11import ConfigSpace as CS
12import pandas as pd
13import pytest
15from mlos_core.spaces.adapters import LlamaTuneAdapter
16from mlos_core.spaces.converters.util import (
17 QUANTIZATION_BINS_META_KEY,
18 monkey_patch_cs_quantization,
19)
21# Explicitly test quantized values with llamatune space adapter.
22# TODO: Add log scale sampling tests as well.
25def construct_parameter_space( # pylint: disable=too-many-arguments
26 *,
27 n_continuous_params: int = 0,
28 n_quantized_continuous_params: int = 0,
29 n_integer_params: int = 0,
30 n_quantized_integer_params: int = 0,
31 n_categorical_params: int = 0,
32 seed: int = 1234,
33) -> CS.ConfigurationSpace:
34 """Helper function for construct an instance of `ConfigSpace.ConfigurationSpace`."""
35 input_space = CS.ConfigurationSpace(
36 seed=seed,
37 space=[
38 *(
39 CS.UniformFloatHyperparameter(name=f"cont_{idx}", lower=0, upper=64)
40 for idx in range(n_continuous_params)
41 ),
42 *(
43 CS.UniformFloatHyperparameter(
44 name=f"cont_{idx}", lower=0, upper=64, meta={QUANTIZATION_BINS_META_KEY: 6}
45 )
46 for idx in range(n_quantized_continuous_params)
47 ),
48 *(
49 CS.UniformIntegerHyperparameter(name=f"int_{idx}", lower=-1, upper=256)
50 for idx in range(n_integer_params)
51 ),
52 *(
53 CS.UniformIntegerHyperparameter(
54 name=f"int_{idx}", lower=0, upper=256, meta={QUANTIZATION_BINS_META_KEY: 17}
55 )
56 for idx in range(n_quantized_integer_params)
57 ),
58 *(
59 CS.CategoricalHyperparameter(
60 name=f"str_{idx}", choices=[f"option_{idx}" for idx in range(5)]
61 )
62 for idx in range(n_categorical_params)
63 ),
64 ],
65 )
66 return monkey_patch_cs_quantization(input_space)
69@pytest.mark.parametrize(
70 ("num_target_space_dims", "param_space_kwargs"),
71 (
72 [
73 (num_target_space_dims, param_space_kwargs)
74 for num_target_space_dims in (2, 4)
75 for num_orig_space_factor in (1.5, 4)
76 for param_space_kwargs in (
77 {"n_continuous_params": int(num_target_space_dims * num_orig_space_factor)},
78 {"n_integer_params": int(num_target_space_dims * num_orig_space_factor)},
79 {"n_categorical_params": int(num_target_space_dims * num_orig_space_factor)},
80 {"n_categorical_params": int(num_target_space_dims * num_orig_space_factor)},
81 {"n_quantized_integer_params": int(num_target_space_dims * num_orig_space_factor)},
82 {
83 "n_quantized_continuous_params": int(
84 num_target_space_dims * num_orig_space_factor
85 )
86 },
87 # Mix of all three types
88 {
89 "n_continuous_params": int(num_target_space_dims * num_orig_space_factor / 3),
90 "n_integer_params": int(num_target_space_dims * num_orig_space_factor / 3),
91 "n_categorical_params": int(num_target_space_dims * num_orig_space_factor / 3),
92 },
93 )
94 ]
95 ),
96)
97def test_num_low_dims(
98 num_target_space_dims: int,
99 param_space_kwargs: dict,
100) -> None: # pylint: disable=too-many-locals
101 """Tests LlamaTune's low-to-high space projection method."""
102 input_space = construct_parameter_space(**param_space_kwargs)
104 # Number of target parameter space dimensions should be fewer than those of the original space
105 with pytest.raises(ValueError):
106 LlamaTuneAdapter(
107 orig_parameter_space=input_space, num_low_dims=len(list(input_space.keys()))
108 )
110 # Enable only low-dimensional space projections
111 adapter = LlamaTuneAdapter(
112 orig_parameter_space=input_space,
113 num_low_dims=num_target_space_dims,
114 special_param_values=None,
115 max_unique_values_per_param=None,
116 )
118 sampled_configs = adapter.target_parameter_space.sample_configuration(size=100)
119 for sampled_config in sampled_configs: # pylint: disable=not-an-iterable # (false positive)
120 # Transform low-dim config to high-dim point/config
121 sampled_config_df = pd.DataFrame(
122 [sampled_config.values()], columns=list(sampled_config.keys())
123 )
124 orig_config_df = adapter.transform(sampled_config_df)
126 # High-dim (i.e., original) config should be valid
127 orig_config = CS.Configuration(input_space, values=orig_config_df.iloc[0].to_dict())
128 orig_config.check_valid_configuration()
130 # Transform high-dim config back to low-dim
131 target_config_df = adapter.inverse_transform(orig_config_df)
133 # Sampled config and this should be the same
134 target_config = CS.Configuration(
135 adapter.target_parameter_space,
136 values=target_config_df.iloc[0].to_dict(),
137 )
138 assert target_config == sampled_config
140 # Try inverse projection (i.e., high-to-low) for previously unseen configs
141 unseen_sampled_configs = adapter.target_parameter_space.sample_configuration(size=25)
142 for (
143 unseen_sampled_config
144 ) in unseen_sampled_configs: # pylint: disable=not-an-iterable # (false positive)
145 if (
146 unseen_sampled_config in sampled_configs
147 ): # pylint: disable=unsupported-membership-test # (false positive)
148 continue
150 unseen_sampled_config_df = pd.DataFrame(
151 [unseen_sampled_config.values()], columns=list(unseen_sampled_config.keys())
152 )
153 with pytest.raises(ValueError):
154 _ = adapter.inverse_transform(
155 unseen_sampled_config_df
156 ) # pylint: disable=redefined-variable-type
159def test_special_parameter_values_validation() -> None:
160 """Tests LlamaTune's validation process of user-provided special parameter values
161 dictionary.
162 """
163 input_space = CS.ConfigurationSpace(seed=1234)
164 input_space.add(
165 CS.CategoricalHyperparameter(name="str", choices=[f"choice_{idx}" for idx in range(5)])
166 )
167 input_space.add(CS.UniformFloatHyperparameter(name="cont", lower=-1, upper=100))
168 input_space.add(CS.UniformIntegerHyperparameter(name="int", lower=0, upper=100))
170 # Only UniformIntegerHyperparameters are currently supported
171 with pytest.raises(NotImplementedError):
172 special_param_values_dict_1 = {"str": "choice_1"}
173 LlamaTuneAdapter(
174 orig_parameter_space=input_space,
175 num_low_dims=2,
176 special_param_values=special_param_values_dict_1,
177 max_unique_values_per_param=None,
178 )
180 with pytest.raises(NotImplementedError):
181 special_param_values_dict_2 = {"cont": -1}
182 LlamaTuneAdapter(
183 orig_parameter_space=input_space,
184 num_low_dims=2,
185 special_param_values=special_param_values_dict_2,
186 max_unique_values_per_param=None,
187 )
189 # Special value should belong to parameter value domain
190 with pytest.raises(ValueError, match="value domain"):
191 special_param_values_dict = {"int": -1}
192 LlamaTuneAdapter(
193 orig_parameter_space=input_space,
194 num_low_dims=2,
195 special_param_values=special_param_values_dict,
196 max_unique_values_per_param=None,
197 )
199 # Invalid dicts; ValueError should be thrown
200 invalid_special_param_values_dicts: List[Dict[str, Any]] = [
201 {"int-Q": 0}, # parameter does not exist
202 {"int": {0: 0.2}}, # invalid definition
203 {"int": 0.2}, # invalid parameter value
204 {"int": (0.4, 0)}, # (biasing %, special value) instead of (special value, biasing %)
205 {"int": [0, 0]}, # duplicate special values
206 {"int": []}, # empty list
207 {"int": [{0: 0.2}]},
208 {"int": [(0.4, 0), (1, 0.7)]}, # first tuple is inverted; second is correct
209 {"int": [(0, 0.1), (0, 0.2)]}, # duplicate special values
210 ]
211 for spv_dict in invalid_special_param_values_dicts:
212 with pytest.raises(ValueError):
213 LlamaTuneAdapter(
214 orig_parameter_space=input_space,
215 num_low_dims=2,
216 special_param_values=spv_dict,
217 max_unique_values_per_param=None,
218 )
220 # Biasing percentage of special value(s) are invalid
221 invalid_special_param_values_dicts = [
222 {"int": (0, 1.1)}, # >1 probability
223 {"int": (0, 0)}, # Zero probability
224 {"int": (0, -0.1)}, # Negative probability
225 {"int": (0, 20)}, # 2,000% instead of 20%
226 {"int": [0, 1, 2, 3, 4, 5]}, # default biasing is 20%; 6 values * 20% > 100%
227 {"int": [(0, 0.4), (1, 0.7)]}, # combined probability >100%
228 {"int": [(0, -0.4), (1, 0.7)]}, # probability for value 0 is invalid.
229 ]
231 for spv_dict in invalid_special_param_values_dicts:
232 with pytest.raises(ValueError):
233 LlamaTuneAdapter(
234 orig_parameter_space=input_space,
235 num_low_dims=2,
236 special_param_values=spv_dict,
237 max_unique_values_per_param=None,
238 )
241def gen_random_configs(adapter: LlamaTuneAdapter, num_configs: int) -> Iterator[CS.Configuration]:
242 for sampled_config in adapter.target_parameter_space.sample_configuration(size=num_configs):
243 # Transform low-dim config to high-dim config
244 sampled_config_df = pd.DataFrame(
245 [sampled_config.values()], columns=list(sampled_config.keys())
246 )
247 orig_config_df = adapter.transform(sampled_config_df)
248 orig_config = CS.Configuration(
249 adapter.orig_parameter_space,
250 values=orig_config_df.iloc[0].to_dict(),
251 )
252 yield orig_config
255def test_special_parameter_values_biasing() -> None: # pylint: disable=too-complex
256 """Tests LlamaTune's special parameter values biasing methodology."""
257 input_space = CS.ConfigurationSpace(seed=1234)
258 input_space.add(CS.UniformIntegerHyperparameter(name="int_1", lower=0, upper=100))
259 input_space.add(CS.UniformIntegerHyperparameter(name="int_2", lower=0, upper=100))
261 num_configs = 400
262 bias_percentage = LlamaTuneAdapter.DEFAULT_SPECIAL_PARAM_VALUE_BIASING_PERCENTAGE
263 eps = 0.2
265 # Single parameter; single special value
266 special_param_value_dicts: List[Dict[str, Any]] = [
267 {"int_1": 0},
268 {"int_1": (0, bias_percentage)},
269 {"int_1": [0]},
270 {"int_1": [(0, bias_percentage)]},
271 ]
273 for spv_dict in special_param_value_dicts:
274 adapter = LlamaTuneAdapter(
275 orig_parameter_space=input_space,
276 num_low_dims=1,
277 special_param_values=spv_dict,
278 max_unique_values_per_param=None,
279 )
281 special_value_occurrences = sum(
282 1 for config in gen_random_configs(adapter, num_configs) if config["int_1"] == 0
283 )
284 assert (1 - eps) * int(num_configs * bias_percentage) <= special_value_occurrences
286 # Single parameter; multiple special values
287 special_param_value_dicts = [
288 {"int_1": [0, 1]},
289 {"int_1": [(0, bias_percentage), (1, bias_percentage)]},
290 ]
292 for spv_dict in special_param_value_dicts:
293 adapter = LlamaTuneAdapter(
294 orig_parameter_space=input_space,
295 num_low_dims=1,
296 special_param_values=spv_dict,
297 max_unique_values_per_param=None,
298 )
300 special_values_occurrences = {0: 0, 1: 0}
301 for config in gen_random_configs(adapter, num_configs):
302 if config["int_1"] == 0:
303 special_values_occurrences[0] += 1
304 elif config["int_1"] == 1:
305 special_values_occurrences[1] += 1
307 assert (1 - eps) * int(num_configs * bias_percentage) <= special_values_occurrences[0]
308 assert (1 - eps) * int(num_configs * bias_percentage) <= special_values_occurrences[1]
310 # Multiple parameters; multiple special values; different biasing percentage
311 spv_dict = {
312 "int_1": [(0, bias_percentage), (1, bias_percentage / 2)],
313 "int_2": [(2, bias_percentage / 2), (100, bias_percentage * 1.5)],
314 }
315 adapter = LlamaTuneAdapter(
316 orig_parameter_space=input_space,
317 num_low_dims=1,
318 special_param_values=spv_dict,
319 max_unique_values_per_param=None,
320 )
322 special_values_instances: Dict[str, Dict[int, int]] = {
323 "int_1": {0: 0, 1: 0},
324 "int_2": {2: 0, 100: 0},
325 }
326 for config in gen_random_configs(adapter, num_configs):
327 if config["int_1"] == 0:
328 special_values_instances["int_1"][0] += 1
329 elif config["int_1"] == 1:
330 special_values_instances["int_1"][1] += 1
332 if config["int_2"] == 2:
333 special_values_instances["int_2"][2] += 1
334 elif config["int_2"] == 100:
335 special_values_instances["int_2"][100] += 1
337 assert (1 - eps) * int(num_configs * bias_percentage) <= special_values_instances["int_1"][0]
338 assert (1 - eps) * int(num_configs * bias_percentage / 2) <= (
339 special_values_instances["int_1"][1]
340 )
341 assert (1 - eps) * int(num_configs * bias_percentage / 2) <= (
342 special_values_instances["int_2"][2]
343 )
344 assert (1 - eps) * int(num_configs * bias_percentage * 1.5) <= (
345 special_values_instances["int_2"][100]
346 )
349def test_max_unique_values_per_param() -> None:
350 """Tests LlamaTune's parameter values discretization implementation."""
351 # Define config space with a mix of different parameter types
352 input_space = CS.ConfigurationSpace(seed=1234)
353 input_space.add(
354 CS.UniformFloatHyperparameter(name="cont_1", lower=0, upper=5),
355 )
356 input_space.add(CS.UniformFloatHyperparameter(name="cont_2", lower=1, upper=100))
357 input_space.add(CS.UniformIntegerHyperparameter(name="int_1", lower=1, upper=10))
358 input_space.add(CS.UniformIntegerHyperparameter(name="int_2", lower=0, upper=2048))
359 input_space.add(CS.CategoricalHyperparameter(name="str_1", choices=["on", "off"]))
360 input_space.add(
361 CS.CategoricalHyperparameter(name="str_2", choices=[f"choice_{idx}" for idx in range(10)])
362 )
364 # Restrict the number of unique parameter values
365 num_configs = 200
366 for max_unique_values_per_param in (5, 25, 100):
367 adapter = LlamaTuneAdapter(
368 orig_parameter_space=input_space,
369 num_low_dims=3,
370 special_param_values=None,
371 max_unique_values_per_param=max_unique_values_per_param,
372 )
374 # Keep track of unique values generated for each parameter
375 unique_values_dict: Dict[str, set] = {param: set() for param in list(input_space.keys())}
376 for config in gen_random_configs(adapter, num_configs):
377 for param, value in config.items():
378 unique_values_dict[param].add(value)
380 # Ensure that their number is less than the maximum number allowed
381 for _, unique_values in unique_values_dict.items():
382 assert len(unique_values) <= max_unique_values_per_param
385@pytest.mark.parametrize(
386 ("num_target_space_dims", "param_space_kwargs"),
387 (
388 [
389 (num_target_space_dims, param_space_kwargs)
390 for num_target_space_dims in (2, 4)
391 for num_orig_space_factor in (1.5, 4)
392 for param_space_kwargs in (
393 {"n_continuous_params": int(num_target_space_dims * num_orig_space_factor)},
394 {"n_integer_params": int(num_target_space_dims * num_orig_space_factor)},
395 {"n_categorical_params": int(num_target_space_dims * num_orig_space_factor)},
396 {"n_quantized_integer_params": int(num_target_space_dims * num_orig_space_factor)},
397 {
398 "n_quantized_continuous_params": int(
399 num_target_space_dims * num_orig_space_factor
400 )
401 },
402 # Mix of all three types
403 {
404 "n_continuous_params": int(num_target_space_dims * num_orig_space_factor / 3),
405 "n_integer_params": int(num_target_space_dims * num_orig_space_factor / 3),
406 "n_categorical_params": int(num_target_space_dims * num_orig_space_factor / 3),
407 },
408 )
409 ]
410 ),
411)
412def test_approx_inverse_mapping(
413 num_target_space_dims: int,
414 param_space_kwargs: dict,
415) -> None: # pylint: disable=too-many-locals
416 """Tests LlamaTune's approximate high-to-low space projection method, using pseudo-
417 inverse.
418 """
419 input_space = construct_parameter_space(**param_space_kwargs)
421 # Enable low-dimensional space projection, but disable reverse mapping
422 adapter = LlamaTuneAdapter(
423 orig_parameter_space=input_space,
424 num_low_dims=num_target_space_dims,
425 special_param_values=None,
426 max_unique_values_per_param=None,
427 use_approximate_reverse_mapping=False,
428 )
430 sampled_config = input_space.sample_configuration() # size=1)
431 with pytest.raises(ValueError):
432 sampled_config_df = pd.DataFrame(
433 [sampled_config.values()], columns=list(sampled_config.keys())
434 )
435 _ = adapter.inverse_transform(sampled_config_df)
437 # Enable low-dimensional space projection *and* reverse mapping
438 adapter = LlamaTuneAdapter(
439 orig_parameter_space=input_space,
440 num_low_dims=num_target_space_dims,
441 special_param_values=None,
442 max_unique_values_per_param=None,
443 use_approximate_reverse_mapping=True,
444 )
446 # Warning should be printed the first time
447 sampled_config = input_space.sample_configuration() # size=1)
448 with pytest.warns(UserWarning):
449 sampled_config_df = pd.DataFrame(
450 [sampled_config.values()], columns=list(sampled_config.keys())
451 )
452 target_config_df = adapter.inverse_transform(sampled_config_df)
453 # Low-dim (i.e., target) config should be valid
454 target_config = CS.Configuration(
455 adapter.target_parameter_space,
456 values=target_config_df.iloc[0].to_dict(),
457 )
458 target_config.check_valid_configuration()
460 # Test inverse transform with 100 random configs
461 for _ in range(100):
462 sampled_config = input_space.sample_configuration() # size=1)
463 sampled_config_df = pd.DataFrame(
464 [sampled_config.values()], columns=list(sampled_config.keys())
465 )
466 target_config_df = adapter.inverse_transform(sampled_config_df)
467 # Low-dim (i.e., target) config should be valid
468 target_config = CS.Configuration(
469 adapter.target_parameter_space,
470 values=target_config_df.iloc[0].to_dict(),
471 )
472 target_config.check_valid_configuration()
475@pytest.mark.parametrize(
476 ("num_low_dims", "special_param_values", "max_unique_values_per_param"),
477 (
478 [
479 (num_low_dims, special_param_values, max_unique_values_per_param)
480 for num_low_dims in (8, 16)
481 for special_param_values in (
482 {"int_1": -1, "int_2": -1, "int_3": -1, "int_4": [-1, 0]},
483 {
484 "int_1": (-1, 0.1),
485 "int_2": -1,
486 "int_3": (-1, 0.3),
487 "int_4": [(-1, 0.1), (0, 0.2)],
488 },
489 )
490 for max_unique_values_per_param in (50, 250)
491 ]
492 ),
493)
494def test_llamatune_pipeline(
495 num_low_dims: int,
496 special_param_values: dict,
497 max_unique_values_per_param: int,
498) -> None:
499 """Tests LlamaTune space adapter when all components are active."""
500 # pylint: disable=too-many-locals
502 # Define config space with a mix of different parameter types
503 input_space = construct_parameter_space(
504 n_continuous_params=10,
505 n_integer_params=10,
506 n_categorical_params=5,
507 )
508 adapter = LlamaTuneAdapter(
509 orig_parameter_space=input_space,
510 num_low_dims=num_low_dims,
511 special_param_values=special_param_values,
512 max_unique_values_per_param=max_unique_values_per_param,
513 )
515 special_value_occurrences = {
516 # pylint: disable=protected-access
517 param: {special_value: 0 for special_value, _ in tuples_list}
518 for param, tuples_list in adapter._special_param_values_dict.items()
519 }
520 unique_values_dict: Dict[str, Set] = {param: set() for param in input_space.keys()}
522 num_configs = 1000
523 for config in adapter.target_parameter_space.sample_configuration(
524 size=num_configs
525 ): # pylint: disable=not-an-iterable
526 # Transform low-dim config to high-dim point/config
527 sampled_config_df = pd.DataFrame([config.values()], columns=list(config.keys()))
528 orig_config_df = adapter.transform(sampled_config_df)
529 # High-dim (i.e., original) config should be valid
530 orig_config = CS.Configuration(input_space, values=orig_config_df.iloc[0].to_dict())
531 orig_config.check_valid_configuration()
533 # Transform high-dim config back to low-dim
534 target_config_df = adapter.inverse_transform(orig_config_df)
535 # Sampled config and this should be the same
536 target_config = CS.Configuration(
537 adapter.target_parameter_space,
538 values=target_config_df.iloc[0].to_dict(),
539 )
540 assert target_config == config
542 for param, value in orig_config.items():
543 # Keep track of special value occurrences
544 if param in special_value_occurrences:
545 if value in special_value_occurrences[param]:
546 special_value_occurrences[param][value] += 1
548 # Keep track of unique values generated for each parameter
549 unique_values_dict[param].add(value)
551 # Ensure that occurrences of special values do not significantly deviate from expected
552 eps = 0.2
553 for (
554 param,
555 tuples_list,
556 ) in adapter._special_param_values_dict.items(): # pylint: disable=protected-access
557 for value, bias_percentage in tuples_list:
558 assert (1 - eps) * int(num_configs * bias_percentage) <= special_value_occurrences[
559 param
560 ][value]
562 # Ensure that number of unique values is less than the maximum number allowed
563 for _, unique_values in unique_values_dict.items():
564 assert len(unique_values) <= max_unique_values_per_param
567@pytest.mark.parametrize(
568 ("num_target_space_dims", "param_space_kwargs"),
569 (
570 [
571 (num_target_space_dims, param_space_kwargs)
572 for num_target_space_dims in (2, 4)
573 for num_orig_space_factor in (1.5, 4)
574 for param_space_kwargs in (
575 {"n_continuous_params": int(num_target_space_dims * num_orig_space_factor)},
576 {"n_integer_params": int(num_target_space_dims * num_orig_space_factor)},
577 {"n_categorical_params": int(num_target_space_dims * num_orig_space_factor)},
578 # Mix of all three types
579 {
580 "n_continuous_params": int(num_target_space_dims * num_orig_space_factor / 3),
581 "n_integer_params": int(num_target_space_dims * num_orig_space_factor / 3),
582 "n_categorical_params": int(num_target_space_dims * num_orig_space_factor / 3),
583 },
584 )
585 ]
586 ),
587)
588def test_deterministic_behavior_for_same_seed(
589 num_target_space_dims: int,
590 param_space_kwargs: dict,
591) -> None:
592 """Tests LlamaTune's space adapter deterministic behavior when given same seed in
593 the input parameter space.
594 """
596 def generate_target_param_space_configs(seed: int) -> List[CS.Configuration]:
597 input_space = construct_parameter_space(**param_space_kwargs, seed=seed)
599 # Init adapter and sample points in the low-dim space
600 adapter = LlamaTuneAdapter(
601 orig_parameter_space=input_space,
602 num_low_dims=num_target_space_dims,
603 special_param_values=None,
604 max_unique_values_per_param=None,
605 use_approximate_reverse_mapping=False,
606 )
608 sample_configs: List[CS.Configuration] = (
609 adapter.target_parameter_space.sample_configuration(size=100)
610 )
611 return sample_configs
613 assert generate_target_param_space_configs(42) == generate_target_param_space_configs(42)
614 assert generate_target_param_space_configs(1234) != generate_target_param_space_configs(42)