Coverage for mlos_core/mlos_core/tests/spaces/adapters/llamatune_test.py: 99%
172 statements
« prev ^ index » next coverage.py v7.6.10, created at 2025-01-21 01:50 +0000
1#
2# Copyright (c) Microsoft Corporation.
3# Licensed under the MIT License.
4#
5"""Tests for LlamaTune space adapter."""
7# pylint: disable=missing-function-docstring
9from collections.abc import Iterator
10from typing import Any
12import ConfigSpace as CS
13import pandas as pd
14import pytest
16from mlos_core.spaces.adapters import LlamaTuneAdapter
17from mlos_core.spaces.converters.util import (
18 QUANTIZATION_BINS_META_KEY,
19 monkey_patch_cs_quantization,
20)
22# Explicitly test quantized values with llamatune space adapter.
23# TODO: Add log scale sampling tests as well.
26def construct_parameter_space( # pylint: disable=too-many-arguments
27 *,
28 n_continuous_params: int = 0,
29 n_quantized_continuous_params: int = 0,
30 n_integer_params: int = 0,
31 n_quantized_integer_params: int = 0,
32 n_categorical_params: int = 0,
33 seed: int = 1234,
34) -> CS.ConfigurationSpace:
35 """Helper function for construct an instance of `ConfigSpace.ConfigurationSpace`."""
36 input_space = CS.ConfigurationSpace(
37 seed=seed,
38 space=[
39 *(
40 CS.UniformFloatHyperparameter(name=f"cont_{idx}", lower=0, upper=64)
41 for idx in range(n_continuous_params)
42 ),
43 *(
44 CS.UniformFloatHyperparameter(
45 name=f"cont_{idx}", lower=0, upper=64, meta={QUANTIZATION_BINS_META_KEY: 6}
46 )
47 for idx in range(n_quantized_continuous_params)
48 ),
49 *(
50 CS.UniformIntegerHyperparameter(name=f"int_{idx}", lower=-1, upper=256)
51 for idx in range(n_integer_params)
52 ),
53 *(
54 CS.UniformIntegerHyperparameter(
55 name=f"int_{idx}", lower=0, upper=256, meta={QUANTIZATION_BINS_META_KEY: 17}
56 )
57 for idx in range(n_quantized_integer_params)
58 ),
59 *(
60 CS.CategoricalHyperparameter(
61 name=f"str_{idx}", choices=[f"option_{idx}" for idx in range(5)]
62 )
63 for idx in range(n_categorical_params)
64 ),
65 ],
66 )
67 return monkey_patch_cs_quantization(input_space)
70@pytest.mark.parametrize(
71 ("num_target_space_dims", "param_space_kwargs"),
72 (
73 [
74 (num_target_space_dims, param_space_kwargs)
75 for num_target_space_dims in (2, 4)
76 for num_orig_space_factor in (1.5, 4)
77 for param_space_kwargs in (
78 {"n_continuous_params": int(num_target_space_dims * num_orig_space_factor)},
79 {"n_integer_params": int(num_target_space_dims * num_orig_space_factor)},
80 {"n_categorical_params": int(num_target_space_dims * num_orig_space_factor)},
81 {"n_categorical_params": int(num_target_space_dims * num_orig_space_factor)},
82 {"n_quantized_integer_params": int(num_target_space_dims * num_orig_space_factor)},
83 {
84 "n_quantized_continuous_params": int(
85 num_target_space_dims * num_orig_space_factor
86 )
87 },
88 # Mix of all three types
89 {
90 "n_continuous_params": int(num_target_space_dims * num_orig_space_factor / 3),
91 "n_integer_params": int(num_target_space_dims * num_orig_space_factor / 3),
92 "n_categorical_params": int(num_target_space_dims * num_orig_space_factor / 3),
93 },
94 )
95 ]
96 ),
97)
98def test_num_low_dims(
99 num_target_space_dims: int,
100 param_space_kwargs: dict,
101) -> None: # pylint: disable=too-many-locals
102 """Tests LlamaTune's low-to-high space projection method."""
103 input_space = construct_parameter_space(**param_space_kwargs)
105 # Number of target parameter space dimensions should be fewer than those of the original space
106 with pytest.raises(ValueError):
107 LlamaTuneAdapter(
108 orig_parameter_space=input_space, num_low_dims=len(list(input_space.keys()))
109 )
111 # Enable only low-dimensional space projections
112 adapter = LlamaTuneAdapter(
113 orig_parameter_space=input_space,
114 num_low_dims=num_target_space_dims,
115 special_param_values=None,
116 max_unique_values_per_param=None,
117 )
119 sampled_configs = adapter.target_parameter_space.sample_configuration(size=100)
120 for sampled_config in sampled_configs: # pylint: disable=not-an-iterable # (false positive)
121 # Transform low-dim config to high-dim point/config
122 sampled_config_sr = pd.Series(dict(sampled_config))
123 orig_config_sr = adapter.transform(sampled_config_sr)
125 # High-dim (i.e., original) config should be valid
126 orig_config = CS.Configuration(input_space, values=orig_config_sr.to_dict())
127 orig_config.check_valid_configuration()
129 # Transform high-dim config back to low-dim
130 target_config_sr = adapter.inverse_transform(orig_config_sr)
132 # Sampled config and this should be the same
133 target_config = CS.Configuration(
134 adapter.target_parameter_space,
135 values=target_config_sr.to_dict(),
136 )
137 assert target_config == sampled_config
139 # Try inverse projection (i.e., high-to-low) for previously unseen configs
140 unseen_sampled_configs = adapter.target_parameter_space.sample_configuration(size=25)
141 for (
142 unseen_sampled_config
143 ) in unseen_sampled_configs: # pylint: disable=not-an-iterable # (false positive)
144 if (
145 unseen_sampled_config
146 in sampled_configs # pylint: disable=unsupported-membership-test # (false positive)
147 ):
148 continue
150 unseen_sampled_config_sr = pd.Series(dict(unseen_sampled_config))
151 with pytest.raises(ValueError):
152 _ = adapter.inverse_transform(
153 unseen_sampled_config_sr
154 ) # pylint: disable=redefined-variable-type
157def test_special_parameter_values_validation() -> None:
158 """Tests LlamaTune's validation process of user-provided special parameter values
159 dictionary.
160 """
161 input_space = CS.ConfigurationSpace(seed=1234)
162 input_space.add(
163 CS.CategoricalHyperparameter(name="str", choices=[f"choice_{idx}" for idx in range(5)])
164 )
165 input_space.add(CS.UniformFloatHyperparameter(name="cont", lower=-1, upper=100))
166 input_space.add(CS.UniformIntegerHyperparameter(name="int", lower=0, upper=100))
168 # Only UniformIntegerHyperparameters are currently supported
169 with pytest.raises(NotImplementedError):
170 special_param_values_dict_1 = {"str": "choice_1"}
171 LlamaTuneAdapter(
172 orig_parameter_space=input_space,
173 num_low_dims=2,
174 special_param_values=special_param_values_dict_1,
175 max_unique_values_per_param=None,
176 )
178 with pytest.raises(NotImplementedError):
179 special_param_values_dict_2 = {"cont": -1}
180 LlamaTuneAdapter(
181 orig_parameter_space=input_space,
182 num_low_dims=2,
183 special_param_values=special_param_values_dict_2,
184 max_unique_values_per_param=None,
185 )
187 # Special value should belong to parameter value domain
188 with pytest.raises(ValueError, match="value domain"):
189 special_param_values_dict = {"int": -1}
190 LlamaTuneAdapter(
191 orig_parameter_space=input_space,
192 num_low_dims=2,
193 special_param_values=special_param_values_dict,
194 max_unique_values_per_param=None,
195 )
197 # Invalid dicts; ValueError should be thrown
198 invalid_special_param_values_dicts: list[dict[str, Any]] = [
199 {"int-Q": 0}, # parameter does not exist
200 {"int": {0: 0.2}}, # invalid definition
201 {"int": 0.2}, # invalid parameter value
202 {"int": (0.4, 0)}, # (biasing %, special value) instead of (special value, biasing %)
203 {"int": [0, 0]}, # duplicate special values
204 {"int": []}, # empty list
205 {"int": [{0: 0.2}]},
206 {"int": [(0.4, 0), (1, 0.7)]}, # first tuple is inverted; second is correct
207 {"int": [(0, 0.1), (0, 0.2)]}, # duplicate special values
208 ]
209 for spv_dict in invalid_special_param_values_dicts:
210 with pytest.raises(ValueError):
211 LlamaTuneAdapter(
212 orig_parameter_space=input_space,
213 num_low_dims=2,
214 special_param_values=spv_dict,
215 max_unique_values_per_param=None,
216 )
218 # Biasing percentage of special value(s) are invalid
219 invalid_special_param_values_dicts = [
220 {"int": (0, 1.1)}, # >1 probability
221 {"int": (0, 0)}, # Zero probability
222 {"int": (0, -0.1)}, # Negative probability
223 {"int": (0, 20)}, # 2,000% instead of 20%
224 {"int": [0, 1, 2, 3, 4, 5]}, # default biasing is 20%; 6 values * 20% > 100%
225 {"int": [(0, 0.4), (1, 0.7)]}, # combined probability >100%
226 {"int": [(0, -0.4), (1, 0.7)]}, # probability for value 0 is invalid.
227 ]
229 for spv_dict in invalid_special_param_values_dicts:
230 with pytest.raises(ValueError):
231 LlamaTuneAdapter(
232 orig_parameter_space=input_space,
233 num_low_dims=2,
234 special_param_values=spv_dict,
235 max_unique_values_per_param=None,
236 )
239def gen_random_configs(adapter: LlamaTuneAdapter, num_configs: int) -> Iterator[CS.Configuration]:
240 for sampled_config in adapter.target_parameter_space.sample_configuration(size=num_configs):
241 # Transform low-dim config to high-dim config
242 sampled_config_sr = pd.Series(dict(sampled_config))
243 orig_config_sr = adapter.transform(sampled_config_sr)
244 orig_config = CS.Configuration(
245 adapter.orig_parameter_space,
246 values=orig_config_sr.to_dict(),
247 )
248 yield orig_config
251def test_special_parameter_values_biasing() -> None: # pylint: disable=too-complex
252 """Tests LlamaTune's special parameter values biasing methodology."""
253 input_space = CS.ConfigurationSpace(seed=1234)
254 input_space.add(CS.UniformIntegerHyperparameter(name="int_1", lower=0, upper=100))
255 input_space.add(CS.UniformIntegerHyperparameter(name="int_2", lower=0, upper=100))
257 num_configs = 400
258 bias_percentage = LlamaTuneAdapter.DEFAULT_SPECIAL_PARAM_VALUE_BIASING_PERCENTAGE
259 eps = 0.2
261 # Single parameter; single special value
262 special_param_value_dicts: list[dict[str, Any]] = [
263 {"int_1": 0},
264 {"int_1": (0, bias_percentage)},
265 {"int_1": [0]},
266 {"int_1": [(0, bias_percentage)]},
267 ]
269 for spv_dict in special_param_value_dicts:
270 adapter = LlamaTuneAdapter(
271 orig_parameter_space=input_space,
272 num_low_dims=1,
273 special_param_values=spv_dict,
274 max_unique_values_per_param=None,
275 )
277 special_value_occurrences = sum(
278 1 for config in gen_random_configs(adapter, num_configs) if config["int_1"] == 0
279 )
280 assert (1 - eps) * int(num_configs * bias_percentage) <= special_value_occurrences
282 # Single parameter; multiple special values
283 special_param_value_dicts = [
284 {"int_1": [0, 1]},
285 {"int_1": [(0, bias_percentage), (1, bias_percentage)]},
286 ]
288 for spv_dict in special_param_value_dicts:
289 adapter = LlamaTuneAdapter(
290 orig_parameter_space=input_space,
291 num_low_dims=1,
292 special_param_values=spv_dict,
293 max_unique_values_per_param=None,
294 )
296 special_values_occurrences = {0: 0, 1: 0}
297 for config in gen_random_configs(adapter, num_configs):
298 if config["int_1"] == 0:
299 special_values_occurrences[0] += 1
300 elif config["int_1"] == 1:
301 special_values_occurrences[1] += 1
303 assert (1 - eps) * int(num_configs * bias_percentage) <= special_values_occurrences[0]
304 assert (1 - eps) * int(num_configs * bias_percentage) <= special_values_occurrences[1]
306 # Multiple parameters; multiple special values; different biasing percentage
307 spv_dict = {
308 "int_1": [(0, bias_percentage), (1, bias_percentage / 2)],
309 "int_2": [(2, bias_percentage / 2), (100, bias_percentage * 1.5)],
310 }
311 adapter = LlamaTuneAdapter(
312 orig_parameter_space=input_space,
313 num_low_dims=1,
314 special_param_values=spv_dict,
315 max_unique_values_per_param=None,
316 )
318 special_values_instances: dict[str, dict[int, int]] = {
319 "int_1": {0: 0, 1: 0},
320 "int_2": {2: 0, 100: 0},
321 }
322 for config in gen_random_configs(adapter, num_configs):
323 if config["int_1"] == 0:
324 special_values_instances["int_1"][0] += 1
325 elif config["int_1"] == 1:
326 special_values_instances["int_1"][1] += 1
328 if config["int_2"] == 2:
329 special_values_instances["int_2"][2] += 1
330 elif config["int_2"] == 100:
331 special_values_instances["int_2"][100] += 1
333 assert (1 - eps) * int(num_configs * bias_percentage) <= special_values_instances["int_1"][0]
334 assert (1 - eps) * int(num_configs * bias_percentage / 2) <= (
335 special_values_instances["int_1"][1]
336 )
337 assert (1 - eps) * int(num_configs * bias_percentage / 2) <= (
338 special_values_instances["int_2"][2]
339 )
340 assert (1 - eps) * int(num_configs * bias_percentage * 1.5) <= (
341 special_values_instances["int_2"][100]
342 )
345def test_max_unique_values_per_param() -> None:
346 """Tests LlamaTune's parameter values discretization implementation."""
347 # Define config space with a mix of different parameter types
348 input_space = CS.ConfigurationSpace(seed=1234)
349 input_space.add(
350 CS.UniformFloatHyperparameter(name="cont_1", lower=0, upper=5),
351 )
352 input_space.add(CS.UniformFloatHyperparameter(name="cont_2", lower=1, upper=100))
353 input_space.add(CS.UniformIntegerHyperparameter(name="int_1", lower=1, upper=10))
354 input_space.add(CS.UniformIntegerHyperparameter(name="int_2", lower=0, upper=2048))
355 input_space.add(CS.CategoricalHyperparameter(name="str_1", choices=["on", "off"]))
356 input_space.add(
357 CS.CategoricalHyperparameter(name="str_2", choices=[f"choice_{idx}" for idx in range(10)])
358 )
360 # Restrict the number of unique parameter values
361 num_configs = 200
362 for max_unique_values_per_param in (5, 25, 100):
363 adapter = LlamaTuneAdapter(
364 orig_parameter_space=input_space,
365 num_low_dims=3,
366 special_param_values=None,
367 max_unique_values_per_param=max_unique_values_per_param,
368 )
370 # Keep track of unique values generated for each parameter
371 unique_values_dict: dict[str, set] = {param: set() for param in list(input_space.keys())}
372 for config in gen_random_configs(adapter, num_configs):
373 for param, value in config.items():
374 unique_values_dict[param].add(value)
376 # Ensure that their number is less than the maximum number allowed
377 for _, unique_values in unique_values_dict.items():
378 assert len(unique_values) <= max_unique_values_per_param
381@pytest.mark.parametrize(
382 ("num_target_space_dims", "param_space_kwargs"),
383 (
384 [
385 (num_target_space_dims, param_space_kwargs)
386 for num_target_space_dims in (2, 4)
387 for num_orig_space_factor in (1.5, 4)
388 for param_space_kwargs in (
389 {"n_continuous_params": int(num_target_space_dims * num_orig_space_factor)},
390 {"n_integer_params": int(num_target_space_dims * num_orig_space_factor)},
391 {"n_categorical_params": int(num_target_space_dims * num_orig_space_factor)},
392 {"n_quantized_integer_params": int(num_target_space_dims * num_orig_space_factor)},
393 {
394 "n_quantized_continuous_params": int(
395 num_target_space_dims * num_orig_space_factor
396 )
397 },
398 # Mix of all three types
399 {
400 "n_continuous_params": int(num_target_space_dims * num_orig_space_factor / 3),
401 "n_integer_params": int(num_target_space_dims * num_orig_space_factor / 3),
402 "n_categorical_params": int(num_target_space_dims * num_orig_space_factor / 3),
403 },
404 )
405 ]
406 ),
407)
408def test_approx_inverse_mapping(
409 num_target_space_dims: int,
410 param_space_kwargs: dict,
411) -> None: # pylint: disable=too-many-locals
412 """Tests LlamaTune's approximate high-to-low space projection method, using pseudo-
413 inverse.
414 """
415 input_space = construct_parameter_space(**param_space_kwargs)
417 # Enable low-dimensional space projection, but disable reverse mapping
418 adapter = LlamaTuneAdapter(
419 orig_parameter_space=input_space,
420 num_low_dims=num_target_space_dims,
421 special_param_values=None,
422 max_unique_values_per_param=None,
423 use_approximate_reverse_mapping=False,
424 )
426 sampled_config = input_space.sample_configuration() # size=1)
427 with pytest.raises(ValueError):
428 sampled_config_sr = pd.Series(dict(sampled_config))
429 _ = adapter.inverse_transform(sampled_config_sr)
431 # Enable low-dimensional space projection *and* reverse mapping
432 adapter = LlamaTuneAdapter(
433 orig_parameter_space=input_space,
434 num_low_dims=num_target_space_dims,
435 special_param_values=None,
436 max_unique_values_per_param=None,
437 use_approximate_reverse_mapping=True,
438 )
440 # Warning should be printed the first time
441 sampled_config = input_space.sample_configuration() # size=1)
442 with pytest.warns(UserWarning):
443 sampled_config_sr = pd.Series(dict(sampled_config))
444 target_config_sr = adapter.inverse_transform(sampled_config_sr)
445 # Low-dim (i.e., target) config should be valid
446 target_config = CS.Configuration(
447 adapter.target_parameter_space,
448 values=target_config_sr.to_dict(),
449 )
450 target_config.check_valid_configuration()
452 # Test inverse transform with 100 random configs
453 for _ in range(100):
454 sampled_config = input_space.sample_configuration() # size=1)
455 sampled_config_sr = pd.Series(dict(sampled_config))
456 target_config_sr = adapter.inverse_transform(sampled_config_sr)
457 # Low-dim (i.e., target) config should be valid
458 target_config = CS.Configuration(
459 adapter.target_parameter_space,
460 values=target_config_sr.to_dict(),
461 )
462 target_config.check_valid_configuration()
465@pytest.mark.parametrize(
466 ("num_low_dims", "special_param_values", "max_unique_values_per_param"),
467 (
468 [
469 (num_low_dims, special_param_values, max_unique_values_per_param)
470 for num_low_dims in (8, 16)
471 for special_param_values in (
472 {"int_1": -1, "int_2": -1, "int_3": -1, "int_4": [-1, 0]},
473 {
474 "int_1": (-1, 0.1),
475 "int_2": -1,
476 "int_3": (-1, 0.3),
477 "int_4": [(-1, 0.1), (0, 0.2)],
478 },
479 )
480 for max_unique_values_per_param in (50, 250)
481 ]
482 ),
483)
484def test_llamatune_pipeline(
485 num_low_dims: int,
486 special_param_values: dict,
487 max_unique_values_per_param: int,
488) -> None:
489 """Tests LlamaTune space adapter when all components are active."""
490 # pylint: disable=too-many-locals
492 # Define config space with a mix of different parameter types
493 input_space = construct_parameter_space(
494 n_continuous_params=10,
495 n_integer_params=10,
496 n_categorical_params=5,
497 )
498 adapter = LlamaTuneAdapter(
499 orig_parameter_space=input_space,
500 num_low_dims=num_low_dims,
501 special_param_values=special_param_values,
502 max_unique_values_per_param=max_unique_values_per_param,
503 )
505 special_value_occurrences = {
506 # pylint: disable=protected-access
507 param: {special_value: 0 for special_value, _ in tuples_list}
508 for param, tuples_list in adapter._special_param_values_dict.items()
509 }
510 unique_values_dict: dict[str, set] = {param: set() for param in input_space.keys()}
512 num_configs = 1000
513 for (
514 config
515 ) in adapter.target_parameter_space.sample_configuration( # pylint: disable=not-an-iterable
516 size=num_configs
517 ):
518 # Transform low-dim config to high-dim point/config
519 sampled_config_sr = pd.Series(dict(config))
520 orig_config_sr = adapter.transform(sampled_config_sr)
521 # High-dim (i.e., original) config should be valid
522 orig_config = CS.Configuration(input_space, values=orig_config_sr.to_dict())
523 orig_config.check_valid_configuration()
525 # Transform high-dim config back to low-dim
526 target_config_sr = adapter.inverse_transform(orig_config_sr)
527 # Sampled config and this should be the same
528 target_config = CS.Configuration(
529 adapter.target_parameter_space,
530 values=target_config_sr.to_dict(),
531 )
532 assert target_config == config
534 for param, value in orig_config.items():
535 # Keep track of special value occurrences
536 if param in special_value_occurrences:
537 if value in special_value_occurrences[param]:
538 special_value_occurrences[param][value] += 1
540 # Keep track of unique values generated for each parameter
541 unique_values_dict[param].add(value)
543 # Ensure that occurrences of special values do not significantly deviate from expected
544 eps = 0.2
545 for (
546 param,
547 tuples_list,
548 ) in adapter._special_param_values_dict.items(): # pylint: disable=protected-access
549 for value, bias_percentage in tuples_list:
550 assert (1 - eps) * int(num_configs * bias_percentage) <= special_value_occurrences[
551 param
552 ][value]
554 # Ensure that number of unique values is less than the maximum number allowed
555 for _, unique_values in unique_values_dict.items():
556 assert len(unique_values) <= max_unique_values_per_param
559@pytest.mark.parametrize(
560 ("num_target_space_dims", "param_space_kwargs"),
561 (
562 [
563 (num_target_space_dims, param_space_kwargs)
564 for num_target_space_dims in (2, 4)
565 for num_orig_space_factor in (1.5, 4)
566 for param_space_kwargs in (
567 {"n_continuous_params": int(num_target_space_dims * num_orig_space_factor)},
568 {"n_integer_params": int(num_target_space_dims * num_orig_space_factor)},
569 {"n_categorical_params": int(num_target_space_dims * num_orig_space_factor)},
570 # Mix of all three types
571 {
572 "n_continuous_params": int(num_target_space_dims * num_orig_space_factor / 3),
573 "n_integer_params": int(num_target_space_dims * num_orig_space_factor / 3),
574 "n_categorical_params": int(num_target_space_dims * num_orig_space_factor / 3),
575 },
576 )
577 ]
578 ),
579)
580def test_deterministic_behavior_for_same_seed(
581 num_target_space_dims: int,
582 param_space_kwargs: dict,
583) -> None:
584 """Tests LlamaTune's space adapter deterministic behavior when given same seed in
585 the input parameter space.
586 """
588 def generate_target_param_space_configs(seed: int) -> list[CS.Configuration]:
589 input_space = construct_parameter_space(**param_space_kwargs, seed=seed)
591 # Init adapter and sample points in the low-dim space
592 adapter = LlamaTuneAdapter(
593 orig_parameter_space=input_space,
594 num_low_dims=num_target_space_dims,
595 special_param_values=None,
596 max_unique_values_per_param=None,
597 use_approximate_reverse_mapping=False,
598 )
600 sample_configs: list[CS.Configuration] = (
601 adapter.target_parameter_space.sample_configuration(size=100)
602 )
603 return sample_configs
605 assert generate_target_param_space_configs(42) == generate_target_param_space_configs(42)
606 assert generate_target_param_space_configs(1234) != generate_target_param_space_configs(42)