Coverage for mlos_bench/mlos_bench/tests/tunables/tunable_to_configspace_test.py: 100%
43 statements
« prev ^ index » next coverage.py v7.6.1, created at 2024-10-07 01:52 +0000
1#
2# Copyright (c) Microsoft Corporation.
3# Licensed under the MIT License.
4#
5"""Unit tests for Tunable to ConfigSpace conversion."""
7import pytest
8from ConfigSpace import (
9 CategoricalHyperparameter,
10 ConfigurationSpace,
11 EqualsCondition,
12 Integer,
13 UniformFloatHyperparameter,
14 UniformIntegerHyperparameter,
15)
17from mlos_bench.optimizers.convert_configspace import (
18 TunableValueKind,
19 _tunable_to_configspace,
20 special_param_names,
21 tunable_groups_to_configspace,
22)
23from mlos_bench.tunables.tunable import Tunable
24from mlos_bench.tunables.tunable_groups import TunableGroups
25from mlos_core.spaces.converters.util import (
26 QUANTIZATION_BINS_META_KEY,
27 monkey_patch_cs_quantization,
28)
30# pylint: disable=redefined-outer-name
33@pytest.fixture
34def configuration_space() -> ConfigurationSpace:
35 """
36 A test fixture that produces a mock ConfigurationSpace object matching the
37 tunable_groups fixture.
39 Returns
40 -------
41 configuration_space : ConfigurationSpace
42 A new ConfigurationSpace object for testing.
43 """
44 (kernel_sched_migration_cost_ns_special, kernel_sched_migration_cost_ns_type) = (
45 special_param_names("kernel_sched_migration_cost_ns")
46 )
48 # NOTE: FLAML requires distribution to be uniform
49 spaces = ConfigurationSpace(
50 space=[
51 CategoricalHyperparameter(
52 name="vmSize",
53 choices=["Standard_B2s", "Standard_B2ms", "Standard_B4ms"],
54 default_value="Standard_B4ms",
55 meta={"group": "provision", "cost": 0},
56 ),
57 CategoricalHyperparameter(
58 name="idle",
59 choices=["halt", "mwait", "noidle"],
60 default_value="halt",
61 meta={"group": "boot", "cost": 0},
62 ),
63 Integer(
64 name="kernel_sched_latency_ns",
65 bounds=(0, 1000000000),
66 log=False,
67 default=2000000,
68 meta={
69 "group": "kernel",
70 "cost": 0,
71 QUANTIZATION_BINS_META_KEY: 11,
72 },
73 ),
74 Integer(
75 name="kernel_sched_migration_cost_ns",
76 bounds=(0, 500000),
77 log=False,
78 default=250000,
79 meta={"group": "kernel", "cost": 0},
80 ),
81 CategoricalHyperparameter(
82 name=kernel_sched_migration_cost_ns_special,
83 choices=[-1, 0],
84 weights=[0.5, 0.5],
85 default_value=-1,
86 meta={"group": "kernel", "cost": 0},
87 ),
88 CategoricalHyperparameter(
89 name=kernel_sched_migration_cost_ns_type,
90 choices=[TunableValueKind.SPECIAL, TunableValueKind.RANGE],
91 weights=[0.5, 0.5],
92 default_value=TunableValueKind.SPECIAL,
93 ),
94 ]
95 )
96 spaces.add(
97 [
98 EqualsCondition(
99 spaces[kernel_sched_migration_cost_ns_special],
100 spaces[kernel_sched_migration_cost_ns_type],
101 TunableValueKind.SPECIAL,
102 ),
103 EqualsCondition(
104 spaces["kernel_sched_migration_cost_ns"],
105 spaces[kernel_sched_migration_cost_ns_type],
106 TunableValueKind.RANGE,
107 ),
108 ]
109 )
110 return monkey_patch_cs_quantization(spaces)
113def _cmp_tunable_hyperparameter_categorical(tunable: Tunable, space: ConfigurationSpace) -> None:
114 """Check if categorical Tunable and ConfigSpace Hyperparameter actually match."""
115 param = space[tunable.name]
116 assert isinstance(param, CategoricalHyperparameter)
117 assert set(param.choices) == set(tunable.categories)
118 assert param.default_value == tunable.value
121def _cmp_tunable_hyperparameter_numerical(tunable: Tunable, space: ConfigurationSpace) -> None:
122 """Check if integer Tunable and ConfigSpace Hyperparameter actually match."""
123 param = space[tunable.name]
124 assert isinstance(param, (UniformIntegerHyperparameter, UniformFloatHyperparameter))
125 assert (param.lower, param.upper) == tuple(tunable.range)
126 if tunable.in_range(tunable.value):
127 assert param.default_value == tunable.value
128 assert (param.meta or {}).get(QUANTIZATION_BINS_META_KEY) == tunable.quantization_bins
129 if tunable.quantization_bins:
130 assert param.sample_value() in list(tunable.quantized_values or [])
133def test_tunable_to_configspace_categorical(tunable_categorical: Tunable) -> None:
134 """Check the conversion of Tunable to CategoricalHyperparameter."""
135 cs_param = _tunable_to_configspace(tunable_categorical)
136 _cmp_tunable_hyperparameter_categorical(tunable_categorical, cs_param)
139def test_tunable_to_configspace_int(tunable_int: Tunable) -> None:
140 """Check the conversion of Tunable to UniformIntegerHyperparameter."""
141 cs_param = _tunable_to_configspace(tunable_int)
142 _cmp_tunable_hyperparameter_numerical(tunable_int, cs_param)
145def test_tunable_to_configspace_float(tunable_float: Tunable) -> None:
146 """Check the conversion of Tunable to UniformFloatHyperparameter."""
147 cs_param = _tunable_to_configspace(tunable_float)
148 _cmp_tunable_hyperparameter_numerical(tunable_float, cs_param)
151_CMP_FUNC = {
152 "int": _cmp_tunable_hyperparameter_numerical,
153 "float": _cmp_tunable_hyperparameter_numerical,
154 "categorical": _cmp_tunable_hyperparameter_categorical,
155}
158def test_tunable_groups_to_hyperparameters(tunable_groups: TunableGroups) -> None:
159 """
160 Check the conversion of TunableGroups to ConfigurationSpace.
162 Make sure that the corresponding Tunable and Hyperparameter objects match.
163 """
164 space = tunable_groups_to_configspace(tunable_groups)
165 for tunable, _group in tunable_groups:
166 _CMP_FUNC[tunable.type](tunable, space)
169def test_tunable_groups_to_configspace(
170 tunable_groups: TunableGroups,
171 configuration_space: ConfigurationSpace,
172) -> None:
173 """Check the conversion of the entire TunableGroups collection to a single
174 ConfigurationSpace object.
175 """
176 space = tunable_groups_to_configspace(tunable_groups)
177 assert space == configuration_space