Coverage for mlos_bench/mlos_bench/tests/storage/exp_load_test.py: 100%
71 statements
« prev ^ index » next coverage.py v7.6.1, created at 2024-10-07 01:52 +0000
1#
2# Copyright (c) Microsoft Corporation.
3# Licensed under the MIT License.
4#
5"""Unit tests for the storage subsystem."""
6from datetime import datetime, tzinfo
7from typing import Optional
9import pytest
10from pytz import UTC
12from mlos_bench.environments.status import Status
13from mlos_bench.storage.base_storage import Storage
14from mlos_bench.tests import ZONE_INFO
15from mlos_bench.tunables.tunable_groups import TunableGroups
18def test_exp_load_empty(exp_storage: Storage.Experiment) -> None:
19 """Try to retrieve old experimental data from the empty storage."""
20 (trial_ids, configs, scores, status) = exp_storage.load()
21 assert not trial_ids
22 assert not configs
23 assert not scores
24 assert not status
27def test_exp_pending_empty(exp_storage: Storage.Experiment) -> None:
28 """Try to retrieve pending experiments from the empty storage."""
29 trials = list(exp_storage.pending_trials(datetime.now(UTC), running=True))
30 assert not trials
33@pytest.mark.parametrize(("zone_info"), ZONE_INFO)
34def test_exp_trial_pending(
35 exp_storage: Storage.Experiment,
36 tunable_groups: TunableGroups,
37 zone_info: Optional[tzinfo],
38) -> None:
39 """Start a trial and check that it is pending."""
40 trial = exp_storage.new_trial(tunable_groups)
41 (pending,) = list(exp_storage.pending_trials(datetime.now(zone_info), running=True))
42 assert pending.trial_id == trial.trial_id
43 assert pending.tunables == tunable_groups
46@pytest.mark.parametrize(("zone_info"), ZONE_INFO)
47def test_exp_trial_pending_many(
48 exp_storage: Storage.Experiment,
49 tunable_groups: TunableGroups,
50 zone_info: Optional[tzinfo],
51) -> None:
52 """Start THREE trials and check that both are pending."""
53 config1 = tunable_groups.copy().assign({"idle": "mwait"})
54 config2 = tunable_groups.copy().assign({"idle": "noidle"})
55 trial_ids = {
56 exp_storage.new_trial(config1).trial_id,
57 exp_storage.new_trial(config2).trial_id,
58 exp_storage.new_trial(config2).trial_id, # Submit same config twice
59 }
60 pending_ids = {
61 pending.trial_id
62 for pending in exp_storage.pending_trials(datetime.now(zone_info), running=True)
63 }
64 assert len(pending_ids) == 3
65 assert trial_ids == pending_ids
68@pytest.mark.parametrize(("zone_info"), ZONE_INFO)
69def test_exp_trial_pending_fail(
70 exp_storage: Storage.Experiment,
71 tunable_groups: TunableGroups,
72 zone_info: Optional[tzinfo],
73) -> None:
74 """Start a trial, fail it, and and check that it is NOT pending."""
75 trial = exp_storage.new_trial(tunable_groups)
76 trial.update(Status.FAILED, datetime.now(zone_info))
77 trials = list(exp_storage.pending_trials(datetime.now(zone_info), running=True))
78 assert not trials
81@pytest.mark.parametrize(("zone_info"), ZONE_INFO)
82def test_exp_trial_success(
83 exp_storage: Storage.Experiment,
84 tunable_groups: TunableGroups,
85 zone_info: Optional[tzinfo],
86) -> None:
87 """Start a trial, finish it successfully, and and check that it is NOT pending."""
88 trial = exp_storage.new_trial(tunable_groups)
89 trial.update(Status.SUCCEEDED, datetime.now(zone_info), {"score": 99.9})
90 trials = list(exp_storage.pending_trials(datetime.now(zone_info), running=True))
91 assert not trials
94@pytest.mark.parametrize(("zone_info"), ZONE_INFO)
95def test_exp_trial_update_categ(
96 exp_storage: Storage.Experiment,
97 tunable_groups: TunableGroups,
98 zone_info: Optional[tzinfo],
99) -> None:
100 """Update the trial with multiple metrics, some of which are categorical."""
101 trial = exp_storage.new_trial(tunable_groups)
102 trial.update(Status.SUCCEEDED, datetime.now(zone_info), {"score": 99.9, "benchmark": "test"})
103 assert exp_storage.load() == (
104 [trial.trial_id],
105 [
106 {
107 "idle": "halt",
108 "kernel_sched_latency_ns": "2000000",
109 "kernel_sched_migration_cost_ns": "-1",
110 "vmSize": "Standard_B4ms",
111 }
112 ],
113 [{"score": "99.9", "benchmark": "test"}],
114 [Status.SUCCEEDED],
115 )
118@pytest.mark.parametrize(("zone_info"), ZONE_INFO)
119def test_exp_trial_update_twice(
120 exp_storage: Storage.Experiment,
121 tunable_groups: TunableGroups,
122 zone_info: Optional[tzinfo],
123) -> None:
124 """Update the trial status twice and receive an error."""
125 trial = exp_storage.new_trial(tunable_groups)
126 trial.update(Status.FAILED, datetime.now(zone_info))
127 with pytest.raises(RuntimeError):
128 trial.update(Status.SUCCEEDED, datetime.now(UTC), {"score": 99.9})
131@pytest.mark.parametrize(("zone_info"), ZONE_INFO)
132def test_exp_trial_pending_3(
133 exp_storage: Storage.Experiment,
134 tunable_groups: TunableGroups,
135 zone_info: Optional[tzinfo],
136) -> None:
137 """
138 Start THREE trials, let one succeed, another one fail and keep one not updated.
140 Check that one is still pending another one can be loaded into the optimizer.
141 """
142 score = 99.9
144 trial_fail = exp_storage.new_trial(tunable_groups)
145 trial_succ = exp_storage.new_trial(tunable_groups)
146 trial_pend = exp_storage.new_trial(tunable_groups)
148 trial_fail.update(Status.FAILED, datetime.now(zone_info))
149 trial_succ.update(Status.SUCCEEDED, datetime.now(zone_info), {"score": score})
151 (pending,) = list(exp_storage.pending_trials(datetime.now(UTC), running=True))
152 assert pending.trial_id == trial_pend.trial_id
154 (trial_ids, configs, scores, status) = exp_storage.load()
155 assert trial_ids == [trial_fail.trial_id, trial_succ.trial_id]
156 assert len(configs) == 2
157 assert scores == [None, {"score": f"{score}"}]
158 assert status == [Status.FAILED, Status.SUCCEEDED]
159 assert tunable_groups.copy().assign(configs[0]).reset() == trial_fail.tunables
160 assert tunable_groups.copy().assign(configs[1]).reset() == trial_succ.tunables