Coverage for mlos_bench/mlos_bench/tests/environments/mock_env_test.py: 100%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5"""Unit tests for mock benchmark environment.""" 

6import pytest 

7 

8from mlos_bench.environments.mock_env import MockEnv 

9from mlos_bench.tunables.tunable_groups import TunableGroups 

10 

11 

12def test_mock_env_default(mock_env: MockEnv, tunable_groups: TunableGroups) -> None: 

13 """Check the default values of the mock environment.""" 

14 with mock_env as env_context: 

15 assert env_context.setup(tunable_groups) 

16 (status, _ts, data) = env_context.run() 

17 assert status.is_succeeded() 

18 assert data is not None 

19 assert data["score"] == pytest.approx(73.97, 0.01) 

20 # Second time, results should differ because of the noise. 

21 (status, _ts, data) = env_context.run() 

22 assert status.is_succeeded() 

23 assert data is not None 

24 assert data["score"] == pytest.approx(72.92, 0.01) 

25 

26 

27def test_mock_env_no_noise(mock_env_no_noise: MockEnv, tunable_groups: TunableGroups) -> None: 

28 """Check the default values of the mock environment.""" 

29 with mock_env_no_noise as env_context: 

30 assert env_context.setup(tunable_groups) 

31 for _ in range(10): 

32 # Noise-free results should be the same every time. 

33 (status, _ts, data) = env_context.run() 

34 assert status.is_succeeded() 

35 assert data is not None 

36 assert data["score"] == pytest.approx(75.0, 0.01) 

37 

38 

39@pytest.mark.parametrize( 

40 ("tunable_values", "expected_score"), 

41 [ 

42 ( 

43 {"vmSize": "Standard_B2ms", "idle": "halt", "kernel_sched_migration_cost_ns": 250000}, 

44 66.4, 

45 ), 

46 ( 

47 {"vmSize": "Standard_B4ms", "idle": "halt", "kernel_sched_migration_cost_ns": 40000}, 

48 74.06, 

49 ), 

50 ], 

51) 

52def test_mock_env_assign( 

53 mock_env: MockEnv, 

54 tunable_groups: TunableGroups, 

55 tunable_values: dict, 

56 expected_score: float, 

57) -> None: 

58 """Check the benchmark values of the mock environment after the assignment.""" 

59 with mock_env as env_context: 

60 tunable_groups.assign(tunable_values) 

61 assert env_context.setup(tunable_groups) 

62 (status, _ts, data) = env_context.run() 

63 assert status.is_succeeded() 

64 assert data is not None 

65 assert data["score"] == pytest.approx(expected_score, 0.01) 

66 

67 

68@pytest.mark.parametrize( 

69 ("tunable_values", "expected_score"), 

70 [ 

71 ( 

72 {"vmSize": "Standard_B2ms", "idle": "halt", "kernel_sched_migration_cost_ns": 250000}, 

73 67.5, 

74 ), 

75 ( 

76 {"vmSize": "Standard_B4ms", "idle": "halt", "kernel_sched_migration_cost_ns": 40000}, 

77 75.1, 

78 ), 

79 ], 

80) 

81def test_mock_env_no_noise_assign( 

82 mock_env_no_noise: MockEnv, 

83 tunable_groups: TunableGroups, 

84 tunable_values: dict, 

85 expected_score: float, 

86) -> None: 

87 """Check the benchmark values of the noiseless mock environment after the 

88 assignment. 

89 """ 

90 with mock_env_no_noise as env_context: 

91 tunable_groups.assign(tunable_values) 

92 assert env_context.setup(tunable_groups) 

93 for _ in range(10): 

94 # Noise-free environment should produce the same results every time. 

95 (status, _ts, data) = env_context.run() 

96 assert status.is_succeeded() 

97 assert data is not None 

98 assert data["score"] == pytest.approx(expected_score, 0.01)