Coverage for mlos_bench/mlos_bench/tests/environments/local/composite_local_env_test.py: 100%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5"""Unit tests for the composition of several LocalEnv benchmark environments.""" 

6import sys 

7from datetime import datetime, timedelta, tzinfo 

8 

9import pytest 

10from pytz import UTC 

11 

12from mlos_bench.tests import ZONE_INFO 

13from mlos_bench.tests.environments import check_env_success 

14from mlos_bench.tests.environments.local import create_composite_local_env 

15from mlos_bench.tunables.tunable_groups import TunableGroups 

16 

17 

18def _format_str(zone_info: tzinfo | None) -> str: 

19 if zone_info is not None: 

20 return "%Y-%m-%d %H:%M:%S %z" 

21 return "%Y-%m-%d %H:%M:%S" 

22 

23 

24# FIXME: This fails with zone_info = None when run with `TZ="America/Chicago pytest -n0 ...` 

25@pytest.mark.parametrize(("zone_info"), ZONE_INFO) 

26def test_composite_env(tunable_groups: TunableGroups, zone_info: tzinfo | None) -> None: 

27 """ 

28 Produce benchmark and telemetry data in TWO local environments and combine the 

29 results. 

30 

31 Also checks that global configs flow down at least one level of CompositeEnv 

32 to its children without being explicitly specified in the CompositeEnv so they 

33 can be used in the shell_envs by its children. 

34 See Also: http://github.com/microsoft/MLOS/issues/501 

35 """ 

36 ts1 = datetime.now(zone_info) 

37 ts1 -= timedelta(microseconds=ts1.microsecond) # Round to a second 

38 ts2 = ts1 + timedelta(minutes=2) 

39 

40 format_str = _format_str(zone_info) 

41 time_str1 = ts1.strftime(format_str) 

42 time_str2 = ts2.strftime(format_str) 

43 

44 (var_prefix, var_suffix) = ("%", "%") if sys.platform == "win32" else ("$", "") 

45 

46 env = create_composite_local_env( 

47 tunable_groups=tunable_groups, 

48 global_config={ 

49 "reads": 2222, 

50 "writes": 1111, 

51 }, 

52 params={ 

53 "const_args": { 

54 "latency": 4.2, 

55 "throughput": 768, 

56 "errors": 0, 

57 } 

58 }, 

59 local_configs=[ 

60 { 

61 "const_args": { 

62 "latency": 3.3, 

63 "reads": 0, 

64 }, 

65 "required_args": ["errors", "reads"], 

66 "shell_env_params": [ 

67 # const_args overridden by the composite env 

68 "latency", 

69 # Comes from the parent const_args 

70 "errors", 

71 # const_args overridden by the global config 

72 "reads", 

73 ], 

74 "run": [ 

75 "echo 'metric,value' > output.csv", 

76 f"echo 'latency,{var_prefix}latency{var_suffix}' >> output.csv", 

77 f"echo 'errors,{var_prefix}errors{var_suffix}' >> output.csv", 

78 f"echo 'reads,{var_prefix}reads{var_suffix}' >> output.csv", 

79 "echo '-------------------'", # This output does not go anywhere 

80 "echo 'timestamp,metric,value' > telemetry.csv", 

81 f"echo {time_str1},cpu_load,0.64 >> telemetry.csv", 

82 f"echo {time_str1},mem_usage,5120 >> telemetry.csv", 

83 ], 

84 "read_results_file": "output.csv", 

85 "read_telemetry_file": "telemetry.csv", 

86 }, 

87 { 

88 "const_args": { 

89 "throughput": 999, 

90 "score": 0.97, 

91 }, 

92 "required_args": ["writes"], 

93 "shell_env_params": [ 

94 # const_args overridden by the composite env 

95 "throughput", 

96 # Comes from the local const_args 

97 "score", 

98 # Comes straight from the global config 

99 "writes", 

100 ], 

101 "run": [ 

102 "echo 'metric,value' > output.csv", 

103 f"echo 'throughput,{var_prefix}throughput{var_suffix}' >> output.csv", 

104 f"echo 'score,{var_prefix}score{var_suffix}' >> output.csv", 

105 f"echo 'writes,{var_prefix}writes{var_suffix}' >> output.csv", 

106 "echo '-------------------'", # This output does not go anywhere 

107 "echo 'timestamp,metric,value' > telemetry.csv", 

108 f"echo {time_str2},cpu_load,0.79 >> telemetry.csv", 

109 f"echo {time_str2},mem_usage,40960 >> telemetry.csv", 

110 ], 

111 "read_results_file": "output.csv", 

112 "read_telemetry_file": "telemetry.csv", 

113 }, 

114 ], 

115 ) 

116 

117 check_env_success( 

118 env, 

119 tunable_groups, 

120 expected_results={ 

121 "latency": 4.2, 

122 "throughput": 768.0, 

123 "score": 0.97, 

124 "errors": 0.0, 

125 "reads": 2222.0, 

126 "writes": 1111.0, 

127 }, 

128 expected_telemetry=[ 

129 (ts1.astimezone(UTC), "cpu_load", 0.64), 

130 (ts1.astimezone(UTC), "mem_usage", 5120.0), 

131 (ts2.astimezone(UTC), "cpu_load", 0.79), 

132 (ts2.astimezone(UTC), "mem_usage", 40960.0), 

133 ], 

134 )