Coverage for mlos_bench/mlos_bench/tests/environments/local/local_env_telemetry_test.py: 100%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5"""Unit tests for telemetry and status of LocalEnv benchmark environment.""" 

6from datetime import datetime, timedelta, tzinfo 

7from typing import Optional 

8 

9import pytest 

10from pytz import UTC 

11 

12from mlos_bench.tests import ZONE_INFO 

13from mlos_bench.tests.environments import check_env_fail_telemetry, check_env_success 

14from mlos_bench.tests.environments.local import create_local_env 

15from mlos_bench.tunables.tunable_groups import TunableGroups 

16 

17 

18def _format_str(zone_info: Optional[tzinfo]) -> str: 

19 if zone_info is not None: 

20 return "%Y-%m-%d %H:%M:%S %z" 

21 return "%Y-%m-%d %H:%M:%S" 

22 

23 

24# FIXME: This fails with zone_info = None when run with `TZ="America/Chicago pytest -n0 ...` 

25@pytest.mark.parametrize(("zone_info"), ZONE_INFO) 

26def test_local_env_telemetry(tunable_groups: TunableGroups, zone_info: Optional[tzinfo]) -> None: 

27 """Produce benchmark and telemetry data in a local script and read it.""" 

28 ts1 = datetime.now(zone_info) 

29 ts1 -= timedelta(microseconds=ts1.microsecond) # Round to a second 

30 ts2 = ts1 + timedelta(minutes=1) 

31 

32 format_str = _format_str(zone_info) 

33 time_str1 = ts1.strftime(format_str) 

34 time_str2 = ts2.strftime(format_str) 

35 

36 local_env = create_local_env( 

37 tunable_groups, 

38 { 

39 "run": [ 

40 "echo 'metric,value' > output.csv", 

41 "echo 'latency,4.1' >> output.csv", 

42 "echo 'throughput,512' >> output.csv", 

43 "echo 'score,0.95' >> output.csv", 

44 "echo '-------------------'", # This output does not go anywhere 

45 "echo 'timestamp,metric,value' > telemetry.csv", 

46 f"echo {time_str1},cpu_load,0.65 >> telemetry.csv", 

47 f"echo {time_str1},mem_usage,10240 >> telemetry.csv", 

48 f"echo {time_str2},cpu_load,0.8 >> telemetry.csv", 

49 f"echo {time_str2},mem_usage,20480 >> telemetry.csv", 

50 ], 

51 "read_results_file": "output.csv", 

52 "read_telemetry_file": "telemetry.csv", 

53 }, 

54 ) 

55 

56 check_env_success( 

57 local_env, 

58 tunable_groups, 

59 expected_results={ 

60 "latency": 4.1, 

61 "throughput": 512.0, 

62 "score": 0.95, 

63 }, 

64 expected_telemetry=[ 

65 (ts1.astimezone(UTC), "cpu_load", 0.65), 

66 (ts1.astimezone(UTC), "mem_usage", 10240.0), 

67 (ts2.astimezone(UTC), "cpu_load", 0.8), 

68 (ts2.astimezone(UTC), "mem_usage", 20480.0), 

69 ], 

70 ) 

71 

72 

73# FIXME: This fails with zone_info = None when run with `TZ="America/Chicago pytest -n0 ...` 

74@pytest.mark.parametrize(("zone_info"), ZONE_INFO) 

75def test_local_env_telemetry_no_header( 

76 tunable_groups: TunableGroups, 

77 zone_info: Optional[tzinfo], 

78) -> None: 

79 """Read the telemetry data with no header.""" 

80 ts1 = datetime.now(zone_info) 

81 ts1 -= timedelta(microseconds=ts1.microsecond) # Round to a second 

82 ts2 = ts1 + timedelta(minutes=1) 

83 

84 format_str = _format_str(zone_info) 

85 time_str1 = ts1.strftime(format_str) 

86 time_str2 = ts2.strftime(format_str) 

87 

88 local_env = create_local_env( 

89 tunable_groups, 

90 { 

91 "run": [ 

92 f"echo {time_str1},cpu_load,0.65 > telemetry.csv", 

93 f"echo {time_str1},mem_usage,10240 >> telemetry.csv", 

94 f"echo {time_str2},cpu_load,0.8 >> telemetry.csv", 

95 f"echo {time_str2},mem_usage,20480 >> telemetry.csv", 

96 ], 

97 "read_telemetry_file": "telemetry.csv", 

98 }, 

99 ) 

100 

101 check_env_success( 

102 local_env, 

103 tunable_groups, 

104 expected_results={}, 

105 expected_telemetry=[ 

106 (ts1.astimezone(UTC), "cpu_load", 0.65), 

107 (ts1.astimezone(UTC), "mem_usage", 10240.0), 

108 (ts2.astimezone(UTC), "cpu_load", 0.8), 

109 (ts2.astimezone(UTC), "mem_usage", 20480.0), 

110 ], 

111 ) 

112 

113 

114@pytest.mark.filterwarnings( 

115 ( 

116 "ignore:.*(Could not infer format, so each element will be parsed individually, " 

117 "falling back to `dateutil`).*:UserWarning::0" 

118 ) 

119) # pylint: disable=line-too-long # noqa 

120@pytest.mark.parametrize(("zone_info"), ZONE_INFO) 

121def test_local_env_telemetry_wrong_header( 

122 tunable_groups: TunableGroups, 

123 zone_info: Optional[tzinfo], 

124) -> None: 

125 """Read the telemetry data with incorrect header.""" 

126 ts1 = datetime.now(zone_info) 

127 ts1 -= timedelta(microseconds=ts1.microsecond) # Round to a second 

128 ts2 = ts1 + timedelta(minutes=1) 

129 

130 format_str = _format_str(zone_info) 

131 time_str1 = ts1.strftime(format_str) 

132 time_str2 = ts2.strftime(format_str) 

133 

134 local_env = create_local_env( 

135 tunable_groups, 

136 { 

137 "run": [ 

138 # Error: the data is correct, but the header has unexpected column names 

139 "echo 'ts,metric_name,metric_value' > telemetry.csv", 

140 f"echo {time_str1},cpu_load,0.65 >> telemetry.csv", 

141 f"echo {time_str1},mem_usage,10240 >> telemetry.csv", 

142 f"echo {time_str2},cpu_load,0.8 >> telemetry.csv", 

143 f"echo {time_str2},mem_usage,20480 >> telemetry.csv", 

144 ], 

145 "read_telemetry_file": "telemetry.csv", 

146 }, 

147 ) 

148 

149 check_env_fail_telemetry(local_env, tunable_groups) 

150 

151 

152def test_local_env_telemetry_invalid(tunable_groups: TunableGroups) -> None: 

153 """Fail when the telemetry data has wrong format.""" 

154 zone_info = UTC 

155 ts1 = datetime.now(zone_info) 

156 ts1 -= timedelta(microseconds=ts1.microsecond) # Round to a second 

157 ts2 = ts1 + timedelta(minutes=1) 

158 

159 format_str = _format_str(zone_info) 

160 time_str1 = ts1.strftime(format_str) 

161 time_str2 = ts2.strftime(format_str) 

162 

163 local_env = create_local_env( 

164 tunable_groups, 

165 { 

166 "run": [ 

167 # Error: too many columns 

168 f"echo {time_str1},EXTRA,cpu_load,0.65 > telemetry.csv", 

169 f"echo {time_str1},EXTRA,mem_usage,10240 >> telemetry.csv", 

170 f"echo {time_str2},EXTRA,cpu_load,0.8 >> telemetry.csv", 

171 f"echo {time_str2},EXTRA,mem_usage,20480 >> telemetry.csv", 

172 ], 

173 "read_telemetry_file": "telemetry.csv", 

174 }, 

175 ) 

176 

177 check_env_fail_telemetry(local_env, tunable_groups) 

178 

179 

180def test_local_env_telemetry_invalid_ts(tunable_groups: TunableGroups) -> None: 

181 """Fail when the telemetry data has wrong format.""" 

182 local_env = create_local_env( 

183 tunable_groups, 

184 { 

185 "run": [ 

186 # Error: field 1 must be a timestamp 

187 "echo 1,cpu_load,0.65 > telemetry.csv", 

188 "echo 2,mem_usage,10240 >> telemetry.csv", 

189 "echo 3,cpu_load,0.8 >> telemetry.csv", 

190 "echo 4,mem_usage,20480 >> telemetry.csv", 

191 ], 

192 "read_telemetry_file": "telemetry.csv", 

193 }, 

194 ) 

195 

196 check_env_fail_telemetry(local_env, tunable_groups)