Coverage for mlos_bench/mlos_bench/tests/environments/local/local_env_telemetry_test.py: 100%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5"""Unit tests for telemetry and status of LocalEnv benchmark environment.""" 

6from datetime import datetime, timedelta, tzinfo 

7 

8import pytest 

9from pytz import UTC 

10 

11from mlos_bench.tests import ZONE_INFO 

12from mlos_bench.tests.environments import check_env_fail_telemetry, check_env_success 

13from mlos_bench.tests.environments.local import create_local_env 

14from mlos_bench.tunables.tunable_groups import TunableGroups 

15 

16 

17def _format_str(zone_info: tzinfo | None) -> str: 

18 if zone_info is not None: 

19 return "%Y-%m-%d %H:%M:%S %z" 

20 return "%Y-%m-%d %H:%M:%S" 

21 

22 

23# FIXME: This fails with zone_info = None when run with `TZ="America/Chicago pytest -n0 ...` 

24@pytest.mark.parametrize(("zone_info"), ZONE_INFO) 

25def test_local_env_telemetry(tunable_groups: TunableGroups, zone_info: tzinfo | None) -> None: 

26 """Produce benchmark and telemetry data in a local script and read it.""" 

27 ts1 = datetime.now(zone_info) 

28 ts1 -= timedelta(microseconds=ts1.microsecond) # Round to a second 

29 ts2 = ts1 + timedelta(minutes=1) 

30 

31 format_str = _format_str(zone_info) 

32 time_str1 = ts1.strftime(format_str) 

33 time_str2 = ts2.strftime(format_str) 

34 

35 local_env = create_local_env( 

36 tunable_groups, 

37 { 

38 "run": [ 

39 "echo 'metric,value' > output.csv", 

40 "echo 'latency,4.1' >> output.csv", 

41 "echo 'throughput,512' >> output.csv", 

42 "echo 'score,0.95' >> output.csv", 

43 "echo '-------------------'", # This output does not go anywhere 

44 "echo 'timestamp,metric,value' > telemetry.csv", 

45 f"echo {time_str1},cpu_load,0.65 >> telemetry.csv", 

46 f"echo {time_str1},mem_usage,10240 >> telemetry.csv", 

47 f"echo {time_str2},cpu_load,0.8 >> telemetry.csv", 

48 f"echo {time_str2},mem_usage,20480 >> telemetry.csv", 

49 ], 

50 "read_results_file": "output.csv", 

51 "read_telemetry_file": "telemetry.csv", 

52 }, 

53 ) 

54 

55 check_env_success( 

56 local_env, 

57 tunable_groups, 

58 expected_results={ 

59 "latency": 4.1, 

60 "throughput": 512.0, 

61 "score": 0.95, 

62 }, 

63 expected_telemetry=[ 

64 (ts1.astimezone(UTC), "cpu_load", 0.65), 

65 (ts1.astimezone(UTC), "mem_usage", 10240.0), 

66 (ts2.astimezone(UTC), "cpu_load", 0.8), 

67 (ts2.astimezone(UTC), "mem_usage", 20480.0), 

68 ], 

69 ) 

70 

71 

72# FIXME: This fails with zone_info = None when run with `TZ="America/Chicago pytest -n0 ...` 

73@pytest.mark.parametrize(("zone_info"), ZONE_INFO) 

74def test_local_env_telemetry_no_header( 

75 tunable_groups: TunableGroups, 

76 zone_info: tzinfo | None, 

77) -> None: 

78 """Read the telemetry data with no header.""" 

79 ts1 = datetime.now(zone_info) 

80 ts1 -= timedelta(microseconds=ts1.microsecond) # Round to a second 

81 ts2 = ts1 + timedelta(minutes=1) 

82 

83 format_str = _format_str(zone_info) 

84 time_str1 = ts1.strftime(format_str) 

85 time_str2 = ts2.strftime(format_str) 

86 

87 local_env = create_local_env( 

88 tunable_groups, 

89 { 

90 "run": [ 

91 f"echo {time_str1},cpu_load,0.65 > telemetry.csv", 

92 f"echo {time_str1},mem_usage,10240 >> telemetry.csv", 

93 f"echo {time_str2},cpu_load,0.8 >> telemetry.csv", 

94 f"echo {time_str2},mem_usage,20480 >> telemetry.csv", 

95 ], 

96 "read_telemetry_file": "telemetry.csv", 

97 }, 

98 ) 

99 

100 check_env_success( 

101 local_env, 

102 tunable_groups, 

103 expected_results={}, 

104 expected_telemetry=[ 

105 (ts1.astimezone(UTC), "cpu_load", 0.65), 

106 (ts1.astimezone(UTC), "mem_usage", 10240.0), 

107 (ts2.astimezone(UTC), "cpu_load", 0.8), 

108 (ts2.astimezone(UTC), "mem_usage", 20480.0), 

109 ], 

110 ) 

111 

112 

113@pytest.mark.filterwarnings( 

114 "ignore:.*(Could not infer format, so each element will be parsed individually, " 

115 "falling back to `dateutil`).*:UserWarning::0" 

116) # pylint: disable=line-too-long # noqa 

117@pytest.mark.parametrize(("zone_info"), ZONE_INFO) 

118def test_local_env_telemetry_wrong_header( 

119 tunable_groups: TunableGroups, 

120 zone_info: tzinfo | None, 

121) -> None: 

122 """Read the telemetry data with incorrect header.""" 

123 ts1 = datetime.now(zone_info) 

124 ts1 -= timedelta(microseconds=ts1.microsecond) # Round to a second 

125 ts2 = ts1 + timedelta(minutes=1) 

126 

127 format_str = _format_str(zone_info) 

128 time_str1 = ts1.strftime(format_str) 

129 time_str2 = ts2.strftime(format_str) 

130 

131 local_env = create_local_env( 

132 tunable_groups, 

133 { 

134 "run": [ 

135 # Error: the data is correct, but the header has unexpected column names 

136 "echo 'ts,metric_name,metric_value' > telemetry.csv", 

137 f"echo {time_str1},cpu_load,0.65 >> telemetry.csv", 

138 f"echo {time_str1},mem_usage,10240 >> telemetry.csv", 

139 f"echo {time_str2},cpu_load,0.8 >> telemetry.csv", 

140 f"echo {time_str2},mem_usage,20480 >> telemetry.csv", 

141 ], 

142 "read_telemetry_file": "telemetry.csv", 

143 }, 

144 ) 

145 

146 check_env_fail_telemetry(local_env, tunable_groups) 

147 

148 

149def test_local_env_telemetry_invalid(tunable_groups: TunableGroups) -> None: 

150 """Fail when the telemetry data has wrong format.""" 

151 zone_info = UTC 

152 ts1 = datetime.now(zone_info) 

153 ts1 -= timedelta(microseconds=ts1.microsecond) # Round to a second 

154 ts2 = ts1 + timedelta(minutes=1) 

155 

156 format_str = _format_str(zone_info) 

157 time_str1 = ts1.strftime(format_str) 

158 time_str2 = ts2.strftime(format_str) 

159 

160 local_env = create_local_env( 

161 tunable_groups, 

162 { 

163 "run": [ 

164 # Error: too many columns 

165 f"echo {time_str1},EXTRA,cpu_load,0.65 > telemetry.csv", 

166 f"echo {time_str1},EXTRA,mem_usage,10240 >> telemetry.csv", 

167 f"echo {time_str2},EXTRA,cpu_load,0.8 >> telemetry.csv", 

168 f"echo {time_str2},EXTRA,mem_usage,20480 >> telemetry.csv", 

169 ], 

170 "read_telemetry_file": "telemetry.csv", 

171 }, 

172 ) 

173 

174 check_env_fail_telemetry(local_env, tunable_groups) 

175 

176 

177def test_local_env_telemetry_invalid_ts(tunable_groups: TunableGroups) -> None: 

178 """Fail when the telemetry data has wrong format.""" 

179 local_env = create_local_env( 

180 tunable_groups, 

181 { 

182 "run": [ 

183 # Error: field 1 must be a timestamp 

184 "echo 1,cpu_load,0.65 > telemetry.csv", 

185 "echo 2,mem_usage,10240 >> telemetry.csv", 

186 "echo 3,cpu_load,0.8 >> telemetry.csv", 

187 "echo 4,mem_usage,20480 >> telemetry.csv", 

188 ], 

189 "read_telemetry_file": "telemetry.csv", 

190 }, 

191 ) 

192 

193 check_env_fail_telemetry(local_env, tunable_groups)