Coverage for mlos_bench/mlos_bench/tests/environments/local/composite_local_env_test.py: 100%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5"""Unit tests for the composition of several LocalEnv benchmark environments.""" 

6import sys 

7from datetime import datetime, timedelta, tzinfo 

8from typing import Optional 

9 

10import pytest 

11from pytz import UTC 

12 

13from mlos_bench.tests import ZONE_INFO 

14from mlos_bench.tests.environments import check_env_success 

15from mlos_bench.tests.environments.local import create_composite_local_env 

16from mlos_bench.tunables.tunable_groups import TunableGroups 

17 

18 

19def _format_str(zone_info: Optional[tzinfo]) -> str: 

20 if zone_info is not None: 

21 return "%Y-%m-%d %H:%M:%S %z" 

22 return "%Y-%m-%d %H:%M:%S" 

23 

24 

25# FIXME: This fails with zone_info = None when run with `TZ="America/Chicago pytest -n0 ...` 

26@pytest.mark.parametrize(("zone_info"), ZONE_INFO) 

27def test_composite_env(tunable_groups: TunableGroups, zone_info: Optional[tzinfo]) -> None: 

28 """ 

29 Produce benchmark and telemetry data in TWO local environments and combine the 

30 results. 

31 

32 Also checks that global configs flow down at least one level of CompositeEnv 

33 to its children without being explicitly specified in the CompositeEnv so they 

34 can be used in the shell_envs by its children. 

35 See Also: http://github.com/microsoft/MLOS/issues/501 

36 """ 

37 ts1 = datetime.now(zone_info) 

38 ts1 -= timedelta(microseconds=ts1.microsecond) # Round to a second 

39 ts2 = ts1 + timedelta(minutes=2) 

40 

41 format_str = _format_str(zone_info) 

42 time_str1 = ts1.strftime(format_str) 

43 time_str2 = ts2.strftime(format_str) 

44 

45 (var_prefix, var_suffix) = ("%", "%") if sys.platform == "win32" else ("$", "") 

46 

47 env = create_composite_local_env( 

48 tunable_groups=tunable_groups, 

49 global_config={ 

50 "reads": 2222, 

51 "writes": 1111, 

52 }, 

53 params={ 

54 "const_args": { 

55 "latency": 4.2, 

56 "throughput": 768, 

57 "errors": 0, 

58 } 

59 }, 

60 local_configs=[ 

61 { 

62 "const_args": { 

63 "latency": 3.3, 

64 "reads": 0, 

65 }, 

66 "required_args": ["errors", "reads"], 

67 "shell_env_params": [ 

68 # const_args overridden by the composite env 

69 "latency", 

70 # Comes from the parent const_args 

71 "errors", 

72 # const_args overridden by the global config 

73 "reads", 

74 ], 

75 "run": [ 

76 "echo 'metric,value' > output.csv", 

77 f"echo 'latency,{var_prefix}latency{var_suffix}' >> output.csv", 

78 f"echo 'errors,{var_prefix}errors{var_suffix}' >> output.csv", 

79 f"echo 'reads,{var_prefix}reads{var_suffix}' >> output.csv", 

80 "echo '-------------------'", # This output does not go anywhere 

81 "echo 'timestamp,metric,value' > telemetry.csv", 

82 f"echo {time_str1},cpu_load,0.64 >> telemetry.csv", 

83 f"echo {time_str1},mem_usage,5120 >> telemetry.csv", 

84 ], 

85 "read_results_file": "output.csv", 

86 "read_telemetry_file": "telemetry.csv", 

87 }, 

88 { 

89 "const_args": { 

90 "throughput": 999, 

91 "score": 0.97, 

92 }, 

93 "required_args": ["writes"], 

94 "shell_env_params": [ 

95 # const_args overridden by the composite env 

96 "throughput", 

97 # Comes from the local const_args 

98 "score", 

99 # Comes straight from the global config 

100 "writes", 

101 ], 

102 "run": [ 

103 "echo 'metric,value' > output.csv", 

104 f"echo 'throughput,{var_prefix}throughput{var_suffix}' >> output.csv", 

105 f"echo 'score,{var_prefix}score{var_suffix}' >> output.csv", 

106 f"echo 'writes,{var_prefix}writes{var_suffix}' >> output.csv", 

107 "echo '-------------------'", # This output does not go anywhere 

108 "echo 'timestamp,metric,value' > telemetry.csv", 

109 f"echo {time_str2},cpu_load,0.79 >> telemetry.csv", 

110 f"echo {time_str2},mem_usage,40960 >> telemetry.csv", 

111 ], 

112 "read_results_file": "output.csv", 

113 "read_telemetry_file": "telemetry.csv", 

114 }, 

115 ], 

116 ) 

117 

118 check_env_success( 

119 env, 

120 tunable_groups, 

121 expected_results={ 

122 "latency": 4.2, 

123 "throughput": 768.0, 

124 "score": 0.97, 

125 "errors": 0.0, 

126 "reads": 2222.0, 

127 "writes": 1111.0, 

128 }, 

129 expected_telemetry=[ 

130 (ts1.astimezone(UTC), "cpu_load", 0.64), 

131 (ts1.astimezone(UTC), "mem_usage", 5120.0), 

132 (ts2.astimezone(UTC), "cpu_load", 0.79), 

133 (ts2.astimezone(UTC), "mem_usage", 40960.0), 

134 ], 

135 )