Coverage for mlos_bench/mlos_bench/tests/environments/local/local_env_stdout_test.py: 100%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5"""Unit tests for extracting data from LocalEnv stdout.""" 

6 

7import sys 

8 

9from mlos_bench.tests.environments import check_env_success 

10from mlos_bench.tests.environments.local import create_local_env 

11from mlos_bench.tunables.tunable_groups import TunableGroups 

12 

13 

14def test_local_env_stdout(tunable_groups: TunableGroups) -> None: 

15 """Print benchmark results to stdout and capture them in the LocalEnv.""" 

16 local_env = create_local_env( 

17 tunable_groups, 

18 { 

19 "run": [ 

20 "echo 'Benchmark results:'", # This line should be ignored 

21 "echo 'latency,111'", 

22 "echo 'throughput,222'", 

23 "echo 'score,0.999'", 

24 "echo 'a,0,b,1'", 

25 ], 

26 "results_stdout_pattern": r"(\w+),([0-9.]+)", 

27 }, 

28 ) 

29 

30 check_env_success( 

31 local_env, 

32 tunable_groups, 

33 expected_results={ 

34 "latency": 111.0, 

35 "throughput": 222.0, 

36 "score": 0.999, 

37 "a": 0, 

38 "b": 1, 

39 }, 

40 expected_telemetry=[], 

41 ) 

42 

43 

44def test_local_env_stdout_anchored(tunable_groups: TunableGroups) -> None: 

45 """Print benchmark results to stdout and capture them in the LocalEnv.""" 

46 local_env = create_local_env( 

47 tunable_groups, 

48 { 

49 "run": [ 

50 "echo 'Benchmark results:'", # This line should be ignored 

51 "echo 'latency,111'", 

52 "echo 'throughput,222'", 

53 "echo 'score,0.999'", 

54 "echo 'a,0,b,1'", # This line should be ignored in the case of anchored pattern 

55 ], 

56 "results_stdout_pattern": r"^(\w+),([0-9.]+)$", 

57 }, 

58 ) 

59 

60 check_env_success( 

61 local_env, 

62 tunable_groups, 

63 expected_results={ 

64 "latency": 111.0, 

65 "throughput": 222.0, 

66 "score": 0.999, 

67 # a, b are missing here 

68 }, 

69 expected_telemetry=[], 

70 ) 

71 

72 

73def test_local_env_file_stdout(tunable_groups: TunableGroups) -> None: 

74 """Print benchmark results to *BOTH* stdout and a file and extract the results from 

75 both. 

76 """ 

77 local_env = create_local_env( 

78 tunable_groups, 

79 { 

80 "run": [ 

81 "echo 'latency,111'", 

82 "echo 'throughput,222'", 

83 "echo 'score,0.999'", 

84 "echo 'stdout-msg,string'", 

85 "echo '-------------------'", # Should be ignored 

86 "echo 'metric,value' > output.csv", 

87 "echo 'extra1,333' >> output.csv", 

88 "echo 'extra2,444' >> output.csv", 

89 "echo 'file-msg,string' >> output.csv", 

90 ], 

91 "results_stdout_pattern": r"([a-zA-Z0-9_-]+),([a-z0-9.]+)", 

92 "read_results_file": "output.csv", 

93 }, 

94 ) 

95 

96 check_env_success( 

97 local_env, 

98 tunable_groups, 

99 expected_results={ 

100 "latency": 111.0, 

101 "throughput": 222.0, 

102 "score": 0.999, 

103 "stdout-msg": "string", 

104 "extra1": 333.0, 

105 "extra2": 444.0, 

106 "file-msg": "string " if sys.platform == "win32" else "string", 

107 }, 

108 expected_telemetry=[], 

109 )