Coverage for mlos_bench/mlos_bench/run.py: 87%

1#!/usr/bin/env python3 

2# 

3# Copyright (c) Microsoft Corporation. 

4# Licensed under the MIT License. 

5# 

6""" 

7mlos_bench main optimization loop and benchmark runner CLI. 

8 

9Note: this script is also available as a CLI tool via ``pip`` under the name ``mlos_bench``. 

10 

11See the current ``--help`` `output for details <../../../mlos_bench.run.usage.html>`_. 

12 

13See Also 

14-------- 

15mlos_bench.config : documentation on the configuration system. 

16mlos_bench.launcher.Launcher : class is responsible for processing the CLI args. 

17""" 

18 

19import logging 

20import sys 

21 

22import numpy as np 

23 

24from mlos_bench.launcher import Launcher 

25from mlos_bench.tunables.tunable_groups import TunableGroups 

26 

27_LOG = logging.getLogger(__name__) 

28 

29 

30def _sanity_check_results(launcher: Launcher) -> None: 

31 """Do some sanity checking on the results and throw an exception if it looks like 

32 something went wrong. 

33 """ 

34 basic_err_msg = "Check configuration, scripts, and logs for details." 

35 

36 # Check if the scheduler has any trials. 

37 if not launcher.scheduler.trial_count: 

38 raise RuntimeError(f"No trials were run. {basic_err_msg}") 

39 

40 # Check if the scheduler ran the expected number of trials. 

41 expected_trial_count = min( 

42 launcher.scheduler.max_trials if launcher.scheduler.max_trials > 0 else np.inf, 

43 launcher.scheduler.trial_config_repeat_count * launcher.optimizer.max_suggestions, 

44 ) 

45 if launcher.scheduler.trial_count < expected_trial_count: 

46 raise RuntimeError( 

47 f"Expected {expected_trial_count} trials, " 

48 f"but only {launcher.scheduler.trial_count} were run. {basic_err_msg}" 

49 ) 

50 

51 # Check to see if "too many" trials seem to have failed (#523). 

52 unsuccessful_trials = [t for t in launcher.scheduler.ran_trials if not t.status.is_succeeded()] 

53 if len(unsuccessful_trials) > 0.2 * launcher.scheduler.trial_count: 

54 raise RuntimeWarning( 

55 "Too many trials failed: " 

56 f"{len(unsuccessful_trials)} out of {launcher.scheduler.trial_count}. " 

57 f"{basic_err_msg}" 

58 ) 

59 

60 

61def _main( 

62 argv: list[str] | None = None, 

63) -> tuple[dict[str, float] | None, TunableGroups | None]: 

64 launcher = Launcher("mlos_bench", "Systems autotuning and benchmarking tool", argv=argv) 

65 

66 with launcher.scheduler as scheduler_context: 

67 scheduler_context.start() 

68 scheduler_context.teardown() 

69 

70 _sanity_check_results(launcher) 

71 

72 (score, _config) = result = launcher.scheduler.get_best_observation() 

73 # NOTE: This log line is used in test_launch_main_app_* unit tests: 

74 _LOG.info("Final score: %s", score) 

75 return result 

76 

77 

78def _shell_main( 

79 argv: list[str] | None = None, 

80) -> int: 

81 (best_score, best_config) = _main(argv) 

82 # Exit zero if it looks like the overall operation was successful. 

83 # TODO: Improve this sanity check to be more robust. 

84 if ( 

85 best_score 

86 and best_config 

87 and all(isinstance(score_value, float) for score_value in best_score.values()) 

88 ): 

89 return 0 

90 else: 

91 raise ValueError(f"Unexpected result: {best_score=}, {best_config=}") 

92 

93 

94if __name__ == "__main__": 

95 sys.exit(_shell_main())