Coverage for mlos_bench/mlos_bench/storage/base_tunable_config_trial_group_data.py: 97%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5"""Base interface for accessing the stored benchmark config trial group data.""" 

6 

7from abc import ABCMeta, abstractmethod 

8from typing import TYPE_CHECKING, Any, Dict, Optional 

9 

10import pandas 

11 

12from mlos_bench.storage.base_tunable_config_data import TunableConfigData 

13 

14if TYPE_CHECKING: 

15 from mlos_bench.storage.base_trial_data import TrialData 

16 

17 

18class TunableConfigTrialGroupData(metaclass=ABCMeta): 

19 """ 

20 Base interface for accessing the stored experiment benchmark tunable config trial 

21 group data. 

22 

23 A (tunable) config is used to define an instance of values for a set of tunable 

24 parameters for a given experiment and can be used by one or more trial instances 

25 (e.g., for repeats), which we call a (tunable) config trial group. 

26 """ 

27 

28 def __init__( 

29 self, 

30 *, 

31 experiment_id: str, 

32 tunable_config_id: int, 

33 tunable_config_trial_group_id: Optional[int] = None, 

34 ): 

35 self._experiment_id = experiment_id 

36 self._tunable_config_id = tunable_config_id 

37 # can be lazily initialized as necessary: 

38 self._tunable_config_trial_group_id: Optional[int] = tunable_config_trial_group_id 

39 

40 @property 

41 def experiment_id(self) -> str: 

42 """ID of the experiment.""" 

43 return self._experiment_id 

44 

45 @property 

46 def tunable_config_id(self) -> int: 

47 """ID of the config.""" 

48 return self._tunable_config_id 

49 

50 @abstractmethod 

51 def _get_tunable_config_trial_group_id(self) -> int: 

52 """Retrieve the trial's config_trial_group_id from the storage.""" 

53 raise NotImplementedError("subclass must implement") 

54 

55 @property 

56 def tunable_config_trial_group_id(self) -> int: 

57 """ 

58 The unique ID (within this experiment) of the (tunable) config trial group. 

59 

60 This is a unique identifier for all trials in this experiment using the given 

61 config_id, and typically defined as the the minimum trial_id for the given 

62 config_id. 

63 """ 

64 if self._tunable_config_trial_group_id is None: 

65 self._tunable_config_trial_group_id = self._get_tunable_config_trial_group_id() 

66 assert self._tunable_config_trial_group_id is not None 

67 return self._tunable_config_trial_group_id 

68 

69 def __repr__(self) -> str: 

70 return f"TunableConfigTrialGroup :: {self._experiment_id} cid:{self.tunable_config_id}" 

71 

72 def __eq__(self, other: Any) -> bool: 

73 if not isinstance(other, self.__class__): 

74 return False 

75 return ( 

76 self._tunable_config_id == other._tunable_config_id 

77 and self._experiment_id == other._experiment_id 

78 ) 

79 

80 @property 

81 @abstractmethod 

82 def tunable_config(self) -> TunableConfigData: 

83 """ 

84 Retrieve the (tunable) config data for this (tunable) config trial group from 

85 the storage. 

86 

87 Returns 

88 ------- 

89 TunableConfigData 

90 """ 

91 

92 @property 

93 @abstractmethod 

94 def trials(self) -> Dict[int, "TrialData"]: 

95 """ 

96 Retrieve the trials' data for this (tunable) config trial group from the 

97 storage. 

98 

99 Returns 

100 ------- 

101 trials : Dict[int, TrialData] 

102 A dictionary of the trials' data, keyed by trial id. 

103 """ 

104 

105 @property 

106 @abstractmethod 

107 def results_df(self) -> pandas.DataFrame: 

108 """ 

109 Retrieve all results for this (tunable) config trial group as a single 

110 DataFrame. 

111 

112 Returns 

113 ------- 

114 results : pandas.DataFrame 

115 A DataFrame with configurations and results from all trials of the experiment. 

116 Has columns [trial_id, config_id, ts_start, ts_end, status] 

117 followed by tunable config parameters (prefixed with "config.") and 

118 trial results (prefixed with "result."). The latter can be NULLs if the 

119 trial was not successful. 

120 

121 See Also 

122 -------- 

123 ExperimentData.results 

124 """