Coverage for mlos_bench/mlos_bench/tests/tunables/tunable_definition_test.py: 100%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5"""Unit tests for checking tunable definition rules.""" 

6 

7import json5 as json 

8import pytest 

9 

10from mlos_bench.tunables.tunable import Tunable, TunableValueTypeName 

11 

12 

13def test_tunable_name() -> None: 

14 """Check that tunable name is valid.""" 

15 with pytest.raises(ValueError): 

16 # ! characters are currently disallowed in tunable names 

17 Tunable(name="test!tunable", config={"type": "float", "range": [0, 1], "default": 0}) 

18 

19 

20def test_categorical_required_params() -> None: 

21 """Check that required parameters are present for categorical tunables.""" 

22 json_config = """ 

23 { 

24 "type": "categorical", 

25 "values_missing": ["foo", "bar", "baz"], 

26 "default": "foo" 

27 } 

28 """ 

29 config = json.loads(json_config) 

30 assert isinstance(config, dict) 

31 with pytest.raises(ValueError): 

32 Tunable(name="test", config=config) 

33 

34 

35def test_categorical_weights() -> None: 

36 """Instantiate a categorical tunable with weights.""" 

37 json_config = """ 

38 { 

39 "type": "categorical", 

40 "values": ["foo", "bar", "baz"], 

41 "values_weights": [25, 25, 50], 

42 "default": "foo" 

43 } 

44 """ 

45 config = json.loads(json_config) 

46 assert isinstance(config, dict) 

47 tunable = Tunable(name="test", config=config) 

48 assert tunable.weights == [25, 25, 50] 

49 

50 

51def test_categorical_weights_wrong_count() -> None: 

52 """Try to instantiate a categorical tunable with incorrect number of weights.""" 

53 json_config = """ 

54 { 

55 "type": "categorical", 

56 "values": ["foo", "bar", "baz"], 

57 "values_weights": [50, 50], 

58 "default": "foo" 

59 } 

60 """ 

61 config = json.loads(json_config) 

62 assert isinstance(config, dict) 

63 with pytest.raises(ValueError): 

64 Tunable(name="test", config=config) 

65 

66 

67def test_categorical_weights_wrong_values() -> None: 

68 """Try to instantiate a categorical tunable with invalid weights.""" 

69 json_config = """ 

70 { 

71 "type": "categorical", 

72 "values": ["foo", "bar", "baz"], 

73 "values_weights": [-1, 50, 50], 

74 "default": "foo" 

75 } 

76 """ 

77 config = json.loads(json_config) 

78 assert isinstance(config, dict) 

79 with pytest.raises(ValueError): 

80 Tunable(name="test", config=config) 

81 

82 

83def test_categorical_wrong_params() -> None: 

84 """Disallow range param for categorical tunables.""" 

85 json_config = """ 

86 { 

87 "type": "categorical", 

88 "values": ["foo", "bar", "foo"], 

89 "range": [0, 1], 

90 "default": "foo" 

91 } 

92 """ 

93 config = json.loads(json_config) 

94 assert isinstance(config, dict) 

95 with pytest.raises(ValueError): 

96 Tunable(name="test", config=config) 

97 

98 

99def test_categorical_disallow_special_values() -> None: 

100 """Disallow special values for categorical values.""" 

101 json_config = """ 

102 { 

103 "type": "categorical", 

104 "values": ["foo", "bar", "foo"], 

105 "special": ["baz"], 

106 "default": "foo" 

107 } 

108 """ 

109 config = json.loads(json_config) 

110 assert isinstance(config, dict) 

111 with pytest.raises(ValueError): 

112 Tunable(name="test", config=config) 

113 

114 

115def test_categorical_tunable_disallow_repeats() -> None: 

116 """Disallow duplicate values in categorical tunables.""" 

117 with pytest.raises(ValueError): 

118 Tunable( 

119 name="test", 

120 config={ 

121 "type": "categorical", 

122 "values": ["foo", "bar", "foo"], 

123 "default": "foo", 

124 }, 

125 ) 

126 

127 

128@pytest.mark.parametrize("tunable_type", ["int", "float"]) 

129def test_numerical_tunable_disallow_null_default(tunable_type: TunableValueTypeName) -> None: 

130 """Disallow null values as default for numerical tunables.""" 

131 with pytest.raises(ValueError): 

132 Tunable( 

133 name=f"test_{tunable_type}", 

134 config={ 

135 "type": tunable_type, 

136 "range": [0, 10], 

137 "default": None, 

138 }, 

139 ) 

140 

141 

142@pytest.mark.parametrize("tunable_type", ["int", "float"]) 

143def test_numerical_tunable_disallow_out_of_range(tunable_type: TunableValueTypeName) -> None: 

144 """Disallow out of range values as default for numerical tunables.""" 

145 with pytest.raises(ValueError): 

146 Tunable( 

147 name=f"test_{tunable_type}", 

148 config={ 

149 "type": tunable_type, 

150 "range": [0, 10], 

151 "default": 11, 

152 }, 

153 ) 

154 

155 

156@pytest.mark.parametrize("tunable_type", ["int", "float"]) 

157def test_numerical_tunable_wrong_params(tunable_type: TunableValueTypeName) -> None: 

158 """Disallow values param for numerical tunables.""" 

159 with pytest.raises(ValueError): 

160 Tunable( 

161 name=f"test_{tunable_type}", 

162 config={ 

163 "type": tunable_type, 

164 "range": [0, 10], 

165 "values": ["foo", "bar"], 

166 "default": 0, 

167 }, 

168 ) 

169 

170 

171@pytest.mark.parametrize("tunable_type", ["int", "float"]) 

172def test_numerical_tunable_required_params(tunable_type: TunableValueTypeName) -> None: 

173 """Disallow null values param for numerical tunables.""" 

174 json_config = f""" 

175 { 

176 "type": "{tunable_type}", 

177 "range_missing": [0, 10], 

178 "default": 0 

179 } 

180 """ 

181 config = json.loads(json_config) 

182 assert isinstance(config, dict) 

183 with pytest.raises(ValueError): 

184 Tunable(name=f"test_{tunable_type}", config=config) 

185 

186 

187@pytest.mark.parametrize("tunable_type", ["int", "float"]) 

188def test_numerical_tunable_invalid_range(tunable_type: TunableValueTypeName) -> None: 

189 """Disallow invalid range param for numerical tunables.""" 

190 json_config = f""" 

191 { 

192 "type": "{tunable_type}", 

193 "range": [0, 10, 7], 

194 "default": 0 

195 } 

196 """ 

197 config = json.loads(json_config) 

198 assert isinstance(config, dict) 

199 with pytest.raises(AssertionError): 

200 Tunable(name=f"test_{tunable_type}", config=config) 

201 

202 

203@pytest.mark.parametrize("tunable_type", ["int", "float"]) 

204def test_numerical_tunable_reversed_range(tunable_type: TunableValueTypeName) -> None: 

205 """Disallow reverse range param for numerical tunables.""" 

206 json_config = f""" 

207 { 

208 "type": "{tunable_type}", 

209 "range": [10, 0], 

210 "default": 0 

211 } 

212 """ 

213 config = json.loads(json_config) 

214 assert isinstance(config, dict) 

215 with pytest.raises(ValueError): 

216 Tunable(name=f"test_{tunable_type}", config=config) 

217 

218 

219@pytest.mark.parametrize("tunable_type", ["int", "float"]) 

220def test_numerical_weights(tunable_type: TunableValueTypeName) -> None: 

221 """Instantiate a numerical tunable with weighted special values.""" 

222 json_config = f""" 

223 { 

224 "type": "{tunable_type}", 

225 "range": [0, 100], 

226 "special": [0], 

227 "special_weights": [0.1], 

228 "range_weight": 0.9, 

229 "default": 0 

230 } 

231 """ 

232 config = json.loads(json_config) 

233 assert isinstance(config, dict) 

234 tunable = Tunable(name="test", config=config) 

235 assert tunable.special == [0] 

236 assert tunable.weights == [0.1] 

237 assert tunable.range_weight == 0.9 

238 

239 

240@pytest.mark.parametrize("tunable_type", ["int", "float"]) 

241def test_numerical_quantization(tunable_type: TunableValueTypeName) -> None: 

242 """Instantiate a numerical tunable with quantization.""" 

243 json_config = f""" 

244 { 

245 "type": "{tunable_type}", 

246 "range": [0, 100], 

247 "quantization_bins": 11, 

248 "default": 0 

249 } 

250 """ 

251 config = json.loads(json_config) 

252 assert isinstance(config, dict) 

253 tunable = Tunable(name="test", config=config) 

254 expected = [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100] 

255 assert tunable.quantization_bins == len(expected) 

256 assert pytest.approx(list(tunable.quantized_values or []), 1e-8) == expected 

257 assert not tunable.is_log 

258 

259 

260@pytest.mark.parametrize("tunable_type", ["int", "float"]) 

261def test_numerical_log(tunable_type: TunableValueTypeName) -> None: 

262 """Instantiate a numerical tunable with log scale.""" 

263 json_config = f""" 

264 { 

265 "type": "{tunable_type}", 

266 "range": [0, 100], 

267 "log": true, 

268 "default": 0 

269 } 

270 """ 

271 config = json.loads(json_config) 

272 assert isinstance(config, dict) 

273 tunable = Tunable(name="test", config=config) 

274 assert tunable.is_log 

275 

276 

277@pytest.mark.parametrize("tunable_type", ["int", "float"]) 

278def test_numerical_weights_no_specials(tunable_type: TunableValueTypeName) -> None: 

279 """Raise an error if special_weights are specified but no special values.""" 

280 json_config = f""" 

281 { 

282 "type": "{tunable_type}", 

283 "range": [0, 100], 

284 "special_weights": [0.1, 0.9], 

285 "default": 0 

286 } 

287 """ 

288 config = json.loads(json_config) 

289 assert isinstance(config, dict) 

290 with pytest.raises(ValueError): 

291 Tunable(name="test", config=config) 

292 

293 

294@pytest.mark.parametrize("tunable_type", ["int", "float"]) 

295def test_numerical_weights_non_normalized(tunable_type: TunableValueTypeName) -> None: 

296 """Instantiate a numerical tunable with non-normalized weights of the special 

297 values. 

298 """ 

299 json_config = f""" 

300 { 

301 "type": "{tunable_type}", 

302 "range": [0, 100], 

303 "special": [-1, 0], 

304 "special_weights": [0, 10], 

305 "range_weight": 90, 

306 "default": 0 

307 } 

308 """ 

309 config = json.loads(json_config) 

310 assert isinstance(config, dict) 

311 tunable = Tunable(name="test", config=config) 

312 assert tunable.special == [-1, 0] 

313 assert tunable.weights == [0, 10] # Zero weights are ok 

314 assert tunable.range_weight == 90 

315 

316 

317@pytest.mark.parametrize("tunable_type", ["int", "float"]) 

318def test_numerical_weights_wrong_count(tunable_type: TunableValueTypeName) -> None: 

319 """Try to instantiate a numerical tunable with incorrect number of weights.""" 

320 json_config = f""" 

321 { 

322 "type": "{tunable_type}", 

323 "range": [0, 100], 

324 "special": [0], 

325 "special_weights": [0.1, 0.1, 0.8], 

326 "range_weight": 0.1, 

327 "default": 0 

328 } 

329 """ 

330 config = json.loads(json_config) 

331 assert isinstance(config, dict) 

332 with pytest.raises(ValueError): 

333 Tunable(name="test", config=config) 

334 

335 

336@pytest.mark.parametrize("tunable_type", ["int", "float"]) 

337def test_numerical_weights_no_range_weight(tunable_type: TunableValueTypeName) -> None: 

338 """Try to instantiate a numerical tunable with weights but no range_weight.""" 

339 json_config = f""" 

340 { 

341 "type": "{tunable_type}", 

342 "range": [0, 100], 

343 "special": [0, -1], 

344 "special_weights": [0.1, 0.2], 

345 "default": 0 

346 } 

347 """ 

348 config = json.loads(json_config) 

349 assert isinstance(config, dict) 

350 with pytest.raises(ValueError): 

351 Tunable(name="test", config=config) 

352 

353 

354@pytest.mark.parametrize("tunable_type", ["int", "float"]) 

355def test_numerical_range_weight_no_weights(tunable_type: TunableValueTypeName) -> None: 

356 """Try to instantiate a numerical tunable with specials but no range_weight.""" 

357 json_config = f""" 

358 { 

359 "type": "{tunable_type}", 

360 "range": [0, 100], 

361 "special": [0, -1], 

362 "range_weight": 0.3, 

363 "default": 0 

364 } 

365 """ 

366 config = json.loads(json_config) 

367 assert isinstance(config, dict) 

368 with pytest.raises(ValueError): 

369 Tunable(name="test", config=config) 

370 

371 

372@pytest.mark.parametrize("tunable_type", ["int", "float"]) 

373def test_numerical_range_weight_no_specials(tunable_type: TunableValueTypeName) -> None: 

374 """Try to instantiate a numerical tunable with specials but no range_weight.""" 

375 json_config = f""" 

376 { 

377 "type": "{tunable_type}", 

378 "range": [0, 100], 

379 "range_weight": 0.3, 

380 "default": 0 

381 } 

382 """ 

383 config = json.loads(json_config) 

384 assert isinstance(config, dict) 

385 with pytest.raises(ValueError): 

386 Tunable(name="test", config=config) 

387 

388 

389@pytest.mark.parametrize("tunable_type", ["int", "float"]) 

390def test_numerical_weights_wrong_values(tunable_type: TunableValueTypeName) -> None: 

391 """Try to instantiate a numerical tunable with incorrect number of weights.""" 

392 json_config = f""" 

393 { 

394 "type": "{tunable_type}", 

395 "range": [0, 100], 

396 "special": [0], 

397 "special_weights": [-1], 

398 "range_weight": 10, 

399 "default": 0 

400 } 

401 """ 

402 config = json.loads(json_config) 

403 assert isinstance(config, dict) 

404 with pytest.raises(ValueError): 

405 Tunable(name="test", config=config) 

406 

407 

408@pytest.mark.parametrize("tunable_type", ["int", "float"]) 

409def test_numerical_quantization_wrong(tunable_type: TunableValueTypeName) -> None: 

410 """Instantiate a numerical tunable with invalid number of quantization points.""" 

411 json_config = f""" 

412 { 

413 "type": "{tunable_type}", 

414 "range": [0, 100], 

415 "quantization_bins": 0, 

416 "default": 0 

417 } 

418 """ 

419 config = json.loads(json_config) 

420 assert isinstance(config, dict) 

421 with pytest.raises(ValueError): 

422 Tunable(name="test", config=config) 

423 

424 

425def test_bad_type() -> None: 

426 """Disallow bad types.""" 

427 json_config = """ 

428 { 

429 "type": "foo", 

430 "range": [0, 10], 

431 "default": 0 

432 } 

433 """ 

434 config = json.loads(json_config) 

435 assert isinstance(config, dict) 

436 with pytest.raises(ValueError): 

437 Tunable(name="test_bad_type", config=config)