Coverage for mlos_bench/mlos_bench/tests/config/schemas/storage/test_storage_schemas.py: 100%

1# 

2# Copyright (c) Microsoft Corporation. 

3# Licensed under the MIT License. 

4# 

5"""Tests for storage schema validation.""" 

6 

7from os import path 

8 

9import pytest 

10 

11from mlos_bench.config.schemas import ConfigSchema 

12from mlos_bench.storage.base_storage import Storage 

13from mlos_bench.tests import try_resolve_class_name 

14from mlos_bench.tests.config.schemas import ( 

15 check_test_case_against_schema, 

16 check_test_case_config_with_extra_param, 

17 get_schema_test_cases, 

18) 

19from mlos_core.tests import get_all_concrete_subclasses 

20 

21# General testing strategy: 

22# - hand code a set of good/bad configs (useful to test editor schema checking) 

23# - for each config, load and validate against expected schema 

24 

25TEST_CASES = get_schema_test_cases(path.join(path.dirname(__file__), "test-cases")) 

26 

27# Dynamically enumerate some of the cases we want to make sure we cover. 

28 

29expected_mlos_bench_storage_class_names = [ 

30 subclass.__module__ + "." + subclass.__name__ 

31 for subclass in get_all_concrete_subclasses( 

32 Storage, # type: ignore[type-abstract] 

33 pkg_name="mlos_bench", 

34 ) 

35] 

36assert expected_mlos_bench_storage_class_names 

37 

38# Do the full cross product of all the test cases and all the storage types. 

39 

40 

41@pytest.mark.parametrize("test_case_subtype", sorted(TEST_CASES.by_subtype)) 

42@pytest.mark.parametrize("mlos_bench_storage_type", expected_mlos_bench_storage_class_names) 

43def test_case_coverage_mlos_bench_storage_type( 

44 test_case_subtype: str, 

45 mlos_bench_storage_type: str, 

46) -> None: 

47 """Checks to see if there is a given type of test case for the given mlos_bench 

48 storage type. 

49 """ 

50 for test_case in TEST_CASES.by_subtype[test_case_subtype].values(): 

51 if try_resolve_class_name(test_case.config.get("class")) == mlos_bench_storage_type: 

52 return 

53 raise NotImplementedError( 

54 f"Missing test case for subtype {test_case_subtype} " 

55 f"for Storage class {mlos_bench_storage_type}" 

56 ) 

57 

58 

59# Now we actually perform all of those validation tests. 

60 

61 

62@pytest.mark.parametrize("test_case_name", sorted(TEST_CASES.by_path)) 

63def test_storage_configs_against_schema(test_case_name: str) -> None: 

64 """Checks that the storage config validates against the schema.""" 

65 check_test_case_against_schema(TEST_CASES.by_path[test_case_name], ConfigSchema.STORAGE) 

66 check_test_case_against_schema(TEST_CASES.by_path[test_case_name], ConfigSchema.UNIFIED) 

67 

68 

69@pytest.mark.parametrize("test_case_name", sorted(TEST_CASES.by_type["good"])) 

70def test_storage_configs_with_extra_param(test_case_name: str) -> None: 

71 """Checks that the storage config fails to validate if extra params are present in 

72 certain places. 

73 """ 

74 check_test_case_config_with_extra_param( 

75 TEST_CASES.by_type["good"][test_case_name], 

76 ConfigSchema.STORAGE, 

77 ) 

78 check_test_case_config_with_extra_param( 

79 TEST_CASES.by_type["good"][test_case_name], 

80 ConfigSchema.UNIFIED, 

81 ) 

82 

83 

84if __name__ == "__main__": 

85 pytest.main( 

86 [__file__, "-n0"], 

87 )