Coverage for /home/ubuntu/hidebound/python/hidebound/core/config.py: 100%

1from typing import Union # noqa F401 

2 

3from copy import copy 

4from importlib import import_module 

5import inspect 

6import os 

7from pathlib import Path 

8import sys 

9 

10from schematics.exceptions import ValidationError 

11from schematics.types import ( 

12 BaseType, BooleanType, DictType, IntType, ListType, ModelType, StringType, 

13 URLType 

14) 

15from schematics import Model 

16 

17from hidebound.core.connection import DaskConnectionConfig 

18from hidebound.core.specification_base import SpecificationBase 

19from hidebound.exporters.disk_exporter import DiskConfig 

20from hidebound.exporters.girder_exporter import GirderConfig 

21from hidebound.exporters.s3_exporter import S3Config 

22import hidebound.core.validators as vd 

23# ------------------------------------------------------------------------------ 

24 

25 

26# must be in here to avoid circular import 

27def is_specification_file(filepath): 

28 # type: (Union[str, Path]) -> None 

29 ''' 

30 Validator for specification files given to Database. 

31 

32 Args: 

33 filepath (str or Path): Filepath of python specification file. 

34 

35 Raises: 

36 ValidationError: If module could not be imported. 

37 ValidationError: If module has no SPECIFICATIONS attribute. 

38 ValidationError: If module SPECIFICATIONS attribute is not a list. 

39 ValidationError: If modules classes in SPECIFICATIONS attribute are not 

40 subclasses of SpecificationBase. 

41 ValidationError: If keys in SPECIFICATIONS attribute are not lowercase 

42 versions of class names. 

43 ''' 

44 # get module name 

45 filepath = Path(filepath) 

46 filename = filepath.name 

47 filename, _ = os.path.splitext(filename) 

48 

49 # append module parent to sys.path 

50 parent = filepath.parent.as_posix() 

51 temp = copy(sys.path) 

52 sys.path.append(parent) 

53 

54 # import module 

55 mod = None 

56 try: 

57 mod = import_module(filename, filepath) # type: ignore 

58 except Exception: 

59 sys.path = temp 

60 msg = f'{filepath.as_posix()} could not be imported.' 

61 raise ValidationError(msg) 

62 

63 # get SPECIFICATIONS 

64 if not hasattr(mod, 'SPECIFICATIONS'): 

65 sys.path = temp 

66 msg = f'{filepath.as_posix()} has no SPECIFICATIONS attribute.' 

67 raise ValidationError(msg) 

68 

69 # ensure SPECIFICATIONS is a list 

70 if not isinstance(mod.SPECIFICATIONS, list): 

71 sys.path = temp 

72 msg = f'{filepath.as_posix()} SPECIFICATIONS attribute is not a list.' 

73 raise ValidationError(msg) 

74 

75 # ensure all SPECIFICATIONS are subclasses of SpecificationBase 

76 errors = list(filter( 

77 lambda x: not inspect.isclass(x) or not issubclass(x, SpecificationBase), 

78 mod.SPECIFICATIONS 

79 )) 

80 if len(errors) > 0: 

81 sys.path = temp 

82 msg = f'{errors} are not subclasses of SpecificationBase.' 

83 raise ValidationError(msg) 

84 

85 sys.path = temp 

86# ------------------------------------------------------------------------------ 

87 

88 

89class Config(Model): 

90 r''' 

91 A class for validating configurations supplied to Database. 

92 

93 Attributes: 

94 ingress_directory (str or Path): Root directory to recurse. 

95 staging_directory (str or Path): Directory where hidebound data will be 

96 staged. 

97 include_regex (str, optional): Include filenames that match this regex. 

98 Default: ''. 

99 exclude_regex (str, optional): Exclude filenames that match this regex. 

100 Default: '\.DS_Store'. 

101 write_mode (str, optional): How assets will be extracted to 

102 hidebound/content directory. Default: copy. 

103 workflow (list[str], optional): Ordered steps of workflow. Default: 

104 ['delete', 'update', 'create', 'export']. 

105 redact_regex (str, optional): Regex pattern matched to config keys. 

106 Values of matching keys will be redacted. 

107 Default: "(_key|_id|_token|url)$". 

108 redact_hash (bool, optional): Whether to replace redacted values with 

109 "REDACTED" or a hash of the value. Default: True. 

110 specification_files (list[str], optional): List of asset specification 

111 files. Default: []. 

112 exporters (dict, optional): Dictionary of exporter configs, where the 

113 key is the exporter name and the value is its config. Default: {}. 

114 webhooks (list[dict], optional): List of webhooks to be called after 

115 export. Default: []. 

116 dask (dict, optional). Dask configuration. Default: {}. 

117 ''' 

118 ingress_directory = StringType( 

119 required=True, validators=[vd.is_directory] 

120 ) # type: StringType 

121 staging_directory = StringType( 

122 required=True, validators=[vd.is_directory, vd.is_hidebound_directory] 

123 ) # type: StringType 

124 include_regex = StringType(default='', required=True) # type: StringType 

125 exclude_regex = StringType(default=r'\.DS_Store', required=True) # type: StringType 

126 write_mode = StringType( 

127 required=True, 

128 validators=[lambda x: vd.is_in(x, ['copy', 'move'])], 

129 default="copy", 

130 ) # type: StringType 

131 dask = ModelType( 

132 DaskConnectionConfig, default={}, required=True 

133 ) # type: ModelType 

134 workflow = ListType( 

135 StringType(), 

136 required=True, 

137 validators=[vd.is_workflow], 

138 default=['delete', 'update', 'create', 'export'] 

139 ) # type: ListType 

140 redact_regex = StringType(required=True, default='(_key|_id|_token|url)$') # type: StringType 

141 redact_hash = BooleanType(required=True, default=True) # type: BooleanType 

142 specification_files = ListType( 

143 StringType(validators=[is_specification_file, vd.is_file]), 

144 default=[], 

145 required=True 

146 ) # type: ListType 

147 exporters = ListType( 

148 BaseType( 

149 validators=[lambda x: vd.is_one_of( 

150 x, [DiskConfig, S3Config, GirderConfig] 

151 )] 

152 ), 

153 required=False, 

154 default=[], 

155 ) # type: ListType 

156 

157 class WebhookConfig(Model): 

158 url = URLType(required=True, fqdn=False) # type: URLType 

159 method = StringType(required=True, validators=[vd.is_http_method]) # type: StringType 

160 headers = DictType(StringType, required=False) # type: DictType 

161 data = DictType(BaseType, required=False, serialize_when_none=False) # type: DictType 

162 json = DictType(BaseType, required=False, serialize_when_none=False) # type: DictType 

163 params = DictType(BaseType, required=False, serialize_when_none=False) # type: DictType 

164 timeout = IntType(required=False, serialize_when_none=False) # type: IntType 

165 webhooks = ListType(ModelType(WebhookConfig), required=False, default=[]) # type: ListType