Coverage for /home/ubuntu/hidebound/python/hidebound/server/extension.py: 100%

1from typing import Optional, Union # noqa F401 

2import flask # noqa F401 

3 

4from pathlib import Path 

5import os 

6 

7import pyjson5 as jsonc 

8import yaml 

9 

10from hidebound.core.database import Database 

11from hidebound.server.api import API 

12import hidebound.core.config as hbc 

13import hidebound.core.tools as hbt 

14# ------------------------------------------------------------------------------ 

15 

16 

17# inheriting from Singleton breaks init and init_app tests 

18class HideboundExtension: 

19 def __init__(self, app=None): 

20 # type: (Optional[flask.Flask]) -> None 

21 ''' 

22 Initialize flask extension. 

23 

24 Args: 

25 app (flask.Flask, optional): Flask app. 

26 ''' 

27 if app is not None: 

28 self.init_app(app) 

29 

30 def get_config(self, app): 

31 # type: (flask.Flask) -> dict 

32 ''' 

33 Get config from envirnment variables or config file. 

34 

35 Args: 

36 app (flask.Flask): Flask app. 

37 

38 Returns: 

39 dict: Database config. 

40 ''' 

41 config_path = os.environ.get('HIDEBOUND_CONFIG_FILEPATH', None) 

42 if config_path is not None: 

43 config = self._get_config_from_file(config_path) 

44 

45 else: 

46 app.config.from_prefixed_env('HIDEBOUND') 

47 config = self._get_config_from_env(app) 

48 

49 config = hbc.Config(config).to_native() 

50 return config 

51 

52 def _get_config_from_file(self, filepath): 

53 # type: (Union[str, Path]) -> dict 

54 ''' 

55 Get config from envirnment variables or config file. 

56 

57 Args: 

58 filepath (str or Path): Filepath of hidebound config. 

59 

60 Raises: 

61 FileNotFoundError: If HIDEBOUND_CONFIG_FILEPATH is set to a file 

62 that does not end in json, yml or yaml. 

63 

64 Returns: 

65 dict: Database config. 

66 ''' 

67 fp = Path(filepath) 

68 ext = fp.suffix.lower().lstrip('.') 

69 exts = ['json', 'yml', 'yaml'] 

70 if ext not in exts: 

71 msg = 'Hidebound config files must end in one of these extensions: ' 

72 msg += f'{exts}. Given file: {fp.as_posix()}.' 

73 raise FileNotFoundError(msg) 

74 

75 with open(filepath) as f: 

76 if ext in ['yml', 'yaml']: 

77 return yaml.safe_load(f) 

78 return jsonc.load(f) 

79 

80 def _get_config_from_env(self, app): 

81 # type: (flask.Flask) -> dict 

82 ''' 

83 Get config from environment variables. 

84 

85 Args: 

86 app (flask.Flask): Flask app. 

87 

88 Returns: 

89 dict: Database config. 

90 ''' 

91 dask = dict( 

92 cluster_type=app.config.get('DASK_CLUSTER_TYPE'), 

93 num_partitions=app.config.get('DASK_NUM_PARTITIONS'), 

94 local_num_workers=app.config.get('DASK_LOCAL_NUM_WORKERS'), 

95 local_threads_per_worker=app.config.get('DASK_LOCAL_THREADS_PER_WORKER'), 

96 local_multiprocessing=hbt.str_to_bool( 

97 app.config.get('DASK_LOCAL_MULTIPROCESSING', 'True') 

98 ), 

99 gateway_address=app.config.get('DASK_GATEWAY_ADDRESS'), 

100 gateway_proxy_address=app.config.get('DASK_GATEWAY_PROXY_ADDRESS'), 

101 gateway_public_address=app.config.get('DASK_GATEWAY_PUBLIC_ADDRESS'), 

102 gateway_auth_type=app.config.get('DASK_GATEWAY_AUTH_TYPE'), 

103 gateway_api_token=app.config.get('DASK_GATEWAY_API_TOKEN'), 

104 gateway_api_user=app.config.get('DASK_GATEWAY_API_USER'), 

105 gateway_cluster_options=yaml.safe_load( 

106 str(app.config.get('DASK_GATEWAY_CLUSTER_OPTIONS', '[]')) 

107 ), 

108 gateway_min_workers=app.config.get('DASK_GATEWAY_MIN_WORKERS'), 

109 gateway_max_workers=app.config.get('DASK_GATEWAY_MAX_WORKERS'), 

110 gateway_shutdown_on_close=hbt.str_to_bool( 

111 app.config.get('DASK_GATEWAY_SHUTDOWN_ON_CLOSE', 'True') 

112 ), 

113 gateway_timeout=app.config.get('DASK_GATEWAY_TIMEOUT'), 

114 ) 

115 return dict( 

116 ingress_directory=app.config.get('INGRESS_DIRECTORY'), 

117 staging_directory=app.config.get('STAGING_DIRECTORY'), 

118 include_regex=app.config.get('INCLUDE_REGEX', ''), 

119 exclude_regex=app.config.get('EXCLUDE_REGEX', r'\.DS_Store'), 

120 write_mode=app.config.get('WRITE_MODE', 'copy'), 

121 redact_regex=app.config.get('REDACT_REGEX', '(_key|_id|_token|url)$'), 

122 redact_hash=hbt.str_to_bool(app.config.get('REDACT_HASH', 'False')), 

123 specification_files=yaml.safe_load( 

124 str(app.config.get('SPECIFICATION_FILES', '[]')) 

125 ), 

126 workflow=yaml.safe_load(str(app.config.get( 

127 'WORKFLOW', 

128 '["delete", "update", "create", "export"]', 

129 ))), 

130 dask=dask, 

131 exporters=yaml.safe_load(str(app.config.get('EXPORTERS', '{}'))), 

132 webhooks=yaml.safe_load(str(app.config.get('WEBHOOKS', '[]'))), 

133 ) 

134 

135 def init_app(self, app): 

136 # type: (flask.Flask) -> None 

137 ''' 

138 Add endpoints and error handlers to given app. 

139 

140 Args: 

141 app (Flask): Flask app. 

142 ''' 

143 app.extensions['hidebound'] = self 

144 app.register_blueprint(API) 

145 if not app.config['TESTING']: 

146 self.config = self.get_config(app) 

147 self.database = Database.from_config(self.config)