Coverage for /home/ubuntu/hidebound/python/hidebound/server/server_tools.py: 100%
131 statements
« prev ^ index » next coverage.py v7.5.4, created at 2024-07-05 23:50 +0000
1from typing import Any, Dict, Optional, Union # noqa F401
3from collections import OrderedDict
4from dataclasses import dataclass
5from pathlib import Path
6from pprint import pformat
7import base64
8import json
9import os
10import re
11import traceback
13from flask.testing import FlaskClient
14import flask
15import jinja2
16import lunchbox.tools as lbt
17import requests
18import rolling_pin.blob_etl as rpb
19import yaml
21import hidebound.core.logging as hblog
22# ------------------------------------------------------------------------------
25HOST = '0.0.0.0'
26PORT = 8080
29@dataclass
30class EndPoints:
31 '''
32 A convenience class for API endpoints.
33 '''
34 host = HOST
35 port = PORT
36 api = f'http://{HOST}:{PORT}/api'
37 init = f'http://{HOST}:{PORT}/api/initialize'
38 update = f'http://{HOST}:{PORT}/api/update'
39 create = f'http://{HOST}:{PORT}/api/create'
40 export = f'http://{HOST}:{PORT}/api/export'
41 delete = f'http://{HOST}:{PORT}/api/delete'
42 read = f'http://{HOST}:{PORT}/api/read'
43 search = f'http://{HOST}:{PORT}/api/search'
44 workflow = f'http://{HOST}:{PORT}/api/workflow'
45 progress = f'http://{HOST}:{PORT}/api/progress'
46# ------------------------------------------------------------------------------
49def render_template(filename, parameters):
50 # type: (str, Dict[str, Any]) -> bytes
51 '''
52 Renders a jinja2 template given by filename with given parameters.
54 Args:
55 filename (str): Filename of template.
56 parameters (dict): Dictionary of template parameters.
58 Returns:
59 bytes: HTML.
60 '''
61 tempdir = lbt.relative_path(__file__, '../../../templates').as_posix()
62 env = jinja2.Environment(
63 loader=jinja2.FileSystemLoader(tempdir),
64 keep_trailing_newline=True
65 )
66 output = env.get_template(filename).render(parameters).encode('utf-8')
67 return output
70def parse_json_file_content(raw_content):
71 # type: (bytes) -> Dict
72 '''
73 Parses JSON file content as supplied by HTML request.
75 Args:
76 raw_content (bytes): Raw JSON file content.
78 Raises:
79 ValueError: If header is invalid.
80 JSONDecodeError: If JSON is invalid.
82 Returns:
83 dict: JSON content or reponse dict with error.
84 '''
85 header, content = raw_content.split(',') # type: ignore
86 temp = header.split('/')[-1].split(';')[0] # type: ignore
87 if temp != 'json':
88 msg = f'File header is not JSON. Header: {header}.' # type: ignore
89 raise ValueError(msg)
91 output = base64.b64decode(content).decode('utf-8')
92 return json.loads(output)
95def error_to_response(error):
96 # type: (Exception) -> flask.Response
97 '''
98 Convenience function for formatting a given exception as a Flask Response.
100 Args:
101 error (Exception): Error to be formatted.
103 Returns:
104 flask.Response: Flask response.
105 '''
106 args = [] # type: Any
107 for arg in error.args:
108 if hasattr(arg, 'items'):
109 for key, val in arg.items():
110 args.append(pformat({key: pformat(val)}))
111 else:
112 args.append(str(arg))
113 args = [' ' + x for x in args]
114 args = '\n'.join(args)
115 klass = error.__class__.__name__
116 msg = f'{klass}(\n{args}\n)'
117 return flask.Response(
118 response=json.dumps(dict(
119 error=error.__class__.__name__,
120 args=list(map(str, error.args)),
121 message=msg,
122 code=500,
123 traceback=traceback.format_exc(),
124 )),
125 mimetype='application/json',
126 status=500,
127 )
130# SETUP-------------------------------------------------------------------------
131def setup_hidebound_directories(root):
132 # type: (Union[str, Path]) -> None
133 '''
134 Creates [root]/ingress, [root]/hidebound and [root]/archive directories.
136 Args:
137 root (str or Path): Root directory.
138 '''
139 for folder in ['ingress', 'hidebound', 'archive']:
140 os.makedirs(Path(root, folder), exist_ok=True)
143# ERRORS------------------------------------------------------------------------
144def get_config_error():
145 # type: () -> flask.Response
146 '''
147 Convenience function for returning a config error response.
149 Returns:
150 Response: Config error.
151 '''
152 msg = 'Please supply a config dictionary.'
153 error = TypeError(msg)
154 return error_to_response(error)
157def get_initialization_error():
158 # type: () -> flask.Response
159 '''
160 Convenience function for returning a initialization error response.
162 Returns:
163 Response: Initialization error.
164 '''
165 msg = 'Database not initialized. Please call initialize.'
166 error = RuntimeError(msg)
167 return error_to_response(error)
170def get_update_error():
171 # type: () -> flask.Response
172 '''
173 Convenience function for returning a update error response.
175 Returns:
176 Response: Update error.
177 '''
178 msg = 'Database not updated. Please call update.'
179 error = RuntimeError(msg)
180 return error_to_response(error)
183def get_read_error():
184 '''
185 Convenience function for returning a read error response.
187 Returns:
188 Response: Update error.
189 '''
190 msg = 'Please supply valid read params in the form '
191 msg += '{"group_by_asset": BOOL}.'
192 error = ValueError(msg)
193 return error_to_response(error)
196def get_search_error():
197 # type: () -> flask.Response
198 '''
199 Convenience function for returning a search error response.
201 Returns:
202 Response: Update error.
203 '''
204 msg = 'Please supply valid search params in the form '
205 msg += '{"query": SQL query, "group_by_asset": BOOL}.'
206 error = ValueError(msg)
207 return error_to_response(error)
210def get_connection_error():
211 # type: () -> flask.Response
212 '''
213 Convenience function for returning a database connection error response.
215 Returns:
216 Response: Connection error.
217 '''
218 msg = 'Database not connected.'
219 error = RuntimeError(msg)
220 return error_to_response(error)
223# DASH-TOOLS--------------------------------------------------------------------
224def get_progress(logpath=hblog.PROGRESS_LOG_PATH):
225 # type: (Union[str, Path]) -> dict
226 '''
227 Gets current progress state.
229 Args:
230 logpath (str or Path, optional): Filepath of progress log.
231 Default: PROGRESS_LOG_PATH.
233 Returns:
234 dict: Progess.
235 '''
236 state = dict(progress=1.0, message='unknown state')
237 state.update(hblog.get_progress(logpath))
238 return state
241def request(store, url, params=None, client=requests):
242 # type: (dict, str, Optional[dict], Any) -> dict
243 '''
244 Execute search against database and update store with response.
245 Sets store['content'] to response if there is an error.
247 Args:
248 store (dict): Dash store.
249 url (str): API endpoint.
250 params (dict, optional): Request paramaters. Default: None.
251 client (object, optional): Client. Default: requests module.
253 Returns:
254 dict: Store.
255 '''
256 params_ = None
257 if params is not None:
258 params_ = json.dumps(params)
259 headers = {'Content-Type': 'application/json'}
260 response = client.post(url, json=params_, headers=headers)
261 code = response.status_code
262 if isinstance(client, FlaskClient):
263 response = response.json
264 else:
265 response = response.json() # pragma: no cover
266 if code < 200 or code >= 300:
267 store['content'] = response
268 store['ready'] = True
269 return response
272def search(store, query, group_by_asset, client=requests):
273 # type: (dict, str, bool, Any) -> dict
274 '''
275 Execute search against database and update given store with response.
277 Args:
278 store (dict): Dash store.
279 query (str): Query string.
280 group_by_asset (bool): Whether to group the search by asset.
281 client (object, optional): Client. Default: requests module.
283 Returns:
284 dict: Store.
285 '''
286 params = dict(query=query, group_by_asset=group_by_asset)
287 store['content'] = request(store, EndPoints().search, params, client)
288 store['query'] = query
289 store['ready'] = True
290 return store
293def format_config(
294 config, redact_regex='(_key|_id|_token|url)$', redact_hash=False
295):
296 # type: (Dict[str, Any], str, bool) -> OrderedDict[str, Any]
297 '''
298 Redacts credentials of config and formats it for display in component.
300 Args:
301 config (dict): Configuration dictionary.
302 redact_regex (str, optional): Regular expression that matches keys,
303 whose values are to be redacted. Default: "(_key|_id|_token|url)$".
304 redact_hash (bool, optional): Whether to redact values with the string
305 "REDACTED" or a hash of the value. Default: False.
307 Returns:
308 OrderedDict: Formatted config.
309 '''
310 def redact(key, value, as_hash):
311 if as_hash:
312 return 'hash-' + str(hash(value)).lstrip('-')
313 return 'REDACTED'
315 def predicate(key, value):
316 if re.search(redact_regex, key):
317 return True
318 return False
320 config = rpb.BlobETL(config) \
321 .set(predicate, value_setter=lambda k, v: redact(k, v, redact_hash)) \
322 .to_dict()
323 output = OrderedDict()
324 keys = [
325 'ingress_directory',
326 'staging_directory',
327 'include_regex',
328 'exclude_regex',
329 'write_mode',
330 'redact_regex',
331 'redact_hash',
332 'specification_files',
333 'workflow',
334 'dask',
335 'exporters',
336 'webhooks',
337 ]
338 all_ = set(config.keys())
339 cross = all_.intersection(keys)
340 diff = sorted(list(all_.difference(cross)))
341 keys = list(filter(lambda x: x in cross, keys)) + diff
342 for key in keys:
343 val = re.sub(r'\.\.\.$', '', yaml.safe_dump(config[key])).rstrip('\n')
344 output[key] = val
345 return output