Coverage for /home/ubuntu/shekels/python/shekels/server/components.py: 100%

1from typing import Any, Dict, List, Optional # noqa: F401 

2import flask # noqa: F401 

3 

4from copy import copy 

5 

6from lunchbox.enforce import Enforce, EnforceError 

7from pandas import DataFrame, DatetimeIndex 

8from schematics.exceptions import DataError 

9import dash 

10from dash import dash_table 

11from dash import dcc 

12from dash import html 

13import lunchbox.tools as lbt 

14import rolling_pin.blob_etl as rpb 

15 

16import shekels.core.config as cfg 

17import shekels.core.data_tools as sdt 

18# ------------------------------------------------------------------------------ 

19 

20 

21# TODO: refactor components tests to use selnium and be less brittle 

22# TODO: add JSON editor component for config 

23# APP--------------------------------------------------------------------------- 

24def get_dash_app(server, storage_type='memory'): 

25 # type: (flask.Flask, str) -> dash.Dash 

26 ''' 

27 Generate Dash Flask app instance. 

28 

29 Args: 

30 server (Flask): Flask instance. 

31 storage_type (str): Storage type (used for testing). Default: memory. 

32 

33 Returns: 

34 Dash: Dash app instance. 

35 ''' 

36 

37 store = dcc.Store(id='store', storage_type=storage_type) 

38 

39 icon = html.Img(id='icon', src='/assets/icon.svg') 

40 tabs = dcc.Tabs( 

41 id='tabs', 

42 className='tabs', 

43 value='plots', 

44 children=[ 

45 dcc.Tab(className='tab', label='plots', value='plots'), 

46 dcc.Tab(className='tab', label='data', value='data'), 

47 dcc.Tab(className='tab', label='config', value='config'), 

48 dcc.Tab(className='tab', label='api', value='api'), 

49 dcc.Tab(className='tab', label='docs', value='docs'), 

50 dcc.Tab(className='tab', label='monitor', value='monitor'), 

51 ], 

52 ) 

53 tabs = html.Div(id='tabs-container', children=[icon, tabs]) 

54 

55 content = dcc.Loading( 

56 id="content", 

57 className='content', 

58 type="dot", 

59 fullscreen=True, 

60 ) 

61 

62 assets = lbt.relative_path(__file__, "../../../resources") 

63 

64 app = dash.Dash( 

65 name='Shekels', 

66 title='Shekels', 

67 server=server, 

68 external_stylesheets=['/static/style.css'], 

69 assets_folder=assets, 

70 ) 

71 app.layout = html.Div(id='layout', children=[store, tabs, content]) 

72 app.config['suppress_callback_exceptions'] = True 

73 

74 return app 

75 

76 

77# TABS-------------------------------------------------------------------------- 

78def get_data_tab(query=None): 

79 # type: (Optional[str]) -> List 

80 ''' 

81 Get tab element for Shekels data. 

82 

83 Args: 

84 query (str, optional): Query string. Default: None. 

85 

86 Return: 

87 list: List of elements for data tab. 

88 ''' 

89 # dummies must go first for element props behavior to work 

90 content = html.Div(id='lower-content', children=[ 

91 html.Div(id='data-content', className='col', children=[]) 

92 ]) 

93 return [*get_dummy_elements(), get_searchbar(query), content] 

94 

95 

96def get_plots_tab(query=None): 

97 # type: (Optional[str]) -> List 

98 ''' 

99 Get tab element for Shekels plots. 

100 

101 Args: 

102 query (str, optional): Query string. Default: None. 

103 

104 Return: 

105 list: List of elements for plots tab. 

106 ''' 

107 # dummies must go first for element props behavior to work 

108 content = html.Div(id='lower-content', children=[ 

109 html.Div(id='plots-content', className='col', children=[ 

110 dcc.Loading(id="progress-bar", type="circle") 

111 ]) 

112 ]) 

113 return [*get_dummy_elements(), get_searchbar(query), content] 

114 

115 

116def get_config_tab(config): 

117 # type: (Dict) -> List 

118 ''' 

119 Get tab element for Shekels config. 

120 

121 Args: 

122 config (dict): Configuration to be displayed. 

123 

124 Return: 

125 list: List of elements for config tab. 

126 ''' 

127 # dummies must go first for element props behavior to work 

128 content = html.Div(id='lower-content', children=[ 

129 html.Div(id='config-content', className='col', children=[ 

130 get_key_value_table( 

131 config, id_='config', header='config', editable=True 

132 ) 

133 ]) 

134 ]) 

135 return [*get_dummy_elements(), get_configbar(config), content] 

136 

137 

138# MENUBARS---------------------------------------------------------------------- 

139def get_searchbar(query=None): 

140 # type: (Optional[str]) -> html.Div 

141 ''' 

142 Get a row of elements used for querying Shekels data. 

143 

144 Args: 

145 query (str, optional): Query string. Default: None. 

146 

147 Returns: 

148 Div: Div with query field and buttons. 

149 ''' 

150 if query is None: 

151 query = 'select * from data' 

152 

153 spacer = html.Div(className='col spacer') 

154 query = dcc.Input( 

155 id='query', 

156 className='col query', 

157 value=query, 

158 placeholder='SQL query that uses "FROM data"', 

159 type='text', 

160 autoFocus=True, 

161 debounce=True 

162 ) 

163 

164 search = get_button('search') 

165 init = get_button('init') 

166 update = get_button('update') 

167 

168 row = html.Div( 

169 className='row', 

170 children=[query, spacer, search, spacer, init, spacer, update], 

171 ) 

172 searchbar = html.Div(id='searchbar', className='menubar', children=[row]) 

173 return searchbar 

174 

175 

176def get_dummy_elements(): 

177 # type: () -> List 

178 ''' 

179 Returns a list of all elements with callbacks so that the client will not 

180 throw errors in each tab. 

181 

182 Returns: 

183 list: List of html elements. 

184 ''' 

185 return [ 

186 dcc.Input(className='dummy', id='config-query', value=None), 

187 html.Div(className='dummy', children=[dash_table.DataTable(id='config-table')]), 

188 dcc.Input(className='dummy', id='query', value=None), 

189 html.Div(className='dummy', id='config-search-button', n_clicks=None), 

190 html.Div(className='dummy', id='search-button', n_clicks=None), 

191 html.Div(className='dummy', id='init-button', n_clicks=None), 

192 html.Div(className='dummy', id='update-button', n_clicks=None), 

193 dcc.Upload(className='dummy', id='upload', contents=None), 

194 html.Div(className='dummy', id='save-button', n_clicks=None), 

195 ] 

196 

197 

198def get_configbar(config, query='select * from config'): 

199 # type: (Dict, Optional[str]) -> html.Div 

200 ''' 

201 Get a row of elements used for configuring Shekels. 

202 

203 Args: 

204 config (dict): Configuration to be displayed. 

205 query (str, optional): Query string. Default: None. 

206 

207 Returns: 

208 Div: Div with buttons and JSON editor. 

209 ''' 

210 spacer = html.Div(className='col spacer') 

211 query = dcc.Input( 

212 id='config-query', 

213 className='col query', 

214 value=query, 

215 placeholder='SQL query that uses "FROM config"', 

216 type='text', 

217 autoFocus=True, 

218 debounce=True 

219 ) 

220 

221 search = get_button('search') 

222 search.id = 'config-search-button' 

223 init = get_button('init') 

224 upload = dcc.Upload( 

225 id='upload', 

226 children=[get_button('upload')] 

227 ) 

228 save = get_button('save') 

229 row = html.Div( 

230 className='row', 

231 children=[ 

232 query, spacer, search, spacer, init, spacer, upload, spacer, save 

233 ], 

234 ) 

235 configbar = html.Div(id='configbar', className='menubar', children=[row]) 

236 return configbar 

237 

238 

239# ELEMENTS---------------------------------------------------------------------- 

240def get_button(title): 

241 # type: (str) -> html.Button 

242 ''' 

243 Get a html button with a given title. 

244 

245 Args: 

246 title (str): Title of button. 

247 

248 Raises: 

249 TypeError: If title is not a string. 

250 

251 Returns: 

252 Button: Button element. 

253 ''' 

254 if not isinstance(title, str): 

255 msg = f'{title} is not a string.' 

256 raise TypeError(msg) 

257 return html.Button(id=f'{title}-button', children=[title], n_clicks=0) 

258 

259 

260def get_key_value_table( 

261 data, id_='key-value', header='', editable=False, key_order=None 

262): 

263 # type (dict, Optional(str), str, bool, Optional(List[str])) -> DataTable 

264 ''' 

265 Gets a Dash DataTable element representing given dictionary. 

266 

267 Args: 

268 data (dict): Dictionary. 

269 id_ (str, optional): CSS id. Default: 'key-value'. 

270 header (str, optional): Table header title. Default: ''. 

271 editable (bool, optional): Whether table is editable. Default: False. 

272 key_order (list[str], optional): Order in which keys will be displayed. 

273 Default: None. 

274 

275 Returns: 

276 DataTable: Tablular representation of given dictionary. 

277 ''' 

278 data = rpb.BlobETL(data).to_flat_dict() 

279 

280 # determine keys 

281 keys = sorted(list(data.keys())) 

282 if key_order is not None: 

283 diff = set(key_order).difference(keys) 

284 if len(diff) > 0: 

285 diff = list(sorted(diff)) 

286 msg = f'Invalid key order. Keys not found in data: {diff}.' 

287 raise KeyError(msg) 

288 

289 keys = set(keys).difference(key_order) 

290 keys = sorted(list(keys)) 

291 keys = key_order + keys 

292 

293 # transform data 

294 data = [dict(key=k, value=data[k]) for k in keys] 

295 

296 cols = [] 

297 if len(data) > 0: 

298 cols = data[0].keys() 

299 cols = [{'name': x, 'id': x} for x in cols] 

300 

301 table = dash_table.DataTable( 

302 data=data, 

303 data_previous=data, 

304 columns=cols, 

305 id=f'{id_}-table', 

306 sort_action='native', 

307 sort_mode='multi', 

308 page_action='none', 

309 cell_selectable=True, 

310 editable=editable, 

311 ) 

312 head = html.Div(className='key-value-table-header', children=header) 

313 return html.Div( 

314 id=id_, className='key-value-table-container', children=[head, table] 

315 ) 

316 

317 

318def get_datatable(data, color_scheme=cfg.COLOR_SCHEME, editable=False): 

319 # type: (List[Dict], Dict[str, str], bool) -> dash_table.DataTable 

320 ''' 

321 Gets a Dash DataTable element using given data. 

322 Assumes dict element has all columns of table as keys. 

323 

324 Args: 

325 data (list[dict]): List of dicts. 

326 color_scheme (dict, optional): Color scheme dictionary. 

327 Default: COLOR_SCHEME. 

328 editable (bool, optional): Whether table is editable. Default: False. 

329 

330 Returns: 

331 DataTable: Table of data. 

332 ''' 

333 cs = copy(cfg.COLOR_SCHEME) 

334 cs.update(color_scheme) 

335 

336 cols = [] # type: Any 

337 if len(data) > 0: 

338 cols = data[0].keys() 

339 cols = [{'name': x, 'id': x} for x in cols] 

340 

341 return dash_table.DataTable( 

342 data=data, 

343 columns=cols, 

344 id='datatable', 

345 fixed_rows=dict(headers=True), 

346 sort_action='native', 

347 sort_mode='multi', 

348 cell_selectable=editable, 

349 editable=editable, 

350 ) 

351 

352 

353def get_plots(data, plots): 

354 # type: (List[dict], List[dict]) -> List[dcc.Graph] 

355 ''' 

356 Gets a Dash plots using given dicts. 

357 Assumes dict element has all columns of table as keys. 

358 

359 Args: 

360 data (list[dict]): List of dicts defining data. 

361 plots (list[dict]): List of dicts defining plots. 

362 

363 Raises: 

364 EnforceError: If data is not a list of dicts. 

365 EnforceError: If plots is not a list of dicts. 

366 

367 Returns: 

368 list[dcc.Graph]: Plots. 

369 ''' 

370 msg = 'Data must be a list of dictionaries. Given value: {a}.' 

371 Enforce(data, 'instance of', list, message=msg) 

372 for item in data: 

373 Enforce(item, 'instance of', dict, message=msg) 

374 

375 msg = 'Plots must be a list of dictionaries. Given value: {a}.' 

376 Enforce(plots, 'instance of', list, message=msg) 

377 for item in plots: 

378 Enforce(item, 'instance of', dict, message=msg) 

379# -------------------------------------------------------------------------- 

380 

381 data_ = DataFrame(data) 

382 if 'date' in data_.columns: 

383 data_.date = DatetimeIndex(data_.date) 

384 

385 elems = [] 

386 for i, x in enumerate(plots): 

387 plot = cfg.PlotItem(x) 

388 plot.validate() 

389 plot = plot.to_primitive() 

390 min_width = str(plot['min_width']) + '%' 

391 

392 try: 

393 fig = sdt.get_figure( 

394 data_, 

395 filters=plot['filters'], 

396 group=plot['group'], 

397 pivot=plot['pivot'], 

398 **plot['figure'], 

399 ) 

400 fig = dcc.Graph( 

401 id=f'plot-{i:02d}', 

402 className='plot', 

403 figure=fig, 

404 style={'min-width': min_width}, 

405 ) 

406 except (DataError, EnforceError): 

407 fig = html.Div( 

408 id=f'plot-{i:02d}', 

409 className='plot plot-error', 

410 style={'min-width': min_width}, 

411 children=html.Div( 

412 className='plot-error-container', 

413 children=html.Div( 

414 className='plot-error-message', 

415 children='no data found' 

416 ) 

417 ) 

418 ) 

419 elems.append(fig) 

420 return elems