Coverage for /home/ubuntu/hidebound/python/hidebound/core/logging.py: 100%

1from typing import Any, List, Optional, Union # noqa F401 

2 

3from datetime import datetime 

4from pathlib import Path 

5import json 

6import logging 

7import logging.handlers 

8import os 

9 

10import json_logging 

11# ------------------------------------------------------------------------------ 

12 

13 

14PROGRESS_LOG_PATH = '/var/log/hidebound/hidebound-progress.log' 

15 

16 

17class ProgressLogger: 

18 ''' 

19 Logs progress to quasi-JSON files. 

20 ''' 

21 def __init__(self, name, filepath=PROGRESS_LOG_PATH, level=logging.INFO): 

22 # type: (str, Union[str, Path], int) -> None 

23 ''' 

24 Create ProgressLogger instance. 

25 

26 Args: 

27 name (str): Logger name. 

28 filepath (str or Path, optional): Log filepath. 

29 Default: /var/logs/hidebound/hidebound-progress.log. 

30 level (int, optional): Log level. Default: INFO. 

31 ''' 

32 filepath = Path(filepath) 

33 os.makedirs(filepath.parent, exist_ok=True) 

34 filepath = filepath.as_posix() 

35 self._filepath = filepath 

36 self._logger = self._get_logger(name, filepath, level=level) 

37 

38 @staticmethod 

39 def _get_logger(name, filepath, level=logging.INFO): 

40 # type: (str, Union[str, Path], int) -> logging.Logger 

41 ''' 

42 Creates a JSON logger. 

43 

44 Args: 

45 name (str): Name of logger. 

46 filepath (str or Path): Filepath of JSON log. 

47 level (int, optional): Log level. Default: INFO. 

48 

49 Returns: 

50 Logger: JSON logger. 

51 ''' 

52 class Formatter(json_logging.JSONLogFormatter): 

53 def format(self, record): 

54 # get progress numbers 

55 progress = None 

56 step = None 

57 total = None 

58 if hasattr(record, 'props') and isinstance(record.props, dict): 

59 step = record.props.get('step', None) 

60 total = record.props.get('total', None) 

61 

62 message = record.getMessage() 

63 orig = message 

64 if step is not None and total is not None: 

65 progress = 1.0 

66 if total != 0: 

67 progress = float(step) / total 

68 pct = progress * 100 

69 message = f'Progress: {pct:.2f}% ({step} of {total})' 

70 message += f' - {orig}' 

71 

72 log = dict( 

73 args=list(map(str, record.args)), 

74 created=record.created, 

75 exc_info=record.exc_info, 

76 exc_text=record.exc_text, 

77 message=message, 

78 level_name=record.levelname, 

79 level_number=record.levelno, 

80 msecs=record.msecs, 

81 original_message=orig, 

82 name=record.name, 

83 process=record.process, 

84 process_name=record.processName, 

85 relative_created=record.relativeCreated, 

86 stack_info=record.stack_info, 

87 thread=record.thread, 

88 thread_name=record.threadName, 

89 timestamp=datetime.fromtimestamp(record.created).isoformat(), 

90 progress=progress, 

91 step=step, 

92 total=total, 

93 ) 

94 return json.dumps(log) 

95 

96 json_logging.init_non_web(enable_json=True, custom_formatter=Formatter) 

97 logger = logging.getLogger(name) 

98 logger.setLevel(level) 

99 handler = logging.handlers.RotatingFileHandler( 

100 filepath, 

101 encoding='utf-8', 

102 maxBytes=2**20, 

103 backupCount=9, 

104 ) 

105 logger.addHandler(handler) 

106 return logger 

107 

108 @staticmethod 

109 def read(filepath): 

110 # type: (Union[str, Path]) -> List[dict] 

111 ''' 

112 Read a given progress log file. 

113 

114 Args: 

115 filepath (str or Path): Log path. 

116 

117 Returns: 

118 list[dict]: Logs. 

119 ''' 

120 with open(filepath) as f: 

121 log = list(map(json.loads, f.readlines())) 

122 return log 

123 

124 @property 

125 def filepath(self): 

126 # type: () -> str 

127 ''' 

128 str: Filepath of progress log. 

129 ''' 

130 return self._filepath 

131 

132 @property 

133 def logs(self): 

134 # type: () -> List[dict] 

135 ''' 

136 list[dict]: Logs read from filepath. 

137 ''' 

138 return self.read(self.filepath) 

139 

140 def log(self, level, message, step=None, total=None, **kwargs): 

141 # type: (int, str, Optional[int], Optional[int], Any) -> None 

142 ''' 

143 Log given message with given level. 

144 

145 Args: 

146 level (int): Log level. 

147 message (str): Log message. 

148 step (int, optional): Step in progress. Default: None. 

149 total (int, optional): Total number of steps. Default: None. 

150 ''' 

151 self._logger.log( 

152 level, 

153 message, 

154 extra=dict(props=dict(step=step, total=total)), 

155 **kwargs, 

156 ) 

157 

158 def info(self, message, step=None, total=None, **kwargs): 

159 # type: (str, Optional[int], Optional[int], Any) -> None 

160 ''' 

161 Log given message with INFO log level. 

162 

163 Args: 

164 message (str): Log message. 

165 step (int, optional): Step in progress. Default: None. 

166 total (int, optional): Total number of steps. Default: None. 

167 ''' 

168 self.log(logging.INFO, message, step=step, total=total, **kwargs) 

169 

170 def warning(self, message, step=None, total=None, **kwargs): 

171 # type: (str, Optional[int], Optional[int], Any) -> None 

172 ''' 

173 Log given message with WARNING log level. 

174 

175 Args: 

176 message (str): Log message. 

177 step (int, optional): Step in progress. Default: None. 

178 total (int, optional): Total number of steps. Default: None. 

179 ''' 

180 self.log(logging.WARNING, message, step=step, total=total, **kwargs) 

181 

182 def error(self, message, step=None, total=None, **kwargs): 

183 # type: (str, Optional[int], Optional[int], Any) -> None 

184 ''' 

185 Log given message with ERROR log level. 

186 

187 Args: 

188 message (str): Log message. 

189 step (int, optional): Step in progress. Default: None. 

190 total (int, optional): Total number of steps. Default: None. 

191 ''' 

192 self.log(logging.ERROR, message, step=step, total=total, **kwargs) 

193 

194 def debug(self, message, step=None, total=None, **kwargs): 

195 # type: (str, Optional[int], Optional[int], Any) -> None 

196 ''' 

197 Log given message with DEBUG log level. 

198 

199 Args: 

200 message (str): Log message. 

201 step (int, optional): Step in progress. Default: None. 

202 total (int, optional): Total number of steps. Default: None. 

203 ''' 

204 self.log(logging.DEBUG, message, step=step, total=total, **kwargs) 

205 

206 def fatal(self, message, step=None, total=None, **kwargs): 

207 # type: (str, Optional[int], Optional[int], Any) -> None 

208 ''' 

209 Log given message with FATAL log level. 

210 

211 Args: 

212 message (str): Log message. 

213 step (int, optional): Step in progress. Default: None. 

214 total (int, optional): Total number of steps. Default: None. 

215 ''' 

216 self.log(logging.FATAL, message, step=step, total=total, **kwargs) 

217 

218 def critical(self, message, step=None, total=None, **kwargs): 

219 # type: (str, Optional[int], Optional[int], Any) -> None 

220 ''' 

221 Log given message with CRITICAL log level. 

222 

223 Args: 

224 message (str): Log message. 

225 step (int, optional): Step in progress. Default: None. 

226 total (int, optional): Total number of steps. Default: None. 

227 ''' 

228 self.log(logging.CRITICAL, message, step=step, total=total, **kwargs) 

229# ------------------------------------------------------------------------------ 

230 

231 

232def get_progress(logpath=PROGRESS_LOG_PATH): 

233 # type: (Union[str, Path]) -> dict 

234 ''' 

235 Get last line of given progress file. 

236 Returns {} if logpath is not a file. 

237 

238 Args: 

239 logpath (str or Path, optional): Path to log file. 

240 

241 Returns: 

242 dict: Progress dictionary. 

243 ''' 

244 logpath = Path(logpath) 

245 output = {} 

246 if logpath.is_file(): 

247 output = ProgressLogger.read(logpath)[-1] 

248 return output 

249 

250 

251class DummyLogger: 

252 '''Dummy class for logging.''' 

253 def info(self, *args, **kwargs): 

254 # type: (...) -> None 

255 '''Does nothing.''' 

256 pass # pragma: no cover 

257 

258 def warning(self, *args, **kwargs): 

259 # type: (...) -> None 

260 '''Does nothing.''' 

261 pass # pragma: no cover 

262 

263 def error(self, *args, **kwargs): 

264 # type: (...) -> None 

265 '''Does nothing.''' 

266 pass # pragma: no cover 

267 

268 def debug(self, *args, **kwargs): 

269 # type: (...) -> None 

270 '''Does nothing.''' 

271 pass # pragma: no cover 

272 

273 def fatal(self, *args, **kwargs): 

274 # type: (...) -> None 

275 '''Does nothing.''' 

276 pass # pragma: no cover 

277 

278 def critical(self, *args, **kwargs): 

279 # type: (...) -> None 

280 '''Does nothing.''' 

281 pass # pragma: no cover