config.py (10269B)
1 import collections, datetime, functools, json, glob, logging, os, shutil, sys, time 2 3 from threading import Timer 4 5 6 logger = logging.getLogger(__name__) 7 8 9 class Config(collections.MutableMapping): 10 """Configuration JSON storage class""" 11 def __init__(self, filename, default=None, failsafe_backups=0, save_delay=0): 12 self.filename = filename 13 self.default = None 14 self.config = {} 15 self.changed = False 16 self.failsafe_backups = failsafe_backups 17 self.save_delay = save_delay 18 self.load() 19 20 self._timer_save = False 21 22 def _make_failsafe_backup(self): 23 try: 24 json.load(open(self.filename)) 25 except IOError: 26 return False 27 except ValueError: 28 logger.warning("{} is corrupted, aborting backup".format(self.filename)) 29 return False 30 31 existing = sorted(glob.glob(self.filename + ".*.bak")) 32 while len(existing) > (self.failsafe_backups - 1): 33 os.remove(existing.pop(0)) 34 35 backup_file = self.filename + "." + datetime.datetime.now().strftime("%Y%m%d%H%M%S") + ".bak" 36 shutil.copy2(self.filename, backup_file) 37 38 return True 39 40 def _recover_from_failsafe(self): 41 existing = sorted(glob.glob(self.filename + ".*.bak")) 42 while len(existing) > 0: 43 try: 44 recovery_filename = existing.pop() 45 json.load(open(recovery_filename)) 46 shutil.copy2(recovery_filename, self.filename) 47 self.load(recovery=True) 48 logger.info("recovery successful: {}".format(recovery_filename)) 49 return True 50 except IOError: 51 pass 52 except ValueError: 53 logger.error("corrupted recovery: {}".format(self.filename)) 54 return False 55 56 def load(self, recovery=False): 57 """Load config from file""" 58 try: 59 self.config = json.load(open(self.filename)) 60 logger.info("{} read".format(self.filename)) 61 62 except IOError: 63 self.config = {} 64 65 except ValueError: 66 if not recovery and self.failsafe_backups > 0 and self._recover_from_failsafe(): 67 return 68 69 raise 70 71 self.changed = False 72 73 def force_taint(self): 74 self.changed = True 75 76 def loads(self, json_str): 77 """Load config from JSON string""" 78 self.config = json.loads(json_str) 79 self.changed = True 80 81 def save(self, delay=True): 82 if self.save_delay: 83 if delay: 84 if self._timer_save and self._timer_save.is_alive(): 85 self._timer_save.cancel() 86 self._timer_save = Timer(self.save_delay, self.save, [], {"delay": False}) 87 self._timer_save.start() 88 return False 89 90 """Save config to file (only if config has changed)""" 91 if self.changed: 92 start_time = time.time() 93 94 if self.failsafe_backups: 95 self._make_failsafe_backup() 96 97 with open(self.filename, 'w+') as f: 98 json.dump(self.config, f, indent=2, sort_keys=True) 99 self.changed = False 100 interval = time.time() - start_time 101 102 logger.info("{} write {}".format(self.filename, interval)) 103 104 return self.changed 105 106 def flush(self): 107 if self._timer_save and self._timer_save.is_alive(): 108 logger.info("flushing {}".format(self.filename)) 109 self._timer_save.cancel() 110 self.save(delay=False) 111 112 def get_by_path(self, keys_list): 113 """Get item from config by path (list of keys)""" 114 return functools.reduce(lambda d, k: d[int(k) if isinstance(d, list) else k], keys_list, self) 115 116 def set_by_path(self, keys_list, value): 117 """Set item in config by path (list of keys)""" 118 self.get_by_path(keys_list[:-1])[keys_list[-1]] = value 119 self.changed = True 120 121 def pop_by_path(self, keys_list): 122 popped_value = self.get_by_path(keys_list[:-1]).pop(keys_list[-1]) 123 self.changed = True 124 return popped_value 125 126 def get_option(self, keyname): 127 try: 128 value = self.config[keyname] 129 except KeyError: 130 value = None 131 return value 132 133 def get_suboption(self, grouping, groupname, keyname): 134 try: 135 value = self.config[grouping][groupname][keyname] 136 except KeyError: 137 value = self.get_option(keyname) 138 return value 139 140 def exists(self, keys_list): 141 _exists = True 142 143 try: 144 if self.get_by_path(keys_list) is None: 145 _exists = False 146 except (KeyError, TypeError): 147 _exists = False 148 149 return _exists 150 151 def __getitem__(self, key): 152 try: 153 return self.config[key] 154 except KeyError: 155 return self.default 156 157 def __setitem__(self, key, value): 158 self.config[key] = value 159 self.changed = True 160 161 def __delitem__(self, key): 162 del self.config[key] 163 self.changed = True 164 165 def __iter__(self): 166 return iter(self.config) 167 168 def __len__(self): 169 return len(self.config) 170 171 class Memory(collections.MutableMapping): 172 """Configuration JSON storage class""" 173 def __init__(self, filename, default=None, failsafe_backups=0, save_delay=0): 174 self.filename = filename 175 self.default = None 176 self.config = {} 177 self.changed = False 178 self.failsafe_backups = failsafe_backups 179 self.save_delay = save_delay 180 self.load() 181 182 self._timer_save = False 183 184 def _make_failsafe_backup(self): 185 try: 186 json.load(open(self.filename)) 187 except IOError: 188 return False 189 except ValueError: 190 logger.warning("{} is corrupted, aborting backup".format(self.filename)) 191 return False 192 193 existing = sorted(glob.glob(self.filename + ".*.bak")) 194 while len(existing) > (self.failsafe_backups - 1): 195 os.remove(existing.pop(0)) 196 197 backup_file = self.filename + "." + datetime.datetime.now().strftime("%Y%m%d%H%M%S") + ".bak" 198 shutil.copy2(self.filename, backup_file) 199 200 return True 201 202 def _recover_from_failsafe(self): 203 existing = sorted(glob.glob(self.filename + ".*.bak")) 204 while len(existing) > 0: 205 try: 206 recovery_filename = existing.pop() 207 json.load(open(recovery_filename)) 208 shutil.copy2(recovery_filename, self.filename) 209 self.load(recovery=True) 210 logger.info("recovery successful: {}".format(recovery_filename)) 211 return True 212 except IOError: 213 pass 214 except ValueError: 215 logger.error("corrupted recovery: {}".format(self.filename)) 216 return False 217 218 def load(self, recovery=False): 219 """Load config from file""" 220 try: 221 self.config = json.load(open(self.filename)) 222 logger.info("{} read".format(self.filename)) 223 224 except IOError: 225 self.config = {} 226 227 except ValueError: 228 if not recovery and self.failsafe_backups > 0 and self._recover_from_failsafe(): 229 return 230 231 raise 232 233 self.changed = False 234 235 def force_taint(self): 236 self.changed = True 237 238 def loads(self, json_str): 239 """Load config from JSON string""" 240 self.config = json.loads(json_str) 241 self.changed = True 242 243 def save(self, delay=True): 244 if self.save_delay: 245 if delay: 246 if self._timer_save and self._timer_save.is_alive(): 247 self._timer_save.cancel() 248 self._timer_save = Timer(self.save_delay, self.save, [], {"delay": False}) 249 self._timer_save.start() 250 return False 251 252 """Save config to file (only if config has changed)""" 253 if self.changed: 254 start_time = time.time() 255 256 if self.failsafe_backups: 257 self._make_failsafe_backup() 258 259 with open(self.filename, 'w+') as f: 260 json.dump(self.config, f, indent=2, sort_keys=True) 261 self.changed = False 262 interval = time.time() - start_time 263 264 logger.info("{} write {}".format(self.filename, interval)) 265 266 return self.changed 267 268 def flush(self): 269 if self._timer_save and self._timer_save.is_alive(): 270 logger.info("flushing {}".format(self.filename)) 271 self._timer_save.cancel() 272 self.save(delay=False) 273 274 def get_by_path(self, keys_list): 275 """Get item from config by path (list of keys)""" 276 return functools.reduce(lambda d, k: d[int(k) if isinstance(d, list) else k], keys_list, self) 277 278 def set_by_path(self, keys_list, value): 279 """Set item in config by path (list of keys)""" 280 self.get_by_path(keys_list[:-1])[keys_list[-1]] = value 281 self.changed = True 282 283 def pop_by_path(self, keys_list): 284 popped_value = self.get_by_path(keys_list[:-1]).pop(keys_list[-1]) 285 self.changed = True 286 return popped_value 287 288 def get_option(self, keyname): 289 try: 290 value = self.config[keyname] 291 except KeyError: 292 value = None 293 return value 294 295 def get_suboption(self, grouping, groupname, keyname): 296 try: 297 value = self.config[grouping][groupname][keyname] 298 except KeyError: 299 value = self.get_option(keyname) 300 return value 301 302 def exists(self, keys_list): 303 _exists = True 304 305 try: 306 if self.get_by_path(keys_list) is None: 307 _exists = False 308 except (KeyError, TypeError): 309 _exists = False 310 311 return _exists 312 313 def __getitem__(self, key): 314 try: 315 return self.config[key] 316 except KeyError: 317 return self.default 318 319 def __setitem__(self, key, value): 320 self.config[key] = value 321 self.changed = True 322 323 def __delitem__(self, key): 324 del self.config[key] 325 self.changed = True 326 327 def __iter__(self): 328 return iter(self.config) 329 330 def __len__(self): 331 return len(self.config)