commit 326d293707557609967ddc616fac1e1d7c0a8a72
parent fd9f41806efd73682621fc0643c29061aab2c832
Author: Marcel <mtrnord1@gmail.com>
Date: Fri, 27 Jan 2017 09:07:50 +0100
fix sync, configs and memory
Diffstat:
5 files changed, 421 insertions(+), 136 deletions(-)
diff --git a/__init__.py b/__init__.py
@@ -1,94 +1,28 @@
-#Import Modules that are needed
+#Import external Modules that are needed
import telegram.ext
from telegram.ext import Updater, Filters
from slackclient import SlackClient
-import json
-import os.path
+import json, os.path, sys, traceback, functools, time, shutil
from multiprocessing import Pool
-import sys, traceback, functools, time
-
-class Config:
- filename = 'config/config.json'
- def generate(self):
- data = {"TELEGRAM_API_KEY": "INSERT YOUR TELEGRAM API KEY HERE", "SLACK_API_KEY": "INSERT YOUR SLACK API KEY HERE", "plugins": []}
- with open(self.filename, 'w') as config_file:
- outfile.write(json.dumps(data, indent=4))
-
- def get_by_path(self, keys_list):
- with open(self.filename) as config_file:
- data = json.load(config_file)
- return functools.reduce(lambda d, k: d[int(k) if isinstance(d, list) else k], keys_list, data)
-
- def set_by_path(self, keys_list, value):
- with open(self.filename, "r+") as config_file:
- config_data = json.load(config_file)
- config_data[keys_list[-1]] = value
-
- config_file.seek(0) # rewind
- config_file.write(json.dumps(config_data))
- config_file.truncate()
-
- def remove(self, json_object):
- with open(self.filename) as config_file:
- config_data = json.load(config_file)
- for item in config_data:
- item.pop(config_data, None)
- with open(self.filename, mode='w') as f:
- f.write(json.dumps(item, indent=4))
-
- def exists(self, keys_list):
- _exists = True
- try:
- if self.get_by_path(keys_list) is None:
- _exists = False
- except (KeyError, TypeError):
- _exists = False
-
- return _exists
-
-class Memory:
- filename = 'config/memory.json'
- def generate(self):
- data = {}
- with open(self.filename, 'w') as memory_file:
- memory_file.write(json.dumps(data, indent=4))
-
- def get_by_path(self, keys_list):
- with open(self.filename) as memory_file:
- data = json.load(memory_file)
- return functools.reduce(lambda d, k: d[int(k) if isinstance(d, list) else k], keys_list, data)
-
- def set_by_path(self, keys_list, value):
- with open(self.filename, "r+") as memory_file:
- memory_data = json.load(memory_file)
- memory_data[keys_list[:-1]][keys_list[-1]] = value
- memory_file.seek(0) # rewind
- memory_file.write(json.dumps(memory_data, indent=4))
- memory_file.truncate()
-
- def remove(self, json_object):
- with open(self.filename) as memory_file:
- memory_data = json.load(memory_file)
- for item in memory_data:
- item.pop(json_object, None)
- with open(self.filename, mode='w') as f:
- f.write(json.dumps(item, indent=4))
-
- def exists(self, keys_list):
- _exists = True
+#Import internal Modules that are needed
+import config
+class Core:
+ def __init__(self):
try:
- if self.get_by_path(keys_list) is None:
- _exists = False
- except (KeyError, TypeError):
- _exists = False
+ self.config = config.Config("config/config.json")
+ except ValueError:
+ logging.exception("failed to load config, malformed json")
+ sys.exit()
- return _exists
-
-class Core:
+ try:
+ self.memory = config.Memory("config/memory.json")
+ except ValueError:
+ logging.exception("failed to load config, malformed json")
+ sys.exit()
def slack_init(self):
- api_key = Config().get_by_path(['SLACK_API_KEY'])
+ api_key = self.config.get_by_path(['SLACK_API_KEY'])
sc = SlackClient(api_key)
print("slack starting...")
@@ -121,41 +55,48 @@ class Core:
def slacksync(bot, update):
params = update.message.text.split()
-
- if Memory().exists(['tg_sl-sync']['tg2sl']):
- if Memory().exists(['tg_sl-sync']['sl2tg']):
- tg2sl = Memory().get_by_path(['tg_sl-sync']['tg2sl'])
- sl2tg = Memory().get_by_path(['tg_sl-sync']['sl2tg'])
-
- if update.message.chat_id in tg2sl:
- print("tretre")
- else:
- try:
- tg2sl[str(update.message.chat_id)] = params[1]
- sl2tg[str(params[1])] = str(update.message.chat_id)
- new_memory = {'tg2sl': tg2sl, 'sl2tg': sl2tg}
- Memory().set_by_path(['tg_sl-sync'], new_memory)
- update.message.reply_text('Saved sync!')
- except Exception as e:
- #print(e)
- exc_type, exc_obj, exc_tb = sys.exc_info()
- fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
- print(exc_type, fname, exc_tb.tb_lineno)
- traceback.print_exc()
- update.message.reply_text('Failed to save sync!')
- print("sync saved")
+ print(params[1])
+
+ try:
+ tg2sl = self.memory.get_by_path(['tg_sl-sync'])['tg2sl']
+ sl2tg = self.memory.get_by_path(['tg_sl-sync'])['sl2tg']
+ except Exception as e:
+ #print(e)
+ exc_type, exc_obj, exc_tb = sys.exc_info()
+ fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
+ print(exc_type, fname, exc_tb.tb_lineno)
+ traceback.print_exc()
+ update.message.reply_text('Failed to get memory. Please contact Admin!')
+
+ if tg2sl:
+ print("works")
+
+ if update.message.chat_id in tg2sl:
+ print("tretre")
+ else:
+ print("got sync request...")
+ try:
+ tg2sl[str(update.message.chat_id)] = params[1]
+ sl2tg[str(params[1])] = str(update.message.chat_id)
+ new_memory = {'tg2sl': tg2sl, 'sl2tg': sl2tg}
+ self.memory.set_by_path(['tg_sl-sync'], new_memory)
+ self.memory.save()
+ update.message.reply_text('Saved sync!')
+ except Exception as e:
+ #print(e)
+ exc_type, exc_obj, exc_tb = sys.exc_info()
+ fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
+ print(exc_type, fname, exc_tb.tb_lineno)
+ traceback.print_exc()
+ update.message.reply_text('Failed to save sync!')
+ print("sync saved")
print("Telegram starting...")
- api_key = Config().get_by_path(['TELEGRAM_API_KEY'])
- if not Memory().exists(['tg_sl-sync']):
+ api_key = self.config.get_by_path(['TELEGRAM_API_KEY'])
+ if not self.memory.exists(['tg_sl-sync']):
print('tg_sl-sync missing...')
- Memory().set_by_path(['tg_sl-sync'], {})
- # if not Memory().exists(['tg_sl-sync']['sl2tg']):
- # print("sl2tg missing...")
- # Memory().set_by_path(['tg_sl-sync']['sl2tg'], {})
- # if not Memory().exists(['tg_sl-sync']['tg2sl']):
- # print("tg2sl missing...")
- # Memory().set_by_path(['tg_sl-sync']['tg2sl'], {})
+ self.memory.set_by_path(['tg_sl-sync'], {'sl2tg':{}, 'tg2sl': {}})
+ self.memory.save()
updater = Updater(api_key)
updater.dispatcher.add_handler(telegram.ext.MessageHandler(Filters.text, sync_handler))
updater.dispatcher.add_handler(telegram.ext.CommandHandler('slacksync', slacksync))
@@ -173,28 +114,26 @@ class Core:
slack = pool.apply_async(self.slack_init(), []) # evaluate "solve2(B)" asynchronously
if __name__ == "__main__":
-
- check_pass = 0
-
if os.path.isdir("config"):
- if os.path.isfile("config/config.json"):
- check_pass = check_pass+1
- else:
- check_pass = check_pass-1
- Config().generate()
-
- if os.path.isfile("config/memory.json"):
- check_pass = check_pass+1
- else:
- check_pass = check_pass-1
- Memory().generate()
-
- if check_pass == 2:
- Core().init()
- else:
- print("Configs are regenrated. Please check the API Keys and restart the Bot.")
+ if not os.path.isfile('config/config.json'):
+ try:
+ shutil.copy('defaults/config.json', "config/config.json")
+ sys.exit('Please set Api Keys')
+ except (OSError, IOError) as e:
+ sys.exit('Failed to copy default config file: {}'.format(e))
+ if not os.path.isfile('config/memory.json'):
+ try:
+ shutil.copy('defaults/memory.json', "config/memory.json")
+ except (OSError, IOError) as e:
+ sys.exit('Failed to copy default memory file: {}'.format(e))
else:
os.mkdir("config")
- Config().generate()
- Memory().generate()
- print("First run... Configs are genrated. Please add the API Keys and restart the Bot.")
+ if not os.path.isfile('config/config.json'):
+ try:
+ shutil.copy('defaults/config.json', "config/config.json")
+ sys.exit('Please set Api Keys')
+ except (OSError, IOError) as e:
+ sys.exit('Failed to copy default config file: {}'.format(e))
+
+ core = Core()
+ core.init()
diff --git a/__pycache__/config.cpython-35.pyc b/__pycache__/config.cpython-35.pyc
Binary files differ.
diff --git a/config.py b/config.py
@@ -0,0 +1,331 @@
+import collections, datetime, functools, json, glob, logging, os, shutil, sys, time
+
+from threading import Timer
+
+
+logger = logging.getLogger(__name__)
+
+
+class Config(collections.MutableMapping):
+ """Configuration JSON storage class"""
+ def __init__(self, filename, default=None, failsafe_backups=0, save_delay=0):
+ self.filename = filename
+ self.default = None
+ self.config = {}
+ self.changed = False
+ self.failsafe_backups = failsafe_backups
+ self.save_delay = save_delay
+ self.load()
+
+ self._timer_save = False
+
+ def _make_failsafe_backup(self):
+ try:
+ json.load(open(self.filename))
+ except IOError:
+ return False
+ except ValueError:
+ logger.warning("{} is corrupted, aborting backup".format(self.filename))
+ return False
+
+ existing = sorted(glob.glob(self.filename + ".*.bak"))
+ while len(existing) > (self.failsafe_backups - 1):
+ os.remove(existing.pop(0))
+
+ backup_file = self.filename + "." + datetime.datetime.now().strftime("%Y%m%d%H%M%S") + ".bak"
+ shutil.copy2(self.filename, backup_file)
+
+ return True
+
+ def _recover_from_failsafe(self):
+ existing = sorted(glob.glob(self.filename + ".*.bak"))
+ while len(existing) > 0:
+ try:
+ recovery_filename = existing.pop()
+ json.load(open(recovery_filename))
+ shutil.copy2(recovery_filename, self.filename)
+ self.load(recovery=True)
+ logger.info("recovery successful: {}".format(recovery_filename))
+ return True
+ except IOError:
+ pass
+ except ValueError:
+ logger.error("corrupted recovery: {}".format(self.filename))
+ return False
+
+ def load(self, recovery=False):
+ """Load config from file"""
+ try:
+ self.config = json.load(open(self.filename))
+ logger.info("{} read".format(self.filename))
+
+ except IOError:
+ self.config = {}
+
+ except ValueError:
+ if not recovery and self.failsafe_backups > 0 and self._recover_from_failsafe():
+ return
+
+ raise
+
+ self.changed = False
+
+ def force_taint(self):
+ self.changed = True
+
+ def loads(self, json_str):
+ """Load config from JSON string"""
+ self.config = json.loads(json_str)
+ self.changed = True
+
+ def save(self, delay=True):
+ if self.save_delay:
+ if delay:
+ if self._timer_save and self._timer_save.is_alive():
+ self._timer_save.cancel()
+ self._timer_save = Timer(self.save_delay, self.save, [], {"delay": False})
+ self._timer_save.start()
+ return False
+
+ """Save config to file (only if config has changed)"""
+ if self.changed:
+ start_time = time.time()
+
+ if self.failsafe_backups:
+ self._make_failsafe_backup()
+
+ with open(self.filename, 'w') as f:
+ json.dump(self.config, f, indent=2, sort_keys=True)
+ self.changed = False
+ interval = time.time() - start_time
+
+ logger.info("{} write {}".format(self.filename, interval))
+
+ return self.changed
+
+ def flush(self):
+ if self._timer_save and self._timer_save.is_alive():
+ logger.info("flushing {}".format(self.filename))
+ self._timer_save.cancel()
+ self.save(delay=False)
+
+ def get_by_path(self, keys_list):
+ """Get item from config by path (list of keys)"""
+ return functools.reduce(lambda d, k: d[int(k) if isinstance(d, list) else k], keys_list, self)
+
+ def set_by_path(self, keys_list, value):
+ """Set item in config by path (list of keys)"""
+ self.get_by_path(keys_list[:-1])[keys_list[-1]] = value
+ self.changed = True
+
+ def pop_by_path(self, keys_list):
+ popped_value = self.get_by_path(keys_list[:-1]).pop(keys_list[-1])
+ self.changed = True
+ return popped_value
+
+ def get_option(self, keyname):
+ try:
+ value = self.config[keyname]
+ except KeyError:
+ value = None
+ return value
+
+ def get_suboption(self, grouping, groupname, keyname):
+ try:
+ value = self.config[grouping][groupname][keyname]
+ except KeyError:
+ value = self.get_option(keyname)
+ return value
+
+ def exists(self, keys_list):
+ _exists = True
+
+ try:
+ if self.get_by_path(keys_list) is None:
+ _exists = False
+ except (KeyError, TypeError):
+ _exists = False
+
+ return _exists
+
+ def __getitem__(self, key):
+ try:
+ return self.config[key]
+ except KeyError:
+ return self.default
+
+ def __setitem__(self, key, value):
+ self.config[key] = value
+ self.changed = True
+
+ def __delitem__(self, key):
+ del self.config[key]
+ self.changed = True
+
+ def __iter__(self):
+ return iter(self.config)
+
+ def __len__(self):
+ return len(self.config)
+
+class Memory(collections.MutableMapping):
+ """Configuration JSON storage class"""
+ def __init__(self, filename, default=None, failsafe_backups=0, save_delay=0):
+ self.filename = filename
+ self.default = None
+ self.config = {}
+ self.changed = False
+ self.failsafe_backups = failsafe_backups
+ self.save_delay = save_delay
+ self.load()
+
+ self._timer_save = False
+
+ def _make_failsafe_backup(self):
+ try:
+ json.load(open(self.filename))
+ except IOError:
+ return False
+ except ValueError:
+ logger.warning("{} is corrupted, aborting backup".format(self.filename))
+ return False
+
+ existing = sorted(glob.glob(self.filename + ".*.bak"))
+ while len(existing) > (self.failsafe_backups - 1):
+ os.remove(existing.pop(0))
+
+ backup_file = self.filename + "." + datetime.datetime.now().strftime("%Y%m%d%H%M%S") + ".bak"
+ shutil.copy2(self.filename, backup_file)
+
+ return True
+
+ def _recover_from_failsafe(self):
+ existing = sorted(glob.glob(self.filename + ".*.bak"))
+ while len(existing) > 0:
+ try:
+ recovery_filename = existing.pop()
+ json.load(open(recovery_filename))
+ shutil.copy2(recovery_filename, self.filename)
+ self.load(recovery=True)
+ logger.info("recovery successful: {}".format(recovery_filename))
+ return True
+ except IOError:
+ pass
+ except ValueError:
+ logger.error("corrupted recovery: {}".format(self.filename))
+ return False
+
+ def load(self, recovery=False):
+ """Load config from file"""
+ try:
+ self.config = json.load(open(self.filename))
+ logger.info("{} read".format(self.filename))
+
+ except IOError:
+ self.config = {}
+
+ except ValueError:
+ if not recovery and self.failsafe_backups > 0 and self._recover_from_failsafe():
+ return
+
+ raise
+
+ self.changed = False
+
+ def force_taint(self):
+ self.changed = True
+
+ def loads(self, json_str):
+ """Load config from JSON string"""
+ self.config = json.loads(json_str)
+ self.changed = True
+
+ def save(self, delay=True):
+ if self.save_delay:
+ if delay:
+ if self._timer_save and self._timer_save.is_alive():
+ self._timer_save.cancel()
+ self._timer_save = Timer(self.save_delay, self.save, [], {"delay": False})
+ self._timer_save.start()
+ return False
+
+ """Save config to file (only if config has changed)"""
+ if self.changed:
+ start_time = time.time()
+
+ if self.failsafe_backups:
+ self._make_failsafe_backup()
+
+ with open(self.filename, 'w') as f:
+ json.dump(self.config, f, indent=2, sort_keys=True)
+ self.changed = False
+ interval = time.time() - start_time
+
+ logger.info("{} write {}".format(self.filename, interval))
+
+ return self.changed
+
+ def flush(self):
+ if self._timer_save and self._timer_save.is_alive():
+ logger.info("flushing {}".format(self.filename))
+ self._timer_save.cancel()
+ self.save(delay=False)
+
+ def get_by_path(self, keys_list):
+ """Get item from config by path (list of keys)"""
+ return functools.reduce(lambda d, k: d[int(k) if isinstance(d, list) else k], keys_list, self)
+
+ def set_by_path(self, keys_list, value):
+ """Set item in config by path (list of keys)"""
+ self.get_by_path(keys_list[:-1])[keys_list[-1]] = value
+ self.changed = True
+
+ def pop_by_path(self, keys_list):
+ popped_value = self.get_by_path(keys_list[:-1]).pop(keys_list[-1])
+ self.changed = True
+ return popped_value
+
+ def get_option(self, keyname):
+ try:
+ value = self.config[keyname]
+ except KeyError:
+ value = None
+ return value
+
+ def get_suboption(self, grouping, groupname, keyname):
+ try:
+ value = self.config[grouping][groupname][keyname]
+ except KeyError:
+ value = self.get_option(keyname)
+ return value
+
+ def exists(self, keys_list):
+ _exists = True
+
+ try:
+ if self.get_by_path(keys_list) is None:
+ _exists = False
+ except (KeyError, TypeError):
+ _exists = False
+
+ return _exists
+
+ def __getitem__(self, key):
+ try:
+ return self.config[key]
+ except KeyError:
+ return self.default
+
+ def __setitem__(self, key, value):
+ self.config[key] = value
+ self.changed = True
+
+ def __delitem__(self, key):
+ del self.config[key]
+ self.changed = True
+
+ def __iter__(self):
+ return iter(self.config)
+
+ def __len__(self):
+ return len(self.config)
diff --git a/defaults/config.json b/defaults/config.json
@@ -0,0 +1,12 @@
+{
+ "admins": [ ],
+ "commands_user": [ ],
+ "commands_enabled": true,
+ "link_to_guide": "https://github.com/hangoutsbot/hangoutsbot/wiki/User-Guide",
+ "silentmode": false,
+ "plugins": [
+
+ ],
+ "TELEGRAM_API_KEY": "SET TELEGRAM API KEY HERE",
+ "SLACK_API_KEY": "SET SLACK API KEY HERE"
+}
diff --git a/defaults/memory.json b/defaults/memory.json
@@ -0,0 +1,3 @@
+{
+
+}
