give config to package defined actions

fsmi_fsr/actions.py

@@ -0,0 +1,50 @@
+import subprocess
+import os
+import mailbox
+import datetime
+
+import pytz
+from dateutil import parser as dateutilparser
+
+
+def clean_data(config):
+    open(config["top_mbox_file"], 'w').close()
+
+
+    for top in config["pre_tops"]:
+        if "file" in top and os.path.isfile(top["file"]):
+            os.remove(top["file"])
+
+    for top in config["post_tops"]:
+        if "file" in top and os.path.isfile(top["file"]):
+            os.remove(top["file"])
+
+def read_db(config):
+    subprocess.run(["helpers/read_db.sh"], check=True)
+
+def read_topmails(config):
+    in_mbox = mailbox.Maildir(config["top_inbox_maildir"])
+    out_mbox = mailbox.mbox(config["top_mbox_file"])
+
+    out_mbox.clear()
+
+    buffer = []
+
+    timezone = pytz.timezone("Europe/Berlin")
+    last_fsr_date = datetime.date.fromisoformat(open(config["last_date_file"]).read().strip())
+    last_fsr_datetime = datetime.datetime.combine(last_fsr_date, datetime.time(17, 30), timezone)
+
+    for message in in_mbox:
+        if message["List-Id"]:
+            if config["top_list_id"] in generate.decode_header(message["List-Id"]).strip():
+                date = dateutilparser.parse(message["Date"])
+                if date > last_fsr_datetime:
+                    buffer.append(message)
+
+    for message in sorted(buffer, key=lambda x: dateutilparser.parse(x["Date"])):
+        out_mbox.add(message)
+    
+    out_mbox.close()
+
+def compile_presentation(config):
+    subprocess.run(["helpers/compile_presentation.sh"], check=True)

fsmi_fsr/generator.conf

@@ -137,5 +137,6 @@ Beschlussvorlage"
 sendmail: ["/usr/sbin/sendmail"]
 override_file: "@import(personal.conf:override_file)"
 
+package_actions_file: "actions.py"
 
 # vim: filetype=yaml

helpers/generate.py

@@ -1,77 +1,32 @@
 #!/usr/bin/python3
 import mailbox
-import jinja2
 import email.header
 import email.utils
 import yaml
 import datetime
 import sys, os
-import pypandoc
 import argparse
 import quopri
-import requests
 import json
 import subprocess
 import re
 from pprint import pprint
 from pathlib import Path
 
-CONFIG_FILE = "generator.conf"
+import requests
+import pypandoc
+import jinja2
 
-WEEKDAYS = { 0: "Montag",
-             1: "Dienstag",
-             2: "Mittwoch",
-             3: "Donnerstag",
-             4: "Freitag",
-             5: "Samstag",
-             6: "Sonntag" }
-
-IMPORT_RE = re.compile(r"@import\((.*)\)")
-FILE_RE = re.compile(r"@file\((.*)\)")
-
-def decode_header(header):
-    decoded_headers = email.header.decode_header(header)
-    header_strs = []
-    for dheader in decoded_headers:
-        encoding = dheader[1] or "ascii"
-        if encoding == "unknown-8bit":
-            encoding = "ascii"
-        result = dheader[0].decode(encoding, errors="replace") if isinstance(dheader[0], bytes) else dheader[0]
-        header_strs.append(result)
-    header_text = "".join(header_strs)
-    header_text = re.sub(r"\n(\s)", r" ", header_text)
-    #header_text = re.sub(r"\n(\s)", r"", header_text)
-    return header_text
-
-def get_body_text(msg):
-    # from https://stackoverflow.com/a/1463144
-    for part in msg.walk():
-        # each part is a either non-multipart, or another multipart message
-        # that contains further parts... Message is organized like a tree
-        if part.get_content_type() == 'text/plain':
-            payload = part.get_payload()
-            if part["Content-Transfer-Encoding"] == "quoted-printable":
-                payload = quopri.decodestring(payload.encode("ascii")).decode(part.get_content_charset("utf-8"))
-            return payload
-
-# from https://stackoverflow.com/a/49986645
-def deEmojify(text):
-    regrex_pattern = re.compile(pattern = "["
-        u"\U0001F600-\U0001F64F"  # emoticons
-        u"\U0001F300-\U0001F5FF"  # symbols & pictographs
-        u"\U0001F680-\U0001F6FF"  # transport & map symbols
-        u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
-                           "]+", flags = re.UNICODE)
-    return regrex_pattern.sub(r'',text)
+import util
 
 class Top:
     def __init__(self, title=None, sender=None, body=None, protostub=None, message=None):
         if message:
             subject = message["Subject"]
             needs_stripping = subject[:6] == "[top] "
-            self.title = deEmojify(decode_header(subject[6:] if needs_stripping else subject))
+            self.title = util.deEmojify(util.decode_header(subject[6:] if needs_stripping else subject))
             real_name, address = email.utils.parseaddr(message["From"])
-            real_name = decode_header(real_name)
+            real_name = util.decode_header(real_name)
             self.sender = real_name or address
             payload = get_body_text(message)
             self.body = str(payload.rpartition("\n--")[0] if "\n--" in payload else payload)
@@ -89,19 +44,6 @@ class Top:
     def __repr__(self):
         return "<TOP "+self.title+">"
 
-# from https://stackoverflow.com/questions/6558535/find-the-date-for-the-first-monday-after-a-given-a-date
-def next_weekday(d, weekday):
-    days_ahead = weekday - d.weekday()
-    if days_ahead < 0: # Target day already happened this week
-        days_ahead += 7
-    return d + datetime.timedelta(days_ahead)
-
-def last_weekday(d, weekday):
-    days_ahead = weekday - d.weekday()
-    if days_ahead >= 0: # Target day already happened this week
-        days_ahead -= 7
-    return d + datetime.timedelta(days_ahead)
-
 def wiki2latex(intext):
     intext = intext.replace(":\n", ":\n\n")
     return pypandoc.convert_text(intext, 'latex', format='md')
@@ -116,7 +58,7 @@ def time(intime):
     return intime.strftime("%H:%M")
 
 def weekday(indate):
-    return WEEKDAYS[indate.weekday()]
+    return util.WEEKDAYS[indate.weekday()]
 
 def prototop(top):
     result  = ""
@@ -169,57 +111,10 @@ def conf2top(top):
 
     return Top(top["title"], sender, body, protostub)
 
-import_cache = {}
 
-def get_imported_conf_entry(f, key):
-    if f not in import_cache:
-        import_cache[f] = yaml.safe_load(open(f))
-    return import_cache[f][key]
-
-
-def do_imports(entry):
-    if isinstance(entry, dict):
-        d = entry.items()
-        result = {}
-        for key,value in d:
-            try:
-                result[key] = do_imports(value)
-            except KeyError:
-                pass
-            except OSError as e:
-                print(f"Warning: reading {key} yielded error {e}")
-        return result
-
-    if isinstance(entry, list):
-        l = iter(entry)
-        result = []
-        for item in l:
-            result.append(do_imports(item))
-        return result
-
-    if isinstance(entry, str):
-        match = IMPORT_RE.match(entry)
-        if match:
-            f, key = match.group(1).split(":")
-            return get_imported_conf_entry(f, key)
-        match = FILE_RE.match(entry)
-        if match:
-            with open(match.group(1)) as f:
-                return f.read().strip()
-
-    return entry
-
-def get_config(f):
-    raw_config = yaml.safe_load(open(f))
-    config = do_imports(raw_config)
-    if "override_file" in config:
-        override_config = yaml.safe_load(open(config["override_file"]))
-        config.update(override_config)
-    return config
-
-if __name__ == "__main__":
+def main(rawargs=None):
     parser = argparse.ArgumentParser()
-    parser.add_argument("--config", "-c", default=CONFIG_FILE)
+    parser.add_argument("--config", "-c", default=util.CONFIG_FILE)
     mode = parser.add_mutually_exclusive_group(required=True)
     mode.add_argument("--invite", action="store_true")
     mode.add_argument("--mm-invite", action="store_true")
@@ -233,9 +128,10 @@ if __name__ == "__main__":
     parser.add_argument("--save", action="store_true")
     parser.add_argument("--time")
     parser.add_argument("--date")
-    args = parser.parse_args()
+    args = parser.parse_args(rawargs)
 
-    config = get_config(args.config)
+    global config
+    config = util.get_config(util.get_normalized_config_path(args.config))
 
     if args.print_config:
         pprint(config)
@@ -252,6 +148,7 @@ if __name__ == "__main__":
     else:
         raise Exception("Should never happen")
 
+    global j2env
     j2env = jinja2.Environment()
     j2env.filters["wiki2latex"] = wiki2latex
     j2env.filters["j2replace"] = j2replace
@@ -263,14 +160,14 @@ if __name__ == "__main__":
 
     mbox = mailbox.mbox(config["top_mbox_file"])
 
-    current_date = next_weekday(datetime.date.today(), config["default_weekday"])
+    current_date = util.next_weekday(datetime.date.today(), config["default_weekday"])
     if args.date:
         current_date = datetime.date.fromisoformat(args.date)
     #next_date = current_date + datetime.timedelta(days=7)
-    next_date = next_weekday(current_date, config["default_weekday"])
-    last_date = last_weekday(current_date, config["default_weekday"])
+    next_date = util.next_weekday(current_date, config["default_weekday"])
+    last_date = util.last_weekday(current_date, config["default_weekday"])
 
-    time = datetime.time.fromisoformat(args.time or config["default_time"])
+    actual_time = datetime.time.fromisoformat(args.time or config["default_time"])
 
     pre_tops = []
     post_tops = []
@@ -291,17 +188,18 @@ if __name__ == "__main__":
 
     to = pre_tops + email_tops + post_tops
 
+    global context
     context = {"to": to,
                "redeleitung": config["redeleitung"],
                "protokoll": config["protokoll"],
                "date": current_date,
-               "time": time,
+               "time": actual_time,
                "place": config["place"],
                "next_date": next_date,
                "last_date": last_date,
                "meeting_link": config["meeting_link"],
                "email_tops": email_tops,
-               "WEEKDAYS": WEEKDAYS}
+               "WEEKDAYS": util.WEEKDAYS}
 
     if args.debug:
         for top in to:  
@@ -344,3 +242,6 @@ if __name__ == "__main__":
         response = requests.post(config["mm_url"], headers=headers, data=values)
     else:
         print(template.render(context))
+
+if __name__ == "__main__":
+    main()

helpers/sequencer.py

@@ -3,72 +3,25 @@ import sys
 import argparse
 import subprocess
 import os
-import mailbox
-import datetime
-
-import pytz
-from dateutil import parser as dateutilparser
+import importlib.machinery
 
 import generate
+import util
 
-class Actions:
-    @staticmethod
-    def clean_data():
-        open(config["top_mbox_file"], 'w').close()
-
-
-        for top in config["pre_tops"]:
-            if "file" in top and os.path.isfile(top["file"]):
-                os.remove(top["file"])
-
-        for top in config["post_tops"]:
-            if "file" in top and os.path.isfile(top["file"]):
-                os.remove(top["file"])
-
-    @staticmethod
-    def read_db():
-        subprocess.run(["helpers/read_db.sh"], check=True)
-
-    @staticmethod
-    def read_topmails():
-        in_mbox = mailbox.Maildir(config["top_inbox_maildir"])
-        out_mbox = mailbox.mbox(config["top_mbox_file"])
-
-        out_mbox.clear()
-
-        buffer = []
-
-        timezone = pytz.timezone("Europe/Berlin")
-        last_fsr_date = datetime.date.fromisoformat(open(config["last_date_file"]).read().strip())
-        last_fsr_datetime = datetime.datetime.combine(last_fsr_date, datetime.time(17, 30), timezone)
-
-        for message in in_mbox:
-            if message["List-Id"]:
-                if config["top_list_id"] in generate.decode_header(message["List-Id"]).strip():
-                    date = dateutilparser.parse(message["Date"])
-                    if date > last_fsr_datetime:
-                        buffer.append(message)
-
-        for message in sorted(buffer, key=lambda x: dateutilparser.parse(x["Date"])):
-            out_mbox.add(message)
-        
-        out_mbox.close()
-
+class BuiltinActions:
     @staticmethod
     def generate(*args):
-        subprocess.run(["helpers/generate.py", "--config", cliargs.config, *args], check=True)
-    
-    @staticmethod
-    def compile_presentation():
-        subprocess.run(["helpers/compile_presentation.sh"], check=True)
+        generate.main(["--config", config_path, *args])
     
     @staticmethod
     def external(*args):
         subprocess.run(args, check=True)
 
 def dispatch(action, args=[]):
-    if action in dir(Actions):
-        getattr(Actions, action)(*args)
+    if package_actions and action in dir(package_actions):
+        getattr(package_actions, action)(config, *args)
+    elif action in dir(BuiltinActions):
+        getattr(BuiltinActions, action)(*args)
     elif action in config["sequencer"]:
         for item in config["sequencer"][action]:
             ilist = item.split()
@@ -78,12 +31,11 @@ def dispatch(action, args=[]):
         raise ValueError(f"Action {action} not found")
 
 if __name__ == "__main__":
-    commandname = sys.argv[0]
-    commandname = commandname.split("/")[-1]
+    commandname = sys.argv[0].split("/")[-1]
     using_alias = commandname != "sequencer.py"
 
     parser = argparse.ArgumentParser()
-    parser.add_argument("--config", "-c", default=generate.CONFIG_FILE)
+    parser.add_argument("--config", "-c", default=util.CONFIG_FILE)
     parser.add_argument("--skip", action='append')
     if not using_alias:
         parser.add_argument("action", nargs="?")
@@ -92,10 +44,21 @@ if __name__ == "__main__":
     if cliargs.skip is None:
         cliargs.skip = []
 
-    config = generate.get_config(cliargs.config)
+    config_path = util.get_normalized_config_path(cliargs.config)
+    config_path = os.path.abspath(config_path)
 
-    if os.path.dirname(cliargs.config) != "":
-        os.chdir(os.path.dirname(cliargs.config))
+    original_working_dir = os.getcwd()
+    normalized_working_dir = original_working_dir
+
+    if os.path.dirname(config_path) != "":
+        normalized_working_dir = os.path.dirname(config_path)
+    os.chdir(normalized_working_dir)
+
+    config = util.get_config(config_path)
+
+    package_actions = None
+    if "package_actions_file" in config:
+        package_actions = importlib.machinery.SourceFileLoader("package_actions",config["package_actions_file"]).load_module()
 
     if using_alias:
         dispatch(commandname)
@@ -104,4 +67,3 @@ if __name__ == "__main__":
     else:
         print("No action specified")
         sys.exit(1)
-

helpers/util.py

@@ -0,0 +1,131 @@
+#!/usr/bin/python3
+import mailbox
+import jinja2
+import email.header
+import email.utils
+import yaml
+import datetime
+import sys, os
+import argparse
+import quopri
+import requests
+import json
+import subprocess
+import re
+from pprint import pprint
+from pathlib import Path
+
+
+IMPORT_RE = re.compile(r"@import\((.*)\)")
+FILE_RE = re.compile(r"@file\((.*)\)")
+
+CONFIG_FILE = "fsmi_fsr/"
+
+WEEKDAYS = { 0: "Montag",
+             1: "Dienstag",
+             2: "Mittwoch",
+             3: "Donnerstag",
+             4: "Freitag",
+             5: "Samstag",
+             6: "Sonntag" }
+
+def decode_header(header):
+    decoded_headers = email.header.decode_header(header)
+    header_strs = []
+    for dheader in decoded_headers:
+        encoding = dheader[1] or "ascii"
+        if encoding == "unknown-8bit":
+            encoding = "ascii"
+        result = dheader[0].decode(encoding, errors="replace") if isinstance(dheader[0], bytes) else dheader[0]
+        header_strs.append(result)
+    header_text = "".join(header_strs)
+    header_text = re.sub(r"\n(\s)", r" ", header_text)
+    #header_text = re.sub(r"\n(\s)", r"", header_text)
+    return header_text
+
+def get_body_text(msg):
+    # from https://stackoverflow.com/a/1463144
+    for part in msg.walk():
+        # each part is a either non-multipart, or another multipart message
+        # that contains further parts... Message is organized like a tree
+        if part.get_content_type() == 'text/plain':
+            payload = part.get_payload()
+            if part["Content-Transfer-Encoding"] == "quoted-printable":
+                payload = quopri.decodestring(payload.encode("ascii")).decode(part.get_content_charset("utf-8"))
+            return payload
+
+# from https://stackoverflow.com/a/49986645
+def deEmojify(text):
+    regrex_pattern = re.compile(pattern = "["
+        u"\U0001F600-\U0001F64F"  # emoticons
+        u"\U0001F300-\U0001F5FF"  # symbols & pictographs
+        u"\U0001F680-\U0001F6FF"  # transport & map symbols
+        u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
+                           "]+", flags = re.UNICODE)
+    return regrex_pattern.sub(r'',text)
+
+# from https://stackoverflow.com/questions/6558535/find-the-date-for-the-first-monday-after-a-given-a-date
+def next_weekday(d, weekday):
+    days_ahead = weekday - d.weekday()
+    if days_ahead < 0: # Target day already happened this week
+        days_ahead += 7
+    return d + datetime.timedelta(days_ahead)
+
+def last_weekday(d, weekday):
+    days_ahead = weekday - d.weekday()
+    if days_ahead >= 0: # Target day already happened this week
+        days_ahead -= 7
+    return d + datetime.timedelta(days_ahead)
+
+import_cache = {}
+
+def get_imported_conf_entry(f, key):
+    if f not in import_cache:
+        import_cache[f] = yaml.safe_load(open(f))
+    return import_cache[f][key]
+
+def do_imports(entry):
+    if isinstance(entry, dict):
+        d = entry.items()
+        result = {}
+        for key,value in d:
+            try:
+                result[key] = do_imports(value)
+            except KeyError:
+                pass
+            except OSError as e:
+                print(f"Warning: reading {key} yielded error {e}")
+        return result
+
+    if isinstance(entry, list):
+        l = iter(entry)
+        result = []
+        for item in l:
+            result.append(do_imports(item))
+        return result
+
+    if isinstance(entry, str):
+        match = IMPORT_RE.match(entry)
+        if match:
+            f, key = match.group(1).split(":")
+            return get_imported_conf_entry(f, key)
+        match = FILE_RE.match(entry)
+        if match:
+            with open(match.group(1)) as f:
+                return f.read().strip()
+
+    return entry
+
+def get_normalized_config_path(f):
+    if os.path.isdir(f):
+        f = Path(f) / Path("generator.conf")
+    return f
+
+def get_config(f):
+    raw_config = yaml.safe_load(open(f))
+    config = do_imports(raw_config)
+    if "override_file" in config:
+        override_config = yaml.safe_load(open(config["override_file"]))
+        config.update(override_config)
+    return config
+