major refactoring

* move all scripts not intended to be called directly to helpers/ dir
* introduce sequencer.py as replacement for various scripts
* introduce --save option to generate.py
* other smaller changes/bugfixes

clean_data.py

@@ -1,34 +0,0 @@
-#!/usr/bin/python3
-import mailbox
-import jinja2
-import email.header
-import email.utils
-import yaml
-import datetime
-import sys, os
-import pypandoc
-import argparse
-import quopri
-from pprint import pprint
-
-import generate
-
-CONFIG_FILE = "generator.conf"
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--config", "-c", default=CONFIG_FILE)
-    args = parser.parse_args()
-
-    config = generate.get_config(args.config)
-
-    open(config["top_mbox_file"], 'w').close()
-
-
-    for top in config["pre_tops"]:
-        if "file" in top and os.path.isfile(top["file"]):
-            os.remove(top["file"])
-
-    for top in config["post_tops"]:
-        if "file" in top and os.path.isfile(top["file"]):
-            os.remove(top["file"])

generator.conf

@@ -14,6 +14,25 @@ mm_url: "@import(personal.conf:mattermost_url)"
 invite_mail: fsr-einladung@fsmi.uni-karlsruhe.de
 invite_subject: 'Einladung zum Fachschaftsrat am {{date|weekday}}, dem {{date|date}}'
 
+presentation_save_path: data/
+protocol_save_path: protokolle/
+
+sequencer:
+    invite:
+        - clean_data
+        - read_db
+        - read_topmails
+        - generate --invite --send-mail
+    presentation:
+        - read_db
+        - generate --presentation --save
+        - compile_presentation
+    protocol:
+        - clean_data
+        - read_db
+        - read_topmails
+        - generate --protocol --save
+
 pre_tops:
     - title: Begrüßung
       protostub: Der FSR wird begrüßt.
@@ -26,14 +45,14 @@ pre_tops:
 {% endfor %}'
     - title: Unveröffentlichte Protokolle
       file: "data/uvproto.txt"
-      command: ./get_uvproto.sh
+      command: helpers/get_uvproto.sh
       body: "* FSR-Protokoll vom {{last_date|date}}"
     - title: Berichte
       protostub:
 
 post_tops:
     - title: Unbeantwortete E-Mails
-      command: ./read_ubmails.py
+      command: helpers/read_ubmails.py
       protostub: '<intern>
 
       {{top.body}}
@@ -58,8 +77,8 @@ post_tops:
 {{"}}"}}'
     - title: Termine
       file: "data/termine.txt"
-      command: "./list_termine.sh" # if khal is setup correctly, uncomment this to read events from there
-      proto_command: "./list_termine_proto.sh"
+      command: "helpers/list_termine.sh" # if khal is setup correctly, uncomment this to read events from there
+      proto_command: "helpers/list_termine_proto.sh"
       protostub: '{|
 
 {{"{{"}}Termin|was=AAAAAAAAAA|wann=XX.YY.{{"}}"}}
@@ -78,6 +97,7 @@ top_mbox_file: data/fsr_tops.mbox
 mbox_out: data/invitemail.mbox
 top_inbox_maildir: "@import(personal.conf:top_inbox_maildir)"
 ubmails_inbox_maildir: "@import(personal.conf:ubmails_inbox_maildir)"
+top_list_id: top.fsmi.uni-karlsruhe.de
 
 last_date_file: data/last_date
 

helpers/compile_presentation.sh

@@ -1,10 +1,10 @@
 #!/bin/sh
 set -e
-dest_file="$(realpath "$(dirname "$0")")/data/presentation_$(date +%Y-%m-%d).tex"
+dest_file="presentation_$(date +%Y-%m-%d).tex"
 
 echo Compiling
 mkdir -p data/presentation
 cd data/presentation/
 
-latexmk -pdf "$dest_file"
+latexmk -pdf "../$dest_file"
 ln -srnf "presentation_$(date +%Y-%m-%d).pdf" ../../presentation.pdf

helpers/generate.py

@@ -14,6 +14,7 @@ import json
 import subprocess
 import re
 from pprint import pprint
+from pathlib import Path
 
 CONFIG_FILE = "generator.conf"
 
@@ -155,10 +156,10 @@ def conf2top(top):
             print("Warning: Error opening", top["file"], file=sys.stderr)
 
     if "command" in top:
-        body = subprocess.run(top["command"], shell=True, text=True, capture_output=True, check=True).stdout
+        body = subprocess.run(top["command"], shell=True, text=True, capture_output=True, check=args.allowfailcommand).stdout
 
     if "proto_command" in top:
-        protostub = subprocess.run(top["proto_command"], shell=True, text=True, capture_output=True, check=True).stdout
+        protostub = subprocess.run(top["proto_command"], shell=True, text=True, capture_output=True, check=args.allowfailcommand).stdout
 
     return Top(top["title"], sender, body, protostub)
 
@@ -218,6 +219,10 @@ if __name__ == "__main__":
     parser.add_argument("--write-mbox", action="store_true")
     parser.add_argument("--send-mm", action="store_true")
     parser.add_argument("--send-mail", action="store_true")
+    parser.add_argument("--save", action="store_true")
+    parser.add_argument("--allowfailcommand", action="store_false")
+    parser.add_argument("--time")
+    parser.add_argument("--date")
     args = parser.parse_args()
 
     config = get_config(args.config)
@@ -249,10 +254,13 @@ if __name__ == "__main__":
     mbox = mailbox.mbox(config["top_mbox_file"])
 
     current_date = next_weekday(datetime.date.today(), config["default_weekday"])
-    next_date = current_date + datetime.timedelta(days=7)
+    if args.date:
+        current_date = datetime.date.fromisoformat(args.date)
+    #next_date = current_date + datetime.timedelta(days=7)
+    next_date = next_weekday(current_date, config["default_weekday"])
     last_date = last_weekday(current_date, config["default_weekday"])
 
-    time = datetime.time.fromisoformat(config["default_time"])
+    time = datetime.time.fromisoformat(args.time or config["default_time"])
 
     pre_tops = []
     post_tops = []
@@ -289,13 +297,26 @@ if __name__ == "__main__":
         for top in to:  
             pprint(top.__dict__)
         pprint(context)
+    elif args.save:
+        if args.invite:
+            filename = Path(config["invite_save_path"]) / Path("invite_"+datetime.date.today().isoformat()+".txt")
+        elif args.presentation:
+            filename = Path(config["presentation_save_path"]) / Path("presentation_"+datetime.date.today().isoformat()+".tex")
+        elif args.protocol:
+            filename = Path(config["protocol_save_path"]) / Path(datetime.date.today().isoformat())
+        else:
+            raise Exception("Should never happen")
+
+        with open(filename, "w") as file:
+            file.write(template.render(context))
+        
     elif args.send_mail:
         msg = email.message.EmailMessage()
         msg.set_content(template.render(context))
         msg["Subject"] = j2env.from_string(config["invite_subject"]).render(context)
         msg["From"] = email.utils.formataddr((config["redeleitung"]["name"], config["redeleitung"]["email"]))
         msg["To"] = config["invite_mail"]
-        subprocess.run([*config["sendmail"], config["invite_mail"]], input=str(msg), text=True)
+        subprocess.run([*config["sendmail"], config["invite_mail"]], input=str(msg), text=True, check=True)
 
     elif args.write_mbox:
         msg = email.message.EmailMessage()

helpers/list_termine.sh

@@ -0,0 +1,4 @@
+#!/bin/sh -e
+
+khal list --day-format "" --format "* {start} {title}" -a calendars_fsmi today 30d | grep -v Fachschaftsrat | grep -v Feriensprechstunde || true
+

helpers/list_termine_proto.sh

@@ -1,4 +1,4 @@
 #!/bin/sh -e
 echo "{{'{|'}}"
-khal list --day-format "" --format "{{{{'{{{{'}}}}Termin|was={title}|wann={start}{{{{'}}}}'}}}}" -a calendars_fsmi today 30d | grep -v Fachschaftsrat
+khal list --day-format "" --format "{{{{'{{{{'}}}}Termin|was={title}|wann={start}{{{{'}}}}'}}}}" -a calendars_fsmi today 30d | grep -v Fachschaftsrat | grep -v Feriensprechstunde || true
 echo "{{'|}'}}"

helpers/read_ubmails.py

@@ -10,7 +10,7 @@ import pypandoc
 import argparse
 import quopri
 from pprint import pprint
-from dateutil import parser
+from dateutil import parser as dateutilparser
 
 import generate
 
@@ -37,7 +37,7 @@ if __name__ == "__main__":
     for message in mbox:
         if message["Subject"]:
             if decode_header(message["Subject"]).strip() == "Unbeantwortete Mails":
-                date = parser.parse(message["Date"])
+                date = dateutilparser.parse(message["Date"])
                 if latest is None:
                     latest = message
                     latest_date = date

helpers/sequencer.py

@@ -0,0 +1,101 @@
+#!/usr/bin/python3
+import sys
+import argparse
+import subprocess
+import os
+import mailbox
+import datetime
+
+import pytz
+from dateutil import parser as dateutilparser
+
+import generate
+
+class Actions:
+    @staticmethod
+    def clean_data():
+        open(config["top_mbox_file"], 'w').close()
+
+
+        for top in config["pre_tops"]:
+            if "file" in top and os.path.isfile(top["file"]):
+                os.remove(top["file"])
+
+        for top in config["post_tops"]:
+            if "file" in top and os.path.isfile(top["file"]):
+                os.remove(top["file"])
+
+    @staticmethod
+    def read_db():
+        subprocess.run(["helpers/read_db.sh"], check=True)
+
+    @staticmethod
+    def read_topmails():
+        in_mbox = mailbox.Maildir(config["top_inbox_maildir"])
+        out_mbox = mailbox.mbox(config["top_mbox_file"])
+
+        out_mbox.clear()
+
+        buffer = []
+
+        timezone = pytz.timezone("Europe/Berlin")
+        last_fsr_date = datetime.date.fromisoformat(open(config["last_date_file"]).read().strip())
+        last_fsr_datetime = datetime.datetime.combine(last_fsr_date, datetime.time(17, 30), timezone)
+
+        for message in in_mbox:
+            if message["List-Id"]:
+                if config["top_list_id"] in generate.decode_header(message["List-Id"]).strip():
+                    date = dateutilparser.parse(message["Date"])
+                    if date > last_fsr_datetime:
+                        buffer.append(message)
+
+        for message in sorted(buffer, key=lambda x: dateutilparser.parse(x["Date"])):
+            out_mbox.add(message)
+        
+        out_mbox.close()
+
+    @staticmethod
+    def generate(*args):
+        subprocess.run(["helpers/generate.py", "--config", cliargs.config, *args], check=True)
+    
+    @staticmethod
+    def compile_presentation():
+        subprocess.run(["helpers/compile_presentation.sh"], check=True)
+
+def dispatch(action, args=[]):
+    if action in dir(Actions):
+        getattr(Actions, action)(*args)
+    elif action in config["sequencer"]:
+        for item in config["sequencer"][action]:
+            ilist = item.split()
+            if not ilist[0] in cliargs.skip:
+                dispatch(ilist[0], ilist[1:])
+    else:
+        raise ValueError(f"Action {action} not found")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--config", "-c", default=generate.CONFIG_FILE)
+    parser.add_argument("--skip", action='append')
+    parser.add_argument("action", nargs="?")
+    cliargs = parser.parse_args()
+
+    if cliargs.skip is None:
+        cliargs.skip = []
+
+    config = generate.get_config(cliargs.config)
+
+    if os.path.dirname(cliargs.config) != "":
+        os.chdir(os.path.dirname(cliargs.config))
+
+    commandname = sys.argv[0]
+    commandname = commandname.split("/")[-1]
+
+    if commandname != "sequencer.py":
+        dispatch(commandname)
+    elif cliargs.action:
+        dispatch(cliargs.action)
+    else:
+        print("No action specified")
+        sys.exit(1)
+

invite

@@ -0,0 +1 @@
+helpers/sequencer.py

list_termine.sh

@@ -1,3 +0,0 @@
-#!/bin/sh -e
-
-khal list --day-format "" --format "* {start} {title}" -a calendars_fsmi today 30d | grep -v Fachschaftsrat

prepare_presentation.sh

@@ -1,10 +0,0 @@
-#!/bin/sh
-set -e
-dest_file="$(realpath "$(dirname "$0")")/data/presentation_$(date +%Y-%m-%d).tex"
-
-echo Reading old protocols
-./read_db.sh
-echo Generating
-./generate.py --presentation >"$dest_file"
-
-./compile_presentation.sh

prepare_protocol.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-echo Reading old protocols
-./read_db.sh
-echo Getting E-Mail
-./read_topmails.py
-echo Generating
-
-./generate.py --proto > protokolle/$(date +%Y-%m-%d)

presentation

@@ -0,0 +1 @@
+helpers/sequencer.py

protocol

@@ -0,0 +1 @@
+helpers/sequencer.py

read_topmails.py

@@ -1,55 +0,0 @@
-#!/usr/bin/python3
-import mailbox
-import jinja2
-import email.header
-import email.utils
-import yaml
-import datetime
-import sys, os
-import pypandoc
-import argparse
-import quopri
-import pytz
-from pprint import pprint
-from dateutil import parser
-
-import generate
-
-CONFIG_FILE = "generator.conf"
-
-def decode_header(header):
-    decoded_header = email.header.decode_header(header)[0]
-    encoding = decoded_header[1] or "ascii"
-    if encoding == "unknown-8bit":
-        encoding = "ascii"
-    return decoded_header[0].decode(encoding, errors="replace") if isinstance(decoded_header[0], bytes) else decoded_header[0]
-
-if __name__ == "__main__":
-    aparser = argparse.ArgumentParser()
-    aparser.add_argument("--config", "-c", default=CONFIG_FILE)
-    args = aparser.parse_args()
-
-    config = generate.get_config(args.config)
-    in_mbox = mailbox.Maildir(config["top_inbox_maildir"])
-    out_mbox = mailbox.mbox(config["top_mbox_file"])
-
-    out_mbox.clear()
-
-    buffer = []
-
-    timezone = pytz.timezone("Europe/Berlin")
-    last_fsr_date = datetime.date.fromisoformat(open(config["last_date_file"]).read().strip())
-    last_fsr_datetime = datetime.datetime.combine(last_fsr_date, datetime.time(17, 30), timezone)
-
-    for message in in_mbox:
-        if message["List-Id"]:
-            if "top.fsmi.uni-karlsruhe.de" in decode_header(message["List-Id"]).strip():
-                date = parser.parse(message["Date"])
-                if date > last_fsr_datetime:
-                    buffer.append(message)
-
-    for message in sorted(buffer, key=lambda x: parser.parse(x["Date"])):
-        out_mbox.add(message)
-    
-    out_mbox.close()
-

sequencer.py

@@ -0,0 +1 @@
+helpers/sequencer.py