From 151f11d90b07c5b78044aab36487d1945c730d7f Mon Sep 17 00:00:00 2001 From: Yannik Enss Date: Wed, 9 Mar 2022 15:52:02 +0100 Subject: [PATCH] major refactoring * move all scripts not intended to be called directly to helpers/ dir * introduce sequencer.py as replacement for various scripts * introduce --save option to generate.py * other smaller changes/bugfixes --- clean_data.py | 34 ------ generator.conf | 28 ++++- .../compile_presentation.sh | 4 +- generate.py => helpers/generate.py | 31 +++++- get_uvproto.sh => helpers/get_uvproto.sh | 0 helpers/list_termine.sh | 4 + .../list_termine_proto.sh | 2 +- read_db.sh => helpers/read_db.sh | 0 read_ubmails.py => helpers/read_ubmails.py | 4 +- helpers/sequencer.py | 101 ++++++++++++++++++ invite | 1 + list_termine.sh | 3 - prepare_presentation.sh | 10 -- prepare_protocol.sh | 9 -- presentation | 1 + protocol | 1 + read_topmails.py | 55 ---------- sequencer.py | 1 + 18 files changed, 164 insertions(+), 125 deletions(-) delete mode 100755 clean_data.py rename compile_presentation.sh => helpers/compile_presentation.sh (58%) rename generate.py => helpers/generate.py (88%) rename get_uvproto.sh => helpers/get_uvproto.sh (100%) create mode 100755 helpers/list_termine.sh rename list_termine_proto.sh => helpers/list_termine_proto.sh (74%) rename read_db.sh => helpers/read_db.sh (100%) rename read_ubmails.py => helpers/read_ubmails.py (93%) create mode 100755 helpers/sequencer.py create mode 120000 invite delete mode 100755 list_termine.sh delete mode 100755 prepare_presentation.sh delete mode 100755 prepare_protocol.sh create mode 120000 presentation create mode 120000 protocol delete mode 100755 read_topmails.py create mode 120000 sequencer.py diff --git a/clean_data.py b/clean_data.py deleted file mode 100755 index bfc1a3a..0000000 --- a/clean_data.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/python3 -import mailbox -import jinja2 -import email.header -import email.utils -import yaml -import datetime -import sys, os -import pypandoc -import argparse -import quopri -from pprint import pprint - -import generate - -CONFIG_FILE = "generator.conf" - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--config", "-c", default=CONFIG_FILE) - args = parser.parse_args() - - config = generate.get_config(args.config) - - open(config["top_mbox_file"], 'w').close() - - - for top in config["pre_tops"]: - if "file" in top and os.path.isfile(top["file"]): - os.remove(top["file"]) - - for top in config["post_tops"]: - if "file" in top and os.path.isfile(top["file"]): - os.remove(top["file"]) diff --git a/generator.conf b/generator.conf index 0e2cd24..c54e7f3 100644 --- a/generator.conf +++ b/generator.conf @@ -14,6 +14,25 @@ mm_url: "@import(personal.conf:mattermost_url)" invite_mail: fsr-einladung@fsmi.uni-karlsruhe.de invite_subject: 'Einladung zum Fachschaftsrat am {{date|weekday}}, dem {{date|date}}' +presentation_save_path: data/ +protocol_save_path: protokolle/ + +sequencer: + invite: + - clean_data + - read_db + - read_topmails + - generate --invite --send-mail + presentation: + - read_db + - generate --presentation --save + - compile_presentation + protocol: + - clean_data + - read_db + - read_topmails + - generate --protocol --save + pre_tops: - title: Begrüßung protostub: Der FSR wird begrüßt. @@ -26,14 +45,14 @@ pre_tops: {% endfor %}' - title: Unveröffentlichte Protokolle file: "data/uvproto.txt" - command: ./get_uvproto.sh + command: helpers/get_uvproto.sh body: "* FSR-Protokoll vom {{last_date|date}}" - title: Berichte protostub: post_tops: - title: Unbeantwortete E-Mails - command: ./read_ubmails.py + command: helpers/read_ubmails.py protostub: ' {{top.body}} @@ -58,8 +77,8 @@ post_tops: {{"}}"}}' - title: Termine file: "data/termine.txt" - command: "./list_termine.sh" # if khal is setup correctly, uncomment this to read events from there - proto_command: "./list_termine_proto.sh" + command: "helpers/list_termine.sh" # if khal is setup correctly, uncomment this to read events from there + proto_command: "helpers/list_termine_proto.sh" protostub: '{| {{"{{"}}Termin|was=AAAAAAAAAA|wann=XX.YY.{{"}}"}} @@ -78,6 +97,7 @@ top_mbox_file: data/fsr_tops.mbox mbox_out: data/invitemail.mbox top_inbox_maildir: "@import(personal.conf:top_inbox_maildir)" ubmails_inbox_maildir: "@import(personal.conf:ubmails_inbox_maildir)" +top_list_id: top.fsmi.uni-karlsruhe.de last_date_file: data/last_date diff --git a/compile_presentation.sh b/helpers/compile_presentation.sh similarity index 58% rename from compile_presentation.sh rename to helpers/compile_presentation.sh index 8b461a4..85e3abc 100755 --- a/compile_presentation.sh +++ b/helpers/compile_presentation.sh @@ -1,10 +1,10 @@ #!/bin/sh set -e -dest_file="$(realpath "$(dirname "$0")")/data/presentation_$(date +%Y-%m-%d).tex" +dest_file="presentation_$(date +%Y-%m-%d).tex" echo Compiling mkdir -p data/presentation cd data/presentation/ -latexmk -pdf "$dest_file" +latexmk -pdf "../$dest_file" ln -srnf "presentation_$(date +%Y-%m-%d).pdf" ../../presentation.pdf diff --git a/generate.py b/helpers/generate.py similarity index 88% rename from generate.py rename to helpers/generate.py index 179dc2f..674f685 100755 --- a/generate.py +++ b/helpers/generate.py @@ -14,6 +14,7 @@ import json import subprocess import re from pprint import pprint +from pathlib import Path CONFIG_FILE = "generator.conf" @@ -155,10 +156,10 @@ def conf2top(top): print("Warning: Error opening", top["file"], file=sys.stderr) if "command" in top: - body = subprocess.run(top["command"], shell=True, text=True, capture_output=True, check=True).stdout + body = subprocess.run(top["command"], shell=True, text=True, capture_output=True, check=args.allowfailcommand).stdout if "proto_command" in top: - protostub = subprocess.run(top["proto_command"], shell=True, text=True, capture_output=True, check=True).stdout + protostub = subprocess.run(top["proto_command"], shell=True, text=True, capture_output=True, check=args.allowfailcommand).stdout return Top(top["title"], sender, body, protostub) @@ -218,6 +219,10 @@ if __name__ == "__main__": parser.add_argument("--write-mbox", action="store_true") parser.add_argument("--send-mm", action="store_true") parser.add_argument("--send-mail", action="store_true") + parser.add_argument("--save", action="store_true") + parser.add_argument("--allowfailcommand", action="store_false") + parser.add_argument("--time") + parser.add_argument("--date") args = parser.parse_args() config = get_config(args.config) @@ -249,10 +254,13 @@ if __name__ == "__main__": mbox = mailbox.mbox(config["top_mbox_file"]) current_date = next_weekday(datetime.date.today(), config["default_weekday"]) - next_date = current_date + datetime.timedelta(days=7) + if args.date: + current_date = datetime.date.fromisoformat(args.date) + #next_date = current_date + datetime.timedelta(days=7) + next_date = next_weekday(current_date, config["default_weekday"]) last_date = last_weekday(current_date, config["default_weekday"]) - time = datetime.time.fromisoformat(config["default_time"]) + time = datetime.time.fromisoformat(args.time or config["default_time"]) pre_tops = [] post_tops = [] @@ -289,13 +297,26 @@ if __name__ == "__main__": for top in to: pprint(top.__dict__) pprint(context) + elif args.save: + if args.invite: + filename = Path(config["invite_save_path"]) / Path("invite_"+datetime.date.today().isoformat()+".txt") + elif args.presentation: + filename = Path(config["presentation_save_path"]) / Path("presentation_"+datetime.date.today().isoformat()+".tex") + elif args.protocol: + filename = Path(config["protocol_save_path"]) / Path(datetime.date.today().isoformat()) + else: + raise Exception("Should never happen") + + with open(filename, "w") as file: + file.write(template.render(context)) + elif args.send_mail: msg = email.message.EmailMessage() msg.set_content(template.render(context)) msg["Subject"] = j2env.from_string(config["invite_subject"]).render(context) msg["From"] = email.utils.formataddr((config["redeleitung"]["name"], config["redeleitung"]["email"])) msg["To"] = config["invite_mail"] - subprocess.run([*config["sendmail"], config["invite_mail"]], input=str(msg), text=True) + subprocess.run([*config["sendmail"], config["invite_mail"]], input=str(msg), text=True, check=True) elif args.write_mbox: msg = email.message.EmailMessage() diff --git a/get_uvproto.sh b/helpers/get_uvproto.sh similarity index 100% rename from get_uvproto.sh rename to helpers/get_uvproto.sh diff --git a/helpers/list_termine.sh b/helpers/list_termine.sh new file mode 100755 index 0000000..6e5e4f0 --- /dev/null +++ b/helpers/list_termine.sh @@ -0,0 +1,4 @@ +#!/bin/sh -e + +khal list --day-format "" --format "* {start} {title}" -a calendars_fsmi today 30d | grep -v Fachschaftsrat | grep -v Feriensprechstunde || true + diff --git a/list_termine_proto.sh b/helpers/list_termine_proto.sh similarity index 74% rename from list_termine_proto.sh rename to helpers/list_termine_proto.sh index 00f8bf9..a434829 100755 --- a/list_termine_proto.sh +++ b/helpers/list_termine_proto.sh @@ -1,4 +1,4 @@ #!/bin/sh -e echo "{{'{|'}}" -khal list --day-format "" --format "{{{{'{{{{'}}}}Termin|was={title}|wann={start}{{{{'}}}}'}}}}" -a calendars_fsmi today 30d | grep -v Fachschaftsrat +khal list --day-format "" --format "{{{{'{{{{'}}}}Termin|was={title}|wann={start}{{{{'}}}}'}}}}" -a calendars_fsmi today 30d | grep -v Fachschaftsrat | grep -v Feriensprechstunde || true echo "{{'|}'}}" diff --git a/read_db.sh b/helpers/read_db.sh similarity index 100% rename from read_db.sh rename to helpers/read_db.sh diff --git a/read_ubmails.py b/helpers/read_ubmails.py similarity index 93% rename from read_ubmails.py rename to helpers/read_ubmails.py index 86d794e..d93d942 100755 --- a/read_ubmails.py +++ b/helpers/read_ubmails.py @@ -10,7 +10,7 @@ import pypandoc import argparse import quopri from pprint import pprint -from dateutil import parser +from dateutil import parser as dateutilparser import generate @@ -37,7 +37,7 @@ if __name__ == "__main__": for message in mbox: if message["Subject"]: if decode_header(message["Subject"]).strip() == "Unbeantwortete Mails": - date = parser.parse(message["Date"]) + date = dateutilparser.parse(message["Date"]) if latest is None: latest = message latest_date = date diff --git a/helpers/sequencer.py b/helpers/sequencer.py new file mode 100755 index 0000000..eff0784 --- /dev/null +++ b/helpers/sequencer.py @@ -0,0 +1,101 @@ +#!/usr/bin/python3 +import sys +import argparse +import subprocess +import os +import mailbox +import datetime + +import pytz +from dateutil import parser as dateutilparser + +import generate + +class Actions: + @staticmethod + def clean_data(): + open(config["top_mbox_file"], 'w').close() + + + for top in config["pre_tops"]: + if "file" in top and os.path.isfile(top["file"]): + os.remove(top["file"]) + + for top in config["post_tops"]: + if "file" in top and os.path.isfile(top["file"]): + os.remove(top["file"]) + + @staticmethod + def read_db(): + subprocess.run(["helpers/read_db.sh"], check=True) + + @staticmethod + def read_topmails(): + in_mbox = mailbox.Maildir(config["top_inbox_maildir"]) + out_mbox = mailbox.mbox(config["top_mbox_file"]) + + out_mbox.clear() + + buffer = [] + + timezone = pytz.timezone("Europe/Berlin") + last_fsr_date = datetime.date.fromisoformat(open(config["last_date_file"]).read().strip()) + last_fsr_datetime = datetime.datetime.combine(last_fsr_date, datetime.time(17, 30), timezone) + + for message in in_mbox: + if message["List-Id"]: + if config["top_list_id"] in generate.decode_header(message["List-Id"]).strip(): + date = dateutilparser.parse(message["Date"]) + if date > last_fsr_datetime: + buffer.append(message) + + for message in sorted(buffer, key=lambda x: dateutilparser.parse(x["Date"])): + out_mbox.add(message) + + out_mbox.close() + + @staticmethod + def generate(*args): + subprocess.run(["helpers/generate.py", "--config", cliargs.config, *args], check=True) + + @staticmethod + def compile_presentation(): + subprocess.run(["helpers/compile_presentation.sh"], check=True) + +def dispatch(action, args=[]): + if action in dir(Actions): + getattr(Actions, action)(*args) + elif action in config["sequencer"]: + for item in config["sequencer"][action]: + ilist = item.split() + if not ilist[0] in cliargs.skip: + dispatch(ilist[0], ilist[1:]) + else: + raise ValueError(f"Action {action} not found") + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--config", "-c", default=generate.CONFIG_FILE) + parser.add_argument("--skip", action='append') + parser.add_argument("action", nargs="?") + cliargs = parser.parse_args() + + if cliargs.skip is None: + cliargs.skip = [] + + config = generate.get_config(cliargs.config) + + if os.path.dirname(cliargs.config) != "": + os.chdir(os.path.dirname(cliargs.config)) + + commandname = sys.argv[0] + commandname = commandname.split("/")[-1] + + if commandname != "sequencer.py": + dispatch(commandname) + elif cliargs.action: + dispatch(cliargs.action) + else: + print("No action specified") + sys.exit(1) + diff --git a/invite b/invite new file mode 120000 index 0000000..1c0dbba --- /dev/null +++ b/invite @@ -0,0 +1 @@ +helpers/sequencer.py \ No newline at end of file diff --git a/list_termine.sh b/list_termine.sh deleted file mode 100755 index 6b89fde..0000000 --- a/list_termine.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -e - -khal list --day-format "" --format "* {start} {title}" -a calendars_fsmi today 30d | grep -v Fachschaftsrat diff --git a/prepare_presentation.sh b/prepare_presentation.sh deleted file mode 100755 index 9c30191..0000000 --- a/prepare_presentation.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -set -e -dest_file="$(realpath "$(dirname "$0")")/data/presentation_$(date +%Y-%m-%d).tex" - -echo Reading old protocols -./read_db.sh -echo Generating -./generate.py --presentation >"$dest_file" - -./compile_presentation.sh diff --git a/prepare_protocol.sh b/prepare_protocol.sh deleted file mode 100755 index c7d92b1..0000000 --- a/prepare_protocol.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -echo Reading old protocols -./read_db.sh -echo Getting E-Mail -./read_topmails.py -echo Generating - -./generate.py --proto > protokolle/$(date +%Y-%m-%d) diff --git a/presentation b/presentation new file mode 120000 index 0000000..1c0dbba --- /dev/null +++ b/presentation @@ -0,0 +1 @@ +helpers/sequencer.py \ No newline at end of file diff --git a/protocol b/protocol new file mode 120000 index 0000000..1c0dbba --- /dev/null +++ b/protocol @@ -0,0 +1 @@ +helpers/sequencer.py \ No newline at end of file diff --git a/read_topmails.py b/read_topmails.py deleted file mode 100755 index b97337b..0000000 --- a/read_topmails.py +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/python3 -import mailbox -import jinja2 -import email.header -import email.utils -import yaml -import datetime -import sys, os -import pypandoc -import argparse -import quopri -import pytz -from pprint import pprint -from dateutil import parser - -import generate - -CONFIG_FILE = "generator.conf" - -def decode_header(header): - decoded_header = email.header.decode_header(header)[0] - encoding = decoded_header[1] or "ascii" - if encoding == "unknown-8bit": - encoding = "ascii" - return decoded_header[0].decode(encoding, errors="replace") if isinstance(decoded_header[0], bytes) else decoded_header[0] - -if __name__ == "__main__": - aparser = argparse.ArgumentParser() - aparser.add_argument("--config", "-c", default=CONFIG_FILE) - args = aparser.parse_args() - - config = generate.get_config(args.config) - in_mbox = mailbox.Maildir(config["top_inbox_maildir"]) - out_mbox = mailbox.mbox(config["top_mbox_file"]) - - out_mbox.clear() - - buffer = [] - - timezone = pytz.timezone("Europe/Berlin") - last_fsr_date = datetime.date.fromisoformat(open(config["last_date_file"]).read().strip()) - last_fsr_datetime = datetime.datetime.combine(last_fsr_date, datetime.time(17, 30), timezone) - - for message in in_mbox: - if message["List-Id"]: - if "top.fsmi.uni-karlsruhe.de" in decode_header(message["List-Id"]).strip(): - date = parser.parse(message["Date"]) - if date > last_fsr_datetime: - buffer.append(message) - - for message in sorted(buffer, key=lambda x: parser.parse(x["Date"])): - out_mbox.add(message) - - out_mbox.close() - diff --git a/sequencer.py b/sequencer.py new file mode 120000 index 0000000..1c0dbba --- /dev/null +++ b/sequencer.py @@ -0,0 +1 @@ +helpers/sequencer.py \ No newline at end of file