From b3dfd2f59e40d65d8038a5ec214698905f1b89f5 Mon Sep 17 00:00:00 2001 From: Yannik Enss Date: Tue, 15 Mar 2022 15:20:51 +0100 Subject: [PATCH] start work on allowing "packages" --- fslogo.png => fsmi_fsr/fslogo.png | Bin generator.conf => fsmi_fsr/generator.conf | 0 .../helpers}/compile_presentation.sh | 0 {helpers => fsmi_fsr/helpers}/get_uvproto.sh | 0 {helpers => fsmi_fsr/helpers}/list_termine.sh | 0 .../helpers}/list_termine_proto.sh | 0 {helpers => fsmi_fsr/helpers}/read_db.sh | 0 {helpers => fsmi_fsr/helpers}/read_ubmails.py | 0 .../templates}/fsr_einladung.j2 | 0 .../templates}/fsr_einladung_mm.j2 | 0 .../templates}/fsr_presentation.tex.j2 | 0 .../templates}/fsr_protokoll.j2 | 0 helpers/generate.py | 144 +++--------------- helpers/sequencer.py | 18 ++- helpers/util.py | 131 ++++++++++++++++ 15 files changed, 166 insertions(+), 127 deletions(-) rename fslogo.png => fsmi_fsr/fslogo.png (100%) rename generator.conf => fsmi_fsr/generator.conf (100%) rename {helpers => fsmi_fsr/helpers}/compile_presentation.sh (100%) rename {helpers => fsmi_fsr/helpers}/get_uvproto.sh (100%) rename {helpers => fsmi_fsr/helpers}/list_termine.sh (100%) rename {helpers => fsmi_fsr/helpers}/list_termine_proto.sh (100%) rename {helpers => fsmi_fsr/helpers}/read_db.sh (100%) rename {helpers => fsmi_fsr/helpers}/read_ubmails.py (100%) rename {templates => fsmi_fsr/templates}/fsr_einladung.j2 (100%) rename {templates => fsmi_fsr/templates}/fsr_einladung_mm.j2 (100%) rename {templates => fsmi_fsr/templates}/fsr_presentation.tex.j2 (100%) rename {templates => fsmi_fsr/templates}/fsr_protokoll.j2 (100%) create mode 100755 helpers/util.py diff --git a/fslogo.png b/fsmi_fsr/fslogo.png similarity index 100% rename from fslogo.png rename to fsmi_fsr/fslogo.png diff --git a/generator.conf b/fsmi_fsr/generator.conf similarity index 100% rename from generator.conf rename to fsmi_fsr/generator.conf diff --git a/helpers/compile_presentation.sh b/fsmi_fsr/helpers/compile_presentation.sh similarity index 100% rename from helpers/compile_presentation.sh rename to fsmi_fsr/helpers/compile_presentation.sh diff --git a/helpers/get_uvproto.sh b/fsmi_fsr/helpers/get_uvproto.sh similarity index 100% rename from helpers/get_uvproto.sh rename to fsmi_fsr/helpers/get_uvproto.sh diff --git a/helpers/list_termine.sh b/fsmi_fsr/helpers/list_termine.sh similarity index 100% rename from helpers/list_termine.sh rename to fsmi_fsr/helpers/list_termine.sh diff --git a/helpers/list_termine_proto.sh b/fsmi_fsr/helpers/list_termine_proto.sh similarity index 100% rename from helpers/list_termine_proto.sh rename to fsmi_fsr/helpers/list_termine_proto.sh diff --git a/helpers/read_db.sh b/fsmi_fsr/helpers/read_db.sh similarity index 100% rename from helpers/read_db.sh rename to fsmi_fsr/helpers/read_db.sh diff --git a/helpers/read_ubmails.py b/fsmi_fsr/helpers/read_ubmails.py similarity index 100% rename from helpers/read_ubmails.py rename to fsmi_fsr/helpers/read_ubmails.py diff --git a/templates/fsr_einladung.j2 b/fsmi_fsr/templates/fsr_einladung.j2 similarity index 100% rename from templates/fsr_einladung.j2 rename to fsmi_fsr/templates/fsr_einladung.j2 diff --git a/templates/fsr_einladung_mm.j2 b/fsmi_fsr/templates/fsr_einladung_mm.j2 similarity index 100% rename from templates/fsr_einladung_mm.j2 rename to fsmi_fsr/templates/fsr_einladung_mm.j2 diff --git a/templates/fsr_presentation.tex.j2 b/fsmi_fsr/templates/fsr_presentation.tex.j2 similarity index 100% rename from templates/fsr_presentation.tex.j2 rename to fsmi_fsr/templates/fsr_presentation.tex.j2 diff --git a/templates/fsr_protokoll.j2 b/fsmi_fsr/templates/fsr_protokoll.j2 similarity index 100% rename from templates/fsr_protokoll.j2 rename to fsmi_fsr/templates/fsr_protokoll.j2 diff --git a/helpers/generate.py b/helpers/generate.py index 07d97c5..1f67c12 100755 --- a/helpers/generate.py +++ b/helpers/generate.py @@ -1,77 +1,32 @@ #!/usr/bin/python3 import mailbox -import jinja2 import email.header import email.utils import yaml import datetime import sys, os -import pypandoc import argparse import quopri -import requests import json import subprocess import re from pprint import pprint from pathlib import Path -CONFIG_FILE = "generator.conf" +import requests +import pypandoc +import jinja2 -WEEKDAYS = { 0: "Montag", - 1: "Dienstag", - 2: "Mittwoch", - 3: "Donnerstag", - 4: "Freitag", - 5: "Samstag", - 6: "Sonntag" } - -IMPORT_RE = re.compile(r"@import\((.*)\)") -FILE_RE = re.compile(r"@file\((.*)\)") - -def decode_header(header): - decoded_headers = email.header.decode_header(header) - header_strs = [] - for dheader in decoded_headers: - encoding = dheader[1] or "ascii" - if encoding == "unknown-8bit": - encoding = "ascii" - result = dheader[0].decode(encoding, errors="replace") if isinstance(dheader[0], bytes) else dheader[0] - header_strs.append(result) - header_text = "".join(header_strs) - header_text = re.sub(r"\n(\s)", r" ", header_text) - #header_text = re.sub(r"\n(\s)", r"", header_text) - return header_text - -def get_body_text(msg): - # from https://stackoverflow.com/a/1463144 - for part in msg.walk(): - # each part is a either non-multipart, or another multipart message - # that contains further parts... Message is organized like a tree - if part.get_content_type() == 'text/plain': - payload = part.get_payload() - if part["Content-Transfer-Encoding"] == "quoted-printable": - payload = quopri.decodestring(payload.encode("ascii")).decode(part.get_content_charset("utf-8")) - return payload - -# from https://stackoverflow.com/a/49986645 -def deEmojify(text): - regrex_pattern = re.compile(pattern = "[" - u"\U0001F600-\U0001F64F" # emoticons - u"\U0001F300-\U0001F5FF" # symbols & pictographs - u"\U0001F680-\U0001F6FF" # transport & map symbols - u"\U0001F1E0-\U0001F1FF" # flags (iOS) - "]+", flags = re.UNICODE) - return regrex_pattern.sub(r'',text) +import util class Top: def __init__(self, title=None, sender=None, body=None, protostub=None, message=None): if message: subject = message["Subject"] needs_stripping = subject[:6] == "[top] " - self.title = deEmojify(decode_header(subject[6:] if needs_stripping else subject)) + self.title = util.deEmojify(util.decode_header(subject[6:] if needs_stripping else subject)) real_name, address = email.utils.parseaddr(message["From"]) - real_name = decode_header(real_name) + real_name = util.decode_header(real_name) self.sender = real_name or address payload = get_body_text(message) self.body = str(payload.rpartition("\n--")[0] if "\n--" in payload else payload) @@ -89,19 +44,6 @@ class Top: def __repr__(self): return "" -# from https://stackoverflow.com/questions/6558535/find-the-date-for-the-first-monday-after-a-given-a-date -def next_weekday(d, weekday): - days_ahead = weekday - d.weekday() - if days_ahead < 0: # Target day already happened this week - days_ahead += 7 - return d + datetime.timedelta(days_ahead) - -def last_weekday(d, weekday): - days_ahead = weekday - d.weekday() - if days_ahead >= 0: # Target day already happened this week - days_ahead -= 7 - return d + datetime.timedelta(days_ahead) - def wiki2latex(intext): intext = intext.replace(":\n", ":\n\n") return pypandoc.convert_text(intext, 'latex', format='md') @@ -116,7 +58,7 @@ def time(intime): return intime.strftime("%H:%M") def weekday(indate): - return WEEKDAYS[indate.weekday()] + return util.WEEKDAYS[indate.weekday()] def prototop(top): result = "" @@ -169,57 +111,10 @@ def conf2top(top): return Top(top["title"], sender, body, protostub) -import_cache = {} -def get_imported_conf_entry(f, key): - if f not in import_cache: - import_cache[f] = yaml.safe_load(open(f)) - return import_cache[f][key] - - -def do_imports(entry): - if isinstance(entry, dict): - d = entry.items() - result = {} - for key,value in d: - try: - result[key] = do_imports(value) - except KeyError: - pass - except OSError as e: - print(f"Warning: reading {key} yielded error {e}") - return result - - if isinstance(entry, list): - l = iter(entry) - result = [] - for item in l: - result.append(do_imports(item)) - return result - - if isinstance(entry, str): - match = IMPORT_RE.match(entry) - if match: - f, key = match.group(1).split(":") - return get_imported_conf_entry(f, key) - match = FILE_RE.match(entry) - if match: - with open(match.group(1)) as f: - return f.read().strip() - - return entry - -def get_config(f): - raw_config = yaml.safe_load(open(f)) - config = do_imports(raw_config) - if "override_file" in config: - override_config = yaml.safe_load(open(config["override_file"])) - config.update(override_config) - return config - -if __name__ == "__main__": +def main(rawargs=None): parser = argparse.ArgumentParser() - parser.add_argument("--config", "-c", default=CONFIG_FILE) + parser.add_argument("--config", "-c", default=util.CONFIG_FILE) mode = parser.add_mutually_exclusive_group(required=True) mode.add_argument("--invite", action="store_true") mode.add_argument("--mm-invite", action="store_true") @@ -233,9 +128,9 @@ if __name__ == "__main__": parser.add_argument("--save", action="store_true") parser.add_argument("--time") parser.add_argument("--date") - args = parser.parse_args() + args = parser.parse_args(rawargs) - config = get_config(args.config) + config = util.get_config(util.get_normalized_config_path(args.config)) if args.print_config: pprint(config) @@ -252,6 +147,7 @@ if __name__ == "__main__": else: raise Exception("Should never happen") + global j2env j2env = jinja2.Environment() j2env.filters["wiki2latex"] = wiki2latex j2env.filters["j2replace"] = j2replace @@ -263,14 +159,14 @@ if __name__ == "__main__": mbox = mailbox.mbox(config["top_mbox_file"]) - current_date = next_weekday(datetime.date.today(), config["default_weekday"]) + current_date = util.next_weekday(datetime.date.today(), config["default_weekday"]) if args.date: current_date = datetime.date.fromisoformat(args.date) #next_date = current_date + datetime.timedelta(days=7) - next_date = next_weekday(current_date, config["default_weekday"]) - last_date = last_weekday(current_date, config["default_weekday"]) + next_date = util.next_weekday(current_date, config["default_weekday"]) + last_date = util.last_weekday(current_date, config["default_weekday"]) - time = datetime.time.fromisoformat(args.time or config["default_time"]) + actual_time = datetime.time.fromisoformat(args.time or config["default_time"]) pre_tops = [] post_tops = [] @@ -291,17 +187,18 @@ if __name__ == "__main__": to = pre_tops + email_tops + post_tops + global context context = {"to": to, "redeleitung": config["redeleitung"], "protokoll": config["protokoll"], "date": current_date, - "time": time, + "time": actual_time, "place": config["place"], "next_date": next_date, "last_date": last_date, "meeting_link": config["meeting_link"], "email_tops": email_tops, - "WEEKDAYS": WEEKDAYS} + "WEEKDAYS": util.WEEKDAYS} if args.debug: for top in to: @@ -344,3 +241,6 @@ if __name__ == "__main__": response = requests.post(config["mm_url"], headers=headers, data=values) else: print(template.render(context)) + +if __name__ == "__main__": + main() diff --git a/helpers/sequencer.py b/helpers/sequencer.py index f600ace..ee07cbe 100755 --- a/helpers/sequencer.py +++ b/helpers/sequencer.py @@ -10,6 +10,7 @@ import pytz from dateutil import parser as dateutilparser import generate +import util class Actions: @staticmethod @@ -56,7 +57,7 @@ class Actions: @staticmethod def generate(*args): - subprocess.run(["helpers/generate.py", "--config", cliargs.config, *args], check=True) + generate.main(["--config", config_path, *args]) @staticmethod def compile_presentation(): @@ -83,7 +84,7 @@ if __name__ == "__main__": using_alias = commandname != "sequencer.py" parser = argparse.ArgumentParser() - parser.add_argument("--config", "-c", default=generate.CONFIG_FILE) + parser.add_argument("--config", "-c", default=util.CONFIG_FILE) parser.add_argument("--skip", action='append') if not using_alias: parser.add_argument("action", nargs="?") @@ -92,10 +93,17 @@ if __name__ == "__main__": if cliargs.skip is None: cliargs.skip = [] - config = generate.get_config(cliargs.config) + config_path = util.get_normalized_config_path(cliargs.config) + config_path = os.path.abspath(config_path) - if os.path.dirname(cliargs.config) != "": - os.chdir(os.path.dirname(cliargs.config)) + config = util.get_config(config_path) + + original_working_dir = os.getcwd() + normalized_working_dir = original_working_dir + + if os.path.dirname(config_path) != "": + normalized_working_dir = os.path.dirname(config_path) + os.chdir(normalized_working_dir) if using_alias: dispatch(commandname) diff --git a/helpers/util.py b/helpers/util.py new file mode 100755 index 0000000..58292ff --- /dev/null +++ b/helpers/util.py @@ -0,0 +1,131 @@ +#!/usr/bin/python3 +import mailbox +import jinja2 +import email.header +import email.utils +import yaml +import datetime +import sys, os +import argparse +import quopri +import requests +import json +import subprocess +import re +from pprint import pprint +from pathlib import Path + + +IMPORT_RE = re.compile(r"@import\((.*)\)") +FILE_RE = re.compile(r"@file\((.*)\)") + +CONFIG_FILE = "fsmi_fsr/generator.conf" + +WEEKDAYS = { 0: "Montag", + 1: "Dienstag", + 2: "Mittwoch", + 3: "Donnerstag", + 4: "Freitag", + 5: "Samstag", + 6: "Sonntag" } + +def decode_header(header): + decoded_headers = email.header.decode_header(header) + header_strs = [] + for dheader in decoded_headers: + encoding = dheader[1] or "ascii" + if encoding == "unknown-8bit": + encoding = "ascii" + result = dheader[0].decode(encoding, errors="replace") if isinstance(dheader[0], bytes) else dheader[0] + header_strs.append(result) + header_text = "".join(header_strs) + header_text = re.sub(r"\n(\s)", r" ", header_text) + #header_text = re.sub(r"\n(\s)", r"", header_text) + return header_text + +def get_body_text(msg): + # from https://stackoverflow.com/a/1463144 + for part in msg.walk(): + # each part is a either non-multipart, or another multipart message + # that contains further parts... Message is organized like a tree + if part.get_content_type() == 'text/plain': + payload = part.get_payload() + if part["Content-Transfer-Encoding"] == "quoted-printable": + payload = quopri.decodestring(payload.encode("ascii")).decode(part.get_content_charset("utf-8")) + return payload + +# from https://stackoverflow.com/a/49986645 +def deEmojify(text): + regrex_pattern = re.compile(pattern = "[" + u"\U0001F600-\U0001F64F" # emoticons + u"\U0001F300-\U0001F5FF" # symbols & pictographs + u"\U0001F680-\U0001F6FF" # transport & map symbols + u"\U0001F1E0-\U0001F1FF" # flags (iOS) + "]+", flags = re.UNICODE) + return regrex_pattern.sub(r'',text) + +# from https://stackoverflow.com/questions/6558535/find-the-date-for-the-first-monday-after-a-given-a-date +def next_weekday(d, weekday): + days_ahead = weekday - d.weekday() + if days_ahead < 0: # Target day already happened this week + days_ahead += 7 + return d + datetime.timedelta(days_ahead) + +def last_weekday(d, weekday): + days_ahead = weekday - d.weekday() + if days_ahead >= 0: # Target day already happened this week + days_ahead -= 7 + return d + datetime.timedelta(days_ahead) + +import_cache = {} + +def get_imported_conf_entry(f, key): + if f not in import_cache: + import_cache[f] = yaml.safe_load(open(f)) + return import_cache[f][key] + +def do_imports(entry): + if isinstance(entry, dict): + d = entry.items() + result = {} + for key,value in d: + try: + result[key] = do_imports(value) + except KeyError: + pass + except OSError as e: + print(f"Warning: reading {key} yielded error {e}") + return result + + if isinstance(entry, list): + l = iter(entry) + result = [] + for item in l: + result.append(do_imports(item)) + return result + + if isinstance(entry, str): + match = IMPORT_RE.match(entry) + if match: + f, key = match.group(1).split(":") + return get_imported_conf_entry(f, key) + match = FILE_RE.match(entry) + if match: + with open(match.group(1)) as f: + return f.read().strip() + + return entry + +def get_normalized_config_path(f): + if os.path.isdir(f): + f = Path(f) / Path("generator.conf") + return f + +def get_config(f): + raw_config = yaml.safe_load(open(f)) + config = do_imports(raw_config) + if "override_file" in config: + override_config = yaml.safe_load(open(config["override_file"])) + config.update(override_config) + return config +