yannik/fsr_generator
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

start work on allowing "packages"

Browse Source
This commit is contained in:
Yannik Enss
2022-03-15 15:20:51 +01:00
parent e2380fb311
commit b3dfd2f59e
15 changed files with 166 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
helpers/compile_presentation.sh
View File

@@ -1,10 +0,0 @@
#!/bin/sh
set -e
dest_file="presentation_$(date +%Y-%m-%d).tex"
echo Compiling
mkdir -p data/presentation
cd data/presentation/
latexmk -pdf "../$dest_file"
ln -srnf "presentation_$(date +%Y-%m-%d).pdf" ../../presentation.pdf

144
helpers/generate.py
View File

@@ -1,77 +1,32 @@
#!/usr/bin/python3
import mailbox
import jinja2
import email.header
import email.utils
import yaml
import datetime
import sys, os
import pypandoc
import argparse
import quopri
import requests
import json
import subprocess
import re
from pprint import pprint
from pathlib import Path
CONFIG_FILE = "generator.conf"
import requests
import pypandoc
import jinja2
WEEKDAYS = { 0: "Montag",
1: "Dienstag",
2: "Mittwoch",
3: "Donnerstag",
4: "Freitag",
5: "Samstag",
6: "Sonntag" }
IMPORT_RE = re.compile(r"@import\((.*)\)")
FILE_RE = re.compile(r"@file\((.*)\)")
def decode_header(header):
decoded_headers = email.header.decode_header(header)
header_strs = []
for dheader in decoded_headers:
encoding = dheader[1] or "ascii"
if encoding == "unknown-8bit":
encoding = "ascii"
result = dheader[0].decode(encoding, errors="replace") if isinstance(dheader[0], bytes) else dheader[0]
header_strs.append(result)
header_text = "".join(header_strs)
header_text = re.sub(r"\n(\s)", r" ", header_text)
#header_text = re.sub(r"\n(\s)", r"", header_text)
return header_text
def get_body_text(msg):
# from https://stackoverflow.com/a/1463144
for part in msg.walk():
# each part is a either non-multipart, or another multipart message
# that contains further parts... Message is organized like a tree
if part.get_content_type() == 'text/plain':
payload = part.get_payload()
if part["Content-Transfer-Encoding"] == "quoted-printable":
payload = quopri.decodestring(payload.encode("ascii")).decode(part.get_content_charset("utf-8"))
return payload
# from https://stackoverflow.com/a/49986645
def deEmojify(text):
regrex_pattern = re.compile(pattern = "["
u"\U0001F600-\U0001F64F" # emoticons
u"\U0001F300-\U0001F5FF" # symbols & pictographs
u"\U0001F680-\U0001F6FF" # transport & map symbols
u"\U0001F1E0-\U0001F1FF" # flags (iOS)
"]+", flags = re.UNICODE)
return regrex_pattern.sub(r'',text)
import util
class Top:
def __init__(self, title=None, sender=None, body=None, protostub=None, message=None):
if message:
subject = message["Subject"]
needs_stripping = subject[:6] == "[top] "
self.title = deEmojify(decode_header(subject[6:] if needs_stripping else subject))
self.title = util.deEmojify(util.decode_header(subject[6:] if needs_stripping else subject))
real_name, address = email.utils.parseaddr(message["From"])
real_name = decode_header(real_name)
real_name = util.decode_header(real_name)
self.sender = real_name or address
payload = get_body_text(message)
self.body = str(payload.rpartition("\n--")[0] if "\n--" in payload else payload)
@@ -89,19 +44,6 @@ class Top:
def __repr__(self):
return "<TOP "+self.title+">"
# from https://stackoverflow.com/questions/6558535/find-the-date-for-the-first-monday-after-a-given-a-date
def next_weekday(d, weekday):
days_ahead = weekday - d.weekday()
if days_ahead < 0: # Target day already happened this week
days_ahead += 7
return d + datetime.timedelta(days_ahead)
def last_weekday(d, weekday):
days_ahead = weekday - d.weekday()
if days_ahead >= 0: # Target day already happened this week
days_ahead -= 7
return d + datetime.timedelta(days_ahead)
def wiki2latex(intext):
intext = intext.replace(":\n", ":\n\n")
return pypandoc.convert_text(intext, 'latex', format='md')
@@ -116,7 +58,7 @@ def time(intime):
return intime.strftime("%H:%M")
def weekday(indate):
return WEEKDAYS[indate.weekday()]
return util.WEEKDAYS[indate.weekday()]
def prototop(top):
result = ""
@@ -169,57 +111,10 @@ def conf2top(top):
return Top(top["title"], sender, body, protostub)
import_cache = {}
def get_imported_conf_entry(f, key):
if f not in import_cache:
import_cache[f] = yaml.safe_load(open(f))
return import_cache[f][key]
def do_imports(entry):
if isinstance(entry, dict):
d = entry.items()
result = {}
for key,value in d:
try:
result[key] = do_imports(value)
except KeyError:
pass
except OSError as e:
print(f"Warning: reading {key} yielded error {e}")
return result
if isinstance(entry, list):
l = iter(entry)
result = []
for item in l:
result.append(do_imports(item))
return result
if isinstance(entry, str):
match = IMPORT_RE.match(entry)
if match:
f, key = match.group(1).split(":")
return get_imported_conf_entry(f, key)
match = FILE_RE.match(entry)
if match:
with open(match.group(1)) as f:
return f.read().strip()
return entry
def get_config(f):
raw_config = yaml.safe_load(open(f))
config = do_imports(raw_config)
if "override_file" in config:
override_config = yaml.safe_load(open(config["override_file"]))
config.update(override_config)
return config
if __name__ == "__main__":
def main(rawargs=None):
parser = argparse.ArgumentParser()
parser.add_argument("--config", "-c", default=CONFIG_FILE)
parser.add_argument("--config", "-c", default=util.CONFIG_FILE)
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument("--invite", action="store_true")
mode.add_argument("--mm-invite", action="store_true")
@@ -233,9 +128,9 @@ if __name__ == "__main__":
parser.add_argument("--save", action="store_true")
parser.add_argument("--time")
parser.add_argument("--date")
args = parser.parse_args()
args = parser.parse_args(rawargs)
config = get_config(args.config)
config = util.get_config(util.get_normalized_config_path(args.config))
if args.print_config:
pprint(config)
@@ -252,6 +147,7 @@ if __name__ == "__main__":
else:
raise Exception("Should never happen")
global j2env
j2env = jinja2.Environment()
j2env.filters["wiki2latex"] = wiki2latex
j2env.filters["j2replace"] = j2replace
@@ -263,14 +159,14 @@ if __name__ == "__main__":
mbox = mailbox.mbox(config["top_mbox_file"])
current_date = next_weekday(datetime.date.today(), config["default_weekday"])
current_date = util.next_weekday(datetime.date.today(), config["default_weekday"])
if args.date:
current_date = datetime.date.fromisoformat(args.date)
#next_date = current_date + datetime.timedelta(days=7)
next_date = next_weekday(current_date, config["default_weekday"])
last_date = last_weekday(current_date, config["default_weekday"])
next_date = util.next_weekday(current_date, config["default_weekday"])
last_date = util.last_weekday(current_date, config["default_weekday"])
time = datetime.time.fromisoformat(args.time or config["default_time"])
actual_time = datetime.time.fromisoformat(args.time or config["default_time"])
pre_tops = []
post_tops = []
@@ -291,17 +187,18 @@ if __name__ == "__main__":
to = pre_tops + email_tops + post_tops
global context
context = {"to": to,
"redeleitung": config["redeleitung"],
"protokoll": config["protokoll"],
"date": current_date,
"time": time,
"time": actual_time,
"place": config["place"],
"next_date": next_date,
"last_date": last_date,
"meeting_link": config["meeting_link"],
"email_tops": email_tops,
"WEEKDAYS": WEEKDAYS}
"WEEKDAYS": util.WEEKDAYS}
if args.debug:
for top in to:
@@ -344,3 +241,6 @@ if __name__ == "__main__":
response = requests.post(config["mm_url"], headers=headers, data=values)
else:
print(template.render(context))
if __name__ == "__main__":
main()

17
helpers/get_uvproto.sh
View File

@@ -1,17 +0,0 @@
#!/bin/sh
set -e
: "${FSR_GEN_SSH_TO:=fsmi-login.fsmi.uni-karlsruhe.de}"
: "${FSR_GEN_SSH:=$(test "$(hostname -d)" = "fsmi.uni-karlsruhe.de" || echo 1)}"
QUERY="select '* FSR-Protokoll vom '||datum|| case when protokoll like '%TODO%' then ' (hat noch TODOs)' else '' end from protokolle where ist_veroeffentlicht=false and name is null order by datum asc"
cmd="psql --no-align --tuples-only service=fsmi -c \"$QUERY\""
if [ -z "$FSR_GEN_SSH" ] || [ "$FSR_GEN_SSH" -eq 0 ]; then
raw_proto="$(sh -c "$cmd")"
else
raw_proto="$(ssh -- "$FSR_GEN_SSH_TO" "$cmd")"
fi
echo "$raw_proto"

6
helpers/list_termine.sh
View File

@@ -1,6 +0,0 @@
#!/bin/sh -e
khal --version > /dev/null
khal list --day-format "" --format "* {start} {title}" -a calendars_fsmi today 30d | grep -v Fachschaftsrat | grep -v Feriensprechstunde || true

5
helpers/list_termine_proto.sh
View File

@@ -1,5 +0,0 @@
#!/bin/sh -e
khal --version > /dev/null
echo "{{'{|'}}"
khal list --day-format "" --format "{{{{'{{{{'}}}}Termin|was={title}|wann={start}{{{{'}}}}'}}}}" -a calendars_fsmi today 30d | grep -v Fachschaftsrat | grep -v Feriensprechstunde || true
echo "{{'|}'}}"

38
helpers/read_db.sh
View File

@@ -1,38 +0,0 @@
#!/bin/sh
set -e
: "${FSR_GEN_SSH_TO:=fsmi-login.fsmi.uni-karlsruhe.de}"
: "${FSR_GEN_SSH:=$(test "$(hostname -d)" = "fsmi.uni-karlsruhe.de" || echo 1)}"
sql() {
# $1: select
# $2: order_by
select="$1"
order_by="$2"
printf "
SELECT %s FROM protokolle
WHERE ist_veroeffentlicht=false AND name IS NULL
ORDER BY %s
" "$select" "$order_by" | tr '\n' ' '
}
cmd="psql --no-align --tuples-only service=fsmi -c"
cmd_raw="$cmd '$(sql "datum" "datum ASC")'"
cmd_last="$cmd '$(sql "protokoll" "datum DESC LIMIT 1")'"
if [ -z "$FSR_GEN_SSH" ] || [ "$FSR_GEN_SSH" -eq 0 ]; then
raw_proto="$(sh -c "$cmd_raw")"
sh -c "$cmd_last" >data/last_proto
else
raw_proto="$(ssh -- "$FSR_GEN_SSH_TO" "$cmd_raw")"
ssh -- "$FSR_GEN_SSH_TO" "$cmd_last" >data/last_proto
fi
for proto in $raw_proto; do
echo "* FSR-Protokoll vom $proto"
done > data/uvproto.txt
echo "$proto" > data/last_date
grep -ioP '(?<=nächste Redeleitung: ).*(?=</li>)' data/last_proto > data/redeleitung
grep -ioP '(?<=nächstes Protokoll: ).*(?=</li>)' data/last_proto > data/protokoll
grep -ioP '(?<=Ort: ).*(?=</li>)' data/last_proto > data/ort

55
helpers/read_ubmails.py
View File

@@ -1,55 +0,0 @@
#!/usr/bin/python3
import mailbox
import jinja2
import email.header
import email.utils
import yaml
import datetime
import sys, os
import pypandoc
import argparse
import quopri
from pprint import pprint
from dateutil import parser as dateutilparser
import generate
CONFIG_FILE = "generator.conf"
def decode_header(header):
decoded_header = email.header.decode_header(header)[0]
encoding = decoded_header[1] or "ascii"
if encoding == "unknown-8bit":
encoding = "ascii"
return decoded_header[0].decode(encoding, errors="replace") if isinstance(decoded_header[0], bytes) else decoded_header[0]
if __name__ == "__main__":
aparser = argparse.ArgumentParser()
aparser.add_argument("--config", "-c", default=CONFIG_FILE)
args = aparser.parse_args()
config = generate.get_config(args.config)
mbox = mailbox.Maildir(config["ubmails_inbox_maildir"])
latest = None
latest_date = None
for message in mbox:
if message["Subject"]:
if decode_header(message["Subject"]).strip() == "Unbeantwortete Mails":
date = dateutilparser.parse(message["Date"])
if latest is None:
latest = message
latest_date = date
elif latest_date < date:
latest = message
latest_date = date
if not latest:
print("ERROR: No Mail found", file=sys.stderr)
sys.exit(1)
if latest_date.date() != datetime.date.today():
print("WARNING: Mail is not from today", file=sys.stderr)
payload = latest.get_payload(decode=True).decode("utf8").strip()
result = (payload.rpartition("\n--")[0].strip() if "\n--" in payload else payload)
print(result)

18
helpers/sequencer.py
View File

@@ -10,6 +10,7 @@ import pytz
from dateutil import parser as dateutilparser
import generate
import util
class Actions:
@staticmethod
@@ -56,7 +57,7 @@ class Actions:
@staticmethod
def generate(*args):
subprocess.run(["helpers/generate.py", "--config", cliargs.config, *args], check=True)
generate.main(["--config", config_path, *args])
@staticmethod
def compile_presentation():
@@ -83,7 +84,7 @@ if __name__ == "__main__":
using_alias = commandname != "sequencer.py"
parser = argparse.ArgumentParser()
parser.add_argument("--config", "-c", default=generate.CONFIG_FILE)
parser.add_argument("--config", "-c", default=util.CONFIG_FILE)
parser.add_argument("--skip", action='append')
if not using_alias:
parser.add_argument("action", nargs="?")
@@ -92,10 +93,17 @@ if __name__ == "__main__":
if cliargs.skip is None:
cliargs.skip = []
config = generate.get_config(cliargs.config)
config_path = util.get_normalized_config_path(cliargs.config)
config_path = os.path.abspath(config_path)
if os.path.dirname(cliargs.config) != "":
os.chdir(os.path.dirname(cliargs.config))
config = util.get_config(config_path)
original_working_dir = os.getcwd()
normalized_working_dir = original_working_dir
if os.path.dirname(config_path) != "":
normalized_working_dir = os.path.dirname(config_path)
os.chdir(normalized_working_dir)
if using_alias:
dispatch(commandname)

131
helpers/util.py Executable file
View File

@@ -0,0 +1,131 @@
#!/usr/bin/python3
import mailbox
import jinja2
import email.header
import email.utils
import yaml
import datetime
import sys, os
import argparse
import quopri
import requests
import json
import subprocess
import re
from pprint import pprint
from pathlib import Path
IMPORT_RE = re.compile(r"@import\((.*)\)")
FILE_RE = re.compile(r"@file\((.*)\)")
CONFIG_FILE = "fsmi_fsr/generator.conf"
WEEKDAYS = { 0: "Montag",
1: "Dienstag",
2: "Mittwoch",
3: "Donnerstag",
4: "Freitag",
5: "Samstag",
6: "Sonntag" }
def decode_header(header):
decoded_headers = email.header.decode_header(header)
header_strs = []
for dheader in decoded_headers:
encoding = dheader[1] or "ascii"
if encoding == "unknown-8bit":
encoding = "ascii"
result = dheader[0].decode(encoding, errors="replace") if isinstance(dheader[0], bytes) else dheader[0]
header_strs.append(result)
header_text = "".join(header_strs)
header_text = re.sub(r"\n(\s)", r" ", header_text)
#header_text = re.sub(r"\n(\s)", r"", header_text)
return header_text
def get_body_text(msg):
# from https://stackoverflow.com/a/1463144
for part in msg.walk():
# each part is a either non-multipart, or another multipart message
# that contains further parts... Message is organized like a tree
if part.get_content_type() == 'text/plain':
payload = part.get_payload()
if part["Content-Transfer-Encoding"] == "quoted-printable":
payload = quopri.decodestring(payload.encode("ascii")).decode(part.get_content_charset("utf-8"))
return payload
# from https://stackoverflow.com/a/49986645
def deEmojify(text):
regrex_pattern = re.compile(pattern = "["
u"\U0001F600-\U0001F64F" # emoticons
u"\U0001F300-\U0001F5FF" # symbols & pictographs
u"\U0001F680-\U0001F6FF" # transport & map symbols
u"\U0001F1E0-\U0001F1FF" # flags (iOS)
"]+", flags = re.UNICODE)
return regrex_pattern.sub(r'',text)
# from https://stackoverflow.com/questions/6558535/find-the-date-for-the-first-monday-after-a-given-a-date
def next_weekday(d, weekday):
days_ahead = weekday - d.weekday()
if days_ahead < 0: # Target day already happened this week
days_ahead += 7
return d + datetime.timedelta(days_ahead)
def last_weekday(d, weekday):
days_ahead = weekday - d.weekday()
if days_ahead >= 0: # Target day already happened this week
days_ahead -= 7
return d + datetime.timedelta(days_ahead)
import_cache = {}
def get_imported_conf_entry(f, key):
if f not in import_cache:
import_cache[f] = yaml.safe_load(open(f))
return import_cache[f][key]
def do_imports(entry):
if isinstance(entry, dict):
d = entry.items()
result = {}
for key,value in d:
try:
result[key] = do_imports(value)
except KeyError:
pass
except OSError as e:
print(f"Warning: reading {key} yielded error {e}")
return result
if isinstance(entry, list):
l = iter(entry)
result = []
for item in l:
result.append(do_imports(item))
return result
if isinstance(entry, str):
match = IMPORT_RE.match(entry)
if match:
f, key = match.group(1).split(":")
return get_imported_conf_entry(f, key)
match = FILE_RE.match(entry)
if match:
with open(match.group(1)) as f:
return f.read().strip()
return entry
def get_normalized_config_path(f):
if os.path.isdir(f):
f = Path(f) / Path("generator.conf")
return f
def get_config(f):
raw_config = yaml.safe_load(open(f))
config = do_imports(raw_config)
if "override_file" in config:
override_config = yaml.safe_load(open(config["override_file"]))
config.update(override_config)
return config