chore: misc local changes

csv2schedule_deu.py

@@ -146,7 +146,7 @@ def process(acronym, base_id, source_csv_url):
 
             for value in row_iter:
                 value = value.strip()
-                if keys2[i] != '' and value != '' and not('Email' in keys2[i]):
+                if keys2[i] != '' and value != '' and 'Email' not in keys2[i]:
                     try:
                         items[keys[i]][keys2[i]] = value
                     except AttributeError as e:

pyproject.toml

@@ -14,7 +14,7 @@ lxml = "^4.9.1"
 gql = ">=3.0.0a5"
 aiohttp = ">=3.7.4.post0"
 GitPython = "^3.1.41"
-bs4 = "^0.0.1"
+beautifulsoup4 = "^4.12.3"
 html5lib = "^1.1"
 ics = "^0.7.2"
 

requirements.txt

@@ -6,7 +6,7 @@ lxml
 gql>=3.0.0a5
 aiohttp>=3.7.4.post0
 gitpython
-bs4
+beautifulsoup4
 html5lib
 ics
 dataclasses

schedule_36C3.py

@@ -187,7 +187,7 @@ def main():
     # write all events from the stages to an own schedule.json/xml
     write('\nExporting main stages... ')
     stages = full_schedule.copy('Stages')
-    for day in stages._schedule['schedule']['conference']['days']:
+    for day in stages.days():
         i = 0
         room_keys = list(day['rooms'].keys())
         for room_key in room_keys:

validator/test.sh

@@ -2,4 +2,7 @@
 
 # Test validation schemas against some example files
 
+#wget 'https://api.events.ccc.de/congress/2023/schedule.xml' -O 37C3.xml
+#xmlstarlet validate -e -s xsd/schedule.xml.xsd 37C3.xml
+
 ./xsd/validate_schedule_xml.sh https://api.events.ccc.de/congress/2023/schedule.xml

voc/git.py

@@ -0,0 +1,50 @@
+
+import argparse
+import os
+from git import Repo
+from voc.c3data import C3data
+from voc.schedule import Schedule
+from voc.tools import (
+    commit_changes_if_something_relevant_changed,
+    git,
+)
+
+def export_event_files(schedule: Schedule, options: argparse.Namespace, local = False):
+    # to get proper a state, we first have to remove all event files from the previous run
+    if not local or options.git:
+        git("rm events/*  >/dev/null")
+    os.makedirs('events', exist_ok=True)
+
+    # write separate file for each event, to get better git diffs
+    # TODO: use Event.export()
+    def export_event(event: Event):
+        origin_system = None
+        if isinstance(event, Event):
+            origin_system = event.origin.origin_system
+
+        with open("events/{}.json".format(event["guid"]), "w") as fp:
+            json.dump(
+                {
+                    **event,
+                    "room_id": schedule._room_ids.get(event["room"], None),
+                    "origin": origin_system or None,
+                },
+                fp,
+                indent=2,
+                cls=ScheduleEncoder,
+            )
+
+    schedule.foreach_event(export_event)
+
+
+def postprocessing(schedule: Schedule, options: argparse.Namespace, local = False):
+    if not local or options.git:
+        commit_changes_if_something_relevant_changed(schedule)
+        # Attention: This method exits the script, if nothing relevant changed
+        # TODO: make this fact more obvious or refactor code
+
+    if not local and "c3data" in targets:
+        print("\n== Updating c3data via API…")
+
+        c3data = C3data(schedule)
+        c3data.process_changed_events(Repo('.'), options)

voc/schedule.py

@@ -26,10 +26,9 @@
 
 log = Logger(__name__)
 
-root_dir = sys.path[0] or (os.path.dirname(__file__) + '/..')
-# validator = '{path}/validator/xsd/validate_schedule_xml.sh'.format(path=sys.path[0])
-validator = f'xmllint --noout --schema {root_dir}/validator/xsd/schedule.xml.xsd'
-# validator = f"xmllint --noout --schema {root_dir}/validator/xsd/schedule-without-person.xml.xsd"
+# validator = f"{sys.path[0]}/validator/xsd/validate_schedule_xml.sh"
+validator = f"xmllint --noout --schema {sys.path[0]}/validator/xsd/schedule.xml.xsd"
+# validator = f"xmllint --noout --schema {sys.path[0]}/validator/xsd/schedule-without-person.xml.xsd"
 validator_filter = ""
 
 
@@ -38,10 +37,6 @@ def set_validator_filter(filter):
     validator_filter += " | awk '" + " && ".join(["!/" + x + "/" for x in filter]) + "'"
 
 
-class Schedule:
-    pass
-
-
 class ScheduleException(Exception):
     pass
 
@@ -215,7 +210,7 @@ def from_XC3_template(cls, name, congress_nr, start_day, days_count):
         return schedule
 
     @classmethod
-    def empty_copy_of(cls, parent_schedule: Schedule, name: str, start_hour=None):
+    def empty_copy_of(cls, parent_schedule: 'Schedule', name: str, start_hour=None):
         schedule = Schedule(
             version=datetime.now().strftime("%Y:%m-%d %H:%M"),
             conference=copy.deepcopy(parent_schedule.conference()),
@@ -413,7 +408,7 @@ def foreach_day_room(self, func):
 
         return out
 
-    def _generate_stats(self, enable_time_stats=False):
+    def _generate_stats(self, enable_time_stats=False, verbose=False):
         class ScheduleStats:
             min_id = None
             max_id = None
@@ -454,6 +449,13 @@ def calc_stats(event: Event):
 
         self.foreach_event(calc_stats)
 
+        if verbose:
+            print(f"  from {self['conference']['start']} to {self['conference']['end']}")
+            print( "  contains {events_count} events, with local ids from {min_id} to {max_id}".format(**schedule.stats.__dict__))  # noqa
+            print( "    local person ids from {person_min_id} to {person_max_id}".format(**self.stats.__dict__)) # noqa
+            print(f"    rooms: {', '.join(self.rooms())}")
+
+
     def get_day_from_time(self, start_time):
         for i in range(self.conference("daysCount")):
             day = self.day(i + 1)

voc/tools.py

@@ -24,6 +24,15 @@
 
 log = Logger(__name__)
 
+def DefaultOptionParser():
+    optparse.OptionParser()
+    parser.add_option("--online", action="store_true", dest="online", default=False)
+    parser.add_option(
+        "--fail", action="store_true", dest="exit_when_exception_occours", default=local
+    )
+    parser.add_option("--stats", action="store_true", dest="only_stats", default=False)
+    parser.add_option("--git", action="store_true", dest="git", default=False)
+    parser.add_option("--debug", action="store_true", dest="debug", default=local)
 
 def write(x):
     sys.stdout.write(x)