diff --git a/apps/calendar/__init__.py b/apps/calendar/__init__.py deleted file mode 100755 index e69de29..0000000 diff --git a/apps/calendar/drivers/__init__.py b/apps/calendar/drivers/__init__.py deleted file mode 100755 index e69de29..0000000 diff --git a/apps/calendar/drivers/ical.py b/apps/calendar/drivers/ical.py deleted file mode 100755 index 610f37c..0000000 --- a/apps/calendar/drivers/ical.py +++ /dev/null @@ -1,71 +0,0 @@ -from google.appengine.ext import db -from google.appengine.api.labs import taskqueue -from google.appengine.api import urlfetch, memcache - -from tipfy import url_for, Response - - -import logging -import chardet - -from apps.calendar.models import iCalSnippet - -#future improvements: abstract out memcache, database, and taskqueue - - -def split_ical(ical): - first_vevent_start=ical.find('BEGIN:VEVENT') - cal_meta=ical[0:first_vevent_start] - - def next_vevent(start): - end=ical.find('END:VEVENT',start) - if end > -1: - end=end+10 - cal=cal_meta+ical[start:end]+"\nEND:VCALENDAR" - return (cal, end) - else: - return(None, None) - vevent, index=next_vevent(first_vevent_start) - while vevent: - yield vevent - vevent, index=next_vevent(index) - -def process(**kwargs): - if not kwargs.get('phase'): - calendar=db.get(kwargs['calendar'])[0] - logging.warning('started fetching %s'% calendar.title ) - result=urlfetch.fetch(calendar.source_uri, allow_truncated=True, deadline=5) - detection=chardet.detect(result.content) - - memcache.add("%s-source" % kwargs['ticket'][0], result.content.decode(detection['encoding'])) - logging.warning("cached ical source with key %s" % "%s-source" % kwargs['ticket'][0]) - - - task_params={ - 'driver':"apps.calendar.drivers.%s:process" % calendar.driver, - 'ticket':kwargs['ticket'][0], - 'calendar':kwargs['calendar'][0], - 'phase':'split' - } - taskqueue.add(url=url_for("calendar/tasks/process_source_calendar"), params=task_params) - - elif kwargs['phase'][0] == 'split': - ticket=db.Key(kwargs['ticket'][0]) - calendar=db.Key(kwargs['calendar'][0]) - ical_source=memcache.get("%s-source" % kwargs['ticket'][0]) - snippets=[] - for snippet in split_ical(ical_source): - snippets.append(iCalSnippet(ticket=ticket, ical=snippet, calendar=calendar, parent=ticket)) - - def save_and_process_snippets(): - db.put(snippets) - task_params={ - 'ticket':kwargs['ticket'][0], - 'calendar':kwargs['calendar'][0], - } - taskqueue.add(url=url_for("calendar/tasks/process_ical_snippets"), params=task_params, transactional=True) - db.run_in_transaction(save_and_process_snippets) - - - - \ No newline at end of file diff --git a/apps/calendar/forms.py b/apps/calendar/forms.py deleted file mode 100755 index fc3598b..0000000 --- a/apps/calendar/forms.py +++ /dev/null @@ -1,63 +0,0 @@ -from datetime import date -import logging - -from tipfy import RequestHandler, Response, redirect, cached_property -from tipfy.ext.jinja2 import Jinja2Mixin -from tipfy.ext.wtforms import Form, fields, validators, widgets -from tipfy.ext.wtforms.validators import ValidationError - -from tipfy.ext.db import populate_entity - -from wtforms.ext.dateutil.fields import DateField - - -from pytz.gae import pytz - - -# a set of HTML5 widgets for wtforms - -from wtforms.widgets import Input - -from models import SourceCalendar - -class DateWidget(Input): - def __init__(self, input_type='date'): - if input_type is not None: - self.input_type = input_type - -class TimeWidget(Input): - def __init__(self, input_type='time'): - if input_type is not None: - self.input_type = input_type - - - - -class LinkWidget(Input): - def __init__(self, input_type='url'): - if input_type is not None: - self.input_type = input_type - - - - -REQUIRED = validators.required() - -DRIVERS=(('ical', 'iCalendar'), ) - -class AddCalendarForm(Form): - title = fields.TextField('Calendar Name', validators=[REQUIRED]) - source_uri=fields.TextField('Source URL', validators=[validators.URL(), REQUIRED], widget=LinkWidget()) - credit_uri=fields.TextField('Credit URL', validators=[validators.optional(), validators.URL()], widget=LinkWidget()) - driver=fields.SelectField('Driver', choices=DRIVERS, validators=[REQUIRED]) - - def save(self, entity=None): - if entity: - populate_entity(entity, **self.data) - entity.put() - else: - new_calendar=SourceCalendar(**self.data) - new_calendar.put() - - - \ No newline at end of file diff --git a/apps/calendar/handlers.py b/apps/calendar/handlers.py deleted file mode 100755 index 64c89b8..0000000 --- a/apps/calendar/handlers.py +++ /dev/null @@ -1,80 +0,0 @@ -import os, logging -from datetime import date, time, datetime - -from google.appengine.api.labs import taskqueue - -from werkzeug import MultiDict - -from tipfy import RequestHandler, Response, redirect, cached_property, url_for -from tipfy.ext.jinja2 import Jinja2Mixin -from tipfy.ext.db import get_entity_dict - -from models import SourceCalendar -from forms import AddCalendarForm - - -class IndexHandler(RequestHandler, Jinja2Mixin): - - - def get(self): - calendars=SourceCalendar.all().fetch(100) - context=dict(calendars=calendars) - return self.render_response('calendar/index.html', **context) - - -class AddCalendarHandler(RequestHandler, Jinja2Mixin): - - def get(self): - calendars=SourceCalendar.all() - context=dict(form=self.form) - return self.render_response('calendar/add.html', **context) - - - def post(self): - if self.form.validate(): - self.form.save() - return redirect('/calendars/') - - - return self.get() - - @cached_property - def form(self): - return AddCalendarForm(self.request.form) - - -class EditCalendarHandler(RequestHandler, Jinja2Mixin): - - - def get(self, **kwargs): - context=dict(form=self.form) - existing_calendar=self.existing_calendar - return self.render_response('calendar/add.html', **context) - - - def post(self, **kwargs): - if self.form.validate(): - self.form.save(self.existing_calendar) - return redirect('/calendars/') - - - return self.get(**kwargs) - - @cached_property - def existing_calendar(self): - slug=self.request.rule_args['calendar_slug'] - return SourceCalendar.all().filter('slug = ', slug).get() - - - @cached_property - def form(self): - if self.request.method == 'GET': - return AddCalendarForm(MultiDict(get_entity_dict(self.existing_calendar).iteritems())) - if self.request.method == 'POST': - return AddCalendarForm(self.request.form) - - -class FetchCalendarHandler(RequestHandler): - def post(self): - taskqueue.add(url=url_for("calendar/tasks/start_fetch_source"), params=self.request.form) - return redirect('/calendars/') \ No newline at end of file diff --git a/apps/calendar/middleware.py b/apps/calendar/middleware.py deleted file mode 100755 index 313b4df..0000000 --- a/apps/calendar/middleware.py +++ /dev/null @@ -1,6 +0,0 @@ -from tipfy import RequestHandler, Response, redirect, cached_property - -class SiteRequiredMiddleware(object): - def pre_dispatch(self, handler): - if not handler.site: - return Response("I don't know that site!") \ No newline at end of file diff --git a/apps/calendar/models.py b/apps/calendar/models.py deleted file mode 100755 index edbc636..0000000 --- a/apps/calendar/models.py +++ /dev/null @@ -1,58 +0,0 @@ - -from google.appengine.ext import db -from tipfy.ext.db import SlugProperty, TimeZoneProperty -from google.appengine.ext.db.polymodel import PolyModel - - -class CalendarImporter(db.Model): - pass - - -class Calendar(PolyModel): - creator=db.ReferenceProperty() - title=db.StringProperty(required=True) - slug=SlugProperty(title) - subscribed_calendars=db.StringListProperty() - subscribed_users=db.StringListProperty() - - -class ProfileCalendar(Calendar): - pass - -class EditedCalendar(Calendar): - editors=db.StringListProperty() - - -class SourceCalendar(Calendar): - source_uri=db.LinkProperty() - credit_uri=db.LinkProperty() - driver=db.StringProperty(required=True) - - -class SourceProcessTicket(db.Model): - timestamp=db.DateTimeProperty(auto_now_add=True) - source_calendar=db.ReferenceProperty() - -class iCalSnippet(db.Model): - ticket=db.ReferenceProperty(SourceProcessTicket) - ical=db.TextProperty() - calendar=db.ReferenceProperty(Calendar, collection_name='snippets') - - -class Event(PolyModel): - title = db.StringProperty(required=True) - start= db.DateTimeProperty() - end=db.DateTimeProperty(required=True, indexed=True) - allday=db.BooleanProperty(required=False) - location=db.PostalAddressProperty(required=False, indexed=False) - link=db.LinkProperty(required=False, indexed=False) - description=db.TextProperty() - cost=db.TextProperty(required=False) - timezone=TimezoneProperty() - created=db.DateTimeProperty(auto_now_add=True) - updated=db.DateTimeProperty(auto_now_add=True) - -class SourcedEvent(Event): - ticket=db.ReferenceProperty() - last_seen=db.DateTimeProperty(auto_now_add=True) - last_ical=db.TextProperty() \ No newline at end of file diff --git a/apps/calendar/tasks.py b/apps/calendar/tasks.py deleted file mode 100755 index c89aa33..0000000 --- a/apps/calendar/tasks.py +++ /dev/null @@ -1,80 +0,0 @@ -from google.appengine.ext import db -from google.appengine.api.labs import taskqueue -import vobject - -import logging -from resolver import resolve - -from tipfy import RequestHandler, Response, url_for - -from models import SourceProcessTicket, iCalSnippet, SourcedEvent - - - -class ProcessSourceCalendarHandler(RequestHandler): - def post(self): - driver=resolve(self.request.form['driver']) - # handle unknown drivers - driver(**self.request.form) - - return Response() - - - - -class StartFetchSourceCalendar(RequestHandler): - def post(self): - logging.info("StartFetchSourceCalendar running") - calendar_key_str=self.request.form.get('calendar') - if calendar_key_str: - calendar_key=db.Key(calendar_key_str) - ticket=SourceProcessTicket(source_calendar=calendar_key) - ticket.put() - # enqueue ProcessSourceCalendarHandler with calendar, and resolver statement - # pointing to driver-appropriate "process" function - calendar=db.get(calendar_key) - task_params={ - 'driver':"apps.calendar.drivers.%s:process" % calendar.driver, - 'ticket':ticket.key(), - 'calendar':calendar.key() - } - taskqueue.add(url=url_for("calendar/tasks/process_source_calendar"), params=task_params) - - return Response() - -class ProcessiCalSnippets(RequestHandler): - def extract_times(self,vevent): - return (vevent.dtstart.value, vevent.dtend.value) - - def post(self): - ticket_key=db.Key(self.request.form['ticket']) - snippets_q=iCalSnippet.all().filter('ticket =', ticket_key) - - if self.request.form.get('cursor'): - snippets_q.with_cursor(self.request.form.get('cursor')) - - next_snippet=snippets_q.get() - if next_snippet: - parsedCal = vobject.readOne(next_snippet.ical) - vevent=parsedCal.vevent - #todo-- run this in transaction - #todo-- name the task - task_params={ - 'ticket':self.request.form['ticket'], - 'cursor':snippets_q.cursor() - } - taskqueue.add(url=url_for("calendar/tasks/process_ical_snippets"), params=task_params) - dtstart, dtend = self.extract_times(vevent) - title=vevent.summary or "Untitled Event" - start=vevent.dtstart.value - logging.warning(start) - #event=SourcedEvent() - else: - pass - #enque cleanup - - - # if no cursor, get the first snippet - # transform into a Event object with a resonable key - # re-enqueue with cursor - return Response() \ No newline at end of file diff --git a/apps/calendar/urls.py b/apps/calendar/urls.py deleted file mode 100755 index 38c05e6..0000000 --- a/apps/calendar/urls.py +++ /dev/null @@ -1,21 +0,0 @@ -from tipfy import Rule - -def get_rules(app): - rules = [ - Rule('/calendars/', endpoint="calendar/index", handler='apps.calendar.handlers.IndexHandler'), - Rule('/calendars/add', endpoint="calendar/add", handler='apps.calendar.handlers.AddCalendarHandler'), - Rule('/calendars//edit', endpoint="calendar/edit", handler="apps.calendar.handlers.EditCalendarHandler"), - Rule('/calendars/initiate_fetch', endpoint="calendar/fetch", handler="apps.calendar.handlers.FetchCalendarHandler"), - Rule('/tasks/start_fetch_source_calendar', - endpoint="calendar/tasks/start_fetch_source", - handler="apps.calendar.tasks.StartFetchSourceCalendar"), - Rule('/tasks/process_source_calendar', - endpoint="calendar/tasks/process_source_calendar", - handler="apps.calendar.tasks.ProcessSourceCalendarHandler"), - - Rule('/tasks/process_ical_snippets', - endpoint="calendar/tasks/process_ical_snippets", - handler="apps.calendar.tasks.ProcessiCalSnippets"), - ] - - return rules \ No newline at end of file diff --git a/cron.yaml b/cron.yaml index 8a250f6..e972356 100755 --- a/cron.yaml +++ b/cron.yaml @@ -1,8 +1,5 @@ cron: - description: start getting icals url: /sources/start_fetch_icals/ - schedule: every 5 minutes -- description: schedule upcoming newsletters - url: /subscriptions/start_schedule_newsletters/ - schedule: every 12 hours synchronized + schedule: every 2 hours diff --git a/django_templates/base.html b/django_templates/base.html index cb9da77..f1e3544 100755 --- a/django_templates/base.html +++ b/django_templates/base.html @@ -26,8 +26,8 @@ - - + +