From 65aab84656bc4ce6f4673cdfe40ae611f5fdc998 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Miguel=20Neves?= Date: Sun, 20 Sep 2015 10:16:07 +0100 Subject: [PATCH 01/11] setup testing for multiple python versions --- .travis.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.travis.yml b/.travis.yml index b165373..260959d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,13 @@ language: python +python: + - "2.6" + - "2.7" + - "3.2" + - "3.3" + - "3.4" + - "3.5" + cache: directories: - $HOME/.cache/pip From 6d21d11595d9985f62442964560e32be50ff3d94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Miguel=20Neves?= Date: Sun, 20 Sep 2015 13:06:19 +0100 Subject: [PATCH 02/11] add python3 version of vobject to requirements --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d3e3321..8c61e24 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,5 +15,5 @@ regex==2015.5.10 requests==2.7.0 six==1.9.0 Unidecode==0.4.17 -vobject==0.6.6 +-e git+https://github.com/tBaxter/vobject.git@5bdf21751a6f443045e05db862e00be5bbd006f3#egg=vobject wok==1.1.1 From bbb71fd1c298959c7a48c2b858cb15aec212594c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Miguel=20Neves?= Date: Sun, 20 Sep 2015 13:09:37 +0100 Subject: [PATCH 03/11] add python3 version of wok to requirements --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8c61e24..deee50c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,4 +16,4 @@ requests==2.7.0 six==1.9.0 Unidecode==0.4.17 -e git+https://github.com/tBaxter/vobject.git@5bdf21751a6f443045e05db862e00be5bbd006f3#egg=vobject -wok==1.1.1 +-e git+https://github.com/jneves/wok.git@f9cb25775d93207d5cb644962952872954b90d8#egg=wok From dc6c3dcff420b3dc481bde6f354f1ba643a81925 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Miguel=20Neves?= Date: Sun, 20 Sep 2015 14:12:49 +0100 Subject: [PATCH 04/11] update wok version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index deee50c..e848b9f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,4 +16,4 @@ requests==2.7.0 six==1.9.0 Unidecode==0.4.17 -e git+https://github.com/tBaxter/vobject.git@5bdf21751a6f443045e05db862e00be5bbd006f3#egg=vobject --e git+https://github.com/jneves/wok.git@f9cb25775d93207d5cb644962952872954b90d8#egg=wok +-e git+https://github.com/jneves/wok.git@ad9145705014b58a23165d920aebc226818a6834#egg=wok From fce8765476b0ec72dfbea6d8c7ee2dfe4e19c246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Miguel=20Neves?= Date: Sun, 20 Sep 2015 14:41:35 +0100 Subject: [PATCH 05/11] update py3 wok version --- hooks/guidebook.py | 8 ++++++-- hooks/schedule_summary.py | 9 +++++++-- requirements.txt | 12 ++---------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/hooks/guidebook.py b/hooks/guidebook.py index 1f8af5f..520f4b5 100644 --- a/hooks/guidebook.py +++ b/hooks/guidebook.py @@ -4,10 +4,14 @@ import codecs import csv -import cStringIO import io import os +try: + from cStringIO import StringIO +except ImportError: # python 3 + from io import StringIO + from flat_schedule import (mkdirs, read_html_tabular_schedule) @@ -23,7 +27,7 @@ class UnicodeWriter(object): # https://docs.python.org/2.7/library/csv.html#writer-objects def __init__(self, f, dialect=csv.excel, encoding='utf-8', **kwds): - self.queue = cStringIO.StringIO() + self.queue = StringIO() self.writer = csv.writer(self.queue, dialect=dialect, **kwds) self.stream = f self.encoder = codecs.getincrementalencoder(encoding)() diff --git a/hooks/schedule_summary.py b/hooks/schedule_summary.py index ce800fc..72a45e7 100644 --- a/hooks/schedule_summary.py +++ b/hooks/schedule_summary.py @@ -8,10 +8,15 @@ import codecs import csv -import cStringIO import io import os +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO + + from flat_schedule import mkdirs, read_html_tabular_schedule @@ -23,7 +28,7 @@ class UnicodeWriter(object): # https://docs.python.org/2.7/library/csv.html#writer-objects def __init__(self, f, dialect=csv.excel, encoding='utf-8', **kwds): - self.queue = cStringIO.StringIO() + self.queue = StringIO() self.writer = csv.writer(self.queue, dialect=dialect, **kwds) self.stream = f self.encoder = codecs.getincrementalencoder(encoding)() diff --git a/requirements.txt b/requirements.txt index e848b9f..a095ea2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,11 @@ -awesome-slugify==1.4 -docutils==0.8.1 -Jinja2==2.7.3 -LinkChecker==9.3 +#LinkChecker==9.3 # waiting for py3 version lxml==3.4.4 -Markdown==2.1.1 MarkupSafe==0.23 -py==1.4.27 -Pygments==1.4 -pytest==2.5.2 python-dateutil==2.4.2 pytz==2015.4 -PyYAML==3.10 regex==2015.5.10 requests==2.7.0 six==1.9.0 Unidecode==0.4.17 -e git+https://github.com/tBaxter/vobject.git@5bdf21751a6f443045e05db862e00be5bbd006f3#egg=vobject --e git+https://github.com/jneves/wok.git@ad9145705014b58a23165d920aebc226818a6834#egg=wok +-e git+https://github.com/jneves/wok.git@77f067bf7d02b003e29476615d37af2f3fe4b042#egg=wok From 62e0a938c1ba026052172b336e7b1b11e002cee6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Miguel=20Neves?= Date: Sun, 20 Sep 2015 15:01:06 +0100 Subject: [PATCH 06/11] updated wok requirement --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a095ea2..482424e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,4 @@ requests==2.7.0 six==1.9.0 Unidecode==0.4.17 -e git+https://github.com/tBaxter/vobject.git@5bdf21751a6f443045e05db862e00be5bbd006f3#egg=vobject --e git+https://github.com/jneves/wok.git@77f067bf7d02b003e29476615d37af2f3fe4b042#egg=wok +-e git+https://github.com/jneves/wok.git@21ce6ea44483218b2380104b6ada8865122ddf2c#egg=wok From 65bb0a78b3ccb7dc8d819c61f477ed94b9dd7025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Miguel=20Neves?= Date: Sun, 20 Sep 2015 15:51:12 +0100 Subject: [PATCH 07/11] fixed hooks --- hooks/__hooks__.py | 18 ------------------ hooks/flat_schedule.py | 21 +++++++++++++-------- hooks/guidebook.py | 3 +-- hooks/schedule_summary.py | 2 -- 4 files changed, 14 insertions(+), 30 deletions(-) delete mode 100644 hooks/__hooks__.py diff --git a/hooks/__hooks__.py b/hooks/__hooks__.py deleted file mode 100644 index 01ba8d1..0000000 --- a/hooks/__hooks__.py +++ /dev/null @@ -1,18 +0,0 @@ -''' __hooks__.py - -Attach Python functions to wok hooks. -''' - -import flat_schedule -import guidebook -import schedule_summary - -# The `hooks` dictionary that wok will import -# See http://wok.mythmon.com/docs/hooks/ -hooks = { - 'site.done': [ - flat_schedule.create_flat_schedule, - guidebook.create_csv_schedule, - schedule_summary.create_summary_schedule, - ], - } diff --git a/hooks/flat_schedule.py b/hooks/flat_schedule.py index b89ff08..5f52133 100755 --- a/hooks/flat_schedule.py +++ b/hooks/flat_schedule.py @@ -3,7 +3,6 @@ import datetime import errno import io -import itertools import os import re @@ -14,6 +13,10 @@ import vobject #import yaml +try: + from itertools import chain, repeat, izip_longest as zip_longest +except ImportError: # python 3 + from itertools import chain, repeat, zip_longest def parse_date(s): """Returns a datetime.date, from a date in the tabular schedule @@ -66,11 +69,11 @@ def collapse_whitespace(s): def repeat_none(value, times): - for i in xrange(times): + for i in range(times): yield None -def colspan_cells(cells, fillfunc=itertools.repeat): +def colspan_cells(cells, fillfunc=repeat): """Yields td or th elements, repeating them as necessary for colspan=n """ for cell in cells: @@ -80,7 +83,7 @@ def colspan_cells(cells, fillfunc=itertools.repeat): yield item -def rowspan_cells(cells, rowspans, fillfunc=itertools.repeat): +def rowspan_cells(cells, rowspans, fillfunc=repeat): """Yields td or th elements, repeating them as necessary for colspan=n & rowspan=n. """ @@ -109,7 +112,7 @@ def parse_rooms(table): """ row1 = colspan_cells(table.xpath('./thead/tr[1]/th')[1:]) row2 = colspan_cells(table.xpath('./thead/tr[2]/th')[1:]) - for th1, th2 in itertools.izip_longest(row1, row2): + for th1, th2 in zip_longest(row1, row2): text1 = collapse_whitespace(th1.text) text2 = collapse_whitespace(th2.text) if th2 is not None else '' if text2: @@ -200,12 +203,12 @@ def parse_event(td, default_room=None): def stringify_children(node): # http://stackoverflow.com/a/28173933/293340 parts = ([node.text] - + list(itertools.chain(*([lxml.html.tostring(c, with_tail=False), + + list(chain(*([lxml.html.tostring(c, with_tail=False), c.tail] for c in node.getchildren()) )) + [node.tail]) # filter removes possible Nones in texts and tails - return ''.join(part for part in parts if part is not None) + return ''.join(str(part) for part in parts if part is not None) def events(table): @@ -366,6 +369,7 @@ def mkdirs(path): def write_flat_schedule(schedule, config): + print('**** write_flat_scheduled CALLED ****') schedule_html = render_schedule(schedule, config['template_dir']) schedule_dir = os.path.join(config['output_dir'], 'schedule', 'flat') schedule_path = os.path.join(schedule_dir, 'index.html') @@ -376,6 +380,7 @@ def write_flat_schedule(schedule, config): def write_ical_schedule(schedule, config): + print('**** write_ical_scheduled CALLED ****') cal = vobject.iCalendar() cal.add('x-wr-calname').value = 'PyCon UK 2015 Schedule' @@ -414,7 +419,7 @@ def add_tz(dt): ics_path = os.path.join(config['output_dir'], 'schedule.ics') with io.open(ics_path, 'w', encoding='utf-8') as f: - f.write(cal.serialize().decode('utf-8')) + f.write(cal.serialize()) def create_flat_schedule(config): diff --git a/hooks/guidebook.py b/hooks/guidebook.py index 520f4b5..2179058 100644 --- a/hooks/guidebook.py +++ b/hooks/guidebook.py @@ -35,7 +35,6 @@ def __init__(self, f, dialect=csv.excel, encoding='utf-8', **kwds): def writerow(self, row): self.writer.writerow([s.encode('utf-8') for s in row]) data = self.queue.getvalue() - data = data.decode('utf-8') data = self.encoder.encode(data) self.stream.write(data) self.queue.truncate(0) @@ -82,7 +81,7 @@ def extract_description(talk, config): if talk['type'] in EVENT_TYPES: path = os.path.join(config['content_dir'], talk['href'].strip('/') + '.md') with open(path) as f: - text = f.read().decode('utf-8') + text = f.read() # Remove metadata if present. idx = text.find('###') if idx != -1: diff --git a/hooks/schedule_summary.py b/hooks/schedule_summary.py index 72a45e7..7de4eb8 100644 --- a/hooks/schedule_summary.py +++ b/hooks/schedule_summary.py @@ -36,7 +36,6 @@ def __init__(self, f, dialect=csv.excel, encoding='utf-8', **kwds): def writerow(self, row): self.writer.writerow([s.encode('utf-8') for s in row]) data = self.queue.getvalue() - data = data.decode('utf-8') data = self.encoder.encode(data) self.stream.write(data) self.queue.truncate(0) @@ -76,7 +75,6 @@ def extract_speaker(talk, config): path = os.path.join(config['content_dir'], talk['href'].strip('/') + '.md') with open(path) as f: for line in f: - line = line.decode('utf-8') if line[:4] == '### ': return line[4:].strip() return '' From 1b6b741a92fc4675487a6d61aca6e24801690415 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Miguel=20Neves?= Date: Sun, 20 Sep 2015 15:53:30 +0100 Subject: [PATCH 08/11] clean debug --- Makefile | 6 ++++++ hooks/flat_schedule.py | 2 -- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 607d5a8..a007f92 100644 --- a/Makefile +++ b/Makefile @@ -11,9 +11,15 @@ help: @echo '' build: + python hooks/flat_schedule.py + python hooks/guidebook.py + python hooks/schedule_summary.py wok serve: + python hooks/flat_schedule.py + python hooks/guidebook.py + python hooks/schedule_summary.py wok --serve test: diff --git a/hooks/flat_schedule.py b/hooks/flat_schedule.py index 5f52133..8fea85d 100755 --- a/hooks/flat_schedule.py +++ b/hooks/flat_schedule.py @@ -369,7 +369,6 @@ def mkdirs(path): def write_flat_schedule(schedule, config): - print('**** write_flat_scheduled CALLED ****') schedule_html = render_schedule(schedule, config['template_dir']) schedule_dir = os.path.join(config['output_dir'], 'schedule', 'flat') schedule_path = os.path.join(schedule_dir, 'index.html') @@ -380,7 +379,6 @@ def write_flat_schedule(schedule, config): def write_ical_schedule(schedule, config): - print('**** write_ical_scheduled CALLED ****') cal = vobject.iCalendar() cal.add('x-wr-calname').value = 'PyCon UK 2015 Schedule' From 5845de499aa644a6831b4f5844e15c357df1e66f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Miguel=20Neves?= Date: Sun, 20 Sep 2015 16:30:01 +0100 Subject: [PATCH 09/11] added new hooks setup --- hooks/__hooks__.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 hooks/__hooks__.py diff --git a/hooks/__hooks__.py b/hooks/__hooks__.py new file mode 100644 index 0000000..b906b91 --- /dev/null +++ b/hooks/__hooks__.py @@ -0,0 +1,7 @@ +from flat_schedule import create_flat_schedule +from guidebook import create_csv_schedule +from schedule_summary import create_summary_schedule + +hooks = { + 'site.done': [create_flat_schedule, create_csv_schedule, create_summary_schedule], +} From 474f69f29156642240f85f6f889b29d582d12129 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Miguel=20Neves?= Date: Mon, 21 Sep 2015 10:40:46 +0100 Subject: [PATCH 10/11] removed useless changes --- Makefile | 6 ------ hooks/__hooks__.py | 21 ++++++++++++++++----- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index a007f92..607d5a8 100644 --- a/Makefile +++ b/Makefile @@ -11,15 +11,9 @@ help: @echo '' build: - python hooks/flat_schedule.py - python hooks/guidebook.py - python hooks/schedule_summary.py wok serve: - python hooks/flat_schedule.py - python hooks/guidebook.py - python hooks/schedule_summary.py wok --serve test: diff --git a/hooks/__hooks__.py b/hooks/__hooks__.py index b906b91..01ba8d1 100644 --- a/hooks/__hooks__.py +++ b/hooks/__hooks__.py @@ -1,7 +1,18 @@ -from flat_schedule import create_flat_schedule -from guidebook import create_csv_schedule -from schedule_summary import create_summary_schedule +''' __hooks__.py +Attach Python functions to wok hooks. +''' + +import flat_schedule +import guidebook +import schedule_summary + +# The `hooks` dictionary that wok will import +# See http://wok.mythmon.com/docs/hooks/ hooks = { - 'site.done': [create_flat_schedule, create_csv_schedule, create_summary_schedule], -} + 'site.done': [ + flat_schedule.create_flat_schedule, + guidebook.create_csv_schedule, + schedule_summary.create_summary_schedule, + ], + } From 04d1e9d1bdd57835621d95c1c15ca19bd211025a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Miguel=20Neves?= Date: Mon, 21 Sep 2015 12:15:59 +0100 Subject: [PATCH 11/11] work-around for lxml string/byte string behaviour --- hooks/flat_schedule.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/hooks/flat_schedule.py b/hooks/flat_schedule.py index 8fea85d..26e678c 100755 --- a/hooks/flat_schedule.py +++ b/hooks/flat_schedule.py @@ -201,14 +201,21 @@ def parse_event(td, default_room=None): def stringify_children(node): + def make_sure_is_string(obj): + """antidote for lxml returning sometimes strings, sometimes byte strings """ + try: + return obj.decode('utf-8') + except AttributeError: + return obj + # http://stackoverflow.com/a/28173933/293340 parts = ([node.text] - + list(chain(*([lxml.html.tostring(c, with_tail=False), + + list(chain(*([lxml.html.tostring(c, with_tail=False, encoding='utf-8'), c.tail] for c in node.getchildren()) )) + [node.tail]) # filter removes possible Nones in texts and tails - return ''.join(str(part) for part in parts if part is not None) + return ''.join(make_sure_is_string(part) for part in parts if part is not None) def events(table):