let's see what's new in this fork of the warc lib

.gitignore

@@ -4,3 +4,4 @@ docs/_build/
 build/
 .coverage
 htmlcov/
+.ropeproject/

Readme.rst

@@ -1,9 +1,7 @@
-warc: Python library to work with WARC files
-============================================
+warc3: Python3 library to work with WARC files
+==============================================
 
-.. image:: https://secure.travis-ci.org/anandology/warc.png?branch=master
-   :alt: build status
-   :target: http://travis-ci.org/anandology/warc
+Note: This is a fork of the original (now dead) warc repository.
 
 WARC (Web ARChive) is a file format for storing web crawls.
 
@@ -12,18 +10,35 @@ http://bibnum.bnf.fr/WARC/
 This `warc` library makes it very easy to work with WARC files.::
 
     import warc
-    f = warc.open("test.warc")
-    for record in f:
-        print record['WARC-Target-URI'], record['Content-Length']
+    with warc.open("test.warc") as f:
+        for record in f:
+            print record['WARC-Target-URI'], record['Content-Length']
 
 Documentation
 -------------
 
 The documentation of the warc library is available at http://warc.readthedocs.org/.
+
+Apart from the install from pip, which will not work for this warc3 version, the
+interface as described there is unchanged.
 
 License
 -------
 
 This software is licensed under GPL v2. See LICENSE_ file for details.
 
 .. LICENSE: http://github.com/internetarchive/warc/blob/master/LICENSE
+
+Authors
+-------
+
+Original Python2 Versions:
+
+* Anand Chitipothu
+* Noufal Ibrahim
+
+Python3 Port:
+
+* Ryan Chartier 
+* Jan Pieter Bruins Slot
+* Almer S. Tigelaar

docs/conf.py

@@ -40,8 +40,8 @@
 master_doc = 'index'
 
 # General information about the project.
-project = u'warc'
-copyright = u'2012, Internet Archive'
+project = 'warc'
+copyright = '2012, Internet Archive'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -178,8 +178,8 @@
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass [howto/manual]).
 latex_documents = [
-  ('index', 'warc.tex', u'WARC Documentation',
-   u'Internet Archive', 'manual'),
+  ('index', 'warc.tex', 'WARC Documentation',
+   'Internet Archive', 'manual'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -211,6 +211,6 @@
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    ('index', 'warc', u'WARC Documentation',
-     [u'Internet Archive'], 1)
+    ('index', 'warc', 'WARC Documentation',
+     ['Internet Archive'], 1)
 ]

requirements.txt

@@ -1 +1 @@
-pytest
+nose

setup.py

@@ -4,7 +4,7 @@
 
 setup(
     name="warc",
-    version="0.2.0",
+    version="0.2.2",
     description="Python library to work with ARC and WARC files",
     long_description=open('Readme.rst').read(),
     license='GPLv2',
@@ -19,7 +19,7 @@
         'Development Status :: 4 - Beta',
         'Environment :: Web Environment',
         'Intended Audience :: Developers',
-        'License :: OSI Approved :: BSD License',
+        'License :: OSI Approved :: GNU General Public License v2 (GPLv2)',
         'Operating System :: OS Independent',
         'Programming Language :: Python',
     ],

warc/__init__.py

@@ -7,32 +7,35 @@
 :copyright: (c) 2012 Internet Archive
 """
 
-from .arc import ARCFile, ARCRecord, ARCHeader
-from .warc import WARCFile, WARCRecord, WARCHeader, WARCReader
+from .arc import ARCFile
+from .warc import WARCFile
+
 
 def detect_format(filename):
     """Tries to figure out the type of the file. Return 'warc' for
     WARC files and 'arc' for ARC files"""
 
-    if ".arc" in filename:
-        return "arc"
-    if ".warc" in filename:
+    if filename.endswith(".warc") or filename.endswith(".warc.gz"):
         return "warc"
 
+    if filename.endswith('.arc') or filename.endswith('.arc.gz'):
+        return 'arc'
+
     return "unknown"
 
-def open(filename, mode="rb", format = None):
+
+def open(filename, mode="rb", format=None):
     """Shorthand for WARCFile(filename, mode).
 
     Auto detects file and opens it.
 
     """
-    if format == "auto" or format == None:
+    if format == "auto" or format is None:
         format = detect_format(filename)
 
     if format == "warc":
         return WARCFile(filename, mode)
     elif format == "arc":
         return ARCFile(filename, mode)
     else:
-        raise IOError("Don't know how to open '%s' files"%format)
+        raise IOError("Don't know how to open '%s' files" % format)