Add handling of rotated and compressed files

* manage files rotated to "filename.1.gz"
* add management of files rotated and compressed immediately
* fix tests and their execution (travis, tox)

.travis.yml

@@ -5,3 +5,6 @@ python:
   - pypy
   - 3.2
 script: python setup.py test
+
+install:
+  - if [[ $TRAVIS_PYTHON_VERSION == 2.6 ]]; then pip install --use-mirrors unittest2; fi

pygtail/core.py

@@ -27,11 +27,26 @@
 import sys
 import glob
 import string
+import gzip
 from optparse import OptionParser
 
 __version__ = '0.4.0'
 
 
+PY3 = sys.version_info[0] == 3
+
+if PY3:
+    text_type = str
+else:
+    text_type = unicode
+
+
+def force_text(s, encoding='utf-8', errors='strict'):
+    if isinstance(s, text_type):
+        return s
+    return s.decode(encoding, errors)
+
+
 class Pygtail(object):
     """
     Creates an iterable object that returns only unread lines.
@@ -117,18 +132,36 @@ def read(self):
         """
         lines = self.readlines()
         if lines:
-            return ''.join(lines)
+            try:
+                return ''.join(lines)
+            except TypeError:
+                return ''.join(force_text(line) for line in lines)
         else:
             return None
 
+    def _is_closed(self):
+        if not self._fh:
+            return True
+        try:
+            return self._fh.closed
+        except AttributeError:
+            if isinstance(self._fh, gzip.GzipFile):
+                # python 2.6
+                return self._fh.fileobj is None
+            else:
+                raise
+
     def _filehandle(self):
         """
         Return a filehandle to the file being tailed, with the position set
         to the current offset.
         """
-        if not self._fh or self._fh.closed:
+        if not self._fh or self._is_closed():
             filename = self._rotated_logfile or self.filename
-            self._fh = open(filename, "r")
+            if filename.endswith('.gz'):
+                self._fh = gzip.open(filename, 'r')
+            else:
+                self._fh = open(filename, "r")
             self._fh.seek(self._offset)
 
         return self._fh
@@ -178,10 +211,16 @@ def _check_rotated_filename_candidates(self):
             return candidate
 
         # logrotate(8)
+        # with delaycompress
         candidate = "%s.1" % self.filename
         if exists(candidate):
             return candidate
 
+        # without delaycompress
+        candidate = "%s.1.gz" % self.filename
+        if exists(candidate):
+            return candidate
+
         # dateext rotation scheme
         candidates = glob.glob("%s-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]" % self.filename)
         if candidates:

pygtail/test/test_pygtail.py

@@ -1,10 +1,18 @@
 import os
 import sys
-import unittest
+try:
+    # python 2.6
+    import unittest2 as unittest
+except ImportError:
+    import unittest
 import shutil
 import tempfile
+import gzip
+import io
 from pygtail import Pygtail
 
+PY2 = sys.version_info[0] == 2
+
 
 class PygtailTest(unittest.TestCase):
     # TODO:
@@ -32,7 +40,7 @@ def copytruncate(self):
 
     def tearDown(self):
         filename = self.logfile.name
-        for tmpfile in [filename, filename + ".offset", filename + ".1"]:
+        for tmpfile in [filename, filename + ".offset", filename + ".1", filename + ".1.gz"]:
             if os.path.exists(tmpfile):
                 os.remove(tmpfile)
 
@@ -57,7 +65,27 @@ def test_subsequent_read_with_new_data(self):
         new_pygtail = Pygtail(self.logfile.name)
         self.assertEqual(new_pygtail.read(), new_lines)
 
-    def test_logrotate(self):
+    def test_logrotate_without_delay_compress(self):
+        new_lines = ["4\n5\n", "6\n7\n"]
+        pygtail = Pygtail(self.logfile.name)
+        pygtail.read()
+        self.append(new_lines[0])
+
+        # put content to gzip file
+        gzip_handle = gzip.open("%s.1.gz" % self.logfile.name, 'wb')
+        with open(self.logfile.name, 'rb') as logfile:
+            gzip_handle.write(logfile.read())
+        gzip_handle.close()
+
+        with open(self.logfile.name, 'w'):
+            # truncate file
+            pass
+
+        self.append(new_lines[1])
+        pygtail = Pygtail(self.logfile.name)
+        self.assertEqual(pygtail.read(), ''.join(new_lines))
+
+    def test_logrotate_with_delay_compress(self):
         new_lines = ["4\n5\n", "6\n7\n"]
         pygtail = Pygtail(self.logfile.name)
         pygtail.read()
@@ -72,9 +100,14 @@ def test_copytruncate_off_smaller(self):
         self.copytruncate()
         new_lines = "4\n5\n"
         self.append(new_lines)
+
+        sys.stderr = captured = io.BytesIO() if PY2 else io.StringIO()
         pygtail = Pygtail(self.logfile.name, copytruncate=False)
+        captured_value = captured.getvalue()
+        sys.stderr = sys.__stderr__
+
+        self.assertRegexpMatches(captured_value, r".*?\bWARN\b.*?\bshrank\b.*")
         self.assertEqual(pygtail.read(), None)
-        self.assertRegexpMatches(sys.stderr.getvalue(), r".*?\bWARN\b.*?\bshrank\b.*")
 
     def test_copytruncate_on_smaller(self):
         self.test_readlines()

tox.ini

@@ -4,7 +4,10 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py26, py27, py31, py32, pypy
+minversion=1.8.0
+envlist = py26, py27, py31, py32, py33, pypy
 
 [testenv]
 commands = python setup.py test
+deps =
+	py26: unittest2