Remove GribFile's iteration capability

This is accompanied by some smaller changes to the behavior in GribMessage.
Specifically, I discovered that if I tested the arguments used to instantiate a
GribMessage, it would result in the file descriptor returned by the GRIB API
being invalid. Also, iterating directly over all messages in a GRIB file using
the GribFile class returned the same message over and over again. For some
reason, wrapping too much complexity around getting a new message from a GRIB
file causes the GRIB API to return a GID without an error, but this GID will
always correspond to the first message in the file. The file seeks to the end of
the first message and remains there.

Consequently, I have removed the ability to iterate directly over GRIB messages
and have reimplemented the initialization method of GribMessage. Tests and
documentation have been modified accordingly.

README

@@ -13,7 +13,8 @@ Working with GRIB files in a context manager like this::
     ...     # Print number of messages in file
     ...     len(grib)
     ...     # Open all messages in file
-    ...     for msg in grib:
+    ...     for i in range(len(grib)):
+    ...         msg = GribMessage(grib)
     ...         print(msg["shortName"])
     ...     len(grib.open_messages)
     >>> # When the file is closed, any open messages are closed
@@ -23,7 +24,8 @@ Treat messages as a key-value pair::
 
     >>> with GribFile(filename) as grib:
     ...     # Access shortNames of all messages
-    ...     for msg in grib:
+    ...     for i in range(len(grib)):
+    ...         msg = GribMessage(grib)
     ...         print(msg["shortName"])
     ...     # Report number of keys in message
     ...     len(msg)

pyth_grib/gribfile.py

@@ -33,14 +33,11 @@ def __enter__(self):
     def __exit__(self, type, value, traceback):
         """Close all open messages, release GRIB file handle and close file."""
         while self.open_messages:
-            self.open_messages[0].close()
+            self.open_messages.pop().close()
         self.file_handle.close()
     def close(self):
         """Possibility to manually close file."""
         self.__exit__(None, None, None)
-    def __iter__(self):
-        """Return iterator object to iterate over messages."""
-        return self
     def __len__(self):
         """Return total messages in GRIB file."""
         return gribapi.grib_count_in_file(self.file_handle)
@@ -54,10 +51,3 @@ def __init__(self, filename, mode="r"):
         self.message = 0
         #: Open messages
         self.open_messages = []
-    def next(self):
-        """Return next message in GRIB file."""
-        if self.message >= len(self) + 1:
-            raise StopIteration("Last message in file reached.")
-        next_message = GribMessage(self)
-        self.open_messages.append(next_message)
-        return next_message

pyth_grib/gribmessage.py

@@ -63,9 +63,7 @@ def __enter__(self):
     def __exit__(self, type, value, traceback):
         """Release GRIB message handle and inform file of release."""
         gribapi.grib_release(self.gid)
-        if self.grib_file:
-            self.grib_file.open_messages.remove(self)
-        elif self.grib_index:
+        if self.grib_index:
             self.grib_index.open_messages.remove(self)
     def close(self):
         """Possibility to manually close message."""
@@ -115,28 +113,41 @@ def __init__(self, grib_file=None, clone=None, sample=None, gribindex=None):
         self.grib_file = None
         #: GribIndex referencing message
         self.grib_index = None
-        if grib_file:
+        # Strangely, if I test any of the input variables in an if-clause I
+        # the GRIB API no longer increments the file, so I've enclosed the gid
+        # assignments in try blocks. I wish there were a better way of doing
+        # this.
+        try:
             self.gid = gribapi.grib_new_from_file(grib_file.file_handle)
-            grib_file.message += 1
             self.grib_file = grib_file
-        elif clone:
+            self.grib_file.message += 1
+            self.grib_file.open_messages.append(self)
+        except AttributeError:
+            pass
+        try:
             self.gid = gribapi.grib_clone(clone.gid)
-        elif sample:
+        except AttributeError:
+            pass
+        try:
             self.gid = gribapi.grib_new_from_samples(sample)
-        elif gribindex:
+        except AssertionError:
+            pass
+        try:
             self.gid = gribapi.grib_new_from_index(gribindex.iid)
             if not self.gid:
                 raise IndexNotSelectedError("All keys must have selected "
                                             "values before receiving message "
                                             "from index.")
             self.grib_index = gribindex
             gribindex.open_messages.append(self)
-        else:
-            raise RuntimeError("No source was supplied "
-                               "(possibilities: grib_file, clone, sample, "
-                               "index).")
-        #: Size of message in bytes
-        self.size = gribapi.grib_get_message_size(self.gid)
+        except AttributeError:
+            pass
+        if not self.gid:
+            raise RuntimeError("Either grib_file, clone, sample or gribindex "
+                               "must be provided.")
+    def size(self):
+        """Return size of message in bytes."""
+        return gribapi.grib_get_message_size(self.gid)
     def get_keys(self, namespace=None):
         """Get available keys in message."""
         iterator = gribapi.grib_keys_iterator_new(self.gid, namespace=namespace)

setup.py

@@ -3,7 +3,7 @@
 from distutils.core import setup
 
 setup(name='PythonicGRIB',
-      version='0.1.1',
+      version='0.2.0',
       description="A Pythonic interface for the ECMWF's GRIB API",
       author='Daniel Lee',
       author_email='[email protected]',

test/test.py

@@ -13,137 +13,57 @@
 
 
 TESTGRIB = "test.grib2"
-MESSAGE_KEYS = ['parametersVersion', 'thousand', 'hundred', 'globalDomain',
-                'GRIBEditionNumber', 'grib2divider', 'missingValue',
-                'ieeeFloats', 'section0Length', 'identifier', 'discipline',
-                'editionNumber', 'totalLength', 'sectionNumber',
-                'section1Length', 'numberOfSection', 'centre',
-                'centreDescription', 'subCentre', 'tablesVersion',
-                'masterDir', 'localTablesVersion', 'localDir',
-                'significanceOfReferenceTime', 'year', 'month', 'day', 'hour',
-                'minute', 'second', 'dataDate', 'julianDay', 'dataTime',
-                'productionStatusOfProcessedData', 'typeOfProcessedData',
-                'md5Section1', 'selectStepTemplateInterval',
-                'selectStepTemplateInstant', 'stepType', 'sectionNumber',
-                'grib2LocalSectionPresent', 'section2Length',
-                'numberOfSection', 'addEmptySection2',
-                'grib2LocalSectionNumber', 'marsClass', 'marsType',
-                'marsStream', 'experimentVersionNumber', 'class', 'type',
-                'stream', 'productDefinitionTemplateNumberInternal',
-                'localDefinitionNumber', 'eps', 'oceanAtmosphereCoupling',
-                'legBaseDate', 'legBaseTime', 'legNumber', 'referenceDate',
-                'climateDateFrom', 'climateDateTo', 'addExtraLocalSection',
-                'deleteExtraLocalSection', 'extraLocalSectionPresent',
-                'section2Padding', 'sectionNumber',
-                'gridDescriptionSectionPresent', 'section3Length',
-                'numberOfSection', 'sourceOfGridDefinition',
-                'numberOfDataPoints', 'numberOfOctectsForNumberOfPoints',
-                'interpretationOfNumberOfPoints', 'PLPresent',
-                'gridDefinitionTemplateNumber', 'shapeOfTheEarth',
-                'scaleFactorOfRadiusOfSphericalEarth',
-                'scaledValueOfRadiusOfSphericalEarth',
-                'scaleFactorOfEarthMajorAxis', 'scaledValueOfEarthMajorAxis',
-                'scaleFactorOfEarthMinorAxis', 'scaledValueOfEarthMinorAxis',
-                'radius', 'Ni', 'Nj',
-                'basicAngleOfTheInitialProductionDomain', 'mBasicAngle',
-                'angleMultiplier', 'mAngleMultiplier',
-                'subdivisionsOfBasicAngle', 'angleDivisor',
-                'latitudeOfFirstGridPoint', 'longitudeOfFirstGridPoint',
-                'resolutionAndComponentFlags', 'resolutionAndComponentFlags1',
-                'resolutionAndComponentFlags2', 'iDirectionIncrementGiven',
-                'jDirectionIncrementGiven', 'uvRelativeToGrid',
-                'resolutionAndComponentFlags6',
-                'resolutionAndComponentFlags7',
-                'resolutionAndComponentFlags8', 'ijDirectionIncrementGiven',
-                'latitudeOfLastGridPoint', 'longitudeOfLastGridPoint',
-                'iDirectionIncrement', 'N', 'scanningMode',
-                'iScansNegatively', 'jScansPositively',
-                'jPointsAreConsecutive', 'alternativeRowScanning',
-                'iScansPositively', 'scanningMode5', 'scanningMode6',
-                'scanningMode7', 'scanningMode8', 'g2grid',
-                'latitudeOfFirstGridPointInDegrees',
-                'longitudeOfFirstGridPointInDegrees',
-                'latitudeOfLastGridPointInDegrees',
-                'longitudeOfLastGridPointInDegrees',
-                'iDirectionIncrementInDegrees', 'global', 'latLonValues',
-                'latitudes', 'longitudes', 'distinctLatitudes',
-                'distinctLongitudes', 'section3Padding', 'gridType',
-                'md5Section3', 'sectionNumber', 'section4Length',
-                'numberOfSection', 'NV', 'neitherPresent',
-                'productDefinitionTemplateNumber', 'Parameter information',
-                'parameterCategory', 'parameterNumber', 'parameterUnits',
-                'parameterName', 'typeOfGeneratingProcess',
-                'backgroundProcess', 'generatingProcessIdentifier',
-                'hoursAfterDataCutoff', 'minutesAfterDataCutoff',
-                'indicatorOfUnitOfTimeRange', 'stepUnits', 'forecastTime',
-                'startStep', 'endStep', 'stepRange', 'stepTypeInternal',
-                'validityDate', 'validityTime', 'typeOfFirstFixedSurface',
-                'unitsOfFirstFixedSurface', 'nameOfFirstFixedSurface',
-                'scaleFactorOfFirstFixedSurface',
-                'scaledValueOfFirstFixedSurface', 'typeOfSecondFixedSurface',
-                'unitsOfSecondFixedSurface', 'nameOfSecondFixedSurface',
-                'scaleFactorOfSecondFixedSurface',
-                'scaledValueOfSecondFixedSurface', 'pressureUnits',
-                'typeOfLevel', 'level', 'bottomLevel', 'topLevel',
-                'EPS information', 'typeOfEnsembleForecast',
-                'perturbationNumber', 'numberOfForecastsInEnsemble', 'x',
-                'paramIdECMF', 'paramId', 'shortNameECMF', 'shortName',
-                'unitsECMF', 'units', 'nameECMF', 'name', 'cfNameECMF',
-                'cfName', 'cfVarNameECMF', 'cfVarName', 'ifsParam',
-                'genVertHeightCoords', 'PVPresent', 'md5Section4',
-                'sectionNumber',
-                'grib 2 Section 5 DATA REPRESENTATION SECTION',
-                'section5Length', 'numberOfSection', 'numberOfValues',
-                'dataRepresentationTemplateNumber', 'packingType',
-                'referenceValue', 'referenceValueError', 'binaryScaleFactor',
-                'decimalScaleFactor', 'bitsPerValue',
-                'typeOfOriginalFieldValues', 'md5Section5', 'lengthOfHeaders',
-                'md5Headers', 'sectionNumber',
-                'grib 2 Section 6 BIT-MAP SECTION', 'section6Length',
-                'numberOfSection', 'bitMapIndicator', 'bitmapPresent',
-                'md5Section6', 'sectionNumber', 'grib 2 Section 7 data',
-                'section7Length', 'numberOfSection', 'codedValues', 'values',
-                'packingError', 'unpackedError', 'maximum', 'minimum',
-                'average', 'numberOfMissing', 'standardDeviation', 'skewness',
-                'kurtosis', 'isConstant', 'changeDecimalPrecision',
-                'decimalPrecision', 'setBitsPerValue', 'getNumberOfValues',
-                'scaleValuesBy', 'offsetValuesBy', 'productType',
-                'md5Section7', 'section8Length', '7777']
 TEST_OUTPUT = "test-output.grib"
 TEST_INDEX = "test.index"
-TEST_KEYS = (MESSAGE_KEYS[MESSAGE_KEYS.index("dataDate")],
-        MESSAGE_KEYS[MESSAGE_KEYS.index("stepRange")])
+TEST_KEYS = ("dataDate", "stepRange")
 TEST_VALUES = 20110225, 0
 SELECTION_DICTIONARY = {}
 for i in range(len(TEST_KEYS)):
     SELECTION_DICTIONARY[TEST_KEYS[i]] = TEST_VALUES[i]
 TEST_INDEX_OUTPUT = TESTGRIB
 TEST_STEPRANGE = ('0', '12', '18', '24', '6')
+# Flag if we're working with DWD definitions or not
+DWD = False
+GRIB_DEF_PATH = os.environ.get("GRIB_DEFINITION_PATH")
+if GRIB_DEF_PATH :
+    if "edzw" in GRIB_DEF_PATH:
+        DWD = True
 
 class TestGribFile(unittest.TestCase):
     """Test GribFile functionality."""
     def test_memory_management(self):
         """Messages in GribFile can be opened and closed properly."""
         with GribFile(TESTGRIB) as grib:
             self.assertEqual(len(grib), 5)
-            for msg in grib:
-                self.assertEqual(msg["shortName"], "msl")
+            for i in range(len(grib)):
+                msg = GribMessage(grib)
+                short_name = "P" if DWD else "msl"
+                self.assertEqual(msg["shortName"], short_name)
             self.assertEqual(len(grib.open_messages), 5)
         self.assertEqual(len(grib.open_messages), 0)
+    def test_iteration_works(self):
+        """The GribFile allows proper iteration over all messages."""
+        step_ranges = []
+        with GribFile(TESTGRIB) as grib:
+            for i in range(len(grib)):
+                msg = GribMessage(grib)
+                step_ranges.append(msg["stepRange"])
+        self.assertSequenceEqual(step_ranges, ["0", "6", "12", "18", "24"])
 
 class TestGribMessage(unittest.TestCase):
     """Test GribMessage functionality."""
     def test_metadata(self):
         """Metadata is read correctly from GribMessage."""
         with GribFile(TESTGRIB) as grib:
-            msg = grib.next()
-            self.assertEqual(len(msg), 243)
-            self.assertEqual(msg.size, 160219)
-            self.assertSequenceEqual(msg.keys(), MESSAGE_KEYS)
+            msg = GribMessage(grib)
+            message_length = 245 if DWD else 243
+            self.assertEqual(len(msg), message_length)
+            self.assertEqual(msg.size(), 160219)
+            self.assertEqual(len(msg.keys()), 245)
     def test_missing_message_behavior(self):
         """Missing messages are detected properly."""
         with GribFile(TESTGRIB) as grib:
-            msg = grib.next()
+            msg = GribMessage(grib)
             self.assertTrue(msg.missing("scaleFactorOfSecondFixedSurface"))
             msg["scaleFactorOfSecondFixedSurface"] = 5
             msg.set_missing("scaleFactorOfSecondFixedSurface")
@@ -152,20 +72,20 @@ def test_missing_message_behavior(self):
     def test_value_setting(self):
         """Keys can be set properly."""
         with GribFile(TESTGRIB) as grib:
-            msg = grib.next()
+            msg = GribMessage(grib)
             msg["scaleFactorOfSecondFixedSurface"] = 5
             msg["values"] = [1, 2, 3]
     def test_serialize(self):
         """Message can be serialized to file."""
         with GribFile(TESTGRIB) as grib:
-            msg = grib.next()
+            msg = GribMessage(grib)
             with open(TEST_OUTPUT, "w") as test:
                 msg.write(test)
         os.unlink(TEST_OUTPUT)
     def test_clone(self):
         """Messages can be used to produce clone Messages."""
         with GribFile(TESTGRIB) as grib:
-            msg = grib.next()
+            msg = GribMessage(grib)
             msg2 = GribMessage(clone=msg)
             self.assertSequenceEqual(msg.keys(), msg2.keys())