Merge pull request #1273 from haileyajohnson/scale-offset

minor changes for performance

cdm-test/src/test/java/ucar/nc2/dataset/TestProjectionCoordinates.java

@@ -82,7 +82,7 @@ private void testCoordinates(String filename, double startLat, double startLon,
     System.out.printf("end = %f %f%n", start2.getLatitude(), start2.getLongitude());
 
     assert start1.nearlyEquals(LatLonPoint.create(startLat, startLon), 2.0E-4);
-    assert start2.nearlyEquals(LatLonPoint.create(endLat, endLon), 2.0E-4);
+    assert start2.nearlyEquals(LatLonPoint.create(endLat, endLon), 3.0E-4);
 
     ncd.close();
   }

cdm-test/src/test/java/ucar/nc2/dt/grid/TestGridSubset.java

@@ -150,7 +150,7 @@ public void testMSG() throws Exception {
 
       ProjectionRect expected =
           new ProjectionRect(ProjectionPoint.create(-2129.5688, -1793.0041), 4297.8453, 3308.3885);
-      assert prect.nearlyEquals(expected);
+      assert prect.nearlyEquals(expected, 1e-4);
 
       LatLonRect bb2 = p.projToLatLonBB(prect);
       logger.debug("{} -> {}", prect, bb2);

cdm-test/src/test/java/ucar/nc2/ft/coverage/TestCoverageHorizSubset.java

@@ -61,7 +61,7 @@ public void testMSG() throws Exception {
 
       ProjectionRect expected =
           new ProjectionRect(ProjectionPoint.create(-2129.5688, -1793.0041), 4297.8453, 3308.3885);
-      assert prect.nearlyEquals(expected);
+      assert prect.nearlyEquals(expected, 1e-4);
 
       LatLonRect bb2 = p.projToLatLonBB(prect);
       System.out.printf("%s -> %s %n", prect, bb2);

cdm-test/src/test/java/ucar/nc2/ft/coverage/TestCoverageProjection.java

@@ -64,7 +64,7 @@ public void shouldGetBoundingBox() throws IOException {
           .that(projectionRect.nearlyEquals(expectedProjectionRect)).isTrue();
 
       final LatLonRect expectedLatLonRect =
-          new LatLonRect(LatLonPoint.create(18.4184, -143.143), LatLonPoint.create(43.3274, -79.9189));
+          new LatLonRect(LatLonPoint.create(18.41837, -143.143), LatLonPoint.create(43.32738, -79.91893));
       final LatLonRect latLonRect = horizCoordSys.calcLatLonBoundingBox();
       assertWithMessage("actual: " + latLonRect + ", expected: " + expectedLatLonRect)
           .that(latLonRect.nearlyEquals(expectedLatLonRect)).isTrue();

cdm-test/src/test/java/ucar/nc2/ft/coverage/TestGribCoverageRead.java

@@ -68,8 +68,8 @@ public void TestTwoDRead() throws IOException, InvalidRangeException {
       float first = data.getFloat(0);
       float last = data.getFloat((int) data.getSize() - 1);
       logger.debug("data first = {} last = {}", first, last);
-      Assert2.assertNearlyEquals(241.699997, first);
-      Assert2.assertNearlyEquals(225.099991, last);
+      assertThat(Misc.nearlyEquals(241.699997, first));
+      assertThat(Misc.nearlyEquals(225.099991, last));
     }
   }
 
@@ -99,8 +99,8 @@ public void TestBestRead() throws IOException, InvalidRangeException {
       float first = data.getFloat(0);
       float last = data.getFloat((int) data.getSize() - 1);
       logger.debug("data first = {} last = {}", first, last);
-      Assert2.assertNearlyEquals(241.699997, first);
-      Assert2.assertNearlyEquals(225.099991, last);
+      assertThat(Misc.nearlyEquals(241.699997, first));
+      assertThat(Misc.nearlyEquals(225.099991, last));
     }
   }
 
@@ -161,11 +161,11 @@ public void TestMRUTCRead() throws IOException, InvalidRangeException {
 
       float val = data.getFloat(40600);
       logger.debug("data val at {} = {}", 40600, val);
-      Assert2.assertNearlyEquals(281.627563, val);
+      assertThat(Misc.nearlyEquals(281.627563, val));
 
       val = data.getFloat(55583);
       logger.debug("data val at {} = {}", 55583, val);
-      Assert2.assertNearlyEquals(281.690063, val);
+      assertThat(Misc.nearlyEquals(281.690063, val));
     }
   }
 

cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java

@@ -11,12 +11,7 @@
 import ucar.nc2.constants.CDM;
 import ucar.nc2.constants.DataFormatType;
 import ucar.nc2.dataset.NetcdfDataset.Enhance;
-import ucar.nc2.filter.ConvertMissing;
-import ucar.nc2.filter.FilterHelpers;
-import ucar.nc2.filter.ScaleOffset;
-import ucar.nc2.filter.Standardizer;
-import ucar.nc2.filter.Normalizer;
-import ucar.nc2.filter.UnsignedConversion;
+import ucar.nc2.filter.*;
 import ucar.nc2.internal.dataset.CoordinatesHelper;
 import ucar.nc2.iosp.netcdf3.N3iosp;
 import ucar.nc2.util.CancelTask;
@@ -261,23 +256,48 @@ Array convert(Array data, Set<NetcdfDataset.Enhance> enhancements) {
       if (this.isVariableLength) {
         return data;
       }
+      if (!dataType.isNumeric()) {
+        return data;
+      }
+      // datatype of the result depends on what enhancements were applied
+      DataType convertedType = data.getDataType();
+
+      // TODO: change to a provider for extensible Enhancements
+      List<Enhancement> toApply = new ArrayList<>();
       if (enhancements.contains(Enhance.ConvertUnsigned) && unsignedConversion != null) {
-        data = unsignedConversion.convertUnsigned(data);
+        toApply.add(unsignedConversion);
+        convertedType = unsignedConversion.getOutType();
       }
       if (enhancements.contains(Enhance.ApplyScaleOffset) && scaleOffset != null) {
-        data = scaleOffset.removeScaleOffset(data);
+        toApply.add(scaleOffset);
+        convertedType = scaleOffset.getScaledOffsetType();
       }
       if (enhancements.contains(Enhance.ConvertMissing) && convertMissing != null
           && (dataType == DataType.FLOAT || dataType == DataType.DOUBLE)) {
-        data = convertMissing.convertMissing(data);
+        toApply.add(convertMissing);
       }
       if (enhancements.contains(Enhance.ApplyStandardizer) && standardizer != null) {
-        data = standardizer.convert(data);
+        toApply.add(standardizer);
       }
       if (enhancements.contains(Enhance.ApplyNormalizer) && normalizer != null) {
-        data = normalizer.convert(data);
+        toApply.add(normalizer);
+      }
+
+      double[] dataArray = (double[]) data.get1DJavaArray(DataType.DOUBLE);
+
+      dataArray = Arrays.stream(dataArray).parallel().map((num) -> {
+        for (Enhancement e : toApply) {
+          num = e.convert(num);
+        }
+        return num;
+      }).toArray();
+
+      Array out = Array.factory(convertedType, data.getShape());
+      IndexIterator iterOut = out.getIndexIterator();
+      for (int i = 0; i < data.getSize(); i++) {
+        iterOut.setObjectNext(dataArray[i]);
       }
-      return data;
+      return out;
     }
   }
 
@@ -755,17 +775,17 @@ public DataType.Signedness getSignedness() {
 
   @Override
   public double applyScaleOffset(Number value) {
-    return scaleOffset != null ? scaleOffset.removeScaleOffset(value) : value.doubleValue();
+    return scaleOffset != null ? scaleOffset.convert(value.doubleValue()) : value.doubleValue();
   }
 
   @Override
   public Array applyScaleOffset(Array data) {
-    return scaleOffset != null ? scaleOffset.removeScaleOffset(data) : data;
+    return scaleOffset != null ? scaleOffset.convert(data) : data;
   }
 
   @Override
   public Number convertUnsigned(Number value) {
-    return unsignedConversion != null ? unsignedConversion.convertUnsigned(value) : value;
+    return unsignedConversion != null ? unsignedConversion.convert(value.doubleValue()) : value;
   }
 
   public Number convertUnsigned(Number value, DataType dataType) {
@@ -779,7 +799,7 @@ public Array convertUnsigned(Array in) {
 
   @Override
   public Number convertMissing(Number value) {
-    return convertMissing != null ? convertMissing.convertMissing(value) : value;
+    return convertMissing != null ? convertMissing.convert(value.doubleValue()) : value;
   }
 
   @Override

cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java

@@ -8,10 +8,12 @@
 import ucar.nc2.dataset.VariableDS;
 import ucar.nc2.util.Misc;
 
-public class ConvertMissing {
+import java.util.*;
+
+public class ConvertMissing implements Enhancement {
 
   private boolean hasValidMin, hasValidMax;
-  private double validMin, validMax;
+  private double validMin, validMax, fuzzyValidMin, fuzzyValidMax;
 
   private boolean hasFillValue;
   private double fillValue; // LOOK: making it double not really correct. What about CHAR?
@@ -91,7 +93,6 @@ public static ConvertMissing createFromVariable(VariableDS var) {
 
     /// missing_value
     double[] missingValue = null;
-    boolean hasMissingValue = false;
     Attribute missingValueAtt = var.findAttribute(CDM.MISSING_VALUE);
     if (missingValueAtt != null) {
       if (missingValueAtt.isString()) {
@@ -111,42 +112,61 @@ public static ConvertMissing createFromVariable(VariableDS var) {
             // TODO add logger
           }
         }
-        hasMissingValue = true;
       } else { // not a string
         missingValue = new double[missingValueAtt.getLength()];
         DataType missingType = FilterHelpers.getAttributeDataType(missingValueAtt, signedness);
         for (int i = 0; i < missingValue.length; i++) {
           missingValue[i] = var.convertUnsigned(missingValueAtt.getNumericValue(i), missingType).doubleValue();
           missingValue[i] = var.applyScaleOffset(missingValue[i]);
         }
-
-        for (double mv : missingValue) {
-          if (!Double.isNaN(mv)) {
-            hasMissingValue = true; // dont need to do anything if it's already a NaN
-            break;
-          }
-        }
       }
     }
     return new ConvertMissing(var.fillValueIsMissing(), var.invalidDataIsMissing(), var.missingDataIsMissing(),
-        hasValidMin, hasValidMax, validMin, validMax, hasFillValue, fillValue, hasMissingValue, missingValue);
+        hasValidMin, hasValidMax, validMin, validMax, hasFillValue, fillValue, missingValue);
   }
 
 
   public ConvertMissing(boolean fillValueIsMissing, boolean invalidDataIsMissing, boolean missingDataIsMissing,
       boolean hasValidMin, boolean hasValidMax, double validMin, double validMax, boolean hasFillValue,
-      double fillValue, boolean hasMissingValue, double[] missingValue) {
+      double fillValue, double[] missingValue) {
     this.fillValueIsMissing = fillValueIsMissing;
     this.invalidDataIsMissing = invalidDataIsMissing;
     this.missingDataIsMissing = missingDataIsMissing;
     this.hasValidMin = hasValidMin;
     this.hasValidMax = hasValidMax;
     this.validMin = validMin;
+    this.fuzzyValidMin = validMin - Misc.defaultMaxRelativeDiffFloat;
     this.validMax = validMax;
+    this.fuzzyValidMax = validMax + Misc.defaultMaxRelativeDiffFloat;
     this.hasFillValue = hasFillValue;
     this.fillValue = fillValue;
-    this.hasMissingValue = hasMissingValue;
     this.missingValue = missingValue;
+    this.hasMissingValue = false;
+    // clean up missing values: remove NaNs, fill values, and values outside valid range
+    if (this.missingDataIsMissing && this.missingValue != null) {
+      List<Double> missing = new ArrayList();
+      for (double mv : this.missingValue) {
+        if (Double.isNaN(mv)) {
+          continue;
+        }
+        if (fillValueIsMissing && hasFillValue && mv == fillValue) {
+          continue;
+        }
+        if (invalidDataIsMissing && hasValidMin && mv < fuzzyValidMin) {
+          continue;
+        }
+        if (invalidDataIsMissing && hasValidMax && mv > fuzzyValidMax) {
+          continue;
+        }
+        missing.add(mv);
+      }
+      int nMissing = missing.size();
+      this.missingValue = new double[nMissing];
+      for (int i = 0; i < nMissing; i++) {
+        this.missingValue[i] = missing.get(i);
+      }
+      this.hasMissingValue = this.missingValue.length > 0;
+    }
   }
 
   public boolean hasValidData() {
@@ -162,15 +182,16 @@ public double getValidMax() {
   }
 
   public boolean isInvalidData(double val) {
-    // valid_min and valid_max may have been multiplied by scale_factor, which could be a float, not a double.
-    // That potential loss of precision means that we cannot do the nearlyEquals() comparison with
-    // Misc.defaultMaxRelativeDiffDouble.
-    boolean greaterThanOrEqualToValidMin =
-        Misc.nearlyEquals(val, validMin, Misc.defaultMaxRelativeDiffFloat) || val > validMin;
-    boolean lessThanOrEqualToValidMax =
-        Misc.nearlyEquals(val, validMax, Misc.defaultMaxRelativeDiffFloat) || val < validMax;
-
-    return (hasValidMin && !greaterThanOrEqualToValidMin) || (hasValidMax && !lessThanOrEqualToValidMax);
+    if (Double.isNaN(val)) {
+      return true;
+    }
+    if (val > fuzzyValidMax) {
+      return true;
+    }
+    if (val < fuzzyValidMin) {
+      return true;
+    }
+    return false;
   }
 
   public boolean hasFillValue() {
@@ -186,9 +207,6 @@ public double getFillValue() {
   }
 
   public boolean isMissingValue(double val) {
-    if (!hasMissingValue) {
-      return false;
-    }
     for (double aMissingValue : missingValue) {
       if (Misc.nearlyEquals(val, aMissingValue, Misc.defaultMaxRelativeDiffFloat)) {
         return true;
@@ -213,10 +231,10 @@ public boolean hasMissing() {
   public boolean isMissing(double val) {
     if (Double.isNaN(val)) {
       return true;
-    } else {
-      return (missingDataIsMissing && isMissingValue(val)) || (fillValueIsMissing && isFillValue(val))
-          || (invalidDataIsMissing && isInvalidData(val));
     }
+    return (missingDataIsMissing && hasMissingValue && isMissingValue(val))
+        || (fillValueIsMissing && hasFillValue && isFillValue(val))
+        || (invalidDataIsMissing && hasValidData() && isInvalidData(val));
   }
 
   @Deprecated
@@ -234,15 +252,18 @@ public void setMissingDataIsMissing(boolean b) {
     this.missingDataIsMissing = b;
   }
 
-  public Number convertMissing(Number value) {
-    return isMissing(value.doubleValue()) ? Double.NaN : value;
+  public double convert(double value) {
+    return isMissing(value) ? Double.NaN : value;
   }
 
   public Array convertMissing(Array in) {
     DataType type = in.getDataType();
     if (!type.isNumeric()) {
       return in;
     }
+    if (!hasMissing()) {
+      return in;
+    }
 
     Array out = Array.factory(type, in.getShape());
     IndexIterator iterIn = in.getIndexIterator();
@@ -251,7 +272,7 @@ public Array convertMissing(Array in) {
     // iterate and convert elements
     while (iterIn.hasNext()) {
       Number value = (Number) iterIn.getObjectNext();
-      value = convertMissing(value);
+      value = convert(value.doubleValue());
       iterOut.setObjectNext(value);
     }
 

cdm/core/src/main/java/ucar/nc2/filter/Enhancement.java

@@ -0,0 +1,5 @@
+package ucar.nc2.filter;
+
+public interface Enhancement {
+  double convert(double num);
+}

cdm/core/src/main/java/ucar/nc2/filter/Normalizer.java

@@ -3,13 +3,14 @@
 import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
+
 import org.apache.commons.math.stat.descriptive.SummaryStatistics;
 import ucar.ma2.Array;
 import ucar.ma2.DataType;
 import ucar.ma2.IndexIterator;
 import ucar.nc2.dataset.VariableDS;
 
-public class Normalizer {
+public class Normalizer implements Enhancement {
 
   private final ScaleOffset scaleOffset;
   private final double minimum;
@@ -59,6 +60,10 @@ public Array convert(Array arr) {
     return scaleOffset.applyScaleOffset(arr);
   }
 
+  public double convert(double val) {
+    return scaleOffset.applyScaleOffset(val);
+  }
+
   public double getMinimum() {
     return minimum;
   }