fix scale offset applied twice

cdm/core/src/main/java/ucar/nc2/dataset/CoordinateAxis1D.java

@@ -866,6 +866,7 @@ protected void readValues() {
     }
 
     coords = (double[]) data.get1DJavaArray(DataType.DOUBLE);
+    this.wasRead = true;
     // IndexIterator iter = data.getIndexIterator();
     // while (iter.hasNext())
     // coords[count++] = iter.getDoubleNext();

cdm/core/src/main/java/ucar/nc2/dataset/CoordinateAxis1DTime.java

@@ -337,12 +337,17 @@ private CoordinateAxis1DTime(NetcdfDataset ncd, VariableDS org, Formatter errMes
     int ncoords = (int) org.getSize();
     List<CalendarDate> result = new ArrayList<>(ncoords);
 
-    Array data = org.read();
+    if (org instanceof CoordinateAxis1D) {
+      coords = ((CoordinateAxis1D) org).getCoordValues();
+    } else {
+      Array data = org.read();
+      coords = (double[]) data.get1DJavaArray(DataType.DOUBLE);
+    }
+    this.wasRead = true;
 
     int count = 0;
-    IndexIterator ii = data.getIndexIterator();
     for (int i = 0; i < ncoords; i++) {
-      double val = ii.getDoubleNext();
+      double val = coords[i];
       if (Double.isNaN(val))
         continue; // WTF ??
       result.add(helper.makeCalendarDateFromOffset(val));
@@ -355,12 +360,11 @@ private CoordinateAxis1DTime(NetcdfDataset ncd, VariableDS org, Formatter errMes
       setDimension(0, localDim);
 
       // set the shortened values
-      Array shortData = Array.factory(data.getDataType(), new int[] {count});
+      Array shortData = Array.factory(org.getDataType(), new int[] {count});
       Index ima = shortData.getIndex();
       int count2 = 0;
-      ii = data.getIndexIterator();
       for (int i = 0; i < ncoords; i++) {
-        double val = ii.getDoubleNext();
+        double val = coords[i];
         if (Double.isNaN(val))
           continue;
         shortData.setDouble(ima.set0(count2), val);

cdm/core/src/main/java/ucar/nc2/dataset/NetcdfDataset.java

@@ -1344,12 +1344,16 @@ public void addCoordinateTransform(CoordinateTransform ct) {
   public CoordinateAxis addCoordinateAxis(VariableDS v) {
     if (v == null)
       return null;
+
+    final List<CoordinateAxis> coordCopy = new ArrayList<>(coordAxes);
+
     CoordinateAxis oldVar = findCoordinateAxis(v.getFullName());
     if (oldVar != null)
-      coordAxes.remove(oldVar);
+      coordCopy.remove(oldVar);
 
     CoordinateAxis ca = (v instanceof CoordinateAxis) ? (CoordinateAxis) v : CoordinateAxis.factory(this, v);
-    coordAxes.add(ca);
+    coordCopy.add(ca);
+    this.coordAxes = coordCopy;
 
     if (v.isMemberOfStructure()) {
       Structure parentOrg = v.getParentStructure(); // gotta be careful to get the wrapping parent

cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java

@@ -436,12 +436,12 @@ public void setCaching(boolean caching) {
   @Override
   protected Array _read() throws IOException {
     Array result;
-
     // check if already cached - caching in VariableDS only done explicitly by app
-    if (hasCachedData())
+    if (hasCachedData()) {
       result = super._read();
-    else
+    } else {
       result = proxyReader.reallyRead(this, null);
+    }
 
     return convert(result);
   }
@@ -450,15 +450,16 @@ protected Array _read() throws IOException {
   @Override
   protected Array _read(Section section) throws IOException, InvalidRangeException {
     // really a full read
-    if ((null == section) || section.computeSize() == getSize())
+    if ((null == section) || section.computeSize() == getSize()) {
       return _read();
+    }
 
     Array result;
-    if (hasCachedData())
+    if (hasCachedData()) {
       result = super._read(section);
-    else
+    } else {
       result = proxyReader.reallyRead(this, section, null);
-
+    }
     return convert(result);
   }
 
@@ -840,6 +841,7 @@ protected VariableDS(Builder<?> builder, Group parentGroup) {
 
     this.orgFileTypeId = builder.orgFileTypeId;
     this.enhanceProxy = new EnhancementsImpl(this, builder.units, builder.getDescription());
+    this.scaleOffset = builder.scaleOffset;
 
     createEnhancements();
 
@@ -858,7 +860,9 @@ private void createEnhancements() {
       this.dataType = unsignedConversion != null ? unsignedConversion.getOutType() : dataType;
     }
     if (this.enhanceMode.contains(Enhance.ApplyScaleOffset) && (dataType.isNumeric() || dataType == DataType.CHAR)) {
-      this.scaleOffset = ScaleOffset.createFromVariable(this);
+      if (this.scaleOffset == null) {
+        this.scaleOffset = ScaleOffset.createFromVariable(this);
+      }
       this.dataType = scaleOffset != null ? scaleOffset.getScaledOffsetType() : this.dataType;
     }
     Attribute standardizerAtt = findAttribute(CDM.STANDARDIZE);
@@ -902,6 +906,10 @@ protected Builder<?> addLocalFieldsToBuilder(Builder<? extends Builder<?>> build
     builder.setOriginalVariable(this.orgVar).setOriginalDataType(this.orgDataType).setOriginalName(this.orgName)
         .setOriginalFileTypeId(this.orgFileTypeId).setEnhanceMode(this.enhanceMode).setUnits(this.enhanceProxy.units)
         .setDesc(this.enhanceProxy.desc);
+    builder.scaleOffset = this.scaleOffset;
+    if (this.coordSysNames != null) {
+      this.coordSysNames.stream().forEach(s -> builder.addCoordinateSystemName(s));
+    }
 
     return (VariableDS.Builder<?>) super.addLocalFieldsToBuilder(builder);
   }
@@ -944,6 +952,8 @@ public static abstract class Builder<T extends Builder<T>> extends Variable.Buil
     private boolean fillValueIsMissing = NetcdfDataset.fillValueIsMissing;
     private boolean missingDataIsMissing = NetcdfDataset.missingDataIsMissing;
 
+    private ScaleOffset scaleOffset;
+
     private boolean built;
 
     protected abstract T self();
@@ -1040,6 +1050,7 @@ public T copyFrom(VariableDS.Builder<?> builder) {
       setFillValueIsMissing(builder.fillValueIsMissing);
       setInvalidDataIsMissing(builder.invalidDataIsMissing);
       setMissingDataIsMissing(builder.missingDataIsMissing);
+      this.scaleOffset = builder.scaleOffset;
       this.orgVar = builder.orgVar;
       this.orgDataType = builder.orgDataType;
       this.orgFileTypeId = builder.orgFileTypeId;

cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java

@@ -78,13 +78,16 @@ public static ScaleOffset createFromVariable(VariableDS var) {
     if (scaleAtt != null && !scaleAtt.isString()) {
       scaleType = FilterHelpers.getAttributeDataType(scaleAtt, signedness);
       scale = 1 / (var.convertUnsigned(scaleAtt.getNumericValue(), scaleType).doubleValue());
+      var.remove(scaleAtt);
     }
 
     Attribute offsetAtt = var.findAttribute(CDM.ADD_OFFSET);
     if (offsetAtt != null && !offsetAtt.isString()) {
       offsetType = FilterHelpers.getAttributeDataType(offsetAtt, signedness);
       offset = var.convertUnsigned(offsetAtt.getNumericValue(), offsetType).doubleValue();
+      var.remove(offsetAtt);
     }
+
     if (scale != DEFAULT_SCALE || offset != DEFAULT_OFFSET) {
       DataType scaledOffsetType =
           FilterHelpers.largestOf(var.getUnsignedConversionType(), scaleType, offsetType).withSignedness(signedness);

cdm/core/src/main/java/ucar/nc2/ft2/coverage/adapter/DtCoverageDataset.java

@@ -13,11 +13,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import ucar.nc2.Attribute;
-import ucar.nc2.Dimension;
-import ucar.nc2.NetcdfFile;
-import ucar.nc2.Variable;
-import ucar.nc2.VariableSimpleIF;
+
+import ucar.nc2.*;
 import ucar.nc2.constants.CDM;
 import ucar.nc2.constants.FeatureType;
 import ucar.nc2.dataset.CoordinateAxis;
@@ -113,22 +110,25 @@ public DtCoverageDataset(NetcdfDataset ncd) throws IOException {
    * @throws java.io.IOException on read error
    */
   public DtCoverageDataset(NetcdfDataset ncd, Formatter parseInfo) throws IOException {
-    this.ncd = ncd;
-
     Set<NetcdfDataset.Enhance> enhance = ncd.getEnhanceMode();
-    if (enhance == null || enhance.isEmpty())
+    if (enhance == null || enhance.isEmpty()) {
       enhance = NetcdfDataset.getDefaultEnhanceMode();
-    ncd = NetcdfDatasets.enhance(ncd, enhance, null);
+    }
+    this.ncd = NetcdfDatasets.enhance(ncd, enhance, null);
 
-    DtCoverageCSBuilder facc = DtCoverageCSBuilder.classify(ncd, parseInfo);
-    if (facc != null)
-      this.coverageType = facc.type;
+    // sort by largest size first
+    List<CoordinateSystem> csList = new ArrayList<>(ncd.getCoordinateSystems());
+    csList.sort((o1, o2) -> o2.getCoordinateAxes().size() - o1.getCoordinateAxes().size());
 
     Map<String, Gridset> csHash = new HashMap<>();
-    for (CoordinateSystem cs : ncd.getCoordinateSystems()) {
+    for (CoordinateSystem cs : csList) {
       DtCoverageCSBuilder fac = new DtCoverageCSBuilder(ncd, cs, parseInfo);
-      if (fac.type == null)
+      if (fac.type == null) {
         continue;
+      }
+      if (this.coverageType == null) {
+        this.coverageType = fac.type;
+      }
       DtCoverageCS ccs = fac.makeCoordSys();
       if (ccs == null)
         continue;

cdm/core/src/main/java/ucar/nc2/internal/dataset/CoordinatesHelper.java

@@ -56,7 +56,7 @@ public List<CoordinateTransform> getCoordTransforms() {
   private CoordinatesHelper(Builder builder, NetcdfDataset ncd) {
     List<CoordinateAxis> axes = new ArrayList<>();
     addAxes(ncd.getRootGroup(), axes);
-    this.coordAxes = ImmutableList.copyOf(axes);
+    coordAxes = ImmutableList.copyOf(axes);
 
     coordTransforms =
         builder.coordTransforms.stream().map(ct -> ct.build(ncd)).filter(Objects::nonNull).collect(Collectors.toList());

cdm/core/src/test/java/ucar/nc2/dataset/TestScaleOffsetMissingUnsigned.java

@@ -84,7 +84,6 @@ public void testWrite() throws Exception {
     }
 
     Array readEnhanced;
-
     // read the packed form, enhance using scale/offset, compare to original
     try (NetcdfDataset ncd = NetcdfDatasets.openDataset(filename)) {
       VariableDS vs = (VariableDS) ncd.findVariable("packed");
@@ -133,15 +132,14 @@ private void doSubset(String filename) throws IOException, InvalidRangeException
     }
   }
 
-
   // Asserts that "scale_factor" is applied to "_FillValue".
   // This test demonstrated the bug in https://github.com/Unidata/thredds/issues/1065.
   @Test
   public void testScaledFillValue() throws URISyntaxException, IOException {
     File testResource = new File(getClass().getResource("testScaledFillValue.ncml").toURI());
 
     // LOOK removeEnhancement does not work in new
-    try (NetcdfDataset ncd = NetcdfDataset.openDataset(testResource.getAbsolutePath(), true, null)) {
+    try (NetcdfDataset ncd = NetcdfDatasets.openDataset(testResource.getAbsolutePath(), true, null)) {
       VariableDS fooVar = (VariableDS) ncd.findVariable("foo");
 
       double expectedFillValue = .99999;

cdm/core/src/test/java/ucar/nc2/internal/ncml/TestEnhance.java

@@ -72,8 +72,7 @@ public void testStandaloneEnhance() throws IOException {
       Variable scaledvar = ncfile.findVariable("scaledvar");
       assertThat((Object) scaledvar).isNotNull();
       assertThat(scaledvar.getDataType()).isEqualTo(DataType.FLOAT);
-      assertThat(scaledvar.attributes().hasAttribute("scale_factor")).isTrue();
-      assertThat(scaledvar.attributes().findAttributeDouble("scale_factor", 1.0)).isEqualTo(2.0);
+      assertThat(scaledvar.attributes().hasAttribute("scale_factor")).isFalse();
       assertThat(scaledvar.readScalarFloat()).isEqualTo(12.0f);
     }
   }
@@ -91,8 +90,7 @@ public void testStandaloneEnhanceDataset() throws IOException {
       Variable scaledvar = ncfile.findVariable("scaledvar");
       assertThat((Object) scaledvar).isNotNull();
       assertThat(scaledvar.getDataType()).isEqualTo(DataType.FLOAT);
-      assertThat(scaledvar.attributes().hasAttribute("scale_factor")).isTrue();
-      assertThat(scaledvar.attributes().findAttributeDouble("scale_factor", 1.0)).isEqualTo(2.0);
+      assertThat(scaledvar.attributes().hasAttribute("scale_factor")).isFalse();
       assertThat(scaledvar.readScalarFloat()).isEqualTo(12.0f);
     }
   }
@@ -110,8 +108,6 @@ public void testStandaloneDoubleEnhanceDataset() throws IOException {
       Variable scaledvar = ncfile.findVariable("scaledvar");
       assertThat((Object) scaledvar).isNotNull();
       assertThat(scaledvar.getDataType()).isEqualTo(DataType.FLOAT);
-      assertThat(scaledvar.attributes().hasAttribute("scale_factor")).isTrue();
-      assertThat(scaledvar.attributes().findAttributeDouble("scale_factor", 1.0)).isEqualTo(2.0);
       assertThat(scaledvar.readScalarFloat()).isEqualTo(12.0f);
     }
   }

cdm/core/src/test/java/ucar/nc2/ncml/TestEnhance.java

@@ -68,8 +68,7 @@ public void testStandaloneEnhance() throws IOException {
       Variable scaledvar = ncfile.findVariable("scaledvar");
       assertThat((Object) scaledvar).isNotNull();
       assertThat(scaledvar.getDataType()).isEqualTo(DataType.FLOAT);
-      assertThat(scaledvar.attributes().hasAttribute("scale_factor")).isTrue();
-      assertThat(scaledvar.attributes().findAttributeDouble("scale_factor", 1.0)).isEqualTo(2.0);
+      assertThat(scaledvar.attributes().hasAttribute("scale_factor")).isFalse();
       assertThat(scaledvar.readScalarFloat()).isEqualTo(12.0f);
     }
   }
@@ -87,8 +86,7 @@ public void testStandaloneEnhanceDataset() throws IOException {
       Variable scaledvar = ncfile.findVariable("scaledvar");
       assertThat((Object) scaledvar).isNotNull();
       assertThat(scaledvar.getDataType()).isEqualTo(DataType.FLOAT);
-      assertThat(scaledvar.attributes().hasAttribute("scale_factor")).isTrue();
-      assertThat(scaledvar.attributes().findAttributeDouble("scale_factor", 1.0)).isEqualTo(2.0);
+      assertThat(scaledvar.attributes().hasAttribute("scale_factor")).isFalse();
       assertThat(scaledvar.readScalarFloat()).isEqualTo(12.0f);
     }
   }
@@ -106,8 +104,7 @@ public void testStandaloneDoubleEnhanceDataset() throws IOException {
       Variable scaledvar = ncfile.findVariable("scaledvar");
       assertThat((Object) scaledvar).isNotNull();
       assertThat(scaledvar.getDataType()).isEqualTo(DataType.FLOAT);
-      assertThat(scaledvar.attributes().hasAttribute("scale_factor")).isTrue();
-      assertThat(scaledvar.attributes().findAttributeDouble("scale_factor", 1.0)).isEqualTo(2.0);
+      assertThat(scaledvar.attributes().hasAttribute("scale_factor")).isFalse();
       assertThat(scaledvar.readScalarFloat()).isEqualTo(12.0f);
     }
   }

docs/src/test/java/tests/cdmdatasets/TestNetcdfDatasetTutorial.java

@@ -61,13 +61,8 @@ public void testUnpackDataTutorial() throws IOException {
       assertThat((Object) scaledvar).isNotNull();
       assertThat(scaledvar.getDataType()).isEqualTo(DataType.FLOAT);
 
-      assertThat(scaledvar.attributes().hasAttribute("scale_factor")).isTrue();
-      double scale_factor = scaledvar.attributes().findAttributeDouble("scale_factor", 1.0);
-      assertThat(scaledvar.attributes().hasAttribute("add_offset")).isTrue();
-      double add_offset = scaledvar.attributes().findAttributeDouble("add_offset", 1.0);
-
       double unpacked_data =
-          NetcdfDatasetTutorial.unpackData(scaledvar.readScalarShort(), scale_factor, add_offset);
+          NetcdfDatasetTutorial.unpackData(scaledvar.readScalarShort(), 1.0, 0.0);
       assertThat(unpacked_data).isNotNaN();
     }
   }