ARROW-4358: [CI] Restore support for trusty in CI

- also, add travis entry for trusty Author: Pindikura Ravindra <[email protected]> Closes apache#3472 from pravindra/trusty and squashes the following commits: 5380ae2 <Pindikura Ravindra> ARROW-4358: switch back to a good conda label 29e69d9 <Pindikura Ravindra> ARROW-4358: skip llvm install for trusty 8094816 <Pindikura Ravindra> ARROW-4358: fix a typo 8d5a715 <Pindikura Ravindra> ARROW-4358: Skip updating mvn
pcmoritz · Jan 28, 2019 · d93db4a · d93db4a
1 parent 35adb25
commit d93db4a
Show file tree

Hide file tree

Showing 11 changed files with 94 additions and 28 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -108,6 +108,39 @@ matrix:
     - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1
     - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh || travis_terminate 1
     - $TRAVIS_BUILD_DIR/ci/travis_upload_cpp_coverage.sh || travis_terminate 1
+  - name: "C++ unit tests w/ gcc 4.8, trusty"
+    dist: trusty
+    compiler: gcc
+    language: cpp
+    os: linux
+    jdk: openjdk8
+    env:
+    - ARROW_TRAVIS_USE_TOOLCHAIN=1
+    - ARROW_TRAVIS_PLASMA=1
+    - ARROW_TRAVIS_ORC=1
+    - ARROW_TRAVIS_PARQUET=1
+    - ARROW_TRAVIS_GANDIVA=1
+    - ARROW_TRAVIS_GANDIVA_JAVA=1
+    - ARROW_BUILD_WARNING_LEVEL=CHECKIN
+    before_install:
+    - ulimit -c unlimited -S
+    - |
+        if [ $TRAVIS_OS_NAME == "linux" ]; then
+          sudo bash -c "echo -e 'Acquire::Retries 10; Acquire::http::Timeout \"20\";' > /etc/apt/apt.conf.d/99-travis-retry"
+          sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test
+          sudo apt-get update -qq
+        fi
+    - eval `python $TRAVIS_BUILD_DIR/ci/detect-changes.py`
+    before_script:
+    - if [ $ARROW_CI_CPP_AFFECTED != "1" ] && [ $ARROW_CI_JAVA_AFFECTED != "1" ]; then exit; fi
+    - $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
+    - $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
+    # If either C++ or Python changed, we must install the C++ libraries
+    - git submodule update --init
+    - $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
+    script:
+    - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh || travis_terminate 1
+    - $TRAVIS_BUILD_DIR/ci/travis_script_gandiva_java.sh || travis_terminate 1
   - name: "Python 2.7 and 3.6 unit tests w/ Valgrind, gcc 5.4, coverage"
     compiler: gcc
     language: cpp

diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh
@@ -80,8 +80,10 @@ ARROW_CXXFLAGS=""
 # Use Ninja for faster builds when using toolchain
 if [ $ARROW_TRAVIS_USE_TOOLCHAIN == "1" ]; then
   CMAKE_COMMON_FLAGS="$CMAKE_COMMON_FLAGS -GNinja"
-  # Make sure the toolchain linker (from binutils package) is picked up by clang
-  ARROW_CXXFLAGS="$ARROW_CXXFLAGS -B$CPP_TOOLCHAIN/bin"
+  if [ "$DISTRO_CODENAME" != "trusty" ]; then
+    # Make sure the toolchain linker (from binutils package) is picked up by clang
+    ARROW_CXXFLAGS="$ARROW_CXXFLAGS -B$CPP_TOOLCHAIN/bin"
+  fi
 fi
 
 if [ $ARROW_TRAVIS_PLASMA == "1" ]; then

diff --git a/ci/travis_env_common.sh b/ci/travis_env_common.sh
@@ -74,7 +74,12 @@ fi
 
 export PARQUET_TEST_DATA=$TRAVIS_BUILD_DIR/cpp/submodules/parquet-testing/data
 
+# e.g. "trusty" or "xenial"
 if [ $TRAVIS_OS_NAME == "linux" ]; then
+  export DISTRO_CODENAME=`lsb_release -s -c`
+fi
+
+if [ $TRAVIS_OS_NAME == "linux" ] && [ "$DISTRO_CODENAME" != "trusty" ]; then
     # Use the Ubuntu-provided OpenJDK
     unset JAVA_HOME
     export TRAVIS_MVN=/usr/bin/mvn

diff --git a/ci/travis_install_clang_tools.sh b/ci/travis_install_clang_tools.sh
@@ -17,8 +17,10 @@
 # specific language governing permissions and limitations
 # under the License.
 
-# e.g. "trusty" or "xenial"
-DISTRO_CODENAME=`lsb_release -s -c`
+
+set -ex
+
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
 
 wget -O - http://llvm.org/apt/llvm-snapshot.gpg.key|sudo apt-key add -
 sudo apt-add-repository -y \

diff --git a/ci/travis_install_linux.sh b/ci/travis_install_linux.sh
@@ -19,10 +19,16 @@
 
 set -e
 
+source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
+
 sudo apt-get install -y -qq \
     gdb binutils ccache libboost-dev libboost-filesystem-dev \
     libboost-system-dev libboost-regex-dev
 
+if [ "$CXX" == "g++-4.9" ]; then
+    sudo apt-get install -y -qq g++-4.9
+fi
+
 if [ "$ARROW_TRAVIS_VALGRIND" == "1" ]; then
     sudo apt-get install -y -qq valgrind
 fi
@@ -31,21 +37,23 @@ if [ "$ARROW_TRAVIS_COVERAGE" == "1" ]; then
     sudo apt-get install -y -qq lcov
 fi
 
-if [ "$ARROW_TRAVIS_GANDIVA" == "1" ]; then
-    sudo apt-get install -y -qq llvm-6.0-dev
-fi
-
 set -x
+if [ "$DISTRO_CODENAME" != "trusty" ]; then
+    if [ "$ARROW_TRAVIS_GANDIVA" == "1" ]; then
+        sudo apt-get install -y -qq llvm-6.0-dev
+    fi
+
+    sudo apt-get install -y -qq maven
+
+    # Remove Travis-specific versions of Java
+    sudo rm -rf /usr/local/lib/jvm*
+    sudo rm -rf /usr/local/maven*
+    hash -r
+    unset JAVA_HOME
+
+    which java
+    which mvn
+    java -version
+    mvn -v
+fi
 
-sudo apt-get install -y -qq maven
-
-# Remove Travis-specific versions of Java
-sudo rm -rf /usr/local/lib/jvm*
-sudo rm -rf /usr/local/maven*
-hash -r
-unset JAVA_HOME
-
-which java
-which mvn
-java -version
-mvn -v
diff --git a/ci/travis_install_toolchain.sh b/ci/travis_install_toolchain.sh
@@ -23,14 +23,19 @@ source $TRAVIS_BUILD_DIR/ci/travis_install_conda.sh
 
 if [ ! -e $CPP_TOOLCHAIN ]; then
     CONDA_PACKAGES=""
+    CONDA_LABEL=""
 
     if [ $ARROW_TRAVIS_GANDIVA == "1" ] && [ $TRAVIS_OS_NAME == "osx" ]; then
         CONDA_PACKAGES="$CONDA_PACKAGES llvmdev=6.0.1"
     fi
 
     if [ $TRAVIS_OS_NAME == "linux" ]; then
-        # Use newer binutils when linking against conda-provided libraries
-        CONDA_PACKAGES="$CONDA_PACKAGES binutils"
+        if [ "$DISTRO_CODENAME" == "trusty" ]; then
+            CONDA_LABEL=" -c conda-forge/label/cf201901"
+        else
+            # Use newer binutils when linking against conda-provided libraries
+            CONDA_PACKAGES="$CONDA_PACKAGES binutils"
+        fi
     fi
 
     if [ $ARROW_TRAVIS_VALGRIND == "1" ]; then
@@ -39,7 +44,7 @@ if [ ! -e $CPP_TOOLCHAIN ]; then
     fi
 
     # Set up C++ toolchain from conda-forge packages for faster builds
-    conda create -y -q -p $CPP_TOOLCHAIN \
+    conda create -y -q -p $CPP_TOOLCHAIN $CONDA_LABEL \
         --file=$TRAVIS_BUILD_DIR/ci/conda_env_cpp.yml \
         $CONDA_PACKAGES \
         ccache \

diff --git a/dev/tasks/gandiva-jars/build-cpp.sh b/dev/tasks/gandiva-jars/build-cpp.sh
@@ -29,6 +29,7 @@ pushd arrow/cpp
           -DARROW_GANDIVA=ON \
           -DARROW_GANDIVA_JAVA=ON \
           -DARROW_GANDIVA_STATIC_LIBSTDCPP=ON \
+          -DARROW_BUILD_TESTS=ON \
           -DARROW_BUILD_UTILITIES=OFF \
           -DARROW_BOOST_USE_SHARED=OFF \
           ..

diff --git a/dev/tasks/gandiva-jars/build-java.sh b/dev/tasks/gandiva-jars/build-java.sh
@@ -19,11 +19,19 @@
 
 set -e
 
+source arrow/ci/travis_env_common.sh
+
+CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp/build/release
+
 pushd arrow/java
+  if [ $TRAVIS_OS_NAME == "linux" ]; then
+    ldd $CPP_BUILD_DIR/libgandiva_jni.so
+  fi
+
   # build the entire project
-  mvn clean install -DskipTests -P gandiva -Dgandiva.cpp.build.dir=../../cpp/build/release
+  mvn clean install -DskipTests -P gandiva -Dgandiva.cpp.build.dir=$CPP_BUILD_DIR
   # test only gandiva
-  mvn test -P gandiva -pl gandiva -Dgandiva.cpp.build.dir=../../cpp/build/release
+  mvn test -P gandiva -pl gandiva -Dgandiva.cpp.build.dir=$CPP_BUILD_DIR
   # copy the jars to distribution folder
   find gandiva/target/ -name "*.jar" -not -name "*tests*" -exec cp  {} ../../dist/ \;
 popd
diff --git a/dev/tasks/gandiva-jars/travis.linux.yml b/dev/tasks/gandiva-jars/travis.linux.yml
@@ -32,6 +32,8 @@ env:
     - ARROW_TRAVIS_USE_TOOLCHAIN=1
 
 before_install:
+  # gcc 4.9 is required for the static linking of libstdc++
+  - export CC="gcc-4.9" CXX="g++-4.9"
   - ulimit -c unlimited -S
   - |
       sudo bash -c "echo -e 'Acquire::Retries 10; Acquire::http::Timeout \"20\";' > /etc/apt/apt.conf.d/99-travis-retry"
@@ -41,9 +43,9 @@ before_install:
 before_script:
   - git clone -b {{ arrow.branch }} {{ arrow.remote }} arrow
   - git -C arrow checkout {{ arrow.head }}
+  - export TRAVIS_BUILD_DIR=$TRAVIS_BUILD_DIR/arrow
   - arrow/ci/travis_install_linux.sh
   - arrow/ci/travis_install_clang_tools.sh
-  - export TRAVIS_BUILD_DIR=$TRAVIS_BUILD_DIR/arrow
   - arrow/ci/travis_install_toolchain.sh
 
 script:

diff --git a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Filter.java b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Filter.java
@@ -86,7 +86,7 @@ public static Filter make(Schema schema, Condition condition, long configuration
     JniWrapper wrapper = JniLoader.getInstance().getWrapper();
     long moduleId = wrapper.buildFilter(schemaBuf.toByteArray(),
         conditionBuf.toByteArray(), configurationId);
-    logger.info("Created module for the projector with id {}", moduleId);
+    logger.debug("Created module for the filter with id {}", moduleId);
     return new Filter(wrapper, moduleId, schema);
   }
 

diff --git a/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Projector.java b/java/gandiva/src/main/java/org/apache/arrow/gandiva/evaluator/Projector.java
@@ -101,7 +101,7 @@ public static Projector make(Schema schema, List<ExpressionTree> exprs, long
     JniWrapper wrapper = JniLoader.getInstance().getWrapper();
     long moduleId = wrapper.buildProjector(schemaBuf.toByteArray(),
         builder.build().toByteArray(), configurationId);
-    logger.info("Created module for the projector with id {}", moduleId);
+    logger.debug("Created module for the projector with id {}", moduleId);
     return new Projector(wrapper, moduleId, schema, exprs.size());
   }