From 6505e48882551a0fd7364f6b2dde23b10b0c5345 Mon Sep 17 00:00:00 2001 From: Niklas Hambuechen Date: Fri, 7 Nov 2014 22:12:35 +0100 Subject: [PATCH] Use Cabal to enforce dependency versions This uses `cabal configure` to determine which exact dependency versions we are compiling against, and ensures that these versions are used by passing -package-id flags to GHC. The `cabal configure` step makes the build fail before compiling / type checking if the user tries to compile against a dependency version we don't support; before, this case led to type errors which were not clearly user errors. This fixes issue #988. The output of `cabal configure` is also used to generate MIN_VERSION_* macros. MIN_VERSION_* macros are the standard way to build CPP dependency switches in Haskell packages, and they replace our custom macros (like PARALLEL3 and NO_REGEX_PCRE) which had to be hand-built for each dependency. We can now query the version of any Haskell dependency without having to manually add a flag via autoconf. All ghc and hlint invocations were adjusted to take these macros into account. This change introduces a Haskell-build-time dependency on cabal-install (for `cabal configure`) and the Cabal API (for obtaining the configured dependency versions and generating the macros). Any cabal version since Debian Squeeze is supported. Note that our use of Cabal does not imply any downloading of dependencies at build time, hermetic builds are unaffected by this change. For developers we now require hlint >= 1.8.60, to make use of its --cpp-file option. However, hlint >= 1.9.12 is recommended since for hlint >= 1.8.58 && < 1.9.12 the --utf8 flag is non-functional (see https://github.com/ndmitchell/hlint/issues/96); this can be worked though by using the equivalent `--encoding=UTF-8` flag. --- INSTALL | 36 +++++++------------- Makefile.am | 58 ++++++++++++++++++++++++++++++-- cabal/CabalDependenciesMacros.hs | 38 +++++++++++++++++++++ cabal/ganeti.template.cabal | 32 +++++++++++++++--- configure.ac | 8 +++++ 5 files changed, 141 insertions(+), 31 deletions(-) create mode 100644 cabal/CabalDependenciesMacros.hs diff --git a/INSTALL b/INSTALL index 05b24e2ac4..ab2280ec38 100644 --- a/INSTALL +++ b/INSTALL @@ -127,6 +127,9 @@ deploy Ganeti on production machines). More specifically: - or even better, `The Haskell Platform `_ which gives you a simple way to bootstrap Haskell +- `cabal-install `_ and + `Cabal `_, the Common Architecture + for Building Haskell Applications and Libraries (executable and library) - `json `_, a JSON library - `network `_, a basic network library @@ -163,7 +166,8 @@ deploy Ganeti on production machines). More specifically: Some of these are also available as package in Debian/Ubuntu:: - $ apt-get install ghc libghc-json-dev libghc-network-dev \ + $ apt-get install ghc cabal-install libghc-cabal-dev \ + libghc-json-dev libghc-network-dev \ libghc-parallel-dev \ libghc-utf8-string-dev libghc-curl-dev \ libghc-hslogger-dev \ @@ -190,36 +194,19 @@ The most recent Fedora doesn't provide ``crypto``, ``inotify``. So these need to be installed using ``cabal``. If using a distribution which does not provide these libraries, first -install the Haskell platform. You can also install ``cabal`` manually:: +install the Haskell platform. Then run:: - $ apt-get install cabal-install $ cabal update Then install the additional native libraries:: $ apt-get install libpcre3-dev libcurl4-openssl-dev -And finally the libraries required for building the packages (only the -ones not available in your distribution packages) via ``cabal``:: +And finally the libraries required for building the packages via ``cabal`` +(it will automatically pick only those that are not already installed via your +distribution packages):: - $ cabal install json network parallel utf8-string curl hslogger \ - Crypto text hinotify==0.3.2 regex-pcre \ - attoparsec vector base64-bytestring \ - lifted-base==0.2.0.3 lens==3.10 - -(The specified versions are suitable for Debian Wheezy, for other -distributions different versions might be needed.) - -.. _cabal-order-note: -.. note:: - When installing additional libraries using ``cabal``, be sure to first - install all the required libraries available in your distribution and - only then install the rest using ``cabal``. - Otherwise cabal might install different versions of libraries that are - available in your distribution, causing conflicts during the - compilation. - This applies in particular when installing libraries for the optional - features. + $ cabal install --only-dependencies cabal/ganeti.template.cabal Haskell optional features ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -242,7 +229,8 @@ either apt:: or ``cabal``:: - $ cabal install snap-server PSQueue + $ cabal install --only-dependencies cabal/ganeti.template.cabal \ + --flags="confd mond metad" to install them. diff --git a/Makefile.am b/Makefile.am index cc4bcd4f58..2aee8396b8 100644 --- a/Makefile.am +++ b/Makefile.am @@ -261,6 +261,7 @@ BUILDTIME_DIR_AUTOCREATE = \ BUILDTIME_DIRS = \ $(BUILDTIME_DIR_AUTOCREATE) \ apps \ + dist \ doc/html \ doc/man-html @@ -296,8 +297,11 @@ CLEANFILES = \ $(addsuffix /*.o,$(HS_DIRS)) \ $(addsuffix /*.$(HTEST_SUFFIX)_hi,$(HS_DIRS)) \ $(addsuffix /*.$(HTEST_SUFFIX)_o,$(HS_DIRS)) \ + $(HASKELL_PACKAGE_VERSIONS_FILE) \ $(CABAL_EXECUTABLES_APPS_STAMPS) \ ganeti.cabal \ + $(HASKELL_PACKAGE_IDS_FILE) \ + $(HASKELL_PACKAGE_VERSIONS_FILE) \ Makefile.ghc \ Makefile.ghc.bak \ $(PYTHON_BOOTSTRAP) \ @@ -353,6 +357,7 @@ GENERATED_FILES = \ clean-local: rm -rf tools/shebang rm -rf apps + rm -rf dist HS_GENERATED_FILES = $(HS_PROGS) src/hluxid src/ganeti-luxid if ENABLE_CONFD @@ -1270,10 +1275,12 @@ HS_MAKEFILE_GHC_SRCS = $(HS_SRC_PROGS:%=%.hs) if WANT_HSTESTS HS_MAKEFILE_GHC_SRCS += $(HS_TEST_PROGS:%=%.hs) endif -Makefile.ghc: $(HS_MAKEFILE_GHC_SRCS) Makefile \ +Makefile.ghc: $(HS_MAKEFILE_GHC_SRCS) Makefile $(HASKELL_PACKAGE_VERSIONS_FILE) \ | $(built_base_sources) $(HS_BUILT_SRCS) $(GHC) -M -dep-makefile $@ $(DEP_SUFFIXES) $(HFLAGS) $(HFLAGS_DYNAMIC) \ -itest/hs \ + -optP-include -optP$(HASKELL_PACKAGE_VERSIONS_FILE) \ + `cat $(HASKELL_PACKAGE_IDS_FILE)` \ $(HS_PARALLEL3) $(HS_REGEX_PCRE) $(HEXTRA_COMBINED) $(HS_MAKEFILE_GHC_SRCS) # Since ghc -M does not generate dependency line for object files, dependencies # from a target executable seed object (e.g. src/hluxid.o) to objects which @@ -1289,19 +1296,51 @@ Makefile.ghc: $(HS_MAKEFILE_GHC_SRCS) Makefile \ @include_makefile_ghc@ +# Contains the `-package-id package-123hash123` flags for the current build. +# Obtained from the setup-config using the Cabal API +# (CabalDependenciesMacros.hs) after `cabal configure`. +# This file is created along with HASKELL_PACKAGE_VERSIONS_FILE; if you want +# to depend on it in a rule, depend on HASKELL_PACKAGE_VERSIONS_FILE instead. +HASKELL_PACKAGE_IDS_FILE = ganeti.depsflags + +# Defines the MIN_VERSION_* macros for all Haskell packages used in this +# compilation. +# The versions are determined using `cabal configure`, which takes them from +# the ghc-pkg database. +# At the moment, we don't support cabal sandboxes, so we use cabal configure +# with the --user flag. +# Note: `cabal configure` and CabalDependenciesMacros.hs perform no +# downloading (only `cabal install` can do that). +HASKELL_PACKAGE_VERSIONS_FILE = cabal_macros.h + +$(HASKELL_PACKAGE_VERSIONS_FILE): Makefile ganeti.cabal \ + cabal/CabalDependenciesMacros.hs + cabal configure --user \ + -f`test $(ENABLE_CONFD) == True && echo "confd" || echo "-confd"` \ + -f`test $(ENABLE_MOND) == True && echo "mond" || echo "-mond"` \ + -f`test $(ENABLE_METADATA) == True && echo "metad" || echo "-metad"` + runhaskell $(abs_top_srcdir)/cabal/CabalDependenciesMacros.hs \ + ganeti.cabal \ + $(HASKELL_PACKAGE_IDS_FILE) \ + $(HASKELL_PACKAGE_VERSIONS_FILE) + # Like the %.o rule, but allows access to the test/hs directory. # This uses HFLAGS instead of HTEST_FLAGS because it's only for generating # object files (.o for GHC <= 7.6, .o/.so for newer GHCs) that are loaded # in GHCI when evaluating TH. The actual test-with-coverage .hpc_o files # are created in the `%.$(HTEST_SUFFIX)_o` rule. -test/hs/%.o: +test/hs/%.o: $(HASKELL_PACKAGE_VERSIONS_FILE) @echo '[GHC|test]: $@ <- test/hs/$^' @$(GHC) -c $(HFLAGS) -itest/hs $(HFLAGS_DYNAMIC) \ + -optP-include -optP$(HASKELL_PACKAGE_VERSIONS_FILE) \ + `cat $(HASKELL_PACKAGE_IDS_FILE)` \ $(HS_PARALLEL3) $(HS_REGEX_PCRE) $(HEXTRA_COMBINED) $(@:%.o=%.hs) -%.o: +%.o: $(HASKELL_PACKAGE_VERSIONS_FILE) @echo '[GHC]: $@ <- $^' @$(GHC) -c $(HFLAGS) $(HFLAGS_DYNAMIC) \ + -optP-include -optP$(HASKELL_PACKAGE_VERSIONS_FILE) \ + `cat $(HASKELL_PACKAGE_IDS_FILE)` \ $(HS_PARALLEL3) $(HS_REGEX_PCRE) $(HEXTRA_COMBINED) $(@:%.o=%.hs) # For TH+profiling we need to compile twice: Once without profiling, @@ -1312,6 +1351,8 @@ if HPROFILE @echo '[GHC|prof]: $@ <- $^' @$(GHC) -c $(HFLAGS) \ $(HPROFFLAGS) \ + -optP-include -optP$(HASKELL_PACKAGE_VERSIONS_FILE) \ + `cat $(HASKELL_PACKAGE_IDS_FILE)` \ $(HS_PARALLEL3) $(HS_REGEX_PCRE) $(HEXTRA_COMBINED) \ $(@:%.$(HPROF_SUFFIX)_o=%.hs) endif @@ -1323,6 +1364,8 @@ endif %.$(HTEST_SUFFIX)_o: %.o @echo '[GHC|test]: $@ <- $^' @$(GHC) -c $(HTEST_FLAGS) \ + -optP-include -optP$(HASKELL_PACKAGE_VERSIONS_FILE) \ + `cat $(HASKELL_PACKAGE_IDS_FILE)` \ $(HS_PARALLEL3) $(HS_REGEX_PCRE) $(HEXTRA_COMBINED) $(@:%.$(HTEST_SUFFIX)_o=%.hs) %.hi: %.o ; @@ -1333,11 +1376,15 @@ if HPROFILE $(HS_SRC_PROGS): %: %.$(HPROF_SUFFIX)_o | stamp-directories @echo '[GHC-link]: $@' $(GHC) $(HFLAGS) $(HPROFFLAGS) \ + -optP-include -optP$(HASKELL_PACKAGE_VERSIONS_FILE) \ + `cat $(HASKELL_PACKAGE_IDS_FILE)` \ $(HS_PARALLEL3) $(HS_REGEX_PCRE) $(HEXTRA_COMBINED) --make $(@:%=%.hs) else $(HS_SRC_PROGS): %: %.o | stamp-directories @echo '[GHC-link]: $@' $(GHC) $(HFLAGS) $(HFLAGS_DYNAMIC) \ + -optP-include -optP$(HASKELL_PACKAGE_VERSIONS_FILE) \ + `cat $(HASKELL_PACKAGE_IDS_FILE)` \ $(HS_PARALLEL3) $(HS_REGEX_PCRE) $(HEXTRA_COMBINED) --make $(@:%=%.hs) endif @rm -f $(notdir $@).tix @@ -1352,6 +1399,8 @@ $(HS_TEST_PROGS): %: %.$(HTEST_SUFFIX)_o \ fi @echo '[GHC-link|test]: $@' $(GHC) $(HTEST_FLAGS) \ + -optP-include -optP$(HASKELL_PACKAGE_VERSIONS_FILE) \ + `cat $(HASKELL_PACKAGE_IDS_FILE)` \ $(HS_PARALLEL3) $(HS_REGEX_PCRE) $(HEXTRA_COMBINED) --make $(@:%=%.hs) @rm -f $(notdir $@).tix @touch "$@" @@ -1517,6 +1566,7 @@ EXTRA_DIST += \ doc/users/users.in \ ganeti.cabal \ cabal/ganeti.template.cabal \ + cabal/CabalDependenciesMacros.hs \ $(dist_TESTS) \ $(TEST_FILES) \ $(python_test_support) \ @@ -2601,6 +2651,7 @@ hlint: $(HS_BUILT_SRCS) src/lint-hints.hs --ignore "Use void" \ --ignore "Reduce duplication" \ --hint src/lint-hints \ + --cpp-file=$(HASKELL_PACKAGE_VERSIONS_FILE) \ $(filter-out $(HLINT_EXCLUDES),$(HS_LIBTEST_SRCS) $(HS_PROG_SRCS)) @if [ ! -f doc/hs-lint.html ]; then \ echo "All good" > doc/hs-lint.html; \ @@ -2736,6 +2787,7 @@ $(APIDOC_HS_DIR)/index.html: $(HS_LIBTESTBUILT_SRCS) Makefile set -e ; \ export LC_ALL=en_US.UTF-8; \ OPTGHC="--optghc=-isrc --optghc=-itest/hs"; \ + OPTGHC="$$OPTGHC --optghc=-optP-include --optghc=-optP$(HASKELL_PACKAGE_VERSIONS_FILE)"; \ if [ "$(HS_PARALLEL3)" ]; \ then OPTGHC="$$OPTGHC --optghc=$(HS_PARALLEL3)"; \ fi; \ diff --git a/cabal/CabalDependenciesMacros.hs b/cabal/CabalDependenciesMacros.hs new file mode 100644 index 0000000000..e07def7b63 --- /dev/null +++ b/cabal/CabalDependenciesMacros.hs @@ -0,0 +1,38 @@ +module Main where + +import Control.Applicative +import qualified Data.Set as Set +import qualified Distribution.Simple.Build.Macros as Macros +import Distribution.Simple.Configure (maybeGetPersistBuildConfig) +import Distribution.Simple.LocalBuildInfo (externalPackageDeps) +import Distribution.PackageDescription (packageDescription) +import Distribution.PackageDescription.Parse (readPackageDescription) +import Distribution.Text (display) +import Distribution.Verbosity (normal) +import System.Environment (getArgs) + + +main :: IO () +main = do + -- Get paths from program arguments. + (cabalPath, depsPath, macrosPath) <- do + args <- getArgs + case args of + [c, d, m] -> return (c, d, m) + _ -> error "Expected 3 arguments: cabalPath depsPath macrosPath" + + -- Read the cabal file. + pkgDesc <- packageDescription <$> readPackageDescription normal cabalPath + + -- Read the setup-config. + m'conf <- maybeGetPersistBuildConfig "dist" + case m'conf of + Nothing -> error "could not read dist/setup-config" + Just conf -> do + + -- Write package dependencies. + let deps = map (display . fst) $ externalPackageDeps conf + writeFile depsPath (unwords $ map ("-package-id " ++) deps) + + -- Write package MIN_VERSION_* macros. + writeFile macrosPath $ Macros.generate pkgDesc conf diff --git a/cabal/ganeti.template.cabal b/cabal/ganeti.template.cabal index dc1b7b017f..04d3b93fc6 100644 --- a/cabal/ganeti.template.cabal +++ b/cabal/ganeti.template.cabal @@ -16,6 +16,20 @@ description: . See + +flag confd + description: enable the ganeti-confd daemon + default: True + +flag mond + description: enable the ganeti monitoring daemon + default: True + +flag metad + description: enable the ganeti metadata daemon + default: True + + library exposed-modules: -- AUTOGENERATED_MODULES_HERE @@ -53,8 +67,6 @@ library , MonadCatchIO-transformers >= 0.3.0.0 && < 0.4 , network >= 2.3.0.13 && < 2.7 , parallel >= 3.2.0.2 && < 3.3 - , PSQueue >= 1.1 && < 1.2 - , regex-pcre >= 0.94.2 && < 0.95 , temporary >= 1.1.2.3 && < 1.3 , transformers-base >= 0.4.1 && < 0.5 , utf8-string >= 0.3.7 && < 0.4 @@ -66,12 +78,24 @@ library , test-framework-hunit >= 0.2.7 && < 0.4 , test-framework-quickcheck2 >= 0.2.12.1 && < 0.4 - , snap-server >= 0.8.1.1 && < 0.10 - -- Executables: -- , happy -- , hscolour -- , shelltestrunner + + if flag(confd) + build-depends: + regex-pcre >= 0.94.2 && < 0.95 + + if flag(mond) + build-depends: + PSQueue >= 1.1 && < 1.2 + , snap-server >= 0.8.1.1 && < 0.10 + + if flag(metad) + build-depends: + snap-server >= 0.8.1.1 && < 0.10 + hs-source-dirs: src, test/hs build-tools: diff --git a/configure.ac b/configure.ac index 57e28e104f..c930422f5f 100644 --- a/configure.ac +++ b/configure.ac @@ -659,6 +659,13 @@ if test -z "$GHC_PKG"; then AC_MSG_FAILURE([ghc-pkg not found, compilation will not be possible]) fi +# Check for cabal +AC_ARG_VAR(CABAL, [cabal path]) +AC_PATH_PROG(CABAL, [cabal], []) +if test -z "$CABAL"; then + AC_MSG_FAILURE([cabal not found, compilation will not be possible]) +fi + # check for modules, first custom/special checks AC_MSG_NOTICE([checking for required haskell modules]) HS_PARALLEL3= @@ -667,6 +674,7 @@ AC_GHC_PKG_CHECK([parallel-3.*], [HS_PARALLEL3=-DPARALLEL3], AC_SUBST(HS_PARALLEL3) # and now standard modules +AC_GHC_PKG_REQUIRE(Cabal) AC_GHC_PKG_REQUIRE(curl) AC_GHC_PKG_REQUIRE(json) AC_GHC_PKG_REQUIRE(network)