From 56533dcb70817f45b9361137ed1b55888b7a53e1 Mon Sep 17 00:00:00 2001 From: Taylor Smock <45215054+tsmock@users.noreply.github.com> Date: Wed, 30 Dec 2020 11:23:51 -0700 Subject: [PATCH] ExternalData: Use ResourceCache to get data (#398) * Gradle: Add JUnit5 JUnit5 has better support for exceptions (assertThrows, assertDoesNotThrow, etc.). Signed-off-by: Taylor Smock * ExternalData: Allow checks to obtain and use arbitrary external data WaterWayChecks: Add note in documentation about resolution and NASA SRTM ElevationUtilities: External data: Add tests to ensure that data is fetched. ExternalData: * Use ResourceCache to get data * Use check constructors to fetch necessary data or to store the file fetcher to be used later. Each check can decide how it fetches data. * See docs/externalData.md for implementation notes. CheckResourceLoader: Fix all code smells Signed-off-by: Taylor Smock --- build.gradle | 7 + dependencies.gradle | 8 + docs/checks/waterWayChecks.md | 2 + docs/externalData.md | 42 ++++ gradle/quality.gradle | 1 + .../checks/base/CheckResourceLoader.java | 75 ++++++-- .../checks/base/ExternalDataFetcher.java | 179 ++++++++++++++++++ .../distributed/IntegrityCheckSparkJob.java | 23 +-- .../ShardedIntegrityChecksSparkJob.java | 10 +- .../checks/utility/ElevationUtilities.java | 65 ++++--- .../linear/lines/WaterWayCheck.java | 8 +- .../ShardedIntegrityChecksSparkJobTest.java | 3 +- .../utility/ElevationUtilitiesTest.java | 71 ++++++- .../utility/ElevationUtilitiesTestRule.java | 42 ++++ .../linear/lines/WaterWayCheckTest.java | 68 ++++--- 15 files changed, 509 insertions(+), 95 deletions(-) create mode 100644 docs/externalData.md create mode 100644 src/main/java/org/openstreetmap/atlas/checks/base/ExternalDataFetcher.java create mode 100644 src/test/java/org/openstreetmap/atlas/checks/utility/ElevationUtilitiesTestRule.java diff --git a/build.gradle b/build.gradle index 53cee48df..221644e3e 100644 --- a/build.gradle +++ b/build.gradle @@ -97,6 +97,13 @@ dependencies checkstyle packages.atlas_checkstyle shaded project.configurations.getByName('compile') + + // Support Junit 5 tests + testImplementation packages.junit.api + testRuntimeOnly packages.junit.engine + // Support JUnit 3/4 tests + testCompileOnly packages.junit.junit4 + testRuntimeOnly packages.junit.vintage } /** diff --git a/dependencies.gradle b/dependencies.gradle index 6ebec52e0..fcff3f50e 100644 --- a/dependencies.gradle +++ b/dependencies.gradle @@ -9,6 +9,8 @@ project.ext.versions = [ postgres: '42.2.6', spring: '4.2.2.RELEASE', mockito: '2.23.0', + junit4: '4.13.1', + junit: '5.7.0', log4j: '1.2.17' ] @@ -22,5 +24,11 @@ project.ext.packages = [ postgres: "org.postgresql:postgresql:${versions.postgres}", spring: "org.springframework:spring-jdbc:${versions.spring}", mockito: "org.mockito:mockito-core:${versions.mockito}", + junit: [ + junit4: "junit:junit:${versions.junit4}", + vintage: "org.junit.vintage:junit-vintage-engine:${versions.junit}", + api: "org.junit.jupiter:junit-jupiter-api:${versions.junit}", + engine: "org.junit.jupiter:junit-jupiter-engine:${versions.junit}", + ], log4j: "log4j:log4j:${versions.log4j}" ] diff --git a/docs/checks/waterWayChecks.md b/docs/checks/waterWayChecks.md index 1a616edde..ce428c7aa 100644 --- a/docs/checks/waterWayChecks.md +++ b/docs/checks/waterWayChecks.md @@ -21,6 +21,8 @@ This check identifies waterways that are closed (i.e., would have a circular wat } } ``` +If you are using the NASA SRTM elevation files, their resolution is either 30m or 90m. _This means that the elevation checks will ignore uphill waterways by default when using the NASA SRTM files_. + You may also want to look at [ElevationUtilities](../utilities/elevationUtilities.md). #### Code Review diff --git a/docs/externalData.md b/docs/externalData.md new file mode 100644 index 000000000..d6a5609e5 --- /dev/null +++ b/docs/externalData.md @@ -0,0 +1,42 @@ +# External Data +## Common information +External data is defined as any data that is not an atlas file. External data +_MUST_ be placed in the same directory as the atlas files, with the name +`extra`. + +In some cases, if the external data file is _very large_, the data transfer +may be interrupted, and the process will fail, absent error handling. + +## Using external data in a check +There must be a constructor for the check that follows this definition: +`public Check(Configuration configuration, ExternalDataFetcher fetcher)` + +At this point, the `ExternalDataFetcher` can be used to prefetch data, or +stored later to be used to dynamically get data as the check progresses. +The latter is generally recommended, since the data is cached as it is needed. +This means that the data will not be transferred if the check is cancelled or +has an error prior to needing the data. + +### Example code +```java +public Check(Configuration configuration, ExternalDataFetcher fetcher) { + super(configuration); + Optional optionalResource = fetcher.apply("filename"); + if (optionalResource.isPresent()) { + // Congratulations, you have the data + Resource resource = optionalResource.get(); + // At this point, you can get an InputStream. + InputStream inputStream = resource.read(); + /* Currently, there is no good way to get the actual file on the local + * filesystem. This should be implemented as soon as that functionality + * is required. + * + * Implementation note: The returned resource will most likely be a + * InputStreamResource. This, however, can change. + */ + } else { + // You don't have the data. Either error out or log, depending upon + // how critical the data is. + } +} +``` diff --git a/gradle/quality.gradle b/gradle/quality.gradle index 48e08d51f..0702a72d0 100644 --- a/gradle/quality.gradle +++ b/gradle/quality.gradle @@ -24,6 +24,7 @@ sourceSets test { + useJUnitPlatform() testLogging { events "failed" diff --git a/src/main/java/org/openstreetmap/atlas/checks/base/CheckResourceLoader.java b/src/main/java/org/openstreetmap/atlas/checks/base/CheckResourceLoader.java index 375f87b55..fe3580a90 100644 --- a/src/main/java/org/openstreetmap/atlas/checks/base/CheckResourceLoader.java +++ b/src/main/java/org/openstreetmap/atlas/checks/base/CheckResourceLoader.java @@ -61,6 +61,7 @@ public class CheckResourceLoader + BaseCheck.PARAMETER_DENYLIST_COUNTRIES; private final Class checkType; private final Configuration configuration; + private final ExternalDataFetcher fileFetcher; private final MultiMap countryGroups = new MultiMap<>(); private final Boolean enabledByDefault; private final String enabledKeyTemplate; @@ -74,18 +75,30 @@ public class CheckResourceLoader * @param configuration * the {@link Configuration} for loaded checks */ - @SuppressWarnings("unchecked") public CheckResourceLoader(final Configuration configuration) + { + this(configuration, null); + } + + /** + * Default constructor + * + * @param configuration + * the {@link Configuration} for loaded checks + * @param fileFetcher + * the {@link ExternalDataFetcher} to load additional data with + */ + @SuppressWarnings("unchecked") + public CheckResourceLoader(final Configuration configuration, + final ExternalDataFetcher fileFetcher) { this.packages = Collections.unmodifiableSet(Iterables.asSet((Iterable) configuration .get("CheckResourceLoader.scanUrls", Collections.singletonList(DEFAULT_PACKAGE)) .value())); final Map> groups = configuration.get("groups", Collections.emptyMap()) .value(); - groups.keySet().forEach(group -> - { - groups.get(group).forEach(country -> this.countryGroups.add(country, group)); - }); + groups.keySet().forEach(group -> groups.get(group) + .forEach(country -> this.countryGroups.add(country, group))); final ClassLoader loader = Thread.currentThread().getContextClassLoader(); try @@ -106,6 +119,7 @@ public CheckResourceLoader(final Configuration configuration) this.checkPermitList = configuration.get("CheckResourceLoader.checks.permitlist") .valueOption(); this.checkDenyList = configuration.get("CheckResourceLoader.checks.denylist").valueOption(); + this.fileFetcher = fileFetcher; } public Configuration getConfiguration() @@ -137,14 +151,14 @@ public Configuration getConfigurationForCountry(final String country) return specializedConfiguration; } - public Set loadChecks(final Predicate isEnabled) + public Set loadChecks(final Predicate> isEnabled) { - return this.loadChecks(isEnabled, this.configuration); + return this.loadChecks(isEnabled, this.configuration, this.fileFetcher); } public Set loadChecks(final Configuration configuration) { - return this.loadChecks(this::isEnabledByConfiguration, configuration); + return this.loadChecks(this::isEnabledByConfiguration, configuration, this.fileFetcher); } /** @@ -154,21 +168,40 @@ public Set loadChecks(final Configuration configuration) * {@link Predicate} used to determine if a check is enabled * @param configuration * {@link Configuration} used to loadChecks {@link CheckResourceLoader} + * @param fileFetcher + * {@link ExternalDataFetcher} used to load additional data * @param * check type * @return a {@link Set} of checks */ - @SuppressWarnings("unchecked") - public Set loadChecks(final Predicate isEnabled, - final Configuration configuration) + public Set loadChecks(final Predicate> isEnabled, + final Configuration configuration, final ExternalDataFetcher fileFetcher) { - final Class[][] constructorArgumentTypes = new Class[][] { { Configuration.class }, - {} }; - final Object[][] constructorArguments = new Object[][] { { configuration }, {} }; + final Class[][] constructorArgumentTypes = new Class[][] { + { Configuration.class, ExternalDataFetcher.class }, { Configuration.class }, {} }; + final Object[][] constructorArguments = new Object[][] { { configuration, fileFetcher }, + { configuration }, {} }; return this.loadChecksUsingConstructors(isEnabled, constructorArgumentTypes, constructorArguments); } + /** + * Loads checks that are enabled by some other means, defined by {@code isEnabled} + * + * @param isEnabled + * {@link Predicate} used to determine if a check is enabled + * @param configuration + * {@link Configuration} used to loadChecks {@link CheckResourceLoader} + * @param + * check type + * @return a {@link Set} of checks + */ + public Set loadChecks(final Predicate> isEnabled, + final Configuration configuration) + { + return this.loadChecks(isEnabled, configuration, this.fileFetcher); + } + /** * Loads checks that are enabled by configuration * @@ -178,14 +211,15 @@ public Set loadChecks(final Predicate isEnabled, */ public Set loadChecks() { - return this.loadChecks(this::isEnabledByConfiguration, this.configuration); + return this.loadChecks(this::isEnabledByConfiguration, this.configuration, + this.fileFetcher); } public Set loadChecksForCountry(final String country) { final Configuration countryConfiguration = this.getConfigurationForCountry(country); return this.loadChecks(checkClass -> this.isEnabledByConfiguration(countryConfiguration, - checkClass, country), countryConfiguration); + checkClass, country), countryConfiguration, this.fileFetcher); } public Set loadChecksUsingConstructors( @@ -217,7 +251,8 @@ public Set loadChecksUsingConstructors( * Any class that extends Check. * @return A set of enabled, initialized checks. */ - public Set loadChecksUsingConstructors(final Predicate isEnabled, + @SuppressWarnings("unchecked") + public Set loadChecksUsingConstructors(final Predicate> isEnabled, final Class[][] constructorArgumentTypes, final Object[][] constructorArguments) { final Set checks = new HashSet<>(); @@ -301,20 +336,20 @@ private Optional initializeCheckWithArguments(final Class checkClass) { return this.isEnabledByConfiguration(this.configuration, checkClass); } private boolean isEnabledByConfiguration(final Configuration configuration, - final Class checkClass) + final Class checkClass) { final String key = String.format(this.enabledKeyTemplate, checkClass.getSimpleName()); return configuration.get(key, this.enabledByDefault).value(); } private boolean isEnabledByConfiguration(final Configuration configuration, - final Class checkClass, final String country) + final Class checkClass, final String country) { final List countryPermitlist = configuration .get(String.format(COUNTRY_PERMITLIST_TEMPLATE, checkClass.getSimpleName()), diff --git a/src/main/java/org/openstreetmap/atlas/checks/base/ExternalDataFetcher.java b/src/main/java/org/openstreetmap/atlas/checks/base/ExternalDataFetcher.java new file mode 100644 index 000000000..cfe61b0aa --- /dev/null +++ b/src/main/java/org/openstreetmap/atlas/checks/base/ExternalDataFetcher.java @@ -0,0 +1,179 @@ +package org.openstreetmap.atlas.checks.base; + +import java.io.InputStream; +import java.io.Serializable; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; + +import javax.annotation.Nonnull; + +import org.openstreetmap.atlas.checks.distributed.ShardedIntegrityChecksSparkJob; +import org.openstreetmap.atlas.exception.CoreException; +import org.openstreetmap.atlas.generator.tools.filesystem.FileSystemHelper; +import org.openstreetmap.atlas.generator.tools.spark.utilities.SparkFileHelper; +import org.openstreetmap.atlas.streaming.resource.Resource; +import org.openstreetmap.atlas.utilities.caching.ConcurrentResourceCache; +import org.openstreetmap.atlas.utilities.caching.strategies.NamespaceCachingStrategy; +import org.openstreetmap.atlas.utilities.runtime.Retry; +import org.openstreetmap.atlas.utilities.scalars.Duration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The fetcher to use for generic resources. The fetcher uses hadoop cache to reduce remote reads. + * See {@link ShardedIntegrityChecksSparkJob#atlasFetcher}. This a separate class so that it can + * implement {@link Serializable} + * + * @author Taylor Smock + */ +public class ExternalDataFetcher implements Function>, Serializable +{ + /** + * Cache external data sources locally. Does not implement {@link Serializable} since + * {@link ConcurrentResourceCache} does not offer a no-args constructor, and is not serializable + * itself. + * + * @author Taylor Smock + */ + private class ExternalDataResourceCache extends ConcurrentResourceCache + { + ExternalDataResourceCache() + { + super(new NamespaceCachingStrategy(GLOBAL_HADOOP_FILECACHE_NAMESPACE), + new ExternalFileFetcher()); + } + } + + /** + * Create a serializable function to get external files for use with a + * {@link org.openstreetmap.atlas.utilities.caching.ResourceCache}. + * + * @author Taylor Smock + */ + private class ExternalFileFetcher implements Function>, Serializable + { + private static final long serialVersionUID = 1721253891315559418L; + + @Override + public Optional apply(final URI uri) + { + final Retry retry = new Retry(RETRY_ATTEMPTS, Duration.ONE_SECOND).withQuadratic(true); + final boolean exists = retry.run(() -> + { + try (InputStream inputStream = FileSystemHelper + .resource(uri.toString(), ExternalDataFetcher.this.configuration).read()) + { + return true; + } + catch (final Exception e) + { + if (e.getMessage().contains(FileSystemHelper.FILE_NOT_FOUND)) + { + return false; + } + else + { + throw new CoreException("Unable to test existence of {}", uri, e); + } + } + }); + if (!exists) + { + if (!ExternalDataFetcher.this.silent) + { + logger.warn("Fetcher: resource {} does not exist!", uri); + } + return Optional.empty(); + } + return Optional.ofNullable(FileSystemHelper.resource(uri.toString(), + ExternalDataFetcher.this.configuration)); + } + } + + private static final long serialVersionUID = 724339604023082195L; + private static final String GLOBAL_HADOOP_FILECACHE_NAMESPACE = "__HadoopExternalFileCache_global_namespace__"; + private static final Logger logger = LoggerFactory.getLogger(ExternalDataFetcher.class); + /** Maximum number of retries for where an error occurs when getting a file */ + private static final int RETRY_ATTEMPTS = 5; + /** The input folder path (same as the atlas file paths) */ + private final String input; + /** The configuration used to create the cache */ + private final Map configuration; + /** The actual caching object */ + private transient ExternalDataResourceCache cache; + /** + * {@code true} implies that the caller does not want to log that files are not present + */ + private boolean silent; + + /** + * Create the fetcher to use for generic resources. The fetcher uses hadoop cache to reduce + * remote reads. See {@link ShardedIntegrityChecksSparkJob#atlasFetcher}. + * + * @param input + * {@link String} input folder path + * @param configuration + * {@link org.openstreetmap.atlas.generator.tools.spark.SparkJob} configuration map + */ + public ExternalDataFetcher(final String input, final Map configuration) + { + this.input = input; + this.configuration = configuration; + } + + @Override + public Optional apply(final String string) + { + return this.getCache().get(this.getUri(string)); + } + + /** + * Make missed file messages silent (use when checking for files -- please log actual issues + * when this is used) Unfortunately, this does not suppress all messages from missed + * files. + * + * @param silent + * {@code true} suppresses some logging messages from non-existent files + */ + public void setSilent(final boolean silent) + { + this.silent = silent; + } + + /** + * @return The resource cacher to use + */ + @Nonnull + private ExternalDataResourceCache getCache() + { + if (this.cache == null) + { + this.cache = new ExternalDataResourceCache(); + } + return this.cache; + } + + /** + * Get a URI for a path string + * + * @param string + * The path + * @return A URI for the string + */ + private URI getUri(final String string) + { + final String atlasURIString = SparkFileHelper.combine(this.input, string); + try + { + return new URI(atlasURIString); + } + catch (final URISyntaxException exception) + { + throw new CoreException("Bad URI syntax: {}", atlasURIString, exception); + } + } + +} diff --git a/src/main/java/org/openstreetmap/atlas/checks/distributed/IntegrityCheckSparkJob.java b/src/main/java/org/openstreetmap/atlas/checks/distributed/IntegrityCheckSparkJob.java index 266d025d2..a0a1b332b 100644 --- a/src/main/java/org/openstreetmap/atlas/checks/distributed/IntegrityCheckSparkJob.java +++ b/src/main/java/org/openstreetmap/atlas/checks/distributed/IntegrityCheckSparkJob.java @@ -16,6 +16,7 @@ import org.apache.spark.api.java.JavaPairRDD; import org.openstreetmap.atlas.checks.base.BaseCheck; import org.openstreetmap.atlas.checks.base.CheckResourceLoader; +import org.openstreetmap.atlas.checks.base.ExternalDataFetcher; import org.openstreetmap.atlas.checks.configuration.ConfigurationResolver; import org.openstreetmap.atlas.checks.constants.CommonConstants; import org.openstreetmap.atlas.checks.event.CheckFlagFileProcessor; @@ -93,9 +94,8 @@ public static void main(final String[] args) * @param configuration * {@link MapRouletteConfiguration} to create a new {@link MapRouletteClient}s */ - @SuppressWarnings("rawtypes") private static void executeChecks(final String country, final Atlas atlas, - final Set checksToRun, final MapRouletteConfiguration configuration) + final Set> checksToRun, final MapRouletteConfiguration configuration) { final Pool checkExecutionPool = new Pool(checksToRun.size(), "Check execution pool", POOL_DURATION_BEFORE_KILL); @@ -133,21 +133,19 @@ public String getName() return "Integrity Check Spark Job"; } - @SuppressWarnings({ "rawtypes", "unchecked" }) + @SuppressWarnings("unchecked") @Override public void start(final CommandMap commandMap) { - final String atlasDirectory = (String) commandMap.get(ATLAS_FOLDER); + final String atlasDirectory = (String) commandMap.get(SparkJob.INPUT); final String input = Optional.ofNullable(this.input(commandMap)).orElse(atlasDirectory); final String output = this.output(commandMap); - @SuppressWarnings("unchecked") final Set outputFormats = (Set) commandMap .get(OUTPUT_FORMATS); final StringList countries = StringList.split((String) commandMap.get(COUNTRIES), CommonConstants.COMMA); final MapRouletteConfiguration mapRouletteConfiguration = (MapRouletteConfiguration) commandMap .get(MAP_ROULETTE); - @SuppressWarnings("unchecked") final Optional> checkFilter = (Optional>) commandMap .getOption(CHECK_FILTER); @@ -164,14 +162,17 @@ public void start(final CommandMap commandMap) .collect(Collectors.toList())); final boolean saveIntermediateAtlas = (Boolean) commandMap.get(PBF_SAVE_INTERMEDIATE_ATLAS); - @SuppressWarnings("unchecked") final Rectangle pbfBoundary = ((Optional) commandMap.getOption(PBF_BOUNDING_BOX)) .orElse(Rectangle.MAXIMUM); final boolean compressOutput = Boolean .parseBoolean((String) commandMap.get(SparkJob.COMPRESS_OUTPUT)); final Map sparkContext = this.configurationMap(); - final CheckResourceLoader checkLoader = new CheckResourceLoader(checksConfiguration); + + final ExternalDataFetcher fileFetcher = new ExternalDataFetcher(input, + this.configurationMap()); + final CheckResourceLoader checkLoader = new CheckResourceLoader(checksConfiguration, + fileFetcher); // check configuration and country list final Set> preOverriddenChecks = checkLoader.loadChecks(); if (!this.isValidInput(countries, preOverriddenChecks)) @@ -186,7 +187,7 @@ public void start(final CommandMap commandMap) // Create a list of Country to Check tuples // Add priority countries first if they are supplied by parameter - final List>> countryCheckTuples = new ArrayList<>(); + final List>>> countryCheckTuples = new ArrayList<>(); countries.stream().filter(priorityCountries::contains).forEach(country -> countryCheckTuples .add(new Tuple2<>(country, checkLoader.loadChecksForCountry(country)))); @@ -204,7 +205,7 @@ public void start(final CommandMap commandMap) logger.info("Initialized checks: {}", infoMessage2); // Parallelize on the countries - final JavaPairRDD> countryCheckRDD = this.getContext() + final JavaPairRDD>> countryCheckRDD = this.getContext() .parallelizePairs(countryCheckTuples, countryCheckTuples.size()); // Set target and temporary folders @@ -231,7 +232,7 @@ public void start(final CommandMap commandMap) final Time timer = Time.now(); final String country = tuple._1(); - final Set checks = tuple._2(); + final Set> checks = tuple._2(); logger.info("Initialized checks for {}: {}", country, checks.stream().map(BaseCheck::getCheckName).collect(Collectors.joining(","))); diff --git a/src/main/java/org/openstreetmap/atlas/checks/distributed/ShardedIntegrityChecksSparkJob.java b/src/main/java/org/openstreetmap/atlas/checks/distributed/ShardedIntegrityChecksSparkJob.java index 4a4d5e666..005243304 100644 --- a/src/main/java/org/openstreetmap/atlas/checks/distributed/ShardedIntegrityChecksSparkJob.java +++ b/src/main/java/org/openstreetmap/atlas/checks/distributed/ShardedIntegrityChecksSparkJob.java @@ -17,6 +17,7 @@ import org.apache.spark.broadcast.Broadcast; import org.openstreetmap.atlas.checks.base.Check; import org.openstreetmap.atlas.checks.base.CheckResourceLoader; +import org.openstreetmap.atlas.checks.base.ExternalDataFetcher; import org.openstreetmap.atlas.checks.configuration.ConfigurationResolver; import org.openstreetmap.atlas.checks.constants.CommonConstants; import org.openstreetmap.atlas.checks.event.CheckFlagEvent; @@ -31,6 +32,7 @@ import org.openstreetmap.atlas.exception.CoreException; import org.openstreetmap.atlas.generator.sharding.AtlasSharding; import org.openstreetmap.atlas.generator.tools.caching.HadoopAtlasFileCache; +import org.openstreetmap.atlas.generator.tools.spark.SparkJob; import org.openstreetmap.atlas.generator.tools.spark.utilities.SparkFileHelper; import org.openstreetmap.atlas.geography.atlas.Atlas; import org.openstreetmap.atlas.geography.atlas.AtlasResourceLoader; @@ -107,7 +109,7 @@ public String getName() public void start(final CommandMap commandMap) { final Time start = Time.now(); - final String atlasDirectory = (String) commandMap.get(ATLAS_FOLDER); + final String atlasDirectory = (String) commandMap.get(SparkJob.INPUT); final String input = Optional.ofNullable(this.input(commandMap)).orElse(atlasDirectory); // Gather arguments @@ -137,7 +139,11 @@ public void start(final CommandMap commandMap) // File loading helpers final SparkFileHelper fileHelper = new SparkFileHelper(sparkContext); - final CheckResourceLoader checkLoader = new CheckResourceLoader(checksConfiguration); + // Get the file fetcher + final ExternalDataFetcher fileFetcher = new ExternalDataFetcher(input, + this.configurationMap()); + final CheckResourceLoader checkLoader = new CheckResourceLoader(checksConfiguration, + fileFetcher); // Get sharding @SuppressWarnings("unchecked") diff --git a/src/main/java/org/openstreetmap/atlas/checks/utility/ElevationUtilities.java b/src/main/java/org/openstreetmap/atlas/checks/utility/ElevationUtilities.java index a2eb4d22b..ad76d77fd 100644 --- a/src/main/java/org/openstreetmap/atlas/checks/utility/ElevationUtilities.java +++ b/src/main/java/org/openstreetmap/atlas/checks/utility/ElevationUtilities.java @@ -1,21 +1,27 @@ package org.openstreetmap.atlas.checks.utility; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.Serializable; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.function.Function; +import java.util.stream.Stream; + +import javax.annotation.Nonnull; import org.apache.commons.lang3.tuple.Pair; +import org.apache.log4j.Logger; +import org.openstreetmap.atlas.checks.base.ExternalDataFetcher; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Longitude; +import org.openstreetmap.atlas.streaming.resource.Resource; import org.openstreetmap.atlas.utilities.configuration.Configuration; import org.openstreetmap.atlas.utilities.scalars.Distance; @@ -33,6 +39,8 @@ public final class ElevationUtilities implements Serializable * The assumed file extension */ private static final String SRTM_EXTENSION = "hgt"; + /** The logger */ + private static final Logger logger = Logger.getLogger(ElevationUtilities.class); /** * The assumed extent of a HGT SRTM file (lat/lon) in degrees */ @@ -48,6 +56,10 @@ public final class ElevationUtilities implements Serializable /** Just an int for converting a decimal to a percentage */ private static final int DECIMAL_TO_PERCENTAGE = 100; + /** Various compression extensions (including none) */ + private static final String[] POSSIBLE_COMPRESSED_EXTS = new String[] { "", ".zip", ".gz", + ".xz", ".bz", ".bz2", ".tar" }; + /** A map of {lat, lon} pairs with a loaded srtm in a byte array */ private final Map, short[][]> loadedSrtm = new HashMap<>(); @@ -56,6 +68,7 @@ public final class ElevationUtilities implements Serializable private final String srtmExtension; private final String srtmPath; + private ExternalDataFetcher fileFetcher; /** * Configuration Keys in the Integrity Framework are based on the check simple classname. @@ -85,15 +98,19 @@ private static String formatKey(final String name, final String key) * * @param configuration * A configuration which should (at a minimum) have a file path for elevation files. + * @param fileFetcher + * The file fetcher to use to get data files */ - public ElevationUtilities(final Configuration configuration) + public ElevationUtilities(final Configuration configuration, + final ExternalDataFetcher fileFetcher) { this(configurationValue(configuration, "elevation.srtm_extent", SRTM_EXTENT, Function.identity()), configurationValue(configuration, "elevation.srtm_ext", SRTM_EXTENSION, Function.identity()), - configurationValue(configuration, "elevation.path", "elevation", - Function.identity())); + configurationValue(configuration, "elevation.path", "extra/elevation", + Function.identity()), + fileFetcher); } @@ -107,13 +124,17 @@ public ElevationUtilities(final Configuration configuration) * (they will be automatically detected). * @param srtmPath * The path for the files. + * @param fileFetcher + * The file fetcher to use to get data files */ public ElevationUtilities(final double srtmExtent, final String srtmExtension, - final String srtmPath) + final String srtmPath, final ExternalDataFetcher fileFetcher) { this.srtmExtension = srtmExtension; this.srtmExtent = srtmExtent; - this.srtmPath = srtmPath; + this.srtmPath = srtmPath.endsWith(File.separator) ? srtmPath + : srtmPath.concat(File.separator); + this.fileFetcher = fileFetcher; } /** @@ -312,29 +333,29 @@ public void putMap(final Location location, final short[][] map) * The longitude to use * @return A short[latitude][longitude] = height in meters array */ + @Nonnull private synchronized short[][] loadMap(final int lat, final int lon) { - final String filename = this.getSrtmFileName(lat, lon); - Path path = Paths.get(this.srtmPath, filename); - if (!path.toFile().isFile()) + if (this.fileFetcher == null) { - for (final String ext : new String[] { "zip", "gz", "xz", "bz", "bz2", "tar" }) - { - path = Paths.get(this.srtmPath, filename + "." + ext); - if (path.toFile().isFile()) - { - break; - } - } + logger.error("Cannot load maps -- fileFetcher is not initialized or is null"); + return EMPTY_MAP; } - if (!path.toFile().isFile()) + final String filename = this.getSrtmFileName(lat, lon); + final Optional path = Stream.of(POSSIBLE_COMPRESSED_EXTS) + .map(ext -> Paths.get(this.srtmPath, filename.concat(ext))).map(Object::toString) + .map(this.fileFetcher::apply).filter(Optional::isPresent).map(Optional::get) + .findFirst(); + + if (path.isEmpty()) { return EMPTY_MAP; } - try (InputStream is = CompressionUtilities - .getUncompressedInputStream(Files.newInputStream(path))) + try (InputStream temporaryInputStream = path.get().read(); + InputStream inputStream = CompressionUtilities + .getUncompressedInputStream(temporaryInputStream)) { - return this.readStream(is); + return this.readStream(inputStream); } catch (final IOException e) { diff --git a/src/main/java/org/openstreetmap/atlas/checks/validation/linear/lines/WaterWayCheck.java b/src/main/java/org/openstreetmap/atlas/checks/validation/linear/lines/WaterWayCheck.java index 9477a90a5..3f4c9c499 100644 --- a/src/main/java/org/openstreetmap/atlas/checks/validation/linear/lines/WaterWayCheck.java +++ b/src/main/java/org/openstreetmap/atlas/checks/validation/linear/lines/WaterWayCheck.java @@ -15,6 +15,7 @@ import org.apache.commons.lang3.tuple.Pair; import org.openstreetmap.atlas.checks.base.BaseCheck; +import org.openstreetmap.atlas.checks.base.ExternalDataFetcher; import org.openstreetmap.atlas.checks.flag.CheckFlag; import org.openstreetmap.atlas.checks.utility.CommonTagFilters; import org.openstreetmap.atlas.checks.utility.ElevationUtilities; @@ -158,11 +159,13 @@ private static boolean endsWithBoundaryNode(final Atlas atlas, final AtlasObject * * @param configuration * the JSON configuration for this check + * @param fileFetcher + * The file fetcher to use to get data files */ - public WaterWayCheck(final Configuration configuration) + public WaterWayCheck(final Configuration configuration, final ExternalDataFetcher fileFetcher) { super(configuration); - this.elevationUtils = new ElevationUtilities(configuration); + this.elevationUtils = new ElevationUtilities(configuration, fileFetcher); this.waterwaySinkTagFilter = this.configurationValue(configuration, "waterway.sink.tags.filters", WATERWAY_SINK_TAG_FILTER_DEFAULT, TaggableFilter::forDefinition); @@ -584,5 +587,4 @@ private boolean waterwayConnects(final LineItem line, final LineItem potential) } return !locations.isEmpty(); } - } diff --git a/src/test/java/org/openstreetmap/atlas/checks/distributed/ShardedIntegrityChecksSparkJobTest.java b/src/test/java/org/openstreetmap/atlas/checks/distributed/ShardedIntegrityChecksSparkJobTest.java index dad92a485..9e543f2d2 100644 --- a/src/test/java/org/openstreetmap/atlas/checks/distributed/ShardedIntegrityChecksSparkJobTest.java +++ b/src/test/java/org/openstreetmap/atlas/checks/distributed/ShardedIntegrityChecksSparkJobTest.java @@ -122,8 +122,7 @@ private void generateData() */ private void runShardedIntegrityChecksSparkJob() { - final String[] arguments = { - String.format("-inputFolder=%s", INPUT.getAbsolutePathString()), + final String[] arguments = { String.format("-input=%s", INPUT.getAbsolutePathString()), String.format("-startedFolder=%s", INPUT.getAbsolutePathString()), String.format("-output=%s", OUTPUT.getAbsolutePathString()), String.format("-sharding=slippy@%s", ZOOM_LEVEL), "-maxShardLoad=1", diff --git a/src/test/java/org/openstreetmap/atlas/checks/utility/ElevationUtilitiesTest.java b/src/test/java/org/openstreetmap/atlas/checks/utility/ElevationUtilitiesTest.java index d8a60f318..af568a5da 100644 --- a/src/test/java/org/openstreetmap/atlas/checks/utility/ElevationUtilitiesTest.java +++ b/src/test/java/org/openstreetmap/atlas/checks/utility/ElevationUtilitiesTest.java @@ -2,9 +2,12 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.lang.reflect.Field; @@ -14,16 +17,22 @@ import java.nio.ByteOrder; import java.text.MessageFormat; import java.util.Arrays; +import java.util.Collections; import java.util.Map; import org.apache.commons.lang3.tuple.Pair; import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.openstreetmap.atlas.checks.base.ExternalDataFetcher; import org.openstreetmap.atlas.checks.configuration.ConfigurationResolver; import org.openstreetmap.atlas.geography.Latitude; import org.openstreetmap.atlas.geography.Location; import org.openstreetmap.atlas.geography.Longitude; import org.openstreetmap.atlas.utilities.scalars.Distance; +import org.springframework.util.SerializationUtils; /** * Test class for {@link ElevationUtilities}. @@ -35,13 +44,22 @@ public class ElevationUtilitiesTest private static ElevationUtilities elevationUtilities; + @Rule + public ElevationUtilitiesTestRule atlases = new ElevationUtilitiesTestRule(); + + @ClassRule + public static TemporaryFolder temporaryFolder = new TemporaryFolder(); + /** * Perform initial setup of the test */ @BeforeClass public static void setUp() { - elevationUtilities = new ElevationUtilities(ConfigurationResolver.emptyConfiguration()); + final ExternalDataFetcher fetcher = new ExternalDataFetcher( + temporaryFolder.getRoot().getAbsolutePath(), Collections.emptyMap()); + elevationUtilities = new ElevationUtilities(ConfigurationResolver.emptyConfiguration(), + fetcher); prefillData(); } @@ -88,7 +106,7 @@ private static void prefillData() /** * Test method for {@link ElevationUtilities#ElevationUtilities} - * + * * @throws ReflectiveOperationException * when something goes wrong with reflection */ @@ -102,9 +120,9 @@ public void testConstructors() throws ReflectiveOperationException "'{'\"ElevationUtilities\":'{'\"elevation.srtm_extent\": {0}, \"elevation.srtm_ext\": \"{1}\", \"elevation.path\": \"{2}\"'}}'", Double.toString(expectedSrtmExtent), expectedSrtmExtension, expectedSrtmPath); final ElevationUtilities config = new ElevationUtilities( - ConfigurationResolver.inlineConfiguration(configuration)); + ConfigurationResolver.inlineConfiguration(configuration), null); final ElevationUtilities custom = new ElevationUtilities(expectedSrtmExtent, - expectedSrtmExtension, expectedSrtmPath); + expectedSrtmExtension, expectedSrtmPath, null); final Field srtmExtentField = ElevationUtilities.class.getDeclaredField("srtmExtent"); final Field srtmExtensionField = ElevationUtilities.class.getDeclaredField("srtmExtension"); final Field srtmPathField = ElevationUtilities.class.getDeclaredField("srtmPath"); @@ -117,7 +135,8 @@ public void testConstructors() throws ReflectiveOperationException assertEquals(expectedSrtmExtent, srtmExtentField.getDouble(elevationUtility), 0.001); assertEquals(expectedSrtmExtension, srtmExtensionField.get(elevationUtility).toString()); - assertEquals(expectedSrtmPath, srtmPathField.get(elevationUtility).toString()); + assertEquals(expectedSrtmPath + File.separator, + srtmPathField.get(elevationUtility).toString()); } } @@ -201,4 +220,46 @@ public void testInSameDataPoint() assertFalse(elevationUtilities.inSameDataPoint(one, Location.CROSSING_85_280)); } + /** + * This checks to ensure that serialization works properly (i.e., that fetching occurs properly + * for multi-node use) + * + * @throws IOException + * If there is an error writing to a test file in a temporary directory + */ + @Test + public void testSerialization() throws IOException + { + // Ensure that we aren't erroring out if someone passes a null fetcher/null + // atlas + final byte[] serializedElevation = SerializationUtils.serialize(elevationUtilities); + elevationUtilities = (ElevationUtilities) SerializationUtils + .deserialize(serializedElevation); + + // Create a temporary file that will be loaded by the fetcher + final File extra = this.temporaryFolder.newFolder("extra", "elevation"); + final File toWrite = new File(extra, elevationUtilities.getSrtmFileName(28, -90)); + try (FileOutputStream writer = new FileOutputStream(toWrite.getAbsolutePath())) + { + final ByteBuffer byteBuffer = ByteBuffer.allocate(32); + byteBuffer.order(ByteOrder.BIG_ENDIAN); + for (short i = 16; i < 32; i++) + { + byteBuffer.putShort(i); + } + writer.write(byteBuffer.array()); + } + catch (final IOException e) + { + throw e; + } + // Attempt to read the stored file + // We mostly care that the elevation is not the default value for this test. + // There is another test for correct responses. + // Please note that ElevationUtilities caches maps, which means that this should + // not be called earlier in the test. + final Location one = new Location(Latitude.degrees(28.92414725033), + Longitude.degrees(-89.42596868149)); + assertNotEquals(Short.MIN_VALUE, elevationUtilities.getElevation(one)); + } } diff --git a/src/test/java/org/openstreetmap/atlas/checks/utility/ElevationUtilitiesTestRule.java b/src/test/java/org/openstreetmap/atlas/checks/utility/ElevationUtilitiesTestRule.java new file mode 100644 index 000000000..bbea24256 --- /dev/null +++ b/src/test/java/org/openstreetmap/atlas/checks/utility/ElevationUtilitiesTestRule.java @@ -0,0 +1,42 @@ +package org.openstreetmap.atlas.checks.utility; + +import org.openstreetmap.atlas.geography.atlas.Atlas; +import org.openstreetmap.atlas.utilities.testing.CoreTestRule; +import org.openstreetmap.atlas.utilities.testing.TestAtlas; +import org.openstreetmap.atlas.utilities.testing.TestAtlas.Line; +import org.openstreetmap.atlas.utilities.testing.TestAtlas.Loc; +import org.openstreetmap.atlas.utilities.testing.TestAtlas.Node; + +/** + * Create test atlases for use with {@link ElevationUtilitiesTest}. + * + * @author Taylor Smock + */ +public class ElevationUtilitiesTestRule extends CoreTestRule +{ + @TestAtlas(nodes = { + @Node(id = "-101752", coordinates = @Loc(value = "28.92414725033,-89.42596868149")), + @Node(id = "-101754", coordinates = @Loc(value = "28.93350332367,-89.41806350699")), + @Node(id = "-101755", coordinates = @Loc(value = "28.92996541203,-89.40144467423")), + @Node(id = "-101756", coordinates = @Loc(value = "28.91942958225,-89.39012590165")), + @Node(id = "-101757", coordinates = @Loc(value = "28.91345356126,-89.40234298952")), + @Node(id = "-101758", coordinates = @Loc(value = "28.91376809726,-89.43081958402")) }, lines = { + @Line(id = "-101782", coordinates = { + @Loc(value = "28.92414725033,-89.42596868149"), + @Loc(value = "28.93350332367,-89.41806350699"), + @Loc(value = "28.92996541203,-89.40144467423"), + @Loc(value = "28.91942958225,-89.39012590165"), + @Loc(value = "28.91345356126,-89.40234298952"), + @Loc(value = "28.91376809726,-89.43081958402"), + @Loc(value = "28.92414725033,-89.42596868149") }, tags = { + "waterway=river" }) }) + private Atlas circularWaterway; + + /** + * @return A circular waterway (reused from WaterWayCheckTestRule) + */ + public Atlas getCircularWaterway() + { + return this.circularWaterway; + } +} diff --git a/src/test/java/org/openstreetmap/atlas/checks/validation/linear/lines/WaterWayCheckTest.java b/src/test/java/org/openstreetmap/atlas/checks/validation/linear/lines/WaterWayCheckTest.java index 1cdcf6330..9c801b0f2 100644 --- a/src/test/java/org/openstreetmap/atlas/checks/validation/linear/lines/WaterWayCheckTest.java +++ b/src/test/java/org/openstreetmap/atlas/checks/validation/linear/lines/WaterWayCheckTest.java @@ -2,6 +2,7 @@ import java.lang.reflect.Field; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.openstreetmap.atlas.checks.configuration.ConfigurationResolver; @@ -24,14 +25,22 @@ public class WaterWayCheckTest @Rule public ConsumerBasedExpectedCheckVerifier verifier = new ConsumerBasedExpectedCheckVerifier(); + private WaterWayCheck defaultWaterWayCheck; + + @Before + public void setUp() + { + this.defaultWaterWayCheck = new WaterWayCheck(ConfigurationResolver.emptyConfiguration(), + null); + } + /** * Look for waterways that connect with themselves */ @Test public void testCircularWaterway() { - this.verifier.actual(this.atlases.getCircularWaterway(), - new WaterWayCheck(ConfigurationResolver.emptyConfiguration())); + this.verifier.actual(this.atlases.getCircularWaterway(), this.defaultWaterWayCheck); this.verifier.verifyExpectedSize(1); } @@ -41,8 +50,7 @@ public void testCircularWaterway() @Test public void testCoastWaterway() { - this.verifier.actual(this.atlases.getCoastlineWaterway(), - new WaterWayCheck(ConfigurationResolver.emptyConfiguration())); + this.verifier.actual(this.atlases.getCoastlineWaterway(), this.defaultWaterWayCheck); this.verifier.verifyEmpty(); } @@ -53,7 +61,7 @@ public void testCoastWaterway() public void testCoastWaterwayConnected() { this.verifier.actual(this.atlases.getCoastlineWaterwayConnected(), - new WaterWayCheck(ConfigurationResolver.emptyConfiguration())); + this.defaultWaterWayCheck); this.verifier.verifyEmpty(); } @@ -62,8 +70,10 @@ public void testCoastWaterwayConnectedElevation() throws ReflectiveOperationExce { final short[][] map = new short[][] { { 15, 14, 13, 12 }, { 11, 10, 9, 8 }, { 7, 6, 5, 4 }, { 3, 2, 1, 0 } }; - final WaterWayCheck check = new WaterWayCheck(ConfigurationResolver.inlineConfiguration( - "{\"WaterWayCheck.waterway.elevation.resolution.min.uphill\": 30000.0}")); + final WaterWayCheck check = new WaterWayCheck( + ConfigurationResolver.inlineConfiguration( + "{\"WaterWayCheck.waterway.elevation.resolution.min.uphill\": 30000.0}"), + null); final Field elevationUtilsField = WaterWayCheck.class.getDeclaredField("elevationUtils"); elevationUtilsField.setAccessible(true); final ElevationUtilities elevationUtils = (ElevationUtilities) elevationUtilsField @@ -78,8 +88,10 @@ public void testCoastWaterwayConnectedElevationReversed() throws ReflectiveOpera { final short[][] map = new short[][] { { 0, 1, 2, 3 }, { 4, 5, 6, 7 }, { 8, 9, 10, 11 }, { 12, 13, 14, 15 } }; - final WaterWayCheck check = new WaterWayCheck(ConfigurationResolver.inlineConfiguration( - "{\"WaterWayCheck.waterway.elevation.resolution.min.uphill\": 30000.0}")); + final WaterWayCheck check = new WaterWayCheck( + ConfigurationResolver.inlineConfiguration( + "{\"WaterWayCheck.waterway.elevation.resolution.min.uphill\": 30000.0}"), + null); final Field elevationUtilsField = WaterWayCheck.class.getDeclaredField("elevationUtils"); elevationUtilsField.setAccessible(true); @@ -97,7 +109,8 @@ public void testCoastWaterwayConnectedElevationReversedLongDistance() final short[][] map = new short[][] { { 0, 1, 2, 3 }, { 4, 5, 6, 7 }, { 8, 9, 10, 11 }, { 12, 13, 14, 15 } }; final WaterWayCheck check = new WaterWayCheck(ConfigurationResolver.inlineConfiguration( - "{\"WaterWayCheck\": {\"waterway.elevation.resolution.min.uphill\": 30000.0, \"waterway.elevation.distance.min.start.end\": 1.0}}")); + "{\"WaterWayCheck\": {\"waterway.elevation.resolution.min.uphill\": 30000.0, \"waterway.elevation.distance.min.start.end\": 1.0}}"), + null); final Field elevationUtilsField = WaterWayCheck.class.getDeclaredField("elevationUtils"); elevationUtilsField.setAccessible(true); @@ -114,14 +127,14 @@ public void testCoastWaterwayConnectedElevationReversedLowResolution() { final short[][] map = new short[][] { { 0, 1, 2, 3 }, { 4, 5, 6, 7 }, { 8, 9, 10, 11 }, { 12, 13, 14, 15 } }; - final WaterWayCheck check = new WaterWayCheck(ConfigurationResolver.emptyConfiguration()); final Field elevationUtilsField = WaterWayCheck.class.getDeclaredField("elevationUtils"); elevationUtilsField.setAccessible(true); final ElevationUtilities elevationUtils = (ElevationUtilities) elevationUtilsField - .get(check); + .get(this.defaultWaterWayCheck); elevationUtils.putMap(Location.forString("16.9906416,-88.3188021"), map); - this.verifier.actual(this.atlases.getCoastlineWaterwayConnected(), check); + this.verifier.actual(this.atlases.getCoastlineWaterwayConnected(), + this.defaultWaterWayCheck); this.verifier.verifyEmpty(); } @@ -132,7 +145,7 @@ public void testCoastWaterwayConnectedElevationReversedLowResolution() public void testCoastWaterwayReversed() { this.verifier.actual(this.atlases.getCoastlineWaterwayReversed(), - new WaterWayCheck(ConfigurationResolver.emptyConfiguration())); + this.defaultWaterWayCheck); this.verifier.verifyExpectedSize(1); } @@ -142,8 +155,7 @@ public void testCoastWaterwayReversed() @Test public void testCrossingWaterway() { - this.verifier.actual(this.atlases.getCrossingWaterways(), - new WaterWayCheck(ConfigurationResolver.emptyConfiguration())); + this.verifier.actual(this.atlases.getCrossingWaterways(), this.defaultWaterWayCheck); this.verifier.verifyExpectedSize(1); } @@ -154,7 +166,7 @@ public void testCrossingWaterway() public void testCrossingWaterwayConnected() { this.verifier.actual(this.atlases.getCrossingWaterwaysConnected(), - new WaterWayCheck(ConfigurationResolver.emptyConfiguration())); + this.defaultWaterWayCheck); this.verifier.verifyEmpty(); } @@ -165,7 +177,7 @@ public void testCrossingWaterwayConnected() public void testCrossingWaterwayDifferentLayers() { this.verifier.actual(this.atlases.getCrossingWaterwaysDifferentLayers(), - new WaterWayCheck(ConfigurationResolver.emptyConfiguration())); + this.defaultWaterWayCheck); this.verifier.verifyEmpty(); } @@ -175,8 +187,7 @@ public void testCrossingWaterwayDifferentLayers() @Test public void testDeadendWaterway() { - this.verifier.actual(this.atlases.getDeadendWaterway(), - new WaterWayCheck(ConfigurationResolver.emptyConfiguration())); + this.verifier.actual(this.atlases.getDeadendWaterway(), this.defaultWaterWayCheck); this.verifier.verifyExpectedSize(1); } @@ -186,8 +197,7 @@ public void testDeadendWaterway() @Test public void testDeadendWaterways() { - this.verifier.actual(this.atlases.getDeadendWaterways(), - new WaterWayCheck(ConfigurationResolver.emptyConfiguration())); + this.verifier.actual(this.atlases.getDeadendWaterways(), this.defaultWaterWayCheck); this.verifier.verifyExpectedSize(1); } @@ -199,7 +209,7 @@ public void testDeadendWaterways() public void testDifferingLayersConnectedWaterway() { this.verifier.actual(this.atlases.getDifferingLayersConnectedWaterway(), - new WaterWayCheck(ConfigurationResolver.emptyConfiguration())); + this.defaultWaterWayCheck); this.verifier.verifyEmpty(); } @@ -209,8 +219,7 @@ public void testDifferingLayersConnectedWaterway() @Test public void testSinkholeAreaWaterway() { - this.verifier.actual(this.atlases.getSinkholeArea(), - new WaterWayCheck(ConfigurationResolver.emptyConfiguration())); + this.verifier.actual(this.atlases.getSinkholeArea(), this.defaultWaterWayCheck); this.verifier.verifyExpectedSize(1); } @@ -220,8 +229,7 @@ public void testSinkholeAreaWaterway() @Test public void testSinkholePointWaterway() { - this.verifier.actual(this.atlases.getSinkholePoint(), - new WaterWayCheck(ConfigurationResolver.emptyConfiguration())); + this.verifier.actual(this.atlases.getSinkholePoint(), this.defaultWaterWayCheck); this.verifier.verifyEmpty(); } @@ -232,7 +240,7 @@ public void testSinkholePointWaterway() public void testWaterwayEndingInOceanArea() { this.verifier.actual(this.atlases.getWaterwayEndingInOceanArea(), - new WaterWayCheck(ConfigurationResolver.emptyConfiguration())); + this.defaultWaterWayCheck); this.verifier.verifyEmpty(); } @@ -243,7 +251,7 @@ public void testWaterwayEndingInOceanArea() public void testWaterwayEndingInStraitArea() { this.verifier.actual(this.atlases.getWaterwayEndingInStraitArea(), - new WaterWayCheck(ConfigurationResolver.emptyConfiguration())); + this.defaultWaterWayCheck); this.verifier.verifyEmpty(); } @@ -254,7 +262,7 @@ public void testWaterwayEndingInStraitArea() public void testWaterwayEndingOnOtherWaterway() { this.verifier.actual(this.atlases.getWaterwayEndingOnOtherWaterway(), - new WaterWayCheck(ConfigurationResolver.emptyConfiguration())); + this.defaultWaterWayCheck); this.verifier.verifyEmpty(); } }